From eea56c4912ebabf94d366b4ead0d0d0a40a1d682 Mon Sep 17 00:00:00 2001 From: GeorgeA92 Date: Sat, 20 Nov 2021 20:13:54 +0200 Subject: [PATCH 01/55] per slot settings: creation of Downloader.Slot objects from per slot settings added --- scrapy/core/downloader/__init__.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index 4f7ab594f..f680b9082 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -83,6 +83,7 @@ class Downloader: self.middleware = DownloaderMiddlewareManager.from_crawler(crawler) self._slot_gc_loop = task.LoopingCall(self._slot_gc) self._slot_gc_loop.start(60) + self.per_slot_settings = self.settings.getdict('PER_SLOT_SETTINGS', {}) def fetch(self, request, spider): def _deactivate(response): @@ -99,9 +100,13 @@ class Downloader: def _get_slot(self, request, spider): key = self._get_slot_key(request, spider) if key not in self.slots: - conc = self.ip_concurrency if self.ip_concurrency else self.domain_concurrency + conc = self.per_slot_settings.get(key,{}).get('concurrency', self.ip_concurrency if self.ip_concurrency else self.domain_concurrency) conc, delay = _get_concurrency_delay(conc, spider, self.settings) - self.slots[key] = Slot(conc, delay, self.randomize_delay) + delay = self.per_slot_settings.get(key,{}).get('delay', delay) + randomize_delay = self.per_slot_settings.get(key, {}).get('randomize_delay', self.randomize_delay) + new_slot = Slot(conc, delay, randomize_delay) + self.slots[key] = new_slot + return key, self.slots[key] From d5acf88ca55988a6c92046da5ef2bf065b0b3e72 Mon Sep 17 00:00:00 2001 From: GeorgeA92 Date: Sat, 20 Nov 2021 20:14:28 +0200 Subject: [PATCH 02/55] per slot settings: logging added (create Slot) --- scrapy/core/downloader/__init__.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index f680b9082..9aab80171 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -1,3 +1,4 @@ +import logging import random from time import time from datetime import datetime @@ -12,6 +13,7 @@ from scrapy import signals from scrapy.core.downloader.middleware import DownloaderMiddlewareManager from scrapy.core.downloader.handlers import DownloadHandlers +logger = logging.getLogger(__name__) class Slot: """Downloader slot""" @@ -106,7 +108,7 @@ class Downloader: randomize_delay = self.per_slot_settings.get(key, {}).get('randomize_delay', self.randomize_delay) new_slot = Slot(conc, delay, randomize_delay) self.slots[key] = new_slot - + logger.debug(f"Downloader slot created {'from per slot settings' if key in self.per_slot_settings.keys() else ''}: {new_slot}") return key, self.slots[key] From 1b7d7ecfcdb6067f3dccd69fc56e25dc1dbf498f Mon Sep 17 00:00:00 2001 From: GeorgeA92 Date: Sat, 20 Nov 2021 21:05:24 +0200 Subject: [PATCH 03/55] per slot settings: logging updated (create/close Slot) --- scrapy/core/downloader/__init__.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index 9aab80171..461b211ab 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -108,7 +108,8 @@ class Downloader: randomize_delay = self.per_slot_settings.get(key, {}).get('randomize_delay', self.randomize_delay) new_slot = Slot(conc, delay, randomize_delay) self.slots[key] = new_slot - logger.debug(f"Downloader slot created {'from per slot settings' if key in self.per_slot_settings.keys() else ''}: {new_slot}") + logger.debug( + f"Downloader slot '{key}' - created {'(from per slot settings)' if key in self.per_slot_settings.keys() else ''}: {new_slot}") return key, self.slots[key] @@ -205,4 +206,7 @@ class Downloader: mintime = time() - age for key, slot in list(self.slots.items()): if not slot.active and slot.lastseen + slot.delay < mintime: - self.slots.pop(key).close() + inactive_slot = self.slots.pop(key) + inactive_slot.close() + logger.debug( + f"Downloader slot '{key}' - closed {'(from per slot settings)' if key in self.per_slot_settings.keys() else ''} : {inactive_slot}") From 8185aa5265c663326474897c24aa0691a2ee0e5c Mon Sep 17 00:00:00 2001 From: GeorgeA92 Date: Sat, 20 Nov 2021 21:06:08 +0200 Subject: [PATCH 04/55] per slot settings: codestyle fix --- scrapy/core/downloader/__init__.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index 461b211ab..d5809ffa0 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -15,6 +15,7 @@ from scrapy.core.downloader.handlers import DownloadHandlers logger = logging.getLogger(__name__) + class Slot: """Downloader slot""" @@ -102,9 +103,9 @@ class Downloader: def _get_slot(self, request, spider): key = self._get_slot_key(request, spider) if key not in self.slots: - conc = self.per_slot_settings.get(key,{}).get('concurrency', self.ip_concurrency if self.ip_concurrency else self.domain_concurrency) + conc = self.per_slot_settings.get(key, {}).get('concurrency', self.ip_concurrency if self.ip_concurrency else self.domain_concurrency) conc, delay = _get_concurrency_delay(conc, spider, self.settings) - delay = self.per_slot_settings.get(key,{}).get('delay', delay) + delay = self.per_slot_settings.get(key, {}).get('delay', delay) randomize_delay = self.per_slot_settings.get(key, {}).get('randomize_delay', self.randomize_delay) new_slot = Slot(conc, delay, randomize_delay) self.slots[key] = new_slot From 115d35270691176e128bd30f6a2a8bcf1289fe5c Mon Sep 17 00:00:00 2001 From: GeorgeA92 Date: Fri, 11 Feb 2022 01:21:31 +0200 Subject: [PATCH 05/55] per slot settings: setting renamed from `PER_SLOT_SETTINGS` to `DOWNLOAD_SLOTS` --- scrapy/core/downloader/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index d5809ffa0..0e32714a8 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -86,7 +86,7 @@ class Downloader: self.middleware = DownloaderMiddlewareManager.from_crawler(crawler) self._slot_gc_loop = task.LoopingCall(self._slot_gc) self._slot_gc_loop.start(60) - self.per_slot_settings = self.settings.getdict('PER_SLOT_SETTINGS', {}) + self.per_slot_settings = self.settings.getdict('DOWNLOAD_SLOTS', {}) def fetch(self, request, spider): def _deactivate(response): From be97402e46c5e4793e23b713728e1777fbe4d49c Mon Sep 17 00:00:00 2001 From: GeorgeA92 Date: Fri, 11 Feb 2022 01:22:27 +0200 Subject: [PATCH 06/55] per slot settings: log notifications (debug) removed --- scrapy/core/downloader/__init__.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index 0e32714a8..53dc546de 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -109,8 +109,6 @@ class Downloader: randomize_delay = self.per_slot_settings.get(key, {}).get('randomize_delay', self.randomize_delay) new_slot = Slot(conc, delay, randomize_delay) self.slots[key] = new_slot - logger.debug( - f"Downloader slot '{key}' - created {'(from per slot settings)' if key in self.per_slot_settings.keys() else ''}: {new_slot}") return key, self.slots[key] @@ -209,5 +207,3 @@ class Downloader: if not slot.active and slot.lastseen + slot.delay < mintime: inactive_slot = self.slots.pop(key) inactive_slot.close() - logger.debug( - f"Downloader slot '{key}' - closed {'(from per slot settings)' if key in self.per_slot_settings.keys() else ''} : {inactive_slot}") From 2006060688976ca469d794cdd3b753a00bfb83c9 Mon Sep 17 00:00:00 2001 From: GeorgeA92 Date: Wed, 11 May 2022 10:29:53 +0300 Subject: [PATCH 07/55] per slot settings: codestyle(flake8) fix, code line length --- scrapy/core/downloader/__init__.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index 53dc546de..511693830 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -103,7 +103,9 @@ class Downloader: def _get_slot(self, request, spider): key = self._get_slot_key(request, spider) if key not in self.slots: - conc = self.per_slot_settings.get(key, {}).get('concurrency', self.ip_concurrency if self.ip_concurrency else self.domain_concurrency) + conc = self.per_slot_settings.get(key, {}).get( + 'concurrency', self.ip_concurrency if self.ip_concurrency else self.domain_concurrency + ) conc, delay = _get_concurrency_delay(conc, spider, self.settings) delay = self.per_slot_settings.get(key, {}).get('delay', delay) randomize_delay = self.per_slot_settings.get(key, {}).get('randomize_delay', self.randomize_delay) From 0ffc52a491e6e6c46196b3aa92767856f64a8ebc Mon Sep 17 00:00:00 2001 From: GeorgeA92 Date: Wed, 11 May 2022 10:40:12 +0300 Subject: [PATCH 08/55] per slot settings: test added (delays for each download slots) --- tests/test_downloaderslotssettings.py | 75 +++++++++++++++++++++++++++ 1 file changed, 75 insertions(+) create mode 100644 tests/test_downloaderslotssettings.py diff --git a/tests/test_downloaderslotssettings.py b/tests/test_downloaderslotssettings.py new file mode 100644 index 000000000..6ea03fb78 --- /dev/null +++ b/tests/test_downloaderslotssettings.py @@ -0,0 +1,75 @@ +import time + +from scrapy.crawler import CrawlerRunner +from scrapy.http import Request + +from tests.mockserver import MockServer +from tests.spiders import MetaSpider + +from twisted.internet import defer +from twisted.trial.unittest import TestCase + + +class DownloaderSlotsSettingsTestSpider(MetaSpider): + + name = 'downloader_slots' + + custom_settings = { + "DOWNLOAD_DELAY": 1, + "RANDOMIZE_DOWNLOAD_DELAY": False, + "DOWNLOAD_SLOTS": { + 'quotes.toscrape.com': { + 'concurrency': 1, + 'delay': 1.5, + 'randomize_delay': False + }, + 'books.toscrape.com': { + 'delay': 2, + 'randomize_delay': False + } + } + } + + def start_requests(self): + self.times = {None: []} + + slots = list(self.custom_settings.get('DOWNLOAD_SLOTS', {}).keys()) + [None] + + for slot in slots: + url = self.mockserver.url(f"/?downloader_slot={slot}") + self.times[slot] = [] + yield Request(url, callback=self.parse, meta={'download_slot': slot}) + + def parse(self, response): + slot = response.meta.get('download_slot', None) + self.times[slot].append(time.time()) + url = self.mockserver.url(f"/?downloader_slot={slot}&req=2") + yield Request(url, callback=self.not_parse, meta={'download_slot': slot}) + + def not_parse(self, response): + slot = response.meta.get('download_slot', None) + self.times[slot].append(time.time()) + + +class CrawlTestCase(TestCase): + + def setUp(self): + self.mockserver = MockServer() + self.mockserver.__enter__() + self.runner = CrawlerRunner() + + def tearDown(self): + self.mockserver.__exit__(None, None, None) + + @defer.inlineCallbacks + def test_delay(self): + crawler = CrawlerRunner().create_crawler(DownloaderSlotsSettingsTestSpider) + yield crawler.crawl(mockserver=self.mockserver) + slots = crawler.engine.downloader.slots + times = crawler.spider.times + tolerance = 0.3 + + delays_real = {k: v[1] - v[0] for k, v in times.items()} + error_delta = {k: 1 - delays_real[k] / v.delay for k, v in slots.items()} + + self.assertTrue(max(list(error_delta.values())) < tolerance) From 5735e93541d26ad3aef5e4fc45fa3d1c4dc0fa2a Mon Sep 17 00:00:00 2001 From: felipeboffnunes Date: Thu, 28 Jul 2022 18:37:21 -0300 Subject: [PATCH 09/55] fix_post_processing_feed_export --- scrapy/extensions/feedexport.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index e7097b7a1..3e98a5a0b 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -350,13 +350,19 @@ class FeedExporter: return defer.DeferredList(deferred_list) if deferred_list else None def _close_slot(self, slot, spider): + + def get_file(slot_): + if isinstance(slot_.file, PostProcessingManager): + return slot_.file.file + return slot_.file + if not slot.itemcount and not slot.store_empty: # We need to call slot.storage.store nonetheless to get the file # properly closed. - return defer.maybeDeferred(slot.storage.store, slot.file) + return defer.maybeDeferred(slot.storage.store, get_file(slot)) slot.finish_exporting() logmsg = f"{slot.format} feed ({slot.itemcount} items) in: {slot.uri}" - d = defer.maybeDeferred(slot.storage.store, slot.file) + d = defer.maybeDeferred(slot.storage.store, get_file(slot)) d.addCallback( self._handle_store_success, logmsg, spider, type(slot.storage).__name__ From 4be9c969fd4bdc3206bdabfb5c22e524d9e355ac Mon Sep 17 00:00:00 2001 From: GeorgeA92 Date: Fri, 29 Jul 2022 12:15:39 +0300 Subject: [PATCH 10/55] per slot settings: logger deleted as not used --- scrapy/core/downloader/__init__.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index 511693830..fa8ac01e5 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -1,4 +1,3 @@ -import logging import random from time import time from datetime import datetime @@ -13,8 +12,6 @@ from scrapy import signals from scrapy.core.downloader.middleware import DownloaderMiddlewareManager from scrapy.core.downloader.handlers import DownloadHandlers -logger = logging.getLogger(__name__) - class Slot: """Downloader slot""" From d599fff2b97ff96c3face707b980926b6eb48a2b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 29 Jul 2022 19:10:28 +0200 Subject: [PATCH 11/55] Test the life cycle of a storage file --- tests/test_feedexport.py | 55 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index ecd1b59d3..a5fd1e467 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -1517,6 +1517,61 @@ class FeedExportTest(FeedExportTestBase): data = yield self.exported_data(items, settings) self.assertEqual(row['expected'], data[feed_options['format']]) + @defer.inlineCallbacks + def test_storage_file_no_postprocessing(self): + + @implementer(IFeedStorage) + class Storage: + + def __init__(self, uri, *, feed_options=None): + pass + + def open(self, spider): + Storage.open_file = tempfile.NamedTemporaryFile(prefix='feed-') + return Storage.open_file + + def store(self, file): + Storage.store_file = file + file.close() + + settings = { + 'FEEDS': {self._random_temp_filename(): {'format': 'jsonlines'}}, + 'FEED_STORAGES': {'file': Storage}, + } + yield self.exported_no_data(settings) + self.assertIs(Storage.open_file, Storage.store_file) + + @defer.inlineCallbacks + def test_storage_file_postprocessing(self): + + @implementer(IFeedStorage) + class Storage: + + def __init__(self, uri, *, feed_options=None): + pass + + def open(self, spider): + Storage.open_file = tempfile.NamedTemporaryFile(prefix='feed-') + return Storage.open_file + + def store(self, file): + Storage.store_file = file + file.close() + + settings = { + 'FEEDS': { + self._random_temp_filename(): { + 'format': 'jsonlines', + 'postprocessing': [ + 'scrapy.extensions.postprocessing.GzipPlugin', + ], + }, + }, + 'FEED_STORAGES': {'file': Storage}, + } + yield self.exported_no_data(settings) + self.assertIs(Storage.open_file, Storage.store_file) + class FeedPostProcessedExportsTest(FeedExportTestBase): __test__ = True From aabdd0b657e1b4398a5a1aad79044401e9dbf909 Mon Sep 17 00:00:00 2001 From: GeorgeA92 Date: Fri, 12 Aug 2022 15:34:48 +0300 Subject: [PATCH 12/55] per slot settings: logger deleted as not used (step 2) --- scrapy/core/downloader/__init__.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index fa8ac01e5..d908f4d7e 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -204,5 +204,4 @@ class Downloader: mintime = time() - age for key, slot in list(self.slots.items()): if not slot.active and slot.lastseen + slot.delay < mintime: - inactive_slot = self.slots.pop(key) - inactive_slot.close() + self.slots.pop(key).close() From 12d52a4f089798f266cbc6f86df3cc9c1cd58257 Mon Sep 17 00:00:00 2001 From: GeorgeA92 Date: Fri, 18 Nov 2022 14:16:18 +0200 Subject: [PATCH 13/55] per slot settings: code optimized --- scrapy/core/downloader/__init__.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index d908f4d7e..f16afe99b 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -100,12 +100,13 @@ class Downloader: def _get_slot(self, request, spider): key = self._get_slot_key(request, spider) if key not in self.slots: - conc = self.per_slot_settings.get(key, {}).get( + slot_settings = self.per_slot_settings.get(key, {}) + conc = slot_settings.get( 'concurrency', self.ip_concurrency if self.ip_concurrency else self.domain_concurrency ) conc, delay = _get_concurrency_delay(conc, spider, self.settings) - delay = self.per_slot_settings.get(key, {}).get('delay', delay) - randomize_delay = self.per_slot_settings.get(key, {}).get('randomize_delay', self.randomize_delay) + delay = slot_settings.get('delay', delay) + randomize_delay = slot_settings.get('randomize_delay', self.randomize_delay) new_slot = Slot(conc, delay, randomize_delay) self.slots[key] = new_slot From 042012f6bdaf4fbd5b978ff7cc9b6796286ac7c4 Mon Sep 17 00:00:00 2001 From: GeorgeA92 Date: Fri, 18 Nov 2022 14:56:29 +0200 Subject: [PATCH 14/55] per slot settings: error calculation metho updated --- tests/test_downloaderslotssettings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_downloaderslotssettings.py b/tests/test_downloaderslotssettings.py index 6ea03fb78..a092d01bf 100644 --- a/tests/test_downloaderslotssettings.py +++ b/tests/test_downloaderslotssettings.py @@ -70,6 +70,6 @@ class CrawlTestCase(TestCase): tolerance = 0.3 delays_real = {k: v[1] - v[0] for k, v in times.items()} - error_delta = {k: 1 - delays_real[k] / v.delay for k, v in slots.items()} + error_delta = {k: 1 - min(delays_real[k], v.delay) / max(delays_real[k], v.delay) for k, v in slots.items()} self.assertTrue(max(list(error_delta.values())) < tolerance) From b1dd893fbb4d2efed3342e6c98a6bbf4b393d48e Mon Sep 17 00:00:00 2001 From: Alex Date: Fri, 20 Jan 2023 02:17:02 -0800 Subject: [PATCH 15/55] Support Path Objects Issue #5739 --- scrapy/pipelines/files.py | 9 +++++---- tests/test_pipeline_files.py | 17 +++++++++++++++++ 2 files changed, 22 insertions(+), 4 deletions(-) diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index 51aedafe8..4b0c96b25 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -14,7 +14,7 @@ from contextlib import suppress from ftplib import FTP from io import BytesIO from pathlib import Path -from typing import DefaultDict, Optional, Set +from typing import DefaultDict, Optional, Set, Union from urllib.parse import urlparse from itemadapter import ItemAdapter @@ -41,7 +41,8 @@ class FileException(Exception): class FSFilesStore: - def __init__(self, basedir: str): + def __init__(self, basedir: Union[str, os.PathLike]): + basedir = str(basedir) # support Path object if '://' in basedir: basedir = basedir.split('://', 1)[1] self.basedir = basedir @@ -65,8 +66,8 @@ class FSFilesStore: return {'last_modified': last_modified, 'checksum': checksum} - def _get_filesystem_path(self, path: str) -> Path: - path_comps = path.split('/') + def _get_filesystem_path(self, path: Union[str, os.PathLike]) -> Path: + path_comps = str(path).split('/') return Path(self.basedir, *path_comps) def _mkdir(self, dirname: Path, domain: Optional[str] = None): diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py index 4acd29bf7..5280ab0d2 100644 --- a/tests/test_pipeline_files.py +++ b/tests/test_pipeline_files.py @@ -405,6 +405,23 @@ class FilesPipelineTestCaseCustomSettings(unittest.TestCase): self.assertEqual(getattr(pipeline_cls, pipe_inst_attr), expected_value) + def test_file_pipeline_using_pathlike_objects(self): + + class CustomFilesPipelineWithPathLikeDir(FilesPipeline): + def file_path(self, request, response=None, info=None, *, item=None): + return Path('subdir') / Path(request.url).name + + pipeline = CustomFilesPipelineWithPathLikeDir.from_settings( + Settings({'FILES_STORE': Path('./Temp')}) + ) + request = Request("http://example.com/image01.jpg") + self.assertEqual(pipeline.file_path(request), Path('subdir/image01.jpg')) + + def test_files_store_constructor_with_pathlike_object(self): + path = Path('./FileDir') + fs_store = FSFilesStore(path) + self.assertEqual(fs_store.basedir, str(path)) + class TestS3FilesStore(unittest.TestCase): From 84fb234cae1bffa944d9f5870c2fc87c75d35261 Mon Sep 17 00:00:00 2001 From: Alex Date: Tue, 24 Jan 2023 22:47:38 -0800 Subject: [PATCH 16/55] fixed additional trigger in FilesPipeline --- scrapy/pipelines/files.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index 4b0c96b25..a5948525a 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -36,13 +36,17 @@ from scrapy.utils.request import referer_str logger = logging.getLogger(__name__) +def _to_string(path: Union[str, os.PathLike]): + return str(path) # convert a Path object to string + + class FileException(Exception): """General media error exception""" class FSFilesStore: def __init__(self, basedir: Union[str, os.PathLike]): - basedir = str(basedir) # support Path object + basedir = _to_string(basedir) if '://' in basedir: basedir = basedir.split('://', 1)[1] self.basedir = basedir @@ -67,7 +71,7 @@ class FSFilesStore: return {'last_modified': last_modified, 'checksum': checksum} def _get_filesystem_path(self, path: Union[str, os.PathLike]) -> Path: - path_comps = str(path).split('/') + path_comps = _to_string(path).split('/') return Path(self.basedir, *path_comps) def _mkdir(self, dirname: Path, domain: Optional[str] = None): @@ -323,12 +327,12 @@ class FilesPipeline(MediaPipeline): DEFAULT_FILES_RESULT_FIELD = 'files' def __init__(self, store_uri, download_func=None, settings=None): + store_uri = _to_string(store_uri) if not store_uri: raise NotConfigured if isinstance(settings, dict) or settings is None: settings = Settings(settings) - cls_name = "FilesPipeline" self.store = self._get_store(store_uri) resolve = functools.partial(self._key_for_pipe, From 44512bebc8bc74164c73be55a8fd5debafb48f10 Mon Sep 17 00:00:00 2001 From: Alex Date: Tue, 24 Jan 2023 22:57:25 -0800 Subject: [PATCH 17/55] fixed mypy warnings with type declaration for os.PathLike --- scrapy/pipelines/files.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index a5948525a..6a32c8b47 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -36,7 +36,7 @@ from scrapy.utils.request import referer_str logger = logging.getLogger(__name__) -def _to_string(path: Union[str, os.PathLike]): +def _to_string(path: Union[str, os.PathLike[str]]): return str(path) # convert a Path object to string @@ -45,7 +45,7 @@ class FileException(Exception): class FSFilesStore: - def __init__(self, basedir: Union[str, os.PathLike]): + def __init__(self, basedir: Union[str, os.PathLike[str]]): basedir = _to_string(basedir) if '://' in basedir: basedir = basedir.split('://', 1)[1] @@ -70,7 +70,7 @@ class FSFilesStore: return {'last_modified': last_modified, 'checksum': checksum} - def _get_filesystem_path(self, path: Union[str, os.PathLike]) -> Path: + def _get_filesystem_path(self, path: Union[str, os.PathLike[str]]) -> Path: path_comps = _to_string(path).split('/') return Path(self.basedir, *path_comps) From 7c753adbe532f1a027d1d285a7c1e4b59fd9e5a8 Mon Sep 17 00:00:00 2001 From: Alex Date: Tue, 24 Jan 2023 23:05:17 -0800 Subject: [PATCH 18/55] revert mypy fix didn't work --- scrapy/pipelines/files.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index 6a32c8b47..a5948525a 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -36,7 +36,7 @@ from scrapy.utils.request import referer_str logger = logging.getLogger(__name__) -def _to_string(path: Union[str, os.PathLike[str]]): +def _to_string(path: Union[str, os.PathLike]): return str(path) # convert a Path object to string @@ -45,7 +45,7 @@ class FileException(Exception): class FSFilesStore: - def __init__(self, basedir: Union[str, os.PathLike[str]]): + def __init__(self, basedir: Union[str, os.PathLike]): basedir = _to_string(basedir) if '://' in basedir: basedir = basedir.split('://', 1)[1] @@ -70,7 +70,7 @@ class FSFilesStore: return {'last_modified': last_modified, 'checksum': checksum} - def _get_filesystem_path(self, path: Union[str, os.PathLike[str]]) -> Path: + def _get_filesystem_path(self, path: Union[str, os.PathLike]) -> Path: path_comps = _to_string(path).split('/') return Path(self.basedir, *path_comps) From 42e8d5a6157a9d2495bd7a2934efe776eee2a7d6 Mon Sep 17 00:00:00 2001 From: Alex Date: Wed, 25 Jan 2023 01:03:27 -0800 Subject: [PATCH 19/55] fixing type declerations --- scrapy/pipelines/files.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index a5948525a..f9dfa53e3 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -9,6 +9,7 @@ import logging import mimetypes import os import time +from os import PathLike from collections import defaultdict from contextlib import suppress from ftplib import FTP @@ -36,7 +37,7 @@ from scrapy.utils.request import referer_str logger = logging.getLogger(__name__) -def _to_string(path: Union[str, os.PathLike]): +def _to_string(path: Union[str, PathLike]): return str(path) # convert a Path object to string @@ -45,7 +46,7 @@ class FileException(Exception): class FSFilesStore: - def __init__(self, basedir: Union[str, os.PathLike]): + def __init__(self, basedir: Union[str, PathLike]): basedir = _to_string(basedir) if '://' in basedir: basedir = basedir.split('://', 1)[1] @@ -70,7 +71,7 @@ class FSFilesStore: return {'last_modified': last_modified, 'checksum': checksum} - def _get_filesystem_path(self, path: Union[str, os.PathLike]) -> Path: + def _get_filesystem_path(self, path: Union[str, PathLike]) -> Path: path_comps = _to_string(path).split('/') return Path(self.basedir, *path_comps) From 0a21a9457b7aeafef3b9ee1c0206546d6c8fb294 Mon Sep 17 00:00:00 2001 From: Alex Date: Thu, 26 Jan 2023 00:50:29 -0800 Subject: [PATCH 20/55] fixed mypy typing error --- scrapy/pipelines/files.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index f9dfa53e3..94b0b1b70 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -37,7 +37,7 @@ from scrapy.utils.request import referer_str logger = logging.getLogger(__name__) -def _to_string(path: Union[str, PathLike]): +def _to_string(path: Union[str, PathLike]) -> str: return str(path) # convert a Path object to string From 94161d101cd47d2dac4be7e8325a24f0ddea86cf Mon Sep 17 00:00:00 2001 From: Alex Date: Thu, 26 Jan 2023 16:41:06 -0800 Subject: [PATCH 21/55] update --- scrapy/pipelines/files.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index 6a32c8b47..afa237b3d 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -36,7 +36,7 @@ from scrapy.utils.request import referer_str logger = logging.getLogger(__name__) -def _to_string(path: Union[str, os.PathLike[str]]): +def _to_string(path: Union[str, os.PathLike]) -> str: return str(path) # convert a Path object to string @@ -45,10 +45,10 @@ class FileException(Exception): class FSFilesStore: - def __init__(self, basedir: Union[str, os.PathLike[str]]): + def __init__(self, basedir: Union[str, os.PathLike]): basedir = _to_string(basedir) - if '://' in basedir: - basedir = basedir.split('://', 1)[1] + if "://" in basedir: + basedir = basedir.split("://", 1)[1] self.basedir = basedir self._mkdir(Path(self.basedir)) self.created_directories: DefaultDict[str, Set[str]] = defaultdict(set) @@ -70,7 +70,7 @@ class FSFilesStore: return {'last_modified': last_modified, 'checksum': checksum} - def _get_filesystem_path(self, path: Union[str, os.PathLike[str]]) -> Path: + def _get_filesystem_path(self, path: Union[str, os.PathLike]) -> Path: path_comps = _to_string(path).split('/') return Path(self.basedir, *path_comps) From a1e2fbafdcaae9b70cf7c4216c5cb80d1df5268e Mon Sep 17 00:00:00 2001 From: Alex Date: Thu, 26 Jan 2023 16:46:08 -0800 Subject: [PATCH 22/55] applied black to tests --- tests/test_pipeline_files.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py index 76c16e57f..1e00e6d1d 100644 --- a/tests/test_pipeline_files.py +++ b/tests/test_pipeline_files.py @@ -484,19 +484,18 @@ class FilesPipelineTestCaseCustomSettings(unittest.TestCase): self.assertEqual(getattr(pipeline_cls, pipe_inst_attr), expected_value) def test_file_pipeline_using_pathlike_objects(self): - class CustomFilesPipelineWithPathLikeDir(FilesPipeline): def file_path(self, request, response=None, info=None, *, item=None): - return Path('subdir') / Path(request.url).name + return Path("subdir") / Path(request.url).name pipeline = CustomFilesPipelineWithPathLikeDir.from_settings( - Settings({'FILES_STORE': Path('./Temp')}) + Settings({"FILES_STORE": Path("./Temp")}) ) request = Request("http://example.com/image01.jpg") - self.assertEqual(pipeline.file_path(request), Path('subdir/image01.jpg')) + self.assertEqual(pipeline.file_path(request), Path("subdir/image01.jpg")) def test_files_store_constructor_with_pathlike_object(self): - path = Path('./FileDir') + path = Path("./FileDir") fs_store = FSFilesStore(path) self.assertEqual(fs_store.basedir, str(path)) From eecc035f4c2ed69cf9dbceae009c583d849b9f2c Mon Sep 17 00:00:00 2001 From: Alex Date: Thu, 2 Feb 2023 11:27:40 -0800 Subject: [PATCH 23/55] correcting type hints --- scrapy/pipelines/files.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index 1b724ce60..fcd9f9078 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -9,11 +9,11 @@ import logging import mimetypes import os import time -from os import PathLike from collections import defaultdict from contextlib import suppress from ftplib import FTP from io import BytesIO +from os import PathLike from pathlib import Path from typing import DefaultDict, Optional, Set, Union from urllib.parse import urlparse @@ -54,12 +54,14 @@ class FSFilesStore: self._mkdir(Path(self.basedir)) self.created_directories: DefaultDict[str, Set[str]] = defaultdict(set) - def persist_file(self, path: str, buf, info, meta=None, headers=None): + def persist_file( + self, path: Union[str, PathLike], buf, info, meta=None, headers=None + ): absolute_path = self._get_filesystem_path(path) self._mkdir(absolute_path.parent, info) absolute_path.write_bytes(buf.getvalue()) - def stat_file(self, path: str, info): + def stat_file(self, path: Union[str, PathLike], info): absolute_path = self._get_filesystem_path(path) try: last_modified = absolute_path.stat().st_mtime From 80a86de5071aef1ac43277c273beace6bd426e80 Mon Sep 17 00:00:00 2001 From: Georgiy Zatserklianyi Date: Wed, 8 Feb 2023 22:40:22 +0200 Subject: [PATCH 24/55] per_slot_settings: test delays increased --- tests/test_downloaderslotssettings.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_downloaderslotssettings.py b/tests/test_downloaderslotssettings.py index a092d01bf..5ee06d5bc 100644 --- a/tests/test_downloaderslotssettings.py +++ b/tests/test_downloaderslotssettings.py @@ -20,11 +20,11 @@ class DownloaderSlotsSettingsTestSpider(MetaSpider): "DOWNLOAD_SLOTS": { 'quotes.toscrape.com': { 'concurrency': 1, - 'delay': 1.5, + 'delay': 2, 'randomize_delay': False }, 'books.toscrape.com': { - 'delay': 2, + 'delay': 3, 'randomize_delay': False } } From c3033a54b1e37287020d8f1f6b541ce4be8d2971 Mon Sep 17 00:00:00 2001 From: Georgiy Zatserklianyi Date: Wed, 8 Feb 2023 23:55:07 +0200 Subject: [PATCH 25/55] per_slot_settings(docs): settings 1 --- docs/topics/settings.rst | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index d636dc301..bde33a0e9 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -771,6 +771,14 @@ The default HTTPS handler uses HTTP/1.1. To use HTTP/2: .. _http2 faq: https://http2.github.io/faq/#does-http2-require-encryption .. _server pushes: https://tools.ietf.org/html/rfc7540#section-8.2 +.. setting:: DOWNLOAD_SLOTS + +DOWNLOAD_SLOTS +---------------- + +Default: ``{}`` + + .. setting:: DOWNLOAD_TIMEOUT DOWNLOAD_TIMEOUT From 5208d436ae26e61b6b00f305079c0a31b8e95063 Mon Sep 17 00:00:00 2001 From: Georgiy Zatserklianyi Date: Thu, 9 Feb 2023 00:33:21 +0200 Subject: [PATCH 26/55] per_slot_settings(docs): settings 2 --- docs/topics/settings.rst | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index bde33a0e9..477c3068f 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -778,6 +778,24 @@ DOWNLOAD_SLOTS Default: ``{}`` +Allows to define concurrency/delay parameters on per slot(domain) basis: + + .. code-block:: python + + DOWNLOAD_SLOTS = { + 'quotes.toscrape.com': { + 'concurrency': 1, + 'delay': 2, + 'randomize_delay': False + }, + 'books.toscrape.com': { + 'delay': 3, + 'randomize_delay': False + } + } +wrapper around :func:`~scrapy.core.downloader.Slot` + +.. autofunction:: scrapy.core.downloader.Slot .. setting:: DOWNLOAD_TIMEOUT From 474087be6fadf2dd3440fc77ab757a18be0931c8 Mon Sep 17 00:00:00 2001 From: Georgiy Zatserklianyi Date: Thu, 9 Feb 2023 14:52:28 +0200 Subject: [PATCH 27/55] per_slot_settings(docs): settings 3 --- docs/topics/settings.rst | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 477c3068f..5acf09efe 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -793,9 +793,15 @@ Allows to define concurrency/delay parameters on per slot(domain) basis: 'randomize_delay': False } } -wrapper around :func:`~scrapy.core.downloader.Slot` -.. autofunction:: scrapy.core.downloader.Slot +.. note:: + + For other downloader slots default settings values will be used: + + - ``DOWNLOAD_DELAY`` -> ``delay`` + - ``CONCURRENT_REQUESTS_PER_DOMAIN`` -> ``concurency`` + - ``RANDOMIZE_DOWNLOAD_DELAY`` -> ``randomize_delay`` + .. setting:: DOWNLOAD_TIMEOUT From 98a5958687b4fae213bf053e560bd18a69312f3f Mon Sep 17 00:00:00 2001 From: Georgiy Zatserklianyi Date: Thu, 16 Feb 2023 14:52:31 +0200 Subject: [PATCH 28/55] per_slot_settings(docs): per slot settings - the highest priority --- scrapy/core/downloader/__init__.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index 1a1eb819e..3691df48a 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -102,13 +102,13 @@ class Downloader: key = self._get_slot_key(request, spider) if key not in self.slots: slot_settings = self.per_slot_settings.get(key, {}) - conc = slot_settings.get( - 'concurrency', ( - self.ip_concurrency if self.ip_concurrency else self.domain_concurrency - ) - ) + conc = ( + self.ip_concurrency if self.ip_concurrency else self.domain_concurrency) conc, delay = _get_concurrency_delay(conc, spider, self.settings) - delay = slot_settings.get('delay', delay) + conc, delay = ( + slot_settings.get('concurrency', conc), + slot_settings.get('delay', delay) + ) randomize_delay = slot_settings.get('randomize_delay', self.randomize_delay) new_slot = Slot(conc, delay, randomize_delay) self.slots[key] = new_slot From 07e1429877e5128786b24ad146dabcad4784ffc2 Mon Sep 17 00:00:00 2001 From: Georgiy Zatserklianyi Date: Thu, 16 Feb 2023 15:14:01 +0200 Subject: [PATCH 29/55] per_slot_settings(docs): docs `:setting` bindings added --- docs/topics/settings.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 5acf09efe..43069c50d 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -798,9 +798,9 @@ Allows to define concurrency/delay parameters on per slot(domain) basis: For other downloader slots default settings values will be used: - - ``DOWNLOAD_DELAY`` -> ``delay`` - - ``CONCURRENT_REQUESTS_PER_DOMAIN`` -> ``concurency`` - - ``RANDOMIZE_DOWNLOAD_DELAY`` -> ``randomize_delay`` + - :setting:`DOWNLOAD_DELAY`: ``delay`` + - :setting:`CONCURRENT_REQUESTS_PER_DOMAIN`: ``concurency`` + - :setting:`RANDOMIZE_DOWNLOAD_DELAY`: ``randomize_delay`` .. setting:: DOWNLOAD_TIMEOUT From 045092e8d753b9689b24db7d30bcbd7f976fb517 Mon Sep 17 00:00:00 2001 From: Georgiy Zatserklianyi Date: Thu, 16 Feb 2023 17:55:59 +0200 Subject: [PATCH 30/55] per_slot_settings(docs): code style fix (from previous pre-commit check) --- docs/topics/settings.rst | 11 ++------ scrapy/core/downloader/__init__.py | 9 ++++--- tests/test_downloaderslotssettings.py | 37 +++++++++++++++------------ 3 files changed, 27 insertions(+), 30 deletions(-) diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 43069c50d..175e8f7f0 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -783,15 +783,8 @@ Allows to define concurrency/delay parameters on per slot(domain) basis: .. code-block:: python DOWNLOAD_SLOTS = { - 'quotes.toscrape.com': { - 'concurrency': 1, - 'delay': 2, - 'randomize_delay': False - }, - 'books.toscrape.com': { - 'delay': 3, - 'randomize_delay': False - } + "quotes.toscrape.com": {"concurrency": 1, "delay": 2, "randomize_delay": False}, + "books.toscrape.com": {"delay": 3, "randomize_delay": False}, } .. note:: diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index 3691df48a..ace483a23 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -103,13 +103,14 @@ class Downloader: if key not in self.slots: slot_settings = self.per_slot_settings.get(key, {}) conc = ( - self.ip_concurrency if self.ip_concurrency else self.domain_concurrency) + self.ip_concurrency if self.ip_concurrency else self.domain_concurrency + ) conc, delay = _get_concurrency_delay(conc, spider, self.settings) conc, delay = ( - slot_settings.get('concurrency', conc), - slot_settings.get('delay', delay) + slot_settings.get("concurrency", conc), + slot_settings.get("delay", delay), ) - randomize_delay = slot_settings.get('randomize_delay', self.randomize_delay) + randomize_delay = slot_settings.get("randomize_delay", self.randomize_delay) new_slot = Slot(conc, delay, randomize_delay) self.slots[key] = new_slot diff --git a/tests/test_downloaderslotssettings.py b/tests/test_downloaderslotssettings.py index 5ee06d5bc..a167161a1 100644 --- a/tests/test_downloaderslotssettings.py +++ b/tests/test_downloaderslotssettings.py @@ -1,31 +1,31 @@ import time +from twisted.internet import defer +from twisted.trial.unittest import TestCase + from scrapy.crawler import CrawlerRunner from scrapy.http import Request from tests.mockserver import MockServer from tests.spiders import MetaSpider -from twisted.internet import defer -from twisted.trial.unittest import TestCase - class DownloaderSlotsSettingsTestSpider(MetaSpider): - name = 'downloader_slots' + name = "downloader_slots" custom_settings = { "DOWNLOAD_DELAY": 1, "RANDOMIZE_DOWNLOAD_DELAY": False, "DOWNLOAD_SLOTS": { - 'quotes.toscrape.com': { - 'concurrency': 1, - 'delay': 2, - 'randomize_delay': False + "quotes.toscrape.com": { + "concurrency": 1, + "delay": 2, + "randomize_delay": False }, - 'books.toscrape.com': { - 'delay': 3, - 'randomize_delay': False + "books.toscrape.com": { + "delay": 3, + "randomize_delay": False } } } @@ -33,21 +33,21 @@ class DownloaderSlotsSettingsTestSpider(MetaSpider): def start_requests(self): self.times = {None: []} - slots = list(self.custom_settings.get('DOWNLOAD_SLOTS', {}).keys()) + [None] + slots = list(self.custom_settings.get("DOWNLOAD_SLOTS", {}).keys()) + [None] for slot in slots: url = self.mockserver.url(f"/?downloader_slot={slot}") self.times[slot] = [] - yield Request(url, callback=self.parse, meta={'download_slot': slot}) + yield Request(url, callback=self.parse, meta={"download_slot": slot}) def parse(self, response): - slot = response.meta.get('download_slot', None) + slot = response.meta.get("download_slot", None) self.times[slot].append(time.time()) url = self.mockserver.url(f"/?downloader_slot={slot}&req=2") - yield Request(url, callback=self.not_parse, meta={'download_slot': slot}) + yield Request(url, callback=self.not_parse, meta={"download_slot": slot}) def not_parse(self, response): - slot = response.meta.get('download_slot', None) + slot = response.meta.get("download_slot", None) self.times[slot].append(time.time()) @@ -70,6 +70,9 @@ class CrawlTestCase(TestCase): tolerance = 0.3 delays_real = {k: v[1] - v[0] for k, v in times.items()} - error_delta = {k: 1 - min(delays_real[k], v.delay) / max(delays_real[k], v.delay) for k, v in slots.items()} + error_delta = { + k: 1 - min(delays_real[k], v.delay) / max(delays_real[k], v.delay) + for k, v in slots.items() + } self.assertTrue(max(list(error_delta.values())) < tolerance) From dd5524eb986c0ef1d6ce6ec2887ed8aabe55f7c9 Mon Sep 17 00:00:00 2001 From: Georgiy Zatserklianyi Date: Thu, 16 Feb 2023 17:59:36 +0200 Subject: [PATCH 31/55] per_slot_settings(docs): code style fix 2 (from previous pre-commit check) --- tests/test_downloaderslotssettings.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/tests/test_downloaderslotssettings.py b/tests/test_downloaderslotssettings.py index a167161a1..b965233a3 100644 --- a/tests/test_downloaderslotssettings.py +++ b/tests/test_downloaderslotssettings.py @@ -21,13 +21,10 @@ class DownloaderSlotsSettingsTestSpider(MetaSpider): "quotes.toscrape.com": { "concurrency": 1, "delay": 2, - "randomize_delay": False + "randomize_delay": False, }, - "books.toscrape.com": { - "delay": 3, - "randomize_delay": False - } - } + "books.toscrape.com": {"delay": 3, "randomize_delay": False}, + }, } def start_requests(self): From 864eee66c77ceff21ba0c8611aa84402f300fc27 Mon Sep 17 00:00:00 2001 From: Georgiy Zatserklianyi Date: Thu, 16 Feb 2023 18:19:36 +0200 Subject: [PATCH 32/55] per_slot_settings(docs): code style fix 3 (from previous pre-commit check) --- tests/test_downloaderslotssettings.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/tests/test_downloaderslotssettings.py b/tests/test_downloaderslotssettings.py index b965233a3..9d4072d19 100644 --- a/tests/test_downloaderslotssettings.py +++ b/tests/test_downloaderslotssettings.py @@ -5,13 +5,11 @@ from twisted.trial.unittest import TestCase from scrapy.crawler import CrawlerRunner from scrapy.http import Request - from tests.mockserver import MockServer from tests.spiders import MetaSpider class DownloaderSlotsSettingsTestSpider(MetaSpider): - name = "downloader_slots" custom_settings = { @@ -49,7 +47,6 @@ class DownloaderSlotsSettingsTestSpider(MetaSpider): class CrawlTestCase(TestCase): - def setUp(self): self.mockserver = MockServer() self.mockserver.__enter__() From 2b3a8f0d6952e20baffbb2118ca82aa1c8c3cde8 Mon Sep 17 00:00:00 2001 From: Oleg Date: Thu, 2 Mar 2023 12:19:41 +0300 Subject: [PATCH 33/55] Fix a typo in the LOG_FORMAT description (#5839) --- docs/topics/settings.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 420e85d37..219509c1e 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -1090,7 +1090,7 @@ LOG_FORMAT Default: ``'%(asctime)s [%(name)s] %(levelname)s: %(message)s'`` String for formatting log messages. Refer to the -:ref:`Python logging documentation ` for the qwhole +:ref:`Python logging documentation ` for the whole list of available placeholders. .. setting:: LOG_DATEFORMAT From 68ba25cb69cb70c0144c9473c1ada46051c7af5b Mon Sep 17 00:00:00 2001 From: Georgiy Zatserklianyi Date: Sat, 4 Mar 2023 22:15:06 +0200 Subject: [PATCH 34/55] periodic log stats: typo in docs fixed --- conftest.py | 2 +- docs/topics/settings.rst | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/conftest.py b/conftest.py index e1d4b1213..01a83d94d 100644 --- a/conftest.py +++ b/conftest.py @@ -20,7 +20,7 @@ collect_ignore = [ *_py_files("tests/CrawlerRunner"), ] -with Path("tests/ignores.txt").open(encoding="utf-8") as reader: +with Path(r"C:/Users/georg/PycharmProjects/scrapy/tests/ignores.txt").open(encoding="utf-8") as reader: for line in reader: file_path = line.strip() if file_path and file_path[0] != "#": diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 175e8f7f0..fbd32ca7a 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -792,7 +792,7 @@ Allows to define concurrency/delay parameters on per slot(domain) basis: For other downloader slots default settings values will be used: - :setting:`DOWNLOAD_DELAY`: ``delay`` - - :setting:`CONCURRENT_REQUESTS_PER_DOMAIN`: ``concurency`` + - :setting:`CONCURRENT_REQUESTS_PER_DOMAIN`: ``concurrency`` - :setting:`RANDOMIZE_DOWNLOAD_DELAY`: ``randomize_delay`` From be52fe4f67647cea9670197feebbcdb05af6cf96 Mon Sep 17 00:00:00 2001 From: Georgiy Zatserklianyi Date: Mon, 6 Mar 2023 16:24:28 +0200 Subject: [PATCH 35/55] Revert "periodic log stats: typo in docs fixed" This reverts commit 68ba25cb69cb70c0144c9473c1ada46051c7af5b. --- conftest.py | 2 +- docs/topics/settings.rst | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/conftest.py b/conftest.py index 01a83d94d..e1d4b1213 100644 --- a/conftest.py +++ b/conftest.py @@ -20,7 +20,7 @@ collect_ignore = [ *_py_files("tests/CrawlerRunner"), ] -with Path(r"C:/Users/georg/PycharmProjects/scrapy/tests/ignores.txt").open(encoding="utf-8") as reader: +with Path("tests/ignores.txt").open(encoding="utf-8") as reader: for line in reader: file_path = line.strip() if file_path and file_path[0] != "#": diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index fbd32ca7a..175e8f7f0 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -792,7 +792,7 @@ Allows to define concurrency/delay parameters on per slot(domain) basis: For other downloader slots default settings values will be used: - :setting:`DOWNLOAD_DELAY`: ``delay`` - - :setting:`CONCURRENT_REQUESTS_PER_DOMAIN`: ``concurrency`` + - :setting:`CONCURRENT_REQUESTS_PER_DOMAIN`: ``concurency`` - :setting:`RANDOMIZE_DOWNLOAD_DELAY`: ``randomize_delay`` From 218829b1db5e086db640a3f2ef3cb8ae76bb66bb Mon Sep 17 00:00:00 2001 From: Georgiy Zatserklianyi Date: Mon, 6 Mar 2023 16:25:49 +0200 Subject: [PATCH 36/55] per slot settings: typo fixed --- docs/topics/settings.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 175e8f7f0..fbd32ca7a 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -792,7 +792,7 @@ Allows to define concurrency/delay parameters on per slot(domain) basis: For other downloader slots default settings values will be used: - :setting:`DOWNLOAD_DELAY`: ``delay`` - - :setting:`CONCURRENT_REQUESTS_PER_DOMAIN`: ``concurency`` + - :setting:`CONCURRENT_REQUESTS_PER_DOMAIN`: ``concurrency`` - :setting:`RANDOMIZE_DOWNLOAD_DELAY`: ``randomize_delay`` From 8aca47e25dd406a39efebc28019db0226b6ec9aa Mon Sep 17 00:00:00 2001 From: Jalil SA Date: Tue, 7 Mar 2023 03:26:59 -0600 Subject: [PATCH 37/55] Add setdefault to BaseSettings (#5821) --- scrapy/settings/__init__.py | 7 +++++++ tests/test_settings/__init__.py | 13 +++++++++++++ 2 files changed, 20 insertions(+) diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py index fde8fdde4..a3b849f7b 100644 --- a/scrapy/settings/__init__.py +++ b/scrapy/settings/__init__.py @@ -293,6 +293,13 @@ class BaseSettings(MutableMapping): else: self.attributes[name].set(value, priority) + def setdefault(self, name, default=None, priority="project"): + if name not in self: + self.set(name, default, priority) + return default + + return self.attributes[name].value + def setdict(self, values, priority="project"): self.update(values, priority) diff --git a/tests/test_settings/__init__.py b/tests/test_settings/__init__.py index 3e8187096..4a577cd8c 100644 --- a/tests/test_settings/__init__.py +++ b/tests/test_settings/__init__.py @@ -65,6 +65,19 @@ class BaseSettingsTest(unittest.TestCase): def setUp(self): self.settings = BaseSettings() + def test_setdefault_not_existing_value(self): + settings = BaseSettings() + value = settings.setdefault("TEST_OPTION", "value") + self.assertEqual(settings["TEST_OPTION"], "value") + self.assertEqual(value, "value") + self.assertIsNotNone(value) + + def test_setdefault_existing_value(self): + settings = BaseSettings({"TEST_OPTION": "value"}) + value = settings.setdefault("TEST_OPTION", None) + self.assertEqual(settings["TEST_OPTION"], "value") + self.assertEqual(value, "value") + def test_set_new_attribute(self): self.settings.set("TEST_OPTION", "value", 0) self.assertIn("TEST_OPTION", self.settings.attributes) From afafc2781af71e0f32c79ede44be3155709dbd36 Mon Sep 17 00:00:00 2001 From: Mohammadtaher Abbasi Date: Fri, 10 Mar 2023 18:50:15 +0330 Subject: [PATCH 38/55] Fix parse command issues with asyncio (#5824) --- scrapy/commands/parse.py | 28 ++++++++- tests/test_command_parse.py | 110 ++++++++++++++++++++++++++++++++++-- 2 files changed, 130 insertions(+), 8 deletions(-) diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py index 9c3fc86d4..ac937e464 100644 --- a/scrapy/commands/parse.py +++ b/scrapy/commands/parse.py @@ -1,3 +1,4 @@ +import inspect import json import logging from typing import Dict @@ -10,7 +11,11 @@ from scrapy.commands import BaseRunSpiderCommand from scrapy.exceptions import UsageError from scrapy.http import Request from scrapy.utils import display -from scrapy.utils.spider import iterate_spider_output, spidercls_for_request +from scrapy.utils.asyncgen import collect_asyncgen +from scrapy.utils.defer import aiter_errback, deferred_from_coro +from scrapy.utils.log import failure_to_exc_info +from scrapy.utils.misc import arg_to_iter +from scrapy.utils.spider import spidercls_for_request logger = logging.getLogger(__name__) @@ -108,6 +113,25 @@ class Command(BaseRunSpiderCommand): max_requests = max(self.requests) return max(max_items, max_requests) + def handle_exception(self, _failure): + logger.error( + "An error is caught while iterating the async iterable", + exc_info=failure_to_exc_info(_failure), + ) + + def iterate_spider_output(self, result): + if inspect.isasyncgen(result): + d = deferred_from_coro( + collect_asyncgen(aiter_errback(result, self.handle_exception)) + ) + d.addCallback(self.iterate_spider_output) + return d + if inspect.iscoroutine(result): + d = deferred_from_coro(result) + d.addCallback(self.iterate_spider_output) + return d + return arg_to_iter(deferred_from_coro(result)) + def add_items(self, lvl, new_items): old_items = self.items.get(lvl, []) self.items[lvl] = old_items + new_items @@ -165,7 +189,7 @@ class Command(BaseRunSpiderCommand): def run_callback(self, response, callback, cb_kwargs=None): cb_kwargs = cb_kwargs or {} - d = maybeDeferred(iterate_spider_output, callback(response, **cb_kwargs)) + d = maybeDeferred(self.iterate_spider_output, callback(response, **cb_kwargs)) return d def get_callback_from_rules(self, spider, response): diff --git a/tests/test_command_parse.py b/tests/test_command_parse.py index b0fb978e9..037333c03 100644 --- a/tests/test_command_parse.py +++ b/tests/test_command_parse.py @@ -30,14 +30,53 @@ import scrapy from scrapy.linkextractors import LinkExtractor from scrapy.spiders import CrawlSpider, Rule from scrapy.utils.test import get_from_asyncio_queue +import asyncio -class AsyncDefAsyncioSpider(scrapy.Spider): - name = 'asyncdef{self.spider_name}' +class AsyncDefAsyncioReturnSpider(scrapy.Spider): + name = "asyncdef_asyncio_return" async def parse(self, response): + await asyncio.sleep(0.2) status = await get_from_asyncio_queue(response.status) - return [scrapy.Item(), dict(foo='bar')] + self.logger.info(f"Got response {{status}}") + return [{{'id': 1}}, {{'id': 2}}] + +class AsyncDefAsyncioReturnSingleElementSpider(scrapy.Spider): + name = "asyncdef_asyncio_return_single_element" + + async def parse(self, response): + await asyncio.sleep(0.1) + status = await get_from_asyncio_queue(response.status) + self.logger.info(f"Got response {{status}}") + return {{'foo': 42}} + +class AsyncDefAsyncioGenLoopSpider(scrapy.Spider): + name = "asyncdef_asyncio_gen_loop" + + async def parse(self, response): + for i in range(10): + await asyncio.sleep(0.1) + yield {{'foo': i}} + self.logger.info(f"Got response {{response.status}}") + +class AsyncDefAsyncioSpider(scrapy.Spider): + name = "asyncdef_asyncio" + + async def parse(self, response): + await asyncio.sleep(0.2) + status = await get_from_asyncio_queue(response.status) + self.logger.debug(f"Got response {{status}}") + +class AsyncDefAsyncioGenExcSpider(scrapy.Spider): + name = "asyncdef_asyncio_gen_exc" + + async def parse(self, response): + for i in range(10): + await asyncio.sleep(0.1) + yield {{'foo': i}} + if i > 5: + raise ValueError("Stopping the processing") class MySpider(scrapy.Spider): name = '{self.spider_name}' @@ -213,17 +252,76 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}} self.assertIn("INFO: It Works!", _textmode(stderr)) @defer.inlineCallbacks - def test_asyncio_parse_items(self): + def test_async_def_asyncio_parse_items_list(self): status, out, stderr = yield self.execute( [ "--spider", - "asyncdef" + self.spider_name, + "asyncdef_asyncio_return", "-c", "parse", self.url("/html"), ] ) - self.assertIn("""[{}, {'foo': 'bar'}]""", _textmode(out)) + self.assertIn("INFO: Got response 200", _textmode(stderr)) + self.assertIn("{'id': 1}", _textmode(out)) + self.assertIn("{'id': 2}", _textmode(out)) + + @defer.inlineCallbacks + def test_async_def_asyncio_parse_items_single_element(self): + status, out, stderr = yield self.execute( + [ + "--spider", + "asyncdef_asyncio_return_single_element", + "-c", + "parse", + self.url("/html"), + ] + ) + self.assertIn("INFO: Got response 200", _textmode(stderr)) + self.assertIn("{'foo': 42}", _textmode(out)) + + @defer.inlineCallbacks + def test_async_def_asyncgen_parse_loop(self): + status, out, stderr = yield self.execute( + [ + "--spider", + "asyncdef_asyncio_gen_loop", + "-c", + "parse", + self.url("/html"), + ] + ) + self.assertIn("INFO: Got response 200", _textmode(stderr)) + for i in range(10): + self.assertIn(f"{{'foo': {i}}}", _textmode(out)) + + @defer.inlineCallbacks + def test_async_def_asyncgen_parse_exc(self): + status, out, stderr = yield self.execute( + [ + "--spider", + "asyncdef_asyncio_gen_exc", + "-c", + "parse", + self.url("/html"), + ] + ) + self.assertIn("ValueError", _textmode(stderr)) + for i in range(7): + self.assertIn(f"{{'foo': {i}}}", _textmode(out)) + + @defer.inlineCallbacks + def test_async_def_asyncio_parse(self): + _, _, stderr = yield self.execute( + [ + "--spider", + "asyncdef_asyncio", + "-c", + "parse", + self.url("/html"), + ] + ) + self.assertIn("DEBUG: Got response 200", _textmode(stderr)) @defer.inlineCallbacks def test_parse_items(self): From 9411cf4e708ea60c7a6972a6804334f2a799e5c6 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade Date: Mon, 13 Mar 2023 20:03:41 +0200 Subject: [PATCH 39/55] Replace deprecated BadZipfile with BadZipFile (#5849) --- scrapy/downloadermiddlewares/decompression.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/downloadermiddlewares/decompression.py b/scrapy/downloadermiddlewares/decompression.py index 368ca60f7..5839dc243 100644 --- a/scrapy/downloadermiddlewares/decompression.py +++ b/scrapy/downloadermiddlewares/decompression.py @@ -51,7 +51,7 @@ class DecompressionMiddleware: archive = BytesIO(response.body) try: zip_file = zipfile.ZipFile(archive) - except zipfile.BadZipfile: + except zipfile.BadZipFile: return namelist = zip_file.namelist() From 101a0c32d71090989be8d37f7b4a464b433705ba Mon Sep 17 00:00:00 2001 From: Alex Date: Tue, 14 Mar 2023 00:13:44 -0700 Subject: [PATCH 40/55] Support genspider with HTTPS (#5808) --- docs/topics/commands.rst | 3 -- scrapy/commands/genspider.py | 16 +++++-- scrapy/templates/spiders/basic.tmpl | 2 +- scrapy/templates/spiders/crawl.tmpl | 2 +- scrapy/templates/spiders/csvfeed.tmpl | 2 +- scrapy/templates/spiders/xmlfeed.tmpl | 2 +- tests/test_commands.py | 63 ++++++++++++++++++++++++--- 7 files changed, 74 insertions(+), 16 deletions(-) diff --git a/docs/topics/commands.rst b/docs/topics/commands.rst index 54fd5d663..106045fc0 100644 --- a/docs/topics/commands.rst +++ b/docs/topics/commands.rst @@ -238,9 +238,6 @@ genspider Create a new spider in the current folder or in the current project's ``spiders`` folder, if called from inside a project. The ```` parameter is set as the spider's ``name``, while ```` is used to generate the ``allowed_domains`` and ``start_urls`` spider's attributes. -.. note:: Even if an HTTPS URL is specified, the protocol used in - ``start_urls`` is always HTTP. This is a known issue: :issue:`3553`. - Usage example:: $ scrapy genspider -l diff --git a/scrapy/commands/genspider.py b/scrapy/commands/genspider.py index c1565a138..68cbe8ff6 100644 --- a/scrapy/commands/genspider.py +++ b/scrapy/commands/genspider.py @@ -31,6 +31,14 @@ def extract_domain(url): return o.netloc +def verify_url_scheme(url): + """Check url for scheme and insert https if none found.""" + parsed = urlparse(url) + if parsed.scheme == "" and parsed.netloc == "": + parsed = urlparse("//" + url)._replace(scheme="https") + return parsed.geturl() + + class Command(ScrapyCommand): requires_project = False default_settings = {"LOG_ENABLED": False} @@ -91,7 +99,7 @@ class Command(ScrapyCommand): raise UsageError() name, url = args[0:2] - domain = extract_domain(url) + url = verify_url_scheme(url) module = sanitize_module_name(name) if self.settings.get("BOT_NAME") == module: @@ -103,18 +111,20 @@ class Command(ScrapyCommand): template_file = self._find_template(opts.template) if template_file: - self._genspider(module, name, domain, opts.template, template_file) + self._genspider(module, name, url, opts.template, template_file) if opts.edit: self.exitcode = os.system(f'scrapy edit "{name}"') - def _genspider(self, module, name, domain, template_name, template_file): + def _genspider(self, module, name, url, template_name, template_file): """Generate the spider module, based on the given template""" capitalized_module = "".join(s.capitalize() for s in module.split("_")) + domain = extract_domain(url) tvars = { "project_name": self.settings.get("BOT_NAME"), "ProjectName": string_camelcase(self.settings.get("BOT_NAME")), "module": module, "name": name, + "url": url, "domain": domain, "classname": f"{capitalized_module}Spider", } diff --git a/scrapy/templates/spiders/basic.tmpl b/scrapy/templates/spiders/basic.tmpl index d3ba19553..20e777271 100644 --- a/scrapy/templates/spiders/basic.tmpl +++ b/scrapy/templates/spiders/basic.tmpl @@ -4,7 +4,7 @@ import scrapy class $classname(scrapy.Spider): name = "$name" allowed_domains = ["$domain"] - start_urls = ["http://$domain/"] + start_urls = ["$url"] def parse(self, response): pass diff --git a/scrapy/templates/spiders/crawl.tmpl b/scrapy/templates/spiders/crawl.tmpl index 2e467e632..36d05e43a 100644 --- a/scrapy/templates/spiders/crawl.tmpl +++ b/scrapy/templates/spiders/crawl.tmpl @@ -6,7 +6,7 @@ from scrapy.spiders import CrawlSpider, Rule class $classname(CrawlSpider): name = "$name" allowed_domains = ["$domain"] - start_urls = ["http://$domain/"] + start_urls = ["$url"] rules = (Rule(LinkExtractor(allow=r"Items/"), callback="parse_item", follow=True),) diff --git a/scrapy/templates/spiders/csvfeed.tmpl b/scrapy/templates/spiders/csvfeed.tmpl index ce9c1dd20..fe96878dc 100644 --- a/scrapy/templates/spiders/csvfeed.tmpl +++ b/scrapy/templates/spiders/csvfeed.tmpl @@ -4,7 +4,7 @@ from scrapy.spiders import CSVFeedSpider class $classname(CSVFeedSpider): name = "$name" allowed_domains = ["$domain"] - start_urls = ["http://$domain/feed.csv"] + start_urls = ["$url"] #headers = ["id", "name", "description", "image_link"] #delimiter = "\t" diff --git a/scrapy/templates/spiders/xmlfeed.tmpl b/scrapy/templates/spiders/xmlfeed.tmpl index 6b50e4cf4..ac62d78d1 100644 --- a/scrapy/templates/spiders/xmlfeed.tmpl +++ b/scrapy/templates/spiders/xmlfeed.tmpl @@ -4,7 +4,7 @@ from scrapy.spiders import XMLFeedSpider class $classname(XMLFeedSpider): name = "$name" allowed_domains = ["$domain"] - start_urls = ["http://$domain/feed.xml"] + start_urls = ["$url"] iterator = "iternodes" # you can change this; see the docs itertag = "item" # change it accordingly diff --git a/tests/test_commands.py b/tests/test_commands.py index 00ddcdd3e..014f50e92 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -541,7 +541,7 @@ class GenspiderCommandTest(CommandTest): ).group(1), ) self.assertEqual( - f"http://{domain}/", + f"https://{domain}", self.find_in_file( Path(self.proj_mod_path, "spiders", "test_name.py"), r"start_urls\s*=\s*\[['\"](.+)['\"]\]", @@ -549,13 +549,64 @@ class GenspiderCommandTest(CommandTest): ) def test_url_schema(self): - self.test_url("http://test.com", "test.com") + self.test_url("https://test.com", "test.com") - def test_url_path(self): - self.test_url("test.com/some/other/page", "test.com") + def test_template_start_urls( + self, url="test.com", expected="https://test.com", template="basic" + ): + self.assertEqual( + 0, self.call("genspider", "-t", template, "--force", "test_name", url) + ) + self.assertEqual( + expected, + self.find_in_file( + Path(self.proj_mod_path, "spiders", "test_name.py"), + r"start_urls\s*=\s*\[['\"](.+)['\"]\]", + ).group(1), + ) - def test_url_schema_path(self): - self.test_url("https://test.com/some/other/page", "test.com") + def test_genspider_basic_start_urls(self): + self.test_template_start_urls("https://test.com", "https://test.com", "basic") + self.test_template_start_urls("http://test.com", "http://test.com", "basic") + self.test_template_start_urls( + "http://test.com/other/path", "http://test.com/other/path", "basic" + ) + self.test_template_start_urls( + "test.com/other/path", "https://test.com/other/path", "basic" + ) + + def test_genspider_crawl_start_urls(self): + self.test_template_start_urls("https://test.com", "https://test.com", "crawl") + self.test_template_start_urls("http://test.com", "http://test.com", "crawl") + self.test_template_start_urls( + "http://test.com/other/path", "http://test.com/other/path", "crawl" + ) + self.test_template_start_urls( + "test.com/other/path", "https://test.com/other/path", "crawl" + ) + self.test_template_start_urls("test.com", "https://test.com", "crawl") + + def test_genspider_xmlfeed_start_urls(self): + self.test_template_start_urls( + "https://test.com/feed.xml", "https://test.com/feed.xml", "xmlfeed" + ) + self.test_template_start_urls( + "http://test.com/feed.xml", "http://test.com/feed.xml", "xmlfeed" + ) + self.test_template_start_urls( + "test.com/feed.xml", "https://test.com/feed.xml", "xmlfeed" + ) + + def test_genspider_csvfeed_start_urls(self): + self.test_template_start_urls( + "https://test.com/feed.csv", "https://test.com/feed.csv", "csvfeed" + ) + self.test_template_start_urls( + "http://test.com/feed.xml", "http://test.com/feed.xml", "csvfeed" + ) + self.test_template_start_urls( + "test.com/feed.csv", "https://test.com/feed.csv", "csvfeed" + ) class GenspiderStandaloneCommandTest(ProjectTest): From d60b4edd11436e61284615ec7ce89f8ac7e46d9a Mon Sep 17 00:00:00 2001 From: auxsvr Date: Tue, 14 Mar 2023 10:31:13 +0200 Subject: [PATCH 41/55] Prevent an edge case that creates an extra event loop (#5832) --- scrapy/utils/reactor.py | 36 +++++++++++++++++++++++++++++------- tests/test_utils_asyncio.py | 11 ++++++++++- 2 files changed, 39 insertions(+), 8 deletions(-) diff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.py index c20948fd3..7f67d036a 100644 --- a/scrapy/utils/reactor.py +++ b/scrapy/utils/reactor.py @@ -1,10 +1,11 @@ import asyncio import sys from contextlib import suppress -from warnings import catch_warnings, filterwarnings +from warnings import catch_warnings, filterwarnings, warn from twisted.internet import asyncioreactor, error +from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.utils.misc import load_object @@ -54,7 +55,31 @@ class CallLaterOnce: return self._func(*self._a, **self._kw) +def set_asyncio_event_loop_policy(): + """The policy functions from asyncio often behave unexpectedly, + so we restrict their use to the absolutely essential case. + This should only be used to install the reactor. + """ + _get_asyncio_event_loop_policy() + + def get_asyncio_event_loop_policy(): + warn( + "Call to deprecated function " + "scrapy.utils.reactor.get_asyncio_event_loop_policy().\n" + "\n" + "Please use get_event_loop, new_event_loop and set_event_loop" + " from asyncio instead, as the corresponding policy methods may lead" + " to unexpected behaviour.\n" + "This function is replaced by set_asyncio_event_loop_policy and" + " is meant to be used only when the reactor is being installed.", + category=ScrapyDeprecationWarning, + stacklevel=2, + ) + return _get_asyncio_event_loop_policy() + + +def _get_asyncio_event_loop_policy(): policy = asyncio.get_event_loop_policy() if ( sys.version_info >= (3, 8) @@ -63,7 +88,6 @@ def get_asyncio_event_loop_policy(): ): policy = asyncio.WindowsSelectorEventLoopPolicy() asyncio.set_event_loop_policy(policy) - return policy @@ -73,6 +97,7 @@ def install_reactor(reactor_path, event_loop_path=None): path if the asyncio reactor is enabled""" reactor_class = load_object(reactor_path) if reactor_class is asyncioreactor.AsyncioSelectorReactor: + set_asyncio_event_loop_policy() with suppress(error.ReactorAlreadyInstalledError): event_loop = set_asyncio_event_loop(event_loop_path) asyncioreactor.install(eventloop=event_loop) @@ -90,7 +115,6 @@ def _get_asyncio_event_loop(): def set_asyncio_event_loop(event_loop_path): """Sets and returns the event loop with specified import path.""" - policy = get_asyncio_event_loop_policy() if event_loop_path is not None: event_loop_class = load_object(event_loop_path) event_loop = event_loop_class() @@ -109,15 +133,13 @@ def set_asyncio_event_loop(event_loop_path): message="There is no current event loop", category=DeprecationWarning, ) - event_loop = policy.get_event_loop() + event_loop = asyncio.get_event_loop() except RuntimeError: # `get_event_loop` raises RuntimeError when called with no asyncio # event loop yet installed in the following scenarios: - # - From a thread other than the main thread. For example, when - # using ``scrapy shell``. # - Previsibly on Python 3.14 and later. # https://github.com/python/cpython/issues/100160#issuecomment-1345581902 - event_loop = policy.new_event_loop() + event_loop = asyncio.new_event_loop() asyncio.set_event_loop(event_loop) return event_loop diff --git a/tests/test_utils_asyncio.py b/tests/test_utils_asyncio.py index 746731a2e..01d0ee043 100644 --- a/tests/test_utils_asyncio.py +++ b/tests/test_utils_asyncio.py @@ -1,9 +1,14 @@ +import asyncio import warnings from unittest import TestCase from pytest import mark -from scrapy.utils.reactor import install_reactor, is_asyncio_reactor_installed +from scrapy.utils.reactor import ( + install_reactor, + is_asyncio_reactor_installed, + set_asyncio_event_loop, +) @mark.usefixtures("reactor_pytest") @@ -23,3 +28,7 @@ class AsyncioTest(TestCase): from twisted.internet import reactor assert original_reactor == reactor + + async def test_set_asyncio_event_loop(self): + install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor") + assert set_asyncio_event_loop() is asyncio.get_running_loop() From 3e59b0805e8f7a9bd89179203a1307f5cb7fc02a Mon Sep 17 00:00:00 2001 From: felipeboffnunes Date: Thu, 16 Mar 2023 10:02:30 -0300 Subject: [PATCH 42/55] #5109 extend contributing.rst --- docs/contributing.rst | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/docs/contributing.rst b/docs/contributing.rst index 6b1a41339..cbfcc3751 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -11,6 +11,19 @@ Contributing to Scrapy There are many ways to contribute to Scrapy. Here are some of them: +* Contribute to Scrapy by fixing currently available issues in the repository. + Scrapy has a list of `good first issues`_ and `help wanted issues`_ that you + can work on. These issues are a great way to get started with contributing to + Scrapy. If you're new to the codebase, you may want to focus on documentation + or testing-related issues, as they are always useful and can help you get + more familiar with the project. You can also check Scrapy's `test coverage`_ + to see which areas may benefit from more tests. + +* Participate in the `Google Summer of Code (GSoC)`_ program. Scrapy is a + mentoring organization for GSoC, and we welcome students who are interested + in contributing to Scrapy. Visit the `GSoC website`_ for more information on + how to participate. + * Blog about Scrapy. Tell the world how you're using Scrapy. This will help newcomers with more examples and will help the Scrapy project to increase its visibility. @@ -314,3 +327,8 @@ And their unit-tests are in:: .. _PEP 257: https://www.python.org/dev/peps/pep-0257/ .. _pull request: https://help.github.com/en/github/collaborating-with-issues-and-pull-requests/creating-a-pull-request .. _pytest-xdist: https://github.com/pytest-dev/pytest-xdist +.. _good first issues: https://github.com/scrapy/scrapy/issues?q=is%3Aissue+is%3Aopen+label%3A%22good+first+issue%22 +.. _help wanted issues: https://github.com/scrapy/scrapy/issues?q=is%3Aissue+is%3Aopen+label%3A%22help+wanted%22 +.. _Google Summer of Code (GSoC): https://summerofcode.withgoogle.com/ +.. _GSoC website: https://gsoc2022.zyte.com/participate +.. _test coverage: https://app.codecov.io/gh/scrapy/scrapy \ No newline at end of file From a81fb5002bec0091976af255d1c8a189f3fbf864 Mon Sep 17 00:00:00 2001 From: felipeboffnunes Date: Thu, 16 Mar 2023 10:23:06 -0300 Subject: [PATCH 43/55] adjustments, lean approach --- docs/contributing.rst | 24 +++++++++++------------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/docs/contributing.rst b/docs/contributing.rst index cbfcc3751..0b1ab74df 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -12,17 +12,7 @@ Contributing to Scrapy There are many ways to contribute to Scrapy. Here are some of them: * Contribute to Scrapy by fixing currently available issues in the repository. - Scrapy has a list of `good first issues`_ and `help wanted issues`_ that you - can work on. These issues are a great way to get started with contributing to - Scrapy. If you're new to the codebase, you may want to focus on documentation - or testing-related issues, as they are always useful and can help you get - more familiar with the project. You can also check Scrapy's `test coverage`_ - to see which areas may benefit from more tests. - -* Participate in the `Google Summer of Code (GSoC)`_ program. Scrapy is a - mentoring organization for GSoC, and we welcome students who are interested - in contributing to Scrapy. Visit the `GSoC website`_ for more information on - how to participate. + See :ref:`contributing-to-scrapy-codebase` below for more information. * Blog about Scrapy. Tell the world how you're using Scrapy. This will help newcomers with more examples and will help the Scrapy project to increase its @@ -41,6 +31,16 @@ There are many ways to contribute to Scrapy. Here are some of them: * Answer Scrapy questions at `Stack Overflow `__. +=============================== +Contributing to Scrapy codebase +=============================== + +Scrapy has a list of `good first issues`_ and `help wanted issues`_ that you +can work on. These issues are a great way to get started with contributing to +Scrapy. If you're new to the codebase, you may want to focus on documentation +or testing-related issues, as they are always useful and can help you get +more familiar with the project. You can also check Scrapy's `test coverage`_ +to see which areas may benefit from more tests. Reporting bugs ============== @@ -329,6 +329,4 @@ And their unit-tests are in:: .. _pytest-xdist: https://github.com/pytest-dev/pytest-xdist .. _good first issues: https://github.com/scrapy/scrapy/issues?q=is%3Aissue+is%3Aopen+label%3A%22good+first+issue%22 .. _help wanted issues: https://github.com/scrapy/scrapy/issues?q=is%3Aissue+is%3Aopen+label%3A%22help+wanted%22 -.. _Google Summer of Code (GSoC): https://summerofcode.withgoogle.com/ -.. _GSoC website: https://gsoc2022.zyte.com/participate .. _test coverage: https://app.codecov.io/gh/scrapy/scrapy \ No newline at end of file From 7bcbfabdbc87aaf53af3216f48bc8524f2453a4d Mon Sep 17 00:00:00 2001 From: felipeboffnunes Date: Thu, 16 Mar 2023 10:28:12 -0300 Subject: [PATCH 44/55] forgot header bind --- docs/contributing.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/contributing.rst b/docs/contributing.rst index 0b1ab74df..823979d0b 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -31,6 +31,8 @@ There are many ways to contribute to Scrapy. Here are some of them: * Answer Scrapy questions at `Stack Overflow `__. +.. _contributing-to-scrapy-codebase: + =============================== Contributing to Scrapy codebase =============================== From ada917307844950a81226f020b596d5932187f6e Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 27 Mar 2023 14:44:44 +0400 Subject: [PATCH 45/55] Don't call SSL_get_server_tmp_key() if not available (#5858) --- scrapy/utils/ssl.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scrapy/utils/ssl.py b/scrapy/utils/ssl.py index f4b598ac7..3ddceea35 100644 --- a/scrapy/utils/ssl.py +++ b/scrapy/utils/ssl.py @@ -20,6 +20,9 @@ def x509name_to_string(x509name): def get_temp_key_info(ssl_object): # adapted from OpenSSL apps/s_cb.c::ssl_print_tmp_key() + if not hasattr(pyOpenSSLutil.lib, "SSL_get_server_tmp_key"): + # removed in cryptography 40.0.0 + return None temp_key_p = pyOpenSSLutil.ffi.new("EVP_PKEY **") if not pyOpenSSLutil.lib.SSL_get_server_tmp_key(ssl_object, temp_key_p): return None From af730df83c74296ec2e831417278b21952b098f7 Mon Sep 17 00:00:00 2001 From: Felipe Boff Nunes Date: Mon, 27 Mar 2023 08:40:17 -0300 Subject: [PATCH 46/55] adjustments --- docs/contributing.rst | 31 +++++++++++-------------------- 1 file changed, 11 insertions(+), 20 deletions(-) diff --git a/docs/contributing.rst b/docs/contributing.rst index 823979d0b..eef92e148 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -11,13 +11,6 @@ Contributing to Scrapy There are many ways to contribute to Scrapy. Here are some of them: -* Contribute to Scrapy by fixing currently available issues in the repository. - See :ref:`contributing-to-scrapy-codebase` below for more information. - -* Blog about Scrapy. Tell the world how you're using Scrapy. This will help - newcomers with more examples and will help the Scrapy project to increase its - visibility. - * Report bugs and request features in the `issue tracker`_, trying to follow the guidelines detailed in `Reporting bugs`_ below. @@ -25,25 +18,16 @@ There are many ways to contribute to Scrapy. Here are some of them: :ref:`writing-patches` and `Submitting patches`_ below for details on how to write and submit a patch. +* Blog about Scrapy. Tell the world how you're using Scrapy. This will help + newcomers with more examples and will help the Scrapy project to increase its + visibility. + * Join the `Scrapy subreddit`_ and share your ideas on how to improve Scrapy. We're always open to suggestions. * Answer Scrapy questions at `Stack Overflow `__. -.. _contributing-to-scrapy-codebase: - -=============================== -Contributing to Scrapy codebase -=============================== - -Scrapy has a list of `good first issues`_ and `help wanted issues`_ that you -can work on. These issues are a great way to get started with contributing to -Scrapy. If you're new to the codebase, you may want to focus on documentation -or testing-related issues, as they are always useful and can help you get -more familiar with the project. You can also check Scrapy's `test coverage`_ -to see which areas may benefit from more tests. - Reporting bugs ============== @@ -95,6 +79,13 @@ guidelines when you're going to report a new bug. Writing patches =============== +Scrapy has a list of `good first issues`_ and `help wanted issues`_ that you +can work on. These issues are a great way to get started with contributing to +Scrapy. If you're new to the codebase, you may want to focus on documentation +or testing-related issues, as they are always useful and can help you get +more familiar with the project. You can also check Scrapy's `test coverage`_ +to see which areas may benefit from more tests. + The better a patch is written, the higher the chances that it'll get accepted and the sooner it will be merged. Well-written patches should: From c22c7bd82be69799d02d5633a2c34657136a4155 Mon Sep 17 00:00:00 2001 From: Jalil SA <61639983+jxlil@users.noreply.github.com> Date: Mon, 27 Mar 2023 20:41:19 -0600 Subject: [PATCH 47/55] fix: rollback GA code --- docs/_templates/layout.html | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/_templates/layout.html b/docs/_templates/layout.html index 18a5231ee..8d11f3a3a 100644 --- a/docs/_templates/layout.html +++ b/docs/_templates/layout.html @@ -3,6 +3,10 @@ {% block footer %} {{ super() }} -{% endblock %} From 46bb7b31d1d4ea351615ae890286e40b6d0a82c4 Mon Sep 17 00:00:00 2001 From: karza_abhishek Date: Tue, 28 Mar 2023 23:23:32 +0530 Subject: [PATCH 49/55] Fixed Docs Makefile to open build/html/index.html in browser(#5878) --- docs/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/Makefile b/docs/Makefile index 596cb6cef..48401bac8 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -87,7 +87,7 @@ coverage: build htmlview: html $(PYTHON) -c "import webbrowser; from pathlib import Path; \ - webbrowser.open('file://' + Path('build/html/index.html').resolve())" + webbrowser.open(Path('build/html/index.html').resolve().as_uri())" clean: -rm -rf build/* From b83fa60a0a76364a64d4036e826a713a9a47f96d Mon Sep 17 00:00:00 2001 From: Prathm-s Date: Thu, 30 Mar 2023 16:57:56 +0530 Subject: [PATCH 50/55] return added to _sent_failed --- scrapy/mail.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scrapy/mail.py b/scrapy/mail.py index 43115c53e..c11f3898d 100644 --- a/scrapy/mail.py +++ b/scrapy/mail.py @@ -164,6 +164,7 @@ class MailSender: "mailerr": errstr, }, ) + return failure def _sendmail(self, to_addrs, msg): from twisted.internet import reactor From 6d94aa061ca96275ab8b83840ac21f0e72ad1583 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20DIDIER?= <73602526+sdidier-dev@users.noreply.github.com> Date: Fri, 31 Mar 2023 11:05:20 +0200 Subject: [PATCH 51/55] Add missing 'crawl' command in crawl examples --- docs/topics/commands.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/topics/commands.rst b/docs/topics/commands.rst index 362190116..c2f2da4e2 100644 --- a/docs/topics/commands.rst +++ b/docs/topics/commands.rst @@ -288,13 +288,13 @@ Usage examples:: $ scrapy crawl myspider [ ... myspider starts crawling ... ] - $ scrapy -o myfile:csv myspider + $ scrapy crawl -o myfile:csv myspider [ ... myspider starts crawling and appends the result to the file myfile in csv format ... ] - $ scrapy -O myfile:json myspider + $ scrapy crawl -O myfile:json myspider [ ... myspider starts crawling and saves the result in myfile in json format overwriting the original content... ] - $ scrapy -o myfile -t csv myspider + $ scrapy crawl -o myfile -t csv myspider [ ... myspider starts crawling and appends the result to the file myfile in csv format ... ] .. command:: check From a0e2e36b52743c9bf1cc03408bf4eb66e68a1edc Mon Sep 17 00:00:00 2001 From: Felipe Boff Nunes Date: Thu, 6 Apr 2023 14:23:19 -0300 Subject: [PATCH 52/55] adjustments --- scrapy/extensions/feedexport.py | 2 -- tests/test_feedexport.py | 22 +++++++++------------- 2 files changed, 9 insertions(+), 15 deletions(-) diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 67ff26764..0df32083f 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -115,7 +115,6 @@ class BlockingFeedStorage: @implementer(IFeedStorage) class StdoutFeedStorage: - def __init__(self, uri, _stdout=None, *, feed_options=None): if not _stdout: _stdout = sys.stdout.buffer @@ -384,7 +383,6 @@ class FeedExporter: return defer.DeferredList(deferred_list) if deferred_list else None def _close_slot(self, slot, spider): - def get_file(slot_): if isinstance(slot_.file, PostProcessingManager): return slot_.file.file diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 45027171f..3124d9d67 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -1640,15 +1640,13 @@ class FeedExportTest(FeedExportTestBase): @defer.inlineCallbacks def test_storage_file_no_postprocessing(self): - @implementer(IFeedStorage) class Storage: - def __init__(self, uri, *, feed_options=None): pass def open(self, spider): - Storage.open_file = tempfile.NamedTemporaryFile(prefix='feed-') + Storage.open_file = tempfile.NamedTemporaryFile(prefix="feed-") return Storage.open_file def store(self, file): @@ -1656,23 +1654,21 @@ class FeedExportTest(FeedExportTestBase): file.close() settings = { - 'FEEDS': {self._random_temp_filename(): {'format': 'jsonlines'}}, - 'FEED_STORAGES': {'file': Storage}, + "FEEDS": {self._random_temp_filename(): {"format": "jsonlines"}}, + "FEED_STORAGES": {"file": Storage}, } yield self.exported_no_data(settings) self.assertIs(Storage.open_file, Storage.store_file) @defer.inlineCallbacks def test_storage_file_postprocessing(self): - @implementer(IFeedStorage) class Storage: - def __init__(self, uri, *, feed_options=None): pass def open(self, spider): - Storage.open_file = tempfile.NamedTemporaryFile(prefix='feed-') + Storage.open_file = tempfile.NamedTemporaryFile(prefix="feed-") return Storage.open_file def store(self, file): @@ -1680,15 +1676,15 @@ class FeedExportTest(FeedExportTestBase): file.close() settings = { - 'FEEDS': { + "FEEDS": { self._random_temp_filename(): { - 'format': 'jsonlines', - 'postprocessing': [ - 'scrapy.extensions.postprocessing.GzipPlugin', + "format": "jsonlines", + "postprocessing": [ + "scrapy.extensions.postprocessing.GzipPlugin", ], }, }, - 'FEED_STORAGES': {'file': Storage}, + "FEED_STORAGES": {"file": Storage}, } yield self.exported_no_data(settings) self.assertIs(Storage.open_file, Storage.store_file) From 98571eb946e24edfe5b520c0478e72b695d09a9d Mon Sep 17 00:00:00 2001 From: Mojtaba Dashtinejad Date: Mon, 10 Apr 2023 14:14:49 +0200 Subject: [PATCH 53/55] Add missing slash in selecting dynamically-loaded content documentation (#5890) --- docs/topics/dynamic-content.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/dynamic-content.rst b/docs/topics/dynamic-content.rst index d01e0a8d4..a0f4b4411 100644 --- a/docs/topics/dynamic-content.rst +++ b/docs/topics/dynamic-content.rst @@ -276,7 +276,7 @@ The following is a simple snippet to illustrate its usage within a Scrapy spider async with async_playwright() as pw: browser = await pw.chromium.launch() page = await browser.new_page() - await page.goto("https:/example.org") + await page.goto("https://example.org") title = await page.title() return {"title": title} From 23017e6e926116162cc8fbdaf38e02670ad5d5cb Mon Sep 17 00:00:00 2001 From: Felipe Boff Nunes Date: Mon, 10 Apr 2023 12:30:00 -0300 Subject: [PATCH 54/55] adjust --- scrapy/extensions/feedexport.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 0df32083f..da1a88299 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -385,6 +385,7 @@ class FeedExporter: def _close_slot(self, slot, spider): def get_file(slot_): if isinstance(slot_.file, PostProcessingManager): + slot_.file.close() return slot_.file.file return slot_.file From c7730627a0f99afccca11437b0775757c50ca9e8 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 11 Apr 2023 15:47:41 +0400 Subject: [PATCH 55/55] Typing improvements for some core components (#5889) --- scrapy/core/downloader/__init__.py | 9 ++-- scrapy/core/engine.py | 68 ++++++++++++++++++------------ scrapy/core/scheduler.py | 31 ++++++++------ scrapy/core/scraper.py | 31 +++++++++----- scrapy/core/spidermw.py | 25 ++++++----- scrapy/crawler.py | 33 +++++++++------ scrapy/exceptions.py | 2 +- scrapy/logformatter.py | 32 +++++++++++--- scrapy/middleware.py | 10 ++--- scrapy/pipelines/__init__.py | 10 +++-- scrapy/signalmanager.py | 23 +++++----- scrapy/statscollectors.py | 65 +++++++++++++++++----------- scrapy/utils/log.py | 5 ++- scrapy/utils/misc.py | 9 +++- scrapy/utils/reactor.py | 18 ++++---- scrapy/utils/signal.py | 9 +++- tox.ini | 12 +++--- 17 files changed, 245 insertions(+), 147 deletions(-) diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index 1b83c3a8a..dde76a547 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -4,8 +4,9 @@ from datetime import datetime from time import time from twisted.internet import defer, task +from twisted.internet.defer import Deferred -from scrapy import signals +from scrapy import Request, Spider, signals from scrapy.core.downloader.handlers import DownloadHandlers from scrapy.core.downloader.middleware import DownloaderMiddlewareManager from scrapy.resolver import dnscache @@ -86,7 +87,7 @@ class Downloader: self._slot_gc_loop.start(60) self.per_slot_settings = self.settings.getdict("DOWNLOAD_SLOTS", {}) - def fetch(self, request, spider): + def fetch(self, request: Request, spider: Spider) -> Deferred: def _deactivate(response): self.active.remove(request) return response @@ -206,12 +207,12 @@ class Downloader: return dfd.addBoth(finish_transferring) - def close(self): + def close(self) -> None: self._slot_gc_loop.stop() for slot in self.slots.values(): slot.close() - def _slot_gc(self, age=60): + def _slot_gc(self, age: float = 60) -> None: mintime = time() - age for key, slot in list(self.slots.items()): if not slot.active and slot.lastseen + slot.delay < mintime: diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index c6738b531..3e5a281b2 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -8,13 +8,16 @@ import logging import warnings from time import time from typing import ( + TYPE_CHECKING, Any, Callable, Generator, Iterable, Iterator, + List, Optional, Set, + Type, Union, cast, ) @@ -28,30 +31,36 @@ from scrapy.core.downloader import Downloader from scrapy.core.scraper import Scraper from scrapy.exceptions import CloseSpider, DontCloseSpider, ScrapyDeprecationWarning from scrapy.http import Request, Response -from scrapy.settings import BaseSettings +from scrapy.logformatter import LogFormatter +from scrapy.settings import BaseSettings, Settings +from scrapy.signalmanager import SignalManager from scrapy.spiders import Spider from scrapy.utils.log import failure_to_exc_info, logformatter_adapter from scrapy.utils.misc import create_instance, load_object from scrapy.utils.reactor import CallLaterOnce +if TYPE_CHECKING: + from scrapy.core.scheduler import BaseScheduler + from scrapy.crawler import Crawler + logger = logging.getLogger(__name__) class Slot: def __init__( self, - start_requests: Iterable, + start_requests: Iterable[Request], close_if_idle: bool, nextcall: CallLaterOnce, - scheduler, + scheduler: "BaseScheduler", ) -> None: self.closing: Optional[Deferred] = None self.inprogress: Set[Request] = set() - self.start_requests: Optional[Iterator] = iter(start_requests) - self.close_if_idle = close_if_idle - self.nextcall = nextcall - self.scheduler = scheduler - self.heartbeat = LoopingCall(nextcall.schedule) + self.start_requests: Optional[Iterator[Request]] = iter(start_requests) + self.close_if_idle: bool = close_if_idle + self.nextcall: CallLaterOnce = nextcall + self.scheduler: "BaseScheduler" = scheduler + self.heartbeat: LoopingCall = LoopingCall(nextcall.schedule) def add_request(self, request: Request) -> None: self.inprogress.add(request) @@ -75,25 +84,28 @@ class Slot: class ExecutionEngine: - def __init__(self, crawler, spider_closed_callback: Callable) -> None: - self.crawler = crawler - self.settings = crawler.settings - self.signals = crawler.signals - self.logformatter = crawler.logformatter + def __init__(self, crawler: "Crawler", spider_closed_callback: Callable) -> None: + self.crawler: "Crawler" = crawler + self.settings: Settings = crawler.settings + self.signals: SignalManager = crawler.signals + self.logformatter: LogFormatter = crawler.logformatter self.slot: Optional[Slot] = None self.spider: Optional[Spider] = None - self.running = False - self.paused = False - self.scheduler_cls = self._get_scheduler_class(crawler.settings) - downloader_cls = load_object(self.settings["DOWNLOADER"]) + self.running: bool = False + self.paused: bool = False + self.scheduler_cls: Type["BaseScheduler"] = self._get_scheduler_class( + crawler.settings + ) + downloader_cls: Type[Downloader] = load_object(self.settings["DOWNLOADER"]) self.downloader: Downloader = downloader_cls(crawler) self.scraper = Scraper(crawler) - self._spider_closed_callback = spider_closed_callback + self._spider_closed_callback: Callable = spider_closed_callback + self.start_time: Optional[float] = None - def _get_scheduler_class(self, settings: BaseSettings) -> type: + def _get_scheduler_class(self, settings: BaseSettings) -> Type["BaseScheduler"]: from scrapy.core.scheduler import BaseScheduler - scheduler_cls = load_object(settings["SCHEDULER"]) + scheduler_cls: Type = load_object(settings["SCHEDULER"]) if not issubclass(scheduler_cls, BaseScheduler): raise TypeError( f"The provided scheduler class ({settings['SCHEDULER']})" @@ -115,7 +127,7 @@ class ExecutionEngine: """Gracefully stop the execution engine""" @inlineCallbacks - def _finish_stopping_engine(_) -> Generator[Deferred, Any, None]: + def _finish_stopping_engine(_: Any) -> Generator[Deferred, Any, None]: yield self.signals.send_catch_log_deferred(signal=signals.engine_stopped) self._closewait.callback(None) @@ -141,7 +153,8 @@ class ExecutionEngine: return self.close_spider( self.spider, reason="shutdown" ) # will also close downloader - return succeed(self.downloader.close()) + self.downloader.close() + return succeed(None) def pause(self) -> None: self.paused = True @@ -209,7 +222,7 @@ class ExecutionEngine: extra={"spider": self.spider}, ) ) - d.addBoth(lambda _: cast(Slot, self.slot).remove_request(request)) + d.addBoth(lambda _: cast(Slot, self.slot).remove_request(request)) # type: ignore[arg-type] d.addErrback( lambda f: logger.info( "Error while removing request from slot", @@ -339,6 +352,7 @@ class ExecutionEngine: if isinstance(result, Response): if result.request is None: result.request = request + assert spider is not None logkws = self.logformatter.crawled(result.request, result, spider) if logkws is not None: logger.log(*logformatter_adapter(logkws), extra={"spider": spider}) @@ -350,10 +364,12 @@ class ExecutionEngine: ) return result - def _on_complete(_): + def _on_complete(_: Any) -> Any: + assert self.slot is not None self.slot.nextcall.schedule() return _ + assert spider is not None dwld = self.downloader.fetch(request, spider) dwld.addCallbacks(_on_success) dwld.addBoth(_on_complete) @@ -362,7 +378,7 @@ class ExecutionEngine: @inlineCallbacks def open_spider( self, spider: Spider, start_requests: Iterable = (), close_if_idle: bool = True - ): + ) -> Generator[Deferred, Any, None]: if self.slot is not None: raise RuntimeError(f"No free spider slot when opening {spider.name!r}") logger.info("Spider opened", extra={"spider": spider}) @@ -471,7 +487,7 @@ class ExecutionEngine: return dfd @property - def open_spiders(self) -> list: + def open_spiders(self) -> List[Spider]: warnings.warn( "ExecutionEngine.open_spiders is deprecated, please use ExecutionEngine.spider instead", category=ScrapyDeprecationWarning, diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.py index 3c46e3a5f..3fb0bbaff 100644 --- a/scrapy/core/scheduler.py +++ b/scrapy/core/scheduler.py @@ -2,13 +2,15 @@ import json import logging from abc import abstractmethod from pathlib import Path -from typing import Optional, Type, TypeVar +from typing import Any, Optional, Type, TypeVar, cast from twisted.internet.defer import Deferred from scrapy.crawler import Crawler +from scrapy.dupefilters import BaseDupeFilter from scrapy.http.request import Request from scrapy.spiders import Spider +from scrapy.statscollectors import StatsCollector from scrapy.utils.job import job_dir from scrapy.utils.misc import create_instance, load_object @@ -20,10 +22,10 @@ class BaseSchedulerMeta(type): Metaclass to check scheduler classes against the necessary interface """ - def __instancecheck__(cls, instance): + def __instancecheck__(cls, instance: Any) -> bool: return cls.__subclasscheck__(type(instance)) - def __subclasscheck__(cls, subclass): + def __subclasscheck__(cls, subclass: type) -> bool: return ( hasattr(subclass, "has_pending_requests") and callable(subclass.has_pending_requests) @@ -168,26 +170,26 @@ class Scheduler(BaseScheduler): def __init__( self, - dupefilter, + dupefilter: BaseDupeFilter, jobdir: Optional[str] = None, dqclass=None, mqclass=None, logunser: bool = False, - stats=None, + stats: Optional[StatsCollector] = None, pqclass=None, crawler: Optional[Crawler] = None, ): - self.df = dupefilter - self.dqdir = self._dqdir(jobdir) + self.df: BaseDupeFilter = dupefilter + self.dqdir: Optional[str] = self._dqdir(jobdir) self.pqclass = pqclass self.dqclass = dqclass self.mqclass = mqclass - self.logunser = logunser - self.stats = stats - self.crawler = crawler + self.logunser: bool = logunser + self.stats: Optional[StatsCollector] = stats + self.crawler: Optional[Crawler] = crawler @classmethod - def from_crawler(cls: Type[SchedulerTV], crawler) -> SchedulerTV: + def from_crawler(cls: Type[SchedulerTV], crawler: Crawler) -> SchedulerTV: """ Factory method, initializes the scheduler with arguments taken from the crawl settings """ @@ -242,6 +244,7 @@ class Scheduler(BaseScheduler): self.df.log(request, self.spider) return False dqok = self._dqpush(request) + assert self.stats is not None if dqok: self.stats.inc_value("scheduler/enqueued/disk", spider=self.spider) else: @@ -259,7 +262,8 @@ class Scheduler(BaseScheduler): Increment the appropriate stats, such as: ``scheduler/dequeued``, ``scheduler/dequeued/disk``, ``scheduler/dequeued/memory``. """ - request = self.mqs.pop() + request: Optional[Request] = self.mqs.pop() + assert self.stats is not None if request is not None: self.stats.inc_value("scheduler/dequeued/memory", spider=self.spider) else: @@ -295,6 +299,7 @@ class Scheduler(BaseScheduler): extra={"spider": self.spider}, ) self.logunser = False + assert self.stats is not None self.stats.inc_value("scheduler/unserializable", spider=self.spider) return False else: @@ -351,7 +356,7 @@ class Scheduler(BaseScheduler): if not path.exists(): return [] with path.open(encoding="utf-8") as f: - return json.load(f) + return cast(list, json.load(f)) def _write_dqs_state(self, dqdir: str, state: list) -> None: with Path(dqdir, "active.json").open("w", encoding="utf-8") as f: diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index 8468b8419..a85f6a661 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -15,6 +15,7 @@ from typing import ( Optional, Set, Tuple, + Type, Union, ) @@ -26,6 +27,9 @@ from scrapy import Spider, signals from scrapy.core.spidermw import SpiderMiddlewareManager from scrapy.exceptions import CloseSpider, DropItem, IgnoreRequest from scrapy.http import Request, Response +from scrapy.logformatter import LogFormatter +from scrapy.pipelines import ItemPipelineManager +from scrapy.signalmanager import SignalManager from scrapy.utils.defer import ( aiter_errback, defer_fail, @@ -96,16 +100,20 @@ class Slot: class Scraper: def __init__(self, crawler: Crawler) -> None: self.slot: Optional[Slot] = None - self.spidermw = SpiderMiddlewareManager.from_crawler(crawler) - itemproc_cls = load_object(crawler.settings["ITEM_PROCESSOR"]) - self.itemproc = itemproc_cls.from_crawler(crawler) - self.concurrent_items = crawler.settings.getint("CONCURRENT_ITEMS") - self.crawler = crawler - self.signals = crawler.signals - self.logformatter = crawler.logformatter + self.spidermw: SpiderMiddlewareManager = SpiderMiddlewareManager.from_crawler( + crawler + ) + itemproc_cls: Type[ItemPipelineManager] = load_object( + crawler.settings["ITEM_PROCESSOR"] + ) + self.itemproc: ItemPipelineManager = itemproc_cls.from_crawler(crawler) + self.concurrent_items: int = crawler.settings.getint("CONCURRENT_ITEMS") + self.crawler: Crawler = crawler + self.signals: SignalManager = crawler.signals + self.logformatter: LogFormatter = crawler.logformatter @inlineCallbacks - def open_spider(self, spider: Spider): + def open_spider(self, spider: Spider) -> Generator[Deferred, Any, None]: """Open the given spider for scraping and allocate resources for it""" self.slot = Slot(self.crawler.settings.getint("SCRAPER_SLOT_MAX_ACTIVE_SIZE")) yield self.itemproc.open_spider(spider) @@ -135,7 +143,8 @@ class Scraper: raise RuntimeError("Scraper slot not assigned") dfd = self.slot.add_response_request(result, request) - def finish_scraping(_): + def finish_scraping(_: Any) -> Any: + assert self.slot is not None self.slot.finish_response(result, request) self._check_if_closing(spider) self._scrape_next(spider) @@ -205,9 +214,9 @@ class Scraper: else: # result is a Failure # TODO: properly type adding this attribute to a Failure result.request = request # type: ignore[attr-defined] - warn_on_generator_with_return_value(spider, request.errback) dfd = defer_fail(result) if request.errback: + warn_on_generator_with_return_value(spider, request.errback) dfd.addErrback(request.errback) return dfd.addCallback(iterate_spider_output) @@ -338,7 +347,7 @@ class Scraper: def _itemproc_finished( self, output: Any, item: Any, response: Response, spider: Spider - ) -> None: + ) -> Deferred: """ItemProcessor finished for the given ``item`` and returned ``output``""" assert self.slot is not None # typing self.slot.itemproc_size -= 1 diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index c21985b18..dcf1a6dbc 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -13,6 +13,8 @@ from typing import ( Callable, Generator, Iterable, + List, + Optional, Tuple, Union, cast, @@ -25,6 +27,7 @@ from scrapy import Request, Spider from scrapy.exceptions import _InvalidOutput from scrapy.http import Response from scrapy.middleware import MiddlewareManager +from scrapy.settings import BaseSettings from scrapy.utils.asyncgen import as_async_generator, collect_asyncgen from scrapy.utils.conf import build_component_list from scrapy.utils.defer import ( @@ -41,22 +44,22 @@ logger = logging.getLogger(__name__) ScrapeFunc = Callable[[Union[Response, Failure], Request, Spider], Any] -def _isiterable(o) -> bool: +def _isiterable(o: Any) -> bool: return isinstance(o, (Iterable, AsyncIterable)) class SpiderMiddlewareManager(MiddlewareManager): component_name = "spider middleware" - def __init__(self, *middlewares): + def __init__(self, *middlewares: Any): super().__init__(*middlewares) self.downgrade_warning_done = False @classmethod - def _get_mwlist_from_settings(cls, settings): + def _get_mwlist_from_settings(cls, settings: BaseSettings) -> List[Any]: return build_component_list(settings.getwithbase("SPIDER_MIDDLEWARES")) - def _add_middleware(self, mw): + def _add_middleware(self, mw: Any) -> None: super()._add_middleware(mw) if hasattr(mw, "process_spider_input"): self.methods["process_spider_input"].append(mw.process_spider_input) @@ -98,7 +101,7 @@ class SpiderMiddlewareManager(MiddlewareManager): exception_processor_index: int, recover_to: Union[MutableChain, MutableAsyncChain], ) -> Union[Generator, AsyncGenerator]: - def process_sync(iterable: Iterable): + def process_sync(iterable: Iterable) -> Generator: try: for r in iterable: yield r @@ -110,7 +113,7 @@ class SpiderMiddlewareManager(MiddlewareManager): raise recover_to.extend(exception_result) - async def process_async(iterable: AsyncIterable): + async def process_async(iterable: AsyncIterable) -> AsyncGenerator: try: async for r in iterable: yield r @@ -280,7 +283,7 @@ class SpiderMiddlewareManager(MiddlewareManager): if isinstance(recovered, AsyncIterable): recovered_collected = await collect_asyncgen(recovered) recovered = MutableChain(recovered_collected) - return MutableChain(result, recovered) # type: ignore[arg-type] + return MutableChain(result, recovered) def scrape_response( self, @@ -306,7 +309,9 @@ class SpiderMiddlewareManager(MiddlewareManager): ) return dfd - def process_start_requests(self, start_requests, spider: Spider) -> Deferred: + def process_start_requests( + self, start_requests: Iterable[Request], spider: Spider + ) -> Deferred: return self._process_chain("process_start_requests", start_requests, spider) # This method is only needed until _async compatibility methods are removed. @@ -314,9 +319,9 @@ class SpiderMiddlewareManager(MiddlewareManager): def _get_async_method_pair( mw: Any, methodname: str ) -> Union[None, Callable, Tuple[Callable, Callable]]: - normal_method = getattr(mw, methodname, None) + normal_method: Optional[Callable] = getattr(mw, methodname, None) methodname_async = methodname + "_async" - async_method = getattr(mw, methodname_async, None) + async_method: Optional[Callable] = getattr(mw, methodname_async, None) if not async_method: return normal_method if not normal_method: diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 397817d6f..256f6e2c5 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -4,7 +4,7 @@ import logging import pprint import signal import warnings -from typing import TYPE_CHECKING, Optional +from typing import TYPE_CHECKING, Optional, Type, Union from twisted.internet import defer from zope.interface.exceptions import DoesNotImplement @@ -22,8 +22,10 @@ from scrapy.core.engine import ExecutionEngine from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.extension import ExtensionManager from scrapy.interfaces import ISpiderLoader +from scrapy.logformatter import LogFormatter from scrapy.settings import Settings, overridden_settings from scrapy.signalmanager import SignalManager +from scrapy.statscollectors import StatsCollector from scrapy.utils.log import ( LogCounterHandler, configure_logging, @@ -49,20 +51,25 @@ logger = logging.getLogger(__name__) class Crawler: - def __init__(self, spidercls, settings=None, init_reactor: bool = False): + def __init__( + self, + spidercls: Type[Spider], + settings: Union[None, dict, Settings] = None, + init_reactor: bool = False, + ): if isinstance(spidercls, Spider): raise ValueError("The spidercls argument must be a class, not an object") if isinstance(settings, dict) or settings is None: settings = Settings(settings) - self.spidercls = spidercls - self.settings = settings.copy() + self.spidercls: Type[Spider] = spidercls + self.settings: Settings = settings.copy() self.spidercls.update_settings(self.settings) - self.signals = SignalManager(self) + self.signals: SignalManager = SignalManager(self) - self.stats = load_object(self.settings["STATS_CLASS"])(self) + self.stats: StatsCollector = load_object(self.settings["STATS_CLASS"])(self) handler = LogCounterHandler(self, level=self.settings.get("LOG_LEVEL")) logging.root.addHandler(handler) @@ -80,8 +87,8 @@ class Crawler: self.__remove_handler = lambda: logging.root.removeHandler(handler) self.signals.connect(self.__remove_handler, signals.engine_stopped) - lf_cls = load_object(self.settings["LOG_FORMATTER"]) - self.logformatter = lf_cls.from_crawler(self) + lf_cls: Type[LogFormatter] = load_object(self.settings["LOG_FORMATTER"]) + self.logformatter: LogFormatter = lf_cls.from_crawler(self) self.request_fingerprinter: RequestFingerprinter = create_instance( load_object(self.settings["REQUEST_FINGERPRINTER_CLASS"]), @@ -89,8 +96,8 @@ class Crawler: crawler=self, ) - reactor_class = self.settings["TWISTED_REACTOR"] - event_loop = self.settings["ASYNCIO_EVENT_LOOP"] + reactor_class: str = self.settings["TWISTED_REACTOR"] + event_loop: str = self.settings["ASYNCIO_EVENT_LOOP"] if init_reactor: # this needs to be done after the spider settings are merged, # but before something imports twisted.internet.reactor @@ -104,11 +111,11 @@ class Crawler: if is_asyncio_reactor_installed() and event_loop: verify_installed_asyncio_event_loop(event_loop) - self.extensions = ExtensionManager.from_crawler(self) + self.extensions: ExtensionManager = ExtensionManager.from_crawler(self) self.settings.freeze() - self.crawling = False - self.spider = None + self.crawling: bool = False + self.spider: Optional[Spider] = None self.engine: Optional[ExecutionEngine] = None @defer.inlineCallbacks diff --git a/scrapy/exceptions.py b/scrapy/exceptions.py index 7f202b8b8..fedd02805 100644 --- a/scrapy/exceptions.py +++ b/scrapy/exceptions.py @@ -39,7 +39,7 @@ class DontCloseSpider(Exception): class CloseSpider(Exception): """Raise this from callbacks to request the spider to be closed""" - def __init__(self, reason="cancelled"): + def __init__(self, reason: str = "cancelled"): super().__init__() self.reason = reason diff --git a/scrapy/logformatter.py b/scrapy/logformatter.py index 560006c95..7cb379b46 100644 --- a/scrapy/logformatter.py +++ b/scrapy/logformatter.py @@ -1,8 +1,11 @@ import logging import os +from typing import Any, Dict, Optional, Union from twisted.python.failure import Failure +from scrapy import Request, Spider +from scrapy.http import Response from scrapy.utils.request import referer_str SCRAPEDMSG = "Scraped from %(src)s" + os.linesep + "%(item)s" @@ -52,7 +55,7 @@ class LogFormatter: } """ - def crawled(self, request, response, spider): + def crawled(self, request: Request, response: Response, spider: Spider) -> dict: """Logs a message when the crawler finds a webpage.""" request_flags = f" {str(request.flags)}" if request.flags else "" response_flags = f" {str(response.flags)}" if response.flags else "" @@ -70,8 +73,11 @@ class LogFormatter: }, } - def scraped(self, item, response, spider): + def scraped( + self, item: Any, response: Union[Response, Failure], spider: Spider + ) -> dict: """Logs a message when an item is scraped by a spider.""" + src: Any if isinstance(response, Failure): src = response.getErrorMessage() else: @@ -85,7 +91,9 @@ class LogFormatter: }, } - def dropped(self, item, exception, response, spider): + def dropped( + self, item: Any, exception: BaseException, response: Response, spider: Spider + ) -> dict: """Logs a message when an item is dropped while it is passing through the item pipeline.""" return { "level": logging.WARNING, @@ -96,7 +104,9 @@ class LogFormatter: }, } - def item_error(self, item, exception, response, spider): + def item_error( + self, item: Any, exception, response: Response, spider: Spider + ) -> dict: """Logs a message when an item causes an error while it is passing through the item pipeline. @@ -110,7 +120,9 @@ class LogFormatter: }, } - def spider_error(self, failure, request, response, spider): + def spider_error( + self, failure: Failure, request: Request, response: Response, spider: Spider + ) -> dict: """Logs an error message from a spider. .. versionadded:: 2.0 @@ -124,13 +136,19 @@ class LogFormatter: }, } - def download_error(self, failure, request, spider, errmsg=None): + def download_error( + self, + failure: Failure, + request: Request, + spider: Spider, + errmsg: Optional[str] = None, + ) -> dict: """Logs a download error message from a spider (typically coming from the engine). .. versionadded:: 2.0 """ - args = {"request": request} + args: Dict[str, Any] = {"request": request} if errmsg: msg = DOWNLOADERRORMSG_LONG args["errmsg"] = errmsg diff --git a/scrapy/middleware.py b/scrapy/middleware.py index f82d722fa..03e92b565 100644 --- a/scrapy/middleware.py +++ b/scrapy/middleware.py @@ -1,7 +1,7 @@ import logging import pprint from collections import defaultdict, deque -from typing import Any, Callable, Deque, Dict, Iterable, Tuple, Union, cast +from typing import Any, Callable, Deque, Dict, Iterable, List, Tuple, Union, cast from twisted.internet.defer import Deferred @@ -30,7 +30,7 @@ class MiddlewareManager: self._add_middleware(mw) @classmethod - def _get_mwlist_from_settings(cls, settings: Settings) -> list: + def _get_mwlist_from_settings(cls, settings: Settings) -> List[Any]: raise NotImplementedError @classmethod @@ -67,17 +67,17 @@ class MiddlewareManager: def from_crawler(cls, crawler): return cls.from_settings(crawler.settings, crawler) - def _add_middleware(self, mw) -> None: + def _add_middleware(self, mw: Any) -> None: if hasattr(mw, "open_spider"): self.methods["open_spider"].append(mw.open_spider) if hasattr(mw, "close_spider"): self.methods["close_spider"].appendleft(mw.close_spider) - def _process_parallel(self, methodname: str, obj, *args) -> Deferred: + def _process_parallel(self, methodname: str, obj: Any, *args: Any) -> Deferred: methods = cast(Iterable[Callable], self.methods[methodname]) return process_parallel(methods, obj, *args) - def _process_chain(self, methodname: str, obj, *args) -> Deferred: + def _process_chain(self, methodname: str, obj: Any, *args: Any) -> Deferred: methods = cast(Iterable[Callable], self.methods[methodname]) return process_chain(methods, obj, *args) diff --git a/scrapy/pipelines/__init__.py b/scrapy/pipelines/__init__.py index df574a0a1..c97d71fb6 100644 --- a/scrapy/pipelines/__init__.py +++ b/scrapy/pipelines/__init__.py @@ -3,7 +3,11 @@ Item pipeline See documentation in docs/item-pipeline.rst """ +from typing import Any, List +from twisted.internet.defer import Deferred + +from scrapy import Spider from scrapy.middleware import MiddlewareManager from scrapy.utils.conf import build_component_list from scrapy.utils.defer import deferred_f_from_coro_f @@ -13,15 +17,15 @@ class ItemPipelineManager(MiddlewareManager): component_name = "item pipeline" @classmethod - def _get_mwlist_from_settings(cls, settings): + def _get_mwlist_from_settings(cls, settings) -> List[Any]: return build_component_list(settings.getwithbase("ITEM_PIPELINES")) - def _add_middleware(self, pipe): + def _add_middleware(self, pipe: Any) -> None: super()._add_middleware(pipe) if hasattr(pipe, "process_item"): self.methods["process_item"].append( deferred_f_from_coro_f(pipe.process_item) ) - def process_item(self, item, spider): + def process_item(self, item: Any, spider: Spider) -> Deferred: return self._process_chain("process_item", item, spider) diff --git a/scrapy/signalmanager.py b/scrapy/signalmanager.py index d7e3bce91..f6df191d8 100644 --- a/scrapy/signalmanager.py +++ b/scrapy/signalmanager.py @@ -1,13 +1,16 @@ +from typing import Any, List, Tuple + from pydispatch import dispatcher +from twisted.internet.defer import Deferred from scrapy.utils import signal as _signal class SignalManager: - def __init__(self, sender=dispatcher.Anonymous): - self.sender = sender + def __init__(self, sender: Any = dispatcher.Anonymous): + self.sender: Any = sender - def connect(self, receiver, signal, **kwargs): + def connect(self, receiver: Any, signal: Any, **kwargs: Any) -> None: """ Connect a receiver function to a signal. @@ -22,18 +25,18 @@ class SignalManager: :type signal: object """ kwargs.setdefault("sender", self.sender) - return dispatcher.connect(receiver, signal, **kwargs) + dispatcher.connect(receiver, signal, **kwargs) - def disconnect(self, receiver, signal, **kwargs): + def disconnect(self, receiver: Any, signal: Any, **kwargs: Any) -> None: """ Disconnect a receiver function from a signal. This has the opposite effect of the :meth:`connect` method, and the arguments are the same. """ kwargs.setdefault("sender", self.sender) - return dispatcher.disconnect(receiver, signal, **kwargs) + dispatcher.disconnect(receiver, signal, **kwargs) - def send_catch_log(self, signal, **kwargs): + def send_catch_log(self, signal: Any, **kwargs: Any) -> List[Tuple[Any, Any]]: """ Send a signal, catch exceptions and log them. @@ -43,7 +46,7 @@ class SignalManager: kwargs.setdefault("sender", self.sender) return _signal.send_catch_log(signal, **kwargs) - def send_catch_log_deferred(self, signal, **kwargs): + def send_catch_log_deferred(self, signal: Any, **kwargs: Any) -> Deferred: """ Like :meth:`send_catch_log` but supports returning :class:`~twisted.internet.defer.Deferred` objects from signal handlers. @@ -57,7 +60,7 @@ class SignalManager: kwargs.setdefault("sender", self.sender) return _signal.send_catch_log_deferred(signal, **kwargs) - def disconnect_all(self, signal, **kwargs): + def disconnect_all(self, signal: Any, **kwargs: Any) -> None: """ Disconnect all receivers from the given signal. @@ -65,4 +68,4 @@ class SignalManager: :type signal: object """ kwargs.setdefault("sender", self.sender) - return _signal.disconnect_all(signal, **kwargs) + _signal.disconnect_all(signal, **kwargs) diff --git a/scrapy/statscollectors.py b/scrapy/statscollectors.py index dd3c32737..15193aac5 100644 --- a/scrapy/statscollectors.py +++ b/scrapy/statscollectors.py @@ -3,44 +3,57 @@ Scrapy extension for collecting scraping stats """ import logging import pprint +from typing import TYPE_CHECKING, Any, Dict, Optional + +from scrapy import Spider + +if TYPE_CHECKING: + from scrapy.crawler import Crawler logger = logging.getLogger(__name__) -class StatsCollector: - def __init__(self, crawler): - self._dump = crawler.settings.getbool("STATS_DUMP") - self._stats = {} +StatsT = Dict[str, Any] - def get_value(self, key, default=None, spider=None): + +class StatsCollector: + def __init__(self, crawler: "Crawler"): + self._dump: bool = crawler.settings.getbool("STATS_DUMP") + self._stats: StatsT = {} + + def get_value( + self, key: str, default: Any = None, spider: Optional[Spider] = None + ) -> Any: return self._stats.get(key, default) - def get_stats(self, spider=None): + def get_stats(self, spider: Optional[Spider] = None) -> StatsT: return self._stats - def set_value(self, key, value, spider=None): + def set_value(self, key: str, value: Any, spider: Optional[Spider] = None) -> None: self._stats[key] = value - def set_stats(self, stats, spider=None): + def set_stats(self, stats: StatsT, spider: Optional[Spider] = None) -> None: self._stats = stats - def inc_value(self, key, count=1, start=0, spider=None): + def inc_value( + self, key: str, count: int = 1, start: int = 0, spider: Optional[Spider] = None + ) -> None: d = self._stats d[key] = d.setdefault(key, start) + count - def max_value(self, key, value, spider=None): + def max_value(self, key: str, value: Any, spider: Optional[Spider] = None) -> None: self._stats[key] = max(self._stats.setdefault(key, value), value) - def min_value(self, key, value, spider=None): + def min_value(self, key: str, value: Any, spider: Optional[Spider] = None) -> None: self._stats[key] = min(self._stats.setdefault(key, value), value) - def clear_stats(self, spider=None): + def clear_stats(self, spider: Optional[Spider] = None) -> None: self._stats.clear() - def open_spider(self, spider): + def open_spider(self, spider: Spider) -> None: pass - def close_spider(self, spider, reason): + def close_spider(self, spider: Spider, reason: str) -> None: if self._dump: logger.info( "Dumping Scrapy stats:\n" + pprint.pformat(self._stats), @@ -48,34 +61,38 @@ class StatsCollector: ) self._persist_stats(self._stats, spider) - def _persist_stats(self, stats, spider): + def _persist_stats(self, stats: StatsT, spider: Spider) -> None: pass class MemoryStatsCollector(StatsCollector): - def __init__(self, crawler): + def __init__(self, crawler: "Crawler"): super().__init__(crawler) - self.spider_stats = {} + self.spider_stats: Dict[str, StatsT] = {} - def _persist_stats(self, stats, spider): + def _persist_stats(self, stats: StatsT, spider: Spider) -> None: self.spider_stats[spider.name] = stats class DummyStatsCollector(StatsCollector): - def get_value(self, key, default=None, spider=None): + def get_value( + self, key: str, default: Any = None, spider: Optional[Spider] = None + ) -> Any: return default - def set_value(self, key, value, spider=None): + def set_value(self, key: str, value: Any, spider: Optional[Spider] = None) -> None: pass - def set_stats(self, stats, spider=None): + def set_stats(self, stats: StatsT, spider: Optional[Spider] = None) -> None: pass - def inc_value(self, key, count=1, start=0, spider=None): + def inc_value( + self, key: str, count: int = 1, start: int = 0, spider: Optional[Spider] = None + ) -> None: pass - def max_value(self, key, value, spider=None): + def max_value(self, key: str, value: Any, spider: Optional[Spider] = None) -> None: pass - def min_value(self, key, value, spider=None): + def min_value(self, key: str, value: Any, spider: Optional[Spider] = None) -> None: pass diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py index 6ae27dc29..2ce4725f4 100644 --- a/scrapy/utils/log.py +++ b/scrapy/utils/log.py @@ -2,6 +2,7 @@ import logging import sys import warnings from logging.config import dictConfig +from typing import Tuple from twisted.python import log as twisted_log from twisted.python.failure import Failure @@ -14,7 +15,7 @@ from scrapy.utils.versions import scrapy_components_versions logger = logging.getLogger(__name__) -def failure_to_exc_info(failure): +def failure_to_exc_info(failure: Failure): """Extract exc_info from Failure instances""" if isinstance(failure, Failure): return (failure.type, failure.value, failure.getTracebackObject()) @@ -206,7 +207,7 @@ class LogCounterHandler(logging.Handler): self.crawler.stats.inc_value(sname) -def logformatter_adapter(logkws): +def logformatter_adapter(logkws: dict) -> Tuple[int, str, dict]: """ Helper that takes the dictionary output from the methods in LogFormatter and adapts it into a tuple of positional arguments for logger.log calls, diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index f9f9c0d5b..d861c9ab6 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -10,6 +10,7 @@ from contextlib import contextmanager from functools import partial from importlib import import_module from pkgutil import iter_modules +from typing import TYPE_CHECKING, Any, Callable, Union from w3lib.html import replace_entities @@ -18,6 +19,10 @@ from scrapy.utils.datatypes import LocalWeakReferencedCache from scrapy.utils.deprecate import ScrapyDeprecationWarning from scrapy.utils.python import flatten, to_unicode +if TYPE_CHECKING: + from scrapy import Spider + + _ITERABLE_SINGLE_VALUES = dict, Item, str, bytes @@ -34,7 +39,7 @@ def arg_to_iter(arg): return [arg] -def load_object(path): +def load_object(path: Union[str, Callable]) -> Any: """Load an object given its absolute object path, and return it. The object can be the import path of a class, function, variable or an @@ -249,7 +254,7 @@ def is_generator_with_return_value(callable): return _generator_callbacks_cache[callable] -def warn_on_generator_with_return_value(spider, callable): +def warn_on_generator_with_return_value(spider: "Spider", callable: Callable) -> None: """ Logs a warning if a callable is a generator function and includes a 'return' statement with a value different than None diff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.py index 7f67d036a..f1b9239e6 100644 --- a/scrapy/utils/reactor.py +++ b/scrapy/utils/reactor.py @@ -1,9 +1,11 @@ import asyncio import sys from contextlib import suppress +from typing import Any, Callable, Dict, Optional, Sequence from warnings import catch_warnings, filterwarnings, warn from twisted.internet import asyncioreactor, error +from twisted.internet.base import DelayedCall from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.utils.misc import load_object @@ -34,23 +36,23 @@ class CallLaterOnce: it hasn't been already scheduled since the last time it ran. """ - def __init__(self, func, *a, **kw): - self._func = func - self._a = a - self._kw = kw - self._call = None + def __init__(self, func: Callable, *a: Any, **kw: Any): + self._func: Callable = func + self._a: Sequence[Any] = a + self._kw: Dict[str, Any] = kw + self._call: Optional[DelayedCall] = None - def schedule(self, delay=0): + def schedule(self, delay: float = 0) -> None: from twisted.internet import reactor if self._call is None: self._call = reactor.callLater(delay, self) - def cancel(self): + def cancel(self) -> None: if self._call: self._call.cancel() - def __call__(self): + def __call__(self) -> Any: self._call = None return self._func(*self._a, **self._kw) diff --git a/scrapy/utils/signal.py b/scrapy/utils/signal.py index b95786d35..9e7ddd827 100644 --- a/scrapy/utils/signal.py +++ b/scrapy/utils/signal.py @@ -1,6 +1,8 @@ """Helper functions for working with signals""" import collections.abc import logging +from typing import Any as TypingAny +from typing import List, Tuple from pydispatch.dispatcher import ( Anonymous, @@ -20,7 +22,9 @@ from scrapy.utils.log import failure_to_exc_info logger = logging.getLogger(__name__) -def send_catch_log(signal=Any, sender=Anonymous, *arguments, **named): +def send_catch_log( + signal=Any, sender=Anonymous, *arguments, **named +) -> List[Tuple[TypingAny, TypingAny]]: """Like pydispatcher.robust.sendRobust but it also logs errors and returns Failures instead of exceptions. """ @@ -32,8 +36,9 @@ def send_catch_log(signal=Any, sender=Anonymous, *arguments, **named): ) dont_log += (StopDownload,) spider = named.get("spider", None) - responses = [] + responses: List[Tuple[TypingAny, TypingAny]] = [] for receiver in liveReceivers(getAllReceivers(sender, signal)): + result: TypingAny try: response = robustApply( receiver, signal=signal, sender=sender, *arguments, **named diff --git a/tox.ini b/tox.ini index 5a9d9cf29..d96a278ea 100644 --- a/tox.ini +++ b/tox.ini @@ -37,13 +37,13 @@ install_command = [testenv:typing] basepython = python3 deps = - mypy==1.0.1 + mypy==1.2.0 types-attrs==19.1.0 - types-lxml==2023.2.11 - types-Pillow==9.4.0.16 - types-Pygments==2.14.0.5 - types-pyOpenSSL==23.0.0.4 - types-setuptools==67.4.0.1 + types-lxml==2023.3.28 + types-Pillow==9.4.0.19 + types-Pygments==2.14.0.7 + types-pyOpenSSL==23.1.0.1 + types-setuptools==67.6.0.7 commands = mypy --show-error-codes {posargs: scrapy tests}