From eea56c4912ebabf94d366b4ead0d0d0a40a1d682 Mon Sep 17 00:00:00 2001 From: GeorgeA92 Date: Sat, 20 Nov 2021 20:13:54 +0200 Subject: [PATCH 01/62] per slot settings: creation of Downloader.Slot objects from per slot settings added --- scrapy/core/downloader/__init__.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index 4f7ab594f..f680b9082 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -83,6 +83,7 @@ class Downloader: self.middleware = DownloaderMiddlewareManager.from_crawler(crawler) self._slot_gc_loop = task.LoopingCall(self._slot_gc) self._slot_gc_loop.start(60) + self.per_slot_settings = self.settings.getdict('PER_SLOT_SETTINGS', {}) def fetch(self, request, spider): def _deactivate(response): @@ -99,9 +100,13 @@ class Downloader: def _get_slot(self, request, spider): key = self._get_slot_key(request, spider) if key not in self.slots: - conc = self.ip_concurrency if self.ip_concurrency else self.domain_concurrency + conc = self.per_slot_settings.get(key,{}).get('concurrency', self.ip_concurrency if self.ip_concurrency else self.domain_concurrency) conc, delay = _get_concurrency_delay(conc, spider, self.settings) - self.slots[key] = Slot(conc, delay, self.randomize_delay) + delay = self.per_slot_settings.get(key,{}).get('delay', delay) + randomize_delay = self.per_slot_settings.get(key, {}).get('randomize_delay', self.randomize_delay) + new_slot = Slot(conc, delay, randomize_delay) + self.slots[key] = new_slot + return key, self.slots[key] From d5acf88ca55988a6c92046da5ef2bf065b0b3e72 Mon Sep 17 00:00:00 2001 From: GeorgeA92 Date: Sat, 20 Nov 2021 20:14:28 +0200 Subject: [PATCH 02/62] per slot settings: logging added (create Slot) --- scrapy/core/downloader/__init__.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index f680b9082..9aab80171 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -1,3 +1,4 @@ +import logging import random from time import time from datetime import datetime @@ -12,6 +13,7 @@ from scrapy import signals from scrapy.core.downloader.middleware import DownloaderMiddlewareManager from scrapy.core.downloader.handlers import DownloadHandlers +logger = logging.getLogger(__name__) class Slot: """Downloader slot""" @@ -106,7 +108,7 @@ class Downloader: randomize_delay = self.per_slot_settings.get(key, {}).get('randomize_delay', self.randomize_delay) new_slot = Slot(conc, delay, randomize_delay) self.slots[key] = new_slot - + logger.debug(f"Downloader slot created {'from per slot settings' if key in self.per_slot_settings.keys() else ''}: {new_slot}") return key, self.slots[key] From 1b7d7ecfcdb6067f3dccd69fc56e25dc1dbf498f Mon Sep 17 00:00:00 2001 From: GeorgeA92 Date: Sat, 20 Nov 2021 21:05:24 +0200 Subject: [PATCH 03/62] per slot settings: logging updated (create/close Slot) --- scrapy/core/downloader/__init__.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index 9aab80171..461b211ab 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -108,7 +108,8 @@ class Downloader: randomize_delay = self.per_slot_settings.get(key, {}).get('randomize_delay', self.randomize_delay) new_slot = Slot(conc, delay, randomize_delay) self.slots[key] = new_slot - logger.debug(f"Downloader slot created {'from per slot settings' if key in self.per_slot_settings.keys() else ''}: {new_slot}") + logger.debug( + f"Downloader slot '{key}' - created {'(from per slot settings)' if key in self.per_slot_settings.keys() else ''}: {new_slot}") return key, self.slots[key] @@ -205,4 +206,7 @@ class Downloader: mintime = time() - age for key, slot in list(self.slots.items()): if not slot.active and slot.lastseen + slot.delay < mintime: - self.slots.pop(key).close() + inactive_slot = self.slots.pop(key) + inactive_slot.close() + logger.debug( + f"Downloader slot '{key}' - closed {'(from per slot settings)' if key in self.per_slot_settings.keys() else ''} : {inactive_slot}") From 8185aa5265c663326474897c24aa0691a2ee0e5c Mon Sep 17 00:00:00 2001 From: GeorgeA92 Date: Sat, 20 Nov 2021 21:06:08 +0200 Subject: [PATCH 04/62] per slot settings: codestyle fix --- scrapy/core/downloader/__init__.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index 461b211ab..d5809ffa0 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -15,6 +15,7 @@ from scrapy.core.downloader.handlers import DownloadHandlers logger = logging.getLogger(__name__) + class Slot: """Downloader slot""" @@ -102,9 +103,9 @@ class Downloader: def _get_slot(self, request, spider): key = self._get_slot_key(request, spider) if key not in self.slots: - conc = self.per_slot_settings.get(key,{}).get('concurrency', self.ip_concurrency if self.ip_concurrency else self.domain_concurrency) + conc = self.per_slot_settings.get(key, {}).get('concurrency', self.ip_concurrency if self.ip_concurrency else self.domain_concurrency) conc, delay = _get_concurrency_delay(conc, spider, self.settings) - delay = self.per_slot_settings.get(key,{}).get('delay', delay) + delay = self.per_slot_settings.get(key, {}).get('delay', delay) randomize_delay = self.per_slot_settings.get(key, {}).get('randomize_delay', self.randomize_delay) new_slot = Slot(conc, delay, randomize_delay) self.slots[key] = new_slot From 115d35270691176e128bd30f6a2a8bcf1289fe5c Mon Sep 17 00:00:00 2001 From: GeorgeA92 Date: Fri, 11 Feb 2022 01:21:31 +0200 Subject: [PATCH 05/62] per slot settings: setting renamed from `PER_SLOT_SETTINGS` to `DOWNLOAD_SLOTS` --- scrapy/core/downloader/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index d5809ffa0..0e32714a8 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -86,7 +86,7 @@ class Downloader: self.middleware = DownloaderMiddlewareManager.from_crawler(crawler) self._slot_gc_loop = task.LoopingCall(self._slot_gc) self._slot_gc_loop.start(60) - self.per_slot_settings = self.settings.getdict('PER_SLOT_SETTINGS', {}) + self.per_slot_settings = self.settings.getdict('DOWNLOAD_SLOTS', {}) def fetch(self, request, spider): def _deactivate(response): From be97402e46c5e4793e23b713728e1777fbe4d49c Mon Sep 17 00:00:00 2001 From: GeorgeA92 Date: Fri, 11 Feb 2022 01:22:27 +0200 Subject: [PATCH 06/62] per slot settings: log notifications (debug) removed --- scrapy/core/downloader/__init__.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index 0e32714a8..53dc546de 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -109,8 +109,6 @@ class Downloader: randomize_delay = self.per_slot_settings.get(key, {}).get('randomize_delay', self.randomize_delay) new_slot = Slot(conc, delay, randomize_delay) self.slots[key] = new_slot - logger.debug( - f"Downloader slot '{key}' - created {'(from per slot settings)' if key in self.per_slot_settings.keys() else ''}: {new_slot}") return key, self.slots[key] @@ -209,5 +207,3 @@ class Downloader: if not slot.active and slot.lastseen + slot.delay < mintime: inactive_slot = self.slots.pop(key) inactive_slot.close() - logger.debug( - f"Downloader slot '{key}' - closed {'(from per slot settings)' if key in self.per_slot_settings.keys() else ''} : {inactive_slot}") From 2006060688976ca469d794cdd3b753a00bfb83c9 Mon Sep 17 00:00:00 2001 From: GeorgeA92 Date: Wed, 11 May 2022 10:29:53 +0300 Subject: [PATCH 07/62] per slot settings: codestyle(flake8) fix, code line length --- scrapy/core/downloader/__init__.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index 53dc546de..511693830 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -103,7 +103,9 @@ class Downloader: def _get_slot(self, request, spider): key = self._get_slot_key(request, spider) if key not in self.slots: - conc = self.per_slot_settings.get(key, {}).get('concurrency', self.ip_concurrency if self.ip_concurrency else self.domain_concurrency) + conc = self.per_slot_settings.get(key, {}).get( + 'concurrency', self.ip_concurrency if self.ip_concurrency else self.domain_concurrency + ) conc, delay = _get_concurrency_delay(conc, spider, self.settings) delay = self.per_slot_settings.get(key, {}).get('delay', delay) randomize_delay = self.per_slot_settings.get(key, {}).get('randomize_delay', self.randomize_delay) From 0ffc52a491e6e6c46196b3aa92767856f64a8ebc Mon Sep 17 00:00:00 2001 From: GeorgeA92 Date: Wed, 11 May 2022 10:40:12 +0300 Subject: [PATCH 08/62] per slot settings: test added (delays for each download slots) --- tests/test_downloaderslotssettings.py | 75 +++++++++++++++++++++++++++ 1 file changed, 75 insertions(+) create mode 100644 tests/test_downloaderslotssettings.py diff --git a/tests/test_downloaderslotssettings.py b/tests/test_downloaderslotssettings.py new file mode 100644 index 000000000..6ea03fb78 --- /dev/null +++ b/tests/test_downloaderslotssettings.py @@ -0,0 +1,75 @@ +import time + +from scrapy.crawler import CrawlerRunner +from scrapy.http import Request + +from tests.mockserver import MockServer +from tests.spiders import MetaSpider + +from twisted.internet import defer +from twisted.trial.unittest import TestCase + + +class DownloaderSlotsSettingsTestSpider(MetaSpider): + + name = 'downloader_slots' + + custom_settings = { + "DOWNLOAD_DELAY": 1, + "RANDOMIZE_DOWNLOAD_DELAY": False, + "DOWNLOAD_SLOTS": { + 'quotes.toscrape.com': { + 'concurrency': 1, + 'delay': 1.5, + 'randomize_delay': False + }, + 'books.toscrape.com': { + 'delay': 2, + 'randomize_delay': False + } + } + } + + def start_requests(self): + self.times = {None: []} + + slots = list(self.custom_settings.get('DOWNLOAD_SLOTS', {}).keys()) + [None] + + for slot in slots: + url = self.mockserver.url(f"/?downloader_slot={slot}") + self.times[slot] = [] + yield Request(url, callback=self.parse, meta={'download_slot': slot}) + + def parse(self, response): + slot = response.meta.get('download_slot', None) + self.times[slot].append(time.time()) + url = self.mockserver.url(f"/?downloader_slot={slot}&req=2") + yield Request(url, callback=self.not_parse, meta={'download_slot': slot}) + + def not_parse(self, response): + slot = response.meta.get('download_slot', None) + self.times[slot].append(time.time()) + + +class CrawlTestCase(TestCase): + + def setUp(self): + self.mockserver = MockServer() + self.mockserver.__enter__() + self.runner = CrawlerRunner() + + def tearDown(self): + self.mockserver.__exit__(None, None, None) + + @defer.inlineCallbacks + def test_delay(self): + crawler = CrawlerRunner().create_crawler(DownloaderSlotsSettingsTestSpider) + yield crawler.crawl(mockserver=self.mockserver) + slots = crawler.engine.downloader.slots + times = crawler.spider.times + tolerance = 0.3 + + delays_real = {k: v[1] - v[0] for k, v in times.items()} + error_delta = {k: 1 - delays_real[k] / v.delay for k, v in slots.items()} + + self.assertTrue(max(list(error_delta.values())) < tolerance) From 4be9c969fd4bdc3206bdabfb5c22e524d9e355ac Mon Sep 17 00:00:00 2001 From: GeorgeA92 Date: Fri, 29 Jul 2022 12:15:39 +0300 Subject: [PATCH 09/62] per slot settings: logger deleted as not used --- scrapy/core/downloader/__init__.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index 511693830..fa8ac01e5 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -1,4 +1,3 @@ -import logging import random from time import time from datetime import datetime @@ -13,8 +12,6 @@ from scrapy import signals from scrapy.core.downloader.middleware import DownloaderMiddlewareManager from scrapy.core.downloader.handlers import DownloadHandlers -logger = logging.getLogger(__name__) - class Slot: """Downloader slot""" From aabdd0b657e1b4398a5a1aad79044401e9dbf909 Mon Sep 17 00:00:00 2001 From: GeorgeA92 Date: Fri, 12 Aug 2022 15:34:48 +0300 Subject: [PATCH 10/62] per slot settings: logger deleted as not used (step 2) --- scrapy/core/downloader/__init__.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index fa8ac01e5..d908f4d7e 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -204,5 +204,4 @@ class Downloader: mintime = time() - age for key, slot in list(self.slots.items()): if not slot.active and slot.lastseen + slot.delay < mintime: - inactive_slot = self.slots.pop(key) - inactive_slot.close() + self.slots.pop(key).close() From 12d52a4f089798f266cbc6f86df3cc9c1cd58257 Mon Sep 17 00:00:00 2001 From: GeorgeA92 Date: Fri, 18 Nov 2022 14:16:18 +0200 Subject: [PATCH 11/62] per slot settings: code optimized --- scrapy/core/downloader/__init__.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index d908f4d7e..f16afe99b 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -100,12 +100,13 @@ class Downloader: def _get_slot(self, request, spider): key = self._get_slot_key(request, spider) if key not in self.slots: - conc = self.per_slot_settings.get(key, {}).get( + slot_settings = self.per_slot_settings.get(key, {}) + conc = slot_settings.get( 'concurrency', self.ip_concurrency if self.ip_concurrency else self.domain_concurrency ) conc, delay = _get_concurrency_delay(conc, spider, self.settings) - delay = self.per_slot_settings.get(key, {}).get('delay', delay) - randomize_delay = self.per_slot_settings.get(key, {}).get('randomize_delay', self.randomize_delay) + delay = slot_settings.get('delay', delay) + randomize_delay = slot_settings.get('randomize_delay', self.randomize_delay) new_slot = Slot(conc, delay, randomize_delay) self.slots[key] = new_slot From 042012f6bdaf4fbd5b978ff7cc9b6796286ac7c4 Mon Sep 17 00:00:00 2001 From: GeorgeA92 Date: Fri, 18 Nov 2022 14:56:29 +0200 Subject: [PATCH 12/62] per slot settings: error calculation metho updated --- tests/test_downloaderslotssettings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_downloaderslotssettings.py b/tests/test_downloaderslotssettings.py index 6ea03fb78..a092d01bf 100644 --- a/tests/test_downloaderslotssettings.py +++ b/tests/test_downloaderslotssettings.py @@ -70,6 +70,6 @@ class CrawlTestCase(TestCase): tolerance = 0.3 delays_real = {k: v[1] - v[0] for k, v in times.items()} - error_delta = {k: 1 - delays_real[k] / v.delay for k, v in slots.items()} + error_delta = {k: 1 - min(delays_real[k], v.delay) / max(delays_real[k], v.delay) for k, v in slots.items()} self.assertTrue(max(list(error_delta.values())) < tolerance) From b1dd893fbb4d2efed3342e6c98a6bbf4b393d48e Mon Sep 17 00:00:00 2001 From: Alex Date: Fri, 20 Jan 2023 02:17:02 -0800 Subject: [PATCH 13/62] Support Path Objects Issue #5739 --- scrapy/pipelines/files.py | 9 +++++---- tests/test_pipeline_files.py | 17 +++++++++++++++++ 2 files changed, 22 insertions(+), 4 deletions(-) diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index 51aedafe8..4b0c96b25 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -14,7 +14,7 @@ from contextlib import suppress from ftplib import FTP from io import BytesIO from pathlib import Path -from typing import DefaultDict, Optional, Set +from typing import DefaultDict, Optional, Set, Union from urllib.parse import urlparse from itemadapter import ItemAdapter @@ -41,7 +41,8 @@ class FileException(Exception): class FSFilesStore: - def __init__(self, basedir: str): + def __init__(self, basedir: Union[str, os.PathLike]): + basedir = str(basedir) # support Path object if '://' in basedir: basedir = basedir.split('://', 1)[1] self.basedir = basedir @@ -65,8 +66,8 @@ class FSFilesStore: return {'last_modified': last_modified, 'checksum': checksum} - def _get_filesystem_path(self, path: str) -> Path: - path_comps = path.split('/') + def _get_filesystem_path(self, path: Union[str, os.PathLike]) -> Path: + path_comps = str(path).split('/') return Path(self.basedir, *path_comps) def _mkdir(self, dirname: Path, domain: Optional[str] = None): diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py index 4acd29bf7..5280ab0d2 100644 --- a/tests/test_pipeline_files.py +++ b/tests/test_pipeline_files.py @@ -405,6 +405,23 @@ class FilesPipelineTestCaseCustomSettings(unittest.TestCase): self.assertEqual(getattr(pipeline_cls, pipe_inst_attr), expected_value) + def test_file_pipeline_using_pathlike_objects(self): + + class CustomFilesPipelineWithPathLikeDir(FilesPipeline): + def file_path(self, request, response=None, info=None, *, item=None): + return Path('subdir') / Path(request.url).name + + pipeline = CustomFilesPipelineWithPathLikeDir.from_settings( + Settings({'FILES_STORE': Path('./Temp')}) + ) + request = Request("http://example.com/image01.jpg") + self.assertEqual(pipeline.file_path(request), Path('subdir/image01.jpg')) + + def test_files_store_constructor_with_pathlike_object(self): + path = Path('./FileDir') + fs_store = FSFilesStore(path) + self.assertEqual(fs_store.basedir, str(path)) + class TestS3FilesStore(unittest.TestCase): From 84fb234cae1bffa944d9f5870c2fc87c75d35261 Mon Sep 17 00:00:00 2001 From: Alex Date: Tue, 24 Jan 2023 22:47:38 -0800 Subject: [PATCH 14/62] fixed additional trigger in FilesPipeline --- scrapy/pipelines/files.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index 4b0c96b25..a5948525a 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -36,13 +36,17 @@ from scrapy.utils.request import referer_str logger = logging.getLogger(__name__) +def _to_string(path: Union[str, os.PathLike]): + return str(path) # convert a Path object to string + + class FileException(Exception): """General media error exception""" class FSFilesStore: def __init__(self, basedir: Union[str, os.PathLike]): - basedir = str(basedir) # support Path object + basedir = _to_string(basedir) if '://' in basedir: basedir = basedir.split('://', 1)[1] self.basedir = basedir @@ -67,7 +71,7 @@ class FSFilesStore: return {'last_modified': last_modified, 'checksum': checksum} def _get_filesystem_path(self, path: Union[str, os.PathLike]) -> Path: - path_comps = str(path).split('/') + path_comps = _to_string(path).split('/') return Path(self.basedir, *path_comps) def _mkdir(self, dirname: Path, domain: Optional[str] = None): @@ -323,12 +327,12 @@ class FilesPipeline(MediaPipeline): DEFAULT_FILES_RESULT_FIELD = 'files' def __init__(self, store_uri, download_func=None, settings=None): + store_uri = _to_string(store_uri) if not store_uri: raise NotConfigured if isinstance(settings, dict) or settings is None: settings = Settings(settings) - cls_name = "FilesPipeline" self.store = self._get_store(store_uri) resolve = functools.partial(self._key_for_pipe, From 44512bebc8bc74164c73be55a8fd5debafb48f10 Mon Sep 17 00:00:00 2001 From: Alex Date: Tue, 24 Jan 2023 22:57:25 -0800 Subject: [PATCH 15/62] fixed mypy warnings with type declaration for os.PathLike --- scrapy/pipelines/files.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index a5948525a..6a32c8b47 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -36,7 +36,7 @@ from scrapy.utils.request import referer_str logger = logging.getLogger(__name__) -def _to_string(path: Union[str, os.PathLike]): +def _to_string(path: Union[str, os.PathLike[str]]): return str(path) # convert a Path object to string @@ -45,7 +45,7 @@ class FileException(Exception): class FSFilesStore: - def __init__(self, basedir: Union[str, os.PathLike]): + def __init__(self, basedir: Union[str, os.PathLike[str]]): basedir = _to_string(basedir) if '://' in basedir: basedir = basedir.split('://', 1)[1] @@ -70,7 +70,7 @@ class FSFilesStore: return {'last_modified': last_modified, 'checksum': checksum} - def _get_filesystem_path(self, path: Union[str, os.PathLike]) -> Path: + def _get_filesystem_path(self, path: Union[str, os.PathLike[str]]) -> Path: path_comps = _to_string(path).split('/') return Path(self.basedir, *path_comps) From 7c753adbe532f1a027d1d285a7c1e4b59fd9e5a8 Mon Sep 17 00:00:00 2001 From: Alex Date: Tue, 24 Jan 2023 23:05:17 -0800 Subject: [PATCH 16/62] revert mypy fix didn't work --- scrapy/pipelines/files.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index 6a32c8b47..a5948525a 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -36,7 +36,7 @@ from scrapy.utils.request import referer_str logger = logging.getLogger(__name__) -def _to_string(path: Union[str, os.PathLike[str]]): +def _to_string(path: Union[str, os.PathLike]): return str(path) # convert a Path object to string @@ -45,7 +45,7 @@ class FileException(Exception): class FSFilesStore: - def __init__(self, basedir: Union[str, os.PathLike[str]]): + def __init__(self, basedir: Union[str, os.PathLike]): basedir = _to_string(basedir) if '://' in basedir: basedir = basedir.split('://', 1)[1] @@ -70,7 +70,7 @@ class FSFilesStore: return {'last_modified': last_modified, 'checksum': checksum} - def _get_filesystem_path(self, path: Union[str, os.PathLike[str]]) -> Path: + def _get_filesystem_path(self, path: Union[str, os.PathLike]) -> Path: path_comps = _to_string(path).split('/') return Path(self.basedir, *path_comps) From 42e8d5a6157a9d2495bd7a2934efe776eee2a7d6 Mon Sep 17 00:00:00 2001 From: Alex Date: Wed, 25 Jan 2023 01:03:27 -0800 Subject: [PATCH 17/62] fixing type declerations --- scrapy/pipelines/files.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index a5948525a..f9dfa53e3 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -9,6 +9,7 @@ import logging import mimetypes import os import time +from os import PathLike from collections import defaultdict from contextlib import suppress from ftplib import FTP @@ -36,7 +37,7 @@ from scrapy.utils.request import referer_str logger = logging.getLogger(__name__) -def _to_string(path: Union[str, os.PathLike]): +def _to_string(path: Union[str, PathLike]): return str(path) # convert a Path object to string @@ -45,7 +46,7 @@ class FileException(Exception): class FSFilesStore: - def __init__(self, basedir: Union[str, os.PathLike]): + def __init__(self, basedir: Union[str, PathLike]): basedir = _to_string(basedir) if '://' in basedir: basedir = basedir.split('://', 1)[1] @@ -70,7 +71,7 @@ class FSFilesStore: return {'last_modified': last_modified, 'checksum': checksum} - def _get_filesystem_path(self, path: Union[str, os.PathLike]) -> Path: + def _get_filesystem_path(self, path: Union[str, PathLike]) -> Path: path_comps = _to_string(path).split('/') return Path(self.basedir, *path_comps) From 73f697f1db835be09bf5b717c25f3565d0c054b1 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 23 Jan 2023 01:10:06 +0400 Subject: [PATCH 18/62] Re-enable follow_imports. --- setup.cfg | 1 - 1 file changed, 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 1fab6fe22..c711e6e72 100644 --- a/setup.cfg +++ b/setup.cfg @@ -6,7 +6,6 @@ universal=1 [mypy] ignore_missing_imports = true -follow_imports = skip # FIXME: remove the following sections once the issues are solved From c0ea7fd4fd2726494adf6a84a9fd6b6d7fcf116f Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 25 Jan 2023 22:14:24 +0400 Subject: [PATCH 19/62] Remove obsolete top-level run code in tests. --- tests/test_dependencies.py | 4 ---- tests/test_utils_iterators.py | 4 ---- tests/test_utils_python.py | 4 ---- 3 files changed, 12 deletions(-) diff --git a/tests/test_dependencies.py b/tests/test_dependencies.py index f9b2d853c..a39ed0694 100644 --- a/tests/test_dependencies.py +++ b/tests/test_dependencies.py @@ -37,7 +37,3 @@ class ScrapyUtilsTest(unittest.TestCase): pinned_twisted_version_string = match[1] self.assertEqual(twisted_version.short(), pinned_twisted_version_string) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py index 893582a32..778f7162d 100644 --- a/tests/test_utils_iterators.py +++ b/tests/test_utils_iterators.py @@ -528,7 +528,3 @@ class TestHelper(unittest.TestCase): type(a) is type(b), f"Got {type(a)}, expected {type(b)} for { obj!r}" ) self.assertEqual(a, b) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py index 5caa5b8f2..8d2695b0d 100644 --- a/tests/test_utils_python.py +++ b/tests/test_utils_python.py @@ -258,7 +258,3 @@ class UtilsPythonTestCase(unittest.TestCase): without_none_values({"one": 1, "none": None, "three": 3, "four": 4}), {"one": 1, "three": 3, "four": 4}, ) - - -if __name__ == "__main__": - unittest.main() From 2e33fb812b227b8f5ded8be9913467c31929a87b Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 25 Jan 2023 22:15:07 +0400 Subject: [PATCH 20/62] Silence improper typing of twisted.internet.reactor in tests. --- scrapy/utils/benchserver.py | 6 +++--- tests/CrawlerRunner/ip_address.py | 6 +++--- tests/mockserver.py | 13 ++++++++----- tests/test_engine.py | 2 +- 4 files changed, 15 insertions(+), 12 deletions(-) diff --git a/scrapy/utils/benchserver.py b/scrapy/utils/benchserver.py index 32bc2e38c..750d3c093 100644 --- a/scrapy/utils/benchserver.py +++ b/scrapy/utils/benchserver.py @@ -37,11 +37,11 @@ if __name__ == "__main__": root = Root() factory = Site(root) - httpPort = reactor.listenTCP(8998, Site(root)) + httpPort = reactor.listenTCP(8998, Site(root)) # type: ignore[attr-defined] def _print_listening(): httpHost = httpPort.getHost() print(f"Bench server at http://{httpHost.host}:{httpHost.port}") - reactor.callWhenRunning(_print_listening) - reactor.run() + reactor.callWhenRunning(_print_listening) # type: ignore[attr-defined] + reactor.run() # type: ignore[attr-defined] diff --git a/tests/CrawlerRunner/ip_address.py b/tests/CrawlerRunner/ip_address.py index 26db16dd6..b9a4485a9 100644 --- a/tests/CrawlerRunner/ip_address.py +++ b/tests/CrawlerRunner/ip_address.py @@ -41,10 +41,10 @@ if __name__ == "__main__": url = f"http://not.a.real.domain:{port}/echo" servers = [(mock_dns_server.host, mock_dns_server.port)] - reactor.installResolver(createResolver(servers=servers)) + reactor.installResolver(createResolver(servers=servers)) # type: ignore[attr-defined] configure_logging() runner = CrawlerRunner() d = runner.crawl(LocalhostSpider, url=url) - d.addBoth(lambda _: reactor.stop()) - reactor.run() + d.addBoth(lambda _: reactor.stop()) # type: ignore[attr-defined] + reactor.run() # type: ignore[attr-defined] diff --git a/tests/mockserver.py b/tests/mockserver.py index e07ae8797..185897373 100644 --- a/tests/mockserver.py +++ b/tests/mockserver.py @@ -10,6 +10,7 @@ from urllib.parse import urlencode from OpenSSL import SSL from twisted.internet import defer, reactor, ssl +from twisted.internet.protocol import ServerFactory from twisted.internet.task import deferLater from twisted.names import dns, error from twisted.names.server import DNSServerFactory @@ -369,12 +370,14 @@ if __name__ == "__main__": ) args = parser.parse_args() + factory: ServerFactory + if args.type == "http": root = Root() factory = Site(root) - httpPort = reactor.listenTCP(0, factory) + httpPort = reactor.listenTCP(0, factory) # type: ignore[attr-defined] contextFactory = ssl_context_factory() - httpsPort = reactor.listenSSL(0, factory, contextFactory) + httpsPort = reactor.listenSSL(0, factory, contextFactory) # type: ignore[attr-defined] def print_listening(): httpHost = httpPort.getHost() @@ -388,11 +391,11 @@ if __name__ == "__main__": clients = [MockDNSResolver()] factory = DNSServerFactory(clients=clients) protocol = dns.DNSDatagramProtocol(controller=factory) - listener = reactor.listenUDP(0, protocol) + listener = reactor.listenUDP(0, protocol) # type: ignore[attr-defined] def print_listening(): host = listener.getHost() print(f"{host.host}:{host.port}") - reactor.callWhenRunning(print_listening) - reactor.run() + reactor.callWhenRunning(print_listening) # type: ignore[attr-defined] + reactor.run() # type: ignore[attr-defined] diff --git a/tests/test_engine.py b/tests/test_engine.py index 7ddb420ba..5fb87424b 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -557,4 +557,4 @@ class EngineTest(unittest.TestCase): if __name__ == "__main__": if len(sys.argv) > 1 and sys.argv[1] == "runserver": start_test_site(debug=True) - reactor.run() + reactor.run() # type: ignore[attr-defined] From afd5d8532093c25a6ed4453e3a10b275335c94bd Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 23 Jan 2023 01:12:59 +0400 Subject: [PATCH 21/62] Fix remaining issues in tests. --- tests/test_downloadermiddleware_httpproxy.py | 2 +- tests/test_http2_client_protocol.py | 5 ++++- tests/test_spidermiddleware.py | 1 + 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/test_downloadermiddleware_httpproxy.py b/tests/test_downloadermiddleware_httpproxy.py index ca125ba36..0040f8d23 100644 --- a/tests/test_downloadermiddleware_httpproxy.py +++ b/tests/test_downloadermiddleware_httpproxy.py @@ -14,7 +14,7 @@ spider = Spider("foo") class TestHttpProxyMiddleware(TestCase): - failureException = AssertionError + failureException = AssertionError # type: ignore[assignment] def setUp(self): self._oldenv = os.environ.copy() diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py index 88345d2bc..2a928eea0 100644 --- a/tests/test_http2_client_protocol.py +++ b/tests/test_http2_client_protocol.py @@ -5,6 +5,7 @@ import shutil import string from ipaddress import IPv4Address from pathlib import Path +from typing import Dict from unittest import mock, skipIf from urllib.parse import urlencode @@ -86,6 +87,7 @@ class GetDataHtmlLarge(LeafResource): class PostDataJsonMixin: @staticmethod def make_response(request: TxRequest, extra_data: str): + assert request.content is not None response = { "request-headers": {}, "request-body": json.loads(request.content.read()), @@ -144,7 +146,8 @@ class QueryParams(LeafResource): request.setHeader("Content-Type", "application/json; charset=UTF-8") request.setHeader("Content-Encoding", "UTF-8") - query_params = {} + query_params: Dict[str, str] = {} + assert request.args is not None for k, v in request.args.items(): query_params[str(k, "utf-8")] = str(v[0], "utf-8") diff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.py index 760ee43df..ca6348913 100644 --- a/tests/test_spidermiddleware.py +++ b/tests/test_spidermiddleware.py @@ -112,6 +112,7 @@ class BaseAsyncSpiderMiddlewareTestCase(SpiderMiddlewareTestCase): Should work for process_spider_output and, when it's supported, process_start_requests. """ + ITEM_TYPE: type RESULT_COUNT = 3 # to simplify checks, let everything return 3 objects @staticmethod From 232aab53b38553262d60259461adf3fa8054e445 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 25 Jan 2023 22:28:15 +0400 Subject: [PATCH 22/62] Fix most of the new typing issues in utils. --- scrapy/utils/defer.py | 19 +++++++++++-------- scrapy/utils/response.py | 7 ++++--- scrapy/utils/testsite.py | 4 ++-- 3 files changed, 17 insertions(+), 13 deletions(-) diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py index 21cd5e78f..8fee5a7ed 100644 --- a/scrapy/utils/defer.py +++ b/scrapy/utils/defer.py @@ -10,6 +10,7 @@ from typing import ( AsyncGenerator, AsyncIterable, Callable, + cast, Coroutine, Generator, Iterable, @@ -38,8 +39,8 @@ def defer_fail(_failure: Failure) -> Deferred: """ from twisted.internet import reactor - d = Deferred() - reactor.callLater(0.1, d.errback, _failure) + d: Deferred = Deferred() + reactor.callLater(0.1, d.errback, _failure) # type: ignore[attr-defined] return d @@ -52,8 +53,8 @@ def defer_succeed(result) -> Deferred: """ from twisted.internet import reactor - d = Deferred() - reactor.callLater(0.1, d.callback, result) + d: Deferred = Deferred() + reactor.callLater(0.1, d.callback, result) # type: ignore[attr-defined] return d @@ -182,7 +183,9 @@ class _AsyncCooperatorAdapter(Iterator): def _call_anext(self) -> None: # This starts waiting for the next result from aiterator. # If aiterator is exhausted, _errback will be called. - self.anext_deferred = deferred_from_coro(self.aiterator.__anext__()) + self.anext_deferred = cast( + Deferred, deferred_from_coro(self.aiterator.__anext__()) + ) self.anext_deferred.addCallbacks(self._callback, self._errback) def __next__(self) -> Deferred: @@ -190,7 +193,7 @@ class _AsyncCooperatorAdapter(Iterator): # It also calls __anext__() if needed. if self.finished: raise StopIteration - d = Deferred() + d: Deferred = Deferred() self.waiting_deferreds.append(d) if not self.anext_deferred: self._call_anext() @@ -209,7 +212,7 @@ def parallel_async( def process_chain(callbacks: Iterable[Callable], input, *a, **kw) -> Deferred: """Return a Deferred built by chaining the given callbacks""" - d = Deferred() + d: Deferred = Deferred() for x in callbacks: d.addCallback(x, *a, **kw) d.callback(input) @@ -220,7 +223,7 @@ def process_chain_both( callbacks: Iterable[Callable], errbacks: Iterable[Callable], input, *a, **kw ) -> Deferred: """Return a Deferred built by chaining the given callbacks and errbacks""" - d = Deferred() + d: Deferred = Deferred() for cb, eb in zip(callbacks, errbacks): d.addCallbacks( callback=cb, diff --git a/scrapy/utils/response.py b/scrapy/utils/response.py index a91a49170..40a32309a 100644 --- a/scrapy/utils/response.py +++ b/scrapy/utils/response.py @@ -6,7 +6,7 @@ import os import re import tempfile import webbrowser -from typing import Any, Callable, Iterable, Optional, Tuple, Union +from typing import Any, Callable, Iterable, Tuple, Union from weakref import WeakKeyDictionary from twisted.web import http @@ -38,12 +38,13 @@ _metaref_cache: "WeakKeyDictionary[Response, Union[Tuple[None, None], Tuple[floa def get_meta_refresh( response: "scrapy.http.response.text.TextResponse", - ignore_tags: Optional[Iterable[str]] = ("script", "noscript"), + ignore_tags: Iterable[str] = ("script", "noscript"), ) -> Union[Tuple[None, None], Tuple[float, str]]: """Parse the http-equiv refresh parameter from the given response""" if response not in _metaref_cache: text = response.text[0:4096] - _metaref_cache[response] = html.get_meta_refresh( + # a w3lib typing bug here + _metaref_cache[response] = html.get_meta_refresh( # type: ignore[assignment] text, response.url, response.encoding, ignore_tags=ignore_tags ) return _metaref_cache[response] diff --git a/scrapy/utils/testsite.py b/scrapy/utils/testsite.py index a47756c4b..119be1dfb 100644 --- a/scrapy/utils/testsite.py +++ b/scrapy/utils/testsite.py @@ -50,6 +50,6 @@ def test_site(): if __name__ == "__main__": from twisted.internet import reactor - port = reactor.listenTCP(0, test_site(), interface="127.0.0.1") + port = reactor.listenTCP(0, test_site(), interface="127.0.0.1") # type: ignore[attr-defined] print(f"http://localhost:{port.getHost().port}/") - reactor.run() + reactor.run() # type: ignore[attr-defined] From 764a9d47bb95e87519a4fc2fe20ddf0c97b11739 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 25 Jan 2023 22:31:12 +0400 Subject: [PATCH 23/62] Fix typing of inlineCallbacks-decorated functions. --- scrapy/core/engine.py | 6 +++--- scrapy/core/spidermw.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index 19696415b..cafa7f6da 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -7,7 +7,7 @@ For more information see docs/topics/architecture.rst import logging import warnings from time import time -from typing import Callable, Iterable, Iterator, Optional, Set, Union +from typing import Any, Callable, Generator, Iterable, Iterator, Optional, Set, Union from twisted.internet.defer import Deferred, inlineCallbacks, succeed from twisted.internet.task import LoopingCall @@ -96,7 +96,7 @@ class ExecutionEngine: return scheduler_cls @inlineCallbacks - def start(self) -> Deferred: + def start(self) -> Generator[Deferred, Any, None]: if self.running: raise RuntimeError("Engine already running") self.start_time = time() @@ -109,7 +109,7 @@ class ExecutionEngine: """Gracefully stop the execution engine""" @inlineCallbacks - def _finish_stopping_engine(_) -> Deferred: + def _finish_stopping_engine(_) -> Generator[Deferred, Any, None]: yield self.signals.send_catch_log_deferred(signal=signals.engine_stopped) self._closewait.callback(None) diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index 1aaed5865..0bc8d54d2 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -182,7 +182,7 @@ class SpiderMiddlewareManager(MiddlewareManager): spider: Spider, result: Union[Iterable, AsyncIterable], start_index: int = 0, - ) -> Deferred: + ) -> Generator[Deferred, Any, Union[MutableChain, MutableAsyncChain]]: # items in this iterable do not need to go through the process_spider_output # chain, they went through it already from the process_spider_exception method recovered: Union[MutableChain, MutableAsyncChain] From 5fde6d533903441fe3d986030319de3415a9b956 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 25 Jan 2023 22:54:46 +0400 Subject: [PATCH 24/62] Don't type-check Twisted interfaces. --- setup.cfg | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/setup.cfg b/setup.cfg index c711e6e72..af9c87945 100644 --- a/setup.cfg +++ b/setup.cfg @@ -7,6 +7,10 @@ universal=1 [mypy] ignore_missing_imports = true +# Interface classes are hard to support +[mypy-twisted.internet.interfaces] +follow_imports = skip + # FIXME: remove the following sections once the issues are solved [mypy-scrapy.downloadermiddlewares.httpproxy] From e9094d1f38f2ed2904a6955004a8069054c8e35b Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 25 Jan 2023 23:22:18 +0400 Subject: [PATCH 25/62] Address remaining typing issues in scrapy.core. --- scrapy/core/downloader/__init__.py | 2 +- scrapy/core/downloader/handlers/http2.py | 8 +++---- scrapy/core/engine.py | 27 ++++++++++++++++++------ scrapy/core/http2/agent.py | 10 +++++---- scrapy/core/http2/protocol.py | 7 ++++++ scrapy/core/http2/stream.py | 2 +- scrapy/core/scraper.py | 8 ++++--- scrapy/core/spidermw.py | 2 +- 8 files changed, 45 insertions(+), 21 deletions(-) diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index 3a7de8072..e66156ba1 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -95,7 +95,7 @@ class Downloader: dfd = self.middleware.download(self._enqueue_request, request, spider) return dfd.addBoth(_deactivate) - def needs_backout(self): + def needs_backout(self) -> bool: return len(self.active) >= self.total_concurrency def _get_slot(self, request, spider): diff --git a/scrapy/core/downloader/handlers/http2.py b/scrapy/core/downloader/handlers/http2.py index 25ac0307b..20cd50c5a 100644 --- a/scrapy/core/downloader/handlers/http2.py +++ b/scrapy/core/downloader/handlers/http2.py @@ -28,7 +28,7 @@ class H2DownloadHandler: from twisted.internet import reactor - self._pool = H2ConnectionPool(reactor, settings) + self._pool = H2ConnectionPool(reactor, settings) # type: ignore[arg-type] self._context_factory = load_context_factory_from_settings(settings, crawler) @classmethod @@ -82,7 +82,7 @@ class ScrapyH2Agent: "Tunneling via CONNECT method using HTTP/2.0 is not yet supported" ) return self._ProxyAgent( - reactor=reactor, + reactor=reactor, # type: ignore[arg-type] context_factory=self._context_factory, proxy_uri=URI.fromBytes(to_bytes(proxy, encoding="ascii")), connect_timeout=timeout, @@ -91,7 +91,7 @@ class ScrapyH2Agent: ) return self._Agent( - reactor=reactor, + reactor=reactor, # type: ignore[arg-type] context_factory=self._context_factory, connect_timeout=timeout, bind_address=bind_address, @@ -108,7 +108,7 @@ class ScrapyH2Agent: d = agent.request(request, spider) d.addCallback(self._cb_latency, request, start_time) - timeout_cl = reactor.callLater(timeout, d.cancel) + timeout_cl = reactor.callLater(timeout, d.cancel) # type: ignore[attr-defined] d.addBoth(self._cb_timeout, request, timeout, timeout_cl) return d diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index cafa7f6da..e44675ef0 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -7,13 +7,24 @@ For more information see docs/topics/architecture.rst import logging import warnings from time import time -from typing import Any, Callable, Generator, Iterable, Iterator, Optional, Set, Union +from typing import ( + Any, + Callable, + cast, + Generator, + Iterable, + Iterator, + Optional, + Set, + Union, +) from twisted.internet.defer import Deferred, inlineCallbacks, succeed from twisted.internet.task import LoopingCall from twisted.python.failure import Failure from scrapy import signals +from scrapy.core.downloader import Downloader from scrapy.core.scraper import Scraper from scrapy.exceptions import ( CloseSpider, @@ -80,7 +91,7 @@ class ExecutionEngine: self.paused = False self.scheduler_cls = self._get_scheduler_class(crawler.settings) downloader_cls = load_object(self.settings["DOWNLOADER"]) - self.downloader = downloader_cls(crawler) + self.downloader: Downloader = downloader_cls(crawler) self.scraper = Scraper(crawler) self._spider_closed_callback = spider_closed_callback @@ -102,7 +113,7 @@ class ExecutionEngine: self.start_time = time() yield self.signals.send_catch_log_deferred(signal=signals.engine_started) self.running = True - self._closewait = Deferred() + self._closewait: Deferred = Deferred() yield self._closewait def stop(self) -> Deferred: @@ -177,11 +188,13 @@ class ExecutionEngine: self._spider_idle() def _needs_backout(self) -> bool: + assert self.slot is not None # typing + assert self.scraper.slot is not None # typing return ( not self.running - or self.slot.closing # type: ignore[union-attr] + or bool(self.slot.closing) or self.downloader.needs_backout() - or self.scraper.slot.needs_backout() # type: ignore[union-attr] + or self.scraper.slot.needs_backout() ) def _next_request_from_scheduler(self) -> Optional[Deferred]: @@ -201,7 +214,7 @@ class ExecutionEngine: extra={"spider": self.spider}, ) ) - d.addBoth(lambda _: self.slot.remove_request(request)) + d.addBoth(lambda _: cast(Slot, self.slot).remove_request(request)) d.addErrback( lambda f: logger.info( "Error while removing request from slot", @@ -429,7 +442,7 @@ class ExecutionEngine: dfd.addErrback(log_failure("Scraper close failure")) if hasattr(self.slot.scheduler, "close"): - dfd.addBoth(lambda _: self.slot.scheduler.close(reason)) + dfd.addBoth(lambda _: cast(Slot, self.slot).scheduler.close(reason)) dfd.addErrback(log_failure("Scheduler close failure")) dfd.addBoth( diff --git a/scrapy/core/http2/agent.py b/scrapy/core/http2/agent.py index 119443c80..3ceda9b61 100644 --- a/scrapy/core/http2/agent.py +++ b/scrapy/core/http2/agent.py @@ -35,7 +35,7 @@ class H2ConnectionPool: # Received a request while connecting to remote # Create a deferred which will fire with the H2ClientProtocol # instance - d = Deferred() + d: Deferred = Deferred() self._pending_requests[key].append(d) return d @@ -53,14 +53,14 @@ class H2ConnectionPool: ) -> Deferred: self._pending_requests[key] = deque() - conn_lost_deferred = Deferred() + conn_lost_deferred: Deferred = Deferred() conn_lost_deferred.addCallback(self._remove_connection, key) factory = H2ClientFactory(uri, self.settings, conn_lost_deferred) conn_d = endpoint.connect(factory) conn_d.addCallback(self.put_connection, key) - d = Deferred() + d: Deferred = Deferred() self._pending_requests[key].append(d) return d @@ -83,7 +83,8 @@ class H2ConnectionPool: pending_requests = self._pending_requests.pop(key, None) while pending_requests: d = pending_requests.popleft() - d.errback(errors) + # TODO: this is incorrect, errback takes a single exception + d.errback(errors) # type: ignore[arg-type] def close_connections(self) -> None: """Close all the HTTP/2 connections and remove them from pool @@ -92,6 +93,7 @@ class H2ConnectionPool: Deferred that fires when all connections have been closed """ for conn in self._connections.values(): + assert conn.transport is not None # typing conn.transport.abortConnection() diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py index 214deeed0..e44aed43c 100644 --- a/scrapy/core/http2/protocol.py +++ b/scrapy/core/http2/protocol.py @@ -141,6 +141,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin): This is used while initiating pending streams to make sure that we initiate stream only during active HTTP/2 Connection """ + assert self.transport is not None # typing return bool(self.transport.connected) and self.metadata["settings_acknowledged"] @property @@ -197,6 +198,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin): """Write data to the underlying transport connection from the HTTP2 connection instance if any """ + assert self.transport is not None # typing # Reset the idle timeout as connection is still actively sending data self.resetTimeout() @@ -227,6 +229,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin): # Initialize the timeout self.setTimeout(self.IDLE_TIMEOUT) + assert self.transport is not None # typing destination = self.transport.getPeer() self.metadata["ip_address"] = ipaddress.ip_address(destination.host) @@ -238,12 +241,14 @@ class H2ClientProtocol(Protocol, TimeoutMixin): """Helper function to lose the connection with the error sent as a reason""" self._conn_lost_errors += errors + assert self.transport is not None # typing self.transport.loseConnection() def handshakeCompleted(self) -> None: """ Close the connection if it's not made via the expected protocol """ + assert self.transport is not None # typing if ( self.transport.negotiatedProtocol is not None and self.transport.negotiatedProtocol != PROTOCOL_NAME @@ -276,6 +281,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin): # hyper-h2 does not drop the connection in this scenario, we # need to abort the connection manually. self._conn_lost_errors += [e] + assert self.transport is not None # typing self.transport.abortConnection() return @@ -389,6 +395,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin): self._send_pending_requests() # Update certificate when our HTTP/2 connection is established + assert self.transport is not None # typing self.metadata["certificate"] = Certificate(self.transport.getPeerCertificate()) def stream_ended(self, event: StreamEnded) -> None: diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py index 2b5c98e5f..5324a5ca1 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/http2/stream.py @@ -151,7 +151,7 @@ class Stream: else: self.close(StreamCloseReason.CANCELLED) - self._deferred_response = Deferred(_cancel) + self._deferred_response: Deferred = Deferred(_cancel) def __repr__(self) -> str: return f"Stream(id={self.stream_id!r})" diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index 7c2eefbe6..c1488a46c 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -66,7 +66,7 @@ class Slot: def add_response_request( self, result: Union[Response, Failure], request: Request ) -> Deferred: - deferred = Deferred() + deferred: Deferred = Deferred() self.queue.append((result, request, deferred)) if isinstance(result, Response): self.active_size += max(len(result.body), self.MIN_RESPONSE_SIZE) @@ -205,10 +205,12 @@ class Scraper: callback=callback, callbackKeywords=result.request.cb_kwargs ) else: # result is a Failure - result.request = request + # TODO: properly type adding this attribute to a Failure + result.request = request # type: ignore[attr-defined] warn_on_generator_with_return_value(spider, request.errback) dfd = defer_fail(result) - dfd.addErrback(request.errback) + if request.errback: + dfd.addErrback(request.errback) return dfd.addCallback(iterate_spider_output) def handle_spider_error( diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index 0bc8d54d2..31e815b19 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -157,7 +157,7 @@ class SpiderMiddlewareManager(MiddlewareManager): # simplified when downgrading is removed. if dfd.called: # the result is available immediately if _process_spider_output didn't do downgrading - return dfd.result + return cast(MutableChain, dfd.result) # we forbid waiting here because otherwise we would need to return a deferred from # _process_spider_exception too, which complicates the architecture msg = f"Async iterable returned from {method.__qualname__} cannot be downgraded" From a4edff31b91c4647a02bb392020110856b669fd9 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 25 Jan 2023 23:30:21 +0400 Subject: [PATCH 26/62] Replace lxml-stubs with types-lxml. --- tox.ini | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tox.ini b/tox.ini index eaf29173a..f05ebc45a 100644 --- a/tox.ini +++ b/tox.ini @@ -37,9 +37,9 @@ install_command = [testenv:typing] basepython = python3 deps = - lxml-stubs==0.2.0 mypy==0.991 types-attrs==19.1.0 + types-lxml==2022.11.8 types-pyOpenSSL==21.0.0 types-setuptools==57.0.0 commands = @@ -201,4 +201,4 @@ deps = black==22.12.0 commands = black {posargs:--check .} - \ No newline at end of file + From 69bb9a78594ed864291d45d36108cc0ad960f048 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 25 Jan 2023 23:52:46 +0400 Subject: [PATCH 27/62] Install types-Pillow and types-Pygments. --- tox.ini | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tox.ini b/tox.ini index f05ebc45a..7048391ac 100644 --- a/tox.ini +++ b/tox.ini @@ -40,6 +40,8 @@ deps = mypy==0.991 types-attrs==19.1.0 types-lxml==2022.11.8 + types-Pillow==9.4.0.5 + types-Pygments==2.14.0.1 types-pyOpenSSL==21.0.0 types-setuptools==57.0.0 commands = From 0a21a9457b7aeafef3b9ee1c0206546d6c8fb294 Mon Sep 17 00:00:00 2001 From: Alex Date: Thu, 26 Jan 2023 00:50:29 -0800 Subject: [PATCH 28/62] fixed mypy typing error --- scrapy/pipelines/files.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index f9dfa53e3..94b0b1b70 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -37,7 +37,7 @@ from scrapy.utils.request import referer_str logger = logging.getLogger(__name__) -def _to_string(path: Union[str, PathLike]): +def _to_string(path: Union[str, PathLike]) -> str: return str(path) # convert a Path object to string From 94161d101cd47d2dac4be7e8325a24f0ddea86cf Mon Sep 17 00:00:00 2001 From: Alex Date: Thu, 26 Jan 2023 16:41:06 -0800 Subject: [PATCH 29/62] update --- scrapy/pipelines/files.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index 6a32c8b47..afa237b3d 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -36,7 +36,7 @@ from scrapy.utils.request import referer_str logger = logging.getLogger(__name__) -def _to_string(path: Union[str, os.PathLike[str]]): +def _to_string(path: Union[str, os.PathLike]) -> str: return str(path) # convert a Path object to string @@ -45,10 +45,10 @@ class FileException(Exception): class FSFilesStore: - def __init__(self, basedir: Union[str, os.PathLike[str]]): + def __init__(self, basedir: Union[str, os.PathLike]): basedir = _to_string(basedir) - if '://' in basedir: - basedir = basedir.split('://', 1)[1] + if "://" in basedir: + basedir = basedir.split("://", 1)[1] self.basedir = basedir self._mkdir(Path(self.basedir)) self.created_directories: DefaultDict[str, Set[str]] = defaultdict(set) @@ -70,7 +70,7 @@ class FSFilesStore: return {'last_modified': last_modified, 'checksum': checksum} - def _get_filesystem_path(self, path: Union[str, os.PathLike[str]]) -> Path: + def _get_filesystem_path(self, path: Union[str, os.PathLike]) -> Path: path_comps = _to_string(path).split('/') return Path(self.basedir, *path_comps) From a1e2fbafdcaae9b70cf7c4216c5cb80d1df5268e Mon Sep 17 00:00:00 2001 From: Alex Date: Thu, 26 Jan 2023 16:46:08 -0800 Subject: [PATCH 30/62] applied black to tests --- tests/test_pipeline_files.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py index 76c16e57f..1e00e6d1d 100644 --- a/tests/test_pipeline_files.py +++ b/tests/test_pipeline_files.py @@ -484,19 +484,18 @@ class FilesPipelineTestCaseCustomSettings(unittest.TestCase): self.assertEqual(getattr(pipeline_cls, pipe_inst_attr), expected_value) def test_file_pipeline_using_pathlike_objects(self): - class CustomFilesPipelineWithPathLikeDir(FilesPipeline): def file_path(self, request, response=None, info=None, *, item=None): - return Path('subdir') / Path(request.url).name + return Path("subdir") / Path(request.url).name pipeline = CustomFilesPipelineWithPathLikeDir.from_settings( - Settings({'FILES_STORE': Path('./Temp')}) + Settings({"FILES_STORE": Path("./Temp")}) ) request = Request("http://example.com/image01.jpg") - self.assertEqual(pipeline.file_path(request), Path('subdir/image01.jpg')) + self.assertEqual(pipeline.file_path(request), Path("subdir/image01.jpg")) def test_files_store_constructor_with_pathlike_object(self): - path = Path('./FileDir') + path = Path("./FileDir") fs_store = FSFilesStore(path) self.assertEqual(fs_store.basedir, str(path)) From c0efb271a23218656b5f629fde6d23c115a0c3de Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 27 Jan 2023 23:19:44 +0400 Subject: [PATCH 31/62] Ignore typing for twisted.internet.reactor globally. --- scrapy/core/downloader/handlers/http2.py | 8 ++++---- scrapy/utils/benchserver.py | 6 +++--- scrapy/utils/defer.py | 4 ++-- scrapy/utils/testsite.py | 4 ++-- setup.cfg | 3 +++ tests/CrawlerRunner/ip_address.py | 6 +++--- tests/mockserver.py | 10 +++++----- tests/test_engine.py | 2 +- 8 files changed, 23 insertions(+), 20 deletions(-) diff --git a/scrapy/core/downloader/handlers/http2.py b/scrapy/core/downloader/handlers/http2.py index 20cd50c5a..25ac0307b 100644 --- a/scrapy/core/downloader/handlers/http2.py +++ b/scrapy/core/downloader/handlers/http2.py @@ -28,7 +28,7 @@ class H2DownloadHandler: from twisted.internet import reactor - self._pool = H2ConnectionPool(reactor, settings) # type: ignore[arg-type] + self._pool = H2ConnectionPool(reactor, settings) self._context_factory = load_context_factory_from_settings(settings, crawler) @classmethod @@ -82,7 +82,7 @@ class ScrapyH2Agent: "Tunneling via CONNECT method using HTTP/2.0 is not yet supported" ) return self._ProxyAgent( - reactor=reactor, # type: ignore[arg-type] + reactor=reactor, context_factory=self._context_factory, proxy_uri=URI.fromBytes(to_bytes(proxy, encoding="ascii")), connect_timeout=timeout, @@ -91,7 +91,7 @@ class ScrapyH2Agent: ) return self._Agent( - reactor=reactor, # type: ignore[arg-type] + reactor=reactor, context_factory=self._context_factory, connect_timeout=timeout, bind_address=bind_address, @@ -108,7 +108,7 @@ class ScrapyH2Agent: d = agent.request(request, spider) d.addCallback(self._cb_latency, request, start_time) - timeout_cl = reactor.callLater(timeout, d.cancel) # type: ignore[attr-defined] + timeout_cl = reactor.callLater(timeout, d.cancel) d.addBoth(self._cb_timeout, request, timeout, timeout_cl) return d diff --git a/scrapy/utils/benchserver.py b/scrapy/utils/benchserver.py index 750d3c093..32bc2e38c 100644 --- a/scrapy/utils/benchserver.py +++ b/scrapy/utils/benchserver.py @@ -37,11 +37,11 @@ if __name__ == "__main__": root = Root() factory = Site(root) - httpPort = reactor.listenTCP(8998, Site(root)) # type: ignore[attr-defined] + httpPort = reactor.listenTCP(8998, Site(root)) def _print_listening(): httpHost = httpPort.getHost() print(f"Bench server at http://{httpHost.host}:{httpHost.port}") - reactor.callWhenRunning(_print_listening) # type: ignore[attr-defined] - reactor.run() # type: ignore[attr-defined] + reactor.callWhenRunning(_print_listening) + reactor.run() diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py index 8fee5a7ed..7f2211877 100644 --- a/scrapy/utils/defer.py +++ b/scrapy/utils/defer.py @@ -40,7 +40,7 @@ def defer_fail(_failure: Failure) -> Deferred: from twisted.internet import reactor d: Deferred = Deferred() - reactor.callLater(0.1, d.errback, _failure) # type: ignore[attr-defined] + reactor.callLater(0.1, d.errback, _failure) return d @@ -54,7 +54,7 @@ def defer_succeed(result) -> Deferred: from twisted.internet import reactor d: Deferred = Deferred() - reactor.callLater(0.1, d.callback, result) # type: ignore[attr-defined] + reactor.callLater(0.1, d.callback, result) return d diff --git a/scrapy/utils/testsite.py b/scrapy/utils/testsite.py index 119be1dfb..a47756c4b 100644 --- a/scrapy/utils/testsite.py +++ b/scrapy/utils/testsite.py @@ -50,6 +50,6 @@ def test_site(): if __name__ == "__main__": from twisted.internet import reactor - port = reactor.listenTCP(0, test_site(), interface="127.0.0.1") # type: ignore[attr-defined] + port = reactor.listenTCP(0, test_site(), interface="127.0.0.1") print(f"http://localhost:{port.getHost().port}/") - reactor.run() # type: ignore[attr-defined] + reactor.run() diff --git a/setup.cfg b/setup.cfg index af9c87945..db79c5821 100644 --- a/setup.cfg +++ b/setup.cfg @@ -11,6 +11,9 @@ ignore_missing_imports = true [mypy-twisted.internet.interfaces] follow_imports = skip +[mypy-twisted.internet.reactor] +follow_imports = skip + # FIXME: remove the following sections once the issues are solved [mypy-scrapy.downloadermiddlewares.httpproxy] diff --git a/tests/CrawlerRunner/ip_address.py b/tests/CrawlerRunner/ip_address.py index b9a4485a9..26db16dd6 100644 --- a/tests/CrawlerRunner/ip_address.py +++ b/tests/CrawlerRunner/ip_address.py @@ -41,10 +41,10 @@ if __name__ == "__main__": url = f"http://not.a.real.domain:{port}/echo" servers = [(mock_dns_server.host, mock_dns_server.port)] - reactor.installResolver(createResolver(servers=servers)) # type: ignore[attr-defined] + reactor.installResolver(createResolver(servers=servers)) configure_logging() runner = CrawlerRunner() d = runner.crawl(LocalhostSpider, url=url) - d.addBoth(lambda _: reactor.stop()) # type: ignore[attr-defined] - reactor.run() # type: ignore[attr-defined] + d.addBoth(lambda _: reactor.stop()) + reactor.run() diff --git a/tests/mockserver.py b/tests/mockserver.py index 185897373..eb1c03db7 100644 --- a/tests/mockserver.py +++ b/tests/mockserver.py @@ -375,9 +375,9 @@ if __name__ == "__main__": if args.type == "http": root = Root() factory = Site(root) - httpPort = reactor.listenTCP(0, factory) # type: ignore[attr-defined] + httpPort = reactor.listenTCP(0, factory) contextFactory = ssl_context_factory() - httpsPort = reactor.listenSSL(0, factory, contextFactory) # type: ignore[attr-defined] + httpsPort = reactor.listenSSL(0, factory, contextFactory) def print_listening(): httpHost = httpPort.getHost() @@ -391,11 +391,11 @@ if __name__ == "__main__": clients = [MockDNSResolver()] factory = DNSServerFactory(clients=clients) protocol = dns.DNSDatagramProtocol(controller=factory) - listener = reactor.listenUDP(0, protocol) # type: ignore[attr-defined] + listener = reactor.listenUDP(0, protocol) def print_listening(): host = listener.getHost() print(f"{host.host}:{host.port}") - reactor.callWhenRunning(print_listening) # type: ignore[attr-defined] - reactor.run() # type: ignore[attr-defined] + reactor.callWhenRunning(print_listening) + reactor.run() diff --git a/tests/test_engine.py b/tests/test_engine.py index 5fb87424b..7ddb420ba 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -557,4 +557,4 @@ class EngineTest(unittest.TestCase): if __name__ == "__main__": if len(sys.argv) > 1 and sys.argv[1] == "runserver": start_test_site(debug=True) - reactor.run() # type: ignore[attr-defined] + reactor.run() From 5fa0f64db5d4b7bf3f3a7c83578ce15ec534d3d3 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 27 Jan 2023 23:29:23 +0400 Subject: [PATCH 32/62] Ru typing on 3.8 as types-lxml isn't available for 3.7. --- .github/workflows/checks.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 740092dab..bd26e8bb2 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -17,7 +17,7 @@ jobs: - python-version: "3.11" env: TOXENV: pylint - - python-version: 3.7 + - python-version: 3.8 env: TOXENV: typing - python-version: "3.11" # Keep in sync with .readthedocs.yml From 80453d53b19bbe288dc1e5b721f2f29acb89706a Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sat, 28 Jan 2023 01:29:09 +0400 Subject: [PATCH 33/62] Type DeferredLists as Deferreds. --- scrapy/utils/defer.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py index 7f2211877..79b96856c 100644 --- a/scrapy/utils/defer.py +++ b/scrapy/utils/defer.py @@ -85,7 +85,7 @@ def mustbe_deferred(f: Callable, *args, **kw) -> Deferred: def parallel( iterable: Iterable, count: int, callable: Callable, *args, **named -) -> DeferredList: +) -> Deferred: """Execute a callable over the objects in the given iterable, in parallel, using no more than ``count`` concurrent calls. @@ -202,11 +202,11 @@ class _AsyncCooperatorAdapter(Iterator): def parallel_async( async_iterable: AsyncIterable, count: int, callable: Callable, *args, **named -) -> DeferredList: +) -> Deferred: """Like parallel but for async iterators""" coop = Cooperator() work = _AsyncCooperatorAdapter(async_iterable, callable, *args, **named) - dl = DeferredList([coop.coiterate(work) for _ in range(count)]) + dl: Deferred = DeferredList([coop.coiterate(work) for _ in range(count)]) return dl @@ -245,7 +245,7 @@ def process_parallel(callbacks: Iterable[Callable], input, *a, **kw) -> Deferred callbacks """ dfds = [defer.succeed(input).addCallback(x, *a, **kw) for x in callbacks] - d = DeferredList(dfds, fireOnOneErrback=True, consumeErrors=True) + d: Deferred = DeferredList(dfds, fireOnOneErrback=True, consumeErrors=True) d.addCallbacks(lambda r: [x[1] for x in r], lambda f: f.value.subFailure) return d From 349fc33cc70aee38d30f7715811829b70ef77492 Mon Sep 17 00:00:00 2001 From: Alex Date: Tue, 31 Jan 2023 14:28:08 -0800 Subject: [PATCH 34/62] added disable_warnings instruction to .coveragerc --- .coveragerc | 1 + 1 file changed, 1 insertion(+) diff --git a/.coveragerc b/.coveragerc index 02acbff8e..ad0ee0f6c 100644 --- a/.coveragerc +++ b/.coveragerc @@ -3,3 +3,4 @@ branch = true include = scrapy/* omit = tests/* +disable_warnings = include-ignored From eecc035f4c2ed69cf9dbceae009c583d849b9f2c Mon Sep 17 00:00:00 2001 From: Alex Date: Thu, 2 Feb 2023 11:27:40 -0800 Subject: [PATCH 35/62] correcting type hints --- scrapy/pipelines/files.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index 1b724ce60..fcd9f9078 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -9,11 +9,11 @@ import logging import mimetypes import os import time -from os import PathLike from collections import defaultdict from contextlib import suppress from ftplib import FTP from io import BytesIO +from os import PathLike from pathlib import Path from typing import DefaultDict, Optional, Set, Union from urllib.parse import urlparse @@ -54,12 +54,14 @@ class FSFilesStore: self._mkdir(Path(self.basedir)) self.created_directories: DefaultDict[str, Set[str]] = defaultdict(set) - def persist_file(self, path: str, buf, info, meta=None, headers=None): + def persist_file( + self, path: Union[str, PathLike], buf, info, meta=None, headers=None + ): absolute_path = self._get_filesystem_path(path) self._mkdir(absolute_path.parent, info) absolute_path.write_bytes(buf.getvalue()) - def stat_file(self, path: str, info): + def stat_file(self, path: Union[str, PathLike], info): absolute_path = self._get_filesystem_path(path) try: last_modified = absolute_path.stat().st_mtime From 80a86de5071aef1ac43277c273beace6bd426e80 Mon Sep 17 00:00:00 2001 From: Georgiy Zatserklianyi Date: Wed, 8 Feb 2023 22:40:22 +0200 Subject: [PATCH 36/62] per_slot_settings: test delays increased --- tests/test_downloaderslotssettings.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_downloaderslotssettings.py b/tests/test_downloaderslotssettings.py index a092d01bf..5ee06d5bc 100644 --- a/tests/test_downloaderslotssettings.py +++ b/tests/test_downloaderslotssettings.py @@ -20,11 +20,11 @@ class DownloaderSlotsSettingsTestSpider(MetaSpider): "DOWNLOAD_SLOTS": { 'quotes.toscrape.com': { 'concurrency': 1, - 'delay': 1.5, + 'delay': 2, 'randomize_delay': False }, 'books.toscrape.com': { - 'delay': 2, + 'delay': 3, 'randomize_delay': False } } From c3033a54b1e37287020d8f1f6b541ce4be8d2971 Mon Sep 17 00:00:00 2001 From: Georgiy Zatserklianyi Date: Wed, 8 Feb 2023 23:55:07 +0200 Subject: [PATCH 37/62] per_slot_settings(docs): settings 1 --- docs/topics/settings.rst | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index d636dc301..bde33a0e9 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -771,6 +771,14 @@ The default HTTPS handler uses HTTP/1.1. To use HTTP/2: .. _http2 faq: https://http2.github.io/faq/#does-http2-require-encryption .. _server pushes: https://tools.ietf.org/html/rfc7540#section-8.2 +.. setting:: DOWNLOAD_SLOTS + +DOWNLOAD_SLOTS +---------------- + +Default: ``{}`` + + .. setting:: DOWNLOAD_TIMEOUT DOWNLOAD_TIMEOUT From 5208d436ae26e61b6b00f305079c0a31b8e95063 Mon Sep 17 00:00:00 2001 From: Georgiy Zatserklianyi Date: Thu, 9 Feb 2023 00:33:21 +0200 Subject: [PATCH 38/62] per_slot_settings(docs): settings 2 --- docs/topics/settings.rst | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index bde33a0e9..477c3068f 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -778,6 +778,24 @@ DOWNLOAD_SLOTS Default: ``{}`` +Allows to define concurrency/delay parameters on per slot(domain) basis: + + .. code-block:: python + + DOWNLOAD_SLOTS = { + 'quotes.toscrape.com': { + 'concurrency': 1, + 'delay': 2, + 'randomize_delay': False + }, + 'books.toscrape.com': { + 'delay': 3, + 'randomize_delay': False + } + } +wrapper around :func:`~scrapy.core.downloader.Slot` + +.. autofunction:: scrapy.core.downloader.Slot .. setting:: DOWNLOAD_TIMEOUT From 474087be6fadf2dd3440fc77ab757a18be0931c8 Mon Sep 17 00:00:00 2001 From: Georgiy Zatserklianyi Date: Thu, 9 Feb 2023 14:52:28 +0200 Subject: [PATCH 39/62] per_slot_settings(docs): settings 3 --- docs/topics/settings.rst | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 477c3068f..5acf09efe 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -793,9 +793,15 @@ Allows to define concurrency/delay parameters on per slot(domain) basis: 'randomize_delay': False } } -wrapper around :func:`~scrapy.core.downloader.Slot` -.. autofunction:: scrapy.core.downloader.Slot +.. note:: + + For other downloader slots default settings values will be used: + + - ``DOWNLOAD_DELAY`` -> ``delay`` + - ``CONCURRENT_REQUESTS_PER_DOMAIN`` -> ``concurency`` + - ``RANDOMIZE_DOWNLOAD_DELAY`` -> ``randomize_delay`` + .. setting:: DOWNLOAD_TIMEOUT From b1f33a68aceb0e0d599f006506ab7d74a23e95f3 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sat, 11 Feb 2023 21:39:27 +0400 Subject: [PATCH 40/62] Bump mypy and types-*. --- tox.ini | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tox.ini b/tox.ini index 7048391ac..b1fd06b09 100644 --- a/tox.ini +++ b/tox.ini @@ -37,13 +37,13 @@ install_command = [testenv:typing] basepython = python3 deps = - mypy==0.991 + mypy==1.0.0 types-attrs==19.1.0 types-lxml==2022.11.8 - types-Pillow==9.4.0.5 - types-Pygments==2.14.0.1 - types-pyOpenSSL==21.0.0 - types-setuptools==57.0.0 + types-Pillow==9.4.0.10 + types-Pygments==2.14.0.2 + types-pyOpenSSL==23.0.0.2 + types-setuptools==67.2.0.1 commands = mypy --show-error-codes {posargs: scrapy tests} From fa9897282fc571bbac64eaba06433362ad233530 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sat, 11 Feb 2023 22:11:30 +0400 Subject: [PATCH 41/62] Add a cast to deferred_from_coro, that code path has different typing. --- scrapy/utils/defer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py index a3bd273e3..d25ebbdf4 100644 --- a/scrapy/utils/defer.py +++ b/scrapy/utils/defer.py @@ -288,7 +288,7 @@ def deferred_from_coro(o) -> Any: if not is_asyncio_reactor_installed(): # wrapping the coroutine directly into a Deferred, this doesn't work correctly with coroutines # that use asyncio, e.g. "await asyncio.sleep(1)" - return ensureDeferred(o) + return ensureDeferred(cast(Coroutine[Deferred, Any, Any], o)) # wrapping the coroutine into a Future and then into a Deferred, this requires AsyncioSelectorReactor event_loop = _get_asyncio_event_loop() return Deferred.fromFuture(asyncio.ensure_future(o, loop=event_loop)) From b7daa2624d698a2fcf64923b4d1c51d859673006 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sat, 11 Feb 2023 23:46:33 +0400 Subject: [PATCH 42/62] Remove a workaround for old lxml. --- scrapy/http/request/form.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index bdc6a3e39..342e0c7ad 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -198,12 +198,6 @@ def _select_value(ele: SelectElement, n: str, v: str): # And for select tags without options o = ele.value_options return (n, o[0]) if o else (None, None) - if v is not None and multiple: - # This is a workaround to bug in lxml fixed 2.3.1 - # fix https://github.com/lxml/lxml/commit/57f49eed82068a20da3db8f1b18ae00c1bab8b12#L1L1139 - selected_options = ele.xpath(".//option[@selected]") - values = [(o.get("value") or o.text or "").strip() for o in selected_options] - return n, values return n, v From a1717aa48c42fdeb9c45739a82b00b9cadf7b58c Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sun, 12 Feb 2023 00:42:39 +0400 Subject: [PATCH 43/62] Fix most of FormRequest typing issues. --- scrapy/http/request/form.py | 33 +++++++++++++++++++++++---------- scrapy/utils/python.py | 4 ++-- 2 files changed, 25 insertions(+), 12 deletions(-) diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index 342e0c7ad..ef6589361 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -5,10 +5,17 @@ This module implements the FormRequest class which is a more convenient class See documentation in docs/topics/request-response.rst """ -from typing import Iterable, List, Optional, Tuple, Type, TypeVar, Union +from typing import Iterable, List, Optional, Tuple, Type, TypeVar, Union, cast from urllib.parse import urlencode, urljoin, urlsplit, urlunsplit -from lxml.html import FormElement, HtmlElement, HTMLParser, SelectElement +from lxml.html import ( + FormElement, + HTMLParser, + InputElement, + MultipleSelectOptions, + SelectElement, + TextareaElement, +) from parsel.selector import create_root_node from w3lib.html import strip_html5_whitespace @@ -19,7 +26,8 @@ from scrapy.utils.response import get_base_url FormRequestTypeVar = TypeVar("FormRequestTypeVar", bound="FormRequest") -FormdataType = Optional[Union[dict, List[Tuple[str, str]]]] +FormdataKVType = Tuple[str, Union[str, Iterable[str]]] +FormdataType = Optional[Union[dict, List[FormdataKVType]]] class FormRequest(Request): @@ -79,6 +87,7 @@ class FormRequest(Request): def _get_form_url(form: FormElement, url: Optional[str]) -> str: + assert form.base_url is not None # typing if url is None: action = form.get("action") if action is None: @@ -87,11 +96,11 @@ def _get_form_url(form: FormElement, url: Optional[str]) -> str: return urljoin(form.base_url, url) -def _urlencode(seq: Iterable, enc: str) -> str: +def _urlencode(seq: Iterable[FormdataKVType], enc: str) -> str: values = [ (to_bytes(k, enc), to_bytes(v, enc)) for k, vs in seq - for v in (vs if is_listlike(vs) else [vs]) + for v in (cast(Iterable[str], vs) if is_listlike(vs) else [cast(str, vs)]) ] return urlencode(values, doseq=True) @@ -147,7 +156,7 @@ def _get_inputs( formdata: FormdataType, dont_click: bool, clickdata: Optional[dict], -) -> List[Tuple[str, str]]: +) -> List[FormdataKVType]: """Return a list of key-value pairs for the inputs found in the given form.""" try: formdata_keys = dict(formdata or ()).keys() @@ -165,7 +174,7 @@ def _get_inputs( ' not(re:test(., "^(?:checkbox|radio)$", "i")))]]', namespaces={"re": "http://exslt.org/regular-expressions"}, ) - values = [ + values: List[FormdataKVType] = [ (k, "" if v is None else v) for k, v in (_value(e) for e in inputs) if k and k not in formdata_keys @@ -183,15 +192,19 @@ def _get_inputs( return values -def _value(ele: HtmlElement): +def _value( + ele: Union[InputElement, SelectElement, TextareaElement] +) -> Tuple[Optional[str], Union[None, str, MultipleSelectOptions]]: n = ele.name v = ele.value if ele.tag == "select": - return _select_value(ele, n, v) + return _select_value(cast(SelectElement, ele), n, v) return n, v -def _select_value(ele: SelectElement, n: str, v: str): +def _select_value( + ele: SelectElement, n: Optional[str], v: Union[None, str, MultipleSelectOptions] +) -> Tuple[Optional[str], Union[None, str, MultipleSelectOptions]]: multiple = ele.multiple if v is None and not multiple: # Match browser behaviour on simple select tag without options selected diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index 1771e5550..fc50e0f12 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -8,7 +8,7 @@ import sys import weakref from functools import partial, wraps from itertools import chain -from typing import AsyncGenerator, AsyncIterable, Iterable, Union +from typing import Any, AsyncGenerator, AsyncIterable, Iterable, Union from scrapy.utils.asyncgen import as_async_generator @@ -45,7 +45,7 @@ def iflatten(x): yield el -def is_listlike(x): +def is_listlike(x: Any) -> bool: """ >>> is_listlike("foo") False From 98a5958687b4fae213bf053e560bd18a69312f3f Mon Sep 17 00:00:00 2001 From: Georgiy Zatserklianyi Date: Thu, 16 Feb 2023 14:52:31 +0200 Subject: [PATCH 44/62] per_slot_settings(docs): per slot settings - the highest priority --- scrapy/core/downloader/__init__.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index 1a1eb819e..3691df48a 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -102,13 +102,13 @@ class Downloader: key = self._get_slot_key(request, spider) if key not in self.slots: slot_settings = self.per_slot_settings.get(key, {}) - conc = slot_settings.get( - 'concurrency', ( - self.ip_concurrency if self.ip_concurrency else self.domain_concurrency - ) - ) + conc = ( + self.ip_concurrency if self.ip_concurrency else self.domain_concurrency) conc, delay = _get_concurrency_delay(conc, spider, self.settings) - delay = slot_settings.get('delay', delay) + conc, delay = ( + slot_settings.get('concurrency', conc), + slot_settings.get('delay', delay) + ) randomize_delay = slot_settings.get('randomize_delay', self.randomize_delay) new_slot = Slot(conc, delay, randomize_delay) self.slots[key] = new_slot From 07e1429877e5128786b24ad146dabcad4784ffc2 Mon Sep 17 00:00:00 2001 From: Georgiy Zatserklianyi Date: Thu, 16 Feb 2023 15:14:01 +0200 Subject: [PATCH 45/62] per_slot_settings(docs): docs `:setting` bindings added --- docs/topics/settings.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 5acf09efe..43069c50d 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -798,9 +798,9 @@ Allows to define concurrency/delay parameters on per slot(domain) basis: For other downloader slots default settings values will be used: - - ``DOWNLOAD_DELAY`` -> ``delay`` - - ``CONCURRENT_REQUESTS_PER_DOMAIN`` -> ``concurency`` - - ``RANDOMIZE_DOWNLOAD_DELAY`` -> ``randomize_delay`` + - :setting:`DOWNLOAD_DELAY`: ``delay`` + - :setting:`CONCURRENT_REQUESTS_PER_DOMAIN`: ``concurency`` + - :setting:`RANDOMIZE_DOWNLOAD_DELAY`: ``randomize_delay`` .. setting:: DOWNLOAD_TIMEOUT From 045092e8d753b9689b24db7d30bcbd7f976fb517 Mon Sep 17 00:00:00 2001 From: Georgiy Zatserklianyi Date: Thu, 16 Feb 2023 17:55:59 +0200 Subject: [PATCH 46/62] per_slot_settings(docs): code style fix (from previous pre-commit check) --- docs/topics/settings.rst | 11 ++------ scrapy/core/downloader/__init__.py | 9 ++++--- tests/test_downloaderslotssettings.py | 37 +++++++++++++++------------ 3 files changed, 27 insertions(+), 30 deletions(-) diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 43069c50d..175e8f7f0 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -783,15 +783,8 @@ Allows to define concurrency/delay parameters on per slot(domain) basis: .. code-block:: python DOWNLOAD_SLOTS = { - 'quotes.toscrape.com': { - 'concurrency': 1, - 'delay': 2, - 'randomize_delay': False - }, - 'books.toscrape.com': { - 'delay': 3, - 'randomize_delay': False - } + "quotes.toscrape.com": {"concurrency": 1, "delay": 2, "randomize_delay": False}, + "books.toscrape.com": {"delay": 3, "randomize_delay": False}, } .. note:: diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index 3691df48a..ace483a23 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -103,13 +103,14 @@ class Downloader: if key not in self.slots: slot_settings = self.per_slot_settings.get(key, {}) conc = ( - self.ip_concurrency if self.ip_concurrency else self.domain_concurrency) + self.ip_concurrency if self.ip_concurrency else self.domain_concurrency + ) conc, delay = _get_concurrency_delay(conc, spider, self.settings) conc, delay = ( - slot_settings.get('concurrency', conc), - slot_settings.get('delay', delay) + slot_settings.get("concurrency", conc), + slot_settings.get("delay", delay), ) - randomize_delay = slot_settings.get('randomize_delay', self.randomize_delay) + randomize_delay = slot_settings.get("randomize_delay", self.randomize_delay) new_slot = Slot(conc, delay, randomize_delay) self.slots[key] = new_slot diff --git a/tests/test_downloaderslotssettings.py b/tests/test_downloaderslotssettings.py index 5ee06d5bc..a167161a1 100644 --- a/tests/test_downloaderslotssettings.py +++ b/tests/test_downloaderslotssettings.py @@ -1,31 +1,31 @@ import time +from twisted.internet import defer +from twisted.trial.unittest import TestCase + from scrapy.crawler import CrawlerRunner from scrapy.http import Request from tests.mockserver import MockServer from tests.spiders import MetaSpider -from twisted.internet import defer -from twisted.trial.unittest import TestCase - class DownloaderSlotsSettingsTestSpider(MetaSpider): - name = 'downloader_slots' + name = "downloader_slots" custom_settings = { "DOWNLOAD_DELAY": 1, "RANDOMIZE_DOWNLOAD_DELAY": False, "DOWNLOAD_SLOTS": { - 'quotes.toscrape.com': { - 'concurrency': 1, - 'delay': 2, - 'randomize_delay': False + "quotes.toscrape.com": { + "concurrency": 1, + "delay": 2, + "randomize_delay": False }, - 'books.toscrape.com': { - 'delay': 3, - 'randomize_delay': False + "books.toscrape.com": { + "delay": 3, + "randomize_delay": False } } } @@ -33,21 +33,21 @@ class DownloaderSlotsSettingsTestSpider(MetaSpider): def start_requests(self): self.times = {None: []} - slots = list(self.custom_settings.get('DOWNLOAD_SLOTS', {}).keys()) + [None] + slots = list(self.custom_settings.get("DOWNLOAD_SLOTS", {}).keys()) + [None] for slot in slots: url = self.mockserver.url(f"/?downloader_slot={slot}") self.times[slot] = [] - yield Request(url, callback=self.parse, meta={'download_slot': slot}) + yield Request(url, callback=self.parse, meta={"download_slot": slot}) def parse(self, response): - slot = response.meta.get('download_slot', None) + slot = response.meta.get("download_slot", None) self.times[slot].append(time.time()) url = self.mockserver.url(f"/?downloader_slot={slot}&req=2") - yield Request(url, callback=self.not_parse, meta={'download_slot': slot}) + yield Request(url, callback=self.not_parse, meta={"download_slot": slot}) def not_parse(self, response): - slot = response.meta.get('download_slot', None) + slot = response.meta.get("download_slot", None) self.times[slot].append(time.time()) @@ -70,6 +70,9 @@ class CrawlTestCase(TestCase): tolerance = 0.3 delays_real = {k: v[1] - v[0] for k, v in times.items()} - error_delta = {k: 1 - min(delays_real[k], v.delay) / max(delays_real[k], v.delay) for k, v in slots.items()} + error_delta = { + k: 1 - min(delays_real[k], v.delay) / max(delays_real[k], v.delay) + for k, v in slots.items() + } self.assertTrue(max(list(error_delta.values())) < tolerance) From dd5524eb986c0ef1d6ce6ec2887ed8aabe55f7c9 Mon Sep 17 00:00:00 2001 From: Georgiy Zatserklianyi Date: Thu, 16 Feb 2023 17:59:36 +0200 Subject: [PATCH 47/62] per_slot_settings(docs): code style fix 2 (from previous pre-commit check) --- tests/test_downloaderslotssettings.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/tests/test_downloaderslotssettings.py b/tests/test_downloaderslotssettings.py index a167161a1..b965233a3 100644 --- a/tests/test_downloaderslotssettings.py +++ b/tests/test_downloaderslotssettings.py @@ -21,13 +21,10 @@ class DownloaderSlotsSettingsTestSpider(MetaSpider): "quotes.toscrape.com": { "concurrency": 1, "delay": 2, - "randomize_delay": False + "randomize_delay": False, }, - "books.toscrape.com": { - "delay": 3, - "randomize_delay": False - } - } + "books.toscrape.com": {"delay": 3, "randomize_delay": False}, + }, } def start_requests(self): From 864eee66c77ceff21ba0c8611aa84402f300fc27 Mon Sep 17 00:00:00 2001 From: Georgiy Zatserklianyi Date: Thu, 16 Feb 2023 18:19:36 +0200 Subject: [PATCH 48/62] per_slot_settings(docs): code style fix 3 (from previous pre-commit check) --- tests/test_downloaderslotssettings.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/tests/test_downloaderslotssettings.py b/tests/test_downloaderslotssettings.py index b965233a3..9d4072d19 100644 --- a/tests/test_downloaderslotssettings.py +++ b/tests/test_downloaderslotssettings.py @@ -5,13 +5,11 @@ from twisted.trial.unittest import TestCase from scrapy.crawler import CrawlerRunner from scrapy.http import Request - from tests.mockserver import MockServer from tests.spiders import MetaSpider class DownloaderSlotsSettingsTestSpider(MetaSpider): - name = "downloader_slots" custom_settings = { @@ -49,7 +47,6 @@ class DownloaderSlotsSettingsTestSpider(MetaSpider): class CrawlTestCase(TestCase): - def setUp(self): self.mockserver = MockServer() self.mockserver.__enter__() From 733309affa0c01239841899a790ec4734ea173dc Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 24 Feb 2023 00:05:59 +0400 Subject: [PATCH 49/62] Don't assume formnumber can be None. --- scrapy/http/request/form.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index ef6589361..8da779b7c 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -58,7 +58,7 @@ class FormRequest(Request): response: TextResponse, formname: Optional[str] = None, formid: Optional[str] = None, - formnumber: Optional[int] = 0, + formnumber: int = 0, formdata: FormdataType = None, clickdata: Optional[dict] = None, dont_click: bool = False, @@ -109,7 +109,7 @@ def _get_form( response: TextResponse, formname: Optional[str], formid: Optional[str], - formnumber: Optional[int], + formnumber: int, formxpath: Optional[str], ) -> FormElement: """Find the wanted form element within the given response.""" @@ -142,13 +142,12 @@ def _get_form( raise ValueError(f"No
element found with {formxpath}") # If we get here, it means that either formname was None or invalid - if formnumber is not None: - try: - form = forms[formnumber] - except IndexError: - raise IndexError(f"Form number {formnumber} not found in {response}") - else: - return form + try: + form = forms[formnumber] + except IndexError: + raise IndexError(f"Form number {formnumber} not found in {response}") + else: + return form def _get_inputs( From 3a40c06ed9e4e7bf110f0eebc17a9ec77195bd1b Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 24 Feb 2023 00:16:49 +0400 Subject: [PATCH 50/62] Bump mypy and types-*. --- tox.ini | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tox.ini b/tox.ini index b78b7c125..7ca26ec49 100644 --- a/tox.ini +++ b/tox.ini @@ -37,13 +37,13 @@ install_command = [testenv:typing] basepython = python3 deps = - mypy==1.0.0 + mypy==1.0.1 types-attrs==19.1.0 - types-lxml==2022.11.8 - types-Pillow==9.4.0.10 - types-Pygments==2.14.0.2 - types-pyOpenSSL==23.0.0.2 - types-setuptools==67.2.0.1 + types-lxml==2023.2.11 + types-Pillow==9.4.0.16 + types-Pygments==2.14.0.5 + types-pyOpenSSL==23.0.0.4 + types-setuptools==67.4.0.1 commands = mypy --show-error-codes {posargs: scrapy tests} From d7bf39ee78acdf6342fbbad3d4ba0fdb6d8c17de Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 24 Feb 2023 00:28:06 +0400 Subject: [PATCH 51/62] Fix a merge problem. --- tox.ini | 6 ------ 1 file changed, 6 deletions(-) diff --git a/tox.ini b/tox.ini index 7ca26ec49..5a9d9cf29 100644 --- a/tox.ini +++ b/tox.ini @@ -187,9 +187,3 @@ deps = {[docs]deps} setenv = {[docs]setenv} commands = sphinx-build -W -b linkcheck . {envtmpdir}/linkcheck - -[testenv:black] -deps = - black==22.12.0 -commands = - black {posargs:--check .} From d20f2788822eabc513a4c55a341cfe65b6bce37d Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 24 Feb 2023 00:28:24 +0400 Subject: [PATCH 52/62] Add a link to the w3lib typing fix. --- scrapy/utils/response.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/utils/response.py b/scrapy/utils/response.py index b26eeb0d4..794678c48 100644 --- a/scrapy/utils/response.py +++ b/scrapy/utils/response.py @@ -42,7 +42,7 @@ def get_meta_refresh( """Parse the http-equiv refresh parameter from the given response""" if response not in _metaref_cache: text = response.text[0:4096] - # a w3lib typing bug here + # a w3lib typing bug here, fixed in https://github.com/scrapy/w3lib/pull/211 _metaref_cache[response] = html.get_meta_refresh( # type: ignore[assignment] text, response.url, response.encoding, ignore_tags=ignore_tags ) From 3dd9d71c328fea0f4b2ab4bc668e65ecd1eb1d8e Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 24 Feb 2023 01:13:48 +0400 Subject: [PATCH 53/62] Correctly wrap connection errors into ResponseFailed. --- scrapy/core/http2/agent.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/scrapy/core/http2/agent.py b/scrapy/core/http2/agent.py index 75731f525..215ea9716 100644 --- a/scrapy/core/http2/agent.py +++ b/scrapy/core/http2/agent.py @@ -6,7 +6,12 @@ from twisted.internet.base import ReactorBase from twisted.internet.defer import Deferred from twisted.internet.endpoints import HostnameEndpoint from twisted.python.failure import Failure -from twisted.web.client import URI, BrowserLikePolicyForHTTPS, _StandardEndpointFactory +from twisted.web.client import ( + URI, + BrowserLikePolicyForHTTPS, + ResponseFailed, + _StandardEndpointFactory, +) from twisted.web.error import SchemeNotSupported from scrapy.core.downloader.contextfactory import AcceptableProtocolsContextFactory @@ -83,8 +88,7 @@ class H2ConnectionPool: pending_requests = self._pending_requests.pop(key, None) while pending_requests: d = pending_requests.popleft() - # TODO: this is incorrect, errback takes a single exception - d.errback(errors) # type: ignore[arg-type] + d.errback(ResponseFailed(errors)) def close_connections(self) -> None: """Close all the HTTP/2 connections and remove them from pool From 2b3a8f0d6952e20baffbb2118ca82aa1c8c3cde8 Mon Sep 17 00:00:00 2001 From: Oleg Date: Thu, 2 Mar 2023 12:19:41 +0300 Subject: [PATCH 54/62] Fix a typo in the LOG_FORMAT description (#5839) --- docs/topics/settings.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 420e85d37..219509c1e 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -1090,7 +1090,7 @@ LOG_FORMAT Default: ``'%(asctime)s [%(name)s] %(levelname)s: %(message)s'`` String for formatting log messages. Refer to the -:ref:`Python logging documentation ` for the qwhole +:ref:`Python logging documentation ` for the whole list of available placeholders. .. setting:: LOG_DATEFORMAT From 68ba25cb69cb70c0144c9473c1ada46051c7af5b Mon Sep 17 00:00:00 2001 From: Georgiy Zatserklianyi Date: Sat, 4 Mar 2023 22:15:06 +0200 Subject: [PATCH 55/62] periodic log stats: typo in docs fixed --- conftest.py | 2 +- docs/topics/settings.rst | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/conftest.py b/conftest.py index e1d4b1213..01a83d94d 100644 --- a/conftest.py +++ b/conftest.py @@ -20,7 +20,7 @@ collect_ignore = [ *_py_files("tests/CrawlerRunner"), ] -with Path("tests/ignores.txt").open(encoding="utf-8") as reader: +with Path(r"C:/Users/georg/PycharmProjects/scrapy/tests/ignores.txt").open(encoding="utf-8") as reader: for line in reader: file_path = line.strip() if file_path and file_path[0] != "#": diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 175e8f7f0..fbd32ca7a 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -792,7 +792,7 @@ Allows to define concurrency/delay parameters on per slot(domain) basis: For other downloader slots default settings values will be used: - :setting:`DOWNLOAD_DELAY`: ``delay`` - - :setting:`CONCURRENT_REQUESTS_PER_DOMAIN`: ``concurency`` + - :setting:`CONCURRENT_REQUESTS_PER_DOMAIN`: ``concurrency`` - :setting:`RANDOMIZE_DOWNLOAD_DELAY`: ``randomize_delay`` From be52fe4f67647cea9670197feebbcdb05af6cf96 Mon Sep 17 00:00:00 2001 From: Georgiy Zatserklianyi Date: Mon, 6 Mar 2023 16:24:28 +0200 Subject: [PATCH 56/62] Revert "periodic log stats: typo in docs fixed" This reverts commit 68ba25cb69cb70c0144c9473c1ada46051c7af5b. --- conftest.py | 2 +- docs/topics/settings.rst | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/conftest.py b/conftest.py index 01a83d94d..e1d4b1213 100644 --- a/conftest.py +++ b/conftest.py @@ -20,7 +20,7 @@ collect_ignore = [ *_py_files("tests/CrawlerRunner"), ] -with Path(r"C:/Users/georg/PycharmProjects/scrapy/tests/ignores.txt").open(encoding="utf-8") as reader: +with Path("tests/ignores.txt").open(encoding="utf-8") as reader: for line in reader: file_path = line.strip() if file_path and file_path[0] != "#": diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index fbd32ca7a..175e8f7f0 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -792,7 +792,7 @@ Allows to define concurrency/delay parameters on per slot(domain) basis: For other downloader slots default settings values will be used: - :setting:`DOWNLOAD_DELAY`: ``delay`` - - :setting:`CONCURRENT_REQUESTS_PER_DOMAIN`: ``concurrency`` + - :setting:`CONCURRENT_REQUESTS_PER_DOMAIN`: ``concurency`` - :setting:`RANDOMIZE_DOWNLOAD_DELAY`: ``randomize_delay`` From 218829b1db5e086db640a3f2ef3cb8ae76bb66bb Mon Sep 17 00:00:00 2001 From: Georgiy Zatserklianyi Date: Mon, 6 Mar 2023 16:25:49 +0200 Subject: [PATCH 57/62] per slot settings: typo fixed --- docs/topics/settings.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 175e8f7f0..fbd32ca7a 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -792,7 +792,7 @@ Allows to define concurrency/delay parameters on per slot(domain) basis: For other downloader slots default settings values will be used: - :setting:`DOWNLOAD_DELAY`: ``delay`` - - :setting:`CONCURRENT_REQUESTS_PER_DOMAIN`: ``concurency`` + - :setting:`CONCURRENT_REQUESTS_PER_DOMAIN`: ``concurrency`` - :setting:`RANDOMIZE_DOWNLOAD_DELAY`: ``randomize_delay`` From 8aca47e25dd406a39efebc28019db0226b6ec9aa Mon Sep 17 00:00:00 2001 From: Jalil SA Date: Tue, 7 Mar 2023 03:26:59 -0600 Subject: [PATCH 58/62] Add setdefault to BaseSettings (#5821) --- scrapy/settings/__init__.py | 7 +++++++ tests/test_settings/__init__.py | 13 +++++++++++++ 2 files changed, 20 insertions(+) diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py index fde8fdde4..a3b849f7b 100644 --- a/scrapy/settings/__init__.py +++ b/scrapy/settings/__init__.py @@ -293,6 +293,13 @@ class BaseSettings(MutableMapping): else: self.attributes[name].set(value, priority) + def setdefault(self, name, default=None, priority="project"): + if name not in self: + self.set(name, default, priority) + return default + + return self.attributes[name].value + def setdict(self, values, priority="project"): self.update(values, priority) diff --git a/tests/test_settings/__init__.py b/tests/test_settings/__init__.py index 3e8187096..4a577cd8c 100644 --- a/tests/test_settings/__init__.py +++ b/tests/test_settings/__init__.py @@ -65,6 +65,19 @@ class BaseSettingsTest(unittest.TestCase): def setUp(self): self.settings = BaseSettings() + def test_setdefault_not_existing_value(self): + settings = BaseSettings() + value = settings.setdefault("TEST_OPTION", "value") + self.assertEqual(settings["TEST_OPTION"], "value") + self.assertEqual(value, "value") + self.assertIsNotNone(value) + + def test_setdefault_existing_value(self): + settings = BaseSettings({"TEST_OPTION": "value"}) + value = settings.setdefault("TEST_OPTION", None) + self.assertEqual(settings["TEST_OPTION"], "value") + self.assertEqual(value, "value") + def test_set_new_attribute(self): self.settings.set("TEST_OPTION", "value", 0) self.assertIn("TEST_OPTION", self.settings.attributes) From afafc2781af71e0f32c79ede44be3155709dbd36 Mon Sep 17 00:00:00 2001 From: Mohammadtaher Abbasi Date: Fri, 10 Mar 2023 18:50:15 +0330 Subject: [PATCH 59/62] Fix parse command issues with asyncio (#5824) --- scrapy/commands/parse.py | 28 ++++++++- tests/test_command_parse.py | 110 ++++++++++++++++++++++++++++++++++-- 2 files changed, 130 insertions(+), 8 deletions(-) diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py index 9c3fc86d4..ac937e464 100644 --- a/scrapy/commands/parse.py +++ b/scrapy/commands/parse.py @@ -1,3 +1,4 @@ +import inspect import json import logging from typing import Dict @@ -10,7 +11,11 @@ from scrapy.commands import BaseRunSpiderCommand from scrapy.exceptions import UsageError from scrapy.http import Request from scrapy.utils import display -from scrapy.utils.spider import iterate_spider_output, spidercls_for_request +from scrapy.utils.asyncgen import collect_asyncgen +from scrapy.utils.defer import aiter_errback, deferred_from_coro +from scrapy.utils.log import failure_to_exc_info +from scrapy.utils.misc import arg_to_iter +from scrapy.utils.spider import spidercls_for_request logger = logging.getLogger(__name__) @@ -108,6 +113,25 @@ class Command(BaseRunSpiderCommand): max_requests = max(self.requests) return max(max_items, max_requests) + def handle_exception(self, _failure): + logger.error( + "An error is caught while iterating the async iterable", + exc_info=failure_to_exc_info(_failure), + ) + + def iterate_spider_output(self, result): + if inspect.isasyncgen(result): + d = deferred_from_coro( + collect_asyncgen(aiter_errback(result, self.handle_exception)) + ) + d.addCallback(self.iterate_spider_output) + return d + if inspect.iscoroutine(result): + d = deferred_from_coro(result) + d.addCallback(self.iterate_spider_output) + return d + return arg_to_iter(deferred_from_coro(result)) + def add_items(self, lvl, new_items): old_items = self.items.get(lvl, []) self.items[lvl] = old_items + new_items @@ -165,7 +189,7 @@ class Command(BaseRunSpiderCommand): def run_callback(self, response, callback, cb_kwargs=None): cb_kwargs = cb_kwargs or {} - d = maybeDeferred(iterate_spider_output, callback(response, **cb_kwargs)) + d = maybeDeferred(self.iterate_spider_output, callback(response, **cb_kwargs)) return d def get_callback_from_rules(self, spider, response): diff --git a/tests/test_command_parse.py b/tests/test_command_parse.py index b0fb978e9..037333c03 100644 --- a/tests/test_command_parse.py +++ b/tests/test_command_parse.py @@ -30,14 +30,53 @@ import scrapy from scrapy.linkextractors import LinkExtractor from scrapy.spiders import CrawlSpider, Rule from scrapy.utils.test import get_from_asyncio_queue +import asyncio -class AsyncDefAsyncioSpider(scrapy.Spider): - name = 'asyncdef{self.spider_name}' +class AsyncDefAsyncioReturnSpider(scrapy.Spider): + name = "asyncdef_asyncio_return" async def parse(self, response): + await asyncio.sleep(0.2) status = await get_from_asyncio_queue(response.status) - return [scrapy.Item(), dict(foo='bar')] + self.logger.info(f"Got response {{status}}") + return [{{'id': 1}}, {{'id': 2}}] + +class AsyncDefAsyncioReturnSingleElementSpider(scrapy.Spider): + name = "asyncdef_asyncio_return_single_element" + + async def parse(self, response): + await asyncio.sleep(0.1) + status = await get_from_asyncio_queue(response.status) + self.logger.info(f"Got response {{status}}") + return {{'foo': 42}} + +class AsyncDefAsyncioGenLoopSpider(scrapy.Spider): + name = "asyncdef_asyncio_gen_loop" + + async def parse(self, response): + for i in range(10): + await asyncio.sleep(0.1) + yield {{'foo': i}} + self.logger.info(f"Got response {{response.status}}") + +class AsyncDefAsyncioSpider(scrapy.Spider): + name = "asyncdef_asyncio" + + async def parse(self, response): + await asyncio.sleep(0.2) + status = await get_from_asyncio_queue(response.status) + self.logger.debug(f"Got response {{status}}") + +class AsyncDefAsyncioGenExcSpider(scrapy.Spider): + name = "asyncdef_asyncio_gen_exc" + + async def parse(self, response): + for i in range(10): + await asyncio.sleep(0.1) + yield {{'foo': i}} + if i > 5: + raise ValueError("Stopping the processing") class MySpider(scrapy.Spider): name = '{self.spider_name}' @@ -213,17 +252,76 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}} self.assertIn("INFO: It Works!", _textmode(stderr)) @defer.inlineCallbacks - def test_asyncio_parse_items(self): + def test_async_def_asyncio_parse_items_list(self): status, out, stderr = yield self.execute( [ "--spider", - "asyncdef" + self.spider_name, + "asyncdef_asyncio_return", "-c", "parse", self.url("/html"), ] ) - self.assertIn("""[{}, {'foo': 'bar'}]""", _textmode(out)) + self.assertIn("INFO: Got response 200", _textmode(stderr)) + self.assertIn("{'id': 1}", _textmode(out)) + self.assertIn("{'id': 2}", _textmode(out)) + + @defer.inlineCallbacks + def test_async_def_asyncio_parse_items_single_element(self): + status, out, stderr = yield self.execute( + [ + "--spider", + "asyncdef_asyncio_return_single_element", + "-c", + "parse", + self.url("/html"), + ] + ) + self.assertIn("INFO: Got response 200", _textmode(stderr)) + self.assertIn("{'foo': 42}", _textmode(out)) + + @defer.inlineCallbacks + def test_async_def_asyncgen_parse_loop(self): + status, out, stderr = yield self.execute( + [ + "--spider", + "asyncdef_asyncio_gen_loop", + "-c", + "parse", + self.url("/html"), + ] + ) + self.assertIn("INFO: Got response 200", _textmode(stderr)) + for i in range(10): + self.assertIn(f"{{'foo': {i}}}", _textmode(out)) + + @defer.inlineCallbacks + def test_async_def_asyncgen_parse_exc(self): + status, out, stderr = yield self.execute( + [ + "--spider", + "asyncdef_asyncio_gen_exc", + "-c", + "parse", + self.url("/html"), + ] + ) + self.assertIn("ValueError", _textmode(stderr)) + for i in range(7): + self.assertIn(f"{{'foo': {i}}}", _textmode(out)) + + @defer.inlineCallbacks + def test_async_def_asyncio_parse(self): + _, _, stderr = yield self.execute( + [ + "--spider", + "asyncdef_asyncio", + "-c", + "parse", + self.url("/html"), + ] + ) + self.assertIn("DEBUG: Got response 200", _textmode(stderr)) @defer.inlineCallbacks def test_parse_items(self): From 9411cf4e708ea60c7a6972a6804334f2a799e5c6 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade Date: Mon, 13 Mar 2023 20:03:41 +0200 Subject: [PATCH 60/62] Replace deprecated BadZipfile with BadZipFile (#5849) --- scrapy/downloadermiddlewares/decompression.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/downloadermiddlewares/decompression.py b/scrapy/downloadermiddlewares/decompression.py index 368ca60f7..5839dc243 100644 --- a/scrapy/downloadermiddlewares/decompression.py +++ b/scrapy/downloadermiddlewares/decompression.py @@ -51,7 +51,7 @@ class DecompressionMiddleware: archive = BytesIO(response.body) try: zip_file = zipfile.ZipFile(archive) - except zipfile.BadZipfile: + except zipfile.BadZipFile: return namelist = zip_file.namelist() From 101a0c32d71090989be8d37f7b4a464b433705ba Mon Sep 17 00:00:00 2001 From: Alex Date: Tue, 14 Mar 2023 00:13:44 -0700 Subject: [PATCH 61/62] Support genspider with HTTPS (#5808) --- docs/topics/commands.rst | 3 -- scrapy/commands/genspider.py | 16 +++++-- scrapy/templates/spiders/basic.tmpl | 2 +- scrapy/templates/spiders/crawl.tmpl | 2 +- scrapy/templates/spiders/csvfeed.tmpl | 2 +- scrapy/templates/spiders/xmlfeed.tmpl | 2 +- tests/test_commands.py | 63 ++++++++++++++++++++++++--- 7 files changed, 74 insertions(+), 16 deletions(-) diff --git a/docs/topics/commands.rst b/docs/topics/commands.rst index 54fd5d663..106045fc0 100644 --- a/docs/topics/commands.rst +++ b/docs/topics/commands.rst @@ -238,9 +238,6 @@ genspider Create a new spider in the current folder or in the current project's ``spiders`` folder, if called from inside a project. The ```` parameter is set as the spider's ``name``, while ```` is used to generate the ``allowed_domains`` and ``start_urls`` spider's attributes. -.. note:: Even if an HTTPS URL is specified, the protocol used in - ``start_urls`` is always HTTP. This is a known issue: :issue:`3553`. - Usage example:: $ scrapy genspider -l diff --git a/scrapy/commands/genspider.py b/scrapy/commands/genspider.py index c1565a138..68cbe8ff6 100644 --- a/scrapy/commands/genspider.py +++ b/scrapy/commands/genspider.py @@ -31,6 +31,14 @@ def extract_domain(url): return o.netloc +def verify_url_scheme(url): + """Check url for scheme and insert https if none found.""" + parsed = urlparse(url) + if parsed.scheme == "" and parsed.netloc == "": + parsed = urlparse("//" + url)._replace(scheme="https") + return parsed.geturl() + + class Command(ScrapyCommand): requires_project = False default_settings = {"LOG_ENABLED": False} @@ -91,7 +99,7 @@ class Command(ScrapyCommand): raise UsageError() name, url = args[0:2] - domain = extract_domain(url) + url = verify_url_scheme(url) module = sanitize_module_name(name) if self.settings.get("BOT_NAME") == module: @@ -103,18 +111,20 @@ class Command(ScrapyCommand): template_file = self._find_template(opts.template) if template_file: - self._genspider(module, name, domain, opts.template, template_file) + self._genspider(module, name, url, opts.template, template_file) if opts.edit: self.exitcode = os.system(f'scrapy edit "{name}"') - def _genspider(self, module, name, domain, template_name, template_file): + def _genspider(self, module, name, url, template_name, template_file): """Generate the spider module, based on the given template""" capitalized_module = "".join(s.capitalize() for s in module.split("_")) + domain = extract_domain(url) tvars = { "project_name": self.settings.get("BOT_NAME"), "ProjectName": string_camelcase(self.settings.get("BOT_NAME")), "module": module, "name": name, + "url": url, "domain": domain, "classname": f"{capitalized_module}Spider", } diff --git a/scrapy/templates/spiders/basic.tmpl b/scrapy/templates/spiders/basic.tmpl index d3ba19553..20e777271 100644 --- a/scrapy/templates/spiders/basic.tmpl +++ b/scrapy/templates/spiders/basic.tmpl @@ -4,7 +4,7 @@ import scrapy class $classname(scrapy.Spider): name = "$name" allowed_domains = ["$domain"] - start_urls = ["http://$domain/"] + start_urls = ["$url"] def parse(self, response): pass diff --git a/scrapy/templates/spiders/crawl.tmpl b/scrapy/templates/spiders/crawl.tmpl index 2e467e632..36d05e43a 100644 --- a/scrapy/templates/spiders/crawl.tmpl +++ b/scrapy/templates/spiders/crawl.tmpl @@ -6,7 +6,7 @@ from scrapy.spiders import CrawlSpider, Rule class $classname(CrawlSpider): name = "$name" allowed_domains = ["$domain"] - start_urls = ["http://$domain/"] + start_urls = ["$url"] rules = (Rule(LinkExtractor(allow=r"Items/"), callback="parse_item", follow=True),) diff --git a/scrapy/templates/spiders/csvfeed.tmpl b/scrapy/templates/spiders/csvfeed.tmpl index ce9c1dd20..fe96878dc 100644 --- a/scrapy/templates/spiders/csvfeed.tmpl +++ b/scrapy/templates/spiders/csvfeed.tmpl @@ -4,7 +4,7 @@ from scrapy.spiders import CSVFeedSpider class $classname(CSVFeedSpider): name = "$name" allowed_domains = ["$domain"] - start_urls = ["http://$domain/feed.csv"] + start_urls = ["$url"] #headers = ["id", "name", "description", "image_link"] #delimiter = "\t" diff --git a/scrapy/templates/spiders/xmlfeed.tmpl b/scrapy/templates/spiders/xmlfeed.tmpl index 6b50e4cf4..ac62d78d1 100644 --- a/scrapy/templates/spiders/xmlfeed.tmpl +++ b/scrapy/templates/spiders/xmlfeed.tmpl @@ -4,7 +4,7 @@ from scrapy.spiders import XMLFeedSpider class $classname(XMLFeedSpider): name = "$name" allowed_domains = ["$domain"] - start_urls = ["http://$domain/feed.xml"] + start_urls = ["$url"] iterator = "iternodes" # you can change this; see the docs itertag = "item" # change it accordingly diff --git a/tests/test_commands.py b/tests/test_commands.py index 00ddcdd3e..014f50e92 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -541,7 +541,7 @@ class GenspiderCommandTest(CommandTest): ).group(1), ) self.assertEqual( - f"http://{domain}/", + f"https://{domain}", self.find_in_file( Path(self.proj_mod_path, "spiders", "test_name.py"), r"start_urls\s*=\s*\[['\"](.+)['\"]\]", @@ -549,13 +549,64 @@ class GenspiderCommandTest(CommandTest): ) def test_url_schema(self): - self.test_url("http://test.com", "test.com") + self.test_url("https://test.com", "test.com") - def test_url_path(self): - self.test_url("test.com/some/other/page", "test.com") + def test_template_start_urls( + self, url="test.com", expected="https://test.com", template="basic" + ): + self.assertEqual( + 0, self.call("genspider", "-t", template, "--force", "test_name", url) + ) + self.assertEqual( + expected, + self.find_in_file( + Path(self.proj_mod_path, "spiders", "test_name.py"), + r"start_urls\s*=\s*\[['\"](.+)['\"]\]", + ).group(1), + ) - def test_url_schema_path(self): - self.test_url("https://test.com/some/other/page", "test.com") + def test_genspider_basic_start_urls(self): + self.test_template_start_urls("https://test.com", "https://test.com", "basic") + self.test_template_start_urls("http://test.com", "http://test.com", "basic") + self.test_template_start_urls( + "http://test.com/other/path", "http://test.com/other/path", "basic" + ) + self.test_template_start_urls( + "test.com/other/path", "https://test.com/other/path", "basic" + ) + + def test_genspider_crawl_start_urls(self): + self.test_template_start_urls("https://test.com", "https://test.com", "crawl") + self.test_template_start_urls("http://test.com", "http://test.com", "crawl") + self.test_template_start_urls( + "http://test.com/other/path", "http://test.com/other/path", "crawl" + ) + self.test_template_start_urls( + "test.com/other/path", "https://test.com/other/path", "crawl" + ) + self.test_template_start_urls("test.com", "https://test.com", "crawl") + + def test_genspider_xmlfeed_start_urls(self): + self.test_template_start_urls( + "https://test.com/feed.xml", "https://test.com/feed.xml", "xmlfeed" + ) + self.test_template_start_urls( + "http://test.com/feed.xml", "http://test.com/feed.xml", "xmlfeed" + ) + self.test_template_start_urls( + "test.com/feed.xml", "https://test.com/feed.xml", "xmlfeed" + ) + + def test_genspider_csvfeed_start_urls(self): + self.test_template_start_urls( + "https://test.com/feed.csv", "https://test.com/feed.csv", "csvfeed" + ) + self.test_template_start_urls( + "http://test.com/feed.xml", "http://test.com/feed.xml", "csvfeed" + ) + self.test_template_start_urls( + "test.com/feed.csv", "https://test.com/feed.csv", "csvfeed" + ) class GenspiderStandaloneCommandTest(ProjectTest): From d60b4edd11436e61284615ec7ce89f8ac7e46d9a Mon Sep 17 00:00:00 2001 From: auxsvr Date: Tue, 14 Mar 2023 10:31:13 +0200 Subject: [PATCH 62/62] Prevent an edge case that creates an extra event loop (#5832) --- scrapy/utils/reactor.py | 36 +++++++++++++++++++++++++++++------- tests/test_utils_asyncio.py | 11 ++++++++++- 2 files changed, 39 insertions(+), 8 deletions(-) diff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.py index c20948fd3..7f67d036a 100644 --- a/scrapy/utils/reactor.py +++ b/scrapy/utils/reactor.py @@ -1,10 +1,11 @@ import asyncio import sys from contextlib import suppress -from warnings import catch_warnings, filterwarnings +from warnings import catch_warnings, filterwarnings, warn from twisted.internet import asyncioreactor, error +from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.utils.misc import load_object @@ -54,7 +55,31 @@ class CallLaterOnce: return self._func(*self._a, **self._kw) +def set_asyncio_event_loop_policy(): + """The policy functions from asyncio often behave unexpectedly, + so we restrict their use to the absolutely essential case. + This should only be used to install the reactor. + """ + _get_asyncio_event_loop_policy() + + def get_asyncio_event_loop_policy(): + warn( + "Call to deprecated function " + "scrapy.utils.reactor.get_asyncio_event_loop_policy().\n" + "\n" + "Please use get_event_loop, new_event_loop and set_event_loop" + " from asyncio instead, as the corresponding policy methods may lead" + " to unexpected behaviour.\n" + "This function is replaced by set_asyncio_event_loop_policy and" + " is meant to be used only when the reactor is being installed.", + category=ScrapyDeprecationWarning, + stacklevel=2, + ) + return _get_asyncio_event_loop_policy() + + +def _get_asyncio_event_loop_policy(): policy = asyncio.get_event_loop_policy() if ( sys.version_info >= (3, 8) @@ -63,7 +88,6 @@ def get_asyncio_event_loop_policy(): ): policy = asyncio.WindowsSelectorEventLoopPolicy() asyncio.set_event_loop_policy(policy) - return policy @@ -73,6 +97,7 @@ def install_reactor(reactor_path, event_loop_path=None): path if the asyncio reactor is enabled""" reactor_class = load_object(reactor_path) if reactor_class is asyncioreactor.AsyncioSelectorReactor: + set_asyncio_event_loop_policy() with suppress(error.ReactorAlreadyInstalledError): event_loop = set_asyncio_event_loop(event_loop_path) asyncioreactor.install(eventloop=event_loop) @@ -90,7 +115,6 @@ def _get_asyncio_event_loop(): def set_asyncio_event_loop(event_loop_path): """Sets and returns the event loop with specified import path.""" - policy = get_asyncio_event_loop_policy() if event_loop_path is not None: event_loop_class = load_object(event_loop_path) event_loop = event_loop_class() @@ -109,15 +133,13 @@ def set_asyncio_event_loop(event_loop_path): message="There is no current event loop", category=DeprecationWarning, ) - event_loop = policy.get_event_loop() + event_loop = asyncio.get_event_loop() except RuntimeError: # `get_event_loop` raises RuntimeError when called with no asyncio # event loop yet installed in the following scenarios: - # - From a thread other than the main thread. For example, when - # using ``scrapy shell``. # - Previsibly on Python 3.14 and later. # https://github.com/python/cpython/issues/100160#issuecomment-1345581902 - event_loop = policy.new_event_loop() + event_loop = asyncio.new_event_loop() asyncio.set_event_loop(event_loop) return event_loop diff --git a/tests/test_utils_asyncio.py b/tests/test_utils_asyncio.py index 746731a2e..01d0ee043 100644 --- a/tests/test_utils_asyncio.py +++ b/tests/test_utils_asyncio.py @@ -1,9 +1,14 @@ +import asyncio import warnings from unittest import TestCase from pytest import mark -from scrapy.utils.reactor import install_reactor, is_asyncio_reactor_installed +from scrapy.utils.reactor import ( + install_reactor, + is_asyncio_reactor_installed, + set_asyncio_event_loop, +) @mark.usefixtures("reactor_pytest") @@ -23,3 +28,7 @@ class AsyncioTest(TestCase): from twisted.internet import reactor assert original_reactor == reactor + + async def test_set_asyncio_event_loop(self): + install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor") + assert set_asyncio_event_loop() is asyncio.get_running_loop()