From 9eea22fb0ca99193b7f38f9f9398b278d64ea977 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 29 Apr 2024 18:59:31 +0500 Subject: [PATCH 01/12] Full typing for scrapy/cmdline.py. --- scrapy/cmdline.py | 47 ++++++++++++++++++++++++++++++++--------------- 1 file changed, 32 insertions(+), 15 deletions(-) diff --git a/scrapy/cmdline.py b/scrapy/cmdline.py index 6580ba9ce..4df5698a6 100644 --- a/scrapy/cmdline.py +++ b/scrapy/cmdline.py @@ -4,18 +4,22 @@ import inspect import os import sys from importlib.metadata import entry_points +from typing import Any, Callable, Dict, Iterable, List, Optional, Tuple, Type import scrapy from scrapy.commands import BaseRunSpiderCommand, ScrapyCommand, ScrapyHelpFormatter from scrapy.crawler import CrawlerProcess from scrapy.exceptions import UsageError +from scrapy.settings import BaseSettings, Settings from scrapy.utils.misc import walk_modules from scrapy.utils.project import get_project_settings, inside_project from scrapy.utils.python import garbage_collect class ScrapyArgumentParser(argparse.ArgumentParser): - def _parse_optional(self, arg_string): + def _parse_optional( + self, arg_string: str + ) -> Optional[Tuple[Optional[argparse.Action], str, Optional[str]]]: # if starts with -: it means that is a parameter not a argument if arg_string[:2] == "-:": return None @@ -23,7 +27,7 @@ class ScrapyArgumentParser(argparse.ArgumentParser): return super()._parse_optional(arg_string) -def _iter_command_classes(module_name): +def _iter_command_classes(module_name: str) -> Iterable[Type[ScrapyCommand]]: # TODO: add `name` attribute to commands and merge this function with # scrapy.utils.spider.iter_spider_classes for module in walk_modules(module_name): @@ -37,8 +41,8 @@ def _iter_command_classes(module_name): yield obj -def _get_commands_from_module(module, inproject): - d = {} +def _get_commands_from_module(module: str, inproject: bool) -> Dict[str, ScrapyCommand]: + d: Dict[str, ScrapyCommand] = {} for cmd in _iter_command_classes(module): if inproject or not cmd.requires_project: cmdname = cmd.__module__.split(".")[-1] @@ -46,8 +50,10 @@ def _get_commands_from_module(module, inproject): return d -def _get_commands_from_entry_points(inproject, group="scrapy.commands"): - cmds = {} +def _get_commands_from_entry_points( + inproject: bool, group: str = "scrapy.commands" +) -> Dict[str, ScrapyCommand]: + cmds: Dict[str, ScrapyCommand] = {} if sys.version_info >= (3, 10): eps = entry_points(group=group) else: @@ -61,7 +67,9 @@ def _get_commands_from_entry_points(inproject, group="scrapy.commands"): return cmds -def _get_commands_dict(settings, inproject): +def _get_commands_dict( + settings: BaseSettings, inproject: bool +) -> Dict[str, ScrapyCommand]: cmds = _get_commands_from_module("scrapy.commands", inproject) cmds.update(_get_commands_from_entry_points(inproject)) cmds_module = settings["COMMANDS_MODULE"] @@ -70,16 +78,17 @@ def _get_commands_dict(settings, inproject): return cmds -def _pop_command_name(argv): +def _pop_command_name(argv: List[str]) -> Optional[str]: i = 0 for arg in argv[1:]: if not arg.startswith("-"): del argv[i] return arg i += 1 + return None -def _print_header(settings, inproject): +def _print_header(settings: BaseSettings, inproject: bool) -> None: version = scrapy.__version__ if inproject: print(f"Scrapy {version} - active project: {settings['BOT_NAME']}\n") @@ -88,7 +97,7 @@ def _print_header(settings, inproject): print(f"Scrapy {version} - no active project\n") -def _print_commands(settings, inproject): +def _print_commands(settings: BaseSettings, inproject: bool) -> None: _print_header(settings, inproject) print("Usage:") print(" scrapy [options] [args]\n") @@ -103,13 +112,17 @@ def _print_commands(settings, inproject): print('Use "scrapy -h" to see more info about a command') -def _print_unknown_command(settings, cmdname, inproject): +def _print_unknown_command( + settings: BaseSettings, cmdname: str, inproject: bool +) -> None: _print_header(settings, inproject) print(f"Unknown command: {cmdname}\n") print('Use "scrapy" to see available commands') -def _run_print_help(parser, func, *a, **kw): +def _run_print_help( + parser: argparse.ArgumentParser, func: Callable, *a: Any, **kw: Any +) -> None: try: func(*a, **kw) except UsageError as e: @@ -120,7 +133,9 @@ def _run_print_help(parser, func, *a, **kw): sys.exit(2) -def execute(argv=None, settings=None): +def execute( + argv: Optional[List[str]] = None, settings: Optional[Settings] = None +) -> None: if argv is None: argv = sys.argv @@ -162,14 +177,16 @@ def execute(argv=None, settings=None): sys.exit(cmd.exitcode) -def _run_command(cmd, args, opts): +def _run_command(cmd: ScrapyCommand, args: List[str], opts: argparse.Namespace) -> None: if opts.profile: _run_command_profiled(cmd, args, opts) else: cmd.run(args, opts) -def _run_command_profiled(cmd, args, opts): +def _run_command_profiled( + cmd: ScrapyCommand, args: List[str], opts: argparse.Namespace +) -> None: if opts.profile: sys.stderr.write(f"scrapy: writing cProfile stats to {opts.profile!r}\n") loc = locals() From fc1a83e7c42dc5142eb9190fdca887385254a7a6 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 29 Apr 2024 19:07:31 +0500 Subject: [PATCH 02/12] Full typing for scrapy/item.py. --- scrapy/item.py | 40 ++++++++++++++++++++++++---------------- 1 file changed, 24 insertions(+), 16 deletions(-) diff --git a/scrapy/item.py b/scrapy/item.py index d3eb90b7b..e04e994ef 100644 --- a/scrapy/item.py +++ b/scrapy/item.py @@ -4,14 +4,20 @@ Scrapy Item See documentation in docs/topics/item.rst """ +from __future__ import annotations + from abc import ABCMeta from collections.abc import MutableMapping from copy import deepcopy from pprint import pformat -from typing import Dict +from typing import TYPE_CHECKING, Any, Dict, Iterator, KeysView, NoReturn, Tuple from scrapy.utils.trackref import object_ref +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self + class Field(dict): """Container of field metadata""" @@ -23,7 +29,9 @@ class ItemMeta(ABCMeta): .. _metaclass: https://realpython.com/python-metaclasses """ - def __new__(mcs, class_name, bases, attrs): + def __new__( + mcs, class_name: str, bases: Tuple[type, ...], attrs: Dict[str, Any] + ) -> ItemMeta: classcell = attrs.pop("__classcell__", None) new_bases = tuple(base._class for base in bases if hasattr(base, "_class")) _class = super().__new__(mcs, "x_" + class_name, new_bases, attrs) @@ -44,7 +52,7 @@ class ItemMeta(ABCMeta): return super().__new__(mcs, class_name, bases, new_attrs) -class Item(MutableMapping, object_ref, metaclass=ItemMeta): +class Item(MutableMapping[str, Any], object_ref, metaclass=ItemMeta): """ Base class for scraped items. @@ -69,51 +77,51 @@ class Item(MutableMapping, object_ref, metaclass=ItemMeta): fields: Dict[str, Field] - def __init__(self, *args, **kwargs): - self._values = {} + def __init__(self, *args: Any, **kwargs: Any): + self._values: Dict[str, Any] = {} if args or kwargs: # avoid creating dict for most common case for k, v in dict(*args, **kwargs).items(): self[k] = v - def __getitem__(self, key): + def __getitem__(self, key: str) -> Any: return self._values[key] - def __setitem__(self, key, value): + def __setitem__(self, key: str, value: Any) -> None: if key in self.fields: self._values[key] = value else: raise KeyError(f"{self.__class__.__name__} does not support field: {key}") - def __delitem__(self, key): + def __delitem__(self, key: str) -> None: del self._values[key] - def __getattr__(self, name): + def __getattr__(self, name: str) -> NoReturn: if name in self.fields: raise AttributeError(f"Use item[{name!r}] to get field value") raise AttributeError(name) - def __setattr__(self, name, value): + def __setattr__(self, name: str, value: Any) -> None: if not name.startswith("_"): raise AttributeError(f"Use item[{name!r}] = {value!r} to set field value") super().__setattr__(name, value) - def __len__(self): + def __len__(self) -> int: return len(self._values) - def __iter__(self): + def __iter__(self) -> Iterator[str]: return iter(self._values) __hash__ = object_ref.__hash__ - def keys(self): + def keys(self) -> KeysView[str]: return self._values.keys() - def __repr__(self): + def __repr__(self) -> str: return pformat(dict(self)) - def copy(self): + def copy(self) -> Self: return self.__class__(self) - def deepcopy(self): + def deepcopy(self) -> Self: """Return a :func:`~copy.deepcopy` of this item.""" return deepcopy(self) From 08a265b6ff9bc47774173238b06715154b39e534 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 29 Apr 2024 19:10:00 +0500 Subject: [PATCH 03/12] Full typing for scrapy/extension.py. --- scrapy/extension.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/scrapy/extension.py b/scrapy/extension.py index 6be14450c..8221b675e 100644 --- a/scrapy/extension.py +++ b/scrapy/extension.py @@ -4,7 +4,10 @@ The Extension Manager See documentation in docs/topics/extensions.rst """ +from typing import Any, List + from scrapy.middleware import MiddlewareManager +from scrapy.settings import Settings from scrapy.utils.conf import build_component_list @@ -12,5 +15,5 @@ class ExtensionManager(MiddlewareManager): component_name = "extension" @classmethod - def _get_mwlist_from_settings(cls, settings): + def _get_mwlist_from_settings(cls, settings: Settings) -> List[Any]: return build_component_list(settings.getwithbase("EXTENSIONS")) From 38020e0b0481d2b15792757669272d6d3bf4b14f Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 29 Apr 2024 20:12:30 +0500 Subject: [PATCH 04/12] Full typing for scrapy/mail.py. --- scrapy/mail.py | 98 +++++++++++++++++++++++++++++++------------------- 1 file changed, 62 insertions(+), 36 deletions(-) diff --git a/scrapy/mail.py b/scrapy/mail.py index 7cb5ef454..56adba934 100644 --- a/scrapy/mail.py +++ b/scrapy/mail.py @@ -14,11 +14,23 @@ from email.mime.nonmultipart import MIMENonMultipart from email.mime.text import MIMEText from email.utils import formatdate from io import BytesIO -from typing import TYPE_CHECKING, Optional +from typing import ( + IO, + TYPE_CHECKING, + Any, + Callable, + Dict, + List, + Optional, + Sequence, + Tuple, + Union, +) from twisted import version as twisted_version from twisted.internet import ssl from twisted.internet.defer import Deferred +from twisted.python.failure import Failure from twisted.python.versions import Version from scrapy.settings import BaseSettings @@ -26,6 +38,9 @@ from scrapy.utils.misc import arg_to_iter from scrapy.utils.python import to_bytes if TYPE_CHECKING: + # imports twisted.internet.reactor + from twisted.mail.smtp import ESMTPSenderFactory + # typing.Self requires Python 3.11 from typing_extensions import Self @@ -37,7 +52,7 @@ logger = logging.getLogger(__name__) COMMASPACE = ", " -def _to_bytes_or_none(text): +def _to_bytes_or_none(text: Union[str, bytes, None]) -> Optional[bytes]: if text is None: return None return to_bytes(text) @@ -46,23 +61,23 @@ def _to_bytes_or_none(text): class MailSender: def __init__( self, - smtphost="localhost", - mailfrom="scrapy@localhost", - smtpuser=None, - smtppass=None, - smtpport=25, - smtptls=False, - smtpssl=False, - debug=False, + smtphost: str = "localhost", + mailfrom: str = "scrapy@localhost", + smtpuser: Optional[str] = None, + smtppass: Optional[str] = None, + smtpport: int = 25, + smtptls: bool = False, + smtpssl: bool = False, + debug: bool = False, ): - self.smtphost = smtphost - self.smtpport = smtpport - self.smtpuser = _to_bytes_or_none(smtpuser) - self.smtppass = _to_bytes_or_none(smtppass) - self.smtptls = smtptls - self.smtpssl = smtpssl - self.mailfrom = mailfrom - self.debug = debug + self.smtphost: str = smtphost + self.smtpport: int = smtpport + self.smtpuser: Optional[bytes] = _to_bytes_or_none(smtpuser) + self.smtppass: Optional[bytes] = _to_bytes_or_none(smtppass) + self.smtptls: bool = smtptls + self.smtpssl: bool = smtpssl + self.mailfrom: str = mailfrom + self.debug: bool = debug @classmethod def from_settings(cls, settings: BaseSettings) -> Self: @@ -78,14 +93,14 @@ class MailSender: def send( self, - to, - subject, - body, - cc=None, - attachs=(), - mimetype="text/plain", - charset=None, - _callback=None, + to: Union[str, List[str]], + subject: str, + body: str, + cc: Union[str, List[str], None] = None, + attachs: Sequence[Tuple[str, str, IO]] = (), + mimetype: str = "text/plain", + charset: Optional[str] = None, + _callback: Optional[Callable[..., None]] = None, ) -> Optional[Deferred]: from twisted.internet import reactor @@ -142,13 +157,15 @@ class MailSender: dfd.addCallbacks( callback=self._sent_ok, errback=self._sent_failed, - callbackArgs=[to, cc, subject, len(attachs)], - errbackArgs=[to, cc, subject, len(attachs)], + callbackArgs=(to, cc, subject, len(attachs)), + errbackArgs=(to, cc, subject, len(attachs)), ) reactor.addSystemEventTrigger("before", "shutdown", lambda: dfd) return dfd - def _sent_ok(self, result, to, cc, subject, nattachs): + def _sent_ok( + self, result: Any, to: List[str], cc: List[str], subject: str, nattachs: int + ) -> None: logger.info( "Mail sent OK: To=%(mailto)s Cc=%(mailcc)s " 'Subject="%(mailsubject)s" Attachs=%(mailattachs)d', @@ -160,7 +177,14 @@ class MailSender: }, ) - def _sent_failed(self, failure, to, cc, subject, nattachs): + def _sent_failed( + self, + failure: Failure, + to: List[str], + cc: List[str], + subject: str, + nattachs: int, + ) -> Failure: errstr = str(failure.value) logger.error( "Unable to send mail: To=%(mailto)s Cc=%(mailcc)s " @@ -176,13 +200,13 @@ class MailSender: ) return failure - def _sendmail(self, to_addrs, msg): + def _sendmail(self, to_addrs: List[str], msg: bytes) -> Deferred: from twisted.internet import reactor - msg = BytesIO(msg) - d = Deferred() + msg_io = BytesIO(msg) + d: Deferred = Deferred() - factory = self._create_sender_factory(to_addrs, msg, d) + factory = self._create_sender_factory(to_addrs, msg_io, d) if self.smtpssl: reactor.connectSSL( @@ -193,10 +217,12 @@ class MailSender: return d - def _create_sender_factory(self, to_addrs, msg, d): + def _create_sender_factory( + self, to_addrs: List[str], msg: IO, d: Deferred + ) -> ESMTPSenderFactory: from twisted.mail.smtp import ESMTPSenderFactory - factory_keywords = { + factory_keywords: Dict[str, Any] = { "heloFallback": True, "requireAuthentication": False, "requireTransportSecurity": self.smtptls, From 0c8e21b8acfcac2d6d057f3c67b2252d0fa660e3 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 29 Apr 2024 20:57:31 +0500 Subject: [PATCH 05/12] Full typing for scrapy/pqueues.py. --- scrapy/core/downloader/__init__.py | 2 +- scrapy/pqueues.py | 138 +++++++++++++++++++++-------- 2 files changed, 100 insertions(+), 40 deletions(-) diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index ecd3e8b56..f88da41ea 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -143,7 +143,7 @@ class Downloader: return key, self.slots[key] - def _get_slot_key(self, request: Request, spider: Spider) -> str: + def _get_slot_key(self, request: Request, spider: Any) -> str: if self.DOWNLOAD_SLOT in request.meta: return cast(str, request.meta[self.DOWNLOAD_SLOT]) diff --git a/scrapy/pqueues.py b/scrapy/pqueues.py index 593667f1f..213ad590d 100644 --- a/scrapy/pqueues.py +++ b/scrapy/pqueues.py @@ -1,12 +1,32 @@ +from __future__ import annotations + import hashlib import logging +from typing import ( + TYPE_CHECKING, + Dict, + Iterable, + List, + Optional, + Protocol, + Tuple, + Type, + cast, +) +from scrapy import Request +from scrapy.core.downloader import Downloader +from scrapy.crawler import Crawler from scrapy.utils.misc import build_from_crawler +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self + logger = logging.getLogger(__name__) -def _path_safe(text): +def _path_safe(text: str) -> str: """ Return a filesystem-safe version of a string ``text`` @@ -24,6 +44,18 @@ def _path_safe(text): return "-".join([pathable_slot, unique_slot]) +class QueueProtocol(Protocol): + """Protocol for downstream queues of ``ScrapyPriorityQueue``.""" + + def push(self, request: Request) -> None: ... + + def pop(self) -> Optional[Request]: ... + + def close(self) -> None: ... + + def __len__(self) -> int: ... + + class ScrapyPriorityQueue: """A priority queue implemented using multiple internal queues (typically, FIFO queues). It uses one internal queue for each priority value. The internal @@ -51,18 +83,30 @@ class ScrapyPriorityQueue: """ @classmethod - def from_crawler(cls, crawler, downstream_queue_cls, key, startprios=()): + def from_crawler( + cls, + crawler: Crawler, + downstream_queue_cls: Type[QueueProtocol], + key: str, + startprios: Iterable[int] = (), + ) -> Self: return cls(crawler, downstream_queue_cls, key, startprios) - def __init__(self, crawler, downstream_queue_cls, key, startprios=()): - self.crawler = crawler - self.downstream_queue_cls = downstream_queue_cls - self.key = key - self.queues = {} - self.curprio = None + def __init__( + self, + crawler: Crawler, + downstream_queue_cls: Type[QueueProtocol], + key: str, + startprios: Iterable[int] = (), + ): + self.crawler: Crawler = crawler + self.downstream_queue_cls: Type[QueueProtocol] = downstream_queue_cls + self.key: str = key + self.queues: Dict[int, QueueProtocol] = {} + self.curprio: Optional[int] = None self.init_prios(startprios) - def init_prios(self, startprios): + def init_prios(self, startprios: Iterable[int]) -> None: if not startprios: return @@ -71,17 +115,17 @@ class ScrapyPriorityQueue: self.curprio = min(startprios) - def qfactory(self, key): + def qfactory(self, key: int) -> QueueProtocol: return build_from_crawler( self.downstream_queue_cls, self.crawler, self.key + "/" + str(key), ) - def priority(self, request): + def priority(self, request: Request) -> int: return -request.priority - def push(self, request): + def push(self, request: Request) -> None: priority = self.priority(request) if priority not in self.queues: self.queues[priority] = self.qfactory(priority) @@ -90,9 +134,9 @@ class ScrapyPriorityQueue: if self.curprio is None or priority < self.curprio: self.curprio = priority - def pop(self): + def pop(self) -> Optional[Request]: if self.curprio is None: - return + return None q = self.queues[self.curprio] m = q.pop() if not q: @@ -102,7 +146,7 @@ class ScrapyPriorityQueue: self.curprio = min(prios) if prios else None return m - def peek(self): + def peek(self) -> Optional[Request]: """Returns the next object to be returned by :meth:`pop`, but without removing it from the queue. @@ -112,30 +156,32 @@ class ScrapyPriorityQueue: if self.curprio is None: return None queue = self.queues[self.curprio] - return queue.peek() + # Protocols can't declare optional members + return cast(Request, queue.peek()) # type: ignore[attr-defined] - def close(self): - active = [] + def close(self) -> List[int]: + active: List[int] = [] for p, q in self.queues.items(): active.append(p) q.close() return active - def __len__(self): + def __len__(self) -> int: return sum(len(x) for x in self.queues.values()) if self.queues else 0 class DownloaderInterface: - def __init__(self, crawler): - self.downloader = crawler.engine.downloader + def __init__(self, crawler: Crawler): + assert crawler.engine + self.downloader: Downloader = crawler.engine.downloader - def stats(self, possible_slots): + def stats(self, possible_slots: Iterable[str]) -> List[Tuple[int, str]]: return [(self._active_downloads(slot), slot) for slot in possible_slots] - def get_slot_key(self, request): + def get_slot_key(self, request: Request) -> str: return self.downloader._get_slot_key(request, None) - def _active_downloads(self, slot): + def _active_downloads(self, slot: str) -> int: """Return a number of requests in a Downloader for a given slot""" if slot not in self.downloader.slots: return 0 @@ -149,10 +195,22 @@ class DownloaderAwarePriorityQueue: """ @classmethod - def from_crawler(cls, crawler, downstream_queue_cls, key, startprios=()): + def from_crawler( + cls, + crawler: Crawler, + downstream_queue_cls: Type[QueueProtocol], + key: str, + startprios: Optional[Dict[str, Iterable[int]]] = None, + ) -> Self: return cls(crawler, downstream_queue_cls, key, startprios) - def __init__(self, crawler, downstream_queue_cls, key, slot_startprios=()): + def __init__( + self, + crawler: Crawler, + downstream_queue_cls: Type[QueueProtocol], + key: str, + slot_startprios: Optional[Dict[str, Iterable[int]]] = None, + ): if crawler.settings.getint("CONCURRENT_REQUESTS_PER_IP") != 0: raise ValueError( f'"{self.__class__}" does not support CONCURRENT_REQUESTS_PER_IP' @@ -169,16 +227,18 @@ class DownloaderAwarePriorityQueue: "queue class can be resumed." ) - self._downloader_interface = DownloaderInterface(crawler) - self.downstream_queue_cls = downstream_queue_cls - self.key = key - self.crawler = crawler + self._downloader_interface: DownloaderInterface = DownloaderInterface(crawler) + self.downstream_queue_cls: Type[QueueProtocol] = downstream_queue_cls + self.key: str = key + self.crawler: Crawler = crawler - self.pqueues = {} # slot -> priority queue + self.pqueues: Dict[str, ScrapyPriorityQueue] = {} # slot -> priority queue for slot, startprios in (slot_startprios or {}).items(): self.pqueues[slot] = self.pqfactory(slot, startprios) - def pqfactory(self, slot, startprios=()): + def pqfactory( + self, slot: str, startprios: Iterable[int] = () + ) -> ScrapyPriorityQueue: return ScrapyPriorityQueue( self.crawler, self.downstream_queue_cls, @@ -186,11 +246,11 @@ class DownloaderAwarePriorityQueue: startprios, ) - def pop(self): + def pop(self) -> Optional[Request]: stats = self._downloader_interface.stats(self.pqueues) if not stats: - return + return None slot = min(stats)[1] queue = self.pqueues[slot] @@ -199,14 +259,14 @@ class DownloaderAwarePriorityQueue: del self.pqueues[slot] return request - def push(self, request): + def push(self, request: Request) -> None: slot = self._downloader_interface.get_slot_key(request) if slot not in self.pqueues: self.pqueues[slot] = self.pqfactory(slot) queue = self.pqueues[slot] queue.push(request) - def peek(self): + def peek(self) -> Optional[Request]: """Returns the next object to be returned by :meth:`pop`, but without removing it from the queue. @@ -220,13 +280,13 @@ class DownloaderAwarePriorityQueue: queue = self.pqueues[slot] return queue.peek() - def close(self): + def close(self) -> Dict[str, List[int]]: active = {slot: queue.close() for slot, queue in self.pqueues.items()} self.pqueues.clear() return active - def __len__(self): + def __len__(self) -> int: return sum(len(x) for x in self.pqueues.values()) if self.pqueues else 0 - def __contains__(self, slot): + def __contains__(self, slot: str) -> bool: return slot in self.pqueues From 21fa0761818c158ae9fc35b49ea8d2300f0fa510 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 29 Apr 2024 22:41:58 +0500 Subject: [PATCH 06/12] Fix MutableMapping import for Python 3.8. --- scrapy/item.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/scrapy/item.py b/scrapy/item.py index e04e994ef..2daea64cc 100644 --- a/scrapy/item.py +++ b/scrapy/item.py @@ -7,10 +7,18 @@ See documentation in docs/topics/item.rst from __future__ import annotations from abc import ABCMeta -from collections.abc import MutableMapping from copy import deepcopy from pprint import pformat -from typing import TYPE_CHECKING, Any, Dict, Iterator, KeysView, NoReturn, Tuple +from typing import ( + TYPE_CHECKING, + Any, + Dict, + Iterator, + KeysView, + MutableMapping, + NoReturn, + Tuple, +) from scrapy.utils.trackref import object_ref From ad35ffdb0da052d0df194ce5dc1ba7e8d823190f Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 29 Apr 2024 23:10:03 +0500 Subject: [PATCH 07/12] Full typing for scrapy/resolver.py. --- scrapy/resolver.py | 67 +++++++++++++++++++++++++++------------------- 1 file changed, 39 insertions(+), 28 deletions(-) diff --git a/scrapy/resolver.py b/scrapy/resolver.py index e2e8beff4..ba7cd716b 100644 --- a/scrapy/resolver.py +++ b/scrapy/resolver.py @@ -1,8 +1,12 @@ -from typing import Any +from __future__ import annotations + +from typing import TYPE_CHECKING, Any, List, Optional, Sequence, Type from twisted.internet import defer -from twisted.internet.base import ThreadedResolver +from twisted.internet.base import ReactorBase, ThreadedResolver +from twisted.internet.defer import Deferred from twisted.internet.interfaces import ( + IAddress, IHostnameResolver, IHostResolution, IResolutionReceiver, @@ -12,6 +16,12 @@ from zope.interface.declarations import implementer, provider from scrapy.utils.datatypes import LocalCache +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self + + from scrapy.crawler import Crawler + # TODO: cache misses dnscache: LocalCache[str, Any] = LocalCache(10000) @@ -22,65 +32,66 @@ class CachingThreadedResolver(ThreadedResolver): Default caching resolver. IPv4 only, supports setting a timeout value for DNS requests. """ - def __init__(self, reactor, cache_size, timeout): + def __init__(self, reactor: ReactorBase, cache_size: int, timeout: float): super().__init__(reactor) dnscache.limit = cache_size self.timeout = timeout @classmethod - def from_crawler(cls, crawler, reactor): + def from_crawler(cls, crawler: Crawler, reactor: ReactorBase) -> Self: if crawler.settings.getbool("DNSCACHE_ENABLED"): cache_size = crawler.settings.getint("DNSCACHE_SIZE") else: cache_size = 0 return cls(reactor, cache_size, crawler.settings.getfloat("DNS_TIMEOUT")) - def install_on_reactor(self): + def install_on_reactor(self) -> None: self.reactor.installResolver(self) - def getHostByName(self, name: str, timeout=None): + def getHostByName(self, name: str, timeout: Sequence[int] = ()) -> Deferred[str]: if name in dnscache: return defer.succeed(dnscache[name]) # in Twisted<=16.6, getHostByName() is always called with # a default timeout of 60s (actually passed as (1, 3, 11, 45) tuple), # so the input argument above is simply overridden # to enforce Scrapy's DNS_TIMEOUT setting's value - timeout = (self.timeout,) + # The timeout arg is typed as Sequence[int] but supports floats. + timeout = (self.timeout,) # type: ignore[assignment] d = super().getHostByName(name, timeout) if dnscache.limit: d.addCallback(self._cache_result, name) return d - def _cache_result(self, result, name): + def _cache_result(self, result: Any, name: str) -> Any: dnscache[name] = result return result @implementer(IHostResolution) class HostResolution: - def __init__(self, name): - self.name = name + def __init__(self, name: str): + self.name: str = name - def cancel(self): + def cancel(self) -> None: raise NotImplementedError() @provider(IResolutionReceiver) class _CachingResolutionReceiver: - def __init__(self, resolutionReceiver, hostName): - self.resolutionReceiver = resolutionReceiver - self.hostName = hostName - self.addresses = [] + def __init__(self, resolutionReceiver: IResolutionReceiver, hostName: str): + self.resolutionReceiver: IResolutionReceiver = resolutionReceiver + self.hostName: str = hostName + self.addresses: List[IAddress] = [] - def resolutionBegan(self, resolution): + def resolutionBegan(self, resolution: IHostResolution) -> None: self.resolutionReceiver.resolutionBegan(resolution) self.resolution = resolution - def addressResolved(self, address): + def addressResolved(self, address: IAddress) -> None: self.resolutionReceiver.addressResolved(address) self.addresses.append(address) - def resolutionComplete(self): + def resolutionComplete(self) -> None: self.resolutionReceiver.resolutionComplete() if self.addresses: dnscache[self.hostName] = self.addresses @@ -93,30 +104,30 @@ class CachingHostnameResolver: does not support setting a timeout value for DNS requests. """ - def __init__(self, reactor, cache_size): - self.reactor = reactor - self.original_resolver = reactor.nameResolver + def __init__(self, reactor: ReactorBase, cache_size: int): + self.reactor: ReactorBase = reactor + self.original_resolver: IHostnameResolver = reactor.nameResolver dnscache.limit = cache_size @classmethod - def from_crawler(cls, crawler, reactor): + def from_crawler(cls, crawler: Crawler, reactor: ReactorBase) -> Self: if crawler.settings.getbool("DNSCACHE_ENABLED"): cache_size = crawler.settings.getint("DNSCACHE_SIZE") else: cache_size = 0 return cls(reactor, cache_size) - def install_on_reactor(self): + def install_on_reactor(self) -> None: self.reactor.installNameResolver(self) def resolveHostName( self, - resolutionReceiver, + resolutionReceiver: IResolutionReceiver, hostName: str, - portNumber=0, - addressTypes=None, - transportSemantics="TCP", - ): + portNumber: int = 0, + addressTypes: Optional[Sequence[Type[IAddress]]] = None, + transportSemantics: str = "TCP", + ) -> IHostResolution: try: addresses = dnscache[hostName] except KeyError: From b749db92e5b15c974b0d77280c22b63000ad4263 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 29 Apr 2024 23:17:59 +0500 Subject: [PATCH 08/12] Full typing for scrapy/robotstxt.py. --- scrapy/robotstxt.py | 65 +++++++++++++++++++++++---------------------- 1 file changed, 33 insertions(+), 32 deletions(-) diff --git a/scrapy/robotstxt.py b/scrapy/robotstxt.py index ad06137e2..a33f73306 100644 --- a/scrapy/robotstxt.py +++ b/scrapy/robotstxt.py @@ -3,9 +3,10 @@ from __future__ import annotations import logging import sys from abc import ABCMeta, abstractmethod -from typing import TYPE_CHECKING, Union +from typing import TYPE_CHECKING, Optional, Union from warnings import warn +from scrapy import Spider from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.utils.python import to_unicode @@ -18,12 +19,14 @@ if TYPE_CHECKING: logger = logging.getLogger(__name__) -def decode_robotstxt(robotstxt_body, spider, to_native_str_type=False): +def decode_robotstxt( + robotstxt_body: bytes, spider: Optional[Spider], to_native_str_type: bool = False +) -> str: try: if to_native_str_type: - robotstxt_body = to_unicode(robotstxt_body) + body_decoded = to_unicode(robotstxt_body) else: - robotstxt_body = robotstxt_body.decode("utf-8", errors="ignore") + body_decoded = robotstxt_body.decode("utf-8", errors="ignore") except UnicodeDecodeError: # If we found garbage or robots.txt in an encoding other than UTF-8, disregard it. # Switch to 'allow all' state. @@ -33,8 +36,8 @@ def decode_robotstxt(robotstxt_body, spider, to_native_str_type=False): exc_info=sys.exc_info(), extra={"spider": spider}, ) - robotstxt_body = "" - return robotstxt_body + body_decoded = "" + return body_decoded class RobotParser(metaclass=ABCMeta): @@ -66,82 +69,80 @@ class RobotParser(metaclass=ABCMeta): class PythonRobotParser(RobotParser): - def __init__(self, robotstxt_body, spider): + def __init__(self, robotstxt_body: bytes, spider: Optional[Spider]): from urllib.robotparser import RobotFileParser - self.spider = spider - robotstxt_body = decode_robotstxt( - robotstxt_body, spider, to_native_str_type=True - ) - self.rp = RobotFileParser() - self.rp.parse(robotstxt_body.splitlines()) + self.spider: Optional[Spider] = spider + body_decoded = decode_robotstxt(robotstxt_body, spider, to_native_str_type=True) + self.rp: RobotFileParser = RobotFileParser() + self.rp.parse(body_decoded.splitlines()) @classmethod - def from_crawler(cls, crawler, robotstxt_body): + def from_crawler(cls, crawler: Crawler, robotstxt_body: bytes) -> Self: spider = None if not crawler else crawler.spider o = cls(robotstxt_body, spider) return o - def allowed(self, url, user_agent): + def allowed(self, url: Union[str, bytes], user_agent: Union[str, bytes]) -> bool: user_agent = to_unicode(user_agent) url = to_unicode(url) return self.rp.can_fetch(user_agent, url) class ReppyRobotParser(RobotParser): - def __init__(self, robotstxt_body, spider): + def __init__(self, robotstxt_body: bytes, spider: Optional[Spider]): warn("ReppyRobotParser is deprecated.", ScrapyDeprecationWarning, stacklevel=2) from reppy.robots import Robots - self.spider = spider + self.spider: Optional[Spider] = spider self.rp = Robots.parse("", robotstxt_body) @classmethod - def from_crawler(cls, crawler, robotstxt_body): + def from_crawler(cls, crawler: Crawler, robotstxt_body: bytes) -> Self: spider = None if not crawler else crawler.spider o = cls(robotstxt_body, spider) return o - def allowed(self, url, user_agent): + def allowed(self, url: Union[str, bytes], user_agent: Union[str, bytes]) -> bool: return self.rp.allowed(url, user_agent) class RerpRobotParser(RobotParser): - def __init__(self, robotstxt_body, spider): + def __init__(self, robotstxt_body: bytes, spider: Optional[Spider]): from robotexclusionrulesparser import RobotExclusionRulesParser - self.spider = spider - self.rp = RobotExclusionRulesParser() - robotstxt_body = decode_robotstxt(robotstxt_body, spider) - self.rp.parse(robotstxt_body) + self.spider: Optional[Spider] = spider + self.rp: RobotExclusionRulesParser = RobotExclusionRulesParser() + body_decoded = decode_robotstxt(robotstxt_body, spider) + self.rp.parse(body_decoded) @classmethod - def from_crawler(cls, crawler, robotstxt_body): + def from_crawler(cls, crawler: Crawler, robotstxt_body: bytes) -> Self: spider = None if not crawler else crawler.spider o = cls(robotstxt_body, spider) return o - def allowed(self, url, user_agent): + def allowed(self, url: Union[str, bytes], user_agent: Union[str, bytes]) -> bool: user_agent = to_unicode(user_agent) url = to_unicode(url) return self.rp.is_allowed(user_agent, url) class ProtegoRobotParser(RobotParser): - def __init__(self, robotstxt_body, spider): + def __init__(self, robotstxt_body: bytes, spider: Optional[Spider]): from protego import Protego - self.spider = spider - robotstxt_body = decode_robotstxt(robotstxt_body, spider) - self.rp = Protego.parse(robotstxt_body) + self.spider: Optional[Spider] = spider + body_decoded = decode_robotstxt(robotstxt_body, spider) + self.rp = Protego.parse(body_decoded) @classmethod - def from_crawler(cls, crawler, robotstxt_body): + def from_crawler(cls, crawler: Crawler, robotstxt_body: bytes) -> Self: spider = None if not crawler else crawler.spider o = cls(robotstxt_body, spider) return o - def allowed(self, url, user_agent): + def allowed(self, url: Union[str, bytes], user_agent: Union[str, bytes]) -> bool: user_agent = to_unicode(user_agent) url = to_unicode(url) return self.rp.can_fetch(url, user_agent) From 5f7fd2a653407da3eb3e53c853209d9bfb6b275f Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 29 Apr 2024 23:39:31 +0500 Subject: [PATCH 09/12] Full typing for scrapy/squeues.py. --- scrapy/squeues.py | 58 +++++++++++++++++++++++++++++++---------------- 1 file changed, 39 insertions(+), 19 deletions(-) diff --git a/scrapy/squeues.py b/scrapy/squeues.py index e20f60f06..4676b058e 100644 --- a/scrapy/squeues.py +++ b/scrapy/squeues.py @@ -2,20 +2,28 @@ Scheduler queues """ +from __future__ import annotations + import marshal import pickle # nosec from os import PathLike from pathlib import Path -from typing import Union +from typing import TYPE_CHECKING, Any, Callable, Optional, Type, Union from queuelib import queue +from scrapy import Request +from scrapy.crawler import Crawler from scrapy.utils.request import request_from_dict +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self -def _with_mkdir(queue_class): + +def _with_mkdir(queue_class: Type[queue.BaseQueue]) -> Type[queue.BaseQueue]: class DirectoriesCreated(queue_class): - def __init__(self, path: Union[str, PathLike], *args, **kwargs): + def __init__(self, path: Union[str, PathLike], *args: Any, **kwargs: Any): dirname = Path(path).parent if not dirname.exists(): dirname.mkdir(parents=True, exist_ok=True) @@ -24,18 +32,23 @@ def _with_mkdir(queue_class): return DirectoriesCreated -def _serializable_queue(queue_class, serialize, deserialize): +def _serializable_queue( + queue_class: Type[queue.BaseQueue], + serialize: Callable[[Any], bytes], + deserialize: Callable[[bytes], Any], +) -> Type[queue.BaseQueue]: class SerializableQueue(queue_class): - def push(self, obj): + def push(self, obj: Any) -> None: s = serialize(obj) super().push(s) - def pop(self): + def pop(self) -> Optional[Any]: s = super().pop() if s: return deserialize(s) + return None - def peek(self): + def peek(self) -> Optional[Any]: """Returns the next object to be returned by :meth:`pop`, but without removing it from the queue. @@ -50,31 +63,36 @@ def _serializable_queue(queue_class, serialize, deserialize): ) from ex if s: return deserialize(s) + return None return SerializableQueue -def _scrapy_serialization_queue(queue_class): +def _scrapy_serialization_queue( + queue_class: Type[queue.BaseQueue], +) -> Type[queue.BaseQueue]: class ScrapyRequestQueue(queue_class): - def __init__(self, crawler, key): + def __init__(self, crawler: Crawler, key: str): self.spider = crawler.spider super().__init__(key) @classmethod - def from_crawler(cls, crawler, key, *args, **kwargs): + def from_crawler( + cls, crawler: Crawler, key: str, *args: Any, **kwargs: Any + ) -> Self: return cls(crawler, key) - def push(self, request): - request = request.to_dict(spider=self.spider) - return super().push(request) + def push(self, request: Request) -> None: + request_dict = request.to_dict(spider=self.spider) + super().push(request_dict) - def pop(self): + def pop(self) -> Optional[Request]: request = super().pop() if not request: return None return request_from_dict(request, spider=self.spider) - def peek(self): + def peek(self) -> Optional[Request]: """Returns the next object to be returned by :meth:`pop`, but without removing it from the queue. @@ -89,13 +107,15 @@ def _scrapy_serialization_queue(queue_class): return ScrapyRequestQueue -def _scrapy_non_serialization_queue(queue_class): +def _scrapy_non_serialization_queue( + queue_class: Type[queue.BaseQueue], +) -> Type[queue.BaseQueue]: class ScrapyRequestQueue(queue_class): @classmethod - def from_crawler(cls, crawler, *args, **kwargs): + def from_crawler(cls, crawler: Crawler, *args: Any, **kwargs: Any) -> Self: return cls() - def peek(self): + def peek(self) -> Optional[Any]: """Returns the next object to be returned by :meth:`pop`, but without removing it from the queue. @@ -113,7 +133,7 @@ def _scrapy_non_serialization_queue(queue_class): return ScrapyRequestQueue -def _pickle_serialize(obj): +def _pickle_serialize(obj: Any) -> bytes: try: return pickle.dumps(obj, protocol=4) # Both pickle.PicklingError and AttributeError can be raised by pickle.dump(s) From 203fa9667fb69f6251c0a44b14cf0450ce769a32 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 29 Apr 2024 23:47:55 +0500 Subject: [PATCH 10/12] Add queue typing to scrapy/core/scheduler.py. --- scrapy/core/scheduler.py | 33 +++++++++++++++++---------------- scrapy/pqueues.py | 3 ++- 2 files changed, 19 insertions(+), 17 deletions(-) diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.py index b2209e53f..ab59c0d14 100644 --- a/scrapy/core/scheduler.py +++ b/scrapy/core/scheduler.py @@ -4,13 +4,15 @@ import json import logging from abc import abstractmethod from pathlib import Path -from typing import TYPE_CHECKING, Any, Optional, Type, TypeVar, cast +from typing import TYPE_CHECKING, Any, Optional, Type, cast +from queuelib.queue import BaseQueue from twisted.internet.defer import Deferred from scrapy.crawler import Crawler from scrapy.dupefilters import BaseDupeFilter from scrapy.http.request import Request +from scrapy.pqueues import ScrapyPriorityQueue from scrapy.spiders import Spider from scrapy.statscollectors import StatsCollector from scrapy.utils.job import job_dir @@ -121,9 +123,6 @@ class BaseScheduler(metaclass=BaseSchedulerMeta): raise NotImplementedError() -SchedulerTV = TypeVar("SchedulerTV", bound="Scheduler") - - class Scheduler(BaseScheduler): """ Default Scrapy scheduler. This implementation also handles duplication @@ -179,24 +178,24 @@ class Scheduler(BaseScheduler): self, dupefilter: BaseDupeFilter, jobdir: Optional[str] = None, - dqclass=None, - mqclass=None, + dqclass: Optional[Type[BaseQueue]] = None, + mqclass: Optional[Type[BaseQueue]] = None, logunser: bool = False, stats: Optional[StatsCollector] = None, - pqclass=None, + pqclass: Optional[Type[ScrapyPriorityQueue]] = None, crawler: Optional[Crawler] = None, ): self.df: BaseDupeFilter = dupefilter self.dqdir: Optional[str] = self._dqdir(jobdir) - self.pqclass = pqclass - self.dqclass = dqclass - self.mqclass = mqclass + self.pqclass: Optional[Type[ScrapyPriorityQueue]] = pqclass + self.dqclass: Optional[Type[BaseQueue]] = dqclass + self.mqclass: Optional[Type[BaseQueue]] = mqclass self.logunser: bool = logunser self.stats: Optional[StatsCollector] = stats self.crawler: Optional[Crawler] = crawler @classmethod - def from_crawler(cls: Type[SchedulerTV], crawler: Crawler) -> SchedulerTV: + def from_crawler(cls, crawler: Crawler) -> Self: """ Factory method, initializes the scheduler with arguments taken from the crawl settings """ @@ -221,9 +220,9 @@ class Scheduler(BaseScheduler): (2) initialize the disk queue if the ``jobdir`` attribute is a valid directory (3) return the result of the dupefilter's ``open`` method """ - self.spider = spider - self.mqs = self._mq() - self.dqs = self._dq() if self.dqdir else None + self.spider: Spider = spider + self.mqs: ScrapyPriorityQueue = self._mq() + self.dqs: Optional[ScrapyPriorityQueue] = self._dq() if self.dqdir else None return self.df.open() def close(self, reason: str) -> Optional[Deferred]: @@ -320,9 +319,10 @@ class Scheduler(BaseScheduler): return self.dqs.pop() return None - def _mq(self): + def _mq(self) -> ScrapyPriorityQueue: """Create a new priority queue instance, with in-memory storage""" assert self.crawler + assert self.pqclass return build_from_crawler( self.pqclass, self.crawler, @@ -330,10 +330,11 @@ class Scheduler(BaseScheduler): key="", ) - def _dq(self): + def _dq(self) -> ScrapyPriorityQueue: """Create a new priority queue instance, with disk storage""" assert self.crawler assert self.dqdir + assert self.pqclass state = self._read_dqs_state(self.dqdir) q = build_from_crawler( self.pqclass, diff --git a/scrapy/pqueues.py b/scrapy/pqueues.py index 213ad590d..773825c5e 100644 --- a/scrapy/pqueues.py +++ b/scrapy/pqueues.py @@ -16,13 +16,14 @@ from typing import ( from scrapy import Request from scrapy.core.downloader import Downloader -from scrapy.crawler import Crawler from scrapy.utils.misc import build_from_crawler if TYPE_CHECKING: # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy.crawler import Crawler + logger = logging.getLogger(__name__) From bd0d4cee885744c7ea38185ec42f0137e7632b79 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sat, 4 May 2024 16:12:44 +0500 Subject: [PATCH 11/12] Fixes for queuelib. --- scrapy/core/scheduler.py | 4 +++- scrapy/squeues.py | 21 +++++++++++---------- 2 files changed, 14 insertions(+), 11 deletions(-) diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.py index ab59c0d14..f30a5d9c9 100644 --- a/scrapy/core/scheduler.py +++ b/scrapy/core/scheduler.py @@ -6,7 +6,6 @@ from abc import abstractmethod from pathlib import Path from typing import TYPE_CHECKING, Any, Optional, Type, cast -from queuelib.queue import BaseQueue from twisted.internet.defer import Deferred from scrapy.crawler import Crawler @@ -19,6 +18,9 @@ from scrapy.utils.job import job_dir from scrapy.utils.misc import build_from_crawler, load_object if TYPE_CHECKING: + # requires queuelib >= 1.6.2 + from queuelib.queue import BaseQueue + # typing.Self requires Python 3.11 from typing_extensions import Self diff --git a/scrapy/squeues.py b/scrapy/squeues.py index 4676b058e..6f80ee388 100644 --- a/scrapy/squeues.py +++ b/scrapy/squeues.py @@ -22,7 +22,7 @@ if TYPE_CHECKING: def _with_mkdir(queue_class: Type[queue.BaseQueue]) -> Type[queue.BaseQueue]: - class DirectoriesCreated(queue_class): + class DirectoriesCreated(queue_class): # type: ignore[valid-type,misc] def __init__(self, path: Union[str, PathLike], *args: Any, **kwargs: Any): dirname = Path(path).parent if not dirname.exists(): @@ -37,7 +37,7 @@ def _serializable_queue( serialize: Callable[[Any], bytes], deserialize: Callable[[bytes], Any], ) -> Type[queue.BaseQueue]: - class SerializableQueue(queue_class): + class SerializableQueue(queue_class): # type: ignore[valid-type,misc] def push(self, obj: Any) -> None: s = serialize(obj) super().push(s) @@ -71,7 +71,7 @@ def _serializable_queue( def _scrapy_serialization_queue( queue_class: Type[queue.BaseQueue], ) -> Type[queue.BaseQueue]: - class ScrapyRequestQueue(queue_class): + class ScrapyRequestQueue(queue_class): # type: ignore[valid-type,misc] def __init__(self, crawler: Crawler, key: str): self.spider = crawler.spider super().__init__(key) @@ -110,7 +110,7 @@ def _scrapy_serialization_queue( def _scrapy_non_serialization_queue( queue_class: Type[queue.BaseQueue], ) -> Type[queue.BaseQueue]: - class ScrapyRequestQueue(queue_class): + class ScrapyRequestQueue(queue_class): # type: ignore[valid-type,misc] @classmethod def from_crawler(cls, crawler: Crawler, *args: Any, **kwargs: Any) -> Self: return cls() @@ -142,17 +142,18 @@ def _pickle_serialize(obj: Any) -> bytes: raise ValueError(str(e)) from e +# queue.*Queue aren't subclasses of queue.BaseQueue _PickleFifoSerializationDiskQueue = _serializable_queue( - _with_mkdir(queue.FifoDiskQueue), _pickle_serialize, pickle.loads + _with_mkdir(queue.FifoDiskQueue), _pickle_serialize, pickle.loads # type: ignore[arg-type] ) _PickleLifoSerializationDiskQueue = _serializable_queue( - _with_mkdir(queue.LifoDiskQueue), _pickle_serialize, pickle.loads + _with_mkdir(queue.LifoDiskQueue), _pickle_serialize, pickle.loads # type: ignore[arg-type] ) _MarshalFifoSerializationDiskQueue = _serializable_queue( - _with_mkdir(queue.FifoDiskQueue), marshal.dumps, marshal.loads + _with_mkdir(queue.FifoDiskQueue), marshal.dumps, marshal.loads # type: ignore[arg-type] ) _MarshalLifoSerializationDiskQueue = _serializable_queue( - _with_mkdir(queue.LifoDiskQueue), marshal.dumps, marshal.loads + _with_mkdir(queue.LifoDiskQueue), marshal.dumps, marshal.loads # type: ignore[arg-type] ) # public queue classes @@ -160,5 +161,5 @@ PickleFifoDiskQueue = _scrapy_serialization_queue(_PickleFifoSerializationDiskQu PickleLifoDiskQueue = _scrapy_serialization_queue(_PickleLifoSerializationDiskQueue) MarshalFifoDiskQueue = _scrapy_serialization_queue(_MarshalFifoSerializationDiskQueue) MarshalLifoDiskQueue = _scrapy_serialization_queue(_MarshalLifoSerializationDiskQueue) -FifoMemoryQueue = _scrapy_non_serialization_queue(queue.FifoMemoryQueue) -LifoMemoryQueue = _scrapy_non_serialization_queue(queue.LifoMemoryQueue) +FifoMemoryQueue = _scrapy_non_serialization_queue(queue.FifoMemoryQueue) # type: ignore[arg-type] +LifoMemoryQueue = _scrapy_non_serialization_queue(queue.LifoMemoryQueue) # type: ignore[arg-type] From 2cba7896d26dda51ff2e598300363531ebd328b8 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 6 May 2024 14:31:24 +0500 Subject: [PATCH 12/12] Small fix for _get_slot_key(). --- scrapy/core/downloader/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index f88da41ea..98e1af6fb 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -143,7 +143,7 @@ class Downloader: return key, self.slots[key] - def _get_slot_key(self, request: Request, spider: Any) -> str: + def _get_slot_key(self, request: Request, spider: Optional[Spider]) -> str: if self.DOWNLOAD_SLOT in request.meta: return cast(str, request.meta[self.DOWNLOAD_SLOT])