RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #6333 from wRAR/typing-scrapy 

Full typing for scrapy/*.py
This commit is contained in:
Andrey Rakhmatullin 2024-05-06 14:04:16 +04:00 committed by GitHub
parent d338982580 2cba7896d2
commit 563ecbe966
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
10 changed files with 373 additions and 214 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

47
scrapy/cmdline.py
View File

@ -4,18 +4,22 @@ import inspect
import os
import sys
from importlib.metadata import entry_points
from typing import Any, Callable, Dict, Iterable, List, Optional, Tuple, Type
import scrapy
from scrapy.commands import BaseRunSpiderCommand, ScrapyCommand, ScrapyHelpFormatter
from scrapy.crawler import CrawlerProcess
from scrapy.exceptions import UsageError
from scrapy.settings import BaseSettings, Settings
from scrapy.utils.misc import walk_modules
from scrapy.utils.project import get_project_settings, inside_project
from scrapy.utils.python import garbage_collect
class ScrapyArgumentParser(argparse.ArgumentParser):
def _parse_optional(self, arg_string):
def _parse_optional(
self, arg_string: str
) -> Optional[Tuple[Optional[argparse.Action], str, Optional[str]]]:
# if starts with -: it means that is a parameter not a argument
if arg_string[:2] == "-:":
return None
@ -23,7 +27,7 @@ class ScrapyArgumentParser(argparse.ArgumentParser):
return super()._parse_optional(arg_string)
def _iter_command_classes(module_name):
def _iter_command_classes(module_name: str) -> Iterable[Type[ScrapyCommand]]:
# TODO: add `name` attribute to commands and merge this function with
# scrapy.utils.spider.iter_spider_classes
for module in walk_modules(module_name):
@ -37,8 +41,8 @@ def _iter_command_classes(module_name):
yield obj
def _get_commands_from_module(module, inproject):
d = {}
def _get_commands_from_module(module: str, inproject: bool) -> Dict[str, ScrapyCommand]:
d: Dict[str, ScrapyCommand] = {}
for cmd in _iter_command_classes(module):
if inproject or not cmd.requires_project:
cmdname = cmd.__module__.split(".")[-1]
@ -46,8 +50,10 @@ def _get_commands_from_module(module, inproject):
return d
def _get_commands_from_entry_points(inproject, group="scrapy.commands"):
cmds = {}
def _get_commands_from_entry_points(
inproject: bool, group: str = "scrapy.commands"
) -> Dict[str, ScrapyCommand]:
cmds: Dict[str, ScrapyCommand] = {}
if sys.version_info >= (3, 10):
eps = entry_points(group=group)
else:
@ -61,7 +67,9 @@ def _get_commands_from_entry_points(inproject, group="scrapy.commands"):
return cmds
def _get_commands_dict(settings, inproject):
def _get_commands_dict(
settings: BaseSettings, inproject: bool
) -> Dict[str, ScrapyCommand]:
cmds = _get_commands_from_module("scrapy.commands", inproject)
cmds.update(_get_commands_from_entry_points(inproject))
cmds_module = settings["COMMANDS_MODULE"]
@ -70,16 +78,17 @@ def _get_commands_dict(settings, inproject):
return cmds
def _pop_command_name(argv):
def _pop_command_name(argv: List[str]) -> Optional[str]:
i = 0
for arg in argv[1:]:
if not arg.startswith("-"):
del argv[i]
return arg
i += 1
return None
def _print_header(settings, inproject):
def _print_header(settings: BaseSettings, inproject: bool) -> None:
version = scrapy.__version__
if inproject:
print(f"Scrapy {version} - active project: {settings['BOT_NAME']}\n")
@ -88,7 +97,7 @@ def _print_header(settings, inproject):
print(f"Scrapy {version} - no active project\n")
def _print_commands(settings, inproject):
def _print_commands(settings: BaseSettings, inproject: bool) -> None:
_print_header(settings, inproject)
print("Usage:")
print(" scrapy <command> [options] [args]\n")
@ -103,13 +112,17 @@ def _print_commands(settings, inproject):
print('Use "scrapy <command> -h" to see more info about a command')
def _print_unknown_command(settings, cmdname, inproject):
def _print_unknown_command(
settings: BaseSettings, cmdname: str, inproject: bool
) -> None:
_print_header(settings, inproject)
print(f"Unknown command: {cmdname}\n")
print('Use "scrapy" to see available commands')
def _run_print_help(parser, func, *a, **kw):
def _run_print_help(
parser: argparse.ArgumentParser, func: Callable, *a: Any, **kw: Any
) -> None:
try:
func(*a, **kw)
except UsageError as e:
@ -120,7 +133,9 @@ def _run_print_help(parser, func, *a, **kw):
sys.exit(2)
def execute(argv=None, settings=None):
def execute(
argv: Optional[List[str]] = None, settings: Optional[Settings] = None
) -> None:
if argv is None:
argv = sys.argv
@ -162,14 +177,16 @@ def execute(argv=None, settings=None):
sys.exit(cmd.exitcode)
def _run_command(cmd, args, opts):
def _run_command(cmd: ScrapyCommand, args: List[str], opts: argparse.Namespace) -> None:
if opts.profile:
_run_command_profiled(cmd, args, opts)
else:
cmd.run(args, opts)
def _run_command_profiled(cmd, args, opts):
def _run_command_profiled(
cmd: ScrapyCommand, args: List[str], opts: argparse.Namespace
) -> None:
if opts.profile:
sys.stderr.write(f"scrapy: writing cProfile stats to {opts.profile!r}\n")
loc = locals()

2
scrapy/core/downloader/__init__.py
View File

@ -143,7 +143,7 @@ class Downloader:
return key, self.slots[key]
def _get_slot_key(self, request: Request, spider: Spider) -> str:
def _get_slot_key(self, request: Request, spider: Optional[Spider]) -> str:
if self.DOWNLOAD_SLOT in request.meta:
return cast(str, request.meta[self.DOWNLOAD_SLOT])

35
scrapy/core/scheduler.py
View File

@ -4,19 +4,23 @@ import json
import logging
from abc import abstractmethod
from pathlib import Path
from typing import TYPE_CHECKING, Any, Optional, Type, TypeVar, cast
from typing import TYPE_CHECKING, Any, Optional, Type, cast
from twisted.internet.defer import Deferred
from scrapy.crawler import Crawler
from scrapy.dupefilters import BaseDupeFilter
from scrapy.http.request import Request
from scrapy.pqueues import ScrapyPriorityQueue
from scrapy.spiders import Spider
from scrapy.statscollectors import StatsCollector
from scrapy.utils.job import job_dir
from scrapy.utils.misc import build_from_crawler, load_object
if TYPE_CHECKING:
# requires queuelib >= 1.6.2
from queuelib.queue import BaseQueue
# typing.Self requires Python 3.11
from typing_extensions import Self
@ -121,9 +125,6 @@ class BaseScheduler(metaclass=BaseSchedulerMeta):
raise NotImplementedError()
SchedulerTV = TypeVar("SchedulerTV", bound="Scheduler")
class Scheduler(BaseScheduler):
"""
Default Scrapy scheduler. This implementation also handles duplication
@ -179,24 +180,24 @@ class Scheduler(BaseScheduler):
self,
dupefilter: BaseDupeFilter,
jobdir: Optional[str] = None,
dqclass=None,
mqclass=None,
dqclass: Optional[Type[BaseQueue]] = None,
mqclass: Optional[Type[BaseQueue]] = None,
logunser: bool = False,
stats: Optional[StatsCollector] = None,
pqclass=None,
pqclass: Optional[Type[ScrapyPriorityQueue]] = None,
crawler: Optional[Crawler] = None,
):
self.df: BaseDupeFilter = dupefilter
self.dqdir: Optional[str] = self._dqdir(jobdir)
self.pqclass = pqclass
self.dqclass = dqclass
self.mqclass = mqclass
self.pqclass: Optional[Type[ScrapyPriorityQueue]] = pqclass
self.dqclass: Optional[Type[BaseQueue]] = dqclass
self.mqclass: Optional[Type[BaseQueue]] = mqclass
self.logunser: bool = logunser
self.stats: Optional[StatsCollector] = stats
self.crawler: Optional[Crawler] = crawler
@classmethod
def from_crawler(cls: Type[SchedulerTV], crawler: Crawler) -> SchedulerTV:
def from_crawler(cls, crawler: Crawler) -> Self:
"""
Factory method, initializes the scheduler with arguments taken from the crawl settings
"""
@ -221,9 +222,9 @@ class Scheduler(BaseScheduler):
(2) initialize the disk queue if the ``jobdir`` attribute is a valid directory
(3) return the result of the dupefilter's ``open`` method
"""
self.spider = spider
self.mqs = self._mq()
self.dqs = self._dq() if self.dqdir else None
self.spider: Spider = spider
self.mqs: ScrapyPriorityQueue = self._mq()
self.dqs: Optional[ScrapyPriorityQueue] = self._dq() if self.dqdir else None
return self.df.open()
def close(self, reason: str) -> Optional[Deferred]:
@ -320,9 +321,10 @@ class Scheduler(BaseScheduler):
return self.dqs.pop()
return None
def _mq(self):
def _mq(self) -> ScrapyPriorityQueue:
"""Create a new priority queue instance, with in-memory storage"""
assert self.crawler
assert self.pqclass
return build_from_crawler(
self.pqclass,
self.crawler,
@ -330,10 +332,11 @@ class Scheduler(BaseScheduler):
key="",
)
def _dq(self):
def _dq(self) -> ScrapyPriorityQueue:
"""Create a new priority queue instance, with disk storage"""
assert self.crawler
assert self.dqdir
assert self.pqclass
state = self._read_dqs_state(self.dqdir)
q = build_from_crawler(
self.pqclass,

5
scrapy/extension.py
View File

@ -4,7 +4,10 @@ The Extension Manager
See documentation in docs/topics/extensions.rst
"""
from typing import Any, List
from scrapy.middleware import MiddlewareManager
from scrapy.settings import Settings
from scrapy.utils.conf import build_component_list
@ -12,5 +15,5 @@ class ExtensionManager(MiddlewareManager):
component_name = "extension"
@classmethod
def _get_mwlist_from_settings(cls, settings):
def _get_mwlist_from_settings(cls, settings: Settings) -> List[Any]:
return build_component_list(settings.getwithbase("EXTENSIONS"))

50
scrapy/item.py
View File

@ -4,14 +4,28 @@ Scrapy Item
See documentation in docs/topics/item.rst
"""
from __future__ import annotations
from abc import ABCMeta
from collections.abc import MutableMapping
from copy import deepcopy
from pprint import pformat
from typing import Dict
from typing import (
TYPE_CHECKING,
Any,
Dict,
Iterator,
KeysView,
MutableMapping,
NoReturn,
Tuple,
)
from scrapy.utils.trackref import object_ref
if TYPE_CHECKING:
# typing.Self requires Python 3.11
from typing_extensions import Self
class Field(dict):
"""Container of field metadata"""
@ -23,7 +37,9 @@ class ItemMeta(ABCMeta):
.. _metaclass: https://realpython.com/python-metaclasses
"""
def __new__(mcs, class_name, bases, attrs):
def __new__(
mcs, class_name: str, bases: Tuple[type, ...], attrs: Dict[str, Any]
) -> ItemMeta:
classcell = attrs.pop("__classcell__", None)
new_bases = tuple(base._class for base in bases if hasattr(base, "_class"))
_class = super().__new__(mcs, "x_" + class_name, new_bases, attrs)
@ -44,7 +60,7 @@ class ItemMeta(ABCMeta):
return super().__new__(mcs, class_name, bases, new_attrs)
class Item(MutableMapping, object_ref, metaclass=ItemMeta):
class Item(MutableMapping[str, Any], object_ref, metaclass=ItemMeta):
"""
Base class for scraped items.
@ -69,51 +85,51 @@ class Item(MutableMapping, object_ref, metaclass=ItemMeta):
fields: Dict[str, Field]
def __init__(self, *args, **kwargs):
self._values = {}
def __init__(self, *args: Any, **kwargs: Any):
self._values: Dict[str, Any] = {}
if args or kwargs: # avoid creating dict for most common case
for k, v in dict(*args, **kwargs).items():
self[k] = v
def __getitem__(self, key):
def __getitem__(self, key: str) -> Any:
return self._values[key]
def __setitem__(self, key, value):
def __setitem__(self, key: str, value: Any) -> None:
if key in self.fields:
self._values[key] = value
else:
raise KeyError(f"{self.__class__.__name__} does not support field: {key}")
def __delitem__(self, key):
def __delitem__(self, key: str) -> None:
del self._values[key]
def __getattr__(self, name):
def __getattr__(self, name: str) -> NoReturn:
if name in self.fields:
raise AttributeError(f"Use item[{name!r}] to get field value")
raise AttributeError(name)
def __setattr__(self, name, value):
def __setattr__(self, name: str, value: Any) -> None:
if not name.startswith("_"):
raise AttributeError(f"Use item[{name!r}] = {value!r} to set field value")
super().__setattr__(name, value)
def __len__(self):
def __len__(self) -> int:
return len(self._values)
def __iter__(self):
def __iter__(self) -> Iterator[str]:
return iter(self._values)
__hash__ = object_ref.__hash__
def keys(self):
def keys(self) -> KeysView[str]:
return self._values.keys()
def __repr__(self):
def __repr__(self) -> str:
return pformat(dict(self))
def copy(self):
def copy(self) -> Self:
return self.__class__(self)
def deepcopy(self):
def deepcopy(self) -> Self:
"""Return a :func:`~copy.deepcopy` of this item."""
return deepcopy(self)

98
scrapy/mail.py
View File

@ -14,11 +14,23 @@ from email.mime.nonmultipart import MIMENonMultipart
from email.mime.text import MIMEText
from email.utils import formatdate
from io import BytesIO
from typing import TYPE_CHECKING, Optional
from typing import (
IO,
TYPE_CHECKING,
Any,
Callable,
Dict,
List,
Optional,
Sequence,
Tuple,
Union,
)
from twisted import version as twisted_version
from twisted.internet import ssl
from twisted.internet.defer import Deferred
from twisted.python.failure import Failure
from twisted.python.versions import Version
from scrapy.settings import BaseSettings
@ -26,6 +38,9 @@ from scrapy.utils.misc import arg_to_iter
from scrapy.utils.python import to_bytes
if TYPE_CHECKING:
# imports twisted.internet.reactor
from twisted.mail.smtp import ESMTPSenderFactory
# typing.Self requires Python 3.11
from typing_extensions import Self
@ -37,7 +52,7 @@ logger = logging.getLogger(__name__)
COMMASPACE = ", "
def _to_bytes_or_none(text):
def _to_bytes_or_none(text: Union[str, bytes, None]) -> Optional[bytes]:
if text is None:
return None
return to_bytes(text)
@ -46,23 +61,23 @@ def _to_bytes_or_none(text):
class MailSender:
def __init__(
self,
smtphost="localhost",
mailfrom="scrapy@localhost",
smtpuser=None,
smtppass=None,
smtpport=25,
smtptls=False,
smtpssl=False,
debug=False,
smtphost: str = "localhost",
mailfrom: str = "scrapy@localhost",
smtpuser: Optional[str] = None,
smtppass: Optional[str] = None,
smtpport: int = 25,
smtptls: bool = False,
smtpssl: bool = False,
debug: bool = False,
):
self.smtphost = smtphost
self.smtpport = smtpport
self.smtpuser = _to_bytes_or_none(smtpuser)
self.smtppass = _to_bytes_or_none(smtppass)
self.smtptls = smtptls
self.smtpssl = smtpssl
self.mailfrom = mailfrom
self.debug = debug
self.smtphost: str = smtphost
self.smtpport: int = smtpport
self.smtpuser: Optional[bytes] = _to_bytes_or_none(smtpuser)
self.smtppass: Optional[bytes] = _to_bytes_or_none(smtppass)
self.smtptls: bool = smtptls
self.smtpssl: bool = smtpssl
self.mailfrom: str = mailfrom
self.debug: bool = debug
@classmethod
def from_settings(cls, settings: BaseSettings) -> Self:
@ -78,14 +93,14 @@ class MailSender:
def send(
self,
to,
subject,
body,
cc=None,
attachs=(),
mimetype="text/plain",
charset=None,
_callback=None,
to: Union[str, List[str]],
subject: str,
body: str,
cc: Union[str, List[str], None] = None,
attachs: Sequence[Tuple[str, str, IO]] = (),
mimetype: str = "text/plain",
charset: Optional[str] = None,
_callback: Optional[Callable[..., None]] = None,
) -> Optional[Deferred]:
from twisted.internet import reactor
@ -142,13 +157,15 @@ class MailSender:
dfd.addCallbacks(
callback=self._sent_ok,
errback=self._sent_failed,
callbackArgs=[to, cc, subject, len(attachs)],
errbackArgs=[to, cc, subject, len(attachs)],
callbackArgs=(to, cc, subject, len(attachs)),
errbackArgs=(to, cc, subject, len(attachs)),
)
reactor.addSystemEventTrigger("before", "shutdown", lambda: dfd)
return dfd
def _sent_ok(self, result, to, cc, subject, nattachs):
def _sent_ok(
self, result: Any, to: List[str], cc: List[str], subject: str, nattachs: int
) -> None:
logger.info(
"Mail sent OK: To=%(mailto)s Cc=%(mailcc)s "
'Subject="%(mailsubject)s" Attachs=%(mailattachs)d',
@ -160,7 +177,14 @@ class MailSender:
},
)
def _sent_failed(self, failure, to, cc, subject, nattachs):
def _sent_failed(
self,
failure: Failure,
to: List[str],
cc: List[str],
subject: str,
nattachs: int,
) -> Failure:
errstr = str(failure.value)
logger.error(
"Unable to send mail: To=%(mailto)s Cc=%(mailcc)s "
@ -176,13 +200,13 @@ class MailSender:
)
return failure
def _sendmail(self, to_addrs, msg):
def _sendmail(self, to_addrs: List[str], msg: bytes) -> Deferred:
from twisted.internet import reactor
msg = BytesIO(msg)
d = Deferred()
msg_io = BytesIO(msg)
d: Deferred = Deferred()
factory = self._create_sender_factory(to_addrs, msg, d)
factory = self._create_sender_factory(to_addrs, msg_io, d)
if self.smtpssl:
reactor.connectSSL(
@ -193,10 +217,12 @@ class MailSender:
return d
def _create_sender_factory(self, to_addrs, msg, d):
def _create_sender_factory(
self, to_addrs: List[str], msg: IO, d: Deferred
) -> ESMTPSenderFactory:
from twisted.mail.smtp import ESMTPSenderFactory
factory_keywords = {
factory_keywords: Dict[str, Any] = {
"heloFallback": True,
"requireAuthentication": False,
"requireTransportSecurity": self.smtptls,

139
scrapy/pqueues.py
View File

@ -1,12 +1,33 @@
from __future__ import annotations
import hashlib
import logging
from typing import (
TYPE_CHECKING,
Dict,
Iterable,
List,
Optional,
Protocol,
Tuple,
Type,
cast,
)
from scrapy import Request
from scrapy.core.downloader import Downloader
from scrapy.utils.misc import build_from_crawler
if TYPE_CHECKING:
# typing.Self requires Python 3.11
from typing_extensions import Self
from scrapy.crawler import Crawler
logger = logging.getLogger(__name__)
def _path_safe(text):
def _path_safe(text: str) -> str:
"""
Return a filesystem-safe version of a string ``text``
@ -24,6 +45,18 @@ def _path_safe(text):
return "-".join([pathable_slot, unique_slot])
class QueueProtocol(Protocol):
"""Protocol for downstream queues of ``ScrapyPriorityQueue``."""
def push(self, request: Request) -> None: ...
def pop(self) -> Optional[Request]: ...
def close(self) -> None: ...
def __len__(self) -> int: ...
class ScrapyPriorityQueue:
"""A priority queue implemented using multiple internal queues (typically,
FIFO queues). It uses one internal queue for each priority value. The internal
@ -51,18 +84,30 @@ class ScrapyPriorityQueue:
"""
@classmethod
def from_crawler(cls, crawler, downstream_queue_cls, key, startprios=()):
def from_crawler(
cls,
crawler: Crawler,
downstream_queue_cls: Type[QueueProtocol],
key: str,
startprios: Iterable[int] = (),
) -> Self:
return cls(crawler, downstream_queue_cls, key, startprios)
def __init__(self, crawler, downstream_queue_cls, key, startprios=()):
self.crawler = crawler
self.downstream_queue_cls = downstream_queue_cls
self.key = key
self.queues = {}
self.curprio = None
def __init__(
self,
crawler: Crawler,
downstream_queue_cls: Type[QueueProtocol],
key: str,
startprios: Iterable[int] = (),
):
self.crawler: Crawler = crawler
self.downstream_queue_cls: Type[QueueProtocol] = downstream_queue_cls
self.key: str = key
self.queues: Dict[int, QueueProtocol] = {}
self.curprio: Optional[int] = None
self.init_prios(startprios)
def init_prios(self, startprios):
def init_prios(self, startprios: Iterable[int]) -> None:
if not startprios:
return
@ -71,17 +116,17 @@ class ScrapyPriorityQueue:
self.curprio = min(startprios)
def qfactory(self, key):
def qfactory(self, key: int) -> QueueProtocol:
return build_from_crawler(
self.downstream_queue_cls,
self.crawler,
self.key + "/" + str(key),
)
def priority(self, request):
def priority(self, request: Request) -> int:
return -request.priority
def push(self, request):
def push(self, request: Request) -> None:
priority = self.priority(request)
if priority not in self.queues:
self.queues[priority] = self.qfactory(priority)
@ -90,9 +135,9 @@ class ScrapyPriorityQueue:
if self.curprio is None or priority < self.curprio:
self.curprio = priority
def pop(self):
def pop(self) -> Optional[Request]:
if self.curprio is None:
return
return None
q = self.queues[self.curprio]
m = q.pop()
if not q:
@ -102,7 +147,7 @@ class ScrapyPriorityQueue:
self.curprio = min(prios) if prios else None
return m
def peek(self):
def peek(self) -> Optional[Request]:
"""Returns the next object to be returned by :meth:`pop`,
but without removing it from the queue.
@ -112,30 +157,32 @@ class ScrapyPriorityQueue:
if self.curprio is None:
return None
queue = self.queues[self.curprio]
return queue.peek()
# Protocols can't declare optional members
return cast(Request, queue.peek()) # type: ignore[attr-defined]
def close(self):
active = []
def close(self) -> List[int]:
active: List[int] = []
for p, q in self.queues.items():
active.append(p)
q.close()
return active
def __len__(self):
def __len__(self) -> int:
return sum(len(x) for x in self.queues.values()) if self.queues else 0
class DownloaderInterface:
def __init__(self, crawler):
self.downloader = crawler.engine.downloader
def __init__(self, crawler: Crawler):
assert crawler.engine
self.downloader: Downloader = crawler.engine.downloader
def stats(self, possible_slots):
def stats(self, possible_slots: Iterable[str]) -> List[Tuple[int, str]]:
return [(self._active_downloads(slot), slot) for slot in possible_slots]
def get_slot_key(self, request):
def get_slot_key(self, request: Request) -> str:
return self.downloader._get_slot_key(request, None)
def _active_downloads(self, slot):
def _active_downloads(self, slot: str) -> int:
"""Return a number of requests in a Downloader for a given slot"""
if slot not in self.downloader.slots:
return 0
@ -149,10 +196,22 @@ class DownloaderAwarePriorityQueue:
"""
@classmethod
def from_crawler(cls, crawler, downstream_queue_cls, key, startprios=()):
def from_crawler(
cls,
crawler: Crawler,
downstream_queue_cls: Type[QueueProtocol],
key: str,
startprios: Optional[Dict[str, Iterable[int]]] = None,
) -> Self:
return cls(crawler, downstream_queue_cls, key, startprios)
def __init__(self, crawler, downstream_queue_cls, key, slot_startprios=()):
def __init__(
self,
crawler: Crawler,
downstream_queue_cls: Type[QueueProtocol],
key: str,
slot_startprios: Optional[Dict[str, Iterable[int]]] = None,
):
if crawler.settings.getint("CONCURRENT_REQUESTS_PER_IP") != 0:
raise ValueError(
f'"{self.__class__}" does not support CONCURRENT_REQUESTS_PER_IP'
@ -169,16 +228,18 @@ class DownloaderAwarePriorityQueue:
"queue class can be resumed."
)
self._downloader_interface = DownloaderInterface(crawler)
self.downstream_queue_cls = downstream_queue_cls
self.key = key
self.crawler = crawler
self._downloader_interface: DownloaderInterface = DownloaderInterface(crawler)
self.downstream_queue_cls: Type[QueueProtocol] = downstream_queue_cls
self.key: str = key
self.crawler: Crawler = crawler
self.pqueues = {} # slot -> priority queue
self.pqueues: Dict[str, ScrapyPriorityQueue] = {} # slot -> priority queue
for slot, startprios in (slot_startprios or {}).items():
self.pqueues[slot] = self.pqfactory(slot, startprios)
def pqfactory(self, slot, startprios=()):
def pqfactory(
self, slot: str, startprios: Iterable[int] = ()
) -> ScrapyPriorityQueue:
return ScrapyPriorityQueue(
self.crawler,
self.downstream_queue_cls,
@ -186,11 +247,11 @@ class DownloaderAwarePriorityQueue:
startprios,
)
def pop(self):
def pop(self) -> Optional[Request]:
stats = self._downloader_interface.stats(self.pqueues)
if not stats:
return
return None
slot = min(stats)[1]
queue = self.pqueues[slot]
@ -199,14 +260,14 @@ class DownloaderAwarePriorityQueue:
del self.pqueues[slot]
return request
def push(self, request):
def push(self, request: Request) -> None:
slot = self._downloader_interface.get_slot_key(request)
if slot not in self.pqueues:
self.pqueues[slot] = self.pqfactory(slot)
queue = self.pqueues[slot]
queue.push(request)
def peek(self):
def peek(self) -> Optional[Request]:
"""Returns the next object to be returned by :meth:`pop`,
but without removing it from the queue.
@ -220,13 +281,13 @@ class DownloaderAwarePriorityQueue:
queue = self.pqueues[slot]
return queue.peek()
def close(self):
def close(self) -> Dict[str, List[int]]:
active = {slot: queue.close() for slot, queue in self.pqueues.items()}
self.pqueues.clear()
return active
def __len__(self):
def __len__(self) -> int:
return sum(len(x) for x in self.pqueues.values()) if self.pqueues else 0
def __contains__(self, slot):
def __contains__(self, slot: str) -> bool:
return slot in self.pqueues

67
scrapy/resolver.py
View File

@ -1,8 +1,12 @@
from typing import Any
from __future__ import annotations
from typing import TYPE_CHECKING, Any, List, Optional, Sequence, Type
from twisted.internet import defer
from twisted.internet.base import ThreadedResolver
from twisted.internet.base import ReactorBase, ThreadedResolver
from twisted.internet.defer import Deferred
from twisted.internet.interfaces import (
IAddress,
IHostnameResolver,
IHostResolution,
IResolutionReceiver,
@ -12,6 +16,12 @@ from zope.interface.declarations import implementer, provider
from scrapy.utils.datatypes import LocalCache
if TYPE_CHECKING:
# typing.Self requires Python 3.11
from typing_extensions import Self
from scrapy.crawler import Crawler
# TODO: cache misses
dnscache: LocalCache[str, Any] = LocalCache(10000)
@ -22,65 +32,66 @@ class CachingThreadedResolver(ThreadedResolver):
Default caching resolver. IPv4 only, supports setting a timeout value for DNS requests.
"""
def __init__(self, reactor, cache_size, timeout):
def __init__(self, reactor: ReactorBase, cache_size: int, timeout: float):
super().__init__(reactor)
dnscache.limit = cache_size
self.timeout = timeout
@classmethod
def from_crawler(cls, crawler, reactor):
def from_crawler(cls, crawler: Crawler, reactor: ReactorBase) -> Self:
if crawler.settings.getbool("DNSCACHE_ENABLED"):
cache_size = crawler.settings.getint("DNSCACHE_SIZE")
else:
cache_size = 0
return cls(reactor, cache_size, crawler.settings.getfloat("DNS_TIMEOUT"))
def install_on_reactor(self):
def install_on_reactor(self) -> None:
self.reactor.installResolver(self)
def getHostByName(self, name: str, timeout=None):
def getHostByName(self, name: str, timeout: Sequence[int] = ()) -> Deferred[str]:
if name in dnscache:
return defer.succeed(dnscache[name])
# in Twisted<=16.6, getHostByName() is always called with
# a default timeout of 60s (actually passed as (1, 3, 11, 45) tuple),
# so the input argument above is simply overridden
# to enforce Scrapy's DNS_TIMEOUT setting's value
timeout = (self.timeout,)
# The timeout arg is typed as Sequence[int] but supports floats.
timeout = (self.timeout,) # type: ignore[assignment]
d = super().getHostByName(name, timeout)
if dnscache.limit:
d.addCallback(self._cache_result, name)
return d
def _cache_result(self, result, name):
def _cache_result(self, result: Any, name: str) -> Any:
dnscache[name] = result
return result
@implementer(IHostResolution)
class HostResolution:
def __init__(self, name):
self.name = name
def __init__(self, name: str):
self.name: str = name
def cancel(self):
def cancel(self) -> None:
raise NotImplementedError()
@provider(IResolutionReceiver)
class _CachingResolutionReceiver:
def __init__(self, resolutionReceiver, hostName):
self.resolutionReceiver = resolutionReceiver
self.hostName = hostName
self.addresses = []
def __init__(self, resolutionReceiver: IResolutionReceiver, hostName: str):
self.resolutionReceiver: IResolutionReceiver = resolutionReceiver
self.hostName: str = hostName
self.addresses: List[IAddress] = []
def resolutionBegan(self, resolution):
def resolutionBegan(self, resolution: IHostResolution) -> None:
self.resolutionReceiver.resolutionBegan(resolution)
self.resolution = resolution
def addressResolved(self, address):
def addressResolved(self, address: IAddress) -> None:
self.resolutionReceiver.addressResolved(address)
self.addresses.append(address)
def resolutionComplete(self):
def resolutionComplete(self) -> None:
self.resolutionReceiver.resolutionComplete()
if self.addresses:
dnscache[self.hostName] = self.addresses
@ -93,30 +104,30 @@ class CachingHostnameResolver:
does not support setting a timeout value for DNS requests.
"""
def __init__(self, reactor, cache_size):
self.reactor = reactor
self.original_resolver = reactor.nameResolver
def __init__(self, reactor: ReactorBase, cache_size: int):
self.reactor: ReactorBase = reactor
self.original_resolver: IHostnameResolver = reactor.nameResolver
dnscache.limit = cache_size
@classmethod
def from_crawler(cls, crawler, reactor):
def from_crawler(cls, crawler: Crawler, reactor: ReactorBase) -> Self:
if crawler.settings.getbool("DNSCACHE_ENABLED"):
cache_size = crawler.settings.getint("DNSCACHE_SIZE")
else:
cache_size = 0
return cls(reactor, cache_size)
def install_on_reactor(self):
def install_on_reactor(self) -> None:
self.reactor.installNameResolver(self)
def resolveHostName(
self,
resolutionReceiver,
resolutionReceiver: IResolutionReceiver,
hostName: str,
portNumber=0,
addressTypes=None,
transportSemantics="TCP",
):
portNumber: int = 0,
addressTypes: Optional[Sequence[Type[IAddress]]] = None,
transportSemantics: str = "TCP",
) -> IHostResolution:
try:
addresses = dnscache[hostName]
except KeyError:

65
scrapy/robotstxt.py
View File

@ -3,9 +3,10 @@ from __future__ import annotations
import logging
import sys
from abc import ABCMeta, abstractmethod
from typing import TYPE_CHECKING, Union
from typing import TYPE_CHECKING, Optional, Union
from warnings import warn
from scrapy import Spider
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.utils.python import to_unicode
@ -18,12 +19,14 @@ if TYPE_CHECKING:
logger = logging.getLogger(__name__)
def decode_robotstxt(robotstxt_body, spider, to_native_str_type=False):
def decode_robotstxt(
robotstxt_body: bytes, spider: Optional[Spider], to_native_str_type: bool = False
) -> str:
try:
if to_native_str_type:
robotstxt_body = to_unicode(robotstxt_body)
body_decoded = to_unicode(robotstxt_body)
else:
robotstxt_body = robotstxt_body.decode("utf-8", errors="ignore")
body_decoded = robotstxt_body.decode("utf-8", errors="ignore")
except UnicodeDecodeError:
# If we found garbage or robots.txt in an encoding other than UTF-8, disregard it.
# Switch to 'allow all' state.
@ -33,8 +36,8 @@ def decode_robotstxt(robotstxt_body, spider, to_native_str_type=False):
exc_info=sys.exc_info(),
extra={"spider": spider},
)
robotstxt_body = ""
return robotstxt_body
body_decoded = ""
return body_decoded
class RobotParser(metaclass=ABCMeta):
@ -66,82 +69,80 @@ class RobotParser(metaclass=ABCMeta):
class PythonRobotParser(RobotParser):
def __init__(self, robotstxt_body, spider):
def __init__(self, robotstxt_body: bytes, spider: Optional[Spider]):
from urllib.robotparser import RobotFileParser
self.spider = spider
robotstxt_body = decode_robotstxt(
robotstxt_body, spider, to_native_str_type=True
)
self.rp = RobotFileParser()
self.rp.parse(robotstxt_body.splitlines())
self.spider: Optional[Spider] = spider
body_decoded = decode_robotstxt(robotstxt_body, spider, to_native_str_type=True)
self.rp: RobotFileParser = RobotFileParser()
self.rp.parse(body_decoded.splitlines())
@classmethod
def from_crawler(cls, crawler, robotstxt_body):
def from_crawler(cls, crawler: Crawler, robotstxt_body: bytes) -> Self:
spider = None if not crawler else crawler.spider
o = cls(robotstxt_body, spider)
return o
def allowed(self, url, user_agent):
def allowed(self, url: Union[str, bytes], user_agent: Union[str, bytes]) -> bool:
user_agent = to_unicode(user_agent)
url = to_unicode(url)
return self.rp.can_fetch(user_agent, url)
class ReppyRobotParser(RobotParser):
def __init__(self, robotstxt_body, spider):
def __init__(self, robotstxt_body: bytes, spider: Optional[Spider]):
warn("ReppyRobotParser is deprecated.", ScrapyDeprecationWarning, stacklevel=2)
from reppy.robots import Robots
self.spider = spider
self.spider: Optional[Spider] = spider
self.rp = Robots.parse("", robotstxt_body)
@classmethod
def from_crawler(cls, crawler, robotstxt_body):
def from_crawler(cls, crawler: Crawler, robotstxt_body: bytes) -> Self:
spider = None if not crawler else crawler.spider
o = cls(robotstxt_body, spider)
return o
def allowed(self, url, user_agent):
def allowed(self, url: Union[str, bytes], user_agent: Union[str, bytes]) -> bool:
return self.rp.allowed(url, user_agent)
class RerpRobotParser(RobotParser):
def __init__(self, robotstxt_body, spider):
def __init__(self, robotstxt_body: bytes, spider: Optional[Spider]):
from robotexclusionrulesparser import RobotExclusionRulesParser
self.spider = spider
self.rp = RobotExclusionRulesParser()
robotstxt_body = decode_robotstxt(robotstxt_body, spider)
self.rp.parse(robotstxt_body)
self.spider: Optional[Spider] = spider
self.rp: RobotExclusionRulesParser = RobotExclusionRulesParser()
body_decoded = decode_robotstxt(robotstxt_body, spider)
self.rp.parse(body_decoded)
@classmethod
def from_crawler(cls, crawler, robotstxt_body):
def from_crawler(cls, crawler: Crawler, robotstxt_body: bytes) -> Self:
spider = None if not crawler else crawler.spider
o = cls(robotstxt_body, spider)
return o
def allowed(self, url, user_agent):
def allowed(self, url: Union[str, bytes], user_agent: Union[str, bytes]) -> bool:
user_agent = to_unicode(user_agent)
url = to_unicode(url)
return self.rp.is_allowed(user_agent, url)
class ProtegoRobotParser(RobotParser):
def __init__(self, robotstxt_body, spider):
def __init__(self, robotstxt_body: bytes, spider: Optional[Spider]):
from protego import Protego
self.spider = spider
robotstxt_body = decode_robotstxt(robotstxt_body, spider)
self.rp = Protego.parse(robotstxt_body)
self.spider: Optional[Spider] = spider
body_decoded = decode_robotstxt(robotstxt_body, spider)
self.rp = Protego.parse(body_decoded)
@classmethod
def from_crawler(cls, crawler, robotstxt_body):
def from_crawler(cls, crawler: Crawler, robotstxt_body: bytes) -> Self:
spider = None if not crawler else crawler.spider
o = cls(robotstxt_body, spider)
return o
def allowed(self, url, user_agent):
def allowed(self, url: Union[str, bytes], user_agent: Union[str, bytes]) -> bool:
user_agent = to_unicode(user_agent)
url = to_unicode(url)
return self.rp.can_fetch(url, user_agent)

79
scrapy/squeues.py
View File

@ -2,20 +2,28 @@
Scheduler queues
"""
from __future__ import annotations
import marshal
import pickle # nosec
from os import PathLike
from pathlib import Path
from typing import Union
from typing import TYPE_CHECKING, Any, Callable, Optional, Type, Union
from queuelib import queue
from scrapy import Request
from scrapy.crawler import Crawler
from scrapy.utils.request import request_from_dict
if TYPE_CHECKING:
# typing.Self requires Python 3.11
from typing_extensions import Self
def _with_mkdir(queue_class):
class DirectoriesCreated(queue_class):
def __init__(self, path: Union[str, PathLike], *args, **kwargs):
def _with_mkdir(queue_class: Type[queue.BaseQueue]) -> Type[queue.BaseQueue]:
class DirectoriesCreated(queue_class): # type: ignore[valid-type,misc]
def __init__(self, path: Union[str, PathLike], *args: Any, **kwargs: Any):
dirname = Path(path).parent
if not dirname.exists():
dirname.mkdir(parents=True, exist_ok=True)
@ -24,18 +32,23 @@ def _with_mkdir(queue_class):
return DirectoriesCreated
def _serializable_queue(queue_class, serialize, deserialize):
class SerializableQueue(queue_class):
def push(self, obj):
def _serializable_queue(
queue_class: Type[queue.BaseQueue],
serialize: Callable[[Any], bytes],
deserialize: Callable[[bytes], Any],
) -> Type[queue.BaseQueue]:
class SerializableQueue(queue_class): # type: ignore[valid-type,misc]
def push(self, obj: Any) -> None:
s = serialize(obj)
super().push(s)
def pop(self):
def pop(self) -> Optional[Any]:
s = super().pop()
if s:
return deserialize(s)
return None
def peek(self):
def peek(self) -> Optional[Any]:
"""Returns the next object to be returned by :meth:`pop`,
but without removing it from the queue.
@ -50,31 +63,36 @@ def _serializable_queue(queue_class, serialize, deserialize):
) from ex
if s:
return deserialize(s)
return None
return SerializableQueue
def _scrapy_serialization_queue(queue_class):
class ScrapyRequestQueue(queue_class):
def __init__(self, crawler, key):
def _scrapy_serialization_queue(
queue_class: Type[queue.BaseQueue],
) -> Type[queue.BaseQueue]:
class ScrapyRequestQueue(queue_class): # type: ignore[valid-type,misc]
def __init__(self, crawler: Crawler, key: str):
self.spider = crawler.spider
super().__init__(key)
@classmethod
def from_crawler(cls, crawler, key, *args, **kwargs):
def from_crawler(
cls, crawler: Crawler, key: str, *args: Any, **kwargs: Any
) -> Self:
return cls(crawler, key)
def push(self, request):
request = request.to_dict(spider=self.spider)
return super().push(request)
def push(self, request: Request) -> None:
request_dict = request.to_dict(spider=self.spider)
super().push(request_dict)
def pop(self):
def pop(self) -> Optional[Request]:
request = super().pop()
if not request:
return None
return request_from_dict(request, spider=self.spider)
def peek(self):
def peek(self) -> Optional[Request]:
"""Returns the next object to be returned by :meth:`pop`,
but without removing it from the queue.
@ -89,13 +107,15 @@ def _scrapy_serialization_queue(queue_class):
return ScrapyRequestQueue
def _scrapy_non_serialization_queue(queue_class):
class ScrapyRequestQueue(queue_class):
def _scrapy_non_serialization_queue(
queue_class: Type[queue.BaseQueue],
) -> Type[queue.BaseQueue]:
class ScrapyRequestQueue(queue_class): # type: ignore[valid-type,misc]
@classmethod
def from_crawler(cls, crawler, *args, **kwargs):
def from_crawler(cls, crawler: Crawler, *args: Any, **kwargs: Any) -> Self:
return cls()
def peek(self):
def peek(self) -> Optional[Any]:
"""Returns the next object to be returned by :meth:`pop`,
but without removing it from the queue.
@ -113,7 +133,7 @@ def _scrapy_non_serialization_queue(queue_class):
return ScrapyRequestQueue
def _pickle_serialize(obj):
def _pickle_serialize(obj: Any) -> bytes:
try:
return pickle.dumps(obj, protocol=4)
# Both pickle.PicklingError and AttributeError can be raised by pickle.dump(s)
@ -122,17 +142,18 @@ def _pickle_serialize(obj):
raise ValueError(str(e)) from e
# queue.*Queue aren't subclasses of queue.BaseQueue
_PickleFifoSerializationDiskQueue = _serializable_queue(
_with_mkdir(queue.FifoDiskQueue), _pickle_serialize, pickle.loads
_with_mkdir(queue.FifoDiskQueue), _pickle_serialize, pickle.loads # type: ignore[arg-type]
)
_PickleLifoSerializationDiskQueue = _serializable_queue(
_with_mkdir(queue.LifoDiskQueue), _pickle_serialize, pickle.loads
_with_mkdir(queue.LifoDiskQueue), _pickle_serialize, pickle.loads # type: ignore[arg-type]
)
_MarshalFifoSerializationDiskQueue = _serializable_queue(
_with_mkdir(queue.FifoDiskQueue), marshal.dumps, marshal.loads
_with_mkdir(queue.FifoDiskQueue), marshal.dumps, marshal.loads # type: ignore[arg-type]
)
_MarshalLifoSerializationDiskQueue = _serializable_queue(
_with_mkdir(queue.LifoDiskQueue), marshal.dumps, marshal.loads
_with_mkdir(queue.LifoDiskQueue), marshal.dumps, marshal.loads # type: ignore[arg-type]
)
# public queue classes
@ -140,5 +161,5 @@ PickleFifoDiskQueue = _scrapy_serialization_queue(_PickleFifoSerializationDiskQu
PickleLifoDiskQueue = _scrapy_serialization_queue(_PickleLifoSerializationDiskQueue)
MarshalFifoDiskQueue = _scrapy_serialization_queue(_MarshalFifoSerializationDiskQueue)
MarshalLifoDiskQueue = _scrapy_serialization_queue(_MarshalLifoSerializationDiskQueue)
FifoMemoryQueue = _scrapy_non_serialization_queue(queue.FifoMemoryQueue)
LifoMemoryQueue = _scrapy_non_serialization_queue(queue.LifoMemoryQueue)
FifoMemoryQueue = _scrapy_non_serialization_queue(queue.FifoMemoryQueue) # type: ignore[arg-type]
LifoMemoryQueue = _scrapy_non_serialization_queue(queue.LifoMemoryQueue) # type: ignore[arg-type]