diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 03298e3cc..ff279e9fd 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -15,10 +15,10 @@ jobs: - python-version: "3.13" env: TOXENV: pylint - - python-version: 3.8 + - python-version: "3.9" env: TOXENV: typing - - python-version: 3.8 + - python-version: "3.9" env: TOXENV: typing-tests - python-version: "3.12" # Keep in sync with .readthedocs.yml diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 4c7bde147..2cd556516 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -20,6 +20,6 @@ jobs: pip install --upgrade build twine python -m build - name: Publish to PyPI - uses: pypa/gh-action-pypi-publish@v1.9.0 + uses: pypa/gh-action-pypi-publish@v1.10.3 with: password: ${{ secrets.PYPI_TOKEN }} diff --git a/.github/workflows/tests-macos.yml b/.github/workflows/tests-macos.yml index 1f123824b..9e78e26e3 100644 --- a/.github/workflows/tests-macos.yml +++ b/.github/workflows/tests-macos.yml @@ -11,7 +11,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13"] + python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"] steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml index 9db2ad897..b2a5681df 100644 --- a/.github/workflows/tests-ubuntu.yml +++ b/.github/workflows/tests-ubuntu.yml @@ -12,7 +12,7 @@ jobs: fail-fast: false matrix: include: - - python-version: 3.9 + - python-version: "3.9" env: TOXENV: py - python-version: "3.10" @@ -38,19 +38,19 @@ jobs: TOXENV: pypy3 # pinned deps - - python-version: 3.8.17 + - python-version: 3.9.19 env: TOXENV: pinned - - python-version: 3.8.17 + - python-version: 3.9.19 env: TOXENV: asyncio-pinned - - python-version: pypy3.8 + - python-version: pypy3.9 env: TOXENV: pypy3-pinned - - python-version: 3.8.17 + - python-version: 3.9.19 env: TOXENV: extra-deps-pinned - - python-version: 3.8.17 + - python-version: 3.9.19 env: TOXENV: botocore-pinned diff --git a/.github/workflows/tests-windows.yml b/.github/workflows/tests-windows.yml index 4e1034d77..67a32aac6 100644 --- a/.github/workflows/tests-windows.yml +++ b/.github/workflows/tests-windows.yml @@ -12,12 +12,9 @@ jobs: fail-fast: false matrix: include: - - python-version: 3.8 + - python-version: "3.9" env: TOXENV: windows-pinned - - python-version: 3.9 - env: - TOXENV: py - python-version: "3.10" env: TOXENV: py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index addad838f..75529be05 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -33,4 +33,4 @@ repos: rev: v3.16.0 hooks: - id: pyupgrade - args: [--py38-plus, --keep-runtime-typing] + args: [--py39-plus, --keep-runtime-typing] diff --git a/README.rst b/README.rst index 14adff648..e640bce35 100644 --- a/README.rst +++ b/README.rst @@ -59,7 +59,7 @@ including a list of features. Requirements ============ -* Python 3.8+ +* Python 3.9+ * Works on Linux, Windows, macOS, BSD Install diff --git a/docs/intro/install.rst b/docs/intro/install.rst index e6c9a683b..ef541368a 100644 --- a/docs/intro/install.rst +++ b/docs/intro/install.rst @@ -9,7 +9,7 @@ Installation guide Supported Python versions ========================= -Scrapy requires Python 3.8+, either the CPython implementation (default) or +Scrapy requires Python 3.9+, either the CPython implementation (default) or the PyPy implementation (see :ref:`python:implementations`). .. _intro-install-scrapy: diff --git a/docs/news.rst b/docs/news.rst index 758b22d80..58b51c9ea 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -8,6 +8,12 @@ Release notes Scrapy VERSION (YYYY-MM-DD) --------------------------- +New features +~~~~~~~~~~~~ + +- If :setting:`SPIDER_LOADER_WARN_ONLY` is set to ``True``, + ``SpiderLoader`` does not raise :exc:`SyntaxError` but emits a warning instead. + Deprecations ~~~~~~~~~~~~ diff --git a/docs/topics/contracts.rst b/docs/topics/contracts.rst index 2d61026e9..61aef4bbb 100644 --- a/docs/topics/contracts.rst +++ b/docs/topics/contracts.rst @@ -20,13 +20,13 @@ following example: This function parses a sample response. Some contracts are mingled with this docstring. - @url http://www.amazon.com/s?field-keywords=selfish+gene + @url http://www.example.com/s?field-keywords=selfish+gene @returns items 1 16 @returns requests 0 0 @scrapes Title Author Year Price """ -This callback is tested using three built-in contracts: +You can use the following contracts: .. module:: scrapy.contracts.default @@ -46,6 +46,14 @@ This callback is tested using three built-in contracts: @cb_kwargs {"arg1": "value1", "arg2": "value2", ...} +.. class:: MetadataContract + + This contract (``@meta``) sets the :attr:`meta ` + attribute for the sample request. It must be a valid JSON dictionary. + :: + + @meta {"arg1": "value1", "arg2": "value2", ...} + .. class:: ReturnsContract This contract (``@returns``) sets lower and upper bounds for the items and diff --git a/docs/topics/items.rst b/docs/topics/items.rst index 97ed7a900..f13a7b5b1 100644 --- a/docs/topics/items.rst +++ b/docs/topics/items.rst @@ -48,7 +48,7 @@ make it the most feature-complete item type: :class:`Item` objects replicate the standard :class:`dict` API, including its ``__init__`` method. - :class:`Item` allows defining field names, so that: + :class:`Item` allows the defining of field names, so that: - :class:`KeyError` is raised when using undefined field names (i.e. prevents typos going unnoticed) @@ -57,7 +57,7 @@ make it the most feature-complete item type: default even if the first scraped object does not have values for all of them - :class:`Item` also allows defining field metadata, which can be used to + :class:`Item` also allows the defining of field metadata, which can be used to :ref:`customize serialization `. :mod:`trackref` tracks :class:`Item` objects to help find memory leaks @@ -94,11 +94,11 @@ Dataclass objects .. versionadded:: 2.2 -:func:`~dataclasses.dataclass` allows defining item classes with field names, +:func:`~dataclasses.dataclass` allows the defining of item classes with field names, so that :ref:`item exporters ` can export all fields by default even if the first scraped object does not have values for all of them. -Additionally, ``dataclass`` items also allow to: +Additionally, ``dataclass`` items also allow you to: * define the type and default value of each defined field. @@ -126,7 +126,7 @@ attr.s objects .. versionadded:: 2.2 -:func:`attr.s` allows defining item classes with field names, +:func:`attr.s` allows the defining of item classes with field names, so that :ref:`item exporters ` can export all fields by default even if the first scraped object does not have values for all of them. diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 904bd7ecc..02fca7ff4 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -1580,7 +1580,7 @@ SPIDER_LOADER_WARN_ONLY Default: ``False`` By default, when Scrapy tries to import spider classes from :setting:`SPIDER_MODULES`, -it will fail loudly if there is any ``ImportError`` exception. +it will fail loudly if there is any ``ImportError`` or ``SyntaxError`` exception. But you can choose to silence this exception and turn it into a simple warning by setting ``SPIDER_LOADER_WARN_ONLY = True``. diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst index 13e636055..b45b12540 100644 --- a/docs/topics/signals.rst +++ b/docs/topics/signals.rst @@ -159,8 +159,9 @@ item_scraped :param spider: the spider which scraped the item :type spider: :class:`~scrapy.Spider` object - :param response: the response from where the item was scraped - :type response: :class:`~scrapy.http.Response` object + :param response: the response from where the item was scraped, or ``None`` + if it was yielded from :meth:`~scrapy.Spider.start_requests`. + :type response: :class:`~scrapy.http.Response` | ``None`` item_dropped ~~~~~~~~~~~~ @@ -179,8 +180,9 @@ item_dropped :param spider: the spider which scraped the item :type spider: :class:`~scrapy.Spider` object - :param response: the response from where the item was dropped - :type response: :class:`~scrapy.http.Response` object + :param response: the response from where the item was dropped, or ``None`` + if it was yielded from :meth:`~scrapy.Spider.start_requests`. + :type response: :class:`~scrapy.http.Response` | ``None`` :param exception: the exception (which must be a :exc:`~scrapy.exceptions.DropItem` subclass) which caused the item @@ -201,8 +203,10 @@ item_error :param item: the item that caused the error in the :ref:`topics-item-pipeline` :type item: :ref:`item object ` - :param response: the response being processed when the exception was raised - :type response: :class:`~scrapy.http.Response` object + :param response: the response being processed when the exception was + raised, or ``None`` if it was yielded from + :meth:`~scrapy.Spider.start_requests`. + :type response: :class:`~scrapy.http.Response` | ``None`` :param spider: the spider which raised the exception :type spider: :class:`~scrapy.Spider` object diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index 8ddf17a14..8f39bcd53 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -176,7 +176,7 @@ object gives you access, for example, to the :ref:`settings `. items). It receives an iterable (in the ``start_requests`` parameter) and must - return another iterable of :class:`~scrapy.Request` objects. + return another iterable of :class:`~scrapy.Request` objects and/or :ref:`item objects `. .. note:: When implementing this method in your spider middleware, you should always return an iterable (that follows the input one) and diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index 8a0102a51..e1b1c5ad6 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -203,7 +203,8 @@ scrapy.Spider .. method:: start_requests() - This method must return an iterable with the first Requests to crawl for + This method must return an iterable with the first Requests to crawl and/or with :ref:`item objects + ` for this spider. It is called by Scrapy when the spider is opened for scraping. Scrapy calls it only once, so it is safe to implement :meth:`start_requests` as a generator. diff --git a/scrapy/addons.py b/scrapy/addons.py index f9ec58cea..7a1da3afc 100644 --- a/scrapy/addons.py +++ b/scrapy/addons.py @@ -1,7 +1,7 @@ from __future__ import annotations import logging -from typing import TYPE_CHECKING, Any, List +from typing import TYPE_CHECKING, Any from scrapy.exceptions import NotConfigured from scrapy.utils.conf import build_component_list @@ -20,7 +20,7 @@ class AddonManager: def __init__(self, crawler: Crawler) -> None: self.crawler: Crawler = crawler - self.addons: List[Any] = [] + self.addons: list[Any] = [] def load_settings(self, settings: Settings) -> None: """Load add-ons and configurations from a settings object and apply them. diff --git a/scrapy/cmdline.py b/scrapy/cmdline.py index e010b159a..b820eb7f9 100644 --- a/scrapy/cmdline.py +++ b/scrapy/cmdline.py @@ -6,7 +6,7 @@ import inspect import os import sys from importlib.metadata import entry_points -from typing import TYPE_CHECKING, Callable, Dict, Iterable, List, Optional, Tuple, Type +from typing import TYPE_CHECKING, Optional import scrapy from scrapy.commands import BaseRunSpiderCommand, ScrapyCommand, ScrapyHelpFormatter @@ -17,6 +17,8 @@ from scrapy.utils.project import get_project_settings, inside_project from scrapy.utils.python import garbage_collect if TYPE_CHECKING: + from collections.abc import Callable, Iterable + # typing.ParamSpec requires Python 3.10 from typing_extensions import ParamSpec @@ -28,7 +30,7 @@ if TYPE_CHECKING: class ScrapyArgumentParser(argparse.ArgumentParser): def _parse_optional( self, arg_string: str - ) -> Optional[Tuple[Optional[argparse.Action], str, Optional[str]]]: + ) -> Optional[tuple[Optional[argparse.Action], str, Optional[str]]]: # if starts with -: it means that is a parameter not a argument if arg_string[:2] == "-:": return None @@ -36,7 +38,7 @@ class ScrapyArgumentParser(argparse.ArgumentParser): return super()._parse_optional(arg_string) -def _iter_command_classes(module_name: str) -> Iterable[Type[ScrapyCommand]]: +def _iter_command_classes(module_name: str) -> Iterable[type[ScrapyCommand]]: # TODO: add `name` attribute to commands and merge this function with # scrapy.utils.spider.iter_spider_classes for module in walk_modules(module_name): @@ -50,8 +52,8 @@ def _iter_command_classes(module_name: str) -> Iterable[Type[ScrapyCommand]]: yield obj -def _get_commands_from_module(module: str, inproject: bool) -> Dict[str, ScrapyCommand]: - d: Dict[str, ScrapyCommand] = {} +def _get_commands_from_module(module: str, inproject: bool) -> dict[str, ScrapyCommand]: + d: dict[str, ScrapyCommand] = {} for cmd in _iter_command_classes(module): if inproject or not cmd.requires_project: cmdname = cmd.__module__.split(".")[-1] @@ -61,8 +63,8 @@ def _get_commands_from_module(module: str, inproject: bool) -> Dict[str, ScrapyC def _get_commands_from_entry_points( inproject: bool, group: str = "scrapy.commands" -) -> Dict[str, ScrapyCommand]: - cmds: Dict[str, ScrapyCommand] = {} +) -> dict[str, ScrapyCommand]: + cmds: dict[str, ScrapyCommand] = {} if sys.version_info >= (3, 10): eps = entry_points(group=group) else: @@ -78,7 +80,7 @@ def _get_commands_from_entry_points( def _get_commands_dict( settings: BaseSettings, inproject: bool -) -> Dict[str, ScrapyCommand]: +) -> dict[str, ScrapyCommand]: cmds = _get_commands_from_module("scrapy.commands", inproject) cmds.update(_get_commands_from_entry_points(inproject)) cmds_module = settings["COMMANDS_MODULE"] @@ -87,7 +89,7 @@ def _get_commands_dict( return cmds -def _pop_command_name(argv: List[str]) -> Optional[str]: +def _pop_command_name(argv: list[str]) -> Optional[str]: i = 0 for arg in argv[1:]: if not arg.startswith("-"): @@ -146,7 +148,7 @@ def _run_print_help( def execute( - argv: Optional[List[str]] = None, settings: Optional[Settings] = None + argv: Optional[list[str]] = None, settings: Optional[Settings] = None ) -> None: if argv is None: argv = sys.argv @@ -189,7 +191,7 @@ def execute( sys.exit(cmd.exitcode) -def _run_command(cmd: ScrapyCommand, args: List[str], opts: argparse.Namespace) -> None: +def _run_command(cmd: ScrapyCommand, args: list[str], opts: argparse.Namespace) -> None: if opts.profile: _run_command_profiled(cmd, args, opts) else: @@ -197,7 +199,7 @@ def _run_command(cmd: ScrapyCommand, args: List[str], opts: argparse.Namespace) def _run_command_profiled( - cmd: ScrapyCommand, args: List[str], opts: argparse.Namespace + cmd: ScrapyCommand, args: list[str], opts: argparse.Namespace ) -> None: if opts.profile: sys.stderr.write(f"scrapy: writing cProfile stats to {opts.profile!r}\n") diff --git a/scrapy/commands/__init__.py b/scrapy/commands/__init__.py index 0322390e5..a94db90b1 100644 --- a/scrapy/commands/__init__.py +++ b/scrapy/commands/__init__.py @@ -8,7 +8,7 @@ import argparse import builtins import os from pathlib import Path -from typing import TYPE_CHECKING, Any, Dict, Iterable, List, Optional +from typing import TYPE_CHECKING, Any, Optional from twisted.python import failure @@ -16,6 +16,8 @@ from scrapy.exceptions import UsageError from scrapy.utils.conf import arglist_to_dict, feed_process_params_from_cli if TYPE_CHECKING: + from collections.abc import Iterable + from scrapy.crawler import Crawler, CrawlerProcess @@ -24,7 +26,7 @@ class ScrapyCommand: crawler_process: Optional[CrawlerProcess] = None # default settings to be used for this command instead of global defaults - default_settings: Dict[str, Any] = {} + default_settings: dict[str, Any] = {} exitcode: int = 0 @@ -97,7 +99,7 @@ class ScrapyCommand: ) group.add_argument("--pdb", action="store_true", help="enable pdb on failure") - def process_options(self, args: List[str], opts: argparse.Namespace) -> None: + def process_options(self, args: list[str], opts: argparse.Namespace) -> None: try: self.settings.setdict(arglist_to_dict(opts.set), priority="cmdline") except ValueError: @@ -122,7 +124,7 @@ class ScrapyCommand: if opts.pdb: failure.startDebugMode() - def run(self, args: List[str], opts: argparse.Namespace) -> None: + def run(self, args: list[str], opts: argparse.Namespace) -> None: """ Entry point for running commands """ @@ -167,7 +169,7 @@ class BaseRunSpiderCommand(ScrapyCommand): help="format to use for dumping items", ) - def process_options(self, args: List[str], opts: argparse.Namespace) -> None: + def process_options(self, args: list[str], opts: argparse.Namespace) -> None: super().process_options(args, opts) try: opts.spargs = arglist_to_dict(opts.spargs) @@ -207,7 +209,7 @@ class ScrapyHelpFormatter(argparse.HelpFormatter): parts = self.format_part_strings(builtins.list(part_strings)) return super()._join_parts(parts) - def format_part_strings(self, part_strings: List[str]) -> List[str]: + def format_part_strings(self, part_strings: list[str]) -> list[str]: """ Underline and title case command line help message headers. """ diff --git a/scrapy/commands/bench.py b/scrapy/commands/bench.py index f91fec57e..4f6933006 100644 --- a/scrapy/commands/bench.py +++ b/scrapy/commands/bench.py @@ -4,7 +4,7 @@ import argparse import subprocess # nosec import sys import time -from typing import TYPE_CHECKING, Any, Iterable, List +from typing import TYPE_CHECKING, Any from urllib.parse import urlencode import scrapy @@ -13,6 +13,8 @@ from scrapy.http import Response, TextResponse from scrapy.linkextractors import LinkExtractor if TYPE_CHECKING: + from collections.abc import Iterable + from scrapy import Request @@ -26,7 +28,7 @@ class Command(ScrapyCommand): def short_desc(self) -> str: return "Run quick benchmark test" - def run(self, args: List[str], opts: argparse.Namespace) -> None: + def run(self, args: list[str], opts: argparse.Namespace) -> None: with _BenchServer(): assert self.crawler_process self.crawler_process.crawl(_BenchSpider, total=100000) diff --git a/scrapy/commands/check.py b/scrapy/commands/check.py index 22c8abf7a..c7946605b 100644 --- a/scrapy/commands/check.py +++ b/scrapy/commands/check.py @@ -1,7 +1,6 @@ import argparse import time from collections import defaultdict -from typing import List from unittest import TextTestResult as _TextTestResult from unittest import TextTestRunner @@ -69,7 +68,7 @@ class Command(ScrapyCommand): help="print contract tests for all spiders", ) - def run(self, args: List[str], opts: argparse.Namespace) -> None: + def run(self, args: list[str], opts: argparse.Namespace) -> None: # load contracts contracts = build_component_list(self.settings.getwithbase("SPIDER_CONTRACTS")) conman = ContractsManager(load_object(c) for c in contracts) diff --git a/scrapy/commands/crawl.py b/scrapy/commands/crawl.py index fe1864372..6b6a80bb5 100644 --- a/scrapy/commands/crawl.py +++ b/scrapy/commands/crawl.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import TYPE_CHECKING, List, cast +from typing import TYPE_CHECKING, cast from twisted.python.failure import Failure @@ -20,7 +20,7 @@ class Command(BaseRunSpiderCommand): def short_desc(self) -> str: return "Run a spider" - def run(self, args: List[str], opts: argparse.Namespace) -> None: + def run(self, args: list[str], opts: argparse.Namespace) -> None: if len(args) < 1: raise UsageError() elif len(args) > 1: diff --git a/scrapy/commands/edit.py b/scrapy/commands/edit.py index 04012bee8..34313d731 100644 --- a/scrapy/commands/edit.py +++ b/scrapy/commands/edit.py @@ -1,7 +1,6 @@ import argparse import os import sys -from typing import List from scrapy.commands import ScrapyCommand from scrapy.exceptions import UsageError @@ -27,7 +26,7 @@ class Command(ScrapyCommand): sys.stderr.write(msg + os.linesep) self.exitcode = 1 - def run(self, args: List[str], opts: argparse.Namespace) -> None: + def run(self, args: list[str], opts: argparse.Namespace) -> None: if len(args) != 1: raise UsageError() diff --git a/scrapy/commands/fetch.py b/scrapy/commands/fetch.py index 0bdc429da..a1806f626 100644 --- a/scrapy/commands/fetch.py +++ b/scrapy/commands/fetch.py @@ -1,7 +1,7 @@ from __future__ import annotations import sys -from typing import TYPE_CHECKING, Dict, List, Type +from typing import TYPE_CHECKING from w3lib.url import is_url @@ -48,7 +48,7 @@ class Command(ScrapyCommand): help="do not handle HTTP 3xx status codes and print response as-is", ) - def _print_headers(self, headers: Dict[bytes, List[bytes]], prefix: bytes) -> None: + def _print_headers(self, headers: dict[bytes, list[bytes]], prefix: bytes) -> None: for key, values in headers.items(): for value in values: self._print_bytes(prefix + b" " + key + b": " + value) @@ -65,7 +65,7 @@ class Command(ScrapyCommand): def _print_bytes(self, bytes_: bytes) -> None: sys.stdout.buffer.write(bytes_ + b"\n") - def run(self, args: List[str], opts: Namespace) -> None: + def run(self, args: list[str], opts: Namespace) -> None: if len(args) != 1 or not is_url(args[0]): raise UsageError() request = Request( @@ -81,7 +81,7 @@ class Command(ScrapyCommand): else: request.meta["handle_httpstatus_all"] = True - spidercls: Type[Spider] = DefaultSpider + spidercls: type[Spider] = DefaultSpider assert self.crawler_process spider_loader = self.crawler_process.spider_loader if opts.spider: diff --git a/scrapy/commands/genspider.py b/scrapy/commands/genspider.py index 2649fb23d..a9b7a6eee 100644 --- a/scrapy/commands/genspider.py +++ b/scrapy/commands/genspider.py @@ -4,7 +4,7 @@ import shutil import string from importlib import import_module from pathlib import Path -from typing import List, Optional, Union, cast +from typing import Optional, Union, cast from urllib.parse import urlparse import scrapy @@ -87,7 +87,7 @@ class Command(ScrapyCommand): help="If the spider already exists, overwrite it with the template", ) - def run(self, args: List[str], opts: argparse.Namespace) -> None: + def run(self, args: list[str], opts: argparse.Namespace) -> None: if opts.list: self._list_templates() return @@ -116,6 +116,24 @@ class Command(ScrapyCommand): if opts.edit: self.exitcode = os.system(f'scrapy edit "{name}"') # nosec + def _generate_template_variables( + self, + module: str, + name: str, + url: str, + template_name: str, + ): + capitalized_module = "".join(s.capitalize() for s in module.split("_")) + return { + "project_name": self.settings.get("BOT_NAME"), + "ProjectName": string_camelcase(self.settings.get("BOT_NAME")), + "module": module, + "name": name, + "url": url, + "domain": extract_domain(url), + "classname": f"{capitalized_module}Spider", + } + def _genspider( self, module: str, @@ -125,17 +143,7 @@ class Command(ScrapyCommand): template_file: Union[str, os.PathLike], ) -> None: """Generate the spider module, based on the given template""" - capitalized_module = "".join(s.capitalize() for s in module.split("_")) - domain = extract_domain(url) - tvars = { - "project_name": self.settings.get("BOT_NAME"), - "ProjectName": string_camelcase(self.settings.get("BOT_NAME")), - "module": module, - "name": name, - "url": url, - "domain": domain, - "classname": f"{capitalized_module}Spider", - } + tvars = self._generate_template_variables(module, name, url, template_name) if self.settings.get("NEWSPIDER_MODULE"): spiders_module = import_module(self.settings["NEWSPIDER_MODULE"]) assert spiders_module.__file__ diff --git a/scrapy/commands/list.py b/scrapy/commands/list.py index 10330c92a..3b2f127c2 100644 --- a/scrapy/commands/list.py +++ b/scrapy/commands/list.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import TYPE_CHECKING, List +from typing import TYPE_CHECKING from scrapy.commands import ScrapyCommand @@ -15,7 +15,7 @@ class Command(ScrapyCommand): def short_desc(self) -> str: return "List available spiders" - def run(self, args: List[str], opts: argparse.Namespace) -> None: + def run(self, args: list[str], opts: argparse.Namespace) -> None: assert self.crawler_process for s in sorted(self.crawler_process.spider_loader.list()): print(s) diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py index fbd200d88..bd1fad14b 100644 --- a/scrapy/commands/parse.py +++ b/scrapy/commands/parse.py @@ -5,20 +5,7 @@ import functools import inspect import json import logging -from typing import ( - TYPE_CHECKING, - Any, - AsyncGenerator, - Coroutine, - Dict, - Iterable, - List, - Optional, - Tuple, - TypeVar, - Union, - overload, -) +from typing import TYPE_CHECKING, Any, Optional, TypeVar, Union, overload from itemadapter import ItemAdapter, is_item from twisted.internet.defer import Deferred, maybeDeferred @@ -35,6 +22,8 @@ from scrapy.utils.misc import arg_to_iter from scrapy.utils.spider import spidercls_for_request if TYPE_CHECKING: + from collections.abc import AsyncGenerator, Coroutine, Iterable + from twisted.python.failure import Failure from scrapy.http.request import CallbackT @@ -50,8 +39,8 @@ class Command(BaseRunSpiderCommand): requires_project = True spider = None - items: Dict[int, List[Any]] = {} - requests: Dict[int, List[Request]] = {} + items: dict[int, list[Any]] = {} + requests: dict[int, list[Request]] = {} first_response = None @@ -166,11 +155,11 @@ class Command(BaseRunSpiderCommand): return d return arg_to_iter(deferred_from_coro(result)) - def add_items(self, lvl: int, new_items: List[Any]) -> None: + def add_items(self, lvl: int, new_items: list[Any]) -> None: old_items = self.items.get(lvl, []) self.items[lvl] = old_items + new_items - def add_requests(self, lvl: int, new_reqs: List[Request]) -> None: + def add_requests(self, lvl: int, new_reqs: list[Request]) -> None: old_reqs = self.requests.get(lvl, []) self.requests[lvl] = old_reqs + new_reqs @@ -219,7 +208,7 @@ class Command(BaseRunSpiderCommand): depth: int, spider: Spider, callback: CallbackT, - ) -> Tuple[List[Any], List[Request], argparse.Namespace, int, Spider, CallbackT]: + ) -> tuple[list[Any], list[Request], argparse.Namespace, int, Spider, CallbackT]: items, requests = [], [] for x in spider_output: if is_item(x): @@ -232,7 +221,7 @@ class Command(BaseRunSpiderCommand): self, response: Response, callback: CallbackT, - cb_kwargs: Optional[Dict[str, Any]] = None, + cb_kwargs: Optional[dict[str, Any]] = None, ) -> Deferred[Any]: cb_kwargs = cb_kwargs or {} d = maybeDeferred(self.iterate_spider_output, callback(response, **cb_kwargs)) @@ -285,10 +274,10 @@ class Command(BaseRunSpiderCommand): def scraped_data( self, - args: Tuple[ - List[Any], List[Request], argparse.Namespace, int, Spider, CallbackT + args: tuple[ + list[Any], list[Request], argparse.Namespace, int, Spider, CallbackT ], - ) -> List[Any]: + ) -> list[Any]: items, requests, opts, depth, spider, callback = args if opts.pipelines: itemproc = self.pcrawler.engine.scraper.itemproc @@ -345,7 +334,7 @@ class Command(BaseRunSpiderCommand): def prepare_request( self, spider: Spider, request: Request, opts: argparse.Namespace ) -> Request: - def callback(response: Response, **cb_kwargs: Any) -> Deferred[List[Any]]: + def callback(response: Response, **cb_kwargs: Any) -> Deferred[list[Any]]: # memorize first request if not self.first_response: self.first_response = response @@ -376,7 +365,7 @@ class Command(BaseRunSpiderCommand): request.callback = callback return request - def process_options(self, args: List[str], opts: argparse.Namespace) -> None: + def process_options(self, args: list[str], opts: argparse.Namespace) -> None: super().process_options(args, opts) self.process_request_meta(opts) @@ -404,7 +393,7 @@ class Command(BaseRunSpiderCommand): print_help=False, ) - def run(self, args: List[str], opts: argparse.Namespace) -> None: + def run(self, args: list[str], opts: argparse.Namespace) -> None: # parse arguments if not len(args) == 1 or not is_url(args[0]): raise UsageError() diff --git a/scrapy/commands/runspider.py b/scrapy/commands/runspider.py index 87acf9a01..14d58f311 100644 --- a/scrapy/commands/runspider.py +++ b/scrapy/commands/runspider.py @@ -4,7 +4,7 @@ import argparse import sys from importlib import import_module from pathlib import Path -from typing import TYPE_CHECKING, List, Union +from typing import TYPE_CHECKING, Union from scrapy.commands import BaseRunSpiderCommand from scrapy.exceptions import UsageError @@ -41,7 +41,7 @@ class Command(BaseRunSpiderCommand): def long_desc(self) -> str: return "Run the spider defined in the given file" - def run(self, args: List[str], opts: argparse.Namespace) -> None: + def run(self, args: list[str], opts: argparse.Namespace) -> None: if len(args) != 1: raise UsageError() filename = Path(args[0]) diff --git a/scrapy/commands/settings.py b/scrapy/commands/settings.py index dbda73b44..59f86b9a7 100644 --- a/scrapy/commands/settings.py +++ b/scrapy/commands/settings.py @@ -1,6 +1,5 @@ import argparse import json -from typing import List from scrapy.commands import ScrapyCommand from scrapy.settings import BaseSettings @@ -46,7 +45,7 @@ class Command(ScrapyCommand): help="print setting value, interpreted as a list", ) - def run(self, args: List[str], opts: argparse.Namespace) -> None: + def run(self, args: list[str], opts: argparse.Namespace) -> None: assert self.crawler_process settings = self.crawler_process.settings if opts.get: diff --git a/scrapy/commands/shell.py b/scrapy/commands/shell.py index f03cf997a..27e6d68ee 100644 --- a/scrapy/commands/shell.py +++ b/scrapy/commands/shell.py @@ -7,7 +7,7 @@ See documentation in docs/topics/shell.rst from __future__ import annotations from threading import Thread -from typing import TYPE_CHECKING, Any, Dict, List, Type +from typing import TYPE_CHECKING, Any from scrapy import Spider from scrapy.commands import ScrapyCommand @@ -56,13 +56,13 @@ class Command(ScrapyCommand): help="do not handle HTTP 3xx status codes and print response as-is", ) - def update_vars(self, vars: Dict[str, Any]) -> None: + def update_vars(self, vars: dict[str, Any]) -> None: """You can use this function to update the Scrapy objects that will be available in the shell """ pass - def run(self, args: List[str], opts: Namespace) -> None: + def run(self, args: list[str], opts: Namespace) -> None: url = args[0] if args else None if url: # first argument may be a local file @@ -71,7 +71,7 @@ class Command(ScrapyCommand): assert self.crawler_process spider_loader = self.crawler_process.spider_loader - spidercls: Type[Spider] = DefaultSpider + spidercls: type[Spider] = DefaultSpider if opts.spider: spidercls = spider_loader.load(opts.spider) elif url: diff --git a/scrapy/commands/startproject.py b/scrapy/commands/startproject.py index 58c1aa28f..f7052cd18 100644 --- a/scrapy/commands/startproject.py +++ b/scrapy/commands/startproject.py @@ -6,14 +6,14 @@ from importlib.util import find_spec from pathlib import Path from shutil import copy2, copystat, ignore_patterns, move from stat import S_IWUSR as OWNER_WRITE_PERMISSION -from typing import List, Tuple, Union +from typing import Union import scrapy from scrapy.commands import ScrapyCommand from scrapy.exceptions import UsageError from scrapy.utils.template import render_templatefile, string_camelcase -TEMPLATES_TO_RENDER: Tuple[Tuple[str, ...], ...] = ( +TEMPLATES_TO_RENDER: tuple[tuple[str, ...], ...] = ( ("scrapy.cfg",), ("${project_name}", "settings.py.tmpl"), ("${project_name}", "items.py.tmpl"), @@ -86,7 +86,7 @@ class Command(ScrapyCommand): copystat(src, dst) _make_writable(dst) - def run(self, args: List[str], opts: argparse.Namespace) -> None: + def run(self, args: list[str], opts: argparse.Namespace) -> None: if len(args) not in (1, 2): raise UsageError() @@ -107,9 +107,7 @@ class Command(ScrapyCommand): return self._copytree(Path(self.templates_dir), project_dir.resolve()) - # On 3.8 shutil.move doesn't fully support Path args, but it supports our use case - # See https://bugs.python.org/issue32689 - move(project_dir / "module", project_dir / project_name) # type: ignore[arg-type] + move(project_dir / "module", project_dir / project_name) for paths in TEMPLATES_TO_RENDER: tplfile = Path( project_dir, diff --git a/scrapy/commands/version.py b/scrapy/commands/version.py index f057e8544..571f4fda8 100644 --- a/scrapy/commands/version.py +++ b/scrapy/commands/version.py @@ -1,5 +1,4 @@ import argparse -from typing import List import scrapy from scrapy.commands import ScrapyCommand @@ -25,7 +24,7 @@ class Command(ScrapyCommand): help="also display twisted/python/platform info (useful for bug reports)", ) - def run(self, args: List[str], opts: argparse.Namespace) -> None: + def run(self, args: list[str], opts: argparse.Namespace) -> None: if opts.verbose: versions = scrapy_components_versions() width = max(len(n) for (n, _) in versions) diff --git a/scrapy/contracts/__init__.py b/scrapy/contracts/__init__.py index a7e129948..ffe5053de 100644 --- a/scrapy/contracts/__init__.py +++ b/scrapy/contracts/__init__.py @@ -2,22 +2,11 @@ from __future__ import annotations import re import sys +from collections.abc import AsyncGenerator, Iterable from functools import wraps from inspect import getmembers from types import CoroutineType -from typing import ( - TYPE_CHECKING, - Any, - AsyncGenerator, - Callable, - Dict, - Iterable, - List, - Optional, - Tuple, - Type, - cast, -) +from typing import TYPE_CHECKING, Any, Optional, cast from unittest import TestCase, TestResult from scrapy.http import Request, Response @@ -25,6 +14,8 @@ from scrapy.utils.python import get_spec from scrapy.utils.spider import iterate_spider_output if TYPE_CHECKING: + from collections.abc import Callable + from twisted.python.failure import Failure from scrapy import Spider @@ -33,13 +24,13 @@ if TYPE_CHECKING: class Contract: """Abstract class for contracts""" - request_cls: Optional[Type[Request]] = None + request_cls: Optional[type[Request]] = None name: str def __init__(self, method: Callable, *args: Any): self.testcase_pre = _create_testcase(method, f"@{self.name} pre-hook") self.testcase_post = _create_testcase(method, f"@{self.name} post-hook") - self.args: Tuple[Any, ...] = args + self.args: tuple[Any, ...] = args def add_pre_hook(self, request: Request, results: TestResult) -> Request: if hasattr(self, "pre_process"): @@ -47,7 +38,7 @@ class Contract: assert cb is not None @wraps(cb) - def wrapper(response: Response, **cb_kwargs: Any) -> List[Any]: + def wrapper(response: Response, **cb_kwargs: Any) -> list[Any]: try: results.startTest(self.testcase_pre) self.pre_process(response) @@ -76,7 +67,7 @@ class Contract: assert cb is not None @wraps(cb) - def wrapper(response: Response, **cb_kwargs: Any) -> List[Any]: + def wrapper(response: Response, **cb_kwargs: Any) -> list[Any]: cb_result = cb(response, **cb_kwargs) if isinstance(cb_result, (AsyncGenerator, CoroutineType)): raise TypeError("Contracts don't support async callbacks") @@ -98,18 +89,18 @@ class Contract: return request - def adjust_request_args(self, args: Dict[str, Any]) -> Dict[str, Any]: + def adjust_request_args(self, args: dict[str, Any]) -> dict[str, Any]: return args class ContractsManager: - contracts: Dict[str, Type[Contract]] = {} + contracts: dict[str, type[Contract]] = {} - def __init__(self, contracts: Iterable[Type[Contract]]): + def __init__(self, contracts: Iterable[type[Contract]]): for contract in contracts: self.contracts[contract.name] = contract - def tested_methods_from_spidercls(self, spidercls: Type[Spider]) -> List[str]: + def tested_methods_from_spidercls(self, spidercls: type[Spider]) -> list[str]: is_method = re.compile(r"^\s*@", re.MULTILINE).search methods = [] for key, value in getmembers(spidercls): @@ -118,8 +109,8 @@ class ContractsManager: return methods - def extract_contracts(self, method: Callable) -> List[Contract]: - contracts: List[Contract] = [] + def extract_contracts(self, method: Callable) -> list[Contract]: + contracts: list[Contract] = [] assert method.__doc__ is not None for line in method.__doc__.split("\n"): line = line.strip() @@ -137,8 +128,8 @@ class ContractsManager: def from_spider( self, spider: Spider, results: TestResult - ) -> List[Optional[Request]]: - requests: List[Optional[Request]] = [] + ) -> list[Optional[Request]]: + requests: list[Optional[Request]] = [] for method in self.tested_methods_from_spidercls(type(spider)): bound_method = spider.__getattribute__(method) try: diff --git a/scrapy/contracts/default.py b/scrapy/contracts/default.py index 71ca4168a..87170d3c1 100644 --- a/scrapy/contracts/default.py +++ b/scrapy/contracts/default.py @@ -1,5 +1,5 @@ import json -from typing import Any, Callable, Dict, List, Optional +from typing import Any, Callable, Optional from itemadapter import ItemAdapter, is_item @@ -16,7 +16,7 @@ class UrlContract(Contract): name = "url" - def adjust_request_args(self, args: Dict[str, Any]) -> Dict[str, Any]: + def adjust_request_args(self, args: dict[str, Any]) -> dict[str, Any]: args["url"] = self.args[0] return args @@ -30,11 +30,25 @@ class CallbackKeywordArgumentsContract(Contract): name = "cb_kwargs" - def adjust_request_args(self, args: Dict[str, Any]) -> Dict[str, Any]: + def adjust_request_args(self, args: dict[str, Any]) -> dict[str, Any]: args["cb_kwargs"] = json.loads(" ".join(self.args)) return args +class MetadataContract(Contract): + """Contract to set metadata arguments for the request. + The value should be JSON-encoded dictionary, e.g.: + + @meta {"arg1": "some value"} + """ + + name = "meta" + + def adjust_request_args(self, args: dict[str, Any]) -> dict[str, Any]: + args["meta"] = json.loads(" ".join(self.args)) + return args + + class ReturnsContract(Contract): """Contract to check the output of a callback @@ -49,7 +63,7 @@ class ReturnsContract(Contract): """ name = "returns" - object_type_verifiers: Dict[Optional[str], Callable[[Any], bool]] = { + object_type_verifiers: dict[Optional[str], Callable[[Any], bool]] = { "request": lambda x: isinstance(x, Request), "requests": lambda x: isinstance(x, Request), "item": is_item, @@ -76,7 +90,7 @@ class ReturnsContract(Contract): except IndexError: self.max_bound = float("inf") - def post_process(self, output: List[Any]) -> None: + def post_process(self, output: list[Any]) -> None: occurrences = 0 for x in output: if self.obj_type_verifier(x): @@ -102,7 +116,7 @@ class ScrapesContract(Contract): name = "scrapes" - def post_process(self, output: List[Any]) -> None: + def post_process(self, output: list[Any]) -> None: for x in output: if is_item(x): missing = [arg for arg in self.args if arg not in ItemAdapter(x)] diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index 6786d7acf..77d57a8d8 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -5,18 +5,7 @@ import warnings from collections import deque from datetime import datetime from time import time -from typing import ( - TYPE_CHECKING, - Any, - Deque, - Dict, - Optional, - Set, - Tuple, - TypeVar, - Union, - cast, -) +from typing import TYPE_CHECKING, Any, Optional, TypeVar, Union, cast from twisted.internet import task from twisted.internet.defer import Deferred @@ -55,9 +44,9 @@ class Slot: self.randomize_delay: bool = randomize_delay self.throttle = throttle - self.active: Set[Request] = set() - self.queue: Deque[Tuple[Request, Deferred[Response]]] = deque() - self.transferring: Set[Request] = set() + self.active: set[Request] = set() + self.queue: deque[tuple[Request, Deferred[Response]]] = deque() + self.transferring: set[Request] = set() self.lastseen: float = 0 self.latercall = None @@ -95,7 +84,7 @@ class Slot: def _get_concurrency_delay( concurrency: int, spider: Spider, settings: BaseSettings -) -> Tuple[int, float]: +) -> tuple[int, float]: delay: float = settings.getfloat("DOWNLOAD_DELAY") if hasattr(spider, "download_delay"): delay = spider.download_delay @@ -112,8 +101,8 @@ class Downloader: def __init__(self, crawler: Crawler): self.settings: BaseSettings = crawler.settings self.signals: SignalManager = crawler.signals - self.slots: Dict[str, Slot] = {} - self.active: Set[Request] = set() + self.slots: dict[str, Slot] = {} + self.active: set[Request] = set() self.handlers: DownloadHandlers = DownloadHandlers(crawler) self.total_concurrency: int = self.settings.getint("CONCURRENT_REQUESTS") self.domain_concurrency: int = self.settings.getint( @@ -126,7 +115,7 @@ class Downloader: ) self._slot_gc_loop: task.LoopingCall = task.LoopingCall(self._slot_gc) self._slot_gc_loop.start(60) - self.per_slot_settings: Dict[str, Dict[str, Any]] = self.settings.getdict( + self.per_slot_settings: dict[str, dict[str, Any]] = self.settings.getdict( "DOWNLOAD_SLOTS", {} ) @@ -146,7 +135,7 @@ class Downloader: def needs_backout(self) -> bool: return len(self.active) >= self.total_concurrency - def _get_slot(self, request: Request, spider: Spider) -> Tuple[str, Slot]: + def _get_slot(self, request: Request, spider: Spider) -> tuple[str, Slot]: key = self.get_slot_key(request) if key not in self.slots: slot_settings = self.per_slot_settings.get(key, {}) diff --git a/scrapy/core/downloader/contextfactory.py b/scrapy/core/downloader/contextfactory.py index 2b388a9f5..ba20c3c2c 100644 --- a/scrapy/core/downloader/contextfactory.py +++ b/scrapy/core/downloader/contextfactory.py @@ -1,7 +1,7 @@ from __future__ import annotations import warnings -from typing import TYPE_CHECKING, Any, List, Optional +from typing import TYPE_CHECKING, Any, Optional from OpenSSL import SSL from twisted.internet._sslverify import _setAcceptableProtocols @@ -154,10 +154,10 @@ class AcceptableProtocolsContextFactory: negotiation. """ - def __init__(self, context_factory: Any, acceptable_protocols: List[bytes]): + def __init__(self, context_factory: Any, acceptable_protocols: list[bytes]): verifyObject(IPolicyForHTTPS, context_factory) self._wrapped_context_factory: Any = context_factory - self._acceptable_protocols: List[bytes] = acceptable_protocols + self._acceptable_protocols: list[bytes] = acceptable_protocols def creatorForNetloc(self, hostname: bytes, port: int) -> ClientTLSOptions: options: ClientTLSOptions = self._wrapped_context_factory.creatorForNetloc( diff --git a/scrapy/core/downloader/handlers/__init__.py b/scrapy/core/downloader/handlers/__init__.py index 70d356b83..c39e480f1 100644 --- a/scrapy/core/downloader/handlers/__init__.py +++ b/scrapy/core/downloader/handlers/__init__.py @@ -3,18 +3,8 @@ from __future__ import annotations import logging -from typing import ( - TYPE_CHECKING, - Any, - Callable, - Dict, - Generator, - Optional, - Protocol, - Type, - Union, - cast, -) +from collections.abc import Callable +from typing import TYPE_CHECKING, Any, Optional, Protocol, Union, cast from twisted.internet import defer @@ -25,6 +15,8 @@ from scrapy.utils.misc import build_from_crawler, load_object from scrapy.utils.python import without_none_values if TYPE_CHECKING: + from collections.abc import Generator + from twisted.internet.defer import Deferred from scrapy.crawler import Crawler @@ -43,16 +35,16 @@ class DownloadHandlerProtocol(Protocol): class DownloadHandlers: def __init__(self, crawler: Crawler): self._crawler: Crawler = crawler - self._schemes: Dict[str, Union[str, Callable[..., Any]]] = ( + self._schemes: dict[str, Union[str, Callable[..., Any]]] = ( {} ) # stores acceptable schemes on instancing - self._handlers: Dict[str, DownloadHandlerProtocol] = ( + self._handlers: dict[str, DownloadHandlerProtocol] = ( {} ) # stores instanced handlers for schemes - self._notconfigured: Dict[str, str] = {} # remembers failed handlers - handlers: Dict[str, Union[str, Callable[..., Any]]] = without_none_values( + self._notconfigured: dict[str, str] = {} # remembers failed handlers + handlers: dict[str, Union[str, Callable[..., Any]]] = without_none_values( cast( - Dict[str, Union[str, Callable[..., Any]]], + dict[str, Union[str, Callable[..., Any]]], crawler.settings.getwithbase("DOWNLOAD_HANDLERS"), ) ) @@ -81,7 +73,7 @@ class DownloadHandlers: ) -> Optional[DownloadHandlerProtocol]: path = self._schemes[scheme] try: - dhcls: Type[DownloadHandlerProtocol] = load_object(path) + dhcls: type[DownloadHandlerProtocol] = load_object(path) if skip_lazy and getattr(dhcls, "lazy", True): return None dh = build_from_crawler( diff --git a/scrapy/core/downloader/handlers/datauri.py b/scrapy/core/downloader/handlers/datauri.py index bf6879521..b3f286d87 100644 --- a/scrapy/core/downloader/handlers/datauri.py +++ b/scrapy/core/downloader/handlers/datauri.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Any, Dict +from typing import TYPE_CHECKING, Any from w3lib.url import parse_data_uri @@ -20,7 +20,7 @@ class DataURIDownloadHandler: uri = parse_data_uri(request.url) respcls = responsetypes.from_mimetype(uri.media_type) - resp_kwargs: Dict[str, Any] = {} + resp_kwargs: dict[str, Any] = {} if issubclass(respcls, TextResponse) and uri.media_type.split("/")[0] == "text": charset = uri.media_type_parameters.get("charset") resp_kwargs["encoding"] = charset diff --git a/scrapy/core/downloader/handlers/ftp.py b/scrapy/core/downloader/handlers/ftp.py index 69c2d88e1..bc06c7ef4 100644 --- a/scrapy/core/downloader/handlers/ftp.py +++ b/scrapy/core/downloader/handlers/ftp.py @@ -32,7 +32,7 @@ from __future__ import annotations import re from io import BytesIO -from typing import TYPE_CHECKING, Any, BinaryIO, Dict, Optional +from typing import TYPE_CHECKING, Any, BinaryIO, Optional from urllib.parse import unquote from twisted.internet.protocol import ClientCreator, Protocol @@ -79,7 +79,7 @@ _CODE_RE = re.compile(r"\d+") class FTPDownloadHandler: lazy = False - CODE_MAPPING: Dict[str, int] = { + CODE_MAPPING: dict[str, int] = { "550": 404, "default": 503, } diff --git a/scrapy/core/downloader/handlers/http10.py b/scrapy/core/downloader/handlers/http10.py index 8d7b0635c..58f7ad577 100644 --- a/scrapy/core/downloader/handlers/http10.py +++ b/scrapy/core/downloader/handlers/http10.py @@ -1,9 +1,8 @@ -"""Download handlers for http and https schemes -""" +"""Download handlers for http and https schemes""" from __future__ import annotations -from typing import TYPE_CHECKING, Type +from typing import TYPE_CHECKING from scrapy.utils.misc import build_from_crawler, load_object from scrapy.utils.python import to_unicode @@ -27,10 +26,10 @@ class HTTP10DownloadHandler: lazy = False def __init__(self, settings: BaseSettings, crawler: Crawler): - self.HTTPClientFactory: Type[ScrapyHTTPClientFactory] = load_object( + self.HTTPClientFactory: type[ScrapyHTTPClientFactory] = load_object( settings["DOWNLOADER_HTTPCLIENTFACTORY"] ) - self.ClientContextFactory: Type[ScrapyClientContextFactory] = load_object( + self.ClientContextFactory: type[ScrapyClientContextFactory] = load_object( settings["DOWNLOADER_CLIENTCONTEXTFACTORY"] ) self._settings: BaseSettings = settings diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index c06d90f01..f96dc7c98 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -8,7 +8,7 @@ import re from contextlib import suppress from io import BytesIO from time import time -from typing import TYPE_CHECKING, Any, List, Optional, Tuple, TypedDict, TypeVar, Union +from typing import TYPE_CHECKING, Any, Optional, TypedDict, TypeVar, Union from urllib.parse import urldefrag, urlunparse from twisted.internet import ssl @@ -52,7 +52,7 @@ _T = TypeVar("_T") class _ResultT(TypedDict): txresponse: TxResponse body: bytes - flags: Optional[List[str]] + flags: Optional[list[str]] certificate: Optional[ssl.Certificate] ip_address: Union[ipaddress.IPv4Address, ipaddress.IPv6Address, None] failure: NotRequired[Optional[Failure]] @@ -143,10 +143,10 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint): reactor: ReactorBase, host: str, port: int, - proxyConf: Tuple[str, int, Optional[bytes]], + proxyConf: tuple[str, int, Optional[bytes]], contextFactory: IPolicyForHTTPS, timeout: float = 30, - bindAddress: Optional[Tuple[str, int]] = None, + bindAddress: Optional[tuple[str, int]] = None, ): proxyHost, proxyPort, self._proxyAuthHeader = proxyConf super().__init__(reactor, proxyHost, proxyPort, timeout, bindAddress) @@ -254,14 +254,14 @@ class TunnelingAgent(Agent): self, *, reactor: ReactorBase, - proxyConf: Tuple[str, int, Optional[bytes]], + proxyConf: tuple[str, int, Optional[bytes]], contextFactory: IPolicyForHTTPS, connectTimeout: Optional[float] = None, bindAddress: Optional[bytes] = None, pool: Optional[HTTPConnectionPool] = None, ): super().__init__(reactor, contextFactory, connectTimeout, bindAddress, pool) - self._proxyConf: Tuple[str, int, Optional[bytes]] = proxyConf + self._proxyConf: tuple[str, int, Optional[bytes]] = proxyConf self._contextFactory: IPolicyForHTTPS = contextFactory def _getEndpoint(self, uri: URI) -> TunnelingTCP4ClientEndpoint: @@ -621,7 +621,7 @@ class _ResponseReader(Protocol): self._crawler: Crawler = crawler def _finish_response( - self, flags: Optional[List[str]] = None, failure: Optional[Failure] = None + self, flags: Optional[list[str]] = None, failure: Optional[Failure] = None ) -> None: self._finished.callback( { diff --git a/scrapy/core/downloader/handlers/s3.py b/scrapy/core/downloader/handlers/s3.py index edf370193..fa660c63c 100644 --- a/scrapy/core/downloader/handlers/s3.py +++ b/scrapy/core/downloader/handlers/s3.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Any, Optional, Type +from typing import TYPE_CHECKING, Any, Optional from scrapy.core.downloader.handlers.http import HTTPDownloadHandler from scrapy.exceptions import NotConfigured @@ -29,7 +29,7 @@ class S3DownloadHandler: aws_access_key_id: Optional[str] = None, aws_secret_access_key: Optional[str] = None, aws_session_token: Optional[str] = None, - httpdownloadhandler: Type[HTTPDownloadHandler] = HTTPDownloadHandler, + httpdownloadhandler: type[HTTPDownloadHandler] = HTTPDownloadHandler, **kw: Any, ): if not is_botocore_available(): diff --git a/scrapy/core/downloader/middleware.py b/scrapy/core/downloader/middleware.py index 0bdb756c8..00d3bd1b0 100644 --- a/scrapy/core/downloader/middleware.py +++ b/scrapy/core/downloader/middleware.py @@ -6,7 +6,8 @@ See documentation in docs/topics/downloader-middleware.rst from __future__ import annotations -from typing import TYPE_CHECKING, Any, Callable, Generator, List, Union, cast +from collections.abc import Callable +from typing import TYPE_CHECKING, Any, Union, cast from twisted.internet.defer import Deferred, inlineCallbacks @@ -17,6 +18,8 @@ from scrapy.utils.conf import build_component_list from scrapy.utils.defer import deferred_from_coro, mustbe_deferred if TYPE_CHECKING: + from collections.abc import Generator + from twisted.python.failure import Failure from scrapy import Spider @@ -27,7 +30,7 @@ class DownloaderMiddlewareManager(MiddlewareManager): component_name = "downloader middleware" @classmethod - def _get_mwlist_from_settings(cls, settings: BaseSettings) -> List[Any]: + def _get_mwlist_from_settings(cls, settings: BaseSettings) -> list[Any]: return build_component_list(settings.getwithbase("DOWNLOADER_MIDDLEWARES")) def _add_middleware(self, mw: Any) -> None: diff --git a/scrapy/core/downloader/tls.py b/scrapy/core/downloader/tls.py index 33cea7263..1ae66f614 100644 --- a/scrapy/core/downloader/tls.py +++ b/scrapy/core/downloader/tls.py @@ -1,5 +1,5 @@ import logging -from typing import Any, Dict +from typing import Any from OpenSSL import SSL from service_identity.exceptions import CertificateError @@ -21,7 +21,7 @@ METHOD_TLSv11 = "TLSv1.1" METHOD_TLSv12 = "TLSv1.2" -openssl_methods: Dict[str, int] = { +openssl_methods: dict[str, int] = { METHOD_TLS: SSL.SSLv23_METHOD, # protocol negotiation (recommended) METHOD_TLSv10: SSL.TLSv1_METHOD, # TLS 1.0 only METHOD_TLSv11: SSL.TLSv1_1_METHOD, # TLS 1.1 only diff --git a/scrapy/core/downloader/webclient.py b/scrapy/core/downloader/webclient.py index 99502f0d2..509bda4e4 100644 --- a/scrapy/core/downloader/webclient.py +++ b/scrapy/core/downloader/webclient.py @@ -2,7 +2,7 @@ from __future__ import annotations import re from time import time -from typing import TYPE_CHECKING, Optional, Tuple +from typing import TYPE_CHECKING, Optional from urllib.parse import ParseResult, urldefrag, urlparse, urlunparse from twisted.internet import defer @@ -18,7 +18,7 @@ if TYPE_CHECKING: from scrapy import Request -def _parsed_url_args(parsed: ParseResult) -> Tuple[bytes, bytes, bytes, int, bytes]: +def _parsed_url_args(parsed: ParseResult) -> tuple[bytes, bytes, bytes, int, bytes]: # Assume parsed is urlparse-d from Request.url, # which was passed via safe_url_string and is ascii-only. path_str = urlunparse(("", "", parsed.path or "/", parsed.params, parsed.query, "")) @@ -33,7 +33,7 @@ def _parsed_url_args(parsed: ParseResult) -> Tuple[bytes, bytes, bytes, int, byt return scheme, netloc, host, port, path -def _parse(url: str) -> Tuple[bytes, bytes, bytes, int, bytes]: +def _parse(url: str) -> tuple[bytes, bytes, bytes, int, bytes]: """Return tuple of (scheme, netloc, host, port, path), all in bytes except for port which is int. Assume url is from Request.url, which was passed via safe_url_string diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index 5318cbd64..bb09d066f 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -9,21 +9,9 @@ from __future__ import annotations import logging from time import time -from typing import ( - TYPE_CHECKING, - Any, - Callable, - Generator, - Iterable, - Iterator, - Optional, - Set, - Type, - TypeVar, - Union, - cast, -) +from typing import TYPE_CHECKING, Any, Optional, TypeVar, Union, cast +from itemadapter import is_item from twisted.internet.defer import Deferred, inlineCallbacks, succeed from twisted.internet.task import LoopingCall from twisted.python.failure import Failure @@ -38,10 +26,11 @@ from scrapy.settings import Settings from scrapy.signalmanager import SignalManager from scrapy.utils.log import failure_to_exc_info, logformatter_adapter from scrapy.utils.misc import build_from_crawler, load_object -from scrapy.utils.python import global_object_name from scrapy.utils.reactor import CallLaterOnce if TYPE_CHECKING: + from collections.abc import Callable, Generator, Iterable, Iterator + from scrapy.core.scheduler import BaseScheduler from scrapy.core.scraper import _HandleOutputDeferred from scrapy.crawler import Crawler @@ -63,7 +52,7 @@ class Slot: scheduler: BaseScheduler, ) -> None: self.closing: Optional[Deferred[None]] = None - self.inprogress: Set[Request] = set() + self.inprogress: set[Request] = set() self.start_requests: Optional[Iterator[Request]] = iter(start_requests) self.close_if_idle: bool = close_if_idle self.nextcall: CallLaterOnce[None] = nextcall @@ -106,10 +95,10 @@ class ExecutionEngine: self.spider: Optional[Spider] = None self.running: bool = False self.paused: bool = False - self.scheduler_cls: Type[BaseScheduler] = self._get_scheduler_class( + self.scheduler_cls: type[BaseScheduler] = self._get_scheduler_class( crawler.settings ) - downloader_cls: Type[Downloader] = load_object(self.settings["DOWNLOADER"]) + downloader_cls: type[Downloader] = load_object(self.settings["DOWNLOADER"]) self.downloader: Downloader = downloader_cls(crawler) self.scraper = Scraper(crawler) self._spider_closed_callback: Callable[[Spider], Optional[Deferred[None]]] = ( @@ -117,10 +106,10 @@ class ExecutionEngine: ) self.start_time: Optional[float] = None - def _get_scheduler_class(self, settings: BaseSettings) -> Type[BaseScheduler]: + def _get_scheduler_class(self, settings: BaseSettings) -> type[BaseScheduler]: from scrapy.core.scheduler import BaseScheduler - scheduler_cls: Type[BaseScheduler] = load_object(settings["SCHEDULER"]) + scheduler_cls: type[BaseScheduler] = load_object(settings["SCHEDULER"]) if not issubclass(scheduler_cls, BaseScheduler): raise TypeError( f"The provided scheduler class ({settings['SCHEDULER']})" @@ -194,7 +183,7 @@ class ExecutionEngine: if self.slot.start_requests is not None and not self._needs_backout(): try: - request = next(self.slot.start_requests) + request_or_item = next(self.slot.start_requests) except StopIteration: self.slot.start_requests = None except Exception: @@ -205,7 +194,16 @@ class ExecutionEngine: extra={"spider": self.spider}, ) else: - self.crawl(request) + if isinstance(request_or_item, Request): + self.crawl(request_or_item) + elif is_item(request_or_item): + self.scraper.start_itemproc(request_or_item, response=None) + else: + logger.error( + f"Got {request_or_item!r} among start requests. Only " + f"requests and items are supported. It will be " + f"ignored." + ) if self.spider_is_idle() and self.slot.close_if_idle: self._spider_idle() @@ -315,10 +313,6 @@ class ExecutionEngine: ) for handler, result in request_scheduled_result: if isinstance(result, Failure) and isinstance(result.value, IgnoreRequest): - logger.debug( - f"Signal handler {global_object_name(handler)} dropped " - f"request {request} before it reached the scheduler." - ) return if not self.slot.scheduler.enqueue_request(request): # type: ignore[union-attr] self.signals.send_catch_log( diff --git a/scrapy/core/http2/agent.py b/scrapy/core/http2/agent.py index 640fb7129..b5ff55eb0 100644 --- a/scrapy/core/http2/agent.py +++ b/scrapy/core/http2/agent.py @@ -1,7 +1,7 @@ from __future__ import annotations from collections import deque -from typing import TYPE_CHECKING, Deque, Dict, List, Optional, Tuple +from typing import TYPE_CHECKING, Optional from twisted.internet import defer from twisted.internet.defer import Deferred @@ -26,7 +26,7 @@ if TYPE_CHECKING: from scrapy.spiders import Spider -ConnectionKeyT = Tuple[bytes, bytes, int] +ConnectionKeyT = tuple[bytes, bytes, int] class H2ConnectionPool: @@ -36,11 +36,11 @@ class H2ConnectionPool: # Store a dictionary which is used to get the respective # H2ClientProtocolInstance using the key as Tuple(scheme, hostname, port) - self._connections: Dict[ConnectionKeyT, H2ClientProtocol] = {} + self._connections: dict[ConnectionKeyT, H2ClientProtocol] = {} # Save all requests that arrive before the connection is established - self._pending_requests: Dict[ - ConnectionKeyT, Deque[Deferred[H2ClientProtocol]] + self._pending_requests: dict[ + ConnectionKeyT, deque[Deferred[H2ClientProtocol]] ] = {} def get_connection( @@ -68,7 +68,7 @@ class H2ConnectionPool: ) -> Deferred[H2ClientProtocol]: self._pending_requests[key] = deque() - conn_lost_deferred: Deferred[List[BaseException]] = Deferred() + conn_lost_deferred: Deferred[list[BaseException]] = Deferred() conn_lost_deferred.addCallback(self._remove_connection, key) factory = H2ClientFactory(uri, self.settings, conn_lost_deferred) @@ -94,7 +94,7 @@ class H2ConnectionPool: return conn def _remove_connection( - self, errors: List[BaseException], key: ConnectionKeyT + self, errors: list[BaseException], key: ConnectionKeyT ) -> None: self._connections.pop(key) diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py index 8aebbaab4..618423218 100644 --- a/scrapy/core/http2/protocol.py +++ b/scrapy/core/http2/protocol.py @@ -4,7 +4,7 @@ import ipaddress import itertools import logging from collections import deque -from typing import TYPE_CHECKING, Any, Deque, Dict, List, Optional, Union +from typing import TYPE_CHECKING, Any, Optional, Union from h2.config import H2Configuration from h2.connection import H2Connection @@ -91,7 +91,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin): self, uri: URI, settings: Settings, - conn_lost_deferred: Deferred[List[BaseException]], + conn_lost_deferred: Deferred[list[BaseException]], ) -> None: """ Arguments: @@ -102,7 +102,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin): conn_lost_deferred -- Deferred fires with the reason: Failure to notify that connection was lost """ - self._conn_lost_deferred: Deferred[List[BaseException]] = conn_lost_deferred + self._conn_lost_deferred: Deferred[list[BaseException]] = conn_lost_deferred config = H2Configuration(client_side=True, header_encoding="utf-8") self.conn = H2Connection(config=config) @@ -113,19 +113,19 @@ class H2ClientProtocol(Protocol, TimeoutMixin): self._stream_id_generator = itertools.count(start=1, step=2) # Streams are stored in a dictionary keyed off their stream IDs - self.streams: Dict[int, Stream] = {} + self.streams: dict[int, Stream] = {} # If requests are received before connection is made we keep # all requests in a pool and send them as the connection is made - self._pending_request_stream_pool: Deque[Stream] = deque() + self._pending_request_stream_pool: deque[Stream] = deque() # Save an instance of errors raised which lead to losing the connection # We pass these instances to the streams ResponseFailed() failure - self._conn_lost_errors: List[BaseException] = [] + self._conn_lost_errors: list[BaseException] = [] # Some meta data of this connection # initialized when connection is successfully made - self.metadata: Dict[str, Any] = { + self.metadata: dict[str, Any] = { # Peer certificate instance "certificate": None, # Address of the server we are connected to which @@ -250,7 +250,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin): self.conn.initiate_connection() self._write_to_transport() - def _lose_connection_with_error(self, errors: List[BaseException]) -> None: + def _lose_connection_with_error(self, errors: list[BaseException]) -> None: """Helper function to lose the connection with the error sent as a reason""" self._conn_lost_errors += errors @@ -353,7 +353,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin): self._pending_request_stream_pool.clear() self.conn.close_connection() - def _handle_events(self, events: List[Event]) -> None: + def _handle_events(self, events: list[Event]) -> None: """Private method which acts as a bridge between the events received from the HTTP/2 data and IH2EventsHandler @@ -442,7 +442,7 @@ class H2ClientFactory(Factory): self, uri: URI, settings: Settings, - conn_lost_deferred: Deferred[List[BaseException]], + conn_lost_deferred: Deferred[list[BaseException]], ) -> None: self.uri = uri self.settings = settings @@ -451,5 +451,5 @@ class H2ClientFactory(Factory): def buildProtocol(self, addr: IAddress) -> H2ClientProtocol: return H2ClientProtocol(self.uri, self.settings, self.conn_lost_deferred) - def acceptableProtocols(self) -> List[bytes]: + def acceptableProtocols(self) -> list[bytes]: return [PROTOCOL_NAME] diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py index d8b5cc8eb..51ebdf489 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/http2/stream.py @@ -3,7 +3,7 @@ from __future__ import annotations import logging from enum import Enum from io import BytesIO -from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple +from typing import TYPE_CHECKING, Any, Optional from h2.errors import ErrorCodes from h2.exceptions import H2Error, ProtocolError, StreamClosedError @@ -113,7 +113,7 @@ class Stream: # Metadata of an HTTP/2 connection stream # initialized when stream is instantiated - self.metadata: Dict[str, Any] = { + self.metadata: dict[str, Any] = { "request_content_length": ( 0 if self._request.body is None else len(self._request.body) ), @@ -134,7 +134,7 @@ class Stream: # Private variable used to build the response # this response is then converted to appropriate Response class # passed to the response deferred callback - self._response: Dict[str, Any] = { + self._response: dict[str, Any] = { # Data received frame by frame from the server is appended # and passed to the response Deferred when completely received. "body": BytesIO(), @@ -196,7 +196,7 @@ class Stream: == f'{self._protocol.metadata["ip_address"]}:{self._protocol.metadata["uri"].port}' ) - def _get_request_headers(self) -> List[Tuple[str, str]]: + def _get_request_headers(self) -> list[tuple[str, str]]: url = urlparse_cached(self._request) path = url.path @@ -349,7 +349,7 @@ class Stream: self._response["flow_controlled_size"], self.stream_id ) - def receive_headers(self, headers: List[HeaderTuple]) -> None: + def receive_headers(self, headers: list[HeaderTuple]) -> None: for name, value in headers: self._response["headers"].appendlist(name, value) @@ -382,7 +382,7 @@ class Stream: def close( self, reason: StreamCloseReason, - errors: Optional[List[BaseException]] = None, + errors: Optional[list[BaseException]] = None, from_protocol: bool = False, ) -> None: """Based on the reason sent we will handle each case.""" diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.py index d4286c874..ced18fc05 100644 --- a/scrapy/core/scheduler.py +++ b/scrapy/core/scheduler.py @@ -4,7 +4,7 @@ import json import logging from abc import abstractmethod from pathlib import Path -from typing import TYPE_CHECKING, Any, List, Optional, Type, cast +from typing import TYPE_CHECKING, Any, Optional, cast # working around https://github.com/sphinx-doc/sphinx/issues/10400 from twisted.internet.defer import Deferred # noqa: TC002 @@ -182,18 +182,18 @@ class Scheduler(BaseScheduler): self, dupefilter: BaseDupeFilter, jobdir: Optional[str] = None, - dqclass: Optional[Type[BaseQueue]] = None, - mqclass: Optional[Type[BaseQueue]] = None, + dqclass: Optional[type[BaseQueue]] = None, + mqclass: Optional[type[BaseQueue]] = None, logunser: bool = False, stats: Optional[StatsCollector] = None, - pqclass: Optional[Type[ScrapyPriorityQueue]] = None, + pqclass: Optional[type[ScrapyPriorityQueue]] = None, crawler: Optional[Crawler] = None, ): self.df: BaseDupeFilter = dupefilter self.dqdir: Optional[str] = self._dqdir(jobdir) - self.pqclass: Optional[Type[ScrapyPriorityQueue]] = pqclass - self.dqclass: Optional[Type[BaseQueue]] = dqclass - self.mqclass: Optional[Type[BaseQueue]] = mqclass + self.pqclass: Optional[type[ScrapyPriorityQueue]] = pqclass + self.dqclass: Optional[type[BaseQueue]] = dqclass + self.mqclass: Optional[type[BaseQueue]] = mqclass self.logunser: bool = logunser self.stats: Optional[StatsCollector] = stats self.crawler: Optional[Crawler] = crawler @@ -364,13 +364,13 @@ class Scheduler(BaseScheduler): return str(dqdir) return None - def _read_dqs_state(self, dqdir: str) -> List[int]: + def _read_dqs_state(self, dqdir: str) -> list[int]: path = Path(dqdir, "active.json") if not path.exists(): return [] with path.open(encoding="utf-8") as f: - return cast(List[int], json.load(f)) + return cast(list[int], json.load(f)) - def _write_dqs_state(self, dqdir: str, state: List[int]) -> None: + def _write_dqs_state(self, dqdir: str, state: list[int]) -> None: with Path(dqdir, "active.json").open("w", encoding="utf-8") as f: json.dump(state, f) diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index a7d65e1e3..29d7cb0c8 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -5,23 +5,8 @@ from __future__ import annotations import logging from collections import deque -from typing import ( - TYPE_CHECKING, - Any, - AsyncIterable, - Deque, - Generator, - Iterable, - Iterator, - List, - Optional, - Set, - Tuple, - Type, - TypeVar, - Union, - cast, -) +from collections.abc import AsyncIterable, Iterator +from typing import TYPE_CHECKING, Any, Optional, TypeVar, Union, cast from itemadapter import is_item from twisted.internet.defer import Deferred, inlineCallbacks @@ -47,6 +32,8 @@ from scrapy.utils.misc import load_object, warn_on_generator_with_return_value from scrapy.utils.spider import iterate_spider_output if TYPE_CHECKING: + from collections.abc import Generator, Iterable + from scrapy.crawler import Crawler @@ -54,12 +41,12 @@ logger = logging.getLogger(__name__) _T = TypeVar("_T") -_ParallelResult = List[Tuple[bool, Iterator[Any]]] +_ParallelResult = list[tuple[bool, Iterator[Any]]] if TYPE_CHECKING: # parameterized Deferreds require Twisted 21.7.0 _HandleOutputDeferred = Deferred[Union[_ParallelResult, None]] - QueueTuple = Tuple[Union[Response, Failure], Request, _HandleOutputDeferred] + QueueTuple = tuple[Union[Response, Failure], Request, _HandleOutputDeferred] class Slot: @@ -69,8 +56,8 @@ class Slot: def __init__(self, max_active_size: int = 5000000): self.max_active_size = max_active_size - self.queue: Deque[QueueTuple] = deque() - self.active: Set[Request] = set() + self.queue: deque[QueueTuple] = deque() + self.active: set[Request] = set() self.active_size: int = 0 self.itemproc_size: int = 0 self.closing: Optional[Deferred[Spider]] = None @@ -113,7 +100,7 @@ class Scraper: self.spidermw: SpiderMiddlewareManager = SpiderMiddlewareManager.from_crawler( crawler ) - itemproc_cls: Type[ItemPipelineManager] = load_object( + itemproc_cls: type[ItemPipelineManager] = load_object( crawler.settings["ITEM_PROCESSOR"] ) self.itemproc: ItemPipelineManager = itemproc_cls.from_crawler(crawler) @@ -313,15 +300,11 @@ class Scraper: """Process each Request/Item (given in the output parameter) returned from the given spider """ - assert self.slot is not None # typing if isinstance(output, Request): assert self.crawler.engine is not None # typing self.crawler.engine.crawl(request=output) elif is_item(output): - self.slot.itemproc_size += 1 - dfd = self.itemproc.process_item(output, spider) - dfd.addBoth(self._itemproc_finished, output, response, spider) - return dfd + return self.start_itemproc(output, response=response) elif output is None: pass else: @@ -333,6 +316,19 @@ class Scraper: ) return None + def start_itemproc(self, item, *, response: Optional[Response]) -> Deferred[Any]: + """Send *item* to the item pipelines for processing. + + *response* is the source of the item data. If the item does not come + from response data, e.g. it was hard-coded, set it to ``None``. + """ + assert self.slot is not None # typing + assert self.crawler.spider is not None # typing + self.slot.itemproc_size += 1 + dfd = self.itemproc.process_item(item, self.crawler.spider) + dfd.addBoth(self._itemproc_finished, item, response, self.crawler.spider) + return dfd + def _log_download_errors( self, spider_failure: Failure, @@ -373,7 +369,7 @@ class Scraper: return None def _itemproc_finished( - self, output: Any, item: Any, response: Response, spider: Spider + self, output: Any, item: Any, response: Optional[Response], spider: Spider ) -> Deferred[Any]: """ItemProcessor finished for the given ``item`` and returned ``output``""" assert self.slot is not None # typing diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index c9feac29c..223e4192e 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -7,22 +7,10 @@ See documentation in docs/topics/spider-middleware.rst from __future__ import annotations import logging +from collections.abc import AsyncIterable, Callable, Iterable from inspect import isasyncgenfunction, iscoroutine from itertools import islice -from typing import ( - TYPE_CHECKING, - Any, - AsyncIterable, - Callable, - Generator, - Iterable, - List, - Optional, - Tuple, - TypeVar, - Union, - cast, -) +from typing import TYPE_CHECKING, Any, Optional, TypeVar, Union, cast from twisted.internet.defer import Deferred, inlineCallbacks from twisted.python.failure import Failure @@ -42,6 +30,8 @@ from scrapy.utils.defer import ( from scrapy.utils.python import MutableAsyncChain, MutableChain if TYPE_CHECKING: + from collections.abc import Generator + from scrapy.settings import BaseSettings @@ -66,7 +56,7 @@ class SpiderMiddlewareManager(MiddlewareManager): self.downgrade_warning_done = False @classmethod - def _get_mwlist_from_settings(cls, settings: BaseSettings) -> List[Any]: + def _get_mwlist_from_settings(cls, settings: BaseSettings) -> list[Any]: return build_component_list(settings.getwithbase("SPIDER_MIDDLEWARES")) def _add_middleware(self, mw: Any) -> None: @@ -349,7 +339,7 @@ class SpiderMiddlewareManager(MiddlewareManager): @staticmethod def _get_async_method_pair( mw: Any, methodname: str - ) -> Union[None, Callable, Tuple[Callable, Callable]]: + ) -> Union[None, Callable, tuple[Callable, Callable]]: normal_method: Optional[Callable] = getattr(mw, methodname, None) methodname_async = methodname + "_async" async_method: Optional[Callable] = getattr(mw, methodname_async, None) diff --git a/scrapy/crawler.py b/scrapy/crawler.py index ecb0a8150..b0a4932e1 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -4,18 +4,7 @@ import logging import pprint import signal import warnings -from typing import ( - TYPE_CHECKING, - Any, - Dict, - Generator, - Optional, - Set, - Type, - TypeVar, - Union, - cast, -) +from typing import TYPE_CHECKING, Any, Optional, TypeVar, Union, cast from twisted.internet.defer import ( Deferred, @@ -53,6 +42,8 @@ from scrapy.utils.reactor import ( ) if TYPE_CHECKING: + from collections.abc import Generator + from scrapy.utils.request import RequestFingerprinter @@ -64,8 +55,8 @@ _T = TypeVar("_T") class Crawler: def __init__( self, - spidercls: Type[Spider], - settings: Union[None, Dict[str, Any], Settings] = None, + spidercls: type[Spider], + settings: Union[None, dict[str, Any], Settings] = None, init_reactor: bool = False, ): if isinstance(spidercls, Spider): @@ -74,7 +65,7 @@ class Crawler: if isinstance(settings, dict) or settings is None: settings = Settings(settings) - self.spidercls: Type[Spider] = spidercls + self.spidercls: type[Spider] = spidercls self.settings: Settings = settings.copy() self.spidercls.update_settings(self.settings) self._update_root_log_handler() @@ -112,7 +103,7 @@ class Crawler: self.__remove_handler = lambda: logging.root.removeHandler(handler) self.signals.connect(self.__remove_handler, signals.engine_stopped) - lf_cls: Type[LogFormatter] = load_object(self.settings["LOG_FORMATTER"]) + lf_cls: type[LogFormatter] = load_object(self.settings["LOG_FORMATTER"]) self.logformatter = lf_cls.from_crawler(self) self.request_fingerprinter = build_from_crawler( @@ -256,18 +247,18 @@ class CrawlerRunner: verifyClass(ISpiderLoader, loader_cls) return loader_cls.from_settings(settings.frozencopy()) - def __init__(self, settings: Union[Dict[str, Any], Settings, None] = None): + def __init__(self, settings: Union[dict[str, Any], Settings, None] = None): if isinstance(settings, dict) or settings is None: settings = Settings(settings) self.settings = settings self.spider_loader = self._get_spider_loader(settings) - self._crawlers: Set[Crawler] = set() - self._active: Set[Deferred[None]] = set() + self._crawlers: set[Crawler] = set() + self._active: set[Deferred[None]] = set() self.bootstrap_failed = False def crawl( self, - crawler_or_spidercls: Union[Type[Spider], str, Crawler], + crawler_or_spidercls: Union[type[Spider], str, Crawler], *args: Any, **kwargs: Any, ) -> Deferred[None]: @@ -314,7 +305,7 @@ class CrawlerRunner: return d.addBoth(_done) def create_crawler( - self, crawler_or_spidercls: Union[Type[Spider], str, Crawler] + self, crawler_or_spidercls: Union[type[Spider], str, Crawler] ) -> Crawler: """ Return a :class:`~scrapy.crawler.Crawler` object. @@ -335,11 +326,11 @@ class CrawlerRunner: return crawler_or_spidercls return self._create_crawler(crawler_or_spidercls) - def _create_crawler(self, spidercls: Union[str, Type[Spider]]) -> Crawler: + def _create_crawler(self, spidercls: Union[str, type[Spider]]) -> Crawler: if isinstance(spidercls, str): spidercls = self.spider_loader.load(spidercls) # temporary cast until self.spider_loader is typed - return Crawler(cast(Type[Spider], spidercls), self.settings) + return Crawler(cast(type[Spider], spidercls), self.settings) def stop(self) -> Deferred[Any]: """ @@ -387,7 +378,7 @@ class CrawlerProcess(CrawlerRunner): def __init__( self, - settings: Union[Dict[str, Any], Settings, None] = None, + settings: Union[dict[str, Any], Settings, None] = None, install_root_handler: bool = True, ): super().__init__(settings) @@ -416,14 +407,14 @@ class CrawlerProcess(CrawlerRunner): ) reactor.callFromThread(self._stop_reactor) - def _create_crawler(self, spidercls: Union[Type[Spider], str]) -> Crawler: + def _create_crawler(self, spidercls: Union[type[Spider], str]) -> Crawler: if isinstance(spidercls, str): spidercls = self.spider_loader.load(spidercls) init_reactor = not self._initialized_reactor self._initialized_reactor = True # temporary cast until self.spider_loader is typed return Crawler( - cast(Type[Spider], spidercls), self.settings, init_reactor=init_reactor + cast(type[Spider], spidercls), self.settings, init_reactor=init_reactor ) def start( diff --git a/scrapy/downloadermiddlewares/cookies.py b/scrapy/downloadermiddlewares/cookies.py index 23140d263..e384793ee 100644 --- a/scrapy/downloadermiddlewares/cookies.py +++ b/scrapy/downloadermiddlewares/cookies.py @@ -2,7 +2,7 @@ from __future__ import annotations import logging from collections import defaultdict -from typing import TYPE_CHECKING, Any, DefaultDict, Iterable, Optional, Sequence, Union +from typing import TYPE_CHECKING, Any, Optional, Union from tldextract import TLDExtract @@ -13,6 +13,7 @@ from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.python import to_unicode if TYPE_CHECKING: + from collections.abc import Iterable, Sequence from http.cookiejar import Cookie # typing.Self requires Python 3.11 @@ -39,7 +40,7 @@ class CookiesMiddleware: """This middleware enables working with sites that need cookies""" def __init__(self, debug: bool = False): - self.jars: DefaultDict[Any, CookieJar] = defaultdict(CookieJar) + self.jars: defaultdict[Any, CookieJar] = defaultdict(CookieJar) self.debug: bool = debug @classmethod diff --git a/scrapy/downloadermiddlewares/defaultheaders.py b/scrapy/downloadermiddlewares/defaultheaders.py index 49b9fdc05..312c1e026 100644 --- a/scrapy/downloadermiddlewares/defaultheaders.py +++ b/scrapy/downloadermiddlewares/defaultheaders.py @@ -6,11 +6,13 @@ See documentation in docs/topics/downloader-middleware.rst from __future__ import annotations -from typing import TYPE_CHECKING, Iterable, Tuple, Union +from typing import TYPE_CHECKING, Union from scrapy.utils.python import without_none_values if TYPE_CHECKING: + from collections.abc import Iterable + # typing.Self requires Python 3.11 from typing_extensions import Self @@ -20,8 +22,8 @@ if TYPE_CHECKING: class DefaultHeadersMiddleware: - def __init__(self, headers: Iterable[Tuple[str, str]]): - self._headers: Iterable[Tuple[str, str]] = headers + def __init__(self, headers: Iterable[tuple[str, str]]): + self._headers: Iterable[tuple[str, str]] = headers @classmethod def from_crawler(cls, crawler: Crawler) -> Self: diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py index 6b0a56f7f..b0cede97d 100644 --- a/scrapy/downloadermiddlewares/httpcompression.py +++ b/scrapy/downloadermiddlewares/httpcompression.py @@ -3,7 +3,7 @@ from __future__ import annotations import warnings from itertools import chain from logging import getLogger -from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union +from typing import TYPE_CHECKING, Any, Optional, Union from scrapy import Request, Spider, signals from scrapy.exceptions import IgnoreRequest, NotConfigured @@ -28,7 +28,7 @@ if TYPE_CHECKING: logger = getLogger(__name__) -ACCEPTED_ENCODINGS: List[bytes] = [b"gzip", b"deflate"] +ACCEPTED_ENCODINGS: list[bytes] = [b"gzip", b"deflate"] try: try: @@ -50,7 +50,7 @@ else: class HttpCompressionMiddleware: """This middleware allows compressed (gzip, deflate) traffic to be - sent/received from web sites""" + sent/received from websites""" def __init__( self, @@ -140,7 +140,7 @@ class HttpCompressionMiddleware: respcls = responsetypes.from_args( headers=response.headers, url=response.url, body=decoded_body ) - kwargs: Dict[str, Any] = {"body": decoded_body} + kwargs: dict[str, Any] = {"body": decoded_body} if issubclass(respcls, TextResponse): # force recalculating the encoding until we make sure the # responsetypes guessing is reliable @@ -152,23 +152,23 @@ class HttpCompressionMiddleware: return response def _handle_encoding( - self, body: bytes, content_encoding: List[bytes], max_size: int - ) -> Tuple[bytes, List[bytes]]: + self, body: bytes, content_encoding: list[bytes], max_size: int + ) -> tuple[bytes, list[bytes]]: to_decode, to_keep = self._split_encodings(content_encoding) for encoding in to_decode: body = self._decode(body, encoding, max_size) return body, to_keep def _split_encodings( - self, content_encoding: List[bytes] - ) -> Tuple[List[bytes], List[bytes]]: - to_keep: List[bytes] = [ + self, content_encoding: list[bytes] + ) -> tuple[list[bytes], list[bytes]]: + to_keep: list[bytes] = [ encoding.strip().lower() for encoding in chain.from_iterable( encodings.split(b",") for encodings in content_encoding ) ] - to_decode: List[bytes] = [] + to_decode: list[bytes] = [] while to_keep: encoding = to_keep.pop() if encoding not in ACCEPTED_ENCODINGS: diff --git a/scrapy/downloadermiddlewares/httpproxy.py b/scrapy/downloadermiddlewares/httpproxy.py index a7af83f7d..b35ecbd54 100644 --- a/scrapy/downloadermiddlewares/httpproxy.py +++ b/scrapy/downloadermiddlewares/httpproxy.py @@ -1,7 +1,7 @@ from __future__ import annotations import base64 -from typing import TYPE_CHECKING, Dict, Optional, Tuple, Union +from typing import TYPE_CHECKING, Optional, Union from urllib.parse import unquote, urlunparse from urllib.request import ( # type: ignore[attr-defined] _parse_proxy, @@ -25,7 +25,7 @@ if TYPE_CHECKING: class HttpProxyMiddleware: def __init__(self, auth_encoding: Optional[str] = "latin-1"): self.auth_encoding: Optional[str] = auth_encoding - self.proxies: Dict[str, Tuple[Optional[bytes], str]] = {} + self.proxies: dict[str, tuple[Optional[bytes], str]] = {} for type_, url in getproxies().items(): try: self.proxies[type_] = self._get_proxy(url, type_) @@ -47,7 +47,7 @@ class HttpProxyMiddleware: ) return base64.b64encode(user_pass) - def _get_proxy(self, url: str, orig_type: str) -> Tuple[Optional[bytes], str]: + def _get_proxy(self, url: str, orig_type: str) -> tuple[Optional[bytes], str]: proxy_type, user, password, hostport = _parse_proxy(url) proxy_url = urlunparse((proxy_type or orig_type, hostport, "", "", "", "")) diff --git a/scrapy/downloadermiddlewares/offsite.py b/scrapy/downloadermiddlewares/offsite.py index 6f67e3975..05ec4cad4 100644 --- a/scrapy/downloadermiddlewares/offsite.py +++ b/scrapy/downloadermiddlewares/offsite.py @@ -3,7 +3,7 @@ from __future__ import annotations import logging import re import warnings -from typing import TYPE_CHECKING, Set +from typing import TYPE_CHECKING from scrapy import Request, Spider, signals from scrapy.exceptions import IgnoreRequest @@ -31,7 +31,7 @@ class OffsiteMiddleware: def __init__(self, stats: StatsCollector): self.stats = stats - self.domains_seen: Set[str] = set() + self.domains_seen: set[str] = set() def spider_opened(self, spider: Spider) -> None: self.host_regex: re.Pattern[str] = self.get_host_regex(spider) diff --git a/scrapy/downloadermiddlewares/redirect.py b/scrapy/downloadermiddlewares/redirect.py index 53081237c..6437485cf 100644 --- a/scrapy/downloadermiddlewares/redirect.py +++ b/scrapy/downloadermiddlewares/redirect.py @@ -1,7 +1,7 @@ from __future__ import annotations import logging -from typing import TYPE_CHECKING, Any, List, Union, cast +from typing import TYPE_CHECKING, Any, Union, cast from urllib.parse import urljoin from w3lib.url import safe_url_string @@ -180,7 +180,7 @@ class MetaRefreshMiddleware(BaseRedirectMiddleware): def __init__(self, settings: BaseSettings): super().__init__(settings) - self._ignore_tags: List[str] = settings.getlist("METAREFRESH_IGNORE_TAGS") + self._ignore_tags: list[str] = settings.getlist("METAREFRESH_IGNORE_TAGS") self._maxdelay: int = settings.getint("METAREFRESH_MAXDELAY") def process_response( diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.py index 8d7b7293c..c32624371 100644 --- a/scrapy/downloadermiddlewares/retry.py +++ b/scrapy/downloadermiddlewares/retry.py @@ -7,14 +7,14 @@ RETRY_TIMES - how many times to retry a failed page RETRY_HTTP_CODES - which HTTP response codes to retry Failed pages are collected on the scraping process and rescheduled at the end, -once the spider has finished crawling all regular (non failed) pages. +once the spider has finished crawling all regular (non-failed) pages. """ from __future__ import annotations import warnings from logging import Logger, getLogger -from typing import TYPE_CHECKING, Any, Optional, Tuple, Type, Union +from typing import TYPE_CHECKING, Any, Optional, Union from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning from scrapy.settings import BaseSettings, Settings @@ -35,7 +35,7 @@ if TYPE_CHECKING: retry_logger = getLogger(__name__) -def backwards_compatibility_getattr(self: Any, name: str) -> Tuple[Any, ...]: +def backwards_compatibility_getattr(self: Any, name: str) -> tuple[Any, ...]: if name == "EXCEPTIONS_TO_RETRY": warnings.warn( "Attribute RetryMiddleware.EXCEPTIONS_TO_RETRY is deprecated. " @@ -60,7 +60,7 @@ def get_retry_request( request: Request, *, spider: Spider, - reason: Union[str, Exception, Type[Exception]] = "unspecified", + reason: Union[str, Exception, type[Exception]] = "unspecified", max_retry_times: Optional[int] = None, priority_adjust: Optional[int] = None, logger: Logger = retry_logger, @@ -187,7 +187,7 @@ class RetryMiddleware(metaclass=BackwardsCompatibilityMetaclass): def _retry( self, request: Request, - reason: Union[str, Exception, Type[Exception]], + reason: Union[str, Exception, type[Exception]], spider: Spider, ) -> Optional[Request]: max_retry_times = request.meta.get("max_retry_times", self.max_retry_times) diff --git a/scrapy/downloadermiddlewares/robotstxt.py b/scrapy/downloadermiddlewares/robotstxt.py index 73757162f..421c58e68 100644 --- a/scrapy/downloadermiddlewares/robotstxt.py +++ b/scrapy/downloadermiddlewares/robotstxt.py @@ -7,7 +7,7 @@ enable this middleware and enable the ROBOTSTXT_OBEY setting. from __future__ import annotations import logging -from typing import TYPE_CHECKING, Dict, Optional, TypeVar, Union +from typing import TYPE_CHECKING, Optional, TypeVar, Union from twisted.internet.defer import Deferred, maybeDeferred @@ -45,7 +45,7 @@ class RobotsTxtMiddleware: "ROBOTSTXT_USER_AGENT", None ) self.crawler: Crawler = crawler - self._parsers: Dict[ + self._parsers: dict[ str, Union[RobotParser, Deferred[Optional[RobotParser]], None] ] = {} self._parserimpl: RobotParser = load_object( diff --git a/scrapy/downloadermiddlewares/stats.py b/scrapy/downloadermiddlewares/stats.py index 0faae7b5a..ab5655393 100644 --- a/scrapy/downloadermiddlewares/stats.py +++ b/scrapy/downloadermiddlewares/stats.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Dict, List, Tuple, Union +from typing import TYPE_CHECKING, Union from twisted.web import http @@ -19,7 +19,7 @@ if TYPE_CHECKING: def get_header_size( - headers: Dict[str, Union[List[Union[str, bytes]], Tuple[Union[str, bytes], ...]]] + headers: dict[str, Union[list[Union[str, bytes]], tuple[Union[str, bytes], ...]]] ) -> int: size = 0 for key, value in headers.items(): diff --git a/scrapy/dupefilters.py b/scrapy/dupefilters.py index 40ea48510..28118977d 100644 --- a/scrapy/dupefilters.py +++ b/scrapy/dupefilters.py @@ -2,7 +2,7 @@ from __future__ import annotations import logging from pathlib import Path -from typing import TYPE_CHECKING, Optional, Set +from typing import TYPE_CHECKING, Optional from scrapy.utils.job import job_dir from scrapy.utils.request import ( @@ -56,7 +56,7 @@ class RFPDupeFilter(BaseDupeFilter): self.fingerprinter: RequestFingerprinterProtocol = ( fingerprinter or RequestFingerprinter() ) - self.fingerprints: Set[str] = set() + self.fingerprints: set[str] = set() self.logdupes = True self.debug = debug self.logger = logging.getLogger(__name__) diff --git a/scrapy/exporters.py b/scrapy/exporters.py index fb4998099..ee0033dfb 100644 --- a/scrapy/exporters.py +++ b/scrapy/exporters.py @@ -6,9 +6,10 @@ import csv import marshal import pickle # nosec import pprint +from collections.abc import Callable, Iterable, Mapping from io import BytesIO, TextIOWrapper from json import JSONEncoder -from typing import Any, Callable, Dict, Iterable, Mapping, Optional, Tuple, Union +from typing import Any, Optional, Union from xml.sax.saxutils import XMLGenerator # nosec from xml.sax.xmlreader import AttributesImpl # nosec @@ -32,10 +33,10 @@ __all__ = [ class BaseItemExporter: def __init__(self, *, dont_fail: bool = False, **kwargs: Any): - self._kwargs: Dict[str, Any] = kwargs + self._kwargs: dict[str, Any] = kwargs self._configure(kwargs, dont_fail=dont_fail) - def _configure(self, options: Dict[str, Any], dont_fail: bool = False) -> None: + def _configure(self, options: dict[str, Any], dont_fail: bool = False) -> None: """Configure the exporter by popping options from the ``options`` dict. If dont_fail is set, it won't raise an exception on unexpected options (useful for using with keyword arguments in subclasses ``__init__`` methods) @@ -66,7 +67,7 @@ class BaseItemExporter: def _get_serialized_fields( self, item: Any, default_value: Any = None, include_empty: Optional[bool] = None - ) -> Iterable[Tuple[str, Any]]: + ) -> Iterable[tuple[str, Any]]: """Return the fields to export as an iterable of tuples (name, serialized_value) """ @@ -339,7 +340,7 @@ class PythonItemExporter(BaseItemExporter): .. _msgpack: https://pypi.org/project/msgpack/ """ - def _configure(self, options: Dict[str, Any], dont_fail: bool = False) -> None: + def _configure(self, options: dict[str, Any], dont_fail: bool = False) -> None: super()._configure(options, dont_fail) if not self.encoding: self.encoding = "utf-8" @@ -363,10 +364,10 @@ class PythonItemExporter(BaseItemExporter): return to_unicode(value, encoding=self.encoding) return value - def _serialize_item(self, item: Any) -> Iterable[Tuple[Union[str, bytes], Any]]: + def _serialize_item(self, item: Any) -> Iterable[tuple[Union[str, bytes], Any]]: for key, value in ItemAdapter(item).items(): yield key, self._serialize_value(value) - def export_item(self, item: Any) -> Dict[Union[str, bytes], Any]: # type: ignore[override] - result: Dict[Union[str, bytes], Any] = dict(self._get_serialized_fields(item)) + def export_item(self, item: Any) -> dict[Union[str, bytes], Any]: # type: ignore[override] + result: dict[Union[str, bytes], Any] = dict(self._get_serialized_fields(item)) return result diff --git a/scrapy/extension.py b/scrapy/extension.py index 8c81ab356..9f978fa32 100644 --- a/scrapy/extension.py +++ b/scrapy/extension.py @@ -6,7 +6,7 @@ See documentation in docs/topics/extensions.rst from __future__ import annotations -from typing import TYPE_CHECKING, Any, List +from typing import TYPE_CHECKING, Any from scrapy.middleware import MiddlewareManager from scrapy.utils.conf import build_component_list @@ -19,5 +19,5 @@ class ExtensionManager(MiddlewareManager): component_name = "extension" @classmethod - def _get_mwlist_from_settings(cls, settings: Settings) -> List[Any]: + def _get_mwlist_from_settings(cls, settings: Settings) -> list[Any]: return build_component_list(settings.getwithbase("EXTENSIONS")) diff --git a/scrapy/extensions/closespider.py b/scrapy/extensions/closespider.py index cef5527b7..dff8bc97e 100644 --- a/scrapy/extensions/closespider.py +++ b/scrapy/extensions/closespider.py @@ -8,7 +8,7 @@ from __future__ import annotations import logging from collections import defaultdict -from typing import TYPE_CHECKING, Any, DefaultDict, Dict +from typing import TYPE_CHECKING, Any from scrapy import Request, Spider, signals from scrapy.exceptions import NotConfigured @@ -30,7 +30,7 @@ class CloseSpider: def __init__(self, crawler: Crawler): self.crawler: Crawler = crawler - self.close_on: Dict[str, Any] = { + self.close_on: dict[str, Any] = { "timeout": crawler.settings.getfloat("CLOSESPIDER_TIMEOUT"), "itemcount": crawler.settings.getint("CLOSESPIDER_ITEMCOUNT"), "pagecount": crawler.settings.getint("CLOSESPIDER_PAGECOUNT"), @@ -44,7 +44,7 @@ class CloseSpider: if not any(self.close_on.values()): raise NotConfigured - self.counter: DefaultDict[str, int] = defaultdict(int) + self.counter: defaultdict[str, int] = defaultdict(int) if self.close_on.get("errorcount"): crawler.signals.connect(self.error_count, signal=signals.spider_error) diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 0d7f5bfd4..b1001dabb 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -10,25 +10,11 @@ import logging import re import sys import warnings +from collections.abc import Callable from datetime import datetime, timezone from pathlib import Path, PureWindowsPath from tempfile import NamedTemporaryFile -from typing import ( - IO, - TYPE_CHECKING, - Any, - Callable, - Dict, - Iterable, - List, - Optional, - Protocol, - Tuple, - Type, - TypeVar, - Union, - cast, -) +from typing import IO, TYPE_CHECKING, Any, Optional, Protocol, TypeVar, Union, cast from urllib.parse import unquote, urlparse from twisted.internet.defer import Deferred, DeferredList, maybeDeferred @@ -50,6 +36,8 @@ from scrapy.utils.misc import build_from_crawler, load_object from scrapy.utils.python import without_none_values if TYPE_CHECKING: + from collections.abc import Iterable + from _typeshed import OpenBinaryMode from twisted.python.failure import Failure @@ -70,7 +58,7 @@ except ImportError: logger = logging.getLogger(__name__) -UriParamsCallableT = Callable[[Dict[str, Any], Spider], Optional[Dict[str, Any]]] +UriParamsCallableT = Callable[[dict[str, Any], Spider], Optional[dict[str, Any]]] _StorageT = TypeVar("_StorageT", bound="FeedStorageProtocol") @@ -79,7 +67,7 @@ def build_storage( builder: Callable[..., _StorageT], uri: str, *args: Any, - feed_options: Optional[Dict[str, Any]] = None, + feed_options: Optional[dict[str, Any]] = None, preargs: Iterable[Any] = (), **kwargs: Any, ) -> _StorageT: @@ -96,10 +84,10 @@ class ItemFilter: :type feed_options: dict """ - feed_options: Optional[Dict[str, Any]] - item_classes: Tuple[type, ...] + feed_options: Optional[dict[str, Any]] + item_classes: tuple[type, ...] - def __init__(self, feed_options: Optional[Dict[str, Any]]) -> None: + def __init__(self, feed_options: Optional[dict[str, Any]]) -> None: self.feed_options = feed_options if feed_options is not None: self.item_classes = tuple( @@ -141,7 +129,7 @@ class IFeedStorage(Interface): class FeedStorageProtocol(Protocol): """Reimplementation of ``IFeedStorage`` that can be used in type hints.""" - def __init__(self, uri: str, *, feed_options: Optional[Dict[str, Any]] = None): + def __init__(self, uri: str, *, feed_options: Optional[dict[str, Any]] = None): """Initialize the storage with the parameters given in the URI and the feed-specific options (see :setting:`FEEDS`)""" @@ -176,7 +164,7 @@ class StdoutFeedStorage: uri: str, _stdout: Optional[IO[bytes]] = None, *, - feed_options: Optional[Dict[str, Any]] = None, + feed_options: Optional[dict[str, Any]] = None, ): if not _stdout: _stdout = sys.stdout.buffer @@ -198,7 +186,7 @@ class StdoutFeedStorage: @implementer(IFeedStorage) class FileFeedStorage: - def __init__(self, uri: str, *, feed_options: Optional[Dict[str, Any]] = None): + def __init__(self, uri: str, *, feed_options: Optional[dict[str, Any]] = None): self.path: str = file_uri_to_path(uri) feed_options = feed_options or {} self.write_mode: OpenBinaryMode = ( @@ -225,7 +213,7 @@ class S3FeedStorage(BlockingFeedStorage): acl: Optional[str] = None, endpoint_url: Optional[str] = None, *, - feed_options: Optional[Dict[str, Any]] = None, + feed_options: Optional[dict[str, Any]] = None, session_token: Optional[str] = None, region_name: Optional[str] = None, ): @@ -291,7 +279,7 @@ class S3FeedStorage(BlockingFeedStorage): crawler: Crawler, uri: str, *, - feed_options: Optional[Dict[str, Any]] = None, + feed_options: Optional[dict[str, Any]] = None, ) -> Self: return build_storage( cls, @@ -307,7 +295,7 @@ class S3FeedStorage(BlockingFeedStorage): def _store_in_thread(self, file: IO[bytes]) -> None: file.seek(0) - kwargs: Dict[str, Any] + kwargs: dict[str, Any] if IS_BOTO3_AVAILABLE: kwargs = {"ExtraArgs": {"ACL": self.acl}} if self.acl else {} self.s3_client.upload_fileobj( @@ -354,7 +342,7 @@ class FTPFeedStorage(BlockingFeedStorage): uri: str, use_active_mode: bool = False, *, - feed_options: Optional[Dict[str, Any]] = None, + feed_options: Optional[dict[str, Any]] = None, ): u = urlparse(uri) if not u.hostname: @@ -373,7 +361,7 @@ class FTPFeedStorage(BlockingFeedStorage): crawler: Crawler, uri: str, *, - feed_options: Optional[Dict[str, Any]] = None, + feed_options: Optional[dict[str, Any]] = None, ) -> Self: return build_storage( cls, @@ -405,9 +393,9 @@ class FeedSlot: batch_id: int, uri_template: str, filter: ItemFilter, - feed_options: Dict[str, Any], + feed_options: dict[str, Any], spider: Spider, - exporters: Dict[str, Type[BaseItemExporter]], + exporters: dict[str, type[BaseItemExporter]], settings: BaseSettings, crawler: Crawler, ): @@ -422,9 +410,9 @@ class FeedSlot: self.uri: str = uri self.filter: ItemFilter = filter # exporter params - self.feed_options: Dict[str, Any] = feed_options + self.feed_options: dict[str, Any] = feed_options self.spider: Spider = spider - self.exporters: Dict[str, Type[BaseItemExporter]] = exporters + self.exporters: dict[str, type[BaseItemExporter]] = exporters self.settings: BaseSettings = settings self.crawler: Crawler = crawler # flags @@ -460,7 +448,7 @@ class FeedSlot: self._exporting = True def _get_instance( - self, objcls: Type[BaseItemExporter], *args: Any, **kwargs: Any + self, objcls: type[BaseItemExporter], *args: Any, **kwargs: Any ) -> BaseItemExporter: return build_from_crawler(objcls, self.crawler, *args, **kwargs) @@ -483,7 +471,7 @@ _FeedSlot = create_deprecated_class( class FeedExporter: - _pending_deferreds: List[Deferred[None]] = [] + _pending_deferreds: list[Deferred[None]] = [] @classmethod def from_crawler(cls, crawler: Crawler) -> Self: @@ -497,8 +485,8 @@ class FeedExporter: self.crawler: Crawler = crawler self.settings: Settings = crawler.settings self.feeds = {} - self.slots: List[FeedSlot] = [] - self.filters: Dict[str, ItemFilter] = {} + self.slots: list[FeedSlot] = [] + self.filters: dict[str, ItemFilter] = {} if not self.settings["FEEDS"] and not self.settings["FEED_URI"]: raise NotConfigured @@ -530,10 +518,10 @@ class FeedExporter: ) self.filters[uri] = self._load_filter(feed_options) - self.storages: Dict[str, Type[FeedStorageProtocol]] = self._load_components( + self.storages: dict[str, type[FeedStorageProtocol]] = self._load_components( "FEED_STORAGES" ) - self.exporters: Dict[str, Type[BaseItemExporter]] = self._load_components( + self.exporters: dict[str, type[BaseItemExporter]] = self._load_components( "FEED_EXPORTERS" ) for uri, feed_options in self.feeds.items(): @@ -631,7 +619,7 @@ class FeedExporter: self, batch_id: int, uri: str, - feed_options: Dict[str, Any], + feed_options: dict[str, Any], spider: Spider, uri_template: str, ) -> FeedSlot: @@ -696,9 +684,9 @@ class FeedExporter: slots.append(slot) self.slots = slots - def _load_components(self, setting_prefix: str) -> Dict[str, Any]: + def _load_components(self, setting_prefix: str) -> dict[str, Any]: conf = without_none_values( - cast(Dict[str, str], self.settings.getwithbase(setting_prefix)) + cast(dict[str, str], self.settings.getwithbase(setting_prefix)) ) d = {} for k, v in conf.items(): @@ -732,7 +720,7 @@ class FeedExporter: return False return True - def _storage_supported(self, uri: str, feed_options: Dict[str, Any]) -> bool: + def _storage_supported(self, uri: str, feed_options: dict[str, Any]) -> bool: scheme = urlparse(uri).scheme if scheme in self.storages or PureWindowsPath(uri).drive: try: @@ -748,7 +736,7 @@ class FeedExporter: return False def _get_storage( - self, uri: str, feed_options: Dict[str, Any] + self, uri: str, feed_options: dict[str, Any] ) -> FeedStorageProtocol: """Fork of create_instance specific to feed storage classes @@ -759,7 +747,7 @@ class FeedExporter: crawler = getattr(self, "crawler", None) def build_instance( - builder: Type[FeedStorageProtocol], *preargs: Any + builder: type[FeedStorageProtocol], *preargs: Any ) -> FeedStorageProtocol: return build_storage( builder, uri, feed_options=feed_options, preargs=preargs @@ -784,7 +772,7 @@ class FeedExporter: spider: Spider, uri_params_function: Union[str, UriParamsCallableT, None], slot: Optional[FeedSlot] = None, - ) -> Dict[str, Any]: + ) -> dict[str, Any]: params = {} for k in dir(spider): params[k] = getattr(spider, k) @@ -800,9 +788,9 @@ class FeedExporter: new_params = uripar_function(params, spider) return new_params if new_params is not None else params - def _load_filter(self, feed_options: Dict[str, Any]) -> ItemFilter: + def _load_filter(self, feed_options: dict[str, Any]) -> ItemFilter: # load the item filter if declared else load the default filter class - item_filter_class: Type[ItemFilter] = load_object( + item_filter_class: type[ItemFilter] = load_object( feed_options.get("item_filter", ItemFilter) ) return item_filter_class(feed_options) diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.py index 448d5f1ab..a72f9db51 100644 --- a/scrapy/extensions/httpcache.py +++ b/scrapy/extensions/httpcache.py @@ -9,7 +9,7 @@ from importlib import import_module from pathlib import Path from time import time from types import ModuleType -from typing import IO, TYPE_CHECKING, Any, Callable, Dict, List, Optional, Union, cast +from typing import IO, TYPE_CHECKING, Any, Optional, Union, cast from weakref import WeakKeyDictionary from w3lib.http import headers_dict_to_raw, headers_raw_to_dict @@ -22,6 +22,8 @@ from scrapy.utils.python import to_bytes, to_unicode from scrapy.utils.request import RequestFingerprinter if TYPE_CHECKING: + from collections.abc import Callable + # typing.Concatenate requires Python 3.10 from typing_extensions import Concatenate @@ -35,8 +37,8 @@ logger = logging.getLogger(__name__) class DummyPolicy: def __init__(self, settings: BaseSettings): - self.ignore_schemes: List[str] = settings.getlist("HTTPCACHE_IGNORE_SCHEMES") - self.ignore_http_codes: List[int] = [ + self.ignore_schemes: list[str] = settings.getlist("HTTPCACHE_IGNORE_SCHEMES") + self.ignore_http_codes: list[int] = [ int(x) for x in settings.getlist("HTTPCACHE_IGNORE_HTTP_CODES") ] @@ -62,18 +64,18 @@ class RFC2616Policy: def __init__(self, settings: BaseSettings): self.always_store: bool = settings.getbool("HTTPCACHE_ALWAYS_STORE") - self.ignore_schemes: List[str] = settings.getlist("HTTPCACHE_IGNORE_SCHEMES") + self.ignore_schemes: list[str] = settings.getlist("HTTPCACHE_IGNORE_SCHEMES") self._cc_parsed: WeakKeyDictionary[ - Union[Request, Response], Dict[bytes, Optional[bytes]] + Union[Request, Response], dict[bytes, Optional[bytes]] ] = WeakKeyDictionary() - self.ignore_response_cache_controls: List[bytes] = [ + self.ignore_response_cache_controls: list[bytes] = [ to_bytes(cc) for cc in settings.getlist("HTTPCACHE_IGNORE_RESPONSE_CACHE_CONTROLS") ] def _parse_cachecontrol( self, r: Union[Request, Response] - ) -> Dict[bytes, Optional[bytes]]: + ) -> dict[bytes, Optional[bytes]]: if r not in self._cc_parsed: cch = r.headers.get(b"Cache-Control", b"") assert cch is not None @@ -189,7 +191,7 @@ class RFC2616Policy: if b"ETag" in cachedresponse.headers: request.headers[b"If-None-Match"] = cachedresponse.headers[b"ETag"] - def _get_max_age(self, cc: Dict[bytes, Optional[bytes]]) -> Optional[int]: + def _get_max_age(self, cc: dict[bytes, Optional[bytes]]) -> Optional[int]: try: return max(0, int(cc[b"max-age"])) # type: ignore[arg-type] except (KeyError, ValueError): @@ -298,7 +300,7 @@ class DbmCacheStorage: self.db[f"{key}_data"] = pickle.dumps(data, protocol=4) self.db[f"{key}_time"] = str(time()) - def _read_data(self, spider: Spider, request: Request) -> Optional[Dict[str, Any]]: + def _read_data(self, spider: Spider, request: Request) -> Optional[dict[str, Any]]: key = self._fingerprinter.fingerprint(request).hex() db = self.db tkey = f"{key}_time" @@ -309,7 +311,7 @@ class DbmCacheStorage: if 0 < self.expiration_secs < time() - float(ts): return None # expired - return cast(Dict[str, Any], pickle.loads(db[f"{key}_data"])) # nosec + return cast(dict[str, Any], pickle.loads(db[f"{key}_data"])) # nosec class FilesystemCacheStorage: @@ -385,7 +387,7 @@ class FilesystemCacheStorage: key = self._fingerprinter.fingerprint(request).hex() return str(Path(self.cachedir, spider.name, key[0:2], key)) - def _read_meta(self, spider: Spider, request: Request) -> Optional[Dict[str, Any]]: + def _read_meta(self, spider: Spider, request: Request) -> Optional[dict[str, Any]]: rpath = Path(self._get_request_path(spider, request)) metapath = rpath / "pickled_meta" if not metapath.exists(): @@ -394,10 +396,10 @@ class FilesystemCacheStorage: if 0 < self.expiration_secs < time() - mtime: return None # expired with self._open(metapath, "rb") as f: - return cast(Dict[str, Any], pickle.load(f)) # nosec + return cast(dict[str, Any], pickle.load(f)) # nosec -def parse_cachecontrol(header: bytes) -> Dict[bytes, Optional[bytes]]: +def parse_cachecontrol(header: bytes) -> dict[bytes, Optional[bytes]]: """Parse Cache-Control header https://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.9 diff --git a/scrapy/extensions/logstats.py b/scrapy/extensions/logstats.py index c4f43482d..01484481b 100644 --- a/scrapy/extensions/logstats.py +++ b/scrapy/extensions/logstats.py @@ -1,7 +1,7 @@ from __future__ import annotations import logging -from typing import TYPE_CHECKING, Optional, Tuple, Union +from typing import TYPE_CHECKING, Optional, Union from twisted.internet import task @@ -81,7 +81,7 @@ class LogStats: def calculate_final_stats( self, spider: Spider - ) -> Union[Tuple[None, None], Tuple[float, float]]: + ) -> Union[tuple[None, None], tuple[float, float]]: start_time = self.stats.get_value("start_time") finished_time = self.stats.get_value("finished_time") diff --git a/scrapy/extensions/memusage.py b/scrapy/extensions/memusage.py index 25f63ecc6..73d864d5d 100644 --- a/scrapy/extensions/memusage.py +++ b/scrapy/extensions/memusage.py @@ -11,7 +11,7 @@ import socket import sys from importlib import import_module from pprint import pformat -from typing import TYPE_CHECKING, List +from typing import TYPE_CHECKING from twisted.internet import task @@ -42,7 +42,7 @@ class MemoryUsage: self.crawler: Crawler = crawler self.warned: bool = False - self.notify_mails: List[str] = crawler.settings.getlist("MEMUSAGE_NOTIFY_MAIL") + self.notify_mails: list[str] = crawler.settings.getlist("MEMUSAGE_NOTIFY_MAIL") self.limit: int = crawler.settings.getint("MEMUSAGE_LIMIT_MB") * 1024 * 1024 self.warning: int = crawler.settings.getint("MEMUSAGE_WARNING_MB") * 1024 * 1024 self.check_interval: float = crawler.settings.getfloat( @@ -66,7 +66,7 @@ class MemoryUsage: def engine_started(self) -> None: assert self.crawler.stats self.crawler.stats.set_value("memusage/startup", self.get_virtual_size()) - self.tasks: List[task.LoopingCall] = [] + self.tasks: list[task.LoopingCall] = [] tsk = task.LoopingCall(self.update) self.tasks.append(tsk) tsk.start(self.check_interval, now=True) @@ -141,7 +141,7 @@ class MemoryUsage: self.crawler.stats.set_value("memusage/warning_notified", 1) self.warned = True - def _send_report(self, rcpts: List[str], subject: str) -> None: + def _send_report(self, rcpts: list[str], subject: str) -> None: """send notification mail with some additional useful info""" assert self.crawler.engine assert self.crawler.stats diff --git a/scrapy/extensions/periodic_log.py b/scrapy/extensions/periodic_log.py index 80c0a3b26..fba12bec7 100644 --- a/scrapy/extensions/periodic_log.py +++ b/scrapy/extensions/periodic_log.py @@ -3,7 +3,7 @@ from __future__ import annotations import logging from datetime import datetime, timezone from json import JSONEncoder -from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union +from typing import TYPE_CHECKING, Any, Optional, Union from twisted.internet import task @@ -29,8 +29,8 @@ class PeriodicLog: self, stats: StatsCollector, interval: float = 60.0, - ext_stats: Dict[str, Any] = {}, - ext_delta: Dict[str, Any] = {}, + ext_stats: dict[str, Any] = {}, + ext_delta: dict[str, Any] = {}, ext_timing_enabled: bool = False, ): self.stats: StatsCollector = stats @@ -39,11 +39,11 @@ class PeriodicLog: self.task: Optional[task.LoopingCall] = None self.encoder: JSONEncoder = ScrapyJSONEncoder(sort_keys=True, indent=4) self.ext_stats_enabled: bool = bool(ext_stats) - self.ext_stats_include: List[str] = ext_stats.get("include", []) - self.ext_stats_exclude: List[str] = ext_stats.get("exclude", []) + self.ext_stats_include: list[str] = ext_stats.get("include", []) + self.ext_stats_exclude: list[str] = ext_stats.get("exclude", []) self.ext_delta_enabled: bool = bool(ext_delta) - self.ext_delta_include: List[str] = ext_delta.get("include", []) - self.ext_delta_exclude: List[str] = ext_delta.get("exclude", []) + self.ext_delta_include: list[str] = ext_delta.get("include", []) + self.ext_delta_exclude: list[str] = ext_delta.get("exclude", []) self.ext_timing_enabled: bool = ext_timing_enabled @classmethod @@ -52,7 +52,7 @@ class PeriodicLog: if not interval: raise NotConfigured try: - ext_stats: Optional[Dict[str, Any]] = crawler.settings.getdict( + ext_stats: Optional[dict[str, Any]] = crawler.settings.getdict( "PERIODIC_LOG_STATS" ) except (TypeError, ValueError): @@ -62,7 +62,7 @@ class PeriodicLog: else None ) try: - ext_delta: Optional[Dict[str, Any]] = crawler.settings.getdict( + ext_delta: Optional[dict[str, Any]] = crawler.settings.getdict( "PERIODIC_LOG_DELTA" ) except (TypeError, ValueError): @@ -93,14 +93,14 @@ class PeriodicLog: def spider_opened(self, spider: Spider) -> None: self.time_prev: datetime = datetime.now(tz=timezone.utc) - self.delta_prev: Dict[str, Union[int, float]] = {} - self.stats_prev: Dict[str, Union[int, float]] = {} + self.delta_prev: dict[str, Union[int, float]] = {} + self.stats_prev: dict[str, Union[int, float]] = {} self.task = task.LoopingCall(self.log) self.task.start(self.interval) def log(self) -> None: - data: Dict[str, Any] = {} + data: dict[str, Any] = {} if self.ext_timing_enabled: data.update(self.log_timing()) if self.ext_delta_enabled: @@ -109,8 +109,8 @@ class PeriodicLog: data.update(self.log_crawler_stats()) logger.info(self.encoder.encode(data)) - def log_delta(self) -> Dict[str, Any]: - num_stats: Dict[str, Union[int, float]] = { + def log_delta(self) -> dict[str, Any]: + num_stats: dict[str, Union[int, float]] = { k: v for k, v in self.stats._stats.items() if isinstance(v, (int, float)) @@ -120,7 +120,7 @@ class PeriodicLog: self.delta_prev = num_stats return {"delta": delta} - def log_timing(self) -> Dict[str, Any]: + def log_timing(self) -> dict[str, Any]: now = datetime.now(tz=timezone.utc) time = { "log_interval": self.interval, @@ -132,7 +132,7 @@ class PeriodicLog: self.time_prev = now return {"time": time} - def log_crawler_stats(self) -> Dict[str, Any]: + def log_crawler_stats(self) -> dict[str, Any]: stats = { k: v for k, v in self.stats._stats.items() @@ -141,7 +141,7 @@ class PeriodicLog: return {"stats": stats} def param_allowed( - self, stat_name: str, include: List[str], exclude: List[str] + self, stat_name: str, include: list[str], exclude: list[str] ) -> bool: if not include and not exclude: return True diff --git a/scrapy/extensions/postprocessing.py b/scrapy/extensions/postprocessing.py index ac12ad829..16067f82b 100644 --- a/scrapy/extensions/postprocessing.py +++ b/scrapy/extensions/postprocessing.py @@ -6,7 +6,7 @@ from bz2 import BZ2File from gzip import GzipFile from io import IOBase from lzma import LZMAFile -from typing import IO, Any, BinaryIO, Dict, List, cast +from typing import IO, Any, BinaryIO, cast from scrapy.utils.misc import load_object @@ -24,7 +24,7 @@ class GzipPlugin: See :py:class:`gzip.GzipFile` for more info about parameters. """ - def __init__(self, file: BinaryIO, feed_options: Dict[str, Any]) -> None: + def __init__(self, file: BinaryIO, feed_options: dict[str, Any]) -> None: self.file = file self.feed_options = feed_options compress_level = self.feed_options.get("gzip_compresslevel", 9) @@ -56,7 +56,7 @@ class Bz2Plugin: See :py:class:`bz2.BZ2File` for more info about parameters. """ - def __init__(self, file: BinaryIO, feed_options: Dict[str, Any]) -> None: + def __init__(self, file: BinaryIO, feed_options: dict[str, Any]) -> None: self.file = file self.feed_options = feed_options compress_level = self.feed_options.get("bz2_compresslevel", 9) @@ -88,7 +88,7 @@ class LZMAPlugin: See :py:class:`lzma.LZMAFile` for more info about parameters. """ - def __init__(self, file: BinaryIO, feed_options: Dict[str, Any]) -> None: + def __init__(self, file: BinaryIO, feed_options: dict[str, Any]) -> None: self.file = file self.feed_options = feed_options @@ -126,7 +126,7 @@ class PostProcessingManager(IOBase): """ def __init__( - self, plugins: List[Any], file: IO[bytes], feed_options: Dict[str, Any] + self, plugins: list[Any], file: IO[bytes], feed_options: dict[str, Any] ) -> None: self.plugins = self._load_plugins(plugins) self.file = file @@ -156,7 +156,7 @@ class PostProcessingManager(IOBase): def writable(self) -> bool: return True - def _load_plugins(self, plugins: List[Any]) -> List[Any]: + def _load_plugins(self, plugins: list[Any]) -> list[Any]: plugins = [load_object(plugin) for plugin in plugins] return plugins diff --git a/scrapy/extensions/statsmailer.py b/scrapy/extensions/statsmailer.py index cad607514..c8fefe792 100644 --- a/scrapy/extensions/statsmailer.py +++ b/scrapy/extensions/statsmailer.py @@ -6,7 +6,7 @@ Use STATSMAILER_RCPTS setting to enable and give the recipient mail address from __future__ import annotations -from typing import TYPE_CHECKING, List, Optional +from typing import TYPE_CHECKING, Optional from scrapy import Spider, signals from scrapy.exceptions import NotConfigured @@ -23,14 +23,14 @@ if TYPE_CHECKING: class StatsMailer: - def __init__(self, stats: StatsCollector, recipients: List[str], mail: MailSender): + def __init__(self, stats: StatsCollector, recipients: list[str], mail: MailSender): self.stats: StatsCollector = stats - self.recipients: List[str] = recipients + self.recipients: list[str] = recipients self.mail: MailSender = mail @classmethod def from_crawler(cls, crawler: Crawler) -> Self: - recipients: List[str] = crawler.settings.getlist("STATSMAILER_RCPTS") + recipients: list[str] = crawler.settings.getlist("STATSMAILER_RCPTS") if not recipients: raise NotConfigured mail: MailSender = MailSender.from_settings(crawler.settings) diff --git a/scrapy/extensions/telnet.py b/scrapy/extensions/telnet.py index c64a0b417..07dc5880b 100644 --- a/scrapy/extensions/telnet.py +++ b/scrapy/extensions/telnet.py @@ -10,7 +10,7 @@ import binascii import logging import os import pprint -from typing import TYPE_CHECKING, Any, Dict, List +from typing import TYPE_CHECKING, Any from twisted.internet import protocol from twisted.internet.tcp import Port @@ -45,7 +45,7 @@ class TelnetConsole(protocol.ServerFactory): self.crawler: Crawler = crawler self.noisy: bool = False - self.portrange: List[int] = [ + self.portrange: list[int] = [ int(x) for x in crawler.settings.getlist("TELNETCONSOLE_PORT") ] self.host: str = crawler.settings["TELNETCONSOLE_HOST"] @@ -98,10 +98,10 @@ class TelnetConsole(protocol.ServerFactory): return telnet.TelnetTransport(telnet.AuthenticatingTelnetProtocol, Portal()) - def _get_telnet_vars(self) -> Dict[str, Any]: + def _get_telnet_vars(self) -> dict[str, Any]: # Note: if you add entries here also update topics/telnetconsole.rst assert self.crawler.engine - telnet_vars: Dict[str, Any] = { + telnet_vars: dict[str, Any] = { "engine": self.crawler.engine, "spider": self.crawler.engine.spider, "slot": self.crawler.engine.slot, diff --git a/scrapy/extensions/throttle.py b/scrapy/extensions/throttle.py index 6ce9ce63a..6b5fd181d 100644 --- a/scrapy/extensions/throttle.py +++ b/scrapy/extensions/throttle.py @@ -1,7 +1,7 @@ from __future__ import annotations import logging -from typing import TYPE_CHECKING, Optional, Tuple +from typing import TYPE_CHECKING, Optional from scrapy import Request, Spider, signals from scrapy.exceptions import NotConfigured @@ -90,7 +90,7 @@ class AutoThrottle: def _get_slot( self, request: Request, spider: Spider - ) -> Tuple[Optional[str], Optional[Slot]]: + ) -> tuple[Optional[str], Optional[Slot]]: key: Optional[str] = request.meta.get("download_slot") if key is None: return None, None diff --git a/scrapy/http/cookies.py b/scrapy/http/cookies.py index cc88a9420..b5388a918 100644 --- a/scrapy/http/cookies.py +++ b/scrapy/http/cookies.py @@ -5,22 +5,14 @@ import time from http.cookiejar import Cookie from http.cookiejar import CookieJar as _CookieJar from http.cookiejar import CookiePolicy, DefaultCookiePolicy -from typing import ( - TYPE_CHECKING, - Any, - Dict, - Iterator, - List, - Optional, - Sequence, - Tuple, - cast, -) +from typing import TYPE_CHECKING, Any, Optional, cast from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.python import to_unicode if TYPE_CHECKING: + from collections.abc import Iterator, Sequence + # typing.Self requires Python 3.11 from typing_extensions import Self @@ -83,7 +75,7 @@ class CookieJar: self.jar.clear_expired_cookies() @property - def _cookies(self) -> Dict[str, Dict[str, Dict[str, Cookie]]]: + def _cookies(self) -> dict[str, dict[str, dict[str, Cookie]]]: return self.jar._cookies # type: ignore[attr-defined,no-any-return] def clear_session_cookies(self) -> None: @@ -118,7 +110,7 @@ class CookieJar: self.jar.set_cookie_if_ok(cookie, WrappedRequest(request)) # type: ignore[arg-type] -def potential_domain_matches(domain: str) -> List[str]: +def potential_domain_matches(domain: str) -> list[str]: """Potential domain matches for a cookie >>> potential_domain_matches('www.example.com') @@ -200,7 +192,7 @@ class WrappedRequest: value = self.request.headers.get(name, default) return to_unicode(value, errors="replace") if value is not None else None - def header_items(self) -> List[Tuple[str, List[str]]]: + def header_items(self) -> list[tuple[str, list[str]]]: return [ ( to_unicode(k, errors="replace"), @@ -220,7 +212,7 @@ class WrappedResponse: def info(self) -> Self: return self - def get_all(self, name: str, default: Any = None) -> List[str]: + def get_all(self, name: str, default: Any = None) -> list[str]: return [ to_unicode(v, errors="replace") for v in self.response.headers.getlist(name) ] diff --git a/scrapy/http/headers.py b/scrapy/http/headers.py index 85b9229d3..1dcbcb966 100644 --- a/scrapy/http/headers.py +++ b/scrapy/http/headers.py @@ -1,18 +1,7 @@ from __future__ import annotations from collections.abc import Mapping -from typing import ( - TYPE_CHECKING, - Any, - AnyStr, - Dict, - Iterable, - List, - Optional, - Tuple, - Union, - cast, -) +from typing import TYPE_CHECKING, Any, AnyStr, Optional, Union, cast from w3lib.http import headers_dict_to_raw @@ -20,6 +9,8 @@ from scrapy.utils.datatypes import CaseInsensitiveDict, CaselessDict from scrapy.utils.python import to_unicode if TYPE_CHECKING: + from collections.abc import Iterable + # typing.Self requires Python 3.11 from typing_extensions import Self @@ -34,17 +25,17 @@ class Headers(CaselessDict): def __init__( self, - seq: Union[Mapping[AnyStr, Any], Iterable[Tuple[AnyStr, Any]], None] = None, + seq: Union[Mapping[AnyStr, Any], Iterable[tuple[AnyStr, Any]], None] = None, encoding: str = "utf-8", ): self.encoding: str = encoding super().__init__(seq) def update( # type: ignore[override] - self, seq: Union[Mapping[AnyStr, Any], Iterable[Tuple[AnyStr, Any]]] + self, seq: Union[Mapping[AnyStr, Any], Iterable[tuple[AnyStr, Any]]] ) -> None: seq = seq.items() if isinstance(seq, Mapping) else seq - iseq: Dict[bytes, List[bytes]] = {} + iseq: dict[bytes, list[bytes]] = {} for k, v in seq: iseq.setdefault(self.normkey(k), []).extend(self.normvalue(v)) super().update(iseq) @@ -53,7 +44,7 @@ class Headers(CaselessDict): """Normalize key to bytes""" return self._tobytes(key.title()) - def normvalue(self, value: Union[_RawValueT, Iterable[_RawValueT]]) -> List[bytes]: + def normvalue(self, value: Union[_RawValueT, Iterable[_RawValueT]]) -> list[bytes]: """Normalize values to bytes""" _value: Iterable[_RawValueT] if value is None: @@ -78,19 +69,19 @@ class Headers(CaselessDict): def __getitem__(self, key: AnyStr) -> Optional[bytes]: try: - return cast(List[bytes], super().__getitem__(key))[-1] + return cast(list[bytes], super().__getitem__(key))[-1] except IndexError: return None def get(self, key: AnyStr, def_val: Any = None) -> Optional[bytes]: try: - return cast(List[bytes], super().get(key, def_val))[-1] + return cast(list[bytes], super().get(key, def_val))[-1] except IndexError: return None - def getlist(self, key: AnyStr, def_val: Any = None) -> List[bytes]: + def getlist(self, key: AnyStr, def_val: Any = None) -> list[bytes]: try: - return cast(List[bytes], super().__getitem__(key)) + return cast(list[bytes], super().__getitem__(key)) except KeyError: if def_val is not None: return self.normvalue(def_val) @@ -109,10 +100,10 @@ class Headers(CaselessDict): lst.extend(self.normvalue(value)) self[key] = lst - def items(self) -> Iterable[Tuple[bytes, List[bytes]]]: # type: ignore[override] + def items(self) -> Iterable[tuple[bytes, list[bytes]]]: # type: ignore[override] return ((k, self.getlist(k)) for k in self.keys()) - def values(self) -> List[Optional[bytes]]: # type: ignore[override] + def values(self) -> list[Optional[bytes]]: # type: ignore[override] return [ self[k] for k in self.keys() # pylint: disable=consider-using-dict-items ] diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py index 9381a6cb3..aac8d3e50 100644 --- a/scrapy/http/request/__init__.py +++ b/scrapy/http/request/__init__.py @@ -12,14 +12,8 @@ from typing import ( TYPE_CHECKING, Any, AnyStr, - Dict, - Iterable, - List, - Mapping, NoReturn, Optional, - Tuple, - Type, TypedDict, TypeVar, Union, @@ -36,7 +30,7 @@ from scrapy.utils.trackref import object_ref from scrapy.utils.url import escape_ajax if TYPE_CHECKING: - from collections.abc import Callable + from collections.abc import Callable, Iterable, Mapping from twisted.python.failure import Failure @@ -57,7 +51,7 @@ class VerboseCookie(TypedDict): secure: NotRequired[bool] -CookiesT = Union[Dict[str, str], List[VerboseCookie]] +CookiesT = Union[dict[str, str], list[VerboseCookie]] RequestTypeVar = TypeVar("RequestTypeVar", bound="Request") @@ -92,7 +86,7 @@ class Request(object_ref): executed by the Downloader, thus generating a :class:`Response`. """ - attributes: Tuple[str, ...] = ( + attributes: tuple[str, ...] = ( "url", "callback", "method", @@ -120,16 +114,16 @@ class Request(object_ref): url: str, callback: Optional[CallbackT] = None, method: str = "GET", - headers: Union[Mapping[AnyStr, Any], Iterable[Tuple[AnyStr, Any]], None] = None, + headers: Union[Mapping[AnyStr, Any], Iterable[tuple[AnyStr, Any]], None] = None, body: Optional[Union[bytes, str]] = None, cookies: Optional[CookiesT] = None, - meta: Optional[Dict[str, Any]] = None, + meta: Optional[dict[str, Any]] = None, encoding: str = "utf-8", priority: int = 0, dont_filter: bool = False, errback: Optional[Callable[[Failure], Any]] = None, - flags: Optional[List[str]] = None, - cb_kwargs: Optional[Dict[str, Any]] = None, + flags: Optional[list[str]] = None, + cb_kwargs: Optional[dict[str, Any]] = None, ) -> None: self._encoding: str = encoding # this one has to be set first self.method: str = str(method).upper() @@ -152,20 +146,20 @@ class Request(object_ref): self.headers: Headers = Headers(headers or {}, encoding=encoding) self.dont_filter: bool = dont_filter - self._meta: Optional[Dict[str, Any]] = dict(meta) if meta else None - self._cb_kwargs: Optional[Dict[str, Any]] = ( + self._meta: Optional[dict[str, Any]] = dict(meta) if meta else None + self._cb_kwargs: Optional[dict[str, Any]] = ( dict(cb_kwargs) if cb_kwargs else None ) - self.flags: List[str] = [] if flags is None else list(flags) + self.flags: list[str] = [] if flags is None else list(flags) @property - def cb_kwargs(self) -> Dict[str, Any]: + def cb_kwargs(self) -> dict[str, Any]: if self._cb_kwargs is None: self._cb_kwargs = {} return self._cb_kwargs @property - def meta(self) -> Dict[str, Any]: + def meta(self) -> dict[str, Any]: if self._meta is None: self._meta = {} return self._meta @@ -207,14 +201,14 @@ class Request(object_ref): @overload def replace( - self, *args: Any, cls: Type[RequestTypeVar], **kwargs: Any + self, *args: Any, cls: type[RequestTypeVar], **kwargs: Any ) -> RequestTypeVar: ... @overload def replace(self, *args: Any, cls: None = None, **kwargs: Any) -> Self: ... def replace( - self, *args: Any, cls: Optional[Type[Request]] = None, **kwargs: Any + self, *args: Any, cls: Optional[type[Request]] = None, **kwargs: Any ) -> Request: """Create a new Request with the same attributes except for those given new values""" for x in self.attributes: @@ -261,7 +255,7 @@ class Request(object_ref): request_kwargs.update(kwargs) return cls(**request_kwargs) - def to_dict(self, *, spider: Optional[scrapy.Spider] = None) -> Dict[str, Any]: + def to_dict(self, *, spider: Optional[scrapy.Spider] = None) -> dict[str, Any]: """Return a dictionary containing the Request's data. Use :func:`~scrapy.utils.request.request_from_dict` to convert back into a :class:`~scrapy.Request` object. diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index a8c242e8b..d9c913672 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -7,17 +7,8 @@ See documentation in docs/topics/request-response.rst from __future__ import annotations -from typing import ( - TYPE_CHECKING, - Any, - Dict, - Iterable, - List, - Optional, - Tuple, - Union, - cast, -) +from collections.abc import Iterable +from typing import TYPE_CHECKING, Any, Optional, Union, cast from urllib.parse import urlencode, urljoin, urlsplit, urlunsplit from lxml.html import FormElement # nosec @@ -31,6 +22,7 @@ from scrapy.http.request import Request from scrapy.utils.python import is_listlike, to_bytes if TYPE_CHECKING: + # typing.Self requires Python 3.11 from typing_extensions import Self @@ -38,8 +30,8 @@ if TYPE_CHECKING: FormdataVType = Union[str, Iterable[str]] -FormdataKVType = Tuple[str, FormdataVType] -FormdataType = Optional[Union[Dict[str, FormdataVType], List[FormdataKVType]]] +FormdataKVType = tuple[str, FormdataVType] +FormdataType = Optional[Union[dict[str, FormdataVType], list[FormdataKVType]]] class FormRequest(Request): @@ -74,7 +66,7 @@ class FormRequest(Request): formid: Optional[str] = None, formnumber: int = 0, formdata: FormdataType = None, - clickdata: Optional[Dict[str, Union[str, int]]] = None, + clickdata: Optional[dict[str, Union[str, int]]] = None, dont_click: bool = False, formxpath: Optional[str] = None, formcss: Optional[str] = None, @@ -168,8 +160,8 @@ def _get_inputs( form: FormElement, formdata: FormdataType, dont_click: bool, - clickdata: Optional[Dict[str, Union[str, int]]], -) -> List[FormdataKVType]: + clickdata: Optional[dict[str, Union[str, int]]], +) -> list[FormdataKVType]: """Return a list of key-value pairs for the inputs found in the given form.""" try: formdata_keys = dict(formdata or ()).keys() @@ -187,7 +179,7 @@ def _get_inputs( ' not(re:test(., "^(?:checkbox|radio)$", "i")))]]', namespaces={"re": "http://exslt.org/regular-expressions"}, ) - values: List[FormdataKVType] = [ + values: list[FormdataKVType] = [ (k, "" if v is None else v) for k, v in (_value(e) for e in inputs) if k and k not in formdata_keys @@ -205,7 +197,7 @@ def _get_inputs( def _value( ele: Union[InputElement, SelectElement, TextareaElement] -) -> Tuple[Optional[str], Union[None, str, MultipleSelectOptions]]: +) -> tuple[Optional[str], Union[None, str, MultipleSelectOptions]]: n = ele.name v = ele.value if ele.tag == "select": @@ -215,7 +207,7 @@ def _value( def _select_value( ele: SelectElement, n: Optional[str], v: Union[None, str, MultipleSelectOptions] -) -> Tuple[Optional[str], Union[None, str, MultipleSelectOptions]]: +) -> tuple[Optional[str], Union[None, str, MultipleSelectOptions]]: multiple = ele.multiple if v is None and not multiple: # Match browser behaviour on simple select tag without options selected @@ -226,8 +218,8 @@ def _select_value( def _get_clickable( - clickdata: Optional[Dict[str, Union[str, int]]], form: FormElement -) -> Optional[Tuple[str, str]]: + clickdata: Optional[dict[str, Union[str, int]]], form: FormElement +) -> Optional[tuple[str, str]]: """ Returns the clickable element specified in clickdata, if the latter is given. If not, it returns the first diff --git a/scrapy/http/request/json_request.py b/scrapy/http/request/json_request.py index 057a4f897..48862534e 100644 --- a/scrapy/http/request/json_request.py +++ b/scrapy/http/request/json_request.py @@ -10,7 +10,7 @@ from __future__ import annotations import copy import json import warnings -from typing import TYPE_CHECKING, Any, Dict, Optional, Tuple, Type, overload +from typing import TYPE_CHECKING, Any, Optional, overload from scrapy.http.request import Request, RequestTypeVar @@ -20,14 +20,14 @@ if TYPE_CHECKING: class JsonRequest(Request): - attributes: Tuple[str, ...] = Request.attributes + ("dumps_kwargs",) + attributes: tuple[str, ...] = Request.attributes + ("dumps_kwargs",) def __init__( - self, *args: Any, dumps_kwargs: Optional[Dict[str, Any]] = None, **kwargs: Any + self, *args: Any, dumps_kwargs: Optional[dict[str, Any]] = None, **kwargs: Any ) -> None: dumps_kwargs = copy.deepcopy(dumps_kwargs) if dumps_kwargs is not None else {} dumps_kwargs.setdefault("sort_keys", True) - self._dumps_kwargs: Dict[str, Any] = dumps_kwargs + self._dumps_kwargs: dict[str, Any] = dumps_kwargs body_passed = kwargs.get("body", None) is not None data: Any = kwargs.pop("data", None) @@ -47,19 +47,19 @@ class JsonRequest(Request): ) @property - def dumps_kwargs(self) -> Dict[str, Any]: + def dumps_kwargs(self) -> dict[str, Any]: return self._dumps_kwargs @overload def replace( - self, *args: Any, cls: Type[RequestTypeVar], **kwargs: Any + self, *args: Any, cls: type[RequestTypeVar], **kwargs: Any ) -> RequestTypeVar: ... @overload def replace(self, *args: Any, cls: None = None, **kwargs: Any) -> Self: ... def replace( - self, *args: Any, cls: Optional[Type[Request]] = None, **kwargs: Any + self, *args: Any, cls: Optional[type[Request]] = None, **kwargs: Any ) -> Request: body_passed = kwargs.get("body", None) is not None data: Any = kwargs.pop("data", None) diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py index 92e4852b6..c69945e2d 100644 --- a/scrapy/http/response/__init__.py +++ b/scrapy/http/response/__init__.py @@ -7,22 +7,7 @@ See documentation in docs/topics/request-response.rst from __future__ import annotations -from typing import ( - TYPE_CHECKING, - Any, - AnyStr, - Callable, - Dict, - Iterable, - List, - Mapping, - Optional, - Tuple, - Type, - TypeVar, - Union, - overload, -) +from typing import TYPE_CHECKING, Any, AnyStr, Optional, TypeVar, Union, overload from urllib.parse import urljoin from scrapy.exceptions import NotSupported @@ -32,6 +17,7 @@ from scrapy.link import Link from scrapy.utils.trackref import object_ref if TYPE_CHECKING: + from collections.abc import Callable, Iterable, Mapping from ipaddress import IPv4Address, IPv6Address from twisted.internet.ssl import Certificate @@ -52,7 +38,7 @@ class Response(object_ref): downloaded (by the Downloader) and fed to the Spiders for processing. """ - attributes: Tuple[str, ...] = ( + attributes: tuple[str, ...] = ( "url", "status", "headers", @@ -74,9 +60,9 @@ class Response(object_ref): self, url: str, status: int = 200, - headers: Union[Mapping[AnyStr, Any], Iterable[Tuple[AnyStr, Any]], None] = None, + headers: Union[Mapping[AnyStr, Any], Iterable[tuple[AnyStr, Any]], None] = None, body: bytes = b"", - flags: Optional[List[str]] = None, + flags: Optional[list[str]] = None, request: Optional[Request] = None, certificate: Optional[Certificate] = None, ip_address: Union[IPv4Address, IPv6Address, None] = None, @@ -87,13 +73,13 @@ class Response(object_ref): self._set_body(body) self._set_url(url) self.request: Optional[Request] = request - self.flags: List[str] = [] if flags is None else list(flags) + self.flags: list[str] = [] if flags is None else list(flags) self.certificate: Optional[Certificate] = certificate self.ip_address: Union[IPv4Address, IPv6Address, None] = ip_address self.protocol: Optional[str] = protocol @property - def cb_kwargs(self) -> Dict[str, Any]: + def cb_kwargs(self) -> dict[str, Any]: try: return self.request.cb_kwargs # type: ignore[union-attr] except AttributeError: @@ -103,7 +89,7 @@ class Response(object_ref): ) @property - def meta(self) -> Dict[str, Any]: + def meta(self) -> dict[str, Any]: try: return self.request.meta # type: ignore[union-attr] except AttributeError: @@ -149,14 +135,14 @@ class Response(object_ref): @overload def replace( - self, *args: Any, cls: Type[ResponseTypeVar], **kwargs: Any + self, *args: Any, cls: type[ResponseTypeVar], **kwargs: Any ) -> ResponseTypeVar: ... @overload def replace(self, *args: Any, cls: None = None, **kwargs: Any) -> Self: ... def replace( - self, *args: Any, cls: Optional[Type[Response]] = None, **kwargs: Any + self, *args: Any, cls: Optional[type[Response]] = None, **kwargs: Any ) -> Response: """Create a new Response with the same attributes except for those given new values""" for x in self.attributes: @@ -200,16 +186,16 @@ class Response(object_ref): url: Union[str, Link], callback: Optional[CallbackT] = None, method: str = "GET", - headers: Union[Mapping[AnyStr, Any], Iterable[Tuple[AnyStr, Any]], None] = None, + headers: Union[Mapping[AnyStr, Any], Iterable[tuple[AnyStr, Any]], None] = None, body: Optional[Union[bytes, str]] = None, cookies: Optional[CookiesT] = None, - meta: Optional[Dict[str, Any]] = None, + meta: Optional[dict[str, Any]] = None, encoding: Optional[str] = "utf-8", priority: int = 0, dont_filter: bool = False, errback: Optional[Callable[[Failure], Any]] = None, - cb_kwargs: Optional[Dict[str, Any]] = None, - flags: Optional[List[str]] = None, + cb_kwargs: Optional[dict[str, Any]] = None, + flags: Optional[list[str]] = None, ) -> Request: """ Return a :class:`~.Request` instance to follow a link ``url``. @@ -253,16 +239,16 @@ class Response(object_ref): urls: Iterable[Union[str, Link]], callback: Optional[CallbackT] = None, method: str = "GET", - headers: Union[Mapping[AnyStr, Any], Iterable[Tuple[AnyStr, Any]], None] = None, + headers: Union[Mapping[AnyStr, Any], Iterable[tuple[AnyStr, Any]], None] = None, body: Optional[Union[bytes, str]] = None, cookies: Optional[CookiesT] = None, - meta: Optional[Dict[str, Any]] = None, + meta: Optional[dict[str, Any]] = None, encoding: Optional[str] = "utf-8", priority: int = 0, dont_filter: bool = False, errback: Optional[Callable[[Failure], Any]] = None, - cb_kwargs: Optional[Dict[str, Any]] = None, - flags: Optional[List[str]] = None, + cb_kwargs: Optional[dict[str, Any]] = None, + flags: Optional[list[str]] = None, ) -> Iterable[Request]: """ .. versionadded:: 2.0 diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py index 588695002..680c1f602 100644 --- a/scrapy/http/response/text.py +++ b/scrapy/http/response/text.py @@ -8,21 +8,9 @@ See documentation in docs/topics/request-response.rst from __future__ import annotations import json +from collections.abc import Iterable from contextlib import suppress -from typing import ( - TYPE_CHECKING, - Any, - AnyStr, - Callable, - Dict, - Iterable, - List, - Mapping, - Optional, - Tuple, - Union, - cast, -) +from typing import TYPE_CHECKING, Any, AnyStr, Optional, Union, cast from urllib.parse import urljoin import parsel @@ -41,6 +29,8 @@ from scrapy.utils.python import memoizemethod_noargs, to_unicode from scrapy.utils.response import get_base_url if TYPE_CHECKING: + from collections.abc import Callable, Mapping + from twisted.python.failure import Failure from scrapy.http.request import CallbackT, CookiesT, Request @@ -54,7 +44,7 @@ class TextResponse(Response): _DEFAULT_ENCODING = "ascii" _cached_decoded_json = _NONE - attributes: Tuple[str, ...] = Response.attributes + ("encoding",) + attributes: tuple[str, ...] = Response.attributes + ("encoding",) def __init__(self, *args: Any, **kwargs: Any): self._encoding: Optional[str] = kwargs.pop("encoding", None) @@ -183,16 +173,16 @@ class TextResponse(Response): url: Union[str, Link, parsel.Selector], callback: Optional[CallbackT] = None, method: str = "GET", - headers: Union[Mapping[AnyStr, Any], Iterable[Tuple[AnyStr, Any]], None] = None, + headers: Union[Mapping[AnyStr, Any], Iterable[tuple[AnyStr, Any]], None] = None, body: Optional[Union[bytes, str]] = None, cookies: Optional[CookiesT] = None, - meta: Optional[Dict[str, Any]] = None, + meta: Optional[dict[str, Any]] = None, encoding: Optional[str] = None, priority: int = 0, dont_filter: bool = False, errback: Optional[Callable[[Failure], Any]] = None, - cb_kwargs: Optional[Dict[str, Any]] = None, - flags: Optional[List[str]] = None, + cb_kwargs: Optional[dict[str, Any]] = None, + flags: Optional[list[str]] = None, ) -> Request: """ Return a :class:`~.Request` instance to follow a link ``url``. @@ -236,16 +226,16 @@ class TextResponse(Response): urls: Union[Iterable[Union[str, Link]], parsel.SelectorList, None] = None, callback: Optional[CallbackT] = None, method: str = "GET", - headers: Union[Mapping[AnyStr, Any], Iterable[Tuple[AnyStr, Any]], None] = None, + headers: Union[Mapping[AnyStr, Any], Iterable[tuple[AnyStr, Any]], None] = None, body: Optional[Union[bytes, str]] = None, cookies: Optional[CookiesT] = None, - meta: Optional[Dict[str, Any]] = None, + meta: Optional[dict[str, Any]] = None, encoding: Optional[str] = None, priority: int = 0, dont_filter: bool = False, errback: Optional[Callable[[Failure], Any]] = None, - cb_kwargs: Optional[Dict[str, Any]] = None, - flags: Optional[List[str]] = None, + cb_kwargs: Optional[dict[str, Any]] = None, + flags: Optional[list[str]] = None, css: Optional[str] = None, xpath: Optional[str] = None, ) -> Iterable[Request]: diff --git a/scrapy/item.py b/scrapy/item.py index 3f93809e7..f77002d18 100644 --- a/scrapy/item.py +++ b/scrapy/item.py @@ -7,27 +7,21 @@ See documentation in docs/topics/item.rst from __future__ import annotations from abc import ABCMeta +from collections.abc import MutableMapping from copy import deepcopy from pprint import pformat -from typing import ( - TYPE_CHECKING, - Any, - Dict, - Iterator, - KeysView, - MutableMapping, - NoReturn, - Tuple, -) +from typing import TYPE_CHECKING, Any, NoReturn from scrapy.utils.trackref import object_ref if TYPE_CHECKING: + from collections.abc import Iterator, KeysView + # typing.Self requires Python 3.11 from typing_extensions import Self -class Field(Dict[str, Any]): +class Field(dict[str, Any]): """Container of field metadata""" @@ -38,7 +32,7 @@ class ItemMeta(ABCMeta): """ def __new__( - mcs, class_name: str, bases: Tuple[type, ...], attrs: Dict[str, Any] + mcs, class_name: str, bases: tuple[type, ...], attrs: dict[str, Any] ) -> ItemMeta: classcell = attrs.pop("__classcell__", None) new_bases = tuple(base._class for base in bases if hasattr(base, "_class")) @@ -83,10 +77,10 @@ class Item(MutableMapping[str, Any], object_ref, metaclass=ItemMeta): :ref:`tracked ` to debug memory leaks. """ - fields: Dict[str, Field] + fields: dict[str, Field] def __init__(self, *args: Any, **kwargs: Any): - self._values: Dict[str, Any] = {} + self._values: dict[str, Any] = {} if args or kwargs: # avoid creating dict for most common case for k, v in dict(*args, **kwargs).items(): self[k] = v diff --git a/scrapy/linkextractors/__init__.py b/scrapy/linkextractors/__init__.py index d59005edd..1c7e96ae0 100644 --- a/scrapy/linkextractors/__init__.py +++ b/scrapy/linkextractors/__init__.py @@ -6,8 +6,13 @@ This package contains a collection of Link Extractors. For more info see docs/topics/link-extractors.rst """ -import re -from typing import Iterable, Pattern +from __future__ import annotations + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from collections.abc import Iterable + from re import Pattern # common file extensions that are not followed if they occur in links IGNORED_EXTENSIONS = [ diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py index d27a132b3..73673b1c6 100644 --- a/scrapy/linkextractors/lxmlhtml.py +++ b/scrapy/linkextractors/lxmlhtml.py @@ -6,20 +6,10 @@ from __future__ import annotations import logging import operator +import re +from collections.abc import Callable, Iterable from functools import partial -from typing import ( - TYPE_CHECKING, - Any, - Callable, - Iterable, - List, - Optional, - Pattern, - Set, - Tuple, - Union, - cast, -) +from typing import TYPE_CHECKING, Any, Optional, Union, cast from urllib.parse import urljoin, urlparse from lxml import etree # nosec @@ -28,13 +18,14 @@ from w3lib.html import strip_html5_whitespace from w3lib.url import canonicalize_url, safe_url_string from scrapy.link import Link -from scrapy.linkextractors import IGNORED_EXTENSIONS, _is_valid_url, _matches, re +from scrapy.linkextractors import IGNORED_EXTENSIONS, _is_valid_url, _matches from scrapy.utils.misc import arg_to_iter, rel_has_nofollow from scrapy.utils.python import unique as unique_list from scrapy.utils.response import get_base_url from scrapy.utils.url import url_has_any_extension, url_is_from_any_domain if TYPE_CHECKING: + from lxml.html import HtmlElement # nosec from scrapy import Selector @@ -98,7 +89,7 @@ class LxmlParserLinkExtractor: def _iter_links( self, document: HtmlElement - ) -> Iterable[Tuple[HtmlElement, str, str]]: + ) -> Iterable[tuple[HtmlElement, str, str]]: for el in document.iter(etree.Element): if not self.scan_tag(_nons(el.tag)): continue @@ -114,8 +105,8 @@ class LxmlParserLinkExtractor: response_url: str, response_encoding: str, base_url: str, - ) -> List[Link]: - links: List[Link] = [] + ) -> list[Link]: + links: list[Link] = [] # hacky way to get the underlying lxml parsed document for el, attr, attr_val in self._iter_links(selector.root): # pseudo lxml.html.HtmlElement.make_links_absolute(base_url) @@ -145,26 +136,26 @@ class LxmlParserLinkExtractor: links.append(link) return self._deduplicate_if_needed(links) - def extract_links(self, response: TextResponse) -> List[Link]: + def extract_links(self, response: TextResponse) -> list[Link]: base_url = get_base_url(response) return self._extract_links( response.selector, response.url, response.encoding, base_url ) - def _process_links(self, links: List[Link]) -> List[Link]: + def _process_links(self, links: list[Link]) -> list[Link]: """Normalize and filter extracted links The subclass should override it if necessary """ return self._deduplicate_if_needed(links) - def _deduplicate_if_needed(self, links: List[Link]) -> List[Link]: + def _deduplicate_if_needed(self, links: list[Link]) -> list[Link]: if self.unique: return unique_list(links, key=self.link_key) return links -_RegexT = Union[str, Pattern[str]] +_RegexT = Union[str, re.Pattern[str]] _RegexOrSeveralT = Union[_RegexT, Iterable[_RegexT]] @@ -197,13 +188,13 @@ class LxmlLinkExtractor: strip=strip, canonicalized=not canonicalize, ) - self.allow_res: List[Pattern[str]] = self._compile_regexes(allow) - self.deny_res: List[Pattern[str]] = self._compile_regexes(deny) + self.allow_res: list[re.Pattern[str]] = self._compile_regexes(allow) + self.deny_res: list[re.Pattern[str]] = self._compile_regexes(deny) - self.allow_domains: Set[str] = set(arg_to_iter(allow_domains)) - self.deny_domains: Set[str] = set(arg_to_iter(deny_domains)) + self.allow_domains: set[str] = set(arg_to_iter(allow_domains)) + self.deny_domains: set[str] = set(arg_to_iter(deny_domains)) - self.restrict_xpaths: Tuple[str, ...] = tuple(arg_to_iter(restrict_xpaths)) + self.restrict_xpaths: tuple[str, ...] = tuple(arg_to_iter(restrict_xpaths)) self.restrict_xpaths += tuple( map(self._csstranslator.css_to_xpath, arg_to_iter(restrict_css)) ) @@ -211,11 +202,11 @@ class LxmlLinkExtractor: if deny_extensions is None: deny_extensions = IGNORED_EXTENSIONS self.canonicalize: bool = canonicalize - self.deny_extensions: Set[str] = {"." + e for e in arg_to_iter(deny_extensions)} - self.restrict_text: List[Pattern[str]] = self._compile_regexes(restrict_text) + self.deny_extensions: set[str] = {"." + e for e in arg_to_iter(deny_extensions)} + self.restrict_text: list[re.Pattern[str]] = self._compile_regexes(restrict_text) @staticmethod - def _compile_regexes(value: Optional[_RegexOrSeveralT]) -> List[Pattern[str]]: + def _compile_regexes(value: Optional[_RegexOrSeveralT]) -> list[re.Pattern[str]]: return [ x if isinstance(x, re.Pattern) else re.compile(x) for x in arg_to_iter(value) @@ -257,7 +248,7 @@ class LxmlLinkExtractor: denied = (regex.search(url) for regex in self.deny_res) if self.deny_res else [] return any(allowed) and not any(denied) - def _process_links(self, links: List[Link]) -> List[Link]: + def _process_links(self, links: list[Link]) -> list[Link]: links = [x for x in links if self._link_allowed(x)] if self.canonicalize: for link in links: @@ -265,10 +256,10 @@ class LxmlLinkExtractor: links = self.link_extractor._process_links(links) return links - def _extract_links(self, *args: Any, **kwargs: Any) -> List[Link]: + def _extract_links(self, *args: Any, **kwargs: Any) -> list[Link]: return self.link_extractor._extract_links(*args, **kwargs) - def extract_links(self, response: TextResponse) -> List[Link]: + def extract_links(self, response: TextResponse) -> list[Link]: """Returns a list of :class:`~scrapy.link.Link` objects from the specified :class:`response `. diff --git a/scrapy/logformatter.py b/scrapy/logformatter.py index 601209fb0..2b838d8e2 100644 --- a/scrapy/logformatter.py +++ b/scrapy/logformatter.py @@ -2,13 +2,14 @@ from __future__ import annotations import logging import os -from typing import TYPE_CHECKING, Any, Dict, Optional, Tuple, TypedDict, Union +from typing import TYPE_CHECKING, Any, Optional, TypedDict, Union from twisted.python.failure import Failure # working around https://github.com/sphinx-doc/sphinx/issues/10400 from scrapy import Request, Spider # noqa: TC001 from scrapy.http import Response # noqa: TC001 +from scrapy.utils.python import global_object_name from scrapy.utils.request import referer_str if TYPE_CHECKING: @@ -30,7 +31,7 @@ DOWNLOADERRORMSG_LONG = "Error downloading %(request)s: %(errmsg)s" class LogFormatterResult(TypedDict): level: int msg: str - args: Union[Dict[str, Any], Tuple[Any, ...]] + args: Union[dict[str, Any], tuple[Any, ...]] class LogFormatter: @@ -92,11 +93,13 @@ class LogFormatter: } def scraped( - self, item: Any, response: Union[Response, Failure], spider: Spider + self, item: Any, response: Union[Response, Failure, None], spider: Spider ) -> LogFormatterResult: """Logs a message when an item is scraped by a spider.""" src: Any - if isinstance(response, Failure): + if response is None: + src = f"{global_object_name(spider.__class__)}.start_requests" + elif isinstance(response, Failure): src = response.getErrorMessage() else: src = response @@ -110,7 +113,11 @@ class LogFormatter: } def dropped( - self, item: Any, exception: BaseException, response: Response, spider: Spider + self, + item: Any, + exception: BaseException, + response: Optional[Response], + spider: Spider, ) -> LogFormatterResult: """Logs a message when an item is dropped while it is passing through the item pipeline.""" return { @@ -123,7 +130,11 @@ class LogFormatter: } def item_error( - self, item: Any, exception: BaseException, response: Response, spider: Spider + self, + item: Any, + exception: BaseException, + response: Optional[Response], + spider: Spider, ) -> LogFormatterResult: """Logs a message when an item causes an error while it is passing through the item pipeline. @@ -170,7 +181,7 @@ class LogFormatter: .. versionadded:: 2.0 """ - args: Dict[str, Any] = {"request": request} + args: dict[str, Any] = {"request": request} if errmsg: msg = DOWNLOADERRORMSG_LONG args["errmsg"] = errmsg diff --git a/scrapy/mail.py b/scrapy/mail.py index c020732f9..f33cf2939 100644 --- a/scrapy/mail.py +++ b/scrapy/mail.py @@ -14,18 +14,7 @@ from email.mime.nonmultipart import MIMENonMultipart from email.mime.text import MIMEText from email.utils import formatdate from io import BytesIO -from typing import ( - IO, - TYPE_CHECKING, - Any, - Callable, - Dict, - List, - Optional, - Sequence, - Tuple, - Union, -) +from typing import IO, TYPE_CHECKING, Any, Optional, Union from twisted import version as twisted_version from twisted.internet import ssl @@ -36,6 +25,8 @@ from scrapy.utils.misc import arg_to_iter from scrapy.utils.python import to_bytes if TYPE_CHECKING: + from collections.abc import Callable, Sequence + # imports twisted.internet.reactor from twisted.mail.smtp import ESMTPSenderFactory from twisted.python.failure import Failure @@ -95,11 +86,11 @@ class MailSender: def send( self, - to: Union[str, List[str]], + to: Union[str, list[str]], subject: str, body: str, - cc: Union[str, List[str], None] = None, - attachs: Sequence[Tuple[str, str, IO[Any]]] = (), + cc: Union[str, list[str], None] = None, + attachs: Sequence[tuple[str, str, IO[Any]]] = (), mimetype: str = "text/plain", charset: Optional[str] = None, _callback: Optional[Callable[..., None]] = None, @@ -164,7 +155,7 @@ class MailSender: return dfd def _sent_ok( - self, result: Any, to: List[str], cc: List[str], subject: str, nattachs: int + self, result: Any, to: list[str], cc: list[str], subject: str, nattachs: int ) -> None: logger.info( "Mail sent OK: To=%(mailto)s Cc=%(mailcc)s " @@ -180,8 +171,8 @@ class MailSender: def _sent_failed( self, failure: Failure, - to: List[str], - cc: List[str], + to: list[str], + cc: list[str], subject: str, nattachs: int, ) -> Failure: @@ -200,7 +191,7 @@ class MailSender: ) return failure - def _sendmail(self, to_addrs: List[str], msg: bytes) -> Deferred[Any]: + def _sendmail(self, to_addrs: list[str], msg: bytes) -> Deferred[Any]: from twisted.internet import reactor msg_io = BytesIO(msg) @@ -218,11 +209,11 @@ class MailSender: return d def _create_sender_factory( - self, to_addrs: List[str], msg: IO[bytes], d: Deferred[Any] + self, to_addrs: list[str], msg: IO[bytes], d: Deferred[Any] ) -> ESMTPSenderFactory: from twisted.mail.smtp import ESMTPSenderFactory - factory_keywords: Dict[str, Any] = { + factory_keywords: dict[str, Any] = { "heloFallback": True, "requireAuthentication": False, "requireTransportSecurity": self.smtptls, diff --git a/scrapy/middleware.py b/scrapy/middleware.py index 2296db90e..825d6b4c8 100644 --- a/scrapy/middleware.py +++ b/scrapy/middleware.py @@ -3,26 +3,15 @@ from __future__ import annotations import logging import pprint from collections import defaultdict, deque -from typing import ( - TYPE_CHECKING, - Any, - Callable, - Deque, - Dict, - Iterable, - List, - Optional, - Tuple, - TypeVar, - Union, - cast, -) +from typing import TYPE_CHECKING, Any, Optional, TypeVar, Union, cast from scrapy.exceptions import NotConfigured from scrapy.utils.defer import process_chain, process_parallel from scrapy.utils.misc import build_from_crawler, build_from_settings, load_object if TYPE_CHECKING: + from collections.abc import Callable, Iterable + from twisted.internet.defer import Deferred # typing.Concatenate and typing.ParamSpec require Python 3.10 @@ -51,14 +40,14 @@ class MiddlewareManager: self.middlewares = middlewares # Only process_spider_output and process_spider_exception can be None. # Only process_spider_output can be a tuple, and only until _async compatibility methods are removed. - self.methods: Dict[ - str, Deque[Union[None, Callable, Tuple[Callable, Callable]]] + self.methods: dict[ + str, deque[Union[None, Callable, tuple[Callable, Callable]]] ] = defaultdict(deque) for mw in middlewares: self._add_middleware(mw) @classmethod - def _get_mwlist_from_settings(cls, settings: Settings) -> List[Any]: + def _get_mwlist_from_settings(cls, settings: Settings) -> list[Any]: raise NotImplementedError @classmethod @@ -107,7 +96,7 @@ class MiddlewareManager: def _process_parallel( self, methodname: str, obj: _T, *args: Any - ) -> Deferred[List[_T2]]: + ) -> Deferred[list[_T2]]: methods = cast( "Iterable[Callable[Concatenate[_T, _P], _T2]]", self.methods[methodname] ) @@ -119,8 +108,8 @@ class MiddlewareManager: ) return process_chain(methods, obj, *args) - def open_spider(self, spider: Spider) -> Deferred[List[None]]: + def open_spider(self, spider: Spider) -> Deferred[list[None]]: return self._process_parallel("open_spider", spider) - def close_spider(self, spider: Spider) -> Deferred[List[None]]: + def close_spider(self, spider: Spider) -> Deferred[list[None]]: return self._process_parallel("close_spider", spider) diff --git a/scrapy/pipelines/__init__.py b/scrapy/pipelines/__init__.py index 480a5a58c..01f8bd2c8 100644 --- a/scrapy/pipelines/__init__.py +++ b/scrapy/pipelines/__init__.py @@ -6,7 +6,7 @@ See documentation in docs/item-pipeline.rst from __future__ import annotations -from typing import TYPE_CHECKING, Any, List +from typing import TYPE_CHECKING, Any from scrapy.middleware import MiddlewareManager from scrapy.utils.conf import build_component_list @@ -23,7 +23,7 @@ class ItemPipelineManager(MiddlewareManager): component_name = "item pipeline" @classmethod - def _get_mwlist_from_settings(cls, settings: Settings) -> List[Any]: + def _get_mwlist_from_settings(cls, settings: Settings) -> list[Any]: return build_component_list(settings.getwithbase("ITEM_PIPELINES")) def _add_middleware(self, pipe: Any) -> None: diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index 1a13aeaf2..9314856c1 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -21,15 +21,9 @@ from typing import ( IO, TYPE_CHECKING, Any, - Callable, - DefaultDict, - Dict, - List, NoReturn, Optional, Protocol, - Set, - Type, TypedDict, Union, cast, @@ -53,6 +47,7 @@ from scrapy.utils.python import to_bytes from scrapy.utils.request import referer_str if TYPE_CHECKING: + from collections.abc import Callable from os import PathLike from twisted.python.failure import Failure @@ -104,8 +99,8 @@ class FilesStoreProtocol(Protocol): path: str, buf: BytesIO, info: MediaPipeline.SpiderInfo, - meta: Optional[Dict[str, Any]] = None, - headers: Optional[Dict[str, str]] = None, + meta: Optional[dict[str, Any]] = None, + headers: Optional[dict[str, str]] = None, ) -> Optional[Deferred[Any]]: ... def stat_file( @@ -120,7 +115,7 @@ class FSFilesStore: basedir = basedir.split("://", 1)[1] self.basedir: str = basedir self._mkdir(Path(self.basedir)) - self.created_directories: DefaultDict[MediaPipeline.SpiderInfo, Set[str]] = ( + self.created_directories: defaultdict[MediaPipeline.SpiderInfo, set[str]] = ( defaultdict(set) ) @@ -129,8 +124,8 @@ class FSFilesStore: path: Union[str, PathLike[str]], buf: BytesIO, info: MediaPipeline.SpiderInfo, - meta: Optional[Dict[str, Any]] = None, - headers: Optional[Dict[str, str]] = None, + meta: Optional[dict[str, Any]] = None, + headers: Optional[dict[str, str]] = None, ) -> None: absolute_path = self._get_filesystem_path(path) self._mkdir(absolute_path.parent, info) @@ -157,7 +152,7 @@ class FSFilesStore: def _mkdir( self, dirname: Path, domain: Optional[MediaPipeline.SpiderInfo] = None ) -> None: - seen: Set[str] = self.created_directories[domain] if domain else set() + seen: set[str] = self.created_directories[domain] if domain else set() if str(dirname) not in seen: if not dirname.exists(): dirname.mkdir(parents=True) @@ -201,7 +196,7 @@ class S3FilesStore: def stat_file( self, path: str, info: MediaPipeline.SpiderInfo ) -> Deferred[StatInfo]: - def _onsuccess(boto_key: Dict[str, Any]) -> StatInfo: + def _onsuccess(boto_key: dict[str, Any]) -> StatInfo: checksum = boto_key["ETag"].strip('"') last_modified = boto_key["LastModified"] modified_stamp = time.mktime(last_modified.timetuple()) @@ -209,10 +204,10 @@ class S3FilesStore: return self._get_boto_key(path).addCallback(_onsuccess) - def _get_boto_key(self, path: str) -> Deferred[Dict[str, Any]]: + def _get_boto_key(self, path: str) -> Deferred[dict[str, Any]]: key_name = f"{self.prefix}{path}" return cast( - "Deferred[Dict[str, Any]]", + "Deferred[dict[str, Any]]", deferToThread( self.s3_client.head_object, Bucket=self.bucket, Key=key_name # type: ignore[attr-defined] ), @@ -223,8 +218,8 @@ class S3FilesStore: path: str, buf: BytesIO, info: MediaPipeline.SpiderInfo, - meta: Optional[Dict[str, Any]] = None, - headers: Optional[Dict[str, str]] = None, + meta: Optional[dict[str, Any]] = None, + headers: Optional[dict[str, str]] = None, ) -> Deferred[Any]: """Upload file to S3 storage""" key_name = f"{self.prefix}{path}" @@ -242,7 +237,7 @@ class S3FilesStore: **extra, ) - def _headers_to_botocore_kwargs(self, headers: Dict[str, Any]) -> Dict[str, Any]: + def _headers_to_botocore_kwargs(self, headers: dict[str, Any]) -> dict[str, Any]: """Convert headers to botocore keyword arguments.""" # This is required while we need to support both boto and botocore. mapping = CaseInsensitiveDict( @@ -274,7 +269,7 @@ class S3FilesStore: "X-Amz-Website-Redirect-Location": "WebsiteRedirectLocation", } ) - extra: Dict[str, Any] = {} + extra: dict[str, Any] = {} for key, value in headers.items(): try: kwarg = mapping[key] @@ -332,7 +327,7 @@ class GCSFilesStore: deferToThread(self.bucket.get_blob, blob_path).addCallback(_onsuccess), ) - def _get_content_type(self, headers: Optional[Dict[str, str]]) -> str: + def _get_content_type(self, headers: Optional[dict[str, str]]) -> str: if headers and "Content-Type" in headers: return headers["Content-Type"] return "application/octet-stream" @@ -345,8 +340,8 @@ class GCSFilesStore: path: str, buf: BytesIO, info: MediaPipeline.SpiderInfo, - meta: Optional[Dict[str, Any]] = None, - headers: Optional[Dict[str, str]] = None, + meta: Optional[dict[str, Any]] = None, + headers: Optional[dict[str, str]] = None, ) -> Deferred[Any]: blob_path = self._get_blob_path(path) blob = self.bucket.blob(blob_path) @@ -385,8 +380,8 @@ class FTPFilesStore: path: str, buf: BytesIO, info: MediaPipeline.SpiderInfo, - meta: Optional[Dict[str, Any]] = None, - headers: Optional[Dict[str, str]] = None, + meta: Optional[dict[str, Any]] = None, + headers: Optional[dict[str, str]] = None, ) -> Deferred[Any]: path = f"{self.basedir}/{path}" return deferToThread( @@ -443,7 +438,7 @@ class FilesPipeline(MediaPipeline): MEDIA_NAME: str = "file" EXPIRES: int = 90 - STORE_SCHEMES: Dict[str, Type[FilesStoreProtocol]] = { + STORE_SCHEMES: dict[str, type[FilesStoreProtocol]] = { "": FSFilesStore, "file": FSFilesStore, "s3": S3FilesStore, @@ -457,7 +452,7 @@ class FilesPipeline(MediaPipeline): self, store_uri: Union[str, PathLike[str]], download_func: Optional[Callable[[Request, Spider], Response]] = None, - settings: Union[Settings, Dict[str, Any], None] = None, + settings: Union[Settings, dict[str, Any], None] = None, ): store_uri = _to_string(store_uri) if not store_uri: @@ -486,7 +481,7 @@ class FilesPipeline(MediaPipeline): @classmethod def from_settings(cls, settings: Settings) -> Self: - s3store: Type[S3FilesStore] = cast(Type[S3FilesStore], cls.STORE_SCHEMES["s3"]) + s3store: type[S3FilesStore] = cast(type[S3FilesStore], cls.STORE_SCHEMES["s3"]) s3store.AWS_ACCESS_KEY_ID = settings["AWS_ACCESS_KEY_ID"] s3store.AWS_SECRET_ACCESS_KEY = settings["AWS_SECRET_ACCESS_KEY"] s3store.AWS_SESSION_TOKEN = settings["AWS_SESSION_TOKEN"] @@ -496,14 +491,14 @@ class FilesPipeline(MediaPipeline): s3store.AWS_VERIFY = settings["AWS_VERIFY"] s3store.POLICY = settings["FILES_STORE_S3_ACL"] - gcs_store: Type[GCSFilesStore] = cast( - Type[GCSFilesStore], cls.STORE_SCHEMES["gs"] + gcs_store: type[GCSFilesStore] = cast( + type[GCSFilesStore], cls.STORE_SCHEMES["gs"] ) gcs_store.GCS_PROJECT_ID = settings["GCS_PROJECT_ID"] gcs_store.POLICY = settings["FILES_STORE_GCS_ACL"] or None - ftp_store: Type[FTPFilesStore] = cast( - Type[FTPFilesStore], cls.STORE_SCHEMES["ftp"] + ftp_store: type[FTPFilesStore] = cast( + type[FTPFilesStore], cls.STORE_SCHEMES["ftp"] ) ftp_store.FTP_USERNAME = settings["FTP_USER"] ftp_store.FTP_PASSWORD = settings["FTP_PASSWORD"] @@ -660,7 +655,7 @@ class FilesPipeline(MediaPipeline): # Overridable Interface def get_media_requests( self, item: Any, info: MediaPipeline.SpiderInfo - ) -> List[Request]: + ) -> list[Request]: urls = ItemAdapter(item).get(self.files_urls_field, []) return [Request(u, callback=NO_CALLBACK) for u in urls] @@ -680,7 +675,7 @@ class FilesPipeline(MediaPipeline): return checksum def item_completed( - self, results: List[FileInfoOrError], item: Any, info: MediaPipeline.SpiderInfo + self, results: list[FileInfoOrError], item: Any, info: MediaPipeline.SpiderInfo ) -> Any: with suppress(KeyError): ItemAdapter(item)[self.files_result_field] = [x for ok, x in results if ok] diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py index 166f81314..f2fe4396b 100644 --- a/scrapy/pipelines/images.py +++ b/scrapy/pipelines/images.py @@ -11,19 +11,7 @@ import hashlib import warnings from contextlib import suppress from io import BytesIO -from typing import ( - TYPE_CHECKING, - Any, - Callable, - Dict, - Iterable, - List, - Optional, - Tuple, - Type, - Union, - cast, -) +from typing import TYPE_CHECKING, Any, Optional, Union, cast from itemadapter import ItemAdapter @@ -42,6 +30,7 @@ from scrapy.settings import Settings from scrapy.utils.python import get_func_args, to_bytes if TYPE_CHECKING: + from collections.abc import Callable, Iterable from os import PathLike from PIL import Image @@ -79,7 +68,7 @@ class ImagesPipeline(FilesPipeline): MIN_WIDTH: int = 0 MIN_HEIGHT: int = 0 EXPIRES: int = 90 - THUMBS: Dict[str, Tuple[int, int]] = {} + THUMBS: dict[str, tuple[int, int]] = {} DEFAULT_IMAGES_URLS_FIELD = "image_urls" DEFAULT_IMAGES_RESULT_FIELD = "images" @@ -87,7 +76,7 @@ class ImagesPipeline(FilesPipeline): self, store_uri: Union[str, PathLike[str]], download_func: Optional[Callable[[Request, Spider], Response]] = None, - settings: Union[Settings, Dict[str, Any], None] = None, + settings: Union[Settings, dict[str, Any], None] = None, ): try: from PIL import Image @@ -127,7 +116,7 @@ class ImagesPipeline(FilesPipeline): self.min_height: int = settings.getint( resolve("IMAGES_MIN_HEIGHT"), self.MIN_HEIGHT ) - self.thumbs: Dict[str, Tuple[int, int]] = settings.get( + self.thumbs: dict[str, tuple[int, int]] = settings.get( resolve("IMAGES_THUMBS"), self.THUMBS ) @@ -135,7 +124,7 @@ class ImagesPipeline(FilesPipeline): @classmethod def from_settings(cls, settings: Settings) -> Self: - s3store: Type[S3FilesStore] = cast(Type[S3FilesStore], cls.STORE_SCHEMES["s3"]) + s3store: type[S3FilesStore] = cast(type[S3FilesStore], cls.STORE_SCHEMES["s3"]) s3store.AWS_ACCESS_KEY_ID = settings["AWS_ACCESS_KEY_ID"] s3store.AWS_SECRET_ACCESS_KEY = settings["AWS_SECRET_ACCESS_KEY"] s3store.AWS_SESSION_TOKEN = settings["AWS_SESSION_TOKEN"] @@ -145,14 +134,14 @@ class ImagesPipeline(FilesPipeline): s3store.AWS_VERIFY = settings["AWS_VERIFY"] s3store.POLICY = settings["IMAGES_STORE_S3_ACL"] - gcs_store: Type[GCSFilesStore] = cast( - Type[GCSFilesStore], cls.STORE_SCHEMES["gs"] + gcs_store: type[GCSFilesStore] = cast( + type[GCSFilesStore], cls.STORE_SCHEMES["gs"] ) gcs_store.GCS_PROJECT_ID = settings["GCS_PROJECT_ID"] gcs_store.POLICY = settings["IMAGES_STORE_GCS_ACL"] or None - ftp_store: Type[FTPFilesStore] = cast( - Type[FTPFilesStore], cls.STORE_SCHEMES["ftp"] + ftp_store: type[FTPFilesStore] = cast( + type[FTPFilesStore], cls.STORE_SCHEMES["ftp"] ) ftp_store.FTP_USERNAME = settings["FTP_USER"] ftp_store.FTP_PASSWORD = settings["FTP_PASSWORD"] @@ -202,7 +191,7 @@ class ImagesPipeline(FilesPipeline): info: MediaPipeline.SpiderInfo, *, item: Any = None, - ) -> Iterable[Tuple[str, Image.Image, BytesIO]]: + ) -> Iterable[tuple[str, Image.Image, BytesIO]]: path = self.file_path(request, response=response, info=info, item=item) orig_image = self._Image.open(BytesIO(response.body)) @@ -246,9 +235,9 @@ class ImagesPipeline(FilesPipeline): def convert_image( self, image: Image.Image, - size: Optional[Tuple[int, int]] = None, + size: Optional[tuple[int, int]] = None, response_body: Optional[BytesIO] = None, - ) -> Tuple[Image.Image, BytesIO]: + ) -> tuple[Image.Image, BytesIO]: if response_body is None: warnings.warn( f"{self.__class__.__name__}.convert_image() method called in a deprecated way, " @@ -288,12 +277,12 @@ class ImagesPipeline(FilesPipeline): def get_media_requests( self, item: Any, info: MediaPipeline.SpiderInfo - ) -> List[Request]: + ) -> list[Request]: urls = ItemAdapter(item).get(self.images_urls_field, []) return [Request(u, callback=NO_CALLBACK) for u in urls] def item_completed( - self, results: List[FileInfoOrError], item: Any, info: MediaPipeline.SpiderInfo + self, results: list[FileInfoOrError], item: Any, info: MediaPipeline.SpiderInfo ) -> Any: with suppress(KeyError): ItemAdapter(item)[self.images_result_field] = [x for ok, x in results if ok] diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.py index ea36a9e8a..b30cf9264 100644 --- a/scrapy/pipelines/media.py +++ b/scrapy/pipelines/media.py @@ -7,15 +7,9 @@ from collections import defaultdict from typing import ( TYPE_CHECKING, Any, - Callable, - DefaultDict, - Dict, - List, Literal, NoReturn, Optional, - Set, - Tuple, TypedDict, TypeVar, Union, @@ -33,6 +27,8 @@ from scrapy.utils.log import failure_to_exc_info from scrapy.utils.misc import arg_to_iter if TYPE_CHECKING: + from collections.abc import Callable + # typing.Self requires Python 3.11 from typing_extensions import Self @@ -52,7 +48,7 @@ class FileInfo(TypedDict): status: str -FileInfoOrError = Union[Tuple[Literal[True], FileInfo], Tuple[Literal[False], Failure]] +FileInfoOrError = Union[tuple[Literal[True], FileInfo], tuple[Literal[False], Failure]] logger = logging.getLogger(__name__) @@ -67,16 +63,16 @@ class MediaPipeline(ABC): class SpiderInfo: def __init__(self, spider: Spider): self.spider: Spider = spider - self.downloading: Set[bytes] = set() - self.downloaded: Dict[bytes, Union[FileInfo, Failure]] = {} - self.waiting: DefaultDict[bytes, List[Deferred[FileInfo]]] = defaultdict( + self.downloading: set[bytes] = set() + self.downloaded: dict[bytes, Union[FileInfo, Failure]] = {} + self.waiting: defaultdict[bytes, list[Deferred[FileInfo]]] = defaultdict( list ) def __init__( self, download_func: Optional[Callable[[Request, Spider], Response]] = None, - settings: Union[Settings, Dict[str, Any], None] = None, + settings: Union[Settings, dict[str, Any], None] = None, ): self.download_func = download_func @@ -129,12 +125,12 @@ class MediaPipeline(ABC): def process_item( self, item: Any, spider: Spider - ) -> Deferred[List[FileInfoOrError]]: + ) -> Deferred[list[FileInfoOrError]]: info = self.spiderinfo requests = arg_to_iter(self.get_media_requests(item, info)) dlist = [self._process_request(r, info, item) for r in requests] dfd = cast( - "Deferred[List[FileInfoOrError]]", DeferredList(dlist, consumeErrors=True) + "Deferred[list[FileInfoOrError]]", DeferredList(dlist, consumeErrors=True) ) return dfd.addCallback(self.item_completed, item, info) @@ -211,7 +207,7 @@ class MediaPipeline(ABC): # minimize cached information for failure result.cleanFailure() result.frames = [] - result.stack = None + result.stack = [] # This code fixes a memory leak by avoiding to keep references to # the Request and Response objects on the Media Pipeline cache. @@ -252,7 +248,7 @@ class MediaPipeline(ABC): raise NotImplementedError() @abstractmethod - def get_media_requests(self, item: Any, info: SpiderInfo) -> List[Request]: + def get_media_requests(self, item: Any, info: SpiderInfo) -> list[Request]: """Returns the media requests to download""" raise NotImplementedError() @@ -276,7 +272,7 @@ class MediaPipeline(ABC): raise NotImplementedError() def item_completed( - self, results: List[FileInfoOrError], item: Any, info: SpiderInfo + self, results: list[FileInfoOrError], item: Any, info: SpiderInfo ) -> Any: """Called per item when all media requests has been processed""" if self.LOG_FAILED_RESULTS: diff --git a/scrapy/pqueues.py b/scrapy/pqueues.py index 58a47ef0f..e1bb21fb1 100644 --- a/scrapy/pqueues.py +++ b/scrapy/pqueues.py @@ -2,23 +2,15 @@ from __future__ import annotations import hashlib import logging -from typing import ( - TYPE_CHECKING, - Dict, - Iterable, - List, - Optional, - Protocol, - Tuple, - Type, - cast, -) +from typing import TYPE_CHECKING, Optional, Protocol, cast from scrapy import Request from scrapy.core.downloader import Downloader from scrapy.utils.misc import build_from_crawler if TYPE_CHECKING: + from collections.abc import Iterable + # typing.Self requires Python 3.11 from typing_extensions import Self @@ -87,7 +79,7 @@ class ScrapyPriorityQueue: def from_crawler( cls, crawler: Crawler, - downstream_queue_cls: Type[QueueProtocol], + downstream_queue_cls: type[QueueProtocol], key: str, startprios: Iterable[int] = (), ) -> Self: @@ -96,14 +88,14 @@ class ScrapyPriorityQueue: def __init__( self, crawler: Crawler, - downstream_queue_cls: Type[QueueProtocol], + downstream_queue_cls: type[QueueProtocol], key: str, startprios: Iterable[int] = (), ): self.crawler: Crawler = crawler - self.downstream_queue_cls: Type[QueueProtocol] = downstream_queue_cls + self.downstream_queue_cls: type[QueueProtocol] = downstream_queue_cls self.key: str = key - self.queues: Dict[int, QueueProtocol] = {} + self.queues: dict[int, QueueProtocol] = {} self.curprio: Optional[int] = None self.init_prios(startprios) @@ -160,8 +152,8 @@ class ScrapyPriorityQueue: # Protocols can't declare optional members return cast(Request, queue.peek()) # type: ignore[attr-defined] - def close(self) -> List[int]: - active: List[int] = [] + def close(self) -> list[int]: + active: list[int] = [] for p, q in self.queues.items(): active.append(p) q.close() @@ -176,7 +168,7 @@ class DownloaderInterface: assert crawler.engine self.downloader: Downloader = crawler.engine.downloader - def stats(self, possible_slots: Iterable[str]) -> List[Tuple[int, str]]: + def stats(self, possible_slots: Iterable[str]) -> list[tuple[int, str]]: return [(self._active_downloads(slot), slot) for slot in possible_slots] def get_slot_key(self, request: Request) -> str: @@ -199,18 +191,18 @@ class DownloaderAwarePriorityQueue: def from_crawler( cls, crawler: Crawler, - downstream_queue_cls: Type[QueueProtocol], + downstream_queue_cls: type[QueueProtocol], key: str, - startprios: Optional[Dict[str, Iterable[int]]] = None, + startprios: Optional[dict[str, Iterable[int]]] = None, ) -> Self: return cls(crawler, downstream_queue_cls, key, startprios) def __init__( self, crawler: Crawler, - downstream_queue_cls: Type[QueueProtocol], + downstream_queue_cls: type[QueueProtocol], key: str, - slot_startprios: Optional[Dict[str, Iterable[int]]] = None, + slot_startprios: Optional[dict[str, Iterable[int]]] = None, ): if crawler.settings.getint("CONCURRENT_REQUESTS_PER_IP") != 0: raise ValueError( @@ -229,11 +221,11 @@ class DownloaderAwarePriorityQueue: ) self._downloader_interface: DownloaderInterface = DownloaderInterface(crawler) - self.downstream_queue_cls: Type[QueueProtocol] = downstream_queue_cls + self.downstream_queue_cls: type[QueueProtocol] = downstream_queue_cls self.key: str = key self.crawler: Crawler = crawler - self.pqueues: Dict[str, ScrapyPriorityQueue] = {} # slot -> priority queue + self.pqueues: dict[str, ScrapyPriorityQueue] = {} # slot -> priority queue for slot, startprios in (slot_startprios or {}).items(): self.pqueues[slot] = self.pqfactory(slot, startprios) @@ -281,7 +273,7 @@ class DownloaderAwarePriorityQueue: queue = self.pqueues[slot] return queue.peek() - def close(self) -> Dict[str, List[int]]: + def close(self) -> dict[str, list[int]]: active = {slot: queue.close() for slot, queue in self.pqueues.items()} self.pqueues.clear() return active diff --git a/scrapy/resolver.py b/scrapy/resolver.py index d5eedf9b1..97fa74bc2 100644 --- a/scrapy/resolver.py +++ b/scrapy/resolver.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Any, List, Optional, Sequence, Type +from typing import TYPE_CHECKING, Any, Optional from twisted.internet import defer from twisted.internet.base import ReactorBase, ThreadedResolver @@ -16,6 +16,8 @@ from zope.interface.declarations import implementer, provider from scrapy.utils.datatypes import LocalCache if TYPE_CHECKING: + from collections.abc import Sequence + from twisted.internet.defer import Deferred # typing.Self requires Python 3.11 @@ -82,7 +84,7 @@ class _CachingResolutionReceiver: def __init__(self, resolutionReceiver: IResolutionReceiver, hostName: str): self.resolutionReceiver: IResolutionReceiver = resolutionReceiver self.hostName: str = hostName - self.addresses: List[IAddress] = [] + self.addresses: list[IAddress] = [] def resolutionBegan(self, resolution: IHostResolution) -> None: self.resolutionReceiver.resolutionBegan(resolution) @@ -126,7 +128,7 @@ class CachingHostnameResolver: resolutionReceiver: IResolutionReceiver, hostName: str, portNumber: int = 0, - addressTypes: Optional[Sequence[Type[IAddress]]] = None, + addressTypes: Optional[Sequence[type[IAddress]]] = None, transportSemantics: str = "TCP", ) -> IHostResolution: try: diff --git a/scrapy/responsetypes.py b/scrapy/responsetypes.py index 702e50536..7154f2b95 100644 --- a/scrapy/responsetypes.py +++ b/scrapy/responsetypes.py @@ -3,15 +3,20 @@ This module implements a class which returns the appropriate Response class based on different criteria. """ +from __future__ import annotations + from io import StringIO from mimetypes import MimeTypes from pkgutil import get_data -from typing import Dict, Mapping, Optional, Type, Union +from typing import TYPE_CHECKING, Optional, Union from scrapy.http import Response from scrapy.utils.misc import load_object from scrapy.utils.python import binary_is_text, to_bytes, to_unicode +if TYPE_CHECKING: + from collections.abc import Mapping + class ResponseTypes: CLASSES = { @@ -32,7 +37,7 @@ class ResponseTypes: } def __init__(self) -> None: - self.classes: Dict[str, Type[Response]] = {} + self.classes: dict[str, type[Response]] = {} self.mimetypes: MimeTypes = MimeTypes() mimedata = get_data("scrapy", "mime.types") if not mimedata: @@ -43,7 +48,7 @@ class ResponseTypes: for mimetype, cls in self.CLASSES.items(): self.classes[mimetype] = load_object(cls) - def from_mimetype(self, mimetype: str) -> Type[Response]: + def from_mimetype(self, mimetype: str) -> type[Response]: """Return the most appropriate Response class for the given mimetype""" if mimetype is None: return Response @@ -54,7 +59,7 @@ class ResponseTypes: def from_content_type( self, content_type: Union[str, bytes], content_encoding: Optional[bytes] = None - ) -> Type[Response]: + ) -> type[Response]: """Return the most appropriate Response class from an HTTP Content-Type header""" if content_encoding: @@ -66,7 +71,7 @@ class ResponseTypes: def from_content_disposition( self, content_disposition: Union[str, bytes] - ) -> Type[Response]: + ) -> type[Response]: try: filename = ( to_unicode(content_disposition, encoding="latin-1", errors="replace") @@ -78,7 +83,7 @@ class ResponseTypes: except IndexError: return Response - def from_headers(self, headers: Mapping[bytes, bytes]) -> Type[Response]: + def from_headers(self, headers: Mapping[bytes, bytes]) -> type[Response]: """Return the most appropriate Response class by looking at the HTTP headers""" cls = Response @@ -91,14 +96,14 @@ class ResponseTypes: cls = self.from_content_disposition(headers[b"Content-Disposition"]) return cls - def from_filename(self, filename: str) -> Type[Response]: + def from_filename(self, filename: str) -> type[Response]: """Return the most appropriate Response class from a file name""" mimetype, encoding = self.mimetypes.guess_type(filename) if mimetype and not encoding: return self.from_mimetype(mimetype) return Response - def from_body(self, body: bytes) -> Type[Response]: + def from_body(self, body: bytes) -> type[Response]: """Try to guess the appropriate response based on the body content. This method is a bit magic and could be improved in the future, but it's not meant to be used except for special cases where response types @@ -122,7 +127,7 @@ class ResponseTypes: url: Optional[str] = None, filename: Optional[str] = None, body: Optional[bytes] = None, - ) -> Type[Response]: + ) -> type[Response]: """Guess the most appropriate Response class based on the given arguments.""" cls = Response diff --git a/scrapy/selector/unified.py b/scrapy/selector/unified.py index bfddb87cb..0a3eae409 100644 --- a/scrapy/selector/unified.py +++ b/scrapy/selector/unified.py @@ -2,7 +2,7 @@ XPath selectors based on lxml """ -from typing import Any, Optional, Type, Union +from typing import Any, Optional, Union from parsel import Selector as _ParselSelector @@ -23,7 +23,7 @@ def _st(response: Optional[TextResponse], st: Optional[str]) -> str: def _response_from_text(text: Union[str, bytes], st: Optional[str]) -> TextResponse: - rt: Type[TextResponse] = XmlResponse if st == "xml" else HtmlResponse + rt: type[TextResponse] = XmlResponse if st == "xml" else HtmlResponse return rt(url="about:blank", encoding="utf-8", body=to_bytes(text, "utf-8")) diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py index 6703c569f..b7e3763fb 100644 --- a/scrapy/settings/__init__.py +++ b/scrapy/settings/__init__.py @@ -2,22 +2,10 @@ from __future__ import annotations import copy import json +from collections.abc import Iterable, Iterator, Mapping, MutableMapping from importlib import import_module from pprint import pformat -from typing import ( - TYPE_CHECKING, - Any, - Dict, - Iterable, - Iterator, - List, - Mapping, - MutableMapping, - Optional, - Tuple, - Union, - cast, -) +from typing import TYPE_CHECKING, Any, Optional, Union, cast from scrapy.settings import default_settings @@ -37,7 +25,7 @@ if TYPE_CHECKING: _SettingsInputT = Union[SupportsItems[_SettingsKeyT, Any], str, None] -SETTINGS_PRIORITIES: Dict[str, int] = { +SETTINGS_PRIORITIES: dict[str, int] = { "default": 0, "command": 10, "addon": 15, @@ -192,8 +180,8 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]): return float(self.get(name, default)) def getlist( - self, name: _SettingsKeyT, default: Optional[List[Any]] = None - ) -> List[Any]: + self, name: _SettingsKeyT, default: Optional[list[Any]] = None + ) -> list[Any]: """ Get a setting value as a list. If the setting original type is a list, a copy of it will be returned. If it's a string it will be split by ",". @@ -213,8 +201,8 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]): return list(value) def getdict( - self, name: _SettingsKeyT, default: Optional[Dict[Any, Any]] = None - ) -> Dict[Any, Any]: + self, name: _SettingsKeyT, default: Optional[dict[Any, Any]] = None + ) -> dict[Any, Any]: """ Get a setting value as a dictionary. If the setting original type is a dictionary, a copy of it will be returned. If it is a string it will be @@ -238,8 +226,8 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]): def getdictorlist( self, name: _SettingsKeyT, - default: Union[Dict[Any, Any], List[Any], Tuple[Any], None] = None, - ) -> Union[Dict[Any, Any], List[Any]]: + default: Union[dict[Any, Any], list[Any], tuple[Any], None] = None, + ) -> Union[dict[Any, Any], list[Any]]: """Get a setting value as either a :class:`dict` or a :class:`list`. If the setting is already a dict or a list, a copy of it will be @@ -412,7 +400,7 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]): """ self._assert_mutability() if isinstance(values, str): - values = cast(Dict[_SettingsKeyT, Any], json.loads(values)) + values = cast(dict[_SettingsKeyT, Any], json.loads(values)) if values is not None: if isinstance(values, BaseSettings): for name, value in values.items(): @@ -477,7 +465,7 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]): def __len__(self) -> int: return len(self.attributes) - def _to_dict(self) -> Dict[_SettingsKeyT, Any]: + def _to_dict(self) -> dict[_SettingsKeyT, Any]: return { self._get_key(k): (v._to_dict() if isinstance(v, BaseSettings) else v) for k, v in self.items() @@ -490,7 +478,7 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]): else str(key_value) ) - def copy_to_dict(self) -> Dict[_SettingsKeyT, Any]: + def copy_to_dict(self) -> dict[_SettingsKeyT, Any]: """ Make a copy of current settings and convert to a dict. @@ -553,7 +541,7 @@ class Settings(BaseSettings): self.update(values, priority) -def iter_default_settings() -> Iterable[Tuple[str, Any]]: +def iter_default_settings() -> Iterable[tuple[str, Any]]: """Return the default settings as an iterator of (name, value) tuples""" for name in dir(default_settings): if name.isupper(): @@ -562,7 +550,7 @@ def iter_default_settings() -> Iterable[Tuple[str, Any]]: def overridden_settings( settings: Mapping[_SettingsKeyT, Any] -) -> Iterable[Tuple[str, Any]]: +) -> Iterable[tuple[str, Any]]: """Return an iterable of the settings that have been overridden""" for name, defvalue in iter_default_settings(): value = settings[name] diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 932475fb5..7ba0128a5 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -333,6 +333,7 @@ SPIDER_CONTRACTS = {} SPIDER_CONTRACTS_BASE = { "scrapy.contracts.default.UrlContract": 1, "scrapy.contracts.default.CallbackKeywordArgumentsContract": 1, + "scrapy.contracts.default.MetadataContract": 1, "scrapy.contracts.default.ReturnsContract": 2, "scrapy.contracts.default.ScrapesContract": 3, } diff --git a/scrapy/shell.py b/scrapy/shell.py index b7e46274f..dc402e678 100644 --- a/scrapy/shell.py +++ b/scrapy/shell.py @@ -8,7 +8,7 @@ from __future__ import annotations import os import signal -from typing import Any, Callable, Dict, Optional, Tuple, Union +from typing import TYPE_CHECKING, Any, Optional, Union from itemadapter import is_item from twisted.internet import defer, threads @@ -27,25 +27,28 @@ from scrapy.utils.misc import load_object from scrapy.utils.reactor import is_asyncio_reactor_installed, set_asyncio_event_loop from scrapy.utils.response import open_in_browser +if TYPE_CHECKING: + from collections.abc import Callable + class Shell: - relevant_classes: Tuple[type, ...] = (Crawler, Spider, Request, Response, Settings) + relevant_classes: tuple[type, ...] = (Crawler, Spider, Request, Response, Settings) def __init__( self, crawler: Crawler, - update_vars: Optional[Callable[[Dict[str, Any]], None]] = None, + update_vars: Optional[Callable[[dict[str, Any]], None]] = None, code: Optional[str] = None, ): self.crawler: Crawler = crawler - self.update_vars: Callable[[Dict[str, Any]], None] = update_vars or ( + self.update_vars: Callable[[dict[str, Any]], None] = update_vars or ( lambda x: None ) self.item_class: type = load_object(crawler.settings["DEFAULT_ITEM_CLASS"]) self.spider: Optional[Spider] = None self.inthread: bool = not threadable.isInIOThread() self.code: Optional[str] = code - self.vars: Dict[str, Any] = {} + self.vars: dict[str, Any] = {} def start( self, diff --git a/scrapy/signalmanager.py b/scrapy/signalmanager.py index b2c6dea5d..e106418d6 100644 --- a/scrapy/signalmanager.py +++ b/scrapy/signalmanager.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Any, List, Tuple +from typing import TYPE_CHECKING, Any from pydispatch import dispatcher @@ -40,7 +40,7 @@ class SignalManager: kwargs.setdefault("sender", self.sender) dispatcher.disconnect(receiver, signal, **kwargs) - def send_catch_log(self, signal: Any, **kwargs: Any) -> List[Tuple[Any, Any]]: + def send_catch_log(self, signal: Any, **kwargs: Any) -> list[tuple[Any, Any]]: """ Send a signal, catch exceptions and log them. @@ -52,7 +52,7 @@ class SignalManager: def send_catch_log_deferred( self, signal: Any, **kwargs: Any - ) -> Deferred[List[Tuple[Any, Any]]]: + ) -> Deferred[list[tuple[Any, Any]]]: """ Like :meth:`send_catch_log` but supports returning :class:`~twisted.internet.defer.Deferred` objects from signal handlers. diff --git a/scrapy/spiderloader.py b/scrapy/spiderloader.py index b8fe65668..210e729a1 100644 --- a/scrapy/spiderloader.py +++ b/scrapy/spiderloader.py @@ -3,7 +3,7 @@ from __future__ import annotations import traceback import warnings from collections import defaultdict -from typing import TYPE_CHECKING, DefaultDict, Dict, List, Tuple, Type +from typing import TYPE_CHECKING from zope.interface import implementer @@ -29,10 +29,10 @@ class SpiderLoader: """ def __init__(self, settings: BaseSettings): - self.spider_modules: List[str] = settings.getlist("SPIDER_MODULES") + self.spider_modules: list[str] = settings.getlist("SPIDER_MODULES") self.warn_only: bool = settings.getbool("SPIDER_LOADER_WARN_ONLY") - self._spiders: Dict[str, Type[Spider]] = {} - self._found: DefaultDict[str, List[Tuple[str, str]]] = defaultdict(list) + self._spiders: dict[str, type[Spider]] = {} + self._found: defaultdict[str, list[tuple[str, str]]] = defaultdict(list) self._load_all_spiders() def _check_name_duplicates(self) -> None: @@ -64,7 +64,7 @@ class SpiderLoader: try: for module in walk_modules(name): self._load_spiders(module) - except ImportError: + except (ImportError, SyntaxError): if self.warn_only: warnings.warn( f"\n{traceback.format_exc()}Could not load spiders " @@ -80,7 +80,7 @@ class SpiderLoader: def from_settings(cls, settings: BaseSettings) -> Self: return cls(settings) - def load(self, spider_name: str) -> Type[Spider]: + def load(self, spider_name: str) -> type[Spider]: """ Return the Spider class for the given spider name. If the spider name is not found, raise a KeyError. @@ -90,7 +90,7 @@ class SpiderLoader: except KeyError: raise KeyError(f"Spider not found: {spider_name}") - def find_by_request(self, request: Request) -> List[str]: + def find_by_request(self, request: Request) -> list[str]: """ Return the list of spider names that can handle the given request. """ @@ -98,7 +98,7 @@ class SpiderLoader: name for name, cls in self._spiders.items() if cls.handles_request(request) ] - def list(self) -> List[str]: + def list(self) -> list[str]: """ Return a list with the names of all spiders available in the project. """ diff --git a/scrapy/spidermiddlewares/depth.py b/scrapy/spidermiddlewares/depth.py index c5b7f0749..3164c1c03 100644 --- a/scrapy/spidermiddlewares/depth.py +++ b/scrapy/spidermiddlewares/depth.py @@ -7,11 +7,13 @@ See documentation in docs/topics/spider-middleware.rst from __future__ import annotations import logging -from typing import TYPE_CHECKING, Any, AsyncIterable, Iterable +from typing import TYPE_CHECKING, Any from scrapy.http import Request, Response if TYPE_CHECKING: + from collections.abc import AsyncIterable, Iterable + # typing.Self requires Python 3.11 from typing_extensions import Self diff --git a/scrapy/spidermiddlewares/httperror.py b/scrapy/spidermiddlewares/httperror.py index ea1686c25..afab2eac2 100644 --- a/scrapy/spidermiddlewares/httperror.py +++ b/scrapy/spidermiddlewares/httperror.py @@ -7,11 +7,13 @@ See documentation in docs/topics/spider-middleware.rst from __future__ import annotations import logging -from typing import TYPE_CHECKING, Any, Iterable, List, Optional +from typing import TYPE_CHECKING, Any, Optional from scrapy.exceptions import IgnoreRequest if TYPE_CHECKING: + from collections.abc import Iterable + # typing.Self requires Python 3.11 from typing_extensions import Self @@ -39,7 +41,7 @@ class HttpErrorMiddleware: def __init__(self, settings: BaseSettings): self.handle_httpstatus_all: bool = settings.getbool("HTTPERROR_ALLOW_ALL") - self.handle_httpstatus_list: List[int] = settings.getlist( + self.handle_httpstatus_list: list[int] = settings.getlist( "HTTPERROR_ALLOWED_CODES" ) diff --git a/scrapy/spidermiddlewares/offsite.py b/scrapy/spidermiddlewares/offsite.py index 379c5d0a3..d3ed64ef5 100644 --- a/scrapy/spidermiddlewares/offsite.py +++ b/scrapy/spidermiddlewares/offsite.py @@ -9,7 +9,7 @@ from __future__ import annotations import logging import re import warnings -from typing import TYPE_CHECKING, Any, AsyncIterable, Iterable, Set +from typing import TYPE_CHECKING, Any from scrapy import Spider, signals from scrapy.exceptions import ScrapyDeprecationWarning @@ -23,6 +23,8 @@ warnings.warn( ) if TYPE_CHECKING: + from collections.abc import AsyncIterable, Iterable + # typing.Self requires Python 3.11 from typing_extensions import Self @@ -109,7 +111,7 @@ class OffsiteMiddleware: def spider_opened(self, spider: Spider) -> None: self.host_regex: re.Pattern[str] = self.get_host_regex(spider) - self.domains_seen: Set[str] = set() + self.domains_seen: set[str] = set() class URLWarning(Warning): diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py index d35cf8f71..8784e4b05 100644 --- a/scrapy/spidermiddlewares/referer.py +++ b/scrapy/spidermiddlewares/referer.py @@ -6,18 +6,7 @@ originated it. from __future__ import annotations import warnings -from typing import ( - TYPE_CHECKING, - Any, - AsyncIterable, - Dict, - Iterable, - Optional, - Tuple, - Type, - Union, - cast, -) +from typing import TYPE_CHECKING, Any, Optional, Union, cast from urllib.parse import urlparse from w3lib.url import safe_url_string @@ -30,6 +19,8 @@ from scrapy.utils.python import to_unicode from scrapy.utils.url import strip_url if TYPE_CHECKING: + from collections.abc import AsyncIterable, Iterable + # typing.Self requires Python 3.11 from typing_extensions import Self @@ -37,7 +28,7 @@ if TYPE_CHECKING: from scrapy.settings import BaseSettings -LOCAL_SCHEMES: Tuple[str, ...] = ( +LOCAL_SCHEMES: tuple[str, ...] = ( "about", "blob", "data", @@ -56,7 +47,7 @@ POLICY_SCRAPY_DEFAULT = "scrapy-default" class ReferrerPolicy: - NOREFERRER_SCHEMES: Tuple[str, ...] = LOCAL_SCHEMES + NOREFERRER_SCHEMES: tuple[str, ...] = LOCAL_SCHEMES name: str def referrer(self, response_url: str, request_url: str) -> Optional[str]: @@ -291,11 +282,11 @@ class DefaultReferrerPolicy(NoReferrerWhenDowngradePolicy): using ``file://`` or ``s3://`` scheme. """ - NOREFERRER_SCHEMES: Tuple[str, ...] = LOCAL_SCHEMES + ("file", "s3") + NOREFERRER_SCHEMES: tuple[str, ...] = LOCAL_SCHEMES + ("file", "s3") name: str = POLICY_SCRAPY_DEFAULT -_policy_classes: Dict[str, Type[ReferrerPolicy]] = { +_policy_classes: dict[str, type[ReferrerPolicy]] = { p.name: p for p in ( NoReferrerPolicy, @@ -316,14 +307,14 @@ _policy_classes[""] = NoReferrerWhenDowngradePolicy def _load_policy_class( policy: str, warning_only: bool = False -) -> Optional[Type[ReferrerPolicy]]: +) -> Optional[type[ReferrerPolicy]]: """ Expect a string for the path to the policy class, otherwise try to interpret the string as a standard value from https://www.w3.org/TR/referrer-policy/#referrer-policies """ try: - return cast(Type[ReferrerPolicy], load_object(policy)) + return cast(type[ReferrerPolicy], load_object(policy)) except ValueError: tokens = [token.strip() for token in policy.lower().split(",")] # https://www.w3.org/TR/referrer-policy/#parse-referrer-policy-from-header @@ -341,7 +332,7 @@ def _load_policy_class( class RefererMiddleware: def __init__(self, settings: Optional[BaseSettings] = None): - self.default_policy: Type[ReferrerPolicy] = DefaultReferrerPolicy + self.default_policy: type[ReferrerPolicy] = DefaultReferrerPolicy if settings is not None: settings_policy = _load_policy_class(settings.get("REFERRER_POLICY")) assert settings_policy diff --git a/scrapy/spidermiddlewares/urllength.py b/scrapy/spidermiddlewares/urllength.py index 34df54ca7..191adb6cd 100644 --- a/scrapy/spidermiddlewares/urllength.py +++ b/scrapy/spidermiddlewares/urllength.py @@ -7,12 +7,14 @@ See documentation in docs/topics/spider-middleware.rst from __future__ import annotations import logging -from typing import TYPE_CHECKING, Any, AsyncIterable, Iterable +from typing import TYPE_CHECKING, Any from scrapy.exceptions import NotConfigured from scrapy.http import Request, Response if TYPE_CHECKING: + from collections.abc import AsyncIterable, Iterable + # typing.Self requires Python 3.11 from typing_extensions import Self diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.py index d977acd26..8220aca28 100644 --- a/scrapy/spiders/__init__.py +++ b/scrapy/spiders/__init__.py @@ -7,7 +7,7 @@ See documentation in docs/topics/spiders.rst from __future__ import annotations import logging -from typing import TYPE_CHECKING, Any, Dict, Iterable, List, Optional, cast +from typing import TYPE_CHECKING, Any, Optional, cast from scrapy import signals from scrapy.http import Request, Response @@ -15,6 +15,8 @@ from scrapy.utils.trackref import object_ref from scrapy.utils.url import url_is_from_spider if TYPE_CHECKING: + from collections.abc import Iterable + from twisted.internet.defer import Deferred # typing.Self requires Python 3.11 @@ -32,7 +34,7 @@ class Spider(object_ref): """ name: str - custom_settings: Optional[Dict[_SettingsKeyT, Any]] = None + custom_settings: Optional[dict[_SettingsKeyT, Any]] = None def __init__(self, name: Optional[str] = None, **kwargs: Any): if name is not None: @@ -41,7 +43,7 @@ class Spider(object_ref): raise ValueError(f"{type(self).__name__} must have a name") self.__dict__.update(kwargs) if not hasattr(self, "start_urls"): - self.start_urls: List[str] = [] + self.start_urls: list[str] = [] @property def logger(self) -> SpiderLoggerAdapter: diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.py index 2639f14b2..d628f49f6 100644 --- a/scrapy/spiders/crawl.py +++ b/scrapy/spiders/crawl.py @@ -1,6 +1,6 @@ """ This modules implements the CrawlSpider which is the recommended spider to use -for scraping typical web sites that requires crawling pages. +for scraping typical websites that requires crawling pages. See documentation in docs/topics/spiders.rst """ @@ -8,22 +8,8 @@ See documentation in docs/topics/spiders.rst from __future__ import annotations import copy -from typing import ( - TYPE_CHECKING, - Any, - AsyncIterable, - Awaitable, - Callable, - Dict, - Iterable, - List, - Optional, - Sequence, - Set, - TypeVar, - Union, - cast, -) +from collections.abc import AsyncIterable, Awaitable, Callable +from typing import TYPE_CHECKING, Any, Optional, TypeVar, Union, cast from twisted.python.failure import Failure @@ -35,6 +21,8 @@ from scrapy.utils.asyncgen import collect_asyncgen from scrapy.utils.spider import iterate_spider_output if TYPE_CHECKING: + from collections.abc import Iterable, Sequence + # typing.Self requires Python 3.11 from typing_extensions import Self @@ -43,7 +31,7 @@ if TYPE_CHECKING: _T = TypeVar("_T") -ProcessLinksT = Callable[[List[Link]], List[Link]] +ProcessLinksT = Callable[[list[Link]], list[Link]] ProcessRequestT = Callable[[Request, Response], Optional[Request]] @@ -75,7 +63,7 @@ class Rule: self, link_extractor: Optional[LinkExtractor] = None, callback: Union[CallbackT, str, None] = None, - cb_kwargs: Optional[Dict[str, Any]] = None, + cb_kwargs: Optional[dict[str, Any]] = None, follow: Optional[bool] = None, process_links: Union[ProcessLinksT, str, None] = None, process_request: Union[ProcessRequestT, str, None] = None, @@ -84,7 +72,7 @@ class Rule: self.link_extractor: LinkExtractor = link_extractor or _default_link_extractor self.callback: Union[CallbackT, str, None] = callback self.errback: Union[Callable[[Failure], Any], str, None] = errback - self.cb_kwargs: Dict[str, Any] = cb_kwargs or {} + self.cb_kwargs: dict[str, Any] = cb_kwargs or {} self.process_links: Union[ProcessLinksT, str] = process_links or _identity self.process_request: Union[ProcessRequestT, str] = ( process_request or _identity_process_request @@ -105,7 +93,7 @@ class Rule: class CrawlSpider(Spider): rules: Sequence[Rule] = () - _rules: List[Rule] + _rules: list[Rule] _follow_links: bool def __init__(self, *a: Any, **kw: Any): @@ -139,9 +127,9 @@ class CrawlSpider(Spider): def _requests_to_follow(self, response: Response) -> Iterable[Optional[Request]]: if not isinstance(response, HtmlResponse): return - seen: Set[Link] = set() + seen: set[Link] = set() for rule_index, rule in enumerate(self._rules): - links: List[Link] = [ + links: list[Link] = [ lnk for lnk in rule.link_extractor.extract_links(response) if lnk not in seen @@ -170,7 +158,7 @@ class CrawlSpider(Spider): self, response: Response, callback: Optional[CallbackT], - cb_kwargs: Dict[str, Any], + cb_kwargs: dict[str, Any], follow: bool = True, ) -> AsyncIterable[Any]: if callback: diff --git a/scrapy/spiders/feed.py b/scrapy/spiders/feed.py index 9dd8a5d68..0ddef1f32 100644 --- a/scrapy/spiders/feed.py +++ b/scrapy/spiders/feed.py @@ -5,7 +5,9 @@ for scraping from an XML feed. See documentation in docs/topics/spiders.rst """ -from typing import Any, Dict, Iterable, List, Optional, Sequence, Tuple +from __future__ import annotations + +from typing import TYPE_CHECKING, Any, Optional from scrapy.exceptions import NotConfigured, NotSupported from scrapy.http import Response, TextResponse @@ -14,6 +16,9 @@ from scrapy.spiders import Spider from scrapy.utils.iterators import csviter, xmliter_lxml from scrapy.utils.spider import iterate_spider_output +if TYPE_CHECKING: + from collections.abc import Iterable, Sequence + class XMLFeedSpider(Spider): """ @@ -27,7 +32,7 @@ class XMLFeedSpider(Spider): iterator: str = "iternodes" itertag: str = "item" - namespaces: Sequence[Tuple[str, str]] = () + namespaces: Sequence[tuple[str, str]] = () def process_results( self, response: Response, results: Iterable[Any] @@ -118,7 +123,7 @@ class CSVFeedSpider(Spider): quotechar: Optional[str] = ( None # When this is None, python's csv module's default quotechar is used ) - headers: Optional[List[str]] = None + headers: Optional[list[str]] = None def process_results( self, response: Response, results: Iterable[Any] @@ -130,7 +135,7 @@ class CSVFeedSpider(Spider): """This method has the same purpose as the one in XMLFeedSpider""" return response - def parse_row(self, response: Response, row: Dict[str, str]) -> Any: + def parse_row(self, response: Response, row: dict[str, str]) -> Any: """This method must be overridden with your custom spider functionality""" raise NotImplementedError diff --git a/scrapy/spiders/init.py b/scrapy/spiders/init.py index ce0f1bbaa..ebe288b83 100644 --- a/scrapy/spiders/init.py +++ b/scrapy/spiders/init.py @@ -1,6 +1,7 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Any, Iterable, Optional, cast +from collections.abc import Iterable +from typing import TYPE_CHECKING, Any, Optional, cast from scrapy import Request from scrapy.spiders import Spider diff --git a/scrapy/spiders/sitemap.py b/scrapy/spiders/sitemap.py index 1542ef79c..945539d7b 100644 --- a/scrapy/spiders/sitemap.py +++ b/scrapy/spiders/sitemap.py @@ -2,18 +2,7 @@ from __future__ import annotations import logging import re -from typing import ( - TYPE_CHECKING, - Any, - Dict, - Iterable, - List, - Optional, - Sequence, - Tuple, - Union, - cast, -) +from typing import TYPE_CHECKING, Any, Optional, Union, cast from scrapy.http import Request, Response, XmlResponse from scrapy.spiders import Spider @@ -22,6 +11,8 @@ from scrapy.utils.gz import gunzip, gzip_magic_number from scrapy.utils.sitemap import Sitemap, sitemap_urls_from_robots if TYPE_CHECKING: + from collections.abc import Iterable, Sequence + # typing.Self requires Python 3.11 from typing_extensions import Self @@ -34,7 +25,7 @@ logger = logging.getLogger(__name__) class SitemapSpider(Spider): sitemap_urls: Sequence[str] = () sitemap_rules: Sequence[ - Tuple[Union[re.Pattern[str], str], Union[str, CallbackT]] + tuple[Union[re.Pattern[str], str], Union[str, CallbackT]] ] = [("", "parse")] sitemap_follow: Sequence[Union[re.Pattern[str], str]] = [""] sitemap_alternate_links: bool = False @@ -54,20 +45,20 @@ class SitemapSpider(Spider): def __init__(self, *a: Any, **kw: Any): super().__init__(*a, **kw) - self._cbs: List[Tuple[re.Pattern[str], CallbackT]] = [] + self._cbs: list[tuple[re.Pattern[str], CallbackT]] = [] for r, c in self.sitemap_rules: if isinstance(c, str): c = cast("CallbackT", getattr(self, c)) self._cbs.append((regex(r), c)) - self._follow: List[re.Pattern[str]] = [regex(x) for x in self.sitemap_follow] + self._follow: list[re.Pattern[str]] = [regex(x) for x in self.sitemap_follow] def start_requests(self) -> Iterable[Request]: for url in self.sitemap_urls: yield Request(url, self._parse_sitemap) def sitemap_filter( - self, entries: Iterable[Dict[str, Any]] - ) -> Iterable[Dict[str, Any]]: + self, entries: Iterable[dict[str, Any]] + ) -> Iterable[dict[str, Any]]: """This method can be used to filter sitemap entries by their attributes, for example, you can filter locs with lastmod greater than a given date (see docs). @@ -142,7 +133,7 @@ def regex(x: Union[re.Pattern[str], str]) -> re.Pattern[str]: return x -def iterloc(it: Iterable[Dict[str, Any]], alt: bool = False) -> Iterable[str]: +def iterloc(it: Iterable[dict[str, Any]], alt: bool = False) -> Iterable[str]: for d in it: yield d["loc"] diff --git a/scrapy/squeues.py b/scrapy/squeues.py index d3e7896c5..767a53db8 100644 --- a/scrapy/squeues.py +++ b/scrapy/squeues.py @@ -7,13 +7,14 @@ from __future__ import annotations import marshal import pickle # nosec from pathlib import Path -from typing import TYPE_CHECKING, Any, Callable, Optional, Type, Union +from typing import TYPE_CHECKING, Any, Optional, Union from queuelib import queue from scrapy.utils.request import request_from_dict if TYPE_CHECKING: + from collections.abc import Callable from os import PathLike # typing.Self requires Python 3.11 @@ -23,7 +24,7 @@ if TYPE_CHECKING: from scrapy.crawler import Crawler -def _with_mkdir(queue_class: Type[queue.BaseQueue]) -> Type[queue.BaseQueue]: +def _with_mkdir(queue_class: type[queue.BaseQueue]) -> type[queue.BaseQueue]: class DirectoriesCreated(queue_class): # type: ignore[valid-type,misc] def __init__(self, path: Union[str, PathLike], *args: Any, **kwargs: Any): dirname = Path(path).parent @@ -35,10 +36,10 @@ def _with_mkdir(queue_class: Type[queue.BaseQueue]) -> Type[queue.BaseQueue]: def _serializable_queue( - queue_class: Type[queue.BaseQueue], + queue_class: type[queue.BaseQueue], serialize: Callable[[Any], bytes], deserialize: Callable[[bytes], Any], -) -> Type[queue.BaseQueue]: +) -> type[queue.BaseQueue]: class SerializableQueue(queue_class): # type: ignore[valid-type,misc] def push(self, obj: Any) -> None: s = serialize(obj) @@ -71,8 +72,8 @@ def _serializable_queue( def _scrapy_serialization_queue( - queue_class: Type[queue.BaseQueue], -) -> Type[queue.BaseQueue]: + queue_class: type[queue.BaseQueue], +) -> type[queue.BaseQueue]: class ScrapyRequestQueue(queue_class): # type: ignore[valid-type,misc] def __init__(self, crawler: Crawler, key: str): self.spider = crawler.spider @@ -110,8 +111,8 @@ def _scrapy_serialization_queue( def _scrapy_non_serialization_queue( - queue_class: Type[queue.BaseQueue], -) -> Type[queue.BaseQueue]: + queue_class: type[queue.BaseQueue], +) -> type[queue.BaseQueue]: class ScrapyRequestQueue(queue_class): # type: ignore[valid-type,misc] @classmethod def from_crawler(cls, crawler: Crawler, *args: Any, **kwargs: Any) -> Self: diff --git a/scrapy/statscollectors.py b/scrapy/statscollectors.py index 88e72f366..63c82ec6d 100644 --- a/scrapy/statscollectors.py +++ b/scrapy/statscollectors.py @@ -6,7 +6,7 @@ from __future__ import annotations import logging import pprint -from typing import TYPE_CHECKING, Any, Dict, Optional +from typing import TYPE_CHECKING, Any, Optional if TYPE_CHECKING: from scrapy import Spider @@ -16,7 +16,7 @@ if TYPE_CHECKING: logger = logging.getLogger(__name__) -StatsT = Dict[str, Any] +StatsT = dict[str, Any] class StatsCollector: @@ -71,7 +71,7 @@ class StatsCollector: class MemoryStatsCollector(StatsCollector): def __init__(self, crawler: Crawler): super().__init__(crawler) - self.spider_stats: Dict[str, StatsT] = {} + self.spider_stats: dict[str, StatsT] = {} def _persist_stats(self, stats: StatsT, spider: Spider) -> None: self.spider_stats[spider.name] = stats diff --git a/scrapy/utils/asyncgen.py b/scrapy/utils/asyncgen.py index 67c8e1a01..f1505e4bd 100644 --- a/scrapy/utils/asyncgen.py +++ b/scrapy/utils/asyncgen.py @@ -1,9 +1,10 @@ -from typing import AsyncGenerator, AsyncIterable, Iterable, List, TypeVar, Union +from collections.abc import AsyncGenerator, AsyncIterable, Iterable +from typing import TypeVar, Union _T = TypeVar("_T") -async def collect_asyncgen(result: AsyncIterable[_T]) -> List[_T]: +async def collect_asyncgen(result: AsyncIterable[_T]) -> list[_T]: results = [] async for x in result: results.append(x) diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.py index c63b69995..463bbb5df 100644 --- a/scrapy/utils/conf.py +++ b/scrapy/utils/conf.py @@ -1,35 +1,29 @@ +from __future__ import annotations + import numbers import os import sys import warnings +from collections.abc import Iterable from configparser import ConfigParser from operator import itemgetter from pathlib import Path -from typing import ( - Any, - Callable, - Collection, - Dict, - Iterable, - List, - Mapping, - MutableMapping, - Optional, - Union, - cast, -) +from typing import TYPE_CHECKING, Any, Callable, Optional, Union, cast from scrapy.exceptions import ScrapyDeprecationWarning, UsageError from scrapy.settings import BaseSettings from scrapy.utils.deprecate import update_classpath from scrapy.utils.python import without_none_values +if TYPE_CHECKING: + from collections.abc import Collection, Mapping, MutableMapping + def build_component_list( compdict: MutableMapping[Any, Any], custom: Any = None, convert: Callable[[Any], Any] = update_classpath, -) -> List[Any]: +) -> list[Any]: """Compose a component list from a { class: order } dictionary.""" def _check_components(complist: Collection[Any]) -> None: @@ -39,7 +33,7 @@ def build_component_list( "please update your settings" ) - def _map_keys(compdict: Mapping[Any, Any]) -> Union[BaseSettings, Dict[Any, Any]]: + def _map_keys(compdict: Mapping[Any, Any]) -> Union[BaseSettings, dict[Any, Any]]: if isinstance(compdict, BaseSettings): compbs = BaseSettings() for k, v in compdict.items(): @@ -84,7 +78,7 @@ def build_component_list( return [k for k, v in sorted(compdict.items(), key=itemgetter(1))] -def arglist_to_dict(arglist: List[str]) -> Dict[str, str]: +def arglist_to_dict(arglist: list[str]) -> dict[str, str]: """Convert a list of arguments like ['arg1=val1', 'arg2=val2', ...] to a dict """ @@ -130,7 +124,7 @@ def get_config(use_closest: bool = True) -> ConfigParser: return cfg -def get_sources(use_closest: bool = True) -> List[str]: +def get_sources(use_closest: bool = True) -> list[str]: xdg_config_home = ( os.environ.get("XDG_CONFIG_HOME") or Path("~/.config").expanduser() ) @@ -146,8 +140,8 @@ def get_sources(use_closest: bool = True) -> List[str]: def feed_complete_default_values_from_settings( - feed: Dict[str, Any], settings: BaseSettings -) -> Dict[str, Any]: + feed: dict[str, Any], settings: BaseSettings +) -> dict[str, Any]: out = feed.copy() out.setdefault("batch_item_count", settings.getint("FEED_EXPORT_BATCH_ITEM_COUNT")) out.setdefault("encoding", settings["FEED_EXPORT_ENCODING"]) @@ -164,17 +158,17 @@ def feed_complete_default_values_from_settings( def feed_process_params_from_cli( settings: BaseSettings, - output: List[str], + output: list[str], output_format: Optional[str] = None, - overwrite_output: Optional[List[str]] = None, -) -> Dict[str, Dict[str, Any]]: + overwrite_output: Optional[list[str]] = None, +) -> dict[str, dict[str, Any]]: """ Receives feed export params (from the 'crawl' or 'runspider' commands), checks for inconsistencies in their quantities and returns a dictionary suitable to be used as the FEEDS setting. """ valid_output_formats: Iterable[str] = without_none_values( - cast(Dict[str, str], settings.getwithbase("FEED_EXPORTERS")) + cast(dict[str, str], settings.getwithbase("FEED_EXPORTERS")) ).keys() def check_valid_format(output_format: str) -> None: @@ -223,7 +217,7 @@ def feed_process_params_from_cli( "URIs are specified" ) - result: Dict[str, Dict[str, Any]] = {} + result: dict[str, dict[str, Any]] = {} for element in output: try: feed_uri, feed_format = element.rsplit(":", 1) diff --git a/scrapy/utils/console.py b/scrapy/utils/console.py index 328219831..3b5596ab7 100644 --- a/scrapy/utils/console.py +++ b/scrapy/utils/console.py @@ -1,12 +1,18 @@ +from __future__ import annotations + +from collections.abc import Callable from functools import wraps -from typing import Any, Callable, Dict, Iterable, Optional +from typing import TYPE_CHECKING, Any, Optional + +if TYPE_CHECKING: + from collections.abc import Iterable EmbedFuncT = Callable[..., None] -KnownShellsT = Dict[str, Callable[..., EmbedFuncT]] +KnownShellsT = dict[str, Callable[..., EmbedFuncT]] def _embed_ipython_shell( - namespace: Dict[str, Any] = {}, banner: str = "" + namespace: dict[str, Any] = {}, banner: str = "" ) -> EmbedFuncT: """Start an IPython Shell""" try: @@ -21,7 +27,7 @@ def _embed_ipython_shell( ) @wraps(_embed_ipython_shell) - def wrapper(namespace: Dict[str, Any] = namespace, banner: str = "") -> None: + def wrapper(namespace: dict[str, Any] = namespace, banner: str = "") -> None: config = load_default_config() # Always use .instance() to ensure _instance propagation to all parents # this is needed for completion works well for new imports @@ -37,26 +43,26 @@ def _embed_ipython_shell( def _embed_bpython_shell( - namespace: Dict[str, Any] = {}, banner: str = "" + namespace: dict[str, Any] = {}, banner: str = "" ) -> EmbedFuncT: """Start a bpython shell""" import bpython @wraps(_embed_bpython_shell) - def wrapper(namespace: Dict[str, Any] = namespace, banner: str = "") -> None: + def wrapper(namespace: dict[str, Any] = namespace, banner: str = "") -> None: bpython.embed(locals_=namespace, banner=banner) return wrapper def _embed_ptpython_shell( - namespace: Dict[str, Any] = {}, banner: str = "" + namespace: dict[str, Any] = {}, banner: str = "" ) -> EmbedFuncT: """Start a ptpython shell""" import ptpython.repl @wraps(_embed_ptpython_shell) - def wrapper(namespace: Dict[str, Any] = namespace, banner: str = "") -> None: + def wrapper(namespace: dict[str, Any] = namespace, banner: str = "") -> None: print(banner) ptpython.repl.embed(locals=namespace) @@ -64,7 +70,7 @@ def _embed_ptpython_shell( def _embed_standard_shell( - namespace: Dict[str, Any] = {}, banner: str = "" + namespace: dict[str, Any] = {}, banner: str = "" ) -> EmbedFuncT: """Start a standard python shell""" import code @@ -79,7 +85,7 @@ def _embed_standard_shell( readline.parse_and_bind("tab:complete") @wraps(_embed_standard_shell) - def wrapper(namespace: Dict[str, Any] = namespace, banner: str = "") -> None: + def wrapper(namespace: dict[str, Any] = namespace, banner: str = "") -> None: code.interact(banner=banner, local=namespace) return wrapper @@ -114,7 +120,7 @@ def get_shell_embed_func( def start_python_console( - namespace: Optional[Dict[str, Any]] = None, + namespace: Optional[dict[str, Any]] = None, banner: str = "", shells: Optional[Iterable[str]] = None, ) -> None: diff --git a/scrapy/utils/curl.py b/scrapy/utils/curl.py index c10e48511..9c7f63848 100644 --- a/scrapy/utils/curl.py +++ b/scrapy/utils/curl.py @@ -1,12 +1,17 @@ +from __future__ import annotations + import argparse import warnings from http.cookies import SimpleCookie from shlex import split -from typing import Any, Dict, List, NoReturn, Optional, Sequence, Tuple, Union +from typing import TYPE_CHECKING, Any, NoReturn, Optional, Union from urllib.parse import urlparse from w3lib.http import basic_auth_header +if TYPE_CHECKING: + from collections.abc import Sequence + class DataAction(argparse.Action): def __call__( @@ -51,9 +56,9 @@ for argument in safe_to_ignore_arguments: def _parse_headers_and_cookies( parsed_args: argparse.Namespace, -) -> Tuple[List[Tuple[str, bytes]], Dict[str, str]]: - headers: List[Tuple[str, bytes]] = [] - cookies: Dict[str, str] = {} +) -> tuple[list[tuple[str, bytes]], dict[str, str]]: + headers: list[tuple[str, bytes]] = [] + cookies: dict[str, str] = {} for header in parsed_args.headers or (): name, val = header.split(":", 1) name = name.strip() @@ -73,7 +78,7 @@ def _parse_headers_and_cookies( def curl_to_request_kwargs( curl_command: str, ignore_unknown_options: bool = True -) -> Dict[str, Any]: +) -> dict[str, Any]: """Convert a cURL command syntax to Request kwargs. :param str curl_command: string containing the curl command @@ -107,7 +112,7 @@ def curl_to_request_kwargs( method = parsed_args.method or "GET" - result: Dict[str, Any] = {"method": method.upper(), "url": url} + result: dict[str, Any] = {"method": method.upper(), "url": url} headers, cookies = _parse_headers_and_cookies(parsed_args) diff --git a/scrapy/utils/datatypes.py b/scrapy/utils/datatypes.py index d06887610..c78325676 100644 --- a/scrapy/utils/datatypes.py +++ b/scrapy/utils/datatypes.py @@ -10,23 +10,15 @@ from __future__ import annotations import collections import warnings import weakref +from collections import OrderedDict from collections.abc import Mapping -from typing import ( - TYPE_CHECKING, - Any, - AnyStr, - Iterable, - Optional, - OrderedDict, - Sequence, - Tuple, - TypeVar, - Union, -) +from typing import TYPE_CHECKING, Any, AnyStr, Optional, TypeVar, Union from scrapy.exceptions import ScrapyDeprecationWarning if TYPE_CHECKING: + from collections.abc import Iterable, Sequence + # typing.Self requires Python 3.11 from typing_extensions import Self @@ -52,7 +44,7 @@ class CaselessDict(dict): def __init__( self, - seq: Union[Mapping[AnyStr, Any], Iterable[Tuple[AnyStr, Any]], None] = None, + seq: Union[Mapping[AnyStr, Any], Iterable[tuple[AnyStr, Any]], None] = None, ): super().__init__() if seq: @@ -92,7 +84,7 @@ class CaselessDict(dict): return dict.setdefault(self, self.normkey(key), self.normvalue(def_val)) # type: ignore[arg-type] # doesn't fully implement MutableMapping.update() - def update(self, seq: Union[Mapping[AnyStr, Any], Iterable[Tuple[AnyStr, Any]]]) -> None: # type: ignore[override] + def update(self, seq: Union[Mapping[AnyStr, Any], Iterable[tuple[AnyStr, Any]]]) -> None: # type: ignore[override] seq = seq.items() if isinstance(seq, Mapping) else seq iseq = ((self.normkey(k), self.normvalue(v)) for k, v in seq) super().update(iseq) diff --git a/scrapy/utils/decorators.py b/scrapy/utils/decorators.py index 2240f0b58..0f4d0beda 100644 --- a/scrapy/utils/decorators.py +++ b/scrapy/utils/decorators.py @@ -2,7 +2,7 @@ from __future__ import annotations import warnings from functools import wraps -from typing import TYPE_CHECKING, Any, Callable, TypeVar +from typing import TYPE_CHECKING, Any, TypeVar from twisted.internet.defer import Deferred, maybeDeferred from twisted.internet.threads import deferToThread @@ -10,6 +10,8 @@ from twisted.internet.threads import deferToThread from scrapy.exceptions import ScrapyDeprecationWarning if TYPE_CHECKING: + from collections.abc import Callable + # typing.ParamSpec requires Python 3.10 from typing_extensions import ParamSpec diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py index c5763a06c..3a0dee8f1 100644 --- a/scrapy/utils/defer.py +++ b/scrapy/utils/defer.py @@ -8,27 +8,10 @@ import asyncio import inspect import warnings from asyncio import Future +from collections.abc import Awaitable, Coroutine, Iterable, Iterator from functools import wraps from types import CoroutineType -from typing import ( - TYPE_CHECKING, - Any, - AsyncIterable, - AsyncIterator, - Awaitable, - Callable, - Coroutine, - Dict, - Iterable, - Iterator, - List, - Optional, - Tuple, - TypeVar, - Union, - cast, - overload, -) +from typing import TYPE_CHECKING, Any, Generic, Optional, TypeVar, Union, cast, overload from twisted.internet import defer from twisted.internet.defer import Deferred, DeferredList, ensureDeferred @@ -39,6 +22,8 @@ from scrapy.exceptions import IgnoreRequest, ScrapyDeprecationWarning from scrapy.utils.reactor import _get_asyncio_event_loop, is_asyncio_reactor_installed if TYPE_CHECKING: + from collections.abc import AsyncIterable, AsyncIterator, Callable + from twisted.python.failure import Failure # typing.Concatenate and typing.ParamSpec require Python 3.10 @@ -46,6 +31,7 @@ if TYPE_CHECKING: _P = ParamSpec("_P") + _T = TypeVar("_T") _T2 = TypeVar("_T2") @@ -133,7 +119,7 @@ def parallel( callable: Callable[Concatenate[_T, _P], _T2], *args: _P.args, **named: _P.kwargs, -) -> Deferred[List[Tuple[bool, Iterator[_T2]]]]: +) -> Deferred[list[tuple[bool, Iterator[_T2]]]]: """Execute a callable over the objects in the given iterable, in parallel, using no more than ``count`` concurrent calls. @@ -144,7 +130,7 @@ def parallel( return DeferredList([coop.coiterate(work) for _ in range(count)]) -class _AsyncCooperatorAdapter(Iterator[Deferred]): +class _AsyncCooperatorAdapter(Iterator, Generic[_T]): """A class that wraps an async iterable into a normal iterator suitable for using in Cooperator.coiterate(). As it's only needed for parallel_async(), it calls the callable directly in the callback, instead of providing a more @@ -199,10 +185,10 @@ class _AsyncCooperatorAdapter(Iterator[Deferred]): ): self.aiterator: AsyncIterator[_T] = aiterable.__aiter__() self.callable: Callable[Concatenate[_T, _P], Optional[Deferred[Any]]] = callable - self.callable_args: Tuple[Any, ...] = callable_args - self.callable_kwargs: Dict[str, Any] = callable_kwargs + self.callable_args: tuple[Any, ...] = callable_args + self.callable_kwargs: dict[str, Any] = callable_kwargs self.finished: bool = False - self.waiting_deferreds: List[Deferred[Any]] = [] + self.waiting_deferreds: list[Deferred[Any]] = [] self.anext_deferred: Optional[Deferred[_T]] = None def _callback(self, result: _T) -> None: @@ -254,13 +240,13 @@ def parallel_async( callable: Callable[Concatenate[_T, _P], Optional[Deferred[Any]]], *args: _P.args, **named: _P.kwargs, -) -> Deferred[List[Tuple[bool, Iterator[Deferred[Any]]]]]: +) -> Deferred[list[tuple[bool, Iterator[Deferred[Any]]]]]: """Like ``parallel`` but for async iterators""" coop = Cooperator() work: Iterator[Deferred[Any]] = _AsyncCooperatorAdapter( async_iterable, callable, *args, **named ) - dl: Deferred[List[Tuple[bool, Iterator[Deferred[Any]]]]] = DeferredList( + dl: Deferred[list[tuple[bool, Iterator[Deferred[Any]]]]] = DeferredList( [coop.coiterate(work) for _ in range(count)] ) return dl @@ -310,15 +296,15 @@ def process_parallel( input: _T, *a: _P.args, **kw: _P.kwargs, -) -> Deferred[List[_T2]]: +) -> Deferred[list[_T2]]: """Return a Deferred with the output of all successful calls to the given callbacks """ dfds = [defer.succeed(input).addCallback(x, *a, **kw) for x in callbacks] - d: Deferred[List[Tuple[bool, _T2]]] = DeferredList( + d: Deferred[list[tuple[bool, _T2]]] = DeferredList( dfds, fireOnOneErrback=True, consumeErrors=True ) - d2: Deferred[List[_T2]] = d.addCallback(lambda r: [x[1] for x in r]) + d2: Deferred[list[_T2]] = d.addCallback(lambda r: [x[1] for x in r]) d2.addErrback(lambda f: f.value.subFailure) return d2 diff --git a/scrapy/utils/deprecate.py b/scrapy/utils/deprecate.py index e0f2ac763..9b0d476a1 100644 --- a/scrapy/utils/deprecate.py +++ b/scrapy/utils/deprecate.py @@ -2,7 +2,7 @@ import inspect import warnings -from typing import Any, Dict, List, Optional, Tuple, Type, overload +from typing import Any, Optional, overload from scrapy.exceptions import ScrapyDeprecationWarning @@ -20,8 +20,8 @@ def attribute(obj: Any, oldattr: str, newattr: str, version: str = "0.12") -> No def create_deprecated_class( name: str, new_class: type, - clsdict: Optional[Dict[str, Any]] = None, - warn_category: Type[Warning] = ScrapyDeprecationWarning, + clsdict: Optional[dict[str, Any]] = None, + warn_category: type[Warning] = ScrapyDeprecationWarning, warn_once: bool = True, old_class_path: Optional[str] = None, new_class_path: Optional[str] = None, @@ -59,14 +59,14 @@ def create_deprecated_class( warned_on_subclass: bool = False def __new__( - metacls, name: str, bases: Tuple[type, ...], clsdict_: Dict[str, Any] + metacls, name: str, bases: tuple[type, ...], clsdict_: dict[str, Any] ) -> type: cls = super().__new__(metacls, name, bases, clsdict_) if metacls.deprecated_class is None: metacls.deprecated_class = cls return cls - def __init__(cls, name: str, bases: Tuple[type, ...], clsdict_: Dict[str, Any]): + def __init__(cls, name: str, bases: tuple[type, ...], clsdict_: dict[str, Any]): meta = cls.__class__ old = meta.deprecated_class if old in bases and not (warn_once and meta.warned_on_subclass): @@ -134,7 +134,7 @@ def _clspath(cls: type, forced: Optional[str] = None) -> str: return f"{cls.__module__}.{cls.__name__}" -DEPRECATION_RULES: List[Tuple[str, str]] = [] +DEPRECATION_RULES: list[tuple[str, str]] = [] @overload diff --git a/scrapy/utils/engine.py b/scrapy/utils/engine.py index 770ee0b1b..1430ed8d6 100644 --- a/scrapy/utils/engine.py +++ b/scrapy/utils/engine.py @@ -4,13 +4,13 @@ from __future__ import annotations # used in global tests code from time import time # noqa: F401 -from typing import TYPE_CHECKING, Any, List, Tuple +from typing import TYPE_CHECKING, Any if TYPE_CHECKING: from scrapy.core.engine import ExecutionEngine -def get_engine_status(engine: ExecutionEngine) -> List[Tuple[str, Any]]: +def get_engine_status(engine: ExecutionEngine) -> list[tuple[str, Any]]: """Return a report of the current engine status""" tests = [ "time()-engine.start_time", @@ -29,7 +29,7 @@ def get_engine_status(engine: ExecutionEngine) -> List[Tuple[str, Any]]: "engine.scraper.slot.needs_backout()", ] - checks: List[Tuple[str, Any]] = [] + checks: list[tuple[str, Any]] = [] for test in tests: try: checks += [(test, eval(test))] # nosec diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py index 41a842386..a4d339adc 100644 --- a/scrapy/utils/iterators.py +++ b/scrapy/utils/iterators.py @@ -1,19 +1,10 @@ +from __future__ import annotations + import csv import logging import re from io import StringIO -from typing import ( - Any, - Callable, - Dict, - Iterator, - List, - Literal, - Optional, - Union, - cast, - overload, -) +from typing import TYPE_CHECKING, Any, Literal, Optional, Union, cast, overload from warnings import warn from lxml import etree # nosec @@ -23,6 +14,9 @@ from scrapy.http import Response, TextResponse from scrapy.selector import Selector from scrapy.utils.python import re_rsearch +if TYPE_CHECKING: + from collections.abc import Callable, Iterator + logger = logging.getLogger(__name__) @@ -59,7 +53,7 @@ def xmliter(obj: Union[Response, str, bytes], nodename: str) -> Iterator[Selecto ) header_end_idx = re_rsearch(HEADER_END_RE, text) header_end = text[header_end_idx[1] :].strip() if header_end_idx else "" - namespaces: Dict[str, str] = {} + namespaces: dict[str, str] = {} if header_end: for tagname in reversed(re.findall(END_TAG_RE, header_end)): assert header_end_idx @@ -162,10 +156,10 @@ class _StreamReader: def csviter( obj: Union[Response, str, bytes], delimiter: Optional[str] = None, - headers: Optional[List[str]] = None, + headers: Optional[list[str]] = None, encoding: Optional[str] = None, quotechar: Optional[str] = None, -) -> Iterator[Dict[str, str]]: +) -> Iterator[dict[str, str]]: """Returns an iterator of dictionaries from the given csv object obj can be: @@ -191,7 +185,7 @@ def csviter( lines = StringIO(_body_or_str(obj, unicode=True)) - kwargs: Dict[str, Any] = {} + kwargs: dict[str, Any] = {} if delimiter: kwargs["delimiter"] = delimiter if quotechar: diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py index 439b065a9..2b90c6b36 100644 --- a/scrapy/utils/log.py +++ b/scrapy/utils/log.py @@ -2,20 +2,10 @@ from __future__ import annotations import logging import sys +from collections.abc import MutableMapping from logging.config import dictConfig from types import TracebackType -from typing import ( - TYPE_CHECKING, - Any, - Dict, - List, - MutableMapping, - Optional, - Tuple, - Type, - Union, - cast, -) +from typing import TYPE_CHECKING, Any, Optional, Union, cast from twisted.python import log as twisted_log from twisted.python.failure import Failure @@ -25,6 +15,7 @@ from scrapy.settings import Settings, _SettingsKeyT from scrapy.utils.versions import scrapy_components_versions if TYPE_CHECKING: + from scrapy.crawler import Crawler from scrapy.logformatter import LogFormatterResult @@ -34,7 +25,7 @@ logger = logging.getLogger(__name__) def failure_to_exc_info( failure: Failure, -) -> Optional[Tuple[Type[BaseException], BaseException, Optional[TracebackType]]]: +) -> Optional[tuple[type[BaseException], BaseException, Optional[TracebackType]]]: """Extract exc_info from Failure instances""" if isinstance(failure, Failure): assert failure.type @@ -48,7 +39,7 @@ def failure_to_exc_info( class TopLevelFormatter(logging.Filter): - """Keep only top level loggers's name (direct children from root) from + """Keep only top level loggers' name (direct children from root) from records. This filter will replace Scrapy loggers' names with 'scrapy'. This mimics @@ -59,8 +50,8 @@ class TopLevelFormatter(logging.Filter): ``loggers`` list where it should act. """ - def __init__(self, loggers: Optional[List[str]] = None): - self.loggers: List[str] = loggers or [] + def __init__(self, loggers: Optional[list[str]] = None): + self.loggers: list[str] = loggers or [] def filter(self, record: logging.LogRecord) -> bool: if any(record.name.startswith(logger + ".") for logger in self.loggers): @@ -89,7 +80,7 @@ DEFAULT_LOGGING = { def configure_logging( - settings: Union[Settings, Dict[_SettingsKeyT, Any], None] = None, + settings: Union[Settings, dict[_SettingsKeyT, Any], None] = None, install_root_handler: bool = True, ) -> None: """ @@ -128,7 +119,7 @@ def configure_logging( settings = Settings(settings) if settings.getbool("LOG_STDOUT"): - sys.stdout = StreamLogger(logging.getLogger("stdout")) # type: ignore[assignment] + sys.stdout = StreamLogger(logging.getLogger("stdout")) if install_root_handler: install_scrapy_root_handler(settings) @@ -240,7 +231,7 @@ class LogCounterHandler(logging.Handler): def logformatter_adapter( logkws: LogFormatterResult, -) -> Tuple[int, str, Union[Dict[str, Any], Tuple[Any, ...]]]: +) -> tuple[int, str, Union[dict[str, Any], tuple[Any, ...]]]: """ Helper that takes the dictionary output from the methods in LogFormatter and adapts it into a tuple of positional arguments for logger.log calls, @@ -251,7 +242,7 @@ def logformatter_adapter( message = logkws.get("msg") or "" # NOTE: This also handles 'args' being an empty dict, that case doesn't # play well in logger.log calls - args = cast(Dict[str, Any], logkws) if not logkws.get("args") else logkws["args"] + args = cast(dict[str, Any], logkws) if not logkws.get("args") else logkws["args"] return (level, message, args) @@ -259,7 +250,7 @@ def logformatter_adapter( class SpiderLoggerAdapter(logging.LoggerAdapter): def process( self, msg: str, kwargs: MutableMapping[str, Any] - ) -> Tuple[str, MutableMapping[str, Any]]: + ) -> tuple[str, MutableMapping[str, Any]]: """Method that augments logging with additional 'extra' data""" if isinstance(kwargs.get("extra"), MutableMapping): kwargs["extra"].update(self.extra) diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index 3c787e50f..e5e00512a 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -9,31 +9,19 @@ import os import re import warnings from collections import deque +from collections.abc import Iterable from contextlib import contextmanager from functools import partial from importlib import import_module from pkgutil import iter_modules -from typing import ( - IO, - TYPE_CHECKING, - Any, - Callable, - Deque, - Iterable, - Iterator, - List, - Optional, - Type, - TypeVar, - Union, - cast, -) +from typing import IO, TYPE_CHECKING, Any, Optional, TypeVar, Union, cast from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.item import Item from scrapy.utils.datatypes import LocalWeakReferencedCache if TYPE_CHECKING: + from collections.abc import Callable, Iterator from types import ModuleType from scrapy import Spider @@ -91,7 +79,7 @@ def load_object(path: Union[str, Callable[..., Any]]) -> Any: return obj -def walk_modules(path: str) -> List[ModuleType]: +def walk_modules(path: str) -> list[ModuleType]: """Loads a module and all its submodules from the given module path and returns them. If *any* module throws an exception while importing, that exception is thrown back. @@ -99,7 +87,7 @@ def walk_modules(path: str) -> List[ModuleType]: For example: walk_modules('scrapy.utils') """ - mods: List[ModuleType] = [] + mods: list[ModuleType] = [] mod = import_module(path) mods.append(mod) if hasattr(mod, "__path__"): @@ -186,7 +174,7 @@ def create_instance(objcls, settings, crawler, *args, **kwargs): def build_from_crawler( - objcls: Type[T], crawler: Crawler, /, *args: Any, **kwargs: Any + objcls: type[T], crawler: Crawler, /, *args: Any, **kwargs: Any ) -> T: """Construct a class instance using its ``from_crawler`` constructor. @@ -209,7 +197,7 @@ def build_from_crawler( def build_from_settings( - objcls: Type[T], settings: BaseSettings, /, *args: Any, **kwargs: Any + objcls: type[T], settings: BaseSettings, /, *args: Any, **kwargs: Any ) -> T: """Construct a class instance using its ``from_settings`` constructor. @@ -250,7 +238,7 @@ def walk_callable(node: ast.AST) -> Iterable[ast.AST]: """Similar to ``ast.walk``, but walks only function body and skips nested functions defined within the node. """ - todo: Deque[ast.AST] = deque([node]) + todo: deque[ast.AST] = deque([node]) walked_func_def = False while todo: node = todo.popleft() diff --git a/scrapy/utils/ossignal.py b/scrapy/utils/ossignal.py index 5985a847e..cff5eb629 100644 --- a/scrapy/utils/ossignal.py +++ b/scrapy/utils/ossignal.py @@ -1,13 +1,14 @@ import signal +from collections.abc import Callable from types import FrameType -from typing import Any, Callable, Dict, Optional, Union +from typing import Any, Optional, Union # copy of _HANDLER from typeshed/stdlib/signal.pyi SignalHandlerT = Union[ Callable[[int, Optional[FrameType]], Any], int, signal.Handlers, None ] -signal_names: Dict[int, str] = {} +signal_names: dict[int, str] = {} for signame in dir(signal): if signame.startswith("SIG") and not signame.startswith("SIG_"): signum = getattr(signal, signame) diff --git a/scrapy/utils/project.py b/scrapy/utils/project.py index efb6af299..c9e5eb857 100644 --- a/scrapy/utils/project.py +++ b/scrapy/utils/project.py @@ -1,5 +1,3 @@ -from __future__ import annotations - import os import warnings from importlib import import_module diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index f56950fdd..91c5d67f5 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -4,36 +4,22 @@ This module contains essential stuff that should've come with Python itself ;) from __future__ import annotations -import collections.abc import gc import inspect import re import sys import weakref +from collections.abc import AsyncIterable, Iterable, Mapping from functools import partial, wraps from itertools import chain -from typing import ( - TYPE_CHECKING, - Any, - AsyncIterable, - AsyncIterator, - Callable, - Dict, - Iterable, - Iterator, - List, - Mapping, - Optional, - Pattern, - Tuple, - TypeVar, - Union, - overload, -) +from typing import TYPE_CHECKING, Any, Optional, TypeVar, Union, overload from scrapy.utils.asyncgen import as_async_generator if TYPE_CHECKING: + from collections.abc import AsyncIterator, Callable, Iterator + from re import Pattern + # typing.Concatenate and typing.ParamSpec require Python 3.10 from typing_extensions import Concatenate, ParamSpec @@ -44,7 +30,7 @@ _KT = TypeVar("_KT") _VT = TypeVar("_VT") -def flatten(x: Iterable[Any]) -> List[Any]: +def flatten(x: Iterable[Any]) -> list[Any]: """flatten(sequence) -> list Returns a single, flat list which contains all elements retrieved @@ -99,10 +85,10 @@ def is_listlike(x: Any) -> bool: return hasattr(x, "__iter__") and not isinstance(x, (str, bytes)) -def unique(list_: Iterable[_T], key: Callable[[_T], Any] = lambda x: x) -> List[_T]: +def unique(list_: Iterable[_T], key: Callable[[_T], Any] = lambda x: x) -> list[_T]: """efficient function to uniquify a list preserving item order""" seen = set() - result: List[_T] = [] + result: list[_T] = [] for item in list_: seenkey = key(item) if seenkey in seen: @@ -147,7 +133,7 @@ def to_bytes( def re_rsearch( pattern: Union[str, Pattern[str]], text: str, chunk_size: int = 1024 -) -> Optional[Tuple[int, int]]: +) -> Optional[tuple[int, int]]: """ This function does a reverse search in a text using a regular expression given in the attribute 'pattern'. @@ -161,7 +147,7 @@ def re_rsearch( the start position of the match, and the ending (regarding the entire text). """ - def _chunk_iter() -> Iterable[Tuple[str, int]]: + def _chunk_iter() -> Iterable[tuple[str, int]]: offset = len(text) while True: offset -= chunk_size * 1024 @@ -215,12 +201,12 @@ def binary_is_text(data: bytes) -> bool: return all(c not in _BINARYCHARS for c in data) -def get_func_args(func: Callable[..., Any], stripself: bool = False) -> List[str]: +def get_func_args(func: Callable[..., Any], stripself: bool = False) -> list[str]: """Return the argument name list of a callable object""" if not callable(func): raise TypeError(f"func must be callable, got '{type(func).__name__}'") - args: List[str] = [] + args: list[str] = [] try: sig = inspect.signature(func) except ValueError: @@ -245,7 +231,7 @@ def get_func_args(func: Callable[..., Any], stripself: bool = False) -> List[str return args -def get_spec(func: Callable[..., Any]) -> Tuple[List[str], Dict[str, Any]]: +def get_spec(func: Callable[..., Any]) -> tuple[list[str], dict[str, Any]]: """Returns (args, kwargs) tuple for a function >>> import re >>> get_spec(re.match) @@ -274,7 +260,7 @@ def get_spec(func: Callable[..., Any]) -> Tuple[List[str], Dict[str, Any]]: else: raise TypeError(f"{type(func)} is not callable") - defaults: Tuple[Any, ...] = spec.defaults or () + defaults: tuple[Any, ...] = spec.defaults or () firstdefault = len(spec.args) - len(defaults) args = spec.args[:firstdefault] @@ -283,7 +269,7 @@ def get_spec(func: Callable[..., Any]) -> Tuple[List[str], Dict[str, Any]]: def equal_attributes( - obj1: Any, obj2: Any, attributes: Optional[List[Union[str, Callable[[Any], Any]]]] + obj1: Any, obj2: Any, attributes: Optional[list[Union[str, Callable[[Any], Any]]]] ) -> bool: """Compare two objects attributes""" # not attributes given return False by default @@ -303,7 +289,7 @@ def equal_attributes( @overload -def without_none_values(iterable: Mapping[_KT, _VT]) -> Dict[_KT, _VT]: ... +def without_none_values(iterable: Mapping[_KT, _VT]) -> dict[_KT, _VT]: ... @overload @@ -312,13 +298,13 @@ def without_none_values(iterable: Iterable[_KT]) -> Iterable[_KT]: ... def without_none_values( iterable: Union[Mapping[_KT, _VT], Iterable[_KT]] -) -> Union[Dict[_KT, _VT], Iterable[_KT]]: +) -> Union[dict[_KT, _VT], Iterable[_KT]]: """Return a copy of ``iterable`` with all ``None`` entries removed. If ``iterable`` is a mapping, return a dictionary where all pairs that have value ``None`` have been removed. """ - if isinstance(iterable, collections.abc.Mapping): + if isinstance(iterable, Mapping): return {k: v for k, v in iterable.items() if v is not None} else: # the iterable __init__ must take another iterable diff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.py index a627db601..ed2fb5959 100644 --- a/scrapy/utils/reactor.py +++ b/scrapy/utils/reactor.py @@ -3,18 +3,7 @@ from __future__ import annotations import asyncio import sys from contextlib import suppress -from typing import ( - TYPE_CHECKING, - Any, - Callable, - Dict, - Generic, - List, - Optional, - Tuple, - Type, - TypeVar, -) +from typing import TYPE_CHECKING, Any, Generic, Optional, TypeVar from warnings import catch_warnings, filterwarnings, warn from twisted.internet import asyncioreactor, error @@ -25,6 +14,7 @@ from scrapy.utils.misc import load_object if TYPE_CHECKING: from asyncio import AbstractEventLoop, AbstractEventLoopPolicy + from collections.abc import Callable from twisted.internet.protocol import ServerFactory from twisted.internet.tcp import Port @@ -37,7 +27,7 @@ if TYPE_CHECKING: _T = TypeVar("_T") -def listen_tcp(portrange: List[int], host: str, factory: ServerFactory) -> Port: # type: ignore[return] +def listen_tcp(portrange: list[int], host: str, factory: ServerFactory) -> Port: # type: ignore[return] """Like reactor.listenTCP but tries different ports in a range.""" from twisted.internet import reactor @@ -62,8 +52,8 @@ class CallLaterOnce(Generic[_T]): def __init__(self, func: Callable[_P, _T], *a: _P.args, **kw: _P.kwargs): self._func: Callable[_P, _T] = func - self._a: Tuple[Any, ...] = a - self._kw: Dict[str, Any] = kw + self._a: tuple[Any, ...] = a + self._kw: dict[str, Any] = kw self._call: Optional[DelayedCall] = None def schedule(self, delay: float = 0) -> None: @@ -142,7 +132,7 @@ def _get_asyncio_event_loop() -> AbstractEventLoop: def set_asyncio_event_loop(event_loop_path: Optional[str]) -> AbstractEventLoop: """Sets and returns the event loop with specified import path.""" if event_loop_path is not None: - event_loop_class: Type[AbstractEventLoop] = load_object(event_loop_path) + event_loop_class: type[AbstractEventLoop] = load_object(event_loop_path) event_loop = event_loop_class() asyncio.set_event_loop(event_loop) else: diff --git a/scrapy/utils/request.py b/scrapy/utils/request.py index 99ca3b7a0..052a3721a 100644 --- a/scrapy/utils/request.py +++ b/scrapy/utils/request.py @@ -8,18 +8,7 @@ from __future__ import annotations import hashlib import json import warnings -from typing import ( - TYPE_CHECKING, - Any, - Dict, - Iterable, - List, - Optional, - Protocol, - Tuple, - Type, - Union, -) +from typing import TYPE_CHECKING, Any, Optional, Protocol, Union from urllib.parse import urlunparse from weakref import WeakKeyDictionary @@ -33,6 +22,8 @@ from scrapy.utils.misc import load_object from scrapy.utils.python import to_bytes, to_unicode if TYPE_CHECKING: + from collections.abc import Iterable + # typing.Self requires Python 3.11 from typing_extensions import Self @@ -47,7 +38,7 @@ def _serialize_headers(headers: Iterable[bytes], request: Request) -> Iterable[b _fingerprint_cache: WeakKeyDictionary[ - Request, Dict[Tuple[Optional[Tuple[bytes, ...]], bool], bytes] + Request, dict[tuple[Optional[tuple[bytes, ...]], bool], bytes] ] _fingerprint_cache = WeakKeyDictionary() @@ -88,7 +79,7 @@ def fingerprint( If you want to include them, set the keep_fragments argument to True (for instance when handling requests with a headless browser). """ - processed_include_headers: Optional[Tuple[bytes, ...]] = None + processed_include_headers: Optional[tuple[bytes, ...]] = None if include_headers: processed_include_headers = tuple( to_bytes(h.lower()) for h in sorted(include_headers) @@ -98,7 +89,7 @@ def fingerprint( if cache_key not in cache: # To decode bytes reliably (JSON does not support bytes), regardless of # character encoding, we use bytes.hex() - headers: Dict[str, List[str]] = {} + headers: dict[str, list[str]] = {} if processed_include_headers: for header in processed_include_headers: if header in request.headers: @@ -194,13 +185,13 @@ def referer_str(request: Request) -> Optional[str]: return to_unicode(referrer, errors="replace") -def request_from_dict(d: Dict[str, Any], *, spider: Optional[Spider] = None) -> Request: +def request_from_dict(d: dict[str, Any], *, spider: Optional[Spider] = None) -> Request: """Create a :class:`~scrapy.Request` object from a dict. If a spider is given, it will try to resolve the callbacks looking at the spider for methods with the same name. """ - request_cls: Type[Request] = load_object(d["_class"]) if "_class" in d else Request + request_cls: type[Request] = load_object(d["_class"]) if "_class" in d else Request kwargs = {key: value for key, value in d.items() if key in request_cls.attributes} if d.get("callback") and spider: kwargs["callback"] = _get_method(spider, d["callback"]) diff --git a/scrapy/utils/response.py b/scrapy/utils/response.py index 320059b3a..0ca9d07a4 100644 --- a/scrapy/utils/response.py +++ b/scrapy/utils/response.py @@ -9,7 +9,7 @@ import os import re import tempfile import webbrowser -from typing import TYPE_CHECKING, Any, Callable, Iterable, Tuple, Union +from typing import TYPE_CHECKING, Any, Union from weakref import WeakKeyDictionary from twisted.web import http @@ -18,6 +18,8 @@ from w3lib import html from scrapy.utils.python import to_bytes, to_unicode if TYPE_CHECKING: + from collections.abc import Callable, Iterable + from scrapy.http import Response, TextResponse _baseurl_cache: WeakKeyDictionary[Response, str] = WeakKeyDictionary() @@ -34,14 +36,14 @@ def get_base_url(response: TextResponse) -> str: _metaref_cache: WeakKeyDictionary[ - Response, Union[Tuple[None, None], Tuple[float, str]] + Response, Union[tuple[None, None], tuple[float, str]] ] = WeakKeyDictionary() def get_meta_refresh( response: TextResponse, ignore_tags: Iterable[str] = ("script", "noscript"), -) -> Union[Tuple[None, None], Tuple[float, str]]: +) -> Union[tuple[None, None], tuple[float, str]]: """Parse the http-equiv refresh parameter from the given response""" if response not in _metaref_cache: text = response.text[0:4096] diff --git a/scrapy/utils/signal.py b/scrapy/utils/signal.py index 4310c1d56..c1d3bfffb 100644 --- a/scrapy/utils/signal.py +++ b/scrapy/utils/signal.py @@ -2,10 +2,9 @@ from __future__ import annotations -import collections.abc import logging +from collections.abc import Sequence from typing import Any as TypingAny -from typing import List, Tuple from pydispatch.dispatcher import ( Anonymous, @@ -30,19 +29,15 @@ def send_catch_log( sender: TypingAny = Anonymous, *arguments: TypingAny, **named: TypingAny, -) -> List[Tuple[TypingAny, TypingAny]]: +) -> list[tuple[TypingAny, TypingAny]]: """Like pydispatcher.robust.sendRobust but it also logs errors and returns Failures instead of exceptions. """ dont_log = named.pop("dont_log", ()) - dont_log = ( - tuple(dont_log) - if isinstance(dont_log, collections.abc.Sequence) - else (dont_log,) - ) + dont_log = tuple(dont_log) if isinstance(dont_log, Sequence) else (dont_log,) dont_log += (StopDownload,) spider = named.get("spider", None) - responses: List[Tuple[TypingAny, TypingAny]] = [] + responses: list[tuple[TypingAny, TypingAny]] = [] for receiver in liveReceivers(getAllReceivers(sender, signal)): result: TypingAny try: @@ -76,7 +71,7 @@ def send_catch_log_deferred( sender: TypingAny = Anonymous, *arguments: TypingAny, **named: TypingAny, -) -> Deferred[List[Tuple[TypingAny, TypingAny]]]: +) -> Deferred[list[tuple[TypingAny, TypingAny]]]: """Like send_catch_log but supports returning deferreds on signal handlers. Returns a deferred that gets fired once all signal handlers deferreds were fired. @@ -94,14 +89,14 @@ def send_catch_log_deferred( dont_log = named.pop("dont_log", None) spider = named.get("spider", None) - dfds: List[Deferred[Tuple[TypingAny, TypingAny]]] = [] + dfds: list[Deferred[tuple[TypingAny, TypingAny]]] = [] for receiver in liveReceivers(getAllReceivers(sender, signal)): d: Deferred[TypingAny] = maybeDeferred_coro( robustApply, receiver, signal=signal, sender=sender, *arguments, **named ) d.addErrback(logerror, receiver) # TODO https://pylint.readthedocs.io/en/latest/user_guide/messages/warning/cell-var-from-loop.html - d2: Deferred[Tuple[TypingAny, TypingAny]] = d.addBoth( + d2: Deferred[tuple[TypingAny, TypingAny]] = d.addBoth( lambda result: ( receiver, # pylint: disable=cell-var-from-loop # noqa: B023 result, @@ -109,7 +104,7 @@ def send_catch_log_deferred( ) dfds.append(d2) dl = DeferredList(dfds) - d3: Deferred[List[Tuple[TypingAny, TypingAny]]] = dl.addCallback( + d3: Deferred[list[tuple[TypingAny, TypingAny]]] = dl.addCallback( lambda out: [x[1] for x in out] ) return d3 diff --git a/scrapy/utils/sitemap.py b/scrapy/utils/sitemap.py index 7a91afe59..1f70fcf69 100644 --- a/scrapy/utils/sitemap.py +++ b/scrapy/utils/sitemap.py @@ -5,11 +5,16 @@ Note: The main purpose of this module is to provide support for the SitemapSpider, its API is subject to change without notice. """ -from typing import Any, Dict, Iterable, Iterator, Optional, Union +from __future__ import annotations + +from typing import TYPE_CHECKING, Any, Optional, Union from urllib.parse import urljoin import lxml.etree # nosec +if TYPE_CHECKING: + from collections.abc import Iterable, Iterator + class Sitemap: """Class to parse Sitemap (type=urlset) and Sitemap Index @@ -23,9 +28,9 @@ class Sitemap: rt = self._root.tag self.type = self._root.tag.split("}", 1)[1] if "}" in rt else rt - def __iter__(self) -> Iterator[Dict[str, Any]]: + def __iter__(self) -> Iterator[dict[str, Any]]: for elem in self._root.getchildren(): - d: Dict[str, Any] = {} + d: dict[str, Any] = {} for el in elem.getchildren(): tag = el.tag name = tag.split("}", 1)[1] if "}" in tag else tag diff --git a/scrapy/utils/spider.py b/scrapy/utils/spider.py index ce754fad3..02dbb2e90 100644 --- a/scrapy/utils/spider.py +++ b/scrapy/utils/spider.py @@ -2,24 +2,14 @@ from __future__ import annotations import inspect import logging -from typing import ( - TYPE_CHECKING, - Any, - AsyncGenerator, - Iterable, - Literal, - Optional, - Type, - TypeVar, - Union, - overload, -) +from typing import TYPE_CHECKING, Any, Literal, Optional, TypeVar, Union, overload from scrapy.spiders import Spider from scrapy.utils.defer import deferred_from_coro from scrapy.utils.misc import arg_to_iter if TYPE_CHECKING: + from collections.abc import AsyncGenerator, Iterable from types import CoroutineType, ModuleType from twisted.internet.defer import Deferred @@ -58,7 +48,7 @@ def iterate_spider_output( return arg_to_iter(deferred_from_coro(result)) -def iter_spider_classes(module: ModuleType) -> Iterable[Type[Spider]]: +def iter_spider_classes(module: ModuleType) -> Iterable[type[Spider]]: """Return an iterator over all spider classes defined in the given module that can be instantiated (i.e. which have name) """ @@ -80,10 +70,10 @@ def iter_spider_classes(module: ModuleType) -> Iterable[Type[Spider]]: def spidercls_for_request( spider_loader: SpiderLoader, request: Request, - default_spidercls: Type[Spider], + default_spidercls: type[Spider], log_none: bool = ..., log_multiple: bool = ..., -) -> Type[Spider]: ... +) -> type[Spider]: ... @overload @@ -93,7 +83,7 @@ def spidercls_for_request( default_spidercls: Literal[None], log_none: bool = ..., log_multiple: bool = ..., -) -> Optional[Type[Spider]]: ... +) -> Optional[type[Spider]]: ... @overload @@ -103,16 +93,16 @@ def spidercls_for_request( *, log_none: bool = ..., log_multiple: bool = ..., -) -> Optional[Type[Spider]]: ... +) -> Optional[type[Spider]]: ... def spidercls_for_request( spider_loader: SpiderLoader, request: Request, - default_spidercls: Optional[Type[Spider]] = None, + default_spidercls: Optional[type[Spider]] = None, log_none: bool = False, log_multiple: bool = False, -) -> Optional[Type[Spider]]: +) -> Optional[type[Spider]]: """Return a spider class that handles the given Request. This will look for the spiders that can handle the given request (using diff --git a/scrapy/utils/ssl.py b/scrapy/utils/ssl.py index 95611ebd9..2c3a259c1 100644 --- a/scrapy/utils/ssl.py +++ b/scrapy/utils/ssl.py @@ -20,7 +20,7 @@ def x509name_to_string(x509name: X509Name) -> str: # from OpenSSL.crypto.X509Name.__repr__ result_buffer: Any = pyOpenSSLutil.ffi.new("char[]", 512) pyOpenSSLutil.lib.X509_NAME_oneline( - x509name._name, result_buffer, len(result_buffer) # type: ignore[attr-defined] + x509name._name, result_buffer, len(result_buffer) ) return ffi_buf_to_string(result_buffer) diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py index 30f235592..860a2e3dd 100644 --- a/scrapy/utils/test.py +++ b/scrapy/utils/test.py @@ -9,17 +9,7 @@ import os from importlib import import_module from pathlib import Path from posixpath import split -from typing import ( - TYPE_CHECKING, - Any, - Awaitable, - Dict, - List, - Optional, - Tuple, - Type, - TypeVar, -) +from typing import TYPE_CHECKING, Any, Optional, TypeVar from unittest import TestCase, mock from twisted.trial.unittest import SkipTest @@ -29,6 +19,8 @@ from scrapy.crawler import Crawler from scrapy.utils.boto import is_botocore_available if TYPE_CHECKING: + from collections.abc import Awaitable + from twisted.internet.defer import Deferred from twisted.web.client import Response as TxResponse @@ -48,7 +40,7 @@ def skip_if_no_boto() -> None: def get_gcs_content_and_delete( bucket: Any, path: str -) -> Tuple[bytes, List[Dict[str, str]], Any]: +) -> tuple[bytes, list[dict[str, str]], Any]: from google.cloud import storage client = storage.Client(project=os.environ.get("GCS_PROJECT_ID")) @@ -75,7 +67,7 @@ def get_ftp_content_and_delete( ftp.login(username, password) if use_active_mode: ftp.set_pasv(False) - ftp_data: List[bytes] = [] + ftp_data: list[bytes] = [] def buffer_data(data: bytes) -> None: ftp_data.append(data) @@ -92,8 +84,8 @@ class TestSpider(Spider): def get_crawler( - spidercls: Optional[Type[Spider]] = None, - settings_dict: Optional[Dict[str, Any]] = None, + spidercls: Optional[type[Spider]] = None, + settings_dict: Optional[dict[str, Any]] = None, prevent_warnings: bool = True, ) -> Crawler: """Return an unconfigured Crawler object. If settings_dict is given, it @@ -103,7 +95,7 @@ def get_crawler( from scrapy.crawler import CrawlerRunner # Set by default settings that prevent deprecation warnings. - settings: Dict[str, Any] = {} + settings: dict[str, Any] = {} settings.update(settings_dict or {}) runner = CrawlerRunner(settings) crawler = runner.create_crawler(spidercls or TestSpider) @@ -118,7 +110,7 @@ def get_pythonpath() -> str: return str(Path(scrapy_path).parent) + os.pathsep + os.environ.get("PYTHONPATH", "") -def get_testenv() -> Dict[str, str]: +def get_testenv() -> dict[str, str]: """Return a OS environment dict suitable to fork processes that need to import this installation of Scrapy, instead of a system installed one. """ @@ -143,7 +135,7 @@ def get_from_asyncio_queue(value: _T) -> Awaitable[_T]: return getter -def mock_google_cloud_storage() -> Tuple[Any, Any, Any]: +def mock_google_cloud_storage() -> tuple[Any, Any, Any]: """Creates autospec mocks for google-cloud-storage Client, Bucket and Blob classes and set their proper return values. """ diff --git a/scrapy/utils/testproc.py b/scrapy/utils/testproc.py index bb269a9f5..dfc823725 100644 --- a/scrapy/utils/testproc.py +++ b/scrapy/utils/testproc.py @@ -2,13 +2,15 @@ from __future__ import annotations import os import sys -from typing import TYPE_CHECKING, Iterable, List, Optional, Tuple, cast +from typing import TYPE_CHECKING, Optional, cast from twisted.internet.defer import Deferred from twisted.internet.error import ProcessTerminated from twisted.internet.protocol import ProcessProtocol if TYPE_CHECKING: + from collections.abc import Iterable + from twisted.python.failure import Failure @@ -36,8 +38,8 @@ class ProcessTest: return pp.deferred def _process_finished( - self, pp: TestProcessProtocol, cmd: List[str], check_code: bool - ) -> Tuple[int, bytes, bytes]: + self, pp: TestProcessProtocol, cmd: list[str], check_code: bool + ) -> tuple[int, bytes, bytes]: if pp.exitcode and check_code: msg = f"process {cmd} exit with code {pp.exitcode}" msg += f"\n>>> stdout <<<\n{pp.out.decode()}" diff --git a/scrapy/utils/trackref.py b/scrapy/utils/trackref.py index 9ff9a273f..5eec1c10f 100644 --- a/scrapy/utils/trackref.py +++ b/scrapy/utils/trackref.py @@ -9,19 +9,23 @@ and no performance penalty at all when disabled (as object_ref becomes just an alias to object in that case). """ +from __future__ import annotations + from collections import defaultdict from operator import itemgetter from time import time -from typing import TYPE_CHECKING, Any, DefaultDict, Iterable +from typing import TYPE_CHECKING, Any from weakref import WeakKeyDictionary if TYPE_CHECKING: + from collections.abc import Iterable + # typing.Self requires Python 3.11 from typing_extensions import Self NoneType = type(None) -live_refs: DefaultDict[type, WeakKeyDictionary] = defaultdict(WeakKeyDictionary) +live_refs: defaultdict[type, WeakKeyDictionary] = defaultdict(WeakKeyDictionary) class object_ref: @@ -29,7 +33,7 @@ class object_ref: __slots__ = () - def __new__(cls, *args: Any, **kwargs: Any) -> "Self": + def __new__(cls, *args: Any, **kwargs: Any) -> Self: obj = object.__new__(cls) live_refs[cls][obj] = time() return obj diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py index 9d97cb12f..41d268baa 100644 --- a/scrapy/utils/url.py +++ b/scrapy/utils/url.py @@ -6,8 +6,10 @@ Some of the functions that used to be imported from this module have been moved to the w3lib.url module. Always import those from there instead. """ +from __future__ import annotations + import re -from typing import TYPE_CHECKING, Iterable, Optional, Type, Union, cast +from typing import TYPE_CHECKING, Optional, Union, cast from urllib.parse import ParseResult, urldefrag, urlparse, urlunparse # scrapy.utils.url was moved to w3lib.url and import * ensures this @@ -18,6 +20,8 @@ from w3lib.url import _safe_chars, _unquotepath # noqa: F401 from scrapy.utils.python import to_unicode if TYPE_CHECKING: + from collections.abc import Iterable + from scrapy import Spider @@ -33,7 +37,7 @@ def url_is_from_any_domain(url: UrlT, domains: Iterable[str]) -> bool: return any((host == d) or (host.endswith(f".{d}")) for d in domains) -def url_is_from_spider(url: UrlT, spider: Type["Spider"]) -> bool: +def url_is_from_spider(url: UrlT, spider: type[Spider]) -> bool: """Return True if the url belongs to the given spider""" return url_is_from_any_domain( url, [spider.name] + list(getattr(spider, "allowed_domains", [])) diff --git a/scrapy/utils/versions.py b/scrapy/utils/versions.py index 42e5e9be4..4e9e29286 100644 --- a/scrapy/utils/versions.py +++ b/scrapy/utils/versions.py @@ -1,6 +1,5 @@ import platform import sys -from typing import List, Tuple import cryptography import cssselect @@ -13,7 +12,7 @@ import scrapy from scrapy.utils.ssl import get_openssl_version -def scrapy_components_versions() -> List[Tuple[str, str]]: +def scrapy_components_versions() -> list[tuple[str, str]]: lxml_version = ".".join(map(str, lxml.etree.LXML_VERSION)) libxml2_version = ".".join(map(str, lxml.etree.LIBXML_VERSION)) diff --git a/setup.py b/setup.py index 96eacef23..6cc1150a5 100644 --- a/setup.py +++ b/setup.py @@ -6,22 +6,21 @@ version = (Path(__file__).parent / "scrapy/VERSION").read_text("ascii").strip() install_requires = [ - "Twisted>=18.9.0", - "cryptography>=36.0.0", + "Twisted>=21.7.0", + "cryptography>=37.0.0", "cssselect>=0.9.1", "itemloaders>=1.0.1", "parsel>=1.5.0", - "pyOpenSSL>=21.0.0", + "pyOpenSSL>=22.0.0", "queuelib>=1.4.2", "service_identity>=18.1.0", "w3lib>=1.17.0", "zope.interface>=5.1.0", "protego>=0.1.15", "itemadapter>=0.1.0", - "setuptools", "packaging", "tldextract", - "lxml>=4.4.1", + "lxml>=4.6.0", "defusedxml>=0.7.1", ] extras_require = { @@ -59,7 +58,6 @@ setup( "Operating System :: OS Independent", "Programming Language :: Python", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", @@ -71,7 +69,7 @@ setup( "Topic :: Software Development :: Libraries :: Application Frameworks", "Topic :: Software Development :: Libraries :: Python Modules", ], - python_requires=">=3.8", + python_requires=">=3.9", install_requires=install_requires, extras_require=extras_require, ) diff --git a/tests/mocks/dummydbm.py b/tests/mocks/dummydbm.py index bde3de228..a7f7f1356 100644 --- a/tests/mocks/dummydbm.py +++ b/tests/mocks/dummydbm.py @@ -1,7 +1,7 @@ """DBM-like dummy module""" -import collections -from typing import Any, DefaultDict +from collections import defaultdict +from typing import Any class DummyDB(dict): @@ -14,7 +14,7 @@ class DummyDB(dict): error = KeyError -_DATABASES: DefaultDict[Any, DummyDB] = collections.defaultdict(DummyDB) +_DATABASES: defaultdict[Any, DummyDB] = defaultdict(DummyDB) def open(file, flag="r", mode=0o666): diff --git a/tests/mockserver.py b/tests/mockserver.py index 6ec46aa3d..f5c12787a 100644 --- a/tests/mockserver.py +++ b/tests/mockserver.py @@ -9,7 +9,7 @@ from pathlib import Path from shutil import rmtree from subprocess import PIPE, Popen from tempfile import mkdtemp -from typing import TYPE_CHECKING, Dict +from typing import TYPE_CHECKING from urllib.parse import urlencode from OpenSSL import SSL @@ -37,7 +37,7 @@ def getarg(request, name, default=None, type=None): return default -def get_mockserver_env() -> Dict[str, str]: +def get_mockserver_env() -> dict[str, str]: """Return a OS environment dict suitable to run mockserver processes.""" tests_path = Path(__file__).parent.parent diff --git a/tests/spiders.py b/tests/spiders.py index 743811893..5d5792858 100644 --- a/tests/spiders.py +++ b/tests/spiders.py @@ -346,6 +346,19 @@ class BrokenStartRequestsSpider(FollowAllSpider): yield from super().parse(response) +class StartRequestsItemSpider(FollowAllSpider): + def start_requests(self): + yield {"name": "test item"} + + +class StartRequestsGoodAndBadOutput(FollowAllSpider): + def start_requests(self): + yield {"a": "a"} + yield Request("data:,a") + yield "data:,b" + yield object() + + class SingleRequestSpider(MetaSpider): seed = None callback_func = None diff --git a/tests/test_addons.py b/tests/test_addons.py index f1b01bc5c..775f629b3 100644 --- a/tests/test_addons.py +++ b/tests/test_addons.py @@ -1,5 +1,5 @@ import itertools -from typing import Any, Dict +from typing import Any from unittest.mock import patch from twisted.internet.defer import inlineCallbacks @@ -17,7 +17,7 @@ class SimpleAddon: pass -def get_addon_cls(config: Dict[str, Any]) -> type: +def get_addon_cls(config: dict[str, Any]) -> type: class AddonWithConfig: def update_settings(self, settings: BaseSettings): settings.update(config, priority="addon") diff --git a/tests/test_commands.py b/tests/test_commands.py index a23b7f4a9..6ec7c21b0 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import argparse import inspect import json @@ -13,7 +15,7 @@ from shutil import copytree, rmtree from stat import S_IWRITE as ANYONE_WRITE_PERMISSION from tempfile import TemporaryFile, mkdtemp from threading import Timer -from typing import Dict, Iterator, Optional, Union +from typing import TYPE_CHECKING, Optional, Union from unittest import skipIf from pytest import mark @@ -27,6 +29,9 @@ from scrapy.utils.python import to_unicode from scrapy.utils.test import get_testenv from tests.test_crawler import ExceptionSpider, NoRequestsSpider +if TYPE_CHECKING: + from collections.abc import Iterator + class CommandSettings(unittest.TestCase): def setUp(self): @@ -194,7 +199,7 @@ class StartprojectTest(ProjectTest): def get_permissions_dict( path: Union[str, os.PathLike], renamings=None, ignore=None -) -> Dict[str, str]: +) -> dict[str, str]: def get_permissions(path: Path) -> str: return oct(path.stat().st_mode) diff --git a/tests/test_contracts.py b/tests/test_contracts.py index c9c12f0d8..d578b3af4 100644 --- a/tests/test_contracts.py +++ b/tests/test_contracts.py @@ -8,6 +8,7 @@ from scrapy import FormRequest from scrapy.contracts import Contract, ContractsManager from scrapy.contracts.default import ( CallbackKeywordArgumentsContract, + MetadataContract, ReturnsContract, ScrapesContract, UrlContract, @@ -29,6 +30,10 @@ class ResponseMock: url = "http://scrapy.org" +class ResponseMetaMock(ResponseMock): + meta = None + + class CustomSuccessContract(Contract): name = "custom_success_contract" @@ -195,6 +200,33 @@ class TestSpider(Spider): """ pass + def returns_request_meta(self, response): + """method which returns request + @url https://example.org + @meta {"cookiejar": "session1"} + @returns requests 1 + """ + return Request( + "https://example.org", meta=response.meta, callback=self.returns_item_meta + ) + + def returns_item_meta(self, response): + """method which returns item + @url http://scrapy.org + @meta {"key": "example"} + @returns items 1 1 + """ + return TestItem(name="example", url=response.url) + + def returns_error_missing_meta(self, response): + """method which depends of metadata be defined + + @url http://scrapy.org + @returns items 1 + """ + key = response.meta["key"] + yield {key: "value"} + class CustomContractSuccessSpider(Spider): name = "custom_contract_success_spider" @@ -224,6 +256,7 @@ class ContractsManagerTest(unittest.TestCase): contracts = [ UrlContract, CallbackKeywordArgumentsContract, + MetadataContract, ReturnsContract, ScrapesContract, CustomFormContract, @@ -328,6 +361,52 @@ class ContractsManagerTest(unittest.TestCase): request.callback(response, **request.cb_kwargs) self.should_error() + def test_meta(self): + spider = TestSpider() + + # extract contracts correctly + contracts = self.conman.extract_contracts(spider.returns_request_meta) + self.assertEqual(len(contracts), 3) + self.assertEqual( + frozenset(type(x) for x in contracts), + frozenset([UrlContract, MetadataContract, ReturnsContract]), + ) + + contracts = self.conman.extract_contracts(spider.returns_item_meta) + self.assertEqual(len(contracts), 3) + self.assertEqual( + frozenset(type(x) for x in contracts), + frozenset([UrlContract, MetadataContract, ReturnsContract]), + ) + + response = ResponseMetaMock() + + # returns_request + request = self.conman.from_method(spider.returns_request_meta, self.results) + assert request.meta["cookiejar"] == "session1" + response.meta = request.meta + request.callback(response) + assert response.meta["cookiejar"] == "session1" + self.should_succeed() + + response = ResponseMetaMock() + + # returns_item + request = self.conman.from_method(spider.returns_item_meta, self.results) + assert request.meta["key"] == "example" + response.meta = request.meta + request.callback(ResponseMetaMock) + assert response.meta["key"] == "example" + self.should_succeed() + + response = ResponseMetaMock() + + request = self.conman.from_method( + spider.returns_error_missing_meta, self.results + ) + request.callback(response) + self.should_error() + def test_returns(self): spider = TestSpider() response = ResponseMock() diff --git a/tests/test_crawl.py b/tests/test_crawl.py index 6cde4ed8c..125709571 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -1,5 +1,6 @@ import json import logging +import re import unittest from ipaddress import IPv4Address from socket import gethostbyname @@ -49,6 +50,8 @@ from tests.spiders import ( HeadersReceivedErrbackSpider, SimpleSpider, SingleRequestSpider, + StartRequestsGoodAndBadOutput, + StartRequestsItemSpider, ) @@ -184,6 +187,39 @@ class CrawlTestCase(TestCase): self.assertIsNotNone(record.exc_info) self.assertIs(record.exc_info[0], ZeroDivisionError) + @defer.inlineCallbacks + def test_start_requests_items(self): + with LogCapture("scrapy", level=logging.ERROR) as log: + crawler = get_crawler(StartRequestsItemSpider) + yield crawler.crawl(mockserver=self.mockserver) + + self.assertEqual(len(log.records), 0) + + @defer.inlineCallbacks + def test_start_requests_unsupported_output(self): + with LogCapture("scrapy", level=logging.ERROR) as log: + crawler = get_crawler(StartRequestsGoodAndBadOutput) + yield crawler.crawl(mockserver=self.mockserver) + + self.assertEqual(len(log.records), 2) + self.assertEqual( + log.records[0].msg, + ( + "Got 'data:,b' among start requests. Only requests and items " + "are supported. It will be ignored." + ), + ) + self.assertTrue( + re.match( + ( + r"^Got among start " + r"requests\. Only requests and items are supported\. It " + r"will be ignored\.$" + ), + log.records[1].msg, + ) + ) + @defer.inlineCallbacks def test_start_requests_laziness(self): settings = {"CONCURRENT_REQUESTS": 1} diff --git a/tests/test_crawler.py b/tests/test_crawler.py index c87e65758..69bfb7eb3 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -6,7 +6,6 @@ import subprocess import sys import warnings from pathlib import Path -from typing import List import pytest from packaging.version import parse as parse_version @@ -651,7 +650,7 @@ class ScriptRunnerMixin: script_dir: Path cwd = os.getcwd() - def get_script_args(self, script_name: str, *script_args: str) -> List[str]: + def get_script_args(self, script_name: str, *script_args: str) -> list[str]: script_path = self.script_dir / script_name return [sys.executable, str(script_path)] + list(script_args) diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 884491d01..f14a10a32 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -4,7 +4,7 @@ import shutil import sys from pathlib import Path from tempfile import mkdtemp, mkstemp -from typing import Optional, Type +from typing import Optional from unittest import SkipTest, mock from testfixtures import LogCapture @@ -218,7 +218,7 @@ class DuplicateHeaderResource(resource.Resource): class HttpTestCase(unittest.TestCase): scheme = "http" - download_handler_cls: Type = HTTPDownloadHandler + download_handler_cls: type = HTTPDownloadHandler # only used for HTTPS tests keyfile = "keys/localhost.key" @@ -428,7 +428,7 @@ class HttpTestCase(unittest.TestCase): class Http10TestCase(HttpTestCase): """HTTP 1.0 test case""" - download_handler_cls: Type = HTTP10DownloadHandler + download_handler_cls: type = HTTP10DownloadHandler def test_protocol(self): request = Request(self.getURL("host"), method="GET") @@ -445,7 +445,7 @@ class Https10TestCase(Http10TestCase): class Http11TestCase(HttpTestCase): """HTTP 1.1 test case""" - download_handler_cls: Type = HTTP11DownloadHandler + download_handler_cls: type = HTTP11DownloadHandler def test_download_without_maxsize_limit(self): request = Request(self.getURL("file")) @@ -645,7 +645,7 @@ class Https11InvalidDNSPattern(Https11TestCase): class Https11CustomCiphers(unittest.TestCase): scheme = "https" - download_handler_cls: Type = HTTP11DownloadHandler + download_handler_cls: type = HTTP11DownloadHandler keyfile = "keys/localhost.key" certfile = "keys/localhost.crt" @@ -740,7 +740,7 @@ class UriResource(resource.Resource): class HttpProxyTestCase(unittest.TestCase): - download_handler_cls: Type = HTTPDownloadHandler + download_handler_cls: type = HTTPDownloadHandler expected_http_proxy_request_body = b"http://example.com" def setUp(self): @@ -783,14 +783,14 @@ class HttpProxyTestCase(unittest.TestCase): class Http10ProxyTestCase(HttpProxyTestCase): - download_handler_cls: Type = HTTP10DownloadHandler + download_handler_cls: type = HTTP10DownloadHandler def test_download_with_proxy_https_noconnect(self): raise unittest.SkipTest("noconnect is not supported in HTTP10DownloadHandler") class Http11ProxyTestCase(HttpProxyTestCase): - download_handler_cls: Type = HTTP11DownloadHandler + download_handler_cls: type = HTTP11DownloadHandler @defer.inlineCallbacks def test_download_with_proxy_https_timeout(self): @@ -845,7 +845,7 @@ class S3AnonTestCase(unittest.TestCase): class S3TestCase(unittest.TestCase): - download_handler_cls: Type = S3DownloadHandler + download_handler_cls: type = S3DownloadHandler # test use same example keys than amazon developer guide # http://s3.amazonaws.com/awsdocs/S3/20060301/s3-dg-20060301.pdf diff --git a/tests/test_engine.py b/tests/test_engine.py index 86526420f..2ebc0b5e4 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -499,7 +499,6 @@ def test_request_scheduled_signal(caplog): assert scheduler.enqueued == [ keep_request ], f"{scheduler.enqueued!r} != [{keep_request!r}]" - assert "dropped request " in caplog.text crawler.signals.disconnect(signal_handler, request_scheduled) diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py index 7ea3fe8c9..1f998de1a 100644 --- a/tests/test_http2_client_protocol.py +++ b/tests/test_http2_client_protocol.py @@ -8,7 +8,7 @@ import string from ipaddress import IPv4Address from pathlib import Path from tempfile import mkdtemp -from typing import TYPE_CHECKING, Dict +from typing import TYPE_CHECKING from unittest import mock, skipIf from urllib.parse import urlencode @@ -152,7 +152,7 @@ class QueryParams(LeafResource): request.setHeader("Content-Type", "application/json; charset=UTF-8") request.setHeader("Content-Encoding", "UTF-8") - query_params: Dict[str, str] = {} + query_params: dict[str, str] = {} assert request.args is not None for k, v in request.args.items(): query_params[str(k, "utf-8")] = str(v[0], "utf-8") diff --git a/tests/test_http_request.py b/tests/test_http_request.py index 7ce73e6ff..d0fb17f1f 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -3,7 +3,7 @@ import re import unittest import warnings import xmlrpc.client -from typing import Any, Dict, List +from typing import Any from unittest import mock from urllib.parse import parse_qs, unquote_to_bytes @@ -23,8 +23,8 @@ from scrapy.utils.python import to_bytes, to_unicode class RequestTest(unittest.TestCase): request_class = Request default_method = "GET" - default_headers: Dict[bytes, List[bytes]] = {} - default_meta: Dict[str, Any] = {} + default_headers: dict[bytes, list[bytes]] = {} + default_meta: dict[str, Any] = {} def test_init(self): # Request requires url in the __init__ method diff --git a/tests/test_pipeline_crawl.py b/tests/test_pipeline_crawl.py index cd3442dd4..83e22b070 100644 --- a/tests/test_pipeline_crawl.py +++ b/tests/test_pipeline_crawl.py @@ -1,7 +1,7 @@ import shutil from pathlib import Path from tempfile import mkdtemp -from typing import Optional, Set +from typing import Optional from testfixtures import LogCapture from twisted.internet import defer @@ -57,7 +57,7 @@ class FileDownloadCrawlTestCase(TestCase): store_setting_key = "FILES_STORE" media_key = "files" media_urls_key = "file_urls" - expected_checksums: Optional[Set[str]] = { + expected_checksums: Optional[set[str]] = { "5547178b89448faf0015a13f904c936e", "c2281c83670e31d8aaab7cb642b824db", "ed3f6538dc15d4d9179dae57319edc5f", diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py index 0babde4d9..6ce7fc059 100644 --- a/tests/test_pipeline_files.py +++ b/tests/test_pipeline_files.py @@ -7,7 +7,6 @@ from io import BytesIO from pathlib import Path from shutil import rmtree from tempfile import mkdtemp -from typing import Dict, List from unittest import mock from urllib.parse import urlparse @@ -309,11 +308,11 @@ class FilesPipelineTestCaseFieldsDataClass( class FilesPipelineTestAttrsItem: name = attr.ib(default="") # default fields - file_urls: List[str] = attr.ib(default=lambda: []) - files: List[Dict[str, str]] = attr.ib(default=lambda: []) + file_urls: list[str] = attr.ib(default=lambda: []) + files: list[dict[str, str]] = attr.ib(default=lambda: []) # overridden fields - custom_file_urls: List[str] = attr.ib(default=lambda: []) - custom_files: List[Dict[str, str]] = attr.ib(default=lambda: []) + custom_file_urls: list[str] = attr.ib(default=lambda: []) + custom_files: list[dict[str, str]] = attr.ib(default=lambda: []) class FilesPipelineTestCaseFieldsAttrsItem( diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py index 7d7c78920..296a6fae0 100644 --- a/tests/test_pipeline_images.py +++ b/tests/test_pipeline_images.py @@ -5,7 +5,7 @@ import random import warnings from shutil import rmtree from tempfile import mkdtemp -from typing import Dict, List, Optional +from typing import Optional from unittest.mock import patch import attr @@ -406,11 +406,11 @@ class ImagesPipelineTestCaseFieldsDataClass( class ImagesPipelineTestAttrsItem: name = attr.ib(default="") # default fields - image_urls: List[str] = attr.ib(default=lambda: []) - images: List[Dict[str, str]] = attr.ib(default=lambda: []) + image_urls: list[str] = attr.ib(default=lambda: []) + images: list[dict[str, str]] = attr.ib(default=lambda: []) # overridden fields - custom_image_urls: List[str] = attr.ib(default=lambda: []) - custom_images: List[Dict[str, str]] = attr.ib(default=lambda: []) + custom_image_urls: list[str] = attr.ib(default=lambda: []) + custom_images: list[dict[str, str]] = attr.ib(default=lambda: []) class ImagesPipelineTestCaseFieldsAttrsItem( diff --git a/tests/test_request_cb_kwargs.py b/tests/test_request_cb_kwargs.py index 7299972f6..8c0e5764a 100644 --- a/tests/test_request_cb_kwargs.py +++ b/tests/test_request_cb_kwargs.py @@ -1,5 +1,3 @@ -from typing import List - from testfixtures import LogCapture from twisted.internet import defer from twisted.trial.unittest import TestCase @@ -64,7 +62,7 @@ class KeywordArgumentsSpider(MockServerSpider): }, } - checks: List[bool] = [] + checks: list[bool] = [] def start_requests(self): data = {"key": "value", "number": 123, "callback": "some_callback"} diff --git a/tests/test_scheduler_base.py b/tests/test_scheduler_base.py index 5db2e4e50..4fd293ec7 100644 --- a/tests/test_scheduler_base.py +++ b/tests/test_scheduler_base.py @@ -1,4 +1,4 @@ -from typing import Dict, Optional +from typing import Optional from unittest import TestCase from urllib.parse import urljoin @@ -20,7 +20,7 @@ URLS = [urljoin("https://example.org", p) for p in PATHS] class MinimalScheduler: def __init__(self) -> None: - self.requests: Dict[bytes, Request] = {} + self.requests: dict[bytes, Request] = {} def has_pending_requests(self) -> bool: return bool(self.requests) diff --git a/tests/test_settings/__init__.py b/tests/test_settings/__init__.py index 9ee248538..503c29e32 100644 --- a/tests/test_settings/__init__.py +++ b/tests/test_settings/__init__.py @@ -105,9 +105,10 @@ class BaseSettingsTest(unittest.TestCase): def test_set_calls_settings_attributes_methods_on_update(self): attr = SettingsAttribute("value", 10) - with mock.patch.object(attr, "__setattr__") as mock_setattr, mock.patch.object( - attr, "set" - ) as mock_set: + with ( + mock.patch.object(attr, "__setattr__") as mock_setattr, + mock.patch.object(attr, "set") as mock_set, + ): self.settings.attributes = {"TEST_OPTION": attr} for priority in (0, 10, 20): diff --git a/tests/test_spiderloader/__init__.py b/tests/test_spiderloader/__init__.py index f950739f2..32699d837 100644 --- a/tests/test_spiderloader/__init__.py +++ b/tests/test_spiderloader/__init__.py @@ -4,6 +4,7 @@ import tempfile import warnings from pathlib import Path from tempfile import mkdtemp +from unittest import mock from twisted.trial import unittest from zope.interface.verify import verifyObject @@ -136,6 +137,33 @@ class SpiderLoaderTest(unittest.TestCase): spiders = spider_loader.list() self.assertEqual(spiders, []) + def test_syntax_error_exception(self): + module = "tests.test_spiderloader.test_spiders.spider1" + with mock.patch.object(SpiderLoader, "_load_spiders") as m: + m.side_effect = SyntaxError + settings = Settings({"SPIDER_MODULES": [module]}) + self.assertRaises(SyntaxError, SpiderLoader.from_settings, settings) + + def test_syntax_error_warning(self): + with warnings.catch_warnings(record=True) as w, mock.patch.object( + SpiderLoader, "_load_spiders" + ) as m: + m.side_effect = SyntaxError + module = "tests.test_spiderloader.test_spiders.spider1" + settings = Settings( + {"SPIDER_MODULES": [module], "SPIDER_LOADER_WARN_ONLY": True} + ) + spider_loader = SpiderLoader.from_settings(settings) + if str(w[0].message).startswith("_SixMetaPathImporter"): + # needed on 3.10 because of https://github.com/benjaminp/six/issues/349, + # at least until all six versions we can import (including botocore.vendored.six) + # are updated to 1.16.0+ + w.pop(0) + self.assertIn("Could not load spiders from module", str(w[0].message)) + + spiders = spider_loader.list() + self.assertEqual(spiders, []) + class DuplicateSpiderNameLoaderTest(unittest.TestCase): def setUp(self): diff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.py index 38ca8d950..41228b5f2 100644 --- a/tests/test_spidermiddleware.py +++ b/tests/test_spidermiddleware.py @@ -1,5 +1,5 @@ -import collections.abc -from typing import Optional +from collections.abc import AsyncIterator, Iterable +from typing import Optional, Union from unittest import mock from testfixtures import LogCapture @@ -112,7 +112,7 @@ class BaseAsyncSpiderMiddlewareTestCase(SpiderMiddlewareTestCase): Should work for process_spider_output and, when it's supported, process_start_requests. """ - ITEM_TYPE: type + ITEM_TYPE: Union[type, tuple] RESULT_COUNT = 3 # to simplify checks, let everything return 3 objects @staticmethod @@ -147,7 +147,7 @@ class BaseAsyncSpiderMiddlewareTestCase(SpiderMiddlewareTestCase): result = yield self._get_middleware_result( *mw_classes, start_index=start_index ) - self.assertIsInstance(result, collections.abc.Iterable) + self.assertIsInstance(result, Iterable) result_list = list(result) self.assertEqual(len(result_list), self.RESULT_COUNT) self.assertIsInstance(result_list[0], self.ITEM_TYPE) @@ -161,7 +161,7 @@ class BaseAsyncSpiderMiddlewareTestCase(SpiderMiddlewareTestCase): result = yield self._get_middleware_result( *mw_classes, start_index=start_index ) - self.assertIsInstance(result, collections.abc.AsyncIterator) + self.assertIsInstance(result, AsyncIterator) result_list = yield deferred_from_coro(collect_asyncgen(result)) self.assertEqual(len(result_list), self.RESULT_COUNT) self.assertIsInstance(result_list[0], self.ITEM_TYPE) @@ -328,12 +328,13 @@ class ProcessStartRequestsSimpleMiddleware: class ProcessStartRequestsSimple(BaseAsyncSpiderMiddlewareTestCase): """process_start_requests tests for simple start_requests""" - ITEM_TYPE = Request + ITEM_TYPE = (Request, dict) MW_SIMPLE = ProcessStartRequestsSimpleMiddleware def _start_requests(self): - for i in range(3): + for i in range(2): yield Request(f"https://example.com/{i}", dont_filter=True) + yield {"name": "test item"} @defer.inlineCallbacks def _get_middleware_result(self, *mw_classes, start_index: Optional[int] = None): diff --git a/tests/test_spidermiddleware_httperror.py b/tests/test_spidermiddleware_httperror.py index 044455415..01a2b4bb4 100644 --- a/tests/test_spidermiddleware_httperror.py +++ b/tests/test_spidermiddleware_httperror.py @@ -1,5 +1,4 @@ import logging -from typing import Set from unittest import TestCase from testfixtures import LogCapture @@ -17,7 +16,7 @@ from tests.spiders import MockServerSpider class _HttpErrorSpider(MockServerSpider): name = "httperror" - bypass_status_codes: Set[int] = set() + bypass_status_codes: set[int] = set() def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.py index 5797edfbd..e73e7ff4c 100644 --- a/tests/test_spidermiddleware_referer.py +++ b/tests/test_spidermiddleware_referer.py @@ -1,5 +1,5 @@ import warnings -from typing import Any, Dict, List, Optional, Tuple +from typing import Any, Optional from unittest import TestCase from urllib.parse import urlparse @@ -32,10 +32,10 @@ from scrapy.spiders import Spider class TestRefererMiddleware(TestCase): - req_meta: Dict[str, Any] = {} - resp_headers: Dict[str, str] = {} - settings: Dict[str, Any] = {} - scenarii: List[Tuple[str, str, Optional[bytes]]] = [ + req_meta: dict[str, Any] = {} + resp_headers: dict[str, str] = {} + settings: dict[str, Any] = {} + scenarii: list[tuple[str, str, Optional[bytes]]] = [ ("http://scrapytest.org", "http://scrapytest.org/", b"http://scrapytest.org"), ] @@ -65,7 +65,7 @@ class MixinDefault: with some additional filtering of s3:// """ - scenarii: List[Tuple[str, str, Optional[bytes]]] = [ + scenarii: list[tuple[str, str, Optional[bytes]]] = [ ("https://example.com/", "https://scrapy.org/", b"https://example.com/"), ("http://example.com/", "http://scrapy.org/", b"http://example.com/"), ("http://example.com/", "https://scrapy.org/", b"http://example.com/"), @@ -86,7 +86,7 @@ class MixinDefault: class MixinNoReferrer: - scenarii: List[Tuple[str, str, Optional[bytes]]] = [ + scenarii: list[tuple[str, str, Optional[bytes]]] = [ ("https://example.com/page.html", "https://example.com/", None), ("http://www.example.com/", "https://scrapy.org/", None), ("http://www.example.com/", "http://scrapy.org/", None), @@ -96,7 +96,7 @@ class MixinNoReferrer: class MixinNoReferrerWhenDowngrade: - scenarii: List[Tuple[str, str, Optional[bytes]]] = [ + scenarii: list[tuple[str, str, Optional[bytes]]] = [ # TLS to TLS: send non-empty referrer ( "https://example.com/page.html", @@ -178,7 +178,7 @@ class MixinNoReferrerWhenDowngrade: class MixinSameOrigin: - scenarii: List[Tuple[str, str, Optional[bytes]]] = [ + scenarii: list[tuple[str, str, Optional[bytes]]] = [ # Same origin (protocol, host, port): send referrer ( "https://example.com/page.html", @@ -247,7 +247,7 @@ class MixinSameOrigin: class MixinOrigin: - scenarii: List[Tuple[str, str, Optional[bytes]]] = [ + scenarii: list[tuple[str, str, Optional[bytes]]] = [ # TLS or non-TLS to TLS or non-TLS: referrer origin is sent (yes, even for downgrades) ( "https://example.com/page.html", @@ -271,7 +271,7 @@ class MixinOrigin: class MixinStrictOrigin: - scenarii: List[Tuple[str, str, Optional[bytes]]] = [ + scenarii: list[tuple[str, str, Optional[bytes]]] = [ # TLS or non-TLS to TLS or non-TLS: referrer origin is sent but not for downgrades ( "https://example.com/page.html", @@ -299,7 +299,7 @@ class MixinStrictOrigin: class MixinOriginWhenCrossOrigin: - scenarii: List[Tuple[str, str, Optional[bytes]]] = [ + scenarii: list[tuple[str, str, Optional[bytes]]] = [ # Same origin (protocol, host, port): send referrer ( "https://example.com/page.html", @@ -406,7 +406,7 @@ class MixinOriginWhenCrossOrigin: class MixinStrictOriginWhenCrossOrigin: - scenarii: List[Tuple[str, str, Optional[bytes]]] = [ + scenarii: list[tuple[str, str, Optional[bytes]]] = [ # Same origin (protocol, host, port): send referrer ( "https://example.com/page.html", @@ -518,7 +518,7 @@ class MixinStrictOriginWhenCrossOrigin: class MixinUnsafeUrl: - scenarii: List[Tuple[str, str, Optional[bytes]]] = [ + scenarii: list[tuple[str, str, Optional[bytes]]] = [ # TLS to TLS: send referrer ( "https://example.com/sekrit.html", @@ -968,8 +968,8 @@ class TestPolicyHeaderPrecedence004( class TestReferrerOnRedirect(TestRefererMiddleware): settings = {"REFERRER_POLICY": "scrapy.spidermiddlewares.referer.UnsafeUrlPolicy"} - scenarii: List[ - Tuple[str, str, Tuple[Tuple[int, str], ...], Optional[bytes], Optional[bytes]] + scenarii: list[ + tuple[str, str, tuple[tuple[int, str], ...], Optional[bytes], Optional[bytes]] ] = [ # type: ignore[assignment] ( "http://scrapytest.org/1", # parent diff --git a/tests/test_utils_datatypes.py b/tests/test_utils_datatypes.py index be5c6de81..fb7c90f80 100644 --- a/tests/test_utils_datatypes.py +++ b/tests/test_utils_datatypes.py @@ -1,8 +1,7 @@ import copy import unittest import warnings -from collections.abc import Mapping, MutableMapping -from typing import Iterator +from collections.abc import Iterator, Mapping, MutableMapping from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import Request diff --git a/tests/test_utils_log.py b/tests/test_utils_log.py index 0f75bdb5c..76820eabf 100644 --- a/tests/test_utils_log.py +++ b/tests/test_utils_log.py @@ -1,10 +1,12 @@ +from __future__ import annotations + import json import logging import re import sys import unittest from io import StringIO -from typing import Any, Dict, Mapping, MutableMapping +from typing import TYPE_CHECKING, Any from unittest import TestCase import pytest @@ -21,6 +23,9 @@ from scrapy.utils.log import ( from scrapy.utils.test import get_crawler from tests.spiders import LogSpider +if TYPE_CHECKING: + from collections.abc import Mapping, MutableMapping + class FailureToExcInfoTest(unittest.TestCase): def test_failure(self): @@ -133,7 +138,7 @@ class StreamLoggerTest(unittest.TestCase): ), ) def test_spider_logger_adapter_process( - base_extra: Mapping[str, Any], log_extra: MutableMapping, expected_extra: Dict + base_extra: Mapping[str, Any], log_extra: MutableMapping, expected_extra: dict ): logger = logging.getLogger("test") spider_logger_adapter = SpiderLoggerAdapter(logger, base_extra) diff --git a/tests/test_utils_request.py b/tests/test_utils_request.py index 633077eec..ca3bca0b2 100644 --- a/tests/test_utils_request.py +++ b/tests/test_utils_request.py @@ -2,7 +2,7 @@ import json import unittest import warnings from hashlib import sha1 -from typing import Dict, Optional, Tuple, Union +from typing import Optional, Union from weakref import WeakKeyDictionary from scrapy.http import Request @@ -57,11 +57,11 @@ class FingerprintTest(unittest.TestCase): function: staticmethod = staticmethod(fingerprint) cache: Union[ - "WeakKeyDictionary[Request, Dict[Tuple[Optional[Tuple[bytes, ...]], bool], bytes]]", - "WeakKeyDictionary[Request, Dict[Tuple[Optional[Tuple[bytes, ...]], bool], str]]", + "WeakKeyDictionary[Request, dict[tuple[Optional[tuple[bytes, ...]], bool], bytes]]", + "WeakKeyDictionary[Request, dict[tuple[Optional[tuple[bytes, ...]], bool], str]]", ] = _fingerprint_cache default_cache_key = (None, False) - known_hashes: Tuple[Tuple[Request, Union[bytes, str], Dict], ...] = ( + known_hashes: tuple[tuple[Request, Union[bytes, str], dict], ...] = ( ( Request("http://example.org"), b"xs\xd7\x0c3uj\x15\xfe\xd7d\x9b\xa9\t\xe0d\xbf\x9cXD", diff --git a/tests_typing/test_http_request.mypy-testing b/tests_typing/test_http_request.mypy-testing index 665db9088..3926c830f 100644 --- a/tests_typing/test_http_request.mypy-testing +++ b/tests_typing/test_http_request.mypy-testing @@ -16,7 +16,7 @@ class MyRequest2(Request): @pytest.mark.mypy_testing def mypy_test_headers(): - Request("data:,", headers=1) # E: Argument "headers" to "Request" has incompatible type "int"; expected "Union[Mapping[str, Any], Iterable[Tuple[str, Any]], None]" + Request("data:,", headers=1) # E: Argument "headers" to "Request" has incompatible type "int"; expected "Union[Mapping[str, Any], Iterable[tuple[str, Any]], None]" Request("data:,", headers=None) Request("data:,", headers={}) Request("data:,", headers=[]) diff --git a/tests_typing/test_http_response.mypy-testing b/tests_typing/test_http_response.mypy-testing index d58ac1027..88aedbd3e 100644 --- a/tests_typing/test_http_response.mypy-testing +++ b/tests_typing/test_http_response.mypy-testing @@ -7,7 +7,7 @@ from scrapy.http import HtmlResponse, Response, TextResponse @pytest.mark.mypy_testing def mypy_test_headers(): - Response("data:,", headers=1) # E: Argument "headers" to "Response" has incompatible type "int"; expected "Union[Mapping[str, Any], Iterable[Tuple[str, Any]], None]" + Response("data:,", headers=1) # E: Argument "headers" to "Response" has incompatible type "int"; expected "Union[Mapping[str, Any], Iterable[tuple[str, Any]], None]" Response("data:,", headers=None) Response("data:,", headers={}) Response("data:,", headers=[]) diff --git a/tox.ini b/tox.ini index 8f3d23d75..451860653 100644 --- a/tox.ini +++ b/tox.ini @@ -46,23 +46,22 @@ install_command = [testenv:typing] basepython = python3 deps = - mypy==1.10.1 + mypy==1.11.1 typing-extensions==4.12.2 - types-lxml==2024.4.14 + types-lxml==2024.8.7 types-Pygments==2.18.0.20240506 - types-pyOpenSSL==24.1.0.20240425 - types-setuptools==70.3.0.20240710 - botocore-stubs==1.34.143 - boto3-stubs[s3]==1.34.143 + botocore-stubs==1.34.158 + boto3-stubs[s3]==1.34.158 attrs >= 18.2.0 Pillow >= 10.3.0 + pyOpenSSL >= 24.2.1 pytest >= 8.2.0 w3lib >= 2.2.0 commands = mypy {posargs: scrapy tests} [testenv:typing-tests] -basepython = python3.8 +basepython = python3.9 deps = {[test-requirements]deps} {[testenv:typing]deps} @@ -95,21 +94,21 @@ commands = twine check dist/* [pinned] -basepython = python3.8 +basepython = python3.9 deps = - cryptography==36.0.0 + cryptography==37.0.0 cssselect==0.9.1 h2==3.0 itemadapter==0.1.0 parsel==1.5.0 Protego==0.1.15 - pyOpenSSL==21.0.0 + pyOpenSSL==22.0.0 queuelib==1.4.2 service_identity==18.1.0 - Twisted[http2]==18.9.0 + Twisted[http2]==21.7.0 w3lib==1.17.0 zope.interface==5.1.0 - lxml==4.4.1 + lxml==4.6.0 {[test-requirements]deps} # mitmproxy 8.0.0 requires upgrading some of the pinned dependencies @@ -195,7 +194,7 @@ commands = pytest {posargs:--durations=10 docs scrapy tests} [testenv:pypy3-pinned] -basepython = pypy3.8 +basepython = pypy3.9 deps = {[pinned]deps} PyPyDispatcher==2.1.0