diff --git a/.flake8 b/.flake8 index be9d83eaf..c4814f13a 100644 --- a/.flake8 +++ b/.flake8 @@ -1,6 +1,7 @@ [flake8] max-line-length = 119 +extend-select = TC, TC1 ignore = # black disagrees with flake8 about these E203, E501, E701, E704, W503 @@ -58,6 +59,9 @@ ignore = D402 # First word of the first line should be properly capitalized D403 + + # Annotation in typing.cast() should be a string literal + TC006 exclude = docs/conf.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f70effc5d..38526d720 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -14,6 +14,7 @@ repos: - flake8-debugger - flake8-docstrings - flake8-string-format + - flake8-type-checking - repo: https://github.com/psf/black.git rev: 24.2.0 hooks: diff --git a/scrapy/addons.py b/scrapy/addons.py index 65d7a0310..f9ec58cea 100644 --- a/scrapy/addons.py +++ b/scrapy/addons.py @@ -1,13 +1,16 @@ +from __future__ import annotations + import logging from typing import TYPE_CHECKING, Any, List from scrapy.exceptions import NotConfigured -from scrapy.settings import Settings from scrapy.utils.conf import build_component_list from scrapy.utils.misc import build_from_crawler, load_object if TYPE_CHECKING: from scrapy.crawler import Crawler + from scrapy.settings import Settings + logger = logging.getLogger(__name__) @@ -15,8 +18,8 @@ logger = logging.getLogger(__name__) class AddonManager: """This class facilitates loading and storing :ref:`topics-addons`.""" - def __init__(self, crawler: "Crawler") -> None: - self.crawler: "Crawler" = crawler + def __init__(self, crawler: Crawler) -> None: + self.crawler: Crawler = crawler self.addons: List[Any] = [] def load_settings(self, settings: Settings) -> None: diff --git a/scrapy/cmdline.py b/scrapy/cmdline.py index da0e51386..e010b159a 100644 --- a/scrapy/cmdline.py +++ b/scrapy/cmdline.py @@ -12,7 +12,6 @@ import scrapy from scrapy.commands import BaseRunSpiderCommand, ScrapyCommand, ScrapyHelpFormatter from scrapy.crawler import CrawlerProcess from scrapy.exceptions import UsageError -from scrapy.settings import BaseSettings, Settings from scrapy.utils.misc import walk_modules from scrapy.utils.project import get_project_settings, inside_project from scrapy.utils.python import garbage_collect @@ -21,6 +20,8 @@ if TYPE_CHECKING: # typing.ParamSpec requires Python 3.10 from typing_extensions import ParamSpec + from scrapy.settings import BaseSettings, Settings + _P = ParamSpec("_P") diff --git a/scrapy/commands/__init__.py b/scrapy/commands/__init__.py index 9fe803d3c..0322390e5 100644 --- a/scrapy/commands/__init__.py +++ b/scrapy/commands/__init__.py @@ -2,18 +2,22 @@ Base class for Scrapy commands """ +from __future__ import annotations + import argparse import builtins import os from pathlib import Path -from typing import Any, Dict, Iterable, List, Optional +from typing import TYPE_CHECKING, Any, Dict, Iterable, List, Optional from twisted.python import failure -from scrapy.crawler import Crawler, CrawlerProcess from scrapy.exceptions import UsageError from scrapy.utils.conf import arglist_to_dict, feed_process_params_from_cli +if TYPE_CHECKING: + from scrapy.crawler import Crawler, CrawlerProcess + class ScrapyCommand: requires_project: bool = False diff --git a/scrapy/commands/bench.py b/scrapy/commands/bench.py index 0c4ebcd23..f91fec57e 100644 --- a/scrapy/commands/bench.py +++ b/scrapy/commands/bench.py @@ -1,16 +1,20 @@ +from __future__ import annotations + import argparse import subprocess # nosec import sys import time -from typing import Any, Iterable, List +from typing import TYPE_CHECKING, Any, Iterable, List from urllib.parse import urlencode import scrapy -from scrapy import Request from scrapy.commands import ScrapyCommand from scrapy.http import Response, TextResponse from scrapy.linkextractors import LinkExtractor +if TYPE_CHECKING: + from scrapy import Request + class Command(ScrapyCommand): default_settings = { diff --git a/scrapy/commands/crawl.py b/scrapy/commands/crawl.py index 6e023af81..fe1864372 100644 --- a/scrapy/commands/crawl.py +++ b/scrapy/commands/crawl.py @@ -1,11 +1,15 @@ -import argparse -from typing import List, cast +from __future__ import annotations + +from typing import TYPE_CHECKING, List, cast from twisted.python.failure import Failure from scrapy.commands import BaseRunSpiderCommand from scrapy.exceptions import UsageError +if TYPE_CHECKING: + import argparse + class Command(BaseRunSpiderCommand): requires_project = True diff --git a/scrapy/commands/fetch.py b/scrapy/commands/fetch.py index 1acf2d26f..0bdc429da 100644 --- a/scrapy/commands/fetch.py +++ b/scrapy/commands/fetch.py @@ -1,6 +1,7 @@ +from __future__ import annotations + import sys -from argparse import ArgumentParser, Namespace -from typing import Dict, List, Type +from typing import TYPE_CHECKING, Dict, List, Type from w3lib.url import is_url @@ -11,6 +12,9 @@ from scrapy.http import Request, Response from scrapy.utils.datatypes import SequenceExclude from scrapy.utils.spider import DefaultSpider, spidercls_for_request +if TYPE_CHECKING: + from argparse import ArgumentParser, Namespace + class Command(ScrapyCommand): requires_project = False diff --git a/scrapy/commands/list.py b/scrapy/commands/list.py index dcc51a694..10330c92a 100644 --- a/scrapy/commands/list.py +++ b/scrapy/commands/list.py @@ -1,8 +1,12 @@ -import argparse -from typing import List +from __future__ import annotations + +from typing import TYPE_CHECKING, List from scrapy.commands import ScrapyCommand +if TYPE_CHECKING: + import argparse + class Command(ScrapyCommand): requires_project = True diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py index 3320a1ee4..e6c5e2a47 100644 --- a/scrapy/commands/parse.py +++ b/scrapy/commands/parse.py @@ -6,6 +6,7 @@ import inspect import json import logging from typing import ( + TYPE_CHECKING, Any, AsyncGenerator, Callable, @@ -22,13 +23,11 @@ from typing import ( from itemadapter import ItemAdapter, is_item from twisted.internet.defer import Deferred, maybeDeferred -from twisted.python.failure import Failure from w3lib.url import is_url from scrapy.commands import BaseRunSpiderCommand from scrapy.exceptions import UsageError from scrapy.http import Request, Response -from scrapy.spiders import Spider from scrapy.utils import display from scrapy.utils.asyncgen import collect_asyncgen from scrapy.utils.defer import aiter_errback, deferred_from_coro @@ -36,6 +35,12 @@ from scrapy.utils.log import failure_to_exc_info from scrapy.utils.misc import arg_to_iter from scrapy.utils.spider import spidercls_for_request +if TYPE_CHECKING: + from twisted.python.failure import Failure + + from scrapy.spiders import Spider + + logger = logging.getLogger(__name__) _T = TypeVar("_T") diff --git a/scrapy/commands/runspider.py b/scrapy/commands/runspider.py index 77850e7b5..87acf9a01 100644 --- a/scrapy/commands/runspider.py +++ b/scrapy/commands/runspider.py @@ -1,17 +1,21 @@ +from __future__ import annotations + import argparse import sys from importlib import import_module -from os import PathLike from pathlib import Path -from types import ModuleType -from typing import List, Union +from typing import TYPE_CHECKING, List, Union from scrapy.commands import BaseRunSpiderCommand from scrapy.exceptions import UsageError from scrapy.utils.spider import iter_spider_classes +if TYPE_CHECKING: + from os import PathLike + from types import ModuleType -def _import_file(filepath: Union[str, PathLike]) -> ModuleType: + +def _import_file(filepath: Union[str, PathLike[str]]) -> ModuleType: abspath = Path(filepath).resolve() if abspath.suffix not in (".py", ".pyw"): raise ValueError(f"Not a Python source file: {abspath}") diff --git a/scrapy/commands/shell.py b/scrapy/commands/shell.py index 668c95a7b..f03cf997a 100644 --- a/scrapy/commands/shell.py +++ b/scrapy/commands/shell.py @@ -4,9 +4,10 @@ Scrapy Shell See documentation in docs/topics/shell.rst """ -from argparse import ArgumentParser, Namespace +from __future__ import annotations + from threading import Thread -from typing import Any, Dict, List, Type +from typing import TYPE_CHECKING, Any, Dict, List, Type from scrapy import Spider from scrapy.commands import ScrapyCommand @@ -15,6 +16,9 @@ from scrapy.shell import Shell from scrapy.utils.spider import DefaultSpider, spidercls_for_request from scrapy.utils.url import guess_scheme +if TYPE_CHECKING: + from argparse import ArgumentParser, Namespace + class Command(ScrapyCommand): requires_project = False diff --git a/scrapy/contracts/__init__.py b/scrapy/contracts/__init__.py index 27bc2fcba..440e0dc44 100644 --- a/scrapy/contracts/__init__.py +++ b/scrapy/contracts/__init__.py @@ -1,9 +1,12 @@ +from __future__ import annotations + import re import sys from functools import wraps from inspect import getmembers from types import CoroutineType from typing import ( + TYPE_CHECKING, Any, AsyncGenerator, Callable, @@ -16,13 +19,15 @@ from typing import ( ) from unittest import TestCase, TestResult -from twisted.python.failure import Failure - -from scrapy import Spider from scrapy.http import Request, Response from scrapy.utils.python import get_spec from scrapy.utils.spider import iterate_spider_output +if TYPE_CHECKING: + from twisted.python.failure import Failure + + from scrapy import Spider + class Contract: """Abstract class for contracts""" diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index 41f729ed9..6786d7acf 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -25,15 +25,16 @@ from scrapy import Request, Spider, signals from scrapy.core.downloader.handlers import DownloadHandlers from scrapy.core.downloader.middleware import DownloaderMiddlewareManager from scrapy.exceptions import ScrapyDeprecationWarning -from scrapy.http import Response from scrapy.resolver import dnscache -from scrapy.settings import BaseSettings from scrapy.signalmanager import SignalManager from scrapy.utils.defer import mustbe_deferred from scrapy.utils.httpobj import urlparse_cached if TYPE_CHECKING: from scrapy.crawler import Crawler + from scrapy.http import Response + from scrapy.settings import BaseSettings + _T = TypeVar("_T") diff --git a/scrapy/core/downloader/contextfactory.py b/scrapy/core/downloader/contextfactory.py index 9f6edb630..2b388a9f5 100644 --- a/scrapy/core/downloader/contextfactory.py +++ b/scrapy/core/downloader/contextfactory.py @@ -21,8 +21,6 @@ from scrapy.core.downloader.tls import ( ScrapyClientTLSOptions, openssl_methods, ) -from scrapy.crawler import Crawler -from scrapy.settings import BaseSettings from scrapy.utils.misc import build_from_crawler, load_object if TYPE_CHECKING: @@ -31,6 +29,9 @@ if TYPE_CHECKING: # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy.crawler import Crawler + from scrapy.settings import BaseSettings + @implementer(IPolicyForHTTPS) class ScrapyClientContextFactory(BrowserLikePolicyForHTTPS): diff --git a/scrapy/core/downloader/handlers/__init__.py b/scrapy/core/downloader/handlers/__init__.py index ebc4898b5..70d356b83 100644 --- a/scrapy/core/downloader/handlers/__init__.py +++ b/scrapy/core/downloader/handlers/__init__.py @@ -17,17 +17,19 @@ from typing import ( ) from twisted.internet import defer -from twisted.internet.defer import Deferred from scrapy import Request, Spider, signals from scrapy.exceptions import NotConfigured, NotSupported -from scrapy.http import Response from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.misc import build_from_crawler, load_object from scrapy.utils.python import without_none_values if TYPE_CHECKING: + from twisted.internet.defer import Deferred + from scrapy.crawler import Crawler + from scrapy.http import Response + logger = logging.getLogger(__name__) diff --git a/scrapy/core/downloader/handlers/datauri.py b/scrapy/core/downloader/handlers/datauri.py index a7ae56a85..bf6879521 100644 --- a/scrapy/core/downloader/handlers/datauri.py +++ b/scrapy/core/downloader/handlers/datauri.py @@ -1,12 +1,16 @@ -from typing import Any, Dict +from __future__ import annotations + +from typing import TYPE_CHECKING, Any, Dict from w3lib.url import parse_data_uri -from scrapy import Request, Spider from scrapy.http import Response, TextResponse from scrapy.responsetypes import responsetypes from scrapy.utils.decorators import defers +if TYPE_CHECKING: + from scrapy import Request, Spider + class DataURIDownloadHandler: lazy = False diff --git a/scrapy/core/downloader/handlers/file.py b/scrapy/core/downloader/handlers/file.py index 17dd7483b..d55c516f0 100644 --- a/scrapy/core/downloader/handlers/file.py +++ b/scrapy/core/downloader/handlers/file.py @@ -1,12 +1,17 @@ +from __future__ import annotations + from pathlib import Path +from typing import TYPE_CHECKING from w3lib.url import file_uri_to_path -from scrapy import Request, Spider -from scrapy.http import Response from scrapy.responsetypes import responsetypes from scrapy.utils.decorators import defers +if TYPE_CHECKING: + from scrapy import Request, Spider + from scrapy.http import Response + class FileDownloadHandler: lazy = False diff --git a/scrapy/core/downloader/handlers/ftp.py b/scrapy/core/downloader/handlers/ftp.py index 724717ffd..69c2d88e1 100644 --- a/scrapy/core/downloader/handlers/ftp.py +++ b/scrapy/core/downloader/handlers/ftp.py @@ -35,23 +35,25 @@ from io import BytesIO from typing import TYPE_CHECKING, Any, BinaryIO, Dict, Optional from urllib.parse import unquote -from twisted.internet.defer import Deferred from twisted.internet.protocol import ClientCreator, Protocol from twisted.protocols.ftp import CommandFailed, FTPClient -from twisted.python.failure import Failure -from scrapy import Request, Spider -from scrapy.crawler import Crawler from scrapy.http import Response from scrapy.responsetypes import responsetypes -from scrapy.settings import BaseSettings from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.python import to_bytes if TYPE_CHECKING: + from twisted.internet.defer import Deferred + from twisted.python.failure import Failure + # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy import Request, Spider + from scrapy.crawler import Crawler + from scrapy.settings import BaseSettings + class ReceivedDataProtocol(Protocol): def __init__(self, filename: Optional[str] = None): diff --git a/scrapy/core/downloader/handlers/http10.py b/scrapy/core/downloader/handlers/http10.py index 3c4e48abb..98f62efcf 100644 --- a/scrapy/core/downloader/handlers/http10.py +++ b/scrapy/core/downloader/handlers/http10.py @@ -5,21 +5,21 @@ from __future__ import annotations from typing import TYPE_CHECKING, Type -from twisted.internet.defer import Deferred - -from scrapy import Request, Spider -from scrapy.crawler import Crawler -from scrapy.http import Response -from scrapy.settings import BaseSettings from scrapy.utils.misc import build_from_crawler, load_object from scrapy.utils.python import to_unicode if TYPE_CHECKING: + from twisted.internet.defer import Deferred + # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy import Request, Spider from scrapy.core.downloader.contextfactory import ScrapyClientContextFactory from scrapy.core.downloader.webclient import ScrapyHTTPClientFactory + from scrapy.crawler import Crawler + from scrapy.http import Response + from scrapy.settings import BaseSettings class HTTP10DownloadHandler: diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index e2ad8f59a..c06d90f01 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -12,11 +12,9 @@ from typing import TYPE_CHECKING, Any, List, Optional, Tuple, TypedDict, TypeVar from urllib.parse import urldefrag, urlunparse from twisted.internet import ssl -from twisted.internet.base import ReactorBase from twisted.internet.defer import CancelledError, Deferred, succeed from twisted.internet.endpoints import TCP4ClientEndpoint from twisted.internet.error import TimeoutError -from twisted.internet.interfaces import IConsumer from twisted.internet.protocol import Factory, Protocol, connectionDone from twisted.python.failure import Failure from twisted.web.client import URI, Agent, HTTPConnectionPool @@ -30,17 +28,22 @@ from zope.interface import implementer from scrapy import Request, Spider, signals from scrapy.core.downloader.contextfactory import load_context_factory_from_settings from scrapy.core.downloader.webclient import _parse -from scrapy.crawler import Crawler from scrapy.exceptions import StopDownload from scrapy.http import Headers, Response from scrapy.responsetypes import responsetypes -from scrapy.settings import BaseSettings from scrapy.utils.python import to_bytes, to_unicode if TYPE_CHECKING: + from twisted.internet.base import ReactorBase + from twisted.internet.interfaces import IConsumer + # typing.NotRequired and typing.Self require Python 3.11 from typing_extensions import NotRequired, Self + from scrapy.crawler import Crawler + from scrapy.settings import BaseSettings + + logger = logging.getLogger(__name__) _T = TypeVar("_T") diff --git a/scrapy/core/downloader/handlers/http2.py b/scrapy/core/downloader/handlers/http2.py index 2ac4eca86..4722c612d 100644 --- a/scrapy/core/downloader/handlers/http2.py +++ b/scrapy/core/downloader/handlers/http2.py @@ -4,25 +4,27 @@ from time import time from typing import TYPE_CHECKING, Optional from urllib.parse import urldefrag -from twisted.internet.base import DelayedCall -from twisted.internet.defer import Deferred from twisted.internet.error import TimeoutError from twisted.web.client import URI -from twisted.web.iweb import IPolicyForHTTPS from scrapy.core.downloader.contextfactory import load_context_factory_from_settings from scrapy.core.downloader.webclient import _parse from scrapy.core.http2.agent import H2Agent, H2ConnectionPool, ScrapyProxyH2Agent -from scrapy.crawler import Crawler -from scrapy.http import Request, Response -from scrapy.settings import Settings -from scrapy.spiders import Spider from scrapy.utils.python import to_bytes if TYPE_CHECKING: + from twisted.internet.base import DelayedCall + from twisted.internet.defer import Deferred + from twisted.web.iweb import IPolicyForHTTPS + # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy.crawler import Crawler + from scrapy.http import Request, Response + from scrapy.settings import Settings + from scrapy.spiders import Spider + class H2DownloadHandler: def __init__(self, settings: Settings, crawler: Crawler): diff --git a/scrapy/core/downloader/handlers/s3.py b/scrapy/core/downloader/handlers/s3.py index 0ad340721..edf370193 100644 --- a/scrapy/core/downloader/handlers/s3.py +++ b/scrapy/core/downloader/handlers/s3.py @@ -2,22 +2,23 @@ from __future__ import annotations from typing import TYPE_CHECKING, Any, Optional, Type -from twisted.internet.defer import Deferred - -from scrapy import Request, Spider from scrapy.core.downloader.handlers.http import HTTPDownloadHandler -from scrapy.crawler import Crawler from scrapy.exceptions import NotConfigured -from scrapy.http import Response -from scrapy.settings import BaseSettings from scrapy.utils.boto import is_botocore_available from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.misc import build_from_crawler if TYPE_CHECKING: + from twisted.internet.defer import Deferred + # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy import Request, Spider + from scrapy.crawler import Crawler + from scrapy.http import Response + from scrapy.settings import BaseSettings + class S3DownloadHandler: def __init__( diff --git a/scrapy/core/downloader/middleware.py b/scrapy/core/downloader/middleware.py index 2d8af114f..0bdb756c8 100644 --- a/scrapy/core/downloader/middleware.py +++ b/scrapy/core/downloader/middleware.py @@ -6,19 +6,22 @@ See documentation in docs/topics/downloader-middleware.rst from __future__ import annotations -from typing import Any, Callable, Generator, List, Union, cast +from typing import TYPE_CHECKING, Any, Callable, Generator, List, Union, cast from twisted.internet.defer import Deferred, inlineCallbacks -from twisted.python.failure import Failure -from scrapy import Spider from scrapy.exceptions import _InvalidOutput from scrapy.http import Request, Response from scrapy.middleware import MiddlewareManager -from scrapy.settings import BaseSettings from scrapy.utils.conf import build_component_list from scrapy.utils.defer import deferred_from_coro, mustbe_deferred +if TYPE_CHECKING: + from twisted.python.failure import Failure + + from scrapy import Spider + from scrapy.settings import BaseSettings + class DownloaderMiddlewareManager(MiddlewareManager): component_name = "downloader middleware" diff --git a/scrapy/core/downloader/webclient.py b/scrapy/core/downloader/webclient.py index 08a1d7c71..99502f0d2 100644 --- a/scrapy/core/downloader/webclient.py +++ b/scrapy/core/downloader/webclient.py @@ -1,18 +1,22 @@ +from __future__ import annotations + import re from time import time -from typing import Optional, Tuple +from typing import TYPE_CHECKING, Optional, Tuple from urllib.parse import ParseResult, urldefrag, urlparse, urlunparse from twisted.internet import defer from twisted.internet.protocol import ClientFactory from twisted.web.http import HTTPClient -from scrapy import Request from scrapy.http import Headers, Response from scrapy.responsetypes import responsetypes from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.python import to_bytes, to_unicode +if TYPE_CHECKING: + from scrapy import Request + def _parsed_url_args(parsed: ParseResult) -> Tuple[bytes, bytes, bytes, int, bytes]: # Assume parsed is urlparse-d from Request.url, diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index 4ffec78b9..5318cbd64 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -34,9 +34,8 @@ from scrapy.core.scraper import Scraper from scrapy.exceptions import CloseSpider, DontCloseSpider, IgnoreRequest from scrapy.http import Request, Response from scrapy.logformatter import LogFormatter -from scrapy.settings import BaseSettings, Settings +from scrapy.settings import Settings from scrapy.signalmanager import SignalManager -from scrapy.spiders import Spider from scrapy.utils.log import failure_to_exc_info, logformatter_adapter from scrapy.utils.misc import build_from_crawler, load_object from scrapy.utils.python import global_object_name @@ -46,6 +45,9 @@ if TYPE_CHECKING: from scrapy.core.scheduler import BaseScheduler from scrapy.core.scraper import _HandleOutputDeferred from scrapy.crawler import Crawler + from scrapy.settings import BaseSettings + from scrapy.spiders import Spider + logger = logging.getLogger(__name__) diff --git a/scrapy/core/http2/agent.py b/scrapy/core/http2/agent.py index 999764a6e..d291a5b8a 100644 --- a/scrapy/core/http2/agent.py +++ b/scrapy/core/http2/agent.py @@ -1,10 +1,10 @@ +from __future__ import annotations + from collections import deque -from typing import Deque, Dict, List, Optional, Tuple +from typing import TYPE_CHECKING, Deque, Dict, List, Optional, Tuple from twisted.internet import defer -from twisted.internet.base import ReactorBase from twisted.internet.defer import Deferred -from twisted.internet.endpoints import HostnameEndpoint from twisted.python.failure import Failure from twisted.web.client import ( URI, @@ -16,9 +16,15 @@ from twisted.web.error import SchemeNotSupported from scrapy.core.downloader.contextfactory import AcceptableProtocolsContextFactory from scrapy.core.http2.protocol import H2ClientFactory, H2ClientProtocol -from scrapy.http.request import Request -from scrapy.settings import Settings -from scrapy.spiders import Spider + +if TYPE_CHECKING: + from twisted.internet.base import ReactorBase + from twisted.internet.endpoints import HostnameEndpoint + + from scrapy.http.request import Request + from scrapy.settings import Settings + from scrapy.spiders import Spider + ConnectionKeyT = Tuple[bytes, bytes, int] diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py index f2f1cb0b8..a6809102b 100644 --- a/scrapy/core/http2/protocol.py +++ b/scrapy/core/http2/protocol.py @@ -1,9 +1,10 @@ +from __future__ import annotations + import ipaddress import itertools import logging from collections import deque -from ipaddress import IPv4Address, IPv6Address -from typing import Any, Deque, Dict, List, Optional, Union +from typing import TYPE_CHECKING, Any, Deque, Dict, List, Optional, Union from h2.config import H2Configuration from h2.connection import H2Connection @@ -20,7 +21,6 @@ from h2.events import ( WindowUpdated, ) from h2.exceptions import FrameTooLargeError, H2Error -from twisted.internet.defer import Deferred from twisted.internet.error import TimeoutError from twisted.internet.interfaces import ( IAddress, @@ -30,14 +30,21 @@ from twisted.internet.interfaces import ( from twisted.internet.protocol import Factory, Protocol, connectionDone from twisted.internet.ssl import Certificate from twisted.protocols.policies import TimeoutMixin -from twisted.python.failure import Failure -from twisted.web.client import URI from zope.interface import implementer from scrapy.core.http2.stream import Stream, StreamCloseReason from scrapy.http import Request -from scrapy.settings import Settings -from scrapy.spiders import Spider + +if TYPE_CHECKING: + from ipaddress import IPv4Address, IPv6Address + + from twisted.internet.defer import Deferred + from twisted.python.failure import Failure + from twisted.web.client import URI + + from scrapy.settings import Settings + from scrapy.spiders import Spider + logger = logging.getLogger(__name__) diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py index 7c70e86db..a02fbb328 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/http2/stream.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import logging from enum import Enum from io import BytesIO @@ -5,19 +7,20 @@ from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple from h2.errors import ErrorCodes from h2.exceptions import H2Error, ProtocolError, StreamClosedError -from hpack import HeaderTuple from twisted.internet.defer import CancelledError, Deferred from twisted.internet.error import ConnectionClosed from twisted.python.failure import Failure from twisted.web.client import ResponseFailed -from scrapy.http import Request from scrapy.http.headers import Headers from scrapy.responsetypes import responsetypes from scrapy.utils.httpobj import urlparse_cached if TYPE_CHECKING: + from hpack import HeaderTuple + from scrapy.core.http2.protocol import H2ClientProtocol + from scrapy.http import Request logger = logging.getLogger(__name__) @@ -87,7 +90,7 @@ class Stream: self, stream_id: int, request: Request, - protocol: "H2ClientProtocol", + protocol: H2ClientProtocol, download_maxsize: int = 0, download_warnsize: int = 0, ) -> None: @@ -99,7 +102,7 @@ class Stream: """ self.stream_id: int = stream_id self._request: Request = request - self._protocol: "H2ClientProtocol" = protocol + self._protocol: H2ClientProtocol = protocol self._download_maxsize = self._request.meta.get( "download_maxsize", download_maxsize diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.py index 1e586c53a..d4286c874 100644 --- a/scrapy/core/scheduler.py +++ b/scrapy/core/scheduler.py @@ -6,14 +6,10 @@ from abc import abstractmethod from pathlib import Path from typing import TYPE_CHECKING, Any, List, Optional, Type, cast -from twisted.internet.defer import Deferred +# working around https://github.com/sphinx-doc/sphinx/issues/10400 +from twisted.internet.defer import Deferred # noqa: TC002 -from scrapy.crawler import Crawler -from scrapy.dupefilters import BaseDupeFilter -from scrapy.http.request import Request -from scrapy.pqueues import ScrapyPriorityQueue -from scrapy.spiders import Spider -from scrapy.statscollectors import StatsCollector +from scrapy.spiders import Spider # noqa: TC001 from scrapy.utils.job import job_dir from scrapy.utils.misc import build_from_crawler, load_object @@ -24,6 +20,12 @@ if TYPE_CHECKING: # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy.crawler import Crawler + from scrapy.dupefilters import BaseDupeFilter + from scrapy.http.request import Request + from scrapy.pqueues import ScrapyPriorityQueue + from scrapy.statscollectors import StatsCollector + logger = logging.getLogger(__name__) diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index e792f8ca7..37a666605 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -10,6 +10,7 @@ import logging from inspect import isasyncgenfunction, iscoroutine from itertools import islice from typing import ( + TYPE_CHECKING, Any, AsyncIterable, Callable, @@ -30,7 +31,6 @@ from scrapy import Request, Spider from scrapy.exceptions import _InvalidOutput from scrapy.http import Response from scrapy.middleware import MiddlewareManager -from scrapy.settings import BaseSettings from scrapy.utils.asyncgen import as_async_generator, collect_asyncgen from scrapy.utils.conf import build_component_list from scrapy.utils.defer import ( @@ -41,6 +41,10 @@ from scrapy.utils.defer import ( ) from scrapy.utils.python import MutableAsyncChain, MutableChain +if TYPE_CHECKING: + from scrapy.settings import BaseSettings + + logger = logging.getLogger(__name__) diff --git a/scrapy/downloadermiddlewares/ajaxcrawl.py b/scrapy/downloadermiddlewares/ajaxcrawl.py index 0e757e4be..5fc7f31a3 100644 --- a/scrapy/downloadermiddlewares/ajaxcrawl.py +++ b/scrapy/downloadermiddlewares/ajaxcrawl.py @@ -6,16 +6,18 @@ from typing import TYPE_CHECKING, Union from w3lib import html -from scrapy import Request, Spider -from scrapy.crawler import Crawler from scrapy.exceptions import NotConfigured from scrapy.http import HtmlResponse, Response -from scrapy.settings import BaseSettings if TYPE_CHECKING: # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy import Request, Spider + from scrapy.crawler import Crawler + from scrapy.settings import BaseSettings + + logger = logging.getLogger(__name__) diff --git a/scrapy/downloadermiddlewares/cookies.py b/scrapy/downloadermiddlewares/cookies.py index 73c2c57fe..23140d263 100644 --- a/scrapy/downloadermiddlewares/cookies.py +++ b/scrapy/downloadermiddlewares/cookies.py @@ -2,24 +2,26 @@ from __future__ import annotations import logging from collections import defaultdict -from http.cookiejar import Cookie from typing import TYPE_CHECKING, Any, DefaultDict, Iterable, Optional, Sequence, Union from tldextract import TLDExtract -from scrapy import Request, Spider -from scrapy.crawler import Crawler from scrapy.exceptions import NotConfigured from scrapy.http import Response from scrapy.http.cookies import CookieJar -from scrapy.http.request import VerboseCookie from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.python import to_unicode if TYPE_CHECKING: + from http.cookiejar import Cookie + # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy import Request, Spider + from scrapy.crawler import Crawler + from scrapy.http.request import VerboseCookie + logger = logging.getLogger(__name__) diff --git a/scrapy/downloadermiddlewares/defaultheaders.py b/scrapy/downloadermiddlewares/defaultheaders.py index 58fd415b9..49b9fdc05 100644 --- a/scrapy/downloadermiddlewares/defaultheaders.py +++ b/scrapy/downloadermiddlewares/defaultheaders.py @@ -8,15 +8,16 @@ from __future__ import annotations from typing import TYPE_CHECKING, Iterable, Tuple, Union -from scrapy import Request, Spider -from scrapy.crawler import Crawler -from scrapy.http import Response from scrapy.utils.python import without_none_values if TYPE_CHECKING: # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy import Request, Spider + from scrapy.crawler import Crawler + from scrapy.http import Response + class DefaultHeadersMiddleware: def __init__(self, headers: Iterable[Tuple[str, str]]): diff --git a/scrapy/downloadermiddlewares/downloadtimeout.py b/scrapy/downloadermiddlewares/downloadtimeout.py index fd7c03a38..ee7a24825 100644 --- a/scrapy/downloadermiddlewares/downloadtimeout.py +++ b/scrapy/downloadermiddlewares/downloadtimeout.py @@ -9,13 +9,14 @@ from __future__ import annotations from typing import TYPE_CHECKING, Union from scrapy import Request, Spider, signals -from scrapy.crawler import Crawler -from scrapy.http import Response if TYPE_CHECKING: # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy.crawler import Crawler + from scrapy.http import Response + class DownloadTimeoutMiddleware: def __init__(self, timeout: float = 180): diff --git a/scrapy/downloadermiddlewares/httpauth.py b/scrapy/downloadermiddlewares/httpauth.py index 63490a37a..39165e155 100644 --- a/scrapy/downloadermiddlewares/httpauth.py +++ b/scrapy/downloadermiddlewares/httpauth.py @@ -11,14 +11,15 @@ from typing import TYPE_CHECKING, Union from w3lib.http import basic_auth_header from scrapy import Request, Spider, signals -from scrapy.crawler import Crawler -from scrapy.http import Response from scrapy.utils.url import url_is_from_any_domain if TYPE_CHECKING: # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy.crawler import Crawler + from scrapy.http import Response + class HttpAuthMiddleware: """Set Basic HTTP Authorization header diff --git a/scrapy/downloadermiddlewares/httpcache.py b/scrapy/downloadermiddlewares/httpcache.py index 971473403..8377a3c1d 100644 --- a/scrapy/downloadermiddlewares/httpcache.py +++ b/scrapy/downloadermiddlewares/httpcache.py @@ -16,19 +16,20 @@ from twisted.internet.error import ( from twisted.web.client import ResponseFailed from scrapy import signals -from scrapy.crawler import Crawler from scrapy.exceptions import IgnoreRequest, NotConfigured -from scrapy.http.request import Request -from scrapy.http.response import Response -from scrapy.settings import Settings -from scrapy.spiders import Spider -from scrapy.statscollectors import StatsCollector from scrapy.utils.misc import load_object if TYPE_CHECKING: # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy.crawler import Crawler + from scrapy.http.request import Request + from scrapy.http.response import Response + from scrapy.settings import Settings + from scrapy.spiders import Spider + from scrapy.statscollectors import StatsCollector + class HttpCacheMiddleware: DOWNLOAD_EXCEPTIONS = ( diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py index f3647e05f..6b0a56f7f 100644 --- a/scrapy/downloadermiddlewares/httpcompression.py +++ b/scrapy/downloadermiddlewares/httpcompression.py @@ -6,11 +6,9 @@ from logging import getLogger from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union from scrapy import Request, Spider, signals -from scrapy.crawler import Crawler from scrapy.exceptions import IgnoreRequest, NotConfigured from scrapy.http import Response, TextResponse from scrapy.responsetypes import responsetypes -from scrapy.statscollectors import StatsCollector from scrapy.utils._compression import ( _DecompressionMaxSizeExceeded, _inflate, @@ -24,6 +22,10 @@ if TYPE_CHECKING: # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy.crawler import Crawler + from scrapy.statscollectors import StatsCollector + + logger = getLogger(__name__) ACCEPTED_ENCODINGS: List[bytes] = [b"gzip", b"deflate"] diff --git a/scrapy/downloadermiddlewares/httpproxy.py b/scrapy/downloadermiddlewares/httpproxy.py index 5b56ad449..a7af83f7d 100644 --- a/scrapy/downloadermiddlewares/httpproxy.py +++ b/scrapy/downloadermiddlewares/httpproxy.py @@ -9,10 +9,7 @@ from urllib.request import ( # type: ignore[attr-defined] proxy_bypass, ) -from scrapy import Request, Spider -from scrapy.crawler import Crawler from scrapy.exceptions import NotConfigured -from scrapy.http import Response from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.python import to_bytes @@ -20,6 +17,10 @@ if TYPE_CHECKING: # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy import Request, Spider + from scrapy.crawler import Crawler + from scrapy.http import Response + class HttpProxyMiddleware: def __init__(self, auth_encoding: Optional[str] = "latin-1"): diff --git a/scrapy/downloadermiddlewares/offsite.py b/scrapy/downloadermiddlewares/offsite.py index bd8dbe329..6f67e3975 100644 --- a/scrapy/downloadermiddlewares/offsite.py +++ b/scrapy/downloadermiddlewares/offsite.py @@ -6,15 +6,17 @@ import warnings from typing import TYPE_CHECKING, Set from scrapy import Request, Spider, signals -from scrapy.crawler import Crawler from scrapy.exceptions import IgnoreRequest -from scrapy.statscollectors import StatsCollector from scrapy.utils.httpobj import urlparse_cached if TYPE_CHECKING: # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy.crawler import Crawler + from scrapy.statscollectors import StatsCollector + + logger = logging.getLogger(__name__) diff --git a/scrapy/downloadermiddlewares/redirect.py b/scrapy/downloadermiddlewares/redirect.py index 371e2fd3b..53081237c 100644 --- a/scrapy/downloadermiddlewares/redirect.py +++ b/scrapy/downloadermiddlewares/redirect.py @@ -6,11 +6,8 @@ from urllib.parse import urljoin from w3lib.url import safe_url_string -from scrapy import Request, Spider -from scrapy.crawler import Crawler from scrapy.exceptions import IgnoreRequest, NotConfigured from scrapy.http import HtmlResponse, Response -from scrapy.settings import BaseSettings from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.response import get_meta_refresh @@ -18,6 +15,11 @@ if TYPE_CHECKING: # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy import Request, Spider + from scrapy.crawler import Crawler + from scrapy.settings import BaseSettings + + logger = logging.getLogger(__name__) diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.py index 0637f09d4..8d7b7293c 100644 --- a/scrapy/downloadermiddlewares/retry.py +++ b/scrapy/downloadermiddlewares/retry.py @@ -16,12 +16,8 @@ import warnings from logging import Logger, getLogger from typing import TYPE_CHECKING, Any, Optional, Tuple, Type, Union -from scrapy.crawler import Crawler from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning -from scrapy.http import Response -from scrapy.http.request import Request from scrapy.settings import BaseSettings, Settings -from scrapy.spiders import Spider from scrapy.utils.misc import load_object from scrapy.utils.python import global_object_name from scrapy.utils.response import response_status_message @@ -30,6 +26,12 @@ if TYPE_CHECKING: # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy.crawler import Crawler + from scrapy.http import Response + from scrapy.http.request import Request + from scrapy.spiders import Spider + + retry_logger = getLogger(__name__) diff --git a/scrapy/downloadermiddlewares/robotstxt.py b/scrapy/downloadermiddlewares/robotstxt.py index 6a0ecb7bf..70393576b 100644 --- a/scrapy/downloadermiddlewares/robotstxt.py +++ b/scrapy/downloadermiddlewares/robotstxt.py @@ -10,22 +10,24 @@ import logging from typing import TYPE_CHECKING, Any, Dict, Optional, Union from twisted.internet.defer import Deferred, maybeDeferred -from twisted.python.failure import Failure -from scrapy import Spider -from scrapy.crawler import Crawler from scrapy.exceptions import IgnoreRequest, NotConfigured from scrapy.http import Request, Response from scrapy.http.request import NO_CALLBACK -from scrapy.robotstxt import RobotParser from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.log import failure_to_exc_info from scrapy.utils.misc import load_object if TYPE_CHECKING: + from twisted.python.failure import Failure + # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy import Spider + from scrapy.crawler import Crawler + from scrapy.robotstxt import RobotParser + logger = logging.getLogger(__name__) diff --git a/scrapy/downloadermiddlewares/stats.py b/scrapy/downloadermiddlewares/stats.py index 444702757..0faae7b5a 100644 --- a/scrapy/downloadermiddlewares/stats.py +++ b/scrapy/downloadermiddlewares/stats.py @@ -4,11 +4,7 @@ from typing import TYPE_CHECKING, Dict, List, Tuple, Union from twisted.web import http -from scrapy import Request, Spider -from scrapy.crawler import Crawler from scrapy.exceptions import NotConfigured -from scrapy.http import Response -from scrapy.statscollectors import StatsCollector from scrapy.utils.python import global_object_name, to_bytes from scrapy.utils.request import request_httprepr @@ -16,6 +12,11 @@ if TYPE_CHECKING: # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy import Request, Spider + from scrapy.crawler import Crawler + from scrapy.http import Response + from scrapy.statscollectors import StatsCollector + def get_header_size( headers: Dict[str, Union[List[Union[str, bytes]], Tuple[Union[str, bytes], ...]]] diff --git a/scrapy/downloadermiddlewares/useragent.py b/scrapy/downloadermiddlewares/useragent.py index 92f1ec897..109f1a4d9 100644 --- a/scrapy/downloadermiddlewares/useragent.py +++ b/scrapy/downloadermiddlewares/useragent.py @@ -5,13 +5,14 @@ from __future__ import annotations from typing import TYPE_CHECKING, Union from scrapy import Request, Spider, signals -from scrapy.crawler import Crawler -from scrapy.http import Response if TYPE_CHECKING: # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy.crawler import Crawler + from scrapy.http import Response + class UserAgentMiddleware: """This middleware allows spiders to override the user_agent""" diff --git a/scrapy/dupefilters.py b/scrapy/dupefilters.py index dd2420e98..ffaf783a7 100644 --- a/scrapy/dupefilters.py +++ b/scrapy/dupefilters.py @@ -4,11 +4,6 @@ import logging from pathlib import Path from typing import TYPE_CHECKING, Optional, Set -from twisted.internet.defer import Deferred - -from scrapy.http.request import Request -from scrapy.settings import BaseSettings -from scrapy.spiders import Spider from scrapy.utils.job import job_dir from scrapy.utils.request import ( RequestFingerprinter, @@ -17,10 +12,15 @@ from scrapy.utils.request import ( ) if TYPE_CHECKING: + from twisted.internet.defer import Deferred + # typing.Self requires Python 3.11 from typing_extensions import Self from scrapy.crawler import Crawler + from scrapy.http.request import Request + from scrapy.settings import BaseSettings + from scrapy.spiders import Spider class BaseDupeFilter: diff --git a/scrapy/extension.py b/scrapy/extension.py index 8221b675e..8c81ab356 100644 --- a/scrapy/extension.py +++ b/scrapy/extension.py @@ -4,12 +4,16 @@ The Extension Manager See documentation in docs/topics/extensions.rst """ -from typing import Any, List +from __future__ import annotations + +from typing import TYPE_CHECKING, Any, List from scrapy.middleware import MiddlewareManager -from scrapy.settings import Settings from scrapy.utils.conf import build_component_list +if TYPE_CHECKING: + from scrapy.settings import Settings + class ExtensionManager(MiddlewareManager): component_name = "extension" diff --git a/scrapy/extensions/closespider.py b/scrapy/extensions/closespider.py index 812b3553c..4627e7f98 100644 --- a/scrapy/extensions/closespider.py +++ b/scrapy/extensions/closespider.py @@ -10,17 +10,19 @@ import logging from collections import defaultdict from typing import TYPE_CHECKING, Any, DefaultDict, Dict -from twisted.python.failure import Failure - from scrapy import Request, Spider, signals -from scrapy.crawler import Crawler from scrapy.exceptions import NotConfigured -from scrapy.http import Response if TYPE_CHECKING: + from twisted.python.failure import Failure + # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy.crawler import Crawler + from scrapy.http import Response + + logger = logging.getLogger(__name__) diff --git a/scrapy/extensions/corestats.py b/scrapy/extensions/corestats.py index f3ac19623..6ef2d0382 100644 --- a/scrapy/extensions/corestats.py +++ b/scrapy/extensions/corestats.py @@ -8,13 +8,14 @@ from datetime import datetime, timezone from typing import TYPE_CHECKING, Any, Optional from scrapy import Spider, signals -from scrapy.crawler import Crawler -from scrapy.statscollectors import StatsCollector if TYPE_CHECKING: # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy.crawler import Crawler + from scrapy.statscollectors import StatsCollector + class CoreStats: def __init__(self, stats: StatsCollector): diff --git a/scrapy/extensions/debug.py b/scrapy/extensions/debug.py index b360ce48d..c54871e02 100644 --- a/scrapy/extensions/debug.py +++ b/scrapy/extensions/debug.py @@ -12,17 +12,20 @@ import sys import threading import traceback from pdb import Pdb -from types import FrameType from typing import TYPE_CHECKING, Optional -from scrapy.crawler import Crawler from scrapy.utils.engine import format_engine_status from scrapy.utils.trackref import format_live_refs if TYPE_CHECKING: + from types import FrameType + # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy.crawler import Crawler + + logger = logging.getLogger(__name__) diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 941bd4b26..43c2d2815 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -31,18 +31,15 @@ from typing import ( ) from urllib.parse import unquote, urlparse -from twisted.internet import threads from twisted.internet.defer import Deferred, DeferredList, maybeDeferred -from twisted.python.failure import Failure +from twisted.internet.threads import deferToThread from w3lib.url import file_uri_to_path from zope.interface import Interface, implementer from scrapy import Spider, signals -from scrapy.crawler import Crawler from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning -from scrapy.exporters import BaseItemExporter from scrapy.extensions.postprocessing import PostProcessingManager -from scrapy.settings import BaseSettings, Settings +from scrapy.settings import Settings from scrapy.utils.boto import is_botocore_available from scrapy.utils.conf import feed_complete_default_values_from_settings from scrapy.utils.defer import maybe_deferred_to_future @@ -54,11 +51,14 @@ from scrapy.utils.python import without_none_values if TYPE_CHECKING: from _typeshed import OpenBinaryMode + from twisted.python.failure import Failure # typing.Self requires Python 3.11 from typing_extensions import Self -logger = logging.getLogger(__name__) + from scrapy.crawler import Crawler + from scrapy.exporters import BaseItemExporter + from scrapy.settings import BaseSettings try: import boto3 # noqa: F401 @@ -67,6 +67,9 @@ try: except ImportError: IS_BOTO3_AVAILABLE = False + +logger = logging.getLogger(__name__) + UriParamsCallableT = Callable[[Dict[str, Any], Spider], Optional[Dict[str, Any]]] _StorageT = TypeVar("_StorageT", bound="FeedStorageProtocol") @@ -160,7 +163,7 @@ class BlockingFeedStorage: return NamedTemporaryFile(prefix="feed-", dir=path) def store(self, file: IO[bytes]) -> Optional[Deferred]: - return threads.deferToThread(self._store_in_thread, file) + return deferToThread(self._store_in_thread, file) def _store_in_thread(self, file: IO[bytes]) -> None: raise NotImplementedError diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.py index b7219bf07..448d5f1ab 100644 --- a/scrapy/extensions/httpcache.py +++ b/scrapy/extensions/httpcache.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import gzip import logging import os @@ -13,10 +15,7 @@ from weakref import WeakKeyDictionary from w3lib.http import headers_dict_to_raw, headers_raw_to_dict from scrapy.http import Headers, Response -from scrapy.http.request import Request from scrapy.responsetypes import responsetypes -from scrapy.settings import BaseSettings -from scrapy.spiders import Spider from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.project import data_path from scrapy.utils.python import to_bytes, to_unicode @@ -26,6 +25,10 @@ if TYPE_CHECKING: # typing.Concatenate requires Python 3.10 from typing_extensions import Concatenate + from scrapy.http.request import Request + from scrapy.settings import BaseSettings + from scrapy.spiders import Spider + logger = logging.getLogger(__name__) diff --git a/scrapy/extensions/logstats.py b/scrapy/extensions/logstats.py index 2388afa75..c4f43482d 100644 --- a/scrapy/extensions/logstats.py +++ b/scrapy/extensions/logstats.py @@ -6,14 +6,16 @@ from typing import TYPE_CHECKING, Optional, Tuple, Union from twisted.internet import task from scrapy import Spider, signals -from scrapy.crawler import Crawler from scrapy.exceptions import NotConfigured -from scrapy.statscollectors import StatsCollector if TYPE_CHECKING: # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy.crawler import Crawler + from scrapy.statscollectors import StatsCollector + + logger = logging.getLogger(__name__) diff --git a/scrapy/extensions/memdebug.py b/scrapy/extensions/memdebug.py index f304e1bf2..3cbbb64e5 100644 --- a/scrapy/extensions/memdebug.py +++ b/scrapy/extensions/memdebug.py @@ -10,15 +10,16 @@ import gc from typing import TYPE_CHECKING from scrapy import Spider, signals -from scrapy.crawler import Crawler from scrapy.exceptions import NotConfigured -from scrapy.statscollectors import StatsCollector from scrapy.utils.trackref import live_refs if TYPE_CHECKING: # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy.crawler import Crawler + from scrapy.statscollectors import StatsCollector + class MemoryDebugger: def __init__(self, stats: StatsCollector): diff --git a/scrapy/extensions/memusage.py b/scrapy/extensions/memusage.py index 9de06b24d..25f63ecc6 100644 --- a/scrapy/extensions/memusage.py +++ b/scrapy/extensions/memusage.py @@ -16,7 +16,6 @@ from typing import TYPE_CHECKING, List from twisted.internet import task from scrapy import signals -from scrapy.crawler import Crawler from scrapy.exceptions import NotConfigured from scrapy.mail import MailSender from scrapy.utils.engine import get_engine_status @@ -25,6 +24,9 @@ if TYPE_CHECKING: # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy.crawler import Crawler + + logger = logging.getLogger(__name__) diff --git a/scrapy/extensions/periodic_log.py b/scrapy/extensions/periodic_log.py index 9567f948a..80c0a3b26 100644 --- a/scrapy/extensions/periodic_log.py +++ b/scrapy/extensions/periodic_log.py @@ -8,15 +8,17 @@ from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union from twisted.internet import task from scrapy import Spider, signals -from scrapy.crawler import Crawler from scrapy.exceptions import NotConfigured -from scrapy.statscollectors import StatsCollector from scrapy.utils.serialize import ScrapyJSONEncoder if TYPE_CHECKING: # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy.crawler import Crawler + from scrapy.statscollectors import StatsCollector + + logger = logging.getLogger(__name__) diff --git a/scrapy/extensions/spiderstate.py b/scrapy/extensions/spiderstate.py index c6eb20277..567efd7a1 100644 --- a/scrapy/extensions/spiderstate.py +++ b/scrapy/extensions/spiderstate.py @@ -5,7 +5,6 @@ from pathlib import Path from typing import TYPE_CHECKING, Optional from scrapy import Spider, signals -from scrapy.crawler import Crawler from scrapy.exceptions import NotConfigured from scrapy.utils.job import job_dir @@ -13,6 +12,8 @@ if TYPE_CHECKING: # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy.crawler import Crawler + class SpiderState: """Store and load spider state during a scraping job""" diff --git a/scrapy/extensions/statsmailer.py b/scrapy/extensions/statsmailer.py index 20b8f910c..e43de6f5c 100644 --- a/scrapy/extensions/statsmailer.py +++ b/scrapy/extensions/statsmailer.py @@ -8,18 +8,19 @@ from __future__ import annotations from typing import TYPE_CHECKING, List, Optional -from twisted.internet.defer import Deferred - from scrapy import Spider, signals -from scrapy.crawler import Crawler from scrapy.exceptions import NotConfigured from scrapy.mail import MailSender -from scrapy.statscollectors import StatsCollector if TYPE_CHECKING: + from twisted.internet.defer import Deferred + # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy.crawler import Crawler + from scrapy.statscollectors import StatsCollector + class StatsMailer: def __init__(self, stats: StatsCollector, recipients: List[str], mail: MailSender): diff --git a/scrapy/extensions/telnet.py b/scrapy/extensions/telnet.py index 00c69434c..c4e01b3d9 100644 --- a/scrapy/extensions/telnet.py +++ b/scrapy/extensions/telnet.py @@ -26,7 +26,6 @@ except (ImportError, SyntaxError): TWISTED_CONCH_AVAILABLE = False from scrapy import signals -from scrapy.crawler import Crawler from scrapy.exceptions import NotConfigured from scrapy.utils.decorators import defers from scrapy.utils.engine import print_engine_status @@ -36,6 +35,10 @@ from scrapy.utils.trackref import print_live_refs if TYPE_CHECKING: # typing.Self requires Python 3.11 from typing_extensions import Self + + from scrapy.crawler import Crawler + + logger = logging.getLogger(__name__) # signal to update telnet variables diff --git a/scrapy/extensions/throttle.py b/scrapy/extensions/throttle.py index 217e61a81..6ce9ce63a 100644 --- a/scrapy/extensions/throttle.py +++ b/scrapy/extensions/throttle.py @@ -4,15 +4,17 @@ import logging from typing import TYPE_CHECKING, Optional, Tuple from scrapy import Request, Spider, signals -from scrapy.core.downloader import Slot -from scrapy.crawler import Crawler from scrapy.exceptions import NotConfigured -from scrapy.http import Response if TYPE_CHECKING: # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy.core.downloader import Slot + from scrapy.crawler import Crawler + from scrapy.http import Response + + logger = logging.getLogger(__name__) diff --git a/scrapy/http/cookies.py b/scrapy/http/cookies.py index 8af89c74f..cc88a9420 100644 --- a/scrapy/http/cookies.py +++ b/scrapy/http/cookies.py @@ -17,8 +17,6 @@ from typing import ( cast, ) -from scrapy import Request -from scrapy.http import Response from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.python import to_unicode @@ -26,6 +24,10 @@ if TYPE_CHECKING: # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy import Request + from scrapy.http import Response + + # Defined in the http.cookiejar module, but undocumented: # https://github.com/python/cpython/blob/v3.9.0/Lib/http/cookiejar.py#L527 IPV4_RE = re.compile(r"\.\d+$", re.ASCII) diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index ea98ed795..a8c242e8b 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -28,13 +28,14 @@ from lxml.html import TextareaElement # nosec from w3lib.html import strip_html5_whitespace from scrapy.http.request import Request -from scrapy.http.response.text import TextResponse from scrapy.utils.python import is_listlike, to_bytes if TYPE_CHECKING: # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy.http.response.text import TextResponse + FormdataVType = Union[str, Iterable[str]] FormdataKVType = Tuple[str, FormdataVType] diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py index 684439097..ff3581abb 100644 --- a/scrapy/http/response/__init__.py +++ b/scrapy/http/response/__init__.py @@ -7,7 +7,6 @@ See documentation in docs/topics/request-response.rst from __future__ import annotations -from ipaddress import IPv4Address, IPv6Address from typing import ( TYPE_CHECKING, Any, @@ -26,8 +25,6 @@ from typing import ( ) from urllib.parse import urljoin -from twisted.internet.ssl import Certificate - from scrapy.exceptions import NotSupported from scrapy.http.headers import Headers from scrapy.http.request import CookiesT, Request @@ -35,6 +32,10 @@ from scrapy.link import Link from scrapy.utils.trackref import object_ref if TYPE_CHECKING: + from ipaddress import IPv4Address, IPv6Address + + from twisted.internet.ssl import Certificate + # typing.Self requires Python 3.11 from typing_extensions import Self diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py index df4d90829..0635f744f 100644 --- a/scrapy/http/response/text.py +++ b/scrapy/http/response/text.py @@ -35,15 +35,16 @@ from w3lib.encoding import ( ) from w3lib.html import strip_html5_whitespace -from scrapy.http.request import CookiesT, Request from scrapy.http.response import Response from scrapy.link import Link from scrapy.utils.python import memoizemethod_noargs, to_unicode from scrapy.utils.response import get_base_url if TYPE_CHECKING: + from scrapy.http.request import CookiesT, Request from scrapy.selector import Selector, SelectorList + _NONE = object() diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py index 33a10cd6c..d27a132b3 100644 --- a/scrapy/linkextractors/lxmlhtml.py +++ b/scrapy/linkextractors/lxmlhtml.py @@ -2,10 +2,13 @@ Link extractor based on lxml.html """ +from __future__ import annotations + import logging import operator from functools import partial from typing import ( + TYPE_CHECKING, Any, Callable, Iterable, @@ -20,13 +23,10 @@ from typing import ( from urllib.parse import urljoin, urlparse from lxml import etree # nosec -from lxml.html import HtmlElement # nosec from parsel.csstranslator import HTMLTranslator from w3lib.html import strip_html5_whitespace from w3lib.url import canonicalize_url, safe_url_string -from scrapy import Selector -from scrapy.http import TextResponse from scrapy.link import Link from scrapy.linkextractors import IGNORED_EXTENSIONS, _is_valid_url, _matches, re from scrapy.utils.misc import arg_to_iter, rel_has_nofollow @@ -34,6 +34,13 @@ from scrapy.utils.python import unique as unique_list from scrapy.utils.response import get_base_url from scrapy.utils.url import url_has_any_extension, url_is_from_any_domain +if TYPE_CHECKING: + from lxml.html import HtmlElement # nosec + + from scrapy import Selector + from scrapy.http import TextResponse + + logger = logging.getLogger(__name__) # from lxml/src/lxml/html/__init__.py diff --git a/scrapy/loader/__init__.py b/scrapy/loader/__init__.py index db0b4820f..9644cc093 100644 --- a/scrapy/loader/__init__.py +++ b/scrapy/loader/__init__.py @@ -4,14 +4,18 @@ Item Loader See documentation in docs/topics/loaders.rst """ -from typing import Any, Optional +from __future__ import annotations + +from typing import TYPE_CHECKING, Any, Optional import itemloaders -from scrapy.http import TextResponse from scrapy.item import Item from scrapy.selector import Selector +if TYPE_CHECKING: + from scrapy.http import TextResponse + class ItemLoader(itemloaders.ItemLoader): """ @@ -91,7 +95,7 @@ class ItemLoader(itemloaders.ItemLoader): selector: Optional[Selector] = None, response: Optional[TextResponse] = None, parent: Optional[itemloaders.ItemLoader] = None, - **context: Any + **context: Any, ): if selector is None and response is not None: try: diff --git a/scrapy/logformatter.py b/scrapy/logformatter.py index 42a03b560..601209fb0 100644 --- a/scrapy/logformatter.py +++ b/scrapy/logformatter.py @@ -6,8 +6,9 @@ from typing import TYPE_CHECKING, Any, Dict, Optional, Tuple, TypedDict, Union from twisted.python.failure import Failure -from scrapy import Request, Spider -from scrapy.http import Response +# working around https://github.com/sphinx-doc/sphinx/issues/10400 +from scrapy import Request, Spider # noqa: TC001 +from scrapy.http import Response # noqa: TC001 from scrapy.utils.request import referer_str if TYPE_CHECKING: diff --git a/scrapy/mail.py b/scrapy/mail.py index f4ce2800c..3ea20e831 100644 --- a/scrapy/mail.py +++ b/scrapy/mail.py @@ -30,20 +30,22 @@ from typing import ( from twisted import version as twisted_version from twisted.internet import ssl from twisted.internet.defer import Deferred -from twisted.python.failure import Failure from twisted.python.versions import Version -from scrapy.settings import BaseSettings from scrapy.utils.misc import arg_to_iter from scrapy.utils.python import to_bytes if TYPE_CHECKING: # imports twisted.internet.reactor from twisted.mail.smtp import ESMTPSenderFactory + from twisted.python.failure import Failure # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy.settings import BaseSettings + + logger = logging.getLogger(__name__) diff --git a/scrapy/middleware.py b/scrapy/middleware.py index f60c726f9..ea5488ba1 100644 --- a/scrapy/middleware.py +++ b/scrapy/middleware.py @@ -17,19 +17,19 @@ from typing import ( cast, ) -from twisted.internet.defer import Deferred - -from scrapy import Spider from scrapy.exceptions import NotConfigured -from scrapy.settings import Settings from scrapy.utils.defer import process_chain, process_parallel from scrapy.utils.misc import build_from_crawler, build_from_settings, load_object if TYPE_CHECKING: + from twisted.internet.defer import Deferred + # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy import Spider from scrapy.crawler import Crawler + from scrapy.settings import Settings logger = logging.getLogger(__name__) diff --git a/scrapy/pipelines/__init__.py b/scrapy/pipelines/__init__.py index 21d649e3c..480a5a58c 100644 --- a/scrapy/pipelines/__init__.py +++ b/scrapy/pipelines/__init__.py @@ -6,16 +6,18 @@ See documentation in docs/item-pipeline.rst from __future__ import annotations -from typing import Any, List +from typing import TYPE_CHECKING, Any, List -from twisted.internet.defer import Deferred - -from scrapy import Spider from scrapy.middleware import MiddlewareManager -from scrapy.settings import Settings from scrapy.utils.conf import build_component_list from scrapy.utils.defer import deferred_f_from_coro_f +if TYPE_CHECKING: + from twisted.internet.defer import Deferred + + from scrapy import Spider + from scrapy.settings import Settings + class ItemPipelineManager(MiddlewareManager): component_name = "item pipeline" diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index 85a8c77da..1a13aeaf2 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -16,7 +16,6 @@ from collections import defaultdict from contextlib import suppress from ftplib import FTP from io import BytesIO -from os import PathLike from pathlib import Path from typing import ( IO, @@ -38,11 +37,9 @@ from typing import ( from urllib.parse import urlparse from itemadapter import ItemAdapter -from twisted.internet import defer, threads -from twisted.internet.defer import Deferred -from twisted.python.failure import Failure +from twisted.internet.defer import Deferred, maybeDeferred +from twisted.internet.threads import deferToThread -from scrapy import Spider from scrapy.exceptions import IgnoreRequest, NotConfigured from scrapy.http import Request, Response from scrapy.http.request import NO_CALLBACK @@ -56,9 +53,15 @@ from scrapy.utils.python import to_bytes from scrapy.utils.request import referer_str if TYPE_CHECKING: + from os import PathLike + + from twisted.python.failure import Failure + # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy import Spider + logger = logging.getLogger(__name__) @@ -210,7 +213,7 @@ class S3FilesStore: key_name = f"{self.prefix}{path}" return cast( "Deferred[Dict[str, Any]]", - threads.deferToThread( + deferToThread( self.s3_client.head_object, Bucket=self.bucket, Key=key_name # type: ignore[attr-defined] ), ) @@ -229,7 +232,7 @@ class S3FilesStore: extra = self._headers_to_botocore_kwargs(self.HEADERS) if headers: extra.update(self._headers_to_botocore_kwargs(headers)) - return threads.deferToThread( + return deferToThread( self.s3_client.put_object, # type: ignore[attr-defined] Bucket=self.bucket, Key=key_name, @@ -326,9 +329,7 @@ class GCSFilesStore: blob_path = self._get_blob_path(path) return cast( Deferred[StatInfo], - threads.deferToThread(self.bucket.get_blob, blob_path).addCallback( - _onsuccess - ), + deferToThread(self.bucket.get_blob, blob_path).addCallback(_onsuccess), ) def _get_content_type(self, headers: Optional[Dict[str, str]]) -> str: @@ -351,7 +352,7 @@ class GCSFilesStore: blob = self.bucket.blob(blob_path) blob.cache_control = self.CACHE_CONTROL blob.metadata = {k: str(v) for k, v in (meta or {}).items()} - return threads.deferToThread( + return deferToThread( blob.upload_from_string, data=buf.getvalue(), content_type=self._get_content_type(headers), @@ -388,7 +389,7 @@ class FTPFilesStore: headers: Optional[Dict[str, str]] = None, ) -> Deferred[Any]: path = f"{self.basedir}/{path}" - return threads.deferToThread( + return deferToThread( ftp_store_file, path=path, file=buf, @@ -418,7 +419,7 @@ class FTPFilesStore: except Exception: return {} - return cast("Deferred[StatInfo]", threads.deferToThread(_stat_file, path)) + return cast("Deferred[StatInfo]", deferToThread(_stat_file, path)) class FilesPipeline(MediaPipeline): @@ -553,8 +554,8 @@ class FilesPipeline(MediaPipeline): } path = self.file_path(request, info=info, item=item) - # defer.maybeDeferred() overloads don't seem to support a Union[_T, Deferred[_T]] return type - dfd: Deferred[StatInfo] = defer.maybeDeferred(self.store.stat_file, path, info) # type: ignore[arg-type] + # maybeDeferred() overloads don't seem to support a Union[_T, Deferred[_T]] return type + dfd: Deferred[StatInfo] = maybeDeferred(self.store.stat_file, path, info) # type: ignore[arg-type] dfd2: Deferred[Optional[FileInfo]] = dfd.addCallback(_onsuccess) dfd2.addErrback(lambda _: None) dfd2.addErrback( diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py index 27a57b17c..166f81314 100644 --- a/scrapy/pipelines/images.py +++ b/scrapy/pipelines/images.py @@ -11,7 +11,6 @@ import hashlib import warnings from contextlib import suppress from io import BytesIO -from os import PathLike from typing import ( TYPE_CHECKING, Any, @@ -28,7 +27,6 @@ from typing import ( from itemadapter import ItemAdapter -from scrapy import Spider from scrapy.exceptions import DropItem, NotConfigured, ScrapyDeprecationWarning from scrapy.http import Request, Response from scrapy.http.request import NO_CALLBACK @@ -40,15 +38,20 @@ from scrapy.pipelines.files import ( S3FilesStore, _md5sum, ) -from scrapy.pipelines.media import FileInfoOrError, MediaPipeline from scrapy.settings import Settings from scrapy.utils.python import get_func_args, to_bytes if TYPE_CHECKING: - # typing.Self requires Python 3.11 + from os import PathLike + from PIL import Image + + # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy import Spider + from scrapy.pipelines.media import FileInfoOrError, MediaPipeline + class NoimagesDrop(DropItem): """Product with no images exception""" diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.py index 09e95cf5d..ea36a9e8a 100644 --- a/scrapy/pipelines/media.py +++ b/scrapy/pipelines/media.py @@ -25,21 +25,23 @@ from typing import ( from twisted.internet.defer import Deferred, DeferredList from twisted.python.failure import Failure -from scrapy import Spider -from scrapy.crawler import Crawler -from scrapy.http import Response from scrapy.http.request import NO_CALLBACK, Request from scrapy.settings import Settings from scrapy.utils.datatypes import SequenceExclude from scrapy.utils.defer import defer_result, mustbe_deferred from scrapy.utils.log import failure_to_exc_info from scrapy.utils.misc import arg_to_iter -from scrapy.utils.request import RequestFingerprinter if TYPE_CHECKING: # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy import Spider + from scrapy.crawler import Crawler + from scrapy.http import Response + from scrapy.utils.request import RequestFingerprinter + + _T = TypeVar("_T") diff --git a/scrapy/resolver.py b/scrapy/resolver.py index ba7cd716b..d5eedf9b1 100644 --- a/scrapy/resolver.py +++ b/scrapy/resolver.py @@ -4,7 +4,6 @@ from typing import TYPE_CHECKING, Any, List, Optional, Sequence, Type from twisted.internet import defer from twisted.internet.base import ReactorBase, ThreadedResolver -from twisted.internet.defer import Deferred from twisted.internet.interfaces import ( IAddress, IHostnameResolver, @@ -17,6 +16,8 @@ from zope.interface.declarations import implementer, provider from scrapy.utils.datatypes import LocalCache if TYPE_CHECKING: + from twisted.internet.defer import Deferred + # typing.Self requires Python 3.11 from typing_extensions import Self diff --git a/scrapy/robotstxt.py b/scrapy/robotstxt.py index a33f73306..0d282dc37 100644 --- a/scrapy/robotstxt.py +++ b/scrapy/robotstxt.py @@ -6,7 +6,6 @@ from abc import ABCMeta, abstractmethod from typing import TYPE_CHECKING, Optional, Union from warnings import warn -from scrapy import Spider from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.utils.python import to_unicode @@ -14,8 +13,10 @@ if TYPE_CHECKING: # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy import Spider from scrapy.crawler import Crawler + logger = logging.getLogger(__name__) diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py index ea1db03f1..6703c569f 100644 --- a/scrapy/settings/__init__.py +++ b/scrapy/settings/__init__.py @@ -4,7 +4,6 @@ import copy import json from importlib import import_module from pprint import pformat -from types import ModuleType from typing import ( TYPE_CHECKING, Any, @@ -27,6 +26,8 @@ from scrapy.settings import default_settings _SettingsKeyT = Union[bool, float, int, str, None] if TYPE_CHECKING: + from types import ModuleType + # https://github.com/python/typing/issues/445#issuecomment-1131458824 from _typeshed import SupportsItems diff --git a/scrapy/signalmanager.py b/scrapy/signalmanager.py index f6df191d8..3d37b8235 100644 --- a/scrapy/signalmanager.py +++ b/scrapy/signalmanager.py @@ -1,10 +1,14 @@ -from typing import Any, List, Tuple +from __future__ import annotations + +from typing import TYPE_CHECKING, Any, List, Tuple from pydispatch import dispatcher -from twisted.internet.defer import Deferred from scrapy.utils import signal as _signal +if TYPE_CHECKING: + from twisted.internet.defer import Deferred + class SignalManager: def __init__(self, sender: Any = dispatcher.Anonymous): diff --git a/scrapy/spiderloader.py b/scrapy/spiderloader.py index d855c962c..b8fe65668 100644 --- a/scrapy/spiderloader.py +++ b/scrapy/spiderloader.py @@ -3,21 +3,23 @@ from __future__ import annotations import traceback import warnings from collections import defaultdict -from types import ModuleType from typing import TYPE_CHECKING, DefaultDict, Dict, List, Tuple, Type from zope.interface import implementer -from scrapy import Request, Spider from scrapy.interfaces import ISpiderLoader -from scrapy.settings import BaseSettings from scrapy.utils.misc import walk_modules from scrapy.utils.spider import iter_spider_classes if TYPE_CHECKING: + from types import ModuleType + # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy import Request, Spider + from scrapy.settings import BaseSettings + @implementer(ISpiderLoader) class SpiderLoader: diff --git a/scrapy/spidermiddlewares/depth.py b/scrapy/spidermiddlewares/depth.py index 1e96654e2..c5b7f0749 100644 --- a/scrapy/spidermiddlewares/depth.py +++ b/scrapy/spidermiddlewares/depth.py @@ -9,15 +9,17 @@ from __future__ import annotations import logging from typing import TYPE_CHECKING, Any, AsyncIterable, Iterable -from scrapy import Spider -from scrapy.crawler import Crawler from scrapy.http import Request, Response -from scrapy.statscollectors import StatsCollector if TYPE_CHECKING: # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy import Spider + from scrapy.crawler import Crawler + from scrapy.statscollectors import StatsCollector + + logger = logging.getLogger(__name__) diff --git a/scrapy/spidermiddlewares/httperror.py b/scrapy/spidermiddlewares/httperror.py index 35c869a75..ea1686c25 100644 --- a/scrapy/spidermiddlewares/httperror.py +++ b/scrapy/spidermiddlewares/httperror.py @@ -9,16 +9,18 @@ from __future__ import annotations import logging from typing import TYPE_CHECKING, Any, Iterable, List, Optional -from scrapy import Spider -from scrapy.crawler import Crawler from scrapy.exceptions import IgnoreRequest -from scrapy.http import Response -from scrapy.settings import BaseSettings if TYPE_CHECKING: # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy import Spider + from scrapy.crawler import Crawler + from scrapy.http import Response + from scrapy.settings import BaseSettings + + logger = logging.getLogger(__name__) diff --git a/scrapy/spidermiddlewares/offsite.py b/scrapy/spidermiddlewares/offsite.py index 50c93ac9f..379c5d0a3 100644 --- a/scrapy/spidermiddlewares/offsite.py +++ b/scrapy/spidermiddlewares/offsite.py @@ -12,10 +12,8 @@ import warnings from typing import TYPE_CHECKING, Any, AsyncIterable, Iterable, Set from scrapy import Spider, signals -from scrapy.crawler import Crawler from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import Request, Response -from scrapy.statscollectors import StatsCollector from scrapy.utils.httpobj import urlparse_cached warnings.warn( @@ -28,6 +26,10 @@ if TYPE_CHECKING: # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy.crawler import Crawler + from scrapy.statscollectors import StatsCollector + + logger = logging.getLogger(__name__) diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py index 8af0bdf5b..d35cf8f71 100644 --- a/scrapy/spidermiddlewares/referer.py +++ b/scrapy/spidermiddlewares/referer.py @@ -23,10 +23,8 @@ from urllib.parse import urlparse from w3lib.url import safe_url_string from scrapy import Spider, signals -from scrapy.crawler import Crawler from scrapy.exceptions import NotConfigured from scrapy.http import Request, Response -from scrapy.settings import BaseSettings from scrapy.utils.misc import load_object from scrapy.utils.python import to_unicode from scrapy.utils.url import strip_url @@ -35,6 +33,10 @@ if TYPE_CHECKING: # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy.crawler import Crawler + from scrapy.settings import BaseSettings + + LOCAL_SCHEMES: Tuple[str, ...] = ( "about", "blob", diff --git a/scrapy/spidermiddlewares/urllength.py b/scrapy/spidermiddlewares/urllength.py index e2aa554a7..34df54ca7 100644 --- a/scrapy/spidermiddlewares/urllength.py +++ b/scrapy/spidermiddlewares/urllength.py @@ -9,15 +9,17 @@ from __future__ import annotations import logging from typing import TYPE_CHECKING, Any, AsyncIterable, Iterable -from scrapy import Spider from scrapy.exceptions import NotConfigured from scrapy.http import Request, Response -from scrapy.settings import BaseSettings if TYPE_CHECKING: # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy import Spider + from scrapy.settings import BaseSettings + + logger = logging.getLogger(__name__) diff --git a/scrapy/spiders/init.py b/scrapy/spiders/init.py index a0898a0cf..ce0f1bbaa 100644 --- a/scrapy/spiders/init.py +++ b/scrapy/spiders/init.py @@ -1,10 +1,14 @@ -from typing import Any, Iterable, Optional, cast +from __future__ import annotations + +from typing import TYPE_CHECKING, Any, Iterable, Optional, cast from scrapy import Request -from scrapy.http import Response from scrapy.spiders import Spider from scrapy.utils.spider import iterate_spider_output +if TYPE_CHECKING: + from scrapy.http import Response + class InitSpider(Spider): """Base Spider with initialization facilities""" diff --git a/scrapy/squeues.py b/scrapy/squeues.py index 6f80ee388..d3e7896c5 100644 --- a/scrapy/squeues.py +++ b/scrapy/squeues.py @@ -6,20 +6,22 @@ from __future__ import annotations import marshal import pickle # nosec -from os import PathLike from pathlib import Path from typing import TYPE_CHECKING, Any, Callable, Optional, Type, Union from queuelib import queue -from scrapy import Request -from scrapy.crawler import Crawler from scrapy.utils.request import request_from_dict if TYPE_CHECKING: + from os import PathLike + # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy import Request + from scrapy.crawler import Crawler + def _with_mkdir(queue_class: Type[queue.BaseQueue]) -> Type[queue.BaseQueue]: class DirectoriesCreated(queue_class): # type: ignore[valid-type,misc] diff --git a/scrapy/statscollectors.py b/scrapy/statscollectors.py index ab571a3ab..88e72f366 100644 --- a/scrapy/statscollectors.py +++ b/scrapy/statscollectors.py @@ -2,15 +2,17 @@ Scrapy extension for collecting scraping stats """ +from __future__ import annotations + import logging import pprint from typing import TYPE_CHECKING, Any, Dict, Optional -from scrapy import Spider - if TYPE_CHECKING: + from scrapy import Spider from scrapy.crawler import Crawler + logger = logging.getLogger(__name__) @@ -18,7 +20,7 @@ StatsT = Dict[str, Any] class StatsCollector: - def __init__(self, crawler: "Crawler"): + def __init__(self, crawler: Crawler): self._dump: bool = crawler.settings.getbool("STATS_DUMP") self._stats: StatsT = {} @@ -67,7 +69,7 @@ class StatsCollector: class MemoryStatsCollector(StatsCollector): - def __init__(self, crawler: "Crawler"): + def __init__(self, crawler: Crawler): super().__init__(crawler) self.spider_stats: Dict[str, StatsT] = {} diff --git a/scrapy/utils/decorators.py b/scrapy/utils/decorators.py index 7e82dd519..2240f0b58 100644 --- a/scrapy/utils/decorators.py +++ b/scrapy/utils/decorators.py @@ -4,8 +4,8 @@ import warnings from functools import wraps from typing import TYPE_CHECKING, Any, Callable, TypeVar -from twisted.internet import defer, threads -from twisted.internet.defer import Deferred +from twisted.internet.defer import Deferred, maybeDeferred +from twisted.internet.threads import deferToThread from scrapy.exceptions import ScrapyDeprecationWarning @@ -48,7 +48,7 @@ def defers(func: Callable[_P, _T]) -> Callable[_P, Deferred[_T]]: @wraps(func) def wrapped(*a: _P.args, **kw: _P.kwargs) -> Deferred[_T]: - return defer.maybeDeferred(func, *a, **kw) + return maybeDeferred(func, *a, **kw) return wrapped @@ -60,6 +60,6 @@ def inthread(func: Callable[_P, _T]) -> Callable[_P, Deferred[_T]]: @wraps(func) def wrapped(*a: _P.args, **kw: _P.kwargs) -> Deferred[_T]: - return threads.deferToThread(func, *a, **kw) + return deferToThread(func, *a, **kw) return wrapped diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py index 877eb4388..1d578e8a3 100644 --- a/scrapy/utils/defer.py +++ b/scrapy/utils/defer.py @@ -34,12 +34,13 @@ from twisted.internet import defer from twisted.internet.defer import Deferred, DeferredList, ensureDeferred from twisted.internet.task import Cooperator from twisted.python import failure -from twisted.python.failure import Failure from scrapy.exceptions import IgnoreRequest, ScrapyDeprecationWarning from scrapy.utils.reactor import _get_asyncio_event_loop, is_asyncio_reactor_installed if TYPE_CHECKING: + from twisted.python.failure import Failure + # typing.Concatenate and typing.ParamSpec require Python 3.10 from typing_extensions import Concatenate, ParamSpec diff --git a/scrapy/utils/engine.py b/scrapy/utils/engine.py index fdcf484d4..770ee0b1b 100644 --- a/scrapy/utils/engine.py +++ b/scrapy/utils/engine.py @@ -4,9 +4,10 @@ from __future__ import annotations # used in global tests code from time import time # noqa: F401 -from typing import Any, List, Tuple +from typing import TYPE_CHECKING, Any, List, Tuple -from scrapy.core.engine import ExecutionEngine +if TYPE_CHECKING: + from scrapy.core.engine import ExecutionEngine def get_engine_status(engine: ExecutionEngine) -> List[Tuple[str, Any]]: diff --git a/scrapy/utils/gz.py b/scrapy/utils/gz.py index 2e487d88b..85324361c 100644 --- a/scrapy/utils/gz.py +++ b/scrapy/utils/gz.py @@ -1,11 +1,15 @@ +from __future__ import annotations + import struct from gzip import GzipFile from io import BytesIO - -from scrapy.http import Response +from typing import TYPE_CHECKING from ._compression import _CHUNK_SIZE, _DecompressionMaxSizeExceeded +if TYPE_CHECKING: + from scrapy.http import Response + def gunzip(data: bytes, *, max_size: int = 0) -> bytes: """Gunzip the given data and return as much data as possible. diff --git a/scrapy/utils/httpobj.py b/scrapy/utils/httpobj.py index d502e8910..3cf9585ec 100644 --- a/scrapy/utils/httpobj.py +++ b/scrapy/utils/httpobj.py @@ -1,12 +1,16 @@ """Helper functions for scrapy.http objects (Request, Response)""" -from typing import Union +from __future__ import annotations + +from typing import TYPE_CHECKING, Union from urllib.parse import ParseResult, urlparse from weakref import WeakKeyDictionary -from scrapy.http import Request, Response +if TYPE_CHECKING: + from scrapy.http import Request, Response -_urlparse_cache: "WeakKeyDictionary[Union[Request, Response], ParseResult]" = ( + +_urlparse_cache: WeakKeyDictionary[Union[Request, Response], ParseResult] = ( WeakKeyDictionary() ) diff --git a/scrapy/utils/job.py b/scrapy/utils/job.py index e230e4235..488c7994b 100644 --- a/scrapy/utils/job.py +++ b/scrapy/utils/job.py @@ -1,7 +1,10 @@ -from pathlib import Path -from typing import Optional +from __future__ import annotations -from scrapy.settings import BaseSettings +from pathlib import Path +from typing import TYPE_CHECKING, Optional + +if TYPE_CHECKING: + from scrapy.settings import BaseSettings def job_dir(settings: BaseSettings) -> Optional[str]: diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py index cbfd170ed..439b065a9 100644 --- a/scrapy/utils/log.py +++ b/scrapy/utils/log.py @@ -21,12 +21,13 @@ from twisted.python import log as twisted_log from twisted.python.failure import Failure import scrapy -from scrapy.logformatter import LogFormatterResult from scrapy.settings import Settings, _SettingsKeyT from scrapy.utils.versions import scrapy_components_versions if TYPE_CHECKING: from scrapy.crawler import Crawler + from scrapy.logformatter import LogFormatterResult + logger = logging.getLogger(__name__) diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index 3d11c1035..3c787e50f 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -13,7 +13,6 @@ from contextlib import contextmanager from functools import partial from importlib import import_module from pkgutil import iter_modules -from types import ModuleType from typing import ( IO, TYPE_CHECKING, @@ -35,10 +34,13 @@ from scrapy.item import Item from scrapy.utils.datatypes import LocalWeakReferencedCache if TYPE_CHECKING: + from types import ModuleType + from scrapy import Spider from scrapy.crawler import Crawler from scrapy.settings import BaseSettings + _ITERABLE_SINGLE_VALUES = dict, Item, str, bytes T = TypeVar("T") diff --git a/scrapy/utils/project.py b/scrapy/utils/project.py index de3c8eaf9..efb6af299 100644 --- a/scrapy/utils/project.py +++ b/scrapy/utils/project.py @@ -1,7 +1,8 @@ +from __future__ import annotations + import os import warnings from importlib import import_module -from os import PathLike from pathlib import Path from typing import Union @@ -46,7 +47,7 @@ def project_data_dir(project: str = "default") -> str: return str(d) -def data_path(path: Union[str, PathLike], createdir: bool = False) -> str: +def data_path(path: Union[str, os.PathLike[str]], createdir: bool = False) -> str: """ Return the given path joined with the .scrapy data directory. If given an absolute path, return it unmodified. diff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.py index 5af6d22eb..a627db601 100644 --- a/scrapy/utils/reactor.py +++ b/scrapy/utils/reactor.py @@ -2,7 +2,6 @@ from __future__ import annotations import asyncio import sys -from asyncio import AbstractEventLoop, AbstractEventLoopPolicy from contextlib import suppress from typing import ( TYPE_CHECKING, @@ -20,13 +19,16 @@ from warnings import catch_warnings, filterwarnings, warn from twisted.internet import asyncioreactor, error from twisted.internet.base import DelayedCall -from twisted.internet.protocol import ServerFactory -from twisted.internet.tcp import Port from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.utils.misc import load_object if TYPE_CHECKING: + from asyncio import AbstractEventLoop, AbstractEventLoopPolicy + + from twisted.internet.protocol import ServerFactory + from twisted.internet.tcp import Port + # typing.ParamSpec requires Python 3.10 from typing_extensions import ParamSpec diff --git a/scrapy/utils/spider.py b/scrapy/utils/spider.py index b05135c04..ce754fad3 100644 --- a/scrapy/utils/spider.py +++ b/scrapy/utils/spider.py @@ -2,7 +2,6 @@ from __future__ import annotations import inspect import logging -from types import CoroutineType, ModuleType from typing import ( TYPE_CHECKING, Any, @@ -16,16 +15,19 @@ from typing import ( overload, ) -from twisted.internet.defer import Deferred - -from scrapy import Request from scrapy.spiders import Spider from scrapy.utils.defer import deferred_from_coro from scrapy.utils.misc import arg_to_iter if TYPE_CHECKING: + from types import CoroutineType, ModuleType + + from twisted.internet.defer import Deferred + + from scrapy import Request from scrapy.spiderloader import SpiderLoader + logger = logging.getLogger(__name__) _T = TypeVar("_T") diff --git a/scrapy/utils/ssl.py b/scrapy/utils/ssl.py index d520ef809..95611ebd9 100644 --- a/scrapy/utils/ssl.py +++ b/scrapy/utils/ssl.py @@ -1,12 +1,16 @@ -from typing import Any, Optional +from __future__ import annotations + +from typing import TYPE_CHECKING, Any, Optional import OpenSSL._util as pyOpenSSLutil import OpenSSL.SSL import OpenSSL.version -from OpenSSL.crypto import X509Name from scrapy.utils.python import to_unicode +if TYPE_CHECKING: + from OpenSSL.crypto import X509Name + def ffi_buf_to_string(buf: Any) -> str: return to_unicode(pyOpenSSLutil.ffi.string(buf)) diff --git a/scrapy/utils/template.py b/scrapy/utils/template.py index 6b22f3bfa..08f3f2dc9 100644 --- a/scrapy/utils/template.py +++ b/scrapy/utils/template.py @@ -1,10 +1,14 @@ """Helper functions for working with templates""" +from __future__ import annotations + import re import string -from os import PathLike from pathlib import Path -from typing import Any, Union +from typing import TYPE_CHECKING, Any, Union + +if TYPE_CHECKING: + from os import PathLike def render_templatefile(path: Union[str, PathLike], **kwargs: Any) -> None: diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py index 268d8d4be..fe2bfa042 100644 --- a/scrapy/utils/test.py +++ b/scrapy/utils/test.py @@ -2,21 +2,36 @@ This module contains some assorted functions used in tests """ +from __future__ import annotations + import asyncio import os from importlib import import_module from pathlib import Path from posixpath import split -from typing import Any, Awaitable, Dict, List, Optional, Tuple, Type, TypeVar +from typing import ( + TYPE_CHECKING, + Any, + Awaitable, + Dict, + List, + Optional, + Tuple, + Type, + TypeVar, +) from unittest import TestCase, mock -from twisted.internet.defer import Deferred from twisted.trial.unittest import SkipTest from scrapy import Spider from scrapy.crawler import Crawler from scrapy.utils.boto import is_botocore_available +if TYPE_CHECKING: + from twisted.internet.defer import Deferred + + _T = TypeVar("_T") diff --git a/scrapy/utils/testproc.py b/scrapy/utils/testproc.py index 3bdffcaa7..8882bfc5f 100644 --- a/scrapy/utils/testproc.py +++ b/scrapy/utils/testproc.py @@ -2,12 +2,14 @@ from __future__ import annotations import os import sys -from typing import Iterable, List, Optional, Tuple, cast +from typing import TYPE_CHECKING, Iterable, List, Optional, Tuple, cast from twisted.internet.defer import Deferred from twisted.internet.error import ProcessTerminated from twisted.internet.protocol import ProcessProtocol -from twisted.python.failure import Failure + +if TYPE_CHECKING: + from twisted.python.failure import Failure class ProcessTest: diff --git a/tests/mockserver.py b/tests/mockserver.py index 233f6b934..6ec46aa3d 100644 --- a/tests/mockserver.py +++ b/tests/mockserver.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import argparse import json import os @@ -7,12 +9,11 @@ from pathlib import Path from shutil import rmtree from subprocess import PIPE, Popen from tempfile import mkdtemp -from typing import Dict +from typing import TYPE_CHECKING, Dict from urllib.parse import urlencode from OpenSSL import SSL from twisted.internet import defer, reactor, ssl -from twisted.internet.protocol import ServerFactory from twisted.internet.task import deferLater from twisted.names import dns, error from twisted.names.server import DNSServerFactory @@ -23,6 +24,9 @@ from twisted.web.util import redirectTo from scrapy.utils.python import to_bytes, to_unicode +if TYPE_CHECKING: + from twisted.internet.protocol import ServerFactory + def getarg(request, name, default=None, type=None): if name in request.args: diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 253987e15..ea3ed3b05 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import bz2 import csv import gzip @@ -14,10 +16,9 @@ from collections import defaultdict from contextlib import ExitStack from io import BytesIO from logging import getLogger -from os import PathLike from pathlib import Path from string import ascii_letters, digits -from typing import Union +from typing import TYPE_CHECKING, Union from unittest import mock from urllib.parse import quote, urljoin from urllib.request import pathname2url @@ -53,6 +54,9 @@ from scrapy.utils.test import get_crawler, mock_google_cloud_storage, skip_if_no from tests.mockserver import MockFTPServer, MockServer from tests.spiders import ItemSpider +if TYPE_CHECKING: + from os import PathLike + def path_to_url(path): return urljoin("file:", pathname2url(str(path))) diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py index 995c02a1a..7ea3fe8c9 100644 --- a/tests/test_http2_client_protocol.py +++ b/tests/test_http2_client_protocol.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import json import random import re @@ -6,7 +8,7 @@ import string from ipaddress import IPv4Address from pathlib import Path from tempfile import mkdtemp -from typing import Dict +from typing import TYPE_CHECKING, Dict from unittest import mock, skipIf from urllib.parse import urlencode @@ -20,7 +22,6 @@ from twisted.internet.defer import ( from twisted.internet.endpoints import SSL4ClientEndpoint, SSL4ServerEndpoint from twisted.internet.error import TimeoutError from twisted.internet.ssl import Certificate, PrivateCertificate, optionsForClientTLS -from twisted.python.failure import Failure from twisted.trial.unittest import TestCase from twisted.web.client import URI, ResponseFailed from twisted.web.http import H2_ENABLED @@ -33,6 +34,9 @@ from scrapy.settings import Settings from scrapy.spiders import Spider from tests.mockserver import LeafResource, Status, ssl_context_factory +if TYPE_CHECKING: + from twisted.python.failure import Failure + def generate_random_string(size): return "".join(random.choices(string.ascii_uppercase + string.digits, k=size))