From f65e64a7243d725d35bbf86ca6f5ae4c350dbcc5 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 16 Oct 2024 21:38:43 +0500 Subject: [PATCH] Misc typing improvements. (#6494) --- scrapy/commands/check.py | 5 ++- scrapy/commands/genspider.py | 4 +-- scrapy/commands/parse.py | 6 ++-- scrapy/core/engine.py | 4 +-- scrapy/core/scraper.py | 6 ++-- scrapy/core/spidermw.py | 4 +-- scrapy/crawler.py | 34 +++++++++---------- .../downloadermiddlewares/httpcompression.py | 2 +- scrapy/downloadermiddlewares/robotstxt.py | 2 +- scrapy/extensions/feedexport.py | 2 +- scrapy/pipelines/files.py | 2 +- scrapy/utils/defer.py | 6 +++- tox.ini | 8 ++--- 13 files changed, 46 insertions(+), 39 deletions(-) diff --git a/scrapy/commands/check.py b/scrapy/commands/check.py index c7946605b..1ce155da7 100644 --- a/scrapy/commands/check.py +++ b/scrapy/commands/check.py @@ -13,8 +13,7 @@ from scrapy.utils.misc import load_object, set_environ class TextTestResult(_TextTestResult): def printSummary(self, start: float, stop: float) -> None: write = self.stream.write - # _WritelnDecorator isn't implemented in typeshed yet - writeln = self.stream.writeln # type: ignore[attr-defined] + writeln = self.stream.writeln run = self.testsRun plural = "s" if run != 1 else "" @@ -84,7 +83,7 @@ class Command(ScrapyCommand): with set_environ(SCRAPY_CHECK="true"): for spidername in args or spider_loader.list(): spidercls = spider_loader.load(spidername) - spidercls.start_requests = lambda s: conman.from_spider(s, result) + spidercls.start_requests = lambda s: conman.from_spider(s, result) # type: ignore[assignment,method-assign,return-value] tested_methods = conman.tested_methods_from_spidercls(spidercls) if opts.list: diff --git a/scrapy/commands/genspider.py b/scrapy/commands/genspider.py index a9b7a6eee..2ac281212 100644 --- a/scrapy/commands/genspider.py +++ b/scrapy/commands/genspider.py @@ -4,7 +4,7 @@ import shutil import string from importlib import import_module from pathlib import Path -from typing import Optional, Union, cast +from typing import Any, Optional, Union, cast from urllib.parse import urlparse import scrapy @@ -122,7 +122,7 @@ class Command(ScrapyCommand): name: str, url: str, template_name: str, - ): + ) -> dict[str, Any]: capitalized_module = "".join(s.capitalize() for s in module.split("_")) return { "project_name": self.settings.get("BOT_NAME"), diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py index bd1fad14b..ff2bb8ab9 100644 --- a/scrapy/commands/parse.py +++ b/scrapy/commands/parse.py @@ -38,9 +38,10 @@ _T = TypeVar("_T") class Command(BaseRunSpiderCommand): requires_project = True - spider = None + spider: Optional[Spider] = None items: dict[int, list[Any]] = {} requests: dict[int, list[Request]] = {} + spidercls: Optional[type[Spider]] first_response = None @@ -261,10 +262,11 @@ class Command(BaseRunSpiderCommand): yield self.prepare_request(spider, Request(url), opts) if self.spidercls: - self.spidercls.start_requests = _start_requests + self.spidercls.start_requests = _start_requests # type: ignore[assignment,method-assign] def start_parsing(self, url: str, opts: argparse.Namespace) -> None: assert self.crawler_process + assert self.spidercls self.crawler_process.crawl(self.spidercls, **opts.spargs) self.pcrawler = list(self.crawler_process.crawlers)[0] self.crawler_process.start() diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index bb09d066f..f3d74eccf 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -100,7 +100,7 @@ class ExecutionEngine: ) downloader_cls: type[Downloader] = load_object(self.settings["DOWNLOADER"]) self.downloader: Downloader = downloader_cls(crawler) - self.scraper = Scraper(crawler) + self.scraper: Scraper = Scraper(crawler) self._spider_closed_callback: Callable[[Spider], Optional[Deferred[None]]] = ( spider_closed_callback ) @@ -325,7 +325,7 @@ class ExecutionEngine: raise RuntimeError(f"No open spider to crawl: {request}") d: Deferred[Union[Response, Request]] = self._download(request) # Deferred.addBoth() overloads don't seem to support a Union[_T, Deferred[_T]] return type - d2: Deferred[Response] = d.addBoth(self._downloaded, request) # type: ignore[arg-type] + d2: Deferred[Response] = d.addBoth(self._downloaded, request) # type: ignore[call-overload] return d2 def _downloaded( diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index 29d7cb0c8..71a0d6aeb 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -55,7 +55,7 @@ class Slot: MIN_RESPONSE_SIZE = 1024 def __init__(self, max_active_size: int = 5000000): - self.max_active_size = max_active_size + self.max_active_size: int = max_active_size self.queue: deque[QueueTuple] = deque() self.active: set[Request] = set() self.active_size: int = 0 @@ -316,7 +316,9 @@ class Scraper: ) return None - def start_itemproc(self, item, *, response: Optional[Response]) -> Deferred[Any]: + def start_itemproc( + self, item: Any, *, response: Optional[Response] + ) -> Deferred[Any]: """Send *item* to the item pipelines for processing. *response* is the source of the item data. If the item does not come diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index 223e4192e..3c8513042 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -72,7 +72,7 @@ class SpiderMiddlewareManager(MiddlewareManager): def _process_spider_input( self, - scrape_func: ScrapeFunc, + scrape_func: ScrapeFunc[_T], response: Response, request: Request, spider: Spider, @@ -306,7 +306,7 @@ class SpiderMiddlewareManager(MiddlewareManager): def scrape_response( self, - scrape_func: ScrapeFunc, + scrape_func: ScrapeFunc[_T], response: Response, request: Request, spider: Spider, diff --git a/scrapy/crawler.py b/scrapy/crawler.py index b0a4932e1..e75ef52ac 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -42,8 +42,9 @@ from scrapy.utils.reactor import ( ) if TYPE_CHECKING: - from collections.abc import Generator + from collections.abc import Generator, Iterable + from scrapy.spiderloader import SpiderLoader from scrapy.utils.request import RequestFingerprinter @@ -178,16 +179,18 @@ class Crawler: yield maybeDeferred(self.engine.stop) @staticmethod - def _get_component(component_class, components): + def _get_component( + component_class: type[_T], components: Iterable[Any] + ) -> Optional[_T]: for component in components: if isinstance(component, component_class): return component return None - def get_addon(self, cls): + def get_addon(self, cls: type[_T]) -> Optional[_T]: return self._get_component(cls, self.addons.addons) - def get_downloader_middleware(self, cls): + def get_downloader_middleware(self, cls: type[_T]) -> Optional[_T]: if not self.engine: raise RuntimeError( "Crawler.get_downloader_middleware() can only be called after " @@ -195,7 +198,7 @@ class Crawler: ) return self._get_component(cls, self.engine.downloader.middleware.middlewares) - def get_extension(self, cls): + def get_extension(self, cls: type[_T]) -> Optional[_T]: if not self.extensions: raise RuntimeError( "Crawler.get_extension() can only be called after the " @@ -203,7 +206,7 @@ class Crawler: ) return self._get_component(cls, self.extensions.middlewares) - def get_item_pipeline(self, cls): + def get_item_pipeline(self, cls: type[_T]) -> Optional[_T]: if not self.engine: raise RuntimeError( "Crawler.get_item_pipeline() can only be called after the " @@ -211,7 +214,7 @@ class Crawler: ) return self._get_component(cls, self.engine.scraper.itemproc.middlewares) - def get_spider_middleware(self, cls): + def get_spider_middleware(self, cls: type[_T]) -> Optional[_T]: if not self.engine: raise RuntimeError( "Crawler.get_spider_middleware() can only be called after the " @@ -240,18 +243,18 @@ class CrawlerRunner: ) @staticmethod - def _get_spider_loader(settings: BaseSettings): + def _get_spider_loader(settings: BaseSettings) -> SpiderLoader: """Get SpiderLoader instance from settings""" cls_path = settings.get("SPIDER_LOADER_CLASS") loader_cls = load_object(cls_path) verifyClass(ISpiderLoader, loader_cls) - return loader_cls.from_settings(settings.frozencopy()) + return cast("SpiderLoader", loader_cls.from_settings(settings.frozencopy())) def __init__(self, settings: Union[dict[str, Any], Settings, None] = None): if isinstance(settings, dict) or settings is None: settings = Settings(settings) - self.settings = settings - self.spider_loader = self._get_spider_loader(settings) + self.settings: Settings = settings + self.spider_loader: SpiderLoader = self._get_spider_loader(settings) self._crawlers: set[Crawler] = set() self._active: set[Deferred[None]] = set() self.bootstrap_failed = False @@ -329,8 +332,7 @@ class CrawlerRunner: def _create_crawler(self, spidercls: Union[str, type[Spider]]) -> Crawler: if isinstance(spidercls, str): spidercls = self.spider_loader.load(spidercls) - # temporary cast until self.spider_loader is typed - return Crawler(cast(type[Spider], spidercls), self.settings) + return Crawler(spidercls, self.settings) def stop(self) -> Deferred[Any]: """ @@ -384,7 +386,7 @@ class CrawlerProcess(CrawlerRunner): super().__init__(settings) configure_logging(self.settings, install_root_handler) log_scrapy_info(self.settings) - self._initialized_reactor = False + self._initialized_reactor: bool = False def _signal_shutdown(self, signum: int, _: Any) -> None: from twisted.internet import reactor @@ -413,9 +415,7 @@ class CrawlerProcess(CrawlerRunner): init_reactor = not self._initialized_reactor self._initialized_reactor = True # temporary cast until self.spider_loader is typed - return Crawler( - cast(type[Spider], spidercls), self.settings, init_reactor=init_reactor - ) + return Crawler(spidercls, self.settings, init_reactor=init_reactor) def start( self, stop_after_crawl: bool = True, install_signal_handlers: bool = True diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py index b0cede97d..d913ca25d 100644 --- a/scrapy/downloadermiddlewares/httpcompression.py +++ b/scrapy/downloadermiddlewares/httpcompression.py @@ -88,7 +88,7 @@ class HttpCompressionMiddleware: crawler.signals.connect(mw.open_spider, signals.spider_opened) return mw - def open_spider(self, spider): + def open_spider(self, spider: Spider) -> None: if hasattr(spider, "download_maxsize"): self._max_size = spider.download_maxsize if hasattr(spider, "download_warnsize"): diff --git a/scrapy/downloadermiddlewares/robotstxt.py b/scrapy/downloadermiddlewares/robotstxt.py index 421c58e68..81ba009d6 100644 --- a/scrapy/downloadermiddlewares/robotstxt.py +++ b/scrapy/downloadermiddlewares/robotstxt.py @@ -67,7 +67,7 @@ class RobotsTxtMiddleware: if request.url.startswith("data:") or request.url.startswith("file:"): return None d: Deferred[Optional[RobotParser]] = maybeDeferred( - self.robot_parser, request, spider # type: ignore[arg-type] + self.robot_parser, request, spider # type: ignore[call-overload] ) d2: Deferred[None] = d.addCallback(self.process_request_2, request, spider) return d2 diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index b1001dabb..7bfcbe6f3 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -578,7 +578,7 @@ class FeedExporter: return None logmsg = f"{slot.format} feed ({slot.itemcount} items) in: {slot.uri}" - d: Deferred[None] = maybeDeferred(slot.storage.store, get_file(slot)) # type: ignore[arg-type] + d: Deferred[None] = maybeDeferred(slot.storage.store, get_file(slot)) # type: ignore[call-overload] d.addCallback( self._handle_store_success, logmsg, spider, type(slot.storage).__name__ diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index 9314856c1..32e9ffe7c 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -550,7 +550,7 @@ class FilesPipeline(MediaPipeline): path = self.file_path(request, info=info, item=item) # maybeDeferred() overloads don't seem to support a Union[_T, Deferred[_T]] return type - dfd: Deferred[StatInfo] = maybeDeferred(self.store.stat_file, path, info) # type: ignore[arg-type] + dfd: Deferred[StatInfo] = maybeDeferred(self.store.stat_file, path, info) # type: ignore[call-overload] dfd2: Deferred[Optional[FileInfo]] = dfd.addCallback(_onsuccess) dfd2.addErrback(lambda _: None) dfd2.addErrback( diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py index 3a0dee8f1..aeacadb1c 100644 --- a/scrapy/utils/defer.py +++ b/scrapy/utils/defer.py @@ -305,7 +305,11 @@ def process_parallel( dfds, fireOnOneErrback=True, consumeErrors=True ) d2: Deferred[list[_T2]] = d.addCallback(lambda r: [x[1] for x in r]) - d2.addErrback(lambda f: f.value.subFailure) + + def eb(failure: Failure) -> Failure: + return failure.value.subFailure + + d2.addErrback(eb) return d2 diff --git a/tox.ini b/tox.ini index dad15c6ab..79f72a0f2 100644 --- a/tox.ini +++ b/tox.ini @@ -46,12 +46,12 @@ install_command = [testenv:typing] basepython = python3 deps = - mypy==1.11.1 + mypy==1.12.0 typing-extensions==4.12.2 - types-lxml==2024.8.7 + types-lxml==2024.9.16 types-Pygments==2.18.0.20240506 - botocore-stubs==1.34.158 - boto3-stubs[s3]==1.34.158 + botocore-stubs==1.35.39 + boto3-stubs[s3]==1.35.39 attrs >= 18.2.0 Pillow >= 10.3.0 pyOpenSSL >= 24.2.1