RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Misc typing improvements. (#6494)

This commit is contained in:
Andrey Rakhmatullin 2024-10-16 21:38:43 +05:00 committed by GitHub
parent 5391663072
commit f65e64a724
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
13 changed files with 46 additions and 39 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
scrapy/commands/check.py
View File

@ -13,8 +13,7 @@ from scrapy.utils.misc import load_object, set_environ
class TextTestResult(_TextTestResult):
def printSummary(self, start: float, stop: float) -> None:
write = self.stream.write
# _WritelnDecorator isn't implemented in typeshed yet
writeln = self.stream.writeln # type: ignore[attr-defined]
writeln = self.stream.writeln
run = self.testsRun
plural = "s" if run != 1 else ""
@ -84,7 +83,7 @@ class Command(ScrapyCommand):
with set_environ(SCRAPY_CHECK="true"):
for spidername in args or spider_loader.list():
spidercls = spider_loader.load(spidername)
spidercls.start_requests = lambda s: conman.from_spider(s, result)
spidercls.start_requests = lambda s: conman.from_spider(s, result) # type: ignore[assignment,method-assign,return-value]
tested_methods = conman.tested_methods_from_spidercls(spidercls)
if opts.list:

4
scrapy/commands/genspider.py
View File

@ -4,7 +4,7 @@ import shutil
import string
from importlib import import_module
from pathlib import Path
from typing import Optional, Union, cast
from typing import Any, Optional, Union, cast
from urllib.parse import urlparse
import scrapy
@ -122,7 +122,7 @@ class Command(ScrapyCommand):
name: str,
url: str,
template_name: str,
):
) -> dict[str, Any]:
capitalized_module = "".join(s.capitalize() for s in module.split("_"))
return {
"project_name": self.settings.get("BOT_NAME"),

6
scrapy/commands/parse.py
View File

@ -38,9 +38,10 @@ _T = TypeVar("_T")
class Command(BaseRunSpiderCommand):
requires_project = True
spider = None
spider: Optional[Spider] = None
items: dict[int, list[Any]] = {}
requests: dict[int, list[Request]] = {}
spidercls: Optional[type[Spider]]
first_response = None
@ -261,10 +262,11 @@ class Command(BaseRunSpiderCommand):
yield self.prepare_request(spider, Request(url), opts)
if self.spidercls:
self.spidercls.start_requests = _start_requests
self.spidercls.start_requests = _start_requests # type: ignore[assignment,method-assign]
def start_parsing(self, url: str, opts: argparse.Namespace) -> None:
assert self.crawler_process
assert self.spidercls
self.crawler_process.crawl(self.spidercls, **opts.spargs)
self.pcrawler = list(self.crawler_process.crawlers)[0]
self.crawler_process.start()

4
scrapy/core/engine.py
View File

@ -100,7 +100,7 @@ class ExecutionEngine:
)
downloader_cls: type[Downloader] = load_object(self.settings["DOWNLOADER"])
self.downloader: Downloader = downloader_cls(crawler)
self.scraper = Scraper(crawler)
self.scraper: Scraper = Scraper(crawler)
self._spider_closed_callback: Callable[[Spider], Optional[Deferred[None]]] = (
spider_closed_callback
)
@ -325,7 +325,7 @@ class ExecutionEngine:
raise RuntimeError(f"No open spider to crawl: {request}")
d: Deferred[Union[Response, Request]] = self._download(request)
# Deferred.addBoth() overloads don't seem to support a Union[_T, Deferred[_T]] return type
d2: Deferred[Response] = d.addBoth(self._downloaded, request) # type: ignore[arg-type]
d2: Deferred[Response] = d.addBoth(self._downloaded, request) # type: ignore[call-overload]
return d2
def _downloaded(

6
scrapy/core/scraper.py
View File

@ -55,7 +55,7 @@ class Slot:
MIN_RESPONSE_SIZE = 1024
def __init__(self, max_active_size: int = 5000000):
self.max_active_size = max_active_size
self.max_active_size: int = max_active_size
self.queue: deque[QueueTuple] = deque()
self.active: set[Request] = set()
self.active_size: int = 0
@ -316,7 +316,9 @@ class Scraper:
)
return None
def start_itemproc(self, item, *, response: Optional[Response]) -> Deferred[Any]:
def start_itemproc(
self, item: Any, *, response: Optional[Response]
) -> Deferred[Any]:
"""Send *item* to the item pipelines for processing.
*response* is the source of the item data. If the item does not come

4
scrapy/core/spidermw.py
View File

@ -72,7 +72,7 @@ class SpiderMiddlewareManager(MiddlewareManager):
def _process_spider_input(
self,
scrape_func: ScrapeFunc,
scrape_func: ScrapeFunc[_T],
response: Response,
request: Request,
spider: Spider,
@ -306,7 +306,7 @@ class SpiderMiddlewareManager(MiddlewareManager):
def scrape_response(
self,
scrape_func: ScrapeFunc,
scrape_func: ScrapeFunc[_T],
response: Response,
request: Request,
spider: Spider,

34
scrapy/crawler.py
View File

@ -42,8 +42,9 @@ from scrapy.utils.reactor import (
)
if TYPE_CHECKING:
from collections.abc import Generator
from collections.abc import Generator, Iterable
from scrapy.spiderloader import SpiderLoader
from scrapy.utils.request import RequestFingerprinter
@ -178,16 +179,18 @@ class Crawler:
yield maybeDeferred(self.engine.stop)
@staticmethod
def _get_component(component_class, components):
def _get_component(
component_class: type[_T], components: Iterable[Any]
) -> Optional[_T]:
for component in components:
if isinstance(component, component_class):
return component
return None
def get_addon(self, cls):
def get_addon(self, cls: type[_T]) -> Optional[_T]:
return self._get_component(cls, self.addons.addons)
def get_downloader_middleware(self, cls):
def get_downloader_middleware(self, cls: type[_T]) -> Optional[_T]:
if not self.engine:
raise RuntimeError(
"Crawler.get_downloader_middleware() can only be called after "
@ -195,7 +198,7 @@ class Crawler:
)
return self._get_component(cls, self.engine.downloader.middleware.middlewares)
def get_extension(self, cls):
def get_extension(self, cls: type[_T]) -> Optional[_T]:
if not self.extensions:
raise RuntimeError(
"Crawler.get_extension() can only be called after the "
@ -203,7 +206,7 @@ class Crawler:
)
return self._get_component(cls, self.extensions.middlewares)
def get_item_pipeline(self, cls):
def get_item_pipeline(self, cls: type[_T]) -> Optional[_T]:
if not self.engine:
raise RuntimeError(
"Crawler.get_item_pipeline() can only be called after the "
@ -211,7 +214,7 @@ class Crawler:
)
return self._get_component(cls, self.engine.scraper.itemproc.middlewares)
def get_spider_middleware(self, cls):
def get_spider_middleware(self, cls: type[_T]) -> Optional[_T]:
if not self.engine:
raise RuntimeError(
"Crawler.get_spider_middleware() can only be called after the "
@ -240,18 +243,18 @@ class CrawlerRunner:
)
@staticmethod
def _get_spider_loader(settings: BaseSettings):
def _get_spider_loader(settings: BaseSettings) -> SpiderLoader:
"""Get SpiderLoader instance from settings"""
cls_path = settings.get("SPIDER_LOADER_CLASS")
loader_cls = load_object(cls_path)
verifyClass(ISpiderLoader, loader_cls)
return loader_cls.from_settings(settings.frozencopy())
return cast("SpiderLoader", loader_cls.from_settings(settings.frozencopy()))
def __init__(self, settings: Union[dict[str, Any], Settings, None] = None):
if isinstance(settings, dict) or settings is None:
settings = Settings(settings)
self.settings = settings
self.spider_loader = self._get_spider_loader(settings)
self.settings: Settings = settings
self.spider_loader: SpiderLoader = self._get_spider_loader(settings)
self._crawlers: set[Crawler] = set()
self._active: set[Deferred[None]] = set()
self.bootstrap_failed = False
@ -329,8 +332,7 @@ class CrawlerRunner:
def _create_crawler(self, spidercls: Union[str, type[Spider]]) -> Crawler:
if isinstance(spidercls, str):
spidercls = self.spider_loader.load(spidercls)
# temporary cast until self.spider_loader is typed
return Crawler(cast(type[Spider], spidercls), self.settings)
return Crawler(spidercls, self.settings)
def stop(self) -> Deferred[Any]:
"""
@ -384,7 +386,7 @@ class CrawlerProcess(CrawlerRunner):
super().__init__(settings)
configure_logging(self.settings, install_root_handler)
log_scrapy_info(self.settings)
self._initialized_reactor = False
self._initialized_reactor: bool = False
def _signal_shutdown(self, signum: int, _: Any) -> None:
from twisted.internet import reactor
@ -413,9 +415,7 @@ class CrawlerProcess(CrawlerRunner):
init_reactor = not self._initialized_reactor
self._initialized_reactor = True
# temporary cast until self.spider_loader is typed
return Crawler(
cast(type[Spider], spidercls), self.settings, init_reactor=init_reactor
)
return Crawler(spidercls, self.settings, init_reactor=init_reactor)
def start(
self, stop_after_crawl: bool = True, install_signal_handlers: bool = True

2
scrapy/downloadermiddlewares/httpcompression.py
View File

@ -88,7 +88,7 @@ class HttpCompressionMiddleware:
crawler.signals.connect(mw.open_spider, signals.spider_opened)
return mw
def open_spider(self, spider):
def open_spider(self, spider: Spider) -> None:
if hasattr(spider, "download_maxsize"):
self._max_size = spider.download_maxsize
if hasattr(spider, "download_warnsize"):

2
scrapy/downloadermiddlewares/robotstxt.py
View File

@ -67,7 +67,7 @@ class RobotsTxtMiddleware:
if request.url.startswith("data:") or request.url.startswith("file:"):
return None
d: Deferred[Optional[RobotParser]] = maybeDeferred(
self.robot_parser, request, spider # type: ignore[arg-type]
self.robot_parser, request, spider # type: ignore[call-overload]
)
d2: Deferred[None] = d.addCallback(self.process_request_2, request, spider)
return d2

2
scrapy/extensions/feedexport.py
View File

@ -578,7 +578,7 @@ class FeedExporter:
return None
logmsg = f"{slot.format} feed ({slot.itemcount} items) in: {slot.uri}"
d: Deferred[None] = maybeDeferred(slot.storage.store, get_file(slot)) # type: ignore[arg-type]
d: Deferred[None] = maybeDeferred(slot.storage.store, get_file(slot)) # type: ignore[call-overload]
d.addCallback(
self._handle_store_success, logmsg, spider, type(slot.storage).__name__

2
scrapy/pipelines/files.py
View File

@ -550,7 +550,7 @@ class FilesPipeline(MediaPipeline):
path = self.file_path(request, info=info, item=item)
# maybeDeferred() overloads don't seem to support a Union[_T, Deferred[_T]] return type
dfd: Deferred[StatInfo] = maybeDeferred(self.store.stat_file, path, info) # type: ignore[arg-type]
dfd: Deferred[StatInfo] = maybeDeferred(self.store.stat_file, path, info) # type: ignore[call-overload]
dfd2: Deferred[Optional[FileInfo]] = dfd.addCallback(_onsuccess)
dfd2.addErrback(lambda _: None)
dfd2.addErrback(

6
scrapy/utils/defer.py
View File

@ -305,7 +305,11 @@ def process_parallel(
dfds, fireOnOneErrback=True, consumeErrors=True
)
d2: Deferred[list[_T2]] = d.addCallback(lambda r: [x[1] for x in r])
d2.addErrback(lambda f: f.value.subFailure)
def eb(failure: Failure) -> Failure:
return failure.value.subFailure
d2.addErrback(eb)
return d2

8
tox.ini
View File

@ -46,12 +46,12 @@ install_command =
[testenv:typing]
basepython = python3
deps =
mypy==1.11.1
mypy==1.12.0
typing-extensions==4.12.2
types-lxml==2024.8.7
types-lxml==2024.9.16
types-Pygments==2.18.0.20240506
botocore-stubs==1.34.158
boto3-stubs[s3]==1.34.158
botocore-stubs==1.35.39
boto3-stubs[s3]==1.35.39
attrs >= 18.2.0
Pillow >= 10.3.0
pyOpenSSL >= 24.2.1