RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Typing improvements for some core components (#5889)

This commit is contained in:
Andrey Rakhmatullin 2023-04-11 15:47:41 +04:00 committed by GitHub
parent bdb78b9aa5
commit c7730627a0
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
17 changed files with 245 additions and 147 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
scrapy/core/downloader/__init__.py
View File

@ -4,8 +4,9 @@ from datetime import datetime
from time import time
from twisted.internet import defer, task
from twisted.internet.defer import Deferred
from scrapy import signals
from scrapy import Request, Spider, signals
from scrapy.core.downloader.handlers import DownloadHandlers
from scrapy.core.downloader.middleware import DownloaderMiddlewareManager
from scrapy.resolver import dnscache
@ -86,7 +87,7 @@ class Downloader:
self._slot_gc_loop.start(60)
self.per_slot_settings = self.settings.getdict("DOWNLOAD_SLOTS", {})
def fetch(self, request, spider):
def fetch(self, request: Request, spider: Spider) -> Deferred:
def _deactivate(response):
self.active.remove(request)
return response
@ -206,12 +207,12 @@ class Downloader:
return dfd.addBoth(finish_transferring)
def close(self):
def close(self) -> None:
self._slot_gc_loop.stop()
for slot in self.slots.values():
slot.close()
def _slot_gc(self, age=60):
def _slot_gc(self, age: float = 60) -> None:
mintime = time() - age
for key, slot in list(self.slots.items()):
if not slot.active and slot.lastseen + slot.delay < mintime:

68
scrapy/core/engine.py
View File

@ -8,13 +8,16 @@ import logging
import warnings
from time import time
from typing import (
TYPE_CHECKING,
Any,
Callable,
Generator,
Iterable,
Iterator,
List,
Optional,
Set,
Type,
Union,
cast,
)
@ -28,30 +31,36 @@ from scrapy.core.downloader import Downloader
from scrapy.core.scraper import Scraper
from scrapy.exceptions import CloseSpider, DontCloseSpider, ScrapyDeprecationWarning
from scrapy.http import Request, Response
from scrapy.settings import BaseSettings
from scrapy.logformatter import LogFormatter
from scrapy.settings import BaseSettings, Settings
from scrapy.signalmanager import SignalManager
from scrapy.spiders import Spider
from scrapy.utils.log import failure_to_exc_info, logformatter_adapter
from scrapy.utils.misc import create_instance, load_object
from scrapy.utils.reactor import CallLaterOnce
if TYPE_CHECKING:
from scrapy.core.scheduler import BaseScheduler
from scrapy.crawler import Crawler
logger = logging.getLogger(__name__)
class Slot:
def __init__(
self,
start_requests: Iterable,
start_requests: Iterable[Request],
close_if_idle: bool,
nextcall: CallLaterOnce,
scheduler,
scheduler: "BaseScheduler",
) -> None:
self.closing: Optional[Deferred] = None
self.inprogress: Set[Request] = set()
self.start_requests: Optional[Iterator] = iter(start_requests)
self.close_if_idle = close_if_idle
self.nextcall = nextcall
self.scheduler = scheduler
self.heartbeat = LoopingCall(nextcall.schedule)
self.start_requests: Optional[Iterator[Request]] = iter(start_requests)
self.close_if_idle: bool = close_if_idle
self.nextcall: CallLaterOnce = nextcall
self.scheduler: "BaseScheduler" = scheduler
self.heartbeat: LoopingCall = LoopingCall(nextcall.schedule)
def add_request(self, request: Request) -> None:
self.inprogress.add(request)
@ -75,25 +84,28 @@ class Slot:
class ExecutionEngine:
def __init__(self, crawler, spider_closed_callback: Callable) -> None:
self.crawler = crawler
self.settings = crawler.settings
self.signals = crawler.signals
self.logformatter = crawler.logformatter
def __init__(self, crawler: "Crawler", spider_closed_callback: Callable) -> None:
self.crawler: "Crawler" = crawler
self.settings: Settings = crawler.settings
self.signals: SignalManager = crawler.signals
self.logformatter: LogFormatter = crawler.logformatter
self.slot: Optional[Slot] = None
self.spider: Optional[Spider] = None
self.running = False
self.paused = False
self.scheduler_cls = self._get_scheduler_class(crawler.settings)
downloader_cls = load_object(self.settings["DOWNLOADER"])
self.running: bool = False
self.paused: bool = False
self.scheduler_cls: Type["BaseScheduler"] = self._get_scheduler_class(
crawler.settings
)
downloader_cls: Type[Downloader] = load_object(self.settings["DOWNLOADER"])
self.downloader: Downloader = downloader_cls(crawler)
self.scraper = Scraper(crawler)
self._spider_closed_callback = spider_closed_callback
self._spider_closed_callback: Callable = spider_closed_callback
self.start_time: Optional[float] = None
def _get_scheduler_class(self, settings: BaseSettings) -> type:
def _get_scheduler_class(self, settings: BaseSettings) -> Type["BaseScheduler"]:
from scrapy.core.scheduler import BaseScheduler
scheduler_cls = load_object(settings["SCHEDULER"])
scheduler_cls: Type = load_object(settings["SCHEDULER"])
if not issubclass(scheduler_cls, BaseScheduler):
raise TypeError(
f"The provided scheduler class ({settings['SCHEDULER']})"
@ -115,7 +127,7 @@ class ExecutionEngine:
"""Gracefully stop the execution engine"""
@inlineCallbacks
def _finish_stopping_engine(_) -> Generator[Deferred, Any, None]:
def _finish_stopping_engine(_: Any) -> Generator[Deferred, Any, None]:
yield self.signals.send_catch_log_deferred(signal=signals.engine_stopped)
self._closewait.callback(None)
@ -141,7 +153,8 @@ class ExecutionEngine:
return self.close_spider(
self.spider, reason="shutdown"
) # will also close downloader
return succeed(self.downloader.close())
self.downloader.close()
return succeed(None)
def pause(self) -> None:
self.paused = True
@ -209,7 +222,7 @@ class ExecutionEngine:
extra={"spider": self.spider},
)
)
d.addBoth(lambda _: cast(Slot, self.slot).remove_request(request))
d.addBoth(lambda _: cast(Slot, self.slot).remove_request(request)) # type: ignore[arg-type]
d.addErrback(
lambda f: logger.info(
"Error while removing request from slot",
@ -339,6 +352,7 @@ class ExecutionEngine:
if isinstance(result, Response):
if result.request is None:
result.request = request
assert spider is not None
logkws = self.logformatter.crawled(result.request, result, spider)
if logkws is not None:
logger.log(*logformatter_adapter(logkws), extra={"spider": spider})
@ -350,10 +364,12 @@ class ExecutionEngine:
)
return result
def _on_complete(_):
def _on_complete(_: Any) -> Any:
assert self.slot is not None
self.slot.nextcall.schedule()
return _
assert spider is not None
dwld = self.downloader.fetch(request, spider)
dwld.addCallbacks(_on_success)
dwld.addBoth(_on_complete)
@ -362,7 +378,7 @@ class ExecutionEngine:
@inlineCallbacks
def open_spider(
self, spider: Spider, start_requests: Iterable = (), close_if_idle: bool = True
):
) -> Generator[Deferred, Any, None]:
if self.slot is not None:
raise RuntimeError(f"No free spider slot when opening {spider.name!r}")
logger.info("Spider opened", extra={"spider": spider})
@ -471,7 +487,7 @@ class ExecutionEngine:
return dfd
@property
def open_spiders(self) -> list:
def open_spiders(self) -> List[Spider]:
warnings.warn(
"ExecutionEngine.open_spiders is deprecated, please use ExecutionEngine.spider instead",
category=ScrapyDeprecationWarning,

31
scrapy/core/scheduler.py
View File

@ -2,13 +2,15 @@ import json
import logging
from abc import abstractmethod
from pathlib import Path
from typing import Optional, Type, TypeVar
from typing import Any, Optional, Type, TypeVar, cast
from twisted.internet.defer import Deferred
from scrapy.crawler import Crawler
from scrapy.dupefilters import BaseDupeFilter
from scrapy.http.request import Request
from scrapy.spiders import Spider
from scrapy.statscollectors import StatsCollector
from scrapy.utils.job import job_dir
from scrapy.utils.misc import create_instance, load_object
@ -20,10 +22,10 @@ class BaseSchedulerMeta(type):
Metaclass to check scheduler classes against the necessary interface
"""
def __instancecheck__(cls, instance):
def __instancecheck__(cls, instance: Any) -> bool:
return cls.__subclasscheck__(type(instance))
def __subclasscheck__(cls, subclass):
def __subclasscheck__(cls, subclass: type) -> bool:
return (
hasattr(subclass, "has_pending_requests")
and callable(subclass.has_pending_requests)
@ -168,26 +170,26 @@ class Scheduler(BaseScheduler):
def __init__(
self,
dupefilter,
dupefilter: BaseDupeFilter,
jobdir: Optional[str] = None,
dqclass=None,
mqclass=None,
logunser: bool = False,
stats=None,
stats: Optional[StatsCollector] = None,
pqclass=None,
crawler: Optional[Crawler] = None,
):
self.df = dupefilter
self.dqdir = self._dqdir(jobdir)
self.df: BaseDupeFilter = dupefilter
self.dqdir: Optional[str] = self._dqdir(jobdir)
self.pqclass = pqclass
self.dqclass = dqclass
self.mqclass = mqclass
self.logunser = logunser
self.stats = stats
self.crawler = crawler
self.logunser: bool = logunser
self.stats: Optional[StatsCollector] = stats
self.crawler: Optional[Crawler] = crawler
@classmethod
def from_crawler(cls: Type[SchedulerTV], crawler) -> SchedulerTV:
def from_crawler(cls: Type[SchedulerTV], crawler: Crawler) -> SchedulerTV:
"""
Factory method, initializes the scheduler with arguments taken from the crawl settings
"""
@ -242,6 +244,7 @@ class Scheduler(BaseScheduler):
self.df.log(request, self.spider)
return False
dqok = self._dqpush(request)
assert self.stats is not None
if dqok:
self.stats.inc_value("scheduler/enqueued/disk", spider=self.spider)
else:
@ -259,7 +262,8 @@ class Scheduler(BaseScheduler):
Increment the appropriate stats, such as: ``scheduler/dequeued``,
``scheduler/dequeued/disk``, ``scheduler/dequeued/memory``.
"""
request = self.mqs.pop()
request: Optional[Request] = self.mqs.pop()
assert self.stats is not None
if request is not None:
self.stats.inc_value("scheduler/dequeued/memory", spider=self.spider)
else:
@ -295,6 +299,7 @@ class Scheduler(BaseScheduler):
extra={"spider": self.spider},
)
self.logunser = False
assert self.stats is not None
self.stats.inc_value("scheduler/unserializable", spider=self.spider)
return False
else:
@ -351,7 +356,7 @@ class Scheduler(BaseScheduler):
if not path.exists():
return []
with path.open(encoding="utf-8") as f:
return json.load(f)
return cast(list, json.load(f))
def _write_dqs_state(self, dqdir: str, state: list) -> None:
with Path(dqdir, "active.json").open("w", encoding="utf-8") as f:

31
scrapy/core/scraper.py
View File

@ -15,6 +15,7 @@ from typing import (
Optional,
Set,
Tuple,
Type,
Union,
)
@ -26,6 +27,9 @@ from scrapy import Spider, signals
from scrapy.core.spidermw import SpiderMiddlewareManager
from scrapy.exceptions import CloseSpider, DropItem, IgnoreRequest
from scrapy.http import Request, Response
from scrapy.logformatter import LogFormatter
from scrapy.pipelines import ItemPipelineManager
from scrapy.signalmanager import SignalManager
from scrapy.utils.defer import (
aiter_errback,
defer_fail,
@ -96,16 +100,20 @@ class Slot:
class Scraper:
def __init__(self, crawler: Crawler) -> None:
self.slot: Optional[Slot] = None
self.spidermw = SpiderMiddlewareManager.from_crawler(crawler)
itemproc_cls = load_object(crawler.settings["ITEM_PROCESSOR"])
self.itemproc = itemproc_cls.from_crawler(crawler)
self.concurrent_items = crawler.settings.getint("CONCURRENT_ITEMS")
self.crawler = crawler
self.signals = crawler.signals
self.logformatter = crawler.logformatter
self.spidermw: SpiderMiddlewareManager = SpiderMiddlewareManager.from_crawler(
crawler
)
itemproc_cls: Type[ItemPipelineManager] = load_object(
crawler.settings["ITEM_PROCESSOR"]
)
self.itemproc: ItemPipelineManager = itemproc_cls.from_crawler(crawler)
self.concurrent_items: int = crawler.settings.getint("CONCURRENT_ITEMS")
self.crawler: Crawler = crawler
self.signals: SignalManager = crawler.signals
self.logformatter: LogFormatter = crawler.logformatter
@inlineCallbacks
def open_spider(self, spider: Spider):
def open_spider(self, spider: Spider) -> Generator[Deferred, Any, None]:
"""Open the given spider for scraping and allocate resources for it"""
self.slot = Slot(self.crawler.settings.getint("SCRAPER_SLOT_MAX_ACTIVE_SIZE"))
yield self.itemproc.open_spider(spider)
@ -135,7 +143,8 @@ class Scraper:
raise RuntimeError("Scraper slot not assigned")
dfd = self.slot.add_response_request(result, request)
def finish_scraping(_):
def finish_scraping(_: Any) -> Any:
assert self.slot is not None
self.slot.finish_response(result, request)
self._check_if_closing(spider)
self._scrape_next(spider)
@ -205,9 +214,9 @@ class Scraper:
else: # result is a Failure
# TODO: properly type adding this attribute to a Failure
result.request = request # type: ignore[attr-defined]
warn_on_generator_with_return_value(spider, request.errback)
dfd = defer_fail(result)
if request.errback:
warn_on_generator_with_return_value(spider, request.errback)
dfd.addErrback(request.errback)
return dfd.addCallback(iterate_spider_output)
@ -338,7 +347,7 @@ class Scraper:
def _itemproc_finished(
self, output: Any, item: Any, response: Response, spider: Spider
) -> None:
) -> Deferred:
"""ItemProcessor finished for the given ``item`` and returned ``output``"""
assert self.slot is not None # typing
self.slot.itemproc_size -= 1

25
scrapy/core/spidermw.py
View File

@ -13,6 +13,8 @@ from typing import (
Callable,
Generator,
Iterable,
List,
Optional,
Tuple,
Union,
cast,
@ -25,6 +27,7 @@ from scrapy import Request, Spider
from scrapy.exceptions import _InvalidOutput
from scrapy.http import Response
from scrapy.middleware import MiddlewareManager
from scrapy.settings import BaseSettings
from scrapy.utils.asyncgen import as_async_generator, collect_asyncgen
from scrapy.utils.conf import build_component_list
from scrapy.utils.defer import (
@ -41,22 +44,22 @@ logger = logging.getLogger(__name__)
ScrapeFunc = Callable[[Union[Response, Failure], Request, Spider], Any]
def _isiterable(o) -> bool:
def _isiterable(o: Any) -> bool:
return isinstance(o, (Iterable, AsyncIterable))
class SpiderMiddlewareManager(MiddlewareManager):
component_name = "spider middleware"
def __init__(self, *middlewares):
def __init__(self, *middlewares: Any):
super().__init__(*middlewares)
self.downgrade_warning_done = False
@classmethod
def _get_mwlist_from_settings(cls, settings):
def _get_mwlist_from_settings(cls, settings: BaseSettings) -> List[Any]:
return build_component_list(settings.getwithbase("SPIDER_MIDDLEWARES"))
def _add_middleware(self, mw):
def _add_middleware(self, mw: Any) -> None:
super()._add_middleware(mw)
if hasattr(mw, "process_spider_input"):
self.methods["process_spider_input"].append(mw.process_spider_input)
@ -98,7 +101,7 @@ class SpiderMiddlewareManager(MiddlewareManager):
exception_processor_index: int,
recover_to: Union[MutableChain, MutableAsyncChain],
) -> Union[Generator, AsyncGenerator]:
def process_sync(iterable: Iterable):
def process_sync(iterable: Iterable) -> Generator:
try:
for r in iterable:
yield r
@ -110,7 +113,7 @@ class SpiderMiddlewareManager(MiddlewareManager):
raise
recover_to.extend(exception_result)
async def process_async(iterable: AsyncIterable):
async def process_async(iterable: AsyncIterable) -> AsyncGenerator:
try:
async for r in iterable:
yield r
@ -280,7 +283,7 @@ class SpiderMiddlewareManager(MiddlewareManager):
if isinstance(recovered, AsyncIterable):
recovered_collected = await collect_asyncgen(recovered)
recovered = MutableChain(recovered_collected)
return MutableChain(result, recovered) # type: ignore[arg-type]
return MutableChain(result, recovered)
def scrape_response(
self,
@ -306,7 +309,9 @@ class SpiderMiddlewareManager(MiddlewareManager):
)
return dfd
def process_start_requests(self, start_requests, spider: Spider) -> Deferred:
def process_start_requests(
self, start_requests: Iterable[Request], spider: Spider
) -> Deferred:
return self._process_chain("process_start_requests", start_requests, spider)
# This method is only needed until _async compatibility methods are removed.
@ -314,9 +319,9 @@ class SpiderMiddlewareManager(MiddlewareManager):
def _get_async_method_pair(
mw: Any, methodname: str
) -> Union[None, Callable, Tuple[Callable, Callable]]:
normal_method = getattr(mw, methodname, None)
normal_method: Optional[Callable] = getattr(mw, methodname, None)
methodname_async = methodname + "_async"
async_method = getattr(mw, methodname_async, None)
async_method: Optional[Callable] = getattr(mw, methodname_async, None)
if not async_method:
return normal_method
if not normal_method:

33
scrapy/crawler.py
View File

@ -4,7 +4,7 @@ import logging
import pprint
import signal
import warnings
from typing import TYPE_CHECKING, Optional
from typing import TYPE_CHECKING, Optional, Type, Union
from twisted.internet import defer
from zope.interface.exceptions import DoesNotImplement
@ -22,8 +22,10 @@ from scrapy.core.engine import ExecutionEngine
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.extension import ExtensionManager
from scrapy.interfaces import ISpiderLoader
from scrapy.logformatter import LogFormatter
from scrapy.settings import Settings, overridden_settings
from scrapy.signalmanager import SignalManager
from scrapy.statscollectors import StatsCollector
from scrapy.utils.log import (
LogCounterHandler,
configure_logging,
@ -49,20 +51,25 @@ logger = logging.getLogger(__name__)
class Crawler:
def __init__(self, spidercls, settings=None, init_reactor: bool = False):
def __init__(
self,
spidercls: Type[Spider],
settings: Union[None, dict, Settings] = None,
init_reactor: bool = False,
):
if isinstance(spidercls, Spider):
raise ValueError("The spidercls argument must be a class, not an object")
if isinstance(settings, dict) or settings is None:
settings = Settings(settings)
self.spidercls = spidercls
self.settings = settings.copy()
self.spidercls: Type[Spider] = spidercls
self.settings: Settings = settings.copy()
self.spidercls.update_settings(self.settings)
self.signals = SignalManager(self)
self.signals: SignalManager = SignalManager(self)
self.stats = load_object(self.settings["STATS_CLASS"])(self)
self.stats: StatsCollector = load_object(self.settings["STATS_CLASS"])(self)
handler = LogCounterHandler(self, level=self.settings.get("LOG_LEVEL"))
logging.root.addHandler(handler)
@ -80,8 +87,8 @@ class Crawler:
self.__remove_handler = lambda: logging.root.removeHandler(handler)
self.signals.connect(self.__remove_handler, signals.engine_stopped)
lf_cls = load_object(self.settings["LOG_FORMATTER"])
self.logformatter = lf_cls.from_crawler(self)
lf_cls: Type[LogFormatter] = load_object(self.settings["LOG_FORMATTER"])
self.logformatter: LogFormatter = lf_cls.from_crawler(self)
self.request_fingerprinter: RequestFingerprinter = create_instance(
load_object(self.settings["REQUEST_FINGERPRINTER_CLASS"]),
@ -89,8 +96,8 @@ class Crawler:
crawler=self,
)
reactor_class = self.settings["TWISTED_REACTOR"]
event_loop = self.settings["ASYNCIO_EVENT_LOOP"]
reactor_class: str = self.settings["TWISTED_REACTOR"]
event_loop: str = self.settings["ASYNCIO_EVENT_LOOP"]
if init_reactor:
# this needs to be done after the spider settings are merged,
# but before something imports twisted.internet.reactor
@ -104,11 +111,11 @@ class Crawler:
if is_asyncio_reactor_installed() and event_loop:
verify_installed_asyncio_event_loop(event_loop)
self.extensions = ExtensionManager.from_crawler(self)
self.extensions: ExtensionManager = ExtensionManager.from_crawler(self)
self.settings.freeze()
self.crawling = False
self.spider = None
self.crawling: bool = False
self.spider: Optional[Spider] = None
self.engine: Optional[ExecutionEngine] = None
@defer.inlineCallbacks

2
scrapy/exceptions.py
View File

@ -39,7 +39,7 @@ class DontCloseSpider(Exception):
class CloseSpider(Exception):
"""Raise this from callbacks to request the spider to be closed"""
def __init__(self, reason="cancelled"):
def __init__(self, reason: str = "cancelled"):
super().__init__()
self.reason = reason

32
scrapy/logformatter.py
View File

@ -1,8 +1,11 @@
import logging
import os
from typing import Any, Dict, Optional, Union
from twisted.python.failure import Failure
from scrapy import Request, Spider
from scrapy.http import Response
from scrapy.utils.request import referer_str
SCRAPEDMSG = "Scraped from %(src)s" + os.linesep + "%(item)s"
@ -52,7 +55,7 @@ class LogFormatter:
}
"""
def crawled(self, request, response, spider):
def crawled(self, request: Request, response: Response, spider: Spider) -> dict:
"""Logs a message when the crawler finds a webpage."""
request_flags = f" {str(request.flags)}" if request.flags else ""
response_flags = f" {str(response.flags)}" if response.flags else ""
@ -70,8 +73,11 @@ class LogFormatter:
},
}
def scraped(self, item, response, spider):
def scraped(
self, item: Any, response: Union[Response, Failure], spider: Spider
) -> dict:
"""Logs a message when an item is scraped by a spider."""
src: Any
if isinstance(response, Failure):
src = response.getErrorMessage()
else:
@ -85,7 +91,9 @@ class LogFormatter:
},
}
def dropped(self, item, exception, response, spider):
def dropped(
self, item: Any, exception: BaseException, response: Response, spider: Spider
) -> dict:
"""Logs a message when an item is dropped while it is passing through the item pipeline."""
return {
"level": logging.WARNING,
@ -96,7 +104,9 @@ class LogFormatter:
},
}
def item_error(self, item, exception, response, spider):
def item_error(
self, item: Any, exception, response: Response, spider: Spider
) -> dict:
"""Logs a message when an item causes an error while it is passing
through the item pipeline.
@ -110,7 +120,9 @@ class LogFormatter:
},
}
def spider_error(self, failure, request, response, spider):
def spider_error(
self, failure: Failure, request: Request, response: Response, spider: Spider
) -> dict:
"""Logs an error message from a spider.
.. versionadded:: 2.0
@ -124,13 +136,19 @@ class LogFormatter:
},
}
def download_error(self, failure, request, spider, errmsg=None):
def download_error(
self,
failure: Failure,
request: Request,
spider: Spider,
errmsg: Optional[str] = None,
) -> dict:
"""Logs a download error message from a spider (typically coming from
the engine).
.. versionadded:: 2.0
"""
args = {"request": request}
args: Dict[str, Any] = {"request": request}
if errmsg:
msg = DOWNLOADERRORMSG_LONG
args["errmsg"] = errmsg

10
scrapy/middleware.py
View File

@ -1,7 +1,7 @@
import logging
import pprint
from collections import defaultdict, deque
from typing import Any, Callable, Deque, Dict, Iterable, Tuple, Union, cast
from typing import Any, Callable, Deque, Dict, Iterable, List, Tuple, Union, cast
from twisted.internet.defer import Deferred
@ -30,7 +30,7 @@ class MiddlewareManager:
self._add_middleware(mw)
@classmethod
def _get_mwlist_from_settings(cls, settings: Settings) -> list:
def _get_mwlist_from_settings(cls, settings: Settings) -> List[Any]:
raise NotImplementedError
@classmethod
@ -67,17 +67,17 @@ class MiddlewareManager:
def from_crawler(cls, crawler):
return cls.from_settings(crawler.settings, crawler)
def _add_middleware(self, mw) -> None:
def _add_middleware(self, mw: Any) -> None:
if hasattr(mw, "open_spider"):
self.methods["open_spider"].append(mw.open_spider)
if hasattr(mw, "close_spider"):
self.methods["close_spider"].appendleft(mw.close_spider)
def _process_parallel(self, methodname: str, obj, *args) -> Deferred:
def _process_parallel(self, methodname: str, obj: Any, *args: Any) -> Deferred:
methods = cast(Iterable[Callable], self.methods[methodname])
return process_parallel(methods, obj, *args)
def _process_chain(self, methodname: str, obj, *args) -> Deferred:
def _process_chain(self, methodname: str, obj: Any, *args: Any) -> Deferred:
methods = cast(Iterable[Callable], self.methods[methodname])
return process_chain(methods, obj, *args)

10
scrapy/pipelines/__init__.py
View File

@ -3,7 +3,11 @@ Item pipeline
See documentation in docs/item-pipeline.rst
"""
from typing import Any, List
from twisted.internet.defer import Deferred
from scrapy import Spider
from scrapy.middleware import MiddlewareManager
from scrapy.utils.conf import build_component_list
from scrapy.utils.defer import deferred_f_from_coro_f
@ -13,15 +17,15 @@ class ItemPipelineManager(MiddlewareManager):
component_name = "item pipeline"
@classmethod
def _get_mwlist_from_settings(cls, settings):
def _get_mwlist_from_settings(cls, settings) -> List[Any]:
return build_component_list(settings.getwithbase("ITEM_PIPELINES"))
def _add_middleware(self, pipe):
def _add_middleware(self, pipe: Any) -> None:
super()._add_middleware(pipe)
if hasattr(pipe, "process_item"):
self.methods["process_item"].append(
deferred_f_from_coro_f(pipe.process_item)
)
def process_item(self, item, spider):
def process_item(self, item: Any, spider: Spider) -> Deferred:
return self._process_chain("process_item", item, spider)

23
scrapy/signalmanager.py
View File

@ -1,13 +1,16 @@
from typing import Any, List, Tuple
from pydispatch import dispatcher
from twisted.internet.defer import Deferred
from scrapy.utils import signal as _signal
class SignalManager:
def __init__(self, sender=dispatcher.Anonymous):
self.sender = sender
def __init__(self, sender: Any = dispatcher.Anonymous):
self.sender: Any = sender
def connect(self, receiver, signal, **kwargs):
def connect(self, receiver: Any, signal: Any, **kwargs: Any) -> None:
"""
Connect a receiver function to a signal.
@ -22,18 +25,18 @@ class SignalManager:
:type signal: object
"""
kwargs.setdefault("sender", self.sender)
return dispatcher.connect(receiver, signal, **kwargs)
dispatcher.connect(receiver, signal, **kwargs)
def disconnect(self, receiver, signal, **kwargs):
def disconnect(self, receiver: Any, signal: Any, **kwargs: Any) -> None:
"""
Disconnect a receiver function from a signal. This has the
opposite effect of the :meth:`connect` method, and the arguments
are the same.
"""
kwargs.setdefault("sender", self.sender)
return dispatcher.disconnect(receiver, signal, **kwargs)
dispatcher.disconnect(receiver, signal, **kwargs)
def send_catch_log(self, signal, **kwargs):
def send_catch_log(self, signal: Any, **kwargs: Any) -> List[Tuple[Any, Any]]:
"""
Send a signal, catch exceptions and log them.
@ -43,7 +46,7 @@ class SignalManager:
kwargs.setdefault("sender", self.sender)
return _signal.send_catch_log(signal, **kwargs)
def send_catch_log_deferred(self, signal, **kwargs):
def send_catch_log_deferred(self, signal: Any, **kwargs: Any) -> Deferred:
"""
Like :meth:`send_catch_log` but supports returning
:class:`~twisted.internet.defer.Deferred` objects from signal handlers.
@ -57,7 +60,7 @@ class SignalManager:
kwargs.setdefault("sender", self.sender)
return _signal.send_catch_log_deferred(signal, **kwargs)
def disconnect_all(self, signal, **kwargs):
def disconnect_all(self, signal: Any, **kwargs: Any) -> None:
"""
Disconnect all receivers from the given signal.
@ -65,4 +68,4 @@ class SignalManager:
:type signal: object
"""
kwargs.setdefault("sender", self.sender)
return _signal.disconnect_all(signal, **kwargs)
_signal.disconnect_all(signal, **kwargs)

65
scrapy/statscollectors.py
View File

@ -3,44 +3,57 @@ Scrapy extension for collecting scraping stats
"""
import logging
import pprint
from typing import TYPE_CHECKING, Any, Dict, Optional
from scrapy import Spider
if TYPE_CHECKING:
from scrapy.crawler import Crawler
logger = logging.getLogger(__name__)
class StatsCollector:
def __init__(self, crawler):
self._dump = crawler.settings.getbool("STATS_DUMP")
self._stats = {}
StatsT = Dict[str, Any]
def get_value(self, key, default=None, spider=None):
class StatsCollector:
def __init__(self, crawler: "Crawler"):
self._dump: bool = crawler.settings.getbool("STATS_DUMP")
self._stats: StatsT = {}
def get_value(
self, key: str, default: Any = None, spider: Optional[Spider] = None
) -> Any:
return self._stats.get(key, default)
def get_stats(self, spider=None):
def get_stats(self, spider: Optional[Spider] = None) -> StatsT:
return self._stats
def set_value(self, key, value, spider=None):
def set_value(self, key: str, value: Any, spider: Optional[Spider] = None) -> None:
self._stats[key] = value
def set_stats(self, stats, spider=None):
def set_stats(self, stats: StatsT, spider: Optional[Spider] = None) -> None:
self._stats = stats
def inc_value(self, key, count=1, start=0, spider=None):
def inc_value(
self, key: str, count: int = 1, start: int = 0, spider: Optional[Spider] = None
) -> None:
d = self._stats
d[key] = d.setdefault(key, start) + count
def max_value(self, key, value, spider=None):
def max_value(self, key: str, value: Any, spider: Optional[Spider] = None) -> None:
self._stats[key] = max(self._stats.setdefault(key, value), value)
def min_value(self, key, value, spider=None):
def min_value(self, key: str, value: Any, spider: Optional[Spider] = None) -> None:
self._stats[key] = min(self._stats.setdefault(key, value), value)
def clear_stats(self, spider=None):
def clear_stats(self, spider: Optional[Spider] = None) -> None:
self._stats.clear()
def open_spider(self, spider):
def open_spider(self, spider: Spider) -> None:
pass
def close_spider(self, spider, reason):
def close_spider(self, spider: Spider, reason: str) -> None:
if self._dump:
logger.info(
"Dumping Scrapy stats:\n" + pprint.pformat(self._stats),
@ -48,34 +61,38 @@ class StatsCollector:
)
self._persist_stats(self._stats, spider)
def _persist_stats(self, stats, spider):
def _persist_stats(self, stats: StatsT, spider: Spider) -> None:
pass
class MemoryStatsCollector(StatsCollector):
def __init__(self, crawler):
def __init__(self, crawler: "Crawler"):
super().__init__(crawler)
self.spider_stats = {}
self.spider_stats: Dict[str, StatsT] = {}
def _persist_stats(self, stats, spider):
def _persist_stats(self, stats: StatsT, spider: Spider) -> None:
self.spider_stats[spider.name] = stats
class DummyStatsCollector(StatsCollector):
def get_value(self, key, default=None, spider=None):
def get_value(
self, key: str, default: Any = None, spider: Optional[Spider] = None
) -> Any:
return default
def set_value(self, key, value, spider=None):
def set_value(self, key: str, value: Any, spider: Optional[Spider] = None) -> None:
pass
def set_stats(self, stats, spider=None):
def set_stats(self, stats: StatsT, spider: Optional[Spider] = None) -> None:
pass
def inc_value(self, key, count=1, start=0, spider=None):
def inc_value(
self, key: str, count: int = 1, start: int = 0, spider: Optional[Spider] = None
) -> None:
pass
def max_value(self, key, value, spider=None):
def max_value(self, key: str, value: Any, spider: Optional[Spider] = None) -> None:
pass
def min_value(self, key, value, spider=None):
def min_value(self, key: str, value: Any, spider: Optional[Spider] = None) -> None:
pass

5
scrapy/utils/log.py
View File

@ -2,6 +2,7 @@ import logging
import sys
import warnings
from logging.config import dictConfig
from typing import Tuple
from twisted.python import log as twisted_log
from twisted.python.failure import Failure
@ -14,7 +15,7 @@ from scrapy.utils.versions import scrapy_components_versions
logger = logging.getLogger(__name__)
def failure_to_exc_info(failure):
def failure_to_exc_info(failure: Failure):
"""Extract exc_info from Failure instances"""
if isinstance(failure, Failure):
return (failure.type, failure.value, failure.getTracebackObject())
@ -206,7 +207,7 @@ class LogCounterHandler(logging.Handler):
self.crawler.stats.inc_value(sname)
def logformatter_adapter(logkws):
def logformatter_adapter(logkws: dict) -> Tuple[int, str, dict]:
"""
Helper that takes the dictionary output from the methods in LogFormatter
and adapts it into a tuple of positional arguments for logger.log calls,

9
scrapy/utils/misc.py
View File

@ -10,6 +10,7 @@ from contextlib import contextmanager
from functools import partial
from importlib import import_module
from pkgutil import iter_modules
from typing import TYPE_CHECKING, Any, Callable, Union
from w3lib.html import replace_entities
@ -18,6 +19,10 @@ from scrapy.utils.datatypes import LocalWeakReferencedCache
from scrapy.utils.deprecate import ScrapyDeprecationWarning
from scrapy.utils.python import flatten, to_unicode
if TYPE_CHECKING:
from scrapy import Spider
_ITERABLE_SINGLE_VALUES = dict, Item, str, bytes
@ -34,7 +39,7 @@ def arg_to_iter(arg):
return [arg]
def load_object(path):
def load_object(path: Union[str, Callable]) -> Any:
"""Load an object given its absolute object path, and return it.
The object can be the import path of a class, function, variable or an
@ -249,7 +254,7 @@ def is_generator_with_return_value(callable):
return _generator_callbacks_cache[callable]
def warn_on_generator_with_return_value(spider, callable):
def warn_on_generator_with_return_value(spider: "Spider", callable: Callable) -> None:
"""
Logs a warning if a callable is a generator function and includes
a 'return' statement with a value different than None

18
scrapy/utils/reactor.py
View File

@ -1,9 +1,11 @@
import asyncio
import sys
from contextlib import suppress
from typing import Any, Callable, Dict, Optional, Sequence
from warnings import catch_warnings, filterwarnings, warn
from twisted.internet import asyncioreactor, error
from twisted.internet.base import DelayedCall
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.utils.misc import load_object
@ -34,23 +36,23 @@ class CallLaterOnce:
it hasn't been already scheduled since the last time it ran.
"""
def __init__(self, func, *a, **kw):
self._func = func
self._a = a
self._kw = kw
self._call = None
def __init__(self, func: Callable, *a: Any, **kw: Any):
self._func: Callable = func
self._a: Sequence[Any] = a
self._kw: Dict[str, Any] = kw
self._call: Optional[DelayedCall] = None
def schedule(self, delay=0):
def schedule(self, delay: float = 0) -> None:
from twisted.internet import reactor
if self._call is None:
self._call = reactor.callLater(delay, self)
def cancel(self):
def cancel(self) -> None:
if self._call:
self._call.cancel()
def __call__(self):
def __call__(self) -> Any:
self._call = None
return self._func(*self._a, **self._kw)

9
scrapy/utils/signal.py
View File

@ -1,6 +1,8 @@
"""Helper functions for working with signals"""
import collections.abc
import logging
from typing import Any as TypingAny
from typing import List, Tuple
from pydispatch.dispatcher import (
Anonymous,
@ -20,7 +22,9 @@ from scrapy.utils.log import failure_to_exc_info
logger = logging.getLogger(__name__)
def send_catch_log(signal=Any, sender=Anonymous, *arguments, **named):
def send_catch_log(
signal=Any, sender=Anonymous, *arguments, **named
) -> List[Tuple[TypingAny, TypingAny]]:
"""Like pydispatcher.robust.sendRobust but it also logs errors and returns
Failures instead of exceptions.
"""
@ -32,8 +36,9 @@ def send_catch_log(signal=Any, sender=Anonymous, *arguments, **named):
)
dont_log += (StopDownload,)
spider = named.get("spider", None)
responses = []
responses: List[Tuple[TypingAny, TypingAny]] = []
for receiver in liveReceivers(getAllReceivers(sender, signal)):
result: TypingAny
try:
response = robustApply(
receiver, signal=signal, sender=sender, *arguments, **named

12
tox.ini
View File

@ -37,13 +37,13 @@ install_command =
[testenv:typing]
basepython = python3
deps =
mypy==1.0.1
mypy==1.2.0
types-attrs==19.1.0
types-lxml==2023.2.11
types-Pillow==9.4.0.16
types-Pygments==2.14.0.5
types-pyOpenSSL==23.0.0.4
types-setuptools==67.4.0.1
types-lxml==2023.3.28
types-Pillow==9.4.0.19
types-Pygments==2.14.0.7
types-pyOpenSSL==23.1.0.1
types-setuptools==67.6.0.7
commands =
mypy --show-error-codes {posargs: scrapy tests}