RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Full typing for scrapy/crawler.py and scrapy/spiders/__init__.py.

This commit is contained in:
Andrey Rakhmatullin 2023-08-06 23:51:24 +04:00
parent 85696d7bab
commit 084a9ba076
5 changed files with 114 additions and 52 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

77
scrapy/crawler.py
View File

@ -4,9 +4,14 @@ import logging
import pprint
import signal
import warnings
from typing import TYPE_CHECKING, Any, Dict, Optional, Set, Type, Union
from typing import TYPE_CHECKING, Any, Dict, Generator, Optional, Set, Type, Union
from twisted.internet import defer
from twisted.internet.defer import (
Deferred,
DeferredList,
inlineCallbacks,
maybeDeferred,
)
from zope.interface.exceptions import DoesNotImplement
try:
@ -24,7 +29,7 @@ from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.extension import ExtensionManager
from scrapy.interfaces import ISpiderLoader
from scrapy.logformatter import LogFormatter
from scrapy.settings import Settings, overridden_settings
from scrapy.settings import BaseSettings, Settings, overridden_settings
from scrapy.signalmanager import SignalManager
from scrapy.spiderloader import SpiderLoader
from scrapy.statscollectors import StatsCollector
@ -123,8 +128,8 @@ class Crawler:
self.spider: Optional[Spider] = None
self.engine: Optional[ExecutionEngine] = None
@defer.inlineCallbacks
def crawl(self, *args, **kwargs):
@inlineCallbacks
def crawl(self, *args: Any, **kwargs: Any) -> Generator[Deferred, Any, None]:
if self.crawling:
raise RuntimeError("Crawling already taking place")
self.crawling = True
@ -134,26 +139,27 @@ class Crawler:
self.engine = self._create_engine()
start_requests = iter(self.spider.start_requests())
yield self.engine.open_spider(self.spider, start_requests)
yield defer.maybeDeferred(self.engine.start)
yield maybeDeferred(self.engine.start)
except Exception:
self.crawling = False
if self.engine is not None:
yield self.engine.close()
raise
def _create_spider(self, *args, **kwargs):
def _create_spider(self, *args: Any, **kwargs: Any) -> Spider:
return self.spidercls.from_crawler(self, *args, **kwargs)
def _create_engine(self):
def _create_engine(self) -> ExecutionEngine:
return ExecutionEngine(self, lambda _: self.stop())
@defer.inlineCallbacks
def stop(self):
@inlineCallbacks
def stop(self) -> Generator[Deferred, Any, None]:
"""Starts a graceful stop of the crawler and returns a deferred that is
fired when the crawler is stopped."""
if self.crawling:
self.crawling = False
yield defer.maybeDeferred(self.engine.stop)
assert self.engine
yield maybeDeferred(self.engine.stop)
class CrawlerRunner:
@ -176,10 +182,10 @@ class CrawlerRunner:
)
@staticmethod
def _get_spider_loader(settings) -> SpiderLoader:
def _get_spider_loader(settings: BaseSettings) -> SpiderLoader:
"""Get SpiderLoader instance from settings"""
cls_path = settings.get("SPIDER_LOADER_CLASS")
loader_cls = load_object(cls_path)
loader_cls: Type[SpiderLoader] = load_object(cls_path)
excs = (
(DoesNotImplement, MultipleInvalid) if MultipleInvalid else DoesNotImplement
)
@ -201,11 +207,11 @@ class CrawlerRunner:
self.settings = settings
self.spider_loader = self._get_spider_loader(settings)
self._crawlers: Set[Crawler] = set()
self._active: Set[defer.Deferred] = set()
self._active: Set[Deferred] = set()
self.bootstrap_failed = False
@property
def spiders(self):
def spiders(self) -> SpiderLoader:
warnings.warn(
"CrawlerRunner.spiders attribute is renamed to "
"CrawlerRunner.spider_loader.",
@ -214,7 +220,12 @@ class CrawlerRunner:
)
return self.spider_loader
def crawl(self, crawler_or_spidercls, *args, **kwargs):
def crawl(
self,
crawler_or_spidercls: Union[Type[Spider], str, Crawler],
*args: Any,
**kwargs: Any,
) -> Deferred:
"""
Run a crawler with the provided arguments.
@ -244,12 +255,12 @@ class CrawlerRunner:
crawler = self.create_crawler(crawler_or_spidercls)
return self._crawl(crawler, *args, **kwargs)
def _crawl(self, crawler, *args, **kwargs):
def _crawl(self, crawler: Crawler, *args: Any, **kwargs: Any) -> Deferred:
self.crawlers.add(crawler)
d = crawler.crawl(*args, **kwargs)
self._active.add(d)
def _done(result):
def _done(result: Any) -> Any:
self.crawlers.discard(crawler)
self._active.discard(d)
self.bootstrap_failed |= not getattr(crawler, "spider", None)
@ -284,16 +295,16 @@ class CrawlerRunner:
spidercls = self.spider_loader.load(spidercls)
return Crawler(spidercls, self.settings)
def stop(self):
def stop(self) -> Deferred:
"""
Stops simultaneously all the crawling jobs taking place.
Returns a deferred that is fired when they all have ended.
"""
return defer.DeferredList([c.stop() for c in list(self.crawlers)])
return DeferredList([c.stop() for c in list(self.crawlers)])
@defer.inlineCallbacks
def join(self):
@inlineCallbacks
def join(self) -> Generator[Deferred, Any, None]:
"""
join()
@ -301,7 +312,7 @@ class CrawlerRunner:
completed their executions.
"""
while self._active:
yield defer.DeferredList(self._active)
yield DeferredList(self._active)
class CrawlerProcess(CrawlerRunner):
@ -328,13 +339,17 @@ class CrawlerProcess(CrawlerRunner):
process. See :ref:`run-from-script` for an example.
"""
def __init__(self, settings=None, install_root_handler=True):
def __init__(
self,
settings: Union[Dict[str, Any], Settings, None] = None,
install_root_handler: bool = True,
):
super().__init__(settings)
configure_logging(self.settings, install_root_handler)
log_scrapy_info(self.settings)
self._initialized_reactor = False
def _signal_shutdown(self, signum, _):
def _signal_shutdown(self, signum: int, _: Any) -> None:
from twisted.internet import reactor
install_shutdown_handlers(self._signal_kill)
@ -345,7 +360,7 @@ class CrawlerProcess(CrawlerRunner):
)
reactor.callFromThread(self._graceful_stop_reactor)
def _signal_kill(self, signum, _):
def _signal_kill(self, signum: int, _: Any) -> None:
from twisted.internet import reactor
install_shutdown_handlers(signal.SIG_IGN)
@ -355,14 +370,16 @@ class CrawlerProcess(CrawlerRunner):
)
reactor.callFromThread(self._stop_reactor)
def _create_crawler(self, spidercls):
def _create_crawler(self, spidercls: Union[Type[Spider], str]) -> Crawler:
if isinstance(spidercls, str):
spidercls = self.spider_loader.load(spidercls)
init_reactor = not self._initialized_reactor
self._initialized_reactor = True
return Crawler(spidercls, self.settings, init_reactor=init_reactor)
def start(self, stop_after_crawl=True, install_signal_handlers=True):
def start(
self, stop_after_crawl: bool = True, install_signal_handlers: bool = True
) -> None:
"""
This method starts a :mod:`~twisted.internet.reactor`, adjusts its pool
size to :setting:`REACTOR_THREADPOOL_MAXSIZE`, and installs a DNS cache
@ -396,12 +413,12 @@ class CrawlerProcess(CrawlerRunner):
reactor.addSystemEventTrigger("before", "shutdown", self.stop)
reactor.run(installSignalHandlers=False) # blocking call
def _graceful_stop_reactor(self):
def _graceful_stop_reactor(self) -> Deferred:
d = self.stop()
d.addBoth(self._stop_reactor)
return d
def _stop_reactor(self, _=None):
def _stop_reactor(self, _: Any = None) -> None:
from twisted.internet import reactor
try:

13
scrapy/logformatter.py
View File

@ -1,6 +1,8 @@
from __future__ import annotations
import logging
import os
from typing import Any, Dict, Optional, Union
from typing import TYPE_CHECKING, Any, Dict, Optional, Union
from twisted.python.failure import Failure
@ -8,6 +10,13 @@ from scrapy import Request, Spider
from scrapy.http import Response
from scrapy.utils.request import referer_str
if TYPE_CHECKING:
# typing.Self requires Python 3.11
from typing_extensions import Self
from scrapy.crawler import Crawler
SCRAPEDMSG = "Scraped from %(src)s" + os.linesep + "%(item)s"
DROPPEDMSG = "Dropped: %(exception)s" + os.linesep + "%(item)s"
CRAWLEDMSG = "Crawled (%(status)s) %(request)s%(request_flags)s (referer: %(referer)s)%(response_flags)s"
@ -161,5 +170,5 @@ class LogFormatter:
}
@classmethod
def from_crawler(cls, crawler):
def from_crawler(cls, crawler: Crawler) -> Self:
return cls()

29
scrapy/middleware.py
View File

@ -1,7 +1,21 @@
from __future__ import annotations
import logging
import pprint
from collections import defaultdict, deque
from typing import Any, Callable, Deque, Dict, Iterable, List, Tuple, Union, cast
from typing import (
TYPE_CHECKING,
Any,
Callable,
Deque,
Dict,
Iterable,
List,
Optional,
Tuple,
Union,
cast,
)
from twisted.internet.defer import Deferred
@ -11,6 +25,13 @@ from scrapy.settings import Settings
from scrapy.utils.defer import process_chain, process_parallel
from scrapy.utils.misc import create_instance, load_object
if TYPE_CHECKING:
# typing.Self requires Python 3.11
from typing_extensions import Self
from scrapy.crawler import Crawler
logger = logging.getLogger(__name__)
@ -34,7 +55,9 @@ class MiddlewareManager:
raise NotImplementedError
@classmethod
def from_settings(cls, settings: Settings, crawler=None):
def from_settings(
cls, settings: Settings, crawler: Optional[Crawler] = None
) -> Self:
mwlist = cls._get_mwlist_from_settings(settings)
middlewares = []
enabled = []
@ -63,7 +86,7 @@ class MiddlewareManager:
return cls(*middlewares)
@classmethod
def from_crawler(cls, crawler):
def from_crawler(cls, crawler: Crawler) -> Self:
return cls.from_settings(crawler.settings, crawler)
def _add_middleware(self, mw: Any) -> None:

10
scrapy/spiderloader.py
View File

@ -1,8 +1,10 @@
from __future__ import annotations
import traceback
import warnings
from collections import defaultdict
from types import ModuleType
from typing import DefaultDict, Dict, List, Tuple, Type
from typing import TYPE_CHECKING, DefaultDict, Dict, List, Tuple, Type
from zope.interface import implementer
@ -12,6 +14,10 @@ from scrapy.settings import BaseSettings
from scrapy.utils.misc import walk_modules
from scrapy.utils.spider import iter_spider_classes
if TYPE_CHECKING:
# typing.Self requires Python 3.11
from typing_extensions import Self
@implementer(ISpiderLoader)
class SpiderLoader:
@ -69,7 +75,7 @@ class SpiderLoader:
self._check_name_duplicates()
@classmethod
def from_settings(cls, settings):
def from_settings(cls, settings: BaseSettings) -> Self:
return cls(settings)
def load(self, spider_name: str) -> Type[Spider]:

37
scrapy/spiders/__init__.py
View File

@ -6,15 +6,21 @@ See documentation in docs/topics/spiders.rst
from __future__ import annotations
import logging
from typing import TYPE_CHECKING, Optional
from typing import TYPE_CHECKING, Any, Iterable, List, Optional, Union, cast
from twisted.internet.defer import Deferred
from scrapy import signals
from scrapy.http import Request
from scrapy.http import Request, Response
from scrapy.utils.trackref import object_ref
from scrapy.utils.url import url_is_from_spider
if TYPE_CHECKING:
# typing.Self requires Python 3.11
from typing_extensions import Self
from scrapy.crawler import Crawler
from scrapy.settings import BaseSettings
class Spider(object_ref):
@ -25,21 +31,21 @@ class Spider(object_ref):
name: str
custom_settings: Optional[dict] = None
def __init__(self, name=None, **kwargs):
def __init__(self, name: Optional[str] = None, **kwargs: Any):
if name is not None:
self.name = name
elif not getattr(self, "name", None):
raise ValueError(f"{type(self).__name__} must have a name")
self.__dict__.update(kwargs)
if not hasattr(self, "start_urls"):
self.start_urls = []
self.start_urls: List[str] = []
@property
def logger(self):
def logger(self) -> logging.LoggerAdapter:
logger = logging.getLogger(self.name)
return logging.LoggerAdapter(logger, {"spider": self})
def log(self, message, level=logging.DEBUG, **kw):
def log(self, message: Any, level: int = logging.DEBUG, **kw: Any) -> None:
"""Log the given message at the given log level
This helper wraps a log call to the logger within the spider, but you
@ -49,17 +55,17 @@ class Spider(object_ref):
self.logger.log(level, message, **kw)
@classmethod
def from_crawler(cls, crawler, *args, **kwargs):
def from_crawler(cls, crawler: Crawler, *args: Any, **kwargs: Any) -> Self:
spider = cls(*args, **kwargs)
spider._set_crawler(crawler)
return spider
def _set_crawler(self, crawler: Crawler):
def _set_crawler(self, crawler: Crawler) -> None:
self.crawler = crawler
self.settings = crawler.settings
crawler.signals.connect(self.close, signals.spider_closed)
def start_requests(self):
def start_requests(self) -> Iterable[Request]:
if not self.start_urls and hasattr(self, "start_url"):
raise AttributeError(
"Crawling could not start: 'start_urls' not found "
@ -69,16 +75,16 @@ class Spider(object_ref):
for url in self.start_urls:
yield Request(url, dont_filter=True)
def _parse(self, response, **kwargs):
def _parse(self, response: Response, **kwargs: Any) -> Any:
return self.parse(response, **kwargs)
def parse(self, response, **kwargs):
def parse(self, response: Response, **kwargs: Any) -> Any:
raise NotImplementedError(
f"{self.__class__.__name__}.parse callback is not defined"
)
@classmethod
def update_settings(cls, settings):
def update_settings(cls, settings: BaseSettings) -> None:
settings.setdict(cls.custom_settings or {}, priority="spider")
@classmethod
@ -86,12 +92,13 @@ class Spider(object_ref):
return url_is_from_spider(request.url, cls)
@staticmethod
def close(spider, reason):
def close(spider: Spider, reason: str) -> Union[Deferred, None]:
closed = getattr(spider, "closed", None)
if callable(closed):
return closed(reason)
return cast(Union[Deferred, None], closed(reason))
return None
def __repr__(self):
def __repr__(self) -> str:
return f"<{type(self).__name__} {self.name!r} at 0x{id(self):0x}>"