mirror of https://github.com/scrapy/scrapy.git
Merge pull request #6005 from wRAR/typing-crawler
More typing for scrapy/*.py
This commit is contained in:
Andrey Rakhmatullin
GitHub
commit
7c497688f8
|
|
@ -1,8 +1,10 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
from abc import abstractmethod
|
||||
from pathlib import Path
|
||||
from typing import Any, Optional, Type, TypeVar, cast
|
||||
from typing import TYPE_CHECKING, Any, Optional, Type, TypeVar, cast
|
||||
|
||||
from twisted.internet.defer import Deferred
|
||||
|
||||
|
|
@ -14,6 +16,11 @@ from scrapy.statscollectors import StatsCollector
|
|||
from scrapy.utils.job import job_dir
|
||||
from scrapy.utils.misc import create_instance, load_object
|
||||
|
||||
if TYPE_CHECKING:
|
||||
# typing.Self requires Python 3.11
|
||||
from typing_extensions import Self
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
|
|
@ -54,7 +61,7 @@ class BaseScheduler(metaclass=BaseSchedulerMeta):
|
|||
"""
|
||||
|
||||
@classmethod
|
||||
def from_crawler(cls, crawler: Crawler):
|
||||
def from_crawler(cls, crawler: Crawler) -> Self:
|
||||
"""
|
||||
Factory method which receives the current :class:`~scrapy.crawler.Crawler` object as argument.
|
||||
"""
|
||||
|
|
@ -325,6 +332,7 @@ class Scheduler(BaseScheduler):
|
|||
|
||||
def _dq(self):
|
||||
"""Create a new priority queue instance, with disk storage"""
|
||||
assert self.dqdir
|
||||
state = self._read_dqs_state(self.dqdir)
|
||||
q = create_instance(
|
||||
self.pqclass,
|
||||
|
|
|
|||
|
|
@ -364,6 +364,7 @@ class Scraper:
|
|||
spider=spider,
|
||||
exception=output.value,
|
||||
)
|
||||
assert ex
|
||||
logkws = self.logformatter.item_error(item, ex, response, spider)
|
||||
logger.log(
|
||||
*logformatter_adapter(logkws),
|
||||
|
|
|
|||
|
|
@ -4,9 +4,14 @@ import logging
|
|||
import pprint
|
||||
import signal
|
||||
import warnings
|
||||
from typing import TYPE_CHECKING, Any, Dict, Optional, Set, Type, Union
|
||||
from typing import TYPE_CHECKING, Any, Dict, Generator, Optional, Set, Type, Union, cast
|
||||
|
||||
from twisted.internet import defer
|
||||
from twisted.internet.defer import (
|
||||
Deferred,
|
||||
DeferredList,
|
||||
inlineCallbacks,
|
||||
maybeDeferred,
|
||||
)
|
||||
from zope.interface.exceptions import DoesNotImplement
|
||||
|
||||
try:
|
||||
|
|
@ -24,9 +29,8 @@ from scrapy.exceptions import ScrapyDeprecationWarning
|
|||
from scrapy.extension import ExtensionManager
|
||||
from scrapy.interfaces import ISpiderLoader
|
||||
from scrapy.logformatter import LogFormatter
|
||||
from scrapy.settings import Settings, overridden_settings
|
||||
from scrapy.settings import BaseSettings, Settings, overridden_settings
|
||||
from scrapy.signalmanager import SignalManager
|
||||
from scrapy.spiderloader import SpiderLoader
|
||||
from scrapy.statscollectors import StatsCollector
|
||||
from scrapy.utils.log import (
|
||||
LogCounterHandler,
|
||||
|
|
@ -123,8 +127,8 @@ class Crawler:
|
|||
self.spider: Optional[Spider] = None
|
||||
self.engine: Optional[ExecutionEngine] = None
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def crawl(self, *args, **kwargs):
|
||||
@inlineCallbacks
|
||||
def crawl(self, *args: Any, **kwargs: Any) -> Generator[Deferred, Any, None]:
|
||||
if self.crawling:
|
||||
raise RuntimeError("Crawling already taking place")
|
||||
self.crawling = True
|
||||
|
|
@ -134,26 +138,27 @@ class Crawler:
|
|||
self.engine = self._create_engine()
|
||||
start_requests = iter(self.spider.start_requests())
|
||||
yield self.engine.open_spider(self.spider, start_requests)
|
||||
yield defer.maybeDeferred(self.engine.start)
|
||||
yield maybeDeferred(self.engine.start)
|
||||
except Exception:
|
||||
self.crawling = False
|
||||
if self.engine is not None:
|
||||
yield self.engine.close()
|
||||
raise
|
||||
|
||||
def _create_spider(self, *args, **kwargs):
|
||||
def _create_spider(self, *args: Any, **kwargs: Any) -> Spider:
|
||||
return self.spidercls.from_crawler(self, *args, **kwargs)
|
||||
|
||||
def _create_engine(self):
|
||||
def _create_engine(self) -> ExecutionEngine:
|
||||
return ExecutionEngine(self, lambda _: self.stop())
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def stop(self):
|
||||
@inlineCallbacks
|
||||
def stop(self) -> Generator[Deferred, Any, None]:
|
||||
"""Starts a graceful stop of the crawler and returns a deferred that is
|
||||
fired when the crawler is stopped."""
|
||||
if self.crawling:
|
||||
self.crawling = False
|
||||
yield defer.maybeDeferred(self.engine.stop)
|
||||
assert self.engine
|
||||
yield maybeDeferred(self.engine.stop)
|
||||
|
||||
|
||||
class CrawlerRunner:
|
||||
|
|
@ -176,7 +181,7 @@ class CrawlerRunner:
|
|||
)
|
||||
|
||||
@staticmethod
|
||||
def _get_spider_loader(settings) -> SpiderLoader:
|
||||
def _get_spider_loader(settings: BaseSettings):
|
||||
"""Get SpiderLoader instance from settings"""
|
||||
cls_path = settings.get("SPIDER_LOADER_CLASS")
|
||||
loader_cls = load_object(cls_path)
|
||||
|
|
@ -201,10 +206,15 @@ class CrawlerRunner:
|
|||
self.settings = settings
|
||||
self.spider_loader = self._get_spider_loader(settings)
|
||||
self._crawlers: Set[Crawler] = set()
|
||||
self._active: Set[defer.Deferred] = set()
|
||||
self._active: Set[Deferred] = set()
|
||||
self.bootstrap_failed = False
|
||||
|
||||
def crawl(self, crawler_or_spidercls, *args, **kwargs):
|
||||
def crawl(
|
||||
self,
|
||||
crawler_or_spidercls: Union[Type[Spider], str, Crawler],
|
||||
*args: Any,
|
||||
**kwargs: Any,
|
||||
) -> Deferred:
|
||||
"""
|
||||
Run a crawler with the provided arguments.
|
||||
|
||||
|
|
@ -234,12 +244,12 @@ class CrawlerRunner:
|
|||
crawler = self.create_crawler(crawler_or_spidercls)
|
||||
return self._crawl(crawler, *args, **kwargs)
|
||||
|
||||
def _crawl(self, crawler, *args, **kwargs):
|
||||
def _crawl(self, crawler: Crawler, *args: Any, **kwargs: Any) -> Deferred:
|
||||
self.crawlers.add(crawler)
|
||||
d = crawler.crawl(*args, **kwargs)
|
||||
self._active.add(d)
|
||||
|
||||
def _done(result):
|
||||
def _done(result: Any) -> Any:
|
||||
self.crawlers.discard(crawler)
|
||||
self._active.discard(d)
|
||||
self.bootstrap_failed |= not getattr(crawler, "spider", None)
|
||||
|
|
@ -272,18 +282,19 @@ class CrawlerRunner:
|
|||
def _create_crawler(self, spidercls: Union[str, Type[Spider]]) -> Crawler:
|
||||
if isinstance(spidercls, str):
|
||||
spidercls = self.spider_loader.load(spidercls)
|
||||
return Crawler(spidercls, self.settings)
|
||||
# temporary cast until self.spider_loader is typed
|
||||
return Crawler(cast(Type[Spider], spidercls), self.settings)
|
||||
|
||||
def stop(self):
|
||||
def stop(self) -> Deferred:
|
||||
"""
|
||||
Stops simultaneously all the crawling jobs taking place.
|
||||
|
||||
Returns a deferred that is fired when they all have ended.
|
||||
"""
|
||||
return defer.DeferredList([c.stop() for c in list(self.crawlers)])
|
||||
return DeferredList([c.stop() for c in list(self.crawlers)])
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def join(self):
|
||||
@inlineCallbacks
|
||||
def join(self) -> Generator[Deferred, Any, None]:
|
||||
"""
|
||||
join()
|
||||
|
||||
|
|
@ -291,7 +302,7 @@ class CrawlerRunner:
|
|||
completed their executions.
|
||||
"""
|
||||
while self._active:
|
||||
yield defer.DeferredList(self._active)
|
||||
yield DeferredList(self._active)
|
||||
|
||||
|
||||
class CrawlerProcess(CrawlerRunner):
|
||||
|
|
@ -318,13 +329,17 @@ class CrawlerProcess(CrawlerRunner):
|
|||
process. See :ref:`run-from-script` for an example.
|
||||
"""
|
||||
|
||||
def __init__(self, settings=None, install_root_handler=True):
|
||||
def __init__(
|
||||
self,
|
||||
settings: Union[Dict[str, Any], Settings, None] = None,
|
||||
install_root_handler: bool = True,
|
||||
):
|
||||
super().__init__(settings)
|
||||
configure_logging(self.settings, install_root_handler)
|
||||
log_scrapy_info(self.settings)
|
||||
self._initialized_reactor = False
|
||||
|
||||
def _signal_shutdown(self, signum, _):
|
||||
def _signal_shutdown(self, signum: int, _: Any) -> None:
|
||||
from twisted.internet import reactor
|
||||
|
||||
install_shutdown_handlers(self._signal_kill)
|
||||
|
|
@ -335,7 +350,7 @@ class CrawlerProcess(CrawlerRunner):
|
|||
)
|
||||
reactor.callFromThread(self._graceful_stop_reactor)
|
||||
|
||||
def _signal_kill(self, signum, _):
|
||||
def _signal_kill(self, signum: int, _: Any) -> None:
|
||||
from twisted.internet import reactor
|
||||
|
||||
install_shutdown_handlers(signal.SIG_IGN)
|
||||
|
|
@ -345,14 +360,19 @@ class CrawlerProcess(CrawlerRunner):
|
|||
)
|
||||
reactor.callFromThread(self._stop_reactor)
|
||||
|
||||
def _create_crawler(self, spidercls):
|
||||
def _create_crawler(self, spidercls: Union[Type[Spider], str]) -> Crawler:
|
||||
if isinstance(spidercls, str):
|
||||
spidercls = self.spider_loader.load(spidercls)
|
||||
init_reactor = not self._initialized_reactor
|
||||
self._initialized_reactor = True
|
||||
return Crawler(spidercls, self.settings, init_reactor=init_reactor)
|
||||
# temporary cast until self.spider_loader is typed
|
||||
return Crawler(
|
||||
cast(Type[Spider], spidercls), self.settings, init_reactor=init_reactor
|
||||
)
|
||||
|
||||
def start(self, stop_after_crawl=True, install_signal_handlers=True):
|
||||
def start(
|
||||
self, stop_after_crawl: bool = True, install_signal_handlers: bool = True
|
||||
) -> None:
|
||||
"""
|
||||
This method starts a :mod:`~twisted.internet.reactor`, adjusts its pool
|
||||
size to :setting:`REACTOR_THREADPOOL_MAXSIZE`, and installs a DNS cache
|
||||
|
|
@ -386,12 +406,12 @@ class CrawlerProcess(CrawlerRunner):
|
|||
reactor.addSystemEventTrigger("before", "shutdown", self.stop)
|
||||
reactor.run(installSignalHandlers=False) # blocking call
|
||||
|
||||
def _graceful_stop_reactor(self):
|
||||
def _graceful_stop_reactor(self) -> Deferred:
|
||||
d = self.stop()
|
||||
d.addBoth(self._stop_reactor)
|
||||
return d
|
||||
|
||||
def _stop_reactor(self, _=None):
|
||||
def _stop_reactor(self, _: Any = None) -> None:
|
||||
from twisted.internet import reactor
|
||||
|
||||
try:
|
||||
|
|
|
|||
|
|
@ -1,6 +1,8 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from pathlib import Path
|
||||
from typing import Optional, Set, Type, TypeVar
|
||||
from typing import TYPE_CHECKING, Optional, Set
|
||||
from warnings import warn
|
||||
|
||||
from twisted.internet.defer import Deferred
|
||||
|
|
@ -10,16 +12,22 @@ from scrapy.settings import BaseSettings
|
|||
from scrapy.spiders import Spider
|
||||
from scrapy.utils.deprecate import ScrapyDeprecationWarning
|
||||
from scrapy.utils.job import job_dir
|
||||
from scrapy.utils.request import RequestFingerprinter, referer_str
|
||||
from scrapy.utils.request import (
|
||||
RequestFingerprinter,
|
||||
RequestFingerprinterProtocol,
|
||||
referer_str,
|
||||
)
|
||||
|
||||
BaseDupeFilterTV = TypeVar("BaseDupeFilterTV", bound="BaseDupeFilter")
|
||||
if TYPE_CHECKING:
|
||||
# typing.Self requires Python 3.11
|
||||
from typing_extensions import Self
|
||||
|
||||
from scrapy.crawler import Crawler
|
||||
|
||||
|
||||
class BaseDupeFilter:
|
||||
@classmethod
|
||||
def from_settings(
|
||||
cls: Type[BaseDupeFilterTV], settings: BaseSettings
|
||||
) -> BaseDupeFilterTV:
|
||||
def from_settings(cls, settings: BaseSettings) -> Self:
|
||||
return cls()
|
||||
|
||||
def request_seen(self, request: Request) -> bool:
|
||||
|
|
@ -36,9 +44,6 @@ class BaseDupeFilter:
|
|||
pass
|
||||
|
||||
|
||||
RFPDupeFilterTV = TypeVar("RFPDupeFilterTV", bound="RFPDupeFilter")
|
||||
|
||||
|
||||
class RFPDupeFilter(BaseDupeFilter):
|
||||
"""Request Fingerprint duplicates filter"""
|
||||
|
||||
|
|
@ -47,10 +52,12 @@ class RFPDupeFilter(BaseDupeFilter):
|
|||
path: Optional[str] = None,
|
||||
debug: bool = False,
|
||||
*,
|
||||
fingerprinter=None,
|
||||
fingerprinter: Optional[RequestFingerprinterProtocol] = None,
|
||||
) -> None:
|
||||
self.file = None
|
||||
self.fingerprinter = fingerprinter or RequestFingerprinter()
|
||||
self.fingerprinter: RequestFingerprinterProtocol = (
|
||||
fingerprinter or RequestFingerprinter()
|
||||
)
|
||||
self.fingerprints: Set[str] = set()
|
||||
self.logdupes = True
|
||||
self.debug = debug
|
||||
|
|
@ -62,8 +69,11 @@ class RFPDupeFilter(BaseDupeFilter):
|
|||
|
||||
@classmethod
|
||||
def from_settings(
|
||||
cls: Type[RFPDupeFilterTV], settings: BaseSettings, *, fingerprinter=None
|
||||
) -> RFPDupeFilterTV:
|
||||
cls,
|
||||
settings: BaseSettings,
|
||||
*,
|
||||
fingerprinter: Optional[RequestFingerprinterProtocol] = None,
|
||||
) -> Self:
|
||||
debug = settings.getbool("DUPEFILTER_DEBUG")
|
||||
try:
|
||||
return cls(job_dir(settings), debug, fingerprinter=fingerprinter)
|
||||
|
|
@ -75,11 +85,11 @@ class RFPDupeFilter(BaseDupeFilter):
|
|||
ScrapyDeprecationWarning,
|
||||
)
|
||||
result = cls(job_dir(settings), debug)
|
||||
result.fingerprinter = fingerprinter
|
||||
result.fingerprinter = fingerprinter or RequestFingerprinter()
|
||||
return result
|
||||
|
||||
@classmethod
|
||||
def from_crawler(cls, crawler):
|
||||
def from_crawler(cls, crawler: Crawler) -> Self:
|
||||
try:
|
||||
return cls.from_settings(
|
||||
crawler.settings,
|
||||
|
|
|
|||
|
|
@ -52,7 +52,7 @@ class StopDownload(Exception):
|
|||
should be handled by the request errback. Note that 'fail' is a keyword-only argument.
|
||||
"""
|
||||
|
||||
def __init__(self, *, fail=True):
|
||||
def __init__(self, *, fail: bool = True):
|
||||
super().__init__()
|
||||
self.fail = fail
|
||||
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ This module defines the Link object used in Link extractors.
|
|||
For actual link extractors implementation see scrapy.linkextractors, or
|
||||
its documentation in: docs/topics/link-extractors.rst
|
||||
"""
|
||||
from typing import Any
|
||||
|
||||
|
||||
class Link:
|
||||
|
|
@ -26,16 +27,20 @@ class Link:
|
|||
|
||||
__slots__ = ["url", "text", "fragment", "nofollow"]
|
||||
|
||||
def __init__(self, url, text="", fragment="", nofollow=False):
|
||||
def __init__(
|
||||
self, url: str, text: str = "", fragment: str = "", nofollow: bool = False
|
||||
):
|
||||
if not isinstance(url, str):
|
||||
got = url.__class__.__name__
|
||||
raise TypeError(f"Link urls must be str objects, got {got}")
|
||||
self.url = url
|
||||
self.text = text
|
||||
self.fragment = fragment
|
||||
self.nofollow = nofollow
|
||||
self.url: str = url
|
||||
self.text: str = text
|
||||
self.fragment: str = fragment
|
||||
self.nofollow: bool = nofollow
|
||||
|
||||
def __eq__(self, other):
|
||||
def __eq__(self, other: Any) -> bool:
|
||||
if not isinstance(other, Link):
|
||||
raise NotImplementedError
|
||||
return (
|
||||
self.url == other.url
|
||||
and self.text == other.text
|
||||
|
|
@ -43,12 +48,12 @@ class Link:
|
|||
and self.nofollow == other.nofollow
|
||||
)
|
||||
|
||||
def __hash__(self):
|
||||
def __hash__(self) -> int:
|
||||
return (
|
||||
hash(self.url) ^ hash(self.text) ^ hash(self.fragment) ^ hash(self.nofollow)
|
||||
)
|
||||
|
||||
def __repr__(self):
|
||||
def __repr__(self) -> str:
|
||||
return (
|
||||
f"Link(url={self.url!r}, text={self.text!r}, "
|
||||
f"fragment={self.fragment!r}, nofollow={self.nofollow!r})"
|
||||
|
|
|
|||
|
|
@ -1,6 +1,8 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import os
|
||||
from typing import Any, Dict, Optional, Union
|
||||
from typing import TYPE_CHECKING, Any, Dict, Optional, Union
|
||||
|
||||
from twisted.python.failure import Failure
|
||||
|
||||
|
|
@ -8,6 +10,13 @@ from scrapy import Request, Spider
|
|||
from scrapy.http import Response
|
||||
from scrapy.utils.request import referer_str
|
||||
|
||||
if TYPE_CHECKING:
|
||||
# typing.Self requires Python 3.11
|
||||
from typing_extensions import Self
|
||||
|
||||
from scrapy.crawler import Crawler
|
||||
|
||||
|
||||
SCRAPEDMSG = "Scraped from %(src)s" + os.linesep + "%(item)s"
|
||||
DROPPEDMSG = "Dropped: %(exception)s" + os.linesep + "%(item)s"
|
||||
CRAWLEDMSG = "Crawled (%(status)s) %(request)s%(request_flags)s (referer: %(referer)s)%(response_flags)s"
|
||||
|
|
@ -105,7 +114,7 @@ class LogFormatter:
|
|||
}
|
||||
|
||||
def item_error(
|
||||
self, item: Any, exception, response: Response, spider: Spider
|
||||
self, item: Any, exception: BaseException, response: Response, spider: Spider
|
||||
) -> dict:
|
||||
"""Logs a message when an item causes an error while it is passing
|
||||
through the item pipeline.
|
||||
|
|
@ -161,5 +170,5 @@ class LogFormatter:
|
|||
}
|
||||
|
||||
@classmethod
|
||||
def from_crawler(cls, crawler):
|
||||
def from_crawler(cls, crawler: Crawler) -> Self:
|
||||
return cls()
|
||||
|
|
|
|||
|
|
@ -1,7 +1,21 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import pprint
|
||||
from collections import defaultdict, deque
|
||||
from typing import Any, Callable, Deque, Dict, Iterable, List, Tuple, Union, cast
|
||||
from typing import (
|
||||
TYPE_CHECKING,
|
||||
Any,
|
||||
Callable,
|
||||
Deque,
|
||||
Dict,
|
||||
Iterable,
|
||||
List,
|
||||
Optional,
|
||||
Tuple,
|
||||
Union,
|
||||
cast,
|
||||
)
|
||||
|
||||
from twisted.internet.defer import Deferred
|
||||
|
||||
|
|
@ -11,6 +25,13 @@ from scrapy.settings import Settings
|
|||
from scrapy.utils.defer import process_chain, process_parallel
|
||||
from scrapy.utils.misc import create_instance, load_object
|
||||
|
||||
if TYPE_CHECKING:
|
||||
# typing.Self requires Python 3.11
|
||||
from typing_extensions import Self
|
||||
|
||||
from scrapy.crawler import Crawler
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
|
|
@ -34,7 +55,9 @@ class MiddlewareManager:
|
|||
raise NotImplementedError
|
||||
|
||||
@classmethod
|
||||
def from_settings(cls, settings: Settings, crawler=None):
|
||||
def from_settings(
|
||||
cls, settings: Settings, crawler: Optional[Crawler] = None
|
||||
) -> Self:
|
||||
mwlist = cls._get_mwlist_from_settings(settings)
|
||||
middlewares = []
|
||||
enabled = []
|
||||
|
|
@ -63,7 +86,7 @@ class MiddlewareManager:
|
|||
return cls(*middlewares)
|
||||
|
||||
@classmethod
|
||||
def from_crawler(cls, crawler):
|
||||
def from_crawler(cls, crawler: Crawler) -> Self:
|
||||
return cls.from_settings(crawler.settings, crawler)
|
||||
|
||||
def _add_middleware(self, mw: Any) -> None:
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ based on different criteria.
|
|||
from io import StringIO
|
||||
from mimetypes import MimeTypes
|
||||
from pkgutil import get_data
|
||||
from typing import Dict, Mapping, Optional, Type, Union
|
||||
|
||||
from scrapy.http import Response
|
||||
from scrapy.utils.misc import load_object
|
||||
|
|
@ -29,15 +30,19 @@ class ResponseTypes:
|
|||
"text/*": "scrapy.http.TextResponse",
|
||||
}
|
||||
|
||||
def __init__(self):
|
||||
self.classes = {}
|
||||
self.mimetypes = MimeTypes()
|
||||
mimedata = get_data("scrapy", "mime.types").decode("utf8")
|
||||
self.mimetypes.readfp(StringIO(mimedata))
|
||||
def __init__(self) -> None:
|
||||
self.classes: Dict[str, Type[Response]] = {}
|
||||
self.mimetypes: MimeTypes = MimeTypes()
|
||||
mimedata = get_data("scrapy", "mime.types")
|
||||
if not mimedata:
|
||||
raise ValueError(
|
||||
"The mime.types file is not found in the Scrapy installation"
|
||||
)
|
||||
self.mimetypes.readfp(StringIO(mimedata.decode("utf8")))
|
||||
for mimetype, cls in self.CLASSES.items():
|
||||
self.classes[mimetype] = load_object(cls)
|
||||
|
||||
def from_mimetype(self, mimetype):
|
||||
def from_mimetype(self, mimetype: str) -> Type[Response]:
|
||||
"""Return the most appropriate Response class for the given mimetype"""
|
||||
if mimetype is None:
|
||||
return Response
|
||||
|
|
@ -46,7 +51,9 @@ class ResponseTypes:
|
|||
basetype = f"{mimetype.split('/')[0]}/*"
|
||||
return self.classes.get(basetype, Response)
|
||||
|
||||
def from_content_type(self, content_type, content_encoding=None):
|
||||
def from_content_type(
|
||||
self, content_type: Union[str, bytes], content_encoding: Optional[bytes] = None
|
||||
) -> Type[Response]:
|
||||
"""Return the most appropriate Response class from an HTTP Content-Type
|
||||
header"""
|
||||
if content_encoding:
|
||||
|
|
@ -56,7 +63,9 @@ class ResponseTypes:
|
|||
)
|
||||
return self.from_mimetype(mimetype)
|
||||
|
||||
def from_content_disposition(self, content_disposition):
|
||||
def from_content_disposition(
|
||||
self, content_disposition: Union[str, bytes]
|
||||
) -> Type[Response]:
|
||||
try:
|
||||
filename = (
|
||||
to_unicode(content_disposition, encoding="latin-1", errors="replace")
|
||||
|
|
@ -68,7 +77,7 @@ class ResponseTypes:
|
|||
except IndexError:
|
||||
return Response
|
||||
|
||||
def from_headers(self, headers):
|
||||
def from_headers(self, headers: Mapping[bytes, bytes]) -> Type[Response]:
|
||||
"""Return the most appropriate Response class by looking at the HTTP
|
||||
headers"""
|
||||
cls = Response
|
||||
|
|
@ -81,14 +90,14 @@ class ResponseTypes:
|
|||
cls = self.from_content_disposition(headers[b"Content-Disposition"])
|
||||
return cls
|
||||
|
||||
def from_filename(self, filename):
|
||||
def from_filename(self, filename: str) -> Type[Response]:
|
||||
"""Return the most appropriate Response class from a file name"""
|
||||
mimetype, encoding = self.mimetypes.guess_type(filename)
|
||||
if mimetype and not encoding:
|
||||
return self.from_mimetype(mimetype)
|
||||
return Response
|
||||
|
||||
def from_body(self, body):
|
||||
def from_body(self, body: bytes) -> Type[Response]:
|
||||
"""Try to guess the appropriate response based on the body content.
|
||||
This method is a bit magic and could be improved in the future, but
|
||||
it's not meant to be used except for special cases where response types
|
||||
|
|
@ -106,7 +115,13 @@ class ResponseTypes:
|
|||
return self.from_mimetype("text/html")
|
||||
return self.from_mimetype("text")
|
||||
|
||||
def from_args(self, headers=None, url=None, filename=None, body=None):
|
||||
def from_args(
|
||||
self,
|
||||
headers: Optional[Mapping[bytes, bytes]] = None,
|
||||
url: Optional[str] = None,
|
||||
filename: Optional[str] = None,
|
||||
body: Optional[bytes] = None,
|
||||
) -> Type[Response]:
|
||||
"""Guess the most appropriate Response class based on
|
||||
the given arguments."""
|
||||
cls = Response
|
||||
|
|
|
|||
|
|
@ -1,8 +1,10 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import traceback
|
||||
import warnings
|
||||
from collections import defaultdict
|
||||
from types import ModuleType
|
||||
from typing import DefaultDict, Dict, List, Tuple, Type
|
||||
from typing import TYPE_CHECKING, DefaultDict, Dict, List, Tuple, Type
|
||||
|
||||
from zope.interface import implementer
|
||||
|
||||
|
|
@ -12,6 +14,10 @@ from scrapy.settings import BaseSettings
|
|||
from scrapy.utils.misc import walk_modules
|
||||
from scrapy.utils.spider import iter_spider_classes
|
||||
|
||||
if TYPE_CHECKING:
|
||||
# typing.Self requires Python 3.11
|
||||
from typing_extensions import Self
|
||||
|
||||
|
||||
@implementer(ISpiderLoader)
|
||||
class SpiderLoader:
|
||||
|
|
@ -21,8 +27,8 @@ class SpiderLoader:
|
|||
"""
|
||||
|
||||
def __init__(self, settings: BaseSettings):
|
||||
self.spider_modules = settings.getlist("SPIDER_MODULES")
|
||||
self.warn_only = settings.getbool("SPIDER_LOADER_WARN_ONLY")
|
||||
self.spider_modules: List[str] = settings.getlist("SPIDER_MODULES")
|
||||
self.warn_only: bool = settings.getbool("SPIDER_LOADER_WARN_ONLY")
|
||||
self._spiders: Dict[str, Type[Spider]] = {}
|
||||
self._found: DefaultDict[str, List[Tuple[str, str]]] = defaultdict(list)
|
||||
self._load_all_spiders()
|
||||
|
|
@ -69,7 +75,7 @@ class SpiderLoader:
|
|||
self._check_name_duplicates()
|
||||
|
||||
@classmethod
|
||||
def from_settings(cls, settings):
|
||||
def from_settings(cls, settings: BaseSettings) -> Self:
|
||||
return cls(settings)
|
||||
|
||||
def load(self, spider_name: str) -> Type[Spider]:
|
||||
|
|
@ -90,7 +96,7 @@ class SpiderLoader:
|
|||
name for name, cls in self._spiders.items() if cls.handles_request(request)
|
||||
]
|
||||
|
||||
def list(self):
|
||||
def list(self) -> List[str]:
|
||||
"""
|
||||
Return a list with the names of all spiders available in the project.
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -6,15 +6,21 @@ See documentation in docs/topics/spiders.rst
|
|||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import TYPE_CHECKING, Optional
|
||||
from typing import TYPE_CHECKING, Any, Iterable, List, Optional, Union, cast
|
||||
|
||||
from twisted.internet.defer import Deferred
|
||||
|
||||
from scrapy import signals
|
||||
from scrapy.http import Request
|
||||
from scrapy.http import Request, Response
|
||||
from scrapy.utils.trackref import object_ref
|
||||
from scrapy.utils.url import url_is_from_spider
|
||||
|
||||
if TYPE_CHECKING:
|
||||
# typing.Self requires Python 3.11
|
||||
from typing_extensions import Self
|
||||
|
||||
from scrapy.crawler import Crawler
|
||||
from scrapy.settings import BaseSettings
|
||||
|
||||
|
||||
class Spider(object_ref):
|
||||
|
|
@ -25,21 +31,21 @@ class Spider(object_ref):
|
|||
name: str
|
||||
custom_settings: Optional[dict] = None
|
||||
|
||||
def __init__(self, name=None, **kwargs):
|
||||
def __init__(self, name: Optional[str] = None, **kwargs: Any):
|
||||
if name is not None:
|
||||
self.name = name
|
||||
elif not getattr(self, "name", None):
|
||||
raise ValueError(f"{type(self).__name__} must have a name")
|
||||
self.__dict__.update(kwargs)
|
||||
if not hasattr(self, "start_urls"):
|
||||
self.start_urls = []
|
||||
self.start_urls: List[str] = []
|
||||
|
||||
@property
|
||||
def logger(self):
|
||||
def logger(self) -> logging.LoggerAdapter:
|
||||
logger = logging.getLogger(self.name)
|
||||
return logging.LoggerAdapter(logger, {"spider": self})
|
||||
|
||||
def log(self, message, level=logging.DEBUG, **kw):
|
||||
def log(self, message: Any, level: int = logging.DEBUG, **kw: Any) -> None:
|
||||
"""Log the given message at the given log level
|
||||
|
||||
This helper wraps a log call to the logger within the spider, but you
|
||||
|
|
@ -49,17 +55,17 @@ class Spider(object_ref):
|
|||
self.logger.log(level, message, **kw)
|
||||
|
||||
@classmethod
|
||||
def from_crawler(cls, crawler, *args, **kwargs):
|
||||
def from_crawler(cls, crawler: Crawler, *args: Any, **kwargs: Any) -> Self:
|
||||
spider = cls(*args, **kwargs)
|
||||
spider._set_crawler(crawler)
|
||||
return spider
|
||||
|
||||
def _set_crawler(self, crawler: Crawler):
|
||||
def _set_crawler(self, crawler: Crawler) -> None:
|
||||
self.crawler = crawler
|
||||
self.settings = crawler.settings
|
||||
crawler.signals.connect(self.close, signals.spider_closed)
|
||||
|
||||
def start_requests(self):
|
||||
def start_requests(self) -> Iterable[Request]:
|
||||
if not self.start_urls and hasattr(self, "start_url"):
|
||||
raise AttributeError(
|
||||
"Crawling could not start: 'start_urls' not found "
|
||||
|
|
@ -69,16 +75,16 @@ class Spider(object_ref):
|
|||
for url in self.start_urls:
|
||||
yield Request(url, dont_filter=True)
|
||||
|
||||
def _parse(self, response, **kwargs):
|
||||
def _parse(self, response: Response, **kwargs: Any) -> Any:
|
||||
return self.parse(response, **kwargs)
|
||||
|
||||
def parse(self, response, **kwargs):
|
||||
def parse(self, response: Response, **kwargs: Any) -> Any:
|
||||
raise NotImplementedError(
|
||||
f"{self.__class__.__name__}.parse callback is not defined"
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def update_settings(cls, settings):
|
||||
def update_settings(cls, settings: BaseSettings) -> None:
|
||||
settings.setdict(cls.custom_settings or {}, priority="spider")
|
||||
|
||||
@classmethod
|
||||
|
|
@ -86,12 +92,13 @@ class Spider(object_ref):
|
|||
return url_is_from_spider(request.url, cls)
|
||||
|
||||
@staticmethod
|
||||
def close(spider, reason):
|
||||
def close(spider: Spider, reason: str) -> Union[Deferred, None]:
|
||||
closed = getattr(spider, "closed", None)
|
||||
if callable(closed):
|
||||
return closed(reason)
|
||||
return cast(Union[Deferred, None], closed(reason))
|
||||
return None
|
||||
|
||||
def __repr__(self):
|
||||
def __repr__(self) -> str:
|
||||
return f"<{type(self).__name__} {self.name!r} at 0x{id(self):0x}>"
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -14,6 +14,7 @@ from typing import (
|
|||
Iterable,
|
||||
List,
|
||||
Optional,
|
||||
Protocol,
|
||||
Tuple,
|
||||
Type,
|
||||
Union,
|
||||
|
|
@ -230,6 +231,11 @@ def fingerprint(
|
|||
return cache[cache_key]
|
||||
|
||||
|
||||
class RequestFingerprinterProtocol(Protocol):
|
||||
def fingerprint(self, request: Request) -> bytes:
|
||||
...
|
||||
|
||||
|
||||
class RequestFingerprinter:
|
||||
"""Default fingerprinter.
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue