From 57acad3c38602f4399c307c2c002f9eddde97cbc Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 29 Apr 2024 12:43:45 +0500 Subject: [PATCH] Full typing for scrapy/extensions, part 3. (#6325) --- scrapy/extensions/feedexport.py | 331 +++++++++++++++++++--------- scrapy/extensions/httpcache.py | 153 ++++++++----- scrapy/extensions/postprocessing.py | 4 +- scrapy/utils/iterators.py | 6 +- tox.ini | 14 +- 5 files changed, 328 insertions(+), 180 deletions(-) diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 3b0dd804e..97f39afe7 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -4,6 +4,8 @@ Feed Exports extension See documentation in docs/topics/feed-exports.rst """ +from __future__ import annotations + import logging import re import sys @@ -11,18 +13,36 @@ import warnings from datetime import datetime, timezone from pathlib import Path, PureWindowsPath from tempfile import NamedTemporaryFile -from typing import IO, Any, Callable, Dict, List, Optional, Tuple, Type, Union +from typing import ( + IO, + TYPE_CHECKING, + Any, + Callable, + Dict, + Iterable, + List, + Optional, + Protocol, + Tuple, + Type, + TypeVar, + Union, + cast, +) from urllib.parse import unquote, urlparse -from twisted.internet import defer, threads -from twisted.internet.defer import DeferredList +from twisted.internet import threads +from twisted.internet.defer import Deferred, DeferredList, maybeDeferred +from twisted.python.failure import Failure from w3lib.url import file_uri_to_path from zope.interface import Interface, implementer from scrapy import Spider, signals +from scrapy.crawler import Crawler from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning from scrapy.exporters import BaseItemExporter from scrapy.extensions.postprocessing import PostProcessingManager +from scrapy.settings import BaseSettings, Settings from scrapy.utils.boto import is_botocore_available from scrapy.utils.conf import feed_complete_default_values_from_settings from scrapy.utils.defer import maybe_deferred_to_future @@ -32,6 +52,12 @@ from scrapy.utils.log import failure_to_exc_info from scrapy.utils.misc import build_from_crawler, load_object from scrapy.utils.python import without_none_values +if TYPE_CHECKING: + from _typeshed import OpenBinaryMode + + # typing.Self requires Python 3.11 + from typing_extensions import Self + logger = logging.getLogger(__name__) try: @@ -41,8 +67,19 @@ try: except ImportError: IS_BOTO3_AVAILABLE = False +UriParamsCallableT = Callable[[Dict[str, Any], Spider], Optional[Dict[str, Any]]] -def build_storage(builder, uri, *args, feed_options=None, preargs=(), **kwargs): +_StorageT = TypeVar("_StorageT", bound="FeedStorageProtocol") + + +def build_storage( + builder: Callable[..., _StorageT], + uri: str, + *args: Any, + feed_options: Optional[Dict[str, Any]] = None, + preargs: Iterable[Any] = (), + **kwargs: Any, +) -> _StorageT: kwargs["feed_options"] = feed_options return builder(*preargs, uri, *args, **kwargs) @@ -56,10 +93,10 @@ class ItemFilter: :type feed_options: dict """ - feed_options: Optional[dict] - item_classes: Tuple + feed_options: Optional[Dict[str, Any]] + item_classes: Tuple[type, ...] - def __init__(self, feed_options: Optional[dict]) -> None: + def __init__(self, feed_options: Optional[Dict[str, Any]]) -> None: self.feed_options = feed_options if feed_options is not None: self.item_classes = tuple( @@ -98,28 +135,49 @@ class IFeedStorage(Interface): """Store the given file stream""" +class FeedStorageProtocol(Protocol): + """Reimplementation of ``IFeedStorage`` that can be used in type hints.""" + + def __init__(self, uri: str, *, feed_options: Optional[Dict[str, Any]] = None): + """Initialize the storage with the parameters given in the URI and the + feed-specific options (see :setting:`FEEDS`)""" + + def open(self, spider: Spider) -> IO[bytes]: + """Open the storage for the given spider. It must return a file-like + object that will be used for the exporters""" + + def store(self, file: IO[bytes]) -> Optional[Deferred]: + """Store the given file stream""" + + @implementer(IFeedStorage) class BlockingFeedStorage: - def open(self, spider): + def open(self, spider: Spider) -> IO[bytes]: path = spider.crawler.settings["FEED_TEMPDIR"] if path and not Path(path).is_dir(): raise OSError("Not a Directory: " + str(path)) return NamedTemporaryFile(prefix="feed-", dir=path) - def store(self, file): + def store(self, file: IO[bytes]) -> Optional[Deferred]: return threads.deferToThread(self._store_in_thread, file) - def _store_in_thread(self, file): + def _store_in_thread(self, file: IO[bytes]) -> None: raise NotImplementedError @implementer(IFeedStorage) class StdoutFeedStorage: - def __init__(self, uri, _stdout=None, *, feed_options=None): + def __init__( + self, + uri: str, + _stdout: Optional[IO[bytes]] = None, + *, + feed_options: Optional[Dict[str, Any]] = None, + ): if not _stdout: _stdout = sys.stdout.buffer - self._stdout = _stdout + self._stdout: IO[bytes] = _stdout if feed_options and feed_options.get("overwrite", False) is True: logger.warning( "Standard output (stdout) storage does not support " @@ -128,54 +186,58 @@ class StdoutFeedStorage: "it to False." ) - def open(self, spider): + def open(self, spider: Spider) -> IO[bytes]: return self._stdout - def store(self, file): + def store(self, file: IO[bytes]) -> Optional[Deferred]: pass @implementer(IFeedStorage) class FileFeedStorage: - def __init__(self, uri, *, feed_options=None): - self.path = file_uri_to_path(uri) + def __init__(self, uri: str, *, feed_options: Optional[Dict[str, Any]] = None): + self.path: str = file_uri_to_path(uri) feed_options = feed_options or {} - self.write_mode = "wb" if feed_options.get("overwrite", False) else "ab" + self.write_mode: OpenBinaryMode = ( + "wb" if feed_options.get("overwrite", False) else "ab" + ) - def open(self, spider) -> IO[Any]: + def open(self, spider: Spider) -> IO[bytes]: dirname = Path(self.path).parent if dirname and not dirname.exists(): dirname.mkdir(parents=True) return Path(self.path).open(self.write_mode) - def store(self, file): + def store(self, file: IO[bytes]) -> Optional[Deferred]: file.close() + return None class S3FeedStorage(BlockingFeedStorage): def __init__( self, - uri, - access_key=None, - secret_key=None, - acl=None, - endpoint_url=None, + uri: str, + access_key: Optional[str] = None, + secret_key: Optional[str] = None, + acl: Optional[str] = None, + endpoint_url: Optional[str] = None, *, - feed_options=None, - session_token=None, - region_name=None, + feed_options: Optional[Dict[str, Any]] = None, + session_token: Optional[str] = None, + region_name: Optional[str] = None, ): if not is_botocore_available(): raise NotConfigured("missing botocore library") u = urlparse(uri) - self.bucketname = u.hostname - self.access_key = u.username or access_key - self.secret_key = u.password or secret_key - self.session_token = session_token - self.keyname = u.path[1:] # remove first "/" - self.acl = acl - self.endpoint_url = endpoint_url - self.region_name = region_name + assert u.hostname + self.bucketname: str = u.hostname + self.access_key: Optional[str] = u.username or access_key + self.secret_key: Optional[str] = u.password or secret_key + self.session_token: Optional[str] = session_token + self.keyname: str = u.path[1:] # remove first "/" + self.acl: Optional[str] = acl + self.endpoint_url: Optional[str] = endpoint_url + self.region_name: Optional[str] = region_name if IS_BOTO3_AVAILABLE: import boto3.session @@ -218,7 +280,13 @@ class S3FeedStorage(BlockingFeedStorage): ) @classmethod - def from_crawler(cls, crawler, uri, *, feed_options=None): + def from_crawler( + cls, + crawler: Crawler, + uri: str, + *, + feed_options: Optional[Dict[str, Any]] = None, + ) -> Self: return build_storage( cls, uri, @@ -231,8 +299,9 @@ class S3FeedStorage(BlockingFeedStorage): feed_options=feed_options, ) - def _store_in_thread(self, file): + def _store_in_thread(self, file: IO[bytes]) -> None: file.seek(0) + kwargs: Dict[str, Any] if IS_BOTO3_AVAILABLE: kwargs = {"ExtraArgs": {"ACL": self.acl}} if self.acl else {} self.s3_client.upload_fileobj( @@ -247,22 +316,23 @@ class S3FeedStorage(BlockingFeedStorage): class GCSFeedStorage(BlockingFeedStorage): - def __init__(self, uri, project_id, acl): - self.project_id = project_id - self.acl = acl + def __init__(self, uri: str, project_id: Optional[str], acl: Optional[str]): + self.project_id: Optional[str] = project_id + self.acl: Optional[str] = acl u = urlparse(uri) - self.bucket_name = u.hostname - self.blob_name = u.path[1:] # remove first "/" + assert u.hostname + self.bucket_name: str = u.hostname + self.blob_name: str = u.path[1:] # remove first "/" @classmethod - def from_crawler(cls, crawler, uri): + def from_crawler(cls, crawler: Crawler, uri: str) -> Self: return cls( uri, crawler.settings["GCS_PROJECT_ID"], crawler.settings["FEED_STORAGE_GCS_ACL"] or None, ) - def _store_in_thread(self, file): + def _store_in_thread(self, file: IO[bytes]) -> None: file.seek(0) from google.cloud.storage import Client @@ -292,7 +362,13 @@ class FTPFeedStorage(BlockingFeedStorage): self.overwrite: bool = not feed_options or feed_options.get("overwrite", True) @classmethod - def from_crawler(cls, crawler, uri, *, feed_options=None): + def from_crawler( + cls, + crawler: Crawler, + uri: str, + *, + feed_options: Optional[Dict[str, Any]] = None, + ) -> Self: return build_storage( cls, uri, @@ -300,7 +376,7 @@ class FTPFeedStorage(BlockingFeedStorage): feed_options=feed_options, ) - def _store_in_thread(self, file): + def _store_in_thread(self, file: IO[bytes]) -> None: ftp_store_file( path=self.path, file=file, @@ -316,46 +392,51 @@ class FTPFeedStorage(BlockingFeedStorage): class FeedSlot: def __init__( self, - storage, - uri, - format, - store_empty, - batch_id, - uri_template, - filter, - feed_options, - spider, + storage: FeedStorageProtocol, + uri: str, + format: str, + store_empty: bool, + batch_id: int, + uri_template: str, + filter: ItemFilter, + feed_options: Dict[str, Any], + spider: Spider, exporters: Dict[str, Type[BaseItemExporter]], - settings, - crawler, + settings: BaseSettings, + crawler: Crawler, ): - self.file = None + self.file: Optional[IO[bytes]] = None self.exporter: Optional[BaseItemExporter] = None - self.storage = storage + self.storage: FeedStorageProtocol = storage # feed params - self.batch_id = batch_id - self.format = format - self.store_empty = store_empty - self.uri_template = uri_template - self.uri = uri - self.filter = filter + self.batch_id: int = batch_id + self.format: str = format + self.store_empty: bool = store_empty + self.uri_template: str = uri_template + self.uri: str = uri + self.filter: ItemFilter = filter # exporter params - self.feed_options = feed_options - self.spider = spider + self.feed_options: Dict[str, Any] = feed_options + self.spider: Spider = spider self.exporters: Dict[str, Type[BaseItemExporter]] = exporters - self.settings = settings - self.crawler = crawler + self.settings: BaseSettings = settings + self.crawler: Crawler = crawler # flags - self.itemcount = 0 - self._exporting = False - self._fileloaded = False + self.itemcount: int = 0 + self._exporting: bool = False + self._fileloaded: bool = False - def start_exporting(self): + def start_exporting(self) -> None: if not self._fileloaded: self.file = self.storage.open(self.spider) if "postprocessing" in self.feed_options: - self.file = PostProcessingManager( - self.feed_options["postprocessing"], self.file, self.feed_options + self.file = cast( + IO[bytes], + PostProcessingManager( + self.feed_options["postprocessing"], + self.file, + self.feed_options, + ), ) self.exporter = self._get_exporter( file=self.file, @@ -368,17 +449,23 @@ class FeedSlot: self._fileloaded = True if not self._exporting: + assert self.exporter self.exporter.start_exporting() self._exporting = True - def _get_instance(self, objcls, *args, **kwargs): + def _get_instance( + self, objcls: Type[BaseItemExporter], *args: Any, **kwargs: Any + ) -> BaseItemExporter: return build_from_crawler(objcls, self.crawler, *args, **kwargs) - def _get_exporter(self, file, format, *args, **kwargs) -> BaseItemExporter: + def _get_exporter( + self, file: IO[bytes], format: str, *args: Any, **kwargs: Any + ) -> BaseItemExporter: return self._get_instance(self.exporters[format], file, *args, **kwargs) - def finish_exporting(self): + def finish_exporting(self) -> None: if self._exporting: + assert self.exporter self.exporter.finish_exporting() self._exporting = False @@ -390,22 +477,22 @@ _FeedSlot = create_deprecated_class( class FeedExporter: - _pending_deferreds: List[defer.Deferred] = [] + _pending_deferreds: List[Deferred] = [] @classmethod - def from_crawler(cls, crawler): + def from_crawler(cls, crawler: Crawler) -> Self: exporter = cls(crawler) crawler.signals.connect(exporter.open_spider, signals.spider_opened) crawler.signals.connect(exporter.close_spider, signals.spider_closed) crawler.signals.connect(exporter.item_scraped, signals.item_scraped) return exporter - def __init__(self, crawler): - self.crawler = crawler - self.settings = crawler.settings + def __init__(self, crawler: Crawler): + self.crawler: Crawler = crawler + self.settings: Settings = crawler.settings self.feeds = {} - self.slots = [] - self.filters = {} + self.slots: List[FeedSlot] = [] + self.filters: Dict[str, ItemFilter] = {} if not self.settings["FEEDS"] and not self.settings["FEED_URI"]: raise NotConfigured @@ -437,8 +524,12 @@ class FeedExporter: ) self.filters[uri] = self._load_filter(feed_options) - self.storages = self._load_components("FEED_STORAGES") - self.exporters = self._load_components("FEED_EXPORTERS") + self.storages: Dict[str, Type[FeedStorageProtocol]] = self._load_components( + "FEED_STORAGES" + ) + self.exporters: Dict[str, Type[BaseItemExporter]] = self._load_components( + "FEED_EXPORTERS" + ) for uri, feed_options in self.feeds.items(): if not self._storage_supported(uri, feed_options): raise NotConfigured @@ -447,7 +538,7 @@ class FeedExporter: if not self._exporter_supported(feed_options["format"]): raise NotConfigured - def open_spider(self, spider): + def open_spider(self, spider: Spider) -> None: for uri, feed_options in self.feeds.items(): uri_params = self._get_uri_params(spider, feed_options["uri_params"]) self.slots.append( @@ -460,7 +551,7 @@ class FeedExporter: ) ) - async def close_spider(self, spider): + async def close_spider(self, spider: Spider) -> None: for slot in self.slots: self._close_slot(slot, spider) @@ -473,8 +564,9 @@ class FeedExporter: self.crawler.signals.send_catch_log_deferred(signals.feed_exporter_closed) ) - def _close_slot(self, slot, spider): - def get_file(slot_): + def _close_slot(self, slot: FeedSlot, spider: Spider) -> Optional[Deferred]: + def get_file(slot_: FeedSlot) -> IO[bytes]: + assert slot_.file if isinstance(slot_.file, PostProcessingManager): slot_.file.close() return slot_.file.file @@ -492,7 +584,7 @@ class FeedExporter: return None logmsg = f"{slot.format} feed ({slot.itemcount} items) in: {slot.uri}" - d = defer.maybeDeferred(slot.storage.store, get_file(slot)) + d: Deferred = maybeDeferred(slot.storage.store, get_file(slot)) d.addCallback( self._handle_store_success, logmsg, spider, type(slot.storage).__name__ @@ -510,20 +602,33 @@ class FeedExporter: return d - def _handle_store_error(self, f, logmsg, spider, slot_type): + def _handle_store_error( + self, f: Failure, logmsg: str, spider: Spider, slot_type: str + ) -> None: logger.error( "Error storing %s", logmsg, exc_info=failure_to_exc_info(f), extra={"spider": spider}, ) + assert self.crawler.stats self.crawler.stats.inc_value(f"feedexport/failed_count/{slot_type}") - def _handle_store_success(self, f, logmsg, spider, slot_type): + def _handle_store_success( + self, f: Failure, logmsg: str, spider: Spider, slot_type: str + ) -> None: logger.info("Stored %s", logmsg, extra={"spider": spider}) + assert self.crawler.stats self.crawler.stats.inc_value(f"feedexport/success_count/{slot_type}") - def _start_new_batch(self, batch_id, uri, feed_options, spider, uri_template): + def _start_new_batch( + self, + batch_id: int, + uri: str, + feed_options: Dict[str, Any], + spider: Spider, + uri_template: str, + ) -> FeedSlot: """ Redirect the output data stream to a new file. Execute multiple times if FEED_EXPORT_BATCH_ITEM_COUNT setting or FEEDS.batch_item_count is specified @@ -546,11 +651,11 @@ class FeedExporter: spider=spider, exporters=self.exporters, settings=self.settings, - crawler=getattr(self, "crawler", None), + crawler=self.crawler, ) return slot - def item_scraped(self, item, spider): + def item_scraped(self, item: Any, spider: Spider) -> None: slots = [] for slot in self.slots: if not slot.filter.accepts(item): @@ -560,6 +665,7 @@ class FeedExporter: continue slot.start_exporting() + assert slot.exporter slot.exporter.export_item(item) slot.itemcount += 1 # create new slot for each slot with itemcount == FEED_EXPORT_BATCH_ITEM_COUNT and close the old one @@ -584,7 +690,7 @@ class FeedExporter: slots.append(slot) self.slots = slots - def _load_components(self, setting_prefix): + def _load_components(self, setting_prefix: str) -> Dict[str, Any]: conf = without_none_values(self.settings.getwithbase(setting_prefix)) d = {} for k, v in conf.items(): @@ -594,12 +700,13 @@ class FeedExporter: pass return d - def _exporter_supported(self, format): + def _exporter_supported(self, format: str) -> bool: if format in self.exporters: return True logger.error("Unknown feed format: %(format)s", {"format": format}) + return False - def _settings_are_valid(self): + def _settings_are_valid(self) -> bool: """ If FEED_EXPORT_BATCH_ITEM_COUNT setting or FEEDS.batch_item_count is specified uri has to contain %(batch_time)s or %(batch_id)d to distinguish different files of partial output @@ -617,7 +724,7 @@ class FeedExporter: return False return True - def _storage_supported(self, uri, feed_options): + def _storage_supported(self, uri: str, feed_options: Dict[str, Any]) -> bool: scheme = urlparse(uri).scheme if scheme in self.storages or PureWindowsPath(uri).drive: try: @@ -630,8 +737,11 @@ class FeedExporter: ) else: logger.error("Unknown feed storage scheme: %(scheme)s", {"scheme": scheme}) + return False - def _get_storage(self, uri, feed_options): + def _get_storage( + self, uri: str, feed_options: Dict[str, Any] + ) -> FeedStorageProtocol: """Fork of create_instance specific to feed storage classes It supports not passing the *feed_options* parameters to classes that @@ -640,11 +750,14 @@ class FeedExporter: feedcls = self.storages.get(urlparse(uri).scheme, self.storages["file"]) crawler = getattr(self, "crawler", None) - def build_instance(builder, *preargs): + def build_instance( + builder: Type[FeedStorageProtocol], *preargs: Any + ) -> FeedStorageProtocol: return build_storage( builder, uri, feed_options=feed_options, preargs=preargs ) + instance: FeedStorageProtocol if crawler and hasattr(feedcls, "from_crawler"): instance = build_instance(feedcls.from_crawler, crawler) method_name = "from_crawler" @@ -661,9 +774,9 @@ class FeedExporter: def _get_uri_params( self, spider: Spider, - uri_params_function: Optional[Union[str, Callable[[dict, Spider], dict]]], + uri_params_function: Union[str, UriParamsCallableT, None], slot: Optional[FeedSlot] = None, - ) -> dict: + ) -> Dict[str, Any]: params = {} for k in dir(spider): params[k] = getattr(spider, k) @@ -671,7 +784,7 @@ class FeedExporter: params["time"] = utc_now.replace(microsecond=0).isoformat().replace(":", "-") params["batch_time"] = utc_now.isoformat().replace(":", "-") params["batch_id"] = slot.batch_id + 1 if slot is not None else 1 - uripar_function = ( + uripar_function: UriParamsCallableT = ( load_object(uri_params_function) if uri_params_function else lambda params, _: params @@ -679,7 +792,9 @@ class FeedExporter: new_params = uripar_function(params, spider) return new_params if new_params is not None else params - def _load_filter(self, feed_options): + def _load_filter(self, feed_options: Dict[str, Any]) -> ItemFilter: # load the item filter if declared else load the default filter class - item_filter_class = load_object(feed_options.get("item_filter", ItemFilter)) + item_filter_class: Type[ItemFilter] = load_object( + feed_options.get("item_filter", ItemFilter) + ) return item_filter_class(feed_options) diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.py index 335728502..dd5bce24f 100644 --- a/scrapy/extensions/httpcache.py +++ b/scrapy/extensions/httpcache.py @@ -1,10 +1,13 @@ import gzip import logging +import os import pickle # nosec from email.utils import mktime_tz, parsedate_tz from importlib import import_module from pathlib import Path from time import time +from types import ModuleType +from typing import IO, TYPE_CHECKING, Any, Callable, Dict, List, Optional, Union, cast from weakref import WeakKeyDictionary from w3lib.http import headers_dict_to_raw, headers_raw_to_dict @@ -12,49 +15,65 @@ from w3lib.http import headers_dict_to_raw, headers_raw_to_dict from scrapy.http import Headers, Response from scrapy.http.request import Request from scrapy.responsetypes import responsetypes +from scrapy.settings import BaseSettings from scrapy.spiders import Spider from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.project import data_path from scrapy.utils.python import to_bytes, to_unicode +from scrapy.utils.request import RequestFingerprinter + +if TYPE_CHECKING: + # typing.Concatenate requires Python 3.10 + from typing_extensions import Concatenate + logger = logging.getLogger(__name__) class DummyPolicy: - def __init__(self, settings): - self.ignore_schemes = settings.getlist("HTTPCACHE_IGNORE_SCHEMES") - self.ignore_http_codes = [ + def __init__(self, settings: BaseSettings): + self.ignore_schemes: List[str] = settings.getlist("HTTPCACHE_IGNORE_SCHEMES") + self.ignore_http_codes: List[int] = [ int(x) for x in settings.getlist("HTTPCACHE_IGNORE_HTTP_CODES") ] - def should_cache_request(self, request): + def should_cache_request(self, request: Request) -> bool: return urlparse_cached(request).scheme not in self.ignore_schemes - def should_cache_response(self, response, request): + def should_cache_response(self, response: Response, request: Request) -> bool: return response.status not in self.ignore_http_codes - def is_cached_response_fresh(self, cachedresponse, request): + def is_cached_response_fresh( + self, cachedresponse: Response, request: Request + ) -> bool: return True - def is_cached_response_valid(self, cachedresponse, response, request): + def is_cached_response_valid( + self, cachedresponse: Response, response: Response, request: Request + ) -> bool: return True class RFC2616Policy: MAXAGE = 3600 * 24 * 365 # one year - def __init__(self, settings): - self.always_store = settings.getbool("HTTPCACHE_ALWAYS_STORE") - self.ignore_schemes = settings.getlist("HTTPCACHE_IGNORE_SCHEMES") - self._cc_parsed = WeakKeyDictionary() - self.ignore_response_cache_controls = [ + def __init__(self, settings: BaseSettings): + self.always_store: bool = settings.getbool("HTTPCACHE_ALWAYS_STORE") + self.ignore_schemes: List[str] = settings.getlist("HTTPCACHE_IGNORE_SCHEMES") + self._cc_parsed: WeakKeyDictionary[ + Union[Request, Response], Dict[bytes, Optional[bytes]] + ] = WeakKeyDictionary() + self.ignore_response_cache_controls: List[bytes] = [ to_bytes(cc) for cc in settings.getlist("HTTPCACHE_IGNORE_RESPONSE_CACHE_CONTROLS") ] - def _parse_cachecontrol(self, r): + def _parse_cachecontrol( + self, r: Union[Request, Response] + ) -> Dict[bytes, Optional[bytes]]: if r not in self._cc_parsed: cch = r.headers.get(b"Cache-Control", b"") + assert cch is not None parsed = parse_cachecontrol(cch) if isinstance(r, Response): for key in self.ignore_response_cache_controls: @@ -62,7 +81,7 @@ class RFC2616Policy: self._cc_parsed[r] = parsed return self._cc_parsed[r] - def should_cache_request(self, request): + def should_cache_request(self, request: Request) -> bool: if urlparse_cached(request).scheme in self.ignore_schemes: return False cc = self._parse_cachecontrol(request) @@ -72,7 +91,7 @@ class RFC2616Policy: # Any other is eligible for caching return True - def should_cache_response(self, response, request): + def should_cache_response(self, response: Response, request: Request) -> bool: # What is cacheable - https://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.9.1 # Response cacheability - https://www.w3.org/Protocols/rfc2616/rfc2616-sec13.html#sec13.4 # Status code 206 is not included because cache can not deal with partial contents @@ -100,7 +119,9 @@ class RFC2616Policy: # info and can not be revalidated return False - def is_cached_response_fresh(self, cachedresponse, request): + def is_cached_response_fresh( + self, cachedresponse: Response, request: Request + ) -> bool: cc = self._parse_cachecontrol(cachedresponse) ccreq = self._parse_cachecontrol(request) if b"no-cache" in cc or b"no-cache" in ccreq: @@ -141,7 +162,9 @@ class RFC2616Policy: self._set_conditional_validators(request, cachedresponse) return False - def is_cached_response_valid(self, cachedresponse, response, request): + def is_cached_response_valid( + self, cachedresponse: Response, response: Response, request: Request + ) -> bool: # Use the cached response if the new response is a server error, # as long as the old response didn't specify must-revalidate. if response.status >= 500: @@ -152,7 +175,9 @@ class RFC2616Policy: # Use the cached response if the server says it hasn't changed. return response.status == 304 - def _set_conditional_validators(self, request, cachedresponse): + def _set_conditional_validators( + self, request: Request, cachedresponse: Response + ) -> None: if b"Last-Modified" in cachedresponse.headers: request.headers[b"If-Modified-Since"] = cachedresponse.headers[ b"Last-Modified" @@ -161,13 +186,15 @@ class RFC2616Policy: if b"ETag" in cachedresponse.headers: request.headers[b"If-None-Match"] = cachedresponse.headers[b"ETag"] - def _get_max_age(self, cc): + def _get_max_age(self, cc: Dict[bytes, Optional[bytes]]) -> Optional[int]: try: - return max(0, int(cc[b"max-age"])) + return max(0, int(cc[b"max-age"])) # type: ignore[arg-type] except (KeyError, ValueError): return None - def _compute_freshness_lifetime(self, response, request, now): + def _compute_freshness_lifetime( + self, response: Response, request: Request, now: float + ) -> float: # Reference nsHttpResponseHead::ComputeFreshnessLifetime # https://dxr.mozilla.org/mozilla-central/source/netwerk/protocol/http/nsHttpResponseHead.cpp#706 cc = self._parse_cachecontrol(response) @@ -198,10 +225,12 @@ class RFC2616Policy: # Insufficient information to compute freshness lifetime return 0 - def _compute_current_age(self, response, request, now): + def _compute_current_age( + self, response: Response, request: Request, now: float + ) -> float: # Reference nsHttpResponseHead::ComputeCurrentAge # https://dxr.mozilla.org/mozilla-central/source/netwerk/protocol/http/nsHttpResponseHead.cpp#658 - currentage = 0 + currentage: float = 0 # If Date header is not set we assume it is a fast connection, and # clock is in sync with the server date = rfc1123_to_epoch(response.headers.get(b"Date")) or now @@ -210,7 +239,7 @@ class RFC2616Policy: if b"Age" in response.headers: try: - age = int(response.headers[b"Age"]) + age = int(response.headers[b"Age"]) # type: ignore[arg-type] currentage = max(currentage, age) except ValueError: pass @@ -219,13 +248,13 @@ class RFC2616Policy: class DbmCacheStorage: - def __init__(self, settings): - self.cachedir = data_path(settings["HTTPCACHE_DIR"], createdir=True) - self.expiration_secs = settings.getint("HTTPCACHE_EXPIRATION_SECS") - self.dbmodule = import_module(settings["HTTPCACHE_DBM_MODULE"]) - self.db = None + def __init__(self, settings: BaseSettings): + self.cachedir: str = data_path(settings["HTTPCACHE_DIR"], createdir=True) + self.expiration_secs: int = settings.getint("HTTPCACHE_EXPIRATION_SECS") + self.dbmodule: ModuleType = import_module(settings["HTTPCACHE_DBM_MODULE"]) + self.db: Any = None # the real type is private - def open_spider(self, spider: Spider): + def open_spider(self, spider: Spider) -> None: dbpath = Path(self.cachedir, f"{spider.name}.db") self.db = self.dbmodule.open(str(dbpath), "c") @@ -235,15 +264,16 @@ class DbmCacheStorage: extra={"spider": spider}, ) - self._fingerprinter = spider.crawler.request_fingerprinter + assert spider.crawler.request_fingerprinter + self._fingerprinter: RequestFingerprinter = spider.crawler.request_fingerprinter - def close_spider(self, spider): + def close_spider(self, spider: Spider) -> None: self.db.close() - def retrieve_response(self, spider, request): + def retrieve_response(self, spider: Spider, request: Request) -> Optional[Response]: data = self._read_data(spider, request) if data is None: - return # not cached + return None # not cached url = data["url"] status = data["status"] headers = Headers(data["headers"]) @@ -252,7 +282,9 @@ class DbmCacheStorage: response = respcls(url=url, headers=headers, status=status, body=body) return response - def store_response(self, spider, request, response): + def store_response( + self, spider: Spider, request: Request, response: Response + ) -> None: key = self._fingerprinter.fingerprint(request).hex() data = { "status": response.status, @@ -263,28 +295,31 @@ class DbmCacheStorage: self.db[f"{key}_data"] = pickle.dumps(data, protocol=4) self.db[f"{key}_time"] = str(time()) - def _read_data(self, spider, request): + def _read_data(self, spider: Spider, request: Request) -> Optional[Dict[str, Any]]: key = self._fingerprinter.fingerprint(request).hex() db = self.db tkey = f"{key}_time" if tkey not in db: - return # not found + return None # not found ts = db[tkey] if 0 < self.expiration_secs < time() - float(ts): - return # expired + return None # expired - return pickle.loads(db[f"{key}_data"]) # nosec + return cast(Dict[str, Any], pickle.loads(db[f"{key}_data"])) # nosec class FilesystemCacheStorage: - def __init__(self, settings): - self.cachedir = data_path(settings["HTTPCACHE_DIR"]) - self.expiration_secs = settings.getint("HTTPCACHE_EXPIRATION_SECS") - self.use_gzip = settings.getbool("HTTPCACHE_GZIP") - self._open = gzip.open if self.use_gzip else open + def __init__(self, settings: BaseSettings): + self.cachedir: str = data_path(settings["HTTPCACHE_DIR"]) + self.expiration_secs: int = settings.getint("HTTPCACHE_EXPIRATION_SECS") + self.use_gzip: bool = settings.getbool("HTTPCACHE_GZIP") + # https://github.com/python/mypy/issues/10740 + self._open: Callable[Concatenate[Union[str, os.PathLike], str, ...], IO] = ( + gzip.open if self.use_gzip else open # type: ignore[assignment] + ) - def open_spider(self, spider: Spider): + def open_spider(self, spider: Spider) -> None: logger.debug( "Using filesystem cache storage in %(cachedir)s", {"cachedir": self.cachedir}, @@ -294,27 +329,29 @@ class FilesystemCacheStorage: assert spider.crawler.request_fingerprinter self._fingerprinter = spider.crawler.request_fingerprinter - def close_spider(self, spider): + def close_spider(self, spider: Spider) -> None: pass - def retrieve_response(self, spider: Spider, request: Request): + def retrieve_response(self, spider: Spider, request: Request) -> Optional[Response]: """Return response if present in cache, or None otherwise.""" metadata = self._read_meta(spider, request) if metadata is None: - return # not cached + return None # not cached rpath = Path(self._get_request_path(spider, request)) with self._open(rpath / "response_body", "rb") as f: body = f.read() with self._open(rpath / "response_headers", "rb") as f: rawheaders = f.read() - url = metadata.get("response_url") + url = metadata["response_url"] status = metadata["status"] headers = Headers(headers_raw_to_dict(rawheaders)) respcls = responsetypes.from_args(headers=headers, url=url, body=body) response = respcls(url=url, headers=headers, status=status, body=body) return response - def store_response(self, spider: Spider, request: Request, response): + def store_response( + self, spider: Spider, request: Request, response: Response + ) -> None: """Store the given response in the cache.""" rpath = Path(self._get_request_path(spider, request)) if not rpath.exists(): @@ -343,19 +380,19 @@ class FilesystemCacheStorage: key = self._fingerprinter.fingerprint(request).hex() return str(Path(self.cachedir, spider.name, key[0:2], key)) - def _read_meta(self, spider: Spider, request: Request): + def _read_meta(self, spider: Spider, request: Request) -> Optional[Dict[str, Any]]: rpath = Path(self._get_request_path(spider, request)) metapath = rpath / "pickled_meta" if not metapath.exists(): - return # not found + return None # not found mtime = metapath.stat().st_mtime if 0 < self.expiration_secs < time() - mtime: - return # expired + return None # expired with self._open(metapath, "rb") as f: - return pickle.load(f) # nosec + return cast(Dict[str, Any], pickle.load(f)) # nosec -def parse_cachecontrol(header): +def parse_cachecontrol(header: bytes) -> Dict[bytes, Optional[bytes]]: """Parse Cache-Control header https://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.9 @@ -375,9 +412,9 @@ def parse_cachecontrol(header): return directives -def rfc1123_to_epoch(date_str): +def rfc1123_to_epoch(date_str: Union[str, bytes, None]) -> Optional[int]: try: - date_str = to_unicode(date_str, encoding="ascii") - return mktime_tz(parsedate_tz(date_str)) + date_str = to_unicode(date_str, encoding="ascii") # type: ignore[arg-type] + return mktime_tz(parsedate_tz(date_str)) # type: ignore[arg-type] except Exception: return None diff --git a/scrapy/extensions/postprocessing.py b/scrapy/extensions/postprocessing.py index 7ffbd8bc3..ac12ad829 100644 --- a/scrapy/extensions/postprocessing.py +++ b/scrapy/extensions/postprocessing.py @@ -6,7 +6,7 @@ from bz2 import BZ2File from gzip import GzipFile from io import IOBase from lzma import LZMAFile -from typing import Any, BinaryIO, Dict, List, cast +from typing import IO, Any, BinaryIO, Dict, List, cast from scrapy.utils.misc import load_object @@ -126,7 +126,7 @@ class PostProcessingManager(IOBase): """ def __init__( - self, plugins: List[Any], file: BinaryIO, feed_options: Dict[str, Any] + self, plugins: List[Any], file: IO[bytes], feed_options: Dict[str, Any] ) -> None: self.plugins = self._load_plugins(plugins) self.file = file diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py index 49493e9c6..cd6e9d04e 100644 --- a/scrapy/utils/iterators.py +++ b/scrapy/utils/iterators.py @@ -3,7 +3,6 @@ import logging import re from io import StringIO from typing import ( - TYPE_CHECKING, Any, Callable, Dict, @@ -25,9 +24,6 @@ from scrapy.http import Response, TextResponse from scrapy.selector import Selector from scrapy.utils.python import re_rsearch, to_unicode -if TYPE_CHECKING: - from lxml._types import SupportsReadClose # nosec - logger = logging.getLogger(__name__) @@ -98,7 +94,7 @@ def xmliter_lxml( reader = _StreamReader(obj) tag = f"{{{namespace}}}{nodename}" if namespace else nodename iterable = etree.iterparse( - cast("SupportsReadClose[bytes]", reader), + reader, encoding=reader.encoding, events=("end", "start-ns"), resolve_entities=False, diff --git a/tox.ini b/tox.ini index b5effb527..d7527bb04 100644 --- a/tox.ini +++ b/tox.ini @@ -43,14 +43,14 @@ install_command = [testenv:typing] basepython = python3 deps = - mypy==1.8.0 - typing-extensions==4.10.0 + mypy==1.10.0 + typing-extensions==4.11.0 types-attrs==19.1.0 - types-lxml==2024.2.9 - types-Pillow==10.2.0.20240213 - types-Pygments==2.17.0.20240106 - types-pyOpenSSL==24.0.0.20240130 - types-setuptools==69.1.0.20240223 + types-lxml==2024.4.14 + types-Pillow==10.2.0.20240423 + types-Pygments==2.17.0.20240310 + types-pyOpenSSL==24.0.0.20240417 + types-setuptools==69.5.0.20240423 # 2.1.2 fixes a typing bug: https://github.com/scrapy/w3lib/pull/211 w3lib >= 2.1.2 commands =