diff --git a/scrapy/downloadermiddlewares/ajaxcrawl.py b/scrapy/downloadermiddlewares/ajaxcrawl.py index 04ae719de..0e757e4be 100644 --- a/scrapy/downloadermiddlewares/ajaxcrawl.py +++ b/scrapy/downloadermiddlewares/ajaxcrawl.py @@ -1,10 +1,20 @@ +from __future__ import annotations + import logging import re +from typing import TYPE_CHECKING, Union from w3lib import html +from scrapy import Request, Spider +from scrapy.crawler import Crawler from scrapy.exceptions import NotConfigured -from scrapy.http import HtmlResponse +from scrapy.http import HtmlResponse, Response +from scrapy.settings import BaseSettings + +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self logger = logging.getLogger(__name__) @@ -15,7 +25,7 @@ class AjaxCrawlMiddleware: For more info see https://developers.google.com/webmasters/ajax-crawling/docs/getting-started. """ - def __init__(self, settings): + def __init__(self, settings: BaseSettings): if not settings.getbool("AJAXCRAWL_ENABLED"): raise NotConfigured @@ -23,13 +33,15 @@ class AjaxCrawlMiddleware: # middleware parses first 4k. 4k turns out to be insufficient # for this middleware, and parsing 100k could be slow. # We use something in between (32K) by default. - self.lookup_bytes = settings.getint("AJAXCRAWL_MAXSIZE", 32768) + self.lookup_bytes: int = settings.getint("AJAXCRAWL_MAXSIZE", 32768) @classmethod - def from_crawler(cls, crawler): + def from_crawler(cls, crawler: Crawler) -> Self: return cls(crawler.settings) - def process_response(self, request, response, spider): + def process_response( + self, request: Request, response: Response, spider: Spider + ) -> Union[Request, Response]: if not isinstance(response, HtmlResponse) or response.status != 200: return response @@ -54,7 +66,7 @@ class AjaxCrawlMiddleware: ajax_crawl_request.meta["ajax_crawlable"] = True return ajax_crawl_request - def _has_ajax_crawlable_variant(self, response): + def _has_ajax_crawlable_variant(self, response: Response) -> bool: """ Return True if a page without hash fragment could be "AJAX crawlable" according to https://developers.google.com/webmasters/ajax-crawling/docs/getting-started. @@ -64,12 +76,12 @@ class AjaxCrawlMiddleware: # XXX: move it to w3lib? -_ajax_crawlable_re = re.compile( +_ajax_crawlable_re: re.Pattern[str] = re.compile( r'' ) -def _has_ajaxcrawlable_meta(text): +def _has_ajaxcrawlable_meta(text: str) -> bool: """ >>> _has_ajaxcrawlable_meta('') True diff --git a/scrapy/downloadermiddlewares/cookies.py b/scrapy/downloadermiddlewares/cookies.py index 6495157d7..85781efd6 100644 --- a/scrapy/downloadermiddlewares/cookies.py +++ b/scrapy/downloadermiddlewares/cookies.py @@ -1,21 +1,41 @@ +from __future__ import annotations + import logging from collections import defaultdict +from http.cookiejar import Cookie +from typing import ( + TYPE_CHECKING, + Any, + DefaultDict, + Dict, + Iterable, + Optional, + Sequence, + Union, +) from tldextract import TLDExtract +from scrapy import Request, Spider +from scrapy.crawler import Crawler from scrapy.exceptions import NotConfigured from scrapy.http import Response from scrapy.http.cookies import CookieJar from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.python import to_unicode +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self + + logger = logging.getLogger(__name__) _split_domain = TLDExtract(include_psl_private_domains=True) -def _is_public_domain(domain): +def _is_public_domain(domain: str) -> bool: parts = _split_domain(domain) return not parts.domain @@ -23,23 +43,27 @@ def _is_public_domain(domain): class CookiesMiddleware: """This middleware enables working with sites that need cookies""" - def __init__(self, debug=False): - self.jars = defaultdict(CookieJar) - self.debug = debug + def __init__(self, debug: bool = False): + self.jars: DefaultDict[Any, CookieJar] = defaultdict(CookieJar) + self.debug: bool = debug @classmethod - def from_crawler(cls, crawler): + def from_crawler(cls, crawler: Crawler) -> Self: if not crawler.settings.getbool("COOKIES_ENABLED"): raise NotConfigured return cls(crawler.settings.getbool("COOKIES_DEBUG")) - def _process_cookies(self, cookies, *, jar, request): + def _process_cookies( + self, cookies: Iterable[Cookie], *, jar: CookieJar, request: Request + ) -> None: for cookie in cookies: cookie_domain = cookie.domain if cookie_domain.startswith("."): cookie_domain = cookie_domain[1:] - request_domain = urlparse_cached(request).hostname.lower() + hostname = urlparse_cached(request).hostname + assert hostname is not None + request_domain = hostname.lower() if cookie_domain and _is_public_domain(cookie_domain): if cookie_domain != request_domain: @@ -48,9 +72,11 @@ class CookiesMiddleware: jar.set_cookie_if_ok(cookie, request) - def process_request(self, request, spider): + def process_request( + self, request: Request, spider: Spider + ) -> Union[Request, Response, None]: if request.meta.get("dont_merge_cookies", False): - return + return None cookiejarkey = request.meta.get("cookiejar") jar = self.jars[cookiejarkey] @@ -61,8 +87,11 @@ class CookiesMiddleware: request.headers.pop("Cookie", None) jar.add_cookie_header(request) self._debug_cookie(request, spider) + return None - def process_response(self, request, response, spider): + def process_response( + self, request: Request, response: Response, spider: Spider + ) -> Union[Request, Response]: if request.meta.get("dont_merge_cookies", False): return response @@ -76,7 +105,7 @@ class CookiesMiddleware: return response - def _debug_cookie(self, request, spider): + def _debug_cookie(self, request: Request, spider: Spider) -> None: if self.debug: cl = [ to_unicode(c, errors="replace") @@ -87,7 +116,7 @@ class CookiesMiddleware: msg = f"Sending cookies to: {request}\n{cookies}" logger.debug(msg, extra={"spider": spider}) - def _debug_set_cookie(self, response, spider): + def _debug_set_cookie(self, response: Response, spider: Spider) -> None: if self.debug: cl = [ to_unicode(c, errors="replace") @@ -98,7 +127,7 @@ class CookiesMiddleware: msg = f"Received cookies from: {response}\n{cookies}" logger.debug(msg, extra={"spider": spider}) - def _format_cookie(self, cookie, request): + def _format_cookie(self, cookie: Dict[str, Any], request: Request) -> Optional[str]: """ Given a dict consisting of cookie components, return its string representation. Decode from bytes if necessary. @@ -109,7 +138,7 @@ class CookiesMiddleware: if key in ("name", "value"): msg = f"Invalid cookie found in request {request}: {cookie} ('{key}' is missing)" logger.warning(msg) - return + return None continue if isinstance(cookie[key], (bool, float, int, str)): decoded[key] = str(cookie[key]) @@ -129,12 +158,15 @@ class CookiesMiddleware: cookie_str += f"; {key.capitalize()}={value}" return cookie_str - def _get_request_cookies(self, jar, request): + def _get_request_cookies( + self, jar: CookieJar, request: Request + ) -> Sequence[Cookie]: """ Extract cookies from the Request.cookies attribute """ if not request.cookies: return [] + cookies: Iterable[Dict[str, Any]] if isinstance(request.cookies, dict): cookies = ({"name": k, "value": v} for k, v in request.cookies.items()) else: diff --git a/scrapy/downloadermiddlewares/defaultheaders.py b/scrapy/downloadermiddlewares/defaultheaders.py index cdacc7368..8aec37cf1 100644 --- a/scrapy/downloadermiddlewares/defaultheaders.py +++ b/scrapy/downloadermiddlewares/defaultheaders.py @@ -3,19 +3,32 @@ DefaultHeaders downloader middleware See documentation in docs/topics/downloader-middleware.rst """ +from __future__ import annotations +from typing import TYPE_CHECKING, Iterable, Tuple, Union + +from scrapy import Request, Spider +from scrapy.crawler import Crawler +from scrapy.http import Response from scrapy.utils.python import without_none_values +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self + class DefaultHeadersMiddleware: - def __init__(self, headers): - self._headers = headers + def __init__(self, headers: Iterable[Tuple[str, str]]): + self._headers: Iterable[Tuple[str, str]] = headers @classmethod - def from_crawler(cls, crawler): + def from_crawler(cls, crawler: Crawler) -> Self: headers = without_none_values(crawler.settings["DEFAULT_REQUEST_HEADERS"]) return cls(headers.items()) - def process_request(self, request, spider): + def process_request( + self, request: Request, spider: Spider + ) -> Union[Request, Response, None]: for k, v in self._headers: request.headers.setdefault(k, v) + return None diff --git a/scrapy/downloadermiddlewares/downloadtimeout.py b/scrapy/downloadermiddlewares/downloadtimeout.py index a926ecf56..1c904c05b 100644 --- a/scrapy/downloadermiddlewares/downloadtimeout.py +++ b/scrapy/downloadermiddlewares/downloadtimeout.py @@ -3,23 +3,35 @@ Download timeout middleware See documentation in docs/topics/downloader-middleware.rst """ +from __future__ import annotations -from scrapy import signals +from typing import TYPE_CHECKING, Union + +from scrapy import Request, Spider, signals +from scrapy.crawler import Crawler +from scrapy.http import Response + +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self class DownloadTimeoutMiddleware: - def __init__(self, timeout=180): - self._timeout = timeout + def __init__(self, timeout: float = 180): + self._timeout: float = timeout @classmethod - def from_crawler(cls, crawler): + def from_crawler(cls, crawler: Crawler) -> Self: o = cls(crawler.settings.getfloat("DOWNLOAD_TIMEOUT")) crawler.signals.connect(o.spider_opened, signal=signals.spider_opened) return o - def spider_opened(self, spider): + def spider_opened(self, spider: Spider) -> None: self._timeout = getattr(spider, "download_timeout", self._timeout) - def process_request(self, request, spider): + def process_request( + self, request: Request, spider: Spider + ) -> Union[Request, Response, None]: if self._timeout: request.meta.setdefault("download_timeout", self._timeout) + return None diff --git a/scrapy/downloadermiddlewares/httpauth.py b/scrapy/downloadermiddlewares/httpauth.py index 5228db786..63490a37a 100644 --- a/scrapy/downloadermiddlewares/httpauth.py +++ b/scrapy/downloadermiddlewares/httpauth.py @@ -4,31 +4,44 @@ HTTP basic auth downloader middleware See documentation in docs/topics/downloader-middleware.rst """ +from __future__ import annotations + +from typing import TYPE_CHECKING, Union + from w3lib.http import basic_auth_header -from scrapy import signals +from scrapy import Request, Spider, signals +from scrapy.crawler import Crawler +from scrapy.http import Response from scrapy.utils.url import url_is_from_any_domain +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self + class HttpAuthMiddleware: """Set Basic HTTP Authorization header (http_user and http_pass spider class attributes)""" @classmethod - def from_crawler(cls, crawler): + def from_crawler(cls, crawler: Crawler) -> Self: o = cls() crawler.signals.connect(o.spider_opened, signal=signals.spider_opened) return o - def spider_opened(self, spider): + def spider_opened(self, spider: Spider) -> None: usr = getattr(spider, "http_user", "") pwd = getattr(spider, "http_pass", "") if usr or pwd: self.auth = basic_auth_header(usr, pwd) - self.domain = spider.http_auth_domain + self.domain = spider.http_auth_domain # type: ignore[attr-defined] - def process_request(self, request, spider): + def process_request( + self, request: Request, spider: Spider + ) -> Union[Request, Response, None]: auth = getattr(self, "auth", None) if auth and b"Authorization" not in request.headers: if not self.domain or url_is_from_any_domain(request.url, [self.domain]): request.headers[b"Authorization"] = auth + return None diff --git a/scrapy/downloadermiddlewares/httpcache.py b/scrapy/downloadermiddlewares/httpcache.py index a521cde7a..971473403 100644 --- a/scrapy/downloadermiddlewares/httpcache.py +++ b/scrapy/downloadermiddlewares/httpcache.py @@ -1,5 +1,7 @@ +from __future__ import annotations + from email.utils import formatdate -from typing import Optional, Type, TypeVar +from typing import TYPE_CHECKING, Optional, Union from twisted.internet import defer from twisted.internet.error import ( @@ -23,7 +25,9 @@ from scrapy.spiders import Spider from scrapy.statscollectors import StatsCollector from scrapy.utils.misc import load_object -HttpCacheMiddlewareTV = TypeVar("HttpCacheMiddlewareTV", bound="HttpCacheMiddleware") +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self class HttpCacheMiddleware: @@ -49,9 +53,7 @@ class HttpCacheMiddleware: self.stats = stats @classmethod - def from_crawler( - cls: Type[HttpCacheMiddlewareTV], crawler: Crawler - ) -> HttpCacheMiddlewareTV: + def from_crawler(cls, crawler: Crawler) -> Self: assert crawler.stats o = cls(crawler.settings, crawler.stats) crawler.signals.connect(o.spider_opened, signal=signals.spider_opened) @@ -64,7 +66,9 @@ class HttpCacheMiddleware: def spider_closed(self, spider: Spider) -> None: self.storage.close_spider(spider) - def process_request(self, request: Request, spider: Spider) -> Optional[Response]: + def process_request( + self, request: Request, spider: Spider + ) -> Union[Request, Response, None]: if request.meta.get("dont_cache", False): return None @@ -74,7 +78,9 @@ class HttpCacheMiddleware: return None # Look for cached response and check if expired - cachedresponse = self.storage.retrieve_response(spider, request) + cachedresponse: Optional[Response] = self.storage.retrieve_response( + spider, request + ) if cachedresponse is None: self.stats.inc_value("httpcache/miss", spider=spider) if self.ignore_missing: @@ -96,7 +102,7 @@ class HttpCacheMiddleware: def process_response( self, request: Request, response: Response, spider: Spider - ) -> Response: + ) -> Union[Request, Response]: if request.meta.get("dont_cache", False): return response @@ -111,7 +117,7 @@ class HttpCacheMiddleware: response.headers["Date"] = formatdate(usegmt=True) # Do not validate first-hand responses - cachedresponse = request.meta.pop("cached_response", None) + cachedresponse: Optional[Response] = request.meta.pop("cached_response", None) if cachedresponse is None: self.stats.inc_value("httpcache/firsthand", spider=spider) self._cache_response(spider, response, request, cachedresponse) @@ -127,8 +133,8 @@ class HttpCacheMiddleware: def process_exception( self, request: Request, exception: Exception, spider: Spider - ) -> Optional[Response]: - cachedresponse = request.meta.pop("cached_response", None) + ) -> Union[Request, Response, None]: + cachedresponse: Optional[Response] = request.meta.pop("cached_response", None) if cachedresponse is not None and isinstance( exception, self.DOWNLOAD_EXCEPTIONS ): diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py index 7b1d3f829..56a58a750 100644 --- a/scrapy/downloadermiddlewares/httpcompression.py +++ b/scrapy/downloadermiddlewares/httpcompression.py @@ -1,12 +1,22 @@ +from __future__ import annotations + import io import zlib +from typing import TYPE_CHECKING, List, Optional, Union +from scrapy import Request, Spider +from scrapy.crawler import Crawler from scrapy.exceptions import NotConfigured from scrapy.http import Response, TextResponse from scrapy.responsetypes import responsetypes +from scrapy.statscollectors import StatsCollector from scrapy.utils.gz import gunzip -ACCEPTED_ENCODINGS = [b"gzip", b"deflate"] +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self + +ACCEPTED_ENCODINGS: List[bytes] = [b"gzip", b"deflate"] try: import brotli @@ -27,19 +37,24 @@ class HttpCompressionMiddleware: """This middleware allows compressed (gzip, deflate) traffic to be sent/received from web sites""" - def __init__(self, stats=None): + def __init__(self, stats: Optional[StatsCollector] = None): self.stats = stats @classmethod - def from_crawler(cls, crawler): + def from_crawler(cls, crawler: Crawler) -> Self: if not crawler.settings.getbool("COMPRESSION_ENABLED"): raise NotConfigured return cls(stats=crawler.stats) - def process_request(self, request, spider): + def process_request( + self, request: Request, spider: Spider + ) -> Union[Request, Response, None]: request.headers.setdefault("Accept-Encoding", b", ".join(ACCEPTED_ENCODINGS)) + return None - def process_response(self, request, response, spider): + def process_response( + self, request: Request, response: Response, spider: Spider + ) -> Union[Request, Response]: if request.method == "HEAD": return response if isinstance(response, Response): @@ -70,7 +85,7 @@ class HttpCompressionMiddleware: return response - def _decode(self, body, encoding): + def _decode(self, body: bytes, encoding: bytes) -> bytes: if encoding == b"gzip" or encoding == b"x-gzip": body = gunzip(body) diff --git a/scrapy/downloadermiddlewares/httpproxy.py b/scrapy/downloadermiddlewares/httpproxy.py index f74d84b69..335896ac1 100644 --- a/scrapy/downloadermiddlewares/httpproxy.py +++ b/scrapy/downloadermiddlewares/httpproxy.py @@ -1,16 +1,30 @@ -import base64 -from urllib.parse import unquote, urlunparse -from urllib.request import _parse_proxy, getproxies, proxy_bypass +from __future__ import annotations +import base64 +from typing import TYPE_CHECKING, Dict, Optional, Tuple, Union +from urllib.parse import unquote, urlunparse +from urllib.request import ( # type: ignore[attr-defined] + _parse_proxy, + getproxies, + proxy_bypass, +) + +from scrapy import Request, Spider +from scrapy.crawler import Crawler from scrapy.exceptions import NotConfigured +from scrapy.http import Response from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.python import to_bytes +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self + class HttpProxyMiddleware: - def __init__(self, auth_encoding="latin-1"): - self.auth_encoding = auth_encoding - self.proxies = {} + def __init__(self, auth_encoding: Optional[str] = "latin-1"): + self.auth_encoding: Optional[str] = auth_encoding + self.proxies: Dict[str, Tuple[Optional[bytes], str]] = {} for type_, url in getproxies().items(): try: self.proxies[type_] = self._get_proxy(url, type_) @@ -20,19 +34,19 @@ class HttpProxyMiddleware: continue @classmethod - def from_crawler(cls, crawler): + def from_crawler(cls, crawler: Crawler) -> Self: if not crawler.settings.getbool("HTTPPROXY_ENABLED"): raise NotConfigured - auth_encoding = crawler.settings.get("HTTPPROXY_AUTH_ENCODING") + auth_encoding: Optional[str] = crawler.settings.get("HTTPPROXY_AUTH_ENCODING") return cls(auth_encoding) - def _basic_auth_header(self, username, password): + def _basic_auth_header(self, username: str, password: str) -> bytes: user_pass = to_bytes( f"{unquote(username)}:{unquote(password)}", encoding=self.auth_encoding ) return base64.b64encode(user_pass) - def _get_proxy(self, url, orig_type): + def _get_proxy(self, url: str, orig_type: str) -> Tuple[Optional[bytes], str]: proxy_type, user, password, hostport = _parse_proxy(url) proxy_url = urlunparse((proxy_type or orig_type, hostport, "", "", "", "")) @@ -43,7 +57,9 @@ class HttpProxyMiddleware: return creds, proxy_url - def process_request(self, request, spider): + def process_request( + self, request: Request, spider: Spider + ) -> Union[Request, Response, None]: creds, proxy_url = None, None if "proxy" in request.meta: if request.meta["proxy"] is not None: @@ -54,13 +70,16 @@ class HttpProxyMiddleware: if ( # 'no_proxy' is only supported by http schemes scheme not in ("http", "https") - or not proxy_bypass(parsed.hostname) + or (parsed.hostname and not proxy_bypass(parsed.hostname)) ) and scheme in self.proxies: creds, proxy_url = self.proxies[scheme] self._set_proxy_and_creds(request, proxy_url, creds) + return None - def _set_proxy_and_creds(self, request, proxy_url, creds): + def _set_proxy_and_creds( + self, request: Request, proxy_url: Optional[str], creds: Optional[bytes] + ) -> None: if proxy_url: request.meta["proxy"] = proxy_url elif request.meta.get("proxy") is not None: diff --git a/scrapy/downloadermiddlewares/redirect.py b/scrapy/downloadermiddlewares/redirect.py index 65f1d2224..814b1a561 100644 --- a/scrapy/downloadermiddlewares/redirect.py +++ b/scrapy/downloadermiddlewares/redirect.py @@ -1,17 +1,29 @@ +from __future__ import annotations + import logging +from typing import TYPE_CHECKING, Any, List, Union, cast from urllib.parse import urljoin, urlparse from w3lib.url import safe_url_string +from scrapy import Request, Spider +from scrapy.crawler import Crawler from scrapy.exceptions import IgnoreRequest, NotConfigured -from scrapy.http import HtmlResponse +from scrapy.http import HtmlResponse, Response +from scrapy.settings import BaseSettings from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.response import get_meta_refresh +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self + logger = logging.getLogger(__name__) -def _build_redirect_request(source_request, *, url, **kwargs): +def _build_redirect_request( + source_request: Request, *, url: str, **kwargs: Any +) -> Request: redirect_request = source_request.replace( url=url, **kwargs, @@ -26,20 +38,22 @@ def _build_redirect_request(source_request, *, url, **kwargs): class BaseRedirectMiddleware: - enabled_setting = "REDIRECT_ENABLED" + enabled_setting: str = "REDIRECT_ENABLED" - def __init__(self, settings): + def __init__(self, settings: BaseSettings): if not settings.getbool(self.enabled_setting): raise NotConfigured - self.max_redirect_times = settings.getint("REDIRECT_MAX_TIMES") - self.priority_adjust = settings.getint("REDIRECT_PRIORITY_ADJUST") + self.max_redirect_times: int = settings.getint("REDIRECT_MAX_TIMES") + self.priority_adjust: int = settings.getint("REDIRECT_PRIORITY_ADJUST") @classmethod - def from_crawler(cls, crawler): + def from_crawler(cls, crawler: Crawler) -> Self: return cls(crawler.settings) - def _redirect(self, redirected, request, spider, reason): + def _redirect( + self, redirected: Request, request: Request, spider: Spider, reason: Any + ) -> Request: ttl = request.meta.setdefault("redirect_ttl", self.max_redirect_times) redirects = request.meta.get("redirect_times", 0) + 1 @@ -67,7 +81,9 @@ class BaseRedirectMiddleware: ) raise IgnoreRequest("max redirections reached") - def _redirect_request_using_get(self, request, redirect_url): + def _redirect_request_using_get( + self, request: Request, redirect_url: str + ) -> Request: redirect_request = _build_redirect_request( request, url=redirect_url, @@ -85,7 +101,9 @@ class RedirectMiddleware(BaseRedirectMiddleware): and meta-refresh html tag. """ - def process_response(self, request, response, spider): + def process_response( + self, request: Request, response: Response, spider: Spider + ) -> Union[Request, Response]: if ( request.meta.get("dont_redirect", False) or response.status in getattr(spider, "handle_httpstatus_list", []) @@ -98,6 +116,7 @@ class RedirectMiddleware(BaseRedirectMiddleware): if "Location" not in response.headers or response.status not in allowed_status: return response + assert response.headers["Location"] is not None location = safe_url_string(response.headers["Location"]) if response.headers["Location"].startswith(b"//"): request_scheme = urlparse(request.url).scheme @@ -116,12 +135,14 @@ class RedirectMiddleware(BaseRedirectMiddleware): class MetaRefreshMiddleware(BaseRedirectMiddleware): enabled_setting = "METAREFRESH_ENABLED" - def __init__(self, settings): + def __init__(self, settings: BaseSettings): super().__init__(settings) - self._ignore_tags = settings.getlist("METAREFRESH_IGNORE_TAGS") - self._maxdelay = settings.getint("METAREFRESH_MAXDELAY") + self._ignore_tags: List[str] = settings.getlist("METAREFRESH_IGNORE_TAGS") + self._maxdelay: int = settings.getint("METAREFRESH_MAXDELAY") - def process_response(self, request, response, spider): + def process_response( + self, request: Request, response: Response, spider: Spider + ) -> Union[Request, Response]: if ( request.meta.get("dont_redirect", False) or request.method == "HEAD" @@ -130,7 +151,7 @@ class MetaRefreshMiddleware(BaseRedirectMiddleware): return response interval, url = get_meta_refresh(response, ignore_tags=self._ignore_tags) - if url and interval < self._maxdelay: + if url and cast(float, interval) < self._maxdelay: redirected = self._redirect_request_using_get(request, url) return self._redirect(redirected, request, spider, "meta refresh") diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.py index 380623cea..3c494de78 100644 --- a/scrapy/downloadermiddlewares/retry.py +++ b/scrapy/downloadermiddlewares/retry.py @@ -9,22 +9,30 @@ RETRY_HTTP_CODES - which HTTP response codes to retry Failed pages are collected on the scraping process and rescheduled at the end, once the spider has finished crawling all regular (non failed) pages. """ +from __future__ import annotations + import warnings from logging import Logger, getLogger -from typing import Optional, Type, Union +from typing import TYPE_CHECKING, Any, Optional, Tuple, Type, Union +from scrapy.crawler import Crawler from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning +from scrapy.http import Response from scrapy.http.request import Request -from scrapy.settings import Settings +from scrapy.settings import BaseSettings, Settings from scrapy.spiders import Spider from scrapy.utils.misc import load_object from scrapy.utils.python import global_object_name from scrapy.utils.response import response_status_message +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self + retry_logger = getLogger(__name__) -def backwards_compatibility_getattr(self, name): +def backwards_compatibility_getattr(self: Any, name: str) -> Tuple[Any, ...]: if name == "EXCEPTIONS_TO_RETRY": warnings.warn( "Attribute RetryMiddleware.EXCEPTIONS_TO_RETRY is deprecated. " @@ -54,7 +62,7 @@ def get_retry_request( priority_adjust: Optional[int] = None, logger: Logger = retry_logger, stats_base_key: str = "retry", -): +) -> Optional[Request]: """ Returns a new :class:`~scrapy.Request` object to retry the specified request, or ``None`` if retries of the specified request have been @@ -134,7 +142,7 @@ def get_retry_request( class RetryMiddleware(metaclass=BackwardsCompatibilityMetaclass): - def __init__(self, settings): + def __init__(self, settings: BaseSettings): if not settings.getbool("RETRY_ENABLED"): raise NotConfigured self.max_retry_times = settings.getint("RETRY_TIMES") @@ -153,10 +161,12 @@ class RetryMiddleware(metaclass=BackwardsCompatibilityMetaclass): ) @classmethod - def from_crawler(cls, crawler): + def from_crawler(cls, crawler: Crawler) -> Self: return cls(crawler.settings) - def process_response(self, request, response, spider): + def process_response( + self, request: Request, response: Response, spider: Spider + ) -> Union[Request, Response]: if request.meta.get("dont_retry", False): return response if response.status in self.retry_http_codes: @@ -164,13 +174,21 @@ class RetryMiddleware(metaclass=BackwardsCompatibilityMetaclass): return self._retry(request, reason, spider) or response return response - def process_exception(self, request, exception, spider): + def process_exception( + self, request: Request, exception: Exception, spider: Spider + ) -> Union[Request, Response, None]: if isinstance(exception, self.exceptions_to_retry) and not request.meta.get( "dont_retry", False ): return self._retry(request, exception, spider) + return None - def _retry(self, request, reason, spider): + def _retry( + self, + request: Request, + reason: Union[str, Exception, Type[Exception]], + spider: Spider, + ) -> Optional[Request]: max_retry_times = request.meta.get("max_retry_times", self.max_retry_times) priority_adjust = request.meta.get("priority_adjust", self.priority_adjust) return get_retry_request( diff --git a/scrapy/downloadermiddlewares/robotstxt.py b/scrapy/downloadermiddlewares/robotstxt.py index 6cab27c5a..6a0ecb7bf 100644 --- a/scrapy/downloadermiddlewares/robotstxt.py +++ b/scrapy/downloadermiddlewares/robotstxt.py @@ -4,65 +4,87 @@ enable this middleware and enable the ROBOTSTXT_OBEY setting. """ +from __future__ import annotations + import logging +from typing import TYPE_CHECKING, Any, Dict, Optional, Union from twisted.internet.defer import Deferred, maybeDeferred +from twisted.python.failure import Failure +from scrapy import Spider +from scrapy.crawler import Crawler from scrapy.exceptions import IgnoreRequest, NotConfigured -from scrapy.http import Request +from scrapy.http import Request, Response from scrapy.http.request import NO_CALLBACK +from scrapy.robotstxt import RobotParser from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.log import failure_to_exc_info from scrapy.utils.misc import load_object +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self + + logger = logging.getLogger(__name__) class RobotsTxtMiddleware: - DOWNLOAD_PRIORITY = 1000 + DOWNLOAD_PRIORITY: int = 1000 - def __init__(self, crawler): + def __init__(self, crawler: Crawler): if not crawler.settings.getbool("ROBOTSTXT_OBEY"): raise NotConfigured - self._default_useragent = crawler.settings.get("USER_AGENT", "Scrapy") - self._robotstxt_useragent = crawler.settings.get("ROBOTSTXT_USER_AGENT", None) - self.crawler = crawler - self._parsers = {} - self._parserimpl = load_object(crawler.settings.get("ROBOTSTXT_PARSER")) + self._default_useragent: str = crawler.settings.get("USER_AGENT", "Scrapy") + self._robotstxt_useragent: Optional[str] = crawler.settings.get( + "ROBOTSTXT_USER_AGENT", None + ) + self.crawler: Crawler = crawler + self._parsers: Dict[str, Union[RobotParser, Deferred, None]] = {} + self._parserimpl: RobotParser = load_object( + crawler.settings.get("ROBOTSTXT_PARSER") + ) # check if parser dependencies are met, this should throw an error otherwise. self._parserimpl.from_crawler(self.crawler, b"") @classmethod - def from_crawler(cls, crawler): + def from_crawler(cls, crawler: Crawler) -> Self: return cls(crawler) - def process_request(self, request, spider): + def process_request(self, request: Request, spider: Spider) -> Optional[Deferred]: if request.meta.get("dont_obey_robotstxt"): - return + return None if request.url.startswith("data:") or request.url.startswith("file:"): - return - d = maybeDeferred(self.robot_parser, request, spider) + return None + d: Deferred = maybeDeferred(self.robot_parser, request, spider) d.addCallback(self.process_request_2, request, spider) return d - def process_request_2(self, rp, request, spider): + def process_request_2( + self, rp: Optional[RobotParser], request: Request, spider: Spider + ) -> None: if rp is None: return - useragent = self._robotstxt_useragent + useragent: Union[str, bytes, None] = self._robotstxt_useragent if not useragent: useragent = request.headers.get(b"User-Agent", self._default_useragent) + assert useragent is not None if not rp.allowed(request.url, useragent): logger.debug( "Forbidden by robots.txt: %(request)s", {"request": request}, extra={"spider": spider}, ) + assert self.crawler.stats self.crawler.stats.inc_value("robotstxt/forbidden") raise IgnoreRequest("Forbidden by robots.txt") - def robot_parser(self, request, spider): + def robot_parser( + self, request: Request, spider: Spider + ) -> Union[RobotParser, Deferred, None]: url = urlparse_cached(request) netloc = url.netloc @@ -75,24 +97,27 @@ class RobotsTxtMiddleware: meta={"dont_obey_robotstxt": True}, callback=NO_CALLBACK, ) + assert self.crawler.engine + assert self.crawler.stats dfd = self.crawler.engine.download(robotsreq) dfd.addCallback(self._parse_robots, netloc, spider) dfd.addErrback(self._logerror, robotsreq, spider) dfd.addErrback(self._robots_error, netloc) self.crawler.stats.inc_value("robotstxt/request_count") - if isinstance(self._parsers[netloc], Deferred): - d = Deferred() + parser = self._parsers[netloc] + if isinstance(parser, Deferred): + d: Deferred = Deferred() - def cb(result): + def cb(result: Any) -> Any: d.callback(result) return result - self._parsers[netloc].addCallback(cb) + parser.addCallback(cb) return d - return self._parsers[netloc] + return parser - def _logerror(self, failure, request, spider): + def _logerror(self, failure: Failure, request: Request, spider: Spider) -> Failure: if failure.type is not IgnoreRequest: logger.error( "Error downloading %(request)s: %(f_exception)s", @@ -102,20 +127,24 @@ class RobotsTxtMiddleware: ) return failure - def _parse_robots(self, response, netloc, spider): + def _parse_robots(self, response: Response, netloc: str, spider: Spider) -> None: + assert self.crawler.stats self.crawler.stats.inc_value("robotstxt/response_count") self.crawler.stats.inc_value( f"robotstxt/response_status_count/{response.status}" ) rp = self._parserimpl.from_crawler(self.crawler, response.body) rp_dfd = self._parsers[netloc] + assert isinstance(rp_dfd, Deferred) self._parsers[netloc] = rp rp_dfd.callback(rp) - def _robots_error(self, failure, netloc): + def _robots_error(self, failure: Failure, netloc: str) -> None: if failure.type is not IgnoreRequest: key = f"robotstxt/exception_count/{failure.type}" + assert self.crawler.stats self.crawler.stats.inc_value(key) rp_dfd = self._parsers[netloc] + assert isinstance(rp_dfd, Deferred) self._parsers[netloc] = None rp_dfd.callback(None) diff --git a/scrapy/downloadermiddlewares/stats.py b/scrapy/downloadermiddlewares/stats.py index a0f62e262..df30e8ca4 100644 --- a/scrapy/downloadermiddlewares/stats.py +++ b/scrapy/downloadermiddlewares/stats.py @@ -1,11 +1,23 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING, Dict, Union + from twisted.web import http +from scrapy import Request, Spider +from scrapy.crawler import Crawler from scrapy.exceptions import NotConfigured +from scrapy.http import Response +from scrapy.statscollectors import StatsCollector from scrapy.utils.python import global_object_name, to_bytes from scrapy.utils.request import request_httprepr +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self -def get_header_size(headers): + +def get_header_size(headers: Dict[str, Union[list, tuple]]) -> int: size = 0 for key, value in headers.items(): if isinstance(value, (list, tuple)): @@ -14,30 +26,36 @@ def get_header_size(headers): return size + len(b"\r\n") * (len(headers.keys()) - 1) -def get_status_size(response_status): +def get_status_size(response_status: int) -> int: return len(to_bytes(http.RESPONSES.get(response_status, b""))) + 15 # resp.status + b"\r\n" + b"HTTP/1.1 <100-599> " class DownloaderStats: - def __init__(self, stats): - self.stats = stats + def __init__(self, stats: StatsCollector): + self.stats: StatsCollector = stats @classmethod - def from_crawler(cls, crawler): + def from_crawler(cls, crawler: Crawler) -> Self: if not crawler.settings.getbool("DOWNLOADER_STATS"): raise NotConfigured + assert crawler.stats return cls(crawler.stats) - def process_request(self, request, spider): + def process_request( + self, request: Request, spider: Spider + ) -> Union[Request, Response, None]: self.stats.inc_value("downloader/request_count", spider=spider) self.stats.inc_value( f"downloader/request_method_count/{request.method}", spider=spider ) reqlen = len(request_httprepr(request)) self.stats.inc_value("downloader/request_bytes", reqlen, spider=spider) + return None - def process_response(self, request, response, spider): + def process_response( + self, request: Request, response: Response, spider: Spider + ) -> Union[Request, Response]: self.stats.inc_value("downloader/response_count", spider=spider) self.stats.inc_value( f"downloader/response_status_count/{response.status}", spider=spider @@ -52,9 +70,12 @@ class DownloaderStats: self.stats.inc_value("downloader/response_bytes", reslen, spider=spider) return response - def process_exception(self, request, exception, spider): + def process_exception( + self, request: Request, exception: Exception, spider: Spider + ) -> Union[Request, Response, None]: ex_class = global_object_name(exception.__class__) self.stats.inc_value("downloader/exception_count", spider=spider) self.stats.inc_value( f"downloader/exception_type_count/{ex_class}", spider=spider ) + return None diff --git a/scrapy/downloadermiddlewares/useragent.py b/scrapy/downloadermiddlewares/useragent.py index 856a275ab..92f1ec897 100644 --- a/scrapy/downloadermiddlewares/useragent.py +++ b/scrapy/downloadermiddlewares/useragent.py @@ -1,23 +1,36 @@ """Set User-Agent header per spider or use a default value from settings""" -from scrapy import signals +from __future__ import annotations + +from typing import TYPE_CHECKING, Union + +from scrapy import Request, Spider, signals +from scrapy.crawler import Crawler +from scrapy.http import Response + +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self class UserAgentMiddleware: """This middleware allows spiders to override the user_agent""" - def __init__(self, user_agent="Scrapy"): + def __init__(self, user_agent: str = "Scrapy"): self.user_agent = user_agent @classmethod - def from_crawler(cls, crawler): + def from_crawler(cls, crawler: Crawler) -> Self: o = cls(crawler.settings["USER_AGENT"]) crawler.signals.connect(o.spider_opened, signal=signals.spider_opened) return o - def spider_opened(self, spider): + def spider_opened(self, spider: Spider) -> None: self.user_agent = getattr(spider, "user_agent", self.user_agent) - def process_request(self, request, spider): + def process_request( + self, request: Request, spider: Spider + ) -> Union[Request, Response, None]: if self.user_agent: request.headers.setdefault(b"User-Agent", self.user_agent) + return None diff --git a/scrapy/http/cookies.py b/scrapy/http/cookies.py index a5329ad51..2595f328e 100644 --- a/scrapy/http/cookies.py +++ b/scrapy/http/cookies.py @@ -1,8 +1,12 @@ import re import time +from http.cookiejar import Cookie from http.cookiejar import CookieJar as _CookieJar from http.cookiejar import DefaultCookiePolicy +from typing import Sequence +from scrapy import Request +from scrapy.http import Response from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.python import to_unicode @@ -24,7 +28,7 @@ class CookieJar: wrsp = WrappedResponse(response) return self.jar.extract_cookies(wrsp, wreq) - def add_cookie_header(self, request): + def add_cookie_header(self, request: Request) -> None: wreq = WrappedRequest(request) self.policy._now = self.jar._now = int(time.time()) @@ -75,7 +79,7 @@ class CookieJar: def set_policy(self, pol): return self.jar.set_policy(pol) - def make_cookies(self, response, request): + def make_cookies(self, response: Response, request: Request) -> Sequence[Cookie]: wreq = WrappedRequest(request) wrsp = WrappedResponse(response) return self.jar.make_cookies(wrsp, wreq) @@ -83,7 +87,7 @@ class CookieJar: def set_cookie(self, cookie): self.jar.set_cookie(cookie) - def set_cookie_if_ok(self, cookie, request): + def set_cookie_if_ok(self, cookie: Cookie, request: Request) -> None: self.jar.set_cookie_if_ok(cookie, WrappedRequest(request)) diff --git a/scrapy/robotstxt.py b/scrapy/robotstxt.py index 5c5ac4e41..6ea2bfd97 100644 --- a/scrapy/robotstxt.py +++ b/scrapy/robotstxt.py @@ -1,11 +1,20 @@ +from __future__ import annotations + import logging import sys from abc import ABCMeta, abstractmethod +from typing import TYPE_CHECKING, Union from warnings import warn from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.utils.python import to_unicode +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self + + from scrapy.crawler import Crawler + logger = logging.getLogger(__name__) @@ -31,7 +40,7 @@ def decode_robotstxt(robotstxt_body, spider, to_native_str_type=False): class RobotParser(metaclass=ABCMeta): @classmethod @abstractmethod - def from_crawler(cls, crawler, robotstxt_body): + def from_crawler(cls, crawler: Crawler, robotstxt_body: bytes) -> Self: """Parse the content of a robots.txt_ file as bytes. This must be a class method. It must return a new instance of the parser backend. @@ -44,14 +53,14 @@ class RobotParser(metaclass=ABCMeta): pass @abstractmethod - def allowed(self, url, user_agent): + def allowed(self, url: Union[str, bytes], user_agent: Union[str, bytes]) -> bool: """Return ``True`` if ``user_agent`` is allowed to crawl ``url``, otherwise return ``False``. :param url: Absolute URL - :type url: str + :type url: str or bytes :param user_agent: User agent - :type user_agent: str + :type user_agent: str or bytes """ pass diff --git a/setup.cfg b/setup.cfg index db79c5821..d23549f10 100644 --- a/setup.cfg +++ b/setup.cfg @@ -16,9 +16,6 @@ follow_imports = skip # FIXME: remove the following sections once the issues are solved -[mypy-scrapy.downloadermiddlewares.httpproxy] -ignore_errors = True - [mypy-scrapy.interfaces] ignore_errors = True