RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Full typing for scrapy/downloadermiddlewares (#6129)

This commit is contained in:
Andrey Rakhmatullin 2023-11-06 11:52:14 +04:00 committed by GitHub
parent eafe828484
commit 7c27c22a98
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
16 changed files with 373 additions and 139 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
scrapy/downloadermiddlewares/ajaxcrawl.py
View File

@ -1,10 +1,20 @@
from __future__ import annotations
import logging
import re
from typing import TYPE_CHECKING, Union
from w3lib import html
from scrapy import Request, Spider
from scrapy.crawler import Crawler
from scrapy.exceptions import NotConfigured
from scrapy.http import HtmlResponse
from scrapy.http import HtmlResponse, Response
from scrapy.settings import BaseSettings
if TYPE_CHECKING:
# typing.Self requires Python 3.11
from typing_extensions import Self
logger = logging.getLogger(__name__)
@ -15,7 +25,7 @@ class AjaxCrawlMiddleware:
For more info see https://developers.google.com/webmasters/ajax-crawling/docs/getting-started.
"""
def __init__(self, settings):
def __init__(self, settings: BaseSettings):
if not settings.getbool("AJAXCRAWL_ENABLED"):
raise NotConfigured
@ -23,13 +33,15 @@ class AjaxCrawlMiddleware:
# middleware parses first 4k. 4k turns out to be insufficient
# for this middleware, and parsing 100k could be slow.
# We use something in between (32K) by default.
self.lookup_bytes = settings.getint("AJAXCRAWL_MAXSIZE", 32768)
self.lookup_bytes: int = settings.getint("AJAXCRAWL_MAXSIZE", 32768)
@classmethod
def from_crawler(cls, crawler):
def from_crawler(cls, crawler: Crawler) -> Self:
return cls(crawler.settings)
def process_response(self, request, response, spider):
def process_response(
self, request: Request, response: Response, spider: Spider
) -> Union[Request, Response]:
if not isinstance(response, HtmlResponse) or response.status != 200:
return response
@ -54,7 +66,7 @@ class AjaxCrawlMiddleware:
ajax_crawl_request.meta["ajax_crawlable"] = True
return ajax_crawl_request
def _has_ajax_crawlable_variant(self, response):
def _has_ajax_crawlable_variant(self, response: Response) -> bool:
"""
Return True if a page without hash fragment could be "AJAX crawlable"
according to https://developers.google.com/webmasters/ajax-crawling/docs/getting-started.
@ -64,12 +76,12 @@ class AjaxCrawlMiddleware:
# XXX: move it to w3lib?
_ajax_crawlable_re = re.compile(
_ajax_crawlable_re: re.Pattern[str] = re.compile(
r'<meta\s+name=["\']fragment["\']\s+content=["\']!["\']/?>'
)
def _has_ajaxcrawlable_meta(text):
def _has_ajaxcrawlable_meta(text: str) -> bool:
"""
>>> _has_ajaxcrawlable_meta('<html><head><meta name="fragment" content="!"/></head><body></body></html>')
True

62
scrapy/downloadermiddlewares/cookies.py
View File

@ -1,21 +1,41 @@
from __future__ import annotations
import logging
from collections import defaultdict
from http.cookiejar import Cookie
from typing import (
TYPE_CHECKING,
Any,
DefaultDict,
Dict,
Iterable,
Optional,
Sequence,
Union,
)
from tldextract import TLDExtract
from scrapy import Request, Spider
from scrapy.crawler import Crawler
from scrapy.exceptions import NotConfigured
from scrapy.http import Response
from scrapy.http.cookies import CookieJar
from scrapy.utils.httpobj import urlparse_cached
from scrapy.utils.python import to_unicode
if TYPE_CHECKING:
# typing.Self requires Python 3.11
from typing_extensions import Self
logger = logging.getLogger(__name__)
_split_domain = TLDExtract(include_psl_private_domains=True)
def _is_public_domain(domain):
def _is_public_domain(domain: str) -> bool:
parts = _split_domain(domain)
return not parts.domain
@ -23,23 +43,27 @@ def _is_public_domain(domain):
class CookiesMiddleware:
"""This middleware enables working with sites that need cookies"""
def __init__(self, debug=False):
self.jars = defaultdict(CookieJar)
self.debug = debug
def __init__(self, debug: bool = False):
self.jars: DefaultDict[Any, CookieJar] = defaultdict(CookieJar)
self.debug: bool = debug
@classmethod
def from_crawler(cls, crawler):
def from_crawler(cls, crawler: Crawler) -> Self:
if not crawler.settings.getbool("COOKIES_ENABLED"):
raise NotConfigured
return cls(crawler.settings.getbool("COOKIES_DEBUG"))
def _process_cookies(self, cookies, *, jar, request):
def _process_cookies(
self, cookies: Iterable[Cookie], *, jar: CookieJar, request: Request
) -> None:
for cookie in cookies:
cookie_domain = cookie.domain
if cookie_domain.startswith("."):
cookie_domain = cookie_domain[1:]
request_domain = urlparse_cached(request).hostname.lower()
hostname = urlparse_cached(request).hostname
assert hostname is not None
request_domain = hostname.lower()
if cookie_domain and _is_public_domain(cookie_domain):
if cookie_domain != request_domain:
@ -48,9 +72,11 @@ class CookiesMiddleware:
jar.set_cookie_if_ok(cookie, request)
def process_request(self, request, spider):
def process_request(
self, request: Request, spider: Spider
) -> Union[Request, Response, None]:
if request.meta.get("dont_merge_cookies", False):
return
return None
cookiejarkey = request.meta.get("cookiejar")
jar = self.jars[cookiejarkey]
@ -61,8 +87,11 @@ class CookiesMiddleware:
request.headers.pop("Cookie", None)
jar.add_cookie_header(request)
self._debug_cookie(request, spider)
return None
def process_response(self, request, response, spider):
def process_response(
self, request: Request, response: Response, spider: Spider
) -> Union[Request, Response]:
if request.meta.get("dont_merge_cookies", False):
return response
@ -76,7 +105,7 @@ class CookiesMiddleware:
return response
def _debug_cookie(self, request, spider):
def _debug_cookie(self, request: Request, spider: Spider) -> None:
if self.debug:
cl = [
to_unicode(c, errors="replace")
@ -87,7 +116,7 @@ class CookiesMiddleware:
msg = f"Sending cookies to: {request}\n{cookies}"
logger.debug(msg, extra={"spider": spider})
def _debug_set_cookie(self, response, spider):
def _debug_set_cookie(self, response: Response, spider: Spider) -> None:
if self.debug:
cl = [
to_unicode(c, errors="replace")
@ -98,7 +127,7 @@ class CookiesMiddleware:
msg = f"Received cookies from: {response}\n{cookies}"
logger.debug(msg, extra={"spider": spider})
def _format_cookie(self, cookie, request):
def _format_cookie(self, cookie: Dict[str, Any], request: Request) -> Optional[str]:
"""
Given a dict consisting of cookie components, return its string representation.
Decode from bytes if necessary.
@ -109,7 +138,7 @@ class CookiesMiddleware:
if key in ("name", "value"):
msg = f"Invalid cookie found in request {request}: {cookie} ('{key}' is missing)"
logger.warning(msg)
return
return None
continue
if isinstance(cookie[key], (bool, float, int, str)):
decoded[key] = str(cookie[key])
@ -129,12 +158,15 @@ class CookiesMiddleware:
cookie_str += f"; {key.capitalize()}={value}"
return cookie_str
def _get_request_cookies(self, jar, request):
def _get_request_cookies(
self, jar: CookieJar, request: Request
) -> Sequence[Cookie]:
"""
Extract cookies from the Request.cookies attribute
"""
if not request.cookies:
return []
cookies: Iterable[Dict[str, Any]]
if isinstance(request.cookies, dict):
cookies = ({"name": k, "value": v} for k, v in request.cookies.items())
else:

21
scrapy/downloadermiddlewares/defaultheaders.py
View File

@ -3,19 +3,32 @@ DefaultHeaders downloader middleware
See documentation in docs/topics/downloader-middleware.rst
"""
from __future__ import annotations
from typing import TYPE_CHECKING, Iterable, Tuple, Union
from scrapy import Request, Spider
from scrapy.crawler import Crawler
from scrapy.http import Response
from scrapy.utils.python import without_none_values
if TYPE_CHECKING:
# typing.Self requires Python 3.11
from typing_extensions import Self
class DefaultHeadersMiddleware:
def __init__(self, headers):
self._headers = headers
def __init__(self, headers: Iterable[Tuple[str, str]]):
self._headers: Iterable[Tuple[str, str]] = headers
@classmethod
def from_crawler(cls, crawler):
def from_crawler(cls, crawler: Crawler) -> Self:
headers = without_none_values(crawler.settings["DEFAULT_REQUEST_HEADERS"])
return cls(headers.items())
def process_request(self, request, spider):
def process_request(
self, request: Request, spider: Spider
) -> Union[Request, Response, None]:
for k, v in self._headers:
request.headers.setdefault(k, v)
return None

24
scrapy/downloadermiddlewares/downloadtimeout.py
View File

@ -3,23 +3,35 @@ Download timeout middleware
See documentation in docs/topics/downloader-middleware.rst
"""
from __future__ import annotations
from scrapy import signals
from typing import TYPE_CHECKING, Union
from scrapy import Request, Spider, signals
from scrapy.crawler import Crawler
from scrapy.http import Response
if TYPE_CHECKING:
# typing.Self requires Python 3.11
from typing_extensions import Self
class DownloadTimeoutMiddleware:
def __init__(self, timeout=180):
self._timeout = timeout
def __init__(self, timeout: float = 180):
self._timeout: float = timeout
@classmethod
def from_crawler(cls, crawler):
def from_crawler(cls, crawler: Crawler) -> Self:
o = cls(crawler.settings.getfloat("DOWNLOAD_TIMEOUT"))
crawler.signals.connect(o.spider_opened, signal=signals.spider_opened)
return o
def spider_opened(self, spider):
def spider_opened(self, spider: Spider) -> None:
self._timeout = getattr(spider, "download_timeout", self._timeout)
def process_request(self, request, spider):
def process_request(
self, request: Request, spider: Spider
) -> Union[Request, Response, None]:
if self._timeout:
request.meta.setdefault("download_timeout", self._timeout)
return None

23
scrapy/downloadermiddlewares/httpauth.py
View File

@ -4,31 +4,44 @@ HTTP basic auth downloader middleware
See documentation in docs/topics/downloader-middleware.rst
"""
from __future__ import annotations
from typing import TYPE_CHECKING, Union
from w3lib.http import basic_auth_header
from scrapy import signals
from scrapy import Request, Spider, signals
from scrapy.crawler import Crawler
from scrapy.http import Response
from scrapy.utils.url import url_is_from_any_domain
if TYPE_CHECKING:
# typing.Self requires Python 3.11
from typing_extensions import Self
class HttpAuthMiddleware:
"""Set Basic HTTP Authorization header
(http_user and http_pass spider class attributes)"""
@classmethod
def from_crawler(cls, crawler):
def from_crawler(cls, crawler: Crawler) -> Self:
o = cls()
crawler.signals.connect(o.spider_opened, signal=signals.spider_opened)
return o
def spider_opened(self, spider):
def spider_opened(self, spider: Spider) -> None:
usr = getattr(spider, "http_user", "")
pwd = getattr(spider, "http_pass", "")
if usr or pwd:
self.auth = basic_auth_header(usr, pwd)
self.domain = spider.http_auth_domain
self.domain = spider.http_auth_domain # type: ignore[attr-defined]
def process_request(self, request, spider):
def process_request(
self, request: Request, spider: Spider
) -> Union[Request, Response, None]:
auth = getattr(self, "auth", None)
if auth and b"Authorization" not in request.headers:
if not self.domain or url_is_from_any_domain(request.url, [self.domain]):
request.headers[b"Authorization"] = auth
return None

28
scrapy/downloadermiddlewares/httpcache.py
View File

@ -1,5 +1,7 @@
from __future__ import annotations
from email.utils import formatdate
from typing import Optional, Type, TypeVar
from typing import TYPE_CHECKING, Optional, Union
from twisted.internet import defer
from twisted.internet.error import (
@ -23,7 +25,9 @@ from scrapy.spiders import Spider
from scrapy.statscollectors import StatsCollector
from scrapy.utils.misc import load_object
HttpCacheMiddlewareTV = TypeVar("HttpCacheMiddlewareTV", bound="HttpCacheMiddleware")
if TYPE_CHECKING:
# typing.Self requires Python 3.11
from typing_extensions import Self
class HttpCacheMiddleware:
@ -49,9 +53,7 @@ class HttpCacheMiddleware:
self.stats = stats
@classmethod
def from_crawler(
cls: Type[HttpCacheMiddlewareTV], crawler: Crawler
) -> HttpCacheMiddlewareTV:
def from_crawler(cls, crawler: Crawler) -> Self:
assert crawler.stats
o = cls(crawler.settings, crawler.stats)
crawler.signals.connect(o.spider_opened, signal=signals.spider_opened)
@ -64,7 +66,9 @@ class HttpCacheMiddleware:
def spider_closed(self, spider: Spider) -> None:
self.storage.close_spider(spider)
def process_request(self, request: Request, spider: Spider) -> Optional[Response]:
def process_request(
self, request: Request, spider: Spider
) -> Union[Request, Response, None]:
if request.meta.get("dont_cache", False):
return None
@ -74,7 +78,9 @@ class HttpCacheMiddleware:
return None
# Look for cached response and check if expired
cachedresponse = self.storage.retrieve_response(spider, request)
cachedresponse: Optional[Response] = self.storage.retrieve_response(
spider, request
)
if cachedresponse is None:
self.stats.inc_value("httpcache/miss", spider=spider)
if self.ignore_missing:
@ -96,7 +102,7 @@ class HttpCacheMiddleware:
def process_response(
self, request: Request, response: Response, spider: Spider
) -> Response:
) -> Union[Request, Response]:
if request.meta.get("dont_cache", False):
return response
@ -111,7 +117,7 @@ class HttpCacheMiddleware:
response.headers["Date"] = formatdate(usegmt=True)
# Do not validate first-hand responses
cachedresponse = request.meta.pop("cached_response", None)
cachedresponse: Optional[Response] = request.meta.pop("cached_response", None)
if cachedresponse is None:
self.stats.inc_value("httpcache/firsthand", spider=spider)
self._cache_response(spider, response, request, cachedresponse)
@ -127,8 +133,8 @@ class HttpCacheMiddleware:
def process_exception(
self, request: Request, exception: Exception, spider: Spider
) -> Optional[Response]:
cachedresponse = request.meta.pop("cached_response", None)
) -> Union[Request, Response, None]:
cachedresponse: Optional[Response] = request.meta.pop("cached_response", None)
if cachedresponse is not None and isinstance(
exception, self.DOWNLOAD_EXCEPTIONS
):

27
scrapy/downloadermiddlewares/httpcompression.py
View File

@ -1,12 +1,22 @@
from __future__ import annotations
import io
import zlib
from typing import TYPE_CHECKING, List, Optional, Union
from scrapy import Request, Spider
from scrapy.crawler import Crawler
from scrapy.exceptions import NotConfigured
from scrapy.http import Response, TextResponse
from scrapy.responsetypes import responsetypes
from scrapy.statscollectors import StatsCollector
from scrapy.utils.gz import gunzip
ACCEPTED_ENCODINGS = [b"gzip", b"deflate"]
if TYPE_CHECKING:
# typing.Self requires Python 3.11
from typing_extensions import Self
ACCEPTED_ENCODINGS: List[bytes] = [b"gzip", b"deflate"]
try:
import brotli
@ -27,19 +37,24 @@ class HttpCompressionMiddleware:
"""This middleware allows compressed (gzip, deflate) traffic to be
sent/received from web sites"""
def __init__(self, stats=None):
def __init__(self, stats: Optional[StatsCollector] = None):
self.stats = stats
@classmethod
def from_crawler(cls, crawler):
def from_crawler(cls, crawler: Crawler) -> Self:
if not crawler.settings.getbool("COMPRESSION_ENABLED"):
raise NotConfigured
return cls(stats=crawler.stats)
def process_request(self, request, spider):
def process_request(
self, request: Request, spider: Spider
) -> Union[Request, Response, None]:
request.headers.setdefault("Accept-Encoding", b", ".join(ACCEPTED_ENCODINGS))
return None
def process_response(self, request, response, spider):
def process_response(
self, request: Request, response: Response, spider: Spider
) -> Union[Request, Response]:
if request.method == "HEAD":
return response
if isinstance(response, Response):
@ -70,7 +85,7 @@ class HttpCompressionMiddleware:
return response
def _decode(self, body, encoding):
def _decode(self, body: bytes, encoding: bytes) -> bytes:
if encoding == b"gzip" or encoding == b"x-gzip":
body = gunzip(body)

45
scrapy/downloadermiddlewares/httpproxy.py
View File

@ -1,16 +1,30 @@
import base64
from urllib.parse import unquote, urlunparse
from urllib.request import _parse_proxy, getproxies, proxy_bypass
from __future__ import annotations
import base64
from typing import TYPE_CHECKING, Dict, Optional, Tuple, Union
from urllib.parse import unquote, urlunparse
from urllib.request import ( # type: ignore[attr-defined]
_parse_proxy,
getproxies,
proxy_bypass,
)
from scrapy import Request, Spider
from scrapy.crawler import Crawler
from scrapy.exceptions import NotConfigured
from scrapy.http import Response
from scrapy.utils.httpobj import urlparse_cached
from scrapy.utils.python import to_bytes
if TYPE_CHECKING:
# typing.Self requires Python 3.11
from typing_extensions import Self
class HttpProxyMiddleware:
def __init__(self, auth_encoding="latin-1"):
self.auth_encoding = auth_encoding
self.proxies = {}
def __init__(self, auth_encoding: Optional[str] = "latin-1"):
self.auth_encoding: Optional[str] = auth_encoding
self.proxies: Dict[str, Tuple[Optional[bytes], str]] = {}
for type_, url in getproxies().items():
try:
self.proxies[type_] = self._get_proxy(url, type_)
@ -20,19 +34,19 @@ class HttpProxyMiddleware:
continue
@classmethod
def from_crawler(cls, crawler):
def from_crawler(cls, crawler: Crawler) -> Self:
if not crawler.settings.getbool("HTTPPROXY_ENABLED"):
raise NotConfigured
auth_encoding = crawler.settings.get("HTTPPROXY_AUTH_ENCODING")
auth_encoding: Optional[str] = crawler.settings.get("HTTPPROXY_AUTH_ENCODING")
return cls(auth_encoding)
def _basic_auth_header(self, username, password):
def _basic_auth_header(self, username: str, password: str) -> bytes:
user_pass = to_bytes(
f"{unquote(username)}:{unquote(password)}", encoding=self.auth_encoding
)
return base64.b64encode(user_pass)
def _get_proxy(self, url, orig_type):
def _get_proxy(self, url: str, orig_type: str) -> Tuple[Optional[bytes], str]:
proxy_type, user, password, hostport = _parse_proxy(url)
proxy_url = urlunparse((proxy_type or orig_type, hostport, "", "", "", ""))
@ -43,7 +57,9 @@ class HttpProxyMiddleware:
return creds, proxy_url
def process_request(self, request, spider):
def process_request(
self, request: Request, spider: Spider
) -> Union[Request, Response, None]:
creds, proxy_url = None, None
if "proxy" in request.meta:
if request.meta["proxy"] is not None:
@ -54,13 +70,16 @@ class HttpProxyMiddleware:
if (
# 'no_proxy' is only supported by http schemes
scheme not in ("http", "https")
or not proxy_bypass(parsed.hostname)
or (parsed.hostname and not proxy_bypass(parsed.hostname))
) and scheme in self.proxies:
creds, proxy_url = self.proxies[scheme]
self._set_proxy_and_creds(request, proxy_url, creds)
return None
def _set_proxy_and_creds(self, request, proxy_url, creds):
def _set_proxy_and_creds(
self, request: Request, proxy_url: Optional[str], creds: Optional[bytes]
) -> None:
if proxy_url:
request.meta["proxy"] = proxy_url
elif request.meta.get("proxy") is not None:

51
scrapy/downloadermiddlewares/redirect.py
View File

@ -1,17 +1,29 @@
from __future__ import annotations
import logging
from typing import TYPE_CHECKING, Any, List, Union, cast
from urllib.parse import urljoin, urlparse
from w3lib.url import safe_url_string
from scrapy import Request, Spider
from scrapy.crawler import Crawler
from scrapy.exceptions import IgnoreRequest, NotConfigured
from scrapy.http import HtmlResponse
from scrapy.http import HtmlResponse, Response
from scrapy.settings import BaseSettings
from scrapy.utils.httpobj import urlparse_cached
from scrapy.utils.response import get_meta_refresh
if TYPE_CHECKING:
# typing.Self requires Python 3.11
from typing_extensions import Self
logger = logging.getLogger(__name__)
def _build_redirect_request(source_request, *, url, **kwargs):
def _build_redirect_request(
source_request: Request, *, url: str, **kwargs: Any
) -> Request:
redirect_request = source_request.replace(
url=url,
**kwargs,
@ -26,20 +38,22 @@ def _build_redirect_request(source_request, *, url, **kwargs):
class BaseRedirectMiddleware:
enabled_setting = "REDIRECT_ENABLED"
enabled_setting: str = "REDIRECT_ENABLED"
def __init__(self, settings):
def __init__(self, settings: BaseSettings):
if not settings.getbool(self.enabled_setting):
raise NotConfigured
self.max_redirect_times = settings.getint("REDIRECT_MAX_TIMES")
self.priority_adjust = settings.getint("REDIRECT_PRIORITY_ADJUST")
self.max_redirect_times: int = settings.getint("REDIRECT_MAX_TIMES")
self.priority_adjust: int = settings.getint("REDIRECT_PRIORITY_ADJUST")
@classmethod
def from_crawler(cls, crawler):
def from_crawler(cls, crawler: Crawler) -> Self:
return cls(crawler.settings)
def _redirect(self, redirected, request, spider, reason):
def _redirect(
self, redirected: Request, request: Request, spider: Spider, reason: Any
) -> Request:
ttl = request.meta.setdefault("redirect_ttl", self.max_redirect_times)
redirects = request.meta.get("redirect_times", 0) + 1
@ -67,7 +81,9 @@ class BaseRedirectMiddleware:
)
raise IgnoreRequest("max redirections reached")
def _redirect_request_using_get(self, request, redirect_url):
def _redirect_request_using_get(
self, request: Request, redirect_url: str
) -> Request:
redirect_request = _build_redirect_request(
request,
url=redirect_url,
@ -85,7 +101,9 @@ class RedirectMiddleware(BaseRedirectMiddleware):
and meta-refresh html tag.
"""
def process_response(self, request, response, spider):
def process_response(
self, request: Request, response: Response, spider: Spider
) -> Union[Request, Response]:
if (
request.meta.get("dont_redirect", False)
or response.status in getattr(spider, "handle_httpstatus_list", [])
@ -98,6 +116,7 @@ class RedirectMiddleware(BaseRedirectMiddleware):
if "Location" not in response.headers or response.status not in allowed_status:
return response
assert response.headers["Location"] is not None
location = safe_url_string(response.headers["Location"])
if response.headers["Location"].startswith(b"//"):
request_scheme = urlparse(request.url).scheme
@ -116,12 +135,14 @@ class RedirectMiddleware(BaseRedirectMiddleware):
class MetaRefreshMiddleware(BaseRedirectMiddleware):
enabled_setting = "METAREFRESH_ENABLED"
def __init__(self, settings):
def __init__(self, settings: BaseSettings):
super().__init__(settings)
self._ignore_tags = settings.getlist("METAREFRESH_IGNORE_TAGS")
self._maxdelay = settings.getint("METAREFRESH_MAXDELAY")
self._ignore_tags: List[str] = settings.getlist("METAREFRESH_IGNORE_TAGS")
self._maxdelay: int = settings.getint("METAREFRESH_MAXDELAY")
def process_response(self, request, response, spider):
def process_response(
self, request: Request, response: Response, spider: Spider
) -> Union[Request, Response]:
if (
request.meta.get("dont_redirect", False)
or request.method == "HEAD"
@ -130,7 +151,7 @@ class MetaRefreshMiddleware(BaseRedirectMiddleware):
return response
interval, url = get_meta_refresh(response, ignore_tags=self._ignore_tags)
if url and interval < self._maxdelay:
if url and cast(float, interval) < self._maxdelay:
redirected = self._redirect_request_using_get(request, url)
return self._redirect(redirected, request, spider, "meta refresh")

36
scrapy/downloadermiddlewares/retry.py
View File

@ -9,22 +9,30 @@ RETRY_HTTP_CODES - which HTTP response codes to retry
Failed pages are collected on the scraping process and rescheduled at the end,
once the spider has finished crawling all regular (non failed) pages.
"""
from __future__ import annotations
import warnings
from logging import Logger, getLogger
from typing import Optional, Type, Union
from typing import TYPE_CHECKING, Any, Optional, Tuple, Type, Union
from scrapy.crawler import Crawler
from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
from scrapy.http import Response
from scrapy.http.request import Request
from scrapy.settings import Settings
from scrapy.settings import BaseSettings, Settings
from scrapy.spiders import Spider
from scrapy.utils.misc import load_object
from scrapy.utils.python import global_object_name
from scrapy.utils.response import response_status_message
if TYPE_CHECKING:
# typing.Self requires Python 3.11
from typing_extensions import Self
retry_logger = getLogger(__name__)
def backwards_compatibility_getattr(self, name):
def backwards_compatibility_getattr(self: Any, name: str) -> Tuple[Any, ...]:
if name == "EXCEPTIONS_TO_RETRY":
warnings.warn(
"Attribute RetryMiddleware.EXCEPTIONS_TO_RETRY is deprecated. "
@ -54,7 +62,7 @@ def get_retry_request(
priority_adjust: Optional[int] = None,
logger: Logger = retry_logger,
stats_base_key: str = "retry",
):
) -> Optional[Request]:
"""
Returns a new :class:`~scrapy.Request` object to retry the specified
request, or ``None`` if retries of the specified request have been
@ -134,7 +142,7 @@ def get_retry_request(
class RetryMiddleware(metaclass=BackwardsCompatibilityMetaclass):
def __init__(self, settings):
def __init__(self, settings: BaseSettings):
if not settings.getbool("RETRY_ENABLED"):
raise NotConfigured
self.max_retry_times = settings.getint("RETRY_TIMES")
@ -153,10 +161,12 @@ class RetryMiddleware(metaclass=BackwardsCompatibilityMetaclass):
)
@classmethod
def from_crawler(cls, crawler):
def from_crawler(cls, crawler: Crawler) -> Self:
return cls(crawler.settings)
def process_response(self, request, response, spider):
def process_response(
self, request: Request, response: Response, spider: Spider
) -> Union[Request, Response]:
if request.meta.get("dont_retry", False):
return response
if response.status in self.retry_http_codes:
@ -164,13 +174,21 @@ class RetryMiddleware(metaclass=BackwardsCompatibilityMetaclass):
return self._retry(request, reason, spider) or response
return response
def process_exception(self, request, exception, spider):
def process_exception(
self, request: Request, exception: Exception, spider: Spider
) -> Union[Request, Response, None]:
if isinstance(exception, self.exceptions_to_retry) and not request.meta.get(
"dont_retry", False
):
return self._retry(request, exception, spider)
return None
def _retry(self, request, reason, spider):
def _retry(
self,
request: Request,
reason: Union[str, Exception, Type[Exception]],
spider: Spider,
) -> Optional[Request]:
max_retry_times = request.meta.get("max_retry_times", self.max_retry_times)
priority_adjust = request.meta.get("priority_adjust", self.priority_adjust)
return get_retry_request(

77
scrapy/downloadermiddlewares/robotstxt.py
View File

@ -4,65 +4,87 @@ enable this middleware and enable the ROBOTSTXT_OBEY setting.
"""
from __future__ import annotations
import logging
from typing import TYPE_CHECKING, Any, Dict, Optional, Union
from twisted.internet.defer import Deferred, maybeDeferred
from twisted.python.failure import Failure
from scrapy import Spider
from scrapy.crawler import Crawler
from scrapy.exceptions import IgnoreRequest, NotConfigured
from scrapy.http import Request
from scrapy.http import Request, Response
from scrapy.http.request import NO_CALLBACK
from scrapy.robotstxt import RobotParser
from scrapy.utils.httpobj import urlparse_cached
from scrapy.utils.log import failure_to_exc_info
from scrapy.utils.misc import load_object
if TYPE_CHECKING:
# typing.Self requires Python 3.11
from typing_extensions import Self
logger = logging.getLogger(__name__)
class RobotsTxtMiddleware:
DOWNLOAD_PRIORITY = 1000
DOWNLOAD_PRIORITY: int = 1000
def __init__(self, crawler):
def __init__(self, crawler: Crawler):
if not crawler.settings.getbool("ROBOTSTXT_OBEY"):
raise NotConfigured
self._default_useragent = crawler.settings.get("USER_AGENT", "Scrapy")
self._robotstxt_useragent = crawler.settings.get("ROBOTSTXT_USER_AGENT", None)
self.crawler = crawler
self._parsers = {}
self._parserimpl = load_object(crawler.settings.get("ROBOTSTXT_PARSER"))
self._default_useragent: str = crawler.settings.get("USER_AGENT", "Scrapy")
self._robotstxt_useragent: Optional[str] = crawler.settings.get(
"ROBOTSTXT_USER_AGENT", None
)
self.crawler: Crawler = crawler
self._parsers: Dict[str, Union[RobotParser, Deferred, None]] = {}
self._parserimpl: RobotParser = load_object(
crawler.settings.get("ROBOTSTXT_PARSER")
)
# check if parser dependencies are met, this should throw an error otherwise.
self._parserimpl.from_crawler(self.crawler, b"")
@classmethod
def from_crawler(cls, crawler):
def from_crawler(cls, crawler: Crawler) -> Self:
return cls(crawler)
def process_request(self, request, spider):
def process_request(self, request: Request, spider: Spider) -> Optional[Deferred]:
if request.meta.get("dont_obey_robotstxt"):
return
return None
if request.url.startswith("data:") or request.url.startswith("file:"):
return
d = maybeDeferred(self.robot_parser, request, spider)
return None
d: Deferred = maybeDeferred(self.robot_parser, request, spider)
d.addCallback(self.process_request_2, request, spider)
return d
def process_request_2(self, rp, request, spider):
def process_request_2(
self, rp: Optional[RobotParser], request: Request, spider: Spider
) -> None:
if rp is None:
return
useragent = self._robotstxt_useragent
useragent: Union[str, bytes, None] = self._robotstxt_useragent
if not useragent:
useragent = request.headers.get(b"User-Agent", self._default_useragent)
assert useragent is not None
if not rp.allowed(request.url, useragent):
logger.debug(
"Forbidden by robots.txt: %(request)s",
{"request": request},
extra={"spider": spider},
)
assert self.crawler.stats
self.crawler.stats.inc_value("robotstxt/forbidden")
raise IgnoreRequest("Forbidden by robots.txt")
def robot_parser(self, request, spider):
def robot_parser(
self, request: Request, spider: Spider
) -> Union[RobotParser, Deferred, None]:
url = urlparse_cached(request)
netloc = url.netloc
@ -75,24 +97,27 @@ class RobotsTxtMiddleware:
meta={"dont_obey_robotstxt": True},
callback=NO_CALLBACK,
)
assert self.crawler.engine
assert self.crawler.stats
dfd = self.crawler.engine.download(robotsreq)
dfd.addCallback(self._parse_robots, netloc, spider)
dfd.addErrback(self._logerror, robotsreq, spider)
dfd.addErrback(self._robots_error, netloc)
self.crawler.stats.inc_value("robotstxt/request_count")
if isinstance(self._parsers[netloc], Deferred):
d = Deferred()
parser = self._parsers[netloc]
if isinstance(parser, Deferred):
d: Deferred = Deferred()
def cb(result):
def cb(result: Any) -> Any:
d.callback(result)
return result
self._parsers[netloc].addCallback(cb)
parser.addCallback(cb)
return d
return self._parsers[netloc]
return parser
def _logerror(self, failure, request, spider):
def _logerror(self, failure: Failure, request: Request, spider: Spider) -> Failure:
if failure.type is not IgnoreRequest:
logger.error(
"Error downloading %(request)s: %(f_exception)s",
@ -102,20 +127,24 @@ class RobotsTxtMiddleware:
)
return failure
def _parse_robots(self, response, netloc, spider):
def _parse_robots(self, response: Response, netloc: str, spider: Spider) -> None:
assert self.crawler.stats
self.crawler.stats.inc_value("robotstxt/response_count")
self.crawler.stats.inc_value(
f"robotstxt/response_status_count/{response.status}"
)
rp = self._parserimpl.from_crawler(self.crawler, response.body)
rp_dfd = self._parsers[netloc]
assert isinstance(rp_dfd, Deferred)
self._parsers[netloc] = rp
rp_dfd.callback(rp)
def _robots_error(self, failure, netloc):
def _robots_error(self, failure: Failure, netloc: str) -> None:
if failure.type is not IgnoreRequest:
key = f"robotstxt/exception_count/{failure.type}"
assert self.crawler.stats
self.crawler.stats.inc_value(key)
rp_dfd = self._parsers[netloc]
assert isinstance(rp_dfd, Deferred)
self._parsers[netloc] = None
rp_dfd.callback(None)

37
scrapy/downloadermiddlewares/stats.py
View File

@ -1,11 +1,23 @@
from __future__ import annotations
from typing import TYPE_CHECKING, Dict, Union
from twisted.web import http
from scrapy import Request, Spider
from scrapy.crawler import Crawler
from scrapy.exceptions import NotConfigured
from scrapy.http import Response
from scrapy.statscollectors import StatsCollector
from scrapy.utils.python import global_object_name, to_bytes
from scrapy.utils.request import request_httprepr
if TYPE_CHECKING:
# typing.Self requires Python 3.11
from typing_extensions import Self
def get_header_size(headers):
def get_header_size(headers: Dict[str, Union[list, tuple]]) -> int:
size = 0
for key, value in headers.items():
if isinstance(value, (list, tuple)):
@ -14,30 +26,36 @@ def get_header_size(headers):
return size + len(b"\r\n") * (len(headers.keys()) - 1)
def get_status_size(response_status):
def get_status_size(response_status: int) -> int:
return len(to_bytes(http.RESPONSES.get(response_status, b""))) + 15
# resp.status + b"\r\n" + b"HTTP/1.1 <100-599> "
class DownloaderStats:
def __init__(self, stats):
self.stats = stats
def __init__(self, stats: StatsCollector):
self.stats: StatsCollector = stats
@classmethod
def from_crawler(cls, crawler):
def from_crawler(cls, crawler: Crawler) -> Self:
if not crawler.settings.getbool("DOWNLOADER_STATS"):
raise NotConfigured
assert crawler.stats
return cls(crawler.stats)
def process_request(self, request, spider):
def process_request(
self, request: Request, spider: Spider
) -> Union[Request, Response, None]:
self.stats.inc_value("downloader/request_count", spider=spider)
self.stats.inc_value(
f"downloader/request_method_count/{request.method}", spider=spider
)
reqlen = len(request_httprepr(request))
self.stats.inc_value("downloader/request_bytes", reqlen, spider=spider)
return None
def process_response(self, request, response, spider):
def process_response(
self, request: Request, response: Response, spider: Spider
) -> Union[Request, Response]:
self.stats.inc_value("downloader/response_count", spider=spider)
self.stats.inc_value(
f"downloader/response_status_count/{response.status}", spider=spider
@ -52,9 +70,12 @@ class DownloaderStats:
self.stats.inc_value("downloader/response_bytes", reslen, spider=spider)
return response
def process_exception(self, request, exception, spider):
def process_exception(
self, request: Request, exception: Exception, spider: Spider
) -> Union[Request, Response, None]:
ex_class = global_object_name(exception.__class__)
self.stats.inc_value("downloader/exception_count", spider=spider)
self.stats.inc_value(
f"downloader/exception_type_count/{ex_class}", spider=spider
)
return None

23
scrapy/downloadermiddlewares/useragent.py
View File

@ -1,23 +1,36 @@
"""Set User-Agent header per spider or use a default value from settings"""
from scrapy import signals
from __future__ import annotations
from typing import TYPE_CHECKING, Union
from scrapy import Request, Spider, signals
from scrapy.crawler import Crawler
from scrapy.http import Response
if TYPE_CHECKING:
# typing.Self requires Python 3.11
from typing_extensions import Self
class UserAgentMiddleware:
"""This middleware allows spiders to override the user_agent"""
def __init__(self, user_agent="Scrapy"):
def __init__(self, user_agent: str = "Scrapy"):
self.user_agent = user_agent
@classmethod
def from_crawler(cls, crawler):
def from_crawler(cls, crawler: Crawler) -> Self:
o = cls(crawler.settings["USER_AGENT"])
crawler.signals.connect(o.spider_opened, signal=signals.spider_opened)
return o
def spider_opened(self, spider):
def spider_opened(self, spider: Spider) -> None:
self.user_agent = getattr(spider, "user_agent", self.user_agent)
def process_request(self, request, spider):
def process_request(
self, request: Request, spider: Spider
) -> Union[Request, Response, None]:
if self.user_agent:
request.headers.setdefault(b"User-Agent", self.user_agent)
return None

10
scrapy/http/cookies.py
View File

@ -1,8 +1,12 @@
import re
import time
from http.cookiejar import Cookie
from http.cookiejar import CookieJar as _CookieJar
from http.cookiejar import DefaultCookiePolicy
from typing import Sequence
from scrapy import Request
from scrapy.http import Response
from scrapy.utils.httpobj import urlparse_cached
from scrapy.utils.python import to_unicode
@ -24,7 +28,7 @@ class CookieJar:
wrsp = WrappedResponse(response)
return self.jar.extract_cookies(wrsp, wreq)
def add_cookie_header(self, request):
def add_cookie_header(self, request: Request) -> None:
wreq = WrappedRequest(request)
self.policy._now = self.jar._now = int(time.time())
@ -75,7 +79,7 @@ class CookieJar:
def set_policy(self, pol):
return self.jar.set_policy(pol)
def make_cookies(self, response, request):
def make_cookies(self, response: Response, request: Request) -> Sequence[Cookie]:
wreq = WrappedRequest(request)
wrsp = WrappedResponse(response)
return self.jar.make_cookies(wrsp, wreq)
@ -83,7 +87,7 @@ class CookieJar:
def set_cookie(self, cookie):
self.jar.set_cookie(cookie)
def set_cookie_if_ok(self, cookie, request):
def set_cookie_if_ok(self, cookie: Cookie, request: Request) -> None:
self.jar.set_cookie_if_ok(cookie, WrappedRequest(request))

17
scrapy/robotstxt.py
View File

@ -1,11 +1,20 @@
from __future__ import annotations
import logging
import sys
from abc import ABCMeta, abstractmethod
from typing import TYPE_CHECKING, Union
from warnings import warn
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.utils.python import to_unicode
if TYPE_CHECKING:
# typing.Self requires Python 3.11
from typing_extensions import Self
from scrapy.crawler import Crawler
logger = logging.getLogger(__name__)
@ -31,7 +40,7 @@ def decode_robotstxt(robotstxt_body, spider, to_native_str_type=False):
class RobotParser(metaclass=ABCMeta):
@classmethod
@abstractmethod
def from_crawler(cls, crawler, robotstxt_body):
def from_crawler(cls, crawler: Crawler, robotstxt_body: bytes) -> Self:
"""Parse the content of a robots.txt_ file as bytes. This must be a class method.
It must return a new instance of the parser backend.
@ -44,14 +53,14 @@ class RobotParser(metaclass=ABCMeta):
pass
@abstractmethod
def allowed(self, url, user_agent):
def allowed(self, url: Union[str, bytes], user_agent: Union[str, bytes]) -> bool:
"""Return ``True`` if ``user_agent`` is allowed to crawl ``url``, otherwise return ``False``.
:param url: Absolute URL
:type url: str
:type url: str or bytes
:param user_agent: User agent
:type user_agent: str
:type user_agent: str or bytes
"""
pass

3
setup.cfg
View File

@ -16,9 +16,6 @@ follow_imports = skip
# FIXME: remove the following sections once the issues are solved
[mypy-scrapy.downloadermiddlewares.httpproxy]
ignore_errors = True
[mypy-scrapy.interfaces]
ignore_errors = True