diff --git a/scrapy/__init__.py b/scrapy/__init__.py index 4326ca4aa..8a8065bf2 100644 --- a/scrapy/__init__.py +++ b/scrapy/__init__.py @@ -22,7 +22,7 @@ __all__ = [ # Scrapy and Twisted versions -__version__ = pkgutil.get_data(__package__, 'VERSION').decode('ascii').strip() +__version__ = (pkgutil.get_data(__package__, "VERSION") or b"").decode("ascii").strip() version_info = tuple(int(v) if v.isdigit() else v for v in __version__.split('.')) twisted_version = (_txv.major, _txv.minor, _txv.micro) diff --git a/scrapy/commands/__init__.py b/scrapy/commands/__init__.py index 23ccffcd9..6e77551c6 100644 --- a/scrapy/commands/__init__.py +++ b/scrapy/commands/__init__.py @@ -3,6 +3,8 @@ Base class for Scrapy commands """ import os from optparse import OptionGroup +from typing import Any, Dict + from twisted.python import failure from scrapy.utils.conf import arglist_to_dict, feed_process_params_from_cli @@ -15,7 +17,7 @@ class ScrapyCommand: crawler_process = None # default settings to be used for this command instead of global defaults - default_settings = {} + default_settings: Dict[str, Any] = {} exitcode = 0 diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py index 83ee074da..52118db1b 100644 --- a/scrapy/commands/parse.py +++ b/scrapy/commands/parse.py @@ -1,5 +1,6 @@ import json import logging +from typing import Dict from itemadapter import is_item, ItemAdapter from w3lib.url import is_url @@ -10,6 +11,7 @@ from scrapy.utils import display from scrapy.utils.spider import iterate_spider_output, spidercls_for_request from scrapy.exceptions import UsageError + logger = logging.getLogger(__name__) @@ -17,8 +19,8 @@ class Command(BaseRunSpiderCommand): requires_project = True spider = None - items = {} - requests = {} + items: Dict[int, list] = {} + requests: Dict[int, list] = {} first_response = None diff --git a/scrapy/contracts/__init__.py b/scrapy/contracts/__init__.py index db0a56e56..b47e55092 100644 --- a/scrapy/contracts/__init__.py +++ b/scrapy/contracts/__init__.py @@ -1,16 +1,77 @@ -import sys import re +import sys from functools import wraps from inspect import getmembers +from typing import Dict from unittest import TestCase from scrapy.http import Request -from scrapy.utils.spider import iterate_spider_output from scrapy.utils.python import get_spec +from scrapy.utils.spider import iterate_spider_output + + +class Contract: + """ Abstract class for contracts """ + request_cls = None + + def __init__(self, method, *args): + self.testcase_pre = _create_testcase(method, f'@{self.name} pre-hook') + self.testcase_post = _create_testcase(method, f'@{self.name} post-hook') + self.args = args + + def add_pre_hook(self, request, results): + if hasattr(self, 'pre_process'): + cb = request.callback + + @wraps(cb) + def wrapper(response, **cb_kwargs): + try: + results.startTest(self.testcase_pre) + self.pre_process(response) + results.stopTest(self.testcase_pre) + except AssertionError: + results.addFailure(self.testcase_pre, sys.exc_info()) + except Exception: + results.addError(self.testcase_pre, sys.exc_info()) + else: + results.addSuccess(self.testcase_pre) + finally: + return list(iterate_spider_output(cb(response, **cb_kwargs))) + + request.callback = wrapper + + return request + + def add_post_hook(self, request, results): + if hasattr(self, 'post_process'): + cb = request.callback + + @wraps(cb) + def wrapper(response, **cb_kwargs): + output = list(iterate_spider_output(cb(response, **cb_kwargs))) + try: + results.startTest(self.testcase_post) + self.post_process(output) + results.stopTest(self.testcase_post) + except AssertionError: + results.addFailure(self.testcase_post, sys.exc_info()) + except Exception: + results.addError(self.testcase_post, sys.exc_info()) + else: + results.addSuccess(self.testcase_post) + finally: + return output + + request.callback = wrapper + + return request + + def adjust_request_args(self, args): + return args class ContractsManager: - contracts = {} + contracts: Dict[str, Contract] = {} def __init__(self, contracts): for contract in contracts: @@ -107,66 +168,6 @@ class ContractsManager: request.errback = eb_wrapper -class Contract: - """ Abstract class for contracts """ - request_cls = None - - def __init__(self, method, *args): - self.testcase_pre = _create_testcase(method, f'@{self.name} pre-hook') - self.testcase_post = _create_testcase(method, f'@{self.name} post-hook') - self.args = args - - def add_pre_hook(self, request, results): - if hasattr(self, 'pre_process'): - cb = request.callback - - @wraps(cb) - def wrapper(response, **cb_kwargs): - try: - results.startTest(self.testcase_pre) - self.pre_process(response) - results.stopTest(self.testcase_pre) - except AssertionError: - results.addFailure(self.testcase_pre, sys.exc_info()) - except Exception: - results.addError(self.testcase_pre, sys.exc_info()) - else: - results.addSuccess(self.testcase_pre) - finally: - return list(iterate_spider_output(cb(response, **cb_kwargs))) - - request.callback = wrapper - - return request - - def add_post_hook(self, request, results): - if hasattr(self, 'post_process'): - cb = request.callback - - @wraps(cb) - def wrapper(response, **cb_kwargs): - output = list(iterate_spider_output(cb(response, **cb_kwargs))) - try: - results.startTest(self.testcase_post) - self.post_process(output) - results.stopTest(self.testcase_post) - except AssertionError: - results.addFailure(self.testcase_post, sys.exc_info()) - except Exception: - results.addError(self.testcase_post, sys.exc_info()) - else: - results.addSuccess(self.testcase_post) - finally: - return output - - request.callback = wrapper - - return request - - def adjust_request_args(self, args): - return args - - def _create_testcase(method, desc): spider = method.__self__.name diff --git a/scrapy/http/cookies.py b/scrapy/http/cookies.py index 0c97e6999..bf4ae7b45 100644 --- a/scrapy/http/cookies.py +++ b/scrapy/http/cookies.py @@ -1,10 +1,16 @@ +import re import time -from http.cookiejar import CookieJar as _CookieJar, DefaultCookiePolicy, IPV4_RE +from http.cookiejar import CookieJar as _CookieJar, DefaultCookiePolicy from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.python import to_unicode +# Defined in the http.cookiejar module, but undocumented: +# https://github.com/python/cpython/blob/v3.9.0/Lib/http/cookiejar.py#L527 +IPV4_RE = re.compile(r"\.\d+$", re.ASCII) + + class CookieJar: def __init__(self, policy=None, check_expired_frequency=10000): self.policy = policy or DefaultCookiePolicy() diff --git a/scrapy/item.py b/scrapy/item.py index af3849302..2ccd7ad18 100644 --- a/scrapy/item.py +++ b/scrapy/item.py @@ -8,6 +8,7 @@ from abc import ABCMeta from collections.abc import MutableMapping from copy import deepcopy from pprint import pformat +from typing import Dict from warnings import warn from scrapy.utils.deprecate import ScrapyDeprecationWarning @@ -75,7 +76,7 @@ class ItemMeta(_BaseItemMeta): class DictItem(MutableMapping, BaseItem): - fields = {} + fields: Dict[str, Field] = {} def __new__(cls, *args, **kwargs): if issubclass(cls, DictItem) and not issubclass(cls, Item): diff --git a/scrapy/mail.py b/scrapy/mail.py index 7d7a2c435..2a25ccd44 100644 --- a/scrapy/mail.py +++ b/scrapy/mail.py @@ -9,7 +9,7 @@ from email.mime.base import MIMEBase from email.mime.multipart import MIMEMultipart from email.mime.nonmultipart import MIMENonMultipart from email.mime.text import MIMEText -from email.utils import COMMASPACE, formatdate +from email.utils import formatdate from io import BytesIO from twisted.internet import defer, ssl @@ -21,6 +21,11 @@ from scrapy.utils.python import to_bytes logger = logging.getLogger(__name__) +# Defined in the email.utils module, but undocumented: +# https://github.com/python/cpython/blob/v3.9.0/Lib/email/utils.py#L42 +COMMASPACE = ", " + + def _to_bytes_or_none(text): if text is None: return None diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py index f81041376..608c0eea5 100644 --- a/scrapy/spidermiddlewares/referer.py +++ b/scrapy/spidermiddlewares/referer.py @@ -3,15 +3,16 @@ RefererMiddleware: populates Request referer field, based on the Response which originated it. """ import warnings +from typing import Tuple from urllib.parse import urlparse from w3lib.url import safe_url_string -from scrapy.http import Request, Response -from scrapy.exceptions import NotConfigured from scrapy import signals -from scrapy.utils.python import to_unicode +from scrapy.exceptions import NotConfigured +from scrapy.http import Request, Response from scrapy.utils.misc import load_object +from scrapy.utils.python import to_unicode from scrapy.utils.url import strip_url @@ -30,7 +31,8 @@ POLICY_SCRAPY_DEFAULT = "scrapy-default" class ReferrerPolicy: - NOREFERRER_SCHEMES = LOCAL_SCHEMES + NOREFERRER_SCHEMES: Tuple[str, ...] = LOCAL_SCHEMES + name: str def referrer(self, response_url, request_url): raise NotImplementedError() @@ -88,7 +90,7 @@ class NoReferrerPolicy(ReferrerPolicy): is to be sent along with requests made from a particular request client to any origin. The header will be omitted entirely. """ - name = POLICY_NO_REFERRER + name: str = POLICY_NO_REFERRER def referrer(self, response_url, request_url): return None @@ -108,7 +110,7 @@ class NoReferrerWhenDowngradePolicy(ReferrerPolicy): This is a user agent's default behavior, if no policy is otherwise specified. """ - name = POLICY_NO_REFERRER_WHEN_DOWNGRADE + name: str = POLICY_NO_REFERRER_WHEN_DOWNGRADE def referrer(self, response_url, request_url): if not self.tls_protected(response_url) or self.tls_protected(request_url): @@ -125,7 +127,7 @@ class SameOriginPolicy(ReferrerPolicy): Cross-origin requests, on the other hand, will contain no referrer information. A Referer HTTP header will not be sent. """ - name = POLICY_SAME_ORIGIN + name: str = POLICY_SAME_ORIGIN def referrer(self, response_url, request_url): if self.origin(response_url) == self.origin(request_url): @@ -141,7 +143,7 @@ class OriginPolicy(ReferrerPolicy): when making both same-origin requests and cross-origin requests from a particular request client. """ - name = POLICY_ORIGIN + name: str = POLICY_ORIGIN def referrer(self, response_url, request_url): return self.origin_referrer(response_url) @@ -160,7 +162,7 @@ class StrictOriginPolicy(ReferrerPolicy): on the other hand, will contain no referrer information. A Referer HTTP header will not be sent. """ - name = POLICY_STRICT_ORIGIN + name: str = POLICY_STRICT_ORIGIN def referrer(self, response_url, request_url): if ( @@ -181,7 +183,7 @@ class OriginWhenCrossOriginPolicy(ReferrerPolicy): is sent as referrer information when making cross-origin requests from a particular request client. """ - name = POLICY_ORIGIN_WHEN_CROSS_ORIGIN + name: str = POLICY_ORIGIN_WHEN_CROSS_ORIGIN def referrer(self, response_url, request_url): origin = self.origin(response_url) @@ -208,7 +210,7 @@ class StrictOriginWhenCrossOriginPolicy(ReferrerPolicy): on the other hand, will contain no referrer information. A Referer HTTP header will not be sent. """ - name = POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN + name: str = POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN def referrer(self, response_url, request_url): origin = self.origin(response_url) @@ -234,7 +236,7 @@ class UnsafeUrlPolicy(ReferrerPolicy): to insecure origins. Carefully consider the impact of setting such a policy for potentially sensitive documents. """ - name = POLICY_UNSAFE_URL + name: str = POLICY_UNSAFE_URL def referrer(self, response_url, request_url): return self.stripped_referrer(response_url) @@ -246,8 +248,8 @@ class DefaultReferrerPolicy(NoReferrerWhenDowngradePolicy): with the addition that "Referer" is not sent if the parent request was using ``file://`` or ``s3://`` scheme. """ - NOREFERRER_SCHEMES = LOCAL_SCHEMES + ('file', 's3') - name = POLICY_SCRAPY_DEFAULT + NOREFERRER_SCHEMES: Tuple[str, ...] = LOCAL_SCHEMES + ('file', 's3') + name: str = POLICY_SCRAPY_DEFAULT _policy_classes = {p.name: p for p in ( diff --git a/scrapy/utils/httpobj.py b/scrapy/utils/httpobj.py index c8d4391b1..a90f1d278 100644 --- a/scrapy/utils/httpobj.py +++ b/scrapy/utils/httpobj.py @@ -1,13 +1,16 @@ """Helper functions for scrapy.http objects (Request, Response)""" -import weakref -from urllib.parse import urlparse +from typing import Union +from urllib.parse import urlparse, ParseResult +from weakref import WeakKeyDictionary + +from scrapy.http import Request, Response -_urlparse_cache = weakref.WeakKeyDictionary() +_urlparse_cache: "WeakKeyDictionary[Union[Request, Response], ParseResult]" = WeakKeyDictionary() -def urlparse_cached(request_or_response): +def urlparse_cached(request_or_response: Union[Request, Response]) -> ParseResult: """Return urlparse.urlparse caching the result, where the argument can be a Request or Response object """ diff --git a/scrapy/utils/request.py b/scrapy/utils/request.py index 12c03d78e..66736b42f 100644 --- a/scrapy/utils/request.py +++ b/scrapy/utils/request.py @@ -4,20 +4,27 @@ scrapy.http.Request objects """ import hashlib -import weakref +from typing import Dict, Iterable, Optional, Tuple, Union from urllib.parse import urlunparse +from weakref import WeakKeyDictionary from w3lib.http import basic_auth_header from w3lib.url import canonicalize_url +from scrapy.http import Request from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.python import to_bytes, to_unicode -_fingerprint_cache = weakref.WeakKeyDictionary() +_fingerprint_cache: "WeakKeyDictionary[Request, Dict[Tuple[Optional[Tuple[bytes, ...]], bool], str]]" +_fingerprint_cache = WeakKeyDictionary() -def request_fingerprint(request, include_headers=None, keep_fragments=False): +def request_fingerprint( + request: Request, + include_headers: Optional[Iterable[Union[bytes, str]]] = None, + keep_fragments: bool = False, +): """ Return the request fingerprint. @@ -49,17 +56,18 @@ def request_fingerprint(request, include_headers=None, keep_fragments=False): (for instance when handling requests with a headless browser). """ + headers: Optional[Tuple[bytes, ...]] = None if include_headers: - include_headers = tuple(to_bytes(h.lower()) for h in sorted(include_headers)) + headers = tuple(to_bytes(h.lower()) for h in sorted(include_headers)) cache = _fingerprint_cache.setdefault(request, {}) - cache_key = (include_headers, keep_fragments) + cache_key = (headers, keep_fragments) if cache_key not in cache: fp = hashlib.sha1() fp.update(to_bytes(request.method)) fp.update(to_bytes(canonicalize_url(request.url, keep_fragments=keep_fragments))) fp.update(request.body or b'') - if include_headers: - for hdr in include_headers: + if headers: + for hdr in headers: if hdr in request.headers: fp.update(hdr) for v in request.headers.getlist(hdr): @@ -68,14 +76,14 @@ def request_fingerprint(request, include_headers=None, keep_fragments=False): return cache[cache_key] -def request_authenticate(request, username, password): +def request_authenticate(request: Request, username: str, password: str) -> None: """Autenticate the given request (in place) using the HTTP basic access authentication mechanism (RFC 2617) and the given username and password """ request.headers['Authorization'] = basic_auth_header(username, password) -def request_httprepr(request): +def request_httprepr(request: Request) -> bytes: """Return the raw HTTP representation (as bytes) of the given request. This is provided only for reference since it's not the actual stream of bytes that will be send when performing the request (that's controlled @@ -92,7 +100,7 @@ def request_httprepr(request): return s -def referer_str(request): +def referer_str(request: Request) -> Optional[str]: """ Return Referer HTTP header suitable for logging. """ referrer = request.headers.get('Referer') if referrer is None: diff --git a/scrapy/utils/response.py b/scrapy/utils/response.py index 99b089b6f..b3ef7b463 100644 --- a/scrapy/utils/response.py +++ b/scrapy/utils/response.py @@ -3,19 +3,23 @@ This module provides some useful functions for working with scrapy.http.Response objects """ import os -import weakref import webbrowser import tempfile +from typing import Any, Callable, Iterable, Optional, Tuple, Union +from weakref import WeakKeyDictionary + +import scrapy +from scrapy.http.response import Response from twisted.web import http from scrapy.utils.python import to_bytes, to_unicode from w3lib import html -_baseurl_cache = weakref.WeakKeyDictionary() +_baseurl_cache: "WeakKeyDictionary[Response, str]" = WeakKeyDictionary() -def get_base_url(response): +def get_base_url(response: "scrapy.http.response.text.TextResponse") -> str: """Return the base url of the given response, joined with the response url""" if response not in _baseurl_cache: text = response.text[0:4096] @@ -23,10 +27,13 @@ def get_base_url(response): return _baseurl_cache[response] -_metaref_cache = weakref.WeakKeyDictionary() +_metaref_cache: "WeakKeyDictionary[Response, Union[Tuple[None, None], Tuple[float, str]]]" = WeakKeyDictionary() -def get_meta_refresh(response, ignore_tags=('script', 'noscript')): +def get_meta_refresh( + response: "scrapy.http.response.text.TextResponse", + ignore_tags: Optional[Iterable[str]] = ('script', 'noscript'), +) -> Union[Tuple[None, None], Tuple[float, str]]: """Parse the http-equiv refrsh parameter from the given response""" if response not in _metaref_cache: text = response.text[0:4096] @@ -35,14 +42,15 @@ def get_meta_refresh(response, ignore_tags=('script', 'noscript')): return _metaref_cache[response] -def response_status_message(status): +def response_status_message(status: Union[bytes, float, int, str]) -> str: """Return status code plus status text descriptive message """ - message = http.RESPONSES.get(int(status), "Unknown Status") - return f'{status} {to_unicode(message)}' + status_int = int(status) + message = http.RESPONSES.get(status_int, "Unknown Status") + return f'{status_int} {to_unicode(message)}' -def response_httprepr(response): +def response_httprepr(response: Response) -> bytes: """Return raw HTTP representation (as bytes) of the given response. This is provided only for reference, since it's not the exact stream of bytes that was received (that's not exposed by Twisted). @@ -60,7 +68,10 @@ def response_httprepr(response): return b"".join(values) -def open_in_browser(response, _openfunc=webbrowser.open): +def open_in_browser( + response: Union["scrapy.http.response.html.HtmlResponse", "scrapy.http.response.text.TextResponse"], + _openfunc: Callable[[str], Any] = webbrowser.open, +) -> Any: """Open the given response in a local web browser, populating the tag for external links to work """ diff --git a/scrapy/utils/trackref.py b/scrapy/utils/trackref.py index 3e40acd69..b0c6a2424 100644 --- a/scrapy/utils/trackref.py +++ b/scrapy/utils/trackref.py @@ -9,14 +9,15 @@ and no performance penalty at all when disabled (as object_ref becomes just an alias to object in that case). """ -import weakref -from time import time -from operator import itemgetter from collections import defaultdict +from operator import itemgetter +from time import time +from typing import DefaultDict +from weakref import WeakKeyDictionary NoneType = type(None) -live_refs = defaultdict(weakref.WeakKeyDictionary) +live_refs: DefaultDict[type, WeakKeyDictionary] = defaultdict(WeakKeyDictionary) class object_ref: diff --git a/setup.cfg b/setup.cfg index 0c9f6b963..89b4ec57e 100644 --- a/setup.cfg +++ b/setup.cfg @@ -10,54 +10,18 @@ follow_imports = skip # FIXME: remove the following sections once the issues are solved -[mypy-scrapy] -ignore_errors = True - -[mypy-scrapy.commands] -ignore_errors = True - -[mypy-scrapy.commands.parse] -ignore_errors = True - [mypy-scrapy.downloadermiddlewares.httpproxy] ignore_errors = True -[mypy-scrapy.contracts] -ignore_errors = True - [mypy-scrapy.interfaces] ignore_errors = True -[mypy-scrapy.item] -ignore_errors = True - -[mypy-scrapy.http.cookies] -ignore_errors = True - -[mypy-scrapy.mail] -ignore_errors = True - [mypy-scrapy.pipelines.images] ignore_errors = True [mypy-scrapy.settings.default_settings] ignore_errors = True -[mypy-scrapy.spidermiddlewares.referer] -ignore_errors = True - -[mypy-scrapy.utils.httpobj] -ignore_errors = True - -[mypy-scrapy.utils.request] -ignore_errors = True - -[mypy-scrapy.utils.response] -ignore_errors = True - -[mypy-scrapy.utils.trackref] -ignore_errors = True - [mypy-tests.mocks.dummydbm] ignore_errors = True