Some type hints

scrapy/__init__.py

@@ -22,7 +22,7 @@ __all__ = [
 
 
 # Scrapy and Twisted versions
-__version__ = pkgutil.get_data(__package__, 'VERSION').decode('ascii').strip()
+__version__ = (pkgutil.get_data(__package__, "VERSION") or b"").decode("ascii").strip()
 version_info = tuple(int(v) if v.isdigit() else v for v in __version__.split('.'))
 twisted_version = (_txv.major, _txv.minor, _txv.micro)
 

scrapy/commands/__init__.py

@@ -3,6 +3,8 @@ Base class for Scrapy commands
 """
 import os
 from optparse import OptionGroup
+from typing import Any, Dict
+
 from twisted.python import failure
 
 from scrapy.utils.conf import arglist_to_dict, feed_process_params_from_cli
@@ -15,7 +17,7 @@ class ScrapyCommand:
     crawler_process = None
 
     # default settings to be used for this command instead of global defaults
-    default_settings = {}
+    default_settings: Dict[str, Any] = {}
 
     exitcode = 0
 

scrapy/commands/parse.py

@@ -1,5 +1,6 @@
 import json
 import logging
+from typing import Dict
 
 from itemadapter import is_item, ItemAdapter
 from w3lib.url import is_url
@@ -10,6 +11,7 @@ from scrapy.utils import display
 from scrapy.utils.spider import iterate_spider_output, spidercls_for_request
 from scrapy.exceptions import UsageError
 
+
 logger = logging.getLogger(__name__)
 
 
@@ -17,8 +19,8 @@ class Command(BaseRunSpiderCommand):
     requires_project = True
 
     spider = None
-    items = {}
+    items: Dict[int, list] = {}
-    requests = {}
+    requests: Dict[int, list] = {}
 
     first_response = None
 

scrapy/contracts/__init__.py

@@ -1,16 +1,77 @@
-import sys
 import re
+import sys
 from functools import wraps
 from inspect import getmembers
+from typing import Dict
 from unittest import TestCase
 
 from scrapy.http import Request
-from scrapy.utils.spider import iterate_spider_output
 from scrapy.utils.python import get_spec
+from scrapy.utils.spider import iterate_spider_output
+
+
+class Contract:
+    """ Abstract class for contracts """
+    request_cls = None
+
+    def __init__(self, method, *args):
+        self.testcase_pre = _create_testcase(method, f'@{self.name} pre-hook')
+        self.testcase_post = _create_testcase(method, f'@{self.name} post-hook')
+        self.args = args
+
+    def add_pre_hook(self, request, results):
+        if hasattr(self, 'pre_process'):
+            cb = request.callback
+
+            @wraps(cb)
+            def wrapper(response, **cb_kwargs):
+                try:
+                    results.startTest(self.testcase_pre)
+                    self.pre_process(response)
+                    results.stopTest(self.testcase_pre)
+                except AssertionError:
+                    results.addFailure(self.testcase_pre, sys.exc_info())
+                except Exception:
+                    results.addError(self.testcase_pre, sys.exc_info())
+                else:
+                    results.addSuccess(self.testcase_pre)
+                finally:
+                    return list(iterate_spider_output(cb(response, **cb_kwargs)))
+
+            request.callback = wrapper
+
+        return request
+
+    def add_post_hook(self, request, results):
+        if hasattr(self, 'post_process'):
+            cb = request.callback
+
+            @wraps(cb)
+            def wrapper(response, **cb_kwargs):
+                output = list(iterate_spider_output(cb(response, **cb_kwargs)))
+                try:
+                    results.startTest(self.testcase_post)
+                    self.post_process(output)
+                    results.stopTest(self.testcase_post)
+                except AssertionError:
+                    results.addFailure(self.testcase_post, sys.exc_info())
+                except Exception:
+                    results.addError(self.testcase_post, sys.exc_info())
+                else:
+                    results.addSuccess(self.testcase_post)
+                finally:
+                    return output
+
+            request.callback = wrapper
+
+        return request
+
+    def adjust_request_args(self, args):
+        return args
 
 
 class ContractsManager:
-    contracts = {}
+    contracts: Dict[str, Contract] = {}
 
     def __init__(self, contracts):
         for contract in contracts:
@@ -107,66 +168,6 @@ class ContractsManager:
         request.errback = eb_wrapper
 
 
-class Contract:
-    """ Abstract class for contracts """
-    request_cls = None
-
-    def __init__(self, method, *args):
-        self.testcase_pre = _create_testcase(method, f'@{self.name} pre-hook')
-        self.testcase_post = _create_testcase(method, f'@{self.name} post-hook')
-        self.args = args
-
-    def add_pre_hook(self, request, results):
-        if hasattr(self, 'pre_process'):
-            cb = request.callback
-
-            @wraps(cb)
-            def wrapper(response, **cb_kwargs):
-                try:
-                    results.startTest(self.testcase_pre)
-                    self.pre_process(response)
-                    results.stopTest(self.testcase_pre)
-                except AssertionError:
-                    results.addFailure(self.testcase_pre, sys.exc_info())
-                except Exception:
-                    results.addError(self.testcase_pre, sys.exc_info())
-                else:
-                    results.addSuccess(self.testcase_pre)
-                finally:
-                    return list(iterate_spider_output(cb(response, **cb_kwargs)))
-
-            request.callback = wrapper
-
-        return request
-
-    def add_post_hook(self, request, results):
-        if hasattr(self, 'post_process'):
-            cb = request.callback
-
-            @wraps(cb)
-            def wrapper(response, **cb_kwargs):
-                output = list(iterate_spider_output(cb(response, **cb_kwargs)))
-                try:
-                    results.startTest(self.testcase_post)
-                    self.post_process(output)
-                    results.stopTest(self.testcase_post)
-                except AssertionError:
-                    results.addFailure(self.testcase_post, sys.exc_info())
-                except Exception:
-                    results.addError(self.testcase_post, sys.exc_info())
-                else:
-                    results.addSuccess(self.testcase_post)
-                finally:
-                    return output
-
-            request.callback = wrapper
-
-        return request
-
-    def adjust_request_args(self, args):
-        return args
-
-
 def _create_testcase(method, desc):
     spider = method.__self__.name
 

scrapy/http/cookies.py

@@ -1,10 +1,16 @@
+import re
 import time
-from http.cookiejar import CookieJar as _CookieJar, DefaultCookiePolicy, IPV4_RE
+from http.cookiejar import CookieJar as _CookieJar, DefaultCookiePolicy
 
 from scrapy.utils.httpobj import urlparse_cached
 from scrapy.utils.python import to_unicode
 
 
+# Defined in the http.cookiejar module, but undocumented:
+# https://github.com/python/cpython/blob/v3.9.0/Lib/http/cookiejar.py#L527
+IPV4_RE = re.compile(r"\.\d+$", re.ASCII)
+
+
 class CookieJar:
     def __init__(self, policy=None, check_expired_frequency=10000):
         self.policy = policy or DefaultCookiePolicy()

scrapy/item.py

@@ -8,6 +8,7 @@ from abc import ABCMeta
 from collections.abc import MutableMapping
 from copy import deepcopy
 from pprint import pformat
+from typing import Dict
 from warnings import warn
 
 from scrapy.utils.deprecate import ScrapyDeprecationWarning
@@ -75,7 +76,7 @@ class ItemMeta(_BaseItemMeta):
 
 class DictItem(MutableMapping, BaseItem):
 
-    fields = {}
+    fields: Dict[str, Field] = {}
 
     def __new__(cls, *args, **kwargs):
         if issubclass(cls, DictItem) and not issubclass(cls, Item):

scrapy/mail.py

@@ -9,7 +9,7 @@ from email.mime.base import MIMEBase
 from email.mime.multipart import MIMEMultipart
 from email.mime.nonmultipart import MIMENonMultipart
 from email.mime.text import MIMEText
-from email.utils import COMMASPACE, formatdate
+from email.utils import formatdate
 from io import BytesIO
 
 from twisted.internet import defer, ssl
@@ -21,6 +21,11 @@ from scrapy.utils.python import to_bytes
 logger = logging.getLogger(__name__)
 
 
+# Defined in the email.utils module, but undocumented:
+# https://github.com/python/cpython/blob/v3.9.0/Lib/email/utils.py#L42
+COMMASPACE = ", "
+
+
 def _to_bytes_or_none(text):
     if text is None:
         return None

scrapy/spidermiddlewares/referer.py

@@ -3,15 +3,16 @@ RefererMiddleware: populates Request referer field, based on the Response which
 originated it.
 """
 import warnings
+from typing import Tuple
 from urllib.parse import urlparse
 
 from w3lib.url import safe_url_string
 
-from scrapy.http import Request, Response
-from scrapy.exceptions import NotConfigured
 from scrapy import signals
-from scrapy.utils.python import to_unicode
+from scrapy.exceptions import NotConfigured
+from scrapy.http import Request, Response
 from scrapy.utils.misc import load_object
+from scrapy.utils.python import to_unicode
 from scrapy.utils.url import strip_url
 
 
@@ -30,7 +31,8 @@ POLICY_SCRAPY_DEFAULT = "scrapy-default"
 
 class ReferrerPolicy:
 
-    NOREFERRER_SCHEMES = LOCAL_SCHEMES
+    NOREFERRER_SCHEMES: Tuple[str, ...] = LOCAL_SCHEMES
+    name: str
 
     def referrer(self, response_url, request_url):
         raise NotImplementedError()
@@ -88,7 +90,7 @@ class NoReferrerPolicy(ReferrerPolicy):
     is to be sent along with requests made from a particular request client to any origin.
     The header will be omitted entirely.
     """
-    name = POLICY_NO_REFERRER
+    name: str = POLICY_NO_REFERRER
 
     def referrer(self, response_url, request_url):
         return None
@@ -108,7 +110,7 @@ class NoReferrerWhenDowngradePolicy(ReferrerPolicy):
 
     This is a user agent's default behavior, if no policy is otherwise specified.
     """
-    name = POLICY_NO_REFERRER_WHEN_DOWNGRADE
+    name: str = POLICY_NO_REFERRER_WHEN_DOWNGRADE
 
     def referrer(self, response_url, request_url):
         if not self.tls_protected(response_url) or self.tls_protected(request_url):
@@ -125,7 +127,7 @@ class SameOriginPolicy(ReferrerPolicy):
     Cross-origin requests, on the other hand, will contain no referrer information.
     A Referer HTTP header will not be sent.
     """
-    name = POLICY_SAME_ORIGIN
+    name: str = POLICY_SAME_ORIGIN
 
     def referrer(self, response_url, request_url):
         if self.origin(response_url) == self.origin(request_url):
@@ -141,7 +143,7 @@ class OriginPolicy(ReferrerPolicy):
     when making both same-origin requests and cross-origin requests
     from a particular request client.
     """
-    name = POLICY_ORIGIN
+    name: str = POLICY_ORIGIN
 
     def referrer(self, response_url, request_url):
         return self.origin_referrer(response_url)
@@ -160,7 +162,7 @@ class StrictOriginPolicy(ReferrerPolicy):
     on the other hand, will contain no referrer information.
     A Referer HTTP header will not be sent.
     """
-    name = POLICY_STRICT_ORIGIN
+    name: str = POLICY_STRICT_ORIGIN
 
     def referrer(self, response_url, request_url):
         if (
@@ -181,7 +183,7 @@ class OriginWhenCrossOriginPolicy(ReferrerPolicy):
     is sent as referrer information when making cross-origin requests
     from a particular request client.
     """
-    name = POLICY_ORIGIN_WHEN_CROSS_ORIGIN
+    name: str = POLICY_ORIGIN_WHEN_CROSS_ORIGIN
 
     def referrer(self, response_url, request_url):
         origin = self.origin(response_url)
@@ -208,7 +210,7 @@ class StrictOriginWhenCrossOriginPolicy(ReferrerPolicy):
     on the other hand, will contain no referrer information.
     A Referer HTTP header will not be sent.
     """
-    name = POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN
+    name: str = POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN
 
     def referrer(self, response_url, request_url):
         origin = self.origin(response_url)
@@ -234,7 +236,7 @@ class UnsafeUrlPolicy(ReferrerPolicy):
     to insecure origins.
     Carefully consider the impact of setting such a policy for potentially sensitive documents.
     """
-    name = POLICY_UNSAFE_URL
+    name: str = POLICY_UNSAFE_URL
 
     def referrer(self, response_url, request_url):
         return self.stripped_referrer(response_url)
@@ -246,8 +248,8 @@ class DefaultReferrerPolicy(NoReferrerWhenDowngradePolicy):
     with the addition that "Referer" is not sent if the parent request was
     using ``file://`` or ``s3://`` scheme.
     """
-    NOREFERRER_SCHEMES = LOCAL_SCHEMES + ('file', 's3')
+    NOREFERRER_SCHEMES: Tuple[str, ...] = LOCAL_SCHEMES + ('file', 's3')
-    name = POLICY_SCRAPY_DEFAULT
+    name: str = POLICY_SCRAPY_DEFAULT
 
 
 _policy_classes = {p.name: p for p in (

scrapy/utils/httpobj.py

@@ -1,13 +1,16 @@
 """Helper functions for scrapy.http objects (Request, Response)"""
 
-import weakref
+from typing import Union
-from urllib.parse import urlparse
+from urllib.parse import urlparse, ParseResult
+from weakref import WeakKeyDictionary
+
+from scrapy.http import Request, Response
 
 
-_urlparse_cache = weakref.WeakKeyDictionary()
+_urlparse_cache: "WeakKeyDictionary[Union[Request, Response], ParseResult]" = WeakKeyDictionary()
 
 
-def urlparse_cached(request_or_response):
+def urlparse_cached(request_or_response: Union[Request, Response]) -> ParseResult:
     """Return urlparse.urlparse caching the result, where the argument can be a
     Request or Response object
     """

scrapy/utils/request.py

@@ -4,20 +4,27 @@ scrapy.http.Request objects
 """
 
 import hashlib
-import weakref
+from typing import Dict, Iterable, Optional, Tuple, Union
 from urllib.parse import urlunparse
+from weakref import WeakKeyDictionary
 
 from w3lib.http import basic_auth_header
 from w3lib.url import canonicalize_url
 
+from scrapy.http import Request
 from scrapy.utils.httpobj import urlparse_cached
 from scrapy.utils.python import to_bytes, to_unicode
 
 
-_fingerprint_cache = weakref.WeakKeyDictionary()
+_fingerprint_cache: "WeakKeyDictionary[Request, Dict[Tuple[Optional[Tuple[bytes, ...]], bool], str]]"
+_fingerprint_cache = WeakKeyDictionary()
 
 
-def request_fingerprint(request, include_headers=None, keep_fragments=False):
+def request_fingerprint(
+    request: Request,
+    include_headers: Optional[Iterable[Union[bytes, str]]] = None,
+    keep_fragments: bool = False,
+):
     """
     Return the request fingerprint.
 
@@ -49,17 +56,18 @@ def request_fingerprint(request, include_headers=None, keep_fragments=False):
     (for instance when handling requests with a headless browser).
 
     """
+    headers: Optional[Tuple[bytes, ...]] = None
     if include_headers:
-        include_headers = tuple(to_bytes(h.lower()) for h in sorted(include_headers))
+        headers = tuple(to_bytes(h.lower()) for h in sorted(include_headers))
     cache = _fingerprint_cache.setdefault(request, {})
-    cache_key = (include_headers, keep_fragments)
+    cache_key = (headers, keep_fragments)
     if cache_key not in cache:
         fp = hashlib.sha1()
         fp.update(to_bytes(request.method))
         fp.update(to_bytes(canonicalize_url(request.url, keep_fragments=keep_fragments)))
         fp.update(request.body or b'')
-        if include_headers:
+        if headers:
-            for hdr in include_headers:
+            for hdr in headers:
                 if hdr in request.headers:
                     fp.update(hdr)
                     for v in request.headers.getlist(hdr):
@@ -68,14 +76,14 @@ def request_fingerprint(request, include_headers=None, keep_fragments=False):
     return cache[cache_key]
 
 
-def request_authenticate(request, username, password):
+def request_authenticate(request: Request, username: str, password: str) -> None:
     """Autenticate the given request (in place) using the HTTP basic access
     authentication mechanism (RFC 2617) and the given username and password
     """
     request.headers['Authorization'] = basic_auth_header(username, password)
 
 
-def request_httprepr(request):
+def request_httprepr(request: Request) -> bytes:
     """Return the raw HTTP representation (as bytes) of the given request.
     This is provided only for reference since it's not the actual stream of
     bytes that will be send when performing the request (that's controlled
@@ -92,7 +100,7 @@ def request_httprepr(request):
     return s
 
 
-def referer_str(request):
+def referer_str(request: Request) -> Optional[str]:
     """ Return Referer HTTP header suitable for logging. """
     referrer = request.headers.get('Referer')
     if referrer is None:

scrapy/utils/response.py

@@ -3,19 +3,23 @@ This module provides some useful functions for working with
 scrapy.http.Response objects
 """
 import os
-import weakref
 import webbrowser
 import tempfile
+from typing import Any, Callable, Iterable, Optional, Tuple, Union
+from weakref import WeakKeyDictionary
+
+import scrapy
+from scrapy.http.response import Response
 
 from twisted.web import http
 from scrapy.utils.python import to_bytes, to_unicode
 from w3lib import html
 
 
-_baseurl_cache = weakref.WeakKeyDictionary()
+_baseurl_cache: "WeakKeyDictionary[Response, str]" = WeakKeyDictionary()
 
 
-def get_base_url(response):
+def get_base_url(response: "scrapy.http.response.text.TextResponse") -> str:
     """Return the base url of the given response, joined with the response url"""
     if response not in _baseurl_cache:
         text = response.text[0:4096]
@@ -23,10 +27,13 @@ def get_base_url(response):
     return _baseurl_cache[response]
 
 
-_metaref_cache = weakref.WeakKeyDictionary()
+_metaref_cache: "WeakKeyDictionary[Response, Union[Tuple[None, None], Tuple[float, str]]]" = WeakKeyDictionary()
 
 
-def get_meta_refresh(response, ignore_tags=('script', 'noscript')):
+def get_meta_refresh(
+    response: "scrapy.http.response.text.TextResponse",
+    ignore_tags: Optional[Iterable[str]] = ('script', 'noscript'),
+) -> Union[Tuple[None, None], Tuple[float, str]]:
     """Parse the http-equiv refrsh parameter from the given response"""
     if response not in _metaref_cache:
         text = response.text[0:4096]
@@ -35,14 +42,15 @@ def get_meta_refresh(response, ignore_tags=('script', 'noscript')):
     return _metaref_cache[response]
 
 
-def response_status_message(status):
+def response_status_message(status: Union[bytes, float, int, str]) -> str:
     """Return status code plus status text descriptive message
     """
-    message = http.RESPONSES.get(int(status), "Unknown Status")
+    status_int = int(status)
-    return f'{status} {to_unicode(message)}'
+    message = http.RESPONSES.get(status_int, "Unknown Status")
+    return f'{status_int} {to_unicode(message)}'
 
 
-def response_httprepr(response):
+def response_httprepr(response: Response) -> bytes:
     """Return raw HTTP representation (as bytes) of the given response. This
     is provided only for reference, since it's not the exact stream of bytes
     that was received (that's not exposed by Twisted).
@@ -60,7 +68,10 @@ def response_httprepr(response):
     return b"".join(values)
 
 
-def open_in_browser(response, _openfunc=webbrowser.open):
+def open_in_browser(
+    response: Union["scrapy.http.response.html.HtmlResponse", "scrapy.http.response.text.TextResponse"],
+    _openfunc: Callable[[str], Any] = webbrowser.open,
+) -> Any:
     """Open the given response in a local web browser, populating the <base>
     tag for external links to work
     """

scrapy/utils/trackref.py

@@ -9,14 +9,15 @@ and no performance penalty at all when disabled (as object_ref becomes just an
 alias to object in that case).
 """
 
-import weakref
-from time import time
-from operator import itemgetter
 from collections import defaultdict
+from operator import itemgetter
+from time import time
+from typing import DefaultDict
+from weakref import WeakKeyDictionary
 
 
 NoneType = type(None)
-live_refs = defaultdict(weakref.WeakKeyDictionary)
+live_refs: DefaultDict[type, WeakKeyDictionary] = defaultdict(WeakKeyDictionary)
 
 
 class object_ref:

setup.cfg

@@ -10,54 +10,18 @@ follow_imports = skip
 
 # FIXME: remove the following sections once the issues are solved
 
-[mypy-scrapy]
-ignore_errors = True
-
-[mypy-scrapy.commands]
-ignore_errors = True
-
-[mypy-scrapy.commands.parse]
-ignore_errors = True
-
 [mypy-scrapy.downloadermiddlewares.httpproxy]
 ignore_errors = True
 
-[mypy-scrapy.contracts]
-ignore_errors = True
-
 [mypy-scrapy.interfaces]
 ignore_errors = True
 
-[mypy-scrapy.item]
-ignore_errors = True
-
-[mypy-scrapy.http.cookies]
-ignore_errors = True
-
-[mypy-scrapy.mail]
-ignore_errors = True
-
 [mypy-scrapy.pipelines.images]
 ignore_errors = True
 
 [mypy-scrapy.settings.default_settings]
 ignore_errors = True
 
-[mypy-scrapy.spidermiddlewares.referer]
-ignore_errors = True
-
-[mypy-scrapy.utils.httpobj]
-ignore_errors = True
-
-[mypy-scrapy.utils.request]
-ignore_errors = True
-
-[mypy-scrapy.utils.response]
-ignore_errors = True
-
-[mypy-scrapy.utils.trackref]
-ignore_errors = True
-
 [mypy-tests.mocks.dummydbm]
 ignore_errors = True