Merge pull request #5805 from scrapy/mypy-imports

Handle typing issues hidden by follow_imports=skip

.github/workflows/checks.yml

@@ -11,7 +11,7 @@ jobs:
         - python-version: "3.11"
           env:
             TOXENV: pylint
-        - python-version: 3.7
+        - python-version: 3.8
           env:
             TOXENV: typing
         - python-version: "3.11"  # Keep in sync with .readthedocs.yml

scrapy/core/downloader/__init__.py

@@ -94,7 +94,7 @@ class Downloader:
         dfd = self.middleware.download(self._enqueue_request, request, spider)
         return dfd.addBoth(_deactivate)
 
-    def needs_backout(self):
+    def needs_backout(self) -> bool:
         return len(self.active) >= self.total_concurrency
 
     def _get_slot(self, request, spider):

scrapy/core/engine.py

@@ -7,13 +7,24 @@ For more information see docs/topics/architecture.rst
 import logging
 import warnings
 from time import time
-from typing import Callable, Iterable, Iterator, Optional, Set, Union
+from typing import (
+    Any,
+    Callable,
+    Generator,
+    Iterable,
+    Iterator,
+    Optional,
+    Set,
+    Union,
+    cast,
+)
 
 from twisted.internet.defer import Deferred, inlineCallbacks, succeed
 from twisted.internet.task import LoopingCall
 from twisted.python.failure import Failure
 
 from scrapy import signals
+from scrapy.core.downloader import Downloader
 from scrapy.core.scraper import Scraper
 from scrapy.exceptions import CloseSpider, DontCloseSpider, ScrapyDeprecationWarning
 from scrapy.http import Request, Response
@@ -75,7 +86,7 @@ class ExecutionEngine:
         self.paused = False
         self.scheduler_cls = self._get_scheduler_class(crawler.settings)
         downloader_cls = load_object(self.settings["DOWNLOADER"])
-        self.downloader = downloader_cls(crawler)
+        self.downloader: Downloader = downloader_cls(crawler)
         self.scraper = Scraper(crawler)
         self._spider_closed_callback = spider_closed_callback
 
@@ -91,20 +102,20 @@ class ExecutionEngine:
         return scheduler_cls
 
     @inlineCallbacks
-    def start(self) -> Deferred:
+    def start(self) -> Generator[Deferred, Any, None]:
         if self.running:
             raise RuntimeError("Engine already running")
         self.start_time = time()
         yield self.signals.send_catch_log_deferred(signal=signals.engine_started)
         self.running = True
-        self._closewait = Deferred()
+        self._closewait: Deferred = Deferred()
         yield self._closewait
 
     def stop(self) -> Deferred:
         """Gracefully stop the execution engine"""
 
         @inlineCallbacks
-        def _finish_stopping_engine(_) -> Deferred:
+        def _finish_stopping_engine(_) -> Generator[Deferred, Any, None]:
             yield self.signals.send_catch_log_deferred(signal=signals.engine_stopped)
             self._closewait.callback(None)
 
@@ -172,11 +183,13 @@ class ExecutionEngine:
             self._spider_idle()
 
     def _needs_backout(self) -> bool:
+        assert self.slot is not None  # typing
+        assert self.scraper.slot is not None  # typing
         return (
             not self.running
-            or self.slot.closing  # type: ignore[union-attr]
+            or bool(self.slot.closing)
             or self.downloader.needs_backout()
-            or self.scraper.slot.needs_backout()  # type: ignore[union-attr]
+            or self.scraper.slot.needs_backout()
         )
 
     def _next_request_from_scheduler(self) -> Optional[Deferred]:
@@ -196,7 +209,7 @@ class ExecutionEngine:
                 extra={"spider": self.spider},
             )
         )
-        d.addBoth(lambda _: self.slot.remove_request(request))
+        d.addBoth(lambda _: cast(Slot, self.slot).remove_request(request))
         d.addErrback(
             lambda f: logger.info(
                 "Error while removing request from slot",
@@ -424,7 +437,7 @@ class ExecutionEngine:
         dfd.addErrback(log_failure("Scraper close failure"))
 
         if hasattr(self.slot.scheduler, "close"):
-            dfd.addBoth(lambda _: self.slot.scheduler.close(reason))
+            dfd.addBoth(lambda _: cast(Slot, self.slot).scheduler.close(reason))
             dfd.addErrback(log_failure("Scheduler close failure"))
 
         dfd.addBoth(

scrapy/core/http2/agent.py

@@ -6,7 +6,12 @@ from twisted.internet.base import ReactorBase
 from twisted.internet.defer import Deferred
 from twisted.internet.endpoints import HostnameEndpoint
 from twisted.python.failure import Failure
-from twisted.web.client import URI, BrowserLikePolicyForHTTPS, _StandardEndpointFactory
+from twisted.web.client import (
+    URI,
+    BrowserLikePolicyForHTTPS,
+    ResponseFailed,
+    _StandardEndpointFactory,
+)
 from twisted.web.error import SchemeNotSupported
 
 from scrapy.core.downloader.contextfactory import AcceptableProtocolsContextFactory
@@ -35,7 +40,7 @@ class H2ConnectionPool:
             # Received a request while connecting to remote
             # Create a deferred which will fire with the H2ClientProtocol
             # instance
-            d = Deferred()
+            d: Deferred = Deferred()
             self._pending_requests[key].append(d)
             return d
 
@@ -53,14 +58,14 @@ class H2ConnectionPool:
     ) -> Deferred:
         self._pending_requests[key] = deque()
 
-        conn_lost_deferred = Deferred()
+        conn_lost_deferred: Deferred = Deferred()
         conn_lost_deferred.addCallback(self._remove_connection, key)
 
         factory = H2ClientFactory(uri, self.settings, conn_lost_deferred)
         conn_d = endpoint.connect(factory)
         conn_d.addCallback(self.put_connection, key)
 
-        d = Deferred()
+        d: Deferred = Deferred()
         self._pending_requests[key].append(d)
         return d
 
@@ -83,7 +88,7 @@ class H2ConnectionPool:
         pending_requests = self._pending_requests.pop(key, None)
         while pending_requests:
             d = pending_requests.popleft()
-            d.errback(errors)
+            d.errback(ResponseFailed(errors))
 
     def close_connections(self) -> None:
         """Close all the HTTP/2 connections and remove them from pool
@@ -92,6 +97,7 @@ class H2ConnectionPool:
             Deferred that fires when all connections have been closed
         """
         for conn in self._connections.values():
+            assert conn.transport is not None  # typing
             conn.transport.abortConnection()
 
 

scrapy/core/http2/protocol.py

@@ -140,6 +140,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin):
         This is used while initiating pending streams to make sure
         that we initiate stream only during active HTTP/2 Connection
         """
+        assert self.transport is not None  # typing
         return bool(self.transport.connected) and self.metadata["settings_acknowledged"]
 
     @property
@@ -196,6 +197,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin):
         """Write data to the underlying transport connection
         from the HTTP2 connection instance if any
         """
+        assert self.transport is not None  # typing
         # Reset the idle timeout as connection is still actively sending data
         self.resetTimeout()
 
@@ -226,6 +228,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin):
         # Initialize the timeout
         self.setTimeout(self.IDLE_TIMEOUT)
 
+        assert self.transport is not None  # typing
         destination = self.transport.getPeer()
         self.metadata["ip_address"] = ipaddress.ip_address(destination.host)
 
@@ -237,12 +240,14 @@ class H2ClientProtocol(Protocol, TimeoutMixin):
         """Helper function to lose the connection with the error sent as a
         reason"""
         self._conn_lost_errors += errors
+        assert self.transport is not None  # typing
         self.transport.loseConnection()
 
     def handshakeCompleted(self) -> None:
         """
         Close the connection if it's not made via the expected protocol
         """
+        assert self.transport is not None  # typing
         if (
             self.transport.negotiatedProtocol is not None
             and self.transport.negotiatedProtocol != PROTOCOL_NAME
@@ -275,6 +280,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin):
                 # hyper-h2 does not drop the connection in this scenario, we
                 # need to abort the connection manually.
                 self._conn_lost_errors += [e]
+                assert self.transport is not None  # typing
                 self.transport.abortConnection()
                 return
 
@@ -388,6 +394,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin):
         self._send_pending_requests()
 
         # Update certificate when our HTTP/2 connection is established
+        assert self.transport is not None  # typing
         self.metadata["certificate"] = Certificate(self.transport.getPeerCertificate())
 
     def stream_ended(self, event: StreamEnded) -> None:

scrapy/core/http2/stream.py

@@ -151,7 +151,7 @@ class Stream:
             else:
                 self.close(StreamCloseReason.CANCELLED)
 
-        self._deferred_response = Deferred(_cancel)
+        self._deferred_response: Deferred = Deferred(_cancel)
 
     def __repr__(self) -> str:
         return f"Stream(id={self.stream_id!r})"

scrapy/core/scraper.py

@@ -64,7 +64,7 @@ class Slot:
     def add_response_request(
         self, result: Union[Response, Failure], request: Request
     ) -> Deferred:
-        deferred = Deferred()
+        deferred: Deferred = Deferred()
         self.queue.append((result, request, deferred))
         if isinstance(result, Response):
             self.active_size += max(len(result.body), self.MIN_RESPONSE_SIZE)
@@ -203,10 +203,12 @@ class Scraper:
                 callback=callback, callbackKeywords=result.request.cb_kwargs
             )
         else:  # result is a Failure
-            result.request = request
+            # TODO: properly type adding this attribute to a Failure
+            result.request = request  # type: ignore[attr-defined]
             warn_on_generator_with_return_value(spider, request.errback)
             dfd = defer_fail(result)
-            dfd.addErrback(request.errback)
+            if request.errback:
+                dfd.addErrback(request.errback)
         return dfd.addCallback(iterate_spider_output)
 
     def handle_spider_error(

scrapy/core/spidermw.py

@@ -155,7 +155,7 @@ class SpiderMiddlewareManager(MiddlewareManager):
                 # simplified when downgrading is removed.
                 if dfd.called:
                     # the result is available immediately if _process_spider_output didn't do downgrading
-                    return dfd.result
+                    return cast(MutableChain, dfd.result)
                 # we forbid waiting here because otherwise we would need to return a deferred from
                 # _process_spider_exception too, which complicates the architecture
                 msg = f"Async iterable returned from {method.__qualname__} cannot be downgraded"
@@ -180,7 +180,7 @@ class SpiderMiddlewareManager(MiddlewareManager):
         spider: Spider,
         result: Union[Iterable, AsyncIterable],
         start_index: int = 0,
-    ) -> Deferred:
+    ) -> Generator[Deferred, Any, Union[MutableChain, MutableAsyncChain]]:
         # items in this iterable do not need to go through the process_spider_output
         # chain, they went through it already from the process_spider_exception method
         recovered: Union[MutableChain, MutableAsyncChain]

scrapy/http/request/form.py

@@ -5,10 +5,17 @@ This module implements the FormRequest class which is a more convenient class
 See documentation in docs/topics/request-response.rst
 """
 
-from typing import Iterable, List, Optional, Tuple, Type, TypeVar, Union
+from typing import Iterable, List, Optional, Tuple, Type, TypeVar, Union, cast
 from urllib.parse import urlencode, urljoin, urlsplit, urlunsplit
 
-from lxml.html import FormElement, HtmlElement, HTMLParser, SelectElement
+from lxml.html import (
+    FormElement,
+    HTMLParser,
+    InputElement,
+    MultipleSelectOptions,
+    SelectElement,
+    TextareaElement,
+)
 from parsel.selector import create_root_node
 from w3lib.html import strip_html5_whitespace
 
@@ -19,7 +26,8 @@ from scrapy.utils.response import get_base_url
 
 FormRequestTypeVar = TypeVar("FormRequestTypeVar", bound="FormRequest")
 
-FormdataType = Optional[Union[dict, List[Tuple[str, str]]]]
+FormdataKVType = Tuple[str, Union[str, Iterable[str]]]
+FormdataType = Optional[Union[dict, List[FormdataKVType]]]
 
 
 class FormRequest(Request):
@@ -50,7 +58,7 @@ class FormRequest(Request):
         response: TextResponse,
         formname: Optional[str] = None,
         formid: Optional[str] = None,
-        formnumber: Optional[int] = 0,
+        formnumber: int = 0,
         formdata: FormdataType = None,
         clickdata: Optional[dict] = None,
         dont_click: bool = False,
@@ -79,6 +87,7 @@ class FormRequest(Request):
 
 
 def _get_form_url(form: FormElement, url: Optional[str]) -> str:
+    assert form.base_url is not None  # typing
     if url is None:
         action = form.get("action")
         if action is None:
@@ -87,11 +96,11 @@ def _get_form_url(form: FormElement, url: Optional[str]) -> str:
     return urljoin(form.base_url, url)
 
 
-def _urlencode(seq: Iterable, enc: str) -> str:
+def _urlencode(seq: Iterable[FormdataKVType], enc: str) -> str:
     values = [
         (to_bytes(k, enc), to_bytes(v, enc))
         for k, vs in seq
-        for v in (vs if is_listlike(vs) else [vs])
+        for v in (cast(Iterable[str], vs) if is_listlike(vs) else [cast(str, vs)])
     ]
     return urlencode(values, doseq=True)
 
@@ -100,7 +109,7 @@ def _get_form(
     response: TextResponse,
     formname: Optional[str],
     formid: Optional[str],
-    formnumber: Optional[int],
+    formnumber: int,
     formxpath: Optional[str],
 ) -> FormElement:
     """Find the wanted form element within the given response."""
@@ -133,13 +142,12 @@ def _get_form(
         raise ValueError(f"No <form> element found with {formxpath}")
 
     # If we get here, it means that either formname was None or invalid
-    if formnumber is not None:
-        try:
-            form = forms[formnumber]
-        except IndexError:
-            raise IndexError(f"Form number {formnumber} not found in {response}")
-        else:
-            return form
+    try:
+        form = forms[formnumber]
+    except IndexError:
+        raise IndexError(f"Form number {formnumber} not found in {response}")
+    else:
+        return form
 
 
 def _get_inputs(
@@ -147,7 +155,7 @@ def _get_inputs(
     formdata: FormdataType,
     dont_click: bool,
     clickdata: Optional[dict],
-) -> List[Tuple[str, str]]:
+) -> List[FormdataKVType]:
     """Return a list of key-value pairs for the inputs found in the given form."""
     try:
         formdata_keys = dict(formdata or ()).keys()
@@ -165,7 +173,7 @@ def _get_inputs(
         '  not(re:test(., "^(?:checkbox|radio)$", "i")))]]',
         namespaces={"re": "http://exslt.org/regular-expressions"},
     )
-    values = [
+    values: List[FormdataKVType] = [
         (k, "" if v is None else v)
         for k, v in (_value(e) for e in inputs)
         if k and k not in formdata_keys
@@ -183,27 +191,25 @@ def _get_inputs(
     return values
 
 
-def _value(ele: HtmlElement):
+def _value(
+    ele: Union[InputElement, SelectElement, TextareaElement]
+) -> Tuple[Optional[str], Union[None, str, MultipleSelectOptions]]:
     n = ele.name
     v = ele.value
     if ele.tag == "select":
-        return _select_value(ele, n, v)
+        return _select_value(cast(SelectElement, ele), n, v)
     return n, v
 
 
-def _select_value(ele: SelectElement, n: str, v: str):
+def _select_value(
+    ele: SelectElement, n: Optional[str], v: Union[None, str, MultipleSelectOptions]
+) -> Tuple[Optional[str], Union[None, str, MultipleSelectOptions]]:
     multiple = ele.multiple
     if v is None and not multiple:
         # Match browser behaviour on simple select tag without options selected
         # And for select tags without options
         o = ele.value_options
         return (n, o[0]) if o else (None, None)
-    if v is not None and multiple:
-        # This is a workaround to bug in lxml fixed 2.3.1
-        # fix https://github.com/lxml/lxml/commit/57f49eed82068a20da3db8f1b18ae00c1bab8b12#L1L1139
-        selected_options = ele.xpath(".//option[@selected]")
-        values = [(o.get("value") or o.text or "").strip() for o in selected_options]
-        return n, values
     return n, v
 
 

scrapy/utils/defer.py

@@ -17,6 +17,7 @@ from typing import (
     List,
     Optional,
     Union,
+    cast,
 )
 
 from twisted.internet import defer
@@ -38,7 +39,7 @@ def defer_fail(_failure: Failure) -> Deferred:
     """
     from twisted.internet import reactor
 
-    d = Deferred()
+    d: Deferred = Deferred()
     reactor.callLater(0.1, d.errback, _failure)
     return d
 
@@ -52,7 +53,7 @@ def defer_succeed(result) -> Deferred:
     """
     from twisted.internet import reactor
 
-    d = Deferred()
+    d: Deferred = Deferred()
     reactor.callLater(0.1, d.callback, result)
     return d
 
@@ -84,7 +85,7 @@ def mustbe_deferred(f: Callable, *args, **kw) -> Deferred:
 
 def parallel(
     iterable: Iterable, count: int, callable: Callable, *args, **named
-) -> DeferredList:
+) -> Deferred:
     """Execute a callable over the objects in the given iterable, in parallel,
     using no more than ``count`` concurrent calls.
 
@@ -182,7 +183,9 @@ class _AsyncCooperatorAdapter(Iterator):
     def _call_anext(self) -> None:
         # This starts waiting for the next result from aiterator.
         # If aiterator is exhausted, _errback will be called.
-        self.anext_deferred = deferred_from_coro(self.aiterator.__anext__())
+        self.anext_deferred = cast(
+            Deferred, deferred_from_coro(self.aiterator.__anext__())
+        )
         self.anext_deferred.addCallbacks(self._callback, self._errback)
 
     def __next__(self) -> Deferred:
@@ -190,7 +193,7 @@ class _AsyncCooperatorAdapter(Iterator):
         # It also calls __anext__() if needed.
         if self.finished:
             raise StopIteration
-        d = Deferred()
+        d: Deferred = Deferred()
         self.waiting_deferreds.append(d)
         if not self.anext_deferred:
             self._call_anext()
@@ -199,17 +202,17 @@ class _AsyncCooperatorAdapter(Iterator):
 
 def parallel_async(
     async_iterable: AsyncIterable, count: int, callable: Callable, *args, **named
-) -> DeferredList:
+) -> Deferred:
     """Like parallel but for async iterators"""
     coop = Cooperator()
     work = _AsyncCooperatorAdapter(async_iterable, callable, *args, **named)
-    dl = DeferredList([coop.coiterate(work) for _ in range(count)])
+    dl: Deferred = DeferredList([coop.coiterate(work) for _ in range(count)])
     return dl
 
 
 def process_chain(callbacks: Iterable[Callable], input, *a, **kw) -> Deferred:
     """Return a Deferred built by chaining the given callbacks"""
-    d = Deferred()
+    d: Deferred = Deferred()
     for x in callbacks:
         d.addCallback(x, *a, **kw)
     d.callback(input)
@@ -220,7 +223,7 @@ def process_chain_both(
     callbacks: Iterable[Callable], errbacks: Iterable[Callable], input, *a, **kw
 ) -> Deferred:
     """Return a Deferred built by chaining the given callbacks and errbacks"""
-    d = Deferred()
+    d: Deferred = Deferred()
     for cb, eb in zip(callbacks, errbacks):
         d.addCallbacks(
             callback=cb,
@@ -242,7 +245,7 @@ def process_parallel(callbacks: Iterable[Callable], input, *a, **kw) -> Deferred
     callbacks
     """
     dfds = [defer.succeed(input).addCallback(x, *a, **kw) for x in callbacks]
-    d = DeferredList(dfds, fireOnOneErrback=True, consumeErrors=True)
+    d: Deferred = DeferredList(dfds, fireOnOneErrback=True, consumeErrors=True)
     d.addCallbacks(lambda r: [x[1] for x in r], lambda f: f.value.subFailure)
     return d
 
@@ -285,7 +288,7 @@ def deferred_from_coro(o) -> Any:
         if not is_asyncio_reactor_installed():
             # wrapping the coroutine directly into a Deferred, this doesn't work correctly with coroutines
             # that use asyncio, e.g. "await asyncio.sleep(1)"
-            return ensureDeferred(o)
+            return ensureDeferred(cast(Coroutine[Deferred, Any, Any], o))
         # wrapping the coroutine into a Future and then into a Deferred, this requires AsyncioSelectorReactor
         event_loop = _get_asyncio_event_loop()
         return Deferred.fromFuture(asyncio.ensure_future(o, loop=event_loop))

scrapy/utils/python.py

@@ -8,7 +8,7 @@ import sys
 import weakref
 from functools import partial, wraps
 from itertools import chain
-from typing import AsyncGenerator, AsyncIterable, Iterable, Union
+from typing import Any, AsyncGenerator, AsyncIterable, Iterable, Union
 
 from scrapy.utils.asyncgen import as_async_generator
 
@@ -45,7 +45,7 @@ def iflatten(x):
             yield el
 
 
-def is_listlike(x):
+def is_listlike(x: Any) -> bool:
     """
     >>> is_listlike("foo")
     False

scrapy/utils/response.py

@@ -6,7 +6,7 @@ import os
 import re
 import tempfile
 import webbrowser
-from typing import Any, Callable, Iterable, Optional, Tuple, Union
+from typing import Any, Callable, Iterable, Tuple, Union
 from weakref import WeakKeyDictionary
 
 from twisted.web import http
@@ -37,12 +37,13 @@ _metaref_cache: "WeakKeyDictionary[Response, Union[Tuple[None, None], Tuple[floa
 
 def get_meta_refresh(
     response: "scrapy.http.response.text.TextResponse",
-    ignore_tags: Optional[Iterable[str]] = ("script", "noscript"),
+    ignore_tags: Iterable[str] = ("script", "noscript"),
 ) -> Union[Tuple[None, None], Tuple[float, str]]:
     """Parse the http-equiv refresh parameter from the given response"""
     if response not in _metaref_cache:
         text = response.text[0:4096]
-        _metaref_cache[response] = html.get_meta_refresh(
+        # a w3lib typing bug here, fixed in https://github.com/scrapy/w3lib/pull/211
+        _metaref_cache[response] = html.get_meta_refresh(  # type: ignore[assignment]
             text, response.url, response.encoding, ignore_tags=ignore_tags
         )
     return _metaref_cache[response]

setup.cfg

@@ -6,6 +6,12 @@ universal=1
 
 [mypy]
 ignore_missing_imports = true
+
+# Interface classes are hard to support
+[mypy-twisted.internet.interfaces]
+follow_imports = skip
+
+[mypy-twisted.internet.reactor]
 follow_imports = skip
 
 # FIXME: remove the following sections once the issues are solved

tests/mockserver.py

@@ -10,6 +10,7 @@ from urllib.parse import urlencode
 
 from OpenSSL import SSL
 from twisted.internet import defer, reactor, ssl
+from twisted.internet.protocol import ServerFactory
 from twisted.internet.task import deferLater
 from twisted.names import dns, error
 from twisted.names.server import DNSServerFactory
@@ -368,6 +369,8 @@ if __name__ == "__main__":
     )
     args = parser.parse_args()
 
+    factory: ServerFactory
+
     if args.type == "http":
         root = Root()
         factory = Site(root)

tests/test_dependencies.py

@@ -37,7 +37,3 @@ class ScrapyUtilsTest(unittest.TestCase):
         pinned_twisted_version_string = match[1]
 
         self.assertEqual(twisted_version.short(), pinned_twisted_version_string)
-
-
-if __name__ == "__main__":
-    unittest.main()

tests/test_downloadermiddleware_httpproxy.py

@@ -13,7 +13,7 @@ spider = Spider("foo")
 
 
 class TestHttpProxyMiddleware(TestCase):
-    failureException = AssertionError
+    failureException = AssertionError  # type: ignore[assignment]
 
     def setUp(self):
         self._oldenv = os.environ.copy()

tests/test_http2_client_protocol.py

@@ -5,6 +5,7 @@ import shutil
 import string
 from ipaddress import IPv4Address
 from pathlib import Path
+from typing import Dict
 from unittest import mock, skipIf
 from urllib.parse import urlencode
 
@@ -87,6 +88,7 @@ class GetDataHtmlLarge(LeafResource):
 class PostDataJsonMixin:
     @staticmethod
     def make_response(request: TxRequest, extra_data: str):
+        assert request.content is not None
         response = {
             "request-headers": {},
             "request-body": json.loads(request.content.read()),
@@ -145,7 +147,8 @@ class QueryParams(LeafResource):
         request.setHeader("Content-Type", "application/json; charset=UTF-8")
         request.setHeader("Content-Encoding", "UTF-8")
 
-        query_params = {}
+        query_params: Dict[str, str] = {}
+        assert request.args is not None
         for k, v in request.args.items():
             query_params[str(k, "utf-8")] = str(v[0], "utf-8")
 

tests/test_spidermiddleware.py

@@ -112,6 +112,7 @@ class BaseAsyncSpiderMiddlewareTestCase(SpiderMiddlewareTestCase):
     Should work for process_spider_output and, when it's supported, process_start_requests.
     """
 
+    ITEM_TYPE: type
     RESULT_COUNT = 3  # to simplify checks, let everything return 3 objects
 
     @staticmethod

tests/test_utils_iterators.py

@@ -527,7 +527,3 @@ class TestHelper(unittest.TestCase):
             type(a) is type(b), f"Got {type(a)}, expected {type(b)} for { obj!r}"
         )
         self.assertEqual(a, b)
-
-
-if __name__ == "__main__":
-    unittest.main()

tests/test_utils_python.py

@@ -257,7 +257,3 @@ class UtilsPythonTestCase(unittest.TestCase):
             without_none_values({"one": 1, "none": None, "three": 3, "four": 4}),
             {"one": 1, "three": 3, "four": 4},
         )
-
-
-if __name__ == "__main__":
-    unittest.main()

tox.ini

@@ -37,11 +37,13 @@ install_command =
 [testenv:typing]
 basepython = python3
 deps =
-    lxml-stubs==0.4.0
-    mypy==0.991
+    mypy==1.0.1
     types-attrs==19.1.0
-    types-pyOpenSSL==23.0.0.2
-    types-setuptools==65.7.0.3
+    types-lxml==2023.2.11
+    types-Pillow==9.4.0.16
+    types-Pygments==2.14.0.5
+    types-pyOpenSSL==23.0.0.4
+    types-setuptools==67.4.0.1
 commands =
     mypy --show-error-codes {posargs: scrapy tests}