Bump mypy, flake8, and pylint (#5738)

scrapy/commands/__init__.py

@@ -23,7 +23,7 @@ class ScrapyCommand:
 
     exitcode = 0
 
-    def __init__(self):
+    def __init__(self) -> None:
         self.settings: Any = None  # set in scrapy.cmdline
 
     def set_crawler(self, crawler):

scrapy/core/scraper.py

@@ -1,8 +1,22 @@
 """This module implements the Scraper component which parses responses and
 extracts information from them"""
+from __future__ import annotations
+
 import logging
 from collections import deque
-from typing import Any, AsyncGenerator, AsyncIterable, Deque, Generator, Iterable, Optional, Set, Tuple, Union
+from typing import (
+    Any,
+    AsyncGenerator,
+    AsyncIterable,
+    Deque,
+    Generator,
+    Iterable,
+    Optional,
+    Set,
+    TYPE_CHECKING,
+    Tuple,
+    Union,
+)
 
 from itemadapter import is_item
 from twisted.internet.defer import Deferred, inlineCallbacks
@@ -26,6 +40,10 @@ from scrapy.utils.misc import load_object, warn_on_generator_with_return_value
 from scrapy.utils.spider import iterate_spider_output
 
 
+if TYPE_CHECKING:
+    from scrapy.crawler import Crawler
+
+
 QueueTuple = Tuple[Union[Response, Failure], Request, Deferred]
 
 
@@ -75,7 +93,7 @@ class Slot:
 
 class Scraper:
 
-    def __init__(self, crawler):
+    def __init__(self, crawler: Crawler) -> None:
         self.slot: Optional[Slot] = None
         self.spidermw = SpiderMiddlewareManager.from_crawler(crawler)
         itemproc_cls = load_object(crawler.settings['ITEM_PROCESSOR'])
@@ -174,6 +192,7 @@ class Scraper:
     def handle_spider_error(self, _failure: Failure, request: Request, response: Response, spider: Spider) -> None:
         exc = _failure.value
         if isinstance(exc, CloseSpider):
+            assert self.crawler.engine is not None  # typing
             self.crawler.engine.close_spider(spider, exc.reason or 'cancelled')
             return
         logkws = self.logformatter.spider_error(_failure, request, response, spider)
@@ -214,6 +233,7 @@ class Scraper:
         """
         assert self.slot is not None  # typing
         if isinstance(output, Request):
+            assert self.crawler.engine is not None  # typing
             self.crawler.engine.crawl(request=output)
         elif is_item(output):
             self.slot.itemproc_size += 1

scrapy/crawler.py

@@ -4,7 +4,7 @@ import logging
 import pprint
 import signal
 import warnings
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Optional
 
 from twisted.internet import defer
 from zope.interface.exceptions import DoesNotImplement
@@ -109,7 +109,7 @@ class Crawler:
         self.settings.freeze()
         self.crawling = False
         self.spider = None
-        self.engine = None
+        self.engine: Optional[ExecutionEngine] = None
 
     @defer.inlineCallbacks
     def crawl(self, *args, **kwargs):

scrapy/http/response/__init__.py

@@ -140,8 +140,7 @@ class Response(object_ref):
 
     def follow(self, url, callback=None, method='GET', headers=None, body=None,
                cookies=None, meta=None, encoding='utf-8', priority=0,
-               dont_filter=False, errback=None, cb_kwargs=None, flags=None):
-        # type: (...) -> Request
+               dont_filter=False, errback=None, cb_kwargs=None, flags=None) -> Request:
         """
         Return a :class:`~.Request` instance to follow a link ``url``.
         It accepts the same arguments as ``Request.__init__`` method,
@@ -179,8 +178,8 @@ class Response(object_ref):
 
     def follow_all(self, urls, callback=None, method='GET', headers=None, body=None,
                    cookies=None, meta=None, encoding='utf-8', priority=0,
-                   dont_filter=False, errback=None, cb_kwargs=None, flags=None):
-        # type: (...) -> Generator[Request, None, None]
+                   dont_filter=False, errback=None, cb_kwargs=None,
+                   flags=None) -> Generator[Request, None, None]:
         """
         .. versionadded:: 2.0
 

scrapy/http/response/text.py

@@ -142,8 +142,7 @@ class TextResponse(Response):
 
     def follow(self, url, callback=None, method='GET', headers=None, body=None,
                cookies=None, meta=None, encoding=None, priority=0,
-               dont_filter=False, errback=None, cb_kwargs=None, flags=None):
-        # type: (...) -> Request
+               dont_filter=False, errback=None, cb_kwargs=None, flags=None) -> Request:
         """
         Return a :class:`~.Request` instance to follow a link ``url``.
         It accepts the same arguments as ``Request.__init__`` method,
@@ -184,8 +183,7 @@ class TextResponse(Response):
     def follow_all(self, urls=None, callback=None, method='GET', headers=None, body=None,
                    cookies=None, meta=None, encoding=None, priority=0,
                    dont_filter=False, errback=None, cb_kwargs=None, flags=None,
-                   css=None, xpath=None):
-        # type: (...) -> Generator[Request, None, None]
+                   css=None, xpath=None) -> Generator[Request, None, None]:
         """
         A generator that produces :class:`~.Request` instances to follow all
         links in ``urls``. It accepts the same arguments as the :class:`~.Request`'s

scrapy/middleware.py

@@ -1,7 +1,7 @@
 import logging
 import pprint
 from collections import defaultdict, deque
-from typing import Callable, Deque, Dict, Iterable, Tuple, Union, cast
+from typing import Any, Callable, Deque, Dict, Iterable, Tuple, Union, cast
 
 from twisted.internet.defer import Deferred
 
@@ -19,7 +19,7 @@ class MiddlewareManager:
 
     component_name = 'foo middleware'
 
-    def __init__(self, *middlewares):
+    def __init__(self, *middlewares: Any) -> None:
         self.middlewares = middlewares
         # Only process_spider_output and process_spider_exception can be None.
         # Only process_spider_output can be a tuple, and only until _async compatibility methods are removed.

tox.ini

@@ -38,7 +38,7 @@ install_command =
 basepython = python3
 deps =
     lxml-stubs==0.2.0
-    mypy==0.982
+    mypy==0.991
     types-attrs==19.1.0
     types-pyOpenSSL==21.0.0
     types-setuptools==57.0.0
@@ -58,7 +58,7 @@ deps =
     {[testenv]deps}
     # Twisted[http2] is required to import some files
     Twisted[http2]>=17.9.0
-    flake8==5.0.4
+    flake8==6.0.0
 commands =
     flake8 {posargs:docs scrapy tests}
 
@@ -67,7 +67,7 @@ commands =
 basepython = python3.8
 deps =
     {[testenv:extra-deps]deps}
-    pylint==2.15.3
+    pylint==2.15.6
 commands =
     pylint conftest.py docs extras scrapy setup.py tests