From 5bd27191a2fd9ec50936c05a377f76bd14aa8266 Mon Sep 17 00:00:00 2001 From: Andrey Rahmatullin Date: Mon, 28 Nov 2022 23:37:57 +0500 Subject: [PATCH] Bump mypy, flake8, and pylint (#5738) --- scrapy/commands/__init__.py | 2 +- scrapy/core/scraper.py | 24 ++++++++++++++++++++++-- scrapy/crawler.py | 4 ++-- scrapy/http/response/__init__.py | 7 +++---- scrapy/http/response/text.py | 6 ++---- scrapy/middleware.py | 4 ++-- tox.ini | 6 +++--- 7 files changed, 35 insertions(+), 18 deletions(-) diff --git a/scrapy/commands/__init__.py b/scrapy/commands/__init__.py index d0fb4efd8..b9ba3335e 100644 --- a/scrapy/commands/__init__.py +++ b/scrapy/commands/__init__.py @@ -23,7 +23,7 @@ class ScrapyCommand: exitcode = 0 - def __init__(self): + def __init__(self) -> None: self.settings: Any = None # set in scrapy.cmdline def set_crawler(self, crawler): diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index 7225e0743..69ac1cdaf 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -1,8 +1,22 @@ """This module implements the Scraper component which parses responses and extracts information from them""" +from __future__ import annotations + import logging from collections import deque -from typing import Any, AsyncGenerator, AsyncIterable, Deque, Generator, Iterable, Optional, Set, Tuple, Union +from typing import ( + Any, + AsyncGenerator, + AsyncIterable, + Deque, + Generator, + Iterable, + Optional, + Set, + TYPE_CHECKING, + Tuple, + Union, +) from itemadapter import is_item from twisted.internet.defer import Deferred, inlineCallbacks @@ -26,6 +40,10 @@ from scrapy.utils.misc import load_object, warn_on_generator_with_return_value from scrapy.utils.spider import iterate_spider_output +if TYPE_CHECKING: + from scrapy.crawler import Crawler + + QueueTuple = Tuple[Union[Response, Failure], Request, Deferred] @@ -75,7 +93,7 @@ class Slot: class Scraper: - def __init__(self, crawler): + def __init__(self, crawler: Crawler) -> None: self.slot: Optional[Slot] = None self.spidermw = SpiderMiddlewareManager.from_crawler(crawler) itemproc_cls = load_object(crawler.settings['ITEM_PROCESSOR']) @@ -174,6 +192,7 @@ class Scraper: def handle_spider_error(self, _failure: Failure, request: Request, response: Response, spider: Spider) -> None: exc = _failure.value if isinstance(exc, CloseSpider): + assert self.crawler.engine is not None # typing self.crawler.engine.close_spider(spider, exc.reason or 'cancelled') return logkws = self.logformatter.spider_error(_failure, request, response, spider) @@ -214,6 +233,7 @@ class Scraper: """ assert self.slot is not None # typing if isinstance(output, Request): + assert self.crawler.engine is not None # typing self.crawler.engine.crawl(request=output) elif is_item(output): self.slot.itemproc_size += 1 diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 04c6891e3..4700a30ab 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -4,7 +4,7 @@ import logging import pprint import signal import warnings -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Optional from twisted.internet import defer from zope.interface.exceptions import DoesNotImplement @@ -109,7 +109,7 @@ class Crawler: self.settings.freeze() self.crawling = False self.spider = None - self.engine = None + self.engine: Optional[ExecutionEngine] = None @defer.inlineCallbacks def crawl(self, *args, **kwargs): diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py index fb2d0f165..7626946ec 100644 --- a/scrapy/http/response/__init__.py +++ b/scrapy/http/response/__init__.py @@ -140,8 +140,7 @@ class Response(object_ref): def follow(self, url, callback=None, method='GET', headers=None, body=None, cookies=None, meta=None, encoding='utf-8', priority=0, - dont_filter=False, errback=None, cb_kwargs=None, flags=None): - # type: (...) -> Request + dont_filter=False, errback=None, cb_kwargs=None, flags=None) -> Request: """ Return a :class:`~.Request` instance to follow a link ``url``. It accepts the same arguments as ``Request.__init__`` method, @@ -179,8 +178,8 @@ class Response(object_ref): def follow_all(self, urls, callback=None, method='GET', headers=None, body=None, cookies=None, meta=None, encoding='utf-8', priority=0, - dont_filter=False, errback=None, cb_kwargs=None, flags=None): - # type: (...) -> Generator[Request, None, None] + dont_filter=False, errback=None, cb_kwargs=None, + flags=None) -> Generator[Request, None, None]: """ .. versionadded:: 2.0 diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py index bfcde878d..da81d0a4a 100644 --- a/scrapy/http/response/text.py +++ b/scrapy/http/response/text.py @@ -142,8 +142,7 @@ class TextResponse(Response): def follow(self, url, callback=None, method='GET', headers=None, body=None, cookies=None, meta=None, encoding=None, priority=0, - dont_filter=False, errback=None, cb_kwargs=None, flags=None): - # type: (...) -> Request + dont_filter=False, errback=None, cb_kwargs=None, flags=None) -> Request: """ Return a :class:`~.Request` instance to follow a link ``url``. It accepts the same arguments as ``Request.__init__`` method, @@ -184,8 +183,7 @@ class TextResponse(Response): def follow_all(self, urls=None, callback=None, method='GET', headers=None, body=None, cookies=None, meta=None, encoding=None, priority=0, dont_filter=False, errback=None, cb_kwargs=None, flags=None, - css=None, xpath=None): - # type: (...) -> Generator[Request, None, None] + css=None, xpath=None) -> Generator[Request, None, None]: """ A generator that produces :class:`~.Request` instances to follow all links in ``urls``. It accepts the same arguments as the :class:`~.Request`'s diff --git a/scrapy/middleware.py b/scrapy/middleware.py index 8d7e5a602..431bd76dc 100644 --- a/scrapy/middleware.py +++ b/scrapy/middleware.py @@ -1,7 +1,7 @@ import logging import pprint from collections import defaultdict, deque -from typing import Callable, Deque, Dict, Iterable, Tuple, Union, cast +from typing import Any, Callable, Deque, Dict, Iterable, Tuple, Union, cast from twisted.internet.defer import Deferred @@ -19,7 +19,7 @@ class MiddlewareManager: component_name = 'foo middleware' - def __init__(self, *middlewares): + def __init__(self, *middlewares: Any) -> None: self.middlewares = middlewares # Only process_spider_output and process_spider_exception can be None. # Only process_spider_output can be a tuple, and only until _async compatibility methods are removed. diff --git a/tox.ini b/tox.ini index eee99cb2d..4d0f0291b 100644 --- a/tox.ini +++ b/tox.ini @@ -38,7 +38,7 @@ install_command = basepython = python3 deps = lxml-stubs==0.2.0 - mypy==0.982 + mypy==0.991 types-attrs==19.1.0 types-pyOpenSSL==21.0.0 types-setuptools==57.0.0 @@ -58,7 +58,7 @@ deps = {[testenv]deps} # Twisted[http2] is required to import some files Twisted[http2]>=17.9.0 - flake8==5.0.4 + flake8==6.0.0 commands = flake8 {posargs:docs scrapy tests} @@ -67,7 +67,7 @@ commands = basepython = python3.8 deps = {[testenv:extra-deps]deps} - pylint==2.15.3 + pylint==2.15.6 commands = pylint conftest.py docs extras scrapy setup.py tests