From b4acf5c827b710fac83fa09fc5832ac76041e5c8 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 7 Nov 2023 12:34:35 +0400 Subject: [PATCH] Fix and remove most of the entries from the mypy ignore list (#6137) --- scrapy/contracts/__init__.py | 4 +- scrapy/loader/__init__.py | 2 +- scrapy/pipelines/images.py | 3 +- scrapy/utils/testproc.py | 2 +- setup.cfg | 73 ++---------------------- tests/mocks/dummydbm.py | 3 +- tests/test_exporters.py | 9 +-- tests/test_http_request.py | 5 +- tests/test_linkextractors.py | 3 +- tests/test_loader.py | 3 +- tests/test_pipeline_crawl.py | 4 +- tests/test_pipeline_files.py | 13 ++--- tests/test_pipeline_images.py | 16 +++--- tests/test_request_cb_kwargs.py | 4 +- tests/test_scheduler.py | 5 +- tests/test_spidermiddleware_httperror.py | 3 +- tests/test_spidermiddleware_referer.py | 31 +++++----- tests/test_utils_serialize.py | 2 +- tests/test_utils_url.py | 4 +- 19 files changed, 67 insertions(+), 122 deletions(-) diff --git a/scrapy/contracts/__init__.py b/scrapy/contracts/__init__.py index 2d9ddd89a..d46eb7c51 100644 --- a/scrapy/contracts/__init__.py +++ b/scrapy/contracts/__init__.py @@ -3,7 +3,7 @@ import sys from functools import wraps from inspect import getmembers from types import CoroutineType -from typing import AsyncGenerator, Dict +from typing import AsyncGenerator, Dict, Optional, Type from unittest import TestCase from scrapy.http import Request @@ -14,7 +14,7 @@ from scrapy.utils.spider import iterate_spider_output class Contract: """Abstract class for contracts""" - request_cls = None + request_cls: Optional[Type[Request]] = None def __init__(self, method, *args): self.testcase_pre = _create_testcase(method, f"@{self.name} pre-hook") diff --git a/scrapy/loader/__init__.py b/scrapy/loader/__init__.py index 91337b949..1042a3d48 100644 --- a/scrapy/loader/__init__.py +++ b/scrapy/loader/__init__.py @@ -78,7 +78,7 @@ class ItemLoader(itemloaders.ItemLoader): read-only. """ - default_item_class = Item + default_item_class: type = Item default_selector_class = Selector def __init__(self, item=None, selector=None, response=None, parent=None, **context): diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py index 9d18144ee..1bd9832a8 100644 --- a/scrapy/pipelines/images.py +++ b/scrapy/pipelines/images.py @@ -8,6 +8,7 @@ import hashlib import warnings from contextlib import suppress from io import BytesIO +from typing import Dict, Tuple from itemadapter import ItemAdapter @@ -48,7 +49,7 @@ class ImagesPipeline(FilesPipeline): MIN_WIDTH = 0 MIN_HEIGHT = 0 EXPIRES = 90 - THUMBS = {} + THUMBS: Dict[str, Tuple[int, int]] = {} DEFAULT_IMAGES_URLS_FIELD = "image_urls" DEFAULT_IMAGES_RESULT_FIELD = "images" diff --git a/scrapy/utils/testproc.py b/scrapy/utils/testproc.py index 0688e014b..3bdffcaa7 100644 --- a/scrapy/utils/testproc.py +++ b/scrapy/utils/testproc.py @@ -11,7 +11,7 @@ from twisted.python.failure import Failure class ProcessTest: - command = None + command: Optional[str] = None prefix = [sys.executable, "-m", "scrapy.cmdline"] cwd = os.getcwd() # trial chdirs to temp dir diff --git a/setup.cfg b/setup.cfg index d23549f10..151e784c6 100644 --- a/setup.cfg +++ b/setup.cfg @@ -8,82 +8,17 @@ universal=1 ignore_missing_imports = true # Interface classes are hard to support + [mypy-twisted.internet.interfaces] follow_imports = skip +[mypy-scrapy.interfaces] +ignore_errors = True + [mypy-twisted.internet.reactor] follow_imports = skip # FIXME: remove the following sections once the issues are solved -[mypy-scrapy.interfaces] -ignore_errors = True - -[mypy-scrapy.pipelines.images] -ignore_errors = True - [mypy-scrapy.settings.default_settings] ignore_errors = True - -[mypy-tests.mocks.dummydbm] -ignore_errors = True - -[mypy-tests.test_command_fetch] -ignore_errors = True - -[mypy-tests.test_command_parse] -ignore_errors = True - -[mypy-tests.test_command_shell] -ignore_errors = True - -[mypy-tests.test_command_version] -ignore_errors = True - -[mypy-tests.test_contracts] -ignore_errors = True - -[mypy-tests.test_downloader_handlers] -ignore_errors = True - -[mypy-tests.test_exporters] -ignore_errors = True - -[mypy-tests.test_http_request] -ignore_errors = True - -[mypy-tests.test_linkextractors] -ignore_errors = True - -[mypy-tests.test_loader] -ignore_errors = True - -[mypy-tests.test_loader_deprecated] -ignore_errors = True - -[mypy-tests.test_pipeline_crawl] -ignore_errors = True - -[mypy-tests.test_pipeline_files] -ignore_errors = True - -[mypy-tests.test_pipeline_images] -ignore_errors = True - -[mypy-tests.test_request_cb_kwargs] -ignore_errors = True - -[mypy-tests.test_scheduler] -ignore_errors = True - -[mypy-tests.test_spidermiddleware_httperror] -ignore_errors = True - -[mypy-tests.test_spidermiddleware_referer] -ignore_errors = True - -[mypy-tests.test_utils_serialize] -ignore_errors = True - -[mypy-tests.test_utils_url] -ignore_errors = True diff --git a/tests/mocks/dummydbm.py b/tests/mocks/dummydbm.py index e77f53666..2869ff8f7 100644 --- a/tests/mocks/dummydbm.py +++ b/tests/mocks/dummydbm.py @@ -1,5 +1,6 @@ """DBM-like dummy module""" import collections +from typing import Any, DefaultDict class DummyDB(dict): @@ -12,7 +13,7 @@ class DummyDB(dict): error = KeyError -_DATABASES = collections.defaultdict(DummyDB) +_DATABASES: DefaultDict[Any, DummyDB] = collections.defaultdict(DummyDB) def open(file, flag="r", mode=0o666): diff --git a/tests/test_exporters.py b/tests/test_exporters.py index f4e82705a..c11913365 100644 --- a/tests/test_exporters.py +++ b/tests/test_exporters.py @@ -7,6 +7,7 @@ import tempfile import unittest from datetime import datetime from io import BytesIO +from typing import Any import lxml.etree from itemadapter import ItemAdapter @@ -53,8 +54,8 @@ class CustomFieldDataclass: class BaseItemExporterTest(unittest.TestCase): - item_class = TestItem - custom_field_item_class = CustomFieldItem + item_class: type = TestItem + custom_field_item_class: type = CustomFieldItem def setUp(self): self.i = self.item_class(name="John\xa3", age="22") @@ -517,7 +518,7 @@ class XmlItemExporterDataclassTest(XmlItemExporterTest): class JsonLinesItemExporterTest(BaseItemExporterTest): - _expected_nested = { + _expected_nested: Any = { "name": "Jesus", "age": {"name": "Maria", "age": {"name": "Joseph", "age": "22"}}, } @@ -665,7 +666,7 @@ class JsonItemExporterDataclassTest(JsonItemExporterTest): class CustomExporterItemTest(unittest.TestCase): - item_class = TestItem + item_class: type = TestItem def setUp(self): if self.item_class is None: diff --git a/tests/test_http_request.py b/tests/test_http_request.py index 2bd68f846..6dc9ec8b7 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -3,6 +3,7 @@ import re import unittest import warnings import xmlrpc.client +from typing import Any, Dict, List from unittest import mock from urllib.parse import parse_qs, unquote_to_bytes, urlparse @@ -21,8 +22,8 @@ from scrapy.utils.python import to_bytes, to_unicode class RequestTest(unittest.TestCase): request_class = Request default_method = "GET" - default_headers = {} - default_meta = {} + default_headers: Dict[bytes, List[bytes]] = {} + default_meta: Dict[str, Any] = {} def test_init(self): # Request requires url in the __init__ method diff --git a/tests/test_linkextractors.py b/tests/test_linkextractors.py index c90065e67..18e9608c1 100644 --- a/tests/test_linkextractors.py +++ b/tests/test_linkextractors.py @@ -1,6 +1,7 @@ import pickle import re import unittest +from typing import Optional from packaging.version import Version from pytest import mark @@ -15,7 +16,7 @@ from tests import get_testdata # a hack to skip base class tests in pytest class Base: class LinkExtractorTestCase(unittest.TestCase): - extractor_cls = None + extractor_cls: Optional[type] = None def setUp(self): body = get_testdata("link_extractor", "linkextractor.html") diff --git a/tests/test_loader.py b/tests/test_loader.py index 0dc2de172..b0b7f8723 100644 --- a/tests/test_loader.py +++ b/tests/test_loader.py @@ -1,5 +1,6 @@ import dataclasses import unittest +from typing import Optional import attr from itemadapter import ItemAdapter @@ -87,7 +88,7 @@ class BasicItemLoaderTest(unittest.TestCase): class InitializationTestMixin: - item_class = None + item_class: Optional[type] = None def test_keep_single_value(self): """Loaded item should contain values from the initial item""" diff --git a/tests/test_pipeline_crawl.py b/tests/test_pipeline_crawl.py index cb9464949..ed8483483 100644 --- a/tests/test_pipeline_crawl.py +++ b/tests/test_pipeline_crawl.py @@ -1,5 +1,6 @@ import shutil from pathlib import Path +from typing import Optional, Set from testfixtures import LogCapture from twisted.internet import defer @@ -54,7 +55,7 @@ class FileDownloadCrawlTestCase(TestCase): store_setting_key = "FILES_STORE" media_key = "files" media_urls_key = "file_urls" - expected_checksums = { + expected_checksums: Optional[Set[str]] = { "5547178b89448faf0015a13f904c936e", "c2281c83670e31d8aaab7cb642b824db", "ed3f6538dc15d4d9179dae57319edc5f", @@ -193,6 +194,7 @@ class FileDownloadCrawlTestCase(TestCase): ) +skip_pillow: Optional[str] try: from PIL import Image # noqa: imported just to check for the import error except ImportError: diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py index 468751446..e7000e314 100644 --- a/tests/test_pipeline_files.py +++ b/tests/test_pipeline_files.py @@ -7,6 +7,7 @@ from io import BytesIO from pathlib import Path from shutil import rmtree from tempfile import mkdtemp +from typing import Dict, List from unittest import mock from urllib.parse import urlparse @@ -308,11 +309,11 @@ class FilesPipelineTestCaseFieldsDataClass( class FilesPipelineTestAttrsItem: name = attr.ib(default="") # default fields - file_urls = attr.ib(default=lambda: []) - files = attr.ib(default=lambda: []) + file_urls: List[str] = attr.ib(default=lambda: []) + files: List[Dict[str, str]] = attr.ib(default=lambda: []) # overridden fields - custom_file_urls = attr.ib(default=lambda: []) - custom_files = attr.ib(default=lambda: []) + custom_file_urls: List[str] = attr.ib(default=lambda: []) + custom_files: List[Dict[str, str]] = attr.ib(default=lambda: []) class FilesPipelineTestCaseFieldsAttrsItem( @@ -690,7 +691,3 @@ def _prepare_request_object(item_url, flags=None): item_url, meta={"response": Response(item_url, status=200, body=b"data", flags=flags)}, ) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py index 8924875d1..2e2e06b89 100644 --- a/tests/test_pipeline_images.py +++ b/tests/test_pipeline_images.py @@ -5,6 +5,7 @@ import random import warnings from shutil import rmtree from tempfile import mkdtemp +from typing import Dict, List, Optional from unittest.mock import patch import attr @@ -18,6 +19,7 @@ from scrapy.pipelines.images import ImageException, ImagesPipeline, NoimagesDrop from scrapy.settings import Settings from scrapy.utils.python import to_bytes +skip_pillow: Optional[str] try: from PIL import Image except ImportError: @@ -26,7 +28,7 @@ except ImportError: ) else: encoders = {"jpeg_encoder", "jpeg_decoder"} - if not encoders.issubset(set(Image.core.__dict__)): + if not encoders.issubset(set(Image.core.__dict__)): # type: ignore[attr-defined] skip_pillow = "Missing JPEG encoders" else: skip_pillow = None @@ -404,11 +406,11 @@ class ImagesPipelineTestCaseFieldsDataClass( class ImagesPipelineTestAttrsItem: name = attr.ib(default="") # default fields - image_urls = attr.ib(default=lambda: []) - images = attr.ib(default=lambda: []) + image_urls: List[str] = attr.ib(default=lambda: []) + images: List[Dict[str, str]] = attr.ib(default=lambda: []) # overridden fields - custom_image_urls = attr.ib(default=lambda: []) - custom_images = attr.ib(default=lambda: []) + custom_image_urls: List[str] = attr.ib(default=lambda: []) + custom_images: List[Dict[str, str]] = attr.ib(default=lambda: []) class ImagesPipelineTestCaseFieldsAttrsItem( @@ -646,7 +648,3 @@ def _create_image(format, *a, **kw): Image.new(*a, **kw).save(buf, format) buf.seek(0) return Image.open(buf), buf - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_request_cb_kwargs.py b/tests/test_request_cb_kwargs.py index adce6cda7..7299972f6 100644 --- a/tests/test_request_cb_kwargs.py +++ b/tests/test_request_cb_kwargs.py @@ -1,3 +1,5 @@ +from typing import List + from testfixtures import LogCapture from twisted.internet import defer from twisted.trial.unittest import TestCase @@ -62,7 +64,7 @@ class KeywordArgumentsSpider(MockServerSpider): }, } - checks = [] + checks: List[bool] = [] def start_requests(self): data = {"key": "value", "number": 123, "callback": "some_callback"} diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py index ef9b360c4..f8465a5ff 100644 --- a/tests/test_scheduler.py +++ b/tests/test_scheduler.py @@ -2,6 +2,7 @@ import collections import shutil import tempfile import unittest +from typing import Optional from twisted.internet import defer from twisted.trial.unittest import TestCase @@ -59,7 +60,7 @@ class MockCrawler(Crawler): class SchedulerHandler: - priority_queue_cls = None + priority_queue_cls: Optional[str] = None jobdir = None def create_scheduler(self): @@ -253,7 +254,7 @@ def _is_scheduling_fair(enqueued_slots, dequeued_slots): class DownloaderAwareSchedulerTestMixin: - priority_queue_cls = "scrapy.pqueues.DownloaderAwarePriorityQueue" + priority_queue_cls: Optional[str] = "scrapy.pqueues.DownloaderAwarePriorityQueue" reopen = False def test_logic(self): diff --git a/tests/test_spidermiddleware_httperror.py b/tests/test_spidermiddleware_httperror.py index 1d5a887cc..044455415 100644 --- a/tests/test_spidermiddleware_httperror.py +++ b/tests/test_spidermiddleware_httperror.py @@ -1,4 +1,5 @@ import logging +from typing import Set from unittest import TestCase from testfixtures import LogCapture @@ -16,7 +17,7 @@ from tests.spiders import MockServerSpider class _HttpErrorSpider(MockServerSpider): name = "httperror" - bypass_status_codes = set() + bypass_status_codes: Set[int] = set() def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.py index 2f0dda269..afffa87fb 100644 --- a/tests/test_spidermiddleware_referer.py +++ b/tests/test_spidermiddleware_referer.py @@ -1,4 +1,5 @@ import warnings +from typing import Any, Dict, List, Optional, Tuple from unittest import TestCase from urllib.parse import urlparse @@ -31,10 +32,10 @@ from scrapy.spiders import Spider class TestRefererMiddleware(TestCase): - req_meta = {} - resp_headers = {} - settings = {} - scenarii = [ + req_meta: Dict[str, Any] = {} + resp_headers: Dict[str, str] = {} + settings: Dict[str, Any] = {} + scenarii: List[Tuple[str, str, Optional[bytes]]] = [ ("http://scrapytest.org", "http://scrapytest.org/", b"http://scrapytest.org"), ] @@ -64,7 +65,7 @@ class MixinDefault: with some additional filtering of s3:// """ - scenarii = [ + scenarii: List[Tuple[str, str, Optional[bytes]]] = [ ("https://example.com/", "https://scrapy.org/", b"https://example.com/"), ("http://example.com/", "http://scrapy.org/", b"http://example.com/"), ("http://example.com/", "https://scrapy.org/", b"http://example.com/"), @@ -85,7 +86,7 @@ class MixinDefault: class MixinNoReferrer: - scenarii = [ + scenarii: List[Tuple[str, str, Optional[bytes]]] = [ ("https://example.com/page.html", "https://example.com/", None), ("http://www.example.com/", "https://scrapy.org/", None), ("http://www.example.com/", "http://scrapy.org/", None), @@ -95,7 +96,7 @@ class MixinNoReferrer: class MixinNoReferrerWhenDowngrade: - scenarii = [ + scenarii: List[Tuple[str, str, Optional[bytes]]] = [ # TLS to TLS: send non-empty referrer ( "https://example.com/page.html", @@ -177,7 +178,7 @@ class MixinNoReferrerWhenDowngrade: class MixinSameOrigin: - scenarii = [ + scenarii: List[Tuple[str, str, Optional[bytes]]] = [ # Same origin (protocol, host, port): send referrer ( "https://example.com/page.html", @@ -246,7 +247,7 @@ class MixinSameOrigin: class MixinOrigin: - scenarii = [ + scenarii: List[Tuple[str, str, Optional[bytes]]] = [ # TLS or non-TLS to TLS or non-TLS: referrer origin is sent (yes, even for downgrades) ( "https://example.com/page.html", @@ -270,7 +271,7 @@ class MixinOrigin: class MixinStrictOrigin: - scenarii = [ + scenarii: List[Tuple[str, str, Optional[bytes]]] = [ # TLS or non-TLS to TLS or non-TLS: referrer origin is sent but not for downgrades ( "https://example.com/page.html", @@ -298,7 +299,7 @@ class MixinStrictOrigin: class MixinOriginWhenCrossOrigin: - scenarii = [ + scenarii: List[Tuple[str, str, Optional[bytes]]] = [ # Same origin (protocol, host, port): send referrer ( "https://example.com/page.html", @@ -405,7 +406,7 @@ class MixinOriginWhenCrossOrigin: class MixinStrictOriginWhenCrossOrigin: - scenarii = [ + scenarii: List[Tuple[str, str, Optional[bytes]]] = [ # Same origin (protocol, host, port): send referrer ( "https://example.com/page.html", @@ -517,7 +518,7 @@ class MixinStrictOriginWhenCrossOrigin: class MixinUnsafeUrl: - scenarii = [ + scenarii: List[Tuple[str, str, Optional[bytes]]] = [ # TLS to TLS: send referrer ( "https://example.com/sekrit.html", @@ -920,7 +921,9 @@ class TestPolicyHeaderPrecedence004( class TestReferrerOnRedirect(TestRefererMiddleware): settings = {"REFERRER_POLICY": "scrapy.spidermiddlewares.referer.UnsafeUrlPolicy"} - scenarii = [ + scenarii: List[ + Tuple[str, str, Tuple[Tuple[int, str], ...], Optional[bytes], Optional[bytes]] + ] = [ # type: ignore[assignment] ( "http://scrapytest.org/1", # parent "http://scrapytest.org/2", # target diff --git a/tests/test_utils_serialize.py b/tests/test_utils_serialize.py index 5cdcc7f7c..055db4e5b 100644 --- a/tests/test_utils_serialize.py +++ b/tests/test_utils_serialize.py @@ -58,7 +58,7 @@ class JsonEncoderTestCase(unittest.TestCase): self.assertIn(r.url, rs) self.assertIn(str(r.status), rs) - def test_encode_dataclass_item(self): + def test_encode_dataclass_item(self) -> None: @dataclasses.dataclass class TestDataClass: name: str diff --git a/tests/test_utils_url.py b/tests/test_utils_url.py index 65522f0fd..59a95b0e2 100644 --- a/tests/test_utils_url.py +++ b/tests/test_utils_url.py @@ -364,7 +364,7 @@ for k, args in enumerate( setattr(GuessSchemeTest, t_method.__name__, t_method) # TODO: the following tests do not pass with current implementation -for k, args in enumerate( +for k, skip_args in enumerate( [ ( r"C:\absolute\path\to\a\file.html", @@ -374,7 +374,7 @@ for k, args in enumerate( ], start=1, ): - t_method = create_skipped_scheme_t(args) + t_method = create_skipped_scheme_t(skip_args) t_method.__name__ = f"test_uri_skipped_{k:03}" setattr(GuessSchemeTest, t_method.__name__, t_method)