Merge pull request #4691 from elacuesta/typing-spider-attributes

Typing: annotate a few Spider attributes

scrapy/spiders/__init__.py

@@ -5,6 +5,7 @@ See documentation in docs/topics/spiders.rst
 """
 import logging
 import warnings
+from typing import Optional
 
 from scrapy import signals
 from scrapy.http import Request
@@ -18,8 +19,8 @@ class Spider(object_ref):
     class.
     """
 
-    name = None
-    custom_settings = None
+    name: Optional[str] = None
+    custom_settings: Optional[dict] = None
 
     def __init__(self, name=None, **kwargs):
         if name is not None:

scrapy/spiders/crawl.py

@@ -7,6 +7,7 @@ See documentation in docs/topics/spiders.rst
 
 import copy
 import warnings
+from typing import Sequence
 
 from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.http import Request, HtmlResponse
@@ -72,7 +73,7 @@ class Rule:
 
 class CrawlSpider(Spider):
 
-    rules = ()
+    rules: Sequence[Rule] = ()
 
     def __init__(self, *a, **kw):
         super().__init__(*a, **kw)

setup.cfg

@@ -16,9 +16,6 @@ ignore_errors = True
 [mypy-scrapy.commands]
 ignore_errors = True
 
-[mypy-scrapy.commands.bench]
-ignore_errors = True
-
 [mypy-scrapy.commands.parse]
 ignore_errors = True
 
@@ -28,9 +25,6 @@ ignore_errors = True
 [mypy-scrapy.contracts]
 ignore_errors = True
 
-[mypy-scrapy.core.spidermw]
-ignore_errors = True
-
 [mypy-scrapy.interfaces]
 ignore_errors = True
 
@@ -70,15 +64,6 @@ ignore_errors = True
 [mypy-tests.mocks.dummydbm]
 ignore_errors = True
 
-[mypy-tests.spiders]
-ignore_errors = True
-
-[mypy-tests.test_cmdline_crawl_with_pipeline.test_spider.spiders.exception]
-ignore_errors = True
-
-[mypy-tests.test_cmdline_crawl_with_pipeline.test_spider.spiders.normal]
-ignore_errors = True
-
 [mypy-tests.test_command_fetch]
 ignore_errors = True
 
@@ -94,9 +79,6 @@ ignore_errors = True
 [mypy-tests.test_contracts]
 ignore_errors = True
 
-[mypy-tests.test_crawler]
-ignore_errors = True
-
 [mypy-tests.test_downloader_handlers]
 ignore_errors = True
 
@@ -127,53 +109,20 @@ ignore_errors = True
 [mypy-tests.test_pipeline_images]
 ignore_errors = True
 
-[mypy-tests.test_pipelines]
-ignore_errors = True
-
-[mypy-tests.test_request_attribute_binding]
-ignore_errors = True
-
 [mypy-tests.test_request_cb_kwargs]
 ignore_errors = True
 
-[mypy-tests.test_request_left]
-ignore_errors = True
-
 [mypy-tests.test_scheduler]
 ignore_errors = True
 
-[mypy-tests.test_signals]
-ignore_errors = True
-
-[mypy-tests.test_spiderloader.test_spiders.nested.spider4]
-ignore_errors = True
-
-[mypy-tests.test_spiderloader.test_spiders.spider1]
-ignore_errors = True
-
-[mypy-tests.test_spiderloader.test_spiders.spider2]
-ignore_errors = True
-
-[mypy-tests.test_spiderloader.test_spiders.spider3]
-ignore_errors = True
-
 [mypy-tests.test_spidermiddleware_httperror]
 ignore_errors = True
 
-[mypy-tests.test_spidermiddleware_output_chain]
-ignore_errors = True
-
 [mypy-tests.test_spidermiddleware_referer]
 ignore_errors = True
 
-[mypy-tests.test_utils_reqser]
-ignore_errors = True
-
 [mypy-tests.test_utils_serialize]
 ignore_errors = True
 
-[mypy-tests.test_utils_spider]
-ignore_errors = True
-
 [mypy-tests.test_utils_url]
 ignore_errors = True

tests/spiders.py

@@ -255,7 +255,7 @@ class CrawlSpiderWithParseMethod(MockServerSpider, CrawlSpider):
     A CrawlSpider which overrides the 'parse' method
     """
     name = 'crawl_spider_with_parse_method'
-    custom_settings = {
+    custom_settings: dict = {
         'RETRY_HTTP_CODES': [],  # no need to retry
     }
     rules = (