From 065db7b56688ce5cbdc1508cfda9603d33b17a87 Mon Sep 17 00:00:00 2001 From: Matthew Donoughe Date: Wed, 19 Oct 2022 08:28:26 -0400 Subject: [PATCH] fix some mypy issues --- scrapy/commands/__init__.py | 4 ++-- scrapy/commands/genspider.py | 5 +++-- scrapy/spiders/__init__.py | 2 +- scrapy/utils/conf.py | 4 ++-- tests/test_commands.py | 1 + tests/test_crawler.py | 2 ++ 6 files changed, 11 insertions(+), 7 deletions(-) diff --git a/scrapy/commands/__init__.py b/scrapy/commands/__init__.py index 8570d90bd..2c205a712 100644 --- a/scrapy/commands/__init__.py +++ b/scrapy/commands/__init__.py @@ -15,7 +15,7 @@ from scrapy.exceptions import UsageError class ScrapyCommand: requires_project = False - crawler_process = None + crawler_process: Any = None # default settings to be used for this command instead of global defaults default_settings: Dict[str, Any] = {} @@ -23,7 +23,7 @@ class ScrapyCommand: exitcode = 0 def __init__(self): - self.settings = None # set in scrapy.cmdline + self.settings: Any = None # set in scrapy.cmdline def set_crawler(self, crawler): if hasattr(self, '_crawler'): diff --git a/scrapy/commands/genspider.py b/scrapy/commands/genspider.py index 01b4a0dbd..facb593da 100644 --- a/scrapy/commands/genspider.py +++ b/scrapy/commands/genspider.py @@ -4,7 +4,7 @@ import string from pathlib import Path from importlib import import_module -from typing import Optional +from typing import Optional, cast from urllib.parse import urlparse import scrapy @@ -116,6 +116,7 @@ class Command(ScrapyCommand): return template_file print(f"Unable to find template: {template}\n") print('Use "scrapy genspider --list" to see all available templates.') + return None def _list_templates(self): print("Available templates:") @@ -144,7 +145,7 @@ class Command(ScrapyCommand): # a file with the same name exists in the target directory spiders_module = import_module(self.settings['NEWSPIDER_MODULE']) - spiders_dir = Path(spiders_module.__file__).parent + spiders_dir = Path(cast(str, spiders_module.__file__)).parent spiders_dir_abs = spiders_dir.resolve() path = spiders_dir_abs / (name + ".py") if path.exists(): diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.py index 9a97e7801..0a9b124b9 100644 --- a/scrapy/spiders/__init__.py +++ b/scrapy/spiders/__init__.py @@ -22,7 +22,7 @@ class Spider(object_ref): class. """ - name: Optional[str] = None + name: str custom_settings: Optional[dict] = None def __init__(self, name=None, **kwargs): diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.py index 0dfa714e8..82defa033 100644 --- a/scrapy/utils/conf.py +++ b/scrapy/utils/conf.py @@ -5,7 +5,7 @@ import warnings from configparser import ConfigParser from operator import itemgetter from pathlib import Path -from typing import List, Optional, Union +from typing import Any, Dict, List, Optional, Union from scrapy.exceptions import ScrapyDeprecationWarning, UsageError @@ -176,7 +176,7 @@ def feed_process_params_from_cli(settings, output: List[str], output_format=None 'URIs are specified' ) - result = {} + result: Dict[str, Dict[str, Any]] = {} for element in output: try: feed_uri, feed_format = element.rsplit(':', 1) diff --git a/tests/test_commands.py b/tests/test_commands.py index b61f314ec..f8081f450 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -112,6 +112,7 @@ class ProjectTest(unittest.TestCase): match = pattern.search(line) if match is not None: return match + return None class StartprojectTest(ProjectTest): diff --git a/tests/test_crawler.py b/tests/test_crawler.py index 19f4229a3..a0703ad47 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -292,6 +292,8 @@ class CrawlerRunnerHasSpider(unittest.TestCase): class ScriptRunnerMixin: + script_dir: Path + def run_script(self, script_name: str, *script_args): script_path = self.script_dir / script_name args = [sys.executable, str(script_path)] + list(script_args)