diff --git a/scrapy/commands/runspider.py b/scrapy/commands/runspider.py index c41135508..a8db1cd86 100644 --- a/scrapy/commands/runspider.py +++ b/scrapy/commands/runspider.py @@ -3,13 +3,14 @@ from os import PathLike from pathlib import Path from importlib import import_module from types import ModuleType +from typing import Union from scrapy.utils.spider import iter_spider_classes from scrapy.exceptions import UsageError from scrapy.commands import BaseRunSpiderCommand -def _import_file(filepath: str | PathLike[str]) -> ModuleType: +def _import_file(filepath: Union[str, PathLike]) -> ModuleType: abspath = Path(filepath).resolve() dirname = str(abspath.parent) if abspath.suffix not in ('.py', '.pyw'): diff --git a/scrapy/squeues.py b/scrapy/squeues.py index 1f2dee55f..2fa84fc00 100644 --- a/scrapy/squeues.py +++ b/scrapy/squeues.py @@ -6,6 +6,7 @@ import marshal import pickle from os import PathLike from pathlib import Path +from typing import Union from queuelib import queue @@ -17,7 +18,7 @@ def _with_mkdir(queue_class): class DirectoriesCreated(queue_class): - def __init__(self, path: str | PathLike[str], *args, **kwargs): + def __init__(self, path: Union[str, PathLike], *args, **kwargs): dirname = Path(path).parent if not dirname.exists(): dirname.mkdir(parents=True, exist_ok=True) diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.py index e247f5999..0dfa714e8 100644 --- a/scrapy/utils/conf.py +++ b/scrapy/utils/conf.py @@ -5,7 +5,7 @@ import warnings from configparser import ConfigParser from operator import itemgetter from pathlib import Path -from typing import Optional +from typing import List, Optional, Union from scrapy.exceptions import ScrapyDeprecationWarning, UsageError @@ -67,7 +67,7 @@ def arglist_to_dict(arglist): return dict(x.split('=', 1) for x in arglist) -def closest_scrapy_cfg(path: str | os.PathLike[str] = '.', prevpath: Optional[str | os.PathLike] = None) -> str: +def closest_scrapy_cfg(path: Union[str, os.PathLike] = '.', prevpath: Optional[Union[str, os.PathLike]] = None) -> str: """Return the path to the closest scrapy.cfg file by traversing the current directory and its parents """ @@ -103,7 +103,7 @@ def get_config(use_closest=True): return cfg -def get_sources(use_closest=True) -> list[str]: +def get_sources(use_closest=True) -> List[str]: xdg_config_home = os.environ.get('XDG_CONFIG_HOME') or Path('~/.config').expanduser() sources = [ '/etc/scrapy.cfg', @@ -131,8 +131,8 @@ def feed_complete_default_values_from_settings(feed, settings): return out -def feed_process_params_from_cli(settings, output: list[str], output_format=None, - overwrite_output: Optional[list[str]] = None): +def feed_process_params_from_cli(settings, output: List[str], output_format=None, + overwrite_output: Optional[List[str]] = None): """ Receives feed export params (from the 'crawl' or 'runspider' commands), checks for inconsistencies in their quantities and returns a dictionary diff --git a/tests/test_commands.py b/tests/test_commands.py index 39f718cce..71ae85094 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -14,7 +14,7 @@ from shutil import rmtree, copytree from stat import S_IWRITE as ANYONE_WRITE_PERMISSION from tempfile import mkdtemp from threading import Timer -from typing import Generator, Optional +from typing import Dict, Generator, Optional, Union from unittest import skipIf from pytest import mark @@ -104,7 +104,7 @@ class ProjectTest(unittest.TestCase): return p, to_unicode(stdout), to_unicode(stderr) - def find_in_file(self, filename: str | os.PathLike[str], regex) -> Optional[re.Match]: + def find_in_file(self, filename: Union[str, os.PathLike], regex) -> Optional[re.Match]: """Find first pattern occurrence in file""" pattern = re.compile(regex) with Path(filename).open("r") as f: @@ -175,7 +175,7 @@ class StartprojectTest(ProjectTest): assert Path(project_path, project_name, 'spiders', '__init__.py').exists() -def get_permissions_dict(path: str | os.PathLike[str], renamings=None, ignore=None) -> dict[str, str]: +def get_permissions_dict(path: Union[str, os.PathLike], renamings=None, ignore=None) -> Dict[str, str]: def get_permissions(path: Path) -> str: return oct(path.stat().st_mode) diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 98905d2c0..97c3a74b3 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -17,6 +17,7 @@ from logging import getLogger from os import PathLike from pathlib import Path from string import ascii_letters, digits +from typing import Union from unittest import mock from urllib.parse import urljoin, quote from urllib.request import pathname2url @@ -63,7 +64,7 @@ def printf_escape(string): return string.replace('%', '%%') -def build_url(path: str | PathLike[str]) -> str: +def build_url(path: Union[str, PathLike]) -> str: path_str = str(path) if path_str[0] != '/': path_str = '/' + path_str