mirror of https://github.com/scrapy/scrapy.git
Replace _SettingsKeyT with str. (#7586)
This commit is contained in:
Andrey Rakhmatullin
GitHub
parent
ddafb37a7c
commit
d59f9b644a
|
|
@ -16,10 +16,6 @@ from scrapy.utils.python import global_object_name
|
|||
|
||||
logger = getLogger(__name__)
|
||||
|
||||
# The key types are restricted in BaseSettings._get_key() to ones supported by JSON,
|
||||
# see https://github.com/scrapy/scrapy/issues/5383.
|
||||
_SettingsKey: TypeAlias = bool | float | int | str | None
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from types import ModuleType
|
||||
|
||||
|
|
@ -29,7 +25,7 @@ if TYPE_CHECKING:
|
|||
# typing.Self requires Python 3.11
|
||||
from typing_extensions import Self
|
||||
|
||||
_SettingsInput: TypeAlias = SupportsItems[_SettingsKey, Any] | str | None
|
||||
_SettingsInput: TypeAlias = SupportsItems[str, Any] | str | None
|
||||
|
||||
|
||||
SETTINGS_PRIORITIES: dict[str, int] = {
|
||||
|
|
@ -80,7 +76,7 @@ class SettingsAttribute:
|
|||
return f"<SettingsAttribute value={self.value!r} priority={self.priority}>"
|
||||
|
||||
|
||||
class BaseSettings(MutableMapping[_SettingsKey, Any]):
|
||||
class BaseSettings(MutableMapping[str, Any]):
|
||||
"""
|
||||
Instances of this class behave like dictionaries, but store priorities
|
||||
along with their ``(key, value)`` pairs, and can be frozen (i.e. marked
|
||||
|
|
@ -106,11 +102,11 @@ class BaseSettings(MutableMapping[_SettingsKey, Any]):
|
|||
|
||||
def __init__(self, values: _SettingsInput = None, priority: int | str = "project"):
|
||||
self.frozen: bool = False
|
||||
self.attributes: dict[_SettingsKey, SettingsAttribute] = {}
|
||||
self.attributes: dict[str, SettingsAttribute] = {}
|
||||
if values:
|
||||
self.update(values, priority)
|
||||
|
||||
def __getitem__(self, opt_name: _SettingsKey) -> Any:
|
||||
def __getitem__(self, opt_name: str) -> Any:
|
||||
if opt_name not in self:
|
||||
return None
|
||||
return self.attributes[opt_name].value
|
||||
|
|
@ -118,7 +114,7 @@ class BaseSettings(MutableMapping[_SettingsKey, Any]):
|
|||
def __contains__(self, name: Any) -> bool:
|
||||
return name in self.attributes
|
||||
|
||||
def add_to_list(self, name: _SettingsKey, item: Any) -> None:
|
||||
def add_to_list(self, name: str, item: Any) -> None:
|
||||
"""Append *item* to the :class:`list` setting with the specified *name*
|
||||
if *item* is not already in that list.
|
||||
|
||||
|
|
@ -129,7 +125,7 @@ class BaseSettings(MutableMapping[_SettingsKey, Any]):
|
|||
if item not in value:
|
||||
self.set(name, [*value, item], self.getpriority(name) or 0)
|
||||
|
||||
def remove_from_list(self, name: _SettingsKey, item: Any) -> None:
|
||||
def remove_from_list(self, name: str, item: Any) -> None:
|
||||
"""Remove *item* from the :class:`list` setting with the specified
|
||||
*name*.
|
||||
|
||||
|
|
@ -143,7 +139,7 @@ class BaseSettings(MutableMapping[_SettingsKey, Any]):
|
|||
raise ValueError(f"{item!r} not found in the {name} setting ({value!r}).")
|
||||
self.set(name, [v for v in value if v != item], self.getpriority(name) or 0)
|
||||
|
||||
def get(self, name: _SettingsKey, default: Any = None) -> Any: # pylint: disable=arguments-renamed
|
||||
def get(self, name: str, default: Any = None) -> Any: # pylint: disable=arguments-renamed
|
||||
"""
|
||||
Get a setting value without affecting its original type.
|
||||
|
||||
|
|
@ -172,7 +168,7 @@ class BaseSettings(MutableMapping[_SettingsKey, Any]):
|
|||
|
||||
return self[name] if self[name] is not None else default
|
||||
|
||||
def getbool(self, name: _SettingsKey, default: bool = False) -> bool:
|
||||
def getbool(self, name: str, default: bool = False) -> bool:
|
||||
"""
|
||||
Get a setting value as a boolean.
|
||||
|
||||
|
|
@ -202,7 +198,7 @@ class BaseSettings(MutableMapping[_SettingsKey, Any]):
|
|||
"'True'/'False' and 'true'/'false'"
|
||||
) from None
|
||||
|
||||
def getint(self, name: _SettingsKey, default: int = 0) -> int:
|
||||
def getint(self, name: str, default: int = 0) -> int:
|
||||
"""
|
||||
Get a setting value as an int.
|
||||
|
||||
|
|
@ -214,7 +210,7 @@ class BaseSettings(MutableMapping[_SettingsKey, Any]):
|
|||
"""
|
||||
return int(self.get(name, default))
|
||||
|
||||
def getfloat(self, name: _SettingsKey, default: float = 0.0) -> float:
|
||||
def getfloat(self, name: str, default: float = 0.0) -> float:
|
||||
"""
|
||||
Get a setting value as a float.
|
||||
|
||||
|
|
@ -226,9 +222,7 @@ class BaseSettings(MutableMapping[_SettingsKey, Any]):
|
|||
"""
|
||||
return float(self.get(name, default))
|
||||
|
||||
def getlist(
|
||||
self, name: _SettingsKey, default: list[Any] | None = None
|
||||
) -> list[Any]:
|
||||
def getlist(self, name: str, default: list[Any] | None = None) -> list[Any]:
|
||||
"""
|
||||
Get a setting value as a list. If the setting original type is a list,
|
||||
a copy of it will be returned. If it's a string it will be split by
|
||||
|
|
@ -251,7 +245,7 @@ class BaseSettings(MutableMapping[_SettingsKey, Any]):
|
|||
return list(value)
|
||||
|
||||
def getdict(
|
||||
self, name: _SettingsKey, default: dict[Any, Any] | None = None
|
||||
self, name: str, default: dict[Any, Any] | None = None
|
||||
) -> dict[Any, Any]:
|
||||
"""
|
||||
Get a setting value as a dictionary. If the setting original type is a
|
||||
|
|
@ -275,7 +269,7 @@ class BaseSettings(MutableMapping[_SettingsKey, Any]):
|
|||
|
||||
def getdictorlist(
|
||||
self,
|
||||
name: _SettingsKey,
|
||||
name: str,
|
||||
default: dict[Any, Any] | list[Any] | tuple[Any] | None = None,
|
||||
) -> dict[Any, Any] | list[Any]:
|
||||
"""Get a setting value as either a :class:`dict` or a :class:`list`.
|
||||
|
|
@ -322,7 +316,7 @@ class BaseSettings(MutableMapping[_SettingsKey, Any]):
|
|||
)
|
||||
return copy.deepcopy(value)
|
||||
|
||||
def getwithbase(self, name: _SettingsKey) -> BaseSettings:
|
||||
def getwithbase(self, name: str) -> BaseSettings:
|
||||
"""Get a composition of a dictionary-like setting and its ``_BASE``
|
||||
counterpart.
|
||||
|
||||
|
|
@ -341,7 +335,7 @@ class BaseSettings(MutableMapping[_SettingsKey, Any]):
|
|||
compbs.update(self[name])
|
||||
return compbs
|
||||
|
||||
def get_component_priority_dict_with_base(self, name: _SettingsKey) -> BaseSettings:
|
||||
def get_component_priority_dict_with_base(self, name: str) -> BaseSettings:
|
||||
"""Get a composition of a component priority dictionary setting and
|
||||
its ``_BASE`` counterpart.
|
||||
|
||||
|
|
@ -389,7 +383,7 @@ class BaseSettings(MutableMapping[_SettingsKey, Any]):
|
|||
{restore_key(k): v for k, v in result.items() if v is not None}
|
||||
)
|
||||
|
||||
def getpriority(self, name: _SettingsKey) -> int | None:
|
||||
def getpriority(self, name: str) -> int | None:
|
||||
"""
|
||||
Return the current numerical priority value of a setting, or ``None`` if
|
||||
the given ``name`` does not exist.
|
||||
|
|
@ -414,7 +408,7 @@ class BaseSettings(MutableMapping[_SettingsKey, Any]):
|
|||
|
||||
def replace_in_component_priority_dict(
|
||||
self,
|
||||
name: _SettingsKey,
|
||||
name: str,
|
||||
old_cls: type,
|
||||
new_cls: type,
|
||||
priority: int | None = None,
|
||||
|
|
@ -453,12 +447,10 @@ class BaseSettings(MutableMapping[_SettingsKey, Any]):
|
|||
)
|
||||
self.set(name, component_priority_dict, priority=self.getpriority(name) or 0)
|
||||
|
||||
def __setitem__(self, name: _SettingsKey, value: Any) -> None:
|
||||
def __setitem__(self, name: str, value: Any) -> None:
|
||||
self.set(name, value)
|
||||
|
||||
def set(
|
||||
self, name: _SettingsKey, value: Any, priority: int | str = "project"
|
||||
) -> None:
|
||||
def set(self, name: str, value: Any, priority: int | str = "project") -> None:
|
||||
"""
|
||||
Store a key/value attribute with a given priority.
|
||||
|
||||
|
|
@ -487,7 +479,7 @@ class BaseSettings(MutableMapping[_SettingsKey, Any]):
|
|||
self.attributes[name].set(value, priority)
|
||||
|
||||
def set_in_component_priority_dict(
|
||||
self, name: _SettingsKey, cls: type, priority: int | None
|
||||
self, name: str, cls: type, priority: int | None
|
||||
) -> None:
|
||||
"""Set the *cls* component in the *name* :ref:`component priority
|
||||
dictionary <component-priority-dictionaries>` setting with *priority*.
|
||||
|
|
@ -512,7 +504,7 @@ class BaseSettings(MutableMapping[_SettingsKey, Any]):
|
|||
|
||||
def setdefault( # pylint: disable=arguments-renamed
|
||||
self,
|
||||
name: _SettingsKey,
|
||||
name: str,
|
||||
default: Any = None,
|
||||
priority: int | str = "project",
|
||||
) -> Any:
|
||||
|
|
@ -523,7 +515,7 @@ class BaseSettings(MutableMapping[_SettingsKey, Any]):
|
|||
return self.attributes[name].value
|
||||
|
||||
def setdefault_in_component_priority_dict(
|
||||
self, name: _SettingsKey, cls: type, priority: int | None
|
||||
self, name: str, cls: type, priority: int | None
|
||||
) -> None:
|
||||
"""Set the *cls* component in the *name* :ref:`component priority
|
||||
dictionary <component-priority-dictionaries>` setting with *priority*
|
||||
|
|
@ -592,7 +584,7 @@ class BaseSettings(MutableMapping[_SettingsKey, Any]):
|
|||
"""
|
||||
self._assert_mutability()
|
||||
if isinstance(values, str):
|
||||
values = cast("dict[_SettingsKey, Any]", json.loads(values))
|
||||
values = cast("dict[str, Any]", json.loads(values))
|
||||
if values is not None:
|
||||
if isinstance(values, BaseSettings):
|
||||
for name, value in values.items():
|
||||
|
|
@ -601,7 +593,7 @@ class BaseSettings(MutableMapping[_SettingsKey, Any]):
|
|||
for name, value in values.items():
|
||||
self.set(name, value, priority)
|
||||
|
||||
def delete(self, name: _SettingsKey, priority: int | str = "project") -> None:
|
||||
def delete(self, name: str, priority: int | str = "project") -> None:
|
||||
if name not in self:
|
||||
raise KeyError(name)
|
||||
self._assert_mutability()
|
||||
|
|
@ -609,7 +601,7 @@ class BaseSettings(MutableMapping[_SettingsKey, Any]):
|
|||
if priority >= cast("int", self.getpriority(name)):
|
||||
del self.attributes[name]
|
||||
|
||||
def __delitem__(self, name: _SettingsKey) -> None:
|
||||
def __delitem__(self, name: str) -> None:
|
||||
self._assert_mutability()
|
||||
del self.attributes[name]
|
||||
|
||||
|
|
@ -649,26 +641,19 @@ class BaseSettings(MutableMapping[_SettingsKey, Any]):
|
|||
copy.freeze()
|
||||
return copy
|
||||
|
||||
def __iter__(self) -> Iterator[_SettingsKey]:
|
||||
def __iter__(self) -> Iterator[str]:
|
||||
return iter(self.attributes)
|
||||
|
||||
def __len__(self) -> int:
|
||||
return len(self.attributes)
|
||||
|
||||
def _to_dict(self) -> dict[_SettingsKey, Any]:
|
||||
def _to_dict(self) -> dict[str, Any]:
|
||||
return {
|
||||
self._get_key(k): (v._to_dict() if isinstance(v, BaseSettings) else v)
|
||||
str(k): (v._to_dict() if isinstance(v, BaseSettings) else v)
|
||||
for k, v in self.items()
|
||||
}
|
||||
|
||||
def _get_key(self, key_value: Any) -> _SettingsKey:
|
||||
return (
|
||||
key_value
|
||||
if isinstance(key_value, (bool, float, int, str, type(None)))
|
||||
else str(key_value)
|
||||
)
|
||||
|
||||
def copy_to_dict(self) -> dict[_SettingsKey, Any]:
|
||||
def copy_to_dict(self) -> dict[str, Any]:
|
||||
"""
|
||||
Make a copy of current settings and convert to a dict.
|
||||
|
||||
|
|
@ -691,7 +676,7 @@ class BaseSettings(MutableMapping[_SettingsKey, Any]):
|
|||
else:
|
||||
p.text(pformat(self.copy_to_dict()))
|
||||
|
||||
def pop(self, name: _SettingsKey, default: Any = __default) -> Any: # pylint: disable=arguments-renamed
|
||||
def pop(self, name: str, default: Any = __default) -> Any: # pylint: disable=arguments-renamed
|
||||
try:
|
||||
value = self.attributes[name].value
|
||||
except KeyError:
|
||||
|
|
@ -735,7 +720,7 @@ def iter_default_settings() -> Iterable[tuple[str, Any]]:
|
|||
|
||||
|
||||
def overridden_settings(
|
||||
settings: Mapping[_SettingsKey, Any],
|
||||
settings: Mapping[str, Any],
|
||||
) -> Iterable[tuple[str, Any]]:
|
||||
"""Return an iterable of the settings that have been overridden"""
|
||||
for name, defvalue in iter_default_settings():
|
||||
|
|
|
|||
|
|
@ -24,7 +24,7 @@ if TYPE_CHECKING:
|
|||
|
||||
from scrapy.crawler import Crawler
|
||||
from scrapy.http.request import CallbackT
|
||||
from scrapy.settings import BaseSettings, _SettingsKey
|
||||
from scrapy.settings import BaseSettings
|
||||
from scrapy.utils.log import SpiderLoggerAdapter
|
||||
|
||||
|
||||
|
|
@ -37,7 +37,7 @@ class Spider(object_ref):
|
|||
"""
|
||||
|
||||
name: str
|
||||
custom_settings: dict[_SettingsKey, Any] | None = None
|
||||
custom_settings: dict[str, Any] | None = None
|
||||
|
||||
#: Start URLs. See :meth:`start`.
|
||||
start_urls: list[str]
|
||||
|
|
|
|||
|
|
@ -12,7 +12,7 @@ from twisted.python import log as twisted_log
|
|||
from twisted.python.failure import Failure
|
||||
|
||||
import scrapy
|
||||
from scrapy.settings import Settings, _SettingsKey
|
||||
from scrapy.settings import Settings
|
||||
from scrapy.utils.versions import get_versions
|
||||
|
||||
if TYPE_CHECKING:
|
||||
|
|
@ -89,7 +89,7 @@ DEFAULT_LOGGING = {
|
|||
|
||||
|
||||
def configure_logging(
|
||||
settings: Settings | dict[_SettingsKey, Any] | None = None,
|
||||
settings: Settings | dict[str, Any] | None = None,
|
||||
install_root_handler: bool = True,
|
||||
) -> None:
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -22,7 +22,6 @@ from scrapy.utils.defer import deferred_to_future, maybe_deferred_to_future
|
|||
from scrapy.utils.test import get_from_asyncio_queue
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from scrapy.settings import _SettingsKey
|
||||
from tests.mockserver.http import MockServer
|
||||
|
||||
|
||||
|
|
@ -418,7 +417,7 @@ class CrawlSpiderWithParseMethod(MockServerSpider, CrawlSpider):
|
|||
"""
|
||||
|
||||
name = "crawl_spider_with_parse_method"
|
||||
custom_settings: dict[_SettingsKey, Any] = {
|
||||
custom_settings: dict[str, Any] = {
|
||||
"RETRY_HTTP_CODES": [], # no need to retry
|
||||
}
|
||||
rules = (Rule(LinkExtractor(), callback="parse", follow=True),)
|
||||
|
|
|
|||
|
|
@ -22,7 +22,7 @@ from scrapy.crawler import (
|
|||
)
|
||||
from scrapy.exceptions import ScrapyDeprecationWarning
|
||||
from scrapy.extensions.throttle import AutoThrottle
|
||||
from scrapy.settings import Settings, _SettingsKey, default_settings
|
||||
from scrapy.settings import Settings, default_settings
|
||||
from scrapy.utils.defer import ensure_awaitable, maybe_deferred_to_future
|
||||
from scrapy.utils.log import (
|
||||
_uninstall_scrapy_root_handler,
|
||||
|
|
@ -56,7 +56,7 @@ class TestBaseCrawler:
|
|||
|
||||
class TestCrawler(TestBaseCrawler):
|
||||
def test_populate_spidercls_settings(self) -> None:
|
||||
spider_settings: dict[_SettingsKey, Any] = {
|
||||
spider_settings: dict[str, Any] = {
|
||||
"TEST1": "spider",
|
||||
"TEST2": "spider",
|
||||
}
|
||||
|
|
|
|||
|
|
@ -384,15 +384,15 @@ class TestBaseSettings:
|
|||
"TEST_STRING": "a string",
|
||||
"TEST_LIST": [1, 2],
|
||||
"TEST_BOOLEAN": False,
|
||||
"TEST_BASE": BaseSettings({1: 1, 2: 2}, "project"),
|
||||
"TEST": BaseSettings({1: 10, 3: 30}, "default"),
|
||||
"HASNOBASE": BaseSettings({3: 3000}, "default"),
|
||||
"TEST_BASE": BaseSettings({"foo": 1, "bar": 2}, "project"),
|
||||
"TEST": BaseSettings({"foo": 10, "baz": 30}, "default"),
|
||||
"HASNOBASE": BaseSettings({"baz": 3000}, "default"),
|
||||
}
|
||||
)
|
||||
assert s.copy_to_dict() == {
|
||||
"HASNOBASE": {3: 3000},
|
||||
"TEST": {1: 10, 3: 30},
|
||||
"TEST_BASE": {1: 1, 2: 2},
|
||||
"HASNOBASE": {"baz": 3000},
|
||||
"TEST": {"foo": 10, "baz": 30},
|
||||
"TEST_BASE": {"foo": 1, "bar": 2},
|
||||
"TEST_LIST": [1, 2],
|
||||
"TEST_BOOLEAN": False,
|
||||
"TEST_STRING": "a string",
|
||||
|
|
|
|||
Loading…
Reference in New Issue