RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Next refactoring pass of test_utils_*. (#7797)

This commit is contained in:
Andrey Rakhmatullin 2026-07-28 17:24:34 +05:00 committed by GitHub
parent ad816d2b3a
commit e7d8b34e73
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
33 changed files with 619 additions and 503 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
pyproject.toml
View File

@ -178,13 +178,6 @@ module = [
"tests.test_squeues",
"tests.test_squeues_request",
"tests.test_stats",
"tests.test_utils_datatypes",
"tests.test_utils_decorators",
"tests.test_utils_defer",
"tests.test_utils_deprecate",
"tests.test_utils_misc.test_return_with_argument_inside_generator",
"tests.test_utils_python",
"tests.test_utils_request",
"tests.utils.bases.http_request",
"tests.utils.bases.http_response",
"tests.utils.bases.spider",

3
scrapy/core/downloader/handlers/ftp.py
View File

@ -126,5 +126,4 @@ class FTPDownloadHandler(BaseDownloadHandler):
headers = {"local filename": protocol.filename or b"", "size": protocol.size}
body = protocol.filename or protocol.body.read()
respcls = responsetypes.from_args(url=request.url, body=body)
# hints for Headers-related types may need to be fixed to not use AnyStr
return respcls(url=request.url, status=200, body=body, headers=headers) # type: ignore[arg-type]
return respcls(url=request.url, status=200, body=body, headers=headers)

26
scrapy/http/headers.py
View File

@ -1,7 +1,7 @@
from __future__ import annotations
from collections.abc import Mapping
from typing import TYPE_CHECKING, Any, AnyStr, TypeAlias, cast
from typing import TYPE_CHECKING, Any, TypeAlias, cast
from w3lib.http import headers_dict_to_raw
@ -25,14 +25,20 @@ class Headers(CaselessDict):
def __init__(
self,
seq: Mapping[AnyStr, Any] | Iterable[tuple[AnyStr, Any]] | None = None,
seq: Mapping[str, Any]
| Mapping[bytes, Any]
| Iterable[tuple[str | bytes, Any]]
| None = None,
encoding: str = "utf-8",
):
self.encoding: str = encoding
super().__init__(seq)
def update( # type: ignore[override]
self, seq: Mapping[AnyStr, Any] | Iterable[tuple[AnyStr, Any]]
self,
seq: Mapping[str, Any]
| Mapping[bytes, Any]
| Iterable[tuple[str | bytes, Any]],
) -> None:
seq = seq.items() if isinstance(seq, Mapping) else seq
iseq: dict[bytes, list[bytes]] = {}
@ -40,7 +46,7 @@ class Headers(CaselessDict):
iseq.setdefault(self.normkey(k), []).extend(self.normvalue(v))
super().update(iseq)
def normkey(self, key: AnyStr) -> bytes: # type: ignore[override]
def normkey(self, key: str | bytes) -> bytes:
"""Normalize key to bytes"""
return self._tobytes(key.title())
@ -67,19 +73,19 @@ class Headers(CaselessDict):
return str(x).encode(self.encoding)
raise TypeError(f"Unsupported value type: {type(x)}")
def __getitem__(self, key: AnyStr) -> bytes | None:
def __getitem__(self, key: str | bytes) -> bytes | None:
try:
return cast("list[bytes]", super().__getitem__(key))[-1]
except IndexError:
return None
def get(self, key: AnyStr, def_val: Any = None) -> bytes | None:
def get(self, key: str | bytes, def_val: Any = None) -> bytes | None:
try:
return cast("list[bytes]", super().get(key, def_val))[-1]
except IndexError:
return None
def getlist(self, key: AnyStr, def_val: Any = None) -> list[bytes]:
def getlist(self, key: str | bytes, def_val: Any = None) -> list[bytes]:
try:
return cast("list[bytes]", super().__getitem__(key))
except KeyError:
@ -87,15 +93,15 @@ class Headers(CaselessDict):
return self.normvalue(def_val)
return []
def setlist(self, key: AnyStr, list_: Iterable[_RawValue]) -> None:
def setlist(self, key: str | bytes, list_: Iterable[_RawValue]) -> None:
self[key] = list_
def setlistdefault(
self, key: AnyStr, default_list: Iterable[_RawValue] = ()
self, key: str | bytes, default_list: Iterable[_RawValue] = ()
) -> Any:
return self.setdefault(key, default_list)
def appendlist(self, key: AnyStr, value: Iterable[_RawValue]) -> None:
def appendlist(self, key: str | bytes, value: Iterable[_RawValue]) -> None:
lst = self.getlist(key)
lst.extend(self.normvalue(value))
self[key] = lst

12
scrapy/http/request/__init__.py
View File

@ -11,7 +11,6 @@ import inspect
from typing import (
TYPE_CHECKING,
Any,
AnyStr,
Concatenate,
NoReturn,
TypeAlias,
@ -125,7 +124,10 @@ class Request(object_ref):
url: str,
callback: CallbackT | None = None,
method: str = "GET",
headers: Mapping[AnyStr, Any] | Iterable[tuple[AnyStr, Any]] | None = None,
headers: Mapping[str, Any]
| Mapping[bytes, Any]
| Iterable[tuple[str | bytes, Any]]
| None = None,
body: bytes | str | None = None,
cookies: CookiesT | None = None,
meta: dict[str, Any] | None = None,
@ -310,7 +312,11 @@ class Request(object_ref):
@headers.setter
def headers(
self, value: Mapping[AnyStr, Any] | Iterable[tuple[AnyStr, Any]] | None
self,
value: Mapping[str, Any]
| Mapping[bytes, Any]
| Iterable[tuple[str | bytes, Any]]
| None,
) -> None:
if isinstance(value, Headers):
self._headers = value

23
scrapy/http/response/__init__.py
View File

@ -7,7 +7,7 @@ See documentation in docs/topics/request-response.rst
from __future__ import annotations
from typing import TYPE_CHECKING, Any, AnyStr, TypeVar, overload
from typing import TYPE_CHECKING, Any, TypeVar, overload
from urllib.parse import urljoin
from scrapy.exceptions import NotSupported
@ -72,7 +72,10 @@ class Response(object_ref):
self,
url: str,
status: int = 200,
headers: Mapping[AnyStr, Any] | Iterable[tuple[AnyStr, Any]] | None = None,
headers: Mapping[str, Any]
| Mapping[bytes, Any]
| Iterable[tuple[str | bytes, Any]]
| None = None,
body: bytes = b"",
flags: list[str] | None = None,
request: Request | None = None,
@ -145,7 +148,11 @@ class Response(object_ref):
@headers.setter
def headers(
self, value: Mapping[AnyStr, Any] | Iterable[tuple[AnyStr, Any]] | None
self,
value: Mapping[str, Any]
| Mapping[bytes, Any]
| Iterable[tuple[str | bytes, Any]]
| None,
) -> None:
if isinstance(value, Headers):
self._headers = value
@ -222,7 +229,10 @@ class Response(object_ref):
url: str | Link,
callback: CallbackT | None = None,
method: str = "GET",
headers: Mapping[AnyStr, Any] | Iterable[tuple[AnyStr, Any]] | None = None,
headers: Mapping[str, Any]
| Mapping[bytes, Any]
| Iterable[tuple[str | bytes, Any]]
| None = None,
body: bytes | str | None = None,
cookies: CookiesT | None = None,
meta: dict[str, Any] | None = None,
@ -272,7 +282,10 @@ class Response(object_ref):
urls: Iterable[str | Link],
callback: CallbackT | None = None,
method: str = "GET",
headers: Mapping[AnyStr, Any] | Iterable[tuple[AnyStr, Any]] | None = None,
headers: Mapping[str, Any]
| Mapping[bytes, Any]
| Iterable[tuple[str | bytes, Any]]
| None = None,
body: bytes | str | None = None,
cookies: CookiesT | None = None,
meta: dict[str, Any] | None = None,

12
scrapy/http/response/text.py
View File

@ -9,7 +9,7 @@ from __future__ import annotations
import json
from contextlib import suppress
from typing import TYPE_CHECKING, Any, AnyStr, cast
from typing import TYPE_CHECKING, Any, cast
from urllib.parse import urljoin
import parsel
@ -170,7 +170,10 @@ class TextResponse(Response):
url: str | Link | parsel.Selector,
callback: CallbackT | None = None,
method: str = "GET",
headers: Mapping[AnyStr, Any] | Iterable[tuple[AnyStr, Any]] | None = None,
headers: Mapping[str, Any]
| Mapping[bytes, Any]
| Iterable[tuple[str | bytes, Any]]
| None = None,
body: bytes | str | None = None,
cookies: CookiesT | None = None,
meta: dict[str, Any] | None = None,
@ -223,7 +226,10 @@ class TextResponse(Response):
urls: Iterable[str | Link] | parsel.SelectorList[Any] | None = None,
callback: CallbackT | None = None,
method: str = "GET",
headers: Mapping[AnyStr, Any] | Iterable[tuple[AnyStr, Any]] | None = None,
headers: Mapping[str, Any]
| Mapping[bytes, Any]
| Iterable[tuple[str | bytes, Any]]
| None = None,
body: bytes | str | None = None,
cookies: CookiesT | None = None,
meta: dict[str, Any] | None = None,

40
scrapy/utils/datatypes.py
View File

@ -11,12 +11,12 @@ import warnings
import weakref
from collections import OrderedDict
from collections.abc import Mapping
from typing import TYPE_CHECKING, Any, AnyStr, TypeVar, cast
from typing import TYPE_CHECKING, Any, TypeVar, cast
from scrapy.exceptions import ScrapyDeprecationWarning
if TYPE_CHECKING:
from collections.abc import Iterable, Sequence
from collections.abc import Container, Iterable
# typing.Self requires Python 3.11
from typing_extensions import Self
@ -44,22 +44,25 @@ class CaselessDict(dict): # type: ignore[type-arg]
def __init__(
self,
seq: Mapping[AnyStr, Any] | Iterable[tuple[AnyStr, Any]] | None = None,
seq: Mapping[str, Any]
| Mapping[bytes, Any]
| Iterable[tuple[str | bytes, Any]]
| None = None,
):
super().__init__()
if seq:
self.update(seq)
def __getitem__(self, key: AnyStr) -> Any:
def __getitem__(self, key: str | bytes) -> Any:
return dict.__getitem__(self, self.normkey(key))
def __setitem__(self, key: AnyStr, value: Any) -> None:
def __setitem__(self, key: str | bytes, value: Any) -> None:
dict.__setitem__(self, self.normkey(key), self.normvalue(value))
def __delitem__(self, key: AnyStr) -> None:
def __delitem__(self, key: str | bytes) -> None:
dict.__delitem__(self, self.normkey(key))
def __contains__(self, key: AnyStr) -> bool: # type: ignore[override]
def __contains__(self, key: str | bytes) -> bool: # type: ignore[override]
return dict.__contains__(self, self.normkey(key))
has_key = __contains__
@ -69,7 +72,7 @@ class CaselessDict(dict): # type: ignore[type-arg]
copy = __copy__
def normkey(self, key: AnyStr) -> AnyStr:
def normkey(self, key: str | bytes) -> str | bytes:
"""Method to normalize dictionary key access"""
return key.lower()
@ -77,23 +80,28 @@ class CaselessDict(dict): # type: ignore[type-arg]
"""Method to normalize values prior to be set"""
return value
def get(self, key: AnyStr, def_val: Any = None) -> Any:
def get(self, key: str | bytes, def_val: Any = None) -> Any:
return dict.get(self, self.normkey(key), self.normvalue(def_val))
def setdefault(self, key: AnyStr, def_val: Any = None) -> Any:
def setdefault(self, key: str | bytes, def_val: Any = None) -> Any:
return dict.setdefault(self, self.normkey(key), self.normvalue(def_val))
# doesn't fully implement MutableMapping.update()
def update(self, seq: Mapping[AnyStr, Any] | Iterable[tuple[AnyStr, Any]]) -> None: # type: ignore[override]
def update( # type: ignore[override]
self,
seq: Mapping[str, Any]
| Mapping[bytes, Any]
| Iterable[tuple[str | bytes, Any]],
) -> None:
seq = seq.items() if isinstance(seq, Mapping) else seq
iseq = ((self.normkey(k), self.normvalue(v)) for k, v in seq)
super().update(iseq)
@classmethod
def fromkeys(cls, keys: Iterable[AnyStr], value: Any = None) -> Self: # type: ignore[override]
return cls((k, value) for k in keys) # type: ignore[misc]
def fromkeys(cls, keys: Iterable[str | bytes], value: Any = None) -> Self: # type: ignore[override]
return cls((k, value) for k in keys)
def pop(self, key: AnyStr, *args: Any) -> Any:
def pop(self, key: str | bytes, *args: Any) -> Any:
return dict.pop(self, self.normkey(key), *args)
@ -205,8 +213,8 @@ class LocalWeakReferencedCache(weakref.WeakKeyDictionary[_KT, _VT | None]):
class SequenceExclude:
"""Object to test if an item is NOT within some sequence."""
def __init__(self, seq: Sequence[Any]):
self.seq: Sequence[Any] = seq
def __init__(self, seq: Container[Any]):
self.seq: Container[Any] = seq
def __contains__(self, item: Any) -> bool:
return item not in self.seq

17
scrapy/utils/decorators.py
View File

@ -19,9 +19,19 @@ _T = TypeVar("_T")
_P = ParamSpec("_P")
@overload
def deprecated(use_instead: Callable[_P, _T]) -> Callable[_P, _T]: ...
@overload
def deprecated(
use_instead: Any = None,
) -> Callable[[Callable[_P, _T]], Callable[_P, _T]]:
use_instead: str | None = None,
) -> Callable[[Callable[_P, _T]], Callable[_P, _T]]: ...
def deprecated(
use_instead: Callable[_P, _T] | str | None = None,
) -> Callable[_P, _T] | Callable[[Callable[_P, _T]], Callable[_P, _T]]:
"""This is a decorator which can be used to mark functions
as deprecated. It will result in a warning being emitted
when the function is used."""
@ -38,8 +48,9 @@ def deprecated(
return wrapped
if callable(use_instead):
deco = deco(use_instead)
func = use_instead
use_instead = None
return deco(func)
return deco

24
tests/test_utils_asyncgen.py
View File

@ -1,16 +1,18 @@
from __future__ import annotations
from scrapy.utils.asyncgen import as_async_generator, collect_asyncgen
from tests.utils.decorators import coroutine_test
class TestAsyncgenUtils:
@coroutine_test
async def test_as_async_generator(self):
ag = as_async_generator(range(42))
results = [i async for i in ag]
assert results == list(range(42))
@coroutine_test
async def test_as_async_generator():
ag = as_async_generator(range(42))
results = [i async for i in ag]
assert results == list(range(42))
@coroutine_test
async def test_collect_asyncgen(self):
ag = as_async_generator(range(42))
results = await collect_asyncgen(ag)
assert results == list(range(42))
@coroutine_test
async def test_collect_asyncgen():
ag = as_async_generator(range(42))
results = await collect_asyncgen(ag)
assert results == list(range(42))

9
tests/test_utils_asyncio.py
View File

@ -20,11 +20,10 @@ if TYPE_CHECKING:
from collections.abc import AsyncGenerator
class TestAsyncio:
@coroutine_test
async def test_is_asyncio_available(self, reactor_pytest: str) -> None:
# the result should depend only on the pytest --reactor argument
assert is_asyncio_available() == (reactor_pytest != "default")
@coroutine_test
async def test_is_asyncio_available(reactor_pytest: str) -> None:
# the result should depend only on the pytest --reactor argument
assert is_asyncio_available() == (reactor_pytest != "default")
@pytest.mark.only_asyncio

2
tests/test_utils_curl.py
View File

@ -1,3 +1,5 @@
from __future__ import annotations
import warnings
from typing import Any

60
tests/test_utils_datatypes.py
View File

@ -1,7 +1,9 @@
from __future__ import annotations
import copy
from abc import ABC, abstractmethod
from collections.abc import Iterator, Mapping, MutableMapping
from typing import Any
from typing import Any, Generic, TypeVar
import pytest
@ -16,11 +18,13 @@ from scrapy.utils.datatypes import (
)
from scrapy.utils.python import garbage_collect
_DictT = TypeVar("_DictT", bound="CaselessDict | CaseInsensitiveDict")
class TestCaseInsensitiveDictBase(ABC):
class TestCaseInsensitiveDictBase(ABC, Generic[_DictT]):
@property
@abstractmethod
def dict_class(self) -> type[MutableMapping[str, Any]]:
def dict_class(self) -> type[_DictT]:
raise NotImplementedError
def test_init_dict(self):
@ -36,17 +40,17 @@ class TestCaseInsensitiveDictBase(ABC):
assert d["black"] == 3
def test_init_mapping(self):
class MyMapping(Mapping):
def __init__(self, **kwargs):
class MyMapping(Mapping[str, int]):
def __init__(self, **kwargs: int) -> None:
self._d = kwargs
def __getitem__(self, key):
def __getitem__(self, key: str) -> int:
return self._d[key]
def __iter__(self):
def __iter__(self) -> Iterator[str]:
return iter(self._d)
def __len__(self):
def __len__(self) -> int:
return len(self._d)
seq = MyMapping(red=1, black=3)
@ -55,23 +59,23 @@ class TestCaseInsensitiveDictBase(ABC):
assert d["black"] == 3
def test_init_mutable_mapping(self):
class MyMutableMapping(MutableMapping):
def __init__(self, **kwargs):
class MyMutableMapping(MutableMapping[str, int]):
def __init__(self, **kwargs: int) -> None:
self._d = kwargs
def __getitem__(self, key):
def __getitem__(self, key: str) -> int:
return self._d[key]
def __setitem__(self, key, value):
def __setitem__(self, key: str, value: int) -> None:
self._d[key] = value
def __delitem__(self, key):
def __delitem__(self, key: str) -> None:
del self._d[key]
def __iter__(self):
def __iter__(self) -> Iterator[str]:
return iter(self._d)
def __len__(self):
def __len__(self) -> int:
return len(self._d)
seq = MyMutableMapping(red=1, black=3)
@ -149,7 +153,7 @@ class TestCaseInsensitiveDictBase(ABC):
d.pop("A")
def test_normkey(self):
class MyDict(self.dict_class):
class MyDict(self.dict_class): # type: ignore[misc,name-defined]
def _normkey(self, key):
return key.title()
@ -160,7 +164,7 @@ class TestCaseInsensitiveDictBase(ABC):
assert list(d.keys()) == ["Key-One"]
def test_normvalue(self):
class MyDict(self.dict_class):
class MyDict(self.dict_class): # type: ignore[misc,name-defined]
def _normvalue(self, value):
if value is not None:
return value + 1
@ -214,8 +218,8 @@ class TestCaseInsensitiveDictBase(ABC):
assert dict(h1) == {"header1": "value1", "header2": "value2"}
class TestCaseInsensitiveDict(TestCaseInsensitiveDictBase):
dict_class = CaseInsensitiveDict # type: ignore[assignment]
class TestCaseInsensitiveDict(TestCaseInsensitiveDictBase[CaseInsensitiveDict]):
dict_class = CaseInsensitiveDict
def test_repr(self):
d1 = self.dict_class({"foo": "bar"})
@ -230,7 +234,7 @@ class TestCaseInsensitiveDict(TestCaseInsensitiveDictBase):
assert list(iterkeys) == ["AsDf", "FoO"]
def test_copy_keeps_values(self):
class MyDict(self.dict_class):
class MyDict(self.dict_class): # type: ignore[misc,name-defined]
def _normvalue(self, value):
return value + 1
@ -253,7 +257,7 @@ class TestCaseInsensitiveDict(TestCaseInsensitiveDictBase):
@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
class TestCaselessDict(TestCaseInsensitiveDictBase):
class TestCaselessDict(TestCaseInsensitiveDictBase[CaselessDict]):
dict_class = CaselessDict
def test_deprecation_message(self):
@ -319,7 +323,7 @@ class TestSequenceExclude:
class TestLocalCache:
def test_cache_with_limit(self):
cache = LocalCache(limit=2)
cache: LocalCache[str, int] = LocalCache(limit=2)
cache["a"] = 1
cache["b"] = 2
cache["c"] = 3
@ -332,7 +336,7 @@ class TestLocalCache:
def test_cache_without_limit(self):
maximum = 10**4
cache = LocalCache()
cache: LocalCache[str, int] = LocalCache()
for x in range(maximum):
cache[str(x)] = x
assert len(cache) == maximum
@ -341,7 +345,7 @@ class TestLocalCache:
assert cache[str(x)] == x
def test_cache_with_zero_limit(self):
cache = LocalCache(limit=0)
cache: LocalCache[str, int] = LocalCache(limit=0)
cache["a"] = 1
cache["b"] = 2
cache["c"] = 3
@ -353,7 +357,9 @@ class TestLocalCache:
class TestLocalWeakReferencedCache:
def test_cache_with_limit(self):
cache = LocalWeakReferencedCache(limit=2)
cache: LocalWeakReferencedCache[Request, int] = LocalWeakReferencedCache(
limit=2
)
r1 = Request("https://example.org")
r2 = Request("https://example.com")
r3 = Request("https://example.net")
@ -375,7 +381,7 @@ class TestLocalWeakReferencedCache:
assert len(cache) == 1
def test_cache_non_weak_referenceable_objects(self):
cache = LocalWeakReferencedCache()
cache: LocalWeakReferencedCache[Any, int] = LocalWeakReferencedCache()
k1 = None
k2 = 1
k3 = [1, 2, 3]
@ -389,7 +395,7 @@ class TestLocalWeakReferencedCache:
def test_cache_without_limit(self):
maximum = 10**4
cache = LocalWeakReferencedCache()
cache: LocalWeakReferencedCache[Request, int] = LocalWeakReferencedCache()
refs = []
for x in range(maximum):
refs.append(Request(f"https://example.org/{x}"))

20
tests/test_utils_decorators.py
View File

@ -1,6 +1,7 @@
from __future__ import annotations
import warnings
from typing import TYPE_CHECKING
import pytest
from twisted.internet.defer import Deferred
@ -10,11 +11,14 @@ from scrapy.utils.decorators import _warn_spider_arg, deprecated, inthread
from scrapy.utils.defer import maybe_deferred_to_future
from tests.utils.decorators import coroutine_test
if TYPE_CHECKING:
from collections.abc import AsyncGenerator
class TestDeprecated:
def test_warns_and_still_calls(self):
@deprecated()
def add(a, b):
def add(a: int, b: int) -> int:
return a + b
with pytest.warns(
@ -26,7 +30,7 @@ class TestDeprecated:
def test_use_instead_in_message(self):
@deprecated(use_instead="other_function")
def old():
def old() -> None:
return None
with pytest.warns(
@ -37,7 +41,7 @@ class TestDeprecated:
def test_applied_without_parentheses(self):
@deprecated
def square(x):
def square(x: int) -> int:
return x * x
with pytest.warns(
@ -65,7 +69,7 @@ class TestInthread:
class TestWarnSpiderArg:
def test_sync_warns_with_spider_arg(self):
@_warn_spider_arg
def parse(response, spider=None):
def parse(response: str, spider: str | None = None) -> str:
return response
with pytest.warns(
@ -75,7 +79,7 @@ class TestWarnSpiderArg:
def test_sync_no_warning_without_spider_arg(self):
@_warn_spider_arg
def parse(response, spider=None):
def parse(response: str, spider: str | None = None) -> str:
return response
with warnings.catch_warnings():
@ -85,7 +89,7 @@ class TestWarnSpiderArg:
@coroutine_test
async def test_async_warns_with_spider_arg(self):
@_warn_spider_arg
async def parse(response, spider=None):
async def parse(response: str, spider: str | None = None) -> str:
return response
with pytest.warns(
@ -96,7 +100,9 @@ class TestWarnSpiderArg:
@coroutine_test
async def test_asyncgen_warns_with_spider_arg(self):
@_warn_spider_arg
async def parse(response, spider=None):
async def parse(
response: str, spider: str | None = None
) -> AsyncGenerator[str]:
yield response
with pytest.warns(

24
tests/test_utils_defer.py
View File

@ -24,6 +24,8 @@ from tests.utils.decorators import coroutine_test, inline_callbacks_test
if TYPE_CHECKING:
from collections.abc import AsyncGenerator, Awaitable, Callable, Generator
from twisted.python.failure import Failure
@pytest.mark.requires_reactor # mustbe_deferred() requires a reactor
@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
@ -70,7 +72,7 @@ class TestIterErrback:
def itergood() -> Generator[int, None, None]:
yield from range(10)
errors = []
errors: list[Failure] = []
out = list(iter_errback(itergood(), errors.append))
assert out == list(range(10))
assert not errors
@ -82,7 +84,7 @@ class TestIterErrback:
1 / 0
yield x
errors = []
errors: list[Failure] = []
out = list(iter_errback(iterbad(), errors.append))
assert out == [0, 1, 2, 3, 4]
assert len(errors) == 1
@ -96,7 +98,7 @@ class TestAiterErrback:
for x in range(10):
yield x
errors = []
errors: list[Failure] = []
out = await collect_asyncgen(aiter_errback(itergood(), errors.append))
assert out == list(range(10))
assert not errors
@ -109,7 +111,7 @@ class TestAiterErrback:
1 / 0
yield x
errors = []
errors: list[Failure] = []
out = await collect_asyncgen(aiter_errback(iterbad(), errors.append))
assert out == [0, 1, 2, 3, 4]
assert len(errors) == 1
@ -202,7 +204,7 @@ class TestParallelAsync:
for length in [20, 50, 100]:
parallel_count = [0]
max_parallel_count = [0]
results = []
results: list[int] = []
ait = self.get_async_iterable(length)
dl = parallel_async(
ait,
@ -222,7 +224,7 @@ class TestParallelAsync:
for length in [20, 50, 100]:
parallel_count = [0]
max_parallel_count = [0]
results = []
results: list[int] = []
ait = self.get_async_iterable_with_delays(length)
dl = parallel_async(
ait,
@ -240,7 +242,7 @@ class TestParallelAsync:
class TestDeferredFromCoro:
def test_deferred(self):
d = Deferred()
d: Deferred[None] = Deferred()
result = deferred_from_coro(d)
assert isinstance(result, Deferred)
assert result is d
@ -274,7 +276,7 @@ class TestDeferredFromCoro:
@pytest.mark.only_asyncio
@inline_callbacks_test
def test_future(self):
future = Future()
future: Future[int] = Future()
result = deferred_from_coro(future)
assert isinstance(result, Deferred)
future.set_result(42)
@ -324,7 +326,7 @@ class TestDeferredFFromCoroF:
class TestDeferredToFuture:
@coroutine_test
async def test_deferred(self):
d = Deferred()
d: Deferred[int] = Deferred()
result = deferred_to_future(d)
assert isinstance(result, Future)
d.callback(42)
@ -359,7 +361,7 @@ class TestDeferredToFuture:
class TestMaybeDeferredToFutureAsyncio:
@coroutine_test
async def test_deferred(self):
d = Deferred()
d: Deferred[int] = Deferred()
result = maybe_deferred_to_future(d)
assert isinstance(result, Future)
d.callback(42)
@ -394,7 +396,7 @@ class TestMaybeDeferredToFutureAsyncio:
class TestMaybeDeferredToFutureNotAsyncio:
@coroutine_test
async def test_deferred(self):
d = Deferred()
d: Deferred[int] = Deferred()
result = maybe_deferred_to_future(d)
assert isinstance(result, Deferred)
assert result is d

46
tests/test_utils_deprecate.py
View File

@ -1,3 +1,5 @@
from __future__ import annotations
import inspect
import warnings
from unittest import mock
@ -38,7 +40,7 @@ class TestWarnWhenSubclassed:
)
with pytest.warns(MyWarning, match=msg) as w:
class UserClass(Deprecated):
class UserClass(Deprecated): # type: ignore[misc, valid-type]
pass
assert w[0].lineno == inspect.getsourcelines(UserClass)[1]
@ -57,7 +59,7 @@ class TestWarnWhenSubclassed:
match=r"UserClass inherits from deprecated class bar\.OldClass, please inherit from foo\.NewClass",
):
class UserClass(Deprecated):
class UserClass(Deprecated): # type: ignore[misc, valid-type]
pass
with pytest.warns(
@ -76,7 +78,7 @@ class TestWarnWhenSubclassed:
match="UserClass inherits from deprecated class",
):
class UserClass(Deprecated):
class UserClass(Deprecated): # type: ignore[misc, valid-type]
pass
with warnings.catch_warnings():
@ -95,16 +97,16 @@ class TestWarnWhenSubclassed:
match="UserClass inherits from deprecated class",
):
class UserClass(Deprecated):
class UserClass(Deprecated): # type: ignore[misc, valid-type]
pass
with warnings.catch_warnings():
warnings.simplefilter("error", MyWarning)
class FooClass(Deprecated):
class FooClass(Deprecated): # type: ignore[misc, valid-type]
pass
class BarClass(Deprecated):
class BarClass(Deprecated): # type: ignore[misc, valid-type]
pass
def test_warning_on_instance(self):
@ -112,22 +114,20 @@ class TestWarnWhenSubclassed:
"Deprecated", NewName, warn_category=MyWarning
)
with pytest.warns(MyWarning) as w:
_, lineno = Deprecated(), inspect.getlineno(inspect.currentframe())
w = [x for x in w if x.category is MyWarning]
with pytest.warns(
MyWarning,
match=r"tests\.test_utils_deprecate\.Deprecated is deprecated, "
r"instantiate tests\.test_utils_deprecate\.NewName instead\.",
) as w:
_, lineno = Deprecated(), inspect.getlineno(inspect.currentframe()) # type: ignore[arg-type]
assert len(w) == 1
assert (
str(w[0].message) == "tests.test_utils_deprecate.Deprecated is deprecated, "
"instantiate tests.test_utils_deprecate.NewName instead."
)
assert w[0].lineno == lineno
# ignore subclassing warnings
with warnings.catch_warnings():
warnings.simplefilter("ignore", MyWarning)
class UserClass(Deprecated):
class UserClass(Deprecated): # type: ignore[misc, valid-type]
pass
with warnings.catch_warnings():
@ -141,7 +141,7 @@ class TestWarnWhenSubclassed:
match=r"UserClass2 inherits from deprecated class tests\.test_utils_deprecate\.Deprecated, please inherit from tests\.test_utils_deprecate\.NewName",
):
class UserClass2(Deprecated):
class UserClass2(Deprecated): # type: ignore[misc, valid-type]
pass
def test_issubclass(self):
@ -155,10 +155,10 @@ class TestWarnWhenSubclassed:
class UpdatedUserClass1a(NewName):
pass
class OutdatedUserClass1(DeprecatedName):
class OutdatedUserClass1(DeprecatedName): # type: ignore[misc, valid-type]
pass
class OutdatedUserClass1a(DeprecatedName):
class OutdatedUserClass1a(DeprecatedName): # type: ignore[misc, valid-type]
pass
class UnrelatedClass:
@ -174,7 +174,7 @@ class TestWarnWhenSubclassed:
assert not issubclass(OutdatedUserClass1a, OutdatedUserClass1)
with pytest.raises(TypeError):
issubclass(object(), DeprecatedName)
issubclass(object(), DeprecatedName) # type: ignore[arg-type]
def test_isinstance(self):
with warnings.catch_warnings():
@ -187,10 +187,10 @@ class TestWarnWhenSubclassed:
class UpdatedUserClass2a(NewName):
pass
class OutdatedUserClass2(DeprecatedName):
class OutdatedUserClass2(DeprecatedName): # type: ignore[misc, valid-type]
pass
class OutdatedUserClass2a(DeprecatedName):
class OutdatedUserClass2a(DeprecatedName): # type: ignore[misc, valid-type]
pass
class UnrelatedClass:
@ -211,7 +211,7 @@ class TestWarnWhenSubclassed:
warnings.simplefilter("ignore", ScrapyDeprecationWarning)
Deprecated = create_deprecated_class("Deprecated", NewName, {"foo": "bar"})
assert Deprecated.foo == "bar"
assert Deprecated.foo == "bar" # type: ignore[attr-defined]
def test_deprecate_a_class_with_custom_metaclass(self):
Meta1 = type("Meta1", (type,), {})
@ -242,7 +242,7 @@ class TestWarnWhenSubclassed:
match=r"UserClass inherits from deprecated class tests\.test_utils_deprecate\.AlsoDeprecated, please inherit from foo\.Bar",
):
class UserClass(AlsoDeprecated):
class UserClass(AlsoDeprecated): # type: ignore[misc, valid-type]
pass
def test_inspect_stack(self):

16
tests/test_utils_display.py
View File

@ -31,13 +31,13 @@ plain_string = "{'a': 1}"
@mock.patch("sys.platform", "linux")
@mock.patch("sys.stdout.isatty")
def test_pformat(isatty):
def test_pformat(isatty: mock.Mock) -> None:
isatty.return_value = True
assert pformat(value) in colorized_strings
@mock.patch("sys.stdout.isatty")
def test_pformat_dont_colorize(isatty):
def test_pformat_dont_colorize(isatty: mock.Mock) -> None:
isatty.return_value = True
assert pformat(value, colorize=False) == plain_string
@ -49,7 +49,7 @@ def test_pformat_not_tty():
@mock.patch("sys.platform", "win32")
@mock.patch("platform.version")
@mock.patch("sys.stdout.isatty")
def test_pformat_old_windows(isatty, version):
def test_pformat_old_windows(isatty: mock.Mock, version: mock.Mock) -> None:
isatty.return_value = True
version.return_value = "10.0.14392"
assert pformat(value) in colorized_strings
@ -59,7 +59,9 @@ def test_pformat_old_windows(isatty, version):
@mock.patch("scrapy.utils.display._enable_windows_terminal_processing")
@mock.patch("platform.version")
@mock.patch("sys.stdout.isatty")
def test_pformat_windows_no_terminal_processing(isatty, version, terminal_processing):
def test_pformat_windows_no_terminal_processing(
isatty: mock.Mock, version: mock.Mock, terminal_processing: mock.Mock
) -> None:
isatty.return_value = True
version.return_value = "10.0.14393"
terminal_processing.return_value = False
@ -70,7 +72,9 @@ def test_pformat_windows_no_terminal_processing(isatty, version, terminal_proces
@mock.patch("scrapy.utils.display._enable_windows_terminal_processing")
@mock.patch("platform.version")
@mock.patch("sys.stdout.isatty")
def test_pformat_windows(isatty, version, terminal_processing):
def test_pformat_windows(
isatty: mock.Mock, version: mock.Mock, terminal_processing: mock.Mock
) -> None:
isatty.return_value = True
version.return_value = "10.0.14393"
terminal_processing.return_value = True
@ -79,7 +83,7 @@ def test_pformat_windows(isatty, version, terminal_processing):
@mock.patch("sys.platform", "linux")
@mock.patch("sys.stdout.isatty")
def test_pformat_no_pygments(isatty):
def test_pformat_no_pygments(isatty: mock.Mock) -> None:
isatty.return_value = True
real_import = builtins.__import__

2
tests/test_utils_gz.py
View File

@ -1,3 +1,5 @@
from __future__ import annotations
from gzip import BadGzipFile
from pathlib import Path

2
tests/test_utils_httpobj.py
View File

@ -1,3 +1,5 @@
from __future__ import annotations
from urllib.parse import urlparse
from scrapy.http import Request

261
tests/test_utils_misc/__init__.py
View File

@ -20,150 +20,157 @@ from scrapy.utils.misc import (
)
class TestUtilsMisc:
def test_load_object_class(self):
obj = load_object(Field)
assert obj is Field
obj = load_object("scrapy.item.Field")
assert obj is Field
def test_load_object_class() -> None:
obj = load_object(Field)
assert obj is Field
obj = load_object("scrapy.item.Field")
assert obj is Field
def test_load_object_function(self):
obj = load_object(load_object)
assert obj is load_object
obj = load_object("scrapy.utils.misc.load_object")
assert obj is load_object
def test_load_object_exceptions(self):
with pytest.raises(ImportError):
load_object("nomodule999.mod.function")
with pytest.raises(NameError):
load_object("scrapy.utils.misc.load_object999")
with pytest.raises(TypeError):
load_object({}) # type: ignore[arg-type]
def test_load_object_function() -> None:
obj = load_object(load_object)
assert obj is load_object
obj = load_object("scrapy.utils.misc.load_object")
assert obj is load_object
def test_walk_modules(self):
mods = walk_modules_iter("tests.test_utils_misc.test_walk_modules")
def test_load_object_exceptions() -> None:
with pytest.raises(ImportError):
load_object("nomodule999.mod.function")
with pytest.raises(NameError):
load_object("scrapy.utils.misc.load_object999")
with pytest.raises(TypeError):
load_object({}) # type: ignore[arg-type]
def test_walk_modules() -> None:
mods = walk_modules_iter("tests.test_utils_misc.test_walk_modules")
expected = [
"tests.test_utils_misc.test_walk_modules",
"tests.test_utils_misc.test_walk_modules.mod",
"tests.test_utils_misc.test_walk_modules.mod.mod0",
"tests.test_utils_misc.test_walk_modules.mod1",
]
assert {m.__name__ for m in mods} == set(expected)
mods = walk_modules_iter("tests.test_utils_misc.test_walk_modules.mod")
expected = [
"tests.test_utils_misc.test_walk_modules.mod",
"tests.test_utils_misc.test_walk_modules.mod.mod0",
]
assert {m.__name__ for m in mods} == set(expected)
mods = walk_modules_iter("tests.test_utils_misc.test_walk_modules.mod1")
expected = [
"tests.test_utils_misc.test_walk_modules.mod1",
]
assert {m.__name__ for m in mods} == set(expected)
with pytest.raises(ImportError):
for _ in walk_modules_iter("nomodule999"):
pass
with (
pytest.raises(ImportError),
pytest.warns(
ScrapyDeprecationWarning,
match="The scrapy.utils.misc.walk_modules function is deprecated and will be "
"removed in a future version of Scrapy. "
"Use scrapy.utils.misc.walk_modules_iter instead.",
),
):
walk_modules("nomodule999")
def test_walk_modules_egg() -> None:
egg = str(Path(__file__).parent / "test.egg")
sys.path.append(egg)
try:
mods = walk_modules_iter("testegg")
expected = [
"tests.test_utils_misc.test_walk_modules",
"tests.test_utils_misc.test_walk_modules.mod",
"tests.test_utils_misc.test_walk_modules.mod.mod0",
"tests.test_utils_misc.test_walk_modules.mod1",
"testegg.spiders",
"testegg.spiders.a",
"testegg.spiders.b",
"testegg",
]
assert {m.__name__ for m in mods} == set(expected)
finally:
sys.path.remove(egg)
mods = walk_modules_iter("tests.test_utils_misc.test_walk_modules.mod")
expected = [
"tests.test_utils_misc.test_walk_modules.mod",
"tests.test_utils_misc.test_walk_modules.mod.mod0",
]
assert {m.__name__ for m in mods} == set(expected)
mods = walk_modules_iter("tests.test_utils_misc.test_walk_modules.mod1")
expected = [
"tests.test_utils_misc.test_walk_modules.mod1",
]
assert {m.__name__ for m in mods} == set(expected)
def test_arg_to_iter() -> None:
class TestItem(Item):
name = Field()
with pytest.raises(ImportError):
for _ in walk_modules_iter("nomodule999"):
pass
with (
pytest.raises(ImportError),
pytest.warns(
ScrapyDeprecationWarning,
match="The scrapy.utils.misc.walk_modules function is deprecated and will be "
"removed in a future version of Scrapy. "
"Use scrapy.utils.misc.walk_modules_iter instead.",
),
):
walk_modules("nomodule999")
assert hasattr(arg_to_iter(None), "__iter__")
assert hasattr(arg_to_iter(100), "__iter__")
assert hasattr(arg_to_iter("lala"), "__iter__")
assert hasattr(arg_to_iter([1, 2, 3]), "__iter__")
assert hasattr(arg_to_iter(c for c in "abcd"), "__iter__")
def test_walk_modules_egg(self):
egg = str(Path(__file__).parent / "test.egg")
sys.path.append(egg)
try:
mods = walk_modules_iter("testegg")
expected = [
"testegg.spiders",
"testegg.spiders.a",
"testegg.spiders.b",
"testegg",
]
assert {m.__name__ for m in mods} == set(expected)
finally:
sys.path.remove(egg)
assert not list(arg_to_iter(None))
assert list(arg_to_iter("lala")) == ["lala"]
assert list(arg_to_iter(100)) == [100]
assert list(arg_to_iter(c for c in "abc")) == ["a", "b", "c"]
assert list(arg_to_iter([1, 2, 3])) == [1, 2, 3]
assert list(arg_to_iter({"a": 1})) == [{"a": 1}]
assert list(arg_to_iter(TestItem(name="john"))) == [TestItem(name="john")]
def test_arg_to_iter(self):
class TestItem(Item):
name = Field()
assert hasattr(arg_to_iter(None), "__iter__")
assert hasattr(arg_to_iter(100), "__iter__")
assert hasattr(arg_to_iter("lala"), "__iter__")
assert hasattr(arg_to_iter([1, 2, 3]), "__iter__")
assert hasattr(arg_to_iter(c for c in "abcd"), "__iter__")
def test_build_from_crawler() -> None:
crawler = mock.MagicMock(spec_set=["settings"])
args = (True, 100.0)
kwargs = {"key": "val"}
assert not list(arg_to_iter(None))
assert list(arg_to_iter("lala")) == ["lala"]
assert list(arg_to_iter(100)) == [100]
assert list(arg_to_iter(c for c in "abc")) == ["a", "b", "c"]
assert list(arg_to_iter([1, 2, 3])) == [1, 2, 3]
assert list(arg_to_iter({"a": 1})) == [{"a": 1}]
assert list(arg_to_iter(TestItem(name="john"))) == [TestItem(name="john")]
def _test_with_crawler(mock: mock.MagicMock, crawler: mock.MagicMock) -> None:
build_from_crawler(mock, crawler, *args, **kwargs)
if hasattr(mock, "from_crawler"):
mock.from_crawler.assert_called_once_with(crawler, *args, **kwargs)
assert mock.call_count == 0
else:
mock.assert_called_once_with(*args, **kwargs)
def test_build_from_crawler(self):
crawler = mock.MagicMock(spec_set=["settings"])
args = (True, 100.0)
kwargs = {"key": "val"}
# Check usage of correct constructor using 2 mocks:
# 1. with no alternative constructors
# 2. with from_crawler() constructor
spec_sets = (
["__qualname__"],
["__qualname__", "from_crawler"],
)
for specs in spec_sets:
m = mock.MagicMock(spec_set=specs)
_test_with_crawler(m, crawler)
m.reset_mock()
def _test_with_crawler(mock: mock.MagicMock, crawler: mock.MagicMock) -> None:
build_from_crawler(mock, crawler, *args, **kwargs)
if hasattr(mock, "from_crawler"):
mock.from_crawler.assert_called_once_with(crawler, *args, **kwargs)
assert mock.call_count == 0
else:
mock.assert_called_once_with(*args, **kwargs)
# Check adoption of crawler
m = mock.MagicMock(spec_set=["__qualname__", "from_crawler"])
m.from_crawler.return_value = None
with pytest.raises(TypeError):
build_from_crawler(m, crawler, *args, **kwargs)
# Check usage of correct constructor using 2 mocks:
# 1. with no alternative constructors
# 2. with from_crawler() constructor
spec_sets = (
["__qualname__"],
["__qualname__", "from_crawler"],
)
for specs in spec_sets:
m = mock.MagicMock(spec_set=specs)
_test_with_crawler(m, crawler)
m.reset_mock()
# Check adoption of crawler
m = mock.MagicMock(spec_set=["__qualname__", "from_crawler"])
m.from_crawler.return_value = None
with pytest.raises(TypeError):
build_from_crawler(m, crawler, *args, **kwargs)
def test_set_environ() -> None:
assert os.environ.get("some_test_environ") is None
with set_environ(some_test_environ="test_value"):
assert os.environ.get("some_test_environ") == "test_value"
assert os.environ.get("some_test_environ") is None
def test_set_environ(self):
assert os.environ.get("some_test_environ") is None
with set_environ(some_test_environ="test_value"):
assert os.environ.get("some_test_environ") == "test_value"
assert os.environ.get("some_test_environ") is None
os.environ["some_test_environ"] = "test"
assert os.environ.get("some_test_environ") == "test"
with set_environ(some_test_environ="test_value"):
assert os.environ.get("some_test_environ") == "test_value"
assert os.environ.get("some_test_environ") == "test"
os.environ["some_test_environ"] = "test"
assert os.environ.get("some_test_environ") == "test"
with set_environ(some_test_environ="test_value"):
assert os.environ.get("some_test_environ") == "test_value"
assert os.environ.get("some_test_environ") == "test"
def test_rel_has_nofollow(self):
assert rel_has_nofollow("ugc nofollow") is True
assert rel_has_nofollow("ugc,nofollow") is True
assert rel_has_nofollow("ugc") is False
assert rel_has_nofollow("nofollow") is True
assert rel_has_nofollow("nofollowfoo") is False
assert rel_has_nofollow("foonofollow") is False
assert rel_has_nofollow("ugc, , nofollow") is True
# rel attribute values are ASCII case-insensitive per the HTML spec
assert rel_has_nofollow("NoFollow") is True
assert rel_has_nofollow("NOFOLLOW") is True
assert rel_has_nofollow("UGC NoFollow") is True
assert rel_has_nofollow("ugc,NoFollow") is True
def test_rel_has_nofollow() -> None:
assert rel_has_nofollow("ugc nofollow") is True
assert rel_has_nofollow("ugc,nofollow") is True
assert rel_has_nofollow("ugc") is False
assert rel_has_nofollow("nofollow") is True
assert rel_has_nofollow("nofollowfoo") is False
assert rel_has_nofollow("foonofollow") is False
assert rel_has_nofollow("ugc, , nofollow") is True
# rel attribute values are ASCII case-insensitive per the HTML spec
assert rel_has_nofollow("NoFollow") is True
assert rel_has_nofollow("NOFOLLOW") is True
assert rel_has_nofollow("UGC NoFollow") is True
assert rel_has_nofollow("ugc,NoFollow") is True

421
tests/test_utils_misc/test_return_with_argument_inside_generator.py
View File

@ -1,5 +1,8 @@
from __future__ import annotations
import warnings
from functools import partial
from typing import TYPE_CHECKING, Any
from unittest import mock
import pytest
@ -9,6 +12,11 @@ from scrapy.utils.misc import (
warn_on_generator_with_return_value,
)
if TYPE_CHECKING:
from collections.abc import Generator
from scrapy import Spider
def _indentation_error(*args, **kwargs):
raise IndentationError
@ -35,244 +43,239 @@ https://example.org
yield url
def generator_that_returns_stuff():
def generator_that_returns_stuff() -> Generator[int, None, int]:
yield 1
yield 2
return 3
class TestUtilsMisc:
@pytest.fixture
def mock_spider(self):
class MockSettings:
def __init__(self, settings_dict=None):
self.settings_dict = settings_dict or {
"WARN_ON_GENERATOR_RETURN_VALUE": True
}
@pytest.fixture
def mock_spider() -> Spider:
class MockSettings:
def __init__(self, settings_dict: dict[str, Any] | None = None):
self.settings_dict = settings_dict or {
"WARN_ON_GENERATOR_RETURN_VALUE": True
}
def getbool(self, name, default=False):
return self.settings_dict.get(name, default)
def getbool(self, name, default=False):
return self.settings_dict.get(name, default)
class MockSpider:
def __init__(self):
self.settings = MockSettings()
class MockSpider:
def __init__(self) -> None:
self.settings = MockSettings()
return MockSpider()
return MockSpider() # type: ignore[return-value]
def test_generators_return_something(self, mock_spider):
def f1():
yield 1
return 2
def g1():
yield 1
return "asdf"
def test_generators_return_something(mock_spider):
def f1():
yield 1
return 2
def h1():
yield 1
def g1():
yield 1
return "asdf"
def helper():
return 0
def h1():
yield 1
yield helper()
return 2
def helper() -> int:
return 0
def i1():
"""
docstring
"""
url = """
https://example.org
yield helper()
return 2
def i1():
"""
yield url
return 1
assert is_generator_with_return_value(top_level_return_something)
assert is_generator_with_return_value(f1)
assert is_generator_with_return_value(g1)
assert is_generator_with_return_value(h1)
assert is_generator_with_return_value(i1)
with pytest.warns(
UserWarning,
match='The "MockSpider.top_level_return_something" method is a generator',
):
warn_on_generator_with_return_value(mock_spider, top_level_return_something)
with pytest.warns(
UserWarning, match='The "MockSpider.f1" method is a generator'
):
warn_on_generator_with_return_value(mock_spider, f1)
with pytest.warns(
UserWarning, match='The "MockSpider.g1" method is a generator'
):
warn_on_generator_with_return_value(mock_spider, g1)
with pytest.warns(
UserWarning, match='The "MockSpider.h1" method is a generator'
):
warn_on_generator_with_return_value(mock_spider, h1)
with pytest.warns(
UserWarning, match='The "MockSpider.i1" method is a generator'
):
warn_on_generator_with_return_value(mock_spider, i1)
def test_generators_return_none(self, mock_spider):
def f2():
yield 1
def g2():
yield 1
def h2():
yield 1
def i2():
yield 1
yield from generator_that_returns_stuff()
def j2():
yield 1
def helper():
return 0
yield helper()
def k2():
"""
docstring
"""
url = """
https://example.org
docstring
"""
yield url
def l2():
return
assert not is_generator_with_return_value(top_level_return_none)
assert not is_generator_with_return_value(f2)
assert not is_generator_with_return_value(g2)
assert not is_generator_with_return_value(h2)
assert not is_generator_with_return_value(i2)
assert not is_generator_with_return_value(j2) # not recursive
assert not is_generator_with_return_value(k2) # not recursive
assert not is_generator_with_return_value(l2)
with warnings.catch_warnings():
warnings.simplefilter("error", UserWarning)
warn_on_generator_with_return_value(mock_spider, top_level_return_none)
warn_on_generator_with_return_value(mock_spider, f2)
warn_on_generator_with_return_value(mock_spider, g2)
warn_on_generator_with_return_value(mock_spider, h2)
warn_on_generator_with_return_value(mock_spider, i2)
warn_on_generator_with_return_value(mock_spider, j2)
warn_on_generator_with_return_value(mock_spider, k2)
warn_on_generator_with_return_value(mock_spider, l2)
def test_generators_return_none_with_decorator(self, mock_spider):
def decorator(func):
def inner_func():
func()
return inner_func
@decorator
def f3():
yield 1
@decorator
def g3():
yield 1
@decorator
def h3():
yield 1
@decorator
def i3():
yield 1
yield from generator_that_returns_stuff()
@decorator
def j3():
yield 1
def helper():
return 0
yield helper()
@decorator
def k3():
"""
docstring
"""
url = """
url = """
https://example.org
"""
yield url
return 1
assert is_generator_with_return_value(top_level_return_something)
assert is_generator_with_return_value(f1)
assert is_generator_with_return_value(g1)
assert is_generator_with_return_value(h1)
assert is_generator_with_return_value(i1)
with pytest.warns(
UserWarning,
match='The "MockSpider.top_level_return_something" method is a generator',
):
warn_on_generator_with_return_value(mock_spider, top_level_return_something)
with pytest.warns(UserWarning, match='The "MockSpider.f1" method is a generator'):
warn_on_generator_with_return_value(mock_spider, f1)
with pytest.warns(UserWarning, match='The "MockSpider.g1" method is a generator'):
warn_on_generator_with_return_value(mock_spider, g1)
with pytest.warns(UserWarning, match='The "MockSpider.h1" method is a generator'):
warn_on_generator_with_return_value(mock_spider, h1)
with pytest.warns(UserWarning, match='The "MockSpider.i1" method is a generator'):
warn_on_generator_with_return_value(mock_spider, i1)
def test_generators_return_none(mock_spider):
def f2():
yield 1
def g2():
yield 1
def h2():
yield 1
def i2():
yield 1
yield from generator_that_returns_stuff()
def j2():
yield 1
def helper() -> int:
return 0
yield helper()
def k2():
"""
yield url
docstring
"""
url = """
https://example.org
"""
yield url
@decorator
def l3():
return
def l2():
return
assert not is_generator_with_return_value(top_level_return_none)
assert not is_generator_with_return_value(f3)
assert not is_generator_with_return_value(g3)
assert not is_generator_with_return_value(h3)
assert not is_generator_with_return_value(i3)
assert not is_generator_with_return_value(j3) # not recursive
assert not is_generator_with_return_value(k3) # not recursive
assert not is_generator_with_return_value(l3)
assert not is_generator_with_return_value(top_level_return_none)
assert not is_generator_with_return_value(f2)
assert not is_generator_with_return_value(g2)
assert not is_generator_with_return_value(h2)
assert not is_generator_with_return_value(i2)
assert not is_generator_with_return_value(j2) # not recursive
assert not is_generator_with_return_value(k2) # not recursive
assert not is_generator_with_return_value(l2)
with warnings.catch_warnings():
warnings.simplefilter("error", UserWarning)
warn_on_generator_with_return_value(mock_spider, top_level_return_none)
warn_on_generator_with_return_value(mock_spider, f3)
warn_on_generator_with_return_value(mock_spider, g3)
warn_on_generator_with_return_value(mock_spider, h3)
warn_on_generator_with_return_value(mock_spider, i3)
warn_on_generator_with_return_value(mock_spider, j3)
warn_on_generator_with_return_value(mock_spider, k3)
warn_on_generator_with_return_value(mock_spider, l3)
with warnings.catch_warnings():
warnings.simplefilter("error", UserWarning)
warn_on_generator_with_return_value(mock_spider, top_level_return_none)
warn_on_generator_with_return_value(mock_spider, f2)
warn_on_generator_with_return_value(mock_spider, g2)
warn_on_generator_with_return_value(mock_spider, h2)
warn_on_generator_with_return_value(mock_spider, i2)
warn_on_generator_with_return_value(mock_spider, j2)
warn_on_generator_with_return_value(mock_spider, k2)
warn_on_generator_with_return_value(mock_spider, l2)
@mock.patch(
"scrapy.utils.misc.is_generator_with_return_value", new=_indentation_error
)
def test_indentation_error(self, mock_spider):
with pytest.warns(UserWarning, match="Unable to determine"):
warn_on_generator_with_return_value(mock_spider, top_level_return_none)
def test_partial(self):
def cb(arg1, arg2):
yield {}
def test_generators_return_none_with_decorator(mock_spider):
def decorator(func):
def inner_func():
func()
partial_cb = partial(cb, arg1=42)
assert not is_generator_with_return_value(partial_cb)
return inner_func
def test_warn_on_generator_with_return_value_settings_disabled(self):
class MockSettings:
def __init__(self, settings_dict=None):
self.settings_dict = settings_dict or {}
@decorator
def f3():
yield 1
def getbool(self, name, default=False):
return self.settings_dict.get(name, default)
@decorator
def g3():
yield 1
class MockSpider:
def __init__(self):
self.settings = MockSettings({"WARN_ON_GENERATOR_RETURN_VALUE": False})
@decorator
def h3():
yield 1
spider = MockSpider()
@decorator
def i3():
yield 1
yield from generator_that_returns_stuff()
def gen_with_return():
yield 1
return "value"
@decorator
def j3():
yield 1
with warnings.catch_warnings():
warnings.simplefilter("error", UserWarning)
warn_on_generator_with_return_value(spider, gen_with_return)
def helper() -> int:
return 0
spider.settings.settings_dict["WARN_ON_GENERATOR_RETURN_VALUE"] = True
yield helper()
with pytest.warns(UserWarning, match="is a generator"):
warn_on_generator_with_return_value(spider, gen_with_return)
@decorator
def k3():
"""
docstring
"""
url = """
https://example.org
"""
yield url
@decorator
def l3():
return
assert not is_generator_with_return_value(top_level_return_none)
assert not is_generator_with_return_value(f3)
assert not is_generator_with_return_value(g3)
assert not is_generator_with_return_value(h3)
assert not is_generator_with_return_value(i3)
assert not is_generator_with_return_value(j3) # not recursive
assert not is_generator_with_return_value(k3) # not recursive
assert not is_generator_with_return_value(l3)
with warnings.catch_warnings():
warnings.simplefilter("error", UserWarning)
warn_on_generator_with_return_value(mock_spider, top_level_return_none)
warn_on_generator_with_return_value(mock_spider, f3)
warn_on_generator_with_return_value(mock_spider, g3)
warn_on_generator_with_return_value(mock_spider, h3)
warn_on_generator_with_return_value(mock_spider, i3)
warn_on_generator_with_return_value(mock_spider, j3)
warn_on_generator_with_return_value(mock_spider, k3)
warn_on_generator_with_return_value(mock_spider, l3)
@mock.patch("scrapy.utils.misc.is_generator_with_return_value", new=_indentation_error)
def test_indentation_error(mock_spider):
with pytest.warns(UserWarning, match="Unable to determine"):
warn_on_generator_with_return_value(mock_spider, top_level_return_none)
def test_partial() -> None:
def cb(arg1, arg2):
yield {}
partial_cb = partial(cb, arg1=42)
assert not is_generator_with_return_value(partial_cb)
def test_warn_on_generator_with_return_value_settings_disabled() -> None:
class MockSettings:
def __init__(self, settings_dict: dict[str, Any] | None = None):
self.settings_dict = settings_dict or {}
def getbool(self, name, default=False):
return self.settings_dict.get(name, default)
class MockSpider:
def __init__(self) -> None:
self.settings = MockSettings({"WARN_ON_GENERATOR_RETURN_VALUE": False})
spider = MockSpider()
def gen_with_return():
yield 1
return "value"
with warnings.catch_warnings():
warnings.simplefilter("error", UserWarning)
warn_on_generator_with_return_value(spider, gen_with_return) # type: ignore[arg-type]
spider.settings.settings_dict["WARN_ON_GENERATOR_RETURN_VALUE"] = True
with pytest.warns(UserWarning, match="is a generator"):
warn_on_generator_with_return_value(spider, gen_with_return) # type: ignore[arg-type]

10
tests/test_utils_project.py
View File

@ -1,14 +1,20 @@
from __future__ import annotations
import os
from pathlib import Path
from typing import TYPE_CHECKING
import pytest
from scrapy.utils.misc import set_environ
from scrapy.utils.project import data_path, get_project_settings
if TYPE_CHECKING:
from collections.abc import Generator
@pytest.fixture
def proj_path(tmp_path):
def proj_path(tmp_path: Path) -> Generator[Path]:
prev_dir = Path.cwd()
project_dir = tmp_path
@ -21,7 +27,7 @@ def proj_path(tmp_path):
os.chdir(prev_dir)
def test_data_path_outside_project():
def test_data_path_outside_project() -> None:
assert str(Path(".scrapy", "somepath")) == data_path("somepath")
abspath = str(Path(os.path.sep, "absolute", "path"))
assert abspath == data_path(abspath)

21
tests/test_utils_python.py
View File

@ -22,8 +22,7 @@ from scrapy.utils.python import (
from tests.utils.decorators import coroutine_test
if TYPE_CHECKING:
from collections.abc import Iterable, Mapping
from collections.abc import AsyncIterator, Iterable, Mapping
_KT = TypeVar("_KT")
_VT = TypeVar("_VT")
@ -31,22 +30,22 @@ _VT = TypeVar("_VT")
class TestMutableAsyncChain:
@staticmethod
async def g1():
async def g1() -> AsyncIterator[int]:
for i in range(3):
yield i
@staticmethod
async def g2():
async def g2() -> AsyncIterator[int]:
return
yield
@staticmethod
async def g3():
async def g3() -> AsyncIterator[int]:
for i in range(7, 10):
yield i
@staticmethod
async def g4():
async def g4() -> AsyncIterator[int]:
for i in range(3, 5):
yield i
1 / 0
@ -85,7 +84,7 @@ class TestToUnicode:
def test_converting_a_strange_object_should_raise_type_error(self):
with pytest.raises(TypeError):
to_unicode(423)
to_unicode(423) # type: ignore[arg-type]
def test_errors_argument(self):
assert to_unicode(b"a\xedb", "utf-8", errors="replace") == "a\ufffdb"
@ -103,7 +102,7 @@ class TestToBytes:
def test_converting_a_strange_object_should_raise_type_error(self):
with pytest.raises(TypeError):
to_bytes(pytest)
to_bytes(pytest) # type: ignore[arg-type]
def test_errors_argument(self):
assert to_bytes("a\ufffdb", "latin-1", errors="replace") == b"a?b"
@ -112,10 +111,10 @@ class TestToBytes:
def test_memoizemethod_noargs():
class A:
@memoizemethod_noargs
def cached(self):
def cached(self) -> object:
return object()
def noncached(self):
def noncached(self) -> object:
return object()
a = A()
@ -150,7 +149,7 @@ def test_get_func_args():
pass
class A:
def __init__(self, a, b, c):
def __init__(self, a: int, b: int, c: int):
pass
def method(self, a, b, c):

2
tests/test_utils_reactor.py
View File

@ -1,3 +1,5 @@
from __future__ import annotations
import asyncio
import pytest

26
tests/test_utils_request.py
View File

@ -2,7 +2,7 @@ from __future__ import annotations
import json
from hashlib import sha1
from typing import Any
from typing import TYPE_CHECKING, Any, Protocol
from weakref import WeakKeyDictionary
import pytest
@ -17,6 +17,9 @@ from scrapy.utils.request import (
)
from scrapy.utils.test import get_crawler
if TYPE_CHECKING:
from collections.abc import Iterable
@pytest.mark.parametrize(
("r", "expected"),
@ -56,8 +59,18 @@ def test_request_httprepr_for_non_http_request(r: Request) -> None:
request_httprepr(r)
class _FingerprintFunction(Protocol):
def __call__(
self,
request: Request,
*,
include_headers: Iterable[bytes | str] | None = None,
keep_fragments: bool = False,
) -> bytes: ...
class TestFingerprint:
function: staticmethod[[Request], bytes] = staticmethod(fingerprint)
function: _FingerprintFunction = staticmethod(fingerprint)
cache: (
WeakKeyDictionary[
Request, dict[tuple[tuple[bytes, ...] | None, bool, bool], bytes]
@ -261,6 +274,7 @@ class TestRequestFingerprinter:
def test_fingerprint(self):
crawler = get_crawler()
request = Request("https://example.com")
assert crawler.request_fingerprinter
assert crawler.request_fingerprinter.fingerprint(request) == fingerprint(
request
)
@ -277,6 +291,7 @@ class TestCustomRequestFingerprinter:
}
crawler = get_crawler(settings_dict=settings)
assert crawler.request_fingerprinter
r1 = Request("http://www.example.com", headers={"X-ID": "1"})
fp1 = crawler.request_fingerprinter.fingerprint(r1)
r2 = Request("http://www.example.com", headers={"X-ID": "2"})
@ -285,9 +300,9 @@ class TestCustomRequestFingerprinter:
def test_dont_canonicalize(self):
class RequestFingerprinter:
cache = WeakKeyDictionary()
cache: WeakKeyDictionary[Request, bytes] = WeakKeyDictionary()
def fingerprint(self, request):
def fingerprint(self, request: Request) -> bytes:
if request not in self.cache:
fp = sha1()
fp.update(to_bytes(request.url))
@ -299,6 +314,7 @@ class TestCustomRequestFingerprinter:
}
crawler = get_crawler(settings_dict=settings)
assert crawler.request_fingerprinter
r1 = Request("http://www.example.com?a=1&a=2")
fp1 = crawler.request_fingerprinter.fingerprint(r1)
r2 = Request("http://www.example.com?a=2&a=1")
@ -317,6 +333,7 @@ class TestCustomRequestFingerprinter:
}
crawler = get_crawler(settings_dict=settings)
assert crawler.request_fingerprinter
r1 = Request("http://www.example.com")
fp1 = crawler.request_fingerprinter.fingerprint(r1)
r2 = Request("http://www.example.com", meta={"fingerprint": "a"})
@ -348,6 +365,7 @@ class TestCustomRequestFingerprinter:
}
crawler = get_crawler(settings_dict=settings)
assert crawler.request_fingerprinter
request = Request("http://www.example.com")
fingerprint = crawler.request_fingerprinter.fingerprint(request)
assert fingerprint == settings["FINGERPRINT"]

6
tests/test_utils_response.py
View File

@ -1,3 +1,5 @@
from __future__ import annotations
from pathlib import Path
from time import process_time
from urllib.parse import urlparse
@ -15,7 +17,7 @@ from scrapy.utils.response import (
)
def _read_browser_output(burl: str):
def _read_browser_output(burl: str) -> bytes:
path = urlparse(burl).path
if not path or not Path(path).exists():
path = burl.replace("file://", "")
@ -224,7 +226,7 @@ def test_open_in_browser_redos_head():
(b"<!-- <head>fake</head> --><head>real</head>", b"<head>real</head>"),
],
)
def test_remove_html_comments(input_body, output_body):
def test_remove_html_comments(input_body: bytes, output_body: bytes) -> None:
assert _remove_html_comments(input_body) == output_body

2
tests/test_utils_serialize.py
View File

@ -1,3 +1,5 @@
from __future__ import annotations
import dataclasses
import datetime
import json

3
tests/test_utils_signal.py
View File

@ -21,9 +21,6 @@ from tests.utils.decorators import coroutine_test
if TYPE_CHECKING:
from collections.abc import Callable
if TYPE_CHECKING:
from collections.abc import Callable
class TestSendCatchLog:
# whether the function being tested returns exceptions or failures

2
tests/test_utils_sitemap.py
View File

@ -1,3 +1,5 @@
from __future__ import annotations
import pytest
from scrapy.exceptions import ScrapyDeprecationWarning

9
tests/test_utils_template.py
View File

@ -1,7 +1,14 @@
from __future__ import annotations
from typing import TYPE_CHECKING
from scrapy.utils.template import render_templatefile
if TYPE_CHECKING:
from pathlib import Path
def test_simple_render(tmp_path):
def test_simple_render(tmp_path: Path) -> None:
context = {"project_name": "proj", "name": "spi", "classname": "TheSpider"}
template = "from ${project_name}.spiders.${name} import ${classname}"
rendered = "from proj.spiders.spi import TheSpider"

6
tests/test_utils_trackref.py
View File

@ -1,3 +1,5 @@
from __future__ import annotations
import sys
from io import StringIO
from unittest import mock
@ -46,13 +48,13 @@ Bar 1 oldest: 0s ago
@mock.patch("sys.stdout", new_callable=StringIO)
def test_print_live_refs_empty(stdout):
def test_print_live_refs_empty(stdout: StringIO) -> None:
trackref.print_live_refs()
assert stdout.getvalue() == "Live References\n\n\n"
@mock.patch("sys.stdout", new_callable=StringIO)
def test_print_live_refs_with_objects(stdout):
def test_print_live_refs_with_objects(stdout: StringIO) -> None:
o1 = Foo() # noqa: F841
trackref.print_live_refs()
assert (

4
tests/test_utils_url.py
View File

@ -1,3 +1,5 @@
from __future__ import annotations
import pytest
from scrapy.linkextractors import IGNORED_EXTENSIONS
@ -191,7 +193,7 @@ def test_guess_scheme(url: str, expected: str):
),
],
)
def test_guess_scheme_skipped(url: str, expected: str, reason: str):
def test_guess_scheme_skipped(url: str, expected: str, reason: str) -> None:
pytest.skip(reason)

2
tests_typing/test_http_request.mypy-testing
View File

@ -16,7 +16,7 @@ class MyRequest2(Request):
@pytest.mark.mypy_testing
def mypy_test_headers() -> None:
Request("data:,", headers=1) # E: Argument "headers" to "Request" has incompatible type "int"; expected "Mapping[str, Any] | Iterable[tuple[str, Any]] | None"
Request("data:,", headers=1) # E: Argument "headers" to "Request" has incompatible type "int"; expected "Mapping[str, Any] | Mapping[bytes, Any] | Iterable[tuple[str | bytes, Any]] | None"
Request("data:,", headers=None)
Request("data:,", headers={})
Request("data:,", headers=[])

2
tests_typing/test_http_response.mypy-testing
View File

@ -7,7 +7,7 @@ from scrapy.http import HtmlResponse, Response, TextResponse
@pytest.mark.mypy_testing
def mypy_test_headers() -> None:
Response("data:,", headers=1) # E: Argument "headers" to "Response" has incompatible type "int"; expected "Mapping[str, Any] | Iterable[tuple[str, Any]] | None"
Response("data:,", headers=1) # E: Argument "headers" to "Response" has incompatible type "int"; expected "Mapping[str, Any] | Mapping[bytes, Any] | Iterable[tuple[str | bytes, Any]] | None"
Response("data:,", headers=None)
Response("data:,", headers={})
Response("data:,", headers=[])