RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into pyupgrade

This commit is contained in:
Andrey Rakhmatullin 2024-06-07 11:20:31 +05:00
parent ed3a7acaf3 a4778d2bdf
commit 5850b8f3e6
25 changed files with 335 additions and 116 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
docs/topics/dynamic-content.rst
View File

@ -115,15 +115,14 @@ Handling different response formats
Once you have a response with the desired data, how you extract the desired
data from it depends on the type of response:
- If the response is HTML or XML, use :ref:`selectors
- If the response is HTML, XML or JSON, use :ref:`selectors
<topics-selectors>` as usual.
- If the response is JSON, use :func:`json.loads` to load the desired data from
:attr:`response.text <scrapy.http.TextResponse.text>`:
- If the response is JSON, use :func:`response.json()` to load the desired data:
.. code-block:: python
data = json.loads(response.text)
data = response.json()
If the desired data is inside HTML or XML code embedded within JSON data,
you can load that HTML or XML code into a

8
docs/topics/selectors.rst
View File

@ -1060,6 +1060,12 @@ Selector objects
For convenience, this method can be called as ``response.css()``
.. automethod:: jmespath
.. note::
For convenience, this method can be called as ``response.jmespath()``
.. automethod:: get
See also: :ref:`old-extraction-api`
@ -1092,6 +1098,8 @@ SelectorList objects
.. automethod:: css
.. automethod:: jmespath
.. automethod:: getall
See also: :ref:`old-extraction-api`

12
scrapy/commands/parse.py
View File

@ -1,13 +1,15 @@
from __future__ import annotations
import argparse
import functools
import inspect
import json
import logging
from types import CoroutineType
from typing import (
Any,
AsyncGenerator,
Callable,
Coroutine,
Dict,
Iterable,
List,
@ -140,13 +142,13 @@ class Command(BaseRunSpiderCommand):
@overload
def iterate_spider_output(
self, result: Union[AsyncGenerator, CoroutineType]
) -> Deferred: ...
self, result: Union[AsyncGenerator[_T, None], Coroutine[Any, Any, _T]]
) -> Deferred[_T]: ...
@overload
def iterate_spider_output(self, result: _T) -> Iterable: ...
def iterate_spider_output(self, result: _T) -> Iterable[Any]: ...
def iterate_spider_output(self, result: Any) -> Union[Iterable, Deferred]:
def iterate_spider_output(self, result: Any) -> Union[Iterable[Any], Deferred]:
if inspect.isasyncgen(result):
d = deferred_from_coro(
collect_asyncgen(aiter_errback(result, self.handle_exception))

5
scrapy/core/engine.py
View File

@ -372,7 +372,10 @@ class ExecutionEngine:
@inlineCallbacks
def open_spider(
self, spider: Spider, start_requests: Iterable = (), close_if_idle: bool = True
self,
spider: Spider,
start_requests: Iterable[Request] = (),
close_if_idle: bool = True,
) -> Generator[Deferred, Any, None]:
if self.slot is not None:
raise RuntimeError(f"No free spider slot when opening {spider.name!r}")

6
scrapy/core/scraper.py
View File

@ -16,6 +16,7 @@ from typing import (
Set,
Tuple,
Type,
TypeVar,
Union,
cast,
)
@ -47,6 +48,7 @@ if TYPE_CHECKING:
from scrapy.crawler import Crawler
_T = TypeVar("_T")
QueueTuple = Tuple[Union[Response, Failure], Request, Deferred]
@ -256,14 +258,14 @@ class Scraper:
def handle_spider_output(
self,
result: Union[Iterable, AsyncIterable],
result: Union[Iterable[_T], AsyncIterable[_T]],
request: Request,
response: Response,
spider: Spider,
) -> Deferred:
if not result:
return defer_succeed(None)
it: Union[Iterable, AsyncIterable]
it: Union[Iterable[_T], AsyncIterable[_T]]
if isinstance(result, AsyncIterable):
it = aiter_errback(
result, self.handle_spider_error, request, response, spider

72
scrapy/core/spidermw.py
View File

@ -4,12 +4,13 @@ Spider Middleware manager
See documentation in docs/topics/spider-middleware.rst
"""
from __future__ import annotations
import logging
from inspect import isasyncgenfunction, iscoroutine
from itertools import islice
from typing import (
Any,
AsyncGenerator,
AsyncIterable,
Callable,
Generator,
@ -17,6 +18,7 @@ from typing import (
List,
Optional,
Tuple,
TypeVar,
Union,
cast,
)
@ -42,6 +44,7 @@ from scrapy.utils.python import MutableAsyncChain, MutableChain
logger = logging.getLogger(__name__)
_T = TypeVar("_T")
ScrapeFunc = Callable[[Union[Response, Failure], Request, Spider], Any]
@ -98,31 +101,39 @@ class SpiderMiddlewareManager(MiddlewareManager):
self,
response: Response,
spider: Spider,
iterable: Union[Iterable, AsyncIterable],
iterable: Union[Iterable[_T], AsyncIterable[_T]],
exception_processor_index: int,
recover_to: Union[MutableChain, MutableAsyncChain],
) -> Union[Generator, AsyncGenerator]:
def process_sync(iterable: Iterable) -> Generator:
recover_to: Union[MutableChain[_T], MutableAsyncChain[_T]],
) -> Union[Iterable[_T], AsyncIterable[_T]]:
def process_sync(iterable: Iterable[_T]) -> Iterable[_T]:
try:
yield from iterable
except Exception as ex:
exception_result = self._process_spider_exception(
response, spider, Failure(ex), exception_processor_index
exception_result = cast(
Union[Failure, MutableChain[_T]],
self._process_spider_exception(
response, spider, Failure(ex), exception_processor_index
),
)
if isinstance(exception_result, Failure):
raise
assert isinstance(recover_to, MutableChain)
recover_to.extend(exception_result)
async def process_async(iterable: AsyncIterable) -> AsyncGenerator:
async def process_async(iterable: AsyncIterable[_T]) -> AsyncIterable[_T]:
try:
async for r in iterable:
yield r
except Exception as ex:
exception_result = self._process_spider_exception(
response, spider, Failure(ex), exception_processor_index
exception_result = cast(
Union[Failure, MutableAsyncChain[_T]],
self._process_spider_exception(
response, spider, Failure(ex), exception_processor_index
),
)
if isinstance(exception_result, Failure):
raise
assert isinstance(recover_to, MutableAsyncChain)
recover_to.extend(exception_result)
if isinstance(iterable, AsyncIterable):
@ -135,7 +146,7 @@ class SpiderMiddlewareManager(MiddlewareManager):
spider: Spider,
_failure: Failure,
start_index: int = 0,
) -> Union[Failure, MutableChain]:
) -> Union[Failure, MutableChain[_T], MutableAsyncChain[_T]]:
exception = _failure.value
# don't handle _InvalidOutput exception
if isinstance(exception, _InvalidOutput):
@ -151,14 +162,18 @@ class SpiderMiddlewareManager(MiddlewareManager):
if _isiterable(result):
# stop exception handling by handing control over to the
# process_spider_output chain if an iterable has been returned
dfd: Deferred = self._process_spider_output(
response, spider, result, method_index + 1
dfd: Deferred[Union[MutableChain[_T], MutableAsyncChain[_T]]] = (
self._process_spider_output(
response, spider, result, method_index + 1
)
)
# _process_spider_output() returns a Deferred only because of downgrading so this can be
# simplified when downgrading is removed.
if dfd.called:
# the result is available immediately if _process_spider_output didn't do downgrading
return cast(MutableChain, dfd.result)
return cast(
Union[MutableChain[_T], MutableAsyncChain[_T]], dfd.result
)
# we forbid waiting here because otherwise we would need to return a deferred from
# _process_spider_exception too, which complicates the architecture
msg = f"Async iterable returned from {method.__qualname__} cannot be downgraded"
@ -181,12 +196,12 @@ class SpiderMiddlewareManager(MiddlewareManager):
self,
response: Response,
spider: Spider,
result: Union[Iterable, AsyncIterable],
result: Union[Iterable[_T], AsyncIterable[_T]],
start_index: int = 0,
) -> Generator[Deferred, Any, Union[MutableChain, MutableAsyncChain]]:
) -> Generator[Deferred[Any], Any, Union[MutableChain[_T], MutableAsyncChain[_T]]]:
# items in this iterable do not need to go through the process_spider_output
# chain, they went through it already from the process_spider_exception method
recovered: Union[MutableChain, MutableAsyncChain]
recovered: Union[MutableChain[_T], MutableAsyncChain[_T]]
last_result_is_async = isinstance(result, AsyncIterable)
if last_result_is_async:
recovered = MutableAsyncChain()
@ -237,7 +252,9 @@ class SpiderMiddlewareManager(MiddlewareManager):
# might fail directly if the output value is not a generator
result = method(response=response, result=result, spider=spider)
except Exception as ex:
exception_result = self._process_spider_exception(
exception_result: Union[
Failure, MutableChain[_T], MutableAsyncChain[_T]
] = self._process_spider_exception(
response, spider, Failure(ex), method_index + 1
)
if isinstance(exception_result, Failure):
@ -267,9 +284,12 @@ class SpiderMiddlewareManager(MiddlewareManager):
return MutableChain(result, recovered) # type: ignore[arg-type]
async def _process_callback_output(
self, response: Response, spider: Spider, result: Union[Iterable, AsyncIterable]
) -> Union[MutableChain, MutableAsyncChain]:
recovered: Union[MutableChain, MutableAsyncChain]
self,
response: Response,
spider: Spider,
result: Union[Iterable[_T], AsyncIterable[_T]],
) -> Union[MutableChain[_T], MutableAsyncChain[_T]]:
recovered: Union[MutableChain[_T], MutableAsyncChain[_T]]
if isinstance(result, AsyncIterable):
recovered = MutableAsyncChain()
else:
@ -293,14 +313,16 @@ class SpiderMiddlewareManager(MiddlewareManager):
spider: Spider,
) -> Deferred:
async def process_callback_output(
result: Union[Iterable, AsyncIterable]
) -> Union[MutableChain, MutableAsyncChain]:
result: Union[Iterable[_T], AsyncIterable[_T]]
) -> Union[MutableChain[_T], MutableAsyncChain[_T]]:
return await self._process_callback_output(response, spider, result)
def process_spider_exception(_failure: Failure) -> Union[Failure, MutableChain]:
def process_spider_exception(
_failure: Failure,
) -> Union[Failure, MutableChain[_T], MutableAsyncChain[_T]]:
return self._process_spider_exception(response, spider, _failure)
dfd = mustbe_deferred(
dfd: Deferred = mustbe_deferred(
self._process_spider_input, scrape_func, response, request, spider
)
dfd.addCallback(deferred_f_from_coro_f(process_callback_output))

6
scrapy/downloadermiddlewares/httpcompression.py
View File

@ -3,7 +3,7 @@ from __future__ import annotations
import warnings
from itertools import chain
from logging import getLogger
from typing import TYPE_CHECKING, List, Optional, Tuple, Union
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union
from scrapy import Request, Spider, signals
from scrapy.crawler import Crawler
@ -138,12 +138,12 @@ class HttpCompressionMiddleware:
respcls = responsetypes.from_args(
headers=response.headers, url=response.url, body=decoded_body
)
kwargs = {"cls": respcls, "body": decoded_body}
kwargs: Dict[str, Any] = {"body": decoded_body}
if issubclass(respcls, TextResponse):
# force recalculating the encoding until we make sure the
# responsetypes guessing is reliable
kwargs["encoding"] = None
response = response.replace(**kwargs)
response = response.replace(cls=respcls, **kwargs)
if not content_encoding:
del response.headers["Content-Encoding"]

1
scrapy/downloadermiddlewares/redirect.py
View File

@ -27,6 +27,7 @@ def _build_redirect_request(
redirect_request = source_request.replace(
url=url,
**kwargs,
cls=None,
cookies=None,
)
if "_scheme_proxy" in redirect_request.meta:

5
scrapy/extensions/httpcache.py
View File

@ -370,12 +370,11 @@ class FilesystemCacheStorage:
with self._open(rpath / "pickled_meta", "wb") as f:
pickle.dump(metadata, f, protocol=4)
with self._open(rpath / "response_headers", "wb") as f:
# headers_dict_to_raw() needs a better type hint
f.write(cast(bytes, headers_dict_to_raw(response.headers)))
f.write(headers_dict_to_raw(response.headers))
with self._open(rpath / "response_body", "wb") as f:
f.write(response.body)
with self._open(rpath / "request_headers", "wb") as f:
f.write(cast(bytes, headers_dict_to_raw(request.headers)))
f.write(headers_dict_to_raw(request.headers))
with self._open(rpath / "request_body", "wb") as f:
f.write(request.body)

3
scrapy/http/headers.py
View File

@ -118,8 +118,7 @@ class Headers(CaselessDict):
]
def to_string(self) -> bytes:
# cast() can be removed if the headers_dict_to_raw() hint is improved
return cast(bytes, headers_dict_to_raw(self))
return headers_dict_to_raw(self)
def to_unicode_dict(self) -> CaseInsensitiveDict:
"""Return headers as a CaseInsensitiveDict with str keys

26
scrapy/http/request/__init__.py
View File

@ -20,9 +20,11 @@ from typing import (
NoReturn,
Optional,
Tuple,
Type,
TypedDict,
TypeVar,
Union,
cast,
overload,
)
from w3lib.url import safe_url_string
@ -50,6 +52,9 @@ class VerboseCookie(TypedDict):
CookiesT = Union[Dict[str, str], List[VerboseCookie]]
RequestTypeVar = TypeVar("RequestTypeVar", bound="Request")
def NO_CALLBACK(*args: Any, **kwargs: Any) -> NoReturn:
"""When assigned to the ``callback`` parameter of
:class:`~scrapy.http.Request`, it indicates that the request is not meant
@ -189,15 +194,26 @@ class Request(object_ref):
def __repr__(self) -> str:
return f"<{self.method} {self.url}>"
def copy(self) -> Request:
def copy(self) -> Self:
return self.replace()
def replace(self, *args: Any, **kwargs: Any) -> Request:
@overload
def replace(
self, *args: Any, cls: Type[RequestTypeVar], **kwargs: Any
) -> RequestTypeVar: ...
@overload
def replace(self, *args: Any, cls: None = None, **kwargs: Any) -> Self: ...
def replace(
self, *args: Any, cls: Optional[Type[Request]] = None, **kwargs: Any
) -> Request:
"""Create a new Request with the same attributes except for those given new values"""
for x in self.attributes:
kwargs.setdefault(x, getattr(self, x))
cls = kwargs.pop("cls", self.__class__)
return cast(Request, cls(*args, **kwargs))
if cls is None:
cls = self.__class__
return cls(*args, **kwargs)
@classmethod
def from_curl(

24
scrapy/http/request/json_request.py
View File

@ -5,12 +5,18 @@ This module implements the JsonRequest class which is a more convenient class
See documentation in docs/topics/request-response.rst
"""
from __future__ import annotations
import copy
import json
import warnings
from typing import Any, Dict, Optional, Tuple
from typing import TYPE_CHECKING, Any, Dict, Optional, Tuple, Type, overload
from scrapy.http.request import Request
from scrapy.http.request import Request, RequestTypeVar
if TYPE_CHECKING:
# typing.Self requires Python 3.11
from typing_extensions import Self
class JsonRequest(Request):
@ -44,7 +50,17 @@ class JsonRequest(Request):
def dumps_kwargs(self) -> Dict[str, Any]:
return self._dumps_kwargs
def replace(self, *args: Any, **kwargs: Any) -> Request:
@overload
def replace(
self, *args: Any, cls: Type[RequestTypeVar], **kwargs: Any
) -> RequestTypeVar: ...
@overload
def replace(self, *args: Any, cls: None = None, **kwargs: Any) -> Self: ...
def replace(
self, *args: Any, cls: Optional[Type[Request]] = None, **kwargs: Any
) -> Request:
body_passed = kwargs.get("body", None) is not None
data: Any = kwargs.pop("data", None)
data_passed: bool = data is not None
@ -54,7 +70,7 @@ class JsonRequest(Request):
elif not body_passed and data_passed:
kwargs["body"] = self._dumps(data)
return super().replace(*args, **kwargs)
return super().replace(*args, cls=cls, **kwargs)
def _dumps(self, data: Any) -> str:
"""Convert to JSON"""

32
scrapy/http/response/__init__.py
View File

@ -14,14 +14,15 @@ from typing import (
AnyStr,
Callable,
Dict,
Generator,
Iterable,
List,
Mapping,
Optional,
Tuple,
Type,
TypeVar,
Union,
cast,
overload,
)
from urllib.parse import urljoin
@ -34,9 +35,15 @@ from scrapy.link import Link
from scrapy.utils.trackref import object_ref
if TYPE_CHECKING:
# typing.Self requires Python 3.11
from typing_extensions import Self
from scrapy.selector import SelectorList
ResponseTypeVar = TypeVar("ResponseTypeVar", bound="Response")
class Response(object_ref):
"""An object that represents an HTTP response, which is usually
downloaded (by the Downloader) and fed to the Spiders for processing.
@ -133,16 +140,27 @@ class Response(object_ref):
def __repr__(self) -> str:
return f"<{self.status} {self.url}>"
def copy(self) -> Response:
def copy(self) -> Self:
"""Return a copy of this Response"""
return self.replace()
def replace(self, *args: Any, **kwargs: Any) -> Response:
@overload
def replace(
self, *args: Any, cls: Type[ResponseTypeVar], **kwargs: Any
) -> ResponseTypeVar: ...
@overload
def replace(self, *args: Any, cls: None = None, **kwargs: Any) -> Self: ...
def replace(
self, *args: Any, cls: Optional[Type[Response]] = None, **kwargs: Any
) -> Response:
"""Create a new Response with the same attributes except for those given new values"""
for x in self.attributes:
kwargs.setdefault(x, getattr(self, x))
cls = kwargs.pop("cls", self.__class__)
return cast(Response, cls(*args, **kwargs))
if cls is None:
cls = self.__class__
return cls(*args, **kwargs)
def urljoin(self, url: str) -> str:
"""Join this Response's url with a possible relative url to form an
@ -242,7 +260,7 @@ class Response(object_ref):
errback: Optional[Callable] = None,
cb_kwargs: Optional[Dict[str, Any]] = None,
flags: Optional[List[str]] = None,
) -> Generator[Request, None, None]:
) -> Iterable[Request]:
"""
.. versionadded:: 2.0

3
scrapy/http/response/text.py
View File

@ -15,7 +15,6 @@ from typing import (
AnyStr,
Callable,
Dict,
Generator,
Iterable,
List,
Mapping,
@ -246,7 +245,7 @@ class TextResponse(Response):
flags: Optional[List[str]] = None,
css: Optional[str] = None,
xpath: Optional[str] = None,
) -> Generator[Request, None, None]:
) -> Iterable[Request]:
"""
A generator that produces :class:`~.Request` instances to follow all
links in ``urls``. It accepts the same arguments as the :class:`~.Request`'s

1
scrapy/selector/unified.py
View File

@ -59,6 +59,7 @@ class Selector(_ParselSelector, object_ref):
* ``"html"`` for :class:`~scrapy.http.HtmlResponse` type
* ``"xml"`` for :class:`~scrapy.http.XmlResponse` type
* ``"json"`` for :class:`~scrapy.http.TextResponse` type
* ``"html"`` for anything else
Otherwise, if ``type`` is set, the selector type will be forced and no

28
scrapy/utils/iterators.py
View File

@ -6,8 +6,7 @@ from typing import (
Any,
Callable,
Dict,
Generator,
Iterable,
Iterator,
List,
Literal,
Optional,
@ -22,14 +21,12 @@ from lxml import etree # nosec
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.http import Response, TextResponse
from scrapy.selector import Selector
from scrapy.utils.python import re_rsearch, to_unicode
from scrapy.utils.python import re_rsearch
logger = logging.getLogger(__name__)
def xmliter(
obj: Union[Response, str, bytes], nodename: str
) -> Generator[Selector, Any, None]:
def xmliter(obj: Union[Response, str, bytes], nodename: str) -> Iterator[Selector]:
"""Return a iterator of Selector's over all nodes of a XML document,
given the name of the node to iterate. Useful for parsing XML feeds.
@ -90,7 +87,7 @@ def xmliter_lxml(
nodename: str,
namespace: Optional[str] = None,
prefix: str = "x",
) -> Generator[Selector, Any, None]:
) -> Iterator[Selector]:
reader = _StreamReader(obj)
tag = f"{{{namespace}}}{nodename}" if namespace else nodename
iterable = etree.iterparse(
@ -168,7 +165,7 @@ def csviter(
headers: Optional[List[str]] = None,
encoding: Optional[str] = None,
quotechar: Optional[str] = None,
) -> Generator[Dict[str, str], Any, None]:
) -> Iterator[Dict[str, str]]:
"""Returns an iterator of dictionaries from the given csv object
obj can be:
@ -184,10 +181,13 @@ def csviter(
quotechar is the character used to enclosure fields on the given obj.
"""
encoding = obj.encoding if isinstance(obj, TextResponse) else encoding or "utf-8"
def row_to_unicode(row_: Iterable) -> List[str]:
return [to_unicode(field, encoding) for field in row_]
if encoding is not None:
warn(
"The encoding argument of csviter() is ignored and will be removed"
" in a future Scrapy version.",
category=ScrapyDeprecationWarning,
stacklevel=2,
)
lines = StringIO(_body_or_str(obj, unicode=True))
@ -200,13 +200,11 @@ def csviter(
if not headers:
try:
row = next(csv_r)
headers = next(csv_r)
except StopIteration:
return
headers = row_to_unicode(row)
for row in csv_r:
row = row_to_unicode(row)
if len(row) != len(headers):
logger.warning(
"ignoring row %(csvlnum)d (length: %(csvrow)d, "

6
scrapy/utils/misc.py
View File

@ -20,8 +20,8 @@ from typing import (
Any,
Callable,
Deque,
Generator,
Iterable,
Iterator,
List,
Optional,
Type,
@ -227,7 +227,7 @@ def build_from_settings(
@contextmanager
def set_environ(**kwargs: str) -> Generator[None, Any, None]:
def set_environ(**kwargs: str) -> Iterator[None]:
"""Temporarily set environment variables inside the context manager and
fully restore previous environment afterwards
"""
@ -244,7 +244,7 @@ def set_environ(**kwargs: str) -> Generator[None, Any, None]:
os.environ[k] = v
def walk_callable(node: ast.AST) -> Generator[ast.AST, Any, None]:
def walk_callable(node: ast.AST) -> Iterable[ast.AST]:
"""Similar to ``ast.walk``, but walks only function body and skips nested
functions defined within the node.
"""

32
scrapy/utils/python.py
View File

@ -15,12 +15,10 @@ from itertools import chain
from typing import (
TYPE_CHECKING,
Any,
AsyncGenerator,
AsyncIterable,
AsyncIterator,
Callable,
Dict,
Generator,
Iterable,
Iterator,
List,
@ -163,7 +161,7 @@ def re_rsearch(
the start position of the match, and the ending (regarding the entire text).
"""
def _chunk_iter() -> Generator[Tuple[str, int], Any, None]:
def _chunk_iter() -> Iterable[Tuple[str, int]]:
offset = len(text)
while True:
offset -= chunk_size * 1024
@ -351,43 +349,45 @@ else:
gc.collect()
class MutableChain(Iterable):
class MutableChain(Iterable[_T]):
"""
Thin wrapper around itertools.chain, allowing to add iterables "in-place"
"""
def __init__(self, *args: Iterable):
self.data = chain.from_iterable(args)
def __init__(self, *args: Iterable[_T]):
self.data: Iterator[_T] = chain.from_iterable(args)
def extend(self, *iterables: Iterable) -> None:
def extend(self, *iterables: Iterable[_T]) -> None:
self.data = chain(self.data, chain.from_iterable(iterables))
def __iter__(self) -> Iterator:
def __iter__(self) -> Iterator[_T]:
return self
def __next__(self) -> Any:
def __next__(self) -> _T:
return next(self.data)
async def _async_chain(*iterables: Union[Iterable, AsyncIterable]) -> AsyncGenerator:
async def _async_chain(
*iterables: Union[Iterable[_T], AsyncIterable[_T]]
) -> AsyncIterator[_T]:
for it in iterables:
async for o in as_async_generator(it):
yield o
class MutableAsyncChain(AsyncIterable):
class MutableAsyncChain(AsyncIterable[_T]):
"""
Similar to MutableChain but for async iterables
"""
def __init__(self, *args: Union[Iterable, AsyncIterable]):
self.data = _async_chain(*args)
def __init__(self, *args: Union[Iterable[_T], AsyncIterable[_T]]):
self.data: AsyncIterator[_T] = _async_chain(*args)
def extend(self, *iterables: Union[Iterable, AsyncIterable]) -> None:
def extend(self, *iterables: Union[Iterable[_T], AsyncIterable[_T]]) -> None:
self.data = _async_chain(self.data, _async_chain(*iterables))
def __aiter__(self) -> AsyncIterator:
def __aiter__(self) -> AsyncIterator[_T]:
return self
async def __anext__(self) -> Any:
async def __anext__(self) -> _T:
return await self.data.__anext__()

5
scrapy/utils/request.py
View File

@ -12,7 +12,6 @@ from typing import (
TYPE_CHECKING,
Any,
Dict,
Generator,
Iterable,
List,
Optional,
@ -40,9 +39,7 @@ if TYPE_CHECKING:
from scrapy.crawler import Crawler
def _serialize_headers(
headers: Iterable[bytes], request: Request
) -> Generator[bytes, Any, None]:
def _serialize_headers(headers: Iterable[bytes], request: Request) -> Iterable[bytes]:
for header in headers:
if header in request.headers:
yield header

4
scrapy/utils/sitemap.py
View File

@ -5,7 +5,7 @@ Note: The main purpose of this module is to provide support for the
SitemapSpider, its API is subject to change without notice.
"""
from typing import Any, Dict, Generator, Iterator, Optional, Union
from typing import Any, Dict, Iterable, Iterator, Optional, Union
from urllib.parse import urljoin
import lxml.etree # nosec
@ -42,7 +42,7 @@ class Sitemap:
def sitemap_urls_from_robots(
robots_text: str, base_url: Optional[str] = None
) -> Generator[str, Any, None]:
) -> Iterable[str]:
"""Return an iterator over all sitemap urls contained in the given
robots.txt file
"""

13
scrapy/utils/spider.py
View File

@ -7,7 +7,6 @@ from typing import (
TYPE_CHECKING,
Any,
AsyncGenerator,
Generator,
Iterable,
Literal,
Optional,
@ -34,18 +33,20 @@ _T = TypeVar("_T")
# https://stackoverflow.com/questions/60222982
@overload
def iterate_spider_output(result: AsyncGenerator) -> AsyncGenerator: ... # type: ignore[overload-overlap]
def iterate_spider_output(result: AsyncGenerator[_T, None]) -> AsyncGenerator[_T, None]: ... # type: ignore[overload-overlap]
@overload
def iterate_spider_output(result: CoroutineType) -> Deferred: ...
def iterate_spider_output(result: CoroutineType[Any, Any, _T]) -> Deferred[_T]: ...
@overload
def iterate_spider_output(result: _T) -> Iterable: ...
def iterate_spider_output(result: _T) -> Iterable[Any]: ...
def iterate_spider_output(result: Any) -> Union[Iterable, AsyncGenerator, Deferred]:
def iterate_spider_output(
result: Any,
) -> Union[Iterable[Any], AsyncGenerator[_T, None], Deferred[_T]]:
if inspect.isasyncgen(result):
return result
if inspect.iscoroutine(result):
@ -55,7 +56,7 @@ def iterate_spider_output(result: Any) -> Union[Iterable, AsyncGenerator, Deferr
return arg_to_iter(deferred_from_coro(result))
def iter_spider_classes(module: ModuleType) -> Generator[Type[Spider], Any, None]:
def iter_spider_classes(module: ModuleType) -> Iterable[Type[Spider]]:
"""Return an iterator over all spider classes defined in the given module
that can be instantiated (i.e. which have name)
"""

4
tests/test_commands.py
View File

@ -13,7 +13,7 @@ from shutil import copytree, rmtree
from stat import S_IWRITE as ANYONE_WRITE_PERMISSION
from tempfile import TemporaryFile, mkdtemp
from threading import Timer
from typing import Dict, Generator, Optional, Union
from typing import Dict, Iterator, Optional, Union
from unittest import skipIf
from pytest import mark
@ -674,7 +674,7 @@ class BadSpider(scrapy.Spider):
"""
@contextmanager
def _create_file(self, content, name=None) -> Generator[str, None, None]:
def _create_file(self, content, name=None) -> Iterator[str]:
tmpdir = Path(self.mktemp())
tmpdir.mkdir()
if name:

80
tests_typing/test_http_request.mypy-testing Normal file
View File

@ -0,0 +1,80 @@
from typing import Any, Dict
import pytest
from scrapy import Request
from scrapy.http import JsonRequest
class MyRequest(Request):
pass
class MyRequest2(Request):
pass
@pytest.mark.mypy_testing
def mypy_test_headers():
Request("data:,", headers=1) # E: Argument "headers" to "Request" has incompatible type "int"; expected "Union[Mapping[str, Any], Iterable[Tuple[str, Any]], None]"
Request("data:,", headers=None)
Request("data:,", headers={})
Request("data:,", headers=[])
Request("data:,", headers={"foo": "bar"})
Request("data:,", headers={b"foo": "bar"})
Request("data:,", headers={"foo": b"bar"})
Request("data:,", headers=[("foo", "bar")])
Request("data:,", headers=[(b"foo", "bar")])
Request("data:,", headers=[("foo", b"bar")])
@pytest.mark.mypy_testing
def mypy_test_copy():
req = Request("data:,")
reveal_type(req) # R: scrapy.http.request.Request
req_copy = req.copy()
reveal_type(req_copy) # R: scrapy.http.request.Request
@pytest.mark.mypy_testing
def mypy_test_copy_subclass():
req = MyRequest("data:,")
reveal_type(req) # R: __main__.MyRequest
req_copy = req.copy()
reveal_type(req_copy) # R: __main__.MyRequest
@pytest.mark.mypy_testing
def mypy_test_replace():
req = Request("data:,")
reveal_type(req) # R: scrapy.http.request.Request
req_copy = req.replace(body=b"a")
reveal_type(req_copy) # R: scrapy.http.request.Request
kwargs: Dict[str, Any] = {}
req_copy2 = req.replace(body=b"a", **kwargs)
reveal_type(req_copy2) # R: Any
@pytest.mark.mypy_testing
def mypy_test_replace_subclass():
req = MyRequest("data:,")
reveal_type(req) # R: __main__.MyRequest
req_copy = req.replace(body=b"a")
reveal_type(req_copy) # R: __main__.MyRequest
req_copy2 = req.replace(body=b"a", cls=MyRequest2)
reveal_type(req_copy2) # R: __main__.MyRequest2
kwargs: Dict[str, Any] = {}
req_copy3 = req.replace(body=b"a", cls=MyRequest2, **kwargs)
reveal_type(req_copy3) # R: __main__.MyRequest2
@pytest.mark.mypy_testing
def mypy_test_jsonrequest_copy_replace():
req = JsonRequest("data:,")
reveal_type(req) # R: scrapy.http.request.json_request.JsonRequest
req_copy = req.copy()
reveal_type(req_copy) # R: scrapy.http.request.json_request.JsonRequest
req_copy = req.replace(body=b"a")
reveal_type(req_copy) # R: scrapy.http.request.json_request.JsonRequest
req_copy_my = req.replace(body=b"a", cls=MyRequest)
reveal_type(req_copy_my) # R: __main__.MyRequest

59
tests_typing/test_http_response.mypy-testing Normal file
View File

@ -0,0 +1,59 @@
from typing import Any, Dict
import pytest
from scrapy.http import HtmlResponse, Response, TextResponse
@pytest.mark.mypy_testing
def mypy_test_headers():
Response("data:,", headers=1) # E: Argument "headers" to "Response" has incompatible type "int"; expected "Union[Mapping[str, Any], Iterable[Tuple[str, Any]], None]"
Response("data:,", headers=None)
Response("data:,", headers={})
Response("data:,", headers=[])
Response("data:,", headers={"foo": "bar"})
Response("data:,", headers={b"foo": "bar"})
Response("data:,", headers={"foo": b"bar"})
Response("data:,", headers=[("foo", "bar")])
Response("data:,", headers=[(b"foo", "bar")])
Response("data:,", headers=[("foo", b"bar")])
@pytest.mark.mypy_testing
def mypy_test_copy():
resp = Response("data:,")
reveal_type(resp) # R: scrapy.http.response.Response
resp_copy = resp.copy()
reveal_type(resp_copy) # R: scrapy.http.response.Response
@pytest.mark.mypy_testing
def mypy_test_copy_subclass():
resp = HtmlResponse("data:,")
reveal_type(resp) # R: scrapy.http.response.html.HtmlResponse
resp_copy = resp.copy()
reveal_type(resp_copy) # R: scrapy.http.response.html.HtmlResponse
@pytest.mark.mypy_testing
def mypy_test_replace():
resp = Response("data:,")
reveal_type(resp) # R: scrapy.http.response.Response
resp_copy = resp.replace(body=b"a")
reveal_type(resp_copy) # R: scrapy.http.response.Response
kwargs: Dict[str, Any] = {}
resp_copy2 = resp.replace(body=b"a", **kwargs)
reveal_type(resp_copy2) # R: Any
@pytest.mark.mypy_testing
def mypy_test_replace_subclass():
resp = HtmlResponse("data:,")
reveal_type(resp) # R: scrapy.http.response.html.HtmlResponse
resp_copy = resp.replace(body=b"a")
reveal_type(resp_copy) # R: scrapy.http.response.html.HtmlResponse
resp_copy2 = resp.replace(body=b"a", cls=TextResponse)
reveal_type(resp_copy2) # R: scrapy.http.response.text.TextResponse
kwargs: Dict[str, Any] = {}
resp_copy3 = resp.replace(body=b"a", cls=TextResponse, **kwargs)
reveal_type(resp_copy3) # R: scrapy.http.response.text.TextResponse

9
tox.ini
View File

@ -47,18 +47,17 @@ install_command =
basepython = python3
deps =
mypy==1.10.0
typing-extensions==4.11.0
typing-extensions==4.12.1
types-lxml==2024.4.14
types-Pygments==2.18.0.20240506
types-pyOpenSSL==24.1.0.20240425
types-setuptools==69.5.0.20240518
types-setuptools==70.0.0.20240524
botocore-stubs==1.34.94
boto3-stubs[s3]==1.34.108
boto3-stubs[s3]==1.34.119
attrs >= 18.2.0
Pillow >= 10.3.0
pytest >= 8.2.0
# 2.1.2 fixes a typing bug: https://github.com/scrapy/w3lib/pull/211
w3lib >= 2.1.2
w3lib >= 2.2.0
commands =
mypy {posargs: scrapy tests}