RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add parameters to iterable generics, replace generators with iterables.

This commit is contained in:
Andrey Rakhmatullin 2024-06-02 01:48:37 +05:00
parent 3f76853bd2
commit 2e214210f6
13 changed files with 103 additions and 84 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
scrapy/commands/parse.py
View File

@ -140,13 +140,13 @@ class Command(BaseRunSpiderCommand):
@overload
def iterate_spider_output(
self, result: Union[AsyncGenerator, CoroutineType]
) -> Deferred: ...
self, result: Union[AsyncGenerator[_T, None], CoroutineType[Any, Any, _T]]
) -> Deferred[_T]: ...
@overload
def iterate_spider_output(self, result: _T) -> Iterable: ...
def iterate_spider_output(self, result: _T) -> Iterable[Any]: ...
def iterate_spider_output(self, result: Any) -> Union[Iterable, Deferred]:
def iterate_spider_output(self, result: Any) -> Union[Iterable[Any], Deferred]:
if inspect.isasyncgen(result):
d = deferred_from_coro(
collect_asyncgen(aiter_errback(result, self.handle_exception))

5
scrapy/core/engine.py
View File

@ -372,7 +372,10 @@ class ExecutionEngine:
@inlineCallbacks
def open_spider(
self, spider: Spider, start_requests: Iterable = (), close_if_idle: bool = True
self,
spider: Spider,
start_requests: Iterable[Request] = (),
close_if_idle: bool = True,
) -> Generator[Deferred, Any, None]:
if self.slot is not None:
raise RuntimeError(f"No free spider slot when opening {spider.name!r}")

6
scrapy/core/scraper.py
View File

@ -16,6 +16,7 @@ from typing import (
Set,
Tuple,
Type,
TypeVar,
Union,
cast,
)
@ -47,6 +48,7 @@ if TYPE_CHECKING:
from scrapy.crawler import Crawler
_T = TypeVar("_T")
QueueTuple = Tuple[Union[Response, Failure], Request, Deferred]
@ -256,14 +258,14 @@ class Scraper:
def handle_spider_output(
self,
result: Union[Iterable, AsyncIterable],
result: Union[Iterable[_T], AsyncIterable[_T]],
request: Request,
response: Response,
spider: Spider,
) -> Deferred:
if not result:
return defer_succeed(None)
it: Union[Iterable, AsyncIterable]
it: Union[Iterable[_T], AsyncIterable[_T]]
if isinstance(result, AsyncIterable):
it = aiter_errback(
result, self.handle_spider_error, request, response, spider

70
scrapy/core/spidermw.py
View File

@ -9,7 +9,6 @@ from inspect import isasyncgenfunction, iscoroutine
from itertools import islice
from typing import (
Any,
AsyncGenerator,
AsyncIterable,
Callable,
Generator,
@ -17,6 +16,7 @@ from typing import (
List,
Optional,
Tuple,
TypeVar,
Union,
cast,
)
@ -42,6 +42,7 @@ from scrapy.utils.python import MutableAsyncChain, MutableChain
logger = logging.getLogger(__name__)
_T = TypeVar("_T")
ScrapeFunc = Callable[[Union[Response, Failure], Request, Spider], Any]
@ -98,31 +99,39 @@ class SpiderMiddlewareManager(MiddlewareManager):
self,
response: Response,
spider: Spider,
iterable: Union[Iterable, AsyncIterable],
iterable: Union[Iterable[_T], AsyncIterable[_T]],
exception_processor_index: int,
recover_to: Union[MutableChain, MutableAsyncChain],
) -> Union[Generator, AsyncGenerator]:
def process_sync(iterable: Iterable) -> Generator:
recover_to: Union[MutableChain[_T], MutableAsyncChain[_T]],
) -> Union[Iterable[_T], AsyncIterable[_T]]:
def process_sync(iterable: Iterable[_T]) -> Iterable[_T]:
try:
yield from iterable
except Exception as ex:
exception_result = self._process_spider_exception(
response, spider, Failure(ex), exception_processor_index
exception_result = cast(
Union[Failure, MutableChain[_T]],
self._process_spider_exception(
response, spider, Failure(ex), exception_processor_index
),
)
if isinstance(exception_result, Failure):
raise
assert isinstance(recover_to, MutableChain)
recover_to.extend(exception_result)
async def process_async(iterable: AsyncIterable) -> AsyncGenerator:
async def process_async(iterable: AsyncIterable[_T]) -> AsyncIterable[_T]:
try:
async for r in iterable:
yield r
except Exception as ex:
exception_result = self._process_spider_exception(
response, spider, Failure(ex), exception_processor_index
exception_result = cast(
Union[Failure, MutableAsyncChain[_T]],
self._process_spider_exception(
response, spider, Failure(ex), exception_processor_index
),
)
if isinstance(exception_result, Failure):
raise
assert isinstance(recover_to, MutableAsyncChain)
recover_to.extend(exception_result)
if isinstance(iterable, AsyncIterable):
@ -135,7 +144,7 @@ class SpiderMiddlewareManager(MiddlewareManager):
spider: Spider,
_failure: Failure,
start_index: int = 0,
) -> Union[Failure, MutableChain]:
) -> Union[Failure, MutableChain[_T], MutableAsyncChain[_T]]:
exception = _failure.value
# don't handle _InvalidOutput exception
if isinstance(exception, _InvalidOutput):
@ -151,14 +160,18 @@ class SpiderMiddlewareManager(MiddlewareManager):
if _isiterable(result):
# stop exception handling by handing control over to the
# process_spider_output chain if an iterable has been returned
dfd: Deferred = self._process_spider_output(
response, spider, result, method_index + 1
dfd: Deferred[Union[MutableChain[_T], MutableAsyncChain[_T]]] = (
self._process_spider_output(
response, spider, result, method_index + 1
)
)
# _process_spider_output() returns a Deferred only because of downgrading so this can be
# simplified when downgrading is removed.
if dfd.called:
# the result is available immediately if _process_spider_output didn't do downgrading
return cast(MutableChain, dfd.result)
return cast(
Union[MutableChain[_T], MutableAsyncChain[_T]], dfd.result
)
# we forbid waiting here because otherwise we would need to return a deferred from
# _process_spider_exception too, which complicates the architecture
msg = f"Async iterable returned from {method.__qualname__} cannot be downgraded"
@ -181,12 +194,12 @@ class SpiderMiddlewareManager(MiddlewareManager):
self,
response: Response,
spider: Spider,
result: Union[Iterable, AsyncIterable],
result: Union[Iterable[_T], AsyncIterable[_T]],
start_index: int = 0,
) -> Generator[Deferred, Any, Union[MutableChain, MutableAsyncChain]]:
) -> Generator[Deferred[Any], Any, Union[MutableChain[_T], MutableAsyncChain[_T]]]:
# items in this iterable do not need to go through the process_spider_output
# chain, they went through it already from the process_spider_exception method
recovered: Union[MutableChain, MutableAsyncChain]
recovered: Union[MutableChain[_T], MutableAsyncChain[_T]]
last_result_is_async = isinstance(result, AsyncIterable)
if last_result_is_async:
recovered = MutableAsyncChain()
@ -237,7 +250,9 @@ class SpiderMiddlewareManager(MiddlewareManager):
# might fail directly if the output value is not a generator
result = method(response=response, result=result, spider=spider)
except Exception as ex:
exception_result = self._process_spider_exception(
exception_result: Union[
Failure, MutableChain[_T], MutableAsyncChain[_T]
] = self._process_spider_exception(
response, spider, Failure(ex), method_index + 1
)
if isinstance(exception_result, Failure):
@ -267,9 +282,12 @@ class SpiderMiddlewareManager(MiddlewareManager):
return MutableChain(result, recovered) # type: ignore[arg-type]
async def _process_callback_output(
self, response: Response, spider: Spider, result: Union[Iterable, AsyncIterable]
) -> Union[MutableChain, MutableAsyncChain]:
recovered: Union[MutableChain, MutableAsyncChain]
self,
response: Response,
spider: Spider,
result: Union[Iterable[_T], AsyncIterable[_T]],
) -> Union[MutableChain[_T], MutableAsyncChain[_T]]:
recovered: Union[MutableChain[_T], MutableAsyncChain[_T]]
if isinstance(result, AsyncIterable):
recovered = MutableAsyncChain()
else:
@ -293,14 +311,16 @@ class SpiderMiddlewareManager(MiddlewareManager):
spider: Spider,
) -> Deferred:
async def process_callback_output(
result: Union[Iterable, AsyncIterable]
) -> Union[MutableChain, MutableAsyncChain]:
result: Union[Iterable[_T], AsyncIterable[_T]]
) -> Union[MutableChain[_T], MutableAsyncChain[_T]]:
return await self._process_callback_output(response, spider, result)
def process_spider_exception(_failure: Failure) -> Union[Failure, MutableChain]:
def process_spider_exception(
_failure: Failure,
) -> Union[Failure, MutableChain[_T], MutableAsyncChain[_T]]:
return self._process_spider_exception(response, spider, _failure)
dfd = mustbe_deferred(
dfd: Deferred = mustbe_deferred(
self._process_spider_input, scrape_func, response, request, spider
)
dfd.addCallback(deferred_f_from_coro_f(process_callback_output))

3
scrapy/http/response/__init__.py
View File

@ -14,7 +14,6 @@ from typing import (
AnyStr,
Callable,
Dict,
Generator,
Iterable,
List,
Mapping,
@ -242,7 +241,7 @@ class Response(object_ref):
errback: Optional[Callable] = None,
cb_kwargs: Optional[Dict[str, Any]] = None,
flags: Optional[List[str]] = None,
) -> Generator[Request, None, None]:
) -> Iterable[Request]:
"""
.. versionadded:: 2.0

3
scrapy/http/response/text.py
View File

@ -15,7 +15,6 @@ from typing import (
AnyStr,
Callable,
Dict,
Generator,
Iterable,
List,
Mapping,
@ -246,7 +245,7 @@ class TextResponse(Response):
flags: Optional[List[str]] = None,
css: Optional[str] = None,
xpath: Optional[str] = None,
) -> Generator[Request, None, None]:
) -> Iterable[Request]:
"""
A generator that produces :class:`~.Request` instances to follow all
links in ``urls``. It accepts the same arguments as the :class:`~.Request`'s

28
scrapy/utils/iterators.py
View File

@ -6,8 +6,7 @@ from typing import (
Any,
Callable,
Dict,
Generator,
Iterable,
Iterator,
List,
Literal,
Optional,
@ -22,14 +21,12 @@ from lxml import etree # nosec
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.http import Response, TextResponse
from scrapy.selector import Selector
from scrapy.utils.python import re_rsearch, to_unicode
from scrapy.utils.python import re_rsearch
logger = logging.getLogger(__name__)
def xmliter(
obj: Union[Response, str, bytes], nodename: str
) -> Generator[Selector, Any, None]:
def xmliter(obj: Union[Response, str, bytes], nodename: str) -> Iterator[Selector]:
"""Return a iterator of Selector's over all nodes of a XML document,
given the name of the node to iterate. Useful for parsing XML feeds.
@ -90,7 +87,7 @@ def xmliter_lxml(
nodename: str,
namespace: Optional[str] = None,
prefix: str = "x",
) -> Generator[Selector, Any, None]:
) -> Iterator[Selector]:
reader = _StreamReader(obj)
tag = f"{{{namespace}}}{nodename}" if namespace else nodename
iterable = etree.iterparse(
@ -168,7 +165,7 @@ def csviter(
headers: Optional[List[str]] = None,
encoding: Optional[str] = None,
quotechar: Optional[str] = None,
) -> Generator[Dict[str, str], Any, None]:
) -> Iterator[Dict[str, str]]:
"""Returns an iterator of dictionaries from the given csv object
obj can be:
@ -184,10 +181,13 @@ def csviter(
quotechar is the character used to enclosure fields on the given obj.
"""
encoding = obj.encoding if isinstance(obj, TextResponse) else encoding or "utf-8"
def row_to_unicode(row_: Iterable) -> List[str]:
return [to_unicode(field, encoding) for field in row_]
if encoding is not None:
warn(
"The encoding argument of csviter() is ignored and will be removed"
" in a future Scrapy version.",
category=ScrapyDeprecationWarning,
stacklevel=2,
)
lines = StringIO(_body_or_str(obj, unicode=True))
@ -200,13 +200,11 @@ def csviter(
if not headers:
try:
row = next(csv_r)
headers = next(csv_r)
except StopIteration:
return
headers = row_to_unicode(row)
for row in csv_r:
row = row_to_unicode(row)
if len(row) != len(headers):
logger.warning(
"ignoring row %(csvlnum)d (length: %(csvrow)d, "

6
scrapy/utils/misc.py
View File

@ -20,8 +20,8 @@ from typing import (
Any,
Callable,
Deque,
Generator,
Iterable,
Iterator,
List,
Optional,
Type,
@ -227,7 +227,7 @@ def build_from_settings(
@contextmanager
def set_environ(**kwargs: str) -> Generator[None, Any, None]:
def set_environ(**kwargs: str) -> Iterator[None]:
"""Temporarily set environment variables inside the context manager and
fully restore previous environment afterwards
"""
@ -244,7 +244,7 @@ def set_environ(**kwargs: str) -> Generator[None, Any, None]:
os.environ[k] = v
def walk_callable(node: ast.AST) -> Generator[ast.AST, Any, None]:
def walk_callable(node: ast.AST) -> Iterable[ast.AST]:
"""Similar to ``ast.walk``, but walks only function body and skips nested
functions defined within the node.
"""

32
scrapy/utils/python.py
View File

@ -15,12 +15,10 @@ from itertools import chain
from typing import (
TYPE_CHECKING,
Any,
AsyncGenerator,
AsyncIterable,
AsyncIterator,
Callable,
Dict,
Generator,
Iterable,
Iterator,
List,
@ -163,7 +161,7 @@ def re_rsearch(
the start position of the match, and the ending (regarding the entire text).
"""
def _chunk_iter() -> Generator[Tuple[str, int], Any, None]:
def _chunk_iter() -> Iterable[Tuple[str, int]]:
offset = len(text)
while True:
offset -= chunk_size * 1024
@ -351,43 +349,45 @@ else:
gc.collect()
class MutableChain(Iterable):
class MutableChain(Iterable[_T]):
"""
Thin wrapper around itertools.chain, allowing to add iterables "in-place"
"""
def __init__(self, *args: Iterable):
self.data = chain.from_iterable(args)
def __init__(self, *args: Iterable[_T]):
self.data: Iterator[_T] = chain.from_iterable(args)
def extend(self, *iterables: Iterable) -> None:
def extend(self, *iterables: Iterable[_T]) -> None:
self.data = chain(self.data, chain.from_iterable(iterables))
def __iter__(self) -> Iterator:
def __iter__(self) -> Iterator[_T]:
return self
def __next__(self) -> Any:
def __next__(self) -> _T:
return next(self.data)
async def _async_chain(*iterables: Union[Iterable, AsyncIterable]) -> AsyncGenerator:
async def _async_chain(
*iterables: Union[Iterable[_T], AsyncIterable[_T]]
) -> AsyncIterator[_T]:
for it in iterables:
async for o in as_async_generator(it):
yield o
class MutableAsyncChain(AsyncIterable):
class MutableAsyncChain(AsyncIterable[_T]):
"""
Similar to MutableChain but for async iterables
"""
def __init__(self, *args: Union[Iterable, AsyncIterable]):
self.data = _async_chain(*args)
def __init__(self, *args: Union[Iterable[_T], AsyncIterable[_T]]):
self.data: AsyncIterator[_T] = _async_chain(*args)
def extend(self, *iterables: Union[Iterable, AsyncIterable]) -> None:
def extend(self, *iterables: Union[Iterable[_T], AsyncIterable[_T]]) -> None:
self.data = _async_chain(self.data, _async_chain(*iterables))
def __aiter__(self) -> AsyncIterator:
def __aiter__(self) -> AsyncIterator[_T]:
return self
async def __anext__(self) -> Any:
async def __anext__(self) -> _T:
return await self.data.__anext__()

5
scrapy/utils/request.py
View File

@ -12,7 +12,6 @@ from typing import (
TYPE_CHECKING,
Any,
Dict,
Generator,
Iterable,
List,
Optional,
@ -40,9 +39,7 @@ if TYPE_CHECKING:
from scrapy.crawler import Crawler
def _serialize_headers(
headers: Iterable[bytes], request: Request
) -> Generator[bytes, Any, None]:
def _serialize_headers(headers: Iterable[bytes], request: Request) -> Iterable[bytes]:
for header in headers:
if header in request.headers:
yield header

4
scrapy/utils/sitemap.py
View File

@ -5,7 +5,7 @@ Note: The main purpose of this module is to provide support for the
SitemapSpider, its API is subject to change without notice.
"""
from typing import Any, Dict, Generator, Iterator, Optional, Union
from typing import Any, Dict, Iterable, Iterator, Optional, Union
from urllib.parse import urljoin
import lxml.etree # nosec
@ -42,7 +42,7 @@ class Sitemap:
def sitemap_urls_from_robots(
robots_text: str, base_url: Optional[str] = None
) -> Generator[str, Any, None]:
) -> Iterable[str]:
"""Return an iterator over all sitemap urls contained in the given
robots.txt file
"""

13
scrapy/utils/spider.py
View File

@ -7,7 +7,6 @@ from typing import (
TYPE_CHECKING,
Any,
AsyncGenerator,
Generator,
Iterable,
Literal,
Optional,
@ -34,18 +33,20 @@ _T = TypeVar("_T")
# https://stackoverflow.com/questions/60222982
@overload
def iterate_spider_output(result: AsyncGenerator) -> AsyncGenerator: ... # type: ignore[overload-overlap]
def iterate_spider_output(result: AsyncGenerator[_T, None]) -> AsyncGenerator[_T, None]: ... # type: ignore[overload-overlap]
@overload
def iterate_spider_output(result: CoroutineType) -> Deferred: ...
def iterate_spider_output(result: CoroutineType[Any, Any, _T]) -> Deferred[_T]: ...
@overload
def iterate_spider_output(result: _T) -> Iterable: ...
def iterate_spider_output(result: _T) -> Iterable[Any]: ...
def iterate_spider_output(result: Any) -> Union[Iterable, AsyncGenerator, Deferred]:
def iterate_spider_output(
result: Any,
) -> Union[Iterable[Any], AsyncGenerator[_T, None], Deferred[_T]]:
if inspect.isasyncgen(result):
return result
if inspect.iscoroutine(result):
@ -55,7 +56,7 @@ def iterate_spider_output(result: Any) -> Union[Iterable, AsyncGenerator, Deferr
return arg_to_iter(deferred_from_coro(result))
def iter_spider_classes(module: ModuleType) -> Generator[Type[Spider], Any, None]:
def iter_spider_classes(module: ModuleType) -> Iterable[Type[Spider]]:
"""Return an iterator over all spider classes defined in the given module
that can be instantiated (i.e. which have name)
"""

4
tests/test_commands.py
View File

@ -13,7 +13,7 @@ from shutil import copytree, rmtree
from stat import S_IWRITE as ANYONE_WRITE_PERMISSION
from tempfile import TemporaryFile, mkdtemp
from threading import Timer
from typing import Dict, Generator, Optional, Union
from typing import Dict, Iterator, Optional, Union
from unittest import skipIf
from pytest import mark
@ -674,7 +674,7 @@ class BadSpider(scrapy.Spider):
"""
@contextmanager
def _create_file(self, content, name=None) -> Generator[str, None, None]:
def _create_file(self, content, name=None) -> Iterator[str]:
tmpdir = Path(self.mktemp())
tmpdir.mkdir()
if name: