mirror of https://github.com/scrapy/scrapy.git
Merge branch 'master' into typing-request-response-cls
This commit is contained in:
Andrey Rakhmatullin
commit
1268b23304
|
|
@ -115,15 +115,14 @@ Handling different response formats
|
|||
Once you have a response with the desired data, how you extract the desired
|
||||
data from it depends on the type of response:
|
||||
|
||||
- If the response is HTML or XML, use :ref:`selectors
|
||||
- If the response is HTML, XML or JSON, use :ref:`selectors
|
||||
<topics-selectors>` as usual.
|
||||
|
||||
- If the response is JSON, use :func:`json.loads` to load the desired data from
|
||||
:attr:`response.text <scrapy.http.TextResponse.text>`:
|
||||
- If the response is JSON, use :func:`response.json()` to load the desired data:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
data = json.loads(response.text)
|
||||
data = response.json()
|
||||
|
||||
If the desired data is inside HTML or XML code embedded within JSON data,
|
||||
you can load that HTML or XML code into a
|
||||
|
|
|
|||
|
|
@ -1060,6 +1060,12 @@ Selector objects
|
|||
|
||||
For convenience, this method can be called as ``response.css()``
|
||||
|
||||
.. automethod:: jmespath
|
||||
|
||||
.. note::
|
||||
|
||||
For convenience, this method can be called as ``response.jmespath()``
|
||||
|
||||
.. automethod:: get
|
||||
|
||||
See also: :ref:`old-extraction-api`
|
||||
|
|
@ -1092,6 +1098,8 @@ SelectorList objects
|
|||
|
||||
.. automethod:: css
|
||||
|
||||
.. automethod:: jmespath
|
||||
|
||||
.. automethod:: getall
|
||||
|
||||
See also: :ref:`old-extraction-api`
|
||||
|
|
|
|||
|
|
@ -1,13 +1,15 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import functools
|
||||
import inspect
|
||||
import json
|
||||
import logging
|
||||
from types import CoroutineType
|
||||
from typing import (
|
||||
Any,
|
||||
AsyncGenerator,
|
||||
Callable,
|
||||
Coroutine,
|
||||
Dict,
|
||||
Iterable,
|
||||
List,
|
||||
|
|
@ -140,13 +142,13 @@ class Command(BaseRunSpiderCommand):
|
|||
|
||||
@overload
|
||||
def iterate_spider_output(
|
||||
self, result: Union[AsyncGenerator, CoroutineType]
|
||||
) -> Deferred: ...
|
||||
self, result: Union[AsyncGenerator[_T, None], Coroutine[Any, Any, _T]]
|
||||
) -> Deferred[_T]: ...
|
||||
|
||||
@overload
|
||||
def iterate_spider_output(self, result: _T) -> Iterable: ...
|
||||
def iterate_spider_output(self, result: _T) -> Iterable[Any]: ...
|
||||
|
||||
def iterate_spider_output(self, result: Any) -> Union[Iterable, Deferred]:
|
||||
def iterate_spider_output(self, result: Any) -> Union[Iterable[Any], Deferred]:
|
||||
if inspect.isasyncgen(result):
|
||||
d = deferred_from_coro(
|
||||
collect_asyncgen(aiter_errback(result, self.handle_exception))
|
||||
|
|
|
|||
|
|
@ -120,7 +120,8 @@ class ContractsManager:
|
|||
|
||||
if line.startswith("@"):
|
||||
m = re.match(r"@(\w+)\s*(.*)", line)
|
||||
assert m is not None
|
||||
if m is None:
|
||||
continue
|
||||
name, args = m.groups()
|
||||
args = re.split(r"\s+", args)
|
||||
|
||||
|
|
|
|||
|
|
@ -1,5 +1,7 @@
|
|||
"""Download handlers for different schemes"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import TYPE_CHECKING, Any, Callable, Dict, Generator, Union, cast
|
||||
|
||||
|
|
@ -19,15 +21,18 @@ logger = logging.getLogger(__name__)
|
|||
|
||||
|
||||
class DownloadHandlers:
|
||||
def __init__(self, crawler: "Crawler"):
|
||||
self._crawler: "Crawler" = crawler
|
||||
self._schemes: Dict[str, Union[str, Callable]] = (
|
||||
def __init__(self, crawler: Crawler):
|
||||
self._crawler: Crawler = crawler
|
||||
self._schemes: Dict[str, Union[str, Callable[..., Any]]] = (
|
||||
{}
|
||||
) # stores acceptable schemes on instancing
|
||||
self._handlers: Dict[str, Any] = {} # stores instanced handlers for schemes
|
||||
self._notconfigured: Dict[str, str] = {} # remembers failed handlers
|
||||
handlers: Dict[str, Union[str, Callable]] = without_none_values(
|
||||
crawler.settings.getwithbase("DOWNLOAD_HANDLERS")
|
||||
handlers: Dict[str, Union[str, Callable[..., Any]]] = without_none_values(
|
||||
cast(
|
||||
Dict[str, Union[str, Callable[..., Any]]],
|
||||
crawler.settings.getwithbase("DOWNLOAD_HANDLERS"),
|
||||
)
|
||||
)
|
||||
for scheme, clspath in handlers.items():
|
||||
self._schemes[scheme] = clspath
|
||||
|
|
|
|||
|
|
@ -5,6 +5,8 @@ For more information see docs/topics/architecture.rst
|
|||
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from time import time
|
||||
from typing import (
|
||||
|
|
@ -51,15 +53,15 @@ class Slot:
|
|||
self,
|
||||
start_requests: Iterable[Request],
|
||||
close_if_idle: bool,
|
||||
nextcall: CallLaterOnce,
|
||||
scheduler: "BaseScheduler",
|
||||
nextcall: CallLaterOnce[None],
|
||||
scheduler: BaseScheduler,
|
||||
) -> None:
|
||||
self.closing: Optional[Deferred] = None
|
||||
self.inprogress: Set[Request] = set()
|
||||
self.start_requests: Optional[Iterator[Request]] = iter(start_requests)
|
||||
self.close_if_idle: bool = close_if_idle
|
||||
self.nextcall: CallLaterOnce = nextcall
|
||||
self.scheduler: "BaseScheduler" = scheduler
|
||||
self.nextcall: CallLaterOnce[None] = nextcall
|
||||
self.scheduler: BaseScheduler = scheduler
|
||||
self.heartbeat: LoopingCall = LoopingCall(nextcall.schedule)
|
||||
|
||||
def add_request(self, request: Request) -> None:
|
||||
|
|
@ -84,8 +86,12 @@ class Slot:
|
|||
|
||||
|
||||
class ExecutionEngine:
|
||||
def __init__(self, crawler: "Crawler", spider_closed_callback: Callable) -> None:
|
||||
self.crawler: "Crawler" = crawler
|
||||
def __init__(
|
||||
self,
|
||||
crawler: Crawler,
|
||||
spider_closed_callback: Callable[[Spider], Optional[Deferred[None]]],
|
||||
) -> None:
|
||||
self.crawler: Crawler = crawler
|
||||
self.settings: Settings = crawler.settings
|
||||
self.signals: SignalManager = crawler.signals
|
||||
assert crawler.logformatter
|
||||
|
|
@ -94,19 +100,21 @@ class ExecutionEngine:
|
|||
self.spider: Optional[Spider] = None
|
||||
self.running: bool = False
|
||||
self.paused: bool = False
|
||||
self.scheduler_cls: Type["BaseScheduler"] = self._get_scheduler_class(
|
||||
self.scheduler_cls: Type[BaseScheduler] = self._get_scheduler_class(
|
||||
crawler.settings
|
||||
)
|
||||
downloader_cls: Type[Downloader] = load_object(self.settings["DOWNLOADER"])
|
||||
self.downloader: Downloader = downloader_cls(crawler)
|
||||
self.scraper = Scraper(crawler)
|
||||
self._spider_closed_callback: Callable = spider_closed_callback
|
||||
self._spider_closed_callback: Callable[[Spider], Optional[Deferred[None]]] = (
|
||||
spider_closed_callback
|
||||
)
|
||||
self.start_time: Optional[float] = None
|
||||
|
||||
def _get_scheduler_class(self, settings: BaseSettings) -> Type["BaseScheduler"]:
|
||||
def _get_scheduler_class(self, settings: BaseSettings) -> Type[BaseScheduler]:
|
||||
from scrapy.core.scheduler import BaseScheduler
|
||||
|
||||
scheduler_cls: Type = load_object(settings["SCHEDULER"])
|
||||
scheduler_cls: Type[BaseScheduler] = load_object(settings["SCHEDULER"])
|
||||
if not issubclass(scheduler_cls, BaseScheduler):
|
||||
raise TypeError(
|
||||
f"The provided scheduler class ({settings['SCHEDULER']})"
|
||||
|
|
@ -364,7 +372,10 @@ class ExecutionEngine:
|
|||
|
||||
@inlineCallbacks
|
||||
def open_spider(
|
||||
self, spider: Spider, start_requests: Iterable = (), close_if_idle: bool = True
|
||||
self,
|
||||
spider: Spider,
|
||||
start_requests: Iterable[Request] = (),
|
||||
close_if_idle: bool = True,
|
||||
) -> Generator[Deferred, Any, None]:
|
||||
if self.slot is not None:
|
||||
raise RuntimeError(f"No free spider slot when opening {spider.name!r}")
|
||||
|
|
@ -425,7 +436,7 @@ class ExecutionEngine:
|
|||
|
||||
dfd = self.slot.close()
|
||||
|
||||
def log_failure(msg: str) -> Callable:
|
||||
def log_failure(msg: str) -> Callable[[Failure], None]:
|
||||
def errback(failure: Failure) -> None:
|
||||
logger.error(
|
||||
msg, exc_info=failure_to_exc_info(failure), extra={"spider": spider}
|
||||
|
|
|
|||
|
|
@ -20,6 +20,8 @@ from scrapy.http.request import Request
|
|||
from scrapy.settings import Settings
|
||||
from scrapy.spiders import Spider
|
||||
|
||||
ConnectionKeyT = Tuple[bytes, bytes, int]
|
||||
|
||||
|
||||
class H2ConnectionPool:
|
||||
def __init__(self, reactor: ReactorBase, settings: Settings) -> None:
|
||||
|
|
@ -28,13 +30,13 @@ class H2ConnectionPool:
|
|||
|
||||
# Store a dictionary which is used to get the respective
|
||||
# H2ClientProtocolInstance using the key as Tuple(scheme, hostname, port)
|
||||
self._connections: Dict[Tuple, H2ClientProtocol] = {}
|
||||
self._connections: Dict[ConnectionKeyT, H2ClientProtocol] = {}
|
||||
|
||||
# Save all requests that arrive before the connection is established
|
||||
self._pending_requests: Dict[Tuple, Deque[Deferred]] = {}
|
||||
self._pending_requests: Dict[ConnectionKeyT, Deque[Deferred]] = {}
|
||||
|
||||
def get_connection(
|
||||
self, key: Tuple, uri: URI, endpoint: HostnameEndpoint
|
||||
self, key: ConnectionKeyT, uri: URI, endpoint: HostnameEndpoint
|
||||
) -> Deferred:
|
||||
if key in self._pending_requests:
|
||||
# Received a request while connecting to remote
|
||||
|
|
@ -54,7 +56,7 @@ class H2ConnectionPool:
|
|||
return self._new_connection(key, uri, endpoint)
|
||||
|
||||
def _new_connection(
|
||||
self, key: Tuple, uri: URI, endpoint: HostnameEndpoint
|
||||
self, key: ConnectionKeyT, uri: URI, endpoint: HostnameEndpoint
|
||||
) -> Deferred:
|
||||
self._pending_requests[key] = deque()
|
||||
|
||||
|
|
@ -69,7 +71,9 @@ class H2ConnectionPool:
|
|||
self._pending_requests[key].append(d)
|
||||
return d
|
||||
|
||||
def put_connection(self, conn: H2ClientProtocol, key: Tuple) -> H2ClientProtocol:
|
||||
def put_connection(
|
||||
self, conn: H2ClientProtocol, key: ConnectionKeyT
|
||||
) -> H2ClientProtocol:
|
||||
self._connections[key] = conn
|
||||
|
||||
# Now as we have established a proper HTTP/2 connection
|
||||
|
|
@ -81,7 +85,9 @@ class H2ConnectionPool:
|
|||
|
||||
return conn
|
||||
|
||||
def _remove_connection(self, errors: List[BaseException], key: Tuple) -> None:
|
||||
def _remove_connection(
|
||||
self, errors: List[BaseException], key: ConnectionKeyT
|
||||
) -> None:
|
||||
self._connections.pop(key)
|
||||
|
||||
# Call the errback of all the pending requests for this connection
|
||||
|
|
@ -122,7 +128,7 @@ class H2Agent:
|
|||
def get_endpoint(self, uri: URI) -> HostnameEndpoint:
|
||||
return self.endpoint_factory.endpointForURI(uri)
|
||||
|
||||
def get_key(self, uri: URI) -> Tuple:
|
||||
def get_key(self, uri: URI) -> ConnectionKeyT:
|
||||
"""
|
||||
Arguments:
|
||||
uri - URI obtained directly from request URL
|
||||
|
|
@ -164,6 +170,6 @@ class ScrapyProxyH2Agent(H2Agent):
|
|||
def get_endpoint(self, uri: URI) -> HostnameEndpoint:
|
||||
return self.endpoint_factory.endpointForURI(self._proxy_uri)
|
||||
|
||||
def get_key(self, uri: URI) -> Tuple:
|
||||
def get_key(self, uri: URI) -> ConnectionKeyT:
|
||||
"""We use the proxy uri instead of uri obtained from request url"""
|
||||
return "http-proxy", self._proxy_uri.host, self._proxy_uri.port
|
||||
return b"http-proxy", self._proxy_uri.host, self._proxy_uri.port
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@ import itertools
|
|||
import logging
|
||||
from collections import deque
|
||||
from ipaddress import IPv4Address, IPv6Address
|
||||
from typing import Dict, List, Optional, Union
|
||||
from typing import Any, Deque, Dict, List, Optional, Union
|
||||
|
||||
from h2.config import H2Configuration
|
||||
from h2.connection import H2Connection
|
||||
|
|
@ -107,7 +107,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin):
|
|||
|
||||
# If requests are received before connection is made we keep
|
||||
# all requests in a pool and send them as the connection is made
|
||||
self._pending_request_stream_pool: deque = deque()
|
||||
self._pending_request_stream_pool: Deque[Stream] = deque()
|
||||
|
||||
# Save an instance of errors raised which lead to losing the connection
|
||||
# We pass these instances to the streams ResponseFailed() failure
|
||||
|
|
@ -115,7 +115,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin):
|
|||
|
||||
# Some meta data of this connection
|
||||
# initialized when connection is successfully made
|
||||
self.metadata: Dict = {
|
||||
self.metadata: Dict[str, Any] = {
|
||||
# Peer certificate instance
|
||||
"certificate": None,
|
||||
# Address of the server we are connected to which
|
||||
|
|
|
|||
|
|
@ -110,7 +110,7 @@ class Stream:
|
|||
|
||||
# Metadata of an HTTP/2 connection stream
|
||||
# initialized when stream is instantiated
|
||||
self.metadata: Dict = {
|
||||
self.metadata: Dict[str, Any] = {
|
||||
"request_content_length": (
|
||||
0 if self._request.body is None else len(self._request.body)
|
||||
),
|
||||
|
|
@ -131,7 +131,7 @@ class Stream:
|
|||
# Private variable used to build the response
|
||||
# this response is then converted to appropriate Response class
|
||||
# passed to the response deferred callback
|
||||
self._response: Dict = {
|
||||
self._response: Dict[str, Any] = {
|
||||
# Data received frame by frame from the server is appended
|
||||
# and passed to the response Deferred when completely received.
|
||||
"body": BytesIO(),
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ import json
|
|||
import logging
|
||||
from abc import abstractmethod
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING, Any, Optional, Type, cast
|
||||
from typing import TYPE_CHECKING, Any, List, Optional, Type, cast
|
||||
|
||||
from twisted.internet.defer import Deferred
|
||||
|
||||
|
|
@ -362,13 +362,13 @@ class Scheduler(BaseScheduler):
|
|||
return str(dqdir)
|
||||
return None
|
||||
|
||||
def _read_dqs_state(self, dqdir: str) -> list:
|
||||
def _read_dqs_state(self, dqdir: str) -> List[int]:
|
||||
path = Path(dqdir, "active.json")
|
||||
if not path.exists():
|
||||
return []
|
||||
with path.open(encoding="utf-8") as f:
|
||||
return cast(list, json.load(f))
|
||||
return cast(List[int], json.load(f))
|
||||
|
||||
def _write_dqs_state(self, dqdir: str, state: list) -> None:
|
||||
def _write_dqs_state(self, dqdir: str, state: List[int]) -> None:
|
||||
with Path(dqdir, "active.json").open("w", encoding="utf-8") as f:
|
||||
json.dump(state, f)
|
||||
|
|
|
|||
|
|
@ -16,6 +16,7 @@ from typing import (
|
|||
Set,
|
||||
Tuple,
|
||||
Type,
|
||||
TypeVar,
|
||||
Union,
|
||||
cast,
|
||||
)
|
||||
|
|
@ -47,6 +48,7 @@ if TYPE_CHECKING:
|
|||
from scrapy.crawler import Crawler
|
||||
|
||||
|
||||
_T = TypeVar("_T")
|
||||
QueueTuple = Tuple[Union[Response, Failure], Request, Deferred]
|
||||
|
||||
|
||||
|
|
@ -256,14 +258,14 @@ class Scraper:
|
|||
|
||||
def handle_spider_output(
|
||||
self,
|
||||
result: Union[Iterable, AsyncIterable],
|
||||
result: Union[Iterable[_T], AsyncIterable[_T]],
|
||||
request: Request,
|
||||
response: Response,
|
||||
spider: Spider,
|
||||
) -> Deferred:
|
||||
if not result:
|
||||
return defer_succeed(None)
|
||||
it: Union[Iterable, AsyncIterable]
|
||||
it: Union[Iterable[_T], AsyncIterable[_T]]
|
||||
if isinstance(result, AsyncIterable):
|
||||
it = aiter_errback(
|
||||
result, self.handle_spider_error, request, response, spider
|
||||
|
|
|
|||
|
|
@ -4,12 +4,13 @@ Spider Middleware manager
|
|||
See documentation in docs/topics/spider-middleware.rst
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from inspect import isasyncgenfunction, iscoroutine
|
||||
from itertools import islice
|
||||
from typing import (
|
||||
Any,
|
||||
AsyncGenerator,
|
||||
AsyncIterable,
|
||||
Callable,
|
||||
Generator,
|
||||
|
|
@ -17,6 +18,7 @@ from typing import (
|
|||
List,
|
||||
Optional,
|
||||
Tuple,
|
||||
TypeVar,
|
||||
Union,
|
||||
cast,
|
||||
)
|
||||
|
|
@ -42,6 +44,7 @@ from scrapy.utils.python import MutableAsyncChain, MutableChain
|
|||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
_T = TypeVar("_T")
|
||||
ScrapeFunc = Callable[[Union[Response, Failure], Request, Spider], Any]
|
||||
|
||||
|
||||
|
|
@ -98,31 +101,39 @@ class SpiderMiddlewareManager(MiddlewareManager):
|
|||
self,
|
||||
response: Response,
|
||||
spider: Spider,
|
||||
iterable: Union[Iterable, AsyncIterable],
|
||||
iterable: Union[Iterable[_T], AsyncIterable[_T]],
|
||||
exception_processor_index: int,
|
||||
recover_to: Union[MutableChain, MutableAsyncChain],
|
||||
) -> Union[Generator, AsyncGenerator]:
|
||||
def process_sync(iterable: Iterable) -> Generator:
|
||||
recover_to: Union[MutableChain[_T], MutableAsyncChain[_T]],
|
||||
) -> Union[Iterable[_T], AsyncIterable[_T]]:
|
||||
def process_sync(iterable: Iterable[_T]) -> Iterable[_T]:
|
||||
try:
|
||||
yield from iterable
|
||||
except Exception as ex:
|
||||
exception_result = self._process_spider_exception(
|
||||
response, spider, Failure(ex), exception_processor_index
|
||||
exception_result = cast(
|
||||
Union[Failure, MutableChain[_T]],
|
||||
self._process_spider_exception(
|
||||
response, spider, Failure(ex), exception_processor_index
|
||||
),
|
||||
)
|
||||
if isinstance(exception_result, Failure):
|
||||
raise
|
||||
assert isinstance(recover_to, MutableChain)
|
||||
recover_to.extend(exception_result)
|
||||
|
||||
async def process_async(iterable: AsyncIterable) -> AsyncGenerator:
|
||||
async def process_async(iterable: AsyncIterable[_T]) -> AsyncIterable[_T]:
|
||||
try:
|
||||
async for r in iterable:
|
||||
yield r
|
||||
except Exception as ex:
|
||||
exception_result = self._process_spider_exception(
|
||||
response, spider, Failure(ex), exception_processor_index
|
||||
exception_result = cast(
|
||||
Union[Failure, MutableAsyncChain[_T]],
|
||||
self._process_spider_exception(
|
||||
response, spider, Failure(ex), exception_processor_index
|
||||
),
|
||||
)
|
||||
if isinstance(exception_result, Failure):
|
||||
raise
|
||||
assert isinstance(recover_to, MutableAsyncChain)
|
||||
recover_to.extend(exception_result)
|
||||
|
||||
if isinstance(iterable, AsyncIterable):
|
||||
|
|
@ -135,7 +146,7 @@ class SpiderMiddlewareManager(MiddlewareManager):
|
|||
spider: Spider,
|
||||
_failure: Failure,
|
||||
start_index: int = 0,
|
||||
) -> Union[Failure, MutableChain]:
|
||||
) -> Union[Failure, MutableChain[_T], MutableAsyncChain[_T]]:
|
||||
exception = _failure.value
|
||||
# don't handle _InvalidOutput exception
|
||||
if isinstance(exception, _InvalidOutput):
|
||||
|
|
@ -151,14 +162,18 @@ class SpiderMiddlewareManager(MiddlewareManager):
|
|||
if _isiterable(result):
|
||||
# stop exception handling by handing control over to the
|
||||
# process_spider_output chain if an iterable has been returned
|
||||
dfd: Deferred = self._process_spider_output(
|
||||
response, spider, result, method_index + 1
|
||||
dfd: Deferred[Union[MutableChain[_T], MutableAsyncChain[_T]]] = (
|
||||
self._process_spider_output(
|
||||
response, spider, result, method_index + 1
|
||||
)
|
||||
)
|
||||
# _process_spider_output() returns a Deferred only because of downgrading so this can be
|
||||
# simplified when downgrading is removed.
|
||||
if dfd.called:
|
||||
# the result is available immediately if _process_spider_output didn't do downgrading
|
||||
return cast(MutableChain, dfd.result)
|
||||
return cast(
|
||||
Union[MutableChain[_T], MutableAsyncChain[_T]], dfd.result
|
||||
)
|
||||
# we forbid waiting here because otherwise we would need to return a deferred from
|
||||
# _process_spider_exception too, which complicates the architecture
|
||||
msg = f"Async iterable returned from {method.__qualname__} cannot be downgraded"
|
||||
|
|
@ -181,12 +196,12 @@ class SpiderMiddlewareManager(MiddlewareManager):
|
|||
self,
|
||||
response: Response,
|
||||
spider: Spider,
|
||||
result: Union[Iterable, AsyncIterable],
|
||||
result: Union[Iterable[_T], AsyncIterable[_T]],
|
||||
start_index: int = 0,
|
||||
) -> Generator[Deferred, Any, Union[MutableChain, MutableAsyncChain]]:
|
||||
) -> Generator[Deferred[Any], Any, Union[MutableChain[_T], MutableAsyncChain[_T]]]:
|
||||
# items in this iterable do not need to go through the process_spider_output
|
||||
# chain, they went through it already from the process_spider_exception method
|
||||
recovered: Union[MutableChain, MutableAsyncChain]
|
||||
recovered: Union[MutableChain[_T], MutableAsyncChain[_T]]
|
||||
last_result_is_async = isinstance(result, AsyncIterable)
|
||||
if last_result_is_async:
|
||||
recovered = MutableAsyncChain()
|
||||
|
|
@ -237,7 +252,9 @@ class SpiderMiddlewareManager(MiddlewareManager):
|
|||
# might fail directly if the output value is not a generator
|
||||
result = method(response=response, result=result, spider=spider)
|
||||
except Exception as ex:
|
||||
exception_result = self._process_spider_exception(
|
||||
exception_result: Union[
|
||||
Failure, MutableChain[_T], MutableAsyncChain[_T]
|
||||
] = self._process_spider_exception(
|
||||
response, spider, Failure(ex), method_index + 1
|
||||
)
|
||||
if isinstance(exception_result, Failure):
|
||||
|
|
@ -267,9 +284,12 @@ class SpiderMiddlewareManager(MiddlewareManager):
|
|||
return MutableChain(result, recovered) # type: ignore[arg-type]
|
||||
|
||||
async def _process_callback_output(
|
||||
self, response: Response, spider: Spider, result: Union[Iterable, AsyncIterable]
|
||||
) -> Union[MutableChain, MutableAsyncChain]:
|
||||
recovered: Union[MutableChain, MutableAsyncChain]
|
||||
self,
|
||||
response: Response,
|
||||
spider: Spider,
|
||||
result: Union[Iterable[_T], AsyncIterable[_T]],
|
||||
) -> Union[MutableChain[_T], MutableAsyncChain[_T]]:
|
||||
recovered: Union[MutableChain[_T], MutableAsyncChain[_T]]
|
||||
if isinstance(result, AsyncIterable):
|
||||
recovered = MutableAsyncChain()
|
||||
else:
|
||||
|
|
@ -293,14 +313,16 @@ class SpiderMiddlewareManager(MiddlewareManager):
|
|||
spider: Spider,
|
||||
) -> Deferred:
|
||||
async def process_callback_output(
|
||||
result: Union[Iterable, AsyncIterable]
|
||||
) -> Union[MutableChain, MutableAsyncChain]:
|
||||
result: Union[Iterable[_T], AsyncIterable[_T]]
|
||||
) -> Union[MutableChain[_T], MutableAsyncChain[_T]]:
|
||||
return await self._process_callback_output(response, spider, result)
|
||||
|
||||
def process_spider_exception(_failure: Failure) -> Union[Failure, MutableChain]:
|
||||
def process_spider_exception(
|
||||
_failure: Failure,
|
||||
) -> Union[Failure, MutableChain[_T], MutableAsyncChain[_T]]:
|
||||
return self._process_spider_exception(response, spider, _failure)
|
||||
|
||||
dfd = mustbe_deferred(
|
||||
dfd: Deferred = mustbe_deferred(
|
||||
self._process_spider_input, scrape_func, response, request, spider
|
||||
)
|
||||
dfd.addCallback(deferred_f_from_coro_f(process_callback_output))
|
||||
|
|
|
|||
|
|
@ -3,16 +3,7 @@ from __future__ import annotations
|
|||
import logging
|
||||
from collections import defaultdict
|
||||
from http.cookiejar import Cookie
|
||||
from typing import (
|
||||
TYPE_CHECKING,
|
||||
Any,
|
||||
DefaultDict,
|
||||
Dict,
|
||||
Iterable,
|
||||
Optional,
|
||||
Sequence,
|
||||
Union,
|
||||
)
|
||||
from typing import TYPE_CHECKING, Any, DefaultDict, Iterable, Optional, Sequence, Union
|
||||
|
||||
from tldextract import TLDExtract
|
||||
|
||||
|
|
@ -21,6 +12,7 @@ from scrapy.crawler import Crawler
|
|||
from scrapy.exceptions import NotConfigured
|
||||
from scrapy.http import Response
|
||||
from scrapy.http.cookies import CookieJar
|
||||
from scrapy.http.request import VerboseCookie
|
||||
from scrapy.utils.httpobj import urlparse_cached
|
||||
from scrapy.utils.python import to_unicode
|
||||
|
||||
|
|
@ -128,7 +120,7 @@ class CookiesMiddleware:
|
|||
msg = f"Received cookies from: {response}\n{cookies}"
|
||||
logger.debug(msg, extra={"spider": spider})
|
||||
|
||||
def _format_cookie(self, cookie: Dict[str, Any], request: Request) -> Optional[str]:
|
||||
def _format_cookie(self, cookie: VerboseCookie, request: Request) -> Optional[str]:
|
||||
"""
|
||||
Given a dict consisting of cookie components, return its string representation.
|
||||
Decode from bytes if necessary.
|
||||
|
|
@ -142,18 +134,19 @@ class CookiesMiddleware:
|
|||
logger.warning(msg)
|
||||
return None
|
||||
continue
|
||||
if isinstance(cookie[key], (bool, float, int, str)):
|
||||
decoded[key] = str(cookie[key])
|
||||
# https://github.com/python/mypy/issues/7178, https://github.com/python/mypy/issues/9168
|
||||
if isinstance(cookie[key], (bool, float, int, str)): # type: ignore[literal-required]
|
||||
decoded[key] = str(cookie[key]) # type: ignore[literal-required]
|
||||
else:
|
||||
try:
|
||||
decoded[key] = cookie[key].decode("utf8")
|
||||
decoded[key] = cookie[key].decode("utf8") # type: ignore[literal-required]
|
||||
except UnicodeDecodeError:
|
||||
logger.warning(
|
||||
"Non UTF-8 encoded cookie found in request %s: %s",
|
||||
request,
|
||||
cookie,
|
||||
)
|
||||
decoded[key] = cookie[key].decode("latin1", errors="replace")
|
||||
decoded[key] = cookie[key].decode("latin1", errors="replace") # type: ignore[literal-required]
|
||||
for flag in ("secure",):
|
||||
value = cookie.get(flag, _UNSET)
|
||||
if value is _UNSET or not value:
|
||||
|
|
@ -174,7 +167,7 @@ class CookiesMiddleware:
|
|||
"""
|
||||
if not request.cookies:
|
||||
return []
|
||||
cookies: Iterable[Dict[str, Any]]
|
||||
cookies: Iterable[VerboseCookie]
|
||||
if isinstance(request.cookies, dict):
|
||||
cookies = tuple({"name": k, "value": v} for k, v in request.cookies.items())
|
||||
else:
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING, Dict, Union
|
||||
from typing import TYPE_CHECKING, Dict, List, Tuple, Union
|
||||
|
||||
from twisted.web import http
|
||||
|
||||
|
|
@ -17,7 +17,9 @@ if TYPE_CHECKING:
|
|||
from typing_extensions import Self
|
||||
|
||||
|
||||
def get_header_size(headers: Dict[str, Union[list, tuple]]) -> int:
|
||||
def get_header_size(
|
||||
headers: Dict[str, Union[List[Union[str, bytes]], Tuple[Union[str, bytes], ...]]]
|
||||
) -> int:
|
||||
size = 0
|
||||
for key, value in headers.items():
|
||||
if isinstance(value, (list, tuple)):
|
||||
|
|
|
|||
|
|
@ -694,7 +694,9 @@ class FeedExporter:
|
|||
self.slots = slots
|
||||
|
||||
def _load_components(self, setting_prefix: str) -> Dict[str, Any]:
|
||||
conf = without_none_values(self.settings.getwithbase(setting_prefix))
|
||||
conf = without_none_values(
|
||||
cast(Dict[str, str], self.settings.getwithbase(setting_prefix))
|
||||
)
|
||||
d = {}
|
||||
for k, v in conf.items():
|
||||
try:
|
||||
|
|
|
|||
|
|
@ -315,7 +315,9 @@ class FilesystemCacheStorage:
|
|||
self.expiration_secs: int = settings.getint("HTTPCACHE_EXPIRATION_SECS")
|
||||
self.use_gzip: bool = settings.getbool("HTTPCACHE_GZIP")
|
||||
# https://github.com/python/mypy/issues/10740
|
||||
self._open: Callable[Concatenate[Union[str, os.PathLike], str, ...], IO] = (
|
||||
self._open: Callable[
|
||||
Concatenate[Union[str, os.PathLike], str, ...], IO[bytes]
|
||||
] = (
|
||||
gzip.open if self.use_gzip else open # type: ignore[assignment]
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -118,8 +118,7 @@ class Headers(CaselessDict):
|
|||
]
|
||||
|
||||
def to_string(self) -> bytes:
|
||||
# cast() can be removed if the headers_dict_to_raw() hint is improved
|
||||
return cast(bytes, headers_dict_to_raw(self))
|
||||
return headers_dict_to_raw(self)
|
||||
|
||||
def to_unicode_dict(self) -> CaseInsensitiveDict:
|
||||
"""Return headers as a CaseInsensitiveDict with str keys
|
||||
|
|
|
|||
|
|
@ -21,6 +21,7 @@ from typing import (
|
|||
Optional,
|
||||
Tuple,
|
||||
Type,
|
||||
TypedDict,
|
||||
TypeVar,
|
||||
Union,
|
||||
overload,
|
||||
|
|
@ -36,8 +37,19 @@ from scrapy.utils.trackref import object_ref
|
|||
from scrapy.utils.url import escape_ajax
|
||||
|
||||
if TYPE_CHECKING:
|
||||
# typing.Self requires Python 3.11
|
||||
from typing_extensions import Self
|
||||
# typing.NotRequired and typing.Self require Python 3.11
|
||||
from typing_extensions import NotRequired, Self
|
||||
|
||||
|
||||
class VerboseCookie(TypedDict):
|
||||
name: str
|
||||
value: str
|
||||
domain: NotRequired[str]
|
||||
path: NotRequired[str]
|
||||
secure: NotRequired[bool]
|
||||
|
||||
|
||||
CookiesT = Union[Dict[str, str], List[VerboseCookie]]
|
||||
|
||||
|
||||
RequestTypeVar = TypeVar("RequestTypeVar", bound="Request")
|
||||
|
|
@ -102,7 +114,7 @@ class Request(object_ref):
|
|||
method: str = "GET",
|
||||
headers: Union[Mapping[AnyStr, Any], Iterable[Tuple[AnyStr, Any]], None] = None,
|
||||
body: Optional[Union[bytes, str]] = None,
|
||||
cookies: Optional[Union[dict, List[dict]]] = None,
|
||||
cookies: Optional[CookiesT] = None,
|
||||
meta: Optional[Dict[str, Any]] = None,
|
||||
encoding: str = "utf-8",
|
||||
priority: int = 0,
|
||||
|
|
@ -128,7 +140,7 @@ class Request(object_ref):
|
|||
self.callback: Optional[Callable] = callback
|
||||
self.errback: Optional[Callable] = errback
|
||||
|
||||
self.cookies: Union[dict, List[dict]] = cookies or {}
|
||||
self.cookies: CookiesT = cookies or {}
|
||||
self.headers: Headers = Headers(headers or {}, encoding=encoding)
|
||||
self.dont_filter: bool = dont_filter
|
||||
|
||||
|
|
@ -270,7 +282,7 @@ class Request(object_ref):
|
|||
return d
|
||||
|
||||
|
||||
def _find_method(obj: Any, func: Callable) -> str:
|
||||
def _find_method(obj: Any, func: Callable[..., Any]) -> str:
|
||||
"""Helper function for Request.to_dict"""
|
||||
# Only instance methods contain ``__func__``
|
||||
if obj and hasattr(func, "__func__"):
|
||||
|
|
|
|||
|
|
@ -7,7 +7,17 @@ See documentation in docs/topics/request-response.rst
|
|||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING, Any, Iterable, List, Optional, Tuple, Union, cast
|
||||
from typing import (
|
||||
TYPE_CHECKING,
|
||||
Any,
|
||||
Dict,
|
||||
Iterable,
|
||||
List,
|
||||
Optional,
|
||||
Tuple,
|
||||
Union,
|
||||
cast,
|
||||
)
|
||||
from urllib.parse import urlencode, urljoin, urlsplit, urlunsplit
|
||||
|
||||
from lxml.html import FormElement # nosec
|
||||
|
|
@ -26,8 +36,9 @@ if TYPE_CHECKING:
|
|||
from typing_extensions import Self
|
||||
|
||||
|
||||
FormdataKVType = Tuple[str, Union[str, Iterable[str]]]
|
||||
FormdataType = Optional[Union[dict, List[FormdataKVType]]]
|
||||
FormdataVType = Union[str, Iterable[str]]
|
||||
FormdataKVType = Tuple[str, FormdataVType]
|
||||
FormdataType = Optional[Union[Dict[str, FormdataVType], List[FormdataKVType]]]
|
||||
|
||||
|
||||
class FormRequest(Request):
|
||||
|
|
@ -62,7 +73,7 @@ class FormRequest(Request):
|
|||
formid: Optional[str] = None,
|
||||
formnumber: int = 0,
|
||||
formdata: FormdataType = None,
|
||||
clickdata: Optional[dict] = None,
|
||||
clickdata: Optional[Dict[str, Union[str, int]]] = None,
|
||||
dont_click: bool = False,
|
||||
formxpath: Optional[str] = None,
|
||||
formcss: Optional[str] = None,
|
||||
|
|
@ -156,7 +167,7 @@ def _get_inputs(
|
|||
form: FormElement,
|
||||
formdata: FormdataType,
|
||||
dont_click: bool,
|
||||
clickdata: Optional[dict],
|
||||
clickdata: Optional[Dict[str, Union[str, int]]],
|
||||
) -> List[FormdataKVType]:
|
||||
"""Return a list of key-value pairs for the inputs found in the given form."""
|
||||
try:
|
||||
|
|
@ -186,10 +197,8 @@ def _get_inputs(
|
|||
if clickable and clickable[0] not in formdata and not clickable[0] is None:
|
||||
values.append(clickable)
|
||||
|
||||
if isinstance(formdata, dict):
|
||||
formdata = formdata.items() # type: ignore[assignment]
|
||||
|
||||
values.extend((k, v) for k, v in formdata if v is not None)
|
||||
formdata_items = formdata.items() if isinstance(formdata, dict) else formdata
|
||||
values.extend((k, v) for k, v in formdata_items if v is not None)
|
||||
return values
|
||||
|
||||
|
||||
|
|
@ -216,7 +225,7 @@ def _select_value(
|
|||
|
||||
|
||||
def _get_clickable(
|
||||
clickdata: Optional[dict], form: FormElement
|
||||
clickdata: Optional[Dict[str, Union[str, int]]], form: FormElement
|
||||
) -> Optional[Tuple[str, str]]:
|
||||
"""
|
||||
Returns the clickable element specified in clickdata,
|
||||
|
|
@ -243,6 +252,7 @@ def _get_clickable(
|
|||
# because that uniquely identifies the element
|
||||
nr = clickdata.get("nr", None)
|
||||
if nr is not None:
|
||||
assert isinstance(nr, int)
|
||||
try:
|
||||
el = list(form.inputs)[nr]
|
||||
except IndexError:
|
||||
|
|
|
|||
|
|
@ -10,7 +10,7 @@ from __future__ import annotations
|
|||
import copy
|
||||
import json
|
||||
import warnings
|
||||
from typing import TYPE_CHECKING, Any, Optional, Tuple, Type, overload
|
||||
from typing import TYPE_CHECKING, Any, Dict, Optional, Tuple, Type, overload
|
||||
|
||||
from scrapy.http.request import Request, RequestTypeVar
|
||||
|
||||
|
|
@ -23,15 +23,15 @@ class JsonRequest(Request):
|
|||
attributes: Tuple[str, ...] = Request.attributes + ("dumps_kwargs",)
|
||||
|
||||
def __init__(
|
||||
self, *args: Any, dumps_kwargs: Optional[dict] = None, **kwargs: Any
|
||||
self, *args: Any, dumps_kwargs: Optional[Dict[str, Any]] = None, **kwargs: Any
|
||||
) -> None:
|
||||
dumps_kwargs = copy.deepcopy(dumps_kwargs) if dumps_kwargs is not None else {}
|
||||
dumps_kwargs.setdefault("sort_keys", True)
|
||||
self._dumps_kwargs = dumps_kwargs
|
||||
self._dumps_kwargs: Dict[str, Any] = dumps_kwargs
|
||||
|
||||
body_passed = kwargs.get("body", None) is not None
|
||||
data = kwargs.pop("data", None)
|
||||
data_passed = data is not None
|
||||
data: Any = kwargs.pop("data", None)
|
||||
data_passed: bool = data is not None
|
||||
|
||||
if body_passed and data_passed:
|
||||
warnings.warn("Both body and data passed. data will be ignored")
|
||||
|
|
@ -47,7 +47,7 @@ class JsonRequest(Request):
|
|||
)
|
||||
|
||||
@property
|
||||
def dumps_kwargs(self) -> dict:
|
||||
def dumps_kwargs(self) -> Dict[str, Any]:
|
||||
return self._dumps_kwargs
|
||||
|
||||
@overload
|
||||
|
|
@ -62,8 +62,8 @@ class JsonRequest(Request):
|
|||
self, *args: Any, cls: Optional[Type[Request]] = None, **kwargs: Any
|
||||
) -> Request:
|
||||
body_passed = kwargs.get("body", None) is not None
|
||||
data = kwargs.pop("data", None)
|
||||
data_passed = data is not None
|
||||
data: Any = kwargs.pop("data", None)
|
||||
data_passed: bool = data is not None
|
||||
|
||||
if body_passed and data_passed:
|
||||
warnings.warn("Both body and data passed. data will be ignored")
|
||||
|
|
@ -72,6 +72,6 @@ class JsonRequest(Request):
|
|||
|
||||
return super().replace(*args, cls=cls, **kwargs)
|
||||
|
||||
def _dumps(self, data: dict) -> str:
|
||||
def _dumps(self, data: Any) -> str:
|
||||
"""Convert to JSON"""
|
||||
return json.dumps(data, **self._dumps_kwargs)
|
||||
|
|
|
|||
|
|
@ -14,7 +14,6 @@ from typing import (
|
|||
AnyStr,
|
||||
Callable,
|
||||
Dict,
|
||||
Generator,
|
||||
Iterable,
|
||||
List,
|
||||
Mapping,
|
||||
|
|
@ -31,7 +30,7 @@ from twisted.internet.ssl import Certificate
|
|||
|
||||
from scrapy.exceptions import NotSupported
|
||||
from scrapy.http.headers import Headers
|
||||
from scrapy.http.request import Request
|
||||
from scrapy.http.request import CookiesT, Request
|
||||
from scrapy.link import Link
|
||||
from scrapy.utils.trackref import object_ref
|
||||
|
||||
|
|
@ -200,7 +199,7 @@ class Response(object_ref):
|
|||
method: str = "GET",
|
||||
headers: Union[Mapping[AnyStr, Any], Iterable[Tuple[AnyStr, Any]], None] = None,
|
||||
body: Optional[Union[bytes, str]] = None,
|
||||
cookies: Optional[Union[dict, List[dict]]] = None,
|
||||
cookies: Optional[CookiesT] = None,
|
||||
meta: Optional[Dict[str, Any]] = None,
|
||||
encoding: Optional[str] = "utf-8",
|
||||
priority: int = 0,
|
||||
|
|
@ -253,7 +252,7 @@ class Response(object_ref):
|
|||
method: str = "GET",
|
||||
headers: Union[Mapping[AnyStr, Any], Iterable[Tuple[AnyStr, Any]], None] = None,
|
||||
body: Optional[Union[bytes, str]] = None,
|
||||
cookies: Optional[Union[dict, List[dict]]] = None,
|
||||
cookies: Optional[CookiesT] = None,
|
||||
meta: Optional[Dict[str, Any]] = None,
|
||||
encoding: Optional[str] = "utf-8",
|
||||
priority: int = 0,
|
||||
|
|
@ -261,7 +260,7 @@ class Response(object_ref):
|
|||
errback: Optional[Callable] = None,
|
||||
cb_kwargs: Optional[Dict[str, Any]] = None,
|
||||
flags: Optional[List[str]] = None,
|
||||
) -> Generator[Request, None, None]:
|
||||
) -> Iterable[Request]:
|
||||
"""
|
||||
.. versionadded:: 2.0
|
||||
|
||||
|
|
|
|||
|
|
@ -15,7 +15,6 @@ from typing import (
|
|||
AnyStr,
|
||||
Callable,
|
||||
Dict,
|
||||
Generator,
|
||||
Iterable,
|
||||
List,
|
||||
Mapping,
|
||||
|
|
@ -36,7 +35,7 @@ from w3lib.encoding import (
|
|||
)
|
||||
from w3lib.html import strip_html5_whitespace
|
||||
|
||||
from scrapy.http import Request
|
||||
from scrapy.http.request import CookiesT, Request
|
||||
from scrapy.http.response import Response
|
||||
from scrapy.link import Link
|
||||
from scrapy.utils.python import memoizemethod_noargs, to_unicode
|
||||
|
|
@ -183,7 +182,7 @@ class TextResponse(Response):
|
|||
method: str = "GET",
|
||||
headers: Union[Mapping[AnyStr, Any], Iterable[Tuple[AnyStr, Any]], None] = None,
|
||||
body: Optional[Union[bytes, str]] = None,
|
||||
cookies: Optional[Union[dict, List[dict]]] = None,
|
||||
cookies: Optional[CookiesT] = None,
|
||||
meta: Optional[Dict[str, Any]] = None,
|
||||
encoding: Optional[str] = None,
|
||||
priority: int = 0,
|
||||
|
|
@ -236,7 +235,7 @@ class TextResponse(Response):
|
|||
method: str = "GET",
|
||||
headers: Union[Mapping[AnyStr, Any], Iterable[Tuple[AnyStr, Any]], None] = None,
|
||||
body: Optional[Union[bytes, str]] = None,
|
||||
cookies: Optional[Union[dict, List[dict]]] = None,
|
||||
cookies: Optional[CookiesT] = None,
|
||||
meta: Optional[Dict[str, Any]] = None,
|
||||
encoding: Optional[str] = None,
|
||||
priority: int = 0,
|
||||
|
|
@ -246,7 +245,7 @@ class TextResponse(Response):
|
|||
flags: Optional[List[str]] = None,
|
||||
css: Optional[str] = None,
|
||||
xpath: Optional[str] = None,
|
||||
) -> Generator[Request, None, None]:
|
||||
) -> Iterable[Request]:
|
||||
"""
|
||||
A generator that produces :class:`~.Request` instances to follow all
|
||||
links in ``urls``. It accepts the same arguments as the :class:`~.Request`'s
|
||||
|
|
|
|||
|
|
@ -27,7 +27,7 @@ if TYPE_CHECKING:
|
|||
from typing_extensions import Self
|
||||
|
||||
|
||||
class Field(dict):
|
||||
class Field(Dict[str, Any]):
|
||||
"""Container of field metadata"""
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@ from __future__ import annotations
|
|||
|
||||
import logging
|
||||
import os
|
||||
from typing import TYPE_CHECKING, Any, Dict, Optional, Union
|
||||
from typing import TYPE_CHECKING, Any, Dict, Optional, Tuple, TypedDict, Union
|
||||
|
||||
from twisted.python.failure import Failure
|
||||
|
||||
|
|
@ -26,6 +26,12 @@ DOWNLOADERRORMSG_SHORT = "Error downloading %(request)s"
|
|||
DOWNLOADERRORMSG_LONG = "Error downloading %(request)s: %(errmsg)s"
|
||||
|
||||
|
||||
class LogFormatterResult(TypedDict):
|
||||
level: int
|
||||
msg: str
|
||||
args: Union[Dict[str, Any], Tuple[Any, ...]]
|
||||
|
||||
|
||||
class LogFormatter:
|
||||
"""Class for generating log messages for different actions.
|
||||
|
||||
|
|
@ -64,7 +70,9 @@ class LogFormatter:
|
|||
}
|
||||
"""
|
||||
|
||||
def crawled(self, request: Request, response: Response, spider: Spider) -> dict:
|
||||
def crawled(
|
||||
self, request: Request, response: Response, spider: Spider
|
||||
) -> LogFormatterResult:
|
||||
"""Logs a message when the crawler finds a webpage."""
|
||||
request_flags = f" {str(request.flags)}" if request.flags else ""
|
||||
response_flags = f" {str(response.flags)}" if response.flags else ""
|
||||
|
|
@ -84,7 +92,7 @@ class LogFormatter:
|
|||
|
||||
def scraped(
|
||||
self, item: Any, response: Union[Response, Failure], spider: Spider
|
||||
) -> dict:
|
||||
) -> LogFormatterResult:
|
||||
"""Logs a message when an item is scraped by a spider."""
|
||||
src: Any
|
||||
if isinstance(response, Failure):
|
||||
|
|
@ -102,7 +110,7 @@ class LogFormatter:
|
|||
|
||||
def dropped(
|
||||
self, item: Any, exception: BaseException, response: Response, spider: Spider
|
||||
) -> dict:
|
||||
) -> LogFormatterResult:
|
||||
"""Logs a message when an item is dropped while it is passing through the item pipeline."""
|
||||
return {
|
||||
"level": logging.WARNING,
|
||||
|
|
@ -115,7 +123,7 @@ class LogFormatter:
|
|||
|
||||
def item_error(
|
||||
self, item: Any, exception: BaseException, response: Response, spider: Spider
|
||||
) -> dict:
|
||||
) -> LogFormatterResult:
|
||||
"""Logs a message when an item causes an error while it is passing
|
||||
through the item pipeline.
|
||||
|
||||
|
|
@ -135,7 +143,7 @@ class LogFormatter:
|
|||
request: Request,
|
||||
response: Union[Response, Failure],
|
||||
spider: Spider,
|
||||
) -> dict:
|
||||
) -> LogFormatterResult:
|
||||
"""Logs an error message from a spider.
|
||||
|
||||
.. versionadded:: 2.0
|
||||
|
|
@ -155,7 +163,7 @@ class LogFormatter:
|
|||
request: Request,
|
||||
spider: Spider,
|
||||
errmsg: Optional[str] = None,
|
||||
) -> dict:
|
||||
) -> LogFormatterResult:
|
||||
"""Logs a download error message from a spider (typically coming from
|
||||
the engine).
|
||||
|
||||
|
|
|
|||
|
|
@ -97,7 +97,7 @@ class MailSender:
|
|||
subject: str,
|
||||
body: str,
|
||||
cc: Union[str, List[str], None] = None,
|
||||
attachs: Sequence[Tuple[str, str, IO]] = (),
|
||||
attachs: Sequence[Tuple[str, str, IO[Any]]] = (),
|
||||
mimetype: str = "text/plain",
|
||||
charset: Optional[str] = None,
|
||||
_callback: Optional[Callable[..., None]] = None,
|
||||
|
|
@ -214,7 +214,7 @@ class MailSender:
|
|||
return d
|
||||
|
||||
def _create_sender_factory(
|
||||
self, to_addrs: List[str], msg: IO, d: Deferred
|
||||
self, to_addrs: List[str], msg: IO[bytes], d: Deferred
|
||||
) -> ESMTPSenderFactory:
|
||||
from twisted.mail.smtp import ESMTPSenderFactory
|
||||
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@ from twisted.internet.defer import Deferred
|
|||
|
||||
from scrapy import Spider
|
||||
from scrapy.middleware import MiddlewareManager
|
||||
from scrapy.settings import Settings
|
||||
from scrapy.utils.conf import build_component_list
|
||||
from scrapy.utils.defer import deferred_f_from_coro_f
|
||||
|
||||
|
|
@ -18,7 +19,7 @@ class ItemPipelineManager(MiddlewareManager):
|
|||
component_name = "item pipeline"
|
||||
|
||||
@classmethod
|
||||
def _get_mwlist_from_settings(cls, settings) -> List[Any]:
|
||||
def _get_mwlist_from_settings(cls, settings: Settings) -> List[Any]:
|
||||
return build_component_list(settings.getwithbase("ITEM_PIPELINES"))
|
||||
|
||||
def _add_middleware(self, pipe: Any) -> None:
|
||||
|
|
|
|||
|
|
@ -18,16 +18,35 @@ from ftplib import FTP
|
|||
from io import BytesIO
|
||||
from os import PathLike
|
||||
from pathlib import Path
|
||||
from typing import IO, TYPE_CHECKING, DefaultDict, Optional, Set, Type, Union, cast
|
||||
from typing import (
|
||||
IO,
|
||||
TYPE_CHECKING,
|
||||
Any,
|
||||
Callable,
|
||||
DefaultDict,
|
||||
Dict,
|
||||
List,
|
||||
NoReturn,
|
||||
Optional,
|
||||
Protocol,
|
||||
Set,
|
||||
Type,
|
||||
TypedDict,
|
||||
Union,
|
||||
cast,
|
||||
)
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from itemadapter import ItemAdapter
|
||||
from twisted.internet import defer, threads
|
||||
from twisted.internet.defer import Deferred
|
||||
from twisted.python.failure import Failure
|
||||
|
||||
from scrapy import Spider
|
||||
from scrapy.exceptions import IgnoreRequest, NotConfigured
|
||||
from scrapy.http import Request
|
||||
from scrapy.http import Request, Response
|
||||
from scrapy.http.request import NO_CALLBACK
|
||||
from scrapy.pipelines.media import MediaPipeline
|
||||
from scrapy.pipelines.media import FileInfo, FileInfoOrError, MediaPipeline
|
||||
from scrapy.settings import Settings
|
||||
from scrapy.utils.boto import is_botocore_available
|
||||
from scrapy.utils.datatypes import CaseInsensitiveDict
|
||||
|
|
@ -40,14 +59,15 @@ if TYPE_CHECKING:
|
|||
# typing.Self requires Python 3.11
|
||||
from typing_extensions import Self
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _to_string(path: Union[str, PathLike]) -> str:
|
||||
def _to_string(path: Union[str, PathLike[str]]) -> str:
|
||||
return str(path) # convert a Path object to string
|
||||
|
||||
|
||||
def _md5sum(file: IO) -> str:
|
||||
def _md5sum(file: IO[bytes]) -> str:
|
||||
"""Calculate the md5 checksum of a file-like object without reading its
|
||||
whole content in memory.
|
||||
|
||||
|
|
@ -68,23 +88,54 @@ class FileException(Exception):
|
|||
"""General media error exception"""
|
||||
|
||||
|
||||
class StatInfo(TypedDict, total=False):
|
||||
checksum: str
|
||||
last_modified: float
|
||||
|
||||
|
||||
class FilesStoreProtocol(Protocol):
|
||||
def __init__(self, basedir: str): ...
|
||||
|
||||
def persist_file(
|
||||
self,
|
||||
path: str,
|
||||
buf: BytesIO,
|
||||
info: MediaPipeline.SpiderInfo,
|
||||
meta: Optional[Dict[str, Any]] = None,
|
||||
headers: Optional[Dict[str, str]] = None,
|
||||
) -> Optional[Deferred[Any]]: ...
|
||||
|
||||
def stat_file(
|
||||
self, path: str, info: MediaPipeline.SpiderInfo
|
||||
) -> Union[StatInfo, Deferred[StatInfo]]: ...
|
||||
|
||||
|
||||
class FSFilesStore:
|
||||
def __init__(self, basedir: Union[str, PathLike]):
|
||||
def __init__(self, basedir: Union[str, PathLike[str]]):
|
||||
basedir = _to_string(basedir)
|
||||
if "://" in basedir:
|
||||
basedir = basedir.split("://", 1)[1]
|
||||
self.basedir = basedir
|
||||
self.basedir: str = basedir
|
||||
self._mkdir(Path(self.basedir))
|
||||
self.created_directories: DefaultDict[str, Set[str]] = defaultdict(set)
|
||||
self.created_directories: DefaultDict[MediaPipeline.SpiderInfo, Set[str]] = (
|
||||
defaultdict(set)
|
||||
)
|
||||
|
||||
def persist_file(
|
||||
self, path: Union[str, PathLike], buf, info, meta=None, headers=None
|
||||
):
|
||||
self,
|
||||
path: Union[str, PathLike[str]],
|
||||
buf: BytesIO,
|
||||
info: MediaPipeline.SpiderInfo,
|
||||
meta: Optional[Dict[str, Any]] = None,
|
||||
headers: Optional[Dict[str, str]] = None,
|
||||
) -> None:
|
||||
absolute_path = self._get_filesystem_path(path)
|
||||
self._mkdir(absolute_path.parent, info)
|
||||
absolute_path.write_bytes(buf.getvalue())
|
||||
|
||||
def stat_file(self, path: Union[str, PathLike], info):
|
||||
def stat_file(
|
||||
self, path: Union[str, PathLike[str]], info: MediaPipeline.SpiderInfo
|
||||
) -> StatInfo:
|
||||
absolute_path = self._get_filesystem_path(path)
|
||||
try:
|
||||
last_modified = absolute_path.stat().st_mtime
|
||||
|
|
@ -96,12 +147,14 @@ class FSFilesStore:
|
|||
|
||||
return {"last_modified": last_modified, "checksum": checksum}
|
||||
|
||||
def _get_filesystem_path(self, path: Union[str, PathLike]) -> Path:
|
||||
def _get_filesystem_path(self, path: Union[str, PathLike[str]]) -> Path:
|
||||
path_comps = _to_string(path).split("/")
|
||||
return Path(self.basedir, *path_comps)
|
||||
|
||||
def _mkdir(self, dirname: Path, domain: Optional[str] = None):
|
||||
seen = self.created_directories[domain] if domain else set()
|
||||
def _mkdir(
|
||||
self, dirname: Path, domain: Optional[MediaPipeline.SpiderInfo] = None
|
||||
) -> None:
|
||||
seen: Set[str] = self.created_directories[domain] if domain else set()
|
||||
if str(dirname) not in seen:
|
||||
if not dirname.exists():
|
||||
dirname.mkdir(parents=True)
|
||||
|
|
@ -122,7 +175,7 @@ class S3FilesStore:
|
|||
"Cache-Control": "max-age=172800",
|
||||
}
|
||||
|
||||
def __init__(self, uri):
|
||||
def __init__(self, uri: str):
|
||||
if not is_botocore_available():
|
||||
raise NotConfigured("missing botocore library")
|
||||
import botocore.session
|
||||
|
|
@ -142,8 +195,10 @@ class S3FilesStore:
|
|||
raise ValueError(f"Incorrect URI scheme in {uri}, expected 's3'")
|
||||
self.bucket, self.prefix = uri[5:].split("/", 1)
|
||||
|
||||
def stat_file(self, path, info):
|
||||
def _onsuccess(boto_key):
|
||||
def stat_file(
|
||||
self, path: str, info: MediaPipeline.SpiderInfo
|
||||
) -> Deferred[StatInfo]:
|
||||
def _onsuccess(boto_key: Dict[str, Any]) -> StatInfo:
|
||||
checksum = boto_key["ETag"].strip('"')
|
||||
last_modified = boto_key["LastModified"]
|
||||
modified_stamp = time.mktime(last_modified.timetuple())
|
||||
|
|
@ -151,13 +206,23 @@ class S3FilesStore:
|
|||
|
||||
return self._get_boto_key(path).addCallback(_onsuccess)
|
||||
|
||||
def _get_boto_key(self, path):
|
||||
def _get_boto_key(self, path: str) -> Deferred[Dict[str, Any]]:
|
||||
key_name = f"{self.prefix}{path}"
|
||||
return threads.deferToThread(
|
||||
self.s3_client.head_object, Bucket=self.bucket, Key=key_name
|
||||
return cast(
|
||||
"Deferred[Dict[str, Any]]",
|
||||
threads.deferToThread(
|
||||
self.s3_client.head_object, Bucket=self.bucket, Key=key_name # type: ignore[attr-defined]
|
||||
),
|
||||
)
|
||||
|
||||
def persist_file(self, path, buf, info, meta=None, headers=None):
|
||||
def persist_file(
|
||||
self,
|
||||
path: str,
|
||||
buf: BytesIO,
|
||||
info: MediaPipeline.SpiderInfo,
|
||||
meta: Optional[Dict[str, Any]] = None,
|
||||
headers: Optional[Dict[str, str]] = None,
|
||||
) -> Deferred[Any]:
|
||||
"""Upload file to S3 storage"""
|
||||
key_name = f"{self.prefix}{path}"
|
||||
buf.seek(0)
|
||||
|
|
@ -165,7 +230,7 @@ class S3FilesStore:
|
|||
if headers:
|
||||
extra.update(self._headers_to_botocore_kwargs(headers))
|
||||
return threads.deferToThread(
|
||||
self.s3_client.put_object,
|
||||
self.s3_client.put_object, # type: ignore[attr-defined]
|
||||
Bucket=self.bucket,
|
||||
Key=key_name,
|
||||
Body=buf,
|
||||
|
|
@ -174,7 +239,7 @@ class S3FilesStore:
|
|||
**extra,
|
||||
)
|
||||
|
||||
def _headers_to_botocore_kwargs(self, headers):
|
||||
def _headers_to_botocore_kwargs(self, headers: Dict[str, Any]) -> Dict[str, Any]:
|
||||
"""Convert headers to botocore keyword arguments."""
|
||||
# This is required while we need to support both boto and botocore.
|
||||
mapping = CaseInsensitiveDict(
|
||||
|
|
@ -206,7 +271,7 @@ class S3FilesStore:
|
|||
"X-Amz-Website-Redirect-Location": "WebsiteRedirectLocation",
|
||||
}
|
||||
)
|
||||
extra = {}
|
||||
extra: Dict[str, Any] = {}
|
||||
for key, value in headers.items():
|
||||
try:
|
||||
kwarg = mapping[key]
|
||||
|
|
@ -226,13 +291,13 @@ class GCSFilesStore:
|
|||
# Overridden from settings.FILES_STORE_GCS_ACL in FilesPipeline.from_settings.
|
||||
POLICY = None
|
||||
|
||||
def __init__(self, uri):
|
||||
def __init__(self, uri: str):
|
||||
from google.cloud import storage
|
||||
|
||||
client = storage.Client(project=self.GCS_PROJECT_ID)
|
||||
bucket, prefix = uri[5:].split("/", 1)
|
||||
self.bucket = client.bucket(bucket)
|
||||
self.prefix = prefix
|
||||
self.prefix: str = prefix
|
||||
permissions = self.bucket.test_iam_permissions(
|
||||
["storage.objects.get", "storage.objects.create"]
|
||||
)
|
||||
|
|
@ -248,8 +313,10 @@ class GCSFilesStore:
|
|||
{"bucket": bucket},
|
||||
)
|
||||
|
||||
def stat_file(self, path, info):
|
||||
def _onsuccess(blob):
|
||||
def stat_file(
|
||||
self, path: str, info: MediaPipeline.SpiderInfo
|
||||
) -> Deferred[StatInfo]:
|
||||
def _onsuccess(blob) -> StatInfo:
|
||||
if blob:
|
||||
checksum = base64.b64decode(blob.md5_hash).hex()
|
||||
last_modified = time.mktime(blob.updated.timetuple())
|
||||
|
|
@ -257,19 +324,29 @@ class GCSFilesStore:
|
|||
return {}
|
||||
|
||||
blob_path = self._get_blob_path(path)
|
||||
return threads.deferToThread(self.bucket.get_blob, blob_path).addCallback(
|
||||
_onsuccess
|
||||
return cast(
|
||||
Deferred[StatInfo],
|
||||
threads.deferToThread(self.bucket.get_blob, blob_path).addCallback(
|
||||
_onsuccess
|
||||
),
|
||||
)
|
||||
|
||||
def _get_content_type(self, headers):
|
||||
def _get_content_type(self, headers: Optional[Dict[str, str]]) -> str:
|
||||
if headers and "Content-Type" in headers:
|
||||
return headers["Content-Type"]
|
||||
return "application/octet-stream"
|
||||
|
||||
def _get_blob_path(self, path):
|
||||
def _get_blob_path(self, path: str) -> str:
|
||||
return self.prefix + path
|
||||
|
||||
def persist_file(self, path, buf, info, meta=None, headers=None):
|
||||
def persist_file(
|
||||
self,
|
||||
path: str,
|
||||
buf: BytesIO,
|
||||
info: MediaPipeline.SpiderInfo,
|
||||
meta: Optional[Dict[str, Any]] = None,
|
||||
headers: Optional[Dict[str, str]] = None,
|
||||
) -> Deferred[Any]:
|
||||
blob_path = self._get_blob_path(path)
|
||||
blob = self.bucket.blob(blob_path)
|
||||
blob.cache_control = self.CACHE_CONTROL
|
||||
|
|
@ -283,22 +360,33 @@ class GCSFilesStore:
|
|||
|
||||
|
||||
class FTPFilesStore:
|
||||
FTP_USERNAME = None
|
||||
FTP_PASSWORD = None
|
||||
USE_ACTIVE_MODE = None
|
||||
FTP_USERNAME: Optional[str] = None
|
||||
FTP_PASSWORD: Optional[str] = None
|
||||
USE_ACTIVE_MODE: Optional[bool] = None
|
||||
|
||||
def __init__(self, uri):
|
||||
def __init__(self, uri: str):
|
||||
if not uri.startswith("ftp://"):
|
||||
raise ValueError(f"Incorrect URI scheme in {uri}, expected 'ftp'")
|
||||
u = urlparse(uri)
|
||||
self.port = u.port
|
||||
self.host = u.hostname
|
||||
assert u.port
|
||||
assert u.hostname
|
||||
self.port: int = u.port
|
||||
self.host: str = u.hostname
|
||||
self.port = int(u.port or 21)
|
||||
self.username = u.username or self.FTP_USERNAME
|
||||
self.password = u.password or self.FTP_PASSWORD
|
||||
self.basedir = u.path.rstrip("/")
|
||||
assert self.FTP_USERNAME
|
||||
assert self.FTP_PASSWORD
|
||||
self.username: str = u.username or self.FTP_USERNAME
|
||||
self.password: str = u.password or self.FTP_PASSWORD
|
||||
self.basedir: str = u.path.rstrip("/")
|
||||
|
||||
def persist_file(self, path, buf, info, meta=None, headers=None):
|
||||
def persist_file(
|
||||
self,
|
||||
path: str,
|
||||
buf: BytesIO,
|
||||
info: MediaPipeline.SpiderInfo,
|
||||
meta: Optional[Dict[str, Any]] = None,
|
||||
headers: Optional[Dict[str, str]] = None,
|
||||
) -> Deferred[Any]:
|
||||
path = f"{self.basedir}/{path}"
|
||||
return threads.deferToThread(
|
||||
ftp_store_file,
|
||||
|
|
@ -311,8 +399,10 @@ class FTPFilesStore:
|
|||
use_active_mode=self.USE_ACTIVE_MODE,
|
||||
)
|
||||
|
||||
def stat_file(self, path, info):
|
||||
def _stat_file(path):
|
||||
def stat_file(
|
||||
self, path: str, info: MediaPipeline.SpiderInfo
|
||||
) -> Deferred[StatInfo]:
|
||||
def _stat_file(path: str) -> StatInfo:
|
||||
try:
|
||||
ftp = FTP()
|
||||
ftp.connect(self.host, self.port)
|
||||
|
|
@ -328,7 +418,7 @@ class FTPFilesStore:
|
|||
except Exception:
|
||||
return {}
|
||||
|
||||
return threads.deferToThread(_stat_file, path)
|
||||
return cast("Deferred[StatInfo]", threads.deferToThread(_stat_file, path))
|
||||
|
||||
|
||||
class FilesPipeline(MediaPipeline):
|
||||
|
|
@ -350,20 +440,23 @@ class FilesPipeline(MediaPipeline):
|
|||
|
||||
"""
|
||||
|
||||
MEDIA_NAME = "file"
|
||||
EXPIRES = 90
|
||||
STORE_SCHEMES = {
|
||||
MEDIA_NAME: str = "file"
|
||||
EXPIRES: int = 90
|
||||
STORE_SCHEMES: Dict[str, Type[FilesStoreProtocol]] = {
|
||||
"": FSFilesStore,
|
||||
"file": FSFilesStore,
|
||||
"s3": S3FilesStore,
|
||||
"gs": GCSFilesStore,
|
||||
"ftp": FTPFilesStore,
|
||||
}
|
||||
DEFAULT_FILES_URLS_FIELD = "file_urls"
|
||||
DEFAULT_FILES_RESULT_FIELD = "files"
|
||||
DEFAULT_FILES_URLS_FIELD: str = "file_urls"
|
||||
DEFAULT_FILES_RESULT_FIELD: str = "files"
|
||||
|
||||
def __init__(
|
||||
self, store_uri: Union[str, PathLike], download_func=None, settings=None
|
||||
self,
|
||||
store_uri: Union[str, PathLike[str]],
|
||||
download_func: Optional[Callable[[Request, Spider], Response]] = None,
|
||||
settings: Union[Settings, Dict[str, Any], None] = None,
|
||||
):
|
||||
store_uri = _to_string(store_uri)
|
||||
if not store_uri:
|
||||
|
|
@ -372,26 +465,26 @@ class FilesPipeline(MediaPipeline):
|
|||
if isinstance(settings, dict) or settings is None:
|
||||
settings = Settings(settings)
|
||||
cls_name = "FilesPipeline"
|
||||
self.store = self._get_store(store_uri)
|
||||
self.store: FilesStoreProtocol = self._get_store(store_uri)
|
||||
resolve = functools.partial(
|
||||
self._key_for_pipe, base_class_name=cls_name, settings=settings
|
||||
)
|
||||
self.expires = settings.getint(resolve("FILES_EXPIRES"), self.EXPIRES)
|
||||
self.expires: int = settings.getint(resolve("FILES_EXPIRES"), self.EXPIRES)
|
||||
if not hasattr(self, "FILES_URLS_FIELD"):
|
||||
self.FILES_URLS_FIELD = self.DEFAULT_FILES_URLS_FIELD
|
||||
if not hasattr(self, "FILES_RESULT_FIELD"):
|
||||
self.FILES_RESULT_FIELD = self.DEFAULT_FILES_RESULT_FIELD
|
||||
self.files_urls_field = settings.get(
|
||||
self.files_urls_field: str = settings.get(
|
||||
resolve("FILES_URLS_FIELD"), self.FILES_URLS_FIELD
|
||||
)
|
||||
self.files_result_field = settings.get(
|
||||
self.files_result_field: str = settings.get(
|
||||
resolve("FILES_RESULT_FIELD"), self.FILES_RESULT_FIELD
|
||||
)
|
||||
|
||||
super().__init__(download_func=download_func, settings=settings)
|
||||
|
||||
@classmethod
|
||||
def from_settings(cls, settings) -> Self:
|
||||
def from_settings(cls, settings: Settings) -> Self:
|
||||
s3store: Type[S3FilesStore] = cast(Type[S3FilesStore], cls.STORE_SCHEMES["s3"])
|
||||
s3store.AWS_ACCESS_KEY_ID = settings["AWS_ACCESS_KEY_ID"]
|
||||
s3store.AWS_SECRET_ACCESS_KEY = settings["AWS_SECRET_ACCESS_KEY"]
|
||||
|
|
@ -418,7 +511,7 @@ class FilesPipeline(MediaPipeline):
|
|||
store_uri = settings["FILES_STORE"]
|
||||
return cls(store_uri, settings=settings)
|
||||
|
||||
def _get_store(self, uri: str):
|
||||
def _get_store(self, uri: str) -> FilesStoreProtocol:
|
||||
if Path(uri).is_absolute(): # to support win32 paths like: C:\\some\dir
|
||||
scheme = "file"
|
||||
else:
|
||||
|
|
@ -426,19 +519,21 @@ class FilesPipeline(MediaPipeline):
|
|||
store_cls = self.STORE_SCHEMES[scheme]
|
||||
return store_cls(uri)
|
||||
|
||||
def media_to_download(self, request, info, *, item=None):
|
||||
def _onsuccess(result):
|
||||
def media_to_download(
|
||||
self, request: Request, info: MediaPipeline.SpiderInfo, *, item: Any = None
|
||||
) -> Deferred[Optional[FileInfo]]:
|
||||
def _onsuccess(result: StatInfo) -> Optional[FileInfo]:
|
||||
if not result:
|
||||
return # returning None force download
|
||||
return None # returning None force download
|
||||
|
||||
last_modified = result.get("last_modified", None)
|
||||
if not last_modified:
|
||||
return # returning None force download
|
||||
return None # returning None force download
|
||||
|
||||
age_seconds = time.time() - last_modified
|
||||
age_days = age_seconds / 60 / 60 / 24
|
||||
if age_days > self.expires:
|
||||
return # returning None force download
|
||||
return None # returning None force download
|
||||
|
||||
referer = referer_str(request)
|
||||
logger.debug(
|
||||
|
|
@ -458,19 +553,22 @@ class FilesPipeline(MediaPipeline):
|
|||
}
|
||||
|
||||
path = self.file_path(request, info=info, item=item)
|
||||
dfd = defer.maybeDeferred(self.store.stat_file, path, info)
|
||||
dfd.addCallback(_onsuccess)
|
||||
dfd.addErrback(lambda _: None)
|
||||
dfd.addErrback(
|
||||
# defer.maybeDeferred() overloads don't seem to support a Union[_T, Deferred[_T]] return type
|
||||
dfd: Deferred[StatInfo] = defer.maybeDeferred(self.store.stat_file, path, info) # type: ignore[arg-type]
|
||||
dfd2: Deferred[Optional[FileInfo]] = dfd.addCallback(_onsuccess)
|
||||
dfd2.addErrback(lambda _: None)
|
||||
dfd2.addErrback(
|
||||
lambda f: logger.error(
|
||||
self.__class__.__name__ + ".store.stat_file",
|
||||
exc_info=failure_to_exc_info(f),
|
||||
extra={"spider": info.spider},
|
||||
)
|
||||
)
|
||||
return dfd
|
||||
return dfd2
|
||||
|
||||
def media_failed(self, failure, request, info):
|
||||
def media_failed(
|
||||
self, failure: Failure, request: Request, info: MediaPipeline.SpiderInfo
|
||||
) -> NoReturn:
|
||||
if not isinstance(failure.value, IgnoreRequest):
|
||||
referer = referer_str(request)
|
||||
logger.warning(
|
||||
|
|
@ -487,7 +585,14 @@ class FilesPipeline(MediaPipeline):
|
|||
|
||||
raise FileException
|
||||
|
||||
def media_downloaded(self, response, request, info, *, item=None):
|
||||
def media_downloaded(
|
||||
self,
|
||||
response: Response,
|
||||
request: Request,
|
||||
info: MediaPipeline.SpiderInfo,
|
||||
*,
|
||||
item: Any = None,
|
||||
) -> FileInfo:
|
||||
referer = referer_str(request)
|
||||
|
||||
if response.status != 200:
|
||||
|
|
@ -546,16 +651,26 @@ class FilesPipeline(MediaPipeline):
|
|||
"status": status,
|
||||
}
|
||||
|
||||
def inc_stats(self, spider, status):
|
||||
def inc_stats(self, spider: Spider, status: str) -> None:
|
||||
assert spider.crawler.stats
|
||||
spider.crawler.stats.inc_value("file_count", spider=spider)
|
||||
spider.crawler.stats.inc_value(f"file_status_count/{status}", spider=spider)
|
||||
|
||||
# Overridable Interface
|
||||
def get_media_requests(self, item, info):
|
||||
def get_media_requests(
|
||||
self, item: Any, info: MediaPipeline.SpiderInfo
|
||||
) -> List[Request]:
|
||||
urls = ItemAdapter(item).get(self.files_urls_field, [])
|
||||
return [Request(u, callback=NO_CALLBACK) for u in urls]
|
||||
|
||||
def file_downloaded(self, response, request, info, *, item=None):
|
||||
def file_downloaded(
|
||||
self,
|
||||
response: Response,
|
||||
request: Request,
|
||||
info: MediaPipeline.SpiderInfo,
|
||||
*,
|
||||
item: Any = None,
|
||||
) -> str:
|
||||
path = self.file_path(request, response=response, info=info, item=item)
|
||||
buf = BytesIO(response.body)
|
||||
checksum = _md5sum(buf)
|
||||
|
|
@ -563,12 +678,21 @@ class FilesPipeline(MediaPipeline):
|
|||
self.store.persist_file(path, buf, info)
|
||||
return checksum
|
||||
|
||||
def item_completed(self, results, item, info):
|
||||
def item_completed(
|
||||
self, results: List[FileInfoOrError], item: Any, info: MediaPipeline.SpiderInfo
|
||||
) -> Any:
|
||||
with suppress(KeyError):
|
||||
ItemAdapter(item)[self.files_result_field] = [x for ok, x in results if ok]
|
||||
return item
|
||||
|
||||
def file_path(self, request, response=None, info=None, *, item=None):
|
||||
def file_path(
|
||||
self,
|
||||
request: Request,
|
||||
response: Optional[Response] = None,
|
||||
info: Optional[MediaPipeline.SpiderInfo] = None,
|
||||
*,
|
||||
item: Any = None,
|
||||
) -> str:
|
||||
media_guid = hashlib.sha1(to_bytes(request.url)).hexdigest() # nosec
|
||||
media_ext = Path(request.url).suffix
|
||||
# Handles empty and wild extensions by trying to guess the
|
||||
|
|
@ -577,5 +701,5 @@ class FilesPipeline(MediaPipeline):
|
|||
media_ext = ""
|
||||
media_type = mimetypes.guess_type(request.url)[0]
|
||||
if media_type:
|
||||
media_ext = mimetypes.guess_extension(media_type)
|
||||
media_ext = cast(str, mimetypes.guess_extension(media_type))
|
||||
return f"full/{media_guid}{media_ext}"
|
||||
|
|
|
|||
|
|
@ -12,12 +12,25 @@ import warnings
|
|||
from contextlib import suppress
|
||||
from io import BytesIO
|
||||
from os import PathLike
|
||||
from typing import TYPE_CHECKING, Dict, Tuple, Type, Union, cast
|
||||
from typing import (
|
||||
TYPE_CHECKING,
|
||||
Any,
|
||||
Callable,
|
||||
Dict,
|
||||
Iterable,
|
||||
List,
|
||||
Optional,
|
||||
Tuple,
|
||||
Type,
|
||||
Union,
|
||||
cast,
|
||||
)
|
||||
|
||||
from itemadapter import ItemAdapter
|
||||
|
||||
from scrapy import Spider
|
||||
from scrapy.exceptions import DropItem, NotConfigured, ScrapyDeprecationWarning
|
||||
from scrapy.http import Request
|
||||
from scrapy.http import Request, Response
|
||||
from scrapy.http.request import NO_CALLBACK
|
||||
from scrapy.pipelines.files import (
|
||||
FileException,
|
||||
|
|
@ -27,20 +40,20 @@ from scrapy.pipelines.files import (
|
|||
S3FilesStore,
|
||||
_md5sum,
|
||||
)
|
||||
|
||||
# TODO: from scrapy.pipelines.media import MediaPipeline
|
||||
from scrapy.pipelines.media import FileInfoOrError, MediaPipeline
|
||||
from scrapy.settings import Settings
|
||||
from scrapy.utils.python import get_func_args, to_bytes
|
||||
|
||||
if TYPE_CHECKING:
|
||||
# typing.Self requires Python 3.11
|
||||
from PIL import Image
|
||||
from typing_extensions import Self
|
||||
|
||||
|
||||
class NoimagesDrop(DropItem):
|
||||
"""Product with no images exception"""
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
def __init__(self, *args: Any, **kwargs: Any):
|
||||
warnings.warn(
|
||||
"The NoimagesDrop class is deprecated",
|
||||
category=ScrapyDeprecationWarning,
|
||||
|
|
@ -56,19 +69,22 @@ class ImageException(FileException):
|
|||
class ImagesPipeline(FilesPipeline):
|
||||
"""Abstract pipeline that implement the image thumbnail generation logic"""
|
||||
|
||||
MEDIA_NAME = "image"
|
||||
MEDIA_NAME: str = "image"
|
||||
|
||||
# Uppercase attributes kept for backward compatibility with code that subclasses
|
||||
# ImagesPipeline. They may be overridden by settings.
|
||||
MIN_WIDTH = 0
|
||||
MIN_HEIGHT = 0
|
||||
EXPIRES = 90
|
||||
MIN_WIDTH: int = 0
|
||||
MIN_HEIGHT: int = 0
|
||||
EXPIRES: int = 90
|
||||
THUMBS: Dict[str, Tuple[int, int]] = {}
|
||||
DEFAULT_IMAGES_URLS_FIELD = "image_urls"
|
||||
DEFAULT_IMAGES_RESULT_FIELD = "images"
|
||||
|
||||
def __init__(
|
||||
self, store_uri: Union[str, PathLike], download_func=None, settings=None
|
||||
self,
|
||||
store_uri: Union[str, PathLike[str]],
|
||||
download_func: Optional[Callable[[Request, Spider], Response]] = None,
|
||||
settings: Union[Settings, Dict[str, Any], None] = None,
|
||||
):
|
||||
try:
|
||||
from PIL import Image
|
||||
|
|
@ -89,27 +105,33 @@ class ImagesPipeline(FilesPipeline):
|
|||
base_class_name="ImagesPipeline",
|
||||
settings=settings,
|
||||
)
|
||||
self.expires = settings.getint(resolve("IMAGES_EXPIRES"), self.EXPIRES)
|
||||
self.expires: int = settings.getint(resolve("IMAGES_EXPIRES"), self.EXPIRES)
|
||||
|
||||
if not hasattr(self, "IMAGES_RESULT_FIELD"):
|
||||
self.IMAGES_RESULT_FIELD = self.DEFAULT_IMAGES_RESULT_FIELD
|
||||
self.IMAGES_RESULT_FIELD: str = self.DEFAULT_IMAGES_RESULT_FIELD
|
||||
if not hasattr(self, "IMAGES_URLS_FIELD"):
|
||||
self.IMAGES_URLS_FIELD = self.DEFAULT_IMAGES_URLS_FIELD
|
||||
self.IMAGES_URLS_FIELD: str = self.DEFAULT_IMAGES_URLS_FIELD
|
||||
|
||||
self.images_urls_field = settings.get(
|
||||
self.images_urls_field: str = settings.get(
|
||||
resolve("IMAGES_URLS_FIELD"), self.IMAGES_URLS_FIELD
|
||||
)
|
||||
self.images_result_field = settings.get(
|
||||
self.images_result_field: str = settings.get(
|
||||
resolve("IMAGES_RESULT_FIELD"), self.IMAGES_RESULT_FIELD
|
||||
)
|
||||
self.min_width = settings.getint(resolve("IMAGES_MIN_WIDTH"), self.MIN_WIDTH)
|
||||
self.min_height = settings.getint(resolve("IMAGES_MIN_HEIGHT"), self.MIN_HEIGHT)
|
||||
self.thumbs = settings.get(resolve("IMAGES_THUMBS"), self.THUMBS)
|
||||
self.min_width: int = settings.getint(
|
||||
resolve("IMAGES_MIN_WIDTH"), self.MIN_WIDTH
|
||||
)
|
||||
self.min_height: int = settings.getint(
|
||||
resolve("IMAGES_MIN_HEIGHT"), self.MIN_HEIGHT
|
||||
)
|
||||
self.thumbs: Dict[str, Tuple[int, int]] = settings.get(
|
||||
resolve("IMAGES_THUMBS"), self.THUMBS
|
||||
)
|
||||
|
||||
self._deprecated_convert_image = None
|
||||
self._deprecated_convert_image: Optional[bool] = None
|
||||
|
||||
@classmethod
|
||||
def from_settings(cls, settings) -> Self:
|
||||
def from_settings(cls, settings: Settings) -> Self:
|
||||
s3store: Type[S3FilesStore] = cast(Type[S3FilesStore], cls.STORE_SCHEMES["s3"])
|
||||
s3store.AWS_ACCESS_KEY_ID = settings["AWS_ACCESS_KEY_ID"]
|
||||
s3store.AWS_SECRET_ACCESS_KEY = settings["AWS_SECRET_ACCESS_KEY"]
|
||||
|
|
@ -136,11 +158,25 @@ class ImagesPipeline(FilesPipeline):
|
|||
store_uri = settings["IMAGES_STORE"]
|
||||
return cls(store_uri, settings=settings)
|
||||
|
||||
def file_downloaded(self, response, request, info, *, item=None):
|
||||
def file_downloaded(
|
||||
self,
|
||||
response: Response,
|
||||
request: Request,
|
||||
info: MediaPipeline.SpiderInfo,
|
||||
*,
|
||||
item: Any = None,
|
||||
) -> str:
|
||||
return self.image_downloaded(response, request, info, item=item)
|
||||
|
||||
def image_downloaded(self, response, request, info, *, item=None):
|
||||
checksum = None
|
||||
def image_downloaded(
|
||||
self,
|
||||
response: Response,
|
||||
request: Request,
|
||||
info: MediaPipeline.SpiderInfo,
|
||||
*,
|
||||
item: Any = None,
|
||||
) -> str:
|
||||
checksum: Optional[str] = None
|
||||
for path, image, buf in self.get_images(response, request, info, item=item):
|
||||
if checksum is None:
|
||||
buf.seek(0)
|
||||
|
|
@ -153,9 +189,17 @@ class ImagesPipeline(FilesPipeline):
|
|||
meta={"width": width, "height": height},
|
||||
headers={"Content-Type": "image/jpeg"},
|
||||
)
|
||||
assert checksum is not None
|
||||
return checksum
|
||||
|
||||
def get_images(self, response, request, info, *, item=None):
|
||||
def get_images(
|
||||
self,
|
||||
response: Response,
|
||||
request: Request,
|
||||
info: MediaPipeline.SpiderInfo,
|
||||
*,
|
||||
item: Any = None,
|
||||
) -> Iterable[Tuple[str, Image.Image, BytesIO]]:
|
||||
path = self.file_path(request, response=response, info=info, item=item)
|
||||
orig_image = self._Image.open(BytesIO(response.body))
|
||||
|
||||
|
|
@ -196,7 +240,12 @@ class ImagesPipeline(FilesPipeline):
|
|||
thumb_image, thumb_buf = self.convert_image(image, size, buf)
|
||||
yield thumb_path, thumb_image, thumb_buf
|
||||
|
||||
def convert_image(self, image, size=None, response_body=None):
|
||||
def convert_image(
|
||||
self,
|
||||
image: Image.Image,
|
||||
size: Optional[Tuple[int, int]] = None,
|
||||
response_body: Optional[BytesIO] = None,
|
||||
) -> Tuple[Image.Image, BytesIO]:
|
||||
if response_body is None:
|
||||
warnings.warn(
|
||||
f"{self.__class__.__name__}.convert_image() method called in a deprecated way, "
|
||||
|
|
@ -225,7 +274,7 @@ class ImagesPipeline(FilesPipeline):
|
|||
# when updating the minimum requirements for Pillow.
|
||||
resampling_filter = self._Image.Resampling.LANCZOS
|
||||
except AttributeError:
|
||||
resampling_filter = self._Image.ANTIALIAS
|
||||
resampling_filter = self._Image.ANTIALIAS # type: ignore[attr-defined]
|
||||
image.thumbnail(size, resampling_filter)
|
||||
elif response_body is not None and image.format == "JPEG":
|
||||
return image, response_body
|
||||
|
|
@ -234,19 +283,38 @@ class ImagesPipeline(FilesPipeline):
|
|||
image.save(buf, "JPEG")
|
||||
return image, buf
|
||||
|
||||
def get_media_requests(self, item, info):
|
||||
def get_media_requests(
|
||||
self, item: Any, info: MediaPipeline.SpiderInfo
|
||||
) -> List[Request]:
|
||||
urls = ItemAdapter(item).get(self.images_urls_field, [])
|
||||
return [Request(u, callback=NO_CALLBACK) for u in urls]
|
||||
|
||||
def item_completed(self, results, item, info):
|
||||
def item_completed(
|
||||
self, results: List[FileInfoOrError], item: Any, info: MediaPipeline.SpiderInfo
|
||||
) -> Any:
|
||||
with suppress(KeyError):
|
||||
ItemAdapter(item)[self.images_result_field] = [x for ok, x in results if ok]
|
||||
return item
|
||||
|
||||
def file_path(self, request, response=None, info=None, *, item=None):
|
||||
def file_path(
|
||||
self,
|
||||
request: Request,
|
||||
response: Optional[Response] = None,
|
||||
info: Optional[MediaPipeline.SpiderInfo] = None,
|
||||
*,
|
||||
item: Any = None,
|
||||
) -> str:
|
||||
image_guid = hashlib.sha1(to_bytes(request.url)).hexdigest() # nosec
|
||||
return f"full/{image_guid}.jpg"
|
||||
|
||||
def thumb_path(self, request, thumb_id, response=None, info=None, *, item=None):
|
||||
def thumb_path(
|
||||
self,
|
||||
request: Request,
|
||||
thumb_id: str,
|
||||
response: Optional[Response] = None,
|
||||
info: Optional[MediaPipeline.SpiderInfo] = None,
|
||||
*,
|
||||
item: Any = None,
|
||||
) -> str:
|
||||
thumb_guid = hashlib.sha1(to_bytes(request.url)).hexdigest() # nosec
|
||||
return f"thumbs/{thumb_id}/{thumb_guid}.jpg"
|
||||
|
|
|
|||
|
|
@ -4,58 +4,101 @@ import functools
|
|||
import logging
|
||||
from abc import ABC, abstractmethod
|
||||
from collections import defaultdict
|
||||
from typing import TYPE_CHECKING
|
||||
from typing import (
|
||||
TYPE_CHECKING,
|
||||
Any,
|
||||
Callable,
|
||||
DefaultDict,
|
||||
Dict,
|
||||
List,
|
||||
Literal,
|
||||
NoReturn,
|
||||
Optional,
|
||||
Set,
|
||||
Tuple,
|
||||
TypedDict,
|
||||
TypeVar,
|
||||
Union,
|
||||
cast,
|
||||
)
|
||||
|
||||
from twisted.internet.defer import Deferred, DeferredList
|
||||
from twisted.python.failure import Failure
|
||||
|
||||
from scrapy.http.request import NO_CALLBACK
|
||||
from scrapy import Spider
|
||||
from scrapy.crawler import Crawler
|
||||
from scrapy.http import Response
|
||||
from scrapy.http.request import NO_CALLBACK, Request
|
||||
from scrapy.settings import Settings
|
||||
from scrapy.utils.datatypes import SequenceExclude
|
||||
from scrapy.utils.defer import defer_result, mustbe_deferred
|
||||
from scrapy.utils.log import failure_to_exc_info
|
||||
from scrapy.utils.misc import arg_to_iter
|
||||
from scrapy.utils.request import RequestFingerprinter
|
||||
|
||||
if TYPE_CHECKING:
|
||||
# typing.Self requires Python 3.11
|
||||
from typing_extensions import Self
|
||||
|
||||
_T = TypeVar("_T")
|
||||
|
||||
|
||||
class FileInfo(TypedDict):
|
||||
url: str
|
||||
path: str
|
||||
checksum: Optional[str]
|
||||
status: str
|
||||
|
||||
|
||||
FileInfoOrError = Union[Tuple[Literal[True], FileInfo], Tuple[Literal[False], Failure]]
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _DUMMY_CALLBACK(response):
|
||||
return response
|
||||
|
||||
|
||||
class MediaPipeline(ABC):
|
||||
LOG_FAILED_RESULTS = True
|
||||
crawler: Crawler
|
||||
_fingerprinter: RequestFingerprinter
|
||||
|
||||
LOG_FAILED_RESULTS: bool = True
|
||||
|
||||
class SpiderInfo:
|
||||
def __init__(self, spider):
|
||||
self.spider = spider
|
||||
self.downloading = set()
|
||||
self.downloaded = {}
|
||||
self.waiting = defaultdict(list)
|
||||
def __init__(self, spider: Spider):
|
||||
self.spider: Spider = spider
|
||||
self.downloading: Set[bytes] = set()
|
||||
self.downloaded: Dict[bytes, Union[FileInfo, Failure]] = {}
|
||||
self.waiting: DefaultDict[bytes, List[Deferred[FileInfo]]] = defaultdict(
|
||||
list
|
||||
)
|
||||
|
||||
def __init__(self, download_func=None, settings=None):
|
||||
def __init__(
|
||||
self,
|
||||
download_func: Optional[Callable[[Request, Spider], Response]] = None,
|
||||
settings: Union[Settings, Dict[str, Any], None] = None,
|
||||
):
|
||||
self.download_func = download_func
|
||||
self._expects_item = {}
|
||||
|
||||
if isinstance(settings, dict) or settings is None:
|
||||
settings = Settings(settings)
|
||||
resolve = functools.partial(
|
||||
self._key_for_pipe, base_class_name="MediaPipeline", settings=settings
|
||||
)
|
||||
self.allow_redirects = settings.getbool(resolve("MEDIA_ALLOW_REDIRECTS"), False)
|
||||
self.allow_redirects: bool = settings.getbool(
|
||||
resolve("MEDIA_ALLOW_REDIRECTS"), False
|
||||
)
|
||||
self._handle_statuses(self.allow_redirects)
|
||||
|
||||
def _handle_statuses(self, allow_redirects):
|
||||
def _handle_statuses(self, allow_redirects: bool) -> None:
|
||||
self.handle_httpstatus_list = None
|
||||
if allow_redirects:
|
||||
self.handle_httpstatus_list = SequenceExclude(range(300, 400))
|
||||
|
||||
def _key_for_pipe(self, key, base_class_name=None, settings=None):
|
||||
def _key_for_pipe(
|
||||
self,
|
||||
key: str,
|
||||
base_class_name: Optional[str] = None,
|
||||
settings: Optional[Settings] = None,
|
||||
) -> str:
|
||||
class_name = self.__class__.__name__
|
||||
formatted_key = f"{class_name.upper()}_{key}"
|
||||
if (
|
||||
|
|
@ -68,31 +111,35 @@ class MediaPipeline(ABC):
|
|||
return formatted_key
|
||||
|
||||
@classmethod
|
||||
def from_crawler(cls, crawler) -> Self:
|
||||
def from_crawler(cls, crawler: Crawler) -> Self:
|
||||
pipe: Self
|
||||
try:
|
||||
pipe = cls.from_settings(crawler.settings) # type: ignore[attr-defined]
|
||||
except AttributeError:
|
||||
pipe = cls()
|
||||
pipe.crawler = crawler
|
||||
assert crawler.request_fingerprinter
|
||||
pipe._fingerprinter = crawler.request_fingerprinter
|
||||
return pipe
|
||||
|
||||
def open_spider(self, spider):
|
||||
def open_spider(self, spider: Spider) -> None:
|
||||
self.spiderinfo = self.SpiderInfo(spider)
|
||||
|
||||
def process_item(self, item, spider):
|
||||
def process_item(
|
||||
self, item: Any, spider: Spider
|
||||
) -> Deferred[List[FileInfoOrError]]:
|
||||
info = self.spiderinfo
|
||||
requests = arg_to_iter(self.get_media_requests(item, info))
|
||||
dlist = [self._process_request(r, info, item) for r in requests]
|
||||
dfd = DeferredList(dlist, consumeErrors=True)
|
||||
dfd = cast(
|
||||
"Deferred[List[FileInfoOrError]]", DeferredList(dlist, consumeErrors=True)
|
||||
)
|
||||
return dfd.addCallback(self.item_completed, item, info)
|
||||
|
||||
def _process_request(self, request, info, item):
|
||||
def _process_request(
|
||||
self, request: Request, info: SpiderInfo, item: Any
|
||||
) -> Deferred[FileInfo]:
|
||||
fp = self._fingerprinter.fingerprint(request)
|
||||
if not request.callback or request.callback is NO_CALLBACK:
|
||||
cb = _DUMMY_CALLBACK
|
||||
else:
|
||||
cb = request.callback
|
||||
eb = request.errback
|
||||
request.callback = NO_CALLBACK
|
||||
request.errback = None
|
||||
|
|
@ -100,14 +147,12 @@ class MediaPipeline(ABC):
|
|||
# Return cached result if request was already seen
|
||||
if fp in info.downloaded:
|
||||
d = defer_result(info.downloaded[fp])
|
||||
d.addCallback(cb)
|
||||
if eb:
|
||||
d.addErrback(eb)
|
||||
return d
|
||||
|
||||
# Otherwise, wait for result
|
||||
wad = Deferred()
|
||||
wad.addCallback(cb)
|
||||
wad: Deferred[FileInfo] = Deferred()
|
||||
if eb:
|
||||
wad.addErrback(eb)
|
||||
info.waiting[fp].append(wad)
|
||||
|
|
@ -118,36 +163,48 @@ class MediaPipeline(ABC):
|
|||
|
||||
# Download request checking media_to_download hook output first
|
||||
info.downloading.add(fp)
|
||||
dfd = mustbe_deferred(self.media_to_download, request, info, item=item)
|
||||
dfd.addCallback(self._check_media_to_download, request, info, item=item)
|
||||
dfd.addErrback(self._log_exception)
|
||||
dfd.addBoth(self._cache_result_and_execute_waiters, fp, info)
|
||||
return dfd.addBoth(lambda _: wad) # it must return wad at last
|
||||
dfd: Deferred[Optional[FileInfo]] = mustbe_deferred(
|
||||
self.media_to_download, request, info, item=item
|
||||
)
|
||||
dfd2: Deferred[FileInfo] = dfd.addCallback(
|
||||
self._check_media_to_download, request, info, item=item
|
||||
)
|
||||
dfd2.addErrback(self._log_exception)
|
||||
dfd2.addBoth(self._cache_result_and_execute_waiters, fp, info)
|
||||
return dfd2.addBoth(lambda _: wad) # it must return wad at last
|
||||
|
||||
def _log_exception(self, result):
|
||||
def _log_exception(self, result: Failure) -> Failure:
|
||||
logger.exception(result)
|
||||
return result
|
||||
|
||||
def _modify_media_request(self, request):
|
||||
def _modify_media_request(self, request: Request) -> None:
|
||||
if self.handle_httpstatus_list:
|
||||
request.meta["handle_httpstatus_list"] = self.handle_httpstatus_list
|
||||
else:
|
||||
request.meta["handle_httpstatus_all"] = True
|
||||
|
||||
def _check_media_to_download(self, result, request, info, item):
|
||||
def _check_media_to_download(
|
||||
self, result: Optional[FileInfo], request: Request, info: SpiderInfo, item: Any
|
||||
) -> Union[FileInfo, Deferred[FileInfo]]:
|
||||
if result is not None:
|
||||
return result
|
||||
dfd: Deferred[Response]
|
||||
if self.download_func:
|
||||
# this ugly code was left only to support tests. TODO: remove
|
||||
dfd = mustbe_deferred(self.download_func, request, info.spider)
|
||||
else:
|
||||
self._modify_media_request(request)
|
||||
assert self.crawler.engine
|
||||
dfd = self.crawler.engine.download(request)
|
||||
dfd.addCallback(self.media_downloaded, request, info, item=item)
|
||||
dfd.addErrback(self.media_failed, request, info)
|
||||
return dfd
|
||||
dfd2: Deferred[FileInfo] = dfd.addCallback(
|
||||
self.media_downloaded, request, info, item=item
|
||||
)
|
||||
dfd2.addErrback(self.media_failed, request, info)
|
||||
return dfd2
|
||||
|
||||
def _cache_result_and_execute_waiters(self, result, fp, info):
|
||||
def _cache_result_and_execute_waiters(
|
||||
self, result: Union[FileInfo, Failure], fp: bytes, info: SpiderInfo
|
||||
) -> None:
|
||||
if isinstance(result, Failure):
|
||||
# minimize cached information for failure
|
||||
result.cleanFailure()
|
||||
|
|
@ -186,30 +243,44 @@ class MediaPipeline(ABC):
|
|||
|
||||
# Overridable Interface
|
||||
@abstractmethod
|
||||
def media_to_download(self, request, info, *, item=None):
|
||||
def media_to_download(
|
||||
self, request: Request, info: SpiderInfo, *, item: Any = None
|
||||
) -> Deferred[Optional[FileInfo]]:
|
||||
"""Check request before starting download"""
|
||||
raise NotImplementedError()
|
||||
|
||||
@abstractmethod
|
||||
def get_media_requests(self, item, info):
|
||||
def get_media_requests(self, item: Any, info: SpiderInfo) -> List[Request]:
|
||||
"""Returns the media requests to download"""
|
||||
raise NotImplementedError()
|
||||
|
||||
@abstractmethod
|
||||
def media_downloaded(self, response, request, info, *, item=None):
|
||||
def media_downloaded(
|
||||
self,
|
||||
response: Response,
|
||||
request: Request,
|
||||
info: SpiderInfo,
|
||||
*,
|
||||
item: Any = None,
|
||||
) -> FileInfo:
|
||||
"""Handler for success downloads"""
|
||||
raise NotImplementedError()
|
||||
|
||||
@abstractmethod
|
||||
def media_failed(self, failure, request, info):
|
||||
def media_failed(
|
||||
self, failure: Failure, request: Request, info: SpiderInfo
|
||||
) -> NoReturn:
|
||||
"""Handler for failed downloads"""
|
||||
raise NotImplementedError()
|
||||
|
||||
def item_completed(self, results, item, info):
|
||||
def item_completed(
|
||||
self, results: List[FileInfoOrError], item: Any, info: SpiderInfo
|
||||
) -> Any:
|
||||
"""Called per item when all media requests has been processed"""
|
||||
if self.LOG_FAILED_RESULTS:
|
||||
for ok, value in results:
|
||||
if not ok:
|
||||
assert isinstance(value, Failure)
|
||||
logger.error(
|
||||
"%(class)s found errors processing %(item)s",
|
||||
{"class": self.__class__.__name__, "item": item},
|
||||
|
|
@ -219,6 +290,13 @@ class MediaPipeline(ABC):
|
|||
return item
|
||||
|
||||
@abstractmethod
|
||||
def file_path(self, request, response=None, info=None, *, item=None):
|
||||
def file_path(
|
||||
self,
|
||||
request: Request,
|
||||
response: Optional[Response] = None,
|
||||
info: Optional[SpiderInfo] = None,
|
||||
*,
|
||||
item: Any = None,
|
||||
) -> str:
|
||||
"""Returns the path where downloaded media should be stored"""
|
||||
raise NotImplementedError()
|
||||
|
|
|
|||
|
|
@ -59,6 +59,7 @@ class Selector(_ParselSelector, object_ref):
|
|||
|
||||
* ``"html"`` for :class:`~scrapy.http.HtmlResponse` type
|
||||
* ``"xml"`` for :class:`~scrapy.http.XmlResponse` type
|
||||
* ``"json"`` for :class:`~scrapy.http.TextResponse` type
|
||||
* ``"html"`` for anything else
|
||||
|
||||
Otherwise, if ``type`` is set, the selector type will be forced and no
|
||||
|
|
|
|||
|
|
@ -411,7 +411,7 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]):
|
|||
"""
|
||||
self._assert_mutability()
|
||||
if isinstance(values, str):
|
||||
values = cast(dict, json.loads(values))
|
||||
values = cast(Dict[_SettingsKeyT, Any], json.loads(values))
|
||||
if values is not None:
|
||||
if isinstance(values, BaseSettings):
|
||||
for name, value in values.items():
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@ See documentation in docs/topics/spiders.rst
|
|||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import TYPE_CHECKING, Any, Iterable, List, Optional, Union, cast
|
||||
from typing import TYPE_CHECKING, Any, Dict, Iterable, List, Optional, Union, cast
|
||||
|
||||
from twisted.internet.defer import Deferred
|
||||
|
||||
|
|
@ -24,7 +24,7 @@ if TYPE_CHECKING:
|
|||
from typing_extensions import Concatenate, Self
|
||||
|
||||
from scrapy.crawler import Crawler
|
||||
from scrapy.settings import BaseSettings
|
||||
from scrapy.settings import BaseSettings, _SettingsKeyT
|
||||
from scrapy.utils.log import SpiderLoggerAdapter
|
||||
|
||||
CallbackT = Callable[Concatenate[Response, ...], Any]
|
||||
|
|
@ -36,7 +36,7 @@ class Spider(object_ref):
|
|||
"""
|
||||
|
||||
name: str
|
||||
custom_settings: Optional[dict] = None
|
||||
custom_settings: Optional[Dict[_SettingsKeyT, Any]] = None
|
||||
|
||||
def __init__(self, name: Optional[str] = None, **kwargs: Any):
|
||||
if name is not None:
|
||||
|
|
|
|||
|
|
@ -1,14 +1,18 @@
|
|||
from typing import AsyncGenerator, AsyncIterable, Iterable, Union
|
||||
from typing import AsyncGenerator, AsyncIterable, Iterable, List, TypeVar, Union
|
||||
|
||||
_T = TypeVar("_T")
|
||||
|
||||
|
||||
async def collect_asyncgen(result: AsyncIterable) -> list:
|
||||
async def collect_asyncgen(result: AsyncIterable[_T]) -> List[_T]:
|
||||
results = []
|
||||
async for x in result:
|
||||
results.append(x)
|
||||
return results
|
||||
|
||||
|
||||
async def as_async_generator(it: Union[Iterable, AsyncIterable]) -> AsyncGenerator:
|
||||
async def as_async_generator(
|
||||
it: Union[Iterable[_T], AsyncIterable[_T]]
|
||||
) -> AsyncGenerator[_T, None]:
|
||||
"""Wraps an iterable (sync or async) into an async generator."""
|
||||
if isinstance(it, AsyncIterable):
|
||||
async for r in it:
|
||||
|
|
|
|||
|
|
@ -16,6 +16,7 @@ from typing import (
|
|||
MutableMapping,
|
||||
Optional,
|
||||
Union,
|
||||
cast,
|
||||
)
|
||||
|
||||
from scrapy.exceptions import ScrapyDeprecationWarning, UsageError
|
||||
|
|
@ -173,7 +174,7 @@ def feed_process_params_from_cli(
|
|||
suitable to be used as the FEEDS setting.
|
||||
"""
|
||||
valid_output_formats: Iterable[str] = without_none_values(
|
||||
settings.getwithbase("FEED_EXPORTERS")
|
||||
cast(Dict[str, str], settings.getwithbase("FEED_EXPORTERS"))
|
||||
).keys()
|
||||
|
||||
def check_valid_format(output_format: str) -> None:
|
||||
|
|
|
|||
|
|
@ -196,8 +196,8 @@ class LocalWeakReferencedCache(weakref.WeakKeyDictionary):
|
|||
class SequenceExclude:
|
||||
"""Object to test if an item is NOT within some sequence."""
|
||||
|
||||
def __init__(self, seq: Sequence):
|
||||
self.seq: Sequence = seq
|
||||
def __init__(self, seq: Sequence[Any]):
|
||||
self.seq: Sequence[Any] = seq
|
||||
|
||||
def __contains__(self, item: Any) -> bool:
|
||||
return item not in self.seq
|
||||
|
|
|
|||
|
|
@ -21,7 +21,7 @@ def ftp_makedirs_cwd(ftp: FTP, path: str, first_call: bool = True) -> None:
|
|||
def ftp_store_file(
|
||||
*,
|
||||
path: str,
|
||||
file: IO,
|
||||
file: IO[bytes],
|
||||
host: str,
|
||||
port: int,
|
||||
username: str,
|
||||
|
|
|
|||
|
|
@ -6,8 +6,7 @@ from typing import (
|
|||
Any,
|
||||
Callable,
|
||||
Dict,
|
||||
Generator,
|
||||
Iterable,
|
||||
Iterator,
|
||||
List,
|
||||
Literal,
|
||||
Optional,
|
||||
|
|
@ -22,14 +21,12 @@ from lxml import etree # nosec
|
|||
from scrapy.exceptions import ScrapyDeprecationWarning
|
||||
from scrapy.http import Response, TextResponse
|
||||
from scrapy.selector import Selector
|
||||
from scrapy.utils.python import re_rsearch, to_unicode
|
||||
from scrapy.utils.python import re_rsearch
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def xmliter(
|
||||
obj: Union[Response, str, bytes], nodename: str
|
||||
) -> Generator[Selector, Any, None]:
|
||||
def xmliter(obj: Union[Response, str, bytes], nodename: str) -> Iterator[Selector]:
|
||||
"""Return a iterator of Selector's over all nodes of a XML document,
|
||||
given the name of the node to iterate. Useful for parsing XML feeds.
|
||||
|
||||
|
|
@ -90,7 +87,7 @@ def xmliter_lxml(
|
|||
nodename: str,
|
||||
namespace: Optional[str] = None,
|
||||
prefix: str = "x",
|
||||
) -> Generator[Selector, Any, None]:
|
||||
) -> Iterator[Selector]:
|
||||
reader = _StreamReader(obj)
|
||||
tag = f"{{{namespace}}}{nodename}" if namespace else nodename
|
||||
iterable = etree.iterparse(
|
||||
|
|
@ -168,7 +165,7 @@ def csviter(
|
|||
headers: Optional[List[str]] = None,
|
||||
encoding: Optional[str] = None,
|
||||
quotechar: Optional[str] = None,
|
||||
) -> Generator[Dict[str, str], Any, None]:
|
||||
) -> Iterator[Dict[str, str]]:
|
||||
"""Returns an iterator of dictionaries from the given csv object
|
||||
|
||||
obj can be:
|
||||
|
|
@ -184,10 +181,13 @@ def csviter(
|
|||
quotechar is the character used to enclosure fields on the given obj.
|
||||
"""
|
||||
|
||||
encoding = obj.encoding if isinstance(obj, TextResponse) else encoding or "utf-8"
|
||||
|
||||
def row_to_unicode(row_: Iterable) -> List[str]:
|
||||
return [to_unicode(field, encoding) for field in row_]
|
||||
if encoding is not None:
|
||||
warn(
|
||||
"The encoding argument of csviter() is ignored and will be removed"
|
||||
" in a future Scrapy version.",
|
||||
category=ScrapyDeprecationWarning,
|
||||
stacklevel=2,
|
||||
)
|
||||
|
||||
lines = StringIO(_body_or_str(obj, unicode=True))
|
||||
|
||||
|
|
@ -200,13 +200,11 @@ def csviter(
|
|||
|
||||
if not headers:
|
||||
try:
|
||||
row = next(csv_r)
|
||||
headers = next(csv_r)
|
||||
except StopIteration:
|
||||
return
|
||||
headers = row_to_unicode(row)
|
||||
|
||||
for row in csv_r:
|
||||
row = row_to_unicode(row)
|
||||
if len(row) != len(headers):
|
||||
logger.warning(
|
||||
"ignoring row %(csvlnum)d (length: %(csvrow)d, "
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ from types import TracebackType
|
|||
from typing import (
|
||||
TYPE_CHECKING,
|
||||
Any,
|
||||
Dict,
|
||||
List,
|
||||
MutableMapping,
|
||||
Optional,
|
||||
|
|
@ -20,7 +21,8 @@ from twisted.python import log as twisted_log
|
|||
from twisted.python.failure import Failure
|
||||
|
||||
import scrapy
|
||||
from scrapy.settings import Settings
|
||||
from scrapy.logformatter import LogFormatterResult
|
||||
from scrapy.settings import Settings, _SettingsKeyT
|
||||
from scrapy.utils.versions import scrapy_components_versions
|
||||
|
||||
if TYPE_CHECKING:
|
||||
|
|
@ -86,7 +88,8 @@ DEFAULT_LOGGING = {
|
|||
|
||||
|
||||
def configure_logging(
|
||||
settings: Union[Settings, dict, None] = None, install_root_handler: bool = True
|
||||
settings: Union[Settings, Dict[_SettingsKeyT, Any], None] = None,
|
||||
install_root_handler: bool = True,
|
||||
) -> None:
|
||||
"""
|
||||
Initialize logging defaults for Scrapy.
|
||||
|
|
@ -234,7 +237,9 @@ class LogCounterHandler(logging.Handler):
|
|||
self.crawler.stats.inc_value(sname)
|
||||
|
||||
|
||||
def logformatter_adapter(logkws: dict) -> Tuple[int, str, dict]:
|
||||
def logformatter_adapter(
|
||||
logkws: LogFormatterResult,
|
||||
) -> Tuple[int, str, Union[Dict[str, Any], Tuple[Any, ...]]]:
|
||||
"""
|
||||
Helper that takes the dictionary output from the methods in LogFormatter
|
||||
and adapts it into a tuple of positional arguments for logger.log calls,
|
||||
|
|
@ -245,7 +250,7 @@ def logformatter_adapter(logkws: dict) -> Tuple[int, str, dict]:
|
|||
message = logkws.get("msg") or ""
|
||||
# NOTE: This also handles 'args' being an empty dict, that case doesn't
|
||||
# play well in logger.log calls
|
||||
args = logkws if not logkws.get("args") else logkws["args"]
|
||||
args = cast(Dict[str, Any], logkws) if not logkws.get("args") else logkws["args"]
|
||||
|
||||
return (level, message, args)
|
||||
|
||||
|
|
|
|||
|
|
@ -20,8 +20,8 @@ from typing import (
|
|||
Any,
|
||||
Callable,
|
||||
Deque,
|
||||
Generator,
|
||||
Iterable,
|
||||
Iterator,
|
||||
List,
|
||||
Optional,
|
||||
Type,
|
||||
|
|
@ -56,7 +56,7 @@ def arg_to_iter(arg: Any) -> Iterable[Any]:
|
|||
return [arg]
|
||||
|
||||
|
||||
def load_object(path: Union[str, Callable]) -> Any:
|
||||
def load_object(path: Union[str, Callable[..., Any]]) -> Any:
|
||||
"""Load an object given its absolute object path, and return it.
|
||||
|
||||
The object can be the import path of a class, function, variable or an
|
||||
|
|
@ -111,7 +111,7 @@ def walk_modules(path: str) -> List[ModuleType]:
|
|||
return mods
|
||||
|
||||
|
||||
def md5sum(file: IO) -> str:
|
||||
def md5sum(file: IO[bytes]) -> str:
|
||||
"""Calculate the md5 checksum of a file-like object without reading its
|
||||
whole content in memory.
|
||||
|
||||
|
|
@ -227,7 +227,7 @@ def build_from_settings(
|
|||
|
||||
|
||||
@contextmanager
|
||||
def set_environ(**kwargs: str) -> Generator[None, Any, None]:
|
||||
def set_environ(**kwargs: str) -> Iterator[None]:
|
||||
"""Temporarily set environment variables inside the context manager and
|
||||
fully restore previous environment afterwards
|
||||
"""
|
||||
|
|
@ -244,7 +244,7 @@ def set_environ(**kwargs: str) -> Generator[None, Any, None]:
|
|||
os.environ[k] = v
|
||||
|
||||
|
||||
def walk_callable(node: ast.AST) -> Generator[ast.AST, Any, None]:
|
||||
def walk_callable(node: ast.AST) -> Iterable[ast.AST]:
|
||||
"""Similar to ``ast.walk``, but walks only function body and skips nested
|
||||
functions defined within the node.
|
||||
"""
|
||||
|
|
@ -263,7 +263,7 @@ def walk_callable(node: ast.AST) -> Generator[ast.AST, Any, None]:
|
|||
_generator_callbacks_cache = LocalWeakReferencedCache(limit=128)
|
||||
|
||||
|
||||
def is_generator_with_return_value(callable: Callable) -> bool:
|
||||
def is_generator_with_return_value(callable: Callable[..., Any]) -> bool:
|
||||
"""
|
||||
Returns True if a callable is a generator function which includes a
|
||||
'return' statement with a value different than None, False otherwise
|
||||
|
|
@ -300,7 +300,9 @@ def is_generator_with_return_value(callable: Callable) -> bool:
|
|||
return bool(_generator_callbacks_cache[callable])
|
||||
|
||||
|
||||
def warn_on_generator_with_return_value(spider: Spider, callable: Callable) -> None:
|
||||
def warn_on_generator_with_return_value(
|
||||
spider: Spider, callable: Callable[..., Any]
|
||||
) -> None:
|
||||
"""
|
||||
Logs a warning if a callable is a generator function and includes
|
||||
a 'return' statement with a value different than None
|
||||
|
|
|
|||
|
|
@ -15,12 +15,10 @@ from itertools import chain
|
|||
from typing import (
|
||||
TYPE_CHECKING,
|
||||
Any,
|
||||
AsyncGenerator,
|
||||
AsyncIterable,
|
||||
AsyncIterator,
|
||||
Callable,
|
||||
Dict,
|
||||
Generator,
|
||||
Iterable,
|
||||
Iterator,
|
||||
List,
|
||||
|
|
@ -42,9 +40,11 @@ if TYPE_CHECKING:
|
|||
_P = ParamSpec("_P")
|
||||
|
||||
_T = TypeVar("_T")
|
||||
_KT = TypeVar("_KT")
|
||||
_VT = TypeVar("_VT")
|
||||
|
||||
|
||||
def flatten(x: Iterable) -> list:
|
||||
def flatten(x: Iterable[Any]) -> List[Any]:
|
||||
"""flatten(sequence) -> list
|
||||
|
||||
Returns a single, flat list which contains all elements retrieved
|
||||
|
|
@ -64,7 +64,7 @@ def flatten(x: Iterable) -> list:
|
|||
return list(iflatten(x))
|
||||
|
||||
|
||||
def iflatten(x: Iterable) -> Iterable:
|
||||
def iflatten(x: Iterable[Any]) -> Iterable[Any]:
|
||||
"""iflatten(sequence) -> iterator
|
||||
|
||||
Similar to ``.flatten()``, but returns iterator instead"""
|
||||
|
|
@ -99,10 +99,10 @@ def is_listlike(x: Any) -> bool:
|
|||
return hasattr(x, "__iter__") and not isinstance(x, (str, bytes))
|
||||
|
||||
|
||||
def unique(list_: Iterable, key: Callable[[Any], Any] = lambda x: x) -> list:
|
||||
def unique(list_: Iterable[_T], key: Callable[[_T], Any] = lambda x: x) -> List[_T]:
|
||||
"""efficient function to uniquify a list preserving item order"""
|
||||
seen = set()
|
||||
result = []
|
||||
result: List[_T] = []
|
||||
for item in list_:
|
||||
seenkey = key(item)
|
||||
if seenkey in seen:
|
||||
|
|
@ -146,7 +146,7 @@ def to_bytes(
|
|||
|
||||
|
||||
def re_rsearch(
|
||||
pattern: Union[str, Pattern], text: str, chunk_size: int = 1024
|
||||
pattern: Union[str, Pattern[str]], text: str, chunk_size: int = 1024
|
||||
) -> Optional[Tuple[int, int]]:
|
||||
"""
|
||||
This function does a reverse search in a text using a regular expression
|
||||
|
|
@ -161,7 +161,7 @@ def re_rsearch(
|
|||
the start position of the match, and the ending (regarding the entire text).
|
||||
"""
|
||||
|
||||
def _chunk_iter() -> Generator[Tuple[str, int], Any, None]:
|
||||
def _chunk_iter() -> Iterable[Tuple[str, int]]:
|
||||
offset = len(text)
|
||||
while True:
|
||||
offset -= chunk_size * 1024
|
||||
|
|
@ -215,7 +215,7 @@ def binary_is_text(data: bytes) -> bool:
|
|||
return all(c not in _BINARYCHARS for c in data)
|
||||
|
||||
|
||||
def get_func_args(func: Callable, stripself: bool = False) -> List[str]:
|
||||
def get_func_args(func: Callable[..., Any], stripself: bool = False) -> List[str]:
|
||||
"""Return the argument name list of a callable object"""
|
||||
if not callable(func):
|
||||
raise TypeError(f"func must be callable, got '{type(func).__name__}'")
|
||||
|
|
@ -245,7 +245,7 @@ def get_func_args(func: Callable, stripself: bool = False) -> List[str]:
|
|||
return args
|
||||
|
||||
|
||||
def get_spec(func: Callable) -> Tuple[List[str], Dict[str, Any]]:
|
||||
def get_spec(func: Callable[..., Any]) -> Tuple[List[str], Dict[str, Any]]:
|
||||
"""Returns (args, kwargs) tuple for a function
|
||||
>>> import re
|
||||
>>> get_spec(re.match)
|
||||
|
|
@ -283,7 +283,7 @@ def get_spec(func: Callable) -> Tuple[List[str], Dict[str, Any]]:
|
|||
|
||||
|
||||
def equal_attributes(
|
||||
obj1: Any, obj2: Any, attributes: Optional[List[Union[str, Callable]]]
|
||||
obj1: Any, obj2: Any, attributes: Optional[List[Union[str, Callable[[Any], Any]]]]
|
||||
) -> bool:
|
||||
"""Compare two objects attributes"""
|
||||
# not attributes given return False by default
|
||||
|
|
@ -303,14 +303,16 @@ def equal_attributes(
|
|||
|
||||
|
||||
@overload
|
||||
def without_none_values(iterable: Mapping) -> dict: ...
|
||||
def without_none_values(iterable: Mapping[_KT, _VT]) -> Dict[_KT, _VT]: ...
|
||||
|
||||
|
||||
@overload
|
||||
def without_none_values(iterable: Iterable) -> Iterable: ...
|
||||
def without_none_values(iterable: Iterable[_KT]) -> Iterable[_KT]: ...
|
||||
|
||||
|
||||
def without_none_values(iterable: Union[Mapping, Iterable]) -> Union[dict, Iterable]:
|
||||
def without_none_values(
|
||||
iterable: Union[Mapping[_KT, _VT], Iterable[_KT]]
|
||||
) -> Union[Dict[_KT, _VT], Iterable[_KT]]:
|
||||
"""Return a copy of ``iterable`` with all ``None`` entries removed.
|
||||
|
||||
If ``iterable`` is a mapping, return a dictionary where all pairs that have
|
||||
|
|
@ -347,43 +349,45 @@ else:
|
|||
gc.collect()
|
||||
|
||||
|
||||
class MutableChain(Iterable):
|
||||
class MutableChain(Iterable[_T]):
|
||||
"""
|
||||
Thin wrapper around itertools.chain, allowing to add iterables "in-place"
|
||||
"""
|
||||
|
||||
def __init__(self, *args: Iterable):
|
||||
self.data = chain.from_iterable(args)
|
||||
def __init__(self, *args: Iterable[_T]):
|
||||
self.data: Iterator[_T] = chain.from_iterable(args)
|
||||
|
||||
def extend(self, *iterables: Iterable) -> None:
|
||||
def extend(self, *iterables: Iterable[_T]) -> None:
|
||||
self.data = chain(self.data, chain.from_iterable(iterables))
|
||||
|
||||
def __iter__(self) -> Iterator:
|
||||
def __iter__(self) -> Iterator[_T]:
|
||||
return self
|
||||
|
||||
def __next__(self) -> Any:
|
||||
def __next__(self) -> _T:
|
||||
return next(self.data)
|
||||
|
||||
|
||||
async def _async_chain(*iterables: Union[Iterable, AsyncIterable]) -> AsyncGenerator:
|
||||
async def _async_chain(
|
||||
*iterables: Union[Iterable[_T], AsyncIterable[_T]]
|
||||
) -> AsyncIterator[_T]:
|
||||
for it in iterables:
|
||||
async for o in as_async_generator(it):
|
||||
yield o
|
||||
|
||||
|
||||
class MutableAsyncChain(AsyncIterable):
|
||||
class MutableAsyncChain(AsyncIterable[_T]):
|
||||
"""
|
||||
Similar to MutableChain but for async iterables
|
||||
"""
|
||||
|
||||
def __init__(self, *args: Union[Iterable, AsyncIterable]):
|
||||
self.data = _async_chain(*args)
|
||||
def __init__(self, *args: Union[Iterable[_T], AsyncIterable[_T]]):
|
||||
self.data: AsyncIterator[_T] = _async_chain(*args)
|
||||
|
||||
def extend(self, *iterables: Union[Iterable, AsyncIterable]) -> None:
|
||||
def extend(self, *iterables: Union[Iterable[_T], AsyncIterable[_T]]) -> None:
|
||||
self.data = _async_chain(self.data, _async_chain(*iterables))
|
||||
|
||||
def __aiter__(self) -> AsyncIterator:
|
||||
def __aiter__(self) -> AsyncIterator[_T]:
|
||||
return self
|
||||
|
||||
async def __anext__(self) -> Any:
|
||||
async def __anext__(self) -> _T:
|
||||
return await self.data.__anext__()
|
||||
|
|
|
|||
|
|
@ -12,7 +12,6 @@ from typing import (
|
|||
TYPE_CHECKING,
|
||||
Any,
|
||||
Dict,
|
||||
Generator,
|
||||
Iterable,
|
||||
List,
|
||||
Optional,
|
||||
|
|
@ -40,9 +39,7 @@ if TYPE_CHECKING:
|
|||
from scrapy.crawler import Crawler
|
||||
|
||||
|
||||
def _serialize_headers(
|
||||
headers: Iterable[bytes], request: Request
|
||||
) -> Generator[bytes, Any, None]:
|
||||
def _serialize_headers(headers: Iterable[bytes], request: Request) -> Iterable[bytes]:
|
||||
for header in headers:
|
||||
if header in request.headers:
|
||||
yield header
|
||||
|
|
@ -197,7 +194,7 @@ def referer_str(request: Request) -> Optional[str]:
|
|||
return to_unicode(referrer, errors="replace")
|
||||
|
||||
|
||||
def request_from_dict(d: dict, *, spider: Optional[Spider] = None) -> Request:
|
||||
def request_from_dict(d: Dict[str, Any], *, spider: Optional[Spider] = None) -> Request:
|
||||
"""Create a :class:`~scrapy.Request` object from a dict.
|
||||
|
||||
If a spider is given, it will try to resolve the callbacks looking at the
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@ Note: The main purpose of this module is to provide support for the
|
|||
SitemapSpider, its API is subject to change without notice.
|
||||
"""
|
||||
|
||||
from typing import Any, Dict, Generator, Iterator, Optional, Union
|
||||
from typing import Any, Dict, Iterable, Iterator, Optional, Union
|
||||
from urllib.parse import urljoin
|
||||
|
||||
import lxml.etree # nosec
|
||||
|
|
@ -42,7 +42,7 @@ class Sitemap:
|
|||
|
||||
def sitemap_urls_from_robots(
|
||||
robots_text: str, base_url: Optional[str] = None
|
||||
) -> Generator[str, Any, None]:
|
||||
) -> Iterable[str]:
|
||||
"""Return an iterator over all sitemap urls contained in the given
|
||||
robots.txt file
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -7,7 +7,6 @@ from typing import (
|
|||
TYPE_CHECKING,
|
||||
Any,
|
||||
AsyncGenerator,
|
||||
Generator,
|
||||
Iterable,
|
||||
Literal,
|
||||
Optional,
|
||||
|
|
@ -34,18 +33,20 @@ _T = TypeVar("_T")
|
|||
|
||||
# https://stackoverflow.com/questions/60222982
|
||||
@overload
|
||||
def iterate_spider_output(result: AsyncGenerator) -> AsyncGenerator: ... # type: ignore[overload-overlap]
|
||||
def iterate_spider_output(result: AsyncGenerator[_T, None]) -> AsyncGenerator[_T, None]: ... # type: ignore[overload-overlap]
|
||||
|
||||
|
||||
@overload
|
||||
def iterate_spider_output(result: CoroutineType) -> Deferred: ...
|
||||
def iterate_spider_output(result: CoroutineType[Any, Any, _T]) -> Deferred[_T]: ...
|
||||
|
||||
|
||||
@overload
|
||||
def iterate_spider_output(result: _T) -> Iterable: ...
|
||||
def iterate_spider_output(result: _T) -> Iterable[Any]: ...
|
||||
|
||||
|
||||
def iterate_spider_output(result: Any) -> Union[Iterable, AsyncGenerator, Deferred]:
|
||||
def iterate_spider_output(
|
||||
result: Any,
|
||||
) -> Union[Iterable[Any], AsyncGenerator[_T, None], Deferred[_T]]:
|
||||
if inspect.isasyncgen(result):
|
||||
return result
|
||||
if inspect.iscoroutine(result):
|
||||
|
|
@ -55,7 +56,7 @@ def iterate_spider_output(result: Any) -> Union[Iterable, AsyncGenerator, Deferr
|
|||
return arg_to_iter(deferred_from_coro(result))
|
||||
|
||||
|
||||
def iter_spider_classes(module: ModuleType) -> Generator[Type[Spider], Any, None]:
|
||||
def iter_spider_classes(module: ModuleType) -> Iterable[Type[Spider]]:
|
||||
"""Return an iterator over all spider classes defined in the given module
|
||||
that can be instantiated (i.e. which have name)
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@ import os
|
|||
from importlib import import_module
|
||||
from pathlib import Path
|
||||
from posixpath import split
|
||||
from typing import Any, Coroutine, Dict, List, Optional, Tuple, Type
|
||||
from typing import Any, Awaitable, Dict, List, Optional, Tuple, Type, TypeVar
|
||||
from unittest import TestCase, mock
|
||||
|
||||
from twisted.internet.defer import Deferred
|
||||
|
|
@ -17,6 +17,8 @@ from scrapy import Spider
|
|||
from scrapy.crawler import Crawler
|
||||
from scrapy.utils.boto import is_botocore_available
|
||||
|
||||
_T = TypeVar("_T")
|
||||
|
||||
|
||||
def assert_gcs_environ() -> None:
|
||||
if "GCS_PROJECT_ID" not in os.environ:
|
||||
|
|
@ -118,8 +120,8 @@ def assert_samelines(
|
|||
testcase.assertEqual(text1.splitlines(), text2.splitlines(), msg)
|
||||
|
||||
|
||||
def get_from_asyncio_queue(value: Any) -> Coroutine:
|
||||
q: asyncio.Queue = asyncio.Queue()
|
||||
def get_from_asyncio_queue(value: _T) -> Awaitable[_T]:
|
||||
q: asyncio.Queue[_T] = asyncio.Queue()
|
||||
getter = q.get()
|
||||
q.put_nowait(value)
|
||||
return getter
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@ from shutil import copytree, rmtree
|
|||
from stat import S_IWRITE as ANYONE_WRITE_PERMISSION
|
||||
from tempfile import TemporaryFile, mkdtemp
|
||||
from threading import Timer
|
||||
from typing import Dict, Generator, Optional, Union
|
||||
from typing import Dict, Iterator, Optional, Union
|
||||
from unittest import skipIf
|
||||
|
||||
from pytest import mark
|
||||
|
|
@ -674,7 +674,7 @@ class BadSpider(scrapy.Spider):
|
|||
"""
|
||||
|
||||
@contextmanager
|
||||
def _create_file(self, content, name=None) -> Generator[str, None, None]:
|
||||
def _create_file(self, content, name=None) -> Iterator[str]:
|
||||
tmpdir = Path(self.mktemp())
|
||||
tmpdir.mkdir()
|
||||
if name:
|
||||
|
|
|
|||
|
|
@ -182,6 +182,19 @@ class TestSpider(Spider):
|
|||
"""
|
||||
pass
|
||||
|
||||
def invalid_regex(self, response):
|
||||
"""method with invalid regex
|
||||
@ Scrapy is awsome
|
||||
"""
|
||||
pass
|
||||
|
||||
def invalid_regex_with_valid_contract(self, response):
|
||||
"""method with invalid regex
|
||||
@ scrapy is awsome
|
||||
@url http://scrapy.org
|
||||
"""
|
||||
pass
|
||||
|
||||
|
||||
class CustomContractSuccessSpider(Spider):
|
||||
name = "custom_contract_success_spider"
|
||||
|
|
@ -385,6 +398,21 @@ class ContractsManagerTest(unittest.TestCase):
|
|||
message = "ContractFail: Missing fields: name, url"
|
||||
assert message in self.results.failures[-1][-1]
|
||||
|
||||
def test_regex(self):
|
||||
spider = TestSpider()
|
||||
response = ResponseMock()
|
||||
|
||||
# invalid regex
|
||||
request = self.conman.from_method(spider.invalid_regex, self.results)
|
||||
self.should_succeed()
|
||||
|
||||
# invalid regex with valid contract
|
||||
request = self.conman.from_method(
|
||||
spider.invalid_regex_with_valid_contract, self.results
|
||||
)
|
||||
self.should_succeed()
|
||||
request.callback(response)
|
||||
|
||||
def test_custom_contracts(self):
|
||||
self.conman.from_spider(CustomContractSuccessSpider(), self.results)
|
||||
self.should_succeed()
|
||||
|
|
|
|||
|
|
@ -211,10 +211,6 @@ class MockedMediaPipeline(UserDefinedPipeline):
|
|||
class MediaPipelineTestCase(BaseMediaPipelineTestCase):
|
||||
pipeline_class = MockedMediaPipeline
|
||||
|
||||
def _callback(self, result):
|
||||
self.pipe._mockcalled.append("request_callback")
|
||||
return result
|
||||
|
||||
def _errback(self, result):
|
||||
self.pipe._mockcalled.append("request_errback")
|
||||
return result
|
||||
|
|
@ -225,7 +221,6 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase):
|
|||
req = Request(
|
||||
"http://url1",
|
||||
meta={"response": rsp},
|
||||
callback=self._callback,
|
||||
errback=self._errback,
|
||||
)
|
||||
item = {"requests": req}
|
||||
|
|
@ -237,7 +232,6 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase):
|
|||
"get_media_requests",
|
||||
"media_to_download",
|
||||
"media_downloaded",
|
||||
"request_callback",
|
||||
"item_completed",
|
||||
],
|
||||
)
|
||||
|
|
@ -249,7 +243,6 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase):
|
|||
req = Request(
|
||||
"http://url1",
|
||||
meta={"response": fail},
|
||||
callback=self._callback,
|
||||
errback=self._errback,
|
||||
)
|
||||
item = {"requests": req}
|
||||
|
|
|
|||
9
tox.ini
9
tox.ini
|
|
@ -47,18 +47,17 @@ install_command =
|
|||
basepython = python3
|
||||
deps =
|
||||
mypy==1.10.0
|
||||
typing-extensions==4.11.0
|
||||
typing-extensions==4.12.1
|
||||
types-lxml==2024.4.14
|
||||
types-Pygments==2.18.0.20240506
|
||||
types-pyOpenSSL==24.1.0.20240425
|
||||
types-setuptools==69.5.0.20240518
|
||||
types-setuptools==70.0.0.20240524
|
||||
botocore-stubs==1.34.94
|
||||
boto3-stubs[s3]==1.34.108
|
||||
boto3-stubs[s3]==1.34.119
|
||||
attrs >= 18.2.0
|
||||
Pillow >= 10.3.0
|
||||
pytest >= 8.2.0
|
||||
# 2.1.2 fixes a typing bug: https://github.com/scrapy/w3lib/pull/211
|
||||
w3lib >= 2.1.2
|
||||
w3lib >= 2.2.0
|
||||
commands =
|
||||
mypy {posargs: scrapy tests}
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue