mirror of https://github.com/scrapy/scrapy.git
Add parameteres to most Deferred instances. (#6414)
This commit is contained in:
Andrey Rakhmatullin
GitHub
parent
d08f559600
commit
e47110f9a5
|
|
@ -153,7 +153,7 @@ class Command(BaseRunSpiderCommand):
|
|||
@overload
|
||||
def iterate_spider_output(self, result: _T) -> Iterable[Any]: ...
|
||||
|
||||
def iterate_spider_output(self, result: Any) -> Union[Iterable[Any], Deferred]:
|
||||
def iterate_spider_output(self, result: Any) -> Union[Iterable[Any], Deferred[Any]]:
|
||||
if inspect.isasyncgen(result):
|
||||
d = deferred_from_coro(
|
||||
collect_asyncgen(aiter_errback(result, self.handle_exception))
|
||||
|
|
@ -233,7 +233,7 @@ class Command(BaseRunSpiderCommand):
|
|||
response: Response,
|
||||
callback: Callable,
|
||||
cb_kwargs: Optional[Dict[str, Any]] = None,
|
||||
) -> Deferred:
|
||||
) -> Deferred[Any]:
|
||||
cb_kwargs = cb_kwargs or {}
|
||||
d = maybeDeferred(self.iterate_spider_output, callback(response, **cb_kwargs))
|
||||
return d
|
||||
|
|
@ -345,7 +345,7 @@ class Command(BaseRunSpiderCommand):
|
|||
def prepare_request(
|
||||
self, spider: Spider, request: Request, opts: argparse.Namespace
|
||||
) -> Request:
|
||||
def callback(response: Response, **cb_kwargs: Any) -> Deferred:
|
||||
def callback(response: Response, **cb_kwargs: Any) -> Deferred[List[Any]]:
|
||||
# memorize first request
|
||||
if not self.first_response:
|
||||
self.first_response = response
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@ from scrapy.utils.python import to_unicode
|
|||
|
||||
if TYPE_CHECKING:
|
||||
from twisted.internet.defer import Deferred
|
||||
from twisted.internet.interfaces import IConnector
|
||||
|
||||
# typing.Self requires Python 3.11
|
||||
from typing_extensions import Self
|
||||
|
|
@ -45,7 +46,7 @@ class HTTP10DownloadHandler:
|
|||
self._connect(factory)
|
||||
return factory.deferred
|
||||
|
||||
def _connect(self, factory: ScrapyHTTPClientFactory) -> Deferred:
|
||||
def _connect(self, factory: ScrapyHTTPClientFactory) -> IConnector:
|
||||
from twisted.internet import reactor
|
||||
|
||||
host, port = to_unicode(factory.host), factory.port
|
||||
|
|
|
|||
|
|
@ -21,7 +21,7 @@ if TYPE_CHECKING:
|
|||
from twisted.internet.base import ReactorBase
|
||||
from twisted.internet.endpoints import HostnameEndpoint
|
||||
|
||||
from scrapy.http.request import Request
|
||||
from scrapy.http import Request, Response
|
||||
from scrapy.settings import Settings
|
||||
from scrapy.spiders import Spider
|
||||
|
||||
|
|
@ -39,16 +39,18 @@ class H2ConnectionPool:
|
|||
self._connections: Dict[ConnectionKeyT, H2ClientProtocol] = {}
|
||||
|
||||
# Save all requests that arrive before the connection is established
|
||||
self._pending_requests: Dict[ConnectionKeyT, Deque[Deferred]] = {}
|
||||
self._pending_requests: Dict[
|
||||
ConnectionKeyT, Deque[Deferred[H2ClientProtocol]]
|
||||
] = {}
|
||||
|
||||
def get_connection(
|
||||
self, key: ConnectionKeyT, uri: URI, endpoint: HostnameEndpoint
|
||||
) -> Deferred:
|
||||
) -> Deferred[H2ClientProtocol]:
|
||||
if key in self._pending_requests:
|
||||
# Received a request while connecting to remote
|
||||
# Create a deferred which will fire with the H2ClientProtocol
|
||||
# instance
|
||||
d: Deferred = Deferred()
|
||||
d: Deferred[H2ClientProtocol] = Deferred()
|
||||
self._pending_requests[key].append(d)
|
||||
return d
|
||||
|
||||
|
|
@ -63,17 +65,17 @@ class H2ConnectionPool:
|
|||
|
||||
def _new_connection(
|
||||
self, key: ConnectionKeyT, uri: URI, endpoint: HostnameEndpoint
|
||||
) -> Deferred:
|
||||
) -> Deferred[H2ClientProtocol]:
|
||||
self._pending_requests[key] = deque()
|
||||
|
||||
conn_lost_deferred: Deferred = Deferred()
|
||||
conn_lost_deferred: Deferred[List[BaseException]] = Deferred()
|
||||
conn_lost_deferred.addCallback(self._remove_connection, key)
|
||||
|
||||
factory = H2ClientFactory(uri, self.settings, conn_lost_deferred)
|
||||
conn_d = endpoint.connect(factory)
|
||||
conn_d.addCallback(self.put_connection, key)
|
||||
|
||||
d: Deferred = Deferred()
|
||||
d: Deferred[H2ClientProtocol] = Deferred()
|
||||
self._pending_requests[key].append(d)
|
||||
return d
|
||||
|
||||
|
|
@ -141,7 +143,7 @@ class H2Agent:
|
|||
"""
|
||||
return uri.scheme, uri.host, uri.port
|
||||
|
||||
def request(self, request: Request, spider: Spider) -> Deferred:
|
||||
def request(self, request: Request, spider: Spider) -> Deferred[Response]:
|
||||
uri = URI.fromBytes(bytes(request.url, encoding="utf-8"))
|
||||
try:
|
||||
endpoint = self.get_endpoint(uri)
|
||||
|
|
@ -149,9 +151,11 @@ class H2Agent:
|
|||
return defer.fail(Failure())
|
||||
|
||||
key = self.get_key(uri)
|
||||
d = self._pool.get_connection(key, uri, endpoint)
|
||||
d.addCallback(lambda conn: conn.request(request, spider))
|
||||
return d
|
||||
d: Deferred[H2ClientProtocol] = self._pool.get_connection(key, uri, endpoint)
|
||||
d2: Deferred[Response] = d.addCallback(
|
||||
lambda conn: conn.request(request, spider)
|
||||
)
|
||||
return d2
|
||||
|
||||
|
||||
class ScrapyProxyH2Agent(H2Agent):
|
||||
|
|
|
|||
|
|
@ -33,7 +33,7 @@ from twisted.protocols.policies import TimeoutMixin
|
|||
from zope.interface import implementer
|
||||
|
||||
from scrapy.core.http2.stream import Stream, StreamCloseReason
|
||||
from scrapy.http import Request
|
||||
from scrapy.http import Request, Response
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from ipaddress import IPv4Address, IPv6Address
|
||||
|
|
@ -88,7 +88,10 @@ class H2ClientProtocol(Protocol, TimeoutMixin):
|
|||
IDLE_TIMEOUT = 240
|
||||
|
||||
def __init__(
|
||||
self, uri: URI, settings: Settings, conn_lost_deferred: Deferred
|
||||
self,
|
||||
uri: URI,
|
||||
settings: Settings,
|
||||
conn_lost_deferred: Deferred[List[BaseException]],
|
||||
) -> None:
|
||||
"""
|
||||
Arguments:
|
||||
|
|
@ -99,7 +102,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin):
|
|||
conn_lost_deferred -- Deferred fires with the reason: Failure to notify
|
||||
that connection was lost
|
||||
"""
|
||||
self._conn_lost_deferred = conn_lost_deferred
|
||||
self._conn_lost_deferred: Deferred[List[BaseException]] = conn_lost_deferred
|
||||
|
||||
config = H2Configuration(client_side=True, header_encoding="utf-8")
|
||||
self.conn = H2Connection(config=config)
|
||||
|
|
@ -215,14 +218,14 @@ class H2ClientProtocol(Protocol, TimeoutMixin):
|
|||
data = self.conn.data_to_send()
|
||||
self.transport.write(data)
|
||||
|
||||
def request(self, request: Request, spider: Spider) -> Deferred:
|
||||
def request(self, request: Request, spider: Spider) -> Deferred[Response]:
|
||||
if not isinstance(request, Request):
|
||||
raise TypeError(
|
||||
f"Expected scrapy.http.Request, received {request.__class__.__qualname__}"
|
||||
)
|
||||
|
||||
stream = self._new_stream(request, spider)
|
||||
d = stream.get_response()
|
||||
d: Deferred[Response] = stream.get_response()
|
||||
|
||||
# Add the stream to the request pool
|
||||
self._pending_request_stream_pool.append(stream)
|
||||
|
|
@ -436,7 +439,10 @@ class H2ClientProtocol(Protocol, TimeoutMixin):
|
|||
@implementer(IProtocolNegotiationFactory)
|
||||
class H2ClientFactory(Factory):
|
||||
def __init__(
|
||||
self, uri: URI, settings: Settings, conn_lost_deferred: Deferred
|
||||
self,
|
||||
uri: URI,
|
||||
settings: Settings,
|
||||
conn_lost_deferred: Deferred[List[BaseException]],
|
||||
) -> None:
|
||||
self.uri = uri
|
||||
self.settings = settings
|
||||
|
|
|
|||
|
|
@ -20,7 +20,7 @@ if TYPE_CHECKING:
|
|||
from hpack import HeaderTuple
|
||||
|
||||
from scrapy.core.http2.protocol import H2ClientProtocol
|
||||
from scrapy.http import Request
|
||||
from scrapy.http import Request, Response
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
|
@ -154,7 +154,7 @@ class Stream:
|
|||
else:
|
||||
self.close(StreamCloseReason.CANCELLED)
|
||||
|
||||
self._deferred_response: Deferred = Deferred(_cancel)
|
||||
self._deferred_response: Deferred[Response] = Deferred(_cancel)
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return f"Stream(id={self.stream_id!r})"
|
||||
|
|
@ -180,7 +180,7 @@ class Stream:
|
|||
and not self.metadata["reached_warnsize"]
|
||||
)
|
||||
|
||||
def get_response(self) -> Deferred:
|
||||
def get_response(self) -> Deferred[Response]:
|
||||
"""Simply return a Deferred which fires when response
|
||||
from the asynchronous request is available
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -13,6 +13,7 @@ from typing import (
|
|||
Generator,
|
||||
Iterable,
|
||||
Iterator,
|
||||
List,
|
||||
Optional,
|
||||
Set,
|
||||
Tuple,
|
||||
|
|
@ -34,7 +35,6 @@ from scrapy.logformatter import LogFormatter
|
|||
from scrapy.pipelines import ItemPipelineManager
|
||||
from scrapy.signalmanager import SignalManager
|
||||
from scrapy.utils.defer import (
|
||||
DeferredListResultListT,
|
||||
aiter_errback,
|
||||
defer_fail,
|
||||
defer_succeed,
|
||||
|
|
@ -54,7 +54,7 @@ logger = logging.getLogger(__name__)
|
|||
|
||||
|
||||
_T = TypeVar("_T")
|
||||
_ParallelResult = DeferredListResultListT[Iterator[Any]]
|
||||
_ParallelResult = List[Tuple[bool, Iterator[Any]]]
|
||||
|
||||
if TYPE_CHECKING:
|
||||
# parameterized Deferreds require Twisted 21.7.0
|
||||
|
|
@ -374,7 +374,7 @@ class Scraper:
|
|||
|
||||
def _itemproc_finished(
|
||||
self, output: Any, item: Any, response: Response, spider: Spider
|
||||
) -> Deferred:
|
||||
) -> Deferred[Any]:
|
||||
"""ItemProcessor finished for the given ``item`` and returned ``output``"""
|
||||
assert self.slot is not None # typing
|
||||
self.slot.itemproc_size -= 1
|
||||
|
|
|
|||
|
|
@ -302,7 +302,10 @@ class SpiderMiddlewareManager(MiddlewareManager):
|
|||
recovered = MutableChain()
|
||||
result = self._evaluate_iterable(response, spider, result, 0, recovered)
|
||||
result = await maybe_deferred_to_future(
|
||||
self._process_spider_output(response, spider, result)
|
||||
cast(
|
||||
"Deferred[Union[Iterable[_T], AsyncIterable[_T]]]",
|
||||
self._process_spider_output(response, spider, result),
|
||||
)
|
||||
)
|
||||
if isinstance(result, AsyncIterable):
|
||||
return MutableAsyncChain(result, recovered)
|
||||
|
|
@ -339,7 +342,7 @@ class SpiderMiddlewareManager(MiddlewareManager):
|
|||
|
||||
def process_start_requests(
|
||||
self, start_requests: Iterable[Request], spider: Spider
|
||||
) -> Deferred:
|
||||
) -> Deferred[Iterable[Request]]:
|
||||
return self._process_chain("process_start_requests", start_requests, spider)
|
||||
|
||||
# This method is only needed until _async compatibility methods are removed.
|
||||
|
|
|
|||
|
|
@ -4,7 +4,18 @@ import logging
|
|||
import pprint
|
||||
import signal
|
||||
import warnings
|
||||
from typing import TYPE_CHECKING, Any, Dict, Generator, Optional, Set, Type, Union, cast
|
||||
from typing import (
|
||||
TYPE_CHECKING,
|
||||
Any,
|
||||
Dict,
|
||||
Generator,
|
||||
Optional,
|
||||
Set,
|
||||
Type,
|
||||
TypeVar,
|
||||
Union,
|
||||
cast,
|
||||
)
|
||||
|
||||
from twisted.internet.defer import (
|
||||
Deferred,
|
||||
|
|
@ -54,6 +65,8 @@ if TYPE_CHECKING:
|
|||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_T = TypeVar("_T")
|
||||
|
||||
|
||||
class Crawler:
|
||||
def __init__(
|
||||
|
|
@ -140,7 +153,7 @@ class Crawler:
|
|||
)
|
||||
|
||||
@inlineCallbacks
|
||||
def crawl(self, *args: Any, **kwargs: Any) -> Generator[Deferred, Any, None]:
|
||||
def crawl(self, *args: Any, **kwargs: Any) -> Generator[Deferred[Any], Any, None]:
|
||||
if self.crawling:
|
||||
raise RuntimeError("Crawling already taking place")
|
||||
if self._started:
|
||||
|
|
@ -172,7 +185,7 @@ class Crawler:
|
|||
return ExecutionEngine(self, lambda _: self.stop())
|
||||
|
||||
@inlineCallbacks
|
||||
def stop(self) -> Generator[Deferred, Any, None]:
|
||||
def stop(self) -> Generator[Deferred[Any], Any, None]:
|
||||
"""Starts a graceful stop of the crawler and returns a deferred that is
|
||||
fired when the crawler is stopped."""
|
||||
if self.crawling:
|
||||
|
|
@ -256,7 +269,7 @@ class CrawlerRunner:
|
|||
self.settings = settings
|
||||
self.spider_loader = self._get_spider_loader(settings)
|
||||
self._crawlers: Set[Crawler] = set()
|
||||
self._active: Set[Deferred] = set()
|
||||
self._active: Set[Deferred[None]] = set()
|
||||
self.bootstrap_failed = False
|
||||
|
||||
def crawl(
|
||||
|
|
@ -264,7 +277,7 @@ class CrawlerRunner:
|
|||
crawler_or_spidercls: Union[Type[Spider], str, Crawler],
|
||||
*args: Any,
|
||||
**kwargs: Any,
|
||||
) -> Deferred:
|
||||
) -> Deferred[None]:
|
||||
"""
|
||||
Run a crawler with the provided arguments.
|
||||
|
||||
|
|
@ -294,12 +307,12 @@ class CrawlerRunner:
|
|||
crawler = self.create_crawler(crawler_or_spidercls)
|
||||
return self._crawl(crawler, *args, **kwargs)
|
||||
|
||||
def _crawl(self, crawler: Crawler, *args: Any, **kwargs: Any) -> Deferred:
|
||||
def _crawl(self, crawler: Crawler, *args: Any, **kwargs: Any) -> Deferred[None]:
|
||||
self.crawlers.add(crawler)
|
||||
d = crawler.crawl(*args, **kwargs)
|
||||
self._active.add(d)
|
||||
|
||||
def _done(result: Any) -> Any:
|
||||
def _done(result: _T) -> _T:
|
||||
self.crawlers.discard(crawler)
|
||||
self._active.discard(d)
|
||||
self.bootstrap_failed |= not getattr(crawler, "spider", None)
|
||||
|
|
@ -335,7 +348,7 @@ class CrawlerRunner:
|
|||
# temporary cast until self.spider_loader is typed
|
||||
return Crawler(cast(Type[Spider], spidercls), self.settings)
|
||||
|
||||
def stop(self) -> Deferred:
|
||||
def stop(self) -> Deferred[Any]:
|
||||
"""
|
||||
Stops simultaneously all the crawling jobs taking place.
|
||||
|
||||
|
|
@ -344,7 +357,7 @@ class CrawlerRunner:
|
|||
return DeferredList([c.stop() for c in list(self.crawlers)])
|
||||
|
||||
@inlineCallbacks
|
||||
def join(self) -> Generator[Deferred, Any, None]:
|
||||
def join(self) -> Generator[Deferred[Any], Any, None]:
|
||||
"""
|
||||
join()
|
||||
|
||||
|
|
@ -460,7 +473,7 @@ class CrawlerProcess(CrawlerRunner):
|
|||
)
|
||||
reactor.run(installSignalHandlers=install_signal_handlers) # blocking call
|
||||
|
||||
def _graceful_stop_reactor(self) -> Deferred:
|
||||
def _graceful_stop_reactor(self) -> Deferred[Any]:
|
||||
d = self.stop()
|
||||
d.addBoth(self._stop_reactor)
|
||||
return d
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@ enable this middleware and enable the ROBOTSTXT_OBEY setting.
|
|||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import TYPE_CHECKING, Any, Dict, Optional, Union
|
||||
from typing import TYPE_CHECKING, Dict, Optional, TypeVar, Union
|
||||
|
||||
from twisted.internet.defer import Deferred, maybeDeferred
|
||||
|
||||
|
|
@ -31,6 +31,8 @@ if TYPE_CHECKING:
|
|||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_T = TypeVar("_T")
|
||||
|
||||
|
||||
class RobotsTxtMiddleware:
|
||||
DOWNLOAD_PRIORITY: int = 1000
|
||||
|
|
@ -43,7 +45,9 @@ class RobotsTxtMiddleware:
|
|||
"ROBOTSTXT_USER_AGENT", None
|
||||
)
|
||||
self.crawler: Crawler = crawler
|
||||
self._parsers: Dict[str, Union[RobotParser, Deferred, None]] = {}
|
||||
self._parsers: Dict[
|
||||
str, Union[RobotParser, Deferred[Optional[RobotParser]], None]
|
||||
] = {}
|
||||
self._parserimpl: RobotParser = load_object(
|
||||
crawler.settings.get("ROBOTSTXT_PARSER")
|
||||
)
|
||||
|
|
@ -55,14 +59,18 @@ class RobotsTxtMiddleware:
|
|||
def from_crawler(cls, crawler: Crawler) -> Self:
|
||||
return cls(crawler)
|
||||
|
||||
def process_request(self, request: Request, spider: Spider) -> Optional[Deferred]:
|
||||
def process_request(
|
||||
self, request: Request, spider: Spider
|
||||
) -> Optional[Deferred[None]]:
|
||||
if request.meta.get("dont_obey_robotstxt"):
|
||||
return None
|
||||
if request.url.startswith("data:") or request.url.startswith("file:"):
|
||||
return None
|
||||
d: Deferred = maybeDeferred(self.robot_parser, request, spider)
|
||||
d.addCallback(self.process_request_2, request, spider)
|
||||
return d
|
||||
d: Deferred[Optional[RobotParser]] = maybeDeferred(
|
||||
self.robot_parser, request, spider # type: ignore[arg-type]
|
||||
)
|
||||
d2: Deferred[None] = d.addCallback(self.process_request_2, request, spider)
|
||||
return d2
|
||||
|
||||
def process_request_2(
|
||||
self, rp: Optional[RobotParser], request: Request, spider: Spider
|
||||
|
|
@ -86,7 +94,7 @@ class RobotsTxtMiddleware:
|
|||
|
||||
def robot_parser(
|
||||
self, request: Request, spider: Spider
|
||||
) -> Union[RobotParser, Deferred, None]:
|
||||
) -> Union[RobotParser, Deferred[Optional[RobotParser]], None]:
|
||||
url = urlparse_cached(request)
|
||||
netloc = url.netloc
|
||||
|
||||
|
|
@ -109,9 +117,9 @@ class RobotsTxtMiddleware:
|
|||
|
||||
parser = self._parsers[netloc]
|
||||
if isinstance(parser, Deferred):
|
||||
d: Deferred = Deferred()
|
||||
d: Deferred[Optional[RobotParser]] = Deferred()
|
||||
|
||||
def cb(result: Any) -> Any:
|
||||
def cb(result: Optional[RobotParser]) -> Optional[RobotParser]:
|
||||
d.callback(result)
|
||||
return result
|
||||
|
||||
|
|
|
|||
|
|
@ -31,10 +31,10 @@ class BaseDupeFilter:
|
|||
def request_seen(self, request: Request) -> bool:
|
||||
return False
|
||||
|
||||
def open(self) -> Optional[Deferred]:
|
||||
def open(self) -> Optional[Deferred[None]]:
|
||||
pass
|
||||
|
||||
def close(self, reason: str) -> Optional[Deferred]:
|
||||
def close(self, reason: str) -> Optional[Deferred[None]]:
|
||||
pass
|
||||
|
||||
def log(self, request: Request, spider: Spider) -> None:
|
||||
|
|
|
|||
|
|
@ -149,7 +149,7 @@ class FeedStorageProtocol(Protocol):
|
|||
"""Open the storage for the given spider. It must return a file-like
|
||||
object that will be used for the exporters"""
|
||||
|
||||
def store(self, file: IO[bytes]) -> Optional[Deferred]:
|
||||
def store(self, file: IO[bytes]) -> Optional[Deferred[None]]:
|
||||
"""Store the given file stream"""
|
||||
|
||||
|
||||
|
|
@ -162,7 +162,7 @@ class BlockingFeedStorage:
|
|||
|
||||
return NamedTemporaryFile(prefix="feed-", dir=path)
|
||||
|
||||
def store(self, file: IO[bytes]) -> Optional[Deferred]:
|
||||
def store(self, file: IO[bytes]) -> Optional[Deferred[None]]:
|
||||
return deferToThread(self._store_in_thread, file)
|
||||
|
||||
def _store_in_thread(self, file: IO[bytes]) -> None:
|
||||
|
|
@ -192,7 +192,7 @@ class StdoutFeedStorage:
|
|||
def open(self, spider: Spider) -> IO[bytes]:
|
||||
return self._stdout
|
||||
|
||||
def store(self, file: IO[bytes]) -> Optional[Deferred]:
|
||||
def store(self, file: IO[bytes]) -> Optional[Deferred[None]]:
|
||||
pass
|
||||
|
||||
|
||||
|
|
@ -211,7 +211,7 @@ class FileFeedStorage:
|
|||
dirname.mkdir(parents=True)
|
||||
return Path(self.path).open(self.write_mode)
|
||||
|
||||
def store(self, file: IO[bytes]) -> Optional[Deferred]:
|
||||
def store(self, file: IO[bytes]) -> Optional[Deferred[None]]:
|
||||
file.close()
|
||||
return None
|
||||
|
||||
|
|
@ -483,7 +483,7 @@ _FeedSlot = create_deprecated_class(
|
|||
|
||||
|
||||
class FeedExporter:
|
||||
_pending_deferreds: List[Deferred] = []
|
||||
_pending_deferreds: List[Deferred[None]] = []
|
||||
|
||||
@classmethod
|
||||
def from_crawler(cls, crawler: Crawler) -> Self:
|
||||
|
|
@ -570,7 +570,7 @@ class FeedExporter:
|
|||
self.crawler.signals.send_catch_log_deferred(signals.feed_exporter_closed)
|
||||
)
|
||||
|
||||
def _close_slot(self, slot: FeedSlot, spider: Spider) -> Optional[Deferred]:
|
||||
def _close_slot(self, slot: FeedSlot, spider: Spider) -> Optional[Deferred[None]]:
|
||||
def get_file(slot_: FeedSlot) -> IO[bytes]:
|
||||
assert slot_.file
|
||||
if isinstance(slot_.file, PostProcessingManager):
|
||||
|
|
@ -590,7 +590,7 @@ class FeedExporter:
|
|||
return None
|
||||
|
||||
logmsg = f"{slot.format} feed ({slot.itemcount} items) in: {slot.uri}"
|
||||
d: Deferred = maybeDeferred(slot.storage.store, get_file(slot))
|
||||
d: Deferred[None] = maybeDeferred(slot.storage.store, get_file(slot)) # type: ignore[arg-type]
|
||||
|
||||
d.addCallback(
|
||||
self._handle_store_success, logmsg, spider, type(slot.storage).__name__
|
||||
|
|
@ -621,7 +621,7 @@ class FeedExporter:
|
|||
self.crawler.stats.inc_value(f"feedexport/failed_count/{slot_type}")
|
||||
|
||||
def _handle_store_success(
|
||||
self, f: Failure, logmsg: str, spider: Spider, slot_type: str
|
||||
self, result: Any, logmsg: str, spider: Spider, slot_type: str
|
||||
) -> None:
|
||||
logger.info("Stored %s", logmsg, extra={"spider": spider})
|
||||
assert self.crawler.stats
|
||||
|
|
|
|||
|
|
@ -39,7 +39,7 @@ class StatsMailer:
|
|||
crawler.signals.connect(o.spider_closed, signal=signals.spider_closed)
|
||||
return o
|
||||
|
||||
def spider_closed(self, spider: Spider) -> Optional[Deferred]:
|
||||
def spider_closed(self, spider: Spider) -> Optional[Deferred[None]]:
|
||||
spider_stats = self.stats.get_stats(spider)
|
||||
body = "Global stats\n\n"
|
||||
body += "\n".join(f"{k:<50} : {v}" for k, v in self.stats.get_stats().items())
|
||||
|
|
|
|||
|
|
@ -103,7 +103,7 @@ class MailSender:
|
|||
mimetype: str = "text/plain",
|
||||
charset: Optional[str] = None,
|
||||
_callback: Optional[Callable[..., None]] = None,
|
||||
) -> Optional[Deferred]:
|
||||
) -> Optional[Deferred[None]]:
|
||||
from twisted.internet import reactor
|
||||
|
||||
msg: MIMEBase
|
||||
|
|
@ -155,7 +155,9 @@ class MailSender:
|
|||
)
|
||||
return None
|
||||
|
||||
dfd = self._sendmail(rcpts, msg.as_string().encode(charset or "utf-8"))
|
||||
dfd: Deferred[Any] = self._sendmail(
|
||||
rcpts, msg.as_string().encode(charset or "utf-8")
|
||||
)
|
||||
dfd.addCallback(self._sent_ok, to, cc, subject, len(attachs))
|
||||
dfd.addErrback(self._sent_failed, to, cc, subject, len(attachs))
|
||||
reactor.addSystemEventTrigger("before", "shutdown", lambda: dfd)
|
||||
|
|
@ -198,11 +200,11 @@ class MailSender:
|
|||
)
|
||||
return failure
|
||||
|
||||
def _sendmail(self, to_addrs: List[str], msg: bytes) -> Deferred:
|
||||
def _sendmail(self, to_addrs: List[str], msg: bytes) -> Deferred[Any]:
|
||||
from twisted.internet import reactor
|
||||
|
||||
msg_io = BytesIO(msg)
|
||||
d: Deferred = Deferred()
|
||||
d: Deferred[Any] = Deferred()
|
||||
|
||||
factory = self._create_sender_factory(to_addrs, msg_io, d)
|
||||
|
||||
|
|
@ -216,7 +218,7 @@ class MailSender:
|
|||
return d
|
||||
|
||||
def _create_sender_factory(
|
||||
self, to_addrs: List[str], msg: IO[bytes], d: Deferred
|
||||
self, to_addrs: List[str], msg: IO[bytes], d: Deferred[Any]
|
||||
) -> ESMTPSenderFactory:
|
||||
from twisted.mail.smtp import ESMTPSenderFactory
|
||||
|
||||
|
|
|
|||
|
|
@ -13,6 +13,7 @@ from typing import (
|
|||
List,
|
||||
Optional,
|
||||
Tuple,
|
||||
TypeVar,
|
||||
Union,
|
||||
cast,
|
||||
)
|
||||
|
|
@ -24,16 +25,22 @@ from scrapy.utils.misc import build_from_crawler, build_from_settings, load_obje
|
|||
if TYPE_CHECKING:
|
||||
from twisted.internet.defer import Deferred
|
||||
|
||||
# typing.Concatenate and typing.ParamSpec require Python 3.10
|
||||
# typing.Self requires Python 3.11
|
||||
from typing_extensions import Self
|
||||
from typing_extensions import Concatenate, ParamSpec, Self
|
||||
|
||||
from scrapy import Spider
|
||||
from scrapy.crawler import Crawler
|
||||
from scrapy.settings import Settings
|
||||
|
||||
_P = ParamSpec("_P")
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_T = TypeVar("_T")
|
||||
_T2 = TypeVar("_T2")
|
||||
|
||||
|
||||
class MiddlewareManager:
|
||||
"""Base class for implementing middleware managers"""
|
||||
|
|
@ -98,16 +105,22 @@ class MiddlewareManager:
|
|||
if hasattr(mw, "close_spider"):
|
||||
self.methods["close_spider"].appendleft(mw.close_spider)
|
||||
|
||||
def _process_parallel(self, methodname: str, obj: Any, *args: Any) -> Deferred:
|
||||
methods = cast(Iterable[Callable], self.methods[methodname])
|
||||
def _process_parallel(
|
||||
self, methodname: str, obj: _T, *args: Any
|
||||
) -> Deferred[List[_T2]]:
|
||||
methods = cast(
|
||||
"Iterable[Callable[Concatenate[_T, _P], _T2]]", self.methods[methodname]
|
||||
)
|
||||
return process_parallel(methods, obj, *args)
|
||||
|
||||
def _process_chain(self, methodname: str, obj: Any, *args: Any) -> Deferred:
|
||||
methods = cast(Iterable[Callable], self.methods[methodname])
|
||||
def _process_chain(self, methodname: str, obj: _T, *args: Any) -> Deferred[_T]:
|
||||
methods = cast(
|
||||
"Iterable[Callable[Concatenate[_T, _P], _T]]", self.methods[methodname]
|
||||
)
|
||||
return process_chain(methods, obj, *args)
|
||||
|
||||
def open_spider(self, spider: Spider) -> Deferred:
|
||||
def open_spider(self, spider: Spider) -> Deferred[List[None]]:
|
||||
return self._process_parallel("open_spider", spider)
|
||||
|
||||
def close_spider(self, spider: Spider) -> Deferred:
|
||||
def close_spider(self, spider: Spider) -> Deferred[List[None]]:
|
||||
return self._process_parallel("close_spider", spider)
|
||||
|
|
|
|||
|
|
@ -4,6 +4,8 @@ See documentation in docs/topics/shell.rst
|
|||
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import signal
|
||||
from typing import Any, Callable, Dict, Optional, Tuple, Union
|
||||
|
|
@ -92,7 +94,9 @@ class Shell:
|
|||
self.vars, shells=shells, banner=self.vars.pop("banner", "")
|
||||
)
|
||||
|
||||
def _schedule(self, request: Request, spider: Optional[Spider]) -> defer.Deferred:
|
||||
def _schedule(
|
||||
self, request: Request, spider: Optional[Spider]
|
||||
) -> defer.Deferred[Any]:
|
||||
if is_asyncio_reactor_installed():
|
||||
# set the asyncio event loop for the current thread
|
||||
event_loop_path = self.crawler.settings["ASYNCIO_EVENT_LOOP"]
|
||||
|
|
@ -209,7 +213,7 @@ def inspect_response(response: Response, spider: Spider) -> None:
|
|||
signal.signal(signal.SIGINT, sigint_handler)
|
||||
|
||||
|
||||
def _request_deferred(request: Request) -> defer.Deferred:
|
||||
def _request_deferred(request: Request) -> defer.Deferred[Any]:
|
||||
"""Wrap a request inside a Deferred.
|
||||
|
||||
This function is harmful, do not use it until you know what you are doing.
|
||||
|
|
@ -228,7 +232,7 @@ def _request_deferred(request: Request) -> defer.Deferred:
|
|||
request.errback = request_errback
|
||||
return result
|
||||
|
||||
d: defer.Deferred = defer.Deferred()
|
||||
d: defer.Deferred[Any] = defer.Deferred()
|
||||
d.addBoth(_restore_callbacks)
|
||||
if request.callback:
|
||||
d.addCallback(request.callback)
|
||||
|
|
|
|||
|
|
@ -50,7 +50,9 @@ class SignalManager:
|
|||
kwargs.setdefault("sender", self.sender)
|
||||
return _signal.send_catch_log(signal, **kwargs)
|
||||
|
||||
def send_catch_log_deferred(self, signal: Any, **kwargs: Any) -> Deferred:
|
||||
def send_catch_log_deferred(
|
||||
self, signal: Any, **kwargs: Any
|
||||
) -> Deferred[List[Tuple[Any, Any]]]:
|
||||
"""
|
||||
Like :meth:`send_catch_log` but supports returning
|
||||
:class:`~twisted.internet.defer.Deferred` objects from signal handlers.
|
||||
|
|
|
|||
|
|
@ -7,9 +7,7 @@ See documentation in docs/topics/spiders.rst
|
|||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import TYPE_CHECKING, Any, Dict, Iterable, List, Optional, Union, cast
|
||||
|
||||
from twisted.internet.defer import Deferred
|
||||
from typing import TYPE_CHECKING, Any, Dict, Iterable, List, Optional, cast
|
||||
|
||||
from scrapy import signals
|
||||
from scrapy.http import Request, Response
|
||||
|
|
@ -19,6 +17,8 @@ from scrapy.utils.url import url_is_from_spider
|
|||
if TYPE_CHECKING:
|
||||
from collections.abc import Callable
|
||||
|
||||
from twisted.internet.defer import Deferred
|
||||
|
||||
# typing.Concatenate requires Python 3.10
|
||||
# typing.Self requires Python 3.11
|
||||
from typing_extensions import Concatenate, Self
|
||||
|
|
@ -105,10 +105,10 @@ class Spider(object_ref):
|
|||
return url_is_from_spider(request.url, cls)
|
||||
|
||||
@staticmethod
|
||||
def close(spider: Spider, reason: str) -> Union[Deferred, None]:
|
||||
def close(spider: Spider, reason: str) -> Optional[Deferred[None]]:
|
||||
closed = getattr(spider, "closed", None)
|
||||
if callable(closed):
|
||||
return cast(Union[Deferred, None], closed(reason))
|
||||
return cast("Optional[Deferred[None]]", closed(reason))
|
||||
return None
|
||||
|
||||
def __repr__(self) -> str:
|
||||
|
|
|
|||
|
|
@ -49,13 +49,8 @@ if TYPE_CHECKING:
|
|||
_T = TypeVar("_T")
|
||||
_T2 = TypeVar("_T2")
|
||||
|
||||
# copied from twisted.internet.defer
|
||||
_SelfResultT = TypeVar("_SelfResultT")
|
||||
_DeferredListResultItemT = Tuple[bool, _SelfResultT]
|
||||
DeferredListResultListT = List[_DeferredListResultItemT[_SelfResultT]]
|
||||
|
||||
|
||||
def defer_fail(_failure: Failure) -> Deferred:
|
||||
def defer_fail(_failure: Failure) -> Deferred[Any]:
|
||||
"""Same as twisted.internet.defer.fail but delay calling errback until
|
||||
next reactor loop
|
||||
|
||||
|
|
@ -64,7 +59,7 @@ def defer_fail(_failure: Failure) -> Deferred:
|
|||
"""
|
||||
from twisted.internet import reactor
|
||||
|
||||
d: Deferred = Deferred()
|
||||
d: Deferred[Any] = Deferred()
|
||||
reactor.callLater(0.1, d.errback, _failure)
|
||||
return d
|
||||
|
||||
|
|
@ -78,12 +73,12 @@ def defer_succeed(result: _T) -> Deferred[_T]:
|
|||
"""
|
||||
from twisted.internet import reactor
|
||||
|
||||
d: Deferred = Deferred()
|
||||
d: Deferred[_T] = Deferred()
|
||||
reactor.callLater(0.1, d.callback, result)
|
||||
return d
|
||||
|
||||
|
||||
def defer_result(result: Any) -> Deferred:
|
||||
def defer_result(result: Any) -> Deferred[Any]:
|
||||
if isinstance(result, Deferred):
|
||||
return result
|
||||
if isinstance(result, failure.Failure):
|
||||
|
|
@ -138,14 +133,14 @@ def parallel(
|
|||
callable: Callable[Concatenate[_T, _P], _T2],
|
||||
*args: _P.args,
|
||||
**named: _P.kwargs,
|
||||
) -> Deferred[DeferredListResultListT[Iterator[_T2]]]:
|
||||
) -> Deferred[List[Tuple[bool, Iterator[_T2]]]]:
|
||||
"""Execute a callable over the objects in the given iterable, in parallel,
|
||||
using no more than ``count`` concurrent calls.
|
||||
|
||||
Taken from: https://jcalderone.livejournal.com/24285.html
|
||||
"""
|
||||
coop = Cooperator()
|
||||
work = (callable(elem, *args, **named) for elem in iterable)
|
||||
work: Iterator[_T2] = (callable(elem, *args, **named) for elem in iterable)
|
||||
return DeferredList([coop.coiterate(work) for _ in range(count)])
|
||||
|
||||
|
||||
|
|
@ -198,16 +193,16 @@ class _AsyncCooperatorAdapter(Iterator[Deferred]):
|
|||
def __init__(
|
||||
self,
|
||||
aiterable: AsyncIterable[_T],
|
||||
callable: Callable[Concatenate[_T, _P], _T2],
|
||||
callable: Callable[Concatenate[_T, _P], Optional[Deferred[Any]]],
|
||||
*callable_args: _P.args,
|
||||
**callable_kwargs: _P.kwargs,
|
||||
):
|
||||
self.aiterator: AsyncIterator[_T] = aiterable.__aiter__()
|
||||
self.callable: Callable[Concatenate[_T, _P], _T2] = callable
|
||||
self.callable: Callable[Concatenate[_T, _P], Optional[Deferred[Any]]] = callable
|
||||
self.callable_args: Tuple[Any, ...] = callable_args
|
||||
self.callable_kwargs: Dict[str, Any] = callable_kwargs
|
||||
self.finished: bool = False
|
||||
self.waiting_deferreds: List[Deferred] = []
|
||||
self.waiting_deferreds: List[Deferred[Any]] = []
|
||||
self.anext_deferred: Optional[Deferred[_T]] = None
|
||||
|
||||
def _callback(self, result: _T) -> None:
|
||||
|
|
@ -241,12 +236,12 @@ class _AsyncCooperatorAdapter(Iterator[Deferred]):
|
|||
self.anext_deferred = deferred_from_coro(self.aiterator.__anext__())
|
||||
self.anext_deferred.addCallbacks(self._callback, self._errback)
|
||||
|
||||
def __next__(self) -> Deferred:
|
||||
def __next__(self) -> Deferred[Any]:
|
||||
# This puts a new Deferred into self.waiting_deferreds and returns it.
|
||||
# It also calls __anext__() if needed.
|
||||
if self.finished:
|
||||
raise StopIteration
|
||||
d: Deferred = Deferred()
|
||||
d: Deferred[Any] = Deferred()
|
||||
self.waiting_deferreds.append(d)
|
||||
if not self.anext_deferred:
|
||||
self._call_anext()
|
||||
|
|
@ -256,25 +251,29 @@ class _AsyncCooperatorAdapter(Iterator[Deferred]):
|
|||
def parallel_async(
|
||||
async_iterable: AsyncIterable[_T],
|
||||
count: int,
|
||||
callable: Callable[Concatenate[_T, _P], _T2],
|
||||
callable: Callable[Concatenate[_T, _P], Optional[Deferred[Any]]],
|
||||
*args: _P.args,
|
||||
**named: _P.kwargs,
|
||||
) -> Deferred[DeferredListResultListT[Iterator[_T2]]]:
|
||||
"""Like parallel but for async iterators"""
|
||||
) -> Deferred[List[Tuple[bool, Iterator[Deferred[Any]]]]]:
|
||||
"""Like ``parallel`` but for async iterators"""
|
||||
coop = Cooperator()
|
||||
work = _AsyncCooperatorAdapter(async_iterable, callable, *args, **named)
|
||||
dl: Deferred = DeferredList([coop.coiterate(work) for _ in range(count)])
|
||||
work: Iterator[Deferred[Any]] = _AsyncCooperatorAdapter(
|
||||
async_iterable, callable, *args, **named
|
||||
)
|
||||
dl: Deferred[List[Tuple[bool, Iterator[Deferred[Any]]]]] = DeferredList(
|
||||
[coop.coiterate(work) for _ in range(count)]
|
||||
)
|
||||
return dl
|
||||
|
||||
|
||||
def process_chain(
|
||||
callbacks: Iterable[Callable[Concatenate[_T, _P], Any]],
|
||||
input: Any,
|
||||
callbacks: Iterable[Callable[Concatenate[_T, _P], _T]],
|
||||
input: _T,
|
||||
*a: _P.args,
|
||||
**kw: _P.kwargs,
|
||||
) -> Deferred:
|
||||
) -> Deferred[_T]:
|
||||
"""Return a Deferred built by chaining the given callbacks"""
|
||||
d: Deferred = Deferred()
|
||||
d: Deferred[_T] = Deferred()
|
||||
for x in callbacks:
|
||||
d.addCallback(x, *a, **kw)
|
||||
d.callback(input)
|
||||
|
|
@ -307,19 +306,21 @@ def process_chain_both(
|
|||
|
||||
|
||||
def process_parallel(
|
||||
callbacks: Iterable[Callable[Concatenate[_T, _P], Any]],
|
||||
input: Any,
|
||||
callbacks: Iterable[Callable[Concatenate[_T, _P], _T2]],
|
||||
input: _T,
|
||||
*a: _P.args,
|
||||
**kw: _P.kwargs,
|
||||
) -> Deferred:
|
||||
) -> Deferred[List[_T2]]:
|
||||
"""Return a Deferred with the output of all successful calls to the given
|
||||
callbacks
|
||||
"""
|
||||
dfds = [defer.succeed(input).addCallback(x, *a, **kw) for x in callbacks]
|
||||
d: Deferred = DeferredList(dfds, fireOnOneErrback=True, consumeErrors=True)
|
||||
d.addCallback(lambda r: [x[1] for x in r])
|
||||
d.addErrback(lambda f: f.value.subFailure)
|
||||
return d
|
||||
d: Deferred[List[Tuple[bool, _T2]]] = DeferredList(
|
||||
dfds, fireOnOneErrback=True, consumeErrors=True
|
||||
)
|
||||
d2: Deferred[List[_T2]] = d.addCallback(lambda r: [x[1] for x in r])
|
||||
d2.addErrback(lambda f: f.value.subFailure)
|
||||
return d2
|
||||
|
||||
|
||||
def iter_errback(
|
||||
|
|
@ -404,7 +405,7 @@ def deferred_f_from_coro_f(
|
|||
|
||||
def maybeDeferred_coro(
|
||||
f: Callable[_P, Any], *args: _P.args, **kw: _P.kwargs
|
||||
) -> Deferred:
|
||||
) -> Deferred[Any]:
|
||||
"""Copy of defer.maybeDeferred that also converts coroutines to Deferreds."""
|
||||
try:
|
||||
result = f(*args, **kw)
|
||||
|
|
@ -420,7 +421,7 @@ def maybeDeferred_coro(
|
|||
return defer.succeed(result)
|
||||
|
||||
|
||||
def deferred_to_future(d: Deferred) -> Future:
|
||||
def deferred_to_future(d: Deferred[_T]) -> Future[_T]:
|
||||
"""
|
||||
.. versionadded:: 2.6.0
|
||||
|
||||
|
|
@ -442,7 +443,7 @@ def deferred_to_future(d: Deferred) -> Future:
|
|||
return d.asFuture(_get_asyncio_event_loop())
|
||||
|
||||
|
||||
def maybe_deferred_to_future(d: Deferred) -> Union[Deferred, Future]:
|
||||
def maybe_deferred_to_future(d: Deferred[_T]) -> Union[Deferred[_T], Future[_T]]:
|
||||
"""
|
||||
.. versionadded:: 2.6.0
|
||||
|
||||
|
|
|
|||
|
|
@ -1,5 +1,7 @@
|
|||
"""Helper functions for working with signals"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import collections.abc
|
||||
import logging
|
||||
from typing import Any as TypingAny
|
||||
|
|
@ -27,7 +29,7 @@ def send_catch_log(
|
|||
signal: TypingAny = Any,
|
||||
sender: TypingAny = Anonymous,
|
||||
*arguments: TypingAny,
|
||||
**named: TypingAny
|
||||
**named: TypingAny,
|
||||
) -> List[Tuple[TypingAny, TypingAny]]:
|
||||
"""Like pydispatcher.robust.sendRobust but it also logs errors and returns
|
||||
Failures instead of exceptions.
|
||||
|
|
@ -73,8 +75,8 @@ def send_catch_log_deferred(
|
|||
signal: TypingAny = Any,
|
||||
sender: TypingAny = Anonymous,
|
||||
*arguments: TypingAny,
|
||||
**named: TypingAny
|
||||
) -> Deferred:
|
||||
**named: TypingAny,
|
||||
) -> Deferred[List[Tuple[TypingAny, TypingAny]]]:
|
||||
"""Like send_catch_log but supports returning deferreds on signal handlers.
|
||||
Returns a deferred that gets fired once all signal handlers deferreds were
|
||||
fired.
|
||||
|
|
@ -92,23 +94,25 @@ def send_catch_log_deferred(
|
|||
|
||||
dont_log = named.pop("dont_log", None)
|
||||
spider = named.get("spider", None)
|
||||
dfds = []
|
||||
dfds: List[Deferred[Tuple[TypingAny, TypingAny]]] = []
|
||||
for receiver in liveReceivers(getAllReceivers(sender, signal)):
|
||||
d = maybeDeferred_coro(
|
||||
d: Deferred[TypingAny] = maybeDeferred_coro(
|
||||
robustApply, receiver, signal=signal, sender=sender, *arguments, **named
|
||||
)
|
||||
d.addErrback(logerror, receiver)
|
||||
# TODO https://pylint.readthedocs.io/en/latest/user_guide/messages/warning/cell-var-from-loop.html
|
||||
d.addBoth(
|
||||
d2: Deferred[Tuple[TypingAny, TypingAny]] = d.addBoth(
|
||||
lambda result: (
|
||||
receiver, # pylint: disable=cell-var-from-loop # noqa: B023
|
||||
result,
|
||||
)
|
||||
)
|
||||
dfds.append(d)
|
||||
d = DeferredList(dfds)
|
||||
d.addCallback(lambda out: [x[1] for x in out])
|
||||
return d
|
||||
dfds.append(d2)
|
||||
dl = DeferredList(dfds)
|
||||
d3: Deferred[List[Tuple[TypingAny, TypingAny]]] = dl.addCallback(
|
||||
lambda out: [x[1] for x in out]
|
||||
)
|
||||
return d3
|
||||
|
||||
|
||||
def disconnect_all(signal: TypingAny = Any, sender: TypingAny = Any) -> None:
|
||||
|
|
|
|||
|
|
@ -30,6 +30,7 @@ from scrapy.utils.boto import is_botocore_available
|
|||
|
||||
if TYPE_CHECKING:
|
||||
from twisted.internet.defer import Deferred
|
||||
from twisted.web.client import Response as TxResponse
|
||||
|
||||
|
||||
_T = TypeVar("_T")
|
||||
|
|
@ -159,7 +160,7 @@ def mock_google_cloud_storage() -> Tuple[Any, Any, Any]:
|
|||
return (client_mock, bucket_mock, blob_mock)
|
||||
|
||||
|
||||
def get_web_client_agent_req(url: str) -> Deferred:
|
||||
def get_web_client_agent_req(url: str) -> Deferred[TxResponse]:
|
||||
from twisted.internet import reactor
|
||||
from twisted.web.client import Agent # imports twisted.internet.reactor
|
||||
|
||||
|
|
|
|||
|
|
@ -22,7 +22,7 @@ class ProcessTest:
|
|||
args: Iterable[str],
|
||||
check_code: bool = True,
|
||||
settings: Optional[str] = None,
|
||||
) -> Deferred:
|
||||
) -> Deferred[TestProcessProtocol]:
|
||||
from twisted.internet import reactor
|
||||
|
||||
env = os.environ.copy()
|
||||
|
|
@ -49,7 +49,7 @@ class ProcessTest:
|
|||
|
||||
class TestProcessProtocol(ProcessProtocol):
|
||||
def __init__(self) -> None:
|
||||
self.deferred: Deferred = Deferred()
|
||||
self.deferred: Deferred[TestProcessProtocol] = Deferred()
|
||||
self.out: bytes = b""
|
||||
self.err: bytes = b""
|
||||
self.exitcode: Optional[int] = None
|
||||
|
|
|
|||
Loading…
Reference in New Issue