mirror of https://github.com/scrapy/scrapy.git
Address typing and linting issues
This commit is contained in:
parent
8613d390d7
commit
15af004382
|
|
@ -641,6 +641,7 @@ class ExecutionEngine:
|
|||
extra={"spider": self.spider},
|
||||
)
|
||||
|
||||
# pylint: disable=too-many-statements
|
||||
async def close_spider_async( # noqa: PLR0912, PLR0915
|
||||
self,
|
||||
*,
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from typing import Literal, cast
|
||||
from typing import Literal
|
||||
|
||||
StopMode = Literal["graceful", "fast", "force"]
|
||||
|
||||
|
|
@ -18,10 +18,9 @@ def normalize_stop_mode(mode: StopMode | None, *, allow_force: bool = True) -> S
|
|||
raise ValueError(
|
||||
f"Unknown stop mode {mode!r}. Expected one of: graceful, fast, force"
|
||||
)
|
||||
normalized = cast("StopMode", mode)
|
||||
if normalized == "force" and not allow_force:
|
||||
if mode == "force" and not allow_force:
|
||||
raise ValueError("The force stop mode is not supported in this context")
|
||||
return normalized
|
||||
return mode
|
||||
|
||||
|
||||
def max_stop_mode(mode1: StopMode, mode2: StopMode) -> StopMode:
|
||||
|
|
|
|||
|
|
@ -31,6 +31,7 @@ from tests.utils.decorators import coroutine_test
|
|||
|
||||
if TYPE_CHECKING:
|
||||
from twisted.internet.ssl import ContextFactory
|
||||
from twisted.python.failure import Failure
|
||||
from twisted.web.iweb import IBodyProducer
|
||||
|
||||
|
||||
|
|
@ -220,7 +221,7 @@ async def test_stop_async_drops_queued_requests() -> None:
|
|||
|
||||
request = Request("https://example.com")
|
||||
queue_dfd: Deferred = Deferred()
|
||||
failures = []
|
||||
failures: list[Failure] = []
|
||||
queue_dfd.addErrback(failures.append)
|
||||
slot.queue.append((request, queue_dfd))
|
||||
|
||||
|
|
|
|||
|
|
@ -794,7 +794,7 @@ async def test_deprecated_crawler_stop() -> None:
|
|||
async def test_crawler_stop_async_invalid_mode() -> None:
|
||||
crawler = get_crawler(DefaultSpider)
|
||||
with pytest.raises(ValueError, match=r"Unknown stop mode"):
|
||||
await crawler.stop_async(mode="invalid")
|
||||
await crawler.stop_async(mode="invalid") # type: ignore[arg-type]
|
||||
|
||||
|
||||
@coroutine_test
|
||||
|
|
@ -809,12 +809,13 @@ async def test_crawler_force_stop_falls_back_to_fast(
|
|||
async def stop_async(self, *, mode: str = "graceful") -> None:
|
||||
self.called_mode = mode
|
||||
|
||||
crawler.engine = DummyEngine() # type: ignore[assignment]
|
||||
dummy_engine = DummyEngine()
|
||||
crawler.engine = dummy_engine # type: ignore[assignment]
|
||||
|
||||
with caplog.at_level(logging.WARNING):
|
||||
await crawler.stop_async(mode="force")
|
||||
|
||||
assert crawler.engine.called_mode == "fast"
|
||||
assert dummy_engine.called_mode == "fast"
|
||||
assert "Falling back to fast stop" in caplog.text
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@ from collections import defaultdict
|
|||
from dataclasses import dataclass
|
||||
from logging import DEBUG
|
||||
from typing import TYPE_CHECKING, Any, cast
|
||||
from unittest.mock import Mock, call
|
||||
from unittest.mock import Mock, call, patch
|
||||
from urllib.parse import urlparse
|
||||
|
||||
import attr
|
||||
|
|
@ -784,9 +784,8 @@ class TestEngineCloseSpider:
|
|||
calls += 1
|
||||
return 3
|
||||
|
||||
engine.downloader.stop_async = fast_stop_downloader
|
||||
|
||||
await engine.close_spider_async(mode="fast")
|
||||
with patch.object(engine.downloader, "stop_async", fast_stop_downloader):
|
||||
await engine.close_spider_async(mode="fast")
|
||||
|
||||
assert calls == 1
|
||||
assert crawler.stats
|
||||
|
|
|
|||
Loading…
Reference in New Issue