mirror of https://github.com/scrapy/scrapy.git
Solve mypy issues
This commit is contained in:
parent
1f619c7411
commit
0ef27ef753
|
|
@ -523,7 +523,7 @@ class FeedExporter:
|
|||
)
|
||||
)
|
||||
|
||||
async def _on_state_loaded(self, state: dict) -> None:
|
||||
async def _on_state_loaded(self, state: dict[str, Any]) -> None:
|
||||
"""Update initial batch-1 slots with the correct resumed batch IDs.
|
||||
|
||||
Called via the spider_state_loaded signal after SpiderState has
|
||||
|
|
@ -533,6 +533,7 @@ class FeedExporter:
|
|||
feed_batch_ids: dict[str, int] = state.get("feed_batch_ids", {})
|
||||
if feed_batch_ids:
|
||||
spider = self.crawler.spider
|
||||
assert spider is not None
|
||||
for slot in self.slots:
|
||||
saved_id = feed_batch_ids.get(slot.uri_template, 0)
|
||||
if saved_id == 0:
|
||||
|
|
@ -558,7 +559,7 @@ class FeedExporter:
|
|||
signals.feed_slots_initialized, slots=self.slots
|
||||
)
|
||||
|
||||
def _on_state_saving(self, state: dict) -> None:
|
||||
def _on_state_saving(self, state: dict[str, Any]) -> None:
|
||||
"""Persist the current batch ID for each feed into spider.state.
|
||||
|
||||
Called via the spider_state_saving signal before SpiderState writes
|
||||
|
|
|
|||
|
|
@ -51,13 +51,15 @@ class SpiderState:
|
|||
self._load_state(spider)
|
||||
assert self.crawler is not None
|
||||
await self.crawler.signals.send_catch_log_async(
|
||||
signals.spider_state_loaded, state=spider.state
|
||||
signals.spider_state_loaded,
|
||||
state=spider.state, # type: ignore[attr-defined]
|
||||
)
|
||||
|
||||
async def _spider_closed(self, spider: Spider) -> None:
|
||||
assert self.crawler is not None
|
||||
await self.crawler.signals.send_catch_log_async(
|
||||
signals.spider_state_saving, state=spider.state
|
||||
signals.spider_state_saving,
|
||||
state=spider.state, # type: ignore[attr-defined]
|
||||
)
|
||||
self._persist_state(spider)
|
||||
|
||||
|
|
@ -71,7 +73,8 @@ class SpiderState:
|
|||
self._load_state(spider)
|
||||
assert self.crawler is not None
|
||||
self.crawler.signals.send_catch_log(
|
||||
signals.spider_state_loaded, state=spider.state
|
||||
signals.spider_state_loaded,
|
||||
state=spider.state, # type: ignore[attr-defined]
|
||||
)
|
||||
|
||||
def spider_closed(self, spider: Spider) -> None:
|
||||
|
|
@ -83,7 +86,8 @@ class SpiderState:
|
|||
)
|
||||
assert self.crawler is not None
|
||||
self.crawler.signals.send_catch_log(
|
||||
signals.spider_state_saving, state=spider.state
|
||||
signals.spider_state_saving,
|
||||
state=spider.state, # type: ignore[attr-defined]
|
||||
)
|
||||
self._persist_state(spider)
|
||||
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timezone
|
||||
from typing import TYPE_CHECKING
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
import pytest
|
||||
|
||||
|
|
@ -60,9 +60,9 @@ async def test_spider_state_loaded_signal_fires(tmp_path: Path) -> None:
|
|||
crawler = get_crawler(Spider, {"JOBDIR": str(tmp_path)})
|
||||
ss = SpiderState.from_crawler(crawler)
|
||||
|
||||
received: list[dict] = []
|
||||
received: list[dict[str, Any]] = []
|
||||
|
||||
def on_loaded(state: dict) -> None:
|
||||
def on_loaded(state: dict[str, Any]) -> None:
|
||||
received.append(dict(state))
|
||||
|
||||
crawler.signals.connect(on_loaded, signal=signals.spider_state_loaded, weak=False)
|
||||
|
|
@ -71,7 +71,7 @@ async def test_spider_state_loaded_signal_fires(tmp_path: Path) -> None:
|
|||
await ss._spider_opened(spider)
|
||||
|
||||
assert received == [{}]
|
||||
assert spider.state == {}
|
||||
assert spider.state == {} # type: ignore[attr-defined]
|
||||
|
||||
|
||||
@coroutine_test
|
||||
|
|
@ -81,11 +81,11 @@ async def test_spider_state_saving_signal_fires(tmp_path: Path) -> None:
|
|||
|
||||
spider = Spider(name="default")
|
||||
await ss._spider_opened(spider)
|
||||
spider.state["key"] = "value"
|
||||
spider.state["key"] = "value" # type: ignore[attr-defined]
|
||||
|
||||
saving_calls: list[dict] = []
|
||||
saving_calls: list[dict[str, Any]] = []
|
||||
|
||||
def on_saving(state: dict) -> None:
|
||||
def on_saving(state: dict[str, Any]) -> None:
|
||||
saving_calls.append(dict(state))
|
||||
|
||||
crawler.signals.connect(on_saving, signal=signals.spider_state_saving, weak=False)
|
||||
|
|
|
|||
Loading…
Reference in New Issue