diff --git a/pyproject.toml b/pyproject.toml index 0bdcf6b51..a3c34a035 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -116,13 +116,9 @@ allow_incomplete_defs = true # 59 errors # TODO [[tool.mypy.overrides]] module = [ - "tests.spiders", - "tests.test_closespider", "tests.test_cmdline", - "tests.test_contracts", "tests.test_downloaderslotssettings", "tests.test_dupefilters", - "tests.test_engine_loop", "tests.test_extension_statsmailer", "tests.test_extension_throttle", "tests.test_linkextractors", @@ -131,15 +127,9 @@ module = [ "tests.test_pqueues", "tests.test_scheduler_base", "tests.test_settings", - "tests.test_spider", - "tests.test_spider_crawl", - "tests.test_spidermiddleware_output_chain", - "tests.test_spidermiddleware_process_start", - "tests.test_spider_sitemap", "tests.test_squeues", "tests.test_squeues_request", "tests.test_stats", - "tests.utils.bases.spider", ] check_untyped_defs = false diff --git a/tests/benchmarks/__init__.py b/tests/benchmarks/__init__.py index a066ed00a..7504ddf18 100644 --- a/tests/benchmarks/__init__.py +++ b/tests/benchmarks/__init__.py @@ -36,7 +36,6 @@ class NullDownloadHandler: async def download_request(self, request: Request) -> Response: self._active += 1 - assert self._crawler.stats self._crawler.stats.max_value("benchmark/peak_concurrency", self._active) try: await asyncio.sleep(0) diff --git a/tests/benchmarks/test_crawl.py b/tests/benchmarks/test_crawl.py index f79b66a5b..4f68e4f6f 100644 --- a/tests/benchmarks/test_crawl.py +++ b/tests/benchmarks/test_crawl.py @@ -128,7 +128,6 @@ class _DelayedPipeline: async def process_item(self, item: Any) -> Any: url = item["url"] self._active[url] += 1 - assert self._crawler.stats self._crawler.stats.max_value("benchmark/peak_items", self._active[url]) try: await asyncio.sleep(DELAY) @@ -147,7 +146,6 @@ def _crawl_tree( pages=pages, items=items, ) - assert crawler.stats assert crawler.stats.get_value("downloader/response_count") == domains * pages assert crawler.stats.get_value("item_scraped_count", 0) == domains * pages * items return crawler @@ -166,7 +164,6 @@ def test_overhead_http(benchmark: BenchmarkFixture, mockserver: MockServer) -> N def run() -> None: crawler = crawl(_FollowSpider, settings, url=url) - assert crawler.stats assert crawler.stats.get_value("item_scraped_count") == PAGES + 1 benchmark(run) @@ -177,7 +174,6 @@ def test_overhead_engine(benchmark: BenchmarkFixture) -> None: def run() -> None: crawler = _crawl_tree({}, domains=1, pages=REQUESTS) - assert crawler.stats assert crawler.stats.get_value("benchmark/peak_concurrency") > 1 benchmark(run) @@ -262,7 +258,6 @@ def test_overhead_item_concurrency(benchmark: BenchmarkFixture) -> None: crawler = _crawl_tree( settings, domains=1, pages=ITEM_REQUESTS, items=ITEMS_PER_RESPONSE ) - assert crawler.stats assert ( crawler.stats.get_value("benchmark/peak_items") == DELAYED_CONCURRENT_ITEMS ) diff --git a/tests/spiders.py b/tests/spiders.py index 7c7d3007c..3ff30ce03 100644 --- a/tests/spiders.py +++ b/tests/spiders.py @@ -13,22 +13,31 @@ from twisted.internet import defer from scrapy import signals from scrapy.exceptions import StopDownload -from scrapy.http import Request +from scrapy.http import Request, TextResponse from scrapy.item import Item from scrapy.linkextractors import LinkExtractor +from scrapy.spidermiddlewares.httperror import HttpError from scrapy.spiders import Spider from scrapy.spiders.crawl import CrawlSpider, Rule from scrapy.utils.defer import deferred_to_future, maybe_deferred_to_future from scrapy.utils.test import get_from_asyncio_queue if TYPE_CHECKING: + from collections.abc import AsyncIterator, Callable, Iterator + + from twisted.python.failure import Failure + from typing_extensions import Self + + from scrapy.crawler import Crawler + from scrapy.http import Headers, Response + from scrapy.http.request import CallbackT from tests.mockserver.http import MockServer class MockServerSpider(Spider): def __init__( self, - *args, + *args: Any, mockserver: MockServer | None = None, is_secure: bool = False, **kwargs: Any, @@ -74,7 +83,7 @@ class MetaSpider(MockServerSpider): super().__init__(*args, **kwargs) self.meta: dict[str, Any] = {} - def closed(self, reason): + def closed(self, reason: str) -> None: self.meta["close_reason"] = reason @@ -83,16 +92,24 @@ class FollowAllSpider(MetaSpider): link_extractor = LinkExtractor() def __init__( - self, total=10, show=20, order="rand", maxlatency=0.0, *args, **kwargs + self, + total: int = 10, + show: int = 20, + order: str = "rand", + maxlatency: float = 0.0, + *args: Any, + **kwargs: Any, ): super().__init__(*args, **kwargs) - self.urls_visited = [] - self.times = [] + self.urls_visited: list[str] = [] + self.times: list[float] = [] qargs = {"total": total, "show": show, "order": order, "maxlatency": maxlatency} + assert self.mockserver url = self.mockserver.url(f"/follow?{urlencode(qargs, doseq=True)}") self.start_urls = [url] - def parse(self, response): + def parse(self, response: Response) -> Iterator[Any]: + assert isinstance(response, TextResponse) self.urls_visited.append(response.url) self.times.append(time.time()) for link in self.link_extractor.extract_links(response): @@ -102,50 +119,52 @@ class FollowAllSpider(MetaSpider): class DelaySpider(MetaSpider): name = "delay" - def __init__(self, n=1, b=0, *args, **kwargs): + def __init__(self, n: float = 1, b: float = 0, *args: Any, **kwargs: Any): super().__init__(*args, **kwargs) self.n = n self.b = b - self.t1 = self.t2 = self.t2_err = 0 + self.t1 = self.t2 = self.t2_err = 0.0 - async def start(self): + async def start(self) -> AsyncIterator[Any]: self.t1 = time.time() + assert self.mockserver url = self.mockserver.url(f"/delay?n={self.n}&b={self.b}") yield Request(url, callback=self.parse, errback=self.errback) - def parse(self, response): + def parse(self, response: Response) -> Any: self.t2 = time.time() - def errback(self, failure): + def errback(self, failure: Failure) -> None: self.t2_err = time.time() class LogSpider(MetaSpider): name = "log_spider" - def log_debug(self, message: str, extra: dict[str, Any] | None = None): + def log_debug(self, message: str, extra: dict[str, Any] | None = None) -> None: self.logger.debug(message, extra=extra) - def log_info(self, message: str, extra: dict[str, Any] | None = None): + def log_info(self, message: str, extra: dict[str, Any] | None = None) -> None: self.logger.info(message, extra=extra) - def log_warning(self, message: str, extra: dict[str, Any] | None = None): + def log_warning(self, message: str, extra: dict[str, Any] | None = None) -> None: self.logger.warning(message, extra=extra) - def log_error(self, message: str, extra: dict[str, Any] | None = None): + def log_error(self, message: str, extra: dict[str, Any] | None = None) -> None: self.logger.error(message, extra=extra) - def log_critical(self, message: str, extra: dict[str, Any] | None = None): + def log_critical(self, message: str, extra: dict[str, Any] | None = None) -> None: self.logger.critical(message, extra=extra) - def parse(self, response): + def parse(self, response: Response) -> None: pass class SlowSpider(DelaySpider): name = "slow" - async def start(self): + async def start(self) -> AsyncIterator[Any]: + assert self.mockserver # 1st response is fast url = self.mockserver.url("/delay?n=0&b=0") yield Request(url, callback=self.parse, errback=self.errback) @@ -154,25 +173,25 @@ class SlowSpider(DelaySpider): url = self.mockserver.url(f"/delay?n={self.n}&b={self.b}") yield Request(url, callback=self.parse, errback=self.errback) - def parse(self, response): + def parse(self, response: Response) -> Iterator[Any]: yield Item() class SimpleSpider(MetaSpider): name = "simple" - def __init__(self, url="http://localhost:8998", *args, **kwargs): + def __init__(self, url: str = "http://localhost:8998", *args: Any, **kwargs: Any): super().__init__(*args, **kwargs) self.start_urls = [url] - def parse(self, response): + def parse(self, response: Response) -> Any: self.logger.info(f"Got response {response.status}") class AsyncDefSpider(SimpleSpider): name = "asyncdef" - async def parse(self, response): + async def parse(self, response: Response) -> None: await defer.succeed(42) self.logger.info(f"Got response {response.status}") @@ -180,7 +199,7 @@ class AsyncDefSpider(SimpleSpider): class AsyncDefAsyncioSpider(SimpleSpider): name = "asyncdef_asyncio" - async def parse(self, response): + async def parse(self, response: Response) -> None: await asyncio.sleep(0.2) status = await get_from_asyncio_queue(response.status) self.logger.info(f"Got response {status}") @@ -189,7 +208,7 @@ class AsyncDefAsyncioSpider(SimpleSpider): class AsyncDefAsyncioReturnSpider(SimpleSpider): name = "asyncdef_asyncio_return" - async def parse(self, response): + async def parse(self, response: Response) -> Any: await asyncio.sleep(0.2) status = await get_from_asyncio_queue(response.status) self.logger.info(f"Got response {status}") @@ -199,7 +218,7 @@ class AsyncDefAsyncioReturnSpider(SimpleSpider): class AsyncDefAsyncioReturnSingleElementSpider(SimpleSpider): name = "asyncdef_asyncio_return_single_element" - async def parse(self, response): + async def parse(self, response: Response) -> Any: await asyncio.sleep(0.1) status = await get_from_asyncio_queue(response.status) self.logger.info(f"Got response {status}") @@ -209,7 +228,7 @@ class AsyncDefAsyncioReturnSingleElementSpider(SimpleSpider): class AsyncDefAsyncioReqsReturnSpider(SimpleSpider): name = "asyncdef_asyncio_reqs_return" - async def parse(self, response): + async def parse(self, response: Response) -> Any: await asyncio.sleep(0.2) req_id = response.meta.get("req_id", 0) status = await get_from_asyncio_queue(response.status) @@ -226,7 +245,7 @@ class AsyncDefAsyncioReqsReturnSpider(SimpleSpider): class AsyncDefAsyncioGenExcSpider(SimpleSpider): name = "asyncdef_asyncio_gen_exc" - async def parse(self, response): + async def parse(self, response: Response) -> AsyncIterator[Any]: for i in range(10): await asyncio.sleep(0.1) yield {"foo": i} @@ -237,7 +256,7 @@ class AsyncDefAsyncioGenExcSpider(SimpleSpider): class AsyncDefDeferredDirectSpider(SimpleSpider): name = "asyncdef_deferred_direct" - async def parse(self, response): + async def parse(self, response: Response) -> AsyncIterator[Any]: await defer.succeed(None) yield {"code": 200} @@ -245,7 +264,7 @@ class AsyncDefDeferredDirectSpider(SimpleSpider): class AsyncDefDeferredWrappedSpider(SimpleSpider): name = "asyncdef_deferred_wrapped" - async def parse(self, response): + async def parse(self, response: Response) -> AsyncIterator[Any]: await deferred_to_future(defer.succeed(None)) yield {"code": 200} @@ -253,7 +272,7 @@ class AsyncDefDeferredWrappedSpider(SimpleSpider): class AsyncDefDeferredMaybeWrappedSpider(SimpleSpider): name = "asyncdef_deferred_maybe_wrapped" - async def parse(self, response): + async def parse(self, response: Response) -> AsyncIterator[Any]: await maybe_deferred_to_future(defer.succeed(None)) yield {"code": 200} @@ -261,7 +280,7 @@ class AsyncDefDeferredMaybeWrappedSpider(SimpleSpider): class AsyncDefAsyncioGenSpider(SimpleSpider): name = "asyncdef_asyncio_gen" - async def parse(self, response): + async def parse(self, response: Response) -> AsyncIterator[Any]: await asyncio.sleep(0.2) yield {"foo": 42} self.logger.info(f"Got response {response.status}") @@ -270,7 +289,7 @@ class AsyncDefAsyncioGenSpider(SimpleSpider): class AsyncDefAsyncioGenLoopSpider(SimpleSpider): name = "asyncdef_asyncio_gen_loop" - async def parse(self, response): + async def parse(self, response: Response) -> AsyncIterator[Any]: for i in range(10): await asyncio.sleep(0.1) yield {"foo": i} @@ -283,7 +302,8 @@ class AsyncDefAsyncioGenComplexSpider(SimpleSpider): following_reqs = 3 depth = 2 - def _get_req(self, index, cb=None): + def _get_req(self, index: int, cb: CallbackT | None = None) -> Request: + assert self.mockserver return Request( self.mockserver.url(f"/status?n=200&request={index}"), meta={"index": index}, @@ -291,11 +311,11 @@ class AsyncDefAsyncioGenComplexSpider(SimpleSpider): callback=cb, ) - async def start(self): + async def start(self) -> AsyncIterator[Any]: for i in range(1, self.initial_reqs + 1): yield self._get_req(i) - async def parse(self, response): + async def parse(self, response: Response) -> AsyncIterator[Any]: index = response.meta["index"] yield {"index": index} if index < 10**self.depth: @@ -305,7 +325,7 @@ class AsyncDefAsyncioGenComplexSpider(SimpleSpider): await asyncio.sleep(0.1) yield {"index": index + 5} - async def parse2(self, response): + async def parse2(self, response: Response) -> AsyncIterator[Any]: await asyncio.sleep(0.1) yield {"index2": response.meta["index"]} @@ -313,7 +333,7 @@ class AsyncDefAsyncioGenComplexSpider(SimpleSpider): class ItemSpider(FollowAllSpider): name = "item" - def parse(self, response): + def parse(self, response: Response) -> Iterator[Any]: for request in super().parse(response): yield request yield Item() @@ -321,12 +341,18 @@ class ItemSpider(FollowAllSpider): class MaxItemsAndRequestsSpider(FollowAllSpider): - def __init__(self, max_items=10, max_requests=10, *args, **kwargs): + def __init__( + self, + max_items: int = 10, + max_requests: int = 10, + *args: Any, + **kwargs: Any, + ): super().__init__(*args, **kwargs) self.max_items = max_items self.max_requests = max_requests - def parse(self, response): + def parse(self, response: Response) -> Iterator[Any]: self.items_scraped = 0 self.pages_crawled = 1 # account for the start url for request in super().parse(response): @@ -344,12 +370,12 @@ class DefaultError(Exception): class ErrorSpider(FollowAllSpider): name = "error" - exception_cls = DefaultError + exception_cls: type[Exception] = DefaultError - def raise_exception(self): + def raise_exception(self) -> None: raise self.exception_cls("Expected exception") - def parse(self, response): + def parse(self, response: Response) -> Iterator[Any]: for request in super().parse(response): yield request self.raise_exception() @@ -359,14 +385,15 @@ class BrokenStartSpider(FollowAllSpider): fail_before_yield = False fail_yielding = False - def __init__(self, *a, **kw): + def __init__(self, *a: Any, **kw: Any): super().__init__(*a, **kw) - self.seedsseen = [] + self.seedsseen: list[Any] = [] - async def start(self): + async def start(self) -> AsyncIterator[Any]: if self.fail_before_yield: 1 / 0 + assert self.mockserver for s in range(100): qargs = {"total": 10, "seed": s} url = self.mockserver.url(f"/follow?{urlencode(qargs, doseq=True)}") @@ -376,18 +403,18 @@ class BrokenStartSpider(FollowAllSpider): assert self.seedsseen, "All seeds consumed before any download happened" - def parse(self, response): + def parse(self, response: Response) -> Iterator[Any]: self.seedsseen.append(response.meta.get("seed")) yield from super().parse(response) class StartItemSpider(FollowAllSpider): - async def start(self): + async def start(self) -> AsyncIterator[Any]: yield {"name": "test item"} class StartGoodAndBadOutput(FollowAllSpider): - async def start(self): + async def start(self) -> AsyncIterator[Any]: yield {"a": "a"} yield Request("data:,a") yield "data:,b" @@ -395,17 +422,18 @@ class StartGoodAndBadOutput(FollowAllSpider): class SingleRequestSpider(MetaSpider): - seed = None - callback_func = None - errback_func = None + seed: Request | str | None = None + callback_func: Callable[[Response], Any] | None = None + errback_func: Callable[[Failure], Any] | None = None - async def start(self): + async def start(self) -> AsyncIterator[Any]: if isinstance(self.seed, Request): yield self.seed.replace(callback=self.parse, errback=self.on_error) else: + assert self.seed yield Request(self.seed, callback=self.parse, errback=self.on_error) - def parse(self, response): + def parse(self, response: Response) -> Any: self.meta.setdefault("responses", []).append(response) if callable(self.callback_func): return self.callback_func(response) @@ -413,7 +441,7 @@ class SingleRequestSpider(MetaSpider): return response.meta["next"] return None - def on_error(self, failure): + def on_error(self, failure: Failure) -> Any: self.meta["failure"] = failure if callable(self.errback_func): return self.errback_func(failure) @@ -426,17 +454,18 @@ class DuplicateStartSpider(MockServerSpider): distinct_urls = 2 dupe_factor = 3 - async def start(self): + async def start(self) -> AsyncIterator[Any]: + assert self.mockserver for i in range(self.distinct_urls): for _ in range(self.dupe_factor): url = self.mockserver.url(f"/echo?headers=1&body=test{i}") yield Request(url, dont_filter=self.dont_filter) - def __init__(self, url="http://localhost:8998", *args, **kwargs): + def __init__(self, url: str = "http://localhost:8998", *args: Any, **kwargs: Any): super().__init__(*args, **kwargs) self.visited = 0 - def parse(self, response): + def parse(self, response: Response) -> None: self.visited += 1 @@ -451,7 +480,7 @@ class CrawlSpiderWithParseMethod(MockServerSpider, CrawlSpider): } rules = (Rule(LinkExtractor(), callback="parse", follow=True),) - async def start(self): + async def start(self) -> AsyncIterator[Any]: test_body = b""" Page title @@ -461,11 +490,13 @@ class CrawlSpiderWithParseMethod(MockServerSpider, CrawlSpider): """ + assert self.mockserver url = self.mockserver.url("/alpayload") yield Request(url, method="POST", body=test_body) - def parse(self, response, foo=None): + def parse(self, response: Response, foo: str | None = None) -> Iterator[Any]: self.logger.info("[parse] status %i (foo: %s)", response.status, foo) + assert self.mockserver yield Request( self.mockserver.url("/status?n=202"), self.parse, cb_kwargs={"foo": "bar"} ) @@ -477,8 +508,9 @@ class CrawlSpiderWithAsyncCallback(CrawlSpiderWithParseMethod): name = "crawl_spider_with_async_callback" rules = (Rule(LinkExtractor(), callback="parse_async", follow=True),) - async def parse_async(self, response, foo=None): + async def parse_async(self, response: Response, foo: str | None = None) -> Request: self.logger.info("[parse_async] status %i (foo: %s)", response.status, foo) + assert self.mockserver return Request( self.mockserver.url("/status?n=202"), self.parse_async, @@ -492,8 +524,11 @@ class CrawlSpiderWithAsyncGeneratorCallback(CrawlSpiderWithParseMethod): name = "crawl_spider_with_async_generator_callback" rules = (Rule(LinkExtractor(), callback="parse_async_gen", follow=True),) - async def parse_async_gen(self, response, foo=None): + async def parse_async_gen( + self, response: Response, foo: str | None = None + ) -> AsyncIterator[Any]: self.logger.info("[parse_async_gen] status %i (foo: %s)", response.status, foo) + assert self.mockserver yield Request( self.mockserver.url("/status?n=202"), self.parse_async_gen, @@ -505,7 +540,7 @@ class CrawlSpiderWithErrback(CrawlSpiderWithParseMethod): name = "crawl_spider_with_errback" rules = (Rule(LinkExtractor(), callback="parse", errback="errback", follow=True),) - async def start(self): + async def start(self) -> AsyncIterator[Any]: test_body = b""" Page title @@ -518,17 +553,19 @@ class CrawlSpiderWithErrback(CrawlSpiderWithParseMethod): """ + assert self.mockserver url = self.mockserver.url("/alpayload") yield Request(url, method="POST", body=test_body) - def errback(self, failure): + def errback(self, failure: Failure) -> None: + assert isinstance(failure.value, HttpError) self.logger.info("[errback] status %i", failure.value.response.status) class CrawlSpiderWithoutErrback(CrawlSpiderWithParseMethod): name = "crawl_spider_without_errback" - async def start(self): + async def start(self) -> AsyncIterator[Any]: test_body = b""" Page title @@ -538,6 +575,7 @@ class CrawlSpiderWithoutErrback(CrawlSpiderWithParseMethod): """ + assert self.mockserver url = self.mockserver.url("/alpayload") yield Request(url, method="POST", body=test_body) @@ -553,7 +591,7 @@ class CrawlSpiderWithProcessRequestCallbackKeywordArguments(CrawlSpiderWithParse ), ) - def process_request(self, request, response): + def process_request(self, request: Request, response: Response) -> Request: request.cb_kwargs["foo"] = "process_request" return request @@ -562,59 +600,65 @@ class BytesReceivedCallbackSpider(MetaSpider): full_response_length = 2**18 @classmethod - def from_crawler(cls, crawler, *args, **kwargs): + def from_crawler(cls, crawler: Crawler, *args: Any, **kwargs: Any) -> Self: spider = super().from_crawler(crawler, *args, **kwargs) crawler.signals.connect(spider.bytes_received, signals.bytes_received) return spider - async def start(self): + async def start(self) -> AsyncIterator[Any]: body = b"a" * self.full_response_length + assert self.mockserver url = self.mockserver.url("/alpayload", is_secure=self.is_secure) yield Request(url, method="POST", body=body, errback=self.errback) - def parse(self, response): + def parse(self, response: Response) -> None: self.meta["response"] = response - def errback(self, failure): + def errback(self, failure: Failure) -> None: self.meta["failure"] = failure - def bytes_received(self, data, request, spider): + def bytes_received(self, data: bytes, request: Request, spider: Spider) -> None: self.meta["bytes_received"] = data raise StopDownload(fail=False) class BytesReceivedErrbackSpider(BytesReceivedCallbackSpider): - def bytes_received(self, data, request, spider): + def bytes_received(self, data: bytes, request: Request, spider: Spider) -> None: self.meta["bytes_received"] = data raise StopDownload(fail=True) class HeadersReceivedCallbackSpider(MetaSpider): @classmethod - def from_crawler(cls, crawler, *args, **kwargs): + def from_crawler(cls, crawler: Crawler, *args: Any, **kwargs: Any) -> Self: spider = super().from_crawler(crawler, *args, **kwargs) crawler.signals.connect(spider.headers_received, signals.headers_received) return spider - async def start(self): + async def start(self) -> AsyncIterator[Any]: + assert self.mockserver yield Request( self.mockserver.url("/status", is_secure=self.is_secure), errback=self.errback, ) - def parse(self, response): + def parse(self, response: Response) -> None: self.meta["response"] = response - def errback(self, failure): + def errback(self, failure: Failure) -> None: self.meta["failure"] = failure - def headers_received(self, headers, body_length, request, spider): + def headers_received( + self, headers: Headers, body_length: int, request: Request, spider: Spider + ) -> None: self.meta["headers_received"] = headers raise StopDownload(fail=False) class HeadersReceivedErrbackSpider(HeadersReceivedCallbackSpider): - def headers_received(self, headers, body_length, request, spider): + def headers_received( + self, headers: Headers, body_length: int, request: Request, spider: Spider + ) -> None: self.meta["headers_received"] = headers raise StopDownload(fail=True) @@ -623,13 +667,13 @@ class ExceptionSpider(Spider): name = "exception" @classmethod - def from_crawler(cls, crawler, *args, **kwargs): + def from_crawler(cls, crawler: Crawler, *args: Any, **kwargs: Any) -> Self: raise ValueError("Exception in from_crawler method") class NoRequestsSpider(Spider): name = "no_request" - async def start(self): + async def start(self) -> AsyncIterator[Any]: return yield diff --git a/tests/test_closespider.py b/tests/test_closespider.py index 9a7447e16..81c122b81 100644 --- a/tests/test_closespider.py +++ b/tests/test_closespider.py @@ -11,6 +11,8 @@ from tests.utils.decorators import inline_callbacks_test class TestCloseSpider: + mockserver: MockServer + @classmethod def setup_class(cls): cls.mockserver = MockServer() @@ -25,6 +27,7 @@ class TestCloseSpider: close_on = 5 crawler = get_crawler(ItemSpider, {"CLOSESPIDER_ITEMCOUNT": close_on}) yield crawler.crawl(mockserver=self.mockserver) + assert isinstance(crawler.spider, ItemSpider) reason = crawler.spider.meta["close_reason"] assert reason == "closespider_itemcount" itemcount = crawler.stats.get_value("item_scraped_count") @@ -35,6 +38,7 @@ class TestCloseSpider: close_on = 5 crawler = get_crawler(FollowAllSpider, {"CLOSESPIDER_PAGECOUNT": close_on}) yield crawler.crawl(mockserver=self.mockserver) + assert isinstance(crawler.spider, FollowAllSpider) reason = crawler.spider.meta["close_reason"] assert reason == "closespider_pagecount" pagecount = crawler.stats.get_value("response_received_count") @@ -54,6 +58,7 @@ class TestCloseSpider: yield crawler.crawl( max_items=max_items, max_requests=max_requests, mockserver=self.mockserver ) + assert isinstance(crawler.spider, MaxItemsAndRequestsSpider) reason = crawler.spider.meta["close_reason"] assert reason == "closespider_pagecount_no_item" pagecount = crawler.stats.get_value("response_received_count") @@ -72,6 +77,7 @@ class TestCloseSpider: }, ) yield crawler.crawl(mockserver=self.mockserver) + assert isinstance(crawler.spider, FollowAllSpider) reason = crawler.spider.meta["close_reason"] assert reason == "closespider_pagecount_no_item" pagecount = crawler.stats.get_value("response_received_count") @@ -82,6 +88,7 @@ class TestCloseSpider: close_on = 5 crawler = get_crawler(ErrorSpider, {"CLOSESPIDER_ERRORCOUNT": close_on}) yield crawler.crawl(total=1000000, mockserver=self.mockserver) + assert isinstance(crawler.spider, ErrorSpider) reason = crawler.spider.meta["close_reason"] assert reason == "closespider_errorcount" key = f"spider_exceptions/{crawler.spider.exception_cls.__name__}" @@ -94,6 +101,7 @@ class TestCloseSpider: close_on = 0.1 crawler = get_crawler(FollowAllSpider, {"CLOSESPIDER_TIMEOUT": close_on}) yield crawler.crawl(total=1000000, mockserver=self.mockserver) + assert isinstance(crawler.spider, FollowAllSpider) reason = crawler.spider.meta["close_reason"] assert reason == "closespider_timeout" total_seconds = crawler.stats.get_value("elapsed_time_seconds") @@ -104,6 +112,7 @@ class TestCloseSpider: timeout = 1 crawler = get_crawler(SlowSpider, {"CLOSESPIDER_TIMEOUT_NO_ITEM": timeout}) yield crawler.crawl(n=3, mockserver=self.mockserver) + assert isinstance(crawler.spider, SlowSpider) reason = crawler.spider.meta["close_reason"] assert reason == "closespider_timeout_no_item" total_seconds = crawler.stats.get_value("elapsed_time_seconds") diff --git a/tests/test_command_runspider.py b/tests/test_command_runspider.py index 11036eaeb..73a85dd5d 100644 --- a/tests/test_command_runspider.py +++ b/tests/test_command_runspider.py @@ -16,6 +16,11 @@ if TYPE_CHECKING: from pathlib import Path +# Header for spider classes copied with inspect.getsource(): the future import +# keeps their annotations from being evaluated in the generated module. +SPIDER_HEADER = "from __future__ import annotations\nfrom scrapy import Spider\n" + + class TestRunSpiderCommand: spider_filename = "myspider.py" @@ -65,14 +70,14 @@ class BadSpider(scrapy.Spider): def test_run_fail_spider(self, tmp_path: Path) -> None: ret, _, _ = self.runspider( - tmp_path, "from scrapy import Spider\n" + inspect.getsource(ExceptionSpider) + tmp_path, SPIDER_HEADER + inspect.getsource(ExceptionSpider) ) assert ret != 0 def test_run_good_spider(self, tmp_path: Path) -> None: ret, _, _ = self.runspider( tmp_path, - "from scrapy import Spider\n" + inspect.getsource(NoRequestsSpider), + SPIDER_HEADER + inspect.getsource(NoRequestsSpider), ) assert ret == 0 diff --git a/tests/test_contracts.py b/tests/test_contracts.py index 698044de7..44b621776 100644 --- a/tests/test_contracts.py +++ b/tests/test_contracts.py @@ -1,3 +1,6 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING, Any from unittest import TextTestResult import pytest @@ -11,7 +14,7 @@ from scrapy.contracts.default import ( ScrapesContract, UrlContract, ) -from scrapy.http import Request +from scrapy.http import Request, Response from scrapy.item import Field, Item from scrapy.spidermiddlewares.httperror import HttpError from scrapy.spiders import Spider @@ -19,6 +22,24 @@ from scrapy.utils.test import get_crawler from tests.mockserver.http import MockServer from tests.utils.decorators import inline_callbacks_test +if TYPE_CHECKING: + from unittest import TestResult + + from scrapy.http.request import CallbackT + + +def _request( + conman: ContractsManager, method: CallbackT, results: TestResult +) -> Request: + request = conman.from_method(method, results) + assert request + return request + + +def _call(request: Request, response: Any) -> Any: + assert request.callback + return request.callback(response, **request.cb_kwargs) + class DemoItem(Item): name = Field() @@ -30,7 +51,7 @@ class ResponseMock: class ResponseMetaMock(ResponseMock): - meta = None + meta: Any = None class TaggedRequest(Request): @@ -289,17 +310,19 @@ class TestContractsManager: def setup_method(self): self.conman = ContractsManager(self.contracts) - self.results = TextTestResult(stream=None, descriptions=False, verbosity=0) + self.results = TextTestResult( # type: ignore[type-var] + stream=None, descriptions=False, verbosity=0 + ) - def should_succeed(self): + def should_succeed(self) -> None: assert not self.results.failures assert not self.results.errors - def should_fail(self): + def should_fail(self) -> None: assert self.results.failures assert not self.results.errors - def should_error(self): + def should_error(self) -> None: assert self.results.errors def test_contracts(self): @@ -313,12 +336,10 @@ class TestContractsManager: ) # returns request for valid method - request = self.conman.from_method(spider.returns_request, self.results) - assert request is not None + assert self.conman.from_method(spider.returns_request, self.results) is not None # no request for missing url - request = self.conman.from_method(spider.parse_no_url, self.results) - assert request is None + assert self.conman.from_method(spider.parse_no_url, self.results) is None def test_cb_kwargs(self): spider = DemoSpider() @@ -354,29 +375,31 @@ class TestContractsManager: ) # returns_request - request = self.conman.from_method( - spider.returns_request_cb_kwargs, self.results - ) - request.callback(response, **request.cb_kwargs) + request = _request(self.conman, spider.returns_request_cb_kwargs, self.results) + _call(request, response) self.should_succeed() # returns_item - request = self.conman.from_method(spider.returns_item_cb_kwargs, self.results) - request.callback(response, **request.cb_kwargs) + request = _request(self.conman, spider.returns_item_cb_kwargs, self.results) + _call(request, response) self.should_succeed() # returns_item (error, callback doesn't take keyword arguments) - request = self.conman.from_method( - spider.returns_item_cb_kwargs_error_unexpected_keyword, self.results + request = _request( + self.conman, + spider.returns_item_cb_kwargs_error_unexpected_keyword, + self.results, ) - request.callback(response, **request.cb_kwargs) + _call(request, response) self.should_error() # returns_item (error, contract doesn't provide keyword arguments) - request = self.conman.from_method( - spider.returns_item_cb_kwargs_error_missing_argument, self.results + request = _request( + self.conman, + spider.returns_item_cb_kwargs_error_missing_argument, + self.results, ) - request.callback(response, **request.cb_kwargs) + _call(request, response) self.should_error() def test_meta(self): @@ -398,29 +421,27 @@ class TestContractsManager: response = ResponseMetaMock() # returns_request - request = self.conman.from_method(spider.returns_request_meta, self.results) + request = _request(self.conman, spider.returns_request_meta, self.results) assert request.meta["cookiejar"] == "session1" response.meta = request.meta - request.callback(response) + _call(request, response) assert response.meta["cookiejar"] == "session1" self.should_succeed() response = ResponseMetaMock() # returns_item - request = self.conman.from_method(spider.returns_item_meta, self.results) + request = _request(self.conman, spider.returns_item_meta, self.results) assert request.meta["key"] == "example" response.meta = request.meta - request.callback(response) + _call(request, response) assert response.meta["key"] == "example" self.should_succeed() response = ResponseMetaMock() - request = self.conman.from_method( - spider.returns_error_missing_meta, self.results - ) - request.callback(response) + request = _request(self.conman, spider.returns_error_missing_meta, self.results) + _call(request, response) self.should_error() def test_returns(self): @@ -428,36 +449,36 @@ class TestContractsManager: response = ResponseMock() # returns_item - request = self.conman.from_method(spider.returns_item, self.results) - request.callback(response) + request = _request(self.conman, spider.returns_item, self.results) + _call(request, response) self.should_succeed() # returns_dict_item - request = self.conman.from_method(spider.returns_dict_item, self.results) - request.callback(response) + request = _request(self.conman, spider.returns_dict_item, self.results) + _call(request, response) self.should_succeed() # returns_request - request = self.conman.from_method(spider.returns_request, self.results) - request.callback(response) + request = _request(self.conman, spider.returns_request, self.results) + _call(request, response) self.should_succeed() # returns_fail - request = self.conman.from_method(spider.returns_fail, self.results) - request.callback(response) + request = _request(self.conman, spider.returns_fail, self.results) + _call(request, response) self.should_fail() # returns_dict_fail - request = self.conman.from_method(spider.returns_dict_fail, self.results) - request.callback(response) + request = _request(self.conman, spider.returns_dict_fail, self.results) + _call(request, response) self.should_fail() def test_returns_async(self): spider = DemoSpider() response = ResponseMock() - request = self.conman.from_method(spider.returns_request_async, self.results) - request.callback(response) + request = _request(self.conman, spider.returns_request_async, self.results) + _call(request, response) self.should_error() def test_returns_invalid_argument_count(self): @@ -477,10 +498,8 @@ class TestContractsManager: spider = DemoSpider() response = ResponseMock() - request = self.conman.from_method( - spider.returns_request_range_fail, self.results - ) - request.callback(response) + request = _request(self.conman, spider.returns_request_range_fail, self.results) + _call(request, response) self.should_fail() assert "expected 2..3" in self.results.failures[-1][-1] @@ -490,8 +509,8 @@ class TestContractsManager: # @returns and @scrapes only count matching output objects and skip # the request that is also yielded. - request = self.conman.from_method(spider.yields_item_and_request, self.results) - request.callback(response) + request = _request(self.conman, spider.yields_item_and_request, self.results) + _call(request, response) self.should_succeed() def test_testcase_str(self): @@ -507,30 +526,30 @@ class TestContractsManager: response = ResponseMock() # scrapes_item_ok - request = self.conman.from_method(spider.scrapes_item_ok, self.results) - request.callback(response) + request = _request(self.conman, spider.scrapes_item_ok, self.results) + _call(request, response) self.should_succeed() # scrapes_dict_item_ok - request = self.conman.from_method(spider.scrapes_dict_item_ok, self.results) - request.callback(response) + request = _request(self.conman, spider.scrapes_dict_item_ok, self.results) + _call(request, response) self.should_succeed() # scrapes_item_fail - request = self.conman.from_method(spider.scrapes_item_fail, self.results) - request.callback(response) + request = _request(self.conman, spider.scrapes_item_fail, self.results) + _call(request, response) self.should_fail() # scrapes_dict_item_fail - request = self.conman.from_method(spider.scrapes_dict_item_fail, self.results) - request.callback(response) + request = _request(self.conman, spider.scrapes_dict_item_fail, self.results) + _call(request, response) self.should_fail() # scrapes_multiple_missing_fields - request = self.conman.from_method( - spider.scrapes_multiple_missing_fields, self.results + request = _request( + self.conman, spider.scrapes_multiple_missing_fields, self.results ) - request.callback(response) + _call(request, response) self.should_fail() message = "ContractFail: Missing fields: name, url" assert message in self.results.failures[-1][-1] @@ -540,14 +559,13 @@ class TestContractsManager: response = ResponseMock() # invalid regex - request = self.conman.from_method(spider.invalid_regex, self.results) - assert request is None + assert self.conman.from_method(spider.invalid_regex, self.results) is None # invalid regex with valid contract - request = self.conman.from_method( - spider.invalid_regex_with_valid_contract, self.results + request = _request( + self.conman, spider.invalid_regex_with_valid_contract, self.results ) - request.callback(response) + _call(request, response) self.should_succeed() def test_custom_contracts(self): @@ -559,14 +577,14 @@ class TestContractsManager: def test_errback(self): spider = DemoSpider() - response = ResponseMock() try: - raise HttpError(response, "Ignoring non-200 response") + raise HttpError(Response("http://scrapy.org"), "Ignoring non-200 response") except HttpError: failure_mock = failure.Failure() - request = self.conman.from_method(spider.returns_request, self.results) + request = _request(self.conman, spider.returns_request, self.results) + assert request.errback request.errback(failure_mock) assert not self.results.failures @@ -602,11 +620,12 @@ class TestContractsManager: crawler = get_crawler(TestSameUrlSpider) yield crawler.crawl() + assert isinstance(crawler.spider, TestSameUrlSpider) assert crawler.spider.visited == 2 def test_custom_tagged_request_contract(self): spider = DemoSpider() - request = self.conman.from_method(spider.custom_tagged_request, self.results) + request = _request(self.conman, spider.custom_tagged_request, self.results) assert request.method == "POST" assert isinstance(request, TaggedRequest) assert request.contract_tag == "custom" @@ -672,18 +691,20 @@ class PostProcessErrorContract(Contract): class TestCustomContractPrePostProcess: def setup_method(self): - self.results = TextTestResult(stream=None, descriptions=False, verbosity=0) + self.results = TextTestResult( # type: ignore[type-var] + stream=None, descriptions=False, verbosity=0 + ) def test_pre_hook_keyboard_interrupt(self): spider = DemoSpider() response = ResponseMock() contract = CustomFailContractPreProcess(spider.returns_request) - conman = ContractsManager([UrlContract, ReturnsContract, contract]) + conman = ContractsManager([UrlContract, ReturnsContract]) - request = conman.from_method(spider.returns_request, self.results) + request = _request(conman, spider.returns_request, self.results) contract.add_pre_hook(request, self.results) with pytest.raises(KeyboardInterrupt, match="Pre-process exception"): - request.callback(response, **request.cb_kwargs) + _call(request, response) assert not self.results.failures assert not self.results.errors @@ -692,12 +713,12 @@ class TestCustomContractPrePostProcess: spider = DemoSpider() response = ResponseMock() contract = CustomFailContractPostProcess(spider.returns_request) - conman = ContractsManager([UrlContract, ReturnsContract, contract]) + conman = ContractsManager([UrlContract, ReturnsContract]) - request = conman.from_method(spider.returns_request, self.results) + request = _request(conman, spider.returns_request, self.results) contract.add_post_hook(request, self.results) with pytest.raises(KeyboardInterrupt, match="Post-process exception"): - request.callback(response, **request.cb_kwargs) + _call(request, response) assert not self.results.failures assert not self.results.errors @@ -706,11 +727,11 @@ class TestCustomContractPrePostProcess: spider = DemoSpider() response = ResponseMock() contract = PreProcessSuccessContract(spider.returns_request) - conman = ContractsManager([UrlContract, ReturnsContract, contract]) + conman = ContractsManager([UrlContract, ReturnsContract]) - request = conman.from_method(spider.returns_request, self.results) + request = _request(conman, spider.returns_request, self.results) contract.add_pre_hook(request, self.results) - request.callback(response, **request.cb_kwargs) + _call(request, response) assert not self.results.failures assert not self.results.errors @@ -719,11 +740,11 @@ class TestCustomContractPrePostProcess: spider = DemoSpider() response = ResponseMock() contract = PreProcessAssertionFailContract(spider.returns_request) - conman = ContractsManager([UrlContract, ReturnsContract, contract]) + conman = ContractsManager([UrlContract, ReturnsContract]) - request = conman.from_method(spider.returns_request, self.results) + request = _request(conman, spider.returns_request, self.results) contract.add_pre_hook(request, self.results) - request.callback(response, **request.cb_kwargs) + _call(request, response) assert self.results.failures assert not self.results.errors @@ -732,11 +753,11 @@ class TestCustomContractPrePostProcess: spider = DemoSpider() response = ResponseMock() contract = PreProcessErrorContract(spider.returns_request) - conman = ContractsManager([UrlContract, ReturnsContract, contract]) + conman = ContractsManager([UrlContract, ReturnsContract]) - request = conman.from_method(spider.returns_request, self.results) + request = _request(conman, spider.returns_request, self.results) contract.add_pre_hook(request, self.results) - request.callback(response, **request.cb_kwargs) + _call(request, response) assert self.results.errors @@ -748,7 +769,7 @@ class TestCustomContractPrePostProcess: contract.add_pre_hook(request, self.results) with pytest.raises(TypeError, match="async callbacks"): - request.callback(response) + _call(request, response) def test_pre_hook_async_generator(self): spider = DemoSpider() @@ -758,7 +779,7 @@ class TestCustomContractPrePostProcess: contract.add_pre_hook(request, self.results) with pytest.raises(TypeError, match="async callbacks"): - request.callback(response) + _call(request, response) def test_post_hook_async_generator(self): spider = DemoSpider() @@ -768,16 +789,16 @@ class TestCustomContractPrePostProcess: contract.add_post_hook(request, self.results) with pytest.raises(TypeError, match="async callbacks"): - request.callback(response) + _call(request, response) def test_post_hook_error(self): spider = DemoSpider() response = ResponseMock() contract = PostProcessErrorContract(spider.returns_request) - conman = ContractsManager([UrlContract, ReturnsContract, contract]) + conman = ContractsManager([UrlContract, ReturnsContract]) - request = conman.from_method(spider.returns_request, self.results) + request = _request(conman, spider.returns_request, self.results) contract.add_post_hook(request, self.results) - request.callback(response, **request.cb_kwargs) + _call(request, response) assert self.results.errors diff --git a/tests/test_crawl.py b/tests/test_crawl.py index d284805be..b277526de 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -359,7 +359,6 @@ with multiples lines est: list[list[tuple[str, Any]]] = [] def cb(response): - assert crawler.engine est.append(get_engine_status(crawler.engine)) crawler = get_crawler(SingleRequestSpider) @@ -377,7 +376,6 @@ with multiples lines est: list[str] = [] def cb(response): - assert crawler.engine est.append(format_engine_status(crawler.engine)) crawler = get_crawler(SingleRequestSpider) @@ -452,7 +450,6 @@ class TestCrawlSpider: await crawler.crawl_async( mockserver.url("/status?n=200"), mockserver=mockserver ) - assert crawler.stats return items, crawler.stats @coroutine_test @@ -518,7 +515,6 @@ class TestCrawlSpider: # so the failure is dropped silently and the crawl finishes normally. assert "[parse] status 200 (foo: None)" in caplog.text assert "[errback]" not in caplog.text - assert crawler.stats assert crawler.stats.get_value("downloader/response_status_count/404") == 1 @coroutine_test diff --git a/tests/test_crawler.py b/tests/test_crawler.py index 17ca02dea..2836a6627 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -536,7 +536,6 @@ class TestSpiderSettings: custom_settings = {"AUTOTHROTTLE_ENABLED": True} crawler = get_crawler(MySpider) - assert crawler.extensions enabled_exts = [e.__class__ for e in crawler.extensions.middlewares] assert AutoThrottle in enabled_exts @@ -568,7 +567,6 @@ class TestCrawlerLogging: } async def start(self): - assert crawler.stats info_count_start = crawler.stats.get_value("log_count/INFO") logging.debug("debug message") # noqa: LOG015 logging.info("info message") # noqa: LOG015 @@ -601,7 +599,6 @@ class TestCrawlerLogging: assert "info message" in logged assert "warning message" in logged assert "error message" in logged - assert crawler.stats assert crawler.stats.get_value("log_count/ERROR") == 1 assert crawler.stats.get_value("log_count/WARNING") == 1 assert info_count == 1 diff --git a/tests/test_downloadermiddleware_httpcache.py b/tests/test_downloadermiddleware_httpcache.py index 6eabe91ad..75affe343 100644 --- a/tests/test_downloadermiddleware_httpcache.py +++ b/tests/test_downloadermiddleware_httpcache.py @@ -72,7 +72,6 @@ class TestBase: settings = self._get_settings(**new_settings) crawler = get_crawler(Spider, settings) crawler.spider = crawler._create_spider("example.com") - assert crawler.stats crawler.stats.open_spider() try: yield crawler @@ -137,7 +136,6 @@ class StorageTestMixin(TestBase): with self._middleware() as mw: spider = mw.crawler.spider assert spider - assert mw.crawler.stats mw.storage.store_response(spider, self.request, self.response) self._corrupt_cache_entry(mw.storage, spider, self.request) @@ -159,7 +157,6 @@ class StorageTestMixin(TestBase): with self._middleware(HTTPCACHE_IGNORE_MISSING=True) as mw: spider = mw.crawler.spider assert spider - assert mw.crawler.stats mw.storage.store_response(spider, self.request, self.response) self._corrupt_cache_entry(mw.storage, spider, self.request) diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py index 4e0fd7c8b..235f29a86 100644 --- a/tests/test_downloadermiddleware_httpcompression.py +++ b/tests/test_downloadermiddleware_httpcompression.py @@ -61,7 +61,6 @@ class TestHttpCompression: def setup_method(self): self.crawler = get_crawler(Spider) self.mw = build_from_crawler(HttpCompressionMiddleware, self.crawler) - assert self.crawler.stats self.crawler.stats.open_spider() def _getresponse(self, coding: str) -> Response: @@ -87,7 +86,6 @@ class TestHttpCompression: return response def assertStatsEqual(self, key: str, value: Any) -> None: - assert self.crawler.stats assert self.crawler.stats.get_value(key) == value, str( self.crawler.stats.get_stats() ) diff --git a/tests/test_downloadermiddleware_offsite.py b/tests/test_downloadermiddleware_offsite.py index 40e09be13..575051526 100644 --- a/tests/test_downloadermiddleware_offsite.py +++ b/tests/test_downloadermiddleware_offsite.py @@ -230,7 +230,6 @@ def test_repeated_offsite_domain(): with pytest.raises(IgnoreRequest): mw.process_request(req1) assert "other.org" in mw.domains_seen - assert crawler.stats assert crawler.stats.get_value("offsite/domains") == 1 assert crawler.stats.get_value("offsite/filtered") == 1 with pytest.raises(IgnoreRequest): diff --git a/tests/test_downloadermiddleware_retry.py b/tests/test_downloadermiddleware_retry.py index 1c8a61aa5..7b684b678 100644 --- a/tests/test_downloadermiddleware_retry.py +++ b/tests/test_downloadermiddleware_retry.py @@ -82,7 +82,6 @@ class TestRetry: # discard it assert self.mw.process_response(req3, rsp) is rsp - assert self.crawler.stats assert self.crawler.stats.get_value("retry/max_reached") == 1 assert ( self.crawler.stats.get_value("retry/reason_count/503 Service Unavailable") @@ -134,7 +133,6 @@ class TestRetry: self._test_retry_exception(req, exc("foo")) stats = self.crawler.stats - assert stats assert stats.get_value("retry/max_reached") == len(exceptions) assert stats.get_value("retry/count") == len(exceptions) * 2 assert ( @@ -315,7 +313,6 @@ class TestGetRetryRequest: assert new_request.meta["retry_times"] == expected_retry_times assert new_request.priority == -1 expected_reason = "unspecified" - assert spider.crawler.stats for stat in ("retry/count", f"retry/reason_count/{expected_reason}"): assert spider.crawler.stats.get_value(stat) == 1 assert ( @@ -336,7 +333,6 @@ class TestGetRetryRequest: max_retry_times=max_retry_times, ) assert new_request is None - assert spider.crawler.stats assert spider.crawler.stats.get_value("retry/max_reached") == 1 failure_count = max_retry_times + 1 expected_reason = "unspecified" @@ -363,7 +359,6 @@ class TestGetRetryRequest: assert new_request.meta["retry_times"] == expected_retry_times assert new_request.priority == -1 expected_reason = "unspecified" - assert spider.crawler.stats for stat in ("retry/count", f"retry/reason_count/{expected_reason}"): assert spider.crawler.stats.get_value(stat) == 1 assert ( @@ -396,7 +391,6 @@ class TestGetRetryRequest: assert new_request.meta["retry_times"] == expected_retry_times assert new_request.priority == -expected_retry_times expected_reason = "unspecified" - assert spider.crawler.stats for stat in ("retry/count", f"retry/reason_count/{expected_reason}"): value = spider.crawler.stats.get_value(stat) assert value == expected_retry_times @@ -521,7 +515,6 @@ class TestGetRetryRequest: reason=expected_reason, ) expected_retry_times = 1 - assert spider.crawler.stats for stat in ("retry/count", f"retry/reason_count/{expected_reason}"): assert spider.crawler.stats.get_value(stat) == 1 assert ( @@ -543,7 +536,6 @@ class TestGetRetryRequest: reason=expected_reason, ) expected_retry_times = 1 - assert spider.crawler.stats stat = spider.crawler.stats.get_value( f"retry/reason_count/{expected_reason_string}" ) @@ -569,7 +561,6 @@ class TestGetRetryRequest: reason=expected_reason, ) expected_retry_times = 1 - assert spider.crawler.stats stat = spider.crawler.stats.get_value( f"retry/reason_count/{expected_reason_string}" ) @@ -593,7 +584,6 @@ class TestGetRetryRequest: reason=expected_reason, ) expected_retry_times = 1 - assert spider.crawler.stats stat = spider.crawler.stats.get_value( f"retry/reason_count/{expected_reason_string}" ) @@ -619,7 +609,6 @@ class TestGetRetryRequest: reason=expected_reason, ) expected_retry_times = 1 - assert spider.crawler.stats stat = spider.crawler.stats.get_value( f"retry/reason_count/{expected_reason_string}" ) @@ -737,7 +726,6 @@ class TestGetRetryRequest: reason=expected_reason, stats_base_key=stats_key, ) - assert spider.crawler.stats for stat in ( f"{stats_key}/count", f"{stats_key}/reason_count/{expected_reason}", diff --git a/tests/test_downloadermiddleware_stats.py b/tests/test_downloadermiddleware_stats.py index 13f6d6fe3..01014ddd6 100644 --- a/tests/test_downloadermiddleware_stats.py +++ b/tests/test_downloadermiddleware_stats.py @@ -17,7 +17,6 @@ class MyException(Exception): class TestDownloaderStats: def setup_method(self) -> None: self.crawler = get_crawler(Spider) - assert self.crawler.stats is not None self.mw = build_from_crawler(DownloaderStats, self.crawler) self.crawler.stats.open_spider() @@ -26,7 +25,6 @@ class TestDownloaderStats: self.res = Response("http://scrapytest.org", status=400) def assertStatsEqual(self, key: str, value: object) -> None: - assert self.crawler.stats is not None assert self.crawler.stats.get_value(key) == value, str( self.crawler.stats.get_stats() ) @@ -53,7 +51,6 @@ class TestDownloaderStats: build_from_crawler(DownloaderStats, crawler) def teardown_method(self) -> None: - assert self.crawler.stats is not None self.crawler.stats.close_spider() diff --git a/tests/test_downloaderslotssettings.py b/tests/test_downloaderslotssettings.py index 9d58a6e09..9f0a03b28 100644 --- a/tests/test_downloaderslotssettings.py +++ b/tests/test_downloaderslotssettings.py @@ -149,7 +149,6 @@ async def test_none_slot_with_priority_queue( assert crawler.spider.default_slot in crawler.spider.times assert len(crawler.spider.times[crawler.spider.default_slot]) == 2 - assert crawler.stats stats = crawler.stats assert stats.get_value("spider_exceptions", 0) == 0 assert stats.get_value("downloader/exception_count", 0) == 0 diff --git a/tests/test_dupefilters.py b/tests/test_dupefilters.py index 348e778cd..4b984406a 100644 --- a/tests/test_dupefilters.py +++ b/tests/test_dupefilters.py @@ -179,7 +179,6 @@ class TestRFPDupeFilter: dupefilter.log(r1, spider) dupefilter.log(r2, spider) - assert crawler.stats assert crawler.stats.get_value("dupefilter/filtered") == 2 assert ( "scrapy.dupefilters", @@ -213,7 +212,6 @@ class TestRFPDupeFilter: dupefilter.log(r1, spider) dupefilter.log(r2, spider) - assert crawler.stats assert crawler.stats.get_value("dupefilter/filtered") == 2 assert ( "scrapy.dupefilters", diff --git a/tests/test_engine_loop.py b/tests/test_engine_loop.py index 8e5197df7..5c3c92d3b 100644 --- a/tests/test_engine_loop.py +++ b/tests/test_engine_loop.py @@ -15,7 +15,7 @@ from tests.mockserver.http import MockServer from tests.utils.decorators import coroutine_test if TYPE_CHECKING: - from collections.abc import Iterator + from collections.abc import AsyncIterator, Callable, Iterator from scrapy.http import Response @@ -85,19 +85,23 @@ class TestMain: class TestSpider(Spider): name = "test" - async def start(self): + async def start(self) -> AsyncIterator[Any]: + assert self.crawler.engine._slot + scheduler = self.crawler.engine._slot.scheduler + assert isinstance(scheduler, MemoryScheduler) + yield Request("data:,a") await sleep(seconds) - self.crawler.engine._slot.scheduler.pause() - self.crawler.engine._slot.scheduler.enqueue_request(Request("data:,b")) + scheduler.pause() + scheduler.enqueue_request(Request("data:,b")) # During this time, the scheduler reports having requests but # returns None. await sleep(seconds) - self.crawler.engine._slot.scheduler.unpause() + scheduler.unpause() # The scheduler request is processed. await sleep(seconds) @@ -106,21 +110,21 @@ class TestMain: await sleep(seconds) - self.crawler.engine._slot.scheduler.pause() - self.crawler.engine._slot.scheduler.enqueue_request(Request("data:,d")) + scheduler.pause() + scheduler.enqueue_request(Request("data:,d")) # The last start request is processed during the time until the # delayed call below, proving that the start iteration can # finish before a scheduler “sleep” without causing the # scheduler to finish. - call_later(seconds, self.crawler.engine._slot.scheduler.unpause) + call_later(seconds, scheduler.unpause) - def parse(self, response): + def parse(self, response: Response) -> None: pass actual_urls = [] - def track_url(request, spider): + def track_url(request: Request, spider: Spider) -> None: actual_urls.append(request.url) settings = {"SCHEDULER": MemoryScheduler} @@ -138,17 +142,16 @@ class TestMain: class TestSpider(Spider): name = "test" - async def start(self): - assert self.crawler.engine is not None + async def start(self) -> AsyncIterator[Any]: await self.crawler.engine.close_async() yield Request("data:,a") - def parse(self, response): + def parse(self, response: Response) -> None: pass actual_urls = [] - def track_url(request, spider): + def track_url(request: Request, spider: Spider) -> None: actual_urls.append(request.url) settings = {"SCHEDULER": MemoryScheduler} @@ -160,7 +163,6 @@ class TestMain: await crawler.crawl_async() assert not caplog.records - assert crawler.stats assert crawler.stats.get_value("finish_reason") == "shutdown" assert not actual_urls @@ -229,7 +231,6 @@ class TestMain: failure, response = errors[0] assert isinstance(failure.value, ValueError) assert response is None - assert crawler.stats assert crawler.stats.get_value("finish_reason") == "start_error" assert crawler.stats.get_value("spider_exceptions/count") == 1 assert crawler.stats.get_value("spider_exceptions/ValueError") == 1 @@ -256,12 +257,13 @@ class TestMain: await crawler.crawl_async() assert not caplog.records - assert crawler.stats assert crawler.stats.get_value("finish_reason") == "my_reason" assert crawler.stats.get_value("spider_exceptions/count") is None class TestRequestSendOrder: + mockserver: MockServer + seconds = 0.1 # increase if flaky @classmethod @@ -273,52 +275,65 @@ class TestRequestSendOrder: def teardown_class(cls): cls.mockserver.__exit__(None, None, None) - def request(self, num, response_seconds, download_slots, priority=0): + def request( + self, + num: int, + response_seconds: float, + download_slots: int, + priority: int = 0, + ) -> Request: url = self.mockserver.url(f"/delay?n={response_seconds}&{num}") meta = {"download_slot": str(num % download_slots)} return Request(url, meta=meta, priority=priority) - def get_num(self, request_or_response: Request | Response): + def get_num(self, request_or_response: Request | Response) -> int: return int(request_or_response.url.rsplit("&", maxsplit=1)[1]) async def _test_request_order( self, - start_nums, - cb_nums=None, - settings=None, - response_seconds=None, - download_slots=1, - start_fn=None, - parse_fn=None, - ): + start_nums: list[int], + cb_nums: list[int] | None = None, + settings: dict[str, Any] | None = None, + response_seconds: float | None = None, + download_slots: int = 1, + start_fn: Callable[[Spider], AsyncIterator[Any]] | None = None, + parse_fn: Callable[..., Iterator[Any]] | None = None, + ) -> None: cb_nums = cb_nums or [] settings = settings or {} - response_seconds = response_seconds or self.seconds + seconds = response_seconds or self.seconds cb_requests = deque( - [self.request(num, response_seconds, download_slots) for num in cb_nums] + [self.request(num, seconds, download_slots) for num in cb_nums] ) if start_fn is None: - async def start_fn(spider): + async def default_start(spider: Spider) -> AsyncIterator[Any]: for num in start_nums: - yield self.request(num, response_seconds, download_slots) + yield self.request(num, seconds, download_slots) + + start_fn = default_start if parse_fn is None: - def parse_fn(spider, response): + def default_parse(spider: Spider, response: Response) -> Iterator[Any]: while cb_requests: yield cb_requests.popleft() + parse_fn = default_parse + + spider_start = start_fn + spider_parse = parse_fn + class TestSpider(Spider): name = "test" - start = start_fn - parse = parse_fn + start = spider_start + parse = spider_parse actual_nums = [] - def track_num(request, spider): + def track_num(request: Request, spider: Spider) -> None: actual_nums.append(self.get_num(request)) crawler = get_crawler(TestSpider, settings_dict=settings) @@ -337,12 +352,13 @@ class TestRequestSendOrder: response_seconds = 0 download_slots = 1 - def _request(num, priority=0): + def _request(num: int, priority: int = 0) -> Request: return self.request( num, response_seconds, download_slots, priority=priority ) - async def start(spider): + async def start(spider: Spider) -> AsyncIterator[Any]: + assert spider.crawler.engine._slot # The first CONCURRENT_REQUESTS start requests are sent # immediately. yield _request(1) @@ -356,7 +372,7 @@ class TestRequestSendOrder: yield _request(3, priority=1) yield _request(4, priority=1) - def parse(spider, response): + def parse(spider: Spider, response: Response) -> Iterator[Any]: return yield @@ -377,12 +393,13 @@ class TestRequestSendOrder: response_seconds = 0 download_slots = 1 - def _request(num, priority=0): + def _request(num: int, priority: int = 0) -> Request: return self.request( num, response_seconds, download_slots, priority=priority ) - async def start(spider): + async def start(spider: Spider) -> AsyncIterator[Any]: + assert spider.crawler.engine._slot # The first CONCURRENT_REQUESTS start requests are sent # immediately. yield _request(1) @@ -396,7 +413,7 @@ class TestRequestSendOrder: yield _request(4, priority=1) yield _request(3, priority=1) - def parse(spider, response): + def parse(spider: Spider, response: Response) -> Iterator[Any]: return yield @@ -420,12 +437,13 @@ class TestRequestSendOrder: response_seconds = 0 download_slots = 1 - def _request(num, priority=0): + def _request(num: int, priority: int = 0) -> Request: return self.request( num, response_seconds, download_slots, priority=priority ) - async def start(spider): + async def start(spider: Spider) -> AsyncIterator[Any]: + assert spider.crawler.engine._slot # The first CONCURRENT_REQUESTS start requests are sent # immediately. yield _request(1) @@ -454,7 +472,7 @@ class TestRequestSendOrder: ): spider.crawler.engine._slot.scheduler.enqueue_request(request) - def parse(spider, response): + def parse(spider: Spider, response: Response) -> Iterator[Any]: return yield @@ -479,7 +497,7 @@ class TestRequestSendOrder: response_seconds = self.seconds * 2**1 # increase if flaky download_slots = 1 - async def start(spider): + async def start(spider: Spider) -> AsyncIterator[Any]: for num in start_nums: if spider.crawler.engine.needs_backout(): await spider.crawler.signals.wait_for(signals.scheduler_empty) diff --git a/tests/test_extension_memdebug.py b/tests/test_extension_memdebug.py index ddcfb5163..0c4f2ad95 100644 --- a/tests/test_extension_memdebug.py +++ b/tests/test_extension_memdebug.py @@ -33,7 +33,6 @@ def test_spider_closed_sets_stats() -> None: ext.spider_closed(DefaultSpider.from_crawler(crawler), "finished") - assert crawler.stats assert crawler.stats.get_value("memdebug/gc_garbage_count") == len(gc.garbage) assert crawler.stats.get_value("memdebug/live_refs/TrackedObject") == len(tracked) assert crawler.stats.get_value("memdebug/live_refs/CollectedObject") is None @@ -47,6 +46,5 @@ async def test_crawl_sets_stats() -> None: crawler = get_crawler(MemDebugSpider, settings_dict={"MEMDEBUG_ENABLED": True}) await crawler.crawl_async() - assert crawler.stats assert crawler.stats.get_value("memdebug/gc_garbage_count") is not None assert crawler.stats.get_value("memdebug/live_refs/MemDebugSpider") == 1 diff --git a/tests/test_extension_memusage.py b/tests/test_extension_memusage.py index dbe28eded..aa0149ff2 100644 --- a/tests/test_extension_memusage.py +++ b/tests/test_extension_memusage.py @@ -99,7 +99,6 @@ async def test_memusage_below_thresholds_logs_peak( with caplog.at_level(logging.INFO, logger="scrapy.extensions.memusage"): await crawler.crawl_async(url="data:,", loops=1) - assert crawler.stats assert crawler.stats.get_value("memusage/limit_reached") is None assert crawler.stats.get_value("memusage/warning_reached") is None assert crawler.stats.get_value("memusage/max") == 25 * MB @@ -129,7 +128,6 @@ async def test_memusage_limit_closes_spider_with_reason_and_error_log( with caplog.at_level(logging.ERROR, logger="scrapy.extensions.memusage"): await crawler.crawl_async(url="data:,", loops=100) - assert crawler.stats assert crawler.stats.get_value("memusage/limit_reached") == 1 assert crawler.stats.get_value("finish_reason") == "memusage_exceeded" assert any( @@ -167,7 +165,6 @@ async def test_memusage_warning_logs_but_allows_normal_finish( await crawler.crawl_async(url="data:,", loops=60) assert warning_signals == [1] - assert crawler.stats assert crawler.stats.get_value("memusage/warning_reached") == 1 assert crawler.stats.get_value("finish_reason") == "finished" warnings_logged = [ diff --git a/tests/test_extension_telnet.py b/tests/test_extension_telnet.py index ad59297b6..1e8771d65 100644 --- a/tests/test_extension_telnet.py +++ b/tests/test_extension_telnet.py @@ -119,7 +119,6 @@ async def test_telnet_vars() -> None: start_urls = ["data:,"] async def parse(self, response: Response) -> None: - assert self.crawler.extensions console = next( ext for ext in self.crawler.extensions.middlewares diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 8560dd364..67de762fc 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -310,7 +310,6 @@ class TestFeedExport(TestFeedExportBase): } crawler = get_crawler(ItemSpider, settings) yield crawler.crawl(mockserver=self.mockserver) - assert crawler.stats is not None assert "feedexport/success_count/FileFeedStorage" in crawler.stats.get_stats() assert crawler.stats.get_value("feedexport/success_count/FileFeedStorage") == 1 @@ -334,7 +333,6 @@ class TestFeedExport(TestFeedExportBase): side_effect=store, ): yield crawler.crawl(mockserver=self.mockserver) - assert crawler.stats is not None assert "feedexport/failed_count/FileFeedStorage" in crawler.stats.get_stats() assert crawler.stats.get_value("feedexport/failed_count/FileFeedStorage") == 1 @@ -352,7 +350,6 @@ class TestFeedExport(TestFeedExportBase): } crawler = get_crawler(ItemSpider, settings) yield crawler.crawl(mockserver=self.mockserver) - assert crawler.stats is not None assert "feedexport/success_count/FileFeedStorage" in crawler.stats.get_stats() assert "feedexport/success_count/StdoutFeedStorage" in crawler.stats.get_stats() assert crawler.stats.get_value("feedexport/success_count/FileFeedStorage") == 1 diff --git a/tests/test_feedexport_batch.py b/tests/test_feedexport_batch.py index 1958e2935..fecb33d07 100644 --- a/tests/test_feedexport_batch.py +++ b/tests/test_feedexport_batch.py @@ -420,7 +420,6 @@ class TestBatchDeliveries(TestFeedExportBase): } crawler = get_crawler(ItemSpider, settings) yield crawler.crawl(total=2, mockserver=self.mockserver) - assert crawler.stats assert "feedexport/success_count/FileFeedStorage" in crawler.stats.get_stats() assert crawler.stats.get_value("feedexport/success_count/FileFeedStorage") == 12 @@ -491,7 +490,6 @@ class TestBatchDeliveries(TestFeedExportBase): assert len(CustomS3FeedStorage.stubs) == len(items) for stub in CustomS3FeedStorage.stubs: stub.assert_no_pending_responses() - assert crawler.stats assert ( "feedexport/success_count/CustomS3FeedStorage" in crawler.stats.get_stats() ) diff --git a/tests/test_logstats.py b/tests/test_logstats.py index 35fd78283..09cafd218 100644 --- a/tests/test_logstats.py +++ b/tests/test_logstats.py @@ -15,7 +15,6 @@ class TestLogStats: def setup_method(self) -> None: self.crawler = get_crawler(SimpleSpider) self.spider = self.crawler._create_spider("spidey") - assert self.crawler.stats is not None self.stats = self.crawler.stats self.stats.set_value("response_received_count", 4802) diff --git a/tests/test_pipeline_crawl.py b/tests/test_pipeline_crawl.py index 8b522255a..529e7ffcb 100644 --- a/tests/test_pipeline_crawl.py +++ b/tests/test_pipeline_crawl.py @@ -140,7 +140,6 @@ class TestFileDownloadCrawl: assert not items[0][self.media_key] # check that there was 1 successful fetch and 3 other responses with non-200 code - assert crawler.stats assert crawler.stats.get_value("downloader/request_method_count/GET") == 4 assert crawler.stats.get_value("downloader/response_count") == 4 assert crawler.stats.get_value("downloader/response_status_count/200") == 1 @@ -208,7 +207,6 @@ class TestFileDownloadCrawl: mockserver=self.mockserver, ) self._assert_files_downloaded(self.items, caplog.text) - assert crawler.stats assert crawler.stats.get_value("downloader/response_status_count/302") == 3 @coroutine_test diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py index 231f6748f..30a42da48 100644 --- a/tests/test_pipeline_media.py +++ b/tests/test_pipeline_media.py @@ -82,7 +82,6 @@ class TestBaseMediaPipeline: self.pipe = build_from_crawler(self.pipeline_class, crawler) self.pipe.open_spider() self.info = self.pipe.spiderinfo - assert crawler.request_fingerprinter is not None self.fingerprint = crawler.request_fingerprinter.fingerprint @property diff --git a/tests/test_request_cb_kwargs.py b/tests/test_request_cb_kwargs.py index 6c26aa878..97dafc87c 100644 --- a/tests/test_request_cb_kwargs.py +++ b/tests/test_request_cb_kwargs.py @@ -70,7 +70,6 @@ class KeywordArgumentsSpider(MockServerSpider): checks: list[bool] = [] def _inc_checks(self, count: int = 1) -> None: - assert self.crawler.stats self.crawler.stats.inc_value("boolean_checks", count) async def start(self): @@ -171,7 +170,6 @@ class TestCallbackKeywordArguments: await crawler.crawl_async(mockserver=mockserver) assert isinstance(crawler.spider, KeywordArgumentsSpider) assert all(crawler.spider.checks) - assert crawler.stats assert len(crawler.spider.checks) == crawler.stats.get_value("boolean_checks") # check exceptions for argument mismatch exceptions = {} diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py index ff92b1133..508147a7c 100644 --- a/tests/test_scheduler.py +++ b/tests/test_scheduler.py @@ -54,7 +54,6 @@ async def create_scheduler( finally: await ensure_awaitable(scheduler.close("finished")) await mock_crawler.stop_async() - assert mock_crawler.engine mock_crawler.engine.downloader.close() @@ -254,7 +253,6 @@ class DownloaderAwareSchedulerTestMixin(TestSchedulerBase): dequeued_slots: list[str] = [] requests: list[Request] = [] assert scheduler.crawler - assert scheduler.crawler.engine downloader = scheduler.crawler.engine.downloader assert isinstance(downloader, MockDownloader) while scheduler.has_pending_requests(): @@ -320,7 +318,6 @@ class TestIntegrationWithDownloaderAwareInMemory: url = mockserver.url("/status?n=200", is_secure=False) start_urls = [url] * 6 yield self.crawler.crawl(start_urls) - assert self.crawler.stats assert self.crawler.stats.get_value("downloader/response_count") == len( start_urls ) diff --git a/tests/test_spider.py b/tests/test_spider.py index 38cb8da18..6f7f171a3 100644 --- a/tests/test_spider.py +++ b/tests/test_spider.py @@ -41,7 +41,7 @@ class TestXMLFeedSpider(TestSpiderBase): """ response = XmlResponse(url="http://example.com/sitemap.xml", body=body) - class _XMLSpider(self.spider_class): + class _XMLSpider(self.spider_class): # type: ignore[name-defined,misc] itertag = "url" namespaces = ( ("a", "http://www.google.com/schemas/sitemap/0.84"), @@ -100,7 +100,6 @@ class TestXMLFeedSpider(TestSpiderBase): items, crawler = await crawl_items(_Spider, mockserver) assert items == [] - assert crawler.stats assert crawler.stats.get_value("spider_exceptions/NotImplementedError") == 1 @coroutine_test @@ -135,7 +134,6 @@ class TestXMLFeedSpider(TestSpiderBase): items, crawler = await crawl_items(_Spider, mockserver) assert items == [] - assert crawler.stats assert crawler.stats.get_value("spider_exceptions/NotSupported") == 1 @pytest.mark.parametrize("feed_iterator", ["xml", "html"]) @@ -155,7 +153,6 @@ class TestXMLFeedSpider(TestSpiderBase): items, crawler = await crawl_items(_Spider, mockserver) assert items == [] - assert crawler.stats assert crawler.stats.get_value("spider_exceptions/ValueError") == 1 @@ -166,7 +163,7 @@ class TestCSVFeedSpider(TestSpiderBase): body = get_testdata("feeds", "feed-sample6.csv") response = Response("http://example.org/dummy.csv", body=body) - class _CrawlSpider(self.spider_class): + class _CrawlSpider(self.spider_class): # type: ignore[name-defined,misc] name = "test" delimiter = "," quotechar = "'" @@ -206,7 +203,6 @@ class TestCSVFeedSpider(TestSpiderBase): items, crawler = await crawl_items(_Spider, mockserver) assert items == [] - assert crawler.stats assert crawler.stats.get_value("spider_exceptions/NotImplementedError") == 1 diff --git a/tests/test_spider_crawl.py b/tests/test_spider_crawl.py index 9d5f0548b..ce556d0d1 100644 --- a/tests/test_spider_crawl.py +++ b/tests/test_spider_crawl.py @@ -2,6 +2,7 @@ from __future__ import annotations import re import warnings +from typing import TYPE_CHECKING, Any import pytest from w3lib.url import safe_url_string @@ -14,6 +15,12 @@ from scrapy.utils.test import get_crawler from tests.utils.bases.spider import TestSpiderBase from tests.utils.decorators import coroutine_test +if TYPE_CHECKING: + from scrapy.http import Response + from scrapy.http.request import CallbackT + +URL = "https://www.example.com" + class TestCrawlSpider(TestSpiderBase): test_body = b"""Page title @@ -33,7 +40,7 @@ class TestCrawlSpider(TestSpiderBase): "http://example.org/somepage/index.html", body=self.test_body ) - class _CrawlSpider(self.spider_class): + class _CrawlSpider(self.spider_class): # type: ignore[name-defined,misc] name = "test" allowed_domains = ["example.org"] rules = (Rule(),) @@ -53,7 +60,7 @@ class TestCrawlSpider(TestSpiderBase): "http://example.org/somepage/index.html", body=self.test_body ) - class _CrawlSpider(self.spider_class): + class _CrawlSpider(self.spider_class): # type: ignore[name-defined,misc] name = "test" allowed_domains = ["example.org"] rules = (Rule(LinkExtractor(), process_links="dummy_process_links"),) @@ -76,7 +83,7 @@ class TestCrawlSpider(TestSpiderBase): "http://example.org/somepage/index.html", body=self.test_body ) - class _CrawlSpider(self.spider_class): + class _CrawlSpider(self.spider_class): # type: ignore[name-defined,misc] name = "test" allowed_domains = ["example.org"] rules = (Rule(LinkExtractor(), process_links="filter_process_links"),) @@ -99,7 +106,7 @@ class TestCrawlSpider(TestSpiderBase): "http://example.org/somepage/index.html", body=self.test_body ) - class _CrawlSpider(self.spider_class): + class _CrawlSpider(self.spider_class): # type: ignore[name-defined,misc] name = "test" allowed_domains = ["example.org"] rules = (Rule(LinkExtractor(), process_links="dummy_process_links"),) @@ -125,7 +132,7 @@ class TestCrawlSpider(TestSpiderBase): def process_request_change_domain(request, response): return request.replace(url=request.url.replace(".org", ".com")) - class _CrawlSpider(self.spider_class): + class _CrawlSpider(self.spider_class): # type: ignore[name-defined,misc] name = "test" allowed_domains = ["example.org"] rules = ( @@ -151,7 +158,7 @@ class TestCrawlSpider(TestSpiderBase): request.meta["response_class"] = response.__class__.__name__ return request - class _CrawlSpider(self.spider_class): + class _CrawlSpider(self.spider_class): # type: ignore[name-defined,misc] name = "test" allowed_domains = ["example.org"] rules = ( @@ -180,7 +187,7 @@ class TestCrawlSpider(TestSpiderBase): "http://example.org/somepage/index.html", body=self.test_body ) - class _CrawlSpider(self.spider_class): + class _CrawlSpider(self.spider_class): # type: ignore[name-defined,misc] name = "test" allowed_domains = ["example.org"] rules = (Rule(LinkExtractor(), process_request="process_request_upper"),) @@ -203,7 +210,7 @@ class TestCrawlSpider(TestSpiderBase): "http://example.org/somepage/index.html", body=self.test_body ) - class _CrawlSpider(self.spider_class): + class _CrawlSpider(self.spider_class): # type: ignore[name-defined,misc] name = "test" allowed_domains = ["example.org"] rules = ( @@ -248,7 +255,6 @@ class TestCrawlSpider(TestSpiderBase): def test_parse_response_use(self): class _CrawlSpider(CrawlSpider): name = "test" - start_urls = "https://www.example.com" _follow_links = False with warnings.catch_warnings(): @@ -258,17 +264,20 @@ class TestCrawlSpider(TestSpiderBase): ScrapyDeprecationWarning, match=r"CrawlSpider\._parse_response method is deprecated", ): - spider._parse_response( - TextResponse(spider.start_urls, body=b""), None, None - ) + spider._parse_response(TextResponse(URL, body=b""), None, {}) def test_parse_response_override(self): class _CrawlSpider(CrawlSpider): - def _parse_response(self, response, callback, cb_kwargs, follow=True): + def _parse_response( + self, + response: Response, + callback: CallbackT | None, + cb_kwargs: dict[str, Any], + follow: bool = True, + ) -> Any: pass name = "test" - start_urls = "https://www.example.com" _follow_links = False with pytest.warns( @@ -278,21 +287,16 @@ class TestCrawlSpider(TestSpiderBase): spider = _CrawlSpider() with warnings.catch_warnings(): warnings.simplefilter("error", category=ScrapyDeprecationWarning) - spider._parse_response( - TextResponse(spider.start_urls, body=b""), None, None - ) + spider._parse_response(TextResponse(URL, body=b""), None, {}) def test_parse_with_rules(self): class _CrawlSpider(CrawlSpider): name = "test" - start_urls = "https://www.example.com" with warnings.catch_warnings(): warnings.simplefilter("error", category=ScrapyDeprecationWarning) spider = _CrawlSpider() - spider.parse_with_rules( - TextResponse(spider.start_urls, body=b""), None, None - ) + spider.parse_with_rules(TextResponse(URL, body=b""), None, {}) @coroutine_test async def test_parse_with_rules_without_callback(self): @@ -327,12 +331,13 @@ class TestCrawlSpider(TestSpiderBase): allowed_domains = ["example.org"] rules = (Rule(),) + def callback(response: Response) -> list[Any]: + return [item] + spider = _CrawlSpider.from_crawler(get_crawler(_CrawlSpider)) results = [ r - async for r in spider.parse_with_rules( - response, lambda response: [item], {}, follow=False - ) + async for r in spider.parse_with_rules(response, callback, {}, follow=False) ] assert results == [item] diff --git a/tests/test_spider_sitemap.py b/tests/test_spider_sitemap.py index 2f1ccab81..630ab5ca4 100644 --- a/tests/test_spider_sitemap.py +++ b/tests/test_spider_sitemap.py @@ -55,7 +55,7 @@ class TestSitemapSpider(TestSpiderBase): assert spider._get_sitemap_body(response) == body def test_get_sitemap_body(self): - r = XmlResponse(url="http://www.example.com/", body=self.BODY) + r: Response = XmlResponse(url="http://www.example.com/", body=self.BODY) self.assertSitemapBody(r, self.BODY) r = HtmlResponse(url="http://www.example.com/", body=self.BODY) @@ -166,7 +166,7 @@ Sitemap: /sitemap-relative-url.xml """ - class FilteredSitemapSpider(self.spider_class): + class FilteredSitemapSpider(self.spider_class): # type: ignore[name-defined,misc] def sitemap_filter(self, entries): for entry in entries: date_time = datetime.strptime(entry["lastmod"], "%Y-%m-%d") @@ -201,7 +201,7 @@ Sitemap: /sitemap-relative-url.xml """ - class FilteredSitemapSpider(self.spider_class): + class FilteredSitemapSpider(self.spider_class): # type: ignore[name-defined,misc] def sitemap_filter(self, entries): for entry in entries: alternate_links = entry.get("alternate", ()) @@ -235,7 +235,7 @@ Sitemap: /sitemap-relative-url.xml """ - class FilteredSitemapSpider(self.spider_class): + class FilteredSitemapSpider(self.spider_class): # type: ignore[name-defined,misc] def sitemap_filter(self, entries): for entry in entries: date_time = datetime.strptime( @@ -285,6 +285,7 @@ Sitemap: /sitemap-relative-url.xml sitemap_rules = [("", parse_item)] def raw_body(self): + assert self.mockserver loc = self.mockserver.url("/text") return ( '' @@ -303,6 +304,7 @@ Sitemap: /sitemap-relative-url.xml yield {"url": response.url} def raw_body(self): + assert self.mockserver loc = self.mockserver.url("/text") return ( '' @@ -363,7 +365,7 @@ Sitemap: /sitemap-relative-url.xml """ r = TextResponse(url="http://www.example.com/sitemap.xml", body=sitemap) - class _FollowSpider(self.spider_class): + class _FollowSpider(self.spider_class): # type: ignore[name-defined,misc] sitemap_follow = [follow] spider = _FollowSpider("example.com") @@ -382,7 +384,7 @@ Sitemap: /sitemap-relative-url.xml @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") def test_compression_bomb_spider_attr(self): - class DownloadMaxSizeSpider(self.spider_class): + class DownloadMaxSizeSpider(self.spider_class): # type: ignore[name-defined,misc] download_maxsize = 10_000_000 crawler = get_crawler() @@ -485,7 +487,7 @@ Sitemap: /sitemap-relative-url.xml @coroutine_test async def test_sitemap_urls(self): - class TestSpider(self.spider_class): + class TestSpider(self.spider_class): # type: ignore[name-defined,misc] name = "test" sitemap_urls = ["https://toscrape.com/sitemap.xml"] diff --git a/tests/test_spider_start.py b/tests/test_spider_start.py index 7cef2fefa..fb72ece6c 100644 --- a/tests/test_spider_start.py +++ b/tests/test_spider_start.py @@ -34,7 +34,6 @@ class TestMain: crawler = get_crawler(spider) crawler.signals.connect(track_item, signals.item_scraped) await crawler.crawl_async() - assert crawler.stats assert crawler.stats.get_value("finish_reason") == "finished" assert actual_items == expected_items diff --git a/tests/test_spidermiddleware_depth.py b/tests/test_spidermiddleware_depth.py index 69a41c0ff..9d6f4666b 100644 --- a/tests/test_spidermiddleware_depth.py +++ b/tests/test_spidermiddleware_depth.py @@ -26,7 +26,6 @@ def crawler() -> Crawler: @pytest.fixture def stats(crawler: Crawler) -> Generator[StatsCollector]: - assert crawler.stats is not None crawler.stats.open_spider() yield crawler.stats @@ -120,7 +119,6 @@ def test_priority_and_non_verbose_stats() -> None: Spider, {"DEPTH_LIMIT": 0, "DEPTH_STATS_VERBOSE": False, "DEPTH_PRIORITY": 10}, ) - assert crawler.stats is not None crawler.stats.open_spider() try: mw = build_from_crawler(DepthMiddleware, crawler) diff --git a/tests/test_spidermiddleware_httperror.py b/tests/test_spidermiddleware_httperror.py index d8d5b278b..43b41ce3f 100644 --- a/tests/test_spidermiddleware_httperror.py +++ b/tests/test_spidermiddleware_httperror.py @@ -205,7 +205,6 @@ class TestHttpErrorMiddlewareIntegrational: assert crawler.spider.parsed == {"200"} assert crawler.spider.failed == {"404", "402", "500"} - assert crawler.stats get_value = crawler.stats.get_value assert get_value("httperror/response_ignored_count") == 3 assert get_value("httperror/response_ignored_status_count/404") == 1 diff --git a/tests/test_spidermiddleware_output_chain.py b/tests/test_spidermiddleware_output_chain.py index dacc90b27..74051d1db 100644 --- a/tests/test_spidermiddleware_output_chain.py +++ b/tests/test_spidermiddleware_output_chain.py @@ -1,16 +1,21 @@ from __future__ import annotations import logging -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any from scrapy import Request, Spider from scrapy.utils.test import get_crawler from tests.mockserver.http import MockServer +from tests.spiders import MockServerSpider from tests.utils.decorators import coroutine_test if TYPE_CHECKING: + from collections.abc import AsyncIterator, Iterator + import pytest + from scrapy.http import Response + class _BaseSpiderMiddleware: def __init__(self, crawler): @@ -41,7 +46,7 @@ class RecoveryMiddleware(_BaseSpiderMiddleware): ] -class RecoverySpider(Spider): +class RecoverySpider(MockServerSpider): name = "RecoverySpider" custom_settings = { "SPIDER_MIDDLEWARES_BASE": {}, @@ -50,10 +55,11 @@ class RecoverySpider(Spider): }, } - async def start(self): + async def start(self) -> AsyncIterator[Any]: + assert self.mockserver yield Request(self.mockserver.url("/status?n=200")) - def parse(self, response): + def parse(self, response: Response) -> Iterator[Any]: yield {"test": 1} self.logger.info("DONT_FAIL: %s", response.meta.get("dont_fail")) if not response.meta.get("dont_fail"): @@ -63,7 +69,7 @@ class RecoverySpider(Spider): class RecoveryAsyncGenSpider(RecoverySpider): name = "RecoveryAsyncGenSpider" - async def parse(self, response): + async def parse(self, response: Response) -> AsyncIterator[Any]: # type: ignore[override] for r in super().parse(response): yield r @@ -76,7 +82,7 @@ class FailProcessSpiderInputMiddleware(_BaseSpiderMiddleware): raise IndexError -class ProcessSpiderInputSpiderWithoutErrback(Spider): +class ProcessSpiderInputSpiderWithoutErrback(MockServerSpider): name = "ProcessSpiderInputSpiderWithoutErrback" custom_settings = { "SPIDER_MIDDLEWARES": { @@ -87,7 +93,8 @@ class ProcessSpiderInputSpiderWithoutErrback(Spider): } } - async def start(self): + async def start(self) -> AsyncIterator[Any]: + assert self.mockserver yield Request(url=self.mockserver.url("/status?n=200"), callback=self.parse) def parse(self, response): @@ -97,7 +104,8 @@ class ProcessSpiderInputSpiderWithoutErrback(Spider): class ProcessSpiderInputSpiderWithErrback(ProcessSpiderInputSpiderWithoutErrback): name = "ProcessSpiderInputSpiderWithErrback" - async def start(self): + async def start(self) -> AsyncIterator[Any]: + assert self.mockserver yield Request( self.mockserver.url("/status?n=200"), self.parse, errback=self.errback ) @@ -109,7 +117,7 @@ class ProcessSpiderInputSpiderWithErrback(ProcessSpiderInputSpiderWithoutErrback # ================================================================================ # (2) exceptions from a spider callback (generator) -class GeneratorCallbackSpider(Spider): +class GeneratorCallbackSpider(MockServerSpider): name = "GeneratorCallbackSpider" custom_settings = { "SPIDER_MIDDLEWARES": { @@ -117,7 +125,8 @@ class GeneratorCallbackSpider(Spider): }, } - async def start(self): + async def start(self) -> AsyncIterator[Any]: + assert self.mockserver yield Request(self.mockserver.url("/status?n=200")) def parse(self, response): @@ -146,7 +155,7 @@ class GeneratorCallbackSpiderMiddlewareRightAfterSpider(GeneratorCallbackSpider) # ================================================================================ # (3) exceptions from a spider callback (not a generator) -class NotGeneratorCallbackSpider(Spider): +class NotGeneratorCallbackSpider(MockServerSpider): name = "NotGeneratorCallbackSpider" custom_settings = { "SPIDER_MIDDLEWARES": { @@ -154,7 +163,8 @@ class NotGeneratorCallbackSpider(Spider): }, } - async def start(self): + async def start(self) -> AsyncIterator[Any]: + assert self.mockserver yield Request(self.mockserver.url("/status?n=200")) def parse(self, response): @@ -224,7 +234,7 @@ class GeneratorDoNothingAfterRecoveryMiddleware(_GeneratorDoNothingMiddleware): pass -class GeneratorOutputChainSpider(Spider): +class GeneratorOutputChainSpider(MockServerSpider): name = "GeneratorOutputChainSpider" custom_settings = { "SPIDER_MIDDLEWARES": { @@ -235,7 +245,8 @@ class GeneratorOutputChainSpider(Spider): }, } - async def start(self): + async def start(self) -> AsyncIterator[Any]: + assert self.mockserver yield Request(self.mockserver.url("/status?n=200")) def parse(self, response): diff --git a/tests/test_spidermiddleware_process_start.py b/tests/test_spidermiddleware_process_start.py index 321139da3..46414bb59 100644 --- a/tests/test_spidermiddleware_process_start.py +++ b/tests/test_spidermiddleware_process_start.py @@ -1,4 +1,7 @@ +from __future__ import annotations + from asyncio import sleep +from typing import TYPE_CHECKING, Any import pytest @@ -9,6 +12,11 @@ from scrapy.utils.test import get_crawler from .utils import twisted_sleep from .utils.decorators import coroutine_test +if TYPE_CHECKING: + from collections.abc import AsyncIterator + + from scrapy.http import Response + ITEM_A = {"id": "a"} ITEM_B = {"id": "b"} ITEM_C = {"id": "c"} @@ -17,20 +25,20 @@ SLEEP_SECONDS = 0.1 class AsyncioSleepSpiderMiddleware: - async def process_start(self, start): + async def process_start(self, start: AsyncIterator[Any]) -> AsyncIterator[Any]: await sleep(SLEEP_SECONDS) async for item_or_request in start: yield item_or_request class NoOpSpiderMiddleware: - async def process_start(self, start): + async def process_start(self, start: AsyncIterator[Any]) -> AsyncIterator[Any]: async for item_or_request in start: yield item_or_request class TwistedSleepSpiderMiddleware: - async def process_start(self, start): + async def process_start(self, start: AsyncIterator[Any]) -> AsyncIterator[Any]: await maybe_deferred_to_future(twisted_sleep(SLEEP_SECONDS)) async for item_or_request in start: yield item_or_request @@ -42,12 +50,12 @@ class TwistedSleepSpiderMiddleware: class ModernWrapSpider(Spider): name = "test" - async def start(self): + async def start(self) -> AsyncIterator[Any]: yield ITEM_B class ModernWrapSpiderMiddleware: - async def process_start(self, start): + async def process_start(self, start: AsyncIterator[Any]) -> AsyncIterator[Any]: yield ITEM_A async for item_or_request in start: yield item_or_request @@ -55,10 +63,15 @@ class ModernWrapSpiderMiddleware: class TestMain: - async def _test(self, spider_middlewares, spider_cls, expected_items): + async def _test( + self, + spider_middlewares: list[type], + spider_cls: type[Spider], + expected_items: list[Any], + ) -> None: actual_items = [] - def track_item(item, response, spider): + def track_item(item: Any, response: Response, spider: Spider) -> None: actual_items.append(item) settings = { @@ -70,7 +83,12 @@ class TestMain: assert crawler.stats.get_value("finish_reason") == "finished" assert actual_items == expected_items, f"{actual_items=} != {expected_items=}" - async def _test_wrap(self, spider_middleware, spider_cls, expected_items=None): + async def _test_wrap( + self, + spider_middleware: type, + spider_cls: type[Spider], + expected_items: list[Any] | None = None, + ) -> None: expected_items = expected_items or [ITEM_A, ITEM_B, ITEM_C] await self._test([spider_middleware], spider_cls, expected_items) @@ -78,11 +96,11 @@ class TestMain: async def test_modern_mw_modern_spider(self): await self._test_wrap(ModernWrapSpiderMiddleware, ModernWrapSpider) - async def _test_sleep(self, spider_middlewares): + async def _test_sleep(self, spider_middlewares: list[type]) -> None: class TestSpider(Spider): name = "test" - async def start(self): + async def start(self) -> AsyncIterator[Any]: yield ITEM_A await self._test(spider_middlewares, TestSpider, [ITEM_A]) diff --git a/tests/test_spidermiddleware_urllength.py b/tests/test_spidermiddleware_urllength.py index 1090b3e84..2620327af 100644 --- a/tests/test_spidermiddleware_urllength.py +++ b/tests/test_spidermiddleware_urllength.py @@ -31,7 +31,6 @@ def crawler() -> Crawler: @pytest.fixture def stats(crawler: Crawler) -> StatsCollector: - assert crawler.stats is not None return crawler.stats diff --git a/tests/test_utils_log.py b/tests/test_utils_log.py index 3ea82ba8c..859eb13ac 100644 --- a/tests/test_utils_log.py +++ b/tests/test_utils_log.py @@ -91,7 +91,6 @@ class TestLogCounterHandler: logger.removeHandler(handler) def test_init(self, crawler: Crawler, logger: logging.Logger) -> None: - assert crawler.stats assert crawler.stats.get_value("log_count/DEBUG") is None assert crawler.stats.get_value("log_count/INFO") is None assert crawler.stats.get_value("log_count/WARNING") is None @@ -100,12 +99,10 @@ class TestLogCounterHandler: def test_accepted_level(self, crawler: Crawler, logger: logging.Logger) -> None: logger.error("test log msg") - assert crawler.stats assert crawler.stats.get_value("log_count/ERROR") == 1 def test_filtered_out_level(self, crawler: Crawler, logger: logging.Logger) -> None: logger.debug("test log msg") - assert crawler.stats assert crawler.stats.get_value("log_count/DEBUG") is None diff --git a/tests/test_utils_request.py b/tests/test_utils_request.py index 935447bc4..6ca608053 100644 --- a/tests/test_utils_request.py +++ b/tests/test_utils_request.py @@ -274,7 +274,6 @@ class TestRequestFingerprinter: def test_fingerprint(self): crawler = get_crawler() request = Request("https://example.com") - assert crawler.request_fingerprinter assert crawler.request_fingerprinter.fingerprint(request) == fingerprint( request ) @@ -291,7 +290,6 @@ class TestCustomRequestFingerprinter: } crawler = get_crawler(settings_dict=settings) - assert crawler.request_fingerprinter r1 = Request("http://www.example.com", headers={"X-ID": "1"}) fp1 = crawler.request_fingerprinter.fingerprint(r1) r2 = Request("http://www.example.com", headers={"X-ID": "2"}) @@ -314,7 +312,6 @@ class TestCustomRequestFingerprinter: } crawler = get_crawler(settings_dict=settings) - assert crawler.request_fingerprinter r1 = Request("http://www.example.com?a=1&a=2") fp1 = crawler.request_fingerprinter.fingerprint(r1) r2 = Request("http://www.example.com?a=2&a=1") @@ -333,7 +330,6 @@ class TestCustomRequestFingerprinter: } crawler = get_crawler(settings_dict=settings) - assert crawler.request_fingerprinter r1 = Request("http://www.example.com") fp1 = crawler.request_fingerprinter.fingerprint(r1) r2 = Request("http://www.example.com", meta={"fingerprint": "a"}) @@ -365,7 +361,6 @@ class TestCustomRequestFingerprinter: } crawler = get_crawler(settings_dict=settings) - assert crawler.request_fingerprinter request = Request("http://www.example.com") fingerprint = crawler.request_fingerprinter.fingerprint(request) assert fingerprint == settings["FINGERPRINT"] diff --git a/tests/utils/bases/spider.py b/tests/utils/bases/spider.py index 799c1820d..224606bf6 100644 --- a/tests/utils/bases/spider.py +++ b/tests/utils/bases/spider.py @@ -32,7 +32,7 @@ class TestSpiderBase(ABC): def test_spider_args(self): """``__init__`` method arguments are assigned to spider attributes""" spider = self.spider_class("example.com", foo="bar") - assert spider.foo == "bar" + assert spider.foo == "bar" # type: ignore[attr-defined] def test_spider_without_name(self): """``__init__`` raises when the name is not provided.""" @@ -58,7 +58,7 @@ class TestSpiderBase(ABC): mock_init.assert_called_once_with("example.com", foo="bar") def test_closed_signal_call(self): - class TestSpider(self.spider_class): + class TestSpider(self.spider_class): # type: ignore[name-defined,misc] closed_called = False def closed(self, reason): @@ -92,7 +92,7 @@ class TestSpiderBase(ABC): **get_reactor_settings(), } - class TestSpider(self.spider_class): + class TestSpider(self.spider_class): # type: ignore[name-defined,misc] name = "test" custom_settings = spider_settings @@ -127,5 +127,5 @@ class TestSpiderBase(ABC): ScrapyDeprecationWarning, match=r"Spider.log\(\) is deprecated" ), ): - spider.log("test log msg", "INFO") + spider.log("test log msg", "INFO") # type: ignore[arg-type] mock_logger.log.assert_called_once_with("INFO", "test log msg")