diff --git a/pyproject.toml b/pyproject.toml index 5ec1d05a4..e996db7e5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -116,13 +116,9 @@ allow_incomplete_defs = true # 59 errors # TODO [[tool.mypy.overrides]] module = [ - "tests.spiders", - "tests.test_closespider", "tests.test_cmdline", - "tests.test_contracts", "tests.test_downloaderslotssettings", "tests.test_dupefilters", - "tests.test_engine_loop", "tests.test_extension_statsmailer", "tests.test_extension_throttle", "tests.test_linkextractors", @@ -131,15 +127,9 @@ module = [ "tests.test_pqueues", "tests.test_scheduler_base", "tests.test_settings", - "tests.test_spider", - "tests.test_spider_crawl", - "tests.test_spidermiddleware_output_chain", - "tests.test_spidermiddleware_process_start", - "tests.test_spider_sitemap", "tests.test_squeues", "tests.test_squeues_request", "tests.test_stats", - "tests.utils.bases.spider", ] check_untyped_defs = false diff --git a/tests/spiders.py b/tests/spiders.py index 7c7d3007c..3ff30ce03 100644 --- a/tests/spiders.py +++ b/tests/spiders.py @@ -13,22 +13,31 @@ from twisted.internet import defer from scrapy import signals from scrapy.exceptions import StopDownload -from scrapy.http import Request +from scrapy.http import Request, TextResponse from scrapy.item import Item from scrapy.linkextractors import LinkExtractor +from scrapy.spidermiddlewares.httperror import HttpError from scrapy.spiders import Spider from scrapy.spiders.crawl import CrawlSpider, Rule from scrapy.utils.defer import deferred_to_future, maybe_deferred_to_future from scrapy.utils.test import get_from_asyncio_queue if TYPE_CHECKING: + from collections.abc import AsyncIterator, Callable, Iterator + + from twisted.python.failure import Failure + from typing_extensions import Self + + from scrapy.crawler import Crawler + from scrapy.http import Headers, Response + from scrapy.http.request import CallbackT from tests.mockserver.http import MockServer class MockServerSpider(Spider): def __init__( self, - *args, + *args: Any, mockserver: MockServer | None = None, is_secure: bool = False, **kwargs: Any, @@ -74,7 +83,7 @@ class MetaSpider(MockServerSpider): super().__init__(*args, **kwargs) self.meta: dict[str, Any] = {} - def closed(self, reason): + def closed(self, reason: str) -> None: self.meta["close_reason"] = reason @@ -83,16 +92,24 @@ class FollowAllSpider(MetaSpider): link_extractor = LinkExtractor() def __init__( - self, total=10, show=20, order="rand", maxlatency=0.0, *args, **kwargs + self, + total: int = 10, + show: int = 20, + order: str = "rand", + maxlatency: float = 0.0, + *args: Any, + **kwargs: Any, ): super().__init__(*args, **kwargs) - self.urls_visited = [] - self.times = [] + self.urls_visited: list[str] = [] + self.times: list[float] = [] qargs = {"total": total, "show": show, "order": order, "maxlatency": maxlatency} + assert self.mockserver url = self.mockserver.url(f"/follow?{urlencode(qargs, doseq=True)}") self.start_urls = [url] - def parse(self, response): + def parse(self, response: Response) -> Iterator[Any]: + assert isinstance(response, TextResponse) self.urls_visited.append(response.url) self.times.append(time.time()) for link in self.link_extractor.extract_links(response): @@ -102,50 +119,52 @@ class FollowAllSpider(MetaSpider): class DelaySpider(MetaSpider): name = "delay" - def __init__(self, n=1, b=0, *args, **kwargs): + def __init__(self, n: float = 1, b: float = 0, *args: Any, **kwargs: Any): super().__init__(*args, **kwargs) self.n = n self.b = b - self.t1 = self.t2 = self.t2_err = 0 + self.t1 = self.t2 = self.t2_err = 0.0 - async def start(self): + async def start(self) -> AsyncIterator[Any]: self.t1 = time.time() + assert self.mockserver url = self.mockserver.url(f"/delay?n={self.n}&b={self.b}") yield Request(url, callback=self.parse, errback=self.errback) - def parse(self, response): + def parse(self, response: Response) -> Any: self.t2 = time.time() - def errback(self, failure): + def errback(self, failure: Failure) -> None: self.t2_err = time.time() class LogSpider(MetaSpider): name = "log_spider" - def log_debug(self, message: str, extra: dict[str, Any] | None = None): + def log_debug(self, message: str, extra: dict[str, Any] | None = None) -> None: self.logger.debug(message, extra=extra) - def log_info(self, message: str, extra: dict[str, Any] | None = None): + def log_info(self, message: str, extra: dict[str, Any] | None = None) -> None: self.logger.info(message, extra=extra) - def log_warning(self, message: str, extra: dict[str, Any] | None = None): + def log_warning(self, message: str, extra: dict[str, Any] | None = None) -> None: self.logger.warning(message, extra=extra) - def log_error(self, message: str, extra: dict[str, Any] | None = None): + def log_error(self, message: str, extra: dict[str, Any] | None = None) -> None: self.logger.error(message, extra=extra) - def log_critical(self, message: str, extra: dict[str, Any] | None = None): + def log_critical(self, message: str, extra: dict[str, Any] | None = None) -> None: self.logger.critical(message, extra=extra) - def parse(self, response): + def parse(self, response: Response) -> None: pass class SlowSpider(DelaySpider): name = "slow" - async def start(self): + async def start(self) -> AsyncIterator[Any]: + assert self.mockserver # 1st response is fast url = self.mockserver.url("/delay?n=0&b=0") yield Request(url, callback=self.parse, errback=self.errback) @@ -154,25 +173,25 @@ class SlowSpider(DelaySpider): url = self.mockserver.url(f"/delay?n={self.n}&b={self.b}") yield Request(url, callback=self.parse, errback=self.errback) - def parse(self, response): + def parse(self, response: Response) -> Iterator[Any]: yield Item() class SimpleSpider(MetaSpider): name = "simple" - def __init__(self, url="http://localhost:8998", *args, **kwargs): + def __init__(self, url: str = "http://localhost:8998", *args: Any, **kwargs: Any): super().__init__(*args, **kwargs) self.start_urls = [url] - def parse(self, response): + def parse(self, response: Response) -> Any: self.logger.info(f"Got response {response.status}") class AsyncDefSpider(SimpleSpider): name = "asyncdef" - async def parse(self, response): + async def parse(self, response: Response) -> None: await defer.succeed(42) self.logger.info(f"Got response {response.status}") @@ -180,7 +199,7 @@ class AsyncDefSpider(SimpleSpider): class AsyncDefAsyncioSpider(SimpleSpider): name = "asyncdef_asyncio" - async def parse(self, response): + async def parse(self, response: Response) -> None: await asyncio.sleep(0.2) status = await get_from_asyncio_queue(response.status) self.logger.info(f"Got response {status}") @@ -189,7 +208,7 @@ class AsyncDefAsyncioSpider(SimpleSpider): class AsyncDefAsyncioReturnSpider(SimpleSpider): name = "asyncdef_asyncio_return" - async def parse(self, response): + async def parse(self, response: Response) -> Any: await asyncio.sleep(0.2) status = await get_from_asyncio_queue(response.status) self.logger.info(f"Got response {status}") @@ -199,7 +218,7 @@ class AsyncDefAsyncioReturnSpider(SimpleSpider): class AsyncDefAsyncioReturnSingleElementSpider(SimpleSpider): name = "asyncdef_asyncio_return_single_element" - async def parse(self, response): + async def parse(self, response: Response) -> Any: await asyncio.sleep(0.1) status = await get_from_asyncio_queue(response.status) self.logger.info(f"Got response {status}") @@ -209,7 +228,7 @@ class AsyncDefAsyncioReturnSingleElementSpider(SimpleSpider): class AsyncDefAsyncioReqsReturnSpider(SimpleSpider): name = "asyncdef_asyncio_reqs_return" - async def parse(self, response): + async def parse(self, response: Response) -> Any: await asyncio.sleep(0.2) req_id = response.meta.get("req_id", 0) status = await get_from_asyncio_queue(response.status) @@ -226,7 +245,7 @@ class AsyncDefAsyncioReqsReturnSpider(SimpleSpider): class AsyncDefAsyncioGenExcSpider(SimpleSpider): name = "asyncdef_asyncio_gen_exc" - async def parse(self, response): + async def parse(self, response: Response) -> AsyncIterator[Any]: for i in range(10): await asyncio.sleep(0.1) yield {"foo": i} @@ -237,7 +256,7 @@ class AsyncDefAsyncioGenExcSpider(SimpleSpider): class AsyncDefDeferredDirectSpider(SimpleSpider): name = "asyncdef_deferred_direct" - async def parse(self, response): + async def parse(self, response: Response) -> AsyncIterator[Any]: await defer.succeed(None) yield {"code": 200} @@ -245,7 +264,7 @@ class AsyncDefDeferredDirectSpider(SimpleSpider): class AsyncDefDeferredWrappedSpider(SimpleSpider): name = "asyncdef_deferred_wrapped" - async def parse(self, response): + async def parse(self, response: Response) -> AsyncIterator[Any]: await deferred_to_future(defer.succeed(None)) yield {"code": 200} @@ -253,7 +272,7 @@ class AsyncDefDeferredWrappedSpider(SimpleSpider): class AsyncDefDeferredMaybeWrappedSpider(SimpleSpider): name = "asyncdef_deferred_maybe_wrapped" - async def parse(self, response): + async def parse(self, response: Response) -> AsyncIterator[Any]: await maybe_deferred_to_future(defer.succeed(None)) yield {"code": 200} @@ -261,7 +280,7 @@ class AsyncDefDeferredMaybeWrappedSpider(SimpleSpider): class AsyncDefAsyncioGenSpider(SimpleSpider): name = "asyncdef_asyncio_gen" - async def parse(self, response): + async def parse(self, response: Response) -> AsyncIterator[Any]: await asyncio.sleep(0.2) yield {"foo": 42} self.logger.info(f"Got response {response.status}") @@ -270,7 +289,7 @@ class AsyncDefAsyncioGenSpider(SimpleSpider): class AsyncDefAsyncioGenLoopSpider(SimpleSpider): name = "asyncdef_asyncio_gen_loop" - async def parse(self, response): + async def parse(self, response: Response) -> AsyncIterator[Any]: for i in range(10): await asyncio.sleep(0.1) yield {"foo": i} @@ -283,7 +302,8 @@ class AsyncDefAsyncioGenComplexSpider(SimpleSpider): following_reqs = 3 depth = 2 - def _get_req(self, index, cb=None): + def _get_req(self, index: int, cb: CallbackT | None = None) -> Request: + assert self.mockserver return Request( self.mockserver.url(f"/status?n=200&request={index}"), meta={"index": index}, @@ -291,11 +311,11 @@ class AsyncDefAsyncioGenComplexSpider(SimpleSpider): callback=cb, ) - async def start(self): + async def start(self) -> AsyncIterator[Any]: for i in range(1, self.initial_reqs + 1): yield self._get_req(i) - async def parse(self, response): + async def parse(self, response: Response) -> AsyncIterator[Any]: index = response.meta["index"] yield {"index": index} if index < 10**self.depth: @@ -305,7 +325,7 @@ class AsyncDefAsyncioGenComplexSpider(SimpleSpider): await asyncio.sleep(0.1) yield {"index": index + 5} - async def parse2(self, response): + async def parse2(self, response: Response) -> AsyncIterator[Any]: await asyncio.sleep(0.1) yield {"index2": response.meta["index"]} @@ -313,7 +333,7 @@ class AsyncDefAsyncioGenComplexSpider(SimpleSpider): class ItemSpider(FollowAllSpider): name = "item" - def parse(self, response): + def parse(self, response: Response) -> Iterator[Any]: for request in super().parse(response): yield request yield Item() @@ -321,12 +341,18 @@ class ItemSpider(FollowAllSpider): class MaxItemsAndRequestsSpider(FollowAllSpider): - def __init__(self, max_items=10, max_requests=10, *args, **kwargs): + def __init__( + self, + max_items: int = 10, + max_requests: int = 10, + *args: Any, + **kwargs: Any, + ): super().__init__(*args, **kwargs) self.max_items = max_items self.max_requests = max_requests - def parse(self, response): + def parse(self, response: Response) -> Iterator[Any]: self.items_scraped = 0 self.pages_crawled = 1 # account for the start url for request in super().parse(response): @@ -344,12 +370,12 @@ class DefaultError(Exception): class ErrorSpider(FollowAllSpider): name = "error" - exception_cls = DefaultError + exception_cls: type[Exception] = DefaultError - def raise_exception(self): + def raise_exception(self) -> None: raise self.exception_cls("Expected exception") - def parse(self, response): + def parse(self, response: Response) -> Iterator[Any]: for request in super().parse(response): yield request self.raise_exception() @@ -359,14 +385,15 @@ class BrokenStartSpider(FollowAllSpider): fail_before_yield = False fail_yielding = False - def __init__(self, *a, **kw): + def __init__(self, *a: Any, **kw: Any): super().__init__(*a, **kw) - self.seedsseen = [] + self.seedsseen: list[Any] = [] - async def start(self): + async def start(self) -> AsyncIterator[Any]: if self.fail_before_yield: 1 / 0 + assert self.mockserver for s in range(100): qargs = {"total": 10, "seed": s} url = self.mockserver.url(f"/follow?{urlencode(qargs, doseq=True)}") @@ -376,18 +403,18 @@ class BrokenStartSpider(FollowAllSpider): assert self.seedsseen, "All seeds consumed before any download happened" - def parse(self, response): + def parse(self, response: Response) -> Iterator[Any]: self.seedsseen.append(response.meta.get("seed")) yield from super().parse(response) class StartItemSpider(FollowAllSpider): - async def start(self): + async def start(self) -> AsyncIterator[Any]: yield {"name": "test item"} class StartGoodAndBadOutput(FollowAllSpider): - async def start(self): + async def start(self) -> AsyncIterator[Any]: yield {"a": "a"} yield Request("data:,a") yield "data:,b" @@ -395,17 +422,18 @@ class StartGoodAndBadOutput(FollowAllSpider): class SingleRequestSpider(MetaSpider): - seed = None - callback_func = None - errback_func = None + seed: Request | str | None = None + callback_func: Callable[[Response], Any] | None = None + errback_func: Callable[[Failure], Any] | None = None - async def start(self): + async def start(self) -> AsyncIterator[Any]: if isinstance(self.seed, Request): yield self.seed.replace(callback=self.parse, errback=self.on_error) else: + assert self.seed yield Request(self.seed, callback=self.parse, errback=self.on_error) - def parse(self, response): + def parse(self, response: Response) -> Any: self.meta.setdefault("responses", []).append(response) if callable(self.callback_func): return self.callback_func(response) @@ -413,7 +441,7 @@ class SingleRequestSpider(MetaSpider): return response.meta["next"] return None - def on_error(self, failure): + def on_error(self, failure: Failure) -> Any: self.meta["failure"] = failure if callable(self.errback_func): return self.errback_func(failure) @@ -426,17 +454,18 @@ class DuplicateStartSpider(MockServerSpider): distinct_urls = 2 dupe_factor = 3 - async def start(self): + async def start(self) -> AsyncIterator[Any]: + assert self.mockserver for i in range(self.distinct_urls): for _ in range(self.dupe_factor): url = self.mockserver.url(f"/echo?headers=1&body=test{i}") yield Request(url, dont_filter=self.dont_filter) - def __init__(self, url="http://localhost:8998", *args, **kwargs): + def __init__(self, url: str = "http://localhost:8998", *args: Any, **kwargs: Any): super().__init__(*args, **kwargs) self.visited = 0 - def parse(self, response): + def parse(self, response: Response) -> None: self.visited += 1 @@ -451,7 +480,7 @@ class CrawlSpiderWithParseMethod(MockServerSpider, CrawlSpider): } rules = (Rule(LinkExtractor(), callback="parse", follow=True),) - async def start(self): + async def start(self) -> AsyncIterator[Any]: test_body = b""" Page title @@ -461,11 +490,13 @@ class CrawlSpiderWithParseMethod(MockServerSpider, CrawlSpider): """ + assert self.mockserver url = self.mockserver.url("/alpayload") yield Request(url, method="POST", body=test_body) - def parse(self, response, foo=None): + def parse(self, response: Response, foo: str | None = None) -> Iterator[Any]: self.logger.info("[parse] status %i (foo: %s)", response.status, foo) + assert self.mockserver yield Request( self.mockserver.url("/status?n=202"), self.parse, cb_kwargs={"foo": "bar"} ) @@ -477,8 +508,9 @@ class CrawlSpiderWithAsyncCallback(CrawlSpiderWithParseMethod): name = "crawl_spider_with_async_callback" rules = (Rule(LinkExtractor(), callback="parse_async", follow=True),) - async def parse_async(self, response, foo=None): + async def parse_async(self, response: Response, foo: str | None = None) -> Request: self.logger.info("[parse_async] status %i (foo: %s)", response.status, foo) + assert self.mockserver return Request( self.mockserver.url("/status?n=202"), self.parse_async, @@ -492,8 +524,11 @@ class CrawlSpiderWithAsyncGeneratorCallback(CrawlSpiderWithParseMethod): name = "crawl_spider_with_async_generator_callback" rules = (Rule(LinkExtractor(), callback="parse_async_gen", follow=True),) - async def parse_async_gen(self, response, foo=None): + async def parse_async_gen( + self, response: Response, foo: str | None = None + ) -> AsyncIterator[Any]: self.logger.info("[parse_async_gen] status %i (foo: %s)", response.status, foo) + assert self.mockserver yield Request( self.mockserver.url("/status?n=202"), self.parse_async_gen, @@ -505,7 +540,7 @@ class CrawlSpiderWithErrback(CrawlSpiderWithParseMethod): name = "crawl_spider_with_errback" rules = (Rule(LinkExtractor(), callback="parse", errback="errback", follow=True),) - async def start(self): + async def start(self) -> AsyncIterator[Any]: test_body = b""" Page title @@ -518,17 +553,19 @@ class CrawlSpiderWithErrback(CrawlSpiderWithParseMethod): """ + assert self.mockserver url = self.mockserver.url("/alpayload") yield Request(url, method="POST", body=test_body) - def errback(self, failure): + def errback(self, failure: Failure) -> None: + assert isinstance(failure.value, HttpError) self.logger.info("[errback] status %i", failure.value.response.status) class CrawlSpiderWithoutErrback(CrawlSpiderWithParseMethod): name = "crawl_spider_without_errback" - async def start(self): + async def start(self) -> AsyncIterator[Any]: test_body = b""" Page title @@ -538,6 +575,7 @@ class CrawlSpiderWithoutErrback(CrawlSpiderWithParseMethod): """ + assert self.mockserver url = self.mockserver.url("/alpayload") yield Request(url, method="POST", body=test_body) @@ -553,7 +591,7 @@ class CrawlSpiderWithProcessRequestCallbackKeywordArguments(CrawlSpiderWithParse ), ) - def process_request(self, request, response): + def process_request(self, request: Request, response: Response) -> Request: request.cb_kwargs["foo"] = "process_request" return request @@ -562,59 +600,65 @@ class BytesReceivedCallbackSpider(MetaSpider): full_response_length = 2**18 @classmethod - def from_crawler(cls, crawler, *args, **kwargs): + def from_crawler(cls, crawler: Crawler, *args: Any, **kwargs: Any) -> Self: spider = super().from_crawler(crawler, *args, **kwargs) crawler.signals.connect(spider.bytes_received, signals.bytes_received) return spider - async def start(self): + async def start(self) -> AsyncIterator[Any]: body = b"a" * self.full_response_length + assert self.mockserver url = self.mockserver.url("/alpayload", is_secure=self.is_secure) yield Request(url, method="POST", body=body, errback=self.errback) - def parse(self, response): + def parse(self, response: Response) -> None: self.meta["response"] = response - def errback(self, failure): + def errback(self, failure: Failure) -> None: self.meta["failure"] = failure - def bytes_received(self, data, request, spider): + def bytes_received(self, data: bytes, request: Request, spider: Spider) -> None: self.meta["bytes_received"] = data raise StopDownload(fail=False) class BytesReceivedErrbackSpider(BytesReceivedCallbackSpider): - def bytes_received(self, data, request, spider): + def bytes_received(self, data: bytes, request: Request, spider: Spider) -> None: self.meta["bytes_received"] = data raise StopDownload(fail=True) class HeadersReceivedCallbackSpider(MetaSpider): @classmethod - def from_crawler(cls, crawler, *args, **kwargs): + def from_crawler(cls, crawler: Crawler, *args: Any, **kwargs: Any) -> Self: spider = super().from_crawler(crawler, *args, **kwargs) crawler.signals.connect(spider.headers_received, signals.headers_received) return spider - async def start(self): + async def start(self) -> AsyncIterator[Any]: + assert self.mockserver yield Request( self.mockserver.url("/status", is_secure=self.is_secure), errback=self.errback, ) - def parse(self, response): + def parse(self, response: Response) -> None: self.meta["response"] = response - def errback(self, failure): + def errback(self, failure: Failure) -> None: self.meta["failure"] = failure - def headers_received(self, headers, body_length, request, spider): + def headers_received( + self, headers: Headers, body_length: int, request: Request, spider: Spider + ) -> None: self.meta["headers_received"] = headers raise StopDownload(fail=False) class HeadersReceivedErrbackSpider(HeadersReceivedCallbackSpider): - def headers_received(self, headers, body_length, request, spider): + def headers_received( + self, headers: Headers, body_length: int, request: Request, spider: Spider + ) -> None: self.meta["headers_received"] = headers raise StopDownload(fail=True) @@ -623,13 +667,13 @@ class ExceptionSpider(Spider): name = "exception" @classmethod - def from_crawler(cls, crawler, *args, **kwargs): + def from_crawler(cls, crawler: Crawler, *args: Any, **kwargs: Any) -> Self: raise ValueError("Exception in from_crawler method") class NoRequestsSpider(Spider): name = "no_request" - async def start(self): + async def start(self) -> AsyncIterator[Any]: return yield diff --git a/tests/test_closespider.py b/tests/test_closespider.py index 9a7447e16..dbb29a8bc 100644 --- a/tests/test_closespider.py +++ b/tests/test_closespider.py @@ -11,6 +11,8 @@ from tests.utils.decorators import inline_callbacks_test class TestCloseSpider: + mockserver: MockServer + @classmethod def setup_class(cls): cls.mockserver = MockServer() @@ -25,6 +27,8 @@ class TestCloseSpider: close_on = 5 crawler = get_crawler(ItemSpider, {"CLOSESPIDER_ITEMCOUNT": close_on}) yield crawler.crawl(mockserver=self.mockserver) + assert isinstance(crawler.spider, ItemSpider) + assert crawler.stats reason = crawler.spider.meta["close_reason"] assert reason == "closespider_itemcount" itemcount = crawler.stats.get_value("item_scraped_count") @@ -35,6 +39,8 @@ class TestCloseSpider: close_on = 5 crawler = get_crawler(FollowAllSpider, {"CLOSESPIDER_PAGECOUNT": close_on}) yield crawler.crawl(mockserver=self.mockserver) + assert isinstance(crawler.spider, FollowAllSpider) + assert crawler.stats reason = crawler.spider.meta["close_reason"] assert reason == "closespider_pagecount" pagecount = crawler.stats.get_value("response_received_count") @@ -54,6 +60,8 @@ class TestCloseSpider: yield crawler.crawl( max_items=max_items, max_requests=max_requests, mockserver=self.mockserver ) + assert isinstance(crawler.spider, MaxItemsAndRequestsSpider) + assert crawler.stats reason = crawler.spider.meta["close_reason"] assert reason == "closespider_pagecount_no_item" pagecount = crawler.stats.get_value("response_received_count") @@ -72,6 +80,8 @@ class TestCloseSpider: }, ) yield crawler.crawl(mockserver=self.mockserver) + assert isinstance(crawler.spider, FollowAllSpider) + assert crawler.stats reason = crawler.spider.meta["close_reason"] assert reason == "closespider_pagecount_no_item" pagecount = crawler.stats.get_value("response_received_count") @@ -82,6 +92,8 @@ class TestCloseSpider: close_on = 5 crawler = get_crawler(ErrorSpider, {"CLOSESPIDER_ERRORCOUNT": close_on}) yield crawler.crawl(total=1000000, mockserver=self.mockserver) + assert isinstance(crawler.spider, ErrorSpider) + assert crawler.stats reason = crawler.spider.meta["close_reason"] assert reason == "closespider_errorcount" key = f"spider_exceptions/{crawler.spider.exception_cls.__name__}" @@ -94,6 +106,8 @@ class TestCloseSpider: close_on = 0.1 crawler = get_crawler(FollowAllSpider, {"CLOSESPIDER_TIMEOUT": close_on}) yield crawler.crawl(total=1000000, mockserver=self.mockserver) + assert isinstance(crawler.spider, FollowAllSpider) + assert crawler.stats reason = crawler.spider.meta["close_reason"] assert reason == "closespider_timeout" total_seconds = crawler.stats.get_value("elapsed_time_seconds") @@ -104,6 +118,8 @@ class TestCloseSpider: timeout = 1 crawler = get_crawler(SlowSpider, {"CLOSESPIDER_TIMEOUT_NO_ITEM": timeout}) yield crawler.crawl(n=3, mockserver=self.mockserver) + assert isinstance(crawler.spider, SlowSpider) + assert crawler.stats reason = crawler.spider.meta["close_reason"] assert reason == "closespider_timeout_no_item" total_seconds = crawler.stats.get_value("elapsed_time_seconds") diff --git a/tests/test_command_runspider.py b/tests/test_command_runspider.py index 11036eaeb..73a85dd5d 100644 --- a/tests/test_command_runspider.py +++ b/tests/test_command_runspider.py @@ -16,6 +16,11 @@ if TYPE_CHECKING: from pathlib import Path +# Header for spider classes copied with inspect.getsource(): the future import +# keeps their annotations from being evaluated in the generated module. +SPIDER_HEADER = "from __future__ import annotations\nfrom scrapy import Spider\n" + + class TestRunSpiderCommand: spider_filename = "myspider.py" @@ -65,14 +70,14 @@ class BadSpider(scrapy.Spider): def test_run_fail_spider(self, tmp_path: Path) -> None: ret, _, _ = self.runspider( - tmp_path, "from scrapy import Spider\n" + inspect.getsource(ExceptionSpider) + tmp_path, SPIDER_HEADER + inspect.getsource(ExceptionSpider) ) assert ret != 0 def test_run_good_spider(self, tmp_path: Path) -> None: ret, _, _ = self.runspider( tmp_path, - "from scrapy import Spider\n" + inspect.getsource(NoRequestsSpider), + SPIDER_HEADER + inspect.getsource(NoRequestsSpider), ) assert ret == 0 diff --git a/tests/test_contracts.py b/tests/test_contracts.py index 698044de7..44b621776 100644 --- a/tests/test_contracts.py +++ b/tests/test_contracts.py @@ -1,3 +1,6 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING, Any from unittest import TextTestResult import pytest @@ -11,7 +14,7 @@ from scrapy.contracts.default import ( ScrapesContract, UrlContract, ) -from scrapy.http import Request +from scrapy.http import Request, Response from scrapy.item import Field, Item from scrapy.spidermiddlewares.httperror import HttpError from scrapy.spiders import Spider @@ -19,6 +22,24 @@ from scrapy.utils.test import get_crawler from tests.mockserver.http import MockServer from tests.utils.decorators import inline_callbacks_test +if TYPE_CHECKING: + from unittest import TestResult + + from scrapy.http.request import CallbackT + + +def _request( + conman: ContractsManager, method: CallbackT, results: TestResult +) -> Request: + request = conman.from_method(method, results) + assert request + return request + + +def _call(request: Request, response: Any) -> Any: + assert request.callback + return request.callback(response, **request.cb_kwargs) + class DemoItem(Item): name = Field() @@ -30,7 +51,7 @@ class ResponseMock: class ResponseMetaMock(ResponseMock): - meta = None + meta: Any = None class TaggedRequest(Request): @@ -289,17 +310,19 @@ class TestContractsManager: def setup_method(self): self.conman = ContractsManager(self.contracts) - self.results = TextTestResult(stream=None, descriptions=False, verbosity=0) + self.results = TextTestResult( # type: ignore[type-var] + stream=None, descriptions=False, verbosity=0 + ) - def should_succeed(self): + def should_succeed(self) -> None: assert not self.results.failures assert not self.results.errors - def should_fail(self): + def should_fail(self) -> None: assert self.results.failures assert not self.results.errors - def should_error(self): + def should_error(self) -> None: assert self.results.errors def test_contracts(self): @@ -313,12 +336,10 @@ class TestContractsManager: ) # returns request for valid method - request = self.conman.from_method(spider.returns_request, self.results) - assert request is not None + assert self.conman.from_method(spider.returns_request, self.results) is not None # no request for missing url - request = self.conman.from_method(spider.parse_no_url, self.results) - assert request is None + assert self.conman.from_method(spider.parse_no_url, self.results) is None def test_cb_kwargs(self): spider = DemoSpider() @@ -354,29 +375,31 @@ class TestContractsManager: ) # returns_request - request = self.conman.from_method( - spider.returns_request_cb_kwargs, self.results - ) - request.callback(response, **request.cb_kwargs) + request = _request(self.conman, spider.returns_request_cb_kwargs, self.results) + _call(request, response) self.should_succeed() # returns_item - request = self.conman.from_method(spider.returns_item_cb_kwargs, self.results) - request.callback(response, **request.cb_kwargs) + request = _request(self.conman, spider.returns_item_cb_kwargs, self.results) + _call(request, response) self.should_succeed() # returns_item (error, callback doesn't take keyword arguments) - request = self.conman.from_method( - spider.returns_item_cb_kwargs_error_unexpected_keyword, self.results + request = _request( + self.conman, + spider.returns_item_cb_kwargs_error_unexpected_keyword, + self.results, ) - request.callback(response, **request.cb_kwargs) + _call(request, response) self.should_error() # returns_item (error, contract doesn't provide keyword arguments) - request = self.conman.from_method( - spider.returns_item_cb_kwargs_error_missing_argument, self.results + request = _request( + self.conman, + spider.returns_item_cb_kwargs_error_missing_argument, + self.results, ) - request.callback(response, **request.cb_kwargs) + _call(request, response) self.should_error() def test_meta(self): @@ -398,29 +421,27 @@ class TestContractsManager: response = ResponseMetaMock() # returns_request - request = self.conman.from_method(spider.returns_request_meta, self.results) + request = _request(self.conman, spider.returns_request_meta, self.results) assert request.meta["cookiejar"] == "session1" response.meta = request.meta - request.callback(response) + _call(request, response) assert response.meta["cookiejar"] == "session1" self.should_succeed() response = ResponseMetaMock() # returns_item - request = self.conman.from_method(spider.returns_item_meta, self.results) + request = _request(self.conman, spider.returns_item_meta, self.results) assert request.meta["key"] == "example" response.meta = request.meta - request.callback(response) + _call(request, response) assert response.meta["key"] == "example" self.should_succeed() response = ResponseMetaMock() - request = self.conman.from_method( - spider.returns_error_missing_meta, self.results - ) - request.callback(response) + request = _request(self.conman, spider.returns_error_missing_meta, self.results) + _call(request, response) self.should_error() def test_returns(self): @@ -428,36 +449,36 @@ class TestContractsManager: response = ResponseMock() # returns_item - request = self.conman.from_method(spider.returns_item, self.results) - request.callback(response) + request = _request(self.conman, spider.returns_item, self.results) + _call(request, response) self.should_succeed() # returns_dict_item - request = self.conman.from_method(spider.returns_dict_item, self.results) - request.callback(response) + request = _request(self.conman, spider.returns_dict_item, self.results) + _call(request, response) self.should_succeed() # returns_request - request = self.conman.from_method(spider.returns_request, self.results) - request.callback(response) + request = _request(self.conman, spider.returns_request, self.results) + _call(request, response) self.should_succeed() # returns_fail - request = self.conman.from_method(spider.returns_fail, self.results) - request.callback(response) + request = _request(self.conman, spider.returns_fail, self.results) + _call(request, response) self.should_fail() # returns_dict_fail - request = self.conman.from_method(spider.returns_dict_fail, self.results) - request.callback(response) + request = _request(self.conman, spider.returns_dict_fail, self.results) + _call(request, response) self.should_fail() def test_returns_async(self): spider = DemoSpider() response = ResponseMock() - request = self.conman.from_method(spider.returns_request_async, self.results) - request.callback(response) + request = _request(self.conman, spider.returns_request_async, self.results) + _call(request, response) self.should_error() def test_returns_invalid_argument_count(self): @@ -477,10 +498,8 @@ class TestContractsManager: spider = DemoSpider() response = ResponseMock() - request = self.conman.from_method( - spider.returns_request_range_fail, self.results - ) - request.callback(response) + request = _request(self.conman, spider.returns_request_range_fail, self.results) + _call(request, response) self.should_fail() assert "expected 2..3" in self.results.failures[-1][-1] @@ -490,8 +509,8 @@ class TestContractsManager: # @returns and @scrapes only count matching output objects and skip # the request that is also yielded. - request = self.conman.from_method(spider.yields_item_and_request, self.results) - request.callback(response) + request = _request(self.conman, spider.yields_item_and_request, self.results) + _call(request, response) self.should_succeed() def test_testcase_str(self): @@ -507,30 +526,30 @@ class TestContractsManager: response = ResponseMock() # scrapes_item_ok - request = self.conman.from_method(spider.scrapes_item_ok, self.results) - request.callback(response) + request = _request(self.conman, spider.scrapes_item_ok, self.results) + _call(request, response) self.should_succeed() # scrapes_dict_item_ok - request = self.conman.from_method(spider.scrapes_dict_item_ok, self.results) - request.callback(response) + request = _request(self.conman, spider.scrapes_dict_item_ok, self.results) + _call(request, response) self.should_succeed() # scrapes_item_fail - request = self.conman.from_method(spider.scrapes_item_fail, self.results) - request.callback(response) + request = _request(self.conman, spider.scrapes_item_fail, self.results) + _call(request, response) self.should_fail() # scrapes_dict_item_fail - request = self.conman.from_method(spider.scrapes_dict_item_fail, self.results) - request.callback(response) + request = _request(self.conman, spider.scrapes_dict_item_fail, self.results) + _call(request, response) self.should_fail() # scrapes_multiple_missing_fields - request = self.conman.from_method( - spider.scrapes_multiple_missing_fields, self.results + request = _request( + self.conman, spider.scrapes_multiple_missing_fields, self.results ) - request.callback(response) + _call(request, response) self.should_fail() message = "ContractFail: Missing fields: name, url" assert message in self.results.failures[-1][-1] @@ -540,14 +559,13 @@ class TestContractsManager: response = ResponseMock() # invalid regex - request = self.conman.from_method(spider.invalid_regex, self.results) - assert request is None + assert self.conman.from_method(spider.invalid_regex, self.results) is None # invalid regex with valid contract - request = self.conman.from_method( - spider.invalid_regex_with_valid_contract, self.results + request = _request( + self.conman, spider.invalid_regex_with_valid_contract, self.results ) - request.callback(response) + _call(request, response) self.should_succeed() def test_custom_contracts(self): @@ -559,14 +577,14 @@ class TestContractsManager: def test_errback(self): spider = DemoSpider() - response = ResponseMock() try: - raise HttpError(response, "Ignoring non-200 response") + raise HttpError(Response("http://scrapy.org"), "Ignoring non-200 response") except HttpError: failure_mock = failure.Failure() - request = self.conman.from_method(spider.returns_request, self.results) + request = _request(self.conman, spider.returns_request, self.results) + assert request.errback request.errback(failure_mock) assert not self.results.failures @@ -602,11 +620,12 @@ class TestContractsManager: crawler = get_crawler(TestSameUrlSpider) yield crawler.crawl() + assert isinstance(crawler.spider, TestSameUrlSpider) assert crawler.spider.visited == 2 def test_custom_tagged_request_contract(self): spider = DemoSpider() - request = self.conman.from_method(spider.custom_tagged_request, self.results) + request = _request(self.conman, spider.custom_tagged_request, self.results) assert request.method == "POST" assert isinstance(request, TaggedRequest) assert request.contract_tag == "custom" @@ -672,18 +691,20 @@ class PostProcessErrorContract(Contract): class TestCustomContractPrePostProcess: def setup_method(self): - self.results = TextTestResult(stream=None, descriptions=False, verbosity=0) + self.results = TextTestResult( # type: ignore[type-var] + stream=None, descriptions=False, verbosity=0 + ) def test_pre_hook_keyboard_interrupt(self): spider = DemoSpider() response = ResponseMock() contract = CustomFailContractPreProcess(spider.returns_request) - conman = ContractsManager([UrlContract, ReturnsContract, contract]) + conman = ContractsManager([UrlContract, ReturnsContract]) - request = conman.from_method(spider.returns_request, self.results) + request = _request(conman, spider.returns_request, self.results) contract.add_pre_hook(request, self.results) with pytest.raises(KeyboardInterrupt, match="Pre-process exception"): - request.callback(response, **request.cb_kwargs) + _call(request, response) assert not self.results.failures assert not self.results.errors @@ -692,12 +713,12 @@ class TestCustomContractPrePostProcess: spider = DemoSpider() response = ResponseMock() contract = CustomFailContractPostProcess(spider.returns_request) - conman = ContractsManager([UrlContract, ReturnsContract, contract]) + conman = ContractsManager([UrlContract, ReturnsContract]) - request = conman.from_method(spider.returns_request, self.results) + request = _request(conman, spider.returns_request, self.results) contract.add_post_hook(request, self.results) with pytest.raises(KeyboardInterrupt, match="Post-process exception"): - request.callback(response, **request.cb_kwargs) + _call(request, response) assert not self.results.failures assert not self.results.errors @@ -706,11 +727,11 @@ class TestCustomContractPrePostProcess: spider = DemoSpider() response = ResponseMock() contract = PreProcessSuccessContract(spider.returns_request) - conman = ContractsManager([UrlContract, ReturnsContract, contract]) + conman = ContractsManager([UrlContract, ReturnsContract]) - request = conman.from_method(spider.returns_request, self.results) + request = _request(conman, spider.returns_request, self.results) contract.add_pre_hook(request, self.results) - request.callback(response, **request.cb_kwargs) + _call(request, response) assert not self.results.failures assert not self.results.errors @@ -719,11 +740,11 @@ class TestCustomContractPrePostProcess: spider = DemoSpider() response = ResponseMock() contract = PreProcessAssertionFailContract(spider.returns_request) - conman = ContractsManager([UrlContract, ReturnsContract, contract]) + conman = ContractsManager([UrlContract, ReturnsContract]) - request = conman.from_method(spider.returns_request, self.results) + request = _request(conman, spider.returns_request, self.results) contract.add_pre_hook(request, self.results) - request.callback(response, **request.cb_kwargs) + _call(request, response) assert self.results.failures assert not self.results.errors @@ -732,11 +753,11 @@ class TestCustomContractPrePostProcess: spider = DemoSpider() response = ResponseMock() contract = PreProcessErrorContract(spider.returns_request) - conman = ContractsManager([UrlContract, ReturnsContract, contract]) + conman = ContractsManager([UrlContract, ReturnsContract]) - request = conman.from_method(spider.returns_request, self.results) + request = _request(conman, spider.returns_request, self.results) contract.add_pre_hook(request, self.results) - request.callback(response, **request.cb_kwargs) + _call(request, response) assert self.results.errors @@ -748,7 +769,7 @@ class TestCustomContractPrePostProcess: contract.add_pre_hook(request, self.results) with pytest.raises(TypeError, match="async callbacks"): - request.callback(response) + _call(request, response) def test_pre_hook_async_generator(self): spider = DemoSpider() @@ -758,7 +779,7 @@ class TestCustomContractPrePostProcess: contract.add_pre_hook(request, self.results) with pytest.raises(TypeError, match="async callbacks"): - request.callback(response) + _call(request, response) def test_post_hook_async_generator(self): spider = DemoSpider() @@ -768,16 +789,16 @@ class TestCustomContractPrePostProcess: contract.add_post_hook(request, self.results) with pytest.raises(TypeError, match="async callbacks"): - request.callback(response) + _call(request, response) def test_post_hook_error(self): spider = DemoSpider() response = ResponseMock() contract = PostProcessErrorContract(spider.returns_request) - conman = ContractsManager([UrlContract, ReturnsContract, contract]) + conman = ContractsManager([UrlContract, ReturnsContract]) - request = conman.from_method(spider.returns_request, self.results) + request = _request(conman, spider.returns_request, self.results) contract.add_post_hook(request, self.results) - request.callback(response, **request.cb_kwargs) + _call(request, response) assert self.results.errors diff --git a/tests/test_engine_loop.py b/tests/test_engine_loop.py index 1ecf8b8de..8f3484115 100644 --- a/tests/test_engine_loop.py +++ b/tests/test_engine_loop.py @@ -13,6 +13,8 @@ from tests.mockserver.http import MockServer from tests.utils.decorators import coroutine_test if TYPE_CHECKING: + from collections.abc import AsyncIterator, Callable, Iterator + import pytest from scrapy.http import Response @@ -62,19 +64,24 @@ class TestMain: class TestSpider(Spider): name = "test" - async def start(self): + async def start(self) -> AsyncIterator[Any]: + assert self.crawler.engine + assert self.crawler.engine._slot + scheduler = self.crawler.engine._slot.scheduler + assert isinstance(scheduler, MemoryScheduler) + yield Request("data:,a") await sleep(seconds) - self.crawler.engine._slot.scheduler.pause() - self.crawler.engine._slot.scheduler.enqueue_request(Request("data:,b")) + scheduler.pause() + scheduler.enqueue_request(Request("data:,b")) # During this time, the scheduler reports having requests but # returns None. await sleep(seconds) - self.crawler.engine._slot.scheduler.unpause() + scheduler.unpause() # The scheduler request is processed. await sleep(seconds) @@ -83,27 +90,28 @@ class TestMain: await sleep(seconds) - self.crawler.engine._slot.scheduler.pause() - self.crawler.engine._slot.scheduler.enqueue_request(Request("data:,d")) + scheduler.pause() + scheduler.enqueue_request(Request("data:,d")) # The last start request is processed during the time until the # delayed call below, proving that the start iteration can # finish before a scheduler “sleep” without causing the # scheduler to finish. - call_later(seconds, self.crawler.engine._slot.scheduler.unpause) + call_later(seconds, scheduler.unpause) - def parse(self, response): + def parse(self, response: Response) -> None: pass actual_urls = [] - def track_url(request, spider): + def track_url(request: Request, spider: Spider) -> None: actual_urls.append(request.url) settings = {"SCHEDULER": MemoryScheduler} crawler = get_crawler(TestSpider, settings_dict=settings) crawler.signals.connect(track_url, signals.request_reached_downloader) await crawler.crawl_async() + assert crawler.stats assert crawler.stats.get_value("finish_reason") == "finished" expected_urls = ["data:,a", "data:,b", "data:,c", "data:,d"] assert actual_urls == expected_urls, f"{actual_urls=} != {expected_urls=}" @@ -115,17 +123,17 @@ class TestMain: class TestSpider(Spider): name = "test" - async def start(self): + async def start(self) -> AsyncIterator[Any]: assert self.crawler.engine is not None await self.crawler.engine.close_async() yield Request("data:,a") - def parse(self, response): + def parse(self, response: Response) -> None: pass actual_urls = [] - def track_url(request, spider): + def track_url(request: Request, spider: Spider) -> None: actual_urls.append(request.url) settings = {"SCHEDULER": MemoryScheduler} @@ -211,6 +219,8 @@ class TestMain: class TestRequestSendOrder: + mockserver: MockServer + seconds = 0.1 # increase if flaky @classmethod @@ -222,57 +232,71 @@ class TestRequestSendOrder: def teardown_class(cls): cls.mockserver.__exit__(None, None, None) - def request(self, num, response_seconds, download_slots, priority=0): + def request( + self, + num: int, + response_seconds: float, + download_slots: int, + priority: int = 0, + ) -> Request: url = self.mockserver.url(f"/delay?n={response_seconds}&{num}") meta = {"download_slot": str(num % download_slots)} return Request(url, meta=meta, priority=priority) - def get_num(self, request_or_response: Request | Response): + def get_num(self, request_or_response: Request | Response) -> int: return int(request_or_response.url.rsplit("&", maxsplit=1)[1]) async def _test_request_order( self, - start_nums, - cb_nums=None, - settings=None, - response_seconds=None, - download_slots=1, - start_fn=None, - parse_fn=None, - ): + start_nums: list[int], + cb_nums: list[int] | None = None, + settings: dict[str, Any] | None = None, + response_seconds: float | None = None, + download_slots: int = 1, + start_fn: Callable[[Spider], AsyncIterator[Any]] | None = None, + parse_fn: Callable[..., Iterator[Any]] | None = None, + ) -> None: cb_nums = cb_nums or [] settings = settings or {} - response_seconds = response_seconds or self.seconds + seconds = response_seconds or self.seconds cb_requests = deque( - [self.request(num, response_seconds, download_slots) for num in cb_nums] + [self.request(num, seconds, download_slots) for num in cb_nums] ) if start_fn is None: - async def start_fn(spider): + async def default_start(spider: Spider) -> AsyncIterator[Any]: for num in start_nums: - yield self.request(num, response_seconds, download_slots) + yield self.request(num, seconds, download_slots) + + start_fn = default_start if parse_fn is None: - def parse_fn(spider, response): + def default_parse(spider: Spider, response: Response) -> Iterator[Any]: while cb_requests: yield cb_requests.popleft() + parse_fn = default_parse + + spider_start = start_fn + spider_parse = parse_fn + class TestSpider(Spider): name = "test" - start = start_fn - parse = parse_fn + start = spider_start + parse = spider_parse actual_nums = [] - def track_num(request, spider): + def track_num(request: Request, spider: Spider) -> None: actual_nums.append(self.get_num(request)) crawler = get_crawler(TestSpider, settings_dict=settings) crawler.signals.connect(track_num, signals.request_reached_downloader) await crawler.crawl_async() + assert crawler.stats assert crawler.stats.get_value("finish_reason") == "finished" expected_nums = sorted(start_nums + cb_nums) assert actual_nums == expected_nums, f"{actual_nums=} != {expected_nums=}" @@ -286,12 +310,14 @@ class TestRequestSendOrder: response_seconds = 0 download_slots = 1 - def _request(num, priority=0): + def _request(num: int, priority: int = 0) -> Request: return self.request( num, response_seconds, download_slots, priority=priority ) - async def start(spider): + async def start(spider: Spider) -> AsyncIterator[Any]: + assert spider.crawler.engine + assert spider.crawler.engine._slot # The first CONCURRENT_REQUESTS start requests are sent # immediately. yield _request(1) @@ -305,7 +331,7 @@ class TestRequestSendOrder: yield _request(3, priority=1) yield _request(4, priority=1) - def parse(spider, response): + def parse(spider: Spider, response: Response) -> Iterator[Any]: return yield @@ -326,12 +352,14 @@ class TestRequestSendOrder: response_seconds = 0 download_slots = 1 - def _request(num, priority=0): + def _request(num: int, priority: int = 0) -> Request: return self.request( num, response_seconds, download_slots, priority=priority ) - async def start(spider): + async def start(spider: Spider) -> AsyncIterator[Any]: + assert spider.crawler.engine + assert spider.crawler.engine._slot # The first CONCURRENT_REQUESTS start requests are sent # immediately. yield _request(1) @@ -345,7 +373,7 @@ class TestRequestSendOrder: yield _request(4, priority=1) yield _request(3, priority=1) - def parse(spider, response): + def parse(spider: Spider, response: Response) -> Iterator[Any]: return yield @@ -369,12 +397,14 @@ class TestRequestSendOrder: response_seconds = 0 download_slots = 1 - def _request(num, priority=0): + def _request(num: int, priority: int = 0) -> Request: return self.request( num, response_seconds, download_slots, priority=priority ) - async def start(spider): + async def start(spider: Spider) -> AsyncIterator[Any]: + assert spider.crawler.engine + assert spider.crawler.engine._slot # The first CONCURRENT_REQUESTS start requests are sent # immediately. yield _request(1) @@ -403,7 +433,7 @@ class TestRequestSendOrder: ): spider.crawler.engine._slot.scheduler.enqueue_request(request) - def parse(spider, response): + def parse(spider: Spider, response: Response) -> Iterator[Any]: return yield @@ -428,8 +458,9 @@ class TestRequestSendOrder: response_seconds = self.seconds * 2**1 # increase if flaky download_slots = 1 - async def start(spider): + async def start(spider: Spider) -> AsyncIterator[Any]: for num in start_nums: + assert spider.crawler.engine if spider.crawler.engine.needs_backout(): await spider.crawler.signals.wait_for(signals.scheduler_empty) request = self.request(num, response_seconds, download_slots) diff --git a/tests/test_spider.py b/tests/test_spider.py index 38cb8da18..118b75422 100644 --- a/tests/test_spider.py +++ b/tests/test_spider.py @@ -41,7 +41,7 @@ class TestXMLFeedSpider(TestSpiderBase): """ response = XmlResponse(url="http://example.com/sitemap.xml", body=body) - class _XMLSpider(self.spider_class): + class _XMLSpider(self.spider_class): # type: ignore[name-defined,misc] itertag = "url" namespaces = ( ("a", "http://www.google.com/schemas/sitemap/0.84"), @@ -166,7 +166,7 @@ class TestCSVFeedSpider(TestSpiderBase): body = get_testdata("feeds", "feed-sample6.csv") response = Response("http://example.org/dummy.csv", body=body) - class _CrawlSpider(self.spider_class): + class _CrawlSpider(self.spider_class): # type: ignore[name-defined,misc] name = "test" delimiter = "," quotechar = "'" diff --git a/tests/test_spider_crawl.py b/tests/test_spider_crawl.py index 9d5f0548b..ce556d0d1 100644 --- a/tests/test_spider_crawl.py +++ b/tests/test_spider_crawl.py @@ -2,6 +2,7 @@ from __future__ import annotations import re import warnings +from typing import TYPE_CHECKING, Any import pytest from w3lib.url import safe_url_string @@ -14,6 +15,12 @@ from scrapy.utils.test import get_crawler from tests.utils.bases.spider import TestSpiderBase from tests.utils.decorators import coroutine_test +if TYPE_CHECKING: + from scrapy.http import Response + from scrapy.http.request import CallbackT + +URL = "https://www.example.com" + class TestCrawlSpider(TestSpiderBase): test_body = b"""Page title @@ -33,7 +40,7 @@ class TestCrawlSpider(TestSpiderBase): "http://example.org/somepage/index.html", body=self.test_body ) - class _CrawlSpider(self.spider_class): + class _CrawlSpider(self.spider_class): # type: ignore[name-defined,misc] name = "test" allowed_domains = ["example.org"] rules = (Rule(),) @@ -53,7 +60,7 @@ class TestCrawlSpider(TestSpiderBase): "http://example.org/somepage/index.html", body=self.test_body ) - class _CrawlSpider(self.spider_class): + class _CrawlSpider(self.spider_class): # type: ignore[name-defined,misc] name = "test" allowed_domains = ["example.org"] rules = (Rule(LinkExtractor(), process_links="dummy_process_links"),) @@ -76,7 +83,7 @@ class TestCrawlSpider(TestSpiderBase): "http://example.org/somepage/index.html", body=self.test_body ) - class _CrawlSpider(self.spider_class): + class _CrawlSpider(self.spider_class): # type: ignore[name-defined,misc] name = "test" allowed_domains = ["example.org"] rules = (Rule(LinkExtractor(), process_links="filter_process_links"),) @@ -99,7 +106,7 @@ class TestCrawlSpider(TestSpiderBase): "http://example.org/somepage/index.html", body=self.test_body ) - class _CrawlSpider(self.spider_class): + class _CrawlSpider(self.spider_class): # type: ignore[name-defined,misc] name = "test" allowed_domains = ["example.org"] rules = (Rule(LinkExtractor(), process_links="dummy_process_links"),) @@ -125,7 +132,7 @@ class TestCrawlSpider(TestSpiderBase): def process_request_change_domain(request, response): return request.replace(url=request.url.replace(".org", ".com")) - class _CrawlSpider(self.spider_class): + class _CrawlSpider(self.spider_class): # type: ignore[name-defined,misc] name = "test" allowed_domains = ["example.org"] rules = ( @@ -151,7 +158,7 @@ class TestCrawlSpider(TestSpiderBase): request.meta["response_class"] = response.__class__.__name__ return request - class _CrawlSpider(self.spider_class): + class _CrawlSpider(self.spider_class): # type: ignore[name-defined,misc] name = "test" allowed_domains = ["example.org"] rules = ( @@ -180,7 +187,7 @@ class TestCrawlSpider(TestSpiderBase): "http://example.org/somepage/index.html", body=self.test_body ) - class _CrawlSpider(self.spider_class): + class _CrawlSpider(self.spider_class): # type: ignore[name-defined,misc] name = "test" allowed_domains = ["example.org"] rules = (Rule(LinkExtractor(), process_request="process_request_upper"),) @@ -203,7 +210,7 @@ class TestCrawlSpider(TestSpiderBase): "http://example.org/somepage/index.html", body=self.test_body ) - class _CrawlSpider(self.spider_class): + class _CrawlSpider(self.spider_class): # type: ignore[name-defined,misc] name = "test" allowed_domains = ["example.org"] rules = ( @@ -248,7 +255,6 @@ class TestCrawlSpider(TestSpiderBase): def test_parse_response_use(self): class _CrawlSpider(CrawlSpider): name = "test" - start_urls = "https://www.example.com" _follow_links = False with warnings.catch_warnings(): @@ -258,17 +264,20 @@ class TestCrawlSpider(TestSpiderBase): ScrapyDeprecationWarning, match=r"CrawlSpider\._parse_response method is deprecated", ): - spider._parse_response( - TextResponse(spider.start_urls, body=b""), None, None - ) + spider._parse_response(TextResponse(URL, body=b""), None, {}) def test_parse_response_override(self): class _CrawlSpider(CrawlSpider): - def _parse_response(self, response, callback, cb_kwargs, follow=True): + def _parse_response( + self, + response: Response, + callback: CallbackT | None, + cb_kwargs: dict[str, Any], + follow: bool = True, + ) -> Any: pass name = "test" - start_urls = "https://www.example.com" _follow_links = False with pytest.warns( @@ -278,21 +287,16 @@ class TestCrawlSpider(TestSpiderBase): spider = _CrawlSpider() with warnings.catch_warnings(): warnings.simplefilter("error", category=ScrapyDeprecationWarning) - spider._parse_response( - TextResponse(spider.start_urls, body=b""), None, None - ) + spider._parse_response(TextResponse(URL, body=b""), None, {}) def test_parse_with_rules(self): class _CrawlSpider(CrawlSpider): name = "test" - start_urls = "https://www.example.com" with warnings.catch_warnings(): warnings.simplefilter("error", category=ScrapyDeprecationWarning) spider = _CrawlSpider() - spider.parse_with_rules( - TextResponse(spider.start_urls, body=b""), None, None - ) + spider.parse_with_rules(TextResponse(URL, body=b""), None, {}) @coroutine_test async def test_parse_with_rules_without_callback(self): @@ -327,12 +331,13 @@ class TestCrawlSpider(TestSpiderBase): allowed_domains = ["example.org"] rules = (Rule(),) + def callback(response: Response) -> list[Any]: + return [item] + spider = _CrawlSpider.from_crawler(get_crawler(_CrawlSpider)) results = [ r - async for r in spider.parse_with_rules( - response, lambda response: [item], {}, follow=False - ) + async for r in spider.parse_with_rules(response, callback, {}, follow=False) ] assert results == [item] diff --git a/tests/test_spider_sitemap.py b/tests/test_spider_sitemap.py index 2f1ccab81..630ab5ca4 100644 --- a/tests/test_spider_sitemap.py +++ b/tests/test_spider_sitemap.py @@ -55,7 +55,7 @@ class TestSitemapSpider(TestSpiderBase): assert spider._get_sitemap_body(response) == body def test_get_sitemap_body(self): - r = XmlResponse(url="http://www.example.com/", body=self.BODY) + r: Response = XmlResponse(url="http://www.example.com/", body=self.BODY) self.assertSitemapBody(r, self.BODY) r = HtmlResponse(url="http://www.example.com/", body=self.BODY) @@ -166,7 +166,7 @@ Sitemap: /sitemap-relative-url.xml """ - class FilteredSitemapSpider(self.spider_class): + class FilteredSitemapSpider(self.spider_class): # type: ignore[name-defined,misc] def sitemap_filter(self, entries): for entry in entries: date_time = datetime.strptime(entry["lastmod"], "%Y-%m-%d") @@ -201,7 +201,7 @@ Sitemap: /sitemap-relative-url.xml """ - class FilteredSitemapSpider(self.spider_class): + class FilteredSitemapSpider(self.spider_class): # type: ignore[name-defined,misc] def sitemap_filter(self, entries): for entry in entries: alternate_links = entry.get("alternate", ()) @@ -235,7 +235,7 @@ Sitemap: /sitemap-relative-url.xml """ - class FilteredSitemapSpider(self.spider_class): + class FilteredSitemapSpider(self.spider_class): # type: ignore[name-defined,misc] def sitemap_filter(self, entries): for entry in entries: date_time = datetime.strptime( @@ -285,6 +285,7 @@ Sitemap: /sitemap-relative-url.xml sitemap_rules = [("", parse_item)] def raw_body(self): + assert self.mockserver loc = self.mockserver.url("/text") return ( '' @@ -303,6 +304,7 @@ Sitemap: /sitemap-relative-url.xml yield {"url": response.url} def raw_body(self): + assert self.mockserver loc = self.mockserver.url("/text") return ( '' @@ -363,7 +365,7 @@ Sitemap: /sitemap-relative-url.xml """ r = TextResponse(url="http://www.example.com/sitemap.xml", body=sitemap) - class _FollowSpider(self.spider_class): + class _FollowSpider(self.spider_class): # type: ignore[name-defined,misc] sitemap_follow = [follow] spider = _FollowSpider("example.com") @@ -382,7 +384,7 @@ Sitemap: /sitemap-relative-url.xml @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") def test_compression_bomb_spider_attr(self): - class DownloadMaxSizeSpider(self.spider_class): + class DownloadMaxSizeSpider(self.spider_class): # type: ignore[name-defined,misc] download_maxsize = 10_000_000 crawler = get_crawler() @@ -485,7 +487,7 @@ Sitemap: /sitemap-relative-url.xml @coroutine_test async def test_sitemap_urls(self): - class TestSpider(self.spider_class): + class TestSpider(self.spider_class): # type: ignore[name-defined,misc] name = "test" sitemap_urls = ["https://toscrape.com/sitemap.xml"] diff --git a/tests/test_spidermiddleware_output_chain.py b/tests/test_spidermiddleware_output_chain.py index dacc90b27..74051d1db 100644 --- a/tests/test_spidermiddleware_output_chain.py +++ b/tests/test_spidermiddleware_output_chain.py @@ -1,16 +1,21 @@ from __future__ import annotations import logging -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any from scrapy import Request, Spider from scrapy.utils.test import get_crawler from tests.mockserver.http import MockServer +from tests.spiders import MockServerSpider from tests.utils.decorators import coroutine_test if TYPE_CHECKING: + from collections.abc import AsyncIterator, Iterator + import pytest + from scrapy.http import Response + class _BaseSpiderMiddleware: def __init__(self, crawler): @@ -41,7 +46,7 @@ class RecoveryMiddleware(_BaseSpiderMiddleware): ] -class RecoverySpider(Spider): +class RecoverySpider(MockServerSpider): name = "RecoverySpider" custom_settings = { "SPIDER_MIDDLEWARES_BASE": {}, @@ -50,10 +55,11 @@ class RecoverySpider(Spider): }, } - async def start(self): + async def start(self) -> AsyncIterator[Any]: + assert self.mockserver yield Request(self.mockserver.url("/status?n=200")) - def parse(self, response): + def parse(self, response: Response) -> Iterator[Any]: yield {"test": 1} self.logger.info("DONT_FAIL: %s", response.meta.get("dont_fail")) if not response.meta.get("dont_fail"): @@ -63,7 +69,7 @@ class RecoverySpider(Spider): class RecoveryAsyncGenSpider(RecoverySpider): name = "RecoveryAsyncGenSpider" - async def parse(self, response): + async def parse(self, response: Response) -> AsyncIterator[Any]: # type: ignore[override] for r in super().parse(response): yield r @@ -76,7 +82,7 @@ class FailProcessSpiderInputMiddleware(_BaseSpiderMiddleware): raise IndexError -class ProcessSpiderInputSpiderWithoutErrback(Spider): +class ProcessSpiderInputSpiderWithoutErrback(MockServerSpider): name = "ProcessSpiderInputSpiderWithoutErrback" custom_settings = { "SPIDER_MIDDLEWARES": { @@ -87,7 +93,8 @@ class ProcessSpiderInputSpiderWithoutErrback(Spider): } } - async def start(self): + async def start(self) -> AsyncIterator[Any]: + assert self.mockserver yield Request(url=self.mockserver.url("/status?n=200"), callback=self.parse) def parse(self, response): @@ -97,7 +104,8 @@ class ProcessSpiderInputSpiderWithoutErrback(Spider): class ProcessSpiderInputSpiderWithErrback(ProcessSpiderInputSpiderWithoutErrback): name = "ProcessSpiderInputSpiderWithErrback" - async def start(self): + async def start(self) -> AsyncIterator[Any]: + assert self.mockserver yield Request( self.mockserver.url("/status?n=200"), self.parse, errback=self.errback ) @@ -109,7 +117,7 @@ class ProcessSpiderInputSpiderWithErrback(ProcessSpiderInputSpiderWithoutErrback # ================================================================================ # (2) exceptions from a spider callback (generator) -class GeneratorCallbackSpider(Spider): +class GeneratorCallbackSpider(MockServerSpider): name = "GeneratorCallbackSpider" custom_settings = { "SPIDER_MIDDLEWARES": { @@ -117,7 +125,8 @@ class GeneratorCallbackSpider(Spider): }, } - async def start(self): + async def start(self) -> AsyncIterator[Any]: + assert self.mockserver yield Request(self.mockserver.url("/status?n=200")) def parse(self, response): @@ -146,7 +155,7 @@ class GeneratorCallbackSpiderMiddlewareRightAfterSpider(GeneratorCallbackSpider) # ================================================================================ # (3) exceptions from a spider callback (not a generator) -class NotGeneratorCallbackSpider(Spider): +class NotGeneratorCallbackSpider(MockServerSpider): name = "NotGeneratorCallbackSpider" custom_settings = { "SPIDER_MIDDLEWARES": { @@ -154,7 +163,8 @@ class NotGeneratorCallbackSpider(Spider): }, } - async def start(self): + async def start(self) -> AsyncIterator[Any]: + assert self.mockserver yield Request(self.mockserver.url("/status?n=200")) def parse(self, response): @@ -224,7 +234,7 @@ class GeneratorDoNothingAfterRecoveryMiddleware(_GeneratorDoNothingMiddleware): pass -class GeneratorOutputChainSpider(Spider): +class GeneratorOutputChainSpider(MockServerSpider): name = "GeneratorOutputChainSpider" custom_settings = { "SPIDER_MIDDLEWARES": { @@ -235,7 +245,8 @@ class GeneratorOutputChainSpider(Spider): }, } - async def start(self): + async def start(self) -> AsyncIterator[Any]: + assert self.mockserver yield Request(self.mockserver.url("/status?n=200")) def parse(self, response): diff --git a/tests/test_spidermiddleware_process_start.py b/tests/test_spidermiddleware_process_start.py index 321139da3..491919759 100644 --- a/tests/test_spidermiddleware_process_start.py +++ b/tests/test_spidermiddleware_process_start.py @@ -1,4 +1,7 @@ +from __future__ import annotations + from asyncio import sleep +from typing import TYPE_CHECKING, Any import pytest @@ -9,6 +12,11 @@ from scrapy.utils.test import get_crawler from .utils import twisted_sleep from .utils.decorators import coroutine_test +if TYPE_CHECKING: + from collections.abc import AsyncIterator + + from scrapy.http import Response + ITEM_A = {"id": "a"} ITEM_B = {"id": "b"} ITEM_C = {"id": "c"} @@ -17,20 +25,20 @@ SLEEP_SECONDS = 0.1 class AsyncioSleepSpiderMiddleware: - async def process_start(self, start): + async def process_start(self, start: AsyncIterator[Any]) -> AsyncIterator[Any]: await sleep(SLEEP_SECONDS) async for item_or_request in start: yield item_or_request class NoOpSpiderMiddleware: - async def process_start(self, start): + async def process_start(self, start: AsyncIterator[Any]) -> AsyncIterator[Any]: async for item_or_request in start: yield item_or_request class TwistedSleepSpiderMiddleware: - async def process_start(self, start): + async def process_start(self, start: AsyncIterator[Any]) -> AsyncIterator[Any]: await maybe_deferred_to_future(twisted_sleep(SLEEP_SECONDS)) async for item_or_request in start: yield item_or_request @@ -42,12 +50,12 @@ class TwistedSleepSpiderMiddleware: class ModernWrapSpider(Spider): name = "test" - async def start(self): + async def start(self) -> AsyncIterator[Any]: yield ITEM_B class ModernWrapSpiderMiddleware: - async def process_start(self, start): + async def process_start(self, start: AsyncIterator[Any]) -> AsyncIterator[Any]: yield ITEM_A async for item_or_request in start: yield item_or_request @@ -55,10 +63,15 @@ class ModernWrapSpiderMiddleware: class TestMain: - async def _test(self, spider_middlewares, spider_cls, expected_items): + async def _test( + self, + spider_middlewares: list[type], + spider_cls: type[Spider], + expected_items: list[Any], + ) -> None: actual_items = [] - def track_item(item, response, spider): + def track_item(item: Any, response: Response, spider: Spider) -> None: actual_items.append(item) settings = { @@ -67,10 +80,16 @@ class TestMain: crawler = get_crawler(spider_cls, settings_dict=settings) crawler.signals.connect(track_item, signals.item_scraped) await crawler.crawl_async() + assert crawler.stats assert crawler.stats.get_value("finish_reason") == "finished" assert actual_items == expected_items, f"{actual_items=} != {expected_items=}" - async def _test_wrap(self, spider_middleware, spider_cls, expected_items=None): + async def _test_wrap( + self, + spider_middleware: type, + spider_cls: type[Spider], + expected_items: list[Any] | None = None, + ) -> None: expected_items = expected_items or [ITEM_A, ITEM_B, ITEM_C] await self._test([spider_middleware], spider_cls, expected_items) @@ -78,11 +97,11 @@ class TestMain: async def test_modern_mw_modern_spider(self): await self._test_wrap(ModernWrapSpiderMiddleware, ModernWrapSpider) - async def _test_sleep(self, spider_middlewares): + async def _test_sleep(self, spider_middlewares: list[type]) -> None: class TestSpider(Spider): name = "test" - async def start(self): + async def start(self) -> AsyncIterator[Any]: yield ITEM_A await self._test(spider_middlewares, TestSpider, [ITEM_A]) diff --git a/tests/utils/bases/spider.py b/tests/utils/bases/spider.py index 799c1820d..224606bf6 100644 --- a/tests/utils/bases/spider.py +++ b/tests/utils/bases/spider.py @@ -32,7 +32,7 @@ class TestSpiderBase(ABC): def test_spider_args(self): """``__init__`` method arguments are assigned to spider attributes""" spider = self.spider_class("example.com", foo="bar") - assert spider.foo == "bar" + assert spider.foo == "bar" # type: ignore[attr-defined] def test_spider_without_name(self): """``__init__`` raises when the name is not provided.""" @@ -58,7 +58,7 @@ class TestSpiderBase(ABC): mock_init.assert_called_once_with("example.com", foo="bar") def test_closed_signal_call(self): - class TestSpider(self.spider_class): + class TestSpider(self.spider_class): # type: ignore[name-defined,misc] closed_called = False def closed(self, reason): @@ -92,7 +92,7 @@ class TestSpiderBase(ABC): **get_reactor_settings(), } - class TestSpider(self.spider_class): + class TestSpider(self.spider_class): # type: ignore[name-defined,misc] name = "test" custom_settings = spider_settings @@ -127,5 +127,5 @@ class TestSpiderBase(ABC): ScrapyDeprecationWarning, match=r"Spider.log\(\) is deprecated" ), ): - spider.log("test log msg", "INFO") + spider.log("test log msg", "INFO") # type: ignore[arg-type] mock_logger.log.assert_called_once_with("INFO", "test log msg")