RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Type spider-related test modules

This commit is contained in:
Adrian Chaves 2026-08-10 10:03:43 +02:00
parent 508bd7faec
commit 3ecb28c93e
12 changed files with 427 additions and 283 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
pyproject.toml
View File

@ -116,13 +116,9 @@ allow_incomplete_defs = true # 59 errors
# TODO
[[tool.mypy.overrides]]
module = [
"tests.spiders",
"tests.test_closespider",
"tests.test_cmdline",
"tests.test_contracts",
"tests.test_downloaderslotssettings",
"tests.test_dupefilters",
"tests.test_engine_loop",
"tests.test_extension_statsmailer",
"tests.test_extension_throttle",
"tests.test_linkextractors",
@ -131,15 +127,9 @@ module = [
"tests.test_pqueues",
"tests.test_scheduler_base",
"tests.test_settings",
"tests.test_spider",
"tests.test_spider_crawl",
"tests.test_spidermiddleware_output_chain",
"tests.test_spidermiddleware_process_start",
"tests.test_spider_sitemap",
"tests.test_squeues",
"tests.test_squeues_request",
"tests.test_stats",
"tests.utils.bases.spider",
]
check_untyped_defs = false

204
tests/spiders.py
View File

@ -13,22 +13,31 @@ from twisted.internet import defer
from scrapy import signals
from scrapy.exceptions import StopDownload
from scrapy.http import Request
from scrapy.http import Request, TextResponse
from scrapy.item import Item
from scrapy.linkextractors import LinkExtractor
from scrapy.spidermiddlewares.httperror import HttpError
from scrapy.spiders import Spider
from scrapy.spiders.crawl import CrawlSpider, Rule
from scrapy.utils.defer import deferred_to_future, maybe_deferred_to_future
from scrapy.utils.test import get_from_asyncio_queue
if TYPE_CHECKING:
from collections.abc import AsyncIterator, Callable, Iterator
from twisted.python.failure import Failure
from typing_extensions import Self
from scrapy.crawler import Crawler
from scrapy.http import Headers, Response
from scrapy.http.request import CallbackT
from tests.mockserver.http import MockServer
class MockServerSpider(Spider):
def __init__(
self,
*args,
*args: Any,
mockserver: MockServer | None = None,
is_secure: bool = False,
**kwargs: Any,
@ -74,7 +83,7 @@ class MetaSpider(MockServerSpider):
super().__init__(*args, **kwargs)
self.meta: dict[str, Any] = {}
def closed(self, reason):
def closed(self, reason: str) -> None:
self.meta["close_reason"] = reason
@ -83,16 +92,24 @@ class FollowAllSpider(MetaSpider):
link_extractor = LinkExtractor()
def __init__(
self, total=10, show=20, order="rand", maxlatency=0.0, *args, **kwargs
self,
total: int = 10,
show: int = 20,
order: str = "rand",
maxlatency: float = 0.0,
*args: Any,
**kwargs: Any,
):
super().__init__(*args, **kwargs)
self.urls_visited = []
self.times = []
self.urls_visited: list[str] = []
self.times: list[float] = []
qargs = {"total": total, "show": show, "order": order, "maxlatency": maxlatency}
assert self.mockserver
url = self.mockserver.url(f"/follow?{urlencode(qargs, doseq=True)}")
self.start_urls = [url]
def parse(self, response):
def parse(self, response: Response) -> Iterator[Any]:
assert isinstance(response, TextResponse)
self.urls_visited.append(response.url)
self.times.append(time.time())
for link in self.link_extractor.extract_links(response):
@ -102,50 +119,52 @@ class FollowAllSpider(MetaSpider):
class DelaySpider(MetaSpider):
name = "delay"
def __init__(self, n=1, b=0, *args, **kwargs):
def __init__(self, n: float = 1, b: float = 0, *args: Any, **kwargs: Any):
super().__init__(*args, **kwargs)
self.n = n
self.b = b
self.t1 = self.t2 = self.t2_err = 0
self.t1 = self.t2 = self.t2_err = 0.0
async def start(self):
async def start(self) -> AsyncIterator[Any]:
self.t1 = time.time()
assert self.mockserver
url = self.mockserver.url(f"/delay?n={self.n}&b={self.b}")
yield Request(url, callback=self.parse, errback=self.errback)
def parse(self, response):
def parse(self, response: Response) -> Any:
self.t2 = time.time()
def errback(self, failure):
def errback(self, failure: Failure) -> None:
self.t2_err = time.time()
class LogSpider(MetaSpider):
name = "log_spider"
def log_debug(self, message: str, extra: dict[str, Any] | None = None):
def log_debug(self, message: str, extra: dict[str, Any] | None = None) -> None:
self.logger.debug(message, extra=extra)
def log_info(self, message: str, extra: dict[str, Any] | None = None):
def log_info(self, message: str, extra: dict[str, Any] | None = None) -> None:
self.logger.info(message, extra=extra)
def log_warning(self, message: str, extra: dict[str, Any] | None = None):
def log_warning(self, message: str, extra: dict[str, Any] | None = None) -> None:
self.logger.warning(message, extra=extra)
def log_error(self, message: str, extra: dict[str, Any] | None = None):
def log_error(self, message: str, extra: dict[str, Any] | None = None) -> None:
self.logger.error(message, extra=extra)
def log_critical(self, message: str, extra: dict[str, Any] | None = None):
def log_critical(self, message: str, extra: dict[str, Any] | None = None) -> None:
self.logger.critical(message, extra=extra)
def parse(self, response):
def parse(self, response: Response) -> None:
pass
class SlowSpider(DelaySpider):
name = "slow"
async def start(self):
async def start(self) -> AsyncIterator[Any]:
assert self.mockserver
# 1st response is fast
url = self.mockserver.url("/delay?n=0&b=0")
yield Request(url, callback=self.parse, errback=self.errback)
@ -154,25 +173,25 @@ class SlowSpider(DelaySpider):
url = self.mockserver.url(f"/delay?n={self.n}&b={self.b}")
yield Request(url, callback=self.parse, errback=self.errback)
def parse(self, response):
def parse(self, response: Response) -> Iterator[Any]:
yield Item()
class SimpleSpider(MetaSpider):
name = "simple"
def __init__(self, url="http://localhost:8998", *args, **kwargs):
def __init__(self, url: str = "http://localhost:8998", *args: Any, **kwargs: Any):
super().__init__(*args, **kwargs)
self.start_urls = [url]
def parse(self, response):
def parse(self, response: Response) -> Any:
self.logger.info(f"Got response {response.status}")
class AsyncDefSpider(SimpleSpider):
name = "asyncdef"
async def parse(self, response):
async def parse(self, response: Response) -> None:
await defer.succeed(42)
self.logger.info(f"Got response {response.status}")
@ -180,7 +199,7 @@ class AsyncDefSpider(SimpleSpider):
class AsyncDefAsyncioSpider(SimpleSpider):
name = "asyncdef_asyncio"
async def parse(self, response):
async def parse(self, response: Response) -> None:
await asyncio.sleep(0.2)
status = await get_from_asyncio_queue(response.status)
self.logger.info(f"Got response {status}")
@ -189,7 +208,7 @@ class AsyncDefAsyncioSpider(SimpleSpider):
class AsyncDefAsyncioReturnSpider(SimpleSpider):
name = "asyncdef_asyncio_return"
async def parse(self, response):
async def parse(self, response: Response) -> Any:
await asyncio.sleep(0.2)
status = await get_from_asyncio_queue(response.status)
self.logger.info(f"Got response {status}")
@ -199,7 +218,7 @@ class AsyncDefAsyncioReturnSpider(SimpleSpider):
class AsyncDefAsyncioReturnSingleElementSpider(SimpleSpider):
name = "asyncdef_asyncio_return_single_element"
async def parse(self, response):
async def parse(self, response: Response) -> Any:
await asyncio.sleep(0.1)
status = await get_from_asyncio_queue(response.status)
self.logger.info(f"Got response {status}")
@ -209,7 +228,7 @@ class AsyncDefAsyncioReturnSingleElementSpider(SimpleSpider):
class AsyncDefAsyncioReqsReturnSpider(SimpleSpider):
name = "asyncdef_asyncio_reqs_return"
async def parse(self, response):
async def parse(self, response: Response) -> Any:
await asyncio.sleep(0.2)
req_id = response.meta.get("req_id", 0)
status = await get_from_asyncio_queue(response.status)
@ -226,7 +245,7 @@ class AsyncDefAsyncioReqsReturnSpider(SimpleSpider):
class AsyncDefAsyncioGenExcSpider(SimpleSpider):
name = "asyncdef_asyncio_gen_exc"
async def parse(self, response):
async def parse(self, response: Response) -> AsyncIterator[Any]:
for i in range(10):
await asyncio.sleep(0.1)
yield {"foo": i}
@ -237,7 +256,7 @@ class AsyncDefAsyncioGenExcSpider(SimpleSpider):
class AsyncDefDeferredDirectSpider(SimpleSpider):
name = "asyncdef_deferred_direct"
async def parse(self, response):
async def parse(self, response: Response) -> AsyncIterator[Any]:
await defer.succeed(None)
yield {"code": 200}
@ -245,7 +264,7 @@ class AsyncDefDeferredDirectSpider(SimpleSpider):
class AsyncDefDeferredWrappedSpider(SimpleSpider):
name = "asyncdef_deferred_wrapped"
async def parse(self, response):
async def parse(self, response: Response) -> AsyncIterator[Any]:
await deferred_to_future(defer.succeed(None))
yield {"code": 200}
@ -253,7 +272,7 @@ class AsyncDefDeferredWrappedSpider(SimpleSpider):
class AsyncDefDeferredMaybeWrappedSpider(SimpleSpider):
name = "asyncdef_deferred_maybe_wrapped"
async def parse(self, response):
async def parse(self, response: Response) -> AsyncIterator[Any]:
await maybe_deferred_to_future(defer.succeed(None))
yield {"code": 200}
@ -261,7 +280,7 @@ class AsyncDefDeferredMaybeWrappedSpider(SimpleSpider):
class AsyncDefAsyncioGenSpider(SimpleSpider):
name = "asyncdef_asyncio_gen"
async def parse(self, response):
async def parse(self, response: Response) -> AsyncIterator[Any]:
await asyncio.sleep(0.2)
yield {"foo": 42}
self.logger.info(f"Got response {response.status}")
@ -270,7 +289,7 @@ class AsyncDefAsyncioGenSpider(SimpleSpider):
class AsyncDefAsyncioGenLoopSpider(SimpleSpider):
name = "asyncdef_asyncio_gen_loop"
async def parse(self, response):
async def parse(self, response: Response) -> AsyncIterator[Any]:
for i in range(10):
await asyncio.sleep(0.1)
yield {"foo": i}
@ -283,7 +302,8 @@ class AsyncDefAsyncioGenComplexSpider(SimpleSpider):
following_reqs = 3
depth = 2
def _get_req(self, index, cb=None):
def _get_req(self, index: int, cb: CallbackT | None = None) -> Request:
assert self.mockserver
return Request(
self.mockserver.url(f"/status?n=200&request={index}"),
meta={"index": index},
@ -291,11 +311,11 @@ class AsyncDefAsyncioGenComplexSpider(SimpleSpider):
callback=cb,
)
async def start(self):
async def start(self) -> AsyncIterator[Any]:
for i in range(1, self.initial_reqs + 1):
yield self._get_req(i)
async def parse(self, response):
async def parse(self, response: Response) -> AsyncIterator[Any]:
index = response.meta["index"]
yield {"index": index}
if index < 10**self.depth:
@ -305,7 +325,7 @@ class AsyncDefAsyncioGenComplexSpider(SimpleSpider):
await asyncio.sleep(0.1)
yield {"index": index + 5}
async def parse2(self, response):
async def parse2(self, response: Response) -> AsyncIterator[Any]:
await asyncio.sleep(0.1)
yield {"index2": response.meta["index"]}
@ -313,7 +333,7 @@ class AsyncDefAsyncioGenComplexSpider(SimpleSpider):
class ItemSpider(FollowAllSpider):
name = "item"
def parse(self, response):
def parse(self, response: Response) -> Iterator[Any]:
for request in super().parse(response):
yield request
yield Item()
@ -321,12 +341,18 @@ class ItemSpider(FollowAllSpider):
class MaxItemsAndRequestsSpider(FollowAllSpider):
def __init__(self, max_items=10, max_requests=10, *args, **kwargs):
def __init__(
self,
max_items: int = 10,
max_requests: int = 10,
*args: Any,
**kwargs: Any,
):
super().__init__(*args, **kwargs)
self.max_items = max_items
self.max_requests = max_requests
def parse(self, response):
def parse(self, response: Response) -> Iterator[Any]:
self.items_scraped = 0
self.pages_crawled = 1 # account for the start url
for request in super().parse(response):
@ -344,12 +370,12 @@ class DefaultError(Exception):
class ErrorSpider(FollowAllSpider):
name = "error"
exception_cls = DefaultError
exception_cls: type[Exception] = DefaultError
def raise_exception(self):
def raise_exception(self) -> None:
raise self.exception_cls("Expected exception")
def parse(self, response):
def parse(self, response: Response) -> Iterator[Any]:
for request in super().parse(response):
yield request
self.raise_exception()
@ -359,14 +385,15 @@ class BrokenStartSpider(FollowAllSpider):
fail_before_yield = False
fail_yielding = False
def __init__(self, *a, **kw):
def __init__(self, *a: Any, **kw: Any):
super().__init__(*a, **kw)
self.seedsseen = []
self.seedsseen: list[Any] = []
async def start(self):
async def start(self) -> AsyncIterator[Any]:
if self.fail_before_yield:
1 / 0
assert self.mockserver
for s in range(100):
qargs = {"total": 10, "seed": s}
url = self.mockserver.url(f"/follow?{urlencode(qargs, doseq=True)}")
@ -376,18 +403,18 @@ class BrokenStartSpider(FollowAllSpider):
assert self.seedsseen, "All seeds consumed before any download happened"
def parse(self, response):
def parse(self, response: Response) -> Iterator[Any]:
self.seedsseen.append(response.meta.get("seed"))
yield from super().parse(response)
class StartItemSpider(FollowAllSpider):
async def start(self):
async def start(self) -> AsyncIterator[Any]:
yield {"name": "test item"}
class StartGoodAndBadOutput(FollowAllSpider):
async def start(self):
async def start(self) -> AsyncIterator[Any]:
yield {"a": "a"}
yield Request("data:,a")
yield "data:,b"
@ -395,17 +422,18 @@ class StartGoodAndBadOutput(FollowAllSpider):
class SingleRequestSpider(MetaSpider):
seed = None
callback_func = None
errback_func = None
seed: Request | str | None = None
callback_func: Callable[[Response], Any] | None = None
errback_func: Callable[[Failure], Any] | None = None
async def start(self):
async def start(self) -> AsyncIterator[Any]:
if isinstance(self.seed, Request):
yield self.seed.replace(callback=self.parse, errback=self.on_error)
else:
assert self.seed
yield Request(self.seed, callback=self.parse, errback=self.on_error)
def parse(self, response):
def parse(self, response: Response) -> Any:
self.meta.setdefault("responses", []).append(response)
if callable(self.callback_func):
return self.callback_func(response)
@ -413,7 +441,7 @@ class SingleRequestSpider(MetaSpider):
return response.meta["next"]
return None
def on_error(self, failure):
def on_error(self, failure: Failure) -> Any:
self.meta["failure"] = failure
if callable(self.errback_func):
return self.errback_func(failure)
@ -426,17 +454,18 @@ class DuplicateStartSpider(MockServerSpider):
distinct_urls = 2
dupe_factor = 3
async def start(self):
async def start(self) -> AsyncIterator[Any]:
assert self.mockserver
for i in range(self.distinct_urls):
for _ in range(self.dupe_factor):
url = self.mockserver.url(f"/echo?headers=1&body=test{i}")
yield Request(url, dont_filter=self.dont_filter)
def __init__(self, url="http://localhost:8998", *args, **kwargs):
def __init__(self, url: str = "http://localhost:8998", *args: Any, **kwargs: Any):
super().__init__(*args, **kwargs)
self.visited = 0
def parse(self, response):
def parse(self, response: Response) -> None:
self.visited += 1
@ -451,7 +480,7 @@ class CrawlSpiderWithParseMethod(MockServerSpider, CrawlSpider):
}
rules = (Rule(LinkExtractor(), callback="parse", follow=True),)
async def start(self):
async def start(self) -> AsyncIterator[Any]:
test_body = b"""
<html>
<head><title>Page title</title></head>
@ -461,11 +490,13 @@ class CrawlSpiderWithParseMethod(MockServerSpider, CrawlSpider):
</body>
</html>
"""
assert self.mockserver
url = self.mockserver.url("/alpayload")
yield Request(url, method="POST", body=test_body)
def parse(self, response, foo=None):
def parse(self, response: Response, foo: str | None = None) -> Iterator[Any]:
self.logger.info("[parse] status %i (foo: %s)", response.status, foo)
assert self.mockserver
yield Request(
self.mockserver.url("/status?n=202"), self.parse, cb_kwargs={"foo": "bar"}
)
@ -477,8 +508,9 @@ class CrawlSpiderWithAsyncCallback(CrawlSpiderWithParseMethod):
name = "crawl_spider_with_async_callback"
rules = (Rule(LinkExtractor(), callback="parse_async", follow=True),)
async def parse_async(self, response, foo=None):
async def parse_async(self, response: Response, foo: str | None = None) -> Request:
self.logger.info("[parse_async] status %i (foo: %s)", response.status, foo)
assert self.mockserver
return Request(
self.mockserver.url("/status?n=202"),
self.parse_async,
@ -492,8 +524,11 @@ class CrawlSpiderWithAsyncGeneratorCallback(CrawlSpiderWithParseMethod):
name = "crawl_spider_with_async_generator_callback"
rules = (Rule(LinkExtractor(), callback="parse_async_gen", follow=True),)
async def parse_async_gen(self, response, foo=None):
async def parse_async_gen(
self, response: Response, foo: str | None = None
) -> AsyncIterator[Any]:
self.logger.info("[parse_async_gen] status %i (foo: %s)", response.status, foo)
assert self.mockserver
yield Request(
self.mockserver.url("/status?n=202"),
self.parse_async_gen,
@ -505,7 +540,7 @@ class CrawlSpiderWithErrback(CrawlSpiderWithParseMethod):
name = "crawl_spider_with_errback"
rules = (Rule(LinkExtractor(), callback="parse", errback="errback", follow=True),)
async def start(self):
async def start(self) -> AsyncIterator[Any]:
test_body = b"""
<html>
<head><title>Page title</title></head>
@ -518,17 +553,19 @@ class CrawlSpiderWithErrback(CrawlSpiderWithParseMethod):
</body>
</html>
"""
assert self.mockserver
url = self.mockserver.url("/alpayload")
yield Request(url, method="POST", body=test_body)
def errback(self, failure):
def errback(self, failure: Failure) -> None:
assert isinstance(failure.value, HttpError)
self.logger.info("[errback] status %i", failure.value.response.status)
class CrawlSpiderWithoutErrback(CrawlSpiderWithParseMethod):
name = "crawl_spider_without_errback"
async def start(self):
async def start(self) -> AsyncIterator[Any]:
test_body = b"""
<html>
<head><title>Page title</title></head>
@ -538,6 +575,7 @@ class CrawlSpiderWithoutErrback(CrawlSpiderWithParseMethod):
</body>
</html>
"""
assert self.mockserver
url = self.mockserver.url("/alpayload")
yield Request(url, method="POST", body=test_body)
@ -553,7 +591,7 @@ class CrawlSpiderWithProcessRequestCallbackKeywordArguments(CrawlSpiderWithParse
),
)
def process_request(self, request, response):
def process_request(self, request: Request, response: Response) -> Request:
request.cb_kwargs["foo"] = "process_request"
return request
@ -562,59 +600,65 @@ class BytesReceivedCallbackSpider(MetaSpider):
full_response_length = 2**18
@classmethod
def from_crawler(cls, crawler, *args, **kwargs):
def from_crawler(cls, crawler: Crawler, *args: Any, **kwargs: Any) -> Self:
spider = super().from_crawler(crawler, *args, **kwargs)
crawler.signals.connect(spider.bytes_received, signals.bytes_received)
return spider
async def start(self):
async def start(self) -> AsyncIterator[Any]:
body = b"a" * self.full_response_length
assert self.mockserver
url = self.mockserver.url("/alpayload", is_secure=self.is_secure)
yield Request(url, method="POST", body=body, errback=self.errback)
def parse(self, response):
def parse(self, response: Response) -> None:
self.meta["response"] = response
def errback(self, failure):
def errback(self, failure: Failure) -> None:
self.meta["failure"] = failure
def bytes_received(self, data, request, spider):
def bytes_received(self, data: bytes, request: Request, spider: Spider) -> None:
self.meta["bytes_received"] = data
raise StopDownload(fail=False)
class BytesReceivedErrbackSpider(BytesReceivedCallbackSpider):
def bytes_received(self, data, request, spider):
def bytes_received(self, data: bytes, request: Request, spider: Spider) -> None:
self.meta["bytes_received"] = data
raise StopDownload(fail=True)
class HeadersReceivedCallbackSpider(MetaSpider):
@classmethod
def from_crawler(cls, crawler, *args, **kwargs):
def from_crawler(cls, crawler: Crawler, *args: Any, **kwargs: Any) -> Self:
spider = super().from_crawler(crawler, *args, **kwargs)
crawler.signals.connect(spider.headers_received, signals.headers_received)
return spider
async def start(self):
async def start(self) -> AsyncIterator[Any]:
assert self.mockserver
yield Request(
self.mockserver.url("/status", is_secure=self.is_secure),
errback=self.errback,
)
def parse(self, response):
def parse(self, response: Response) -> None:
self.meta["response"] = response
def errback(self, failure):
def errback(self, failure: Failure) -> None:
self.meta["failure"] = failure
def headers_received(self, headers, body_length, request, spider):
def headers_received(
self, headers: Headers, body_length: int, request: Request, spider: Spider
) -> None:
self.meta["headers_received"] = headers
raise StopDownload(fail=False)
class HeadersReceivedErrbackSpider(HeadersReceivedCallbackSpider):
def headers_received(self, headers, body_length, request, spider):
def headers_received(
self, headers: Headers, body_length: int, request: Request, spider: Spider
) -> None:
self.meta["headers_received"] = headers
raise StopDownload(fail=True)
@ -623,13 +667,13 @@ class ExceptionSpider(Spider):
name = "exception"
@classmethod
def from_crawler(cls, crawler, *args, **kwargs):
def from_crawler(cls, crawler: Crawler, *args: Any, **kwargs: Any) -> Self:
raise ValueError("Exception in from_crawler method")
class NoRequestsSpider(Spider):
name = "no_request"
async def start(self):
async def start(self) -> AsyncIterator[Any]:
return
yield

16
tests/test_closespider.py
View File

@ -11,6 +11,8 @@ from tests.utils.decorators import inline_callbacks_test
class TestCloseSpider:
mockserver: MockServer
@classmethod
def setup_class(cls):
cls.mockserver = MockServer()
@ -25,6 +27,8 @@ class TestCloseSpider:
close_on = 5
crawler = get_crawler(ItemSpider, {"CLOSESPIDER_ITEMCOUNT": close_on})
yield crawler.crawl(mockserver=self.mockserver)
assert isinstance(crawler.spider, ItemSpider)
assert crawler.stats
reason = crawler.spider.meta["close_reason"]
assert reason == "closespider_itemcount"
itemcount = crawler.stats.get_value("item_scraped_count")
@ -35,6 +39,8 @@ class TestCloseSpider:
close_on = 5
crawler = get_crawler(FollowAllSpider, {"CLOSESPIDER_PAGECOUNT": close_on})
yield crawler.crawl(mockserver=self.mockserver)
assert isinstance(crawler.spider, FollowAllSpider)
assert crawler.stats
reason = crawler.spider.meta["close_reason"]
assert reason == "closespider_pagecount"
pagecount = crawler.stats.get_value("response_received_count")
@ -54,6 +60,8 @@ class TestCloseSpider:
yield crawler.crawl(
max_items=max_items, max_requests=max_requests, mockserver=self.mockserver
)
assert isinstance(crawler.spider, MaxItemsAndRequestsSpider)
assert crawler.stats
reason = crawler.spider.meta["close_reason"]
assert reason == "closespider_pagecount_no_item"
pagecount = crawler.stats.get_value("response_received_count")
@ -72,6 +80,8 @@ class TestCloseSpider:
},
)
yield crawler.crawl(mockserver=self.mockserver)
assert isinstance(crawler.spider, FollowAllSpider)
assert crawler.stats
reason = crawler.spider.meta["close_reason"]
assert reason == "closespider_pagecount_no_item"
pagecount = crawler.stats.get_value("response_received_count")
@ -82,6 +92,8 @@ class TestCloseSpider:
close_on = 5
crawler = get_crawler(ErrorSpider, {"CLOSESPIDER_ERRORCOUNT": close_on})
yield crawler.crawl(total=1000000, mockserver=self.mockserver)
assert isinstance(crawler.spider, ErrorSpider)
assert crawler.stats
reason = crawler.spider.meta["close_reason"]
assert reason == "closespider_errorcount"
key = f"spider_exceptions/{crawler.spider.exception_cls.__name__}"
@ -94,6 +106,8 @@ class TestCloseSpider:
close_on = 0.1
crawler = get_crawler(FollowAllSpider, {"CLOSESPIDER_TIMEOUT": close_on})
yield crawler.crawl(total=1000000, mockserver=self.mockserver)
assert isinstance(crawler.spider, FollowAllSpider)
assert crawler.stats
reason = crawler.spider.meta["close_reason"]
assert reason == "closespider_timeout"
total_seconds = crawler.stats.get_value("elapsed_time_seconds")
@ -104,6 +118,8 @@ class TestCloseSpider:
timeout = 1
crawler = get_crawler(SlowSpider, {"CLOSESPIDER_TIMEOUT_NO_ITEM": timeout})
yield crawler.crawl(n=3, mockserver=self.mockserver)
assert isinstance(crawler.spider, SlowSpider)
assert crawler.stats
reason = crawler.spider.meta["close_reason"]
assert reason == "closespider_timeout_no_item"
total_seconds = crawler.stats.get_value("elapsed_time_seconds")

9
tests/test_command_runspider.py
View File

@ -16,6 +16,11 @@ if TYPE_CHECKING:
from pathlib import Path
# Header for spider classes copied with inspect.getsource(): the future import
# keeps their annotations from being evaluated in the generated module.
SPIDER_HEADER = "from __future__ import annotations\nfrom scrapy import Spider\n"
class TestRunSpiderCommand:
spider_filename = "myspider.py"
@ -65,14 +70,14 @@ class BadSpider(scrapy.Spider):
def test_run_fail_spider(self, tmp_path: Path) -> None:
ret, _, _ = self.runspider(
tmp_path, "from scrapy import Spider\n" + inspect.getsource(ExceptionSpider)
tmp_path, SPIDER_HEADER + inspect.getsource(ExceptionSpider)
)
assert ret != 0
def test_run_good_spider(self, tmp_path: Path) -> None:
ret, _, _ = self.runspider(
tmp_path,
"from scrapy import Spider\n" + inspect.getsource(NoRequestsSpider),
SPIDER_HEADER + inspect.getsource(NoRequestsSpider),
)
assert ret == 0

201
tests/test_contracts.py
View File

@ -1,3 +1,6 @@
from __future__ import annotations
from typing import TYPE_CHECKING, Any
from unittest import TextTestResult
import pytest
@ -11,7 +14,7 @@ from scrapy.contracts.default import (
ScrapesContract,
UrlContract,
)
from scrapy.http import Request
from scrapy.http import Request, Response
from scrapy.item import Field, Item
from scrapy.spidermiddlewares.httperror import HttpError
from scrapy.spiders import Spider
@ -19,6 +22,24 @@ from scrapy.utils.test import get_crawler
from tests.mockserver.http import MockServer
from tests.utils.decorators import inline_callbacks_test
if TYPE_CHECKING:
from unittest import TestResult
from scrapy.http.request import CallbackT
def _request(
conman: ContractsManager, method: CallbackT, results: TestResult
) -> Request:
request = conman.from_method(method, results)
assert request
return request
def _call(request: Request, response: Any) -> Any:
assert request.callback
return request.callback(response, **request.cb_kwargs)
class DemoItem(Item):
name = Field()
@ -30,7 +51,7 @@ class ResponseMock:
class ResponseMetaMock(ResponseMock):
meta = None
meta: Any = None
class TaggedRequest(Request):
@ -289,17 +310,19 @@ class TestContractsManager:
def setup_method(self):
self.conman = ContractsManager(self.contracts)
self.results = TextTestResult(stream=None, descriptions=False, verbosity=0)
self.results = TextTestResult( # type: ignore[type-var]
stream=None, descriptions=False, verbosity=0
)
def should_succeed(self):
def should_succeed(self) -> None:
assert not self.results.failures
assert not self.results.errors
def should_fail(self):
def should_fail(self) -> None:
assert self.results.failures
assert not self.results.errors
def should_error(self):
def should_error(self) -> None:
assert self.results.errors
def test_contracts(self):
@ -313,12 +336,10 @@ class TestContractsManager:
)
# returns request for valid method
request = self.conman.from_method(spider.returns_request, self.results)
assert request is not None
assert self.conman.from_method(spider.returns_request, self.results) is not None
# no request for missing url
request = self.conman.from_method(spider.parse_no_url, self.results)
assert request is None
assert self.conman.from_method(spider.parse_no_url, self.results) is None
def test_cb_kwargs(self):
spider = DemoSpider()
@ -354,29 +375,31 @@ class TestContractsManager:
)
# returns_request
request = self.conman.from_method(
spider.returns_request_cb_kwargs, self.results
)
request.callback(response, **request.cb_kwargs)
request = _request(self.conman, spider.returns_request_cb_kwargs, self.results)
_call(request, response)
self.should_succeed()
# returns_item
request = self.conman.from_method(spider.returns_item_cb_kwargs, self.results)
request.callback(response, **request.cb_kwargs)
request = _request(self.conman, spider.returns_item_cb_kwargs, self.results)
_call(request, response)
self.should_succeed()
# returns_item (error, callback doesn't take keyword arguments)
request = self.conman.from_method(
spider.returns_item_cb_kwargs_error_unexpected_keyword, self.results
request = _request(
self.conman,
spider.returns_item_cb_kwargs_error_unexpected_keyword,
self.results,
)
request.callback(response, **request.cb_kwargs)
_call(request, response)
self.should_error()
# returns_item (error, contract doesn't provide keyword arguments)
request = self.conman.from_method(
spider.returns_item_cb_kwargs_error_missing_argument, self.results
request = _request(
self.conman,
spider.returns_item_cb_kwargs_error_missing_argument,
self.results,
)
request.callback(response, **request.cb_kwargs)
_call(request, response)
self.should_error()
def test_meta(self):
@ -398,29 +421,27 @@ class TestContractsManager:
response = ResponseMetaMock()
# returns_request
request = self.conman.from_method(spider.returns_request_meta, self.results)
request = _request(self.conman, spider.returns_request_meta, self.results)
assert request.meta["cookiejar"] == "session1"
response.meta = request.meta
request.callback(response)
_call(request, response)
assert response.meta["cookiejar"] == "session1"
self.should_succeed()
response = ResponseMetaMock()
# returns_item
request = self.conman.from_method(spider.returns_item_meta, self.results)
request = _request(self.conman, spider.returns_item_meta, self.results)
assert request.meta["key"] == "example"
response.meta = request.meta
request.callback(response)
_call(request, response)
assert response.meta["key"] == "example"
self.should_succeed()
response = ResponseMetaMock()
request = self.conman.from_method(
spider.returns_error_missing_meta, self.results
)
request.callback(response)
request = _request(self.conman, spider.returns_error_missing_meta, self.results)
_call(request, response)
self.should_error()
def test_returns(self):
@ -428,36 +449,36 @@ class TestContractsManager:
response = ResponseMock()
# returns_item
request = self.conman.from_method(spider.returns_item, self.results)
request.callback(response)
request = _request(self.conman, spider.returns_item, self.results)
_call(request, response)
self.should_succeed()
# returns_dict_item
request = self.conman.from_method(spider.returns_dict_item, self.results)
request.callback(response)
request = _request(self.conman, spider.returns_dict_item, self.results)
_call(request, response)
self.should_succeed()
# returns_request
request = self.conman.from_method(spider.returns_request, self.results)
request.callback(response)
request = _request(self.conman, spider.returns_request, self.results)
_call(request, response)
self.should_succeed()
# returns_fail
request = self.conman.from_method(spider.returns_fail, self.results)
request.callback(response)
request = _request(self.conman, spider.returns_fail, self.results)
_call(request, response)
self.should_fail()
# returns_dict_fail
request = self.conman.from_method(spider.returns_dict_fail, self.results)
request.callback(response)
request = _request(self.conman, spider.returns_dict_fail, self.results)
_call(request, response)
self.should_fail()
def test_returns_async(self):
spider = DemoSpider()
response = ResponseMock()
request = self.conman.from_method(spider.returns_request_async, self.results)
request.callback(response)
request = _request(self.conman, spider.returns_request_async, self.results)
_call(request, response)
self.should_error()
def test_returns_invalid_argument_count(self):
@ -477,10 +498,8 @@ class TestContractsManager:
spider = DemoSpider()
response = ResponseMock()
request = self.conman.from_method(
spider.returns_request_range_fail, self.results
)
request.callback(response)
request = _request(self.conman, spider.returns_request_range_fail, self.results)
_call(request, response)
self.should_fail()
assert "expected 2..3" in self.results.failures[-1][-1]
@ -490,8 +509,8 @@ class TestContractsManager:
# @returns and @scrapes only count matching output objects and skip
# the request that is also yielded.
request = self.conman.from_method(spider.yields_item_and_request, self.results)
request.callback(response)
request = _request(self.conman, spider.yields_item_and_request, self.results)
_call(request, response)
self.should_succeed()
def test_testcase_str(self):
@ -507,30 +526,30 @@ class TestContractsManager:
response = ResponseMock()
# scrapes_item_ok
request = self.conman.from_method(spider.scrapes_item_ok, self.results)
request.callback(response)
request = _request(self.conman, spider.scrapes_item_ok, self.results)
_call(request, response)
self.should_succeed()
# scrapes_dict_item_ok
request = self.conman.from_method(spider.scrapes_dict_item_ok, self.results)
request.callback(response)
request = _request(self.conman, spider.scrapes_dict_item_ok, self.results)
_call(request, response)
self.should_succeed()
# scrapes_item_fail
request = self.conman.from_method(spider.scrapes_item_fail, self.results)
request.callback(response)
request = _request(self.conman, spider.scrapes_item_fail, self.results)
_call(request, response)
self.should_fail()
# scrapes_dict_item_fail
request = self.conman.from_method(spider.scrapes_dict_item_fail, self.results)
request.callback(response)
request = _request(self.conman, spider.scrapes_dict_item_fail, self.results)
_call(request, response)
self.should_fail()
# scrapes_multiple_missing_fields
request = self.conman.from_method(
spider.scrapes_multiple_missing_fields, self.results
request = _request(
self.conman, spider.scrapes_multiple_missing_fields, self.results
)
request.callback(response)
_call(request, response)
self.should_fail()
message = "ContractFail: Missing fields: name, url"
assert message in self.results.failures[-1][-1]
@ -540,14 +559,13 @@ class TestContractsManager:
response = ResponseMock()
# invalid regex
request = self.conman.from_method(spider.invalid_regex, self.results)
assert request is None
assert self.conman.from_method(spider.invalid_regex, self.results) is None
# invalid regex with valid contract
request = self.conman.from_method(
spider.invalid_regex_with_valid_contract, self.results
request = _request(
self.conman, spider.invalid_regex_with_valid_contract, self.results
)
request.callback(response)
_call(request, response)
self.should_succeed()
def test_custom_contracts(self):
@ -559,14 +577,14 @@ class TestContractsManager:
def test_errback(self):
spider = DemoSpider()
response = ResponseMock()
try:
raise HttpError(response, "Ignoring non-200 response")
raise HttpError(Response("http://scrapy.org"), "Ignoring non-200 response")
except HttpError:
failure_mock = failure.Failure()
request = self.conman.from_method(spider.returns_request, self.results)
request = _request(self.conman, spider.returns_request, self.results)
assert request.errback
request.errback(failure_mock)
assert not self.results.failures
@ -602,11 +620,12 @@ class TestContractsManager:
crawler = get_crawler(TestSameUrlSpider)
yield crawler.crawl()
assert isinstance(crawler.spider, TestSameUrlSpider)
assert crawler.spider.visited == 2
def test_custom_tagged_request_contract(self):
spider = DemoSpider()
request = self.conman.from_method(spider.custom_tagged_request, self.results)
request = _request(self.conman, spider.custom_tagged_request, self.results)
assert request.method == "POST"
assert isinstance(request, TaggedRequest)
assert request.contract_tag == "custom"
@ -672,18 +691,20 @@ class PostProcessErrorContract(Contract):
class TestCustomContractPrePostProcess:
def setup_method(self):
self.results = TextTestResult(stream=None, descriptions=False, verbosity=0)
self.results = TextTestResult( # type: ignore[type-var]
stream=None, descriptions=False, verbosity=0
)
def test_pre_hook_keyboard_interrupt(self):
spider = DemoSpider()
response = ResponseMock()
contract = CustomFailContractPreProcess(spider.returns_request)
conman = ContractsManager([UrlContract, ReturnsContract, contract])
conman = ContractsManager([UrlContract, ReturnsContract])
request = conman.from_method(spider.returns_request, self.results)
request = _request(conman, spider.returns_request, self.results)
contract.add_pre_hook(request, self.results)
with pytest.raises(KeyboardInterrupt, match="Pre-process exception"):
request.callback(response, **request.cb_kwargs)
_call(request, response)
assert not self.results.failures
assert not self.results.errors
@ -692,12 +713,12 @@ class TestCustomContractPrePostProcess:
spider = DemoSpider()
response = ResponseMock()
contract = CustomFailContractPostProcess(spider.returns_request)
conman = ContractsManager([UrlContract, ReturnsContract, contract])
conman = ContractsManager([UrlContract, ReturnsContract])
request = conman.from_method(spider.returns_request, self.results)
request = _request(conman, spider.returns_request, self.results)
contract.add_post_hook(request, self.results)
with pytest.raises(KeyboardInterrupt, match="Post-process exception"):
request.callback(response, **request.cb_kwargs)
_call(request, response)
assert not self.results.failures
assert not self.results.errors
@ -706,11 +727,11 @@ class TestCustomContractPrePostProcess:
spider = DemoSpider()
response = ResponseMock()
contract = PreProcessSuccessContract(spider.returns_request)
conman = ContractsManager([UrlContract, ReturnsContract, contract])
conman = ContractsManager([UrlContract, ReturnsContract])
request = conman.from_method(spider.returns_request, self.results)
request = _request(conman, spider.returns_request, self.results)
contract.add_pre_hook(request, self.results)
request.callback(response, **request.cb_kwargs)
_call(request, response)
assert not self.results.failures
assert not self.results.errors
@ -719,11 +740,11 @@ class TestCustomContractPrePostProcess:
spider = DemoSpider()
response = ResponseMock()
contract = PreProcessAssertionFailContract(spider.returns_request)
conman = ContractsManager([UrlContract, ReturnsContract, contract])
conman = ContractsManager([UrlContract, ReturnsContract])
request = conman.from_method(spider.returns_request, self.results)
request = _request(conman, spider.returns_request, self.results)
contract.add_pre_hook(request, self.results)
request.callback(response, **request.cb_kwargs)
_call(request, response)
assert self.results.failures
assert not self.results.errors
@ -732,11 +753,11 @@ class TestCustomContractPrePostProcess:
spider = DemoSpider()
response = ResponseMock()
contract = PreProcessErrorContract(spider.returns_request)
conman = ContractsManager([UrlContract, ReturnsContract, contract])
conman = ContractsManager([UrlContract, ReturnsContract])
request = conman.from_method(spider.returns_request, self.results)
request = _request(conman, spider.returns_request, self.results)
contract.add_pre_hook(request, self.results)
request.callback(response, **request.cb_kwargs)
_call(request, response)
assert self.results.errors
@ -748,7 +769,7 @@ class TestCustomContractPrePostProcess:
contract.add_pre_hook(request, self.results)
with pytest.raises(TypeError, match="async callbacks"):
request.callback(response)
_call(request, response)
def test_pre_hook_async_generator(self):
spider = DemoSpider()
@ -758,7 +779,7 @@ class TestCustomContractPrePostProcess:
contract.add_pre_hook(request, self.results)
with pytest.raises(TypeError, match="async callbacks"):
request.callback(response)
_call(request, response)
def test_post_hook_async_generator(self):
spider = DemoSpider()
@ -768,16 +789,16 @@ class TestCustomContractPrePostProcess:
contract.add_post_hook(request, self.results)
with pytest.raises(TypeError, match="async callbacks"):
request.callback(response)
_call(request, response)
def test_post_hook_error(self):
spider = DemoSpider()
response = ResponseMock()
contract = PostProcessErrorContract(spider.returns_request)
conman = ContractsManager([UrlContract, ReturnsContract, contract])
conman = ContractsManager([UrlContract, ReturnsContract])
request = conman.from_method(spider.returns_request, self.results)
request = _request(conman, spider.returns_request, self.results)
contract.add_post_hook(request, self.results)
request.callback(response, **request.cb_kwargs)
_call(request, response)
assert self.results.errors

111
tests/test_engine_loop.py
View File

@ -13,6 +13,8 @@ from tests.mockserver.http import MockServer
from tests.utils.decorators import coroutine_test
if TYPE_CHECKING:
from collections.abc import AsyncIterator, Callable, Iterator
import pytest
from scrapy.http import Response
@ -62,19 +64,24 @@ class TestMain:
class TestSpider(Spider):
name = "test"
async def start(self):
async def start(self) -> AsyncIterator[Any]:
assert self.crawler.engine
assert self.crawler.engine._slot
scheduler = self.crawler.engine._slot.scheduler
assert isinstance(scheduler, MemoryScheduler)
yield Request("data:,a")
await sleep(seconds)
self.crawler.engine._slot.scheduler.pause()
self.crawler.engine._slot.scheduler.enqueue_request(Request("data:,b"))
scheduler.pause()
scheduler.enqueue_request(Request("data:,b"))
# During this time, the scheduler reports having requests but
# returns None.
await sleep(seconds)
self.crawler.engine._slot.scheduler.unpause()
scheduler.unpause()
# The scheduler request is processed.
await sleep(seconds)
@ -83,27 +90,28 @@ class TestMain:
await sleep(seconds)
self.crawler.engine._slot.scheduler.pause()
self.crawler.engine._slot.scheduler.enqueue_request(Request("data:,d"))
scheduler.pause()
scheduler.enqueue_request(Request("data:,d"))
# The last start request is processed during the time until the
# delayed call below, proving that the start iteration can
# finish before a scheduler “sleep” without causing the
# scheduler to finish.
call_later(seconds, self.crawler.engine._slot.scheduler.unpause)
call_later(seconds, scheduler.unpause)
def parse(self, response):
def parse(self, response: Response) -> None:
pass
actual_urls = []
def track_url(request, spider):
def track_url(request: Request, spider: Spider) -> None:
actual_urls.append(request.url)
settings = {"SCHEDULER": MemoryScheduler}
crawler = get_crawler(TestSpider, settings_dict=settings)
crawler.signals.connect(track_url, signals.request_reached_downloader)
await crawler.crawl_async()
assert crawler.stats
assert crawler.stats.get_value("finish_reason") == "finished"
expected_urls = ["data:,a", "data:,b", "data:,c", "data:,d"]
assert actual_urls == expected_urls, f"{actual_urls=} != {expected_urls=}"
@ -115,17 +123,17 @@ class TestMain:
class TestSpider(Spider):
name = "test"
async def start(self):
async def start(self) -> AsyncIterator[Any]:
assert self.crawler.engine is not None
await self.crawler.engine.close_async()
yield Request("data:,a")
def parse(self, response):
def parse(self, response: Response) -> None:
pass
actual_urls = []
def track_url(request, spider):
def track_url(request: Request, spider: Spider) -> None:
actual_urls.append(request.url)
settings = {"SCHEDULER": MemoryScheduler}
@ -211,6 +219,8 @@ class TestMain:
class TestRequestSendOrder:
mockserver: MockServer
seconds = 0.1 # increase if flaky
@classmethod
@ -222,57 +232,71 @@ class TestRequestSendOrder:
def teardown_class(cls):
cls.mockserver.__exit__(None, None, None)
def request(self, num, response_seconds, download_slots, priority=0):
def request(
self,
num: int,
response_seconds: float,
download_slots: int,
priority: int = 0,
) -> Request:
url = self.mockserver.url(f"/delay?n={response_seconds}&{num}")
meta = {"download_slot": str(num % download_slots)}
return Request(url, meta=meta, priority=priority)
def get_num(self, request_or_response: Request | Response):
def get_num(self, request_or_response: Request | Response) -> int:
return int(request_or_response.url.rsplit("&", maxsplit=1)[1])
async def _test_request_order(
self,
start_nums,
cb_nums=None,
settings=None,
response_seconds=None,
download_slots=1,
start_fn=None,
parse_fn=None,
):
start_nums: list[int],
cb_nums: list[int] | None = None,
settings: dict[str, Any] | None = None,
response_seconds: float | None = None,
download_slots: int = 1,
start_fn: Callable[[Spider], AsyncIterator[Any]] | None = None,
parse_fn: Callable[..., Iterator[Any]] | None = None,
) -> None:
cb_nums = cb_nums or []
settings = settings or {}
response_seconds = response_seconds or self.seconds
seconds = response_seconds or self.seconds
cb_requests = deque(
[self.request(num, response_seconds, download_slots) for num in cb_nums]
[self.request(num, seconds, download_slots) for num in cb_nums]
)
if start_fn is None:
async def start_fn(spider):
async def default_start(spider: Spider) -> AsyncIterator[Any]:
for num in start_nums:
yield self.request(num, response_seconds, download_slots)
yield self.request(num, seconds, download_slots)
start_fn = default_start
if parse_fn is None:
def parse_fn(spider, response):
def default_parse(spider: Spider, response: Response) -> Iterator[Any]:
while cb_requests:
yield cb_requests.popleft()
parse_fn = default_parse
spider_start = start_fn
spider_parse = parse_fn
class TestSpider(Spider):
name = "test"
start = start_fn
parse = parse_fn
start = spider_start
parse = spider_parse
actual_nums = []
def track_num(request, spider):
def track_num(request: Request, spider: Spider) -> None:
actual_nums.append(self.get_num(request))
crawler = get_crawler(TestSpider, settings_dict=settings)
crawler.signals.connect(track_num, signals.request_reached_downloader)
await crawler.crawl_async()
assert crawler.stats
assert crawler.stats.get_value("finish_reason") == "finished"
expected_nums = sorted(start_nums + cb_nums)
assert actual_nums == expected_nums, f"{actual_nums=} != {expected_nums=}"
@ -286,12 +310,14 @@ class TestRequestSendOrder:
response_seconds = 0
download_slots = 1
def _request(num, priority=0):
def _request(num: int, priority: int = 0) -> Request:
return self.request(
num, response_seconds, download_slots, priority=priority
)
async def start(spider):
async def start(spider: Spider) -> AsyncIterator[Any]:
assert spider.crawler.engine
assert spider.crawler.engine._slot
# The first CONCURRENT_REQUESTS start requests are sent
# immediately.
yield _request(1)
@ -305,7 +331,7 @@ class TestRequestSendOrder:
yield _request(3, priority=1)
yield _request(4, priority=1)
def parse(spider, response):
def parse(spider: Spider, response: Response) -> Iterator[Any]:
return
yield
@ -326,12 +352,14 @@ class TestRequestSendOrder:
response_seconds = 0
download_slots = 1
def _request(num, priority=0):
def _request(num: int, priority: int = 0) -> Request:
return self.request(
num, response_seconds, download_slots, priority=priority
)
async def start(spider):
async def start(spider: Spider) -> AsyncIterator[Any]:
assert spider.crawler.engine
assert spider.crawler.engine._slot
# The first CONCURRENT_REQUESTS start requests are sent
# immediately.
yield _request(1)
@ -345,7 +373,7 @@ class TestRequestSendOrder:
yield _request(4, priority=1)
yield _request(3, priority=1)
def parse(spider, response):
def parse(spider: Spider, response: Response) -> Iterator[Any]:
return
yield
@ -369,12 +397,14 @@ class TestRequestSendOrder:
response_seconds = 0
download_slots = 1
def _request(num, priority=0):
def _request(num: int, priority: int = 0) -> Request:
return self.request(
num, response_seconds, download_slots, priority=priority
)
async def start(spider):
async def start(spider: Spider) -> AsyncIterator[Any]:
assert spider.crawler.engine
assert spider.crawler.engine._slot
# The first CONCURRENT_REQUESTS start requests are sent
# immediately.
yield _request(1)
@ -403,7 +433,7 @@ class TestRequestSendOrder:
):
spider.crawler.engine._slot.scheduler.enqueue_request(request)
def parse(spider, response):
def parse(spider: Spider, response: Response) -> Iterator[Any]:
return
yield
@ -428,8 +458,9 @@ class TestRequestSendOrder:
response_seconds = self.seconds * 2**1 # increase if flaky
download_slots = 1
async def start(spider):
async def start(spider: Spider) -> AsyncIterator[Any]:
for num in start_nums:
assert spider.crawler.engine
if spider.crawler.engine.needs_backout():
await spider.crawler.signals.wait_for(signals.scheduler_empty)
request = self.request(num, response_seconds, download_slots)

4
tests/test_spider.py
View File

@ -41,7 +41,7 @@ class TestXMLFeedSpider(TestSpiderBase):
</urlset>"""
response = XmlResponse(url="http://example.com/sitemap.xml", body=body)
class _XMLSpider(self.spider_class):
class _XMLSpider(self.spider_class): # type: ignore[name-defined,misc]
itertag = "url"
namespaces = (
("a", "http://www.google.com/schemas/sitemap/0.84"),
@ -166,7 +166,7 @@ class TestCSVFeedSpider(TestSpiderBase):
body = get_testdata("feeds", "feed-sample6.csv")
response = Response("http://example.org/dummy.csv", body=body)
class _CrawlSpider(self.spider_class):
class _CrawlSpider(self.spider_class): # type: ignore[name-defined,misc]
name = "test"
delimiter = ","
quotechar = "'"

53
tests/test_spider_crawl.py
View File

@ -2,6 +2,7 @@ from __future__ import annotations
import re
import warnings
from typing import TYPE_CHECKING, Any
import pytest
from w3lib.url import safe_url_string
@ -14,6 +15,12 @@ from scrapy.utils.test import get_crawler
from tests.utils.bases.spider import TestSpiderBase
from tests.utils.decorators import coroutine_test
if TYPE_CHECKING:
from scrapy.http import Response
from scrapy.http.request import CallbackT
URL = "https://www.example.com"
class TestCrawlSpider(TestSpiderBase):
test_body = b"""<html><head><title>Page title</title></head>
@ -33,7 +40,7 @@ class TestCrawlSpider(TestSpiderBase):
"http://example.org/somepage/index.html", body=self.test_body
)
class _CrawlSpider(self.spider_class):
class _CrawlSpider(self.spider_class): # type: ignore[name-defined,misc]
name = "test"
allowed_domains = ["example.org"]
rules = (Rule(),)
@ -53,7 +60,7 @@ class TestCrawlSpider(TestSpiderBase):
"http://example.org/somepage/index.html", body=self.test_body
)
class _CrawlSpider(self.spider_class):
class _CrawlSpider(self.spider_class): # type: ignore[name-defined,misc]
name = "test"
allowed_domains = ["example.org"]
rules = (Rule(LinkExtractor(), process_links="dummy_process_links"),)
@ -76,7 +83,7 @@ class TestCrawlSpider(TestSpiderBase):
"http://example.org/somepage/index.html", body=self.test_body
)
class _CrawlSpider(self.spider_class):
class _CrawlSpider(self.spider_class): # type: ignore[name-defined,misc]
name = "test"
allowed_domains = ["example.org"]
rules = (Rule(LinkExtractor(), process_links="filter_process_links"),)
@ -99,7 +106,7 @@ class TestCrawlSpider(TestSpiderBase):
"http://example.org/somepage/index.html", body=self.test_body
)
class _CrawlSpider(self.spider_class):
class _CrawlSpider(self.spider_class): # type: ignore[name-defined,misc]
name = "test"
allowed_domains = ["example.org"]
rules = (Rule(LinkExtractor(), process_links="dummy_process_links"),)
@ -125,7 +132,7 @@ class TestCrawlSpider(TestSpiderBase):
def process_request_change_domain(request, response):
return request.replace(url=request.url.replace(".org", ".com"))
class _CrawlSpider(self.spider_class):
class _CrawlSpider(self.spider_class): # type: ignore[name-defined,misc]
name = "test"
allowed_domains = ["example.org"]
rules = (
@ -151,7 +158,7 @@ class TestCrawlSpider(TestSpiderBase):
request.meta["response_class"] = response.__class__.__name__
return request
class _CrawlSpider(self.spider_class):
class _CrawlSpider(self.spider_class): # type: ignore[name-defined,misc]
name = "test"
allowed_domains = ["example.org"]
rules = (
@ -180,7 +187,7 @@ class TestCrawlSpider(TestSpiderBase):
"http://example.org/somepage/index.html", body=self.test_body
)
class _CrawlSpider(self.spider_class):
class _CrawlSpider(self.spider_class): # type: ignore[name-defined,misc]
name = "test"
allowed_domains = ["example.org"]
rules = (Rule(LinkExtractor(), process_request="process_request_upper"),)
@ -203,7 +210,7 @@ class TestCrawlSpider(TestSpiderBase):
"http://example.org/somepage/index.html", body=self.test_body
)
class _CrawlSpider(self.spider_class):
class _CrawlSpider(self.spider_class): # type: ignore[name-defined,misc]
name = "test"
allowed_domains = ["example.org"]
rules = (
@ -248,7 +255,6 @@ class TestCrawlSpider(TestSpiderBase):
def test_parse_response_use(self):
class _CrawlSpider(CrawlSpider):
name = "test"
start_urls = "https://www.example.com"
_follow_links = False
with warnings.catch_warnings():
@ -258,17 +264,20 @@ class TestCrawlSpider(TestSpiderBase):
ScrapyDeprecationWarning,
match=r"CrawlSpider\._parse_response method is deprecated",
):
spider._parse_response(
TextResponse(spider.start_urls, body=b""), None, None
)
spider._parse_response(TextResponse(URL, body=b""), None, {})
def test_parse_response_override(self):
class _CrawlSpider(CrawlSpider):
def _parse_response(self, response, callback, cb_kwargs, follow=True):
def _parse_response(
self,
response: Response,
callback: CallbackT | None,
cb_kwargs: dict[str, Any],
follow: bool = True,
) -> Any:
pass
name = "test"
start_urls = "https://www.example.com"
_follow_links = False
with pytest.warns(
@ -278,21 +287,16 @@ class TestCrawlSpider(TestSpiderBase):
spider = _CrawlSpider()
with warnings.catch_warnings():
warnings.simplefilter("error", category=ScrapyDeprecationWarning)
spider._parse_response(
TextResponse(spider.start_urls, body=b""), None, None
)
spider._parse_response(TextResponse(URL, body=b""), None, {})
def test_parse_with_rules(self):
class _CrawlSpider(CrawlSpider):
name = "test"
start_urls = "https://www.example.com"
with warnings.catch_warnings():
warnings.simplefilter("error", category=ScrapyDeprecationWarning)
spider = _CrawlSpider()
spider.parse_with_rules(
TextResponse(spider.start_urls, body=b""), None, None
)
spider.parse_with_rules(TextResponse(URL, body=b""), None, {})
@coroutine_test
async def test_parse_with_rules_without_callback(self):
@ -327,12 +331,13 @@ class TestCrawlSpider(TestSpiderBase):
allowed_domains = ["example.org"]
rules = (Rule(),)
def callback(response: Response) -> list[Any]:
return [item]
spider = _CrawlSpider.from_crawler(get_crawler(_CrawlSpider))
results = [
r
async for r in spider.parse_with_rules(
response, lambda response: [item], {}, follow=False
)
async for r in spider.parse_with_rules(response, callback, {}, follow=False)
]
assert results == [item]

16
tests/test_spider_sitemap.py
View File

@ -55,7 +55,7 @@ class TestSitemapSpider(TestSpiderBase):
assert spider._get_sitemap_body(response) == body
def test_get_sitemap_body(self):
r = XmlResponse(url="http://www.example.com/", body=self.BODY)
r: Response = XmlResponse(url="http://www.example.com/", body=self.BODY)
self.assertSitemapBody(r, self.BODY)
r = HtmlResponse(url="http://www.example.com/", body=self.BODY)
@ -166,7 +166,7 @@ Sitemap: /sitemap-relative-url.xml
</url>
</urlset>"""
class FilteredSitemapSpider(self.spider_class):
class FilteredSitemapSpider(self.spider_class): # type: ignore[name-defined,misc]
def sitemap_filter(self, entries):
for entry in entries:
date_time = datetime.strptime(entry["lastmod"], "%Y-%m-%d")
@ -201,7 +201,7 @@ Sitemap: /sitemap-relative-url.xml
</url>
</urlset>"""
class FilteredSitemapSpider(self.spider_class):
class FilteredSitemapSpider(self.spider_class): # type: ignore[name-defined,misc]
def sitemap_filter(self, entries):
for entry in entries:
alternate_links = entry.get("alternate", ())
@ -235,7 +235,7 @@ Sitemap: /sitemap-relative-url.xml
</sitemap>
</sitemapindex>"""
class FilteredSitemapSpider(self.spider_class):
class FilteredSitemapSpider(self.spider_class): # type: ignore[name-defined,misc]
def sitemap_filter(self, entries):
for entry in entries:
date_time = datetime.strptime(
@ -285,6 +285,7 @@ Sitemap: /sitemap-relative-url.xml
sitemap_rules = [("", parse_item)]
def raw_body(self):
assert self.mockserver
loc = self.mockserver.url("/text")
return (
'<?xml version="1.0" encoding="UTF-8"?>'
@ -303,6 +304,7 @@ Sitemap: /sitemap-relative-url.xml
yield {"url": response.url}
def raw_body(self):
assert self.mockserver
loc = self.mockserver.url("/text")
return (
'<?xml version="1.0" encoding="UTF-8"?>'
@ -363,7 +365,7 @@ Sitemap: /sitemap-relative-url.xml
</sitemapindex>"""
r = TextResponse(url="http://www.example.com/sitemap.xml", body=sitemap)
class _FollowSpider(self.spider_class):
class _FollowSpider(self.spider_class): # type: ignore[name-defined,misc]
sitemap_follow = [follow]
spider = _FollowSpider("example.com")
@ -382,7 +384,7 @@ Sitemap: /sitemap-relative-url.xml
@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
def test_compression_bomb_spider_attr(self):
class DownloadMaxSizeSpider(self.spider_class):
class DownloadMaxSizeSpider(self.spider_class): # type: ignore[name-defined,misc]
download_maxsize = 10_000_000
crawler = get_crawler()
@ -485,7 +487,7 @@ Sitemap: /sitemap-relative-url.xml
@coroutine_test
async def test_sitemap_urls(self):
class TestSpider(self.spider_class):
class TestSpider(self.spider_class): # type: ignore[name-defined,misc]
name = "test"
sitemap_urls = ["https://toscrape.com/sitemap.xml"]

39
tests/test_spidermiddleware_output_chain.py
View File

@ -1,16 +1,21 @@
from __future__ import annotations
import logging
from typing import TYPE_CHECKING
from typing import TYPE_CHECKING, Any
from scrapy import Request, Spider
from scrapy.utils.test import get_crawler
from tests.mockserver.http import MockServer
from tests.spiders import MockServerSpider
from tests.utils.decorators import coroutine_test
if TYPE_CHECKING:
from collections.abc import AsyncIterator, Iterator
import pytest
from scrapy.http import Response
class _BaseSpiderMiddleware:
def __init__(self, crawler):
@ -41,7 +46,7 @@ class RecoveryMiddleware(_BaseSpiderMiddleware):
]
class RecoverySpider(Spider):
class RecoverySpider(MockServerSpider):
name = "RecoverySpider"
custom_settings = {
"SPIDER_MIDDLEWARES_BASE": {},
@ -50,10 +55,11 @@ class RecoverySpider(Spider):
},
}
async def start(self):
async def start(self) -> AsyncIterator[Any]:
assert self.mockserver
yield Request(self.mockserver.url("/status?n=200"))
def parse(self, response):
def parse(self, response: Response) -> Iterator[Any]:
yield {"test": 1}
self.logger.info("DONT_FAIL: %s", response.meta.get("dont_fail"))
if not response.meta.get("dont_fail"):
@ -63,7 +69,7 @@ class RecoverySpider(Spider):
class RecoveryAsyncGenSpider(RecoverySpider):
name = "RecoveryAsyncGenSpider"
async def parse(self, response):
async def parse(self, response: Response) -> AsyncIterator[Any]: # type: ignore[override]
for r in super().parse(response):
yield r
@ -76,7 +82,7 @@ class FailProcessSpiderInputMiddleware(_BaseSpiderMiddleware):
raise IndexError
class ProcessSpiderInputSpiderWithoutErrback(Spider):
class ProcessSpiderInputSpiderWithoutErrback(MockServerSpider):
name = "ProcessSpiderInputSpiderWithoutErrback"
custom_settings = {
"SPIDER_MIDDLEWARES": {
@ -87,7 +93,8 @@ class ProcessSpiderInputSpiderWithoutErrback(Spider):
}
}
async def start(self):
async def start(self) -> AsyncIterator[Any]:
assert self.mockserver
yield Request(url=self.mockserver.url("/status?n=200"), callback=self.parse)
def parse(self, response):
@ -97,7 +104,8 @@ class ProcessSpiderInputSpiderWithoutErrback(Spider):
class ProcessSpiderInputSpiderWithErrback(ProcessSpiderInputSpiderWithoutErrback):
name = "ProcessSpiderInputSpiderWithErrback"
async def start(self):
async def start(self) -> AsyncIterator[Any]:
assert self.mockserver
yield Request(
self.mockserver.url("/status?n=200"), self.parse, errback=self.errback
)
@ -109,7 +117,7 @@ class ProcessSpiderInputSpiderWithErrback(ProcessSpiderInputSpiderWithoutErrback
# ================================================================================
# (2) exceptions from a spider callback (generator)
class GeneratorCallbackSpider(Spider):
class GeneratorCallbackSpider(MockServerSpider):
name = "GeneratorCallbackSpider"
custom_settings = {
"SPIDER_MIDDLEWARES": {
@ -117,7 +125,8 @@ class GeneratorCallbackSpider(Spider):
},
}
async def start(self):
async def start(self) -> AsyncIterator[Any]:
assert self.mockserver
yield Request(self.mockserver.url("/status?n=200"))
def parse(self, response):
@ -146,7 +155,7 @@ class GeneratorCallbackSpiderMiddlewareRightAfterSpider(GeneratorCallbackSpider)
# ================================================================================
# (3) exceptions from a spider callback (not a generator)
class NotGeneratorCallbackSpider(Spider):
class NotGeneratorCallbackSpider(MockServerSpider):
name = "NotGeneratorCallbackSpider"
custom_settings = {
"SPIDER_MIDDLEWARES": {
@ -154,7 +163,8 @@ class NotGeneratorCallbackSpider(Spider):
},
}
async def start(self):
async def start(self) -> AsyncIterator[Any]:
assert self.mockserver
yield Request(self.mockserver.url("/status?n=200"))
def parse(self, response):
@ -224,7 +234,7 @@ class GeneratorDoNothingAfterRecoveryMiddleware(_GeneratorDoNothingMiddleware):
pass
class GeneratorOutputChainSpider(Spider):
class GeneratorOutputChainSpider(MockServerSpider):
name = "GeneratorOutputChainSpider"
custom_settings = {
"SPIDER_MIDDLEWARES": {
@ -235,7 +245,8 @@ class GeneratorOutputChainSpider(Spider):
},
}
async def start(self):
async def start(self) -> AsyncIterator[Any]:
assert self.mockserver
yield Request(self.mockserver.url("/status?n=200"))
def parse(self, response):

39
tests/test_spidermiddleware_process_start.py
View File

@ -1,4 +1,7 @@
from __future__ import annotations
from asyncio import sleep
from typing import TYPE_CHECKING, Any
import pytest
@ -9,6 +12,11 @@ from scrapy.utils.test import get_crawler
from .utils import twisted_sleep
from .utils.decorators import coroutine_test
if TYPE_CHECKING:
from collections.abc import AsyncIterator
from scrapy.http import Response
ITEM_A = {"id": "a"}
ITEM_B = {"id": "b"}
ITEM_C = {"id": "c"}
@ -17,20 +25,20 @@ SLEEP_SECONDS = 0.1
class AsyncioSleepSpiderMiddleware:
async def process_start(self, start):
async def process_start(self, start: AsyncIterator[Any]) -> AsyncIterator[Any]:
await sleep(SLEEP_SECONDS)
async for item_or_request in start:
yield item_or_request
class NoOpSpiderMiddleware:
async def process_start(self, start):
async def process_start(self, start: AsyncIterator[Any]) -> AsyncIterator[Any]:
async for item_or_request in start:
yield item_or_request
class TwistedSleepSpiderMiddleware:
async def process_start(self, start):
async def process_start(self, start: AsyncIterator[Any]) -> AsyncIterator[Any]:
await maybe_deferred_to_future(twisted_sleep(SLEEP_SECONDS))
async for item_or_request in start:
yield item_or_request
@ -42,12 +50,12 @@ class TwistedSleepSpiderMiddleware:
class ModernWrapSpider(Spider):
name = "test"
async def start(self):
async def start(self) -> AsyncIterator[Any]:
yield ITEM_B
class ModernWrapSpiderMiddleware:
async def process_start(self, start):
async def process_start(self, start: AsyncIterator[Any]) -> AsyncIterator[Any]:
yield ITEM_A
async for item_or_request in start:
yield item_or_request
@ -55,10 +63,15 @@ class ModernWrapSpiderMiddleware:
class TestMain:
async def _test(self, spider_middlewares, spider_cls, expected_items):
async def _test(
self,
spider_middlewares: list[type],
spider_cls: type[Spider],
expected_items: list[Any],
) -> None:
actual_items = []
def track_item(item, response, spider):
def track_item(item: Any, response: Response, spider: Spider) -> None:
actual_items.append(item)
settings = {
@ -67,10 +80,16 @@ class TestMain:
crawler = get_crawler(spider_cls, settings_dict=settings)
crawler.signals.connect(track_item, signals.item_scraped)
await crawler.crawl_async()
assert crawler.stats
assert crawler.stats.get_value("finish_reason") == "finished"
assert actual_items == expected_items, f"{actual_items=} != {expected_items=}"
async def _test_wrap(self, spider_middleware, spider_cls, expected_items=None):
async def _test_wrap(
self,
spider_middleware: type,
spider_cls: type[Spider],
expected_items: list[Any] | None = None,
) -> None:
expected_items = expected_items or [ITEM_A, ITEM_B, ITEM_C]
await self._test([spider_middleware], spider_cls, expected_items)
@ -78,11 +97,11 @@ class TestMain:
async def test_modern_mw_modern_spider(self):
await self._test_wrap(ModernWrapSpiderMiddleware, ModernWrapSpider)
async def _test_sleep(self, spider_middlewares):
async def _test_sleep(self, spider_middlewares: list[type]) -> None:
class TestSpider(Spider):
name = "test"
async def start(self):
async def start(self) -> AsyncIterator[Any]:
yield ITEM_A
await self._test(spider_middlewares, TestSpider, [ITEM_A])

8
tests/utils/bases/spider.py
View File

@ -32,7 +32,7 @@ class TestSpiderBase(ABC):
def test_spider_args(self):
"""``__init__`` method arguments are assigned to spider attributes"""
spider = self.spider_class("example.com", foo="bar")
assert spider.foo == "bar"
assert spider.foo == "bar" # type: ignore[attr-defined]
def test_spider_without_name(self):
"""``__init__`` raises when the name is not provided."""
@ -58,7 +58,7 @@ class TestSpiderBase(ABC):
mock_init.assert_called_once_with("example.com", foo="bar")
def test_closed_signal_call(self):
class TestSpider(self.spider_class):
class TestSpider(self.spider_class): # type: ignore[name-defined,misc]
closed_called = False
def closed(self, reason):
@ -92,7 +92,7 @@ class TestSpiderBase(ABC):
**get_reactor_settings(),
}
class TestSpider(self.spider_class):
class TestSpider(self.spider_class): # type: ignore[name-defined,misc]
name = "test"
custom_settings = spider_settings
@ -127,5 +127,5 @@ class TestSpiderBase(ABC):
ScrapyDeprecationWarning, match=r"Spider.log\(\) is deprecated"
),
):
spider.log("test log msg", "INFO")
spider.log("test log msg", "INFO") # type: ignore[arg-type]
mock_logger.log.assert_called_once_with("INFO", "test log msg")