RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge 86a3c0e0ee into ad43bf0c56

This commit is contained in:
Adrian 2026-08-15 11:31:49 -05:00 committed by GitHub
parent ad43bf0c56 86a3c0e0ee
commit 35827b16d4
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
39 changed files with 411 additions and 362 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
pyproject.toml
View File

@ -116,13 +116,9 @@ allow_incomplete_defs = true # 59 errors
# TODO
[[tool.mypy.overrides]]
module = [
"tests.spiders",
"tests.test_closespider",
"tests.test_cmdline",
"tests.test_contracts",
"tests.test_downloaderslotssettings",
"tests.test_dupefilters",
"tests.test_engine_loop",
"tests.test_extension_statsmailer",
"tests.test_extension_throttle",
"tests.test_linkextractors",
@ -131,15 +127,9 @@ module = [
"tests.test_pqueues",
"tests.test_scheduler_base",
"tests.test_settings",
"tests.test_spider",
"tests.test_spider_crawl",
"tests.test_spidermiddleware_output_chain",
"tests.test_spidermiddleware_process_start",
"tests.test_spider_sitemap",
"tests.test_squeues",
"tests.test_squeues_request",
"tests.test_stats",
"tests.utils.bases.spider",
]
check_untyped_defs = false

1
tests/benchmarks/__init__.py
View File

@ -36,7 +36,6 @@ class NullDownloadHandler:
async def download_request(self, request: Request) -> Response:
self._active += 1
assert self._crawler.stats
self._crawler.stats.max_value("benchmark/peak_concurrency", self._active)
try:
await asyncio.sleep(0)

5
tests/benchmarks/test_crawl.py
View File

@ -128,7 +128,6 @@ class _DelayedPipeline:
async def process_item(self, item: Any) -> Any:
url = item["url"]
self._active[url] += 1
assert self._crawler.stats
self._crawler.stats.max_value("benchmark/peak_items", self._active[url])
try:
await asyncio.sleep(DELAY)
@ -147,7 +146,6 @@ def _crawl_tree(
pages=pages,
items=items,
)
assert crawler.stats
assert crawler.stats.get_value("downloader/response_count") == domains * pages
assert crawler.stats.get_value("item_scraped_count", 0) == domains * pages * items
return crawler
@ -166,7 +164,6 @@ def test_overhead_http(benchmark: BenchmarkFixture, mockserver: MockServer) -> N
def run() -> None:
crawler = crawl(_FollowSpider, settings, url=url)
assert crawler.stats
assert crawler.stats.get_value("item_scraped_count") == PAGES + 1
benchmark(run)
@ -177,7 +174,6 @@ def test_overhead_engine(benchmark: BenchmarkFixture) -> None:
def run() -> None:
crawler = _crawl_tree({}, domains=1, pages=REQUESTS)
assert crawler.stats
assert crawler.stats.get_value("benchmark/peak_concurrency") > 1
benchmark(run)
@ -262,7 +258,6 @@ def test_overhead_item_concurrency(benchmark: BenchmarkFixture) -> None:
crawler = _crawl_tree(
settings, domains=1, pages=ITEM_REQUESTS, items=ITEMS_PER_RESPONSE
)
assert crawler.stats
assert (
crawler.stats.get_value("benchmark/peak_items") == DELAYED_CONCURRENT_ITEMS
)

204
tests/spiders.py
View File

@ -13,22 +13,31 @@ from twisted.internet import defer
from scrapy import signals
from scrapy.exceptions import StopDownload
from scrapy.http import Request
from scrapy.http import Request, TextResponse
from scrapy.item import Item
from scrapy.linkextractors import LinkExtractor
from scrapy.spidermiddlewares.httperror import HttpError
from scrapy.spiders import Spider
from scrapy.spiders.crawl import CrawlSpider, Rule
from scrapy.utils.defer import deferred_to_future, maybe_deferred_to_future
from scrapy.utils.test import get_from_asyncio_queue
if TYPE_CHECKING:
from collections.abc import AsyncIterator, Callable, Iterator
from twisted.python.failure import Failure
from typing_extensions import Self
from scrapy.crawler import Crawler
from scrapy.http import Headers, Response
from scrapy.http.request import CallbackT
from tests.mockserver.http import MockServer
class MockServerSpider(Spider):
def __init__(
self,
*args,
*args: Any,
mockserver: MockServer | None = None,
is_secure: bool = False,
**kwargs: Any,
@ -74,7 +83,7 @@ class MetaSpider(MockServerSpider):
super().__init__(*args, **kwargs)
self.meta: dict[str, Any] = {}
def closed(self, reason):
def closed(self, reason: str) -> None:
self.meta["close_reason"] = reason
@ -83,16 +92,24 @@ class FollowAllSpider(MetaSpider):
link_extractor = LinkExtractor()
def __init__(
self, total=10, show=20, order="rand", maxlatency=0.0, *args, **kwargs
self,
total: int = 10,
show: int = 20,
order: str = "rand",
maxlatency: float = 0.0,
*args: Any,
**kwargs: Any,
):
super().__init__(*args, **kwargs)
self.urls_visited = []
self.times = []
self.urls_visited: list[str] = []
self.times: list[float] = []
qargs = {"total": total, "show": show, "order": order, "maxlatency": maxlatency}
assert self.mockserver
url = self.mockserver.url(f"/follow?{urlencode(qargs, doseq=True)}")
self.start_urls = [url]
def parse(self, response):
def parse(self, response: Response) -> Iterator[Any]:
assert isinstance(response, TextResponse)
self.urls_visited.append(response.url)
self.times.append(time.time())
for link in self.link_extractor.extract_links(response):
@ -102,50 +119,52 @@ class FollowAllSpider(MetaSpider):
class DelaySpider(MetaSpider):
name = "delay"
def __init__(self, n=1, b=0, *args, **kwargs):
def __init__(self, n: float = 1, b: float = 0, *args: Any, **kwargs: Any):
super().__init__(*args, **kwargs)
self.n = n
self.b = b
self.t1 = self.t2 = self.t2_err = 0
self.t1 = self.t2 = self.t2_err = 0.0
async def start(self):
async def start(self) -> AsyncIterator[Any]:
self.t1 = time.time()
assert self.mockserver
url = self.mockserver.url(f"/delay?n={self.n}&b={self.b}")
yield Request(url, callback=self.parse, errback=self.errback)
def parse(self, response):
def parse(self, response: Response) -> Any:
self.t2 = time.time()
def errback(self, failure):
def errback(self, failure: Failure) -> None:
self.t2_err = time.time()
class LogSpider(MetaSpider):
name = "log_spider"
def log_debug(self, message: str, extra: dict[str, Any] | None = None):
def log_debug(self, message: str, extra: dict[str, Any] | None = None) -> None:
self.logger.debug(message, extra=extra)
def log_info(self, message: str, extra: dict[str, Any] | None = None):
def log_info(self, message: str, extra: dict[str, Any] | None = None) -> None:
self.logger.info(message, extra=extra)
def log_warning(self, message: str, extra: dict[str, Any] | None = None):
def log_warning(self, message: str, extra: dict[str, Any] | None = None) -> None:
self.logger.warning(message, extra=extra)
def log_error(self, message: str, extra: dict[str, Any] | None = None):
def log_error(self, message: str, extra: dict[str, Any] | None = None) -> None:
self.logger.error(message, extra=extra)
def log_critical(self, message: str, extra: dict[str, Any] | None = None):
def log_critical(self, message: str, extra: dict[str, Any] | None = None) -> None:
self.logger.critical(message, extra=extra)
def parse(self, response):
def parse(self, response: Response) -> None:
pass
class SlowSpider(DelaySpider):
name = "slow"
async def start(self):
async def start(self) -> AsyncIterator[Any]:
assert self.mockserver
# 1st response is fast
url = self.mockserver.url("/delay?n=0&b=0")
yield Request(url, callback=self.parse, errback=self.errback)
@ -154,25 +173,25 @@ class SlowSpider(DelaySpider):
url = self.mockserver.url(f"/delay?n={self.n}&b={self.b}")
yield Request(url, callback=self.parse, errback=self.errback)
def parse(self, response):
def parse(self, response: Response) -> Iterator[Any]:
yield Item()
class SimpleSpider(MetaSpider):
name = "simple"
def __init__(self, url="http://localhost:8998", *args, **kwargs):
def __init__(self, url: str = "http://localhost:8998", *args: Any, **kwargs: Any):
super().__init__(*args, **kwargs)
self.start_urls = [url]
def parse(self, response):
def parse(self, response: Response) -> Any:
self.logger.info(f"Got response {response.status}")
class AsyncDefSpider(SimpleSpider):
name = "asyncdef"
async def parse(self, response):
async def parse(self, response: Response) -> None:
await defer.succeed(42)
self.logger.info(f"Got response {response.status}")
@ -180,7 +199,7 @@ class AsyncDefSpider(SimpleSpider):
class AsyncDefAsyncioSpider(SimpleSpider):
name = "asyncdef_asyncio"
async def parse(self, response):
async def parse(self, response: Response) -> None:
await asyncio.sleep(0.2)
status = await get_from_asyncio_queue(response.status)
self.logger.info(f"Got response {status}")
@ -189,7 +208,7 @@ class AsyncDefAsyncioSpider(SimpleSpider):
class AsyncDefAsyncioReturnSpider(SimpleSpider):
name = "asyncdef_asyncio_return"
async def parse(self, response):
async def parse(self, response: Response) -> Any:
await asyncio.sleep(0.2)
status = await get_from_asyncio_queue(response.status)
self.logger.info(f"Got response {status}")
@ -199,7 +218,7 @@ class AsyncDefAsyncioReturnSpider(SimpleSpider):
class AsyncDefAsyncioReturnSingleElementSpider(SimpleSpider):
name = "asyncdef_asyncio_return_single_element"
async def parse(self, response):
async def parse(self, response: Response) -> Any:
await asyncio.sleep(0.1)
status = await get_from_asyncio_queue(response.status)
self.logger.info(f"Got response {status}")
@ -209,7 +228,7 @@ class AsyncDefAsyncioReturnSingleElementSpider(SimpleSpider):
class AsyncDefAsyncioReqsReturnSpider(SimpleSpider):
name = "asyncdef_asyncio_reqs_return"
async def parse(self, response):
async def parse(self, response: Response) -> Any:
await asyncio.sleep(0.2)
req_id = response.meta.get("req_id", 0)
status = await get_from_asyncio_queue(response.status)
@ -226,7 +245,7 @@ class AsyncDefAsyncioReqsReturnSpider(SimpleSpider):
class AsyncDefAsyncioGenExcSpider(SimpleSpider):
name = "asyncdef_asyncio_gen_exc"
async def parse(self, response):
async def parse(self, response: Response) -> AsyncIterator[Any]:
for i in range(10):
await asyncio.sleep(0.1)
yield {"foo": i}
@ -237,7 +256,7 @@ class AsyncDefAsyncioGenExcSpider(SimpleSpider):
class AsyncDefDeferredDirectSpider(SimpleSpider):
name = "asyncdef_deferred_direct"
async def parse(self, response):
async def parse(self, response: Response) -> AsyncIterator[Any]:
await defer.succeed(None)
yield {"code": 200}
@ -245,7 +264,7 @@ class AsyncDefDeferredDirectSpider(SimpleSpider):
class AsyncDefDeferredWrappedSpider(SimpleSpider):
name = "asyncdef_deferred_wrapped"
async def parse(self, response):
async def parse(self, response: Response) -> AsyncIterator[Any]:
await deferred_to_future(defer.succeed(None))
yield {"code": 200}
@ -253,7 +272,7 @@ class AsyncDefDeferredWrappedSpider(SimpleSpider):
class AsyncDefDeferredMaybeWrappedSpider(SimpleSpider):
name = "asyncdef_deferred_maybe_wrapped"
async def parse(self, response):
async def parse(self, response: Response) -> AsyncIterator[Any]:
await maybe_deferred_to_future(defer.succeed(None))
yield {"code": 200}
@ -261,7 +280,7 @@ class AsyncDefDeferredMaybeWrappedSpider(SimpleSpider):
class AsyncDefAsyncioGenSpider(SimpleSpider):
name = "asyncdef_asyncio_gen"
async def parse(self, response):
async def parse(self, response: Response) -> AsyncIterator[Any]:
await asyncio.sleep(0.2)
yield {"foo": 42}
self.logger.info(f"Got response {response.status}")
@ -270,7 +289,7 @@ class AsyncDefAsyncioGenSpider(SimpleSpider):
class AsyncDefAsyncioGenLoopSpider(SimpleSpider):
name = "asyncdef_asyncio_gen_loop"
async def parse(self, response):
async def parse(self, response: Response) -> AsyncIterator[Any]:
for i in range(10):
await asyncio.sleep(0.1)
yield {"foo": i}
@ -283,7 +302,8 @@ class AsyncDefAsyncioGenComplexSpider(SimpleSpider):
following_reqs = 3
depth = 2
def _get_req(self, index, cb=None):
def _get_req(self, index: int, cb: CallbackT | None = None) -> Request:
assert self.mockserver
return Request(
self.mockserver.url(f"/status?n=200&request={index}"),
meta={"index": index},
@ -291,11 +311,11 @@ class AsyncDefAsyncioGenComplexSpider(SimpleSpider):
callback=cb,
)
async def start(self):
async def start(self) -> AsyncIterator[Any]:
for i in range(1, self.initial_reqs + 1):
yield self._get_req(i)
async def parse(self, response):
async def parse(self, response: Response) -> AsyncIterator[Any]:
index = response.meta["index"]
yield {"index": index}
if index < 10**self.depth:
@ -305,7 +325,7 @@ class AsyncDefAsyncioGenComplexSpider(SimpleSpider):
await asyncio.sleep(0.1)
yield {"index": index + 5}
async def parse2(self, response):
async def parse2(self, response: Response) -> AsyncIterator[Any]:
await asyncio.sleep(0.1)
yield {"index2": response.meta["index"]}
@ -313,7 +333,7 @@ class AsyncDefAsyncioGenComplexSpider(SimpleSpider):
class ItemSpider(FollowAllSpider):
name = "item"
def parse(self, response):
def parse(self, response: Response) -> Iterator[Any]:
for request in super().parse(response):
yield request
yield Item()
@ -321,12 +341,18 @@ class ItemSpider(FollowAllSpider):
class MaxItemsAndRequestsSpider(FollowAllSpider):
def __init__(self, max_items=10, max_requests=10, *args, **kwargs):
def __init__(
self,
max_items: int = 10,
max_requests: int = 10,
*args: Any,
**kwargs: Any,
):
super().__init__(*args, **kwargs)
self.max_items = max_items
self.max_requests = max_requests
def parse(self, response):
def parse(self, response: Response) -> Iterator[Any]:
self.items_scraped = 0
self.pages_crawled = 1 # account for the start url
for request in super().parse(response):
@ -344,12 +370,12 @@ class DefaultError(Exception):
class ErrorSpider(FollowAllSpider):
name = "error"
exception_cls = DefaultError
exception_cls: type[Exception] = DefaultError
def raise_exception(self):
def raise_exception(self) -> None:
raise self.exception_cls("Expected exception")
def parse(self, response):
def parse(self, response: Response) -> Iterator[Any]:
for request in super().parse(response):
yield request
self.raise_exception()
@ -359,14 +385,15 @@ class BrokenStartSpider(FollowAllSpider):
fail_before_yield = False
fail_yielding = False
def __init__(self, *a, **kw):
def __init__(self, *a: Any, **kw: Any):
super().__init__(*a, **kw)
self.seedsseen = []
self.seedsseen: list[Any] = []
async def start(self):
async def start(self) -> AsyncIterator[Any]:
if self.fail_before_yield:
1 / 0
assert self.mockserver
for s in range(100):
qargs = {"total": 10, "seed": s}
url = self.mockserver.url(f"/follow?{urlencode(qargs, doseq=True)}")
@ -376,18 +403,18 @@ class BrokenStartSpider(FollowAllSpider):
assert self.seedsseen, "All seeds consumed before any download happened"
def parse(self, response):
def parse(self, response: Response) -> Iterator[Any]:
self.seedsseen.append(response.meta.get("seed"))
yield from super().parse(response)
class StartItemSpider(FollowAllSpider):
async def start(self):
async def start(self) -> AsyncIterator[Any]:
yield {"name": "test item"}
class StartGoodAndBadOutput(FollowAllSpider):
async def start(self):
async def start(self) -> AsyncIterator[Any]:
yield {"a": "a"}
yield Request("data:,a")
yield "data:,b"
@ -395,17 +422,18 @@ class StartGoodAndBadOutput(FollowAllSpider):
class SingleRequestSpider(MetaSpider):
seed = None
callback_func = None
errback_func = None
seed: Request | str | None = None
callback_func: Callable[[Response], Any] | None = None
errback_func: Callable[[Failure], Any] | None = None
async def start(self):
async def start(self) -> AsyncIterator[Any]:
if isinstance(self.seed, Request):
yield self.seed.replace(callback=self.parse, errback=self.on_error)
else:
assert self.seed
yield Request(self.seed, callback=self.parse, errback=self.on_error)
def parse(self, response):
def parse(self, response: Response) -> Any:
self.meta.setdefault("responses", []).append(response)
if callable(self.callback_func):
return self.callback_func(response)
@ -413,7 +441,7 @@ class SingleRequestSpider(MetaSpider):
return response.meta["next"]
return None
def on_error(self, failure):
def on_error(self, failure: Failure) -> Any:
self.meta["failure"] = failure
if callable(self.errback_func):
return self.errback_func(failure)
@ -426,17 +454,18 @@ class DuplicateStartSpider(MockServerSpider):
distinct_urls = 2
dupe_factor = 3
async def start(self):
async def start(self) -> AsyncIterator[Any]:
assert self.mockserver
for i in range(self.distinct_urls):
for _ in range(self.dupe_factor):
url = self.mockserver.url(f"/echo?headers=1&body=test{i}")
yield Request(url, dont_filter=self.dont_filter)
def __init__(self, url="http://localhost:8998", *args, **kwargs):
def __init__(self, url: str = "http://localhost:8998", *args: Any, **kwargs: Any):
super().__init__(*args, **kwargs)
self.visited = 0
def parse(self, response):
def parse(self, response: Response) -> None:
self.visited += 1
@ -451,7 +480,7 @@ class CrawlSpiderWithParseMethod(MockServerSpider, CrawlSpider):
}
rules = (Rule(LinkExtractor(), callback="parse", follow=True),)
async def start(self):
async def start(self) -> AsyncIterator[Any]:
test_body = b"""
<html>
<head><title>Page title</title></head>
@ -461,11 +490,13 @@ class CrawlSpiderWithParseMethod(MockServerSpider, CrawlSpider):
</body>
</html>
"""
assert self.mockserver
url = self.mockserver.url("/alpayload")
yield Request(url, method="POST", body=test_body)
def parse(self, response, foo=None):
def parse(self, response: Response, foo: str | None = None) -> Iterator[Any]:
self.logger.info("[parse] status %i (foo: %s)", response.status, foo)
assert self.mockserver
yield Request(
self.mockserver.url("/status?n=202"), self.parse, cb_kwargs={"foo": "bar"}
)
@ -477,8 +508,9 @@ class CrawlSpiderWithAsyncCallback(CrawlSpiderWithParseMethod):
name = "crawl_spider_with_async_callback"
rules = (Rule(LinkExtractor(), callback="parse_async", follow=True),)
async def parse_async(self, response, foo=None):
async def parse_async(self, response: Response, foo: str | None = None) -> Request:
self.logger.info("[parse_async] status %i (foo: %s)", response.status, foo)
assert self.mockserver
return Request(
self.mockserver.url("/status?n=202"),
self.parse_async,
@ -492,8 +524,11 @@ class CrawlSpiderWithAsyncGeneratorCallback(CrawlSpiderWithParseMethod):
name = "crawl_spider_with_async_generator_callback"
rules = (Rule(LinkExtractor(), callback="parse_async_gen", follow=True),)
async def parse_async_gen(self, response, foo=None):
async def parse_async_gen(
self, response: Response, foo: str | None = None
) -> AsyncIterator[Any]:
self.logger.info("[parse_async_gen] status %i (foo: %s)", response.status, foo)
assert self.mockserver
yield Request(
self.mockserver.url("/status?n=202"),
self.parse_async_gen,
@ -505,7 +540,7 @@ class CrawlSpiderWithErrback(CrawlSpiderWithParseMethod):
name = "crawl_spider_with_errback"
rules = (Rule(LinkExtractor(), callback="parse", errback="errback", follow=True),)
async def start(self):
async def start(self) -> AsyncIterator[Any]:
test_body = b"""
<html>
<head><title>Page title</title></head>
@ -518,17 +553,19 @@ class CrawlSpiderWithErrback(CrawlSpiderWithParseMethod):
</body>
</html>
"""
assert self.mockserver
url = self.mockserver.url("/alpayload")
yield Request(url, method="POST", body=test_body)
def errback(self, failure):
def errback(self, failure: Failure) -> None:
assert isinstance(failure.value, HttpError)
self.logger.info("[errback] status %i", failure.value.response.status)
class CrawlSpiderWithoutErrback(CrawlSpiderWithParseMethod):
name = "crawl_spider_without_errback"
async def start(self):
async def start(self) -> AsyncIterator[Any]:
test_body = b"""
<html>
<head><title>Page title</title></head>
@ -538,6 +575,7 @@ class CrawlSpiderWithoutErrback(CrawlSpiderWithParseMethod):
</body>
</html>
"""
assert self.mockserver
url = self.mockserver.url("/alpayload")
yield Request(url, method="POST", body=test_body)
@ -553,7 +591,7 @@ class CrawlSpiderWithProcessRequestCallbackKeywordArguments(CrawlSpiderWithParse
),
)
def process_request(self, request, response):
def process_request(self, request: Request, response: Response) -> Request:
request.cb_kwargs["foo"] = "process_request"
return request
@ -562,59 +600,65 @@ class BytesReceivedCallbackSpider(MetaSpider):
full_response_length = 2**18
@classmethod
def from_crawler(cls, crawler, *args, **kwargs):
def from_crawler(cls, crawler: Crawler, *args: Any, **kwargs: Any) -> Self:
spider = super().from_crawler(crawler, *args, **kwargs)
crawler.signals.connect(spider.bytes_received, signals.bytes_received)
return spider
async def start(self):
async def start(self) -> AsyncIterator[Any]:
body = b"a" * self.full_response_length
assert self.mockserver
url = self.mockserver.url("/alpayload", is_secure=self.is_secure)
yield Request(url, method="POST", body=body, errback=self.errback)
def parse(self, response):
def parse(self, response: Response) -> None:
self.meta["response"] = response
def errback(self, failure):
def errback(self, failure: Failure) -> None:
self.meta["failure"] = failure
def bytes_received(self, data, request, spider):
def bytes_received(self, data: bytes, request: Request, spider: Spider) -> None:
self.meta["bytes_received"] = data
raise StopDownload(fail=False)
class BytesReceivedErrbackSpider(BytesReceivedCallbackSpider):
def bytes_received(self, data, request, spider):
def bytes_received(self, data: bytes, request: Request, spider: Spider) -> None:
self.meta["bytes_received"] = data
raise StopDownload(fail=True)
class HeadersReceivedCallbackSpider(MetaSpider):
@classmethod
def from_crawler(cls, crawler, *args, **kwargs):
def from_crawler(cls, crawler: Crawler, *args: Any, **kwargs: Any) -> Self:
spider = super().from_crawler(crawler, *args, **kwargs)
crawler.signals.connect(spider.headers_received, signals.headers_received)
return spider
async def start(self):
async def start(self) -> AsyncIterator[Any]:
assert self.mockserver
yield Request(
self.mockserver.url("/status", is_secure=self.is_secure),
errback=self.errback,
)
def parse(self, response):
def parse(self, response: Response) -> None:
self.meta["response"] = response
def errback(self, failure):
def errback(self, failure: Failure) -> None:
self.meta["failure"] = failure
def headers_received(self, headers, body_length, request, spider):
def headers_received(
self, headers: Headers, body_length: int, request: Request, spider: Spider
) -> None:
self.meta["headers_received"] = headers
raise StopDownload(fail=False)
class HeadersReceivedErrbackSpider(HeadersReceivedCallbackSpider):
def headers_received(self, headers, body_length, request, spider):
def headers_received(
self, headers: Headers, body_length: int, request: Request, spider: Spider
) -> None:
self.meta["headers_received"] = headers
raise StopDownload(fail=True)
@ -623,13 +667,13 @@ class ExceptionSpider(Spider):
name = "exception"
@classmethod
def from_crawler(cls, crawler, *args, **kwargs):
def from_crawler(cls, crawler: Crawler, *args: Any, **kwargs: Any) -> Self:
raise ValueError("Exception in from_crawler method")
class NoRequestsSpider(Spider):
name = "no_request"
async def start(self):
async def start(self) -> AsyncIterator[Any]:
return
yield

9
tests/test_closespider.py
View File

@ -11,6 +11,8 @@ from tests.utils.decorators import inline_callbacks_test
class TestCloseSpider:
mockserver: MockServer
@classmethod
def setup_class(cls):
cls.mockserver = MockServer()
@ -25,6 +27,7 @@ class TestCloseSpider:
close_on = 5
crawler = get_crawler(ItemSpider, {"CLOSESPIDER_ITEMCOUNT": close_on})
yield crawler.crawl(mockserver=self.mockserver)
assert isinstance(crawler.spider, ItemSpider)
reason = crawler.spider.meta["close_reason"]
assert reason == "closespider_itemcount"
itemcount = crawler.stats.get_value("item_scraped_count")
@ -35,6 +38,7 @@ class TestCloseSpider:
close_on = 5
crawler = get_crawler(FollowAllSpider, {"CLOSESPIDER_PAGECOUNT": close_on})
yield crawler.crawl(mockserver=self.mockserver)
assert isinstance(crawler.spider, FollowAllSpider)
reason = crawler.spider.meta["close_reason"]
assert reason == "closespider_pagecount"
pagecount = crawler.stats.get_value("response_received_count")
@ -54,6 +58,7 @@ class TestCloseSpider:
yield crawler.crawl(
max_items=max_items, max_requests=max_requests, mockserver=self.mockserver
)
assert isinstance(crawler.spider, MaxItemsAndRequestsSpider)
reason = crawler.spider.meta["close_reason"]
assert reason == "closespider_pagecount_no_item"
pagecount = crawler.stats.get_value("response_received_count")
@ -72,6 +77,7 @@ class TestCloseSpider:
},
)
yield crawler.crawl(mockserver=self.mockserver)
assert isinstance(crawler.spider, FollowAllSpider)
reason = crawler.spider.meta["close_reason"]
assert reason == "closespider_pagecount_no_item"
pagecount = crawler.stats.get_value("response_received_count")
@ -82,6 +88,7 @@ class TestCloseSpider:
close_on = 5
crawler = get_crawler(ErrorSpider, {"CLOSESPIDER_ERRORCOUNT": close_on})
yield crawler.crawl(total=1000000, mockserver=self.mockserver)
assert isinstance(crawler.spider, ErrorSpider)
reason = crawler.spider.meta["close_reason"]
assert reason == "closespider_errorcount"
key = f"spider_exceptions/{crawler.spider.exception_cls.__name__}"
@ -94,6 +101,7 @@ class TestCloseSpider:
close_on = 0.1
crawler = get_crawler(FollowAllSpider, {"CLOSESPIDER_TIMEOUT": close_on})
yield crawler.crawl(total=1000000, mockserver=self.mockserver)
assert isinstance(crawler.spider, FollowAllSpider)
reason = crawler.spider.meta["close_reason"]
assert reason == "closespider_timeout"
total_seconds = crawler.stats.get_value("elapsed_time_seconds")
@ -104,6 +112,7 @@ class TestCloseSpider:
timeout = 1
crawler = get_crawler(SlowSpider, {"CLOSESPIDER_TIMEOUT_NO_ITEM": timeout})
yield crawler.crawl(n=3, mockserver=self.mockserver)
assert isinstance(crawler.spider, SlowSpider)
reason = crawler.spider.meta["close_reason"]
assert reason == "closespider_timeout_no_item"
total_seconds = crawler.stats.get_value("elapsed_time_seconds")

9
tests/test_command_runspider.py
View File

@ -16,6 +16,11 @@ if TYPE_CHECKING:
from pathlib import Path
# Header for spider classes copied with inspect.getsource(): the future import
# keeps their annotations from being evaluated in the generated module.
SPIDER_HEADER = "from __future__ import annotations\nfrom scrapy import Spider\n"
class TestRunSpiderCommand:
spider_filename = "myspider.py"
@ -65,14 +70,14 @@ class BadSpider(scrapy.Spider):
def test_run_fail_spider(self, tmp_path: Path) -> None:
ret, _, _ = self.runspider(
tmp_path, "from scrapy import Spider\n" + inspect.getsource(ExceptionSpider)
tmp_path, SPIDER_HEADER + inspect.getsource(ExceptionSpider)
)
assert ret != 0
def test_run_good_spider(self, tmp_path: Path) -> None:
ret, _, _ = self.runspider(
tmp_path,
"from scrapy import Spider\n" + inspect.getsource(NoRequestsSpider),
SPIDER_HEADER + inspect.getsource(NoRequestsSpider),
)
assert ret == 0

201
tests/test_contracts.py
View File

@ -1,3 +1,6 @@
from __future__ import annotations
from typing import TYPE_CHECKING, Any
from unittest import TextTestResult
import pytest
@ -11,7 +14,7 @@ from scrapy.contracts.default import (
ScrapesContract,
UrlContract,
)
from scrapy.http import Request
from scrapy.http import Request, Response
from scrapy.item import Field, Item
from scrapy.spidermiddlewares.httperror import HttpError
from scrapy.spiders import Spider
@ -19,6 +22,24 @@ from scrapy.utils.test import get_crawler
from tests.mockserver.http import MockServer
from tests.utils.decorators import inline_callbacks_test
if TYPE_CHECKING:
from unittest import TestResult
from scrapy.http.request import CallbackT
def _request(
conman: ContractsManager, method: CallbackT, results: TestResult
) -> Request:
request = conman.from_method(method, results)
assert request
return request
def _call(request: Request, response: Any) -> Any:
assert request.callback
return request.callback(response, **request.cb_kwargs)
class DemoItem(Item):
name = Field()
@ -30,7 +51,7 @@ class ResponseMock:
class ResponseMetaMock(ResponseMock):
meta = None
meta: Any = None
class TaggedRequest(Request):
@ -289,17 +310,19 @@ class TestContractsManager:
def setup_method(self):
self.conman = ContractsManager(self.contracts)
self.results = TextTestResult(stream=None, descriptions=False, verbosity=0)
self.results = TextTestResult( # type: ignore[type-var]
stream=None, descriptions=False, verbosity=0
)
def should_succeed(self):
def should_succeed(self) -> None:
assert not self.results.failures
assert not self.results.errors
def should_fail(self):
def should_fail(self) -> None:
assert self.results.failures
assert not self.results.errors
def should_error(self):
def should_error(self) -> None:
assert self.results.errors
def test_contracts(self):
@ -313,12 +336,10 @@ class TestContractsManager:
)
# returns request for valid method
request = self.conman.from_method(spider.returns_request, self.results)
assert request is not None
assert self.conman.from_method(spider.returns_request, self.results) is not None
# no request for missing url
request = self.conman.from_method(spider.parse_no_url, self.results)
assert request is None
assert self.conman.from_method(spider.parse_no_url, self.results) is None
def test_cb_kwargs(self):
spider = DemoSpider()
@ -354,29 +375,31 @@ class TestContractsManager:
)
# returns_request
request = self.conman.from_method(
spider.returns_request_cb_kwargs, self.results
)
request.callback(response, **request.cb_kwargs)
request = _request(self.conman, spider.returns_request_cb_kwargs, self.results)
_call(request, response)
self.should_succeed()
# returns_item
request = self.conman.from_method(spider.returns_item_cb_kwargs, self.results)
request.callback(response, **request.cb_kwargs)
request = _request(self.conman, spider.returns_item_cb_kwargs, self.results)
_call(request, response)
self.should_succeed()
# returns_item (error, callback doesn't take keyword arguments)
request = self.conman.from_method(
spider.returns_item_cb_kwargs_error_unexpected_keyword, self.results
request = _request(
self.conman,
spider.returns_item_cb_kwargs_error_unexpected_keyword,
self.results,
)
request.callback(response, **request.cb_kwargs)
_call(request, response)
self.should_error()
# returns_item (error, contract doesn't provide keyword arguments)
request = self.conman.from_method(
spider.returns_item_cb_kwargs_error_missing_argument, self.results
request = _request(
self.conman,
spider.returns_item_cb_kwargs_error_missing_argument,
self.results,
)
request.callback(response, **request.cb_kwargs)
_call(request, response)
self.should_error()
def test_meta(self):
@ -398,29 +421,27 @@ class TestContractsManager:
response = ResponseMetaMock()
# returns_request
request = self.conman.from_method(spider.returns_request_meta, self.results)
request = _request(self.conman, spider.returns_request_meta, self.results)
assert request.meta["cookiejar"] == "session1"
response.meta = request.meta
request.callback(response)
_call(request, response)
assert response.meta["cookiejar"] == "session1"
self.should_succeed()
response = ResponseMetaMock()
# returns_item
request = self.conman.from_method(spider.returns_item_meta, self.results)
request = _request(self.conman, spider.returns_item_meta, self.results)
assert request.meta["key"] == "example"
response.meta = request.meta
request.callback(response)
_call(request, response)
assert response.meta["key"] == "example"
self.should_succeed()
response = ResponseMetaMock()
request = self.conman.from_method(
spider.returns_error_missing_meta, self.results
)
request.callback(response)
request = _request(self.conman, spider.returns_error_missing_meta, self.results)
_call(request, response)
self.should_error()
def test_returns(self):
@ -428,36 +449,36 @@ class TestContractsManager:
response = ResponseMock()
# returns_item
request = self.conman.from_method(spider.returns_item, self.results)
request.callback(response)
request = _request(self.conman, spider.returns_item, self.results)
_call(request, response)
self.should_succeed()
# returns_dict_item
request = self.conman.from_method(spider.returns_dict_item, self.results)
request.callback(response)
request = _request(self.conman, spider.returns_dict_item, self.results)
_call(request, response)
self.should_succeed()
# returns_request
request = self.conman.from_method(spider.returns_request, self.results)
request.callback(response)
request = _request(self.conman, spider.returns_request, self.results)
_call(request, response)
self.should_succeed()
# returns_fail
request = self.conman.from_method(spider.returns_fail, self.results)
request.callback(response)
request = _request(self.conman, spider.returns_fail, self.results)
_call(request, response)
self.should_fail()
# returns_dict_fail
request = self.conman.from_method(spider.returns_dict_fail, self.results)
request.callback(response)
request = _request(self.conman, spider.returns_dict_fail, self.results)
_call(request, response)
self.should_fail()
def test_returns_async(self):
spider = DemoSpider()
response = ResponseMock()
request = self.conman.from_method(spider.returns_request_async, self.results)
request.callback(response)
request = _request(self.conman, spider.returns_request_async, self.results)
_call(request, response)
self.should_error()
def test_returns_invalid_argument_count(self):
@ -477,10 +498,8 @@ class TestContractsManager:
spider = DemoSpider()
response = ResponseMock()
request = self.conman.from_method(
spider.returns_request_range_fail, self.results
)
request.callback(response)
request = _request(self.conman, spider.returns_request_range_fail, self.results)
_call(request, response)
self.should_fail()
assert "expected 2..3" in self.results.failures[-1][-1]
@ -490,8 +509,8 @@ class TestContractsManager:
# @returns and @scrapes only count matching output objects and skip
# the request that is also yielded.
request = self.conman.from_method(spider.yields_item_and_request, self.results)
request.callback(response)
request = _request(self.conman, spider.yields_item_and_request, self.results)
_call(request, response)
self.should_succeed()
def test_testcase_str(self):
@ -507,30 +526,30 @@ class TestContractsManager:
response = ResponseMock()
# scrapes_item_ok
request = self.conman.from_method(spider.scrapes_item_ok, self.results)
request.callback(response)
request = _request(self.conman, spider.scrapes_item_ok, self.results)
_call(request, response)
self.should_succeed()
# scrapes_dict_item_ok
request = self.conman.from_method(spider.scrapes_dict_item_ok, self.results)
request.callback(response)
request = _request(self.conman, spider.scrapes_dict_item_ok, self.results)
_call(request, response)
self.should_succeed()
# scrapes_item_fail
request = self.conman.from_method(spider.scrapes_item_fail, self.results)
request.callback(response)
request = _request(self.conman, spider.scrapes_item_fail, self.results)
_call(request, response)
self.should_fail()
# scrapes_dict_item_fail
request = self.conman.from_method(spider.scrapes_dict_item_fail, self.results)
request.callback(response)
request = _request(self.conman, spider.scrapes_dict_item_fail, self.results)
_call(request, response)
self.should_fail()
# scrapes_multiple_missing_fields
request = self.conman.from_method(
spider.scrapes_multiple_missing_fields, self.results
request = _request(
self.conman, spider.scrapes_multiple_missing_fields, self.results
)
request.callback(response)
_call(request, response)
self.should_fail()
message = "ContractFail: Missing fields: name, url"
assert message in self.results.failures[-1][-1]
@ -540,14 +559,13 @@ class TestContractsManager:
response = ResponseMock()
# invalid regex
request = self.conman.from_method(spider.invalid_regex, self.results)
assert request is None
assert self.conman.from_method(spider.invalid_regex, self.results) is None
# invalid regex with valid contract
request = self.conman.from_method(
spider.invalid_regex_with_valid_contract, self.results
request = _request(
self.conman, spider.invalid_regex_with_valid_contract, self.results
)
request.callback(response)
_call(request, response)
self.should_succeed()
def test_custom_contracts(self):
@ -559,14 +577,14 @@ class TestContractsManager:
def test_errback(self):
spider = DemoSpider()
response = ResponseMock()
try:
raise HttpError(response, "Ignoring non-200 response")
raise HttpError(Response("http://scrapy.org"), "Ignoring non-200 response")
except HttpError:
failure_mock = failure.Failure()
request = self.conman.from_method(spider.returns_request, self.results)
request = _request(self.conman, spider.returns_request, self.results)
assert request.errback
request.errback(failure_mock)
assert not self.results.failures
@ -602,11 +620,12 @@ class TestContractsManager:
crawler = get_crawler(TestSameUrlSpider)
yield crawler.crawl()
assert isinstance(crawler.spider, TestSameUrlSpider)
assert crawler.spider.visited == 2
def test_custom_tagged_request_contract(self):
spider = DemoSpider()
request = self.conman.from_method(spider.custom_tagged_request, self.results)
request = _request(self.conman, spider.custom_tagged_request, self.results)
assert request.method == "POST"
assert isinstance(request, TaggedRequest)
assert request.contract_tag == "custom"
@ -672,18 +691,20 @@ class PostProcessErrorContract(Contract):
class TestCustomContractPrePostProcess:
def setup_method(self):
self.results = TextTestResult(stream=None, descriptions=False, verbosity=0)
self.results = TextTestResult( # type: ignore[type-var]
stream=None, descriptions=False, verbosity=0
)
def test_pre_hook_keyboard_interrupt(self):
spider = DemoSpider()
response = ResponseMock()
contract = CustomFailContractPreProcess(spider.returns_request)
conman = ContractsManager([UrlContract, ReturnsContract, contract])
conman = ContractsManager([UrlContract, ReturnsContract])
request = conman.from_method(spider.returns_request, self.results)
request = _request(conman, spider.returns_request, self.results)
contract.add_pre_hook(request, self.results)
with pytest.raises(KeyboardInterrupt, match="Pre-process exception"):
request.callback(response, **request.cb_kwargs)
_call(request, response)
assert not self.results.failures
assert not self.results.errors
@ -692,12 +713,12 @@ class TestCustomContractPrePostProcess:
spider = DemoSpider()
response = ResponseMock()
contract = CustomFailContractPostProcess(spider.returns_request)
conman = ContractsManager([UrlContract, ReturnsContract, contract])
conman = ContractsManager([UrlContract, ReturnsContract])
request = conman.from_method(spider.returns_request, self.results)
request = _request(conman, spider.returns_request, self.results)
contract.add_post_hook(request, self.results)
with pytest.raises(KeyboardInterrupt, match="Post-process exception"):
request.callback(response, **request.cb_kwargs)
_call(request, response)
assert not self.results.failures
assert not self.results.errors
@ -706,11 +727,11 @@ class TestCustomContractPrePostProcess:
spider = DemoSpider()
response = ResponseMock()
contract = PreProcessSuccessContract(spider.returns_request)
conman = ContractsManager([UrlContract, ReturnsContract, contract])
conman = ContractsManager([UrlContract, ReturnsContract])
request = conman.from_method(spider.returns_request, self.results)
request = _request(conman, spider.returns_request, self.results)
contract.add_pre_hook(request, self.results)
request.callback(response, **request.cb_kwargs)
_call(request, response)
assert not self.results.failures
assert not self.results.errors
@ -719,11 +740,11 @@ class TestCustomContractPrePostProcess:
spider = DemoSpider()
response = ResponseMock()
contract = PreProcessAssertionFailContract(spider.returns_request)
conman = ContractsManager([UrlContract, ReturnsContract, contract])
conman = ContractsManager([UrlContract, ReturnsContract])
request = conman.from_method(spider.returns_request, self.results)
request = _request(conman, spider.returns_request, self.results)
contract.add_pre_hook(request, self.results)
request.callback(response, **request.cb_kwargs)
_call(request, response)
assert self.results.failures
assert not self.results.errors
@ -732,11 +753,11 @@ class TestCustomContractPrePostProcess:
spider = DemoSpider()
response = ResponseMock()
contract = PreProcessErrorContract(spider.returns_request)
conman = ContractsManager([UrlContract, ReturnsContract, contract])
conman = ContractsManager([UrlContract, ReturnsContract])
request = conman.from_method(spider.returns_request, self.results)
request = _request(conman, spider.returns_request, self.results)
contract.add_pre_hook(request, self.results)
request.callback(response, **request.cb_kwargs)
_call(request, response)
assert self.results.errors
@ -748,7 +769,7 @@ class TestCustomContractPrePostProcess:
contract.add_pre_hook(request, self.results)
with pytest.raises(TypeError, match="async callbacks"):
request.callback(response)
_call(request, response)
def test_pre_hook_async_generator(self):
spider = DemoSpider()
@ -758,7 +779,7 @@ class TestCustomContractPrePostProcess:
contract.add_pre_hook(request, self.results)
with pytest.raises(TypeError, match="async callbacks"):
request.callback(response)
_call(request, response)
def test_post_hook_async_generator(self):
spider = DemoSpider()
@ -768,16 +789,16 @@ class TestCustomContractPrePostProcess:
contract.add_post_hook(request, self.results)
with pytest.raises(TypeError, match="async callbacks"):
request.callback(response)
_call(request, response)
def test_post_hook_error(self):
spider = DemoSpider()
response = ResponseMock()
contract = PostProcessErrorContract(spider.returns_request)
conman = ContractsManager([UrlContract, ReturnsContract, contract])
conman = ContractsManager([UrlContract, ReturnsContract])
request = conman.from_method(spider.returns_request, self.results)
request = _request(conman, spider.returns_request, self.results)
contract.add_post_hook(request, self.results)
request.callback(response, **request.cb_kwargs)
_call(request, response)
assert self.results.errors

4
tests/test_crawl.py
View File

@ -359,7 +359,6 @@ with multiples lines
est: list[list[tuple[str, Any]]] = []
def cb(response):
assert crawler.engine
est.append(get_engine_status(crawler.engine))
crawler = get_crawler(SingleRequestSpider)
@ -377,7 +376,6 @@ with multiples lines
est: list[str] = []
def cb(response):
assert crawler.engine
est.append(format_engine_status(crawler.engine))
crawler = get_crawler(SingleRequestSpider)
@ -452,7 +450,6 @@ class TestCrawlSpider:
await crawler.crawl_async(
mockserver.url("/status?n=200"), mockserver=mockserver
)
assert crawler.stats
return items, crawler.stats
@coroutine_test
@ -518,7 +515,6 @@ class TestCrawlSpider:
# so the failure is dropped silently and the crawl finishes normally.
assert "[parse] status 200 (foo: None)" in caplog.text
assert "[errback]" not in caplog.text
assert crawler.stats
assert crawler.stats.get_value("downloader/response_status_count/404") == 1
@coroutine_test

3
tests/test_crawler.py
View File

@ -536,7 +536,6 @@ class TestSpiderSettings:
custom_settings = {"AUTOTHROTTLE_ENABLED": True}
crawler = get_crawler(MySpider)
assert crawler.extensions
enabled_exts = [e.__class__ for e in crawler.extensions.middlewares]
assert AutoThrottle in enabled_exts
@ -568,7 +567,6 @@ class TestCrawlerLogging:
}
async def start(self):
assert crawler.stats
info_count_start = crawler.stats.get_value("log_count/INFO")
logging.debug("debug message") # noqa: LOG015
logging.info("info message") # noqa: LOG015
@ -601,7 +599,6 @@ class TestCrawlerLogging:
assert "info message" in logged
assert "warning message" in logged
assert "error message" in logged
assert crawler.stats
assert crawler.stats.get_value("log_count/ERROR") == 1
assert crawler.stats.get_value("log_count/WARNING") == 1
assert info_count == 1

3
tests/test_downloadermiddleware_httpcache.py
View File

@ -72,7 +72,6 @@ class TestBase:
settings = self._get_settings(**new_settings)
crawler = get_crawler(Spider, settings)
crawler.spider = crawler._create_spider("example.com")
assert crawler.stats
crawler.stats.open_spider()
try:
yield crawler
@ -137,7 +136,6 @@ class StorageTestMixin(TestBase):
with self._middleware() as mw:
spider = mw.crawler.spider
assert spider
assert mw.crawler.stats
mw.storage.store_response(spider, self.request, self.response)
self._corrupt_cache_entry(mw.storage, spider, self.request)
@ -159,7 +157,6 @@ class StorageTestMixin(TestBase):
with self._middleware(HTTPCACHE_IGNORE_MISSING=True) as mw:
spider = mw.crawler.spider
assert spider
assert mw.crawler.stats
mw.storage.store_response(spider, self.request, self.response)
self._corrupt_cache_entry(mw.storage, spider, self.request)

2
tests/test_downloadermiddleware_httpcompression.py
View File

@ -61,7 +61,6 @@ class TestHttpCompression:
def setup_method(self):
self.crawler = get_crawler(Spider)
self.mw = build_from_crawler(HttpCompressionMiddleware, self.crawler)
assert self.crawler.stats
self.crawler.stats.open_spider()
def _getresponse(self, coding: str) -> Response:
@ -87,7 +86,6 @@ class TestHttpCompression:
return response
def assertStatsEqual(self, key: str, value: Any) -> None:
assert self.crawler.stats
assert self.crawler.stats.get_value(key) == value, str(
self.crawler.stats.get_stats()
)

1
tests/test_downloadermiddleware_offsite.py
View File

@ -230,7 +230,6 @@ def test_repeated_offsite_domain():
with pytest.raises(IgnoreRequest):
mw.process_request(req1)
assert "other.org" in mw.domains_seen
assert crawler.stats
assert crawler.stats.get_value("offsite/domains") == 1
assert crawler.stats.get_value("offsite/filtered") == 1
with pytest.raises(IgnoreRequest):

12
tests/test_downloadermiddleware_retry.py
View File

@ -82,7 +82,6 @@ class TestRetry:
# discard it
assert self.mw.process_response(req3, rsp) is rsp
assert self.crawler.stats
assert self.crawler.stats.get_value("retry/max_reached") == 1
assert (
self.crawler.stats.get_value("retry/reason_count/503 Service Unavailable")
@ -134,7 +133,6 @@ class TestRetry:
self._test_retry_exception(req, exc("foo"))
stats = self.crawler.stats
assert stats
assert stats.get_value("retry/max_reached") == len(exceptions)
assert stats.get_value("retry/count") == len(exceptions) * 2
assert (
@ -315,7 +313,6 @@ class TestGetRetryRequest:
assert new_request.meta["retry_times"] == expected_retry_times
assert new_request.priority == -1
expected_reason = "unspecified"
assert spider.crawler.stats
for stat in ("retry/count", f"retry/reason_count/{expected_reason}"):
assert spider.crawler.stats.get_value(stat) == 1
assert (
@ -336,7 +333,6 @@ class TestGetRetryRequest:
max_retry_times=max_retry_times,
)
assert new_request is None
assert spider.crawler.stats
assert spider.crawler.stats.get_value("retry/max_reached") == 1
failure_count = max_retry_times + 1
expected_reason = "unspecified"
@ -363,7 +359,6 @@ class TestGetRetryRequest:
assert new_request.meta["retry_times"] == expected_retry_times
assert new_request.priority == -1
expected_reason = "unspecified"
assert spider.crawler.stats
for stat in ("retry/count", f"retry/reason_count/{expected_reason}"):
assert spider.crawler.stats.get_value(stat) == 1
assert (
@ -396,7 +391,6 @@ class TestGetRetryRequest:
assert new_request.meta["retry_times"] == expected_retry_times
assert new_request.priority == -expected_retry_times
expected_reason = "unspecified"
assert spider.crawler.stats
for stat in ("retry/count", f"retry/reason_count/{expected_reason}"):
value = spider.crawler.stats.get_value(stat)
assert value == expected_retry_times
@ -521,7 +515,6 @@ class TestGetRetryRequest:
reason=expected_reason,
)
expected_retry_times = 1
assert spider.crawler.stats
for stat in ("retry/count", f"retry/reason_count/{expected_reason}"):
assert spider.crawler.stats.get_value(stat) == 1
assert (
@ -543,7 +536,6 @@ class TestGetRetryRequest:
reason=expected_reason,
)
expected_retry_times = 1
assert spider.crawler.stats
stat = spider.crawler.stats.get_value(
f"retry/reason_count/{expected_reason_string}"
)
@ -569,7 +561,6 @@ class TestGetRetryRequest:
reason=expected_reason,
)
expected_retry_times = 1
assert spider.crawler.stats
stat = spider.crawler.stats.get_value(
f"retry/reason_count/{expected_reason_string}"
)
@ -593,7 +584,6 @@ class TestGetRetryRequest:
reason=expected_reason,
)
expected_retry_times = 1
assert spider.crawler.stats
stat = spider.crawler.stats.get_value(
f"retry/reason_count/{expected_reason_string}"
)
@ -619,7 +609,6 @@ class TestGetRetryRequest:
reason=expected_reason,
)
expected_retry_times = 1
assert spider.crawler.stats
stat = spider.crawler.stats.get_value(
f"retry/reason_count/{expected_reason_string}"
)
@ -737,7 +726,6 @@ class TestGetRetryRequest:
reason=expected_reason,
stats_base_key=stats_key,
)
assert spider.crawler.stats
for stat in (
f"{stats_key}/count",
f"{stats_key}/reason_count/{expected_reason}",

3
tests/test_downloadermiddleware_stats.py
View File

@ -17,7 +17,6 @@ class MyException(Exception):
class TestDownloaderStats:
def setup_method(self) -> None:
self.crawler = get_crawler(Spider)
assert self.crawler.stats is not None
self.mw = build_from_crawler(DownloaderStats, self.crawler)
self.crawler.stats.open_spider()
@ -26,7 +25,6 @@ class TestDownloaderStats:
self.res = Response("http://scrapytest.org", status=400)
def assertStatsEqual(self, key: str, value: object) -> None:
assert self.crawler.stats is not None
assert self.crawler.stats.get_value(key) == value, str(
self.crawler.stats.get_stats()
)
@ -53,7 +51,6 @@ class TestDownloaderStats:
build_from_crawler(DownloaderStats, crawler)
def teardown_method(self) -> None:
assert self.crawler.stats is not None
self.crawler.stats.close_spider()

1
tests/test_downloaderslotssettings.py
View File

@ -149,7 +149,6 @@ async def test_none_slot_with_priority_queue(
assert crawler.spider.default_slot in crawler.spider.times
assert len(crawler.spider.times[crawler.spider.default_slot]) == 2
assert crawler.stats
stats = crawler.stats
assert stats.get_value("spider_exceptions", 0) == 0
assert stats.get_value("downloader/exception_count", 0) == 0

2
tests/test_dupefilters.py
View File

@ -179,7 +179,6 @@ class TestRFPDupeFilter:
dupefilter.log(r1, spider)
dupefilter.log(r2, spider)
assert crawler.stats
assert crawler.stats.get_value("dupefilter/filtered") == 2
assert (
"scrapy.dupefilters",
@ -213,7 +212,6 @@ class TestRFPDupeFilter:
dupefilter.log(r1, spider)
dupefilter.log(r2, spider)
assert crawler.stats
assert crawler.stats.get_value("dupefilter/filtered") == 2
assert (
"scrapy.dupefilters",

108
tests/test_engine_loop.py
View File

@ -15,7 +15,7 @@ from tests.mockserver.http import MockServer
from tests.utils.decorators import coroutine_test
if TYPE_CHECKING:
from collections.abc import Iterator
from collections.abc import AsyncIterator, Callable, Iterator
from scrapy.http import Response
@ -85,19 +85,23 @@ class TestMain:
class TestSpider(Spider):
name = "test"
async def start(self):
async def start(self) -> AsyncIterator[Any]:
assert self.crawler.engine._slot
scheduler = self.crawler.engine._slot.scheduler
assert isinstance(scheduler, MemoryScheduler)
yield Request("data:,a")
await sleep(seconds)
self.crawler.engine._slot.scheduler.pause()
self.crawler.engine._slot.scheduler.enqueue_request(Request("data:,b"))
scheduler.pause()
scheduler.enqueue_request(Request("data:,b"))
# During this time, the scheduler reports having requests but
# returns None.
await sleep(seconds)
self.crawler.engine._slot.scheduler.unpause()
scheduler.unpause()
# The scheduler request is processed.
await sleep(seconds)
@ -106,21 +110,21 @@ class TestMain:
await sleep(seconds)
self.crawler.engine._slot.scheduler.pause()
self.crawler.engine._slot.scheduler.enqueue_request(Request("data:,d"))
scheduler.pause()
scheduler.enqueue_request(Request("data:,d"))
# The last start request is processed during the time until the
# delayed call below, proving that the start iteration can
# finish before a scheduler “sleep” without causing the
# scheduler to finish.
call_later(seconds, self.crawler.engine._slot.scheduler.unpause)
call_later(seconds, scheduler.unpause)
def parse(self, response):
def parse(self, response: Response) -> None:
pass
actual_urls = []
def track_url(request, spider):
def track_url(request: Request, spider: Spider) -> None:
actual_urls.append(request.url)
settings = {"SCHEDULER": MemoryScheduler}
@ -138,17 +142,16 @@ class TestMain:
class TestSpider(Spider):
name = "test"
async def start(self):
assert self.crawler.engine is not None
async def start(self) -> AsyncIterator[Any]:
await self.crawler.engine.close_async()
yield Request("data:,a")
def parse(self, response):
def parse(self, response: Response) -> None:
pass
actual_urls = []
def track_url(request, spider):
def track_url(request: Request, spider: Spider) -> None:
actual_urls.append(request.url)
settings = {"SCHEDULER": MemoryScheduler}
@ -160,7 +163,6 @@ class TestMain:
await crawler.crawl_async()
assert not caplog.records
assert crawler.stats
assert crawler.stats.get_value("finish_reason") == "shutdown"
assert not actual_urls
@ -229,7 +231,6 @@ class TestMain:
failure, response = errors[0]
assert isinstance(failure.value, ValueError)
assert response is None
assert crawler.stats
assert crawler.stats.get_value("finish_reason") == "start_error"
assert crawler.stats.get_value("spider_exceptions/count") == 1
assert crawler.stats.get_value("spider_exceptions/ValueError") == 1
@ -256,12 +257,13 @@ class TestMain:
await crawler.crawl_async()
assert not caplog.records
assert crawler.stats
assert crawler.stats.get_value("finish_reason") == "my_reason"
assert crawler.stats.get_value("spider_exceptions/count") is None
class TestRequestSendOrder:
mockserver: MockServer
seconds = 0.1 # increase if flaky
@classmethod
@ -273,52 +275,65 @@ class TestRequestSendOrder:
def teardown_class(cls):
cls.mockserver.__exit__(None, None, None)
def request(self, num, response_seconds, download_slots, priority=0):
def request(
self,
num: int,
response_seconds: float,
download_slots: int,
priority: int = 0,
) -> Request:
url = self.mockserver.url(f"/delay?n={response_seconds}&{num}")
meta = {"download_slot": str(num % download_slots)}
return Request(url, meta=meta, priority=priority)
def get_num(self, request_or_response: Request | Response):
def get_num(self, request_or_response: Request | Response) -> int:
return int(request_or_response.url.rsplit("&", maxsplit=1)[1])
async def _test_request_order(
self,
start_nums,
cb_nums=None,
settings=None,
response_seconds=None,
download_slots=1,
start_fn=None,
parse_fn=None,
):
start_nums: list[int],
cb_nums: list[int] | None = None,
settings: dict[str, Any] | None = None,
response_seconds: float | None = None,
download_slots: int = 1,
start_fn: Callable[[Spider], AsyncIterator[Any]] | None = None,
parse_fn: Callable[..., Iterator[Any]] | None = None,
) -> None:
cb_nums = cb_nums or []
settings = settings or {}
response_seconds = response_seconds or self.seconds
seconds = response_seconds or self.seconds
cb_requests = deque(
[self.request(num, response_seconds, download_slots) for num in cb_nums]
[self.request(num, seconds, download_slots) for num in cb_nums]
)
if start_fn is None:
async def start_fn(spider):
async def default_start(spider: Spider) -> AsyncIterator[Any]:
for num in start_nums:
yield self.request(num, response_seconds, download_slots)
yield self.request(num, seconds, download_slots)
start_fn = default_start
if parse_fn is None:
def parse_fn(spider, response):
def default_parse(spider: Spider, response: Response) -> Iterator[Any]:
while cb_requests:
yield cb_requests.popleft()
parse_fn = default_parse
spider_start = start_fn
spider_parse = parse_fn
class TestSpider(Spider):
name = "test"
start = start_fn
parse = parse_fn
start = spider_start
parse = spider_parse
actual_nums = []
def track_num(request, spider):
def track_num(request: Request, spider: Spider) -> None:
actual_nums.append(self.get_num(request))
crawler = get_crawler(TestSpider, settings_dict=settings)
@ -337,12 +352,13 @@ class TestRequestSendOrder:
response_seconds = 0
download_slots = 1
def _request(num, priority=0):
def _request(num: int, priority: int = 0) -> Request:
return self.request(
num, response_seconds, download_slots, priority=priority
)
async def start(spider):
async def start(spider: Spider) -> AsyncIterator[Any]:
assert spider.crawler.engine._slot
# The first CONCURRENT_REQUESTS start requests are sent
# immediately.
yield _request(1)
@ -356,7 +372,7 @@ class TestRequestSendOrder:
yield _request(3, priority=1)
yield _request(4, priority=1)
def parse(spider, response):
def parse(spider: Spider, response: Response) -> Iterator[Any]:
return
yield
@ -377,12 +393,13 @@ class TestRequestSendOrder:
response_seconds = 0
download_slots = 1
def _request(num, priority=0):
def _request(num: int, priority: int = 0) -> Request:
return self.request(
num, response_seconds, download_slots, priority=priority
)
async def start(spider):
async def start(spider: Spider) -> AsyncIterator[Any]:
assert spider.crawler.engine._slot
# The first CONCURRENT_REQUESTS start requests are sent
# immediately.
yield _request(1)
@ -396,7 +413,7 @@ class TestRequestSendOrder:
yield _request(4, priority=1)
yield _request(3, priority=1)
def parse(spider, response):
def parse(spider: Spider, response: Response) -> Iterator[Any]:
return
yield
@ -420,12 +437,13 @@ class TestRequestSendOrder:
response_seconds = 0
download_slots = 1
def _request(num, priority=0):
def _request(num: int, priority: int = 0) -> Request:
return self.request(
num, response_seconds, download_slots, priority=priority
)
async def start(spider):
async def start(spider: Spider) -> AsyncIterator[Any]:
assert spider.crawler.engine._slot
# The first CONCURRENT_REQUESTS start requests are sent
# immediately.
yield _request(1)
@ -454,7 +472,7 @@ class TestRequestSendOrder:
):
spider.crawler.engine._slot.scheduler.enqueue_request(request)
def parse(spider, response):
def parse(spider: Spider, response: Response) -> Iterator[Any]:
return
yield
@ -479,7 +497,7 @@ class TestRequestSendOrder:
response_seconds = self.seconds * 2**1 # increase if flaky
download_slots = 1
async def start(spider):
async def start(spider: Spider) -> AsyncIterator[Any]:
for num in start_nums:
if spider.crawler.engine.needs_backout():
await spider.crawler.signals.wait_for(signals.scheduler_empty)

2
tests/test_extension_memdebug.py
View File

@ -33,7 +33,6 @@ def test_spider_closed_sets_stats() -> None:
ext.spider_closed(DefaultSpider.from_crawler(crawler), "finished")
assert crawler.stats
assert crawler.stats.get_value("memdebug/gc_garbage_count") == len(gc.garbage)
assert crawler.stats.get_value("memdebug/live_refs/TrackedObject") == len(tracked)
assert crawler.stats.get_value("memdebug/live_refs/CollectedObject") is None
@ -47,6 +46,5 @@ async def test_crawl_sets_stats() -> None:
crawler = get_crawler(MemDebugSpider, settings_dict={"MEMDEBUG_ENABLED": True})
await crawler.crawl_async()
assert crawler.stats
assert crawler.stats.get_value("memdebug/gc_garbage_count") is not None
assert crawler.stats.get_value("memdebug/live_refs/MemDebugSpider") == 1

3
tests/test_extension_memusage.py
View File

@ -99,7 +99,6 @@ async def test_memusage_below_thresholds_logs_peak(
with caplog.at_level(logging.INFO, logger="scrapy.extensions.memusage"):
await crawler.crawl_async(url="data:,", loops=1)
assert crawler.stats
assert crawler.stats.get_value("memusage/limit_reached") is None
assert crawler.stats.get_value("memusage/warning_reached") is None
assert crawler.stats.get_value("memusage/max") == 25 * MB
@ -129,7 +128,6 @@ async def test_memusage_limit_closes_spider_with_reason_and_error_log(
with caplog.at_level(logging.ERROR, logger="scrapy.extensions.memusage"):
await crawler.crawl_async(url="data:,", loops=100)
assert crawler.stats
assert crawler.stats.get_value("memusage/limit_reached") == 1
assert crawler.stats.get_value("finish_reason") == "memusage_exceeded"
assert any(
@ -167,7 +165,6 @@ async def test_memusage_warning_logs_but_allows_normal_finish(
await crawler.crawl_async(url="data:,", loops=60)
assert warning_signals == [1]
assert crawler.stats
assert crawler.stats.get_value("memusage/warning_reached") == 1
assert crawler.stats.get_value("finish_reason") == "finished"
warnings_logged = [

1
tests/test_extension_telnet.py
View File

@ -119,7 +119,6 @@ async def test_telnet_vars() -> None:
start_urls = ["data:,"]
async def parse(self, response: Response) -> None:
assert self.crawler.extensions
console = next(
ext
for ext in self.crawler.extensions.middlewares

3
tests/test_feedexport.py
View File

@ -310,7 +310,6 @@ class TestFeedExport(TestFeedExportBase):
}
crawler = get_crawler(ItemSpider, settings)
yield crawler.crawl(mockserver=self.mockserver)
assert crawler.stats is not None
assert "feedexport/success_count/FileFeedStorage" in crawler.stats.get_stats()
assert crawler.stats.get_value("feedexport/success_count/FileFeedStorage") == 1
@ -334,7 +333,6 @@ class TestFeedExport(TestFeedExportBase):
side_effect=store,
):
yield crawler.crawl(mockserver=self.mockserver)
assert crawler.stats is not None
assert "feedexport/failed_count/FileFeedStorage" in crawler.stats.get_stats()
assert crawler.stats.get_value("feedexport/failed_count/FileFeedStorage") == 1
@ -352,7 +350,6 @@ class TestFeedExport(TestFeedExportBase):
}
crawler = get_crawler(ItemSpider, settings)
yield crawler.crawl(mockserver=self.mockserver)
assert crawler.stats is not None
assert "feedexport/success_count/FileFeedStorage" in crawler.stats.get_stats()
assert "feedexport/success_count/StdoutFeedStorage" in crawler.stats.get_stats()
assert crawler.stats.get_value("feedexport/success_count/FileFeedStorage") == 1

2
tests/test_feedexport_batch.py
View File

@ -420,7 +420,6 @@ class TestBatchDeliveries(TestFeedExportBase):
}
crawler = get_crawler(ItemSpider, settings)
yield crawler.crawl(total=2, mockserver=self.mockserver)
assert crawler.stats
assert "feedexport/success_count/FileFeedStorage" in crawler.stats.get_stats()
assert crawler.stats.get_value("feedexport/success_count/FileFeedStorage") == 12
@ -491,7 +490,6 @@ class TestBatchDeliveries(TestFeedExportBase):
assert len(CustomS3FeedStorage.stubs) == len(items)
for stub in CustomS3FeedStorage.stubs:
stub.assert_no_pending_responses()
assert crawler.stats
assert (
"feedexport/success_count/CustomS3FeedStorage" in crawler.stats.get_stats()
)

1
tests/test_logstats.py
View File

@ -15,7 +15,6 @@ class TestLogStats:
def setup_method(self) -> None:
self.crawler = get_crawler(SimpleSpider)
self.spider = self.crawler._create_spider("spidey")
assert self.crawler.stats is not None
self.stats = self.crawler.stats
self.stats.set_value("response_received_count", 4802)

2
tests/test_pipeline_crawl.py
View File

@ -140,7 +140,6 @@ class TestFileDownloadCrawl:
assert not items[0][self.media_key]
# check that there was 1 successful fetch and 3 other responses with non-200 code
assert crawler.stats
assert crawler.stats.get_value("downloader/request_method_count/GET") == 4
assert crawler.stats.get_value("downloader/response_count") == 4
assert crawler.stats.get_value("downloader/response_status_count/200") == 1
@ -208,7 +207,6 @@ class TestFileDownloadCrawl:
mockserver=self.mockserver,
)
self._assert_files_downloaded(self.items, caplog.text)
assert crawler.stats
assert crawler.stats.get_value("downloader/response_status_count/302") == 3
@coroutine_test

1
tests/test_pipeline_media.py
View File

@ -82,7 +82,6 @@ class TestBaseMediaPipeline:
self.pipe = build_from_crawler(self.pipeline_class, crawler)
self.pipe.open_spider()
self.info = self.pipe.spiderinfo
assert crawler.request_fingerprinter is not None
self.fingerprint = crawler.request_fingerprinter.fingerprint
@property

2
tests/test_request_cb_kwargs.py
View File

@ -70,7 +70,6 @@ class KeywordArgumentsSpider(MockServerSpider):
checks: list[bool] = []
def _inc_checks(self, count: int = 1) -> None:
assert self.crawler.stats
self.crawler.stats.inc_value("boolean_checks", count)
async def start(self):
@ -171,7 +170,6 @@ class TestCallbackKeywordArguments:
await crawler.crawl_async(mockserver=mockserver)
assert isinstance(crawler.spider, KeywordArgumentsSpider)
assert all(crawler.spider.checks)
assert crawler.stats
assert len(crawler.spider.checks) == crawler.stats.get_value("boolean_checks")
# check exceptions for argument mismatch
exceptions = {}

3
tests/test_scheduler.py
View File

@ -54,7 +54,6 @@ async def create_scheduler(
finally:
await ensure_awaitable(scheduler.close("finished"))
await mock_crawler.stop_async()
assert mock_crawler.engine
mock_crawler.engine.downloader.close()
@ -254,7 +253,6 @@ class DownloaderAwareSchedulerTestMixin(TestSchedulerBase):
dequeued_slots: list[str] = []
requests: list[Request] = []
assert scheduler.crawler
assert scheduler.crawler.engine
downloader = scheduler.crawler.engine.downloader
assert isinstance(downloader, MockDownloader)
while scheduler.has_pending_requests():
@ -320,7 +318,6 @@ class TestIntegrationWithDownloaderAwareInMemory:
url = mockserver.url("/status?n=200", is_secure=False)
start_urls = [url] * 6
yield self.crawler.crawl(start_urls)
assert self.crawler.stats
assert self.crawler.stats.get_value("downloader/response_count") == len(
start_urls
)

8
tests/test_spider.py
View File

@ -41,7 +41,7 @@ class TestXMLFeedSpider(TestSpiderBase):
</urlset>"""
response = XmlResponse(url="http://example.com/sitemap.xml", body=body)
class _XMLSpider(self.spider_class):
class _XMLSpider(self.spider_class): # type: ignore[name-defined,misc]
itertag = "url"
namespaces = (
("a", "http://www.google.com/schemas/sitemap/0.84"),
@ -100,7 +100,6 @@ class TestXMLFeedSpider(TestSpiderBase):
items, crawler = await crawl_items(_Spider, mockserver)
assert items == []
assert crawler.stats
assert crawler.stats.get_value("spider_exceptions/NotImplementedError") == 1
@coroutine_test
@ -135,7 +134,6 @@ class TestXMLFeedSpider(TestSpiderBase):
items, crawler = await crawl_items(_Spider, mockserver)
assert items == []
assert crawler.stats
assert crawler.stats.get_value("spider_exceptions/NotSupported") == 1
@pytest.mark.parametrize("feed_iterator", ["xml", "html"])
@ -155,7 +153,6 @@ class TestXMLFeedSpider(TestSpiderBase):
items, crawler = await crawl_items(_Spider, mockserver)
assert items == []
assert crawler.stats
assert crawler.stats.get_value("spider_exceptions/ValueError") == 1
@ -166,7 +163,7 @@ class TestCSVFeedSpider(TestSpiderBase):
body = get_testdata("feeds", "feed-sample6.csv")
response = Response("http://example.org/dummy.csv", body=body)
class _CrawlSpider(self.spider_class):
class _CrawlSpider(self.spider_class): # type: ignore[name-defined,misc]
name = "test"
delimiter = ","
quotechar = "'"
@ -206,7 +203,6 @@ class TestCSVFeedSpider(TestSpiderBase):
items, crawler = await crawl_items(_Spider, mockserver)
assert items == []
assert crawler.stats
assert crawler.stats.get_value("spider_exceptions/NotImplementedError") == 1

53
tests/test_spider_crawl.py
View File

@ -2,6 +2,7 @@ from __future__ import annotations
import re
import warnings
from typing import TYPE_CHECKING, Any
import pytest
from w3lib.url import safe_url_string
@ -14,6 +15,12 @@ from scrapy.utils.test import get_crawler
from tests.utils.bases.spider import TestSpiderBase
from tests.utils.decorators import coroutine_test
if TYPE_CHECKING:
from scrapy.http import Response
from scrapy.http.request import CallbackT
URL = "https://www.example.com"
class TestCrawlSpider(TestSpiderBase):
test_body = b"""<html><head><title>Page title</title></head>
@ -33,7 +40,7 @@ class TestCrawlSpider(TestSpiderBase):
"http://example.org/somepage/index.html", body=self.test_body
)
class _CrawlSpider(self.spider_class):
class _CrawlSpider(self.spider_class): # type: ignore[name-defined,misc]
name = "test"
allowed_domains = ["example.org"]
rules = (Rule(),)
@ -53,7 +60,7 @@ class TestCrawlSpider(TestSpiderBase):
"http://example.org/somepage/index.html", body=self.test_body
)
class _CrawlSpider(self.spider_class):
class _CrawlSpider(self.spider_class): # type: ignore[name-defined,misc]
name = "test"
allowed_domains = ["example.org"]
rules = (Rule(LinkExtractor(), process_links="dummy_process_links"),)
@ -76,7 +83,7 @@ class TestCrawlSpider(TestSpiderBase):
"http://example.org/somepage/index.html", body=self.test_body
)
class _CrawlSpider(self.spider_class):
class _CrawlSpider(self.spider_class): # type: ignore[name-defined,misc]
name = "test"
allowed_domains = ["example.org"]
rules = (Rule(LinkExtractor(), process_links="filter_process_links"),)
@ -99,7 +106,7 @@ class TestCrawlSpider(TestSpiderBase):
"http://example.org/somepage/index.html", body=self.test_body
)
class _CrawlSpider(self.spider_class):
class _CrawlSpider(self.spider_class): # type: ignore[name-defined,misc]
name = "test"
allowed_domains = ["example.org"]
rules = (Rule(LinkExtractor(), process_links="dummy_process_links"),)
@ -125,7 +132,7 @@ class TestCrawlSpider(TestSpiderBase):
def process_request_change_domain(request, response):
return request.replace(url=request.url.replace(".org", ".com"))
class _CrawlSpider(self.spider_class):
class _CrawlSpider(self.spider_class): # type: ignore[name-defined,misc]
name = "test"
allowed_domains = ["example.org"]
rules = (
@ -151,7 +158,7 @@ class TestCrawlSpider(TestSpiderBase):
request.meta["response_class"] = response.__class__.__name__
return request
class _CrawlSpider(self.spider_class):
class _CrawlSpider(self.spider_class): # type: ignore[name-defined,misc]
name = "test"
allowed_domains = ["example.org"]
rules = (
@ -180,7 +187,7 @@ class TestCrawlSpider(TestSpiderBase):
"http://example.org/somepage/index.html", body=self.test_body
)
class _CrawlSpider(self.spider_class):
class _CrawlSpider(self.spider_class): # type: ignore[name-defined,misc]
name = "test"
allowed_domains = ["example.org"]
rules = (Rule(LinkExtractor(), process_request="process_request_upper"),)
@ -203,7 +210,7 @@ class TestCrawlSpider(TestSpiderBase):
"http://example.org/somepage/index.html", body=self.test_body
)
class _CrawlSpider(self.spider_class):
class _CrawlSpider(self.spider_class): # type: ignore[name-defined,misc]
name = "test"
allowed_domains = ["example.org"]
rules = (
@ -248,7 +255,6 @@ class TestCrawlSpider(TestSpiderBase):
def test_parse_response_use(self):
class _CrawlSpider(CrawlSpider):
name = "test"
start_urls = "https://www.example.com"
_follow_links = False
with warnings.catch_warnings():
@ -258,17 +264,20 @@ class TestCrawlSpider(TestSpiderBase):
ScrapyDeprecationWarning,
match=r"CrawlSpider\._parse_response method is deprecated",
):
spider._parse_response(
TextResponse(spider.start_urls, body=b""), None, None
)
spider._parse_response(TextResponse(URL, body=b""), None, {})
def test_parse_response_override(self):
class _CrawlSpider(CrawlSpider):
def _parse_response(self, response, callback, cb_kwargs, follow=True):
def _parse_response(
self,
response: Response,
callback: CallbackT | None,
cb_kwargs: dict[str, Any],
follow: bool = True,
) -> Any:
pass
name = "test"
start_urls = "https://www.example.com"
_follow_links = False
with pytest.warns(
@ -278,21 +287,16 @@ class TestCrawlSpider(TestSpiderBase):
spider = _CrawlSpider()
with warnings.catch_warnings():
warnings.simplefilter("error", category=ScrapyDeprecationWarning)
spider._parse_response(
TextResponse(spider.start_urls, body=b""), None, None
)
spider._parse_response(TextResponse(URL, body=b""), None, {})
def test_parse_with_rules(self):
class _CrawlSpider(CrawlSpider):
name = "test"
start_urls = "https://www.example.com"
with warnings.catch_warnings():
warnings.simplefilter("error", category=ScrapyDeprecationWarning)
spider = _CrawlSpider()
spider.parse_with_rules(
TextResponse(spider.start_urls, body=b""), None, None
)
spider.parse_with_rules(TextResponse(URL, body=b""), None, {})
@coroutine_test
async def test_parse_with_rules_without_callback(self):
@ -327,12 +331,13 @@ class TestCrawlSpider(TestSpiderBase):
allowed_domains = ["example.org"]
rules = (Rule(),)
def callback(response: Response) -> list[Any]:
return [item]
spider = _CrawlSpider.from_crawler(get_crawler(_CrawlSpider))
results = [
r
async for r in spider.parse_with_rules(
response, lambda response: [item], {}, follow=False
)
async for r in spider.parse_with_rules(response, callback, {}, follow=False)
]
assert results == [item]

16
tests/test_spider_sitemap.py
View File

@ -55,7 +55,7 @@ class TestSitemapSpider(TestSpiderBase):
assert spider._get_sitemap_body(response) == body
def test_get_sitemap_body(self):
r = XmlResponse(url="http://www.example.com/", body=self.BODY)
r: Response = XmlResponse(url="http://www.example.com/", body=self.BODY)
self.assertSitemapBody(r, self.BODY)
r = HtmlResponse(url="http://www.example.com/", body=self.BODY)
@ -166,7 +166,7 @@ Sitemap: /sitemap-relative-url.xml
</url>
</urlset>"""
class FilteredSitemapSpider(self.spider_class):
class FilteredSitemapSpider(self.spider_class): # type: ignore[name-defined,misc]
def sitemap_filter(self, entries):
for entry in entries:
date_time = datetime.strptime(entry["lastmod"], "%Y-%m-%d")
@ -201,7 +201,7 @@ Sitemap: /sitemap-relative-url.xml
</url>
</urlset>"""
class FilteredSitemapSpider(self.spider_class):
class FilteredSitemapSpider(self.spider_class): # type: ignore[name-defined,misc]
def sitemap_filter(self, entries):
for entry in entries:
alternate_links = entry.get("alternate", ())
@ -235,7 +235,7 @@ Sitemap: /sitemap-relative-url.xml
</sitemap>
</sitemapindex>"""
class FilteredSitemapSpider(self.spider_class):
class FilteredSitemapSpider(self.spider_class): # type: ignore[name-defined,misc]
def sitemap_filter(self, entries):
for entry in entries:
date_time = datetime.strptime(
@ -285,6 +285,7 @@ Sitemap: /sitemap-relative-url.xml
sitemap_rules = [("", parse_item)]
def raw_body(self):
assert self.mockserver
loc = self.mockserver.url("/text")
return (
'<?xml version="1.0" encoding="UTF-8"?>'
@ -303,6 +304,7 @@ Sitemap: /sitemap-relative-url.xml
yield {"url": response.url}
def raw_body(self):
assert self.mockserver
loc = self.mockserver.url("/text")
return (
'<?xml version="1.0" encoding="UTF-8"?>'
@ -363,7 +365,7 @@ Sitemap: /sitemap-relative-url.xml
</sitemapindex>"""
r = TextResponse(url="http://www.example.com/sitemap.xml", body=sitemap)
class _FollowSpider(self.spider_class):
class _FollowSpider(self.spider_class): # type: ignore[name-defined,misc]
sitemap_follow = [follow]
spider = _FollowSpider("example.com")
@ -382,7 +384,7 @@ Sitemap: /sitemap-relative-url.xml
@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
def test_compression_bomb_spider_attr(self):
class DownloadMaxSizeSpider(self.spider_class):
class DownloadMaxSizeSpider(self.spider_class): # type: ignore[name-defined,misc]
download_maxsize = 10_000_000
crawler = get_crawler()
@ -485,7 +487,7 @@ Sitemap: /sitemap-relative-url.xml
@coroutine_test
async def test_sitemap_urls(self):
class TestSpider(self.spider_class):
class TestSpider(self.spider_class): # type: ignore[name-defined,misc]
name = "test"
sitemap_urls = ["https://toscrape.com/sitemap.xml"]

1
tests/test_spider_start.py
View File

@ -34,7 +34,6 @@ class TestMain:
crawler = get_crawler(spider)
crawler.signals.connect(track_item, signals.item_scraped)
await crawler.crawl_async()
assert crawler.stats
assert crawler.stats.get_value("finish_reason") == "finished"
assert actual_items == expected_items

2
tests/test_spidermiddleware_depth.py
View File

@ -26,7 +26,6 @@ def crawler() -> Crawler:
@pytest.fixture
def stats(crawler: Crawler) -> Generator[StatsCollector]:
assert crawler.stats is not None
crawler.stats.open_spider()
yield crawler.stats
@ -120,7 +119,6 @@ def test_priority_and_non_verbose_stats() -> None:
Spider,
{"DEPTH_LIMIT": 0, "DEPTH_STATS_VERBOSE": False, "DEPTH_PRIORITY": 10},
)
assert crawler.stats is not None
crawler.stats.open_spider()
try:
mw = build_from_crawler(DepthMiddleware, crawler)

1
tests/test_spidermiddleware_httperror.py
View File

@ -205,7 +205,6 @@ class TestHttpErrorMiddlewareIntegrational:
assert crawler.spider.parsed == {"200"}
assert crawler.spider.failed == {"404", "402", "500"}
assert crawler.stats
get_value = crawler.stats.get_value
assert get_value("httperror/response_ignored_count") == 3
assert get_value("httperror/response_ignored_status_count/404") == 1

39
tests/test_spidermiddleware_output_chain.py
View File

@ -1,16 +1,21 @@
from __future__ import annotations
import logging
from typing import TYPE_CHECKING
from typing import TYPE_CHECKING, Any
from scrapy import Request, Spider
from scrapy.utils.test import get_crawler
from tests.mockserver.http import MockServer
from tests.spiders import MockServerSpider
from tests.utils.decorators import coroutine_test
if TYPE_CHECKING:
from collections.abc import AsyncIterator, Iterator
import pytest
from scrapy.http import Response
class _BaseSpiderMiddleware:
def __init__(self, crawler):
@ -41,7 +46,7 @@ class RecoveryMiddleware(_BaseSpiderMiddleware):
]
class RecoverySpider(Spider):
class RecoverySpider(MockServerSpider):
name = "RecoverySpider"
custom_settings = {
"SPIDER_MIDDLEWARES_BASE": {},
@ -50,10 +55,11 @@ class RecoverySpider(Spider):
},
}
async def start(self):
async def start(self) -> AsyncIterator[Any]:
assert self.mockserver
yield Request(self.mockserver.url("/status?n=200"))
def parse(self, response):
def parse(self, response: Response) -> Iterator[Any]:
yield {"test": 1}
self.logger.info("DONT_FAIL: %s", response.meta.get("dont_fail"))
if not response.meta.get("dont_fail"):
@ -63,7 +69,7 @@ class RecoverySpider(Spider):
class RecoveryAsyncGenSpider(RecoverySpider):
name = "RecoveryAsyncGenSpider"
async def parse(self, response):
async def parse(self, response: Response) -> AsyncIterator[Any]: # type: ignore[override]
for r in super().parse(response):
yield r
@ -76,7 +82,7 @@ class FailProcessSpiderInputMiddleware(_BaseSpiderMiddleware):
raise IndexError
class ProcessSpiderInputSpiderWithoutErrback(Spider):
class ProcessSpiderInputSpiderWithoutErrback(MockServerSpider):
name = "ProcessSpiderInputSpiderWithoutErrback"
custom_settings = {
"SPIDER_MIDDLEWARES": {
@ -87,7 +93,8 @@ class ProcessSpiderInputSpiderWithoutErrback(Spider):
}
}
async def start(self):
async def start(self) -> AsyncIterator[Any]:
assert self.mockserver
yield Request(url=self.mockserver.url("/status?n=200"), callback=self.parse)
def parse(self, response):
@ -97,7 +104,8 @@ class ProcessSpiderInputSpiderWithoutErrback(Spider):
class ProcessSpiderInputSpiderWithErrback(ProcessSpiderInputSpiderWithoutErrback):
name = "ProcessSpiderInputSpiderWithErrback"
async def start(self):
async def start(self) -> AsyncIterator[Any]:
assert self.mockserver
yield Request(
self.mockserver.url("/status?n=200"), self.parse, errback=self.errback
)
@ -109,7 +117,7 @@ class ProcessSpiderInputSpiderWithErrback(ProcessSpiderInputSpiderWithoutErrback
# ================================================================================
# (2) exceptions from a spider callback (generator)
class GeneratorCallbackSpider(Spider):
class GeneratorCallbackSpider(MockServerSpider):
name = "GeneratorCallbackSpider"
custom_settings = {
"SPIDER_MIDDLEWARES": {
@ -117,7 +125,8 @@ class GeneratorCallbackSpider(Spider):
},
}
async def start(self):
async def start(self) -> AsyncIterator[Any]:
assert self.mockserver
yield Request(self.mockserver.url("/status?n=200"))
def parse(self, response):
@ -146,7 +155,7 @@ class GeneratorCallbackSpiderMiddlewareRightAfterSpider(GeneratorCallbackSpider)
# ================================================================================
# (3) exceptions from a spider callback (not a generator)
class NotGeneratorCallbackSpider(Spider):
class NotGeneratorCallbackSpider(MockServerSpider):
name = "NotGeneratorCallbackSpider"
custom_settings = {
"SPIDER_MIDDLEWARES": {
@ -154,7 +163,8 @@ class NotGeneratorCallbackSpider(Spider):
},
}
async def start(self):
async def start(self) -> AsyncIterator[Any]:
assert self.mockserver
yield Request(self.mockserver.url("/status?n=200"))
def parse(self, response):
@ -224,7 +234,7 @@ class GeneratorDoNothingAfterRecoveryMiddleware(_GeneratorDoNothingMiddleware):
pass
class GeneratorOutputChainSpider(Spider):
class GeneratorOutputChainSpider(MockServerSpider):
name = "GeneratorOutputChainSpider"
custom_settings = {
"SPIDER_MIDDLEWARES": {
@ -235,7 +245,8 @@ class GeneratorOutputChainSpider(Spider):
},
}
async def start(self):
async def start(self) -> AsyncIterator[Any]:
assert self.mockserver
yield Request(self.mockserver.url("/status?n=200"))
def parse(self, response):

38
tests/test_spidermiddleware_process_start.py
View File

@ -1,4 +1,7 @@
from __future__ import annotations
from asyncio import sleep
from typing import TYPE_CHECKING, Any
import pytest
@ -9,6 +12,11 @@ from scrapy.utils.test import get_crawler
from .utils import twisted_sleep
from .utils.decorators import coroutine_test
if TYPE_CHECKING:
from collections.abc import AsyncIterator
from scrapy.http import Response
ITEM_A = {"id": "a"}
ITEM_B = {"id": "b"}
ITEM_C = {"id": "c"}
@ -17,20 +25,20 @@ SLEEP_SECONDS = 0.1
class AsyncioSleepSpiderMiddleware:
async def process_start(self, start):
async def process_start(self, start: AsyncIterator[Any]) -> AsyncIterator[Any]:
await sleep(SLEEP_SECONDS)
async for item_or_request in start:
yield item_or_request
class NoOpSpiderMiddleware:
async def process_start(self, start):
async def process_start(self, start: AsyncIterator[Any]) -> AsyncIterator[Any]:
async for item_or_request in start:
yield item_or_request
class TwistedSleepSpiderMiddleware:
async def process_start(self, start):
async def process_start(self, start: AsyncIterator[Any]) -> AsyncIterator[Any]:
await maybe_deferred_to_future(twisted_sleep(SLEEP_SECONDS))
async for item_or_request in start:
yield item_or_request
@ -42,12 +50,12 @@ class TwistedSleepSpiderMiddleware:
class ModernWrapSpider(Spider):
name = "test"
async def start(self):
async def start(self) -> AsyncIterator[Any]:
yield ITEM_B
class ModernWrapSpiderMiddleware:
async def process_start(self, start):
async def process_start(self, start: AsyncIterator[Any]) -> AsyncIterator[Any]:
yield ITEM_A
async for item_or_request in start:
yield item_or_request
@ -55,10 +63,15 @@ class ModernWrapSpiderMiddleware:
class TestMain:
async def _test(self, spider_middlewares, spider_cls, expected_items):
async def _test(
self,
spider_middlewares: list[type],
spider_cls: type[Spider],
expected_items: list[Any],
) -> None:
actual_items = []
def track_item(item, response, spider):
def track_item(item: Any, response: Response, spider: Spider) -> None:
actual_items.append(item)
settings = {
@ -70,7 +83,12 @@ class TestMain:
assert crawler.stats.get_value("finish_reason") == "finished"
assert actual_items == expected_items, f"{actual_items=} != {expected_items=}"
async def _test_wrap(self, spider_middleware, spider_cls, expected_items=None):
async def _test_wrap(
self,
spider_middleware: type,
spider_cls: type[Spider],
expected_items: list[Any] | None = None,
) -> None:
expected_items = expected_items or [ITEM_A, ITEM_B, ITEM_C]
await self._test([spider_middleware], spider_cls, expected_items)
@ -78,11 +96,11 @@ class TestMain:
async def test_modern_mw_modern_spider(self):
await self._test_wrap(ModernWrapSpiderMiddleware, ModernWrapSpider)
async def _test_sleep(self, spider_middlewares):
async def _test_sleep(self, spider_middlewares: list[type]) -> None:
class TestSpider(Spider):
name = "test"
async def start(self):
async def start(self) -> AsyncIterator[Any]:
yield ITEM_A
await self._test(spider_middlewares, TestSpider, [ITEM_A])

1
tests/test_spidermiddleware_urllength.py
View File

@ -31,7 +31,6 @@ def crawler() -> Crawler:
@pytest.fixture
def stats(crawler: Crawler) -> StatsCollector:
assert crawler.stats is not None
return crawler.stats

3
tests/test_utils_log.py
View File

@ -91,7 +91,6 @@ class TestLogCounterHandler:
logger.removeHandler(handler)
def test_init(self, crawler: Crawler, logger: logging.Logger) -> None:
assert crawler.stats
assert crawler.stats.get_value("log_count/DEBUG") is None
assert crawler.stats.get_value("log_count/INFO") is None
assert crawler.stats.get_value("log_count/WARNING") is None
@ -100,12 +99,10 @@ class TestLogCounterHandler:
def test_accepted_level(self, crawler: Crawler, logger: logging.Logger) -> None:
logger.error("test log msg")
assert crawler.stats
assert crawler.stats.get_value("log_count/ERROR") == 1
def test_filtered_out_level(self, crawler: Crawler, logger: logging.Logger) -> None:
logger.debug("test log msg")
assert crawler.stats
assert crawler.stats.get_value("log_count/DEBUG") is None

5
tests/test_utils_request.py
View File

@ -274,7 +274,6 @@ class TestRequestFingerprinter:
def test_fingerprint(self):
crawler = get_crawler()
request = Request("https://example.com")
assert crawler.request_fingerprinter
assert crawler.request_fingerprinter.fingerprint(request) == fingerprint(
request
)
@ -291,7 +290,6 @@ class TestCustomRequestFingerprinter:
}
crawler = get_crawler(settings_dict=settings)
assert crawler.request_fingerprinter
r1 = Request("http://www.example.com", headers={"X-ID": "1"})
fp1 = crawler.request_fingerprinter.fingerprint(r1)
r2 = Request("http://www.example.com", headers={"X-ID": "2"})
@ -314,7 +312,6 @@ class TestCustomRequestFingerprinter:
}
crawler = get_crawler(settings_dict=settings)
assert crawler.request_fingerprinter
r1 = Request("http://www.example.com?a=1&a=2")
fp1 = crawler.request_fingerprinter.fingerprint(r1)
r2 = Request("http://www.example.com?a=2&a=1")
@ -333,7 +330,6 @@ class TestCustomRequestFingerprinter:
}
crawler = get_crawler(settings_dict=settings)
assert crawler.request_fingerprinter
r1 = Request("http://www.example.com")
fp1 = crawler.request_fingerprinter.fingerprint(r1)
r2 = Request("http://www.example.com", meta={"fingerprint": "a"})
@ -365,7 +361,6 @@ class TestCustomRequestFingerprinter:
}
crawler = get_crawler(settings_dict=settings)
assert crawler.request_fingerprinter
request = Request("http://www.example.com")
fingerprint = crawler.request_fingerprinter.fingerprint(request)
assert fingerprint == settings["FINGERPRINT"]

8
tests/utils/bases/spider.py
View File

@ -32,7 +32,7 @@ class TestSpiderBase(ABC):
def test_spider_args(self):
"""``__init__`` method arguments are assigned to spider attributes"""
spider = self.spider_class("example.com", foo="bar")
assert spider.foo == "bar"
assert spider.foo == "bar" # type: ignore[attr-defined]
def test_spider_without_name(self):
"""``__init__`` raises when the name is not provided."""
@ -58,7 +58,7 @@ class TestSpiderBase(ABC):
mock_init.assert_called_once_with("example.com", foo="bar")
def test_closed_signal_call(self):
class TestSpider(self.spider_class):
class TestSpider(self.spider_class): # type: ignore[name-defined,misc]
closed_called = False
def closed(self, reason):
@ -92,7 +92,7 @@ class TestSpiderBase(ABC):
**get_reactor_settings(),
}
class TestSpider(self.spider_class):
class TestSpider(self.spider_class): # type: ignore[name-defined,misc]
name = "test"
custom_settings = spider_settings
@ -127,5 +127,5 @@ class TestSpiderBase(ABC):
ScrapyDeprecationWarning, match=r"Spider.log\(\) is deprecated"
),
):
spider.log("test log msg", "INFO")
spider.log("test log msg", "INFO") # type: ignore[arg-type]
mock_logger.log.assert_called_once_with("INFO", "test log msg")