diff --git a/pyproject.toml b/pyproject.toml index 888f95574..e027ea74d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -225,6 +225,7 @@ disable = [ "too-many-positional-arguments", "too-many-public-methods", "too-many-return-statements", + "undefined-variable", "unused-argument", "unused-variable", "useless-import-alias", # used as a hint to mypy diff --git a/tests/test_crawl.py b/tests/test_crawl.py index 52abb1ccc..ada4c31ce 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -9,7 +9,6 @@ from urllib.parse import urlencode, urlparse import pytest from cryptography.x509 import load_der_x509_certificate -from testfixtures import LogCapture from twisted.internet.defer import succeed from twisted.internet.ssl import Certificate from twisted.python.failure import Failure @@ -24,7 +23,6 @@ from scrapy.utils.engine import format_engine_status, get_engine_status from scrapy.utils.python import to_unicode from scrapy.utils.test import get_crawler, get_reactor_settings from tests import NON_EXISTING_RESOLVABLE -from tests.mockserver.http import MockServer from tests.spiders import ( AsyncDefAsyncioGenComplexSpider, AsyncDefAsyncioGenExcSpider, @@ -56,44 +54,36 @@ from tests.spiders import ( StartGoodAndBadOutput, StartItemSpider, ) -from tests.utils.decorators import coroutine_test, inline_callbacks_test +from tests.utils.decorators import coroutine_test if TYPE_CHECKING: from scrapy.statscollectors import StatsCollector + from tests.mockserver.http import MockServer class TestCrawl: - mockserver: MockServer - - @classmethod - def setup_class(cls): - cls.mockserver = MockServer() - cls.mockserver.__enter__() - - @classmethod - def teardown_class(cls): - cls.mockserver.__exit__(None, None, None) - - @inline_callbacks_test - def test_follow_all(self): + @coroutine_test + async def test_follow_all(self, mockserver: MockServer) -> None: crawler = get_crawler(FollowAllSpider) - yield crawler.crawl(mockserver=self.mockserver) + await crawler.crawl_async(mockserver=mockserver) + assert isinstance(crawler.spider, FollowAllSpider) assert len(crawler.spider.urls_visited) == 11 # 10 + start_url @coroutine_test - async def test_fixed_delay(self): - await self._test_delay(total=3, delay=0.2) + async def test_fixed_delay(self, mockserver: MockServer) -> None: + await self._test_delay(mockserver, total=3, delay=0.2) @coroutine_test - async def test_randomized_delay(self): - await self._test_delay(total=3, delay=0.1, randomize=True) + async def test_randomized_delay(self, mockserver: MockServer) -> None: + await self._test_delay(mockserver, total=3, delay=0.1, randomize=True) + @staticmethod async def _test_delay( - self, total: int, delay: float, randomize: bool = False + mockserver: MockServer, total: int, delay: float, randomize: bool = False ) -> None: crawl_kwargs = { "maxlatency": delay * 2, - "mockserver": self.mockserver, + "mockserver": mockserver, "total": total, } tolerance = 1 - (0.6 if randomize else 0.2) @@ -122,18 +112,20 @@ class TestCrawl: average = total_time / (len(times) - 1) assert average <= delay / tolerance, "test total or delay values are too small" - @inline_callbacks_test - def test_timeout_success(self): + @coroutine_test + async def test_timeout_success(self, mockserver: MockServer) -> None: crawler = get_crawler(DelaySpider) - yield crawler.crawl(n=0.5, mockserver=self.mockserver) + await crawler.crawl_async(n=0.5, mockserver=mockserver) + assert isinstance(crawler.spider, DelaySpider) assert crawler.spider.t1 > 0 assert crawler.spider.t2 > 0 assert crawler.spider.t2 > crawler.spider.t1 - @inline_callbacks_test - def test_timeout_failure(self): + @coroutine_test + async def test_timeout_failure(self, mockserver: MockServer) -> None: crawler = get_crawler(DelaySpider, {"DOWNLOAD_TIMEOUT": 0.35}) - yield crawler.crawl(n=0.5, mockserver=self.mockserver) + await crawler.crawl_async(n=0.5, mockserver=mockserver) + assert isinstance(crawler.spider, DelaySpider) assert crawler.spider.t1 > 0 assert crawler.spider.t2 == 0 assert crawler.spider.t2_err > 0 @@ -141,81 +133,96 @@ class TestCrawl: # server hangs after receiving response headers crawler = get_crawler(DelaySpider, {"DOWNLOAD_TIMEOUT": 0.35}) - yield crawler.crawl(n=0.5, b=1, mockserver=self.mockserver) + await crawler.crawl_async(n=0.5, b=1, mockserver=mockserver) + assert isinstance(crawler.spider, DelaySpider) assert crawler.spider.t1 > 0 assert crawler.spider.t2 == 0 assert crawler.spider.t2_err > 0 assert crawler.spider.t2_err > crawler.spider.t1 - @inline_callbacks_test - def test_retry_503(self): + @coroutine_test + async def test_retry_503( + self, caplog: pytest.LogCaptureFixture, mockserver: MockServer + ) -> None: crawler = get_crawler(SimpleSpider) - with LogCapture() as log: - yield crawler.crawl( - self.mockserver.url("/status?n=503"), mockserver=self.mockserver + with caplog.at_level(logging.DEBUG): + await crawler.crawl_async( + mockserver.url("/status?n=503"), mockserver=mockserver ) - self._assert_retried(log) + self._assert_retried(caplog.text) - @inline_callbacks_test - def test_retry_conn_failed(self): + @coroutine_test + async def test_retry_conn_failed( + self, caplog: pytest.LogCaptureFixture, mockserver: MockServer + ) -> None: crawler = get_crawler(SimpleSpider) - with LogCapture() as log: - yield crawler.crawl( - "http://localhost:65432/status?n=503", mockserver=self.mockserver + with caplog.at_level(logging.DEBUG): + await crawler.crawl_async( + "http://localhost:65432/status?n=503", mockserver=mockserver ) - self._assert_retried(log) + self._assert_retried(caplog.text) - @inline_callbacks_test - def test_retry_dns_error(self): + @coroutine_test + async def test_retry_dns_error( + self, caplog: pytest.LogCaptureFixture, mockserver: MockServer + ) -> None: if NON_EXISTING_RESOLVABLE: pytest.skip("Non-existing hosts are resolvable") crawler = get_crawler(SimpleSpider) - with LogCapture() as log: + with caplog.at_level(logging.DEBUG): # try to fetch the homepage of a nonexistent domain - yield crawler.crawl( - "http://dns.resolution.invalid./", mockserver=self.mockserver + await crawler.crawl_async( + "http://dns.resolution.invalid./", mockserver=mockserver ) - self._assert_retried(log) + self._assert_retried(caplog.text) - @inline_callbacks_test - def test_start_bug_before_yield(self): - with LogCapture("scrapy", level=logging.ERROR) as log: + @coroutine_test + async def test_start_bug_before_yield( + self, caplog: pytest.LogCaptureFixture, mockserver: MockServer + ) -> None: + with caplog.at_level(logging.ERROR): crawler = get_crawler(BrokenStartSpider) - yield crawler.crawl(fail_before_yield=1, mockserver=self.mockserver) + await crawler.crawl_async(fail_before_yield=1, mockserver=mockserver) - assert len(log.records) == 1 - record = log.records[0] + assert len(caplog.records) == 1 + record = caplog.records[0] assert record.exc_info is not None assert record.exc_info[0] is ZeroDivisionError - @inline_callbacks_test - def test_start_bug_yielding(self): - with LogCapture("scrapy", level=logging.ERROR) as log: + @coroutine_test + async def test_start_bug_yielding( + self, caplog: pytest.LogCaptureFixture, mockserver: MockServer + ) -> None: + with caplog.at_level(logging.ERROR): crawler = get_crawler(BrokenStartSpider) - yield crawler.crawl(fail_yielding=1, mockserver=self.mockserver) + await crawler.crawl_async(fail_yielding=1, mockserver=mockserver) - assert len(log.records) == 1 - record = log.records[0] + assert len(caplog.records) == 1 + record = caplog.records[0] assert record.exc_info is not None assert record.exc_info[0] is ZeroDivisionError - @inline_callbacks_test - def test_start_items(self): + @coroutine_test + async def test_start_items( + self, caplog: pytest.LogCaptureFixture, mockserver: MockServer + ) -> None: items = [] def _on_item_scraped(item): items.append(item) - with LogCapture("scrapy", level=logging.ERROR) as log: + with caplog.at_level(logging.ERROR): crawler = get_crawler(StartItemSpider) crawler.signals.connect(_on_item_scraped, signals.item_scraped) - yield crawler.crawl(mockserver=self.mockserver) + await crawler.crawl_async(mockserver=mockserver) - assert len(log.records) == 0 + assert len(caplog.records) == 0 assert items == [{"name": "test item"}] - @inline_callbacks_test - def test_start_unsupported_output(self): + @coroutine_test + async def test_start_unsupported_output( + self, caplog: pytest.LogCaptureFixture, mockserver: MockServer + ) -> None: """Anything that is not a request is assumed to be an item, avoiding a potentially expensive call to itemadapter.is_item(), and letting instead things fail when ItemAdapter is actually used on the @@ -226,35 +233,39 @@ class TestCrawl: def _on_item_scraped(item): items.append(item) - with LogCapture("scrapy", level=logging.ERROR) as log: + with caplog.at_level(logging.ERROR): crawler = get_crawler(StartGoodAndBadOutput) crawler.signals.connect(_on_item_scraped, signals.item_scraped) - yield crawler.crawl(mockserver=self.mockserver) + await crawler.crawl_async(mockserver=mockserver) - assert len(log.records) == 0 + assert len(caplog.records) == 0 assert len(items) == 3 assert not any(isinstance(item, Request) for item in items) - @inline_callbacks_test - def test_start_dupes(self): + @coroutine_test + async def test_start_dupes(self, mockserver: MockServer) -> None: settings = {"CONCURRENT_REQUESTS": 1} crawler = get_crawler(DuplicateStartSpider, settings) - yield crawler.crawl( - dont_filter=True, distinct_urls=2, dupe_factor=3, mockserver=self.mockserver + await crawler.crawl_async( + dont_filter=True, distinct_urls=2, dupe_factor=3, mockserver=mockserver ) + assert isinstance(crawler.spider, DuplicateStartSpider) assert crawler.spider.visited == 6 crawler = get_crawler(DuplicateStartSpider, settings) - yield crawler.crawl( + await crawler.crawl_async( dont_filter=False, distinct_urls=3, dupe_factor=4, - mockserver=self.mockserver, + mockserver=mockserver, ) + assert isinstance(crawler.spider, DuplicateStartSpider) assert crawler.spider.visited == 3 - @inline_callbacks_test - def test_unbounded_response(self): + @coroutine_test + async def test_unbounded_response( + self, caplog: pytest.LogCaptureFixture, mockserver: MockServer + ) -> None: # Completeness of responses without Content-Length or Transfer-Encoding # can not be determined, we treat them as valid but flagged as "partial" query = urlencode( @@ -279,41 +290,45 @@ with multiples lines } ) crawler = get_crawler(SimpleSpider) - with LogCapture() as log: - yield crawler.crawl( - self.mockserver.url(f"/raw?{query}"), mockserver=self.mockserver + with caplog.at_level(logging.INFO): + await crawler.crawl_async( + mockserver.url(f"/raw?{query}"), mockserver=mockserver ) - assert str(log).count("Got response 200") == 1 + assert caplog.text.count("Got response 200") == 1 - @inline_callbacks_test - def test_retry_conn_lost(self): + @coroutine_test + async def test_retry_conn_lost( + self, caplog: pytest.LogCaptureFixture, mockserver: MockServer + ) -> None: # connection lost after receiving data crawler = get_crawler(SimpleSpider) - with LogCapture() as log: - yield crawler.crawl( - self.mockserver.url("/drop?abort=0"), mockserver=self.mockserver + with caplog.at_level(logging.DEBUG): + await crawler.crawl_async( + mockserver.url("/drop?abort=0"), mockserver=mockserver ) - self._assert_retried(log) + self._assert_retried(caplog.text) - @inline_callbacks_test - def test_retry_conn_aborted(self): + @coroutine_test + async def test_retry_conn_aborted( + self, caplog: pytest.LogCaptureFixture, mockserver: MockServer + ) -> None: # connection lost before receiving data crawler = get_crawler(SimpleSpider) - with LogCapture() as log: - yield crawler.crawl( - self.mockserver.url("/drop?abort=1"), mockserver=self.mockserver + with caplog.at_level(logging.DEBUG): + await crawler.crawl_async( + mockserver.url("/drop?abort=1"), mockserver=mockserver ) - self._assert_retried(log) + self._assert_retried(caplog.text) @staticmethod - def _assert_retried(log: LogCapture | str) -> None: + def _assert_retried(log: str) -> None: assert str(log).count("Retrying") == 2 assert str(log).count("Gave up retrying") == 1 - @inline_callbacks_test - def test_referer_header(self): + @coroutine_test + async def test_referer_header(self, mockserver: MockServer) -> None: """Referer header is set by RefererMiddleware unless it is already set""" - req0 = Request(self.mockserver.url("/echo?headers=1&body=0"), dont_filter=1) + req0 = Request(mockserver.url("/echo?headers=1&body=0"), dont_filter=True) req1 = req0.replace() req2 = req0.replace(headers={"Referer": None}) req3 = req0.replace(headers={"Referer": "http://example.com"}) @@ -321,7 +336,8 @@ with multiples lines req1.meta["next"] = req2 req2.meta["next"] = req3 crawler = get_crawler(SingleRequestSpider) - yield crawler.crawl(seed=req0, mockserver=self.mockserver) + await crawler.crawl_async(seed=req0, mockserver=mockserver) + assert isinstance(crawler.spider, SingleRequestSpider) # basic asserts in case of weird communication errors assert "responses" in crawler.spider.meta assert "failures" not in crawler.spider.meta @@ -338,33 +354,35 @@ with multiples lines echo3 = json.loads(to_unicode(crawler.spider.meta["responses"][3].body)) assert echo3["headers"].get("Referer") == ["http://example.com"] - @inline_callbacks_test - def test_engine_status(self): + @coroutine_test + async def test_engine_status(self, mockserver: MockServer) -> None: est = [] def cb(response): est.append(get_engine_status(crawler.engine)) crawler = get_crawler(SingleRequestSpider) - yield crawler.crawl( - seed=self.mockserver.url("/"), callback_func=cb, mockserver=self.mockserver + await crawler.crawl_async( + seed=mockserver.url("/"), callback_func=cb, mockserver=mockserver ) + assert isinstance(crawler.spider, SingleRequestSpider) assert len(est) == 1, est s = dict(est[0]) assert s["engine.spider.name"] == crawler.spider.name assert s["len(engine.scraper.slot.active)"] == 1 - @inline_callbacks_test - def test_format_engine_status(self): + @coroutine_test + async def test_format_engine_status(self, mockserver: MockServer) -> None: est = [] def cb(response): est.append(format_engine_status(crawler.engine)) crawler = get_crawler(SingleRequestSpider) - yield crawler.crawl( - seed=self.mockserver.url("/"), callback_func=cb, mockserver=self.mockserver + await crawler.crawl_async( + seed=mockserver.url("/"), callback_func=cb, mockserver=mockserver ) + assert isinstance(crawler.spider, SingleRequestSpider) assert len(est) == 1, est est = est[0].split("\n")[2:-2] # remove header & footer # convert to dict @@ -377,8 +395,10 @@ with multiples lines assert s["engine.spider.name"] == crawler.spider.name assert s["len(engine.scraper.slot.active)"] == "1" - @inline_callbacks_test - def test_open_spider_error_on_faulty_pipeline(self): + @coroutine_test + async def test_open_spider_error_on_faulty_pipeline( + self, mockserver: MockServer + ) -> None: settings = { "ITEM_PIPELINES": { "tests.pipelines.ZeroDivisionErrorPipeline": 300, @@ -386,25 +406,48 @@ with multiples lines } crawler = get_crawler(SimpleSpider, settings) with pytest.raises(ZeroDivisionError): - yield crawler.crawl( - self.mockserver.url("/status?n=200"), mockserver=self.mockserver + await crawler.crawl_async( + mockserver.url("/status?n=200"), mockserver=mockserver ) assert not crawler.crawling - @inline_callbacks_test - def test_crawlerrunner_accepts_crawler(self): - crawler = get_crawler(SimpleSpider) - runner = CrawlerRunner() - with LogCapture() as log: - yield runner.crawl( - crawler, - self.mockserver.url("/status?n=200"), - mockserver=self.mockserver, + @coroutine_test + async def test_open_spider_error_on_faulty_pipeline_crawl( + self, mockserver: MockServer + ) -> None: + # cover the except block in Crawler.crawl() + settings = { + "ITEM_PIPELINES": { + "tests.pipelines.ZeroDivisionErrorPipeline": 300, + } + } + crawler = get_crawler(SimpleSpider, settings) + with pytest.raises(ZeroDivisionError): + await maybe_deferred_to_future( + crawler.crawl(mockserver.url("/status?n=200"), mockserver=mockserver) ) - assert "Got response 200" in str(log) + assert not crawler.crawling @coroutine_test - async def test_crawl_multiple(self, caplog: pytest.LogCaptureFixture) -> None: + async def test_crawlerrunner_accepts_crawler( + self, caplog: pytest.LogCaptureFixture, mockserver: MockServer + ) -> None: + crawler = get_crawler(SimpleSpider) + runner = CrawlerRunner() + with caplog.at_level(logging.DEBUG): + await maybe_deferred_to_future( + runner.crawl( + crawler, + mockserver.url("/status?n=200"), + mockserver=mockserver, + ) + ) + assert "Got response 200" in caplog.text + + @coroutine_test + async def test_crawl_multiple( + self, caplog: pytest.LogCaptureFixture, mockserver: MockServer + ) -> None: settings_dict = get_reactor_settings() runner_cls = ( CrawlerRunner @@ -414,13 +457,13 @@ with multiples lines runner = runner_cls(settings_dict) runner.crawl( SimpleSpider, - self.mockserver.url("/status?n=200"), - mockserver=self.mockserver, + mockserver.url("/status?n=200"), + mockserver=mockserver, ) runner.crawl( SimpleSpider, - self.mockserver.url("/status?n=503"), - mockserver=self.mockserver, + mockserver.url("/status?n=503"), + mockserver=mockserver, ) with caplog.at_level(logging.DEBUG): @@ -432,25 +475,15 @@ with multiples lines @coroutine_test async def test_unknown_url_scheme(self, caplog: pytest.LogCaptureFixture) -> None: crawler = get_crawler(SimpleSpider) - await maybe_deferred_to_future(crawler.crawl("foo://bar")) + await crawler.crawl_async("foo://bar") assert "NotSupported: Unsupported URL scheme 'foo'" in caplog.text class TestCrawlSpider: - mockserver: MockServer - - @classmethod - def setup_class(cls): - cls.mockserver = MockServer() - cls.mockserver.__enter__() - - @classmethod - def teardown_class(cls): - cls.mockserver.__exit__(None, None, None) - + @staticmethod async def _run_spider( - self, spider_cls: type[Spider] - ) -> tuple[LogCapture, list[Any], StatsCollector]: + spider_cls: type[Spider], mockserver: MockServer + ) -> tuple[list[Any], StatsCollector]: items = [] def _on_item_scraped(item): @@ -458,103 +491,119 @@ class TestCrawlSpider: crawler = get_crawler(spider_cls) crawler.signals.connect(_on_item_scraped, signals.item_scraped) - with LogCapture() as log: - await maybe_deferred_to_future( - crawler.crawl( - self.mockserver.url("/status?n=200"), mockserver=self.mockserver - ) - ) + await crawler.crawl_async( + mockserver.url("/status?n=200"), mockserver=mockserver + ) assert crawler.stats - return log, items, crawler.stats + return items, crawler.stats - @inline_callbacks_test - def test_crawlspider_with_parse(self): + @coroutine_test + async def test_crawlspider_with_parse( + self, caplog: pytest.LogCaptureFixture, mockserver: MockServer + ) -> None: crawler = get_crawler(CrawlSpiderWithParseMethod) - with LogCapture() as log: - yield crawler.crawl(mockserver=self.mockserver) + with caplog.at_level(logging.INFO): + await crawler.crawl_async(mockserver=mockserver) - assert "[parse] status 200 (foo: None)" in str(log) - assert "[parse] status 201 (foo: None)" in str(log) - assert "[parse] status 202 (foo: bar)" in str(log) + assert "[parse] status 200 (foo: None)" in caplog.text + assert "[parse] status 201 (foo: None)" in caplog.text + assert "[parse] status 202 (foo: bar)" in caplog.text - @inline_callbacks_test - def test_crawlspider_with_async_callback(self): + @coroutine_test + async def test_crawlspider_with_async_callback( + self, caplog: pytest.LogCaptureFixture, mockserver: MockServer + ) -> None: crawler = get_crawler(CrawlSpiderWithAsyncCallback) - with LogCapture() as log: - yield crawler.crawl(mockserver=self.mockserver) + with caplog.at_level(logging.INFO): + await crawler.crawl_async(mockserver=mockserver) - assert "[parse_async] status 200 (foo: None)" in str(log) - assert "[parse_async] status 201 (foo: None)" in str(log) - assert "[parse_async] status 202 (foo: bar)" in str(log) + assert "[parse_async] status 200 (foo: None)" in caplog.text + assert "[parse_async] status 201 (foo: None)" in caplog.text + assert "[parse_async] status 202 (foo: bar)" in caplog.text - @inline_callbacks_test - def test_crawlspider_with_async_generator_callback(self): + @coroutine_test + async def test_crawlspider_with_async_generator_callback( + self, caplog: pytest.LogCaptureFixture, mockserver: MockServer + ) -> None: crawler = get_crawler(CrawlSpiderWithAsyncGeneratorCallback) - with LogCapture() as log: - yield crawler.crawl(mockserver=self.mockserver) + with caplog.at_level(logging.INFO): + await crawler.crawl_async(mockserver=mockserver) - assert "[parse_async_gen] status 200 (foo: None)" in str(log) - assert "[parse_async_gen] status 201 (foo: None)" in str(log) - assert "[parse_async_gen] status 202 (foo: bar)" in str(log) + assert "[parse_async_gen] status 200 (foo: None)" in caplog.text + assert "[parse_async_gen] status 201 (foo: None)" in caplog.text + assert "[parse_async_gen] status 202 (foo: bar)" in caplog.text - @inline_callbacks_test - def test_crawlspider_with_errback(self): + @coroutine_test + async def test_crawlspider_with_errback( + self, caplog: pytest.LogCaptureFixture, mockserver: MockServer + ) -> None: crawler = get_crawler(CrawlSpiderWithErrback) - with LogCapture() as log: - yield crawler.crawl(mockserver=self.mockserver) + with caplog.at_level(logging.INFO): + await crawler.crawl_async(mockserver=mockserver) - assert "[parse] status 200 (foo: None)" in str(log) - assert "[parse] status 201 (foo: None)" in str(log) - assert "[parse] status 202 (foo: bar)" in str(log) - assert "[errback] status 404" in str(log) - assert "[errback] status 500" in str(log) - assert "[errback] status 501" in str(log) + assert "[parse] status 200 (foo: None)" in caplog.text + assert "[parse] status 201 (foo: None)" in caplog.text + assert "[parse] status 202 (foo: bar)" in caplog.text + assert "[errback] status 404" in caplog.text + assert "[errback] status 500" in caplog.text + assert "[errback] status 501" in caplog.text - @inline_callbacks_test - def test_crawlspider_process_request_cb_kwargs(self): + @coroutine_test + async def test_crawlspider_process_request_cb_kwargs( + self, caplog: pytest.LogCaptureFixture, mockserver: MockServer + ) -> None: crawler = get_crawler(CrawlSpiderWithProcessRequestCallbackKeywordArguments) - with LogCapture() as log: - yield crawler.crawl(mockserver=self.mockserver) + with caplog.at_level(logging.INFO): + await crawler.crawl_async(mockserver=mockserver) - assert "[parse] status 200 (foo: process_request)" in str(log) - assert "[parse] status 201 (foo: process_request)" in str(log) - assert "[parse] status 202 (foo: bar)" in str(log) + assert "[parse] status 200 (foo: process_request)" in caplog.text + assert "[parse] status 201 (foo: process_request)" in caplog.text + assert "[parse] status 202 (foo: bar)" in caplog.text - @inline_callbacks_test - def test_async_def_parse(self): + @coroutine_test + async def test_async_def_parse( + self, caplog: pytest.LogCaptureFixture, mockserver: MockServer + ) -> None: crawler = get_crawler(AsyncDefSpider) - with LogCapture() as log: - yield crawler.crawl( - self.mockserver.url("/status?n=200"), mockserver=self.mockserver + with caplog.at_level(logging.INFO): + await crawler.crawl_async( + mockserver.url("/status?n=200"), mockserver=mockserver ) - assert "Got response 200" in str(log) + assert "Got response 200" in caplog.text @pytest.mark.only_asyncio - @inline_callbacks_test - def test_async_def_asyncio_parse(self): + @coroutine_test + async def test_async_def_asyncio_parse( + self, caplog: pytest.LogCaptureFixture, mockserver: MockServer + ) -> None: crawler = get_crawler( AsyncDefAsyncioSpider, { "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor" }, ) - with LogCapture() as log: - yield crawler.crawl( - self.mockserver.url("/status?n=200"), mockserver=self.mockserver + with caplog.at_level(logging.INFO): + await crawler.crawl_async( + mockserver.url("/status?n=200"), mockserver=mockserver ) - assert "Got response 200" in str(log) + assert "Got response 200" in caplog.text @pytest.mark.only_asyncio @coroutine_test - async def test_async_def_asyncio_parse_items_list(self): - log, items, _ = await self._run_spider(AsyncDefAsyncioReturnSpider) - assert "Got response 200" in str(log) + async def test_async_def_asyncio_parse_items_list( + self, caplog: pytest.LogCaptureFixture, mockserver: MockServer + ) -> None: + with caplog.at_level(logging.INFO): + items, _ = await self._run_spider(AsyncDefAsyncioReturnSpider, mockserver) + assert "Got response 200" in caplog.text assert {"id": 1} in items assert {"id": 2} in items @pytest.mark.only_asyncio - @inline_callbacks_test - def test_async_def_asyncio_parse_items_single_element(self): + @coroutine_test + async def test_async_def_asyncio_parse_items_single_element( + self, caplog: pytest.LogCaptureFixture, mockserver: MockServer + ) -> None: items = [] def _on_item_scraped(item): @@ -562,26 +611,34 @@ class TestCrawlSpider: crawler = get_crawler(AsyncDefAsyncioReturnSingleElementSpider) crawler.signals.connect(_on_item_scraped, signals.item_scraped) - with LogCapture() as log: - yield crawler.crawl( - self.mockserver.url("/status?n=200"), mockserver=self.mockserver + with caplog.at_level(logging.INFO): + await crawler.crawl_async( + mockserver.url("/status?n=200"), mockserver=mockserver ) - assert "Got response 200" in str(log) + assert "Got response 200" in caplog.text assert {"foo": 42} in items @pytest.mark.only_asyncio @coroutine_test - async def test_async_def_asyncgen_parse(self): - log, _, stats = await self._run_spider(AsyncDefAsyncioGenSpider) - assert "Got response 200" in str(log) + async def test_async_def_asyncgen_parse( + self, caplog: pytest.LogCaptureFixture, mockserver: MockServer + ) -> None: + with caplog.at_level(logging.INFO): + _, stats = await self._run_spider(AsyncDefAsyncioGenSpider, mockserver) + assert "Got response 200" in caplog.text itemcount = stats.get_value("item_scraped_count") assert itemcount == 1 @pytest.mark.only_asyncio @coroutine_test - async def test_async_def_asyncgen_parse_loop(self): - log, items, stats = await self._run_spider(AsyncDefAsyncioGenLoopSpider) - assert "Got response 200" in str(log) + async def test_async_def_asyncgen_parse_loop( + self, caplog: pytest.LogCaptureFixture, mockserver: MockServer + ) -> None: + with caplog.at_level(logging.INFO): + items, stats = await self._run_spider( + AsyncDefAsyncioGenLoopSpider, mockserver + ) + assert "Got response 200" in caplog.text itemcount = stats.get_value("item_scraped_count") assert itemcount == 10 for i in range(10): @@ -589,11 +646,15 @@ class TestCrawlSpider: @pytest.mark.only_asyncio @coroutine_test - async def test_async_def_asyncgen_parse_exc(self): - log, items, stats = await self._run_spider(AsyncDefAsyncioGenExcSpider) - log = str(log) - assert "Spider error processing" in log - assert "ValueError" in log + async def test_async_def_asyncgen_parse_exc( + self, caplog: pytest.LogCaptureFixture, mockserver: MockServer + ) -> None: + with caplog.at_level(logging.INFO): + items, stats = await self._run_spider( + AsyncDefAsyncioGenExcSpider, mockserver + ) + assert "Spider error processing" in caplog.text + assert "ValueError" in caplog.text itemcount = stats.get_value("item_scraped_count") assert itemcount == 7 for i in range(7): @@ -601,8 +662,12 @@ class TestCrawlSpider: @pytest.mark.only_asyncio @coroutine_test - async def test_async_def_asyncgen_parse_complex(self): - _, items, stats = await self._run_spider(AsyncDefAsyncioGenComplexSpider) + async def test_async_def_asyncgen_parse_complex( + self, mockserver: MockServer + ) -> None: + items, stats = await self._run_spider( + AsyncDefAsyncioGenComplexSpider, mockserver + ) itemcount = stats.get_value("item_scraped_count") assert itemcount == 156 # some random items @@ -613,33 +678,41 @@ class TestCrawlSpider: @pytest.mark.only_asyncio @coroutine_test - async def test_async_def_asyncio_parse_reqs_list(self): - log, *_ = await self._run_spider(AsyncDefAsyncioReqsReturnSpider) + async def test_async_def_asyncio_parse_reqs_list( + self, caplog: pytest.LogCaptureFixture, mockserver: MockServer + ) -> None: + with caplog.at_level(logging.INFO): + await self._run_spider(AsyncDefAsyncioReqsReturnSpider, mockserver) for req_id in range(3): - assert f"Got response 200, req_id {req_id}" in str(log) + assert f"Got response 200, req_id {req_id}" in caplog.text @pytest.mark.only_not_asyncio @coroutine_test - async def test_async_def_deferred_direct(self): - _, items, _ = await self._run_spider(AsyncDefDeferredDirectSpider) + async def test_async_def_deferred_direct(self, mockserver: MockServer) -> None: + items, _ = await self._run_spider(AsyncDefDeferredDirectSpider, mockserver) assert items == [{"code": 200}] @pytest.mark.only_asyncio @coroutine_test - async def test_async_def_deferred_wrapped(self): - _, items, _ = await self._run_spider(AsyncDefDeferredWrappedSpider) + async def test_async_def_deferred_wrapped(self, mockserver: MockServer) -> None: + items, _ = await self._run_spider(AsyncDefDeferredWrappedSpider, mockserver) assert items == [{"code": 200}] @coroutine_test - async def test_async_def_deferred_maybe_wrapped(self): - _, items, _ = await self._run_spider(AsyncDefDeferredMaybeWrappedSpider) + async def test_async_def_deferred_maybe_wrapped( + self, mockserver: MockServer + ) -> None: + items, _ = await self._run_spider( + AsyncDefDeferredMaybeWrappedSpider, mockserver + ) assert items == [{"code": 200}] - @inline_callbacks_test - def test_response_ssl_certificate_none(self): + @coroutine_test + async def test_response_ssl_certificate_none(self, mockserver: MockServer) -> None: crawler = get_crawler(SingleRequestSpider) - url = self.mockserver.url("/echo?body=test", is_secure=False) - yield crawler.crawl(seed=url, mockserver=self.mockserver) + url = mockserver.url("/echo?body=test", is_secure=False) + await crawler.crawl_async(seed=url, mockserver=mockserver) + assert isinstance(crawler.spider, SingleRequestSpider) assert crawler.spider.meta["responses"][0].certificate is None @pytest.mark.parametrize( @@ -699,10 +772,13 @@ class TestCrawlSpider: assert isinstance(ip_address, IPv4Address) assert str(ip_address) == gethostbyname(expected_netloc) - @inline_callbacks_test - def test_bytes_received_stop_download_callback(self): + @coroutine_test + async def test_bytes_received_stop_download_callback( + self, mockserver: MockServer + ) -> None: crawler = get_crawler(BytesReceivedCallbackSpider) - yield crawler.crawl(mockserver=self.mockserver) + await crawler.crawl_async(mockserver=mockserver) + assert isinstance(crawler.spider, BytesReceivedCallbackSpider) assert crawler.spider.meta.get("failure") is None assert isinstance(crawler.spider.meta["response"], Response) assert crawler.spider.meta["response"].body == crawler.spider.meta.get( @@ -713,10 +789,13 @@ class TestCrawlSpider: < crawler.spider.full_response_length ) - @inline_callbacks_test - def test_bytes_received_stop_download_errback(self): + @coroutine_test + async def test_bytes_received_stop_download_errback( + self, mockserver: MockServer + ) -> None: crawler = get_crawler(BytesReceivedErrbackSpider) - yield crawler.crawl(mockserver=self.mockserver) + await crawler.crawl_async(mockserver=mockserver) + assert isinstance(crawler.spider, BytesReceivedErrbackSpider) assert crawler.spider.meta.get("response") is None assert isinstance(crawler.spider.meta["failure"], Failure) assert isinstance(crawler.spider.meta["failure"].value, StopDownload) @@ -729,20 +808,26 @@ class TestCrawlSpider: < crawler.spider.full_response_length ) - @inline_callbacks_test - def test_headers_received_stop_download_callback(self): + @coroutine_test + async def test_headers_received_stop_download_callback( + self, mockserver: MockServer + ) -> None: crawler = get_crawler(HeadersReceivedCallbackSpider) - yield crawler.crawl(mockserver=self.mockserver) + await crawler.crawl_async(mockserver=mockserver) + assert isinstance(crawler.spider, HeadersReceivedCallbackSpider) assert crawler.spider.meta.get("failure") is None assert isinstance(crawler.spider.meta["response"], Response) assert crawler.spider.meta["response"].headers == crawler.spider.meta.get( "headers_received" ) - @inline_callbacks_test - def test_headers_received_stop_download_errback(self): + @coroutine_test + async def test_headers_received_stop_download_errback( + self, mockserver: MockServer + ) -> None: crawler = get_crawler(HeadersReceivedErrbackSpider) - yield crawler.crawl(mockserver=self.mockserver) + await crawler.crawl_async(mockserver=mockserver) + assert isinstance(crawler.spider, HeadersReceivedErrbackSpider) assert crawler.spider.meta.get("response") is None assert isinstance(crawler.spider.meta["failure"], Failure) assert isinstance(crawler.spider.meta["failure"].value, StopDownload) @@ -751,8 +836,10 @@ class TestCrawlSpider: "failure" ].value.response.headers == crawler.spider.meta.get("headers_received") - @inline_callbacks_test - def test_spider_callback_deferred_deprecated(self): + @coroutine_test + async def test_spider_callback_deferred_deprecated( + self, mockserver: MockServer + ) -> None: def cb(response: Response) -> Any: return succeed(None) @@ -761,10 +848,12 @@ class TestCrawlSpider: ScrapyDeprecationWarning, match="Returning Deferreds from spider callbacks is deprecated", ): - yield crawler.crawl(seed=self.mockserver.url("/"), callback_func=cb) + await crawler.crawl_async(seed=mockserver.url("/"), callback_func=cb) - @inline_callbacks_test - def test_spider_errback(self): + @coroutine_test + async def test_spider_errback( + self, caplog: pytest.LogCaptureFixture, mockserver: MockServer + ) -> None: failures = [] def eb(failure: Failure) -> Failure: @@ -772,72 +861,82 @@ class TestCrawlSpider: return failure crawler = get_crawler(SingleRequestSpider) - with LogCapture() as log: - yield crawler.crawl( - seed=self.mockserver.url("/status?n=400"), errback_func=eb + with caplog.at_level(logging.INFO): + await crawler.crawl_async( + seed=mockserver.url("/status?n=400"), errback_func=eb ) assert len(failures) == 1 - assert "HTTP status code is not handled or not allowed" in str(log) - assert "Spider error processing" not in str(log) + assert "HTTP status code is not handled or not allowed" in caplog.text + assert "Spider error processing" not in caplog.text - @inline_callbacks_test - def test_spider_errback_silence(self): + @coroutine_test + async def test_spider_errback_silence( + self, caplog: pytest.LogCaptureFixture, mockserver: MockServer + ) -> None: failures = [] def eb(failure: Failure) -> None: failures.append(failure) crawler = get_crawler(SingleRequestSpider) - with LogCapture() as log: - yield crawler.crawl( - seed=self.mockserver.url("/status?n=400"), errback_func=eb + with caplog.at_level(logging.INFO): + await crawler.crawl_async( + seed=mockserver.url("/status?n=400"), errback_func=eb ) assert len(failures) == 1 - assert "HTTP status code is not handled or not allowed" not in str(log) - assert "Spider error processing" not in str(log) + assert "HTTP status code is not handled or not allowed" not in caplog.text + assert "Spider error processing" not in caplog.text - @inline_callbacks_test - def test_spider_errback_exception(self): + @coroutine_test + async def test_spider_errback_exception( + self, caplog: pytest.LogCaptureFixture, mockserver: MockServer + ) -> None: def eb(failure: Failure) -> None: raise ValueError("foo") crawler = get_crawler(SingleRequestSpider) - with LogCapture() as log: - yield crawler.crawl( - seed=self.mockserver.url("/status?n=400"), errback_func=eb + with caplog.at_level(logging.INFO): + await crawler.crawl_async( + seed=mockserver.url("/status?n=400"), errback_func=eb ) - assert "Spider error processing" in str(log) + assert "Spider error processing" in caplog.text - @inline_callbacks_test - def test_spider_errback_item(self): + @coroutine_test + async def test_spider_errback_item( + self, caplog: pytest.LogCaptureFixture, mockserver: MockServer + ) -> None: def eb(failure: Failure) -> Any: return {"foo": "bar"} crawler = get_crawler(SingleRequestSpider) - with LogCapture() as log: - yield crawler.crawl( - seed=self.mockserver.url("/status?n=400"), errback_func=eb + with caplog.at_level(logging.INFO): + await crawler.crawl_async( + seed=mockserver.url("/status?n=400"), errback_func=eb ) - assert "HTTP status code is not handled or not allowed" not in str(log) - assert "Spider error processing" not in str(log) - assert "'item_scraped_count': 1" in str(log) + assert "HTTP status code is not handled or not allowed" not in caplog.text + assert "Spider error processing" not in caplog.text + assert "'item_scraped_count': 1" in caplog.text - @inline_callbacks_test - def test_spider_errback_request(self): + @coroutine_test + async def test_spider_errback_request( + self, caplog: pytest.LogCaptureFixture, mockserver: MockServer + ) -> None: def eb(failure: Failure) -> Request: - return Request(self.mockserver.url("/")) + return Request(mockserver.url("/")) crawler = get_crawler(SingleRequestSpider) - with LogCapture() as log: - yield crawler.crawl( - seed=self.mockserver.url("/status?n=400"), errback_func=eb + with caplog.at_level(logging.DEBUG): + await crawler.crawl_async( + seed=mockserver.url("/status?n=400"), errback_func=eb ) - assert "HTTP status code is not handled or not allowed" not in str(log) - assert "Spider error processing" not in str(log) - assert "Crawled (200)" in str(log) + assert "HTTP status code is not handled or not allowed" not in caplog.text + assert "Spider error processing" not in caplog.text + assert "Crawled (200)" in caplog.text - @inline_callbacks_test - def test_spider_errback_downloader_error(self): + @coroutine_test + async def test_spider_errback_downloader_error( + self, caplog: pytest.LogCaptureFixture, mockserver: MockServer + ) -> None: failures = [] def eb(failure: Failure) -> Failure: @@ -845,57 +944,65 @@ class TestCrawlSpider: return failure crawler = get_crawler(SingleRequestSpider) - with LogCapture() as log: - yield crawler.crawl( - seed=self.mockserver.url("/drop?abort=1"), errback_func=eb + with caplog.at_level(logging.INFO): + await crawler.crawl_async( + seed=mockserver.url("/drop?abort=1"), errback_func=eb ) assert len(failures) == 1 - assert "Error downloading" in str(log) - assert "Spider error processing" not in str(log) + assert "Error downloading" in caplog.text + assert "Spider error processing" not in caplog.text - @inline_callbacks_test - def test_spider_errback_downloader_error_exception(self): + @coroutine_test + async def test_spider_errback_downloader_error_exception( + self, caplog: pytest.LogCaptureFixture, mockserver: MockServer + ) -> None: def eb(failure: Failure) -> None: raise ValueError("foo") crawler = get_crawler(SingleRequestSpider) - with LogCapture() as log: - yield crawler.crawl( - seed=self.mockserver.url("/drop?abort=1"), errback_func=eb + with caplog.at_level(logging.INFO): + await crawler.crawl_async( + seed=mockserver.url("/drop?abort=1"), errback_func=eb ) - assert "Error downloading" in str(log) - assert "Spider error processing" in str(log) + assert "Error downloading" in caplog.text + assert "Spider error processing" in caplog.text - @inline_callbacks_test - def test_spider_errback_downloader_error_item(self): + @coroutine_test + async def test_spider_errback_downloader_error_item( + self, caplog: pytest.LogCaptureFixture, mockserver: MockServer + ) -> None: def eb(failure: Failure) -> Any: return {"foo": "bar"} crawler = get_crawler(SingleRequestSpider) - with LogCapture() as log: - yield crawler.crawl( - seed=self.mockserver.url("/drop?abort=1"), errback_func=eb + with caplog.at_level(logging.INFO): + await crawler.crawl_async( + seed=mockserver.url("/drop?abort=1"), errback_func=eb ) - assert "HTTP status code is not handled or not allowed" not in str(log) - assert "Spider error processing" not in str(log) - assert "'item_scraped_count': 1" in str(log) + assert "HTTP status code is not handled or not allowed" not in caplog.text + assert "Spider error processing" not in caplog.text + assert "'item_scraped_count': 1" in caplog.text - @inline_callbacks_test - def test_spider_errback_downloader_error_request(self): + @coroutine_test + async def test_spider_errback_downloader_error_request( + self, caplog: pytest.LogCaptureFixture, mockserver: MockServer + ) -> None: def eb(failure: Failure) -> Request: - return Request(self.mockserver.url("/")) + return Request(mockserver.url("/")) crawler = get_crawler(SingleRequestSpider) - with LogCapture() as log: - yield crawler.crawl( - seed=self.mockserver.url("/drop?abort=1"), errback_func=eb + with caplog.at_level(logging.DEBUG): + await crawler.crawl_async( + seed=mockserver.url("/drop?abort=1"), errback_func=eb ) - assert "HTTP status code is not handled or not allowed" not in str(log) - assert "Spider error processing" not in str(log) - assert "Crawled (200)" in str(log) + assert "HTTP status code is not handled or not allowed" not in caplog.text + assert "Spider error processing" not in caplog.text + assert "Crawled (200)" in caplog.text - @inline_callbacks_test - def test_spider_errback_deferred_deprecated(self): + @coroutine_test + async def test_spider_errback_deferred_deprecated( + self, mockserver: MockServer + ) -> None: def eb(failure: Failure) -> Any: return succeed(None) @@ -904,28 +1011,32 @@ class TestCrawlSpider: ScrapyDeprecationWarning, match="Returning Deferreds from spider errbacks is deprecated", ): - yield crawler.crawl( - seed=self.mockserver.url("/status?n=400"), errback_func=eb + await crawler.crawl_async( + seed=mockserver.url("/status?n=400"), errback_func=eb ) - @inline_callbacks_test - def test_raise_closespider(self): + @coroutine_test + async def test_raise_closespider( + self, caplog: pytest.LogCaptureFixture, mockserver: MockServer + ) -> None: def cb(response): raise CloseSpider crawler = get_crawler(SingleRequestSpider) - with LogCapture() as log: - yield crawler.crawl(seed=self.mockserver.url("/"), callback_func=cb) - assert "Closing spider (cancelled)" in str(log) - assert "Spider error processing" not in str(log) + with caplog.at_level(logging.INFO): + await crawler.crawl_async(seed=mockserver.url("/"), callback_func=cb) + assert "Closing spider (cancelled)" in caplog.text + assert "Spider error processing" not in caplog.text - @inline_callbacks_test - def test_raise_closespider_reason(self): + @coroutine_test + async def test_raise_closespider_reason( + self, caplog: pytest.LogCaptureFixture, mockserver: MockServer + ) -> None: def cb(response): raise CloseSpider("my_reason") crawler = get_crawler(SingleRequestSpider) - with LogCapture() as log: - yield crawler.crawl(seed=self.mockserver.url("/"), callback_func=cb) - assert "Closing spider (my_reason)" in str(log) - assert "Spider error processing" not in str(log) + with caplog.at_level(logging.INFO): + await crawler.crawl_async(seed=mockserver.url("/"), callback_func=cb) + assert "Closing spider (my_reason)" in caplog.text + assert "Spider error processing" not in caplog.text diff --git a/tests/test_crawler.py b/tests/test_crawler.py index cbcb7e274..853d6cfaa 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -1,13 +1,13 @@ +from __future__ import annotations + import asyncio import logging import re import warnings -from collections.abc import Generator from pathlib import Path -from typing import Any, cast +from typing import Any, ClassVar import pytest -from twisted.internet.defer import Deferred from zope.interface.exceptions import MultipleInvalid import scrapy @@ -22,8 +22,8 @@ from scrapy.crawler import ( ) from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.extensions.throttle import AutoThrottle -from scrapy.settings import Settings, default_settings -from scrapy.utils.defer import deferred_from_coro, maybe_deferred_to_future +from scrapy.settings import Settings, _SettingsKey, default_settings +from scrapy.utils.defer import ensure_awaitable, maybe_deferred_to_future from scrapy.utils.log import ( _uninstall_scrapy_root_handler, configure_logging, @@ -31,12 +31,14 @@ from scrapy.utils.log import ( ) from scrapy.utils.spider import DefaultSpider from scrapy.utils.test import get_crawler, get_reactor_settings -from tests.utils.decorators import coroutine_test, inline_callbacks_test +from tests.utils.decorators import coroutine_test BASE_SETTINGS: dict[str, Any] = {} -def get_raw_crawler(spidercls=None, settings_dict=None): +def get_raw_crawler( + spidercls: type[Spider] | None = None, settings_dict: dict[str, Any] | None = None +) -> Crawler: """get_crawler alternative that only calls the __init__ method of the crawler.""" settings = Settings() @@ -46,14 +48,18 @@ def get_raw_crawler(spidercls=None, settings_dict=None): class TestBaseCrawler: - def assertOptionIsDefault(self, settings: Settings, key: str) -> None: + @staticmethod + def assertOptionIsDefault(settings: Settings, key: str) -> None: assert isinstance(settings, Settings) assert settings[key] == getattr(default_settings, key) class TestCrawler(TestBaseCrawler): - def test_populate_spidercls_settings(self): - spider_settings = {"TEST1": "spider", "TEST2": "spider"} + def test_populate_spidercls_settings(self) -> None: + spider_settings: dict[_SettingsKey, Any] = { + "TEST1": "spider", + "TEST2": "spider", + } project_settings = { **BASE_SETTINGS, "TEST1": "project", @@ -76,47 +82,47 @@ class TestCrawler(TestBaseCrawler): assert not settings.frozen assert crawler.settings.frozen - def test_crawler_accepts_dict(self): + def test_crawler_accepts_dict(self) -> None: crawler = get_crawler(DefaultSpider, {"foo": "bar"}) assert crawler.settings["foo"] == "bar" self.assertOptionIsDefault(crawler.settings, "RETRY_ENABLED") - def test_crawler_accepts_None(self): + def test_crawler_accepts_None(self) -> None: with warnings.catch_warnings(): warnings.simplefilter("ignore", ScrapyDeprecationWarning) crawler = Crawler(DefaultSpider) self.assertOptionIsDefault(crawler.settings, "RETRY_ENABLED") - def test_crawler_rejects_spider_objects(self): + def test_crawler_rejects_spider_objects(self) -> None: with pytest.raises(ValueError, match="spidercls argument must be a class"): - Crawler(DefaultSpider()) - - @inline_callbacks_test - def test_crawler_crawl_twice_seq_unsupported(self): - crawler = get_raw_crawler(NoRequestsSpider, BASE_SETTINGS) - yield crawler.crawl() - with pytest.raises(RuntimeError, match="more than once on the same instance"): - yield crawler.crawl() + Crawler(DefaultSpider()) # type: ignore[arg-type] @coroutine_test - async def test_crawler_crawl_async_twice_seq_unsupported(self): + async def test_crawler_crawl_twice_seq_unsupported(self) -> None: + crawler = get_raw_crawler(NoRequestsSpider, BASE_SETTINGS) + await maybe_deferred_to_future(crawler.crawl()) + with pytest.raises(RuntimeError, match="more than once on the same instance"): + await maybe_deferred_to_future(crawler.crawl()) + + @coroutine_test + async def test_crawler_crawl_async_twice_seq_unsupported(self) -> None: crawler = get_raw_crawler(NoRequestsSpider, BASE_SETTINGS) await crawler.crawl_async() with pytest.raises(RuntimeError, match="more than once on the same instance"): await crawler.crawl_async() - @inline_callbacks_test - def test_crawler_crawl_twice_parallel_unsupported(self): + @coroutine_test + async def test_crawler_crawl_twice_parallel_unsupported(self) -> None: crawler = get_raw_crawler(NoRequestsSpider, BASE_SETTINGS) d1 = crawler.crawl() d2 = crawler.crawl() - yield d1 + await maybe_deferred_to_future(d1) with pytest.raises(RuntimeError, match="Crawling already taking place"): - yield d2 + await maybe_deferred_to_future(d2) @pytest.mark.only_asyncio @coroutine_test - async def test_crawler_crawl_async_twice_parallel_unsupported(self): + async def test_crawler_crawl_async_twice_parallel_unsupported(self) -> None: crawler = get_raw_crawler(NoRequestsSpider, BASE_SETTINGS) t1 = asyncio.create_task(crawler.crawl_async()) t2 = asyncio.create_task(crawler.crawl_async()) @@ -124,12 +130,12 @@ class TestCrawler(TestBaseCrawler): with pytest.raises(RuntimeError, match="Crawling already taking place"): await t2 - def test_get_addon(self): + def test_get_addon(self) -> None: class ParentAddon: pass class TrackingAddon(ParentAddon): - instances = [] + instances: ClassVar[list[TrackingAddon]] = [] def __init__(self): TrackingAddon.instances.append(self) @@ -150,7 +156,7 @@ class TestCrawler(TestBaseCrawler): addon = crawler.get_addon(TrackingAddon) assert addon == expected - addon = crawler.get_addon(DefaultSpider) + addon = crawler.get_addon(DefaultSpider) # type: ignore[assignment] assert addon is None addon = crawler.get_addon(ParentAddon) @@ -162,19 +168,21 @@ class TestCrawler(TestBaseCrawler): addon = crawler.get_addon(ChildAddon) assert addon is None - @inline_callbacks_test - def test_get_downloader_middleware(self): + @coroutine_test + async def test_get_downloader_middleware(self) -> None: class ParentDownloaderMiddleware: pass class TrackingDownloaderMiddleware(ParentDownloaderMiddleware): - instances = [] + instances: ClassVar[list[TrackingDownloaderMiddleware]] = [] def __init__(self): TrackingDownloaderMiddleware.instances.append(self) class MySpider(Spider): name = "myspider" + cls: ClassVar[type[Any]] + result: ClassVar[Any] @classmethod def from_crawler(cls, crawler): @@ -198,18 +206,18 @@ class TestCrawler(TestBaseCrawler): crawler = get_raw_crawler(MySpider, settings) MySpider.cls = TrackingDownloaderMiddleware - yield crawler.crawl() + await crawler.crawl_async() assert len(TrackingDownloaderMiddleware.instances) == 1 assert MySpider.result == TrackingDownloaderMiddleware.instances[-1] crawler = get_raw_crawler(MySpider, settings) MySpider.cls = DefaultSpider - yield crawler.crawl() + await crawler.crawl_async() assert MySpider.result is None crawler = get_raw_crawler(MySpider, settings) MySpider.cls = ParentDownloaderMiddleware - yield crawler.crawl() + await crawler.crawl_async() assert MySpider.result == TrackingDownloaderMiddleware.instances[-1] class ChildDownloaderMiddleware(TrackingDownloaderMiddleware): @@ -217,16 +225,16 @@ class TestCrawler(TestBaseCrawler): crawler = get_raw_crawler(MySpider, settings) MySpider.cls = ChildDownloaderMiddleware - yield crawler.crawl() + await crawler.crawl_async() assert MySpider.result is None - def test_get_downloader_middleware_not_crawling(self): + def test_get_downloader_middleware_not_crawling(self) -> None: crawler = get_raw_crawler(settings_dict=BASE_SETTINGS) with pytest.raises(RuntimeError): crawler.get_downloader_middleware(DefaultSpider) - @inline_callbacks_test - def test_get_downloader_middleware_no_engine(self): + @coroutine_test + async def test_get_downloader_middleware_no_engine(self) -> None: class MySpider(Spider): name = "myspider" @@ -240,21 +248,23 @@ class TestCrawler(TestBaseCrawler): crawler = get_raw_crawler(MySpider, BASE_SETTINGS) with pytest.raises(RuntimeError): - yield crawler.crawl() + await crawler.crawl_async() - @inline_callbacks_test - def test_get_extension(self): + @coroutine_test + async def test_get_extension(self) -> None: class ParentExtension: pass class TrackingExtension(ParentExtension): - instances = [] + instances: ClassVar[list[TrackingExtension]] = [] def __init__(self): TrackingExtension.instances.append(self) class MySpider(Spider): name = "myspider" + cls: ClassVar[type[Any]] + result: ClassVar[Any] @classmethod def from_crawler(cls, crawler): @@ -278,18 +288,18 @@ class TestCrawler(TestBaseCrawler): crawler = get_raw_crawler(MySpider, settings) MySpider.cls = TrackingExtension - yield crawler.crawl() + await crawler.crawl_async() assert len(TrackingExtension.instances) == 1 assert MySpider.result == TrackingExtension.instances[-1] crawler = get_raw_crawler(MySpider, settings) MySpider.cls = DefaultSpider - yield crawler.crawl() + await crawler.crawl_async() assert MySpider.result is None crawler = get_raw_crawler(MySpider, settings) MySpider.cls = ParentExtension - yield crawler.crawl() + await crawler.crawl_async() assert MySpider.result == TrackingExtension.instances[-1] class ChildExtension(TrackingExtension): @@ -297,16 +307,16 @@ class TestCrawler(TestBaseCrawler): crawler = get_raw_crawler(MySpider, settings) MySpider.cls = ChildExtension - yield crawler.crawl() + await crawler.crawl_async() assert MySpider.result is None - def test_get_extension_not_crawling(self): + def test_get_extension_not_crawling(self) -> None: crawler = get_raw_crawler(settings_dict=BASE_SETTINGS) with pytest.raises(RuntimeError): crawler.get_extension(DefaultSpider) - @inline_callbacks_test - def test_get_extension_no_engine(self): + @coroutine_test + async def test_get_extension_no_engine(self) -> None: class MySpider(Spider): name = "myspider" @@ -320,21 +330,23 @@ class TestCrawler(TestBaseCrawler): crawler = get_raw_crawler(MySpider, BASE_SETTINGS) with pytest.raises(RuntimeError): - yield crawler.crawl() + await crawler.crawl_async() - @inline_callbacks_test - def test_get_item_pipeline(self): + @coroutine_test + async def test_get_item_pipeline(self) -> None: class ParentItemPipeline: pass class TrackingItemPipeline(ParentItemPipeline): - instances = [] + instances: ClassVar[list[TrackingItemPipeline]] = [] def __init__(self): TrackingItemPipeline.instances.append(self) class MySpider(Spider): name = "myspider" + cls: ClassVar[type[Any]] + result: ClassVar[Any] @classmethod def from_crawler(cls, crawler): @@ -358,18 +370,18 @@ class TestCrawler(TestBaseCrawler): crawler = get_raw_crawler(MySpider, settings) MySpider.cls = TrackingItemPipeline - yield crawler.crawl() + await crawler.crawl_async() assert len(TrackingItemPipeline.instances) == 1 assert MySpider.result == TrackingItemPipeline.instances[-1] crawler = get_raw_crawler(MySpider, settings) MySpider.cls = DefaultSpider - yield crawler.crawl() + await crawler.crawl_async() assert MySpider.result is None crawler = get_raw_crawler(MySpider, settings) MySpider.cls = ParentItemPipeline - yield crawler.crawl() + await crawler.crawl_async() assert MySpider.result == TrackingItemPipeline.instances[-1] class ChildItemPipeline(TrackingItemPipeline): @@ -377,16 +389,16 @@ class TestCrawler(TestBaseCrawler): crawler = get_raw_crawler(MySpider, settings) MySpider.cls = ChildItemPipeline - yield crawler.crawl() + await crawler.crawl_async() assert MySpider.result is None - def test_get_item_pipeline_not_crawling(self): + def test_get_item_pipeline_not_crawling(self) -> None: crawler = get_raw_crawler(settings_dict=BASE_SETTINGS) with pytest.raises(RuntimeError): crawler.get_item_pipeline(DefaultSpider) - @inline_callbacks_test - def test_get_item_pipeline_no_engine(self): + @coroutine_test + async def test_get_item_pipeline_no_engine(self) -> None: class MySpider(Spider): name = "myspider" @@ -400,21 +412,23 @@ class TestCrawler(TestBaseCrawler): crawler = get_raw_crawler(MySpider, BASE_SETTINGS) with pytest.raises(RuntimeError): - yield crawler.crawl() + await crawler.crawl_async() - @inline_callbacks_test - def test_get_spider_middleware(self): + @coroutine_test + async def test_get_spider_middleware(self) -> None: class ParentSpiderMiddleware: pass class TrackingSpiderMiddleware(ParentSpiderMiddleware): - instances = [] + instances: ClassVar[list[TrackingSpiderMiddleware]] = [] def __init__(self): TrackingSpiderMiddleware.instances.append(self) class MySpider(Spider): name = "myspider" + cls: ClassVar[type[Any]] + result: ClassVar[Any] @classmethod def from_crawler(cls, crawler): @@ -438,18 +452,18 @@ class TestCrawler(TestBaseCrawler): crawler = get_raw_crawler(MySpider, settings) MySpider.cls = TrackingSpiderMiddleware - yield crawler.crawl() + await crawler.crawl_async() assert len(TrackingSpiderMiddleware.instances) == 1 assert MySpider.result == TrackingSpiderMiddleware.instances[-1] crawler = get_raw_crawler(MySpider, settings) MySpider.cls = DefaultSpider - yield crawler.crawl() + await crawler.crawl_async() assert MySpider.result is None crawler = get_raw_crawler(MySpider, settings) MySpider.cls = ParentSpiderMiddleware - yield crawler.crawl() + await crawler.crawl_async() assert MySpider.result == TrackingSpiderMiddleware.instances[-1] class ChildSpiderMiddleware(TrackingSpiderMiddleware): @@ -457,16 +471,16 @@ class TestCrawler(TestBaseCrawler): crawler = get_raw_crawler(MySpider, settings) MySpider.cls = ChildSpiderMiddleware - yield crawler.crawl() + await crawler.crawl_async() assert MySpider.result is None - def test_get_spider_middleware_not_crawling(self): + def test_get_spider_middleware_not_crawling(self) -> None: crawler = get_raw_crawler(settings_dict=BASE_SETTINGS) with pytest.raises(RuntimeError): crawler.get_spider_middleware(DefaultSpider) - @inline_callbacks_test - def test_get_spider_middleware_no_engine(self): + @coroutine_test + async def test_get_spider_middleware_no_engine(self) -> None: class MySpider(Spider): name = "myspider" @@ -480,22 +494,23 @@ class TestCrawler(TestBaseCrawler): crawler = get_raw_crawler(MySpider, BASE_SETTINGS) with pytest.raises(RuntimeError): - yield crawler.crawl() + await crawler.crawl_async() class TestSpiderSettings: - def test_spider_custom_settings(self): + def test_spider_custom_settings(self) -> None: class MySpider(scrapy.Spider): name = "spider" custom_settings = {"AUTOTHROTTLE_ENABLED": True} crawler = get_crawler(MySpider) + assert crawler.extensions enabled_exts = [e.__class__ for e in crawler.extensions.middlewares] assert AutoThrottle in enabled_exts class TestCrawlerLogging: - def test_no_root_handler_installed(self): + def test_no_root_handler_installed(self) -> None: handler = get_scrapy_root_handler() if handler is not None: logging.root.removeHandler(handler) @@ -507,7 +522,7 @@ class TestCrawlerLogging: assert get_scrapy_root_handler() is None @coroutine_test - async def test_spider_custom_settings_log_level(self, tmp_path): + async def test_spider_custom_settings_log_level(self, tmp_path: Path) -> None: log_file = Path(tmp_path, "log.txt") log_file.write_text("previous message\n", encoding="utf-8") @@ -535,9 +550,13 @@ class TestCrawlerLogging: try: configure_logging() - assert get_scrapy_root_handler().level == logging.DEBUG + handler = get_scrapy_root_handler() + assert handler is not None + assert handler.level == logging.DEBUG crawler = get_crawler(MySpider) - assert get_scrapy_root_handler().level == logging.INFO + handler = get_scrapy_root_handler() + assert handler is not None + assert handler.level == logging.INFO await crawler.crawl_async() finally: _uninstall_scrapy_root_handler() @@ -549,12 +568,13 @@ class TestCrawlerLogging: assert "info message" in logged assert "warning message" in logged assert "error message" in logged + assert crawler.stats assert crawler.stats.get_value("log_count/ERROR") == 1 assert crawler.stats.get_value("log_count/WARNING") == 1 assert info_count == 1 assert crawler.stats.get_value("log_count/DEBUG", 0) == 0 - def test_spider_custom_settings_log_append(self, tmp_path): + def test_spider_custom_settings_log_append(self, tmp_path: Path) -> None: log_file = Path(tmp_path, "log.txt") log_file.write_text("previous message\n", encoding="utf-8") @@ -579,12 +599,12 @@ class TestCrawlerLogging: class SpiderLoaderWithWrongInterface: - def unneeded_method(self): + def unneeded_method(self) -> None: pass class TestCrawlerRunner(TestBaseCrawler): - def test_spider_manager_verify_interface(self): + def test_spider_manager_verify_interface(self) -> None: settings = Settings( { "SPIDER_LOADER_CLASS": SpiderLoaderWithWrongInterface, @@ -593,18 +613,18 @@ class TestCrawlerRunner(TestBaseCrawler): with pytest.raises(MultipleInvalid): CrawlerRunner(settings) - def test_crawler_runner_accepts_dict(self): + def test_crawler_runner_accepts_dict(self) -> None: runner = CrawlerRunner({"foo": "bar"}) assert runner.settings["foo"] == "bar" self.assertOptionIsDefault(runner.settings, "RETRY_ENABLED") - def test_crawler_runner_accepts_None(self): + def test_crawler_runner_accepts_None(self) -> None: runner = CrawlerRunner() self.assertOptionIsDefault(runner.settings, "RETRY_ENABLED") class TestAsyncCrawlerRunner(TestBaseCrawler): - def test_spider_manager_verify_interface(self): + def test_spider_manager_verify_interface(self) -> None: settings = Settings( { "SPIDER_LOADER_CLASS": SpiderLoaderWithWrongInterface, @@ -613,23 +633,23 @@ class TestAsyncCrawlerRunner(TestBaseCrawler): with pytest.raises(MultipleInvalid): AsyncCrawlerRunner(settings) - def test_crawler_runner_accepts_dict(self): + def test_crawler_runner_accepts_dict(self) -> None: runner = AsyncCrawlerRunner({"foo": "bar"}) assert runner.settings["foo"] == "bar" self.assertOptionIsDefault(runner.settings, "RETRY_ENABLED") - def test_crawler_runner_accepts_None(self): + def test_crawler_runner_accepts_None(self) -> None: runner = AsyncCrawlerRunner() self.assertOptionIsDefault(runner.settings, "RETRY_ENABLED") class TestCrawlerProcess(TestBaseCrawler): - def test_crawler_process_accepts_dict(self): + def test_crawler_process_accepts_dict(self) -> None: runner = CrawlerProcess({"foo": "bar"}, install_root_handler=False) assert runner.settings["foo"] == "bar" self.assertOptionIsDefault(runner.settings, "RETRY_ENABLED") - def test_crawler_process_accepts_None(self): + def test_crawler_process_accepts_None(self) -> None: runner = CrawlerProcess(install_root_handler=False) self.assertOptionIsDefault(runner.settings, "RETRY_ENABLED") @@ -668,33 +688,35 @@ class NoRequestsSpider(scrapy.Spider): @pytest.mark.requires_reactor # CrawlerRunner requires a reactor class TestCrawlerRunnerHasSpider: - @staticmethod - def _runner() -> CrawlerRunnerBase: + @pytest.fixture + def runner(self) -> CrawlerRunnerBase: return CrawlerRunner(get_reactor_settings()) @staticmethod - def _crawl(runner: CrawlerRunnerBase, spider: type[Spider]) -> Deferred[None]: - return cast("Deferred[None]", runner.crawl(spider)) + async def _crawl(runner: CrawlerRunnerBase, spider: type[Spider]) -> None: + await ensure_awaitable(runner.crawl(spider)) - @inline_callbacks_test - def test_crawler_runner_bootstrap_successful(self): - runner = self._runner() - yield self._crawl(runner, NoRequestsSpider) + @coroutine_test + async def test_crawler_runner_bootstrap_successful( + self, runner: CrawlerRunnerBase + ) -> None: + await self._crawl(runner, NoRequestsSpider) assert not runner.bootstrap_failed - @inline_callbacks_test - def test_crawler_runner_bootstrap_successful_for_several(self): - runner = self._runner() - yield self._crawl(runner, NoRequestsSpider) - yield self._crawl(runner, NoRequestsSpider) + @coroutine_test + async def test_crawler_runner_bootstrap_successful_for_several( + self, runner: CrawlerRunnerBase + ) -> None: + await self._crawl(runner, NoRequestsSpider) + await self._crawl(runner, NoRequestsSpider) assert not runner.bootstrap_failed - @inline_callbacks_test - def test_crawler_runner_bootstrap_failed(self): - runner = self._runner() - + @coroutine_test + async def test_crawler_runner_bootstrap_failed( + self, runner: CrawlerRunnerBase + ) -> None: try: - yield self._crawl(runner, ExceptionSpider) + await self._crawl(runner, ExceptionSpider) except ValueError: pass else: @@ -702,25 +724,25 @@ class TestCrawlerRunnerHasSpider: assert runner.bootstrap_failed - @inline_callbacks_test - def test_crawler_runner_bootstrap_failed_for_several(self): - runner = self._runner() - + @coroutine_test + async def test_crawler_runner_bootstrap_failed_for_several( + self, runner: CrawlerRunnerBase + ) -> None: try: - yield self._crawl(runner, ExceptionSpider) + await self._crawl(runner, ExceptionSpider) except ValueError: pass else: pytest.fail("Exception should be raised from spider") - yield self._crawl(runner, NoRequestsSpider) + await self._crawl(runner, NoRequestsSpider) assert runner.bootstrap_failed - @inline_callbacks_test - def test_crawler_runner_asyncio_enabled_true( + @coroutine_test + async def test_crawler_runner_asyncio_enabled_true( self, reactor_pytest: str - ) -> Generator[Deferred[Any], Any, None]: + ) -> None: if reactor_pytest != "asyncio": runner = CrawlerRunner( settings={ @@ -731,7 +753,7 @@ class TestCrawlerRunnerHasSpider: Exception, match=r"The installed reactor \(.*?\) does not match the requested one \(.*?\)", ): - yield self._crawl(runner, NoRequestsSpider) + await self._crawl(runner, NoRequestsSpider) else: CrawlerRunner( settings={ @@ -746,11 +768,7 @@ class TestAsyncCrawlerRunnerHasSpider(TestCrawlerRunnerHasSpider): def _runner() -> CrawlerRunnerBase: return AsyncCrawlerRunner(get_reactor_settings()) - @staticmethod - def _crawl(runner: CrawlerRunnerBase, spider: type[Spider]) -> Deferred[None]: - return deferred_from_coro(runner.crawl(spider)) - - def test_crawler_runner_asyncio_enabled_true(self): + def test_crawler_runner_asyncio_enabled_true(self) -> None: # type: ignore[override] pytest.skip("This test is only for CrawlerRunner") @@ -762,7 +780,9 @@ class TestAsyncCrawlerRunnerHasSpider(TestCrawlerRunnerHasSpider): ({"LOG_VERSIONS": []}, None), ], ) -def test_log_scrapy_info(settings, items, caplog): +def test_log_scrapy_info( + settings: dict[str, Any], items: list[str] | None, caplog: pytest.LogCaptureFixture +) -> None: with caplog.at_level("INFO"): CrawlerProcess(settings, install_root_handler=False) assert ( diff --git a/tests/test_crawler_subprocess.py b/tests/test_crawler_subprocess.py index beae4f277..146d94ecd 100644 --- a/tests/test_crawler_subprocess.py +++ b/tests/test_crawler_subprocess.py @@ -52,7 +52,7 @@ class TestCrawlerProcessSubprocessBase(ScriptRunnerMixin): with the same file names and expectations. """ - def test_simple(self): + def test_simple(self) -> None: log = self.run_script("simple.py") assert "Spider closed (finished)" in log assert ( @@ -61,7 +61,7 @@ class TestCrawlerProcessSubprocessBase(ScriptRunnerMixin): ) assert "is_reactorless(): False" in log - def test_multi(self): + def test_multi(self) -> None: log = self.run_script("multi.py") assert "Spider closed (finished)" in log assert ( @@ -70,7 +70,7 @@ class TestCrawlerProcessSubprocessBase(ScriptRunnerMixin): ) assert "ReactorAlreadyInstalledError" not in log - def test_reactor_default(self): + def test_reactor_default(self) -> None: log = self.run_script("reactor_default.py") assert "Spider closed (finished)" not in log assert ( @@ -78,7 +78,7 @@ class TestCrawlerProcessSubprocessBase(ScriptRunnerMixin): "(twisted.internet.asyncioreactor.AsyncioSelectorReactor)" ) in log - def test_asyncio_enabled_no_reactor(self): + def test_asyncio_enabled_no_reactor(self) -> None: log = self.run_script("asyncio_enabled_no_reactor.py") assert "Spider closed (finished)" in log assert ( @@ -87,7 +87,7 @@ class TestCrawlerProcessSubprocessBase(ScriptRunnerMixin): ) assert "RuntimeError" not in log - def test_asyncio_enabled_reactor(self): + def test_asyncio_enabled_reactor(self) -> None: log = self.run_script("asyncio_enabled_reactor.py") assert "Spider closed (finished)" in log assert ( @@ -100,7 +100,7 @@ class TestCrawlerProcessSubprocessBase(ScriptRunnerMixin): parse_version(w3lib_version) >= parse_version("2.0.0"), reason="w3lib 2.0.0 and later do not allow invalid domains.", ) - def test_ipv6_default_name_resolver(self): + def test_ipv6_default_name_resolver(self) -> None: log = self.run_script("default_name_resolver.py") assert "Spider closed (finished)" in log assert ( @@ -112,7 +112,7 @@ class TestCrawlerProcessSubprocessBase(ScriptRunnerMixin): in log ) - def test_caching_hostname_resolver_ipv6(self): + def test_caching_hostname_resolver_ipv6(self) -> None: log = self.run_script("caching_hostname_resolver_ipv6.py") assert "Spider closed (finished)" in log assert "scrapy.exceptions.CannotResolveHostError" not in log @@ -126,7 +126,7 @@ class TestCrawlerProcessSubprocessBase(ScriptRunnerMixin): assert "TimeoutError" not in log assert "scrapy.exceptions.CannotResolveHostError" not in log - def test_twisted_reactor_asyncio(self): + def test_twisted_reactor_asyncio(self) -> None: log = self.run_script("twisted_reactor_asyncio.py") assert "Spider closed (finished)" in log assert ( @@ -134,7 +134,7 @@ class TestCrawlerProcessSubprocessBase(ScriptRunnerMixin): in log ) - def test_twisted_reactor_asyncio_custom_settings(self): + def test_twisted_reactor_asyncio_custom_settings(self) -> None: log = self.run_script("twisted_reactor_custom_settings.py") assert "Spider closed (finished)" in log assert ( @@ -142,7 +142,7 @@ class TestCrawlerProcessSubprocessBase(ScriptRunnerMixin): in log ) - def test_twisted_reactor_asyncio_custom_settings_same(self): + def test_twisted_reactor_asyncio_custom_settings_same(self) -> None: log = self.run_script("twisted_reactor_custom_settings_same.py") assert "Spider closed (finished)" in log assert ( @@ -151,7 +151,7 @@ class TestCrawlerProcessSubprocessBase(ScriptRunnerMixin): ) @pytest.mark.requires_uvloop - def test_custom_loop_asyncio(self): + def test_custom_loop_asyncio(self) -> None: log = self.run_script("asyncio_custom_loop.py") assert "Spider closed (finished)" in log assert ( @@ -161,7 +161,7 @@ class TestCrawlerProcessSubprocessBase(ScriptRunnerMixin): assert "Using asyncio event loop: uvloop.Loop" in log @pytest.mark.requires_uvloop - def test_custom_loop_asyncio_deferred_signal(self): + def test_custom_loop_asyncio_deferred_signal(self) -> None: log = self.run_script("asyncio_deferred_signal.py", "uvloop.Loop") assert "Spider closed (finished)" in log assert ( @@ -172,7 +172,7 @@ class TestCrawlerProcessSubprocessBase(ScriptRunnerMixin): assert "async pipeline opened!" in log @pytest.mark.requires_uvloop - def test_asyncio_enabled_reactor_same_loop(self): + def test_asyncio_enabled_reactor_same_loop(self) -> None: log = self.run_script("asyncio_enabled_reactor_same_loop.py") assert "Spider closed (finished)" in log assert ( @@ -182,7 +182,7 @@ class TestCrawlerProcessSubprocessBase(ScriptRunnerMixin): assert "Using asyncio event loop: uvloop.Loop" in log @pytest.mark.requires_uvloop - def test_asyncio_enabled_reactor_different_loop(self): + def test_asyncio_enabled_reactor_different_loop(self) -> None: log = self.run_script("asyncio_enabled_reactor_different_loop.py") assert "Spider closed (finished)" not in log assert ( @@ -190,7 +190,7 @@ class TestCrawlerProcessSubprocessBase(ScriptRunnerMixin): "setting (uvloop.Loop)" ) in log - def test_default_loop_asyncio_deferred_signal(self): + def test_default_loop_asyncio_deferred_signal(self) -> None: log = self.run_script("asyncio_deferred_signal.py") assert "Spider closed (finished)" in log assert ( @@ -200,7 +200,7 @@ class TestCrawlerProcessSubprocessBase(ScriptRunnerMixin): assert "Using asyncio event loop: uvloop.Loop" not in log assert "async pipeline opened!" in log - def test_args_change_settings(self): + def test_args_change_settings(self) -> None: log = self.run_script("args_settings.py") assert "Spider closed (finished)" in log assert "The value of FOO is 42" in log @@ -243,7 +243,7 @@ class TestCrawlerProcessSubprocess(TestCrawlerProcessSubprocessBase): def script_dir(self) -> Path: return self.get_script_dir("CrawlerProcess") - def test_reactor_default_twisted_reactor_select(self): + def test_reactor_default_twisted_reactor_select(self) -> None: log = self.run_script("reactor_default_twisted_reactor_select.py") if platform.system() in ["Windows", "Darwin"]: # The goal of this test function is to test that, when a reactor is @@ -264,7 +264,7 @@ class TestCrawlerProcessSubprocess(TestCrawlerProcessSubprocessBase): "(twisted.internet.selectreactor.SelectReactor)" ) in log - def test_reactor_select(self): + def test_reactor_select(self) -> None: log = self.run_script("reactor_select.py") assert "Spider closed (finished)" not in log assert ( @@ -272,12 +272,12 @@ class TestCrawlerProcessSubprocess(TestCrawlerProcessSubprocessBase): "(twisted.internet.asyncioreactor.AsyncioSelectorReactor)" ) in log - def test_reactor_select_twisted_reactor_select(self): + def test_reactor_select_twisted_reactor_select(self) -> None: log = self.run_script("reactor_select_twisted_reactor_select.py") assert "Spider closed (finished)" in log assert "ReactorAlreadyInstalledError" not in log - def test_reactor_select_subclass_twisted_reactor_select(self): + def test_reactor_select_subclass_twisted_reactor_select(self) -> None: log = self.run_script("reactor_select_subclass_twisted_reactor_select.py") assert "Spider closed (finished)" not in log assert ( @@ -285,7 +285,7 @@ class TestCrawlerProcessSubprocess(TestCrawlerProcessSubprocessBase): "(twisted.internet.selectreactor.SelectReactor)" ) in log - def test_twisted_reactor_select(self): + def test_twisted_reactor_select(self) -> None: log = self.run_script("twisted_reactor_select.py") assert "Spider closed (finished)" in log assert "Using reactor: twisted.internet.selectreactor.SelectReactor" in log @@ -293,12 +293,12 @@ class TestCrawlerProcessSubprocess(TestCrawlerProcessSubprocessBase): @pytest.mark.skipif( platform.system() == "Windows", reason="PollReactor is not supported on Windows" ) - def test_twisted_reactor_poll(self): + def test_twisted_reactor_poll(self) -> None: log = self.run_script("twisted_reactor_poll.py") assert "Spider closed (finished)" in log assert "Using reactor: twisted.internet.pollreactor.PollReactor" in log - def test_twisted_reactor_asyncio_custom_settings_conflict(self): + def test_twisted_reactor_asyncio_custom_settings_conflict(self) -> None: log = self.run_script("twisted_reactor_custom_settings_conflict.py") assert "Using reactor: twisted.internet.selectreactor.SelectReactor" in log assert ( @@ -306,7 +306,7 @@ class TestCrawlerProcessSubprocess(TestCrawlerProcessSubprocessBase): in log ) - def test_reactorless(self): + def test_reactorless(self) -> None: log = self.run_script("reactorless.py") assert ( "RuntimeError: CrawlerProcess doesn't support TWISTED_REACTOR_ENABLED=False" @@ -319,7 +319,7 @@ class TestAsyncCrawlerProcessSubprocess(TestCrawlerProcessSubprocessBase): def script_dir(self) -> Path: return self.get_script_dir("AsyncCrawlerProcess") - def test_twisted_reactor_custom_settings_select(self): + def test_twisted_reactor_custom_settings_select(self) -> None: log = self.run_script("twisted_reactor_custom_settings_select.py") assert "Spider closed (finished)" not in log assert ( @@ -329,7 +329,7 @@ class TestAsyncCrawlerProcessSubprocess(TestCrawlerProcessSubprocessBase): ) in log @pytest.mark.requires_uvloop - def test_asyncio_enabled_reactor_same_loop(self): + def test_asyncio_enabled_reactor_same_loop(self) -> None: log = self.run_script("asyncio_custom_loop_custom_settings_same.py") assert "Spider closed (finished)" in log assert ( @@ -339,7 +339,7 @@ class TestAsyncCrawlerProcessSubprocess(TestCrawlerProcessSubprocessBase): assert "Using asyncio event loop: uvloop.Loop" in log @pytest.mark.requires_uvloop - def test_asyncio_enabled_reactor_different_loop(self): + def test_asyncio_enabled_reactor_different_loop(self) -> None: log = self.run_script("asyncio_custom_loop_custom_settings_different.py") assert "Spider closed (finished)" not in log assert ( @@ -347,7 +347,7 @@ class TestAsyncCrawlerProcessSubprocess(TestCrawlerProcessSubprocessBase): "setting (uvloop.Loop)" ) in log - def test_reactorless_simple(self): + def test_reactorless_simple(self) -> None: log = self.run_script("reactorless_simple.py") assert "Not using a Twisted reactor" in log assert "Spider closed (finished)" in log @@ -356,7 +356,7 @@ class TestAsyncCrawlerProcessSubprocess(TestCrawlerProcessSubprocessBase): assert log.count("WARNING: HttpxDownloadHandler is experimental") == 2 assert log.count("WARNING: ") == 2 - def test_reactorless_custom_settings(self): + def test_reactorless_custom_settings(self) -> None: """Setting TWISTED_REACTOR_ENABLED=False in spider settings is not currently supported, AsyncCrawlerProcess will install a reactor in this case. @@ -368,7 +368,7 @@ class TestAsyncCrawlerProcessSubprocess(TestCrawlerProcessSubprocessBase): in log ) - def test_reactorless_datauri(self): + def test_reactorless_datauri(self) -> None: log = self.run_script("reactorless_datauri.py") assert "Not using a Twisted reactor" in log assert "Spider closed (finished)" in log @@ -378,13 +378,13 @@ class TestAsyncCrawlerProcessSubprocess(TestCrawlerProcessSubprocessBase): assert log.count("WARNING: HttpxDownloadHandler is experimental") == 2 assert log.count("WARNING: ") == 2 - def test_reactorless_import_hook(self): + def test_reactorless_import_hook(self) -> None: log = self.run_script("reactorless_import_hook.py") assert "Not using a Twisted reactor" in log assert "Spider closed (finished)" in log assert "ImportError: Import of twisted.internet.reactor is forbidden" in log - def test_reactorless_telnetconsole_default(self): + def test_reactorless_telnetconsole_default(self) -> None: """By default TWISTED_REACTOR_ENABLED=False silently sets TELNETCONSOLE_ENABLED=False.""" log = self.run_script("reactorless_simple.py") # no need for a separate script assert "Not using a Twisted reactor" in log @@ -392,7 +392,7 @@ class TestAsyncCrawlerProcessSubprocess(TestCrawlerProcessSubprocessBase): assert "The TelnetConsole extension requires a Twisted reactor" not in log assert "scrapy.extensions.telnet.TelnetConsole" not in log - def test_reactorless_telnetconsole_disabled(self): + def test_reactorless_telnetconsole_disabled(self) -> None: """Explicit TELNETCONSOLE_ENABLED=False, there are no warnings.""" log = self.run_script("reactorless_telnetconsole_disabled.py") assert "Not using a Twisted reactor" in log @@ -400,14 +400,14 @@ class TestAsyncCrawlerProcessSubprocess(TestCrawlerProcessSubprocessBase): assert "The TelnetConsole extension requires a Twisted reactor" not in log assert "scrapy.extensions.telnet.TelnetConsole" not in log - def test_reactorless_telnetconsole_enabled(self): + def test_reactorless_telnetconsole_enabled(self) -> None: """Explicit TELNETCONSOLE_ENABLED=True, the user gets a warning.""" log = self.run_script("reactorless_telnetconsole_enabled.py") assert "Not using a Twisted reactor" in log assert "Spider closed (finished)" in log assert "The TelnetConsole extension requires a Twisted reactor" in log - def test_reactorless_reactor(self): + def test_reactorless_reactor(self) -> None: log = self.run_script("reactorless_reactor.py") assert ( "RuntimeError: TWISTED_REACTOR_ENABLED is False but a Twisted reactor is installed" @@ -427,7 +427,7 @@ class TestCrawlerRunnerSubprocessBase(ScriptRunnerMixin): with the same file names and expectations. """ - def test_simple(self): + def test_simple(self) -> None: log = self.run_script("simple.py") assert "Spider closed (finished)" in log assert ( @@ -436,7 +436,7 @@ class TestCrawlerRunnerSubprocessBase(ScriptRunnerMixin): ) assert "is_reactorless(): False" in log - def test_multi_parallel(self): + def test_multi_parallel(self) -> None: log = self.run_script("multi_parallel.py") assert "Spider closed (finished)" in log assert ( @@ -449,7 +449,7 @@ class TestCrawlerRunnerSubprocessBase(ScriptRunnerMixin): re.DOTALL, ) - def test_multi_seq(self): + def test_multi_seq(self) -> None: log = self.run_script("multi_seq.py") assert "Spider closed (finished)" in log assert ( @@ -463,7 +463,7 @@ class TestCrawlerRunnerSubprocessBase(ScriptRunnerMixin): ) @pytest.mark.requires_uvloop - def test_custom_loop_same(self): + def test_custom_loop_same(self) -> None: log = self.run_script("custom_loop_same.py") assert "Spider closed (finished)" in log assert ( @@ -473,7 +473,7 @@ class TestCrawlerRunnerSubprocessBase(ScriptRunnerMixin): assert "Using asyncio event loop: uvloop.Loop" in log @pytest.mark.requires_uvloop - def test_custom_loop_different(self): + def test_custom_loop_different(self) -> None: log = self.run_script("custom_loop_different.py") assert "Spider closed (finished)" not in log assert ( @@ -481,7 +481,7 @@ class TestCrawlerRunnerSubprocessBase(ScriptRunnerMixin): "setting (uvloop.Loop)" ) in log - def test_no_reactor(self): + def test_no_reactor(self) -> None: log = self.run_script("no_reactor.py") assert "Spider closed (finished)" not in log assert ( @@ -495,7 +495,7 @@ class TestCrawlerRunnerSubprocess(TestCrawlerRunnerSubprocessBase): def script_dir(self) -> Path: return self.get_script_dir("CrawlerRunner") - def test_explicit_default_reactor(self): + def test_explicit_default_reactor(self) -> None: log = self.run_script("explicit_default_reactor.py") assert "Spider closed (finished)" in log assert ( @@ -503,14 +503,14 @@ class TestCrawlerRunnerSubprocess(TestCrawlerRunnerSubprocessBase): not in log ) - def test_response_ip_address(self): + def test_response_ip_address(self) -> None: log = self.run_script("ip_address.py") assert "INFO: Spider closed (finished)" in log assert "INFO: Host: not.a.real.domain" in log assert "INFO: Type: " in log assert "INFO: IP address: 127.0.0.1" in log - def test_change_default_reactor(self): + def test_change_default_reactor(self) -> None: log = self.run_script("change_reactor.py") assert ( "DEBUG: Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" @@ -518,7 +518,7 @@ class TestCrawlerRunnerSubprocess(TestCrawlerRunnerSubprocessBase): ) assert "DEBUG: Using asyncio event loop" in log - def test_reactorless(self): + def test_reactorless(self) -> None: log = self.run_script("reactorless.py") assert ( "RuntimeError: CrawlerRunner doesn't support TWISTED_REACTOR_ENABLED=False" @@ -531,7 +531,7 @@ class TestAsyncCrawlerRunnerSubprocess(TestCrawlerRunnerSubprocessBase): def script_dir(self) -> Path: return self.get_script_dir("AsyncCrawlerRunner") - def test_simple_default_reactor(self): + def test_simple_default_reactor(self) -> None: log = self.run_script("simple_default_reactor.py") assert "Spider closed (finished)" not in log assert ( @@ -539,7 +539,7 @@ class TestAsyncCrawlerRunnerSubprocess(TestCrawlerRunnerSubprocessBase): "AsyncCrawlerRunner requires that the installed Twisted reactor" ) in log - def test_reactorless_simple(self): + def test_reactorless_simple(self) -> None: log = self.run_script("reactorless_simple.py") assert "Not using a Twisted reactor" in log assert "Spider closed (finished)" in log @@ -548,7 +548,7 @@ class TestAsyncCrawlerRunnerSubprocess(TestCrawlerRunnerSubprocessBase): assert log.count("WARNING: HttpxDownloadHandler is experimental") == 2 assert log.count("WARNING: ") == 2 - def test_reactorless_custom_settings(self): + def test_reactorless_custom_settings(self) -> None: """Setting TWISTED_REACTOR_ENABLED=False in spider settings is not currently supported, AsyncCrawlerRunner will expect a reactor installed by the user. @@ -557,7 +557,7 @@ class TestAsyncCrawlerRunnerSubprocess(TestCrawlerRunnerSubprocessBase): assert "Spider closed (finished)" not in log assert "We expected a Twisted reactor to be installed but it isn't." in log - def test_reactorless_datauri(self): + def test_reactorless_datauri(self) -> None: log = self.run_script("reactorless_datauri.py") assert "Not using a Twisted reactor" in log assert "Spider closed (finished)" in log @@ -567,7 +567,7 @@ class TestAsyncCrawlerRunnerSubprocess(TestCrawlerRunnerSubprocessBase): assert log.count("WARNING: HttpxDownloadHandler is experimental") == 2 assert log.count("WARNING: ") == 2 - def test_reactorless_reactor(self): + def test_reactorless_reactor(self) -> None: log = self.run_script("reactorless_reactor.py") assert ( "RuntimeError: TWISTED_REACTOR_ENABLED is False but a Twisted reactor is installed"