RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Rename async test decorators. (#7234)

This commit is contained in:
Andrey Rakhmatullin 2026-02-03 00:39:43 +05:00 committed by GitHub
parent c8aa429c9b
commit 11977afba5
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
45 changed files with 513 additions and 515 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
tests/test_addons.py
View File

@ -7,7 +7,7 @@ from scrapy.crawler import AsyncCrawlerRunner, Crawler, CrawlerRunner
from scrapy.exceptions import NotConfigured from scrapy.exceptions import NotConfigured
from scrapy.settings import BaseSettings, Settings from scrapy.settings import BaseSettings, Settings
from scrapy.utils.test import get_crawler, get_reactor_settings from scrapy.utils.test import get_crawler, get_reactor_settings
from tests.utils.decorators import inlineCallbacks from tests.utils.decorators import inline_callbacks_test
class SimpleAddon: class SimpleAddon:
@ -183,7 +183,7 @@ class TestAddonManager:
extra={"crawler": crawler}, extra={"crawler": crawler},
) )
@inlineCallbacks @inline_callbacks_test
def test_enable_addon_in_spider(self): def test_enable_addon_in_spider(self):
class MySpider(Spider): class MySpider(Spider):
name = "myspider" name = "myspider"

16
tests/test_closespider.py
View File

@ -9,7 +9,7 @@ from tests.spiders import (
MaxItemsAndRequestsSpider, MaxItemsAndRequestsSpider,
SlowSpider, SlowSpider,
) )
from tests.utils.decorators import inlineCallbacks from tests.utils.decorators import inline_callbacks_test
@pytest.mark.requires_http_handler @pytest.mark.requires_http_handler
@ -23,7 +23,7 @@ class TestCloseSpider:
def teardown_class(cls): def teardown_class(cls):
cls.mockserver.__exit__(None, None, None) cls.mockserver.__exit__(None, None, None)
@inlineCallbacks @inline_callbacks_test
def test_closespider_itemcount(self): def test_closespider_itemcount(self):
close_on = 5 close_on = 5
crawler = get_crawler(ItemSpider, {"CLOSESPIDER_ITEMCOUNT": close_on}) crawler = get_crawler(ItemSpider, {"CLOSESPIDER_ITEMCOUNT": close_on})
@ -33,7 +33,7 @@ class TestCloseSpider:
itemcount = crawler.stats.get_value("item_scraped_count") itemcount = crawler.stats.get_value("item_scraped_count")
assert itemcount >= close_on assert itemcount >= close_on
@inlineCallbacks @inline_callbacks_test
def test_closespider_pagecount(self): def test_closespider_pagecount(self):
close_on = 5 close_on = 5
crawler = get_crawler(FollowAllSpider, {"CLOSESPIDER_PAGECOUNT": close_on}) crawler = get_crawler(FollowAllSpider, {"CLOSESPIDER_PAGECOUNT": close_on})
@ -43,7 +43,7 @@ class TestCloseSpider:
pagecount = crawler.stats.get_value("response_received_count") pagecount = crawler.stats.get_value("response_received_count")
assert pagecount >= close_on assert pagecount >= close_on
@inlineCallbacks @inline_callbacks_test
def test_closespider_pagecount_no_item(self): def test_closespider_pagecount_no_item(self):
close_on = 5 close_on = 5
max_items = 5 max_items = 5
@ -63,7 +63,7 @@ class TestCloseSpider:
itemcount = crawler.stats.get_value("item_scraped_count") itemcount = crawler.stats.get_value("item_scraped_count")
assert pagecount <= close_on + itemcount assert pagecount <= close_on + itemcount
@inlineCallbacks @inline_callbacks_test
def test_closespider_pagecount_no_item_with_pagecount(self): def test_closespider_pagecount_no_item_with_pagecount(self):
close_on_pagecount_no_item = 5 close_on_pagecount_no_item = 5
close_on_pagecount = 20 close_on_pagecount = 20
@ -80,7 +80,7 @@ class TestCloseSpider:
pagecount = crawler.stats.get_value("response_received_count") pagecount = crawler.stats.get_value("response_received_count")
assert pagecount < close_on_pagecount assert pagecount < close_on_pagecount
@inlineCallbacks @inline_callbacks_test
def test_closespider_errorcount(self): def test_closespider_errorcount(self):
close_on = 5 close_on = 5
crawler = get_crawler(ErrorSpider, {"CLOSESPIDER_ERRORCOUNT": close_on}) crawler = get_crawler(ErrorSpider, {"CLOSESPIDER_ERRORCOUNT": close_on})
@ -92,7 +92,7 @@ class TestCloseSpider:
assert crawler.stats.get_value("spider_exceptions/count") >= close_on assert crawler.stats.get_value("spider_exceptions/count") >= close_on
assert errorcount >= close_on assert errorcount >= close_on
@inlineCallbacks @inline_callbacks_test
def test_closespider_timeout(self): def test_closespider_timeout(self):
close_on = 0.1 close_on = 0.1
crawler = get_crawler(FollowAllSpider, {"CLOSESPIDER_TIMEOUT": close_on}) crawler = get_crawler(FollowAllSpider, {"CLOSESPIDER_TIMEOUT": close_on})
@ -102,7 +102,7 @@ class TestCloseSpider:
total_seconds = crawler.stats.get_value("elapsed_time_seconds") total_seconds = crawler.stats.get_value("elapsed_time_seconds")
assert total_seconds >= close_on assert total_seconds >= close_on
@inlineCallbacks @inline_callbacks_test
def test_closespider_timeout_no_item(self): def test_closespider_timeout_no_item(self):
timeout = 1 timeout = 1
crawler = get_crawler(SlowSpider, {"CLOSESPIDER_TIMEOUT_NO_ITEM": timeout}) crawler = get_crawler(SlowSpider, {"CLOSESPIDER_TIMEOUT_NO_ITEM": timeout})

4
tests/test_contracts.py
View File

@ -18,7 +18,7 @@ from scrapy.spidermiddlewares.httperror import HttpError
from scrapy.spiders import Spider from scrapy.spiders import Spider
from scrapy.utils.test import get_crawler from scrapy.utils.test import get_crawler
from tests.mockserver.http import MockServer from tests.mockserver.http import MockServer
from tests.utils.decorators import inlineCallbacks from tests.utils.decorators import inline_callbacks_test
class DemoItem(Item): class DemoItem(Item):
@ -502,7 +502,7 @@ class TestContractsManager:
assert self.results.errors assert self.results.errors
@pytest.mark.requires_http_handler @pytest.mark.requires_http_handler
@inlineCallbacks @inline_callbacks_test
def test_same_url(self): def test_same_url(self):
class TestSameUrlSpider(Spider): class TestSameUrlSpider(Spider):
name = "test_same_url" name = "test_same_url"

14
tests/test_core_downloader.py
View File

@ -25,7 +25,7 @@ from scrapy.utils.spider import DefaultSpider
from scrapy.utils.test import get_crawler from scrapy.utils.test import get_crawler
from tests.mockserver.http_resources import PayloadResource from tests.mockserver.http_resources import PayloadResource
from tests.mockserver.utils import ssl_context_factory from tests.mockserver.utils import ssl_context_factory
from tests.utils.decorators import deferred_f_from_coro_f from tests.utils.decorators import coroutine_test
if TYPE_CHECKING: if TYPE_CHECKING:
from twisted.internet.defer import Deferred from twisted.internet.defer import Deferred
@ -97,7 +97,7 @@ class TestContextFactoryBase:
class TestContextFactory(TestContextFactoryBase): class TestContextFactory(TestContextFactoryBase):
@deferred_f_from_coro_f @coroutine_test
async def testPayload(self, server_url: str) -> None: async def testPayload(self, server_url: str) -> None:
s = "0123456789" * 10 s = "0123456789" * 10
crawler = get_crawler() crawler = get_crawler()
@ -135,7 +135,7 @@ class TestContextFactoryTLSMethod(TestContextFactoryBase):
) )
assert body == to_bytes(s) assert body == to_bytes(s)
@deferred_f_from_coro_f @coroutine_test
async def test_setting_default(self, server_url: str) -> None: async def test_setting_default(self, server_url: str) -> None:
crawler = get_crawler() crawler = get_crawler()
settings = Settings() settings = Settings()
@ -155,7 +155,7 @@ class TestContextFactoryTLSMethod(TestContextFactoryBase):
with pytest.raises(KeyError): with pytest.raises(KeyError):
load_context_factory_from_settings(settings, crawler) load_context_factory_from_settings(settings, crawler)
@deferred_f_from_coro_f @coroutine_test
async def test_setting_explicit(self, server_url: str) -> None: async def test_setting_explicit(self, server_url: str) -> None:
crawler = get_crawler() crawler = get_crawler()
settings = Settings({"DOWNLOADER_CLIENT_TLS_METHOD": "TLSv1.2"}) settings = Settings({"DOWNLOADER_CLIENT_TLS_METHOD": "TLSv1.2"})
@ -163,7 +163,7 @@ class TestContextFactoryTLSMethod(TestContextFactoryBase):
assert client_context_factory._ssl_method == OpenSSL.SSL.TLSv1_2_METHOD assert client_context_factory._ssl_method == OpenSSL.SSL.TLSv1_2_METHOD
await self._assert_factory_works(server_url, client_context_factory) await self._assert_factory_works(server_url, client_context_factory)
@deferred_f_from_coro_f @coroutine_test
async def test_direct_from_crawler(self, server_url: str) -> None: async def test_direct_from_crawler(self, server_url: str) -> None:
# the setting is ignored # the setting is ignored
crawler = get_crawler(settings_dict={"DOWNLOADER_CLIENT_TLS_METHOD": "bad"}) crawler = get_crawler(settings_dict={"DOWNLOADER_CLIENT_TLS_METHOD": "bad"})
@ -171,14 +171,14 @@ class TestContextFactoryTLSMethod(TestContextFactoryBase):
assert client_context_factory._ssl_method == OpenSSL.SSL.SSLv23_METHOD assert client_context_factory._ssl_method == OpenSSL.SSL.SSLv23_METHOD
await self._assert_factory_works(server_url, client_context_factory) await self._assert_factory_works(server_url, client_context_factory)
@deferred_f_from_coro_f @coroutine_test
async def test_direct_init(self, server_url: str) -> None: async def test_direct_init(self, server_url: str) -> None:
client_context_factory = ScrapyClientContextFactory(OpenSSL.SSL.TLSv1_2_METHOD) client_context_factory = ScrapyClientContextFactory(OpenSSL.SSL.TLSv1_2_METHOD)
assert client_context_factory._ssl_method == OpenSSL.SSL.TLSv1_2_METHOD assert client_context_factory._ssl_method == OpenSSL.SSL.TLSv1_2_METHOD
await self._assert_factory_works(server_url, client_context_factory) await self._assert_factory_works(server_url, client_context_factory)
@deferred_f_from_coro_f @coroutine_test
async def test_fetch_deprecated_spider_arg(): async def test_fetch_deprecated_spider_arg():
class CustomDownloader(Downloader): class CustomDownloader(Downloader):
def fetch(self, request, spider): # pylint: disable=signature-differs def fetch(self, request, spider): # pylint: disable=signature-differs

4
tests/test_core_scraper.py
View File

@ -6,14 +6,14 @@ import pytest
from scrapy.utils.test import get_crawler from scrapy.utils.test import get_crawler
from tests.spiders import SimpleSpider from tests.spiders import SimpleSpider
from tests.utils.decorators import deferred_f_from_coro_f from tests.utils.decorators import coroutine_test
if TYPE_CHECKING: if TYPE_CHECKING:
from tests.mockserver.http import MockServer from tests.mockserver.http import MockServer
@pytest.mark.requires_http_handler @pytest.mark.requires_http_handler
@deferred_f_from_coro_f @coroutine_test
async def test_scraper_exception( async def test_scraper_exception(
mockserver: MockServer, mockserver: MockServer,
caplog: pytest.LogCaptureFixture, caplog: pytest.LogCaptureFixture,

122
tests/test_crawl.py
View File

@ -55,7 +55,7 @@ from tests.spiders import (
StartGoodAndBadOutput, StartGoodAndBadOutput,
StartItemSpider, StartItemSpider,
) )
from tests.utils.decorators import deferred_f_from_coro_f, inlineCallbacks from tests.utils.decorators import coroutine_test, inline_callbacks_test
if TYPE_CHECKING: if TYPE_CHECKING:
from scrapy.statscollectors import StatsCollector from scrapy.statscollectors import StatsCollector
@ -74,17 +74,17 @@ class TestCrawl:
def teardown_class(cls): def teardown_class(cls):
cls.mockserver.__exit__(None, None, None) cls.mockserver.__exit__(None, None, None)
@inlineCallbacks @inline_callbacks_test
def test_follow_all(self): def test_follow_all(self):
crawler = get_crawler(FollowAllSpider) crawler = get_crawler(FollowAllSpider)
yield crawler.crawl(mockserver=self.mockserver) yield crawler.crawl(mockserver=self.mockserver)
assert len(crawler.spider.urls_visited) == 11 # 10 + start_url assert len(crawler.spider.urls_visited) == 11 # 10 + start_url
@deferred_f_from_coro_f @coroutine_test
async def test_fixed_delay(self): async def test_fixed_delay(self):
await self._test_delay(total=3, delay=0.2) await self._test_delay(total=3, delay=0.2)
@deferred_f_from_coro_f @coroutine_test
async def test_randomized_delay(self): async def test_randomized_delay(self):
await self._test_delay(total=3, delay=0.1, randomize=True) await self._test_delay(total=3, delay=0.1, randomize=True)
@ -122,7 +122,7 @@ class TestCrawl:
average = total_time / (len(times) - 1) average = total_time / (len(times) - 1)
assert average <= delay / tolerance, "test total or delay values are too small" assert average <= delay / tolerance, "test total or delay values are too small"
@inlineCallbacks @inline_callbacks_test
def test_timeout_success(self): def test_timeout_success(self):
crawler = get_crawler(DelaySpider) crawler = get_crawler(DelaySpider)
yield crawler.crawl(n=0.5, mockserver=self.mockserver) yield crawler.crawl(n=0.5, mockserver=self.mockserver)
@ -130,7 +130,7 @@ class TestCrawl:
assert crawler.spider.t2 > 0 assert crawler.spider.t2 > 0
assert crawler.spider.t2 > crawler.spider.t1 assert crawler.spider.t2 > crawler.spider.t1
@inlineCallbacks @inline_callbacks_test
def test_timeout_failure(self): def test_timeout_failure(self):
crawler = get_crawler(DelaySpider, {"DOWNLOAD_TIMEOUT": 0.35}) crawler = get_crawler(DelaySpider, {"DOWNLOAD_TIMEOUT": 0.35})
yield crawler.crawl(n=0.5, mockserver=self.mockserver) yield crawler.crawl(n=0.5, mockserver=self.mockserver)
@ -147,7 +147,7 @@ class TestCrawl:
assert crawler.spider.t2_err > 0 assert crawler.spider.t2_err > 0
assert crawler.spider.t2_err > crawler.spider.t1 assert crawler.spider.t2_err > crawler.spider.t1
@inlineCallbacks @inline_callbacks_test
def test_retry_503(self): def test_retry_503(self):
crawler = get_crawler(SimpleSpider) crawler = get_crawler(SimpleSpider)
with LogCapture() as log: with LogCapture() as log:
@ -156,7 +156,7 @@ class TestCrawl:
) )
self._assert_retried(log) self._assert_retried(log)
@inlineCallbacks @inline_callbacks_test
def test_retry_conn_failed(self): def test_retry_conn_failed(self):
crawler = get_crawler(SimpleSpider) crawler = get_crawler(SimpleSpider)
with LogCapture() as log: with LogCapture() as log:
@ -165,7 +165,7 @@ class TestCrawl:
) )
self._assert_retried(log) self._assert_retried(log)
@inlineCallbacks @inline_callbacks_test
def test_retry_dns_error(self): def test_retry_dns_error(self):
if NON_EXISTING_RESOLVABLE: if NON_EXISTING_RESOLVABLE:
pytest.skip("Non-existing hosts are resolvable") pytest.skip("Non-existing hosts are resolvable")
@ -177,7 +177,7 @@ class TestCrawl:
) )
self._assert_retried(log) self._assert_retried(log)
@inlineCallbacks @inline_callbacks_test
def test_start_bug_before_yield(self): def test_start_bug_before_yield(self):
with LogCapture("scrapy", level=logging.ERROR) as log: with LogCapture("scrapy", level=logging.ERROR) as log:
crawler = get_crawler(BrokenStartSpider) crawler = get_crawler(BrokenStartSpider)
@ -188,7 +188,7 @@ class TestCrawl:
assert record.exc_info is not None assert record.exc_info is not None
assert record.exc_info[0] is ZeroDivisionError assert record.exc_info[0] is ZeroDivisionError
@inlineCallbacks @inline_callbacks_test
def test_start_bug_yielding(self): def test_start_bug_yielding(self):
with LogCapture("scrapy", level=logging.ERROR) as log: with LogCapture("scrapy", level=logging.ERROR) as log:
crawler = get_crawler(BrokenStartSpider) crawler = get_crawler(BrokenStartSpider)
@ -199,7 +199,7 @@ class TestCrawl:
assert record.exc_info is not None assert record.exc_info is not None
assert record.exc_info[0] is ZeroDivisionError assert record.exc_info[0] is ZeroDivisionError
@inlineCallbacks @inline_callbacks_test
def test_start_items(self): def test_start_items(self):
items = [] items = []
@ -214,7 +214,7 @@ class TestCrawl:
assert len(log.records) == 0 assert len(log.records) == 0
assert items == [{"name": "test item"}] assert items == [{"name": "test item"}]
@inlineCallbacks @inline_callbacks_test
def test_start_unsupported_output(self): def test_start_unsupported_output(self):
"""Anything that is not a request is assumed to be an item, avoiding a """Anything that is not a request is assumed to be an item, avoiding a
potentially expensive call to itemadapter.is_item(), and letting potentially expensive call to itemadapter.is_item(), and letting
@ -235,7 +235,7 @@ class TestCrawl:
assert len(items) == 3 assert len(items) == 3
assert not any(isinstance(item, Request) for item in items) assert not any(isinstance(item, Request) for item in items)
@inlineCallbacks @inline_callbacks_test
def test_start_dupes(self): def test_start_dupes(self):
settings = {"CONCURRENT_REQUESTS": 1} settings = {"CONCURRENT_REQUESTS": 1}
crawler = get_crawler(DuplicateStartSpider, settings) crawler = get_crawler(DuplicateStartSpider, settings)
@ -253,7 +253,7 @@ class TestCrawl:
) )
assert crawler.spider.visited == 3 assert crawler.spider.visited == 3
@inlineCallbacks @inline_callbacks_test
def test_unbounded_response(self): def test_unbounded_response(self):
# Completeness of responses without Content-Length or Transfer-Encoding # Completeness of responses without Content-Length or Transfer-Encoding
# can not be determined, we treat them as valid but flagged as "partial" # can not be determined, we treat them as valid but flagged as "partial"
@ -285,7 +285,7 @@ with multiples lines
) )
assert str(log).count("Got response 200") == 1 assert str(log).count("Got response 200") == 1
@inlineCallbacks @inline_callbacks_test
def test_retry_conn_lost(self): def test_retry_conn_lost(self):
# connection lost after receiving data # connection lost after receiving data
crawler = get_crawler(SimpleSpider) crawler = get_crawler(SimpleSpider)
@ -295,7 +295,7 @@ with multiples lines
) )
self._assert_retried(log) self._assert_retried(log)
@inlineCallbacks @inline_callbacks_test
def test_retry_conn_aborted(self): def test_retry_conn_aborted(self):
# connection lost before receiving data # connection lost before receiving data
crawler = get_crawler(SimpleSpider) crawler = get_crawler(SimpleSpider)
@ -309,7 +309,7 @@ with multiples lines
assert str(log).count("Retrying") == 2 assert str(log).count("Retrying") == 2
assert str(log).count("Gave up retrying") == 1 assert str(log).count("Gave up retrying") == 1
@inlineCallbacks @inline_callbacks_test
def test_referer_header(self): def test_referer_header(self):
"""Referer header is set by RefererMiddleware unless it is already set""" """Referer header is set by RefererMiddleware unless it is already set"""
req0 = Request(self.mockserver.url("/echo?headers=1&body=0"), dont_filter=1) req0 = Request(self.mockserver.url("/echo?headers=1&body=0"), dont_filter=1)
@ -337,7 +337,7 @@ with multiples lines
echo3 = json.loads(to_unicode(crawler.spider.meta["responses"][3].body)) echo3 = json.loads(to_unicode(crawler.spider.meta["responses"][3].body))
assert echo3["headers"].get("Referer") == ["http://example.com"] assert echo3["headers"].get("Referer") == ["http://example.com"]
@inlineCallbacks @inline_callbacks_test
def test_engine_status(self): def test_engine_status(self):
est = [] est = []
@ -353,7 +353,7 @@ with multiples lines
assert s["engine.spider.name"] == crawler.spider.name assert s["engine.spider.name"] == crawler.spider.name
assert s["len(engine.scraper.slot.active)"] == 1 assert s["len(engine.scraper.slot.active)"] == 1
@inlineCallbacks @inline_callbacks_test
def test_format_engine_status(self): def test_format_engine_status(self):
est = [] est = []
@ -376,7 +376,7 @@ with multiples lines
assert s["engine.spider.name"] == crawler.spider.name assert s["engine.spider.name"] == crawler.spider.name
assert s["len(engine.scraper.slot.active)"] == "1" assert s["len(engine.scraper.slot.active)"] == "1"
@inlineCallbacks @inline_callbacks_test
def test_open_spider_error_on_faulty_pipeline(self): def test_open_spider_error_on_faulty_pipeline(self):
settings = { settings = {
"ITEM_PIPELINES": { "ITEM_PIPELINES": {
@ -390,7 +390,7 @@ with multiples lines
) )
assert not crawler.crawling assert not crawler.crawling
@inlineCallbacks @inline_callbacks_test
def test_crawlerrunner_accepts_crawler(self): def test_crawlerrunner_accepts_crawler(self):
crawler = get_crawler(SimpleSpider) crawler = get_crawler(SimpleSpider)
runner = CrawlerRunner() runner = CrawlerRunner()
@ -402,7 +402,7 @@ with multiples lines
) )
assert "Got response 200" in str(log) assert "Got response 200" in str(log)
@inlineCallbacks @inline_callbacks_test
def test_crawl_multiple(self, caplog: pytest.LogCaptureFixture): def test_crawl_multiple(self, caplog: pytest.LogCaptureFixture):
runner = CrawlerRunner(get_reactor_settings()) runner = CrawlerRunner(get_reactor_settings())
runner.crawl( runner.crawl(
@ -422,7 +422,7 @@ with multiples lines
self._assert_retried(caplog.text) self._assert_retried(caplog.text)
assert "Got response 200" in caplog.text assert "Got response 200" in caplog.text
@deferred_f_from_coro_f @coroutine_test
async def test_unknown_url_scheme(self, caplog: pytest.LogCaptureFixture) -> None: async def test_unknown_url_scheme(self, caplog: pytest.LogCaptureFixture) -> None:
crawler = get_crawler(SimpleSpider) crawler = get_crawler(SimpleSpider)
await maybe_deferred_to_future(crawler.crawl("foo://bar")) await maybe_deferred_to_future(crawler.crawl("foo://bar"))
@ -461,7 +461,7 @@ class TestCrawlSpider:
assert crawler.stats assert crawler.stats
return log, items, crawler.stats return log, items, crawler.stats
@inlineCallbacks @inline_callbacks_test
def test_crawlspider_with_parse(self): def test_crawlspider_with_parse(self):
crawler = get_crawler(CrawlSpiderWithParseMethod) crawler = get_crawler(CrawlSpiderWithParseMethod)
with LogCapture() as log: with LogCapture() as log:
@ -471,7 +471,7 @@ class TestCrawlSpider:
assert "[parse] status 201 (foo: None)" in str(log) assert "[parse] status 201 (foo: None)" in str(log)
assert "[parse] status 202 (foo: bar)" in str(log) assert "[parse] status 202 (foo: bar)" in str(log)
@inlineCallbacks @inline_callbacks_test
def test_crawlspider_with_async_callback(self): def test_crawlspider_with_async_callback(self):
crawler = get_crawler(CrawlSpiderWithAsyncCallback) crawler = get_crawler(CrawlSpiderWithAsyncCallback)
with LogCapture() as log: with LogCapture() as log:
@ -481,7 +481,7 @@ class TestCrawlSpider:
assert "[parse_async] status 201 (foo: None)" in str(log) assert "[parse_async] status 201 (foo: None)" in str(log)
assert "[parse_async] status 202 (foo: bar)" in str(log) assert "[parse_async] status 202 (foo: bar)" in str(log)
@inlineCallbacks @inline_callbacks_test
def test_crawlspider_with_async_generator_callback(self): def test_crawlspider_with_async_generator_callback(self):
crawler = get_crawler(CrawlSpiderWithAsyncGeneratorCallback) crawler = get_crawler(CrawlSpiderWithAsyncGeneratorCallback)
with LogCapture() as log: with LogCapture() as log:
@ -491,7 +491,7 @@ class TestCrawlSpider:
assert "[parse_async_gen] status 201 (foo: None)" in str(log) assert "[parse_async_gen] status 201 (foo: None)" in str(log)
assert "[parse_async_gen] status 202 (foo: bar)" in str(log) assert "[parse_async_gen] status 202 (foo: bar)" in str(log)
@inlineCallbacks @inline_callbacks_test
def test_crawlspider_with_errback(self): def test_crawlspider_with_errback(self):
crawler = get_crawler(CrawlSpiderWithErrback) crawler = get_crawler(CrawlSpiderWithErrback)
with LogCapture() as log: with LogCapture() as log:
@ -504,7 +504,7 @@ class TestCrawlSpider:
assert "[errback] status 500" in str(log) assert "[errback] status 500" in str(log)
assert "[errback] status 501" in str(log) assert "[errback] status 501" in str(log)
@inlineCallbacks @inline_callbacks_test
def test_crawlspider_process_request_cb_kwargs(self): def test_crawlspider_process_request_cb_kwargs(self):
crawler = get_crawler(CrawlSpiderWithProcessRequestCallbackKeywordArguments) crawler = get_crawler(CrawlSpiderWithProcessRequestCallbackKeywordArguments)
with LogCapture() as log: with LogCapture() as log:
@ -514,7 +514,7 @@ class TestCrawlSpider:
assert "[parse] status 201 (foo: process_request)" in str(log) assert "[parse] status 201 (foo: process_request)" in str(log)
assert "[parse] status 202 (foo: bar)" in str(log) assert "[parse] status 202 (foo: bar)" in str(log)
@inlineCallbacks @inline_callbacks_test
def test_async_def_parse(self): def test_async_def_parse(self):
crawler = get_crawler(AsyncDefSpider) crawler = get_crawler(AsyncDefSpider)
with LogCapture() as log: with LogCapture() as log:
@ -524,7 +524,7 @@ class TestCrawlSpider:
assert "Got response 200" in str(log) assert "Got response 200" in str(log)
@pytest.mark.only_asyncio @pytest.mark.only_asyncio
@inlineCallbacks @inline_callbacks_test
def test_async_def_asyncio_parse(self): def test_async_def_asyncio_parse(self):
crawler = get_crawler( crawler = get_crawler(
AsyncDefAsyncioSpider, AsyncDefAsyncioSpider,
@ -539,7 +539,7 @@ class TestCrawlSpider:
assert "Got response 200" in str(log) assert "Got response 200" in str(log)
@pytest.mark.only_asyncio @pytest.mark.only_asyncio
@deferred_f_from_coro_f @coroutine_test
async def test_async_def_asyncio_parse_items_list(self): async def test_async_def_asyncio_parse_items_list(self):
log, items, _ = await self._run_spider(AsyncDefAsyncioReturnSpider) log, items, _ = await self._run_spider(AsyncDefAsyncioReturnSpider)
assert "Got response 200" in str(log) assert "Got response 200" in str(log)
@ -547,7 +547,7 @@ class TestCrawlSpider:
assert {"id": 2} in items assert {"id": 2} in items
@pytest.mark.only_asyncio @pytest.mark.only_asyncio
@inlineCallbacks @inline_callbacks_test
def test_async_def_asyncio_parse_items_single_element(self): def test_async_def_asyncio_parse_items_single_element(self):
items = [] items = []
@ -564,7 +564,7 @@ class TestCrawlSpider:
assert {"foo": 42} in items assert {"foo": 42} in items
@pytest.mark.only_asyncio @pytest.mark.only_asyncio
@deferred_f_from_coro_f @coroutine_test
async def test_async_def_asyncgen_parse(self): async def test_async_def_asyncgen_parse(self):
log, _, stats = await self._run_spider(AsyncDefAsyncioGenSpider) log, _, stats = await self._run_spider(AsyncDefAsyncioGenSpider)
assert "Got response 200" in str(log) assert "Got response 200" in str(log)
@ -572,7 +572,7 @@ class TestCrawlSpider:
assert itemcount == 1 assert itemcount == 1
@pytest.mark.only_asyncio @pytest.mark.only_asyncio
@deferred_f_from_coro_f @coroutine_test
async def test_async_def_asyncgen_parse_loop(self): async def test_async_def_asyncgen_parse_loop(self):
log, items, stats = await self._run_spider(AsyncDefAsyncioGenLoopSpider) log, items, stats = await self._run_spider(AsyncDefAsyncioGenLoopSpider)
assert "Got response 200" in str(log) assert "Got response 200" in str(log)
@ -582,7 +582,7 @@ class TestCrawlSpider:
assert {"foo": i} in items assert {"foo": i} in items
@pytest.mark.only_asyncio @pytest.mark.only_asyncio
@deferred_f_from_coro_f @coroutine_test
async def test_async_def_asyncgen_parse_exc(self): async def test_async_def_asyncgen_parse_exc(self):
log, items, stats = await self._run_spider(AsyncDefAsyncioGenExcSpider) log, items, stats = await self._run_spider(AsyncDefAsyncioGenExcSpider)
log = str(log) log = str(log)
@ -594,7 +594,7 @@ class TestCrawlSpider:
assert {"foo": i} in items assert {"foo": i} in items
@pytest.mark.only_asyncio @pytest.mark.only_asyncio
@deferred_f_from_coro_f @coroutine_test
async def test_async_def_asyncgen_parse_complex(self): async def test_async_def_asyncgen_parse_complex(self):
_, items, stats = await self._run_spider(AsyncDefAsyncioGenComplexSpider) _, items, stats = await self._run_spider(AsyncDefAsyncioGenComplexSpider)
itemcount = stats.get_value("item_scraped_count") itemcount = stats.get_value("item_scraped_count")
@ -606,30 +606,30 @@ class TestCrawlSpider:
assert {"index2": i} in items assert {"index2": i} in items
@pytest.mark.only_asyncio @pytest.mark.only_asyncio
@deferred_f_from_coro_f @coroutine_test
async def test_async_def_asyncio_parse_reqs_list(self): async def test_async_def_asyncio_parse_reqs_list(self):
log, *_ = await self._run_spider(AsyncDefAsyncioReqsReturnSpider) log, *_ = await self._run_spider(AsyncDefAsyncioReqsReturnSpider)
for req_id in range(3): for req_id in range(3):
assert f"Got response 200, req_id {req_id}" in str(log) assert f"Got response 200, req_id {req_id}" in str(log)
@pytest.mark.only_not_asyncio @pytest.mark.only_not_asyncio
@deferred_f_from_coro_f @coroutine_test
async def test_async_def_deferred_direct(self): async def test_async_def_deferred_direct(self):
_, items, _ = await self._run_spider(AsyncDefDeferredDirectSpider) _, items, _ = await self._run_spider(AsyncDefDeferredDirectSpider)
assert items == [{"code": 200}] assert items == [{"code": 200}]
@pytest.mark.only_asyncio @pytest.mark.only_asyncio
@deferred_f_from_coro_f @coroutine_test
async def test_async_def_deferred_wrapped(self): async def test_async_def_deferred_wrapped(self):
_, items, _ = await self._run_spider(AsyncDefDeferredWrappedSpider) _, items, _ = await self._run_spider(AsyncDefDeferredWrappedSpider)
assert items == [{"code": 200}] assert items == [{"code": 200}]
@deferred_f_from_coro_f @coroutine_test
async def test_async_def_deferred_maybe_wrapped(self): async def test_async_def_deferred_maybe_wrapped(self):
_, items, _ = await self._run_spider(AsyncDefDeferredMaybeWrappedSpider) _, items, _ = await self._run_spider(AsyncDefDeferredMaybeWrappedSpider)
assert items == [{"code": 200}] assert items == [{"code": 200}]
@inlineCallbacks @inline_callbacks_test
def test_response_ssl_certificate_none(self): def test_response_ssl_certificate_none(self):
crawler = get_crawler(SingleRequestSpider) crawler = get_crawler(SingleRequestSpider)
url = self.mockserver.url("/echo?body=test", is_secure=False) url = self.mockserver.url("/echo?body=test", is_secure=False)
@ -649,7 +649,7 @@ class TestCrawlSpider:
), ),
], ],
) )
@deferred_f_from_coro_f @coroutine_test
async def test_response_ssl_certificate( async def test_response_ssl_certificate(
self, mockserver: MockServer, url: str self, mockserver: MockServer, url: str
) -> None: ) -> None:
@ -675,7 +675,7 @@ class TestCrawlSpider:
), ),
], ],
) )
@deferred_f_from_coro_f @coroutine_test
async def test_response_ip_address(self, mockserver: MockServer, url: str) -> None: async def test_response_ip_address(self, mockserver: MockServer, url: str) -> None:
crawler = get_crawler(SingleRequestSpider) crawler = get_crawler(SingleRequestSpider)
url = mockserver.url(url) url = mockserver.url(url)
@ -686,7 +686,7 @@ class TestCrawlSpider:
assert isinstance(ip_address, IPv4Address) assert isinstance(ip_address, IPv4Address)
assert str(ip_address) == gethostbyname(expected_netloc) assert str(ip_address) == gethostbyname(expected_netloc)
@inlineCallbacks @inline_callbacks_test
def test_bytes_received_stop_download_callback(self): def test_bytes_received_stop_download_callback(self):
crawler = get_crawler(BytesReceivedCallbackSpider) crawler = get_crawler(BytesReceivedCallbackSpider)
yield crawler.crawl(mockserver=self.mockserver) yield crawler.crawl(mockserver=self.mockserver)
@ -700,7 +700,7 @@ class TestCrawlSpider:
< crawler.spider.full_response_length < crawler.spider.full_response_length
) )
@inlineCallbacks @inline_callbacks_test
def test_bytes_received_stop_download_errback(self): def test_bytes_received_stop_download_errback(self):
crawler = get_crawler(BytesReceivedErrbackSpider) crawler = get_crawler(BytesReceivedErrbackSpider)
yield crawler.crawl(mockserver=self.mockserver) yield crawler.crawl(mockserver=self.mockserver)
@ -716,7 +716,7 @@ class TestCrawlSpider:
< crawler.spider.full_response_length < crawler.spider.full_response_length
) )
@inlineCallbacks @inline_callbacks_test
def test_headers_received_stop_download_callback(self): def test_headers_received_stop_download_callback(self):
crawler = get_crawler(HeadersReceivedCallbackSpider) crawler = get_crawler(HeadersReceivedCallbackSpider)
yield crawler.crawl(mockserver=self.mockserver) yield crawler.crawl(mockserver=self.mockserver)
@ -726,7 +726,7 @@ class TestCrawlSpider:
"headers_received" "headers_received"
) )
@inlineCallbacks @inline_callbacks_test
def test_headers_received_stop_download_errback(self): def test_headers_received_stop_download_errback(self):
crawler = get_crawler(HeadersReceivedErrbackSpider) crawler = get_crawler(HeadersReceivedErrbackSpider)
yield crawler.crawl(mockserver=self.mockserver) yield crawler.crawl(mockserver=self.mockserver)
@ -738,7 +738,7 @@ class TestCrawlSpider:
"failure" "failure"
].value.response.headers == crawler.spider.meta.get("headers_received") ].value.response.headers == crawler.spider.meta.get("headers_received")
@inlineCallbacks @inline_callbacks_test
def test_spider_callback_deferred_deprecated(self): def test_spider_callback_deferred_deprecated(self):
def cb(response: Response) -> Any: def cb(response: Response) -> Any:
return succeed(None) return succeed(None)
@ -750,7 +750,7 @@ class TestCrawlSpider:
): ):
yield crawler.crawl(seed=self.mockserver.url("/"), callback_func=cb) yield crawler.crawl(seed=self.mockserver.url("/"), callback_func=cb)
@inlineCallbacks @inline_callbacks_test
def test_spider_errback(self): def test_spider_errback(self):
failures = [] failures = []
@ -767,7 +767,7 @@ class TestCrawlSpider:
assert "HTTP status code is not handled or not allowed" in str(log) assert "HTTP status code is not handled or not allowed" in str(log)
assert "Spider error processing" not in str(log) assert "Spider error processing" not in str(log)
@inlineCallbacks @inline_callbacks_test
def test_spider_errback_silence(self): def test_spider_errback_silence(self):
failures = [] failures = []
@ -783,7 +783,7 @@ class TestCrawlSpider:
assert "HTTP status code is not handled or not allowed" not in str(log) assert "HTTP status code is not handled or not allowed" not in str(log)
assert "Spider error processing" not in str(log) assert "Spider error processing" not in str(log)
@inlineCallbacks @inline_callbacks_test
def test_spider_errback_exception(self): def test_spider_errback_exception(self):
def eb(failure: Failure) -> None: def eb(failure: Failure) -> None:
raise ValueError("foo") raise ValueError("foo")
@ -795,7 +795,7 @@ class TestCrawlSpider:
) )
assert "Spider error processing" in str(log) assert "Spider error processing" in str(log)
@inlineCallbacks @inline_callbacks_test
def test_spider_errback_item(self): def test_spider_errback_item(self):
def eb(failure: Failure) -> Any: def eb(failure: Failure) -> Any:
return {"foo": "bar"} return {"foo": "bar"}
@ -809,7 +809,7 @@ class TestCrawlSpider:
assert "Spider error processing" not in str(log) assert "Spider error processing" not in str(log)
assert "'item_scraped_count': 1" in str(log) assert "'item_scraped_count': 1" in str(log)
@inlineCallbacks @inline_callbacks_test
def test_spider_errback_request(self): def test_spider_errback_request(self):
def eb(failure: Failure) -> Request: def eb(failure: Failure) -> Request:
return Request(self.mockserver.url("/")) return Request(self.mockserver.url("/"))
@ -823,7 +823,7 @@ class TestCrawlSpider:
assert "Spider error processing" not in str(log) assert "Spider error processing" not in str(log)
assert "Crawled (200)" in str(log) assert "Crawled (200)" in str(log)
@inlineCallbacks @inline_callbacks_test
def test_spider_errback_downloader_error(self): def test_spider_errback_downloader_error(self):
failures = [] failures = []
@ -840,7 +840,7 @@ class TestCrawlSpider:
assert "Error downloading" in str(log) assert "Error downloading" in str(log)
assert "Spider error processing" not in str(log) assert "Spider error processing" not in str(log)
@inlineCallbacks @inline_callbacks_test
def test_spider_errback_downloader_error_exception(self): def test_spider_errback_downloader_error_exception(self):
def eb(failure: Failure) -> None: def eb(failure: Failure) -> None:
raise ValueError("foo") raise ValueError("foo")
@ -853,7 +853,7 @@ class TestCrawlSpider:
assert "Error downloading" in str(log) assert "Error downloading" in str(log)
assert "Spider error processing" in str(log) assert "Spider error processing" in str(log)
@inlineCallbacks @inline_callbacks_test
def test_spider_errback_downloader_error_item(self): def test_spider_errback_downloader_error_item(self):
def eb(failure: Failure) -> Any: def eb(failure: Failure) -> Any:
return {"foo": "bar"} return {"foo": "bar"}
@ -867,7 +867,7 @@ class TestCrawlSpider:
assert "Spider error processing" not in str(log) assert "Spider error processing" not in str(log)
assert "'item_scraped_count': 1" in str(log) assert "'item_scraped_count': 1" in str(log)
@inlineCallbacks @inline_callbacks_test
def test_spider_errback_downloader_error_request(self): def test_spider_errback_downloader_error_request(self):
def eb(failure: Failure) -> Request: def eb(failure: Failure) -> Request:
return Request(self.mockserver.url("/")) return Request(self.mockserver.url("/"))
@ -881,7 +881,7 @@ class TestCrawlSpider:
assert "Spider error processing" not in str(log) assert "Spider error processing" not in str(log)
assert "Crawled (200)" in str(log) assert "Crawled (200)" in str(log)
@inlineCallbacks @inline_callbacks_test
def test_spider_errback_deferred_deprecated(self): def test_spider_errback_deferred_deprecated(self):
def eb(failure: Failure) -> Any: def eb(failure: Failure) -> Any:
return succeed(None) return succeed(None)
@ -895,7 +895,7 @@ class TestCrawlSpider:
seed=self.mockserver.url("/status?n=400"), errback_func=eb seed=self.mockserver.url("/status?n=400"), errback_func=eb
) )
@inlineCallbacks @inline_callbacks_test
def test_raise_closespider(self): def test_raise_closespider(self):
def cb(response): def cb(response):
raise CloseSpider raise CloseSpider
@ -906,7 +906,7 @@ class TestCrawlSpider:
assert "Closing spider (cancelled)" in str(log) assert "Closing spider (cancelled)" in str(log)
assert "Spider error processing" not in str(log) assert "Spider error processing" not in str(log)
@inlineCallbacks @inline_callbacks_test
def test_raise_closespider_reason(self): def test_raise_closespider_reason(self):
def cb(response): def cb(response):
raise CloseSpider("my_reason") raise CloseSpider("my_reason")

42
tests/test_crawler.py
View File

@ -41,7 +41,7 @@ from scrapy.utils.spider import DefaultSpider
from scrapy.utils.test import get_crawler, get_reactor_settings from scrapy.utils.test import get_crawler, get_reactor_settings
from tests.mockserver.http import MockServer from tests.mockserver.http import MockServer
from tests.utils import get_script_run_env from tests.utils import get_script_run_env
from tests.utils.decorators import deferred_f_from_coro_f, inlineCallbacks from tests.utils.decorators import coroutine_test, inline_callbacks_test
BASE_SETTINGS: dict[str, Any] = {} BASE_SETTINGS: dict[str, Any] = {}
@ -101,21 +101,21 @@ class TestCrawler(TestBaseCrawler):
with pytest.raises(ValueError, match="spidercls argument must be a class"): with pytest.raises(ValueError, match="spidercls argument must be a class"):
Crawler(DefaultSpider()) Crawler(DefaultSpider())
@inlineCallbacks @inline_callbacks_test
def test_crawler_crawl_twice_seq_unsupported(self): def test_crawler_crawl_twice_seq_unsupported(self):
crawler = get_raw_crawler(NoRequestsSpider, BASE_SETTINGS) crawler = get_raw_crawler(NoRequestsSpider, BASE_SETTINGS)
yield crawler.crawl() yield crawler.crawl()
with pytest.raises(RuntimeError, match="more than once on the same instance"): with pytest.raises(RuntimeError, match="more than once on the same instance"):
yield crawler.crawl() yield crawler.crawl()
@deferred_f_from_coro_f @coroutine_test
async def test_crawler_crawl_async_twice_seq_unsupported(self): async def test_crawler_crawl_async_twice_seq_unsupported(self):
crawler = get_raw_crawler(NoRequestsSpider, BASE_SETTINGS) crawler = get_raw_crawler(NoRequestsSpider, BASE_SETTINGS)
await crawler.crawl_async() await crawler.crawl_async()
with pytest.raises(RuntimeError, match="more than once on the same instance"): with pytest.raises(RuntimeError, match="more than once on the same instance"):
await crawler.crawl_async() await crawler.crawl_async()
@inlineCallbacks @inline_callbacks_test
def test_crawler_crawl_twice_parallel_unsupported(self): def test_crawler_crawl_twice_parallel_unsupported(self):
crawler = get_raw_crawler(NoRequestsSpider, BASE_SETTINGS) crawler = get_raw_crawler(NoRequestsSpider, BASE_SETTINGS)
d1 = crawler.crawl() d1 = crawler.crawl()
@ -125,7 +125,7 @@ class TestCrawler(TestBaseCrawler):
yield d2 yield d2
@pytest.mark.only_asyncio @pytest.mark.only_asyncio
@deferred_f_from_coro_f @coroutine_test
async def test_crawler_crawl_async_twice_parallel_unsupported(self): async def test_crawler_crawl_async_twice_parallel_unsupported(self):
crawler = get_raw_crawler(NoRequestsSpider, BASE_SETTINGS) crawler = get_raw_crawler(NoRequestsSpider, BASE_SETTINGS)
t1 = asyncio.create_task(crawler.crawl_async()) t1 = asyncio.create_task(crawler.crawl_async())
@ -172,7 +172,7 @@ class TestCrawler(TestBaseCrawler):
addon = crawler.get_addon(ChildAddon) addon = crawler.get_addon(ChildAddon)
assert addon is None assert addon is None
@inlineCallbacks @inline_callbacks_test
def test_get_downloader_middleware(self): def test_get_downloader_middleware(self):
class ParentDownloaderMiddleware: class ParentDownloaderMiddleware:
pass pass
@ -235,7 +235,7 @@ class TestCrawler(TestBaseCrawler):
with pytest.raises(RuntimeError): with pytest.raises(RuntimeError):
crawler.get_downloader_middleware(DefaultSpider) crawler.get_downloader_middleware(DefaultSpider)
@inlineCallbacks @inline_callbacks_test
def test_get_downloader_middleware_no_engine(self): def test_get_downloader_middleware_no_engine(self):
class MySpider(Spider): class MySpider(Spider):
name = "myspider" name = "myspider"
@ -252,7 +252,7 @@ class TestCrawler(TestBaseCrawler):
with pytest.raises(RuntimeError): with pytest.raises(RuntimeError):
yield crawler.crawl() yield crawler.crawl()
@inlineCallbacks @inline_callbacks_test
def test_get_extension(self): def test_get_extension(self):
class ParentExtension: class ParentExtension:
pass pass
@ -315,7 +315,7 @@ class TestCrawler(TestBaseCrawler):
with pytest.raises(RuntimeError): with pytest.raises(RuntimeError):
crawler.get_extension(DefaultSpider) crawler.get_extension(DefaultSpider)
@inlineCallbacks @inline_callbacks_test
def test_get_extension_no_engine(self): def test_get_extension_no_engine(self):
class MySpider(Spider): class MySpider(Spider):
name = "myspider" name = "myspider"
@ -332,7 +332,7 @@ class TestCrawler(TestBaseCrawler):
with pytest.raises(RuntimeError): with pytest.raises(RuntimeError):
yield crawler.crawl() yield crawler.crawl()
@inlineCallbacks @inline_callbacks_test
def test_get_item_pipeline(self): def test_get_item_pipeline(self):
class ParentItemPipeline: class ParentItemPipeline:
pass pass
@ -395,7 +395,7 @@ class TestCrawler(TestBaseCrawler):
with pytest.raises(RuntimeError): with pytest.raises(RuntimeError):
crawler.get_item_pipeline(DefaultSpider) crawler.get_item_pipeline(DefaultSpider)
@inlineCallbacks @inline_callbacks_test
def test_get_item_pipeline_no_engine(self): def test_get_item_pipeline_no_engine(self):
class MySpider(Spider): class MySpider(Spider):
name = "myspider" name = "myspider"
@ -412,7 +412,7 @@ class TestCrawler(TestBaseCrawler):
with pytest.raises(RuntimeError): with pytest.raises(RuntimeError):
yield crawler.crawl() yield crawler.crawl()
@inlineCallbacks @inline_callbacks_test
def test_get_spider_middleware(self): def test_get_spider_middleware(self):
class ParentSpiderMiddleware: class ParentSpiderMiddleware:
pass pass
@ -475,7 +475,7 @@ class TestCrawler(TestBaseCrawler):
with pytest.raises(RuntimeError): with pytest.raises(RuntimeError):
crawler.get_spider_middleware(DefaultSpider) crawler.get_spider_middleware(DefaultSpider)
@inlineCallbacks @inline_callbacks_test
def test_get_spider_middleware_no_engine(self): def test_get_spider_middleware_no_engine(self):
class MySpider(Spider): class MySpider(Spider):
name = "myspider" name = "myspider"
@ -516,7 +516,7 @@ class TestCrawlerLogging:
get_crawler(MySpider) get_crawler(MySpider)
assert get_scrapy_root_handler() is None assert get_scrapy_root_handler() is None
@deferred_f_from_coro_f @coroutine_test
async def test_spider_custom_settings_log_level(self, tmp_path): async def test_spider_custom_settings_log_level(self, tmp_path):
log_file = Path(tmp_path, "log.txt") log_file = Path(tmp_path, "log.txt")
log_file.write_text("previous message\n", encoding="utf-8") log_file.write_text("previous message\n", encoding="utf-8")
@ -686,20 +686,20 @@ class TestCrawlerRunnerHasSpider:
def _crawl(runner, spider): def _crawl(runner, spider):
return runner.crawl(spider) return runner.crawl(spider)
@inlineCallbacks @inline_callbacks_test
def test_crawler_runner_bootstrap_successful(self): def test_crawler_runner_bootstrap_successful(self):
runner = self._runner() runner = self._runner()
yield self._crawl(runner, NoRequestsSpider) yield self._crawl(runner, NoRequestsSpider)
assert not runner.bootstrap_failed assert not runner.bootstrap_failed
@inlineCallbacks @inline_callbacks_test
def test_crawler_runner_bootstrap_successful_for_several(self): def test_crawler_runner_bootstrap_successful_for_several(self):
runner = self._runner() runner = self._runner()
yield self._crawl(runner, NoRequestsSpider) yield self._crawl(runner, NoRequestsSpider)
yield self._crawl(runner, NoRequestsSpider) yield self._crawl(runner, NoRequestsSpider)
assert not runner.bootstrap_failed assert not runner.bootstrap_failed
@inlineCallbacks @inline_callbacks_test
def test_crawler_runner_bootstrap_failed(self): def test_crawler_runner_bootstrap_failed(self):
runner = self._runner() runner = self._runner()
@ -712,7 +712,7 @@ class TestCrawlerRunnerHasSpider:
assert runner.bootstrap_failed assert runner.bootstrap_failed
@inlineCallbacks @inline_callbacks_test
def test_crawler_runner_bootstrap_failed_for_several(self): def test_crawler_runner_bootstrap_failed_for_several(self):
runner = self._runner() runner = self._runner()
@ -727,7 +727,7 @@ class TestCrawlerRunnerHasSpider:
assert runner.bootstrap_failed assert runner.bootstrap_failed
@inlineCallbacks @inline_callbacks_test
def test_crawler_runner_asyncio_enabled_true( def test_crawler_runner_asyncio_enabled_true(
self, reactor_pytest: str self, reactor_pytest: str
) -> Generator[Deferred[Any], Any, None]: ) -> Generator[Deferred[Any], Any, None]:
@ -959,7 +959,7 @@ class TestCrawlerProcessSubprocessBase(ScriptRunnerMixin):
p.expect_exact("Spider closed (shutdown)") p.expect_exact("Spider closed (shutdown)")
p.wait() p.wait()
@inlineCallbacks @inline_callbacks_test
def test_shutdown_forced(self): def test_shutdown_forced(self):
sig = signal.SIGINT if sys.platform != "win32" else signal.SIGBREAK sig = signal.SIGINT if sys.platform != "win32" else signal.SIGBREAK
args = self.get_script_args("sleeping.py", "10") args = self.get_script_args("sleeping.py", "10")
@ -1297,7 +1297,7 @@ def test_log_scrapy_info(settings, items, caplog):
assert re.search(r"^Versions:\n{'" + expected_items_pattern + "'}$", version_string) assert re.search(r"^Versions:\n{'" + expected_items_pattern + "'}$", version_string)
@deferred_f_from_coro_f @coroutine_test
async def test_deprecated_crawler_stop() -> None: async def test_deprecated_crawler_stop() -> None:
crawler = get_crawler(DefaultSpider) crawler = get_crawler(DefaultSpider)
d = crawler.crawl() d = crawler.crawl()

34
tests/test_downloader_handlers.py
View File

@ -21,7 +21,7 @@ from scrapy.responsetypes import responsetypes
from scrapy.utils.boto import is_botocore_available from scrapy.utils.boto import is_botocore_available
from scrapy.utils.misc import build_from_crawler from scrapy.utils.misc import build_from_crawler
from scrapy.utils.test import get_crawler from scrapy.utils.test import get_crawler
from tests.utils.decorators import deferred_f_from_coro_f from tests.utils.decorators import coroutine_test
class DummyDH: class DummyDH:
@ -126,7 +126,7 @@ class TestFile:
os.close(self.fd) os.close(self.fd)
Path(self.tmpname).unlink() Path(self.tmpname).unlink()
@deferred_f_from_coro_f @coroutine_test
async def test_download(self): async def test_download(self):
request = Request(path_to_file_uri(self.tmpname)) request = Request(path_to_file_uri(self.tmpname))
assert request.url.upper().endswith("%5E") assert request.url.upper().endswith("%5E")
@ -136,7 +136,7 @@ class TestFile:
assert response.body == b"0123456789" assert response.body == b"0123456789"
assert response.protocol is None assert response.protocol is None
@deferred_f_from_coro_f @coroutine_test
async def test_non_existent(self): async def test_non_existent(self):
request = Request(path_to_file_uri(mkdtemp())) request = Request(path_to_file_uri(mkdtemp()))
# the specific exception differs between platforms # the specific exception differs between platforms
@ -163,7 +163,7 @@ class TestS3Anon:
self.s3reqh = build_from_crawler(S3DownloadHandler, crawler) self.s3reqh = build_from_crawler(S3DownloadHandler, crawler)
self.download_request = self.s3reqh.download_request self.download_request = self.s3reqh.download_request
@deferred_f_from_coro_f @coroutine_test
async def test_anon_request(self): async def test_anon_request(self):
req = Request("s3://aws-publicdatasets/") req = Request("s3://aws-publicdatasets/")
httpreq = await self.download_request(req) httpreq = await self.download_request(req)
@ -205,7 +205,7 @@ class TestS3:
mock_formatdate.return_value = date mock_formatdate.return_value = date
yield yield
@deferred_f_from_coro_f @coroutine_test
async def test_request_signing1(self): async def test_request_signing1(self):
# gets an object from the johnsmith bucket. # gets an object from the johnsmith bucket.
date = "Tue, 27 Mar 2007 19:36:42 +0000" date = "Tue, 27 Mar 2007 19:36:42 +0000"
@ -217,7 +217,7 @@ class TestS3:
== b"AWS 0PN5J17HBGZHT7JJ3X82:xXjDGYUmKxnwqr5KXNPGldn5LbA=" == b"AWS 0PN5J17HBGZHT7JJ3X82:xXjDGYUmKxnwqr5KXNPGldn5LbA="
) )
@deferred_f_from_coro_f @coroutine_test
async def test_request_signing2(self): async def test_request_signing2(self):
# puts an object into the johnsmith bucket. # puts an object into the johnsmith bucket.
date = "Tue, 27 Mar 2007 21:15:45 +0000" date = "Tue, 27 Mar 2007 21:15:45 +0000"
@ -237,7 +237,7 @@ class TestS3:
== b"AWS 0PN5J17HBGZHT7JJ3X82:hcicpDDvL9SsO6AkvxqmIWkmOuQ=" == b"AWS 0PN5J17HBGZHT7JJ3X82:hcicpDDvL9SsO6AkvxqmIWkmOuQ="
) )
@deferred_f_from_coro_f @coroutine_test
async def test_request_signing3(self): async def test_request_signing3(self):
# lists the content of the johnsmith bucket. # lists the content of the johnsmith bucket.
date = "Tue, 27 Mar 2007 19:42:41 +0000" date = "Tue, 27 Mar 2007 19:42:41 +0000"
@ -256,7 +256,7 @@ class TestS3:
== b"AWS 0PN5J17HBGZHT7JJ3X82:jsRt/rhG+Vtp88HrYL706QhE4w4=" == b"AWS 0PN5J17HBGZHT7JJ3X82:jsRt/rhG+Vtp88HrYL706QhE4w4="
) )
@deferred_f_from_coro_f @coroutine_test
async def test_request_signing4(self): async def test_request_signing4(self):
# fetches the access control policy sub-resource for the 'johnsmith' bucket. # fetches the access control policy sub-resource for the 'johnsmith' bucket.
date = "Tue, 27 Mar 2007 19:44:46 +0000" date = "Tue, 27 Mar 2007 19:44:46 +0000"
@ -268,7 +268,7 @@ class TestS3:
== b"AWS 0PN5J17HBGZHT7JJ3X82:thdUi9VAkzhkniLj96JIrOPGi0g=" == b"AWS 0PN5J17HBGZHT7JJ3X82:thdUi9VAkzhkniLj96JIrOPGi0g="
) )
@deferred_f_from_coro_f @coroutine_test
async def test_request_signing6(self): async def test_request_signing6(self):
# uploads an object to a CNAME style virtual hosted bucket with metadata. # uploads an object to a CNAME style virtual hosted bucket with metadata.
date = "Tue, 27 Mar 2007 21:06:08 +0000" date = "Tue, 27 Mar 2007 21:06:08 +0000"
@ -297,7 +297,7 @@ class TestS3:
== b"AWS 0PN5J17HBGZHT7JJ3X82:C0FlOtU8Ylb9KDTpZqYkZPX91iI=" == b"AWS 0PN5J17HBGZHT7JJ3X82:C0FlOtU8Ylb9KDTpZqYkZPX91iI="
) )
@deferred_f_from_coro_f @coroutine_test
async def test_request_signing7(self): async def test_request_signing7(self):
# ensure that spaces are quoted properly before signing # ensure that spaces are quoted properly before signing
date = "Tue, 27 Mar 2007 19:42:41 +0000" date = "Tue, 27 Mar 2007 19:42:41 +0000"
@ -327,7 +327,7 @@ class TestDataURI:
download_handler = build_from_crawler(DataURIDownloadHandler, crawler) download_handler = build_from_crawler(DataURIDownloadHandler, crawler)
self.download_request = download_handler.download_request self.download_request = download_handler.download_request
@deferred_f_from_coro_f @coroutine_test
async def test_response_attrs(self): async def test_response_attrs(self):
uri = "data:,A%20brief%20note" uri = "data:,A%20brief%20note"
request = Request(uri) request = Request(uri)
@ -335,7 +335,7 @@ class TestDataURI:
assert response.url == uri assert response.url == uri
assert not response.headers assert not response.headers
@deferred_f_from_coro_f @coroutine_test
async def test_default_mediatype_encoding(self): async def test_default_mediatype_encoding(self):
request = Request("data:,A%20brief%20note") request = Request("data:,A%20brief%20note")
response = await self.download_request(request) response = await self.download_request(request)
@ -343,7 +343,7 @@ class TestDataURI:
assert type(response) is responsetypes.from_mimetype("text/plain") # pylint: disable=unidiomatic-typecheck assert type(response) is responsetypes.from_mimetype("text/plain") # pylint: disable=unidiomatic-typecheck
assert response.encoding == "US-ASCII" assert response.encoding == "US-ASCII"
@deferred_f_from_coro_f @coroutine_test
async def test_default_mediatype(self): async def test_default_mediatype(self):
request = Request("data:;charset=iso-8859-7,%be%d3%be") request = Request("data:;charset=iso-8859-7,%be%d3%be")
response = await self.download_request(request) response = await self.download_request(request)
@ -351,7 +351,7 @@ class TestDataURI:
assert type(response) is responsetypes.from_mimetype("text/plain") # pylint: disable=unidiomatic-typecheck assert type(response) is responsetypes.from_mimetype("text/plain") # pylint: disable=unidiomatic-typecheck
assert response.encoding == "iso-8859-7" assert response.encoding == "iso-8859-7"
@deferred_f_from_coro_f @coroutine_test
async def test_text_charset(self): async def test_text_charset(self):
request = Request("data:text/plain;charset=iso-8859-7,%be%d3%be") request = Request("data:text/plain;charset=iso-8859-7,%be%d3%be")
response = await self.download_request(request) response = await self.download_request(request)
@ -359,7 +359,7 @@ class TestDataURI:
assert response.body == b"\xbe\xd3\xbe" assert response.body == b"\xbe\xd3\xbe"
assert response.encoding == "iso-8859-7" assert response.encoding == "iso-8859-7"
@deferred_f_from_coro_f @coroutine_test
async def test_mediatype_parameters(self): async def test_mediatype_parameters(self):
request = Request( request = Request(
"data:text/plain;foo=%22foo;bar%5C%22%22;" "data:text/plain;foo=%22foo;bar%5C%22%22;"
@ -371,13 +371,13 @@ class TestDataURI:
assert type(response) is responsetypes.from_mimetype("text/plain") # pylint: disable=unidiomatic-typecheck assert type(response) is responsetypes.from_mimetype("text/plain") # pylint: disable=unidiomatic-typecheck
assert response.encoding == "utf-8" assert response.encoding == "utf-8"
@deferred_f_from_coro_f @coroutine_test
async def test_base64(self): async def test_base64(self):
request = Request("data:text/plain;base64,SGVsbG8sIHdvcmxkLg%3D%3D") request = Request("data:text/plain;base64,SGVsbG8sIHdvcmxkLg%3D%3D")
response = await self.download_request(request) response = await self.download_request(request)
assert response.text == "Hello, world." assert response.text == "Hello, world."
@deferred_f_from_coro_f @coroutine_test
async def test_protocol(self): async def test_protocol(self):
request = Request("data:,") request = Request("data:,")
response = await self.download_request(request) response = await self.download_request(request)

100
tests/test_downloader_handlers_http_base.py
View File

@ -34,7 +34,7 @@ from tests import NON_EXISTING_RESOLVABLE
from tests.mockserver.proxy_echo import ProxyEchoMockServer from tests.mockserver.proxy_echo import ProxyEchoMockServer
from tests.mockserver.simple_https import SimpleMockServer from tests.mockserver.simple_https import SimpleMockServer
from tests.spiders import SingleRequestSpider from tests.spiders import SingleRequestSpider
from tests.utils.decorators import deferred_f_from_coro_f from tests.utils.decorators import coroutine_test
if TYPE_CHECKING: if TYPE_CHECKING:
from collections.abc import AsyncGenerator, Generator from collections.abc import AsyncGenerator, Generator
@ -63,21 +63,21 @@ class TestHttpBase(ABC):
finally: finally:
await dh.close() await dh.close()
@deferred_f_from_coro_f @coroutine_test
async def test_unsupported_scheme(self) -> None: async def test_unsupported_scheme(self) -> None:
request = Request("ftp://unsupported.scheme") request = Request("ftp://unsupported.scheme")
async with self.get_dh() as download_handler: async with self.get_dh() as download_handler:
with pytest.raises(UnsupportedURLSchemeError): with pytest.raises(UnsupportedURLSchemeError):
await download_handler.download_request(request) await download_handler.download_request(request)
@deferred_f_from_coro_f @coroutine_test
async def test_download(self, mockserver: MockServer) -> None: async def test_download(self, mockserver: MockServer) -> None:
request = Request(mockserver.url("/text", is_secure=self.is_secure)) request = Request(mockserver.url("/text", is_secure=self.is_secure))
async with self.get_dh() as download_handler: async with self.get_dh() as download_handler:
response = await download_handler.download_request(request) response = await download_handler.download_request(request)
assert response.body == b"Works" assert response.body == b"Works"
@deferred_f_from_coro_f @coroutine_test
async def test_download_head(self, mockserver: MockServer) -> None: async def test_download_head(self, mockserver: MockServer) -> None:
request = Request( request = Request(
mockserver.url("/text", is_secure=self.is_secure), method="HEAD" mockserver.url("/text", is_secure=self.is_secure), method="HEAD"
@ -94,7 +94,7 @@ class TestHttpBase(ABC):
if http_status.value == 200 or http_status.value // 100 in (4, 5) if http_status.value == 200 or http_status.value // 100 in (4, 5)
], ],
) )
@deferred_f_from_coro_f @coroutine_test
async def test_download_has_correct_http_status_code( async def test_download_has_correct_http_status_code(
self, mockserver: MockServer, http_status: HTTPStatus self, mockserver: MockServer, http_status: HTTPStatus
) -> None: ) -> None:
@ -105,7 +105,7 @@ class TestHttpBase(ABC):
response = await download_handler.download_request(request) response = await download_handler.download_request(request)
assert response.status == http_status.value assert response.status == http_status.value
@deferred_f_from_coro_f @coroutine_test
async def test_server_receives_correct_request_headers( async def test_server_receives_correct_request_headers(
self, mockserver: MockServer self, mockserver: MockServer
) -> None: ) -> None:
@ -132,7 +132,7 @@ class TestHttpBase(ABC):
assert header_name in body["headers"] assert header_name in body["headers"]
assert body["headers"][header_name] == [header_value] assert body["headers"][header_name] == [header_value]
@deferred_f_from_coro_f @coroutine_test
async def test_request_header_none(self, mockserver: MockServer) -> None: async def test_request_header_none(self, mockserver: MockServer) -> None:
"""Adding a header with None as the value should not send that header.""" """Adding a header with None as the value should not send that header."""
request_headers = { request_headers = {
@ -158,7 +158,7 @@ class TestHttpBase(ABC):
[("X-Custom-Header", "foo"), ("X-Custom-Header", "bar")], [("X-Custom-Header", "foo"), ("X-Custom-Header", "bar")],
], ],
) )
@deferred_f_from_coro_f @coroutine_test
async def test_request_header_duplicate( async def test_request_header_duplicate(
self, mockserver: MockServer, request_headers: Any self, mockserver: MockServer, request_headers: Any
) -> None: ) -> None:
@ -174,7 +174,7 @@ class TestHttpBase(ABC):
assert "headers" in body assert "headers" in body
assert body["headers"]["X-Custom-Header"] == ["foo", "bar"] assert body["headers"]["X-Custom-Header"] == ["foo", "bar"]
@deferred_f_from_coro_f @coroutine_test
async def test_server_receives_correct_request_body( async def test_server_receives_correct_request_body(
self, mockserver: MockServer self, mockserver: MockServer
) -> None: ) -> None:
@ -191,7 +191,7 @@ class TestHttpBase(ABC):
body = json.loads(response.body.decode("utf-8")) body = json.loads(response.body.decode("utf-8"))
assert json.loads(body["body"]) == request_body assert json.loads(body["body"]) == request_body
@deferred_f_from_coro_f @coroutine_test
async def test_download_has_correct_response_headers( async def test_download_has_correct_response_headers(
self, mockserver: MockServer self, mockserver: MockServer
) -> None: ) -> None:
@ -230,14 +230,14 @@ class TestHttpBase(ABC):
header_value, encoding="utf-8" header_value, encoding="utf-8"
) )
@deferred_f_from_coro_f @coroutine_test
async def test_redirect_status(self, mockserver: MockServer) -> None: async def test_redirect_status(self, mockserver: MockServer) -> None:
request = Request(mockserver.url("/redirect", is_secure=self.is_secure)) request = Request(mockserver.url("/redirect", is_secure=self.is_secure))
async with self.get_dh() as download_handler: async with self.get_dh() as download_handler:
response = await download_handler.download_request(request) response = await download_handler.download_request(request)
assert response.status == 302 assert response.status == 302
@deferred_f_from_coro_f @coroutine_test
async def test_redirect_status_head(self, mockserver: MockServer) -> None: async def test_redirect_status_head(self, mockserver: MockServer) -> None:
request = Request( request = Request(
mockserver.url("/redirect", is_secure=self.is_secure), method="HEAD" mockserver.url("/redirect", is_secure=self.is_secure), method="HEAD"
@ -246,7 +246,7 @@ class TestHttpBase(ABC):
response = await download_handler.download_request(request) response = await download_handler.download_request(request)
assert response.status == 302 assert response.status == 302
@deferred_f_from_coro_f @coroutine_test
async def test_timeout_download_from_spider_nodata_rcvd( async def test_timeout_download_from_spider_nodata_rcvd(
self, mockserver: MockServer, reactor_pytest: str self, mockserver: MockServer, reactor_pytest: str
) -> None: ) -> None:
@ -264,7 +264,7 @@ class TestHttpBase(ABC):
with pytest.raises(DownloadTimeoutError): with pytest.raises(DownloadTimeoutError):
await maybe_deferred_to_future(d) await maybe_deferred_to_future(d)
@deferred_f_from_coro_f @coroutine_test
async def test_timeout_download_from_spider_server_hangs( async def test_timeout_download_from_spider_server_hangs(
self, self,
mockserver: MockServer, mockserver: MockServer,
@ -286,7 +286,7 @@ class TestHttpBase(ABC):
await maybe_deferred_to_future(d) await maybe_deferred_to_future(d)
@pytest.mark.parametrize("send_header", [True, False]) @pytest.mark.parametrize("send_header", [True, False])
@deferred_f_from_coro_f @coroutine_test
async def test_host_header(self, send_header: bool, mockserver: MockServer) -> None: async def test_host_header(self, send_header: bool, mockserver: MockServer) -> None:
host_port = f"{mockserver.host}:{mockserver.port(is_secure=self.is_secure)}" host_port = f"{mockserver.host}:{mockserver.port(is_secure=self.is_secure)}"
request = Request( request = Request(
@ -301,7 +301,7 @@ class TestHttpBase(ABC):
else: else:
assert not request.headers assert not request.headers
@deferred_f_from_coro_f @coroutine_test
async def test_content_length_zero_bodyless_post_request_headers( async def test_content_length_zero_bodyless_post_request_headers(
self, mockserver: MockServer self, mockserver: MockServer
) -> None: ) -> None:
@ -322,7 +322,7 @@ class TestHttpBase(ABC):
response = await download_handler.download_request(request) response = await download_handler.download_request(request)
assert response.body == b"0" assert response.body == b"0"
@deferred_f_from_coro_f @coroutine_test
async def test_content_length_zero_bodyless_post_only_one( async def test_content_length_zero_bodyless_post_only_one(
self, mockserver: MockServer self, mockserver: MockServer
) -> None: ) -> None:
@ -336,7 +336,7 @@ class TestHttpBase(ABC):
assert len(contentlengths) == 1 assert len(contentlengths) == 1
assert contentlengths == [b"0"] assert contentlengths == [b"0"]
@deferred_f_from_coro_f @coroutine_test
async def test_payload(self, mockserver: MockServer) -> None: async def test_payload(self, mockserver: MockServer) -> None:
body = b"1" * 100 # PayloadResource requires body length to be 100 body = b"1" * 100 # PayloadResource requires body length to be 100
request = Request( request = Request(
@ -348,7 +348,7 @@ class TestHttpBase(ABC):
response = await download_handler.download_request(request) response = await download_handler.download_request(request)
assert response.body == body assert response.body == body
@deferred_f_from_coro_f @coroutine_test
async def test_response_header_content_length(self, mockserver: MockServer) -> None: async def test_response_header_content_length(self, mockserver: MockServer) -> None:
request = Request( request = Request(
mockserver.url("/text", is_secure=self.is_secure), method="GET" mockserver.url("/text", is_secure=self.is_secure), method="GET"
@ -364,7 +364,7 @@ class TestHttpBase(ABC):
("foo", b"<!DOCTYPE html>\n<title>.</title>", HtmlResponse), ("foo", b"<!DOCTYPE html>\n<title>.</title>", HtmlResponse),
], ],
) )
@deferred_f_from_coro_f @coroutine_test
async def test_response_class( async def test_response_class(
self, self,
filename: str, filename: str,
@ -379,14 +379,14 @@ class TestHttpBase(ABC):
response = await download_handler.download_request(request) response = await download_handler.download_request(request)
assert type(response) is response_class # pylint: disable=unidiomatic-typecheck assert type(response) is response_class # pylint: disable=unidiomatic-typecheck
@deferred_f_from_coro_f @coroutine_test
async def test_get_duplicate_header(self, mockserver: MockServer) -> None: async def test_get_duplicate_header(self, mockserver: MockServer) -> None:
request = Request(mockserver.url("/duplicate-header", is_secure=self.is_secure)) request = Request(mockserver.url("/duplicate-header", is_secure=self.is_secure))
async with self.get_dh() as download_handler: async with self.get_dh() as download_handler:
response = await download_handler.download_request(request) response = await download_handler.download_request(request)
assert response.headers.getlist(b"Set-Cookie") == [b"a=b", b"c=d"] assert response.headers.getlist(b"Set-Cookie") == [b"a=b", b"c=d"]
@deferred_f_from_coro_f @coroutine_test
async def test_download_is_not_automatically_gzip_decoded( async def test_download_is_not_automatically_gzip_decoded(
self, mockserver: MockServer self, mockserver: MockServer
) -> None: ) -> None:
@ -420,7 +420,7 @@ class TestHttpBase(ABC):
expected_decoding = bytes(data, encoding="utf-8") expected_decoding = bytes(data, encoding="utf-8")
assert gzip.decompress(response.body) == expected_decoding assert gzip.decompress(response.body) == expected_decoding
@deferred_f_from_coro_f @coroutine_test
async def test_no_cookie_processing_or_persistence( async def test_no_cookie_processing_or_persistence(
self, mockserver: MockServer self, mockserver: MockServer
) -> None: ) -> None:
@ -451,14 +451,14 @@ class TestHttpBase(ABC):
class TestHttp11Base(TestHttpBase): class TestHttp11Base(TestHttpBase):
"""HTTP 1.1 test case""" """HTTP 1.1 test case"""
@deferred_f_from_coro_f @coroutine_test
async def test_download_without_maxsize_limit(self, mockserver: MockServer) -> None: async def test_download_without_maxsize_limit(self, mockserver: MockServer) -> None:
request = Request(mockserver.url("/text", is_secure=self.is_secure)) request = Request(mockserver.url("/text", is_secure=self.is_secure))
async with self.get_dh() as download_handler: async with self.get_dh() as download_handler:
response = await download_handler.download_request(request) response = await download_handler.download_request(request)
assert response.body == b"Works" assert response.body == b"Works"
@deferred_f_from_coro_f @coroutine_test
async def test_response_class_choosing_request( async def test_response_class_choosing_request(
self, mockserver: MockServer self, mockserver: MockServer
) -> None: ) -> None:
@ -473,7 +473,7 @@ class TestHttp11Base(TestHttpBase):
response = await download_handler.download_request(request) response = await download_handler.download_request(request)
assert type(response) is TextResponse # pylint: disable=unidiomatic-typecheck assert type(response) is TextResponse # pylint: disable=unidiomatic-typecheck
@deferred_f_from_coro_f @coroutine_test
async def test_download_with_maxsize(self, mockserver: MockServer) -> None: async def test_download_with_maxsize(self, mockserver: MockServer) -> None:
request = Request(mockserver.url("/text", is_secure=self.is_secure)) request = Request(mockserver.url("/text", is_secure=self.is_secure))
@ -487,7 +487,7 @@ class TestHttp11Base(TestHttpBase):
with pytest.raises(DownloadCancelledError): with pytest.raises(DownloadCancelledError):
await download_handler.download_request(request) await download_handler.download_request(request)
@deferred_f_from_coro_f @coroutine_test
async def test_download_with_maxsize_very_large_file( async def test_download_with_maxsize_very_large_file(
self, mockserver: MockServer, caplog: pytest.LogCaptureFixture self, mockserver: MockServer, caplog: pytest.LogCaptureFixture
) -> None: ) -> None:
@ -498,7 +498,7 @@ class TestHttp11Base(TestHttpBase):
assert "larger than download max size" in caplog.text assert "larger than download max size" in caplog.text
@deferred_f_from_coro_f @coroutine_test
async def test_download_with_maxsize_per_req(self, mockserver: MockServer) -> None: async def test_download_with_maxsize_per_req(self, mockserver: MockServer) -> None:
meta = {"download_maxsize": 2} meta = {"download_maxsize": 2}
request = Request(mockserver.url("/text", is_secure=self.is_secure), meta=meta) request = Request(mockserver.url("/text", is_secure=self.is_secure), meta=meta)
@ -506,7 +506,7 @@ class TestHttp11Base(TestHttpBase):
with pytest.raises(DownloadCancelledError): with pytest.raises(DownloadCancelledError):
await download_handler.download_request(request) await download_handler.download_request(request)
@deferred_f_from_coro_f @coroutine_test
async def test_download_with_small_maxsize_via_setting( async def test_download_with_small_maxsize_via_setting(
self, mockserver: MockServer self, mockserver: MockServer
) -> None: ) -> None:
@ -515,7 +515,7 @@ class TestHttp11Base(TestHttpBase):
with pytest.raises(DownloadCancelledError): with pytest.raises(DownloadCancelledError):
await download_handler.download_request(request) await download_handler.download_request(request)
@deferred_f_from_coro_f @coroutine_test
async def test_download_with_large_maxsize_via_setting( async def test_download_with_large_maxsize_via_setting(
self, mockserver: MockServer self, mockserver: MockServer
) -> None: ) -> None:
@ -524,7 +524,7 @@ class TestHttp11Base(TestHttpBase):
response = await download_handler.download_request(request) response = await download_handler.download_request(request)
assert response.body == b"Works" assert response.body == b"Works"
@deferred_f_from_coro_f @coroutine_test
async def test_download_chunked_content(self, mockserver: MockServer) -> None: async def test_download_chunked_content(self, mockserver: MockServer) -> None:
request = Request(mockserver.url("/chunked", is_secure=self.is_secure)) request = Request(mockserver.url("/chunked", is_secure=self.is_secure))
async with self.get_dh() as download_handler: async with self.get_dh() as download_handler:
@ -532,7 +532,7 @@ class TestHttp11Base(TestHttpBase):
assert response.body == b"chunked content\n" assert response.body == b"chunked content\n"
@pytest.mark.parametrize("url", ["broken", "broken-chunked"]) @pytest.mark.parametrize("url", ["broken", "broken-chunked"])
@deferred_f_from_coro_f @coroutine_test
async def test_download_cause_data_loss( async def test_download_cause_data_loss(
self, url: str, mockserver: MockServer self, url: str, mockserver: MockServer
) -> None: ) -> None:
@ -541,7 +541,7 @@ class TestHttp11Base(TestHttpBase):
with pytest.raises(ResponseDataLossError): with pytest.raises(ResponseDataLossError):
await download_handler.download_request(request) await download_handler.download_request(request)
@deferred_f_from_coro_f @coroutine_test
async def test_download_cause_data_loss_double_warning( async def test_download_cause_data_loss_double_warning(
self, caplog: pytest.LogCaptureFixture, mockserver: MockServer self, caplog: pytest.LogCaptureFixture, mockserver: MockServer
) -> None: ) -> None:
@ -557,7 +557,7 @@ class TestHttp11Base(TestHttpBase):
assert "Got data loss" not in caplog.text assert "Got data loss" not in caplog.text
@pytest.mark.parametrize("url", ["broken", "broken-chunked"]) @pytest.mark.parametrize("url", ["broken", "broken-chunked"])
@deferred_f_from_coro_f @coroutine_test
async def test_download_allow_data_loss( async def test_download_allow_data_loss(
self, url: str, mockserver: MockServer self, url: str, mockserver: MockServer
) -> None: ) -> None:
@ -570,7 +570,7 @@ class TestHttp11Base(TestHttpBase):
assert response.flags == ["dataloss"] assert response.flags == ["dataloss"]
@pytest.mark.parametrize("url", ["broken", "broken-chunked"]) @pytest.mark.parametrize("url", ["broken", "broken-chunked"])
@deferred_f_from_coro_f @coroutine_test
async def test_download_allow_data_loss_via_setting( async def test_download_allow_data_loss_via_setting(
self, url: str, mockserver: MockServer self, url: str, mockserver: MockServer
) -> None: ) -> None:
@ -581,7 +581,7 @@ class TestHttp11Base(TestHttpBase):
response = await download_handler.download_request(request) response = await download_handler.download_request(request)
assert response.flags == ["dataloss"] assert response.flags == ["dataloss"]
@deferred_f_from_coro_f @coroutine_test
async def test_download_conn_failed(self) -> None: async def test_download_conn_failed(self) -> None:
# copy of TestCrawl.test_retry_conn_failed() # copy of TestCrawl.test_retry_conn_failed()
scheme = "https" if self.is_secure else "http" scheme = "https" if self.is_secure else "http"
@ -590,7 +590,7 @@ class TestHttp11Base(TestHttpBase):
with pytest.raises(DownloadConnectionRefusedError): with pytest.raises(DownloadConnectionRefusedError):
await download_handler.download_request(request) await download_handler.download_request(request)
@deferred_f_from_coro_f @coroutine_test
async def test_download_conn_lost(self, mockserver: MockServer) -> None: async def test_download_conn_lost(self, mockserver: MockServer) -> None:
# copy of TestCrawl.test_retry_conn_lost() # copy of TestCrawl.test_retry_conn_lost()
request = Request(mockserver.url("/drop?abort=0", is_secure=self.is_secure)) request = Request(mockserver.url("/drop?abort=0", is_secure=self.is_secure))
@ -598,7 +598,7 @@ class TestHttp11Base(TestHttpBase):
with pytest.raises(ResponseDataLossError): with pytest.raises(ResponseDataLossError):
await download_handler.download_request(request) await download_handler.download_request(request)
@deferred_f_from_coro_f @coroutine_test
async def test_download_conn_aborted(self, mockserver: MockServer) -> None: async def test_download_conn_aborted(self, mockserver: MockServer) -> None:
# copy of TestCrawl.test_retry_conn_aborted() # copy of TestCrawl.test_retry_conn_aborted()
request = Request(mockserver.url("/drop?abort=1", is_secure=self.is_secure)) request = Request(mockserver.url("/drop?abort=1", is_secure=self.is_secure))
@ -609,7 +609,7 @@ class TestHttp11Base(TestHttpBase):
@pytest.mark.skipif( @pytest.mark.skipif(
NON_EXISTING_RESOLVABLE, reason="Non-existing hosts are resolvable" NON_EXISTING_RESOLVABLE, reason="Non-existing hosts are resolvable"
) )
@deferred_f_from_coro_f @coroutine_test
async def test_download_dns_error(self) -> None: async def test_download_dns_error(self) -> None:
# copy of TestCrawl.test_retry_dns_error() # copy of TestCrawl.test_retry_dns_error()
scheme = "https" if self.is_secure else "http" scheme = "https" if self.is_secure else "http"
@ -618,7 +618,7 @@ class TestHttp11Base(TestHttpBase):
with pytest.raises(CannotResolveHostError): with pytest.raises(CannotResolveHostError):
await download_handler.download_request(request) await download_handler.download_request(request)
@deferred_f_from_coro_f @coroutine_test
async def test_protocol(self, mockserver: MockServer) -> None: async def test_protocol(self, mockserver: MockServer) -> None:
request = Request( request = Request(
mockserver.url("/host", is_secure=self.is_secure), method="GET" mockserver.url("/host", is_secure=self.is_secure), method="GET"
@ -642,7 +642,7 @@ class TestHttps11Base(TestHttp11Base):
# (not just Scrapy) hang on /drop?abort=0. # (not just Scrapy) hang on /drop?abort=0.
pytest.skip("Unable to test on HTTPS") pytest.skip("Unable to test on HTTPS")
@deferred_f_from_coro_f @coroutine_test
async def test_tls_logging( async def test_tls_logging(
self, mockserver: MockServer, caplog: pytest.LogCaptureFixture self, mockserver: MockServer, caplog: pytest.LogCaptureFixture
) -> None: ) -> None:
@ -695,7 +695,7 @@ class TestSimpleHttpsBase(ABC):
finally: finally:
await dh.close() await dh.close()
@deferred_f_from_coro_f @coroutine_test
async def test_download(self, url: str) -> None: async def test_download(self, url: str) -> None:
request = Request(url) request = Request(url)
async with self.get_dh() as download_handler: async with self.get_dh() as download_handler:
@ -738,7 +738,7 @@ class TestHttpWithCrawlerBase(ABC):
is_secure = False is_secure = False
@deferred_f_from_coro_f @coroutine_test
async def test_download_with_content_length(self, mockserver: MockServer) -> None: async def test_download_with_content_length(self, mockserver: MockServer) -> None:
crawler = get_crawler(SingleRequestSpider, self.settings_dict) crawler = get_crawler(SingleRequestSpider, self.settings_dict)
# http://localhost:8998/partial set Content-Length to 1024, use download_maxsize= 1000 to avoid # http://localhost:8998/partial set Content-Length to 1024, use download_maxsize= 1000 to avoid
@ -755,7 +755,7 @@ class TestHttpWithCrawlerBase(ABC):
failure = crawler.spider.meta["failure"] # type: ignore[attr-defined] failure = crawler.spider.meta["failure"] # type: ignore[attr-defined]
assert isinstance(failure.value, DownloadCancelledError) assert isinstance(failure.value, DownloadCancelledError)
@deferred_f_from_coro_f @coroutine_test
async def test_download(self, mockserver: MockServer) -> None: async def test_download(self, mockserver: MockServer) -> None:
crawler = get_crawler(SingleRequestSpider, self.settings_dict) crawler = get_crawler(SingleRequestSpider, self.settings_dict)
await maybe_deferred_to_future( await maybe_deferred_to_future(
@ -769,7 +769,7 @@ class TestHttpWithCrawlerBase(ABC):
reason = crawler.spider.meta["close_reason"] # type: ignore[attr-defined] reason = crawler.spider.meta["close_reason"] # type: ignore[attr-defined]
assert reason == "finished" assert reason == "finished"
@deferred_f_from_coro_f @coroutine_test
async def test_response_ssl_certificate(self, mockserver: MockServer) -> None: async def test_response_ssl_certificate(self, mockserver: MockServer) -> None:
if not self.is_secure: if not self.is_secure:
pytest.skip("Only applies to HTTPS") pytest.skip("Only applies to HTTPS")
@ -784,7 +784,7 @@ class TestHttpWithCrawlerBase(ABC):
assert cert.getSubject().commonName == b"localhost" assert cert.getSubject().commonName == b"localhost"
assert cert.getIssuer().commonName == b"localhost" assert cert.getIssuer().commonName == b"localhost"
@deferred_f_from_coro_f @coroutine_test
async def test_response_ip_address(self, mockserver: MockServer) -> None: async def test_response_ip_address(self, mockserver: MockServer) -> None:
# copy of TestCrawl.test_response_ip_address() # copy of TestCrawl.test_response_ip_address()
crawler = get_crawler(SingleRequestSpider, self.settings_dict) crawler = get_crawler(SingleRequestSpider, self.settings_dict)
@ -821,7 +821,7 @@ class TestHttpProxyBase(ABC):
finally: finally:
await dh.close() await dh.close()
@deferred_f_from_coro_f @coroutine_test
async def test_download_with_proxy( async def test_download_with_proxy(
self, proxy_mockserver: ProxyEchoMockServer self, proxy_mockserver: ProxyEchoMockServer
) -> None: ) -> None:
@ -833,7 +833,7 @@ class TestHttpProxyBase(ABC):
assert response.url == request.url assert response.url == request.url
assert response.body == self.expected_http_proxy_request_body assert response.body == self.expected_http_proxy_request_body
@deferred_f_from_coro_f @coroutine_test
async def test_download_without_proxy( async def test_download_without_proxy(
self, proxy_mockserver: ProxyEchoMockServer self, proxy_mockserver: ProxyEchoMockServer
) -> None: ) -> None:
@ -846,7 +846,7 @@ class TestHttpProxyBase(ABC):
assert response.url == request.url assert response.url == request.url
assert response.body == b"/path/to/resource" assert response.body == b"/path/to/resource"
@deferred_f_from_coro_f @coroutine_test
async def test_download_with_proxy_https_timeout( async def test_download_with_proxy_https_timeout(
self, proxy_mockserver: ProxyEchoMockServer self, proxy_mockserver: ProxyEchoMockServer
) -> None: ) -> None:
@ -860,7 +860,7 @@ class TestHttpProxyBase(ABC):
await download_handler.download_request(request) await download_handler.download_request(request)
assert domain in str(exc_info.value) assert domain in str(exc_info.value)
@deferred_f_from_coro_f @coroutine_test
async def test_download_with_proxy_without_http_scheme( async def test_download_with_proxy_without_http_scheme(
self, proxy_mockserver: ProxyEchoMockServer self, proxy_mockserver: ProxyEchoMockServer
) -> None: ) -> None:

28
tests/test_downloadermiddleware.py
View File

@ -16,7 +16,7 @@ from scrapy.spiders import Spider
from scrapy.utils.defer import maybe_deferred_to_future from scrapy.utils.defer import maybe_deferred_to_future
from scrapy.utils.python import to_bytes from scrapy.utils.python import to_bytes
from scrapy.utils.test import get_crawler, get_from_asyncio_queue from scrapy.utils.test import get_crawler, get_from_asyncio_queue
from tests.utils.decorators import deferred_f_from_coro_f from tests.utils.decorators import coroutine_test
if TYPE_CHECKING: if TYPE_CHECKING:
from collections.abc import AsyncGenerator from collections.abc import AsyncGenerator
@ -60,7 +60,7 @@ class TestManagerBase:
class TestDefaults(TestManagerBase): class TestDefaults(TestManagerBase):
"""Tests default behavior with default settings""" """Tests default behavior with default settings"""
@deferred_f_from_coro_f @coroutine_test
async def test_request_response(self): async def test_request_response(self):
req = Request("http://example.com/index.html") req = Request("http://example.com/index.html")
resp = Response(req.url, status=200) resp = Response(req.url, status=200)
@ -68,7 +68,7 @@ class TestDefaults(TestManagerBase):
ret = await self._download(mwman, req, resp) ret = await self._download(mwman, req, resp)
assert isinstance(ret, Response), "Non-response returned" assert isinstance(ret, Response), "Non-response returned"
@deferred_f_from_coro_f @coroutine_test
async def test_3xx_and_invalid_gzipped_body_must_redirect(self): async def test_3xx_and_invalid_gzipped_body_must_redirect(self):
"""Regression test for a failure when redirecting a compressed """Regression test for a failure when redirecting a compressed
request. request.
@ -101,7 +101,7 @@ class TestDefaults(TestManagerBase):
"Not redirected to location header" "Not redirected to location header"
) )
@deferred_f_from_coro_f @coroutine_test
async def test_200_and_invalid_gzipped_body_must_fail(self): async def test_200_and_invalid_gzipped_body_must_fail(self):
req = Request("http://example.com") req = Request("http://example.com")
body = b"<p>You are being redirected</p>" body = b"<p>You are being redirected</p>"
@ -124,7 +124,7 @@ class TestDefaults(TestManagerBase):
class TestResponseFromProcessRequest(TestManagerBase): class TestResponseFromProcessRequest(TestManagerBase):
"""Tests middleware returning a response from process_request.""" """Tests middleware returning a response from process_request."""
@deferred_f_from_coro_f @coroutine_test
async def test_download_func_not_called(self): async def test_download_func_not_called(self):
req = Request("http://example.com/index.html") req = Request("http://example.com/index.html")
resp = Response("http://example.com/index.html") resp = Response("http://example.com/index.html")
@ -144,7 +144,7 @@ class TestResponseFromProcessRequest(TestManagerBase):
class TestResponseFromProcessException(TestManagerBase): class TestResponseFromProcessException(TestManagerBase):
"""Tests middleware returning a response from process_exception.""" """Tests middleware returning a response from process_exception."""
@deferred_f_from_coro_f @coroutine_test
async def test_process_response_called(self): async def test_process_response_called(self):
req = Request("http://example.com/index.html") req = Request("http://example.com/index.html")
resp = Response("http://example.com/index.html") resp = Response("http://example.com/index.html")
@ -173,7 +173,7 @@ class TestResponseFromProcessException(TestManagerBase):
class TestInvalidOutput(TestManagerBase): class TestInvalidOutput(TestManagerBase):
@deferred_f_from_coro_f @coroutine_test
async def test_invalid_process_request(self): async def test_invalid_process_request(self):
"""Invalid return value for process_request method should raise an exception""" """Invalid return value for process_request method should raise an exception"""
req = Request("http://example.com/index.html") req = Request("http://example.com/index.html")
@ -187,7 +187,7 @@ class TestInvalidOutput(TestManagerBase):
with pytest.raises(_InvalidOutput): with pytest.raises(_InvalidOutput):
await self._download(mwman, req) await self._download(mwman, req)
@deferred_f_from_coro_f @coroutine_test
async def test_invalid_process_response(self): async def test_invalid_process_response(self):
"""Invalid return value for process_response method should raise an exception""" """Invalid return value for process_response method should raise an exception"""
req = Request("http://example.com/index.html") req = Request("http://example.com/index.html")
@ -201,7 +201,7 @@ class TestInvalidOutput(TestManagerBase):
with pytest.raises(_InvalidOutput): with pytest.raises(_InvalidOutput):
await self._download(mwman, req) await self._download(mwman, req)
@deferred_f_from_coro_f @coroutine_test
async def test_invalid_process_exception(self): async def test_invalid_process_exception(self):
"""Invalid return value for process_exception method should raise an exception""" """Invalid return value for process_exception method should raise an exception"""
req = Request("http://example.com/index.html") req = Request("http://example.com/index.html")
@ -222,7 +222,7 @@ class TestInvalidOutput(TestManagerBase):
class TestMiddlewareUsingDeferreds(TestManagerBase): class TestMiddlewareUsingDeferreds(TestManagerBase):
"""Middlewares using Deferreds (deprecated) should work""" """Middlewares using Deferreds (deprecated) should work"""
@deferred_f_from_coro_f @coroutine_test
async def test_deferred(self): async def test_deferred(self):
req = Request("http://example.com/index.html") req = Request("http://example.com/index.html")
resp = Response("http://example.com/index.html") resp = Response("http://example.com/index.html")
@ -252,7 +252,7 @@ class TestMiddlewareUsingDeferreds(TestManagerBase):
class TestMiddlewareUsingCoro(TestManagerBase): class TestMiddlewareUsingCoro(TestManagerBase):
"""Middlewares using asyncio coroutines should work""" """Middlewares using asyncio coroutines should work"""
@deferred_f_from_coro_f @coroutine_test
async def test_asyncdef(self): async def test_asyncdef(self):
req = Request("http://example.com/index.html") req = Request("http://example.com/index.html")
resp = Response("http://example.com/index.html") resp = Response("http://example.com/index.html")
@ -270,7 +270,7 @@ class TestMiddlewareUsingCoro(TestManagerBase):
assert not download_func.called assert not download_func.called
@pytest.mark.only_asyncio @pytest.mark.only_asyncio
@deferred_f_from_coro_f @coroutine_test
async def test_asyncdef_asyncio(self): async def test_asyncdef_asyncio(self):
req = Request("http://example.com/index.html") req = Request("http://example.com/index.html")
resp = Response("http://example.com/index.html") resp = Response("http://example.com/index.html")
@ -289,7 +289,7 @@ class TestMiddlewareUsingCoro(TestManagerBase):
class TestDownloadDeprecated(TestManagerBase): class TestDownloadDeprecated(TestManagerBase):
@deferred_f_from_coro_f @coroutine_test
async def test_mwman_download(self): async def test_mwman_download(self):
req = Request("http://example.com/index.html") req = Request("http://example.com/index.html")
resp = Response(req.url, status=200) resp = Response(req.url, status=200)
@ -309,7 +309,7 @@ class TestDownloadDeprecated(TestManagerBase):
class TestDeprecatedSpiderArg(TestManagerBase): class TestDeprecatedSpiderArg(TestManagerBase):
@deferred_f_from_coro_f @coroutine_test
async def test_deprecated_spider_arg(self): async def test_deprecated_spider_arg(self):
req = Request("http://example.com/index.html") req = Request("http://example.com/index.html")
resp = Response("http://example.com/index.html") resp = Response("http://example.com/index.html")

24
tests/test_downloadermiddleware_robotstxt.py
View File

@ -16,7 +16,7 @@ from scrapy.settings import Settings
from scrapy.utils.asyncio import call_later from scrapy.utils.asyncio import call_later
from scrapy.utils.defer import deferred_from_coro, maybe_deferred_to_future from scrapy.utils.defer import deferred_from_coro, maybe_deferred_to_future
from tests.test_robotstxt_interface import rerp_available from tests.test_robotstxt_interface import rerp_available
from tests.utils.decorators import deferred_f_from_coro_f from tests.utils.decorators import coroutine_test
if TYPE_CHECKING: if TYPE_CHECKING:
from scrapy.crawler import Crawler from scrapy.crawler import Crawler
@ -60,7 +60,7 @@ Disallow: /some/randome/page.html
crawler.engine.download_async.side_effect = return_response crawler.engine.download_async.side_effect = return_response
return crawler return crawler
@deferred_f_from_coro_f @coroutine_test
async def test_robotstxt(self): async def test_robotstxt(self):
middleware = RobotsTxtMiddleware(self._get_successful_crawler()) middleware = RobotsTxtMiddleware(self._get_successful_crawler())
await self.assertNotIgnored(Request("http://site.local/allowed"), middleware) await self.assertNotIgnored(Request("http://site.local/allowed"), middleware)
@ -74,7 +74,7 @@ Disallow: /some/randome/page.html
Request("http://site.local/wiki/Käyttäjä:"), middleware Request("http://site.local/wiki/Käyttäjä:"), middleware
) )
@deferred_f_from_coro_f @coroutine_test
async def test_robotstxt_multiple_reqs(self) -> None: async def test_robotstxt_multiple_reqs(self) -> None:
middleware = RobotsTxtMiddleware(self._get_successful_crawler()) middleware = RobotsTxtMiddleware(self._get_successful_crawler())
d1 = deferred_from_coro( d1 = deferred_from_coro(
@ -86,20 +86,20 @@ Disallow: /some/randome/page.html
await maybe_deferred_to_future(DeferredList([d1, d2], fireOnOneErrback=True)) await maybe_deferred_to_future(DeferredList([d1, d2], fireOnOneErrback=True))
@pytest.mark.only_asyncio @pytest.mark.only_asyncio
@deferred_f_from_coro_f @coroutine_test
async def test_robotstxt_multiple_reqs_asyncio(self) -> None: async def test_robotstxt_multiple_reqs_asyncio(self) -> None:
middleware = RobotsTxtMiddleware(self._get_successful_crawler()) middleware = RobotsTxtMiddleware(self._get_successful_crawler())
c1 = middleware.process_request(Request("http://site.local/allowed1")) c1 = middleware.process_request(Request("http://site.local/allowed1"))
c2 = middleware.process_request(Request("http://site.local/allowed2")) c2 = middleware.process_request(Request("http://site.local/allowed2"))
await asyncio.gather(c1, c2) await asyncio.gather(c1, c2)
@deferred_f_from_coro_f @coroutine_test
async def test_robotstxt_ready_parser(self): async def test_robotstxt_ready_parser(self):
middleware = RobotsTxtMiddleware(self._get_successful_crawler()) middleware = RobotsTxtMiddleware(self._get_successful_crawler())
await self.assertNotIgnored(Request("http://site.local/allowed"), middleware) await self.assertNotIgnored(Request("http://site.local/allowed"), middleware)
await self.assertNotIgnored(Request("http://site.local/allowed"), middleware) await self.assertNotIgnored(Request("http://site.local/allowed"), middleware)
@deferred_f_from_coro_f @coroutine_test
async def test_robotstxt_meta(self): async def test_robotstxt_meta(self):
middleware = RobotsTxtMiddleware(self._get_successful_crawler()) middleware = RobotsTxtMiddleware(self._get_successful_crawler())
meta = {"dont_obey_robotstxt": True} meta = {"dont_obey_robotstxt": True}
@ -128,7 +128,7 @@ Disallow: /some/randome/page.html
crawler.engine.download_async.side_effect = return_response crawler.engine.download_async.side_effect = return_response
return crawler return crawler
@deferred_f_from_coro_f @coroutine_test
async def test_robotstxt_garbage(self): async def test_robotstxt_garbage(self):
# garbage response should be discarded, equal 'allow all' # garbage response should be discarded, equal 'allow all'
middleware = RobotsTxtMiddleware(self._get_garbage_crawler()) middleware = RobotsTxtMiddleware(self._get_garbage_crawler())
@ -150,7 +150,7 @@ Disallow: /some/randome/page.html
crawler.engine.download_async.side_effect = return_response crawler.engine.download_async.side_effect = return_response
return crawler return crawler
@deferred_f_from_coro_f @coroutine_test
async def test_robotstxt_empty_response(self): async def test_robotstxt_empty_response(self):
# empty response should equal 'allow all' # empty response should equal 'allow all'
middleware = RobotsTxtMiddleware(self._get_emptybody_crawler()) middleware = RobotsTxtMiddleware(self._get_emptybody_crawler())
@ -158,7 +158,7 @@ Disallow: /some/randome/page.html
await self.assertNotIgnored(Request("http://site.local/admin/main"), middleware) await self.assertNotIgnored(Request("http://site.local/admin/main"), middleware)
await self.assertNotIgnored(Request("http://site.local/static/"), middleware) await self.assertNotIgnored(Request("http://site.local/static/"), middleware)
@deferred_f_from_coro_f @coroutine_test
async def test_robotstxt_error(self, caplog: pytest.LogCaptureFixture) -> None: async def test_robotstxt_error(self, caplog: pytest.LogCaptureFixture) -> None:
self.crawler.settings.set("ROBOTSTXT_OBEY", True) self.crawler.settings.set("ROBOTSTXT_OBEY", True)
err = CannotResolveHostError("Robotstxt address not found") err = CannotResolveHostError("Robotstxt address not found")
@ -174,7 +174,7 @@ Disallow: /some/randome/page.html
await middleware.process_request(Request("http://site.local")) await middleware.process_request(Request("http://site.local"))
assert "Robotstxt address not found" in caplog.text assert "Robotstxt address not found" in caplog.text
@deferred_f_from_coro_f @coroutine_test
async def test_robotstxt_immediate_error(self): async def test_robotstxt_immediate_error(self):
self.crawler.settings.set("ROBOTSTXT_OBEY", True) self.crawler.settings.set("ROBOTSTXT_OBEY", True)
err = CannotResolveHostError("Robotstxt address not found") err = CannotResolveHostError("Robotstxt address not found")
@ -187,7 +187,7 @@ Disallow: /some/randome/page.html
middleware = RobotsTxtMiddleware(self.crawler) middleware = RobotsTxtMiddleware(self.crawler)
await self.assertNotIgnored(Request("http://site.local"), middleware) await self.assertNotIgnored(Request("http://site.local"), middleware)
@deferred_f_from_coro_f @coroutine_test
async def test_ignore_robotstxt_request(self): async def test_ignore_robotstxt_request(self):
self.crawler.settings.set("ROBOTSTXT_OBEY", True) self.crawler.settings.set("ROBOTSTXT_OBEY", True)
@ -216,7 +216,7 @@ Disallow: /some/randome/page.html
middleware.process_request_2(rp, Request("http://site.local/allowed")) middleware.process_request_2(rp, Request("http://site.local/allowed"))
rp.allowed.assert_called_once_with("http://site.local/allowed", "Examplebot") rp.allowed.assert_called_once_with("http://site.local/allowed", "Examplebot")
@deferred_f_from_coro_f @coroutine_test
async def test_robotstxt_local_file(self): async def test_robotstxt_local_file(self):
middleware = RobotsTxtMiddleware(self._get_emptybody_crawler()) middleware = RobotsTxtMiddleware(self._get_emptybody_crawler())
middleware.process_request_2 = mock.MagicMock() middleware.process_request_2 = mock.MagicMock()

6
tests/test_downloaderslotssettings.py
View File

@ -11,7 +11,7 @@ from scrapy.utils.spider import DefaultSpider
from scrapy.utils.test import get_crawler from scrapy.utils.test import get_crawler
from tests.mockserver.http import MockServer from tests.mockserver.http import MockServer
from tests.spiders import MetaSpider from tests.spiders import MetaSpider
from tests.utils.decorators import deferred_f_from_coro_f, inlineCallbacks from tests.utils.decorators import coroutine_test, inline_callbacks_test
class DownloaderSlotsSettingsTestSpider(MetaSpider): class DownloaderSlotsSettingsTestSpider(MetaSpider):
@ -68,7 +68,7 @@ class TestCrawl:
self.runner = CrawlerRunner() self.runner = CrawlerRunner()
@pytest.mark.requires_http_handler @pytest.mark.requires_http_handler
@inlineCallbacks @inline_callbacks_test
def test_delay(self): def test_delay(self):
crawler = get_crawler(DownloaderSlotsSettingsTestSpider) crawler = get_crawler(DownloaderSlotsSettingsTestSpider)
yield crawler.crawl(mockserver=self.mockserver) yield crawler.crawl(mockserver=self.mockserver)
@ -137,7 +137,7 @@ def test_get_slot_deprecated_spider_arg():
"scrapy.pqueues.DownloaderAwarePriorityQueue", "scrapy.pqueues.DownloaderAwarePriorityQueue",
], ],
) )
@deferred_f_from_coro_f @coroutine_test
async def test_none_slot_with_priority_queue( async def test_none_slot_with_priority_queue(
mockserver: MockServer, priority_queue_class: str mockserver: MockServer, priority_queue_class: str
) -> None: ) -> None:

46
tests/test_engine.py
View File

@ -35,7 +35,7 @@ from scrapy.utils.signal import disconnect_all
from scrapy.utils.spider import DefaultSpider from scrapy.utils.spider import DefaultSpider
from scrapy.utils.test import get_crawler from scrapy.utils.test import get_crawler
from tests import get_testdata from tests import get_testdata
from tests.utils.decorators import deferred_f_from_coro_f, inlineCallbacks from tests.utils.decorators import coroutine_test, inline_callbacks_test
if TYPE_CHECKING: if TYPE_CHECKING:
from scrapy.core.scheduler import Scheduler from scrapy.core.scheduler import Scheduler
@ -374,7 +374,7 @@ class TestEngineBase:
class TestEngine(TestEngineBase): class TestEngine(TestEngineBase):
@pytest.mark.requires_http_handler @pytest.mark.requires_http_handler
@deferred_f_from_coro_f @coroutine_test
async def test_crawler(self, mockserver: MockServer) -> None: async def test_crawler(self, mockserver: MockServer) -> None:
for spider in ( for spider in (
MySpider, MySpider,
@ -392,7 +392,7 @@ class TestEngine(TestEngineBase):
self._assert_bytes_received(run) self._assert_bytes_received(run)
@pytest.mark.requires_http_handler @pytest.mark.requires_http_handler
@deferred_f_from_coro_f @coroutine_test
async def test_crawler_dupefilter(self, mockserver: MockServer) -> None: async def test_crawler_dupefilter(self, mockserver: MockServer) -> None:
run = CrawlerRun(DupeFilterSpider) run = CrawlerRun(DupeFilterSpider)
await run.run(mockserver) await run.run(mockserver)
@ -400,14 +400,14 @@ class TestEngine(TestEngineBase):
self._assert_dropped_requests(run) self._assert_dropped_requests(run)
@pytest.mark.requires_http_handler @pytest.mark.requires_http_handler
@deferred_f_from_coro_f @coroutine_test
async def test_crawler_itemerror(self, mockserver: MockServer) -> None: async def test_crawler_itemerror(self, mockserver: MockServer) -> None:
run = CrawlerRun(ItemZeroDivisionErrorSpider) run = CrawlerRun(ItemZeroDivisionErrorSpider)
await run.run(mockserver) await run.run(mockserver)
self._assert_items_error(run) self._assert_items_error(run)
@pytest.mark.requires_http_handler @pytest.mark.requires_http_handler
@deferred_f_from_coro_f @coroutine_test
async def test_crawler_change_close_reason_on_idle( async def test_crawler_change_close_reason_on_idle(
self, mockserver: MockServer self, mockserver: MockServer
) -> None: ) -> None:
@ -418,7 +418,7 @@ class TestEngine(TestEngineBase):
"reason": "custom_reason", "reason": "custom_reason",
} == run.signals_caught[signals.spider_closed] } == run.signals_caught[signals.spider_closed]
@deferred_f_from_coro_f @coroutine_test
async def test_close_downloader(self): async def test_close_downloader(self):
e = ExecutionEngine(get_crawler(MySpider), lambda _: None) e = ExecutionEngine(get_crawler(MySpider), lambda _: None)
await e.close_async() await e.close_async()
@ -436,7 +436,7 @@ class TestEngine(TestEngineBase):
get_crawler(MySpider, {"DOWNLOADER": BadDownloader}), lambda _: None get_crawler(MySpider, {"DOWNLOADER": BadDownloader}), lambda _: None
) )
@inlineCallbacks @inline_callbacks_test
def test_start_already_running_exception(self): def test_start_already_running_exception(self):
crawler = get_crawler(DefaultSpider) crawler = get_crawler(DefaultSpider)
crawler.spider = crawler._create_spider() crawler.spider = crawler._create_spider()
@ -449,7 +449,7 @@ class TestEngine(TestEngineBase):
yield deferred_from_coro(e.stop_async()) yield deferred_from_coro(e.stop_async())
@pytest.mark.only_asyncio @pytest.mark.only_asyncio
@deferred_f_from_coro_f @coroutine_test
async def test_start_already_running_exception_asyncio(self): async def test_start_already_running_exception_asyncio(self):
crawler = get_crawler(DefaultSpider) crawler = get_crawler(DefaultSpider)
crawler.spider = crawler._create_spider() crawler.spider = crawler._create_spider()
@ -460,7 +460,7 @@ class TestEngine(TestEngineBase):
await asyncio.gather(e.start_async(), e.start_async()) await asyncio.gather(e.start_async(), e.start_async())
await e.stop_async() await e.stop_async()
@inlineCallbacks @inline_callbacks_test
def test_start_request_processing_exception(self): def test_start_request_processing_exception(self):
class BadRequestFingerprinter: class BadRequestFingerprinter:
def fingerprint(self, request): def fingerprint(self, request):
@ -527,7 +527,7 @@ class TestEngineDownloadAsync:
async def _download(engine: ExecutionEngine, request: Request) -> Response: async def _download(engine: ExecutionEngine, request: Request) -> Response:
return await engine.download_async(request) return await engine.download_async(request)
@deferred_f_from_coro_f @coroutine_test
async def test_download_async_success(self, engine): async def test_download_async_success(self, engine):
"""Test basic successful async download of a request.""" """Test basic successful async download of a request."""
request = Request("http://example.com") request = Request("http://example.com")
@ -543,7 +543,7 @@ class TestEngineDownloadAsync:
engine._slot.remove_request.assert_called_once_with(request) engine._slot.remove_request.assert_called_once_with(request)
engine.downloader.fetch.assert_called_once_with(request) engine.downloader.fetch.assert_called_once_with(request)
@deferred_f_from_coro_f @coroutine_test
async def test_download_async_redirect(self, engine): async def test_download_async_redirect(self, engine):
"""Test async download with a redirect request.""" """Test async download with a redirect request."""
original_request = Request("http://example.com") original_request = Request("http://example.com")
@ -569,7 +569,7 @@ class TestEngineDownloadAsync:
[call(original_request), call(redirect_request)] [call(original_request), call(redirect_request)]
) )
@deferred_f_from_coro_f @coroutine_test
async def test_download_async_no_spider(self, engine): async def test_download_async_no_spider(self, engine):
"""Test async download attempt when no spider is available.""" """Test async download attempt when no spider is available."""
request = Request("http://example.com") request = Request("http://example.com")
@ -577,7 +577,7 @@ class TestEngineDownloadAsync:
with pytest.raises(RuntimeError, match="No open spider to crawl:"): with pytest.raises(RuntimeError, match="No open spider to crawl:"):
await self._download(engine, request) await self._download(engine, request)
@deferred_f_from_coro_f @coroutine_test
async def test_download_async_failure(self, engine): async def test_download_async_failure(self, engine):
"""Test async download when the downloader raises an exception.""" """Test async download when the downloader raises an exception."""
request = Request("http://example.com") request = Request("http://example.com")
@ -647,7 +647,7 @@ class TestEngineCloseSpider:
crawler.spider = crawler._create_spider() crawler.spider = crawler._create_spider()
return crawler return crawler
@deferred_f_from_coro_f @coroutine_test
async def test_no_slot(self, crawler: Crawler) -> None: async def test_no_slot(self, crawler: Crawler) -> None:
engine = ExecutionEngine(crawler, lambda _: None) engine = ExecutionEngine(crawler, lambda _: None)
crawler.engine = engine crawler.engine = engine
@ -660,14 +660,14 @@ class TestEngineCloseSpider:
engine._slot = slot engine._slot = slot
await engine.close_spider_async() await engine.close_spider_async()
@deferred_f_from_coro_f @coroutine_test
async def test_no_spider(self, crawler: Crawler) -> None: async def test_no_spider(self, crawler: Crawler) -> None:
engine = ExecutionEngine(crawler, lambda _: None) engine = ExecutionEngine(crawler, lambda _: None)
with pytest.raises(RuntimeError, match="Spider not opened"): with pytest.raises(RuntimeError, match="Spider not opened"):
await engine.close_spider_async() await engine.close_spider_async()
engine.downloader.close() # cleanup engine.downloader.close() # cleanup
@deferred_f_from_coro_f @coroutine_test
async def test_exception_slot( async def test_exception_slot(
self, crawler: Crawler, caplog: pytest.LogCaptureFixture self, crawler: Crawler, caplog: pytest.LogCaptureFixture
) -> None: ) -> None:
@ -679,7 +679,7 @@ class TestEngineCloseSpider:
await engine.close_spider_async() await engine.close_spider_async()
assert "Slot close failure" in caplog.text assert "Slot close failure" in caplog.text
@deferred_f_from_coro_f @coroutine_test
async def test_exception_downloader( async def test_exception_downloader(
self, crawler: Crawler, caplog: pytest.LogCaptureFixture self, crawler: Crawler, caplog: pytest.LogCaptureFixture
) -> None: ) -> None:
@ -690,7 +690,7 @@ class TestEngineCloseSpider:
await engine.close_spider_async() await engine.close_spider_async()
assert "Downloader close failure" in caplog.text assert "Downloader close failure" in caplog.text
@deferred_f_from_coro_f @coroutine_test
async def test_exception_scraper( async def test_exception_scraper(
self, crawler: Crawler, caplog: pytest.LogCaptureFixture self, crawler: Crawler, caplog: pytest.LogCaptureFixture
) -> None: ) -> None:
@ -701,7 +701,7 @@ class TestEngineCloseSpider:
await engine.close_spider_async() await engine.close_spider_async()
assert "Scraper close failure" in caplog.text assert "Scraper close failure" in caplog.text
@deferred_f_from_coro_f @coroutine_test
async def test_exception_scheduler( async def test_exception_scheduler(
self, crawler: Crawler, caplog: pytest.LogCaptureFixture self, crawler: Crawler, caplog: pytest.LogCaptureFixture
) -> None: ) -> None:
@ -713,7 +713,7 @@ class TestEngineCloseSpider:
await engine.close_spider_async() await engine.close_spider_async()
assert "Scheduler close failure" in caplog.text assert "Scheduler close failure" in caplog.text
@deferred_f_from_coro_f @coroutine_test
async def test_exception_signal( async def test_exception_signal(
self, crawler: Crawler, caplog: pytest.LogCaptureFixture self, crawler: Crawler, caplog: pytest.LogCaptureFixture
) -> None: ) -> None:
@ -731,7 +731,7 @@ class TestEngineCloseSpider:
reason="cancelled", reason="cancelled",
) )
@deferred_f_from_coro_f @coroutine_test
async def test_exception_stats( async def test_exception_stats(
self, crawler: Crawler, caplog: pytest.LogCaptureFixture self, crawler: Crawler, caplog: pytest.LogCaptureFixture
) -> None: ) -> None:
@ -742,7 +742,7 @@ class TestEngineCloseSpider:
await engine.close_spider_async() await engine.close_spider_async()
assert "Stats close failure" in caplog.text assert "Stats close failure" in caplog.text
@deferred_f_from_coro_f @coroutine_test
async def test_exception_callback( async def test_exception_callback(
self, crawler: Crawler, caplog: pytest.LogCaptureFixture self, crawler: Crawler, caplog: pytest.LogCaptureFixture
) -> None: ) -> None:
@ -752,7 +752,7 @@ class TestEngineCloseSpider:
await engine.close_spider_async() await engine.close_spider_async()
assert "Error running spider_closed_callback" in caplog.text assert "Error running spider_closed_callback" in caplog.text
@deferred_f_from_coro_f @coroutine_test
async def test_exception_async_callback( async def test_exception_async_callback(
self, crawler: Crawler, caplog: pytest.LogCaptureFixture self, crawler: Crawler, caplog: pytest.LogCaptureFixture
) -> None: ) -> None:

14
tests/test_engine_loop.py
View File

@ -12,7 +12,7 @@ from scrapy.utils.defer import maybe_deferred_to_future
from scrapy.utils.test import get_crawler from scrapy.utils.test import get_crawler
from tests.mockserver.http import MockServer from tests.mockserver.http import MockServer
from tests.test_scheduler import MemoryScheduler from tests.test_scheduler import MemoryScheduler
from tests.utils.decorators import deferred_f_from_coro_f from tests.utils.decorators import coroutine_test
if TYPE_CHECKING: if TYPE_CHECKING:
from scrapy.http import Response from scrapy.http import Response
@ -28,7 +28,7 @@ async def sleep(seconds: float = 0.001) -> None:
class TestMain: class TestMain:
@pytest.mark.requires_reactor # TODO @pytest.mark.requires_reactor # TODO
@deferred_f_from_coro_f @coroutine_test
async def test_sleep(self): async def test_sleep(self):
"""Neither asynchronous sleeps on Spider.start() nor the equivalent on """Neither asynchronous sleeps on Spider.start() nor the equivalent on
the scheduler (returning no requests while also returning True from the scheduler (returning no requests while also returning True from
@ -87,7 +87,7 @@ class TestMain:
expected_urls = ["data:,a", "data:,b", "data:,c", "data:,d"] expected_urls = ["data:,a", "data:,b", "data:,c", "data:,d"]
assert actual_urls == expected_urls, f"{actual_urls=} != {expected_urls=}" assert actual_urls == expected_urls, f"{actual_urls=} != {expected_urls=}"
@deferred_f_from_coro_f @coroutine_test
async def test_close_during_start_iteration( async def test_close_during_start_iteration(
self, caplog: pytest.LogCaptureFixture self, caplog: pytest.LogCaptureFixture
) -> None: ) -> None:
@ -188,7 +188,7 @@ class TestRequestSendOrder:
expected_nums = sorted(start_nums + cb_nums) expected_nums = sorted(start_nums + cb_nums)
assert actual_nums == expected_nums, f"{actual_nums=} != {expected_nums=}" assert actual_nums == expected_nums, f"{actual_nums=} != {expected_nums=}"
@deferred_f_from_coro_f @coroutine_test
async def test_default(self): async def test_default(self):
"""By default, callback requests take priority over start requests and """By default, callback requests take priority over start requests and
are sent in order. Priority matters, but given the same priority, a are sent in order. Priority matters, but given the same priority, a
@ -228,7 +228,7 @@ class TestRequestSendOrder:
parse_fn=parse, parse_fn=parse,
) )
@deferred_f_from_coro_f @coroutine_test
async def test_lifo_start(self): async def test_lifo_start(self):
"""Changing the queues of start requests to LIFO, matching the queues """Changing the queues of start requests to LIFO, matching the queues
of non-start requests, does not cause all requests to be stored in the of non-start requests, does not cause all requests to be stored in the
@ -271,7 +271,7 @@ class TestRequestSendOrder:
parse_fn=parse, parse_fn=parse,
) )
@deferred_f_from_coro_f @coroutine_test
async def test_shared_queues(self): async def test_shared_queues(self):
"""If SCHEDULER_START_*_QUEUE is falsy, start requests and other """If SCHEDULER_START_*_QUEUE is falsy, start requests and other
requests share the same queue, i.e. start requests are not priorized requests share the same queue, i.e. start requests are not priorized
@ -333,7 +333,7 @@ class TestRequestSendOrder:
# spiders. # spiders.
@pytest.mark.requires_http_handler @pytest.mark.requires_http_handler
@deferred_f_from_coro_f @coroutine_test
async def test_lazy(self): async def test_lazy(self):
start_nums = [1, 2, 4] start_nums = [1, 2, 4]
cb_nums = [3] cb_nums = [3]

4
tests/test_engine_stop_download_bytes.py
View File

@ -13,7 +13,7 @@ from tests.test_engine import (
MySpider, MySpider,
TestEngineBase, TestEngineBase,
) )
from tests.utils.decorators import deferred_f_from_coro_f from tests.utils.decorators import coroutine_test
if TYPE_CHECKING: if TYPE_CHECKING:
from tests.mockserver.http import MockServer from tests.mockserver.http import MockServer
@ -27,7 +27,7 @@ class BytesReceivedCrawlerRun(CrawlerRun):
class TestBytesReceivedEngine(TestEngineBase): class TestBytesReceivedEngine(TestEngineBase):
@pytest.mark.requires_http_handler @pytest.mark.requires_http_handler
@deferred_f_from_coro_f @coroutine_test
async def test_crawler( async def test_crawler(
self, mockserver: MockServer, caplog: pytest.LogCaptureFixture self, mockserver: MockServer, caplog: pytest.LogCaptureFixture
) -> None: ) -> None:

4
tests/test_engine_stop_download_headers.py
View File

@ -13,7 +13,7 @@ from tests.test_engine import (
MySpider, MySpider,
TestEngineBase, TestEngineBase,
) )
from tests.utils.decorators import deferred_f_from_coro_f from tests.utils.decorators import coroutine_test
if TYPE_CHECKING: if TYPE_CHECKING:
from tests.mockserver.http import MockServer from tests.mockserver.http import MockServer
@ -27,7 +27,7 @@ class HeadersReceivedCrawlerRun(CrawlerRun):
class TestHeadersReceivedEngine(TestEngineBase): class TestHeadersReceivedEngine(TestEngineBase):
@pytest.mark.requires_http_handler @pytest.mark.requires_http_handler
@deferred_f_from_coro_f @coroutine_test
async def test_crawler( async def test_crawler(
self, mockserver: MockServer, caplog: pytest.LogCaptureFixture self, mockserver: MockServer, caplog: pytest.LogCaptureFixture
) -> None: ) -> None:

8
tests/test_extension_telnet.py
View File

@ -4,7 +4,7 @@ from twisted.cred import credentials
from scrapy.extensions.telnet import TelnetConsole from scrapy.extensions.telnet import TelnetConsole
from scrapy.utils.test import get_crawler from scrapy.utils.test import get_crawler
from tests.utils.decorators import inlineCallbacks from tests.utils.decorators import inline_callbacks_test
pytestmark = pytest.mark.requires_reactor pytestmark = pytest.mark.requires_reactor
@ -23,7 +23,7 @@ class TestTelnetExtension:
return console, portal return console, portal
@inlineCallbacks @inline_callbacks_test
def test_bad_credentials(self): def test_bad_credentials(self):
console, portal = self._get_console_and_portal() console, portal = self._get_console_and_portal()
creds = credentials.UsernamePassword(b"username", b"password") creds = credentials.UsernamePassword(b"username", b"password")
@ -32,7 +32,7 @@ class TestTelnetExtension:
yield d yield d
console.stop_listening() console.stop_listening()
@inlineCallbacks @inline_callbacks_test
def test_good_credentials(self): def test_good_credentials(self):
console, portal = self._get_console_and_portal() console, portal = self._get_console_and_portal()
creds = credentials.UsernamePassword( creds = credentials.UsernamePassword(
@ -42,7 +42,7 @@ class TestTelnetExtension:
yield d yield d
console.stop_listening() console.stop_listening()
@inlineCallbacks @inline_callbacks_test
def test_custom_credentials(self): def test_custom_credentials(self):
settings = { settings = {
"TELNETCONSOLE_USERNAME": "user", "TELNETCONSOLE_USERNAME": "user",

136
tests/test_feedexport.py
View File

@ -55,7 +55,7 @@ from scrapy.utils.test import get_crawler
from tests.mockserver.ftp import MockFTPServer from tests.mockserver.ftp import MockFTPServer
from tests.mockserver.http import MockServer from tests.mockserver.http import MockServer
from tests.spiders import ItemSpider from tests.spiders import ItemSpider
from tests.utils.decorators import deferred_f_from_coro_f, inlineCallbacks from tests.utils.decorators import coroutine_test, inline_callbacks_test
if TYPE_CHECKING: if TYPE_CHECKING:
from collections.abc import Callable, Iterable from collections.abc import Callable, Iterable
@ -192,7 +192,7 @@ class TestFTPFeedStorage:
finally: finally:
path.unlink() path.unlink()
@deferred_f_from_coro_f @coroutine_test
async def test_append(self): async def test_append(self):
with MockFTPServer() as ftp_server: with MockFTPServer() as ftp_server:
filename = "file" filename = "file"
@ -202,7 +202,7 @@ class TestFTPFeedStorage:
await self._store(url, b"bar", feed_options=feed_options) await self._store(url, b"bar", feed_options=feed_options)
self._assert_stored(ftp_server.path / filename, b"foobar") self._assert_stored(ftp_server.path / filename, b"foobar")
@deferred_f_from_coro_f @coroutine_test
async def test_overwrite(self): async def test_overwrite(self):
with MockFTPServer() as ftp_server: with MockFTPServer() as ftp_server:
filename = "file" filename = "file"
@ -211,7 +211,7 @@ class TestFTPFeedStorage:
await self._store(url, b"bar") await self._store(url, b"bar")
self._assert_stored(ftp_server.path / filename, b"bar") self._assert_stored(ftp_server.path / filename, b"bar")
@deferred_f_from_coro_f @coroutine_test
async def test_append_active_mode(self): async def test_append_active_mode(self):
with MockFTPServer() as ftp_server: with MockFTPServer() as ftp_server:
settings = {"FEED_STORAGE_FTP_ACTIVE": True} settings = {"FEED_STORAGE_FTP_ACTIVE": True}
@ -222,7 +222,7 @@ class TestFTPFeedStorage:
await self._store(url, b"bar", feed_options=feed_options, settings=settings) await self._store(url, b"bar", feed_options=feed_options, settings=settings)
self._assert_stored(ftp_server.path / filename, b"foobar") self._assert_stored(ftp_server.path / filename, b"foobar")
@deferred_f_from_coro_f @coroutine_test
async def test_overwrite_active_mode(self): async def test_overwrite_active_mode(self):
with MockFTPServer() as ftp_server: with MockFTPServer() as ftp_server:
settings = {"FEED_STORAGE_FTP_ACTIVE": True} settings = {"FEED_STORAGE_FTP_ACTIVE": True}
@ -314,7 +314,7 @@ class TestS3FeedStorage:
assert storage.access_key == "uri_key" assert storage.access_key == "uri_key"
assert storage.secret_key == "uri_secret" assert storage.secret_key == "uri_secret"
@deferred_f_from_coro_f @coroutine_test
async def test_store(self): async def test_store(self):
settings = { settings = {
"AWS_ACCESS_KEY_ID": "access_key", "AWS_ACCESS_KEY_ID": "access_key",
@ -455,7 +455,7 @@ class TestS3FeedStorage:
assert storage.region_name == region_name assert storage.region_name == region_name
assert storage.s3_client._client_config.region_name == region_name assert storage.s3_client._client_config.region_name == region_name
@deferred_f_from_coro_f @coroutine_test
async def test_store_without_acl(self): async def test_store_without_acl(self):
storage = S3FeedStorage( storage = S3FeedStorage(
"s3://mybucket/export.csv", "s3://mybucket/export.csv",
@ -475,7 +475,7 @@ class TestS3FeedStorage:
) )
assert acl is None assert acl is None
@deferred_f_from_coro_f @coroutine_test
async def test_store_with_acl(self): async def test_store_with_acl(self):
storage = S3FeedStorage( storage = S3FeedStorage(
"s3://mybucket/export.csv", "access_key", "secret_key", "custom-acl" "s3://mybucket/export.csv", "access_key", "secret_key", "custom-acl"
@ -540,7 +540,7 @@ class TestGCSFeedStorage:
storage = GCSFeedStorage.from_crawler(crawler, "gs://mybucket/export.csv") storage = GCSFeedStorage.from_crawler(crawler, "gs://mybucket/export.csv")
assert storage.acl is None assert storage.acl is None
@deferred_f_from_coro_f @coroutine_test
async def test_store(self): async def test_store(self):
try: try:
from google.cloud.storage import Client # noqa: F401,PLC0415 from google.cloud.storage import Client # noqa: F401,PLC0415
@ -1006,7 +1006,7 @@ class TestFeedExport(TestFeedExportBase):
result = self._load_until_eof(data["marshal"], load_func=marshal.load) result = self._load_until_eof(data["marshal"], load_func=marshal.load)
assert result == expected assert result == expected
@inlineCallbacks @inline_callbacks_test
def test_stats_file_success(self): def test_stats_file_success(self):
settings = { settings = {
"FEEDS": { "FEEDS": {
@ -1020,7 +1020,7 @@ class TestFeedExport(TestFeedExportBase):
assert "feedexport/success_count/FileFeedStorage" in crawler.stats.get_stats() assert "feedexport/success_count/FileFeedStorage" in crawler.stats.get_stats()
assert crawler.stats.get_value("feedexport/success_count/FileFeedStorage") == 1 assert crawler.stats.get_value("feedexport/success_count/FileFeedStorage") == 1
@inlineCallbacks @inline_callbacks_test
def test_stats_file_failed(self): def test_stats_file_failed(self):
settings = { settings = {
"FEEDS": { "FEEDS": {
@ -1038,7 +1038,7 @@ class TestFeedExport(TestFeedExportBase):
assert "feedexport/failed_count/FileFeedStorage" in crawler.stats.get_stats() assert "feedexport/failed_count/FileFeedStorage" in crawler.stats.get_stats()
assert crawler.stats.get_value("feedexport/failed_count/FileFeedStorage") == 1 assert crawler.stats.get_value("feedexport/failed_count/FileFeedStorage") == 1
@inlineCallbacks @inline_callbacks_test
def test_stats_multiple_file(self): def test_stats_multiple_file(self):
settings = { settings = {
"FEEDS": { "FEEDS": {
@ -1060,7 +1060,7 @@ class TestFeedExport(TestFeedExportBase):
crawler.stats.get_value("feedexport/success_count/StdoutFeedStorage") == 1 crawler.stats.get_value("feedexport/success_count/StdoutFeedStorage") == 1
) )
@deferred_f_from_coro_f @coroutine_test
async def test_export_items(self): async def test_export_items(self):
# feed exporters use field names from Item # feed exporters use field names from Item
items = [ items = [
@ -1074,7 +1074,7 @@ class TestFeedExport(TestFeedExportBase):
header = self.MyItem.fields.keys() header = self.MyItem.fields.keys()
await self.assertExported(items, header, rows) await self.assertExported(items, header, rows)
@deferred_f_from_coro_f @coroutine_test
async def test_export_no_items_not_store_empty(self): async def test_export_no_items_not_store_empty(self):
for fmt in ("json", "jsonlines", "xml", "csv"): for fmt in ("json", "jsonlines", "xml", "csv"):
settings = { settings = {
@ -1086,7 +1086,7 @@ class TestFeedExport(TestFeedExportBase):
data = await self.exported_no_data(settings) data = await self.exported_no_data(settings)
assert data[fmt] is None assert data[fmt] is None
@deferred_f_from_coro_f @coroutine_test
async def test_start_finish_exporting_items(self): async def test_start_finish_exporting_items(self):
items = [ items = [
self.MyItem({"foo": "bar1", "egg": "spam1"}), self.MyItem({"foo": "bar1", "egg": "spam1"}),
@ -1106,7 +1106,7 @@ class TestFeedExport(TestFeedExportBase):
assert not listener.start_without_finish assert not listener.start_without_finish
assert not listener.finish_without_start assert not listener.finish_without_start
@deferred_f_from_coro_f @coroutine_test
async def test_start_finish_exporting_no_items(self): async def test_start_finish_exporting_no_items(self):
items = [] items = []
settings = { settings = {
@ -1124,7 +1124,7 @@ class TestFeedExport(TestFeedExportBase):
assert not listener.start_without_finish assert not listener.start_without_finish
assert not listener.finish_without_start assert not listener.finish_without_start
@deferred_f_from_coro_f @coroutine_test
async def test_start_finish_exporting_items_exception(self): async def test_start_finish_exporting_items_exception(self):
items = [ items = [
self.MyItem({"foo": "bar1", "egg": "spam1"}), self.MyItem({"foo": "bar1", "egg": "spam1"}),
@ -1145,7 +1145,7 @@ class TestFeedExport(TestFeedExportBase):
assert not listener.start_without_finish assert not listener.start_without_finish
assert not listener.finish_without_start assert not listener.finish_without_start
@deferred_f_from_coro_f @coroutine_test
async def test_start_finish_exporting_no_items_exception(self): async def test_start_finish_exporting_no_items_exception(self):
items = [] items = []
settings = { settings = {
@ -1164,7 +1164,7 @@ class TestFeedExport(TestFeedExportBase):
assert not listener.start_without_finish assert not listener.start_without_finish
assert not listener.finish_without_start assert not listener.finish_without_start
@deferred_f_from_coro_f @coroutine_test
async def test_export_no_items_store_empty(self): async def test_export_no_items_store_empty(self):
formats = ( formats = (
("json", b"[]"), ("json", b"[]"),
@ -1184,7 +1184,7 @@ class TestFeedExport(TestFeedExportBase):
data = await self.exported_no_data(settings) data = await self.exported_no_data(settings)
assert expctd == data[fmt] assert expctd == data[fmt]
@deferred_f_from_coro_f @coroutine_test
async def test_export_no_items_multiple_feeds(self): async def test_export_no_items_multiple_feeds(self):
"""Make sure that `storage.store` is called for every feed.""" """Make sure that `storage.store` is called for every feed."""
settings = { settings = {
@ -1202,7 +1202,7 @@ class TestFeedExport(TestFeedExportBase):
assert str(log).count("Storage.store is called") == 0 assert str(log).count("Storage.store is called") == 0
@deferred_f_from_coro_f @coroutine_test
async def test_export_multiple_item_classes(self): async def test_export_multiple_item_classes(self):
items = [ items = [
self.MyItem({"foo": "bar1", "egg": "spam1"}), self.MyItem({"foo": "bar1", "egg": "spam1"}),
@ -1224,7 +1224,7 @@ class TestFeedExport(TestFeedExportBase):
await self.assertExportedCsv(items, header, rows_csv) await self.assertExportedCsv(items, header, rows_csv)
await self.assertExportedJsonLines(items, rows_jl) await self.assertExportedJsonLines(items, rows_jl)
@deferred_f_from_coro_f @coroutine_test
async def test_export_items_empty_field_list(self): async def test_export_items_empty_field_list(self):
# FEED_EXPORT_FIELDS==[] means the same as default None # FEED_EXPORT_FIELDS==[] means the same as default None
items = [{"foo": "bar"}] items = [{"foo": "bar"}]
@ -1234,7 +1234,7 @@ class TestFeedExport(TestFeedExportBase):
await self.assertExportedCsv(items, header, rows) await self.assertExportedCsv(items, header, rows)
await self.assertExportedJsonLines(items, rows, settings) await self.assertExportedJsonLines(items, rows, settings)
@deferred_f_from_coro_f @coroutine_test
async def test_export_items_field_list(self): async def test_export_items_field_list(self):
items = [{"foo": "bar"}] items = [{"foo": "bar"}]
header = ["foo", "baz"] header = ["foo", "baz"]
@ -1242,7 +1242,7 @@ class TestFeedExport(TestFeedExportBase):
settings = {"FEED_EXPORT_FIELDS": header} settings = {"FEED_EXPORT_FIELDS": header}
await self.assertExported(items, header, rows, settings=settings) await self.assertExported(items, header, rows, settings=settings)
@deferred_f_from_coro_f @coroutine_test
async def test_export_items_comma_separated_field_list(self): async def test_export_items_comma_separated_field_list(self):
items = [{"foo": "bar"}] items = [{"foo": "bar"}]
header = ["foo", "baz"] header = ["foo", "baz"]
@ -1250,7 +1250,7 @@ class TestFeedExport(TestFeedExportBase):
settings = {"FEED_EXPORT_FIELDS": ",".join(header)} settings = {"FEED_EXPORT_FIELDS": ",".join(header)}
await self.assertExported(items, header, rows, settings=settings) await self.assertExported(items, header, rows, settings=settings)
@deferred_f_from_coro_f @coroutine_test
async def test_export_items_json_field_list(self): async def test_export_items_json_field_list(self):
items = [{"foo": "bar"}] items = [{"foo": "bar"}]
header = ["foo", "baz"] header = ["foo", "baz"]
@ -1258,7 +1258,7 @@ class TestFeedExport(TestFeedExportBase):
settings = {"FEED_EXPORT_FIELDS": json.dumps(header)} settings = {"FEED_EXPORT_FIELDS": json.dumps(header)}
await self.assertExported(items, header, rows, settings=settings) await self.assertExported(items, header, rows, settings=settings)
@deferred_f_from_coro_f @coroutine_test
async def test_export_items_field_names(self): async def test_export_items_field_names(self):
items = [{"foo": "bar"}] items = [{"foo": "bar"}]
header = {"foo": "Foo"} header = {"foo": "Foo"}
@ -1266,7 +1266,7 @@ class TestFeedExport(TestFeedExportBase):
settings = {"FEED_EXPORT_FIELDS": header} settings = {"FEED_EXPORT_FIELDS": header}
await self.assertExported(items, list(header.values()), rows, settings=settings) await self.assertExported(items, list(header.values()), rows, settings=settings)
@deferred_f_from_coro_f @coroutine_test
async def test_export_items_dict_field_names(self): async def test_export_items_dict_field_names(self):
items = [{"foo": "bar"}] items = [{"foo": "bar"}]
header = { header = {
@ -1277,7 +1277,7 @@ class TestFeedExport(TestFeedExportBase):
settings = {"FEED_EXPORT_FIELDS": header} settings = {"FEED_EXPORT_FIELDS": header}
await self.assertExported(items, ["Baz", "Foo"], rows, settings=settings) await self.assertExported(items, ["Baz", "Foo"], rows, settings=settings)
@deferred_f_from_coro_f @coroutine_test
async def test_export_items_json_field_names(self): async def test_export_items_json_field_names(self):
items = [{"foo": "bar"}] items = [{"foo": "bar"}]
header = {"foo": "Foo"} header = {"foo": "Foo"}
@ -1285,7 +1285,7 @@ class TestFeedExport(TestFeedExportBase):
settings = {"FEED_EXPORT_FIELDS": json.dumps(header)} settings = {"FEED_EXPORT_FIELDS": json.dumps(header)}
await self.assertExported(items, list(header.values()), rows, settings=settings) await self.assertExported(items, list(header.values()), rows, settings=settings)
@deferred_f_from_coro_f @coroutine_test
async def test_export_based_on_item_classes(self): async def test_export_based_on_item_classes(self):
items = [ items = [
self.MyItem({"foo": "bar1", "egg": "spam1"}), self.MyItem({"foo": "bar1", "egg": "spam1"}),
@ -1331,7 +1331,7 @@ class TestFeedExport(TestFeedExportBase):
for fmt, expected in formats.items(): for fmt, expected in formats.items():
assert data[fmt] == expected assert data[fmt] == expected
@deferred_f_from_coro_f @coroutine_test
async def test_export_based_on_custom_filters(self): async def test_export_based_on_custom_filters(self):
items = [ items = [
self.MyItem({"foo": "bar1", "egg": "spam1"}), self.MyItem({"foo": "bar1", "egg": "spam1"}),
@ -1390,7 +1390,7 @@ class TestFeedExport(TestFeedExportBase):
for fmt, expected in formats.items(): for fmt, expected in formats.items():
assert data[fmt] == expected assert data[fmt] == expected
@deferred_f_from_coro_f @coroutine_test
async def test_export_dicts(self): async def test_export_dicts(self):
# When dicts are used, only keys from the first row are used as # When dicts are used, only keys from the first row are used as
# a header for CSV, and all fields are used for JSON Lines. # a header for CSV, and all fields are used for JSON Lines.
@ -1403,7 +1403,7 @@ class TestFeedExport(TestFeedExportBase):
await self.assertExportedCsv(items, ["foo", "egg"], rows_csv) await self.assertExportedCsv(items, ["foo", "egg"], rows_csv)
await self.assertExportedJsonLines(items, rows_jl) await self.assertExportedJsonLines(items, rows_jl)
@deferred_f_from_coro_f @coroutine_test
async def test_export_tuple(self): async def test_export_tuple(self):
items = [ items = [
{"foo": "bar1", "egg": "spam1"}, {"foo": "bar1", "egg": "spam1"},
@ -1414,7 +1414,7 @@ class TestFeedExport(TestFeedExportBase):
rows = [{"foo": "bar1", "baz": ""}, {"foo": "bar2", "baz": "quux"}] rows = [{"foo": "bar1", "baz": ""}, {"foo": "bar2", "baz": "quux"}]
await self.assertExported(items, ["foo", "baz"], rows, settings=settings) await self.assertExported(items, ["foo", "baz"], rows, settings=settings)
@deferred_f_from_coro_f @coroutine_test
async def test_export_feed_export_fields(self): async def test_export_feed_export_fields(self):
# FEED_EXPORT_FIELDS option allows to order export fields # FEED_EXPORT_FIELDS option allows to order export fields
# and to select a subset of fields to export, both for Items and dicts. # and to select a subset of fields to export, both for Items and dicts.
@ -1440,7 +1440,7 @@ class TestFeedExport(TestFeedExportBase):
rows = [{"egg": "spam1", "baz": ""}, {"egg": "spam2", "baz": "quux2"}] rows = [{"egg": "spam1", "baz": ""}, {"egg": "spam2", "baz": "quux2"}]
await self.assertExported(items, ["egg", "baz"], rows, settings=settings) await self.assertExported(items, ["egg", "baz"], rows, settings=settings)
@deferred_f_from_coro_f @coroutine_test
async def test_export_encoding(self): async def test_export_encoding(self):
items = [{"foo": "Test\xd6"}] items = [{"foo": "Test\xd6"}]
@ -1485,7 +1485,7 @@ class TestFeedExport(TestFeedExportBase):
data = await self.exported_data(items, settings) data = await self.exported_data(items, settings)
assert data[fmt] == expected assert data[fmt] == expected
@deferred_f_from_coro_f @coroutine_test
async def test_export_multiple_configs(self): async def test_export_multiple_configs(self):
items = [{"foo": "FOO", "bar": "BAR"}] items = [{"foo": "FOO", "bar": "BAR"}]
@ -1525,7 +1525,7 @@ class TestFeedExport(TestFeedExportBase):
for fmt, expected in formats.items(): for fmt, expected in formats.items():
assert data[fmt] == expected assert data[fmt] == expected
@deferred_f_from_coro_f @coroutine_test
async def test_export_indentation(self): async def test_export_indentation(self):
items = [ items = [
{"foo": ["bar"]}, {"foo": ["bar"]},
@ -1681,7 +1681,7 @@ class TestFeedExport(TestFeedExportBase):
data = await self.exported_data(items, settings) data = await self.exported_data(items, settings)
assert data[row["format"]] == row["expected"] assert data[row["format"]] == row["expected"]
@deferred_f_from_coro_f @coroutine_test
async def test_init_exporters_storages_with_crawler(self): async def test_init_exporters_storages_with_crawler(self):
settings = { settings = {
"FEED_EXPORTERS": {"csv": FromCrawlerCsvItemExporter}, "FEED_EXPORTERS": {"csv": FromCrawlerCsvItemExporter},
@ -1694,7 +1694,7 @@ class TestFeedExport(TestFeedExportBase):
assert FromCrawlerCsvItemExporter.init_with_crawler assert FromCrawlerCsvItemExporter.init_with_crawler
assert FromCrawlerFileFeedStorage.init_with_crawler assert FromCrawlerFileFeedStorage.init_with_crawler
@deferred_f_from_coro_f @coroutine_test
async def test_str_uri(self): async def test_str_uri(self):
settings = { settings = {
"FEED_STORE_EMPTY": True, "FEED_STORE_EMPTY": True,
@ -1703,7 +1703,7 @@ class TestFeedExport(TestFeedExportBase):
data = await self.exported_no_data(settings) data = await self.exported_no_data(settings)
assert data["csv"] == b"" assert data["csv"] == b""
@deferred_f_from_coro_f @coroutine_test
async def test_multiple_feeds_success_logs_blocking_feed_storage(self): async def test_multiple_feeds_success_logs_blocking_feed_storage(self):
settings = { settings = {
"FEEDS": { "FEEDS": {
@ -1723,7 +1723,7 @@ class TestFeedExport(TestFeedExportBase):
for fmt in ["json", "xml", "csv"]: for fmt in ["json", "xml", "csv"]:
assert f"Stored {fmt} feed (2 items)" in str(log) assert f"Stored {fmt} feed (2 items)" in str(log)
@deferred_f_from_coro_f @coroutine_test
async def test_multiple_feeds_failing_logs_blocking_feed_storage(self): async def test_multiple_feeds_failing_logs_blocking_feed_storage(self):
settings = { settings = {
"FEEDS": { "FEEDS": {
@ -1743,7 +1743,7 @@ class TestFeedExport(TestFeedExportBase):
for fmt in ["json", "xml", "csv"]: for fmt in ["json", "xml", "csv"]:
assert f"Error storing {fmt} feed (2 items)" in str(log) assert f"Error storing {fmt} feed (2 items)" in str(log)
@deferred_f_from_coro_f @coroutine_test
async def test_extend_kwargs(self): async def test_extend_kwargs(self):
items = [{"foo": "FOO", "bar": "BAR"}] items = [{"foo": "FOO", "bar": "BAR"}]
@ -1780,7 +1780,7 @@ class TestFeedExport(TestFeedExportBase):
data = await self.exported_data(items, settings) data = await self.exported_data(items, settings)
assert data[feed_options["format"]] == row["expected"] assert data[feed_options["format"]] == row["expected"]
@deferred_f_from_coro_f @coroutine_test
async def test_storage_file_no_postprocessing(self): async def test_storage_file_no_postprocessing(self):
@implementer(IFeedStorage) @implementer(IFeedStorage)
class Storage: class Storage:
@ -1802,7 +1802,7 @@ class TestFeedExport(TestFeedExportBase):
await self.exported_no_data(settings) await self.exported_no_data(settings)
assert Storage.open_file is Storage.store_file assert Storage.open_file is Storage.store_file
@deferred_f_from_coro_f @coroutine_test
async def test_storage_file_postprocessing(self): async def test_storage_file_postprocessing(self):
@implementer(IFeedStorage) @implementer(IFeedStorage)
class Storage: class Storage:
@ -1901,7 +1901,7 @@ class TestFeedPostProcessedExports(TestFeedExportBase):
data_stream.seek(0) data_stream.seek(0)
return data_stream.read() return data_stream.read()
@deferred_f_from_coro_f @coroutine_test
async def test_gzip_plugin(self): async def test_gzip_plugin(self):
filename = self._named_tempfile("gzip_file") filename = self._named_tempfile("gzip_file")
@ -1920,7 +1920,7 @@ class TestFeedPostProcessedExports(TestFeedExportBase):
except OSError: except OSError:
pytest.fail("Received invalid gzip data.") pytest.fail("Received invalid gzip data.")
@deferred_f_from_coro_f @coroutine_test
async def test_gzip_plugin_compresslevel(self): async def test_gzip_plugin_compresslevel(self):
filename_to_compressed = { filename_to_compressed = {
self._named_tempfile("compresslevel_0"): self.get_gzip_compressed( self._named_tempfile("compresslevel_0"): self.get_gzip_compressed(
@ -1957,7 +1957,7 @@ class TestFeedPostProcessedExports(TestFeedExportBase):
assert compressed == data[filename] assert compressed == data[filename]
assert result == self.expected assert result == self.expected
@deferred_f_from_coro_f @coroutine_test
async def test_gzip_plugin_mtime(self): async def test_gzip_plugin_mtime(self):
filename_to_compressed = { filename_to_compressed = {
self._named_tempfile("mtime_123"): self.get_gzip_compressed( self._named_tempfile("mtime_123"): self.get_gzip_compressed(
@ -1992,7 +1992,7 @@ class TestFeedPostProcessedExports(TestFeedExportBase):
assert compressed == data[filename] assert compressed == data[filename]
assert result == self.expected assert result == self.expected
@deferred_f_from_coro_f @coroutine_test
async def test_gzip_plugin_filename(self): async def test_gzip_plugin_filename(self):
filename_to_compressed = { filename_to_compressed = {
self._named_tempfile("filename_FILE1"): self.get_gzip_compressed( self._named_tempfile("filename_FILE1"): self.get_gzip_compressed(
@ -2027,7 +2027,7 @@ class TestFeedPostProcessedExports(TestFeedExportBase):
assert compressed == data[filename] assert compressed == data[filename]
assert result == self.expected assert result == self.expected
@deferred_f_from_coro_f @coroutine_test
async def test_lzma_plugin(self): async def test_lzma_plugin(self):
filename = self._named_tempfile("lzma_file") filename = self._named_tempfile("lzma_file")
@ -2046,7 +2046,7 @@ class TestFeedPostProcessedExports(TestFeedExportBase):
except lzma.LZMAError: except lzma.LZMAError:
pytest.fail("Received invalid lzma data.") pytest.fail("Received invalid lzma data.")
@deferred_f_from_coro_f @coroutine_test
async def test_lzma_plugin_format(self): async def test_lzma_plugin_format(self):
filename_to_compressed = { filename_to_compressed = {
self._named_tempfile("format_FORMAT_XZ"): lzma.compress( self._named_tempfile("format_FORMAT_XZ"): lzma.compress(
@ -2079,7 +2079,7 @@ class TestFeedPostProcessedExports(TestFeedExportBase):
assert compressed == data[filename] assert compressed == data[filename]
assert result == self.expected assert result == self.expected
@deferred_f_from_coro_f @coroutine_test
async def test_lzma_plugin_check(self): async def test_lzma_plugin_check(self):
filename_to_compressed = { filename_to_compressed = {
self._named_tempfile("check_CHECK_NONE"): lzma.compress( self._named_tempfile("check_CHECK_NONE"): lzma.compress(
@ -2112,7 +2112,7 @@ class TestFeedPostProcessedExports(TestFeedExportBase):
assert compressed == data[filename] assert compressed == data[filename]
assert result == self.expected assert result == self.expected
@deferred_f_from_coro_f @coroutine_test
async def test_lzma_plugin_preset(self): async def test_lzma_plugin_preset(self):
filename_to_compressed = { filename_to_compressed = {
self._named_tempfile("preset_PRESET_0"): lzma.compress( self._named_tempfile("preset_PRESET_0"): lzma.compress(
@ -2145,7 +2145,7 @@ class TestFeedPostProcessedExports(TestFeedExportBase):
assert compressed == data[filename] assert compressed == data[filename]
assert result == self.expected assert result == self.expected
@deferred_f_from_coro_f @coroutine_test
async def test_lzma_plugin_filters(self): async def test_lzma_plugin_filters(self):
if "PyPy" in sys.version: if "PyPy" in sys.version:
# https://foss.heptapod.net/pypy/pypy/-/issues/3527 # https://foss.heptapod.net/pypy/pypy/-/issues/3527
@ -2170,7 +2170,7 @@ class TestFeedPostProcessedExports(TestFeedExportBase):
result = lzma.decompress(data[filename]) result = lzma.decompress(data[filename])
assert result == self.expected assert result == self.expected
@deferred_f_from_coro_f @coroutine_test
async def test_bz2_plugin(self): async def test_bz2_plugin(self):
filename = self._named_tempfile("bz2_file") filename = self._named_tempfile("bz2_file")
@ -2189,7 +2189,7 @@ class TestFeedPostProcessedExports(TestFeedExportBase):
except OSError: except OSError:
pytest.fail("Received invalid bz2 data.") pytest.fail("Received invalid bz2 data.")
@deferred_f_from_coro_f @coroutine_test
async def test_bz2_plugin_compresslevel(self): async def test_bz2_plugin_compresslevel(self):
filename_to_compressed = { filename_to_compressed = {
self._named_tempfile("compresslevel_1"): bz2.compress( self._named_tempfile("compresslevel_1"): bz2.compress(
@ -2222,7 +2222,7 @@ class TestFeedPostProcessedExports(TestFeedExportBase):
assert compressed == data[filename] assert compressed == data[filename]
assert result == self.expected assert result == self.expected
@deferred_f_from_coro_f @coroutine_test
async def test_custom_plugin(self): async def test_custom_plugin(self):
filename = self._named_tempfile("csv_file") filename = self._named_tempfile("csv_file")
@ -2238,7 +2238,7 @@ class TestFeedPostProcessedExports(TestFeedExportBase):
data = await self.exported_data(self.items, settings) data = await self.exported_data(self.items, settings)
assert data[filename] == self.expected assert data[filename] == self.expected
@deferred_f_from_coro_f @coroutine_test
async def test_custom_plugin_with_parameter(self): async def test_custom_plugin_with_parameter(self):
expected = b"foo\r\n\nbar\r\n\n" expected = b"foo\r\n\nbar\r\n\n"
filename = self._named_tempfile("newline") filename = self._named_tempfile("newline")
@ -2256,7 +2256,7 @@ class TestFeedPostProcessedExports(TestFeedExportBase):
data = await self.exported_data(self.items, settings) data = await self.exported_data(self.items, settings)
assert data[filename] == expected assert data[filename] == expected
@deferred_f_from_coro_f @coroutine_test
async def test_custom_plugin_with_compression(self): async def test_custom_plugin_with_compression(self):
expected = b"foo\r\n\nbar\r\n\n" expected = b"foo\r\n\nbar\r\n\n"
@ -2301,7 +2301,7 @@ class TestFeedPostProcessedExports(TestFeedExportBase):
result = decompressor(data[filename]) result = decompressor(data[filename])
assert result == expected assert result == expected
@deferred_f_from_coro_f @coroutine_test
async def test_exports_compatibility_with_postproc(self): async def test_exports_compatibility_with_postproc(self):
filename_to_expected = { filename_to_expected = {
self._named_tempfile("csv"): b"foo\r\nbar\r\n", self._named_tempfile("csv"): b"foo\r\nbar\r\n",
@ -2511,7 +2511,7 @@ class TestBatchDeliveries(TestFeedExportBase):
expected_batch, rows = rows[:batch_size], rows[batch_size:] expected_batch, rows = rows[:batch_size], rows[batch_size:]
assert got_batch == expected_batch assert got_batch == expected_batch
@deferred_f_from_coro_f @coroutine_test
async def test_export_items(self): async def test_export_items(self):
"""Test partial deliveries in all supported formats""" """Test partial deliveries in all supported formats"""
items = [ items = [
@ -2540,7 +2540,7 @@ class TestBatchDeliveries(TestFeedExportBase):
with pytest.raises(NotConfigured): with pytest.raises(NotConfigured):
FeedExporter(crawler) FeedExporter(crawler)
@deferred_f_from_coro_f @coroutine_test
async def test_export_no_items_not_store_empty(self): async def test_export_no_items_not_store_empty(self):
for fmt in ("json", "jsonlines", "xml", "csv"): for fmt in ("json", "jsonlines", "xml", "csv"):
settings = { settings = {
@ -2556,7 +2556,7 @@ class TestBatchDeliveries(TestFeedExportBase):
data = dict(data) data = dict(data)
assert len(data[fmt]) == 0 assert len(data[fmt]) == 0
@deferred_f_from_coro_f @coroutine_test
async def test_export_no_items_store_empty(self): async def test_export_no_items_store_empty(self):
formats = ( formats = (
("json", b"[]"), ("json", b"[]"),
@ -2580,7 +2580,7 @@ class TestBatchDeliveries(TestFeedExportBase):
data = dict(data) data = dict(data)
assert data[fmt][0] == expctd assert data[fmt][0] == expctd
@deferred_f_from_coro_f @coroutine_test
async def test_export_multiple_configs(self): async def test_export_multiple_configs(self):
items = [ items = [
{"foo": "FOO", "bar": "BAR"}, {"foo": "FOO", "bar": "BAR"},
@ -2636,7 +2636,7 @@ class TestBatchDeliveries(TestFeedExportBase):
for expected_batch, got_batch in zip(expected, data[fmt], strict=False): for expected_batch, got_batch in zip(expected, data[fmt], strict=False):
assert got_batch == expected_batch assert got_batch == expected_batch
@deferred_f_from_coro_f @coroutine_test
async def test_batch_item_count_feeds_setting(self): async def test_batch_item_count_feeds_setting(self):
items = [{"foo": "FOO"}, {"foo": "FOO1"}] items = [{"foo": "FOO"}, {"foo": "FOO1"}]
formats = { formats = {
@ -2660,7 +2660,7 @@ class TestBatchDeliveries(TestFeedExportBase):
for expected_batch, got_batch in zip(expected, data[fmt], strict=False): for expected_batch, got_batch in zip(expected, data[fmt], strict=False):
assert got_batch == expected_batch assert got_batch == expected_batch
@deferred_f_from_coro_f @coroutine_test
async def test_batch_path_differ(self): async def test_batch_path_differ(self):
""" """
Test that the name of all batch files differ from each other. Test that the name of all batch files differ from each other.
@ -2682,7 +2682,7 @@ class TestBatchDeliveries(TestFeedExportBase):
data = await self.exported_data(items, settings) data = await self.exported_data(items, settings)
assert len(items) == len(data["json"]) assert len(items) == len(data["json"])
@inlineCallbacks @inline_callbacks_test
def test_stats_batch_file_success(self): def test_stats_batch_file_success(self):
settings = { settings = {
"FEEDS": { "FEEDS": {
@ -2700,7 +2700,7 @@ class TestBatchDeliveries(TestFeedExportBase):
assert crawler.stats.get_value("feedexport/success_count/FileFeedStorage") == 12 assert crawler.stats.get_value("feedexport/success_count/FileFeedStorage") == 12
@pytest.mark.requires_boto3 @pytest.mark.requires_boto3
@inlineCallbacks @inline_callbacks_test
def test_s3_export(self): def test_s3_export(self):
bucket = "mybucket" bucket = "mybucket"
items = [ items = [
@ -2823,7 +2823,7 @@ class TestFeedExporterSignals:
feed_exporter.item_scraped(item, spider) feed_exporter.item_scraped(item, spider)
await feed_exporter.close_spider(spider) await feed_exporter.close_spider(spider)
@deferred_f_from_coro_f @coroutine_test
async def test_feed_exporter_signals_sent(self) -> None: async def test_feed_exporter_signals_sent(self) -> None:
self.feed_exporter_closed_received = False self.feed_exporter_closed_received = False
self.feed_slot_closed_received = False self.feed_slot_closed_received = False
@ -2835,7 +2835,7 @@ class TestFeedExporterSignals:
assert self.feed_slot_closed_received assert self.feed_slot_closed_received
assert self.feed_exporter_closed_received assert self.feed_exporter_closed_received
@deferred_f_from_coro_f @coroutine_test
async def test_feed_exporter_signals_sent_async(self) -> None: async def test_feed_exporter_signals_sent_async(self) -> None:
self.feed_exporter_closed_received = False self.feed_exporter_closed_received = False
self.feed_slot_closed_received = False self.feed_slot_closed_received = False

6
tests/test_logformatter.py
View File

@ -12,7 +12,7 @@ from scrapy.spiders import Spider
from scrapy.utils.test import get_crawler from scrapy.utils.test import get_crawler
from tests.mockserver.http import MockServer from tests.mockserver.http import MockServer
from tests.spiders import ItemSpider from tests.spiders import ItemSpider
from tests.utils.decorators import inlineCallbacks from tests.utils.decorators import inline_callbacks_test
class CustomItem(Item): class CustomItem(Item):
@ -272,7 +272,7 @@ class TestShowOrSkipMessages:
}, },
} }
@inlineCallbacks @inline_callbacks_test
def test_show_messages(self): def test_show_messages(self):
crawler = get_crawler(ItemSpider, self.base_settings) crawler = get_crawler(ItemSpider, self.base_settings)
with LogCapture() as lc: with LogCapture() as lc:
@ -281,7 +281,7 @@ class TestShowOrSkipMessages:
assert "Crawled (200) <GET http://127.0.0.1:" in str(lc) assert "Crawled (200) <GET http://127.0.0.1:" in str(lc)
assert "Dropped: Ignoring item" in str(lc) assert "Dropped: Ignoring item" in str(lc)
@inlineCallbacks @inline_callbacks_test
def test_skip_messages(self): def test_skip_messages(self):
settings = self.base_settings.copy() settings = self.base_settings.copy()
settings["LOG_FORMATTER"] = SkipMessagesLogFormatter settings["LOG_FORMATTER"] = SkipMessagesLogFormatter

12
tests/test_pipeline_crawl.py
View File

@ -14,7 +14,7 @@ from scrapy.utils.misc import load_object
from scrapy.utils.test import get_crawler from scrapy.utils.test import get_crawler
from tests.mockserver.http import MockServer from tests.mockserver.http import MockServer
from tests.spiders import SimpleSpider from tests.spiders import SimpleSpider
from tests.utils.decorators import inlineCallbacks from tests.utils.decorators import inline_callbacks_test
if TYPE_CHECKING: if TYPE_CHECKING:
from scrapy.crawler import Crawler from scrapy.crawler import Crawler
@ -145,7 +145,7 @@ class TestFileDownloadCrawl:
# check that no files were written to the media store # check that no files were written to the media store
assert not list(self.tmpmediastore.iterdir()) assert not list(self.tmpmediastore.iterdir())
@inlineCallbacks @inline_callbacks_test
def test_download_media(self): def test_download_media(self):
crawler = self._create_crawler(MediaDownloadSpider) crawler = self._create_crawler(MediaDownloadSpider)
with LogCapture() as log: with LogCapture() as log:
@ -156,7 +156,7 @@ class TestFileDownloadCrawl:
) )
self._assert_files_downloaded(self.items, str(log)) self._assert_files_downloaded(self.items, str(log))
@inlineCallbacks @inline_callbacks_test
def test_download_media_wrong_urls(self): def test_download_media_wrong_urls(self):
crawler = self._create_crawler(BrokenLinksMediaDownloadSpider) crawler = self._create_crawler(BrokenLinksMediaDownloadSpider)
with LogCapture() as log: with LogCapture() as log:
@ -167,7 +167,7 @@ class TestFileDownloadCrawl:
) )
self._assert_files_download_failure(crawler, self.items, 404, str(log)) self._assert_files_download_failure(crawler, self.items, 404, str(log))
@inlineCallbacks @inline_callbacks_test
def test_download_media_redirected_default_failure(self): def test_download_media_redirected_default_failure(self):
crawler = self._create_crawler(RedirectedMediaDownloadSpider) crawler = self._create_crawler(RedirectedMediaDownloadSpider)
with LogCapture() as log: with LogCapture() as log:
@ -179,7 +179,7 @@ class TestFileDownloadCrawl:
) )
self._assert_files_download_failure(crawler, self.items, 302, str(log)) self._assert_files_download_failure(crawler, self.items, 302, str(log))
@inlineCallbacks @inline_callbacks_test
def test_download_media_redirected_allowed(self): def test_download_media_redirected_allowed(self):
settings = { settings = {
**self.settings, **self.settings,
@ -196,7 +196,7 @@ class TestFileDownloadCrawl:
self._assert_files_downloaded(self.items, str(log)) self._assert_files_downloaded(self.items, str(log))
assert crawler.stats.get_value("downloader/response_status_count/302") == 3 assert crawler.stats.get_value("downloader/response_status_count/302") == 3
@inlineCallbacks @inline_callbacks_test
def test_download_media_file_path_error(self): def test_download_media_file_path_error(self):
cls = load_object(self.pipeline_class) cls = load_object(self.pipeline_class)

18
tests/test_pipeline_files.py
View File

@ -34,7 +34,7 @@ from scrapy.settings import Settings
from scrapy.utils.spider import DefaultSpider from scrapy.utils.spider import DefaultSpider
from scrapy.utils.test import get_crawler from scrapy.utils.test import get_crawler
from tests.mockserver.ftp import MockFTPServer from tests.mockserver.ftp import MockFTPServer
from tests.utils.decorators import deferred_f_from_coro_f, inlineCallbacks from tests.utils.decorators import coroutine_test, inline_callbacks_test
from .test_pipeline_media import _mocked_download_func from .test_pipeline_media import _mocked_download_func
@ -161,7 +161,7 @@ class TestFilesPipeline:
fullpath = Path(self.tempdir, "some", "image", "key.jpg") fullpath = Path(self.tempdir, "some", "image", "key.jpg")
assert self.pipeline.store._get_filesystem_path(path) == fullpath assert self.pipeline.store._get_filesystem_path(path) == fullpath
@deferred_f_from_coro_f @coroutine_test
async def test_file_not_expired(self): async def test_file_not_expired(self):
item_url = "http://example.com/file.pdf" item_url = "http://example.com/file.pdf"
item = _create_item_with_files(item_url) item = _create_item_with_files(item_url)
@ -188,7 +188,7 @@ class TestFilesPipeline:
for p in patchers: for p in patchers:
p.stop() p.stop()
@deferred_f_from_coro_f @coroutine_test
async def test_file_expired(self): async def test_file_expired(self):
item_url = "http://example.com/file2.pdf" item_url = "http://example.com/file2.pdf"
item = _create_item_with_files(item_url) item = _create_item_with_files(item_url)
@ -219,7 +219,7 @@ class TestFilesPipeline:
for p in patchers: for p in patchers:
p.stop() p.stop()
@deferred_f_from_coro_f @coroutine_test
async def test_file_cached(self): async def test_file_cached(self):
item_url = "http://example.com/file3.pdf" item_url = "http://example.com/file3.pdf"
item = _create_item_with_files(item_url) item = _create_item_with_files(item_url)
@ -563,7 +563,7 @@ class TestFilesPipelineCustomSettings:
@pytest.mark.requires_botocore @pytest.mark.requires_botocore
class TestS3FilesStore: class TestS3FilesStore:
@inlineCallbacks @inline_callbacks_test
def test_persist(self): def test_persist(self):
bucket = "mybucket" bucket = "mybucket"
key = "export.csv" key = "export.csv"
@ -603,7 +603,7 @@ class TestS3FilesStore:
# The call to read does not happen with Stubber # The call to read does not happen with Stubber
assert buffer.method_calls == [mock.call.seek(0)] assert buffer.method_calls == [mock.call.seek(0)]
@inlineCallbacks @inline_callbacks_test
def test_stat(self): def test_stat(self):
bucket = "mybucket" bucket = "mybucket"
key = "export.csv" key = "export.csv"
@ -640,7 +640,7 @@ class TestS3FilesStore:
"GCS_PROJECT_ID" not in os.environ, reason="GCS_PROJECT_ID not found" "GCS_PROJECT_ID" not in os.environ, reason="GCS_PROJECT_ID not found"
) )
class TestGCSFilesStore: class TestGCSFilesStore:
@inlineCallbacks @inline_callbacks_test
def test_persist(self): def test_persist(self):
uri = os.environ.get("GCS_TEST_FILE_URI") uri = os.environ.get("GCS_TEST_FILE_URI")
if not uri: if not uri:
@ -665,7 +665,7 @@ class TestGCSFilesStore:
assert blob.content_type == "application/octet-stream" assert blob.content_type == "application/octet-stream"
assert expected_policy in acl assert expected_policy in acl
@inlineCallbacks @inline_callbacks_test
def test_blob_path_consistency(self): def test_blob_path_consistency(self):
"""Test to make sure that paths used to store files is the same as the one used to get """Test to make sure that paths used to store files is the same as the one used to get
already uploaded files. already uploaded files.
@ -693,7 +693,7 @@ class TestGCSFilesStore:
@pytest.mark.requires_reactor # needs a reactor for FTPFilesStore @pytest.mark.requires_reactor # needs a reactor for FTPFilesStore
class TestFTPFileStore: class TestFTPFileStore:
@inlineCallbacks @inline_callbacks_test
def test_persist(self): def test_persist(self):
data = b"TestFTPFilesStore: \xe2\x98\x83" data = b"TestFTPFilesStore: \xe2\x98\x83"
buf = BytesIO(data) buf = BytesIO(data)

22
tests/test_pipeline_media.py
View File

@ -18,7 +18,7 @@ from scrapy.utils.log import failure_to_exc_info
from scrapy.utils.signal import disconnect_all from scrapy.utils.signal import disconnect_all
from scrapy.utils.spider import DefaultSpider from scrapy.utils.spider import DefaultSpider
from scrapy.utils.test import get_crawler from scrapy.utils.test import get_crawler
from tests.utils.decorators import deferred_f_from_coro_f from tests.utils.decorators import coroutine_test
async def _mocked_download_func(request): async def _mocked_download_func(request):
@ -163,7 +163,7 @@ class TestBaseMediaPipeline:
assert new_item is item assert new_item is item
assert len(log.records) == 0 assert len(log.records) == 0
@deferred_f_from_coro_f @coroutine_test
async def test_default_process_item(self): async def test_default_process_item(self):
item = {"name": "name"} item = {"name": "name"}
new_item = await self.pipe.process_item(item) new_item = await self.pipe.process_item(item)
@ -211,7 +211,7 @@ class TestMediaPipeline(TestBaseMediaPipeline):
self.pipe._mockcalled.append("request_errback") self.pipe._mockcalled.append("request_errback")
return result return result
@deferred_f_from_coro_f @coroutine_test
async def test_result_succeed(self): async def test_result_succeed(self):
rsp = Response("http://url1") rsp = Response("http://url1")
req = Request( req = Request(
@ -229,7 +229,7 @@ class TestMediaPipeline(TestBaseMediaPipeline):
"item_completed", "item_completed",
] ]
@deferred_f_from_coro_f @coroutine_test
async def test_result_failure(self): async def test_result_failure(self):
self.pipe.LOG_FAILED_RESULTS = False self.pipe.LOG_FAILED_RESULTS = False
exc = Exception("foo") exc = Exception("foo")
@ -252,7 +252,7 @@ class TestMediaPipeline(TestBaseMediaPipeline):
"item_completed", "item_completed",
] ]
@deferred_f_from_coro_f @coroutine_test
async def test_mix_of_success_and_failure(self): async def test_mix_of_success_and_failure(self):
self.pipe.LOG_FAILED_RESULTS = False self.pipe.LOG_FAILED_RESULTS = False
rsp1 = Response("http://url1") rsp1 = Response("http://url1")
@ -278,7 +278,7 @@ class TestMediaPipeline(TestBaseMediaPipeline):
assert m.count("media_downloaded") == 1 assert m.count("media_downloaded") == 1
assert m.count("media_failed") == 1 assert m.count("media_failed") == 1
@deferred_f_from_coro_f @coroutine_test
async def test_get_media_requests(self): async def test_get_media_requests(self):
# returns single Request (without callback) # returns single Request (without callback)
req = Request("http://url") req = Request("http://url")
@ -296,7 +296,7 @@ class TestMediaPipeline(TestBaseMediaPipeline):
assert self.fingerprint(req1) in self.info.downloaded assert self.fingerprint(req1) in self.info.downloaded
assert self.fingerprint(req2) in self.info.downloaded assert self.fingerprint(req2) in self.info.downloaded
@deferred_f_from_coro_f @coroutine_test
async def test_results_are_cached_across_multiple_items(self): async def test_results_are_cached_across_multiple_items(self):
rsp1 = Response("http://url1") rsp1 = Response("http://url1")
req1 = Request("http://url1", meta={"response": rsp1}) req1 = Request("http://url1", meta={"response": rsp1})
@ -315,7 +315,7 @@ class TestMediaPipeline(TestBaseMediaPipeline):
assert self.fingerprint(req1) == self.fingerprint(req2) assert self.fingerprint(req1) == self.fingerprint(req2)
assert new_item["results"] == [(True, {})] assert new_item["results"] == [(True, {})]
@deferred_f_from_coro_f @coroutine_test
async def test_results_are_cached_for_requests_of_single_item(self): async def test_results_are_cached_for_requests_of_single_item(self):
rsp1 = Response("http://url1") rsp1 = Response("http://url1")
req1 = Request("http://url1", meta={"response": rsp1}) req1 = Request("http://url1", meta={"response": rsp1})
@ -327,7 +327,7 @@ class TestMediaPipeline(TestBaseMediaPipeline):
assert new_item is item assert new_item is item
assert new_item["results"] == [(True, {}), (True, {})] assert new_item["results"] == [(True, {}), (True, {})]
@deferred_f_from_coro_f @coroutine_test
async def test_wait_if_request_is_downloading(self): async def test_wait_if_request_is_downloading(self):
def _check_downloading(response): def _check_downloading(response):
fp = self.fingerprint(req1) fp = self.fingerprint(req1)
@ -352,7 +352,7 @@ class TestMediaPipeline(TestBaseMediaPipeline):
new_item = await self.pipe.process_item(item) new_item = await self.pipe.process_item(item)
assert new_item["results"] == [(True, {}), (True, {})] assert new_item["results"] == [(True, {}), (True, {})]
@deferred_f_from_coro_f @coroutine_test
async def test_use_media_to_download_result(self): async def test_use_media_to_download_result(self):
req = Request("http://url", meta={"result": "ITSME"}) req = Request("http://url", meta={"result": "ITSME"})
item = {"requests": req} item = {"requests": req}
@ -480,7 +480,7 @@ class TestMediaFailedFailure(TestBaseMediaPipeline):
self.pipe._mockcalled.append("request_errback") self.pipe._mockcalled.append("request_errback")
return result return result
@deferred_f_from_coro_f @coroutine_test
async def test_result_failure(self): async def test_result_failure(self):
self.pipe.LOG_FAILED_RESULTS = False self.pipe.LOG_FAILED_RESULTS = False
exc = Exception("foo") exc = Exception("foo")

26
tests/test_pipelines.py
View File

@ -14,7 +14,7 @@ from scrapy.utils.defer import deferred_to_future, maybe_deferred_to_future
from scrapy.utils.spider import DefaultSpider from scrapy.utils.spider import DefaultSpider
from scrapy.utils.test import get_crawler, get_from_asyncio_queue from scrapy.utils.test import get_crawler, get_from_asyncio_queue
from tests.mockserver.http import MockServer from tests.mockserver.http import MockServer
from tests.utils.decorators import deferred_f_from_coro_f from tests.utils.decorators import coroutine_test
class SimplePipeline: class SimplePipeline:
@ -154,13 +154,13 @@ class TestPipeline:
), ),
], ],
) )
@deferred_f_from_coro_f @coroutine_test
async def test_pipeline(self, mockserver: MockServer, pipeline_class: type) -> None: async def test_pipeline(self, mockserver: MockServer, pipeline_class: type) -> None:
crawler = self._create_crawler(pipeline_class) crawler = self._create_crawler(pipeline_class)
await crawler.crawl_async(mockserver=mockserver) await crawler.crawl_async(mockserver=mockserver)
assert len(self.items) == 1 assert len(self.items) == 1
@deferred_f_from_coro_f @coroutine_test
async def test_pipeline_deferred(self, mockserver: MockServer) -> None: async def test_pipeline_deferred(self, mockserver: MockServer) -> None:
crawler = self._create_crawler(DeferredPipeline) crawler = self._create_crawler(DeferredPipeline)
with ( with (
@ -180,7 +180,7 @@ class TestPipeline:
await crawler.crawl_async(mockserver=mockserver) await crawler.crawl_async(mockserver=mockserver)
assert len(self.items) == 1 assert len(self.items) == 1
@deferred_f_from_coro_f @coroutine_test
async def test_deprecated_spider_arg(self, mockserver: MockServer) -> None: async def test_deprecated_spider_arg(self, mockserver: MockServer) -> None:
crawler = self._create_crawler(DeprecatedSpiderArgPipeline) crawler = self._create_crawler(DeprecatedSpiderArgPipeline)
with ( with (
@ -214,7 +214,7 @@ class TestPipeline:
ProcessItemExceptionAsyncPipeline, ProcessItemExceptionAsyncPipeline,
], ],
) )
@deferred_f_from_coro_f @coroutine_test
async def test_process_item_exception( async def test_process_item_exception(
self, self,
caplog: pytest.LogCaptureFixture, caplog: pytest.LogCaptureFixture,
@ -239,7 +239,7 @@ class TestPipeline:
OpenSpiderExceptionAsyncPipeline, OpenSpiderExceptionAsyncPipeline,
], ],
) )
@deferred_f_from_coro_f @coroutine_test
async def test_open_spider_exception( async def test_open_spider_exception(
self, mockserver: MockServer, pipeline_class: type self, mockserver: MockServer, pipeline_class: type
) -> None: ) -> None:
@ -267,7 +267,7 @@ class TestCustomPipelineManager:
itemproc.process_item({}, crawler.spider) itemproc.process_item({}, crawler.spider)
@pytest.mark.requires_http_handler @pytest.mark.requires_http_handler
@deferred_f_from_coro_f @coroutine_test
async def test_integration_recommended(self, mockserver: MockServer) -> None: async def test_integration_recommended(self, mockserver: MockServer) -> None:
class CustomPipelineManager(ItemPipelineManager): class CustomPipelineManager(ItemPipelineManager):
async def process_item_async(self, item): async def process_item_async(self, item):
@ -294,7 +294,7 @@ class TestCustomPipelineManager:
assert len(items) == 1 assert len(items) == 1
@pytest.mark.requires_http_handler @pytest.mark.requires_http_handler
@deferred_f_from_coro_f @coroutine_test
async def test_integration_no_async_subclass(self, mockserver: MockServer) -> None: async def test_integration_no_async_subclass(self, mockserver: MockServer) -> None:
class CustomPipelineManager(ItemPipelineManager): class CustomPipelineManager(ItemPipelineManager):
def open_spider(self, spider): def open_spider(self, spider):
@ -353,7 +353,7 @@ class TestCustomPipelineManager:
assert len(items) == 1 assert len(items) == 1
@pytest.mark.requires_http_handler @pytest.mark.requires_http_handler
@deferred_f_from_coro_f @coroutine_test
async def test_integration_no_async_not_subclass( async def test_integration_no_async_not_subclass(
self, mockserver: MockServer self, mockserver: MockServer
) -> None: ) -> None:
@ -425,7 +425,7 @@ class TestMiddlewareManagerSpider:
def crawler(self) -> Crawler: def crawler(self) -> Crawler:
return get_crawler(Spider) return get_crawler(Spider)
@deferred_f_from_coro_f @coroutine_test
async def test_deprecated_spider_arg_no_crawler_spider( async def test_deprecated_spider_arg_no_crawler_spider(
self, crawler: Crawler self, crawler: Crawler
) -> None: ) -> None:
@ -480,7 +480,7 @@ class TestMiddlewareManagerSpider:
): ):
await mwman.close_spider_async() await mwman.close_spider_async()
@deferred_f_from_coro_f @coroutine_test
async def test_deprecated_spider_arg_with_crawler(self, crawler: Crawler) -> None: async def test_deprecated_spider_arg_with_crawler(self, crawler: Crawler) -> None:
"""Crawler is provided and has a spider, works. The instance passed to a deprecated method """Crawler is provided and has a spider, works. The instance passed to a deprecated method
is ignored, even if mismatched.""" is ignored, even if mismatched."""
@ -497,7 +497,7 @@ class TestMiddlewareManagerSpider:
): ):
await maybe_deferred_to_future(mwman.close_spider(DefaultSpider())) await maybe_deferred_to_future(mwman.close_spider(DefaultSpider()))
@deferred_f_from_coro_f @coroutine_test
async def test_deprecated_spider_arg_without_crawler(self) -> None: async def test_deprecated_spider_arg_without_crawler(self) -> None:
"""The first instance passed to a deprecated method is used. Mismatched ones raise an error.""" """The first instance passed to a deprecated method is used. Mismatched ones raise an error."""
with pytest.warns( with pytest.warns(
@ -527,7 +527,7 @@ class TestMiddlewareManagerSpider:
): ):
await maybe_deferred_to_future(mwman.close_spider(spider)) await maybe_deferred_to_future(mwman.close_spider(spider))
@deferred_f_from_coro_f @coroutine_test
async def test_no_spider_arg_without_crawler(self) -> None: async def test_no_spider_arg_without_crawler(self) -> None:
"""If no crawler and no spider arg, raise an error.""" """If no crawler and no spider arg, raise an error."""
with pytest.warns( with pytest.warns(

8
tests/test_proxy_connect.py
View File

@ -13,7 +13,7 @@ from scrapy.http import Request
from scrapy.utils.test import get_crawler from scrapy.utils.test import get_crawler
from tests.mockserver.http import MockServer from tests.mockserver.http import MockServer
from tests.spiders import SimpleSpider, SingleRequestSpider from tests.spiders import SimpleSpider, SingleRequestSpider
from tests.utils.decorators import inlineCallbacks from tests.utils.decorators import inline_callbacks_test
class MitmProxy: class MitmProxy:
@ -84,14 +84,14 @@ class TestProxyConnect:
self._proxy.stop() self._proxy.stop()
os.environ = self._oldenv os.environ = self._oldenv
@inlineCallbacks @inline_callbacks_test
def test_https_connect_tunnel(self): def test_https_connect_tunnel(self):
crawler = get_crawler(SimpleSpider) crawler = get_crawler(SimpleSpider)
with LogCapture() as log: with LogCapture() as log:
yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True)) yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True))
self._assert_got_response_code(200, log) self._assert_got_response_code(200, log)
@inlineCallbacks @inline_callbacks_test
def test_https_tunnel_auth_error(self): def test_https_tunnel_auth_error(self):
os.environ["https_proxy"] = _wrong_credentials(os.environ["https_proxy"]) os.environ["https_proxy"] = _wrong_credentials(os.environ["https_proxy"])
crawler = get_crawler(SimpleSpider) crawler = get_crawler(SimpleSpider)
@ -101,7 +101,7 @@ class TestProxyConnect:
# he just sees a TunnelError. # he just sees a TunnelError.
self._assert_got_tunnel_error(log) self._assert_got_tunnel_error(log)
@inlineCallbacks @inline_callbacks_test
def test_https_tunnel_without_leak_proxy_authorization_header(self): def test_https_tunnel_without_leak_proxy_authorization_header(self):
request = Request(self.mockserver.url("/echo", is_secure=True)) request = Request(self.mockserver.url("/echo", is_secure=True))
crawler = get_crawler(SingleRequestSpider) crawler = get_crawler(SingleRequestSpider)

16
tests/test_request_attribute_binding.py
View File

@ -6,7 +6,7 @@ from scrapy.http.response import Response
from scrapy.utils.test import get_crawler from scrapy.utils.test import get_crawler
from tests.mockserver.http import MockServer from tests.mockserver.http import MockServer
from tests.spiders import SingleRequestSpider from tests.spiders import SingleRequestSpider
from tests.utils.decorators import inlineCallbacks from tests.utils.decorators import inline_callbacks_test
OVERRIDDEN_URL = "https://example.org" OVERRIDDEN_URL = "https://example.org"
@ -74,7 +74,7 @@ class TestCrawl:
def teardown_class(cls): def teardown_class(cls):
cls.mockserver.__exit__(None, None, None) cls.mockserver.__exit__(None, None, None)
@inlineCallbacks @inline_callbacks_test
def test_response_200(self): def test_response_200(self):
url = self.mockserver.url("/status?n=200") url = self.mockserver.url("/status?n=200")
crawler = get_crawler(SingleRequestSpider) crawler = get_crawler(SingleRequestSpider)
@ -82,7 +82,7 @@ class TestCrawl:
response = crawler.spider.meta["responses"][0] response = crawler.spider.meta["responses"][0]
assert response.request.url == url assert response.request.url == url
@inlineCallbacks @inline_callbacks_test
def test_response_error(self): def test_response_error(self):
for status in ("404", "500"): for status in ("404", "500"):
url = self.mockserver.url(f"/status?n={status}") url = self.mockserver.url(f"/status?n={status}")
@ -93,7 +93,7 @@ class TestCrawl:
assert failure.request.url == url assert failure.request.url == url
assert response.request.url == url assert response.request.url == url
@inlineCallbacks @inline_callbacks_test
def test_downloader_middleware_raise_exception(self): def test_downloader_middleware_raise_exception(self):
url = self.mockserver.url("/status?n=200") url = self.mockserver.url("/status?n=200")
crawler = get_crawler( crawler = get_crawler(
@ -109,7 +109,7 @@ class TestCrawl:
assert failure.request.url == url assert failure.request.url == url
assert isinstance(failure.value, ZeroDivisionError) assert isinstance(failure.value, ZeroDivisionError)
@inlineCallbacks @inline_callbacks_test
def test_downloader_middleware_override_request_in_process_response(self): def test_downloader_middleware_override_request_in_process_response(self):
""" """
Downloader middleware which returns a response with an specific 'request' attribute. Downloader middleware which returns a response with an specific 'request' attribute.
@ -152,7 +152,7 @@ class TestCrawl:
), ),
) )
@inlineCallbacks @inline_callbacks_test
def test_downloader_middleware_override_in_process_exception(self): def test_downloader_middleware_override_in_process_exception(self):
""" """
An exception is raised but caught by the next middleware, which An exception is raised but caught by the next middleware, which
@ -175,7 +175,7 @@ class TestCrawl:
assert response.body == b"Caught ZeroDivisionError" assert response.body == b"Caught ZeroDivisionError"
assert response.request.url == OVERRIDDEN_URL assert response.request.url == OVERRIDDEN_URL
@inlineCallbacks @inline_callbacks_test
def test_downloader_middleware_do_not_override_in_process_exception(self): def test_downloader_middleware_do_not_override_in_process_exception(self):
""" """
An exception is raised but caught by the next middleware, which An exception is raised but caught by the next middleware, which
@ -198,7 +198,7 @@ class TestCrawl:
assert response.body == b"Caught ZeroDivisionError" assert response.body == b"Caught ZeroDivisionError"
assert response.request.url == url assert response.request.url == url
@inlineCallbacks @inline_callbacks_test
def test_downloader_middleware_alternative_callback(self): def test_downloader_middleware_alternative_callback(self):
""" """
Downloader middleware which returns a response with a Downloader middleware which returns a response with a

4
tests/test_request_cb_kwargs.py
View File

@ -5,7 +5,7 @@ from scrapy.http import Request
from scrapy.utils.test import get_crawler from scrapy.utils.test import get_crawler
from tests.mockserver.http import MockServer from tests.mockserver.http import MockServer
from tests.spiders import MockServerSpider from tests.spiders import MockServerSpider
from tests.utils.decorators import inlineCallbacks from tests.utils.decorators import inline_callbacks_test
class InjectArgumentsDownloaderMiddleware: class InjectArgumentsDownloaderMiddleware:
@ -160,7 +160,7 @@ class TestCallbackKeywordArguments:
def teardown_class(cls): def teardown_class(cls):
cls.mockserver.__exit__(None, None, None) cls.mockserver.__exit__(None, None, None)
@inlineCallbacks @inline_callbacks_test
def test_callback_kwargs(self): def test_callback_kwargs(self):
crawler = get_crawler(KeywordArgumentsSpider) crawler = get_crawler(KeywordArgumentsSpider)
with LogCapture() as log: with LogCapture() as log:

10
tests/test_request_left.py
View File

@ -2,7 +2,7 @@ from scrapy.signals import request_left_downloader
from scrapy.spiders import Spider from scrapy.spiders import Spider
from scrapy.utils.test import get_crawler from scrapy.utils.test import get_crawler
from tests.mockserver.http import MockServer from tests.mockserver.http import MockServer
from tests.utils.decorators import inlineCallbacks from tests.utils.decorators import inline_callbacks_test
class SignalCatcherSpider(Spider): class SignalCatcherSpider(Spider):
@ -32,25 +32,25 @@ class TestCatching:
def teardown_class(cls): def teardown_class(cls):
cls.mockserver.__exit__(None, None, None) cls.mockserver.__exit__(None, None, None)
@inlineCallbacks @inline_callbacks_test
def test_success(self): def test_success(self):
crawler = get_crawler(SignalCatcherSpider) crawler = get_crawler(SignalCatcherSpider)
yield crawler.crawl(self.mockserver.url("/status?n=200")) yield crawler.crawl(self.mockserver.url("/status?n=200"))
assert crawler.spider.caught_times == 1 assert crawler.spider.caught_times == 1
@inlineCallbacks @inline_callbacks_test
def test_timeout(self): def test_timeout(self):
crawler = get_crawler(SignalCatcherSpider, {"DOWNLOAD_TIMEOUT": 0.1}) crawler = get_crawler(SignalCatcherSpider, {"DOWNLOAD_TIMEOUT": 0.1})
yield crawler.crawl(self.mockserver.url("/delay?n=0.2")) yield crawler.crawl(self.mockserver.url("/delay?n=0.2"))
assert crawler.spider.caught_times == 1 assert crawler.spider.caught_times == 1
@inlineCallbacks @inline_callbacks_test
def test_disconnect(self): def test_disconnect(self):
crawler = get_crawler(SignalCatcherSpider) crawler = get_crawler(SignalCatcherSpider)
yield crawler.crawl(self.mockserver.url("/drop")) yield crawler.crawl(self.mockserver.url("/drop"))
assert crawler.spider.caught_times == 1 assert crawler.spider.caught_times == 1
@inlineCallbacks @inline_callbacks_test
def test_noconnect(self): def test_noconnect(self):
crawler = get_crawler(SignalCatcherSpider) crawler = get_crawler(SignalCatcherSpider)
yield crawler.crawl("http://thereisdefinetelynosuchdomain.com") yield crawler.crawl("http://thereisdefinetelynosuchdomain.com")

4
tests/test_scheduler.py
View File

@ -19,7 +19,7 @@ from scrapy.utils.httpobj import urlparse_cached
from scrapy.utils.misc import load_object from scrapy.utils.misc import load_object
from scrapy.utils.test import get_crawler from scrapy.utils.test import get_crawler
from tests.mockserver.http import MockServer from tests.mockserver.http import MockServer
from tests.utils.decorators import inlineCallbacks from tests.utils.decorators import inline_callbacks_test
class MemoryScheduler(BaseScheduler): class MemoryScheduler(BaseScheduler):
@ -370,7 +370,7 @@ class TestIntegrationWithDownloaderAwareInMemory:
) )
@pytest.mark.requires_http_handler @pytest.mark.requires_http_handler
@inlineCallbacks @inline_callbacks_test
def test_integration_downloader_aware_priority_queue(self): def test_integration_downloader_aware_priority_queue(self):
with MockServer() as mockserver: with MockServer() as mockserver:
url = mockserver.url("/status?n=200", is_secure=False) url = mockserver.url("/status?n=200", is_secure=False)

6
tests/test_scheduler_base.py
View File

@ -13,7 +13,7 @@ from scrapy.utils.httpobj import urlparse_cached
from scrapy.utils.request import fingerprint from scrapy.utils.request import fingerprint
from scrapy.utils.test import get_crawler from scrapy.utils.test import get_crawler
from tests.mockserver.http import MockServer from tests.mockserver.http import MockServer
from tests.utils.decorators import inlineCallbacks from tests.utils.decorators import inline_callbacks_test
PATHS = ["/a", "/b", "/c"] PATHS = ["/a", "/b", "/c"]
URLS = [urljoin("https://example.org", p) for p in PATHS] URLS = [urljoin("https://example.org", p) for p in PATHS]
@ -118,7 +118,7 @@ class TestSimpleScheduler(InterfaceCheckMixin):
def setup_method(self): def setup_method(self):
self.scheduler = SimpleScheduler() self.scheduler = SimpleScheduler()
@inlineCallbacks @inline_callbacks_test
def test_enqueue_dequeue(self): def test_enqueue_dequeue(self):
open_result = yield self.scheduler.open(Spider("foo")) open_result = yield self.scheduler.open(Spider("foo"))
assert open_result == "open" assert open_result == "open"
@ -148,7 +148,7 @@ class TestSimpleScheduler(InterfaceCheckMixin):
class TestMinimalSchedulerCrawl: class TestMinimalSchedulerCrawl:
scheduler_cls = MinimalScheduler scheduler_cls = MinimalScheduler
@inlineCallbacks @inline_callbacks_test
def test_crawl(self): def test_crawl(self):
with MockServer() as mockserver: with MockServer() as mockserver:
settings = { settings = {

6
tests/test_signals.py
View File

@ -3,7 +3,7 @@ import pytest
from scrapy import Request, Spider, signals from scrapy import Request, Spider, signals
from scrapy.utils.test import get_crawler, get_from_asyncio_queue from scrapy.utils.test import get_crawler, get_from_asyncio_queue
from tests.mockserver.http import MockServer from tests.mockserver.http import MockServer
from tests.utils.decorators import deferred_f_from_coro_f, inlineCallbacks from tests.utils.decorators import coroutine_test, inline_callbacks_test
class ItemSpider(Spider): class ItemSpider(Spider):
@ -20,7 +20,7 @@ class ItemSpider(Spider):
class TestMain: class TestMain:
@deferred_f_from_coro_f @coroutine_test
async def test_scheduler_empty(self): async def test_scheduler_empty(self):
crawler = get_crawler() crawler = get_crawler()
calls = [] calls = []
@ -52,7 +52,7 @@ class TestMockServer:
@pytest.mark.requires_http_handler @pytest.mark.requires_http_handler
@pytest.mark.only_asyncio @pytest.mark.only_asyncio
@inlineCallbacks @inline_callbacks_test
def test_simple_pipeline(self): def test_simple_pipeline(self):
crawler = get_crawler(ItemSpider) crawler = get_crawler(ItemSpider)
crawler.signals.connect(self._on_item_scraped, signals.item_scraped) crawler.signals.connect(self._on_item_scraped, signals.item_scraped)

10
tests/test_spider.py
View File

@ -30,7 +30,7 @@ from scrapy.spiders import (
from scrapy.spiders.init import InitSpider from scrapy.spiders.init import InitSpider
from scrapy.utils.test import get_crawler, get_reactor_settings from scrapy.utils.test import get_crawler, get_reactor_settings
from tests import get_testdata, tests_datadir from tests import get_testdata, tests_datadir
from tests.utils.decorators import deferred_f_from_coro_f, inlineCallbacks from tests.utils.decorators import coroutine_test, inline_callbacks_test
class TestSpider: class TestSpider:
@ -95,7 +95,7 @@ class TestSpider:
assert settings.get("TEST2") == "spider" assert settings.get("TEST2") == "spider"
assert settings.get("TEST3") == "project" assert settings.get("TEST3") == "project"
@inlineCallbacks @inline_callbacks_test
def test_settings_in_from_crawler(self): def test_settings_in_from_crawler(self):
spider_settings = {"TEST1": "spider", "TEST2": "spider"} spider_settings = {"TEST1": "spider", "TEST2": "spider"}
project_settings = { project_settings = {
@ -142,7 +142,7 @@ class TestSpider:
class TestInitSpider(TestSpider): class TestInitSpider(TestSpider):
spider_class = InitSpider spider_class = InitSpider
@deferred_f_from_coro_f @coroutine_test
async def test_start_urls(self): async def test_start_urls(self):
responses = [] responses = []
@ -458,7 +458,7 @@ class TestCrawlSpider(TestSpider):
assert hasattr(spider, "_follow_links") assert hasattr(spider, "_follow_links")
assert not spider._follow_links assert not spider._follow_links
@inlineCallbacks @inline_callbacks_test
def test_start_url(self): def test_start_url(self):
class TestSpider(self.spider_class): class TestSpider(self.spider_class):
name = "test" name = "test"
@ -827,7 +827,7 @@ Sitemap: /sitemap-relative-url.xml
), ),
) )
@deferred_f_from_coro_f @coroutine_test
async def test_sitemap_urls(self): async def test_sitemap_urls(self):
class TestSpider(self.spider_class): class TestSpider(self.spider_class):
name = "test" name = "test"

24
tests/test_spider_start.py
View File

@ -13,7 +13,7 @@ from scrapy.utils.defer import maybe_deferred_to_future
from scrapy.utils.test import get_crawler from scrapy.utils.test import get_crawler
from .utils import twisted_sleep from .utils import twisted_sleep
from .utils.decorators import deferred_f_from_coro_f from .utils.decorators import coroutine_test
SLEEP_SECONDS = 0.1 SLEEP_SECONDS = 0.1
@ -38,7 +38,7 @@ class TestMain:
assert crawler.stats.get_value("finish_reason") == "finished" assert crawler.stats.get_value("finish_reason") == "finished"
assert actual_items == expected_items assert actual_items == expected_items
@deferred_f_from_coro_f @coroutine_test
async def test_start_urls(self): async def test_start_urls(self):
class TestSpider(Spider): class TestSpider(Spider):
name = "test" name = "test"
@ -51,7 +51,7 @@ class TestMain:
warnings.simplefilter("error") warnings.simplefilter("error")
await self._test_spider(TestSpider, [ITEM_A]) await self._test_spider(TestSpider, [ITEM_A])
@deferred_f_from_coro_f @coroutine_test
async def test_start(self): async def test_start(self):
class TestSpider(Spider): class TestSpider(Spider):
name = "test" name = "test"
@ -63,7 +63,7 @@ class TestMain:
warnings.simplefilter("error") warnings.simplefilter("error")
await self._test_spider(TestSpider, [ITEM_A]) await self._test_spider(TestSpider, [ITEM_A])
@deferred_f_from_coro_f @coroutine_test
async def test_start_subclass(self): async def test_start_subclass(self):
class BaseSpider(Spider): class BaseSpider(Spider):
async def start(self): async def start(self):
@ -76,7 +76,7 @@ class TestMain:
warnings.simplefilter("error") warnings.simplefilter("error")
await self._test_spider(TestSpider, [ITEM_A]) await self._test_spider(TestSpider, [ITEM_A])
@deferred_f_from_coro_f @coroutine_test
async def test_deprecated(self): async def test_deprecated(self):
class TestSpider(Spider): class TestSpider(Spider):
name = "test" name = "test"
@ -87,7 +87,7 @@ class TestMain:
with pytest.warns(ScrapyDeprecationWarning): with pytest.warns(ScrapyDeprecationWarning):
await self._test_spider(TestSpider, [ITEM_A]) await self._test_spider(TestSpider, [ITEM_A])
@deferred_f_from_coro_f @coroutine_test
async def test_deprecated_subclass(self): async def test_deprecated_subclass(self):
class BaseSpider(Spider): class BaseSpider(Spider):
def start_requests(self): def start_requests(self):
@ -100,7 +100,7 @@ class TestMain:
with pytest.warns(ScrapyDeprecationWarning, match="BaseSpider"): with pytest.warns(ScrapyDeprecationWarning, match="BaseSpider"):
await self._test_spider(TestSpider, [ITEM_A]) await self._test_spider(TestSpider, [ITEM_A])
@deferred_f_from_coro_f @coroutine_test
async def test_universal(self): async def test_universal(self):
class TestSpider(Spider): class TestSpider(Spider):
name = "test" name = "test"
@ -115,7 +115,7 @@ class TestMain:
warnings.simplefilter("error") warnings.simplefilter("error")
await self._test_spider(TestSpider, [ITEM_A]) await self._test_spider(TestSpider, [ITEM_A])
@deferred_f_from_coro_f @coroutine_test
async def test_universal_subclass(self): async def test_universal_subclass(self):
class BaseSpider(Spider): class BaseSpider(Spider):
async def start(self): async def start(self):
@ -131,7 +131,7 @@ class TestMain:
warnings.simplefilter("error") warnings.simplefilter("error")
await self._test_spider(TestSpider, [ITEM_A]) await self._test_spider(TestSpider, [ITEM_A])
@deferred_f_from_coro_f @coroutine_test
async def test_start_deprecated_super(self): async def test_start_deprecated_super(self):
class TestSpider(Spider): class TestSpider(Spider):
name = "test" name = "test"
@ -154,7 +154,7 @@ class TestMain:
await self._test_spider(TestSpider, expected_items) await self._test_spider(TestSpider, expected_items)
@pytest.mark.only_asyncio @pytest.mark.only_asyncio
@deferred_f_from_coro_f @coroutine_test
async def test_asyncio_delayed(self): async def test_asyncio_delayed(self):
async def start(spider): async def start(spider):
await sleep(SLEEP_SECONDS) await sleep(SLEEP_SECONDS)
@ -163,7 +163,7 @@ class TestMain:
await self._test_start(start, [ITEM_A]) await self._test_start(start, [ITEM_A])
@pytest.mark.requires_reactor # needs a reactor for twisted_sleep() @pytest.mark.requires_reactor # needs a reactor for twisted_sleep()
@deferred_f_from_coro_f @coroutine_test
async def test_twisted_delayed(self): async def test_twisted_delayed(self):
async def start(spider): async def start(spider):
await maybe_deferred_to_future(twisted_sleep(SLEEP_SECONDS)) await maybe_deferred_to_future(twisted_sleep(SLEEP_SECONDS))
@ -173,7 +173,7 @@ class TestMain:
# Exceptions # Exceptions
@deferred_f_from_coro_f @coroutine_test
async def test_deprecated_non_generator_exception(self): async def test_deprecated_non_generator_exception(self):
class TestSpider(Spider): class TestSpider(Spider):
name = "test" name = "test"

90
tests/test_spidermiddleware.py
View File

@ -18,7 +18,7 @@ from scrapy.utils.asyncio import call_later
from scrapy.utils.defer import maybe_deferred_to_future from scrapy.utils.defer import maybe_deferred_to_future
from scrapy.utils.spider import DefaultSpider from scrapy.utils.spider import DefaultSpider
from scrapy.utils.test import get_crawler from scrapy.utils.test import get_crawler
from tests.utils.decorators import deferred_f_from_coro_f from tests.utils.decorators import coroutine_test
if TYPE_CHECKING: if TYPE_CHECKING:
from twisted.python.failure import Failure from twisted.python.failure import Failure
@ -53,7 +53,7 @@ class TestSpiderMiddleware:
class TestProcessSpiderInputInvalidOutput(TestSpiderMiddleware): class TestProcessSpiderInputInvalidOutput(TestSpiderMiddleware):
"""Invalid return value for process_spider_input method""" """Invalid return value for process_spider_input method"""
@deferred_f_from_coro_f @coroutine_test
async def test_invalid_process_spider_input(self): async def test_invalid_process_spider_input(self):
class InvalidProcessSpiderInputMiddleware: class InvalidProcessSpiderInputMiddleware:
def process_spider_input(self, response): def process_spider_input(self, response):
@ -67,7 +67,7 @@ class TestProcessSpiderInputInvalidOutput(TestSpiderMiddleware):
class TestProcessSpiderOutputInvalidOutput(TestSpiderMiddleware): class TestProcessSpiderOutputInvalidOutput(TestSpiderMiddleware):
"""Invalid return value for process_spider_output method""" """Invalid return value for process_spider_output method"""
@deferred_f_from_coro_f @coroutine_test
async def test_invalid_process_spider_output(self): async def test_invalid_process_spider_output(self):
class InvalidProcessSpiderOutputMiddleware: class InvalidProcessSpiderOutputMiddleware:
def process_spider_output(self, response, result): def process_spider_output(self, response, result):
@ -81,7 +81,7 @@ class TestProcessSpiderOutputInvalidOutput(TestSpiderMiddleware):
class TestProcessSpiderExceptionInvalidOutput(TestSpiderMiddleware): class TestProcessSpiderExceptionInvalidOutput(TestSpiderMiddleware):
"""Invalid return value for process_spider_exception method""" """Invalid return value for process_spider_exception method"""
@deferred_f_from_coro_f @coroutine_test
async def test_invalid_process_spider_exception(self): async def test_invalid_process_spider_exception(self):
class InvalidProcessSpiderOutputExceptionMiddleware: class InvalidProcessSpiderOutputExceptionMiddleware:
def process_spider_exception(self, response, exception): def process_spider_exception(self, response, exception):
@ -100,7 +100,7 @@ class TestProcessSpiderExceptionInvalidOutput(TestSpiderMiddleware):
class TestProcessSpiderExceptionReRaise(TestSpiderMiddleware): class TestProcessSpiderExceptionReRaise(TestSpiderMiddleware):
"""Re raise the exception by returning None""" """Re raise the exception by returning None"""
@deferred_f_from_coro_f @coroutine_test
async def test_process_spider_exception_return_none(self): async def test_process_spider_exception_return_none(self):
class ProcessSpiderExceptionReturnNoneMiddleware: class ProcessSpiderExceptionReturnNoneMiddleware:
def process_spider_exception(self, response, exception): def process_spider_exception(self, response, exception):
@ -237,47 +237,47 @@ class TestProcessSpiderOutputSimple(TestBaseAsyncSpiderMiddleware):
MW_ASYNCGEN = ProcessSpiderOutputAsyncGenMiddleware MW_ASYNCGEN = ProcessSpiderOutputAsyncGenMiddleware
MW_UNIVERSAL = ProcessSpiderOutputUniversalMiddleware MW_UNIVERSAL = ProcessSpiderOutputUniversalMiddleware
@deferred_f_from_coro_f @coroutine_test
async def test_simple(self): async def test_simple(self):
"""Simple mw""" """Simple mw"""
await self._test_simple_base(self.MW_SIMPLE) await self._test_simple_base(self.MW_SIMPLE)
@deferred_f_from_coro_f @coroutine_test
async def test_asyncgen(self): async def test_asyncgen(self):
"""Asyncgen mw; upgrade""" """Asyncgen mw; upgrade"""
await self._test_asyncgen_base(self.MW_ASYNCGEN) await self._test_asyncgen_base(self.MW_ASYNCGEN)
@deferred_f_from_coro_f @coroutine_test
async def test_simple_asyncgen(self): async def test_simple_asyncgen(self):
"""Simple mw -> asyncgen mw; upgrade""" """Simple mw -> asyncgen mw; upgrade"""
await self._test_asyncgen_base(self.MW_ASYNCGEN, self.MW_SIMPLE) await self._test_asyncgen_base(self.MW_ASYNCGEN, self.MW_SIMPLE)
@deferred_f_from_coro_f @coroutine_test
async def test_asyncgen_simple(self): async def test_asyncgen_simple(self):
"""Asyncgen mw -> simple mw; upgrade then downgrade""" """Asyncgen mw -> simple mw; upgrade then downgrade"""
await self._test_simple_base(self.MW_SIMPLE, self.MW_ASYNCGEN, downgrade=True) await self._test_simple_base(self.MW_SIMPLE, self.MW_ASYNCGEN, downgrade=True)
@deferred_f_from_coro_f @coroutine_test
async def test_universal(self): async def test_universal(self):
"""Universal mw""" """Universal mw"""
await self._test_simple_base(self.MW_UNIVERSAL) await self._test_simple_base(self.MW_UNIVERSAL)
@deferred_f_from_coro_f @coroutine_test
async def test_universal_simple(self): async def test_universal_simple(self):
"""Universal mw -> simple mw""" """Universal mw -> simple mw"""
await self._test_simple_base(self.MW_SIMPLE, self.MW_UNIVERSAL) await self._test_simple_base(self.MW_SIMPLE, self.MW_UNIVERSAL)
@deferred_f_from_coro_f @coroutine_test
async def test_simple_universal(self): async def test_simple_universal(self):
"""Simple mw -> universal mw""" """Simple mw -> universal mw"""
await self._test_simple_base(self.MW_UNIVERSAL, self.MW_SIMPLE) await self._test_simple_base(self.MW_UNIVERSAL, self.MW_SIMPLE)
@deferred_f_from_coro_f @coroutine_test
async def test_universal_asyncgen(self): async def test_universal_asyncgen(self):
"""Universal mw -> asyncgen mw; upgrade""" """Universal mw -> asyncgen mw; upgrade"""
await self._test_asyncgen_base(self.MW_ASYNCGEN, self.MW_UNIVERSAL) await self._test_asyncgen_base(self.MW_ASYNCGEN, self.MW_UNIVERSAL)
@deferred_f_from_coro_f @coroutine_test
async def test_asyncgen_universal(self): async def test_asyncgen_universal(self):
"""Asyncgen mw -> universal mw; upgrade""" """Asyncgen mw -> universal mw; upgrade"""
await self._test_asyncgen_base(self.MW_UNIVERSAL, self.MW_ASYNCGEN) await self._test_asyncgen_base(self.MW_UNIVERSAL, self.MW_ASYNCGEN)
@ -290,27 +290,27 @@ class TestProcessSpiderOutputAsyncGen(TestProcessSpiderOutputSimple):
for item in super()._callback(): for item in super()._callback():
yield item yield item
@deferred_f_from_coro_f @coroutine_test
async def test_simple(self): async def test_simple(self):
"""Simple mw; downgrade""" """Simple mw; downgrade"""
await self._test_simple_base(self.MW_SIMPLE, downgrade=True) await self._test_simple_base(self.MW_SIMPLE, downgrade=True)
@deferred_f_from_coro_f @coroutine_test
async def test_simple_asyncgen(self): async def test_simple_asyncgen(self):
"""Simple mw -> asyncgen mw; downgrade then upgrade""" """Simple mw -> asyncgen mw; downgrade then upgrade"""
await self._test_asyncgen_base(self.MW_ASYNCGEN, self.MW_SIMPLE, downgrade=True) await self._test_asyncgen_base(self.MW_ASYNCGEN, self.MW_SIMPLE, downgrade=True)
@deferred_f_from_coro_f @coroutine_test
async def test_universal(self): async def test_universal(self):
"""Universal mw""" """Universal mw"""
await self._test_asyncgen_base(self.MW_UNIVERSAL) await self._test_asyncgen_base(self.MW_UNIVERSAL)
@deferred_f_from_coro_f @coroutine_test
async def test_universal_simple(self): async def test_universal_simple(self):
"""Universal mw -> simple mw; downgrade""" """Universal mw -> simple mw; downgrade"""
await self._test_simple_base(self.MW_SIMPLE, self.MW_UNIVERSAL, downgrade=True) await self._test_simple_base(self.MW_SIMPLE, self.MW_UNIVERSAL, downgrade=True)
@deferred_f_from_coro_f @coroutine_test
async def test_simple_universal(self): async def test_simple_universal(self):
"""Simple mw -> universal mw; downgrade""" """Simple mw -> universal mw; downgrade"""
await self._test_simple_base(self.MW_UNIVERSAL, self.MW_SIMPLE, downgrade=True) await self._test_simple_base(self.MW_UNIVERSAL, self.MW_SIMPLE, downgrade=True)
@ -327,7 +327,7 @@ class ProcessSpiderOutputCoroutineMiddleware:
class TestProcessSpiderOutputInvalidResult(TestBaseAsyncSpiderMiddleware): class TestProcessSpiderOutputInvalidResult(TestBaseAsyncSpiderMiddleware):
@deferred_f_from_coro_f @coroutine_test
async def test_non_iterable(self): async def test_non_iterable(self):
with pytest.raises( with pytest.raises(
_InvalidOutput, _InvalidOutput,
@ -335,7 +335,7 @@ class TestProcessSpiderOutputInvalidResult(TestBaseAsyncSpiderMiddleware):
): ):
await self._get_middleware_result(ProcessSpiderOutputNonIterableMiddleware) await self._get_middleware_result(ProcessSpiderOutputNonIterableMiddleware)
@deferred_f_from_coro_f @coroutine_test
async def test_coroutine(self): async def test_coroutine(self):
with pytest.raises( with pytest.raises(
_InvalidOutput, _InvalidOutput,
@ -375,7 +375,7 @@ class TestProcessStartSimple(TestBaseAsyncSpiderMiddleware):
self.mwman = SpiderMiddlewareManager.from_crawler(self.crawler) self.mwman = SpiderMiddlewareManager.from_crawler(self.crawler)
return await self.mwman.process_start() return await self.mwman.process_start()
@deferred_f_from_coro_f @coroutine_test
async def test_simple(self): async def test_simple(self):
"""Simple mw""" """Simple mw"""
start = await self._get_processed_start(self.MW_SIMPLE) start = await self._get_processed_start(self.MW_SIMPLE)
@ -489,33 +489,33 @@ class TestBuiltinMiddlewareSimple(TestBaseAsyncSpiderMiddleware):
self._scrape_func, self.response, self.request self._scrape_func, self.response, self.request
) )
@deferred_f_from_coro_f @coroutine_test
async def test_just_builtin(self): async def test_just_builtin(self):
await self._test_simple_base() await self._test_simple_base()
@deferred_f_from_coro_f @coroutine_test
async def test_builtin_simple(self): async def test_builtin_simple(self):
await self._test_simple_base(self.MW_SIMPLE, start_index=1000) await self._test_simple_base(self.MW_SIMPLE, start_index=1000)
@deferred_f_from_coro_f @coroutine_test
async def test_builtin_async(self): async def test_builtin_async(self):
"""Upgrade""" """Upgrade"""
await self._test_asyncgen_base(self.MW_ASYNCGEN, start_index=1000) await self._test_asyncgen_base(self.MW_ASYNCGEN, start_index=1000)
@deferred_f_from_coro_f @coroutine_test
async def test_builtin_universal(self): async def test_builtin_universal(self):
await self._test_simple_base(self.MW_UNIVERSAL, start_index=1000) await self._test_simple_base(self.MW_UNIVERSAL, start_index=1000)
@deferred_f_from_coro_f @coroutine_test
async def test_simple_builtin(self): async def test_simple_builtin(self):
await self._test_simple_base(self.MW_SIMPLE) await self._test_simple_base(self.MW_SIMPLE)
@deferred_f_from_coro_f @coroutine_test
async def test_async_builtin(self): async def test_async_builtin(self):
"""Upgrade""" """Upgrade"""
await self._test_asyncgen_base(self.MW_ASYNCGEN) await self._test_asyncgen_base(self.MW_ASYNCGEN)
@deferred_f_from_coro_f @coroutine_test
async def test_universal_builtin(self): async def test_universal_builtin(self):
await self._test_simple_base(self.MW_UNIVERSAL) await self._test_simple_base(self.MW_UNIVERSAL)
@ -525,33 +525,33 @@ class TestBuiltinMiddlewareAsyncGen(TestBuiltinMiddlewareSimple):
for item in super()._callback(): for item in super()._callback():
yield item yield item
@deferred_f_from_coro_f @coroutine_test
async def test_just_builtin(self): async def test_just_builtin(self):
await self._test_asyncgen_base() await self._test_asyncgen_base()
@deferred_f_from_coro_f @coroutine_test
async def test_builtin_simple(self): async def test_builtin_simple(self):
"""Downgrade""" """Downgrade"""
await self._test_simple_base(self.MW_SIMPLE, downgrade=True, start_index=1000) await self._test_simple_base(self.MW_SIMPLE, downgrade=True, start_index=1000)
@deferred_f_from_coro_f @coroutine_test
async def test_builtin_async(self): async def test_builtin_async(self):
await self._test_asyncgen_base(self.MW_ASYNCGEN, start_index=1000) await self._test_asyncgen_base(self.MW_ASYNCGEN, start_index=1000)
@deferred_f_from_coro_f @coroutine_test
async def test_builtin_universal(self): async def test_builtin_universal(self):
await self._test_asyncgen_base(self.MW_UNIVERSAL, start_index=1000) await self._test_asyncgen_base(self.MW_UNIVERSAL, start_index=1000)
@deferred_f_from_coro_f @coroutine_test
async def test_simple_builtin(self): async def test_simple_builtin(self):
"""Downgrade""" """Downgrade"""
await self._test_simple_base(self.MW_SIMPLE, downgrade=True) await self._test_simple_base(self.MW_SIMPLE, downgrade=True)
@deferred_f_from_coro_f @coroutine_test
async def test_async_builtin(self): async def test_async_builtin(self):
await self._test_asyncgen_base(self.MW_ASYNCGEN) await self._test_asyncgen_base(self.MW_ASYNCGEN)
@deferred_f_from_coro_f @coroutine_test
async def test_universal_builtin(self): async def test_universal_builtin(self):
await self._test_asyncgen_base(self.MW_UNIVERSAL) await self._test_asyncgen_base(self.MW_UNIVERSAL)
@ -574,39 +574,39 @@ class TestProcessSpiderException(TestBaseAsyncSpiderMiddleware):
): ):
await self._get_middleware_result(*mw_classes) await self._get_middleware_result(*mw_classes)
@deferred_f_from_coro_f @coroutine_test
async def test_exc_simple(self): async def test_exc_simple(self):
"""Simple exc mw""" """Simple exc mw"""
await self._test_simple_base(self.MW_EXC_SIMPLE) await self._test_simple_base(self.MW_EXC_SIMPLE)
@deferred_f_from_coro_f @coroutine_test
async def test_exc_async(self): async def test_exc_async(self):
"""Async exc mw""" """Async exc mw"""
await self._test_asyncgen_base(self.MW_EXC_ASYNCGEN) await self._test_asyncgen_base(self.MW_EXC_ASYNCGEN)
@deferred_f_from_coro_f @coroutine_test
async def test_exc_simple_simple(self): async def test_exc_simple_simple(self):
"""Simple exc mw -> simple output mw""" """Simple exc mw -> simple output mw"""
await self._test_simple_base(self.MW_SIMPLE, self.MW_EXC_SIMPLE) await self._test_simple_base(self.MW_SIMPLE, self.MW_EXC_SIMPLE)
@deferred_f_from_coro_f @coroutine_test
async def test_exc_async_async(self): async def test_exc_async_async(self):
"""Async exc mw -> async output mw""" """Async exc mw -> async output mw"""
await self._test_asyncgen_base(self.MW_ASYNCGEN, self.MW_EXC_ASYNCGEN) await self._test_asyncgen_base(self.MW_ASYNCGEN, self.MW_EXC_ASYNCGEN)
@deferred_f_from_coro_f @coroutine_test
async def test_exc_simple_async(self): async def test_exc_simple_async(self):
"""Simple exc mw -> async output mw; upgrade""" """Simple exc mw -> async output mw; upgrade"""
await self._test_asyncgen_base(self.MW_ASYNCGEN, self.MW_EXC_SIMPLE) await self._test_asyncgen_base(self.MW_ASYNCGEN, self.MW_EXC_SIMPLE)
@deferred_f_from_coro_f @coroutine_test
async def test_exc_async_simple(self): async def test_exc_async_simple(self):
"""Async exc mw -> simple output mw; cannot work as downgrading is not supported""" """Async exc mw -> simple output mw; cannot work as downgrading is not supported"""
await self._test_asyncgen_nodowngrade(self.MW_SIMPLE, self.MW_EXC_ASYNCGEN) await self._test_asyncgen_nodowngrade(self.MW_SIMPLE, self.MW_EXC_ASYNCGEN)
class TestDeprecatedSpiderArg(TestSpiderMiddleware): class TestDeprecatedSpiderArg(TestSpiderMiddleware):
@deferred_f_from_coro_f @coroutine_test
async def test_deprecated_mw_spider_arg(self): async def test_deprecated_mw_spider_arg(self):
class DeprecatedSpiderArgMiddleware: class DeprecatedSpiderArgMiddleware:
def process_spider_input(self, response, spider): def process_spider_input(self, response, spider):
@ -635,7 +635,7 @@ class TestDeprecatedSpiderArg(TestSpiderMiddleware):
self.mwman._add_middleware(DeprecatedSpiderArgMiddleware()) self.mwman._add_middleware(DeprecatedSpiderArgMiddleware())
await self._scrape_response() await self._scrape_response()
@deferred_f_from_coro_f @coroutine_test
async def test_deprecated_mwman_spider_arg(self): async def test_deprecated_mwman_spider_arg(self):
with pytest.warns( with pytest.warns(
ScrapyDeprecationWarning, ScrapyDeprecationWarning,
@ -644,7 +644,7 @@ class TestDeprecatedSpiderArg(TestSpiderMiddleware):
): ):
await self.mwman.process_start(DefaultSpider()) await self.mwman.process_start(DefaultSpider())
@deferred_f_from_coro_f @coroutine_test
async def test_deprecated_mwman_spider_arg_no_crawler(self): async def test_deprecated_mwman_spider_arg_no_crawler(self):
with pytest.warns( with pytest.warns(
ScrapyDeprecationWarning, ScrapyDeprecationWarning,

8
tests/test_spidermiddleware_httperror.py
View File

@ -11,7 +11,7 @@ from scrapy.utils.spider import DefaultSpider
from scrapy.utils.test import get_crawler from scrapy.utils.test import get_crawler
from tests.mockserver.http import MockServer from tests.mockserver.http import MockServer
from tests.spiders import MockServerSpider from tests.spiders import MockServerSpider
from tests.utils.decorators import inlineCallbacks from tests.utils.decorators import inline_callbacks_test
class _HttpErrorSpider(MockServerSpider): class _HttpErrorSpider(MockServerSpider):
@ -202,7 +202,7 @@ class TestHttpErrorMiddlewareIntegrational:
def teardown_class(cls): def teardown_class(cls):
cls.mockserver.__exit__(None, None, None) cls.mockserver.__exit__(None, None, None)
@inlineCallbacks @inline_callbacks_test
def test_middleware_works(self): def test_middleware_works(self):
crawler = get_crawler(_HttpErrorSpider) crawler = get_crawler(_HttpErrorSpider)
yield crawler.crawl(mockserver=self.mockserver) yield crawler.crawl(mockserver=self.mockserver)
@ -216,7 +216,7 @@ class TestHttpErrorMiddlewareIntegrational:
assert get_value("httperror/response_ignored_status_count/402") == 1 assert get_value("httperror/response_ignored_status_count/402") == 1
assert get_value("httperror/response_ignored_status_count/500") == 1 assert get_value("httperror/response_ignored_status_count/500") == 1
@inlineCallbacks @inline_callbacks_test
def test_logging(self): def test_logging(self):
crawler = get_crawler(_HttpErrorSpider) crawler = get_crawler(_HttpErrorSpider)
with LogCapture() as log: with LogCapture() as log:
@ -230,7 +230,7 @@ class TestHttpErrorMiddlewareIntegrational:
assert "Ignoring response <200" not in str(log) assert "Ignoring response <200" not in str(log)
assert "Ignoring response <402" not in str(log) assert "Ignoring response <402" not in str(log)
@inlineCallbacks @inline_callbacks_test
def test_logging_level(self): def test_logging_level(self):
# HttpError logs ignored responses with level INFO # HttpError logs ignored responses with level INFO
crawler = get_crawler(_HttpErrorSpider) crawler = get_crawler(_HttpErrorSpider)

24
tests/test_spidermiddleware_output_chain.py
View File

@ -4,7 +4,7 @@ from testfixtures import LogCapture
from scrapy import Request, Spider from scrapy import Request, Spider
from scrapy.utils.test import get_crawler from scrapy.utils.test import get_crawler
from tests.mockserver.http import MockServer from tests.mockserver.http import MockServer
from tests.utils.decorators import deferred_f_from_coro_f from tests.utils.decorators import coroutine_test
class _BaseSpiderMiddleware: class _BaseSpiderMiddleware:
@ -338,7 +338,7 @@ class TestSpiderMiddleware:
await crawler.crawl_async(mockserver=self.mockserver) await crawler.crawl_async(mockserver=self.mockserver)
return log return log
@deferred_f_from_coro_f @coroutine_test
async def test_recovery(self): async def test_recovery(self):
""" """
(0) Recover from an exception in a spider callback. The final item count should be 3 (0) Recover from an exception in a spider callback. The final item count should be 3
@ -351,7 +351,7 @@ class TestSpiderMiddleware:
assert str(log).count("Middleware: TabError exception caught") == 1 assert str(log).count("Middleware: TabError exception caught") == 1
assert "'item_scraped_count': 3" in str(log) assert "'item_scraped_count': 3" in str(log)
@deferred_f_from_coro_f @coroutine_test
async def test_recovery_asyncgen(self): async def test_recovery_asyncgen(self):
""" """
Same as test_recovery but with an async callback. Same as test_recovery but with an async callback.
@ -361,7 +361,7 @@ class TestSpiderMiddleware:
assert str(log).count("Middleware: TabError exception caught") == 1 assert str(log).count("Middleware: TabError exception caught") == 1
assert "'item_scraped_count': 3" in str(log) assert "'item_scraped_count': 3" in str(log)
@deferred_f_from_coro_f @coroutine_test
async def test_process_spider_input_without_errback(self): async def test_process_spider_input_without_errback(self):
""" """
(1.1) An exception from the process_spider_input chain should be caught by the (1.1) An exception from the process_spider_input chain should be caught by the
@ -371,7 +371,7 @@ class TestSpiderMiddleware:
assert "Middleware: will raise IndexError" in str(log1) assert "Middleware: will raise IndexError" in str(log1)
assert "Middleware: IndexError exception caught" in str(log1) assert "Middleware: IndexError exception caught" in str(log1)
@deferred_f_from_coro_f @coroutine_test
async def test_process_spider_input_with_errback(self): async def test_process_spider_input_with_errback(self):
""" """
(1.2) An exception from the process_spider_input chain should not be caught by the (1.2) An exception from the process_spider_input chain should not be caught by the
@ -385,7 +385,7 @@ class TestSpiderMiddleware:
assert "{'from': 'callback'}" not in str(log1) assert "{'from': 'callback'}" not in str(log1)
assert "'item_scraped_count': 1" in str(log1) assert "'item_scraped_count': 1" in str(log1)
@deferred_f_from_coro_f @coroutine_test
async def test_generator_callback(self): async def test_generator_callback(self):
""" """
(2) An exception from a spider callback (returning a generator) should (2) An exception from a spider callback (returning a generator) should
@ -396,7 +396,7 @@ class TestSpiderMiddleware:
assert "Middleware: ImportError exception caught" in str(log2) assert "Middleware: ImportError exception caught" in str(log2)
assert "'item_scraped_count': 2" in str(log2) assert "'item_scraped_count': 2" in str(log2)
@deferred_f_from_coro_f @coroutine_test
async def test_async_generator_callback(self): async def test_async_generator_callback(self):
""" """
Same as test_generator_callback but with an async callback. Same as test_generator_callback but with an async callback.
@ -405,7 +405,7 @@ class TestSpiderMiddleware:
assert "Middleware: ImportError exception caught" in str(log2) assert "Middleware: ImportError exception caught" in str(log2)
assert "'item_scraped_count': 2" in str(log2) assert "'item_scraped_count': 2" in str(log2)
@deferred_f_from_coro_f @coroutine_test
async def test_generator_callback_right_after_callback(self): async def test_generator_callback_right_after_callback(self):
""" """
(2.1) Special case of (2): Exceptions should be caught (2.1) Special case of (2): Exceptions should be caught
@ -415,7 +415,7 @@ class TestSpiderMiddleware:
assert "Middleware: ImportError exception caught" in str(log21) assert "Middleware: ImportError exception caught" in str(log21)
assert "'item_scraped_count': 2" in str(log21) assert "'item_scraped_count': 2" in str(log21)
@deferred_f_from_coro_f @coroutine_test
async def test_not_a_generator_callback(self): async def test_not_a_generator_callback(self):
""" """
(3) An exception from a spider callback (returning a list) should (3) An exception from a spider callback (returning a list) should
@ -425,7 +425,7 @@ class TestSpiderMiddleware:
assert "Middleware: ZeroDivisionError exception caught" in str(log3) assert "Middleware: ZeroDivisionError exception caught" in str(log3)
assert "item_scraped_count" not in str(log3) assert "item_scraped_count" not in str(log3)
@deferred_f_from_coro_f @coroutine_test
async def test_not_a_generator_callback_right_after_callback(self): async def test_not_a_generator_callback_right_after_callback(self):
""" """
(3.1) Special case of (3): Exceptions should be caught (3.1) Special case of (3): Exceptions should be caught
@ -437,7 +437,7 @@ class TestSpiderMiddleware:
assert "Middleware: ZeroDivisionError exception caught" in str(log31) assert "Middleware: ZeroDivisionError exception caught" in str(log31)
assert "item_scraped_count" not in str(log31) assert "item_scraped_count" not in str(log31)
@deferred_f_from_coro_f @coroutine_test
async def test_generator_output_chain(self): async def test_generator_output_chain(self):
""" """
(4) An exception from a middleware's process_spider_output method should be sent (4) An exception from a middleware's process_spider_output method should be sent
@ -484,7 +484,7 @@ class TestSpiderMiddleware:
assert str(item_recovered) in str(log4) assert str(item_recovered) in str(log4)
assert "parse-second-item" not in str(log4) assert "parse-second-item" not in str(log4)
@deferred_f_from_coro_f @coroutine_test
async def test_not_a_generator_output_chain(self): async def test_not_a_generator_output_chain(self):
""" """
(5) An exception from a middleware's process_spider_output method should be sent (5) An exception from a middleware's process_spider_output method should be sent

48
tests/test_spidermiddleware_process_start.py
View File

@ -10,7 +10,7 @@ from scrapy.utils.test import get_crawler
from tests.test_spider_start import SLEEP_SECONDS from tests.test_spider_start import SLEEP_SECONDS
from .utils import twisted_sleep from .utils import twisted_sleep
from .utils.decorators import deferred_f_from_coro_f from .utils.decorators import coroutine_test
ITEM_A = {"id": "a"} ITEM_A = {"id": "a"}
ITEM_B = {"id": "b"} ITEM_B = {"id": "b"}
@ -130,45 +130,45 @@ class TestMain:
expected_items = expected_items or [ITEM_A, ITEM_A, ITEM_B, ITEM_C, ITEM_C] expected_items = expected_items or [ITEM_A, ITEM_A, ITEM_B, ITEM_C, ITEM_C]
await self._test([smw1, smw2], spider_cls, expected_items) await self._test([smw1, smw2], spider_cls, expected_items)
@deferred_f_from_coro_f @coroutine_test
async def test_modern_mw_modern_spider(self): async def test_modern_mw_modern_spider(self):
with warnings.catch_warnings(): with warnings.catch_warnings():
warnings.simplefilter("error") warnings.simplefilter("error")
await self._test_wrap(ModernWrapSpiderMiddleware, ModernWrapSpider) await self._test_wrap(ModernWrapSpiderMiddleware, ModernWrapSpider)
@deferred_f_from_coro_f @coroutine_test
async def test_modern_mw_universal_spider(self): async def test_modern_mw_universal_spider(self):
with warnings.catch_warnings(): with warnings.catch_warnings():
warnings.simplefilter("error") warnings.simplefilter("error")
await self._test_wrap(ModernWrapSpiderMiddleware, UniversalWrapSpider) await self._test_wrap(ModernWrapSpiderMiddleware, UniversalWrapSpider)
@deferred_f_from_coro_f @coroutine_test
async def test_modern_mw_deprecated_spider(self): async def test_modern_mw_deprecated_spider(self):
with pytest.warns( with pytest.warns(
ScrapyDeprecationWarning, match=r"deprecated start_requests\(\)" ScrapyDeprecationWarning, match=r"deprecated start_requests\(\)"
): ):
await self._test_wrap(ModernWrapSpiderMiddleware, DeprecatedWrapSpider) await self._test_wrap(ModernWrapSpiderMiddleware, DeprecatedWrapSpider)
@deferred_f_from_coro_f @coroutine_test
async def test_universal_mw_modern_spider(self): async def test_universal_mw_modern_spider(self):
with warnings.catch_warnings(): with warnings.catch_warnings():
warnings.simplefilter("error") warnings.simplefilter("error")
await self._test_wrap(UniversalWrapSpiderMiddleware, ModernWrapSpider) await self._test_wrap(UniversalWrapSpiderMiddleware, ModernWrapSpider)
@deferred_f_from_coro_f @coroutine_test
async def test_universal_mw_universal_spider(self): async def test_universal_mw_universal_spider(self):
with warnings.catch_warnings(): with warnings.catch_warnings():
warnings.simplefilter("error") warnings.simplefilter("error")
await self._test_wrap(UniversalWrapSpiderMiddleware, UniversalWrapSpider) await self._test_wrap(UniversalWrapSpiderMiddleware, UniversalWrapSpider)
@deferred_f_from_coro_f @coroutine_test
async def test_universal_mw_deprecated_spider(self): async def test_universal_mw_deprecated_spider(self):
with pytest.warns( with pytest.warns(
ScrapyDeprecationWarning, match=r"deprecated start_requests\(\)" ScrapyDeprecationWarning, match=r"deprecated start_requests\(\)"
): ):
await self._test_wrap(UniversalWrapSpiderMiddleware, DeprecatedWrapSpider) await self._test_wrap(UniversalWrapSpiderMiddleware, DeprecatedWrapSpider)
@deferred_f_from_coro_f @coroutine_test
async def test_deprecated_mw_modern_spider(self): async def test_deprecated_mw_modern_spider(self):
with ( with (
pytest.warns( pytest.warns(
@ -180,7 +180,7 @@ class TestMain:
): ):
await self._test_wrap(DeprecatedWrapSpiderMiddleware, ModernWrapSpider) await self._test_wrap(DeprecatedWrapSpiderMiddleware, ModernWrapSpider)
@deferred_f_from_coro_f @coroutine_test
async def test_deprecated_mw_modern_spider_subclass(self): async def test_deprecated_mw_modern_spider_subclass(self):
with ( with (
pytest.warns( pytest.warns(
@ -195,7 +195,7 @@ class TestMain:
DeprecatedWrapSpiderMiddleware, ModernWrapSpiderSubclass DeprecatedWrapSpiderMiddleware, ModernWrapSpiderSubclass
) )
@deferred_f_from_coro_f @coroutine_test
async def test_deprecated_mw_universal_spider(self): async def test_deprecated_mw_universal_spider(self):
with pytest.warns( with pytest.warns(
ScrapyDeprecationWarning, match=r"deprecated process_start_requests\(\)" ScrapyDeprecationWarning, match=r"deprecated process_start_requests\(\)"
@ -206,7 +206,7 @@ class TestMain:
[ITEM_A, ITEM_D, ITEM_C], [ITEM_A, ITEM_D, ITEM_C],
) )
@deferred_f_from_coro_f @coroutine_test
async def test_deprecated_mw_deprecated_spider(self): async def test_deprecated_mw_deprecated_spider(self):
with ( with (
pytest.warns( pytest.warns(
@ -218,7 +218,7 @@ class TestMain:
): ):
await self._test_wrap(DeprecatedWrapSpiderMiddleware, DeprecatedWrapSpider) await self._test_wrap(DeprecatedWrapSpiderMiddleware, DeprecatedWrapSpider)
@deferred_f_from_coro_f @coroutine_test
async def test_modern_mw_universal_mw_modern_spider(self): async def test_modern_mw_universal_mw_modern_spider(self):
with warnings.catch_warnings(): with warnings.catch_warnings():
warnings.simplefilter("error") warnings.simplefilter("error")
@ -228,7 +228,7 @@ class TestMain:
ModernWrapSpider, ModernWrapSpider,
) )
@deferred_f_from_coro_f @coroutine_test
async def test_modern_mw_deprecated_mw_modern_spider(self): async def test_modern_mw_deprecated_mw_modern_spider(self):
with pytest.raises(ValueError, match=r"trying to combine spider middlewares"): with pytest.raises(ValueError, match=r"trying to combine spider middlewares"):
await self._test_douple_wrap( await self._test_douple_wrap(
@ -237,7 +237,7 @@ class TestMain:
ModernWrapSpider, ModernWrapSpider,
) )
@deferred_f_from_coro_f @coroutine_test
async def test_universal_mw_deprecated_mw_modern_spider(self): async def test_universal_mw_deprecated_mw_modern_spider(self):
with ( with (
pytest.warns( pytest.warns(
@ -253,7 +253,7 @@ class TestMain:
ModernWrapSpider, ModernWrapSpider,
) )
@deferred_f_from_coro_f @coroutine_test
async def test_modern_mw_universal_mw_universal_spider(self): async def test_modern_mw_universal_mw_universal_spider(self):
with warnings.catch_warnings(): with warnings.catch_warnings():
warnings.simplefilter("error") warnings.simplefilter("error")
@ -263,7 +263,7 @@ class TestMain:
UniversalWrapSpider, UniversalWrapSpider,
) )
@deferred_f_from_coro_f @coroutine_test
async def test_modern_mw_deprecated_mw_universal_spider(self): async def test_modern_mw_deprecated_mw_universal_spider(self):
with pytest.raises(ValueError, match=r"trying to combine spider middlewares"): with pytest.raises(ValueError, match=r"trying to combine spider middlewares"):
await self._test_douple_wrap( await self._test_douple_wrap(
@ -272,7 +272,7 @@ class TestMain:
UniversalWrapSpider, UniversalWrapSpider,
) )
@deferred_f_from_coro_f @coroutine_test
async def test_universal_mw_deprecated_mw_universal_spider(self): async def test_universal_mw_deprecated_mw_universal_spider(self):
with pytest.warns( with pytest.warns(
ScrapyDeprecationWarning, match=r"deprecated process_start_requests\(\)" ScrapyDeprecationWarning, match=r"deprecated process_start_requests\(\)"
@ -284,7 +284,7 @@ class TestMain:
[ITEM_A, ITEM_A, ITEM_D, ITEM_C, ITEM_C], [ITEM_A, ITEM_A, ITEM_D, ITEM_C, ITEM_C],
) )
@deferred_f_from_coro_f @coroutine_test
async def test_modern_mw_universal_mw_deprecated_spider(self): async def test_modern_mw_universal_mw_deprecated_spider(self):
with pytest.warns( with pytest.warns(
ScrapyDeprecationWarning, match=r"deprecated start_requests\(\)" ScrapyDeprecationWarning, match=r"deprecated start_requests\(\)"
@ -295,7 +295,7 @@ class TestMain:
DeprecatedWrapSpider, DeprecatedWrapSpider,
) )
@deferred_f_from_coro_f @coroutine_test
async def test_modern_mw_deprecated_mw_deprecated_spider(self): async def test_modern_mw_deprecated_mw_deprecated_spider(self):
with pytest.raises(ValueError, match=r"trying to combine spider middlewares"): with pytest.raises(ValueError, match=r"trying to combine spider middlewares"):
await self._test_douple_wrap( await self._test_douple_wrap(
@ -304,7 +304,7 @@ class TestMain:
DeprecatedWrapSpider, DeprecatedWrapSpider,
) )
@deferred_f_from_coro_f @coroutine_test
async def test_universal_mw_deprecated_mw_deprecated_spider(self): async def test_universal_mw_deprecated_mw_deprecated_spider(self):
with ( with (
pytest.warns( pytest.warns(
@ -330,24 +330,24 @@ class TestMain:
await self._test(spider_middlewares, TestSpider, [ITEM_A]) await self._test(spider_middlewares, TestSpider, [ITEM_A])
@pytest.mark.only_asyncio @pytest.mark.only_asyncio
@deferred_f_from_coro_f @coroutine_test
async def test_asyncio_sleep_single(self): async def test_asyncio_sleep_single(self):
await self._test_sleep([AsyncioSleepSpiderMiddleware]) await self._test_sleep([AsyncioSleepSpiderMiddleware])
@pytest.mark.only_asyncio @pytest.mark.only_asyncio
@deferred_f_from_coro_f @coroutine_test
async def test_asyncio_sleep_multiple(self): async def test_asyncio_sleep_multiple(self):
await self._test_sleep( await self._test_sleep(
[NoOpSpiderMiddleware, AsyncioSleepSpiderMiddleware, NoOpSpiderMiddleware] [NoOpSpiderMiddleware, AsyncioSleepSpiderMiddleware, NoOpSpiderMiddleware]
) )
@pytest.mark.requires_reactor @pytest.mark.requires_reactor
@deferred_f_from_coro_f @coroutine_test
async def test_twisted_sleep_single(self): async def test_twisted_sleep_single(self):
await self._test_sleep([TwistedSleepSpiderMiddleware]) await self._test_sleep([TwistedSleepSpiderMiddleware])
@pytest.mark.requires_reactor @pytest.mark.requires_reactor
@deferred_f_from_coro_f @coroutine_test
async def test_twisted_sleep_multiple(self): async def test_twisted_sleep_multiple(self):
await self._test_sleep( await self._test_sleep(
[NoOpSpiderMiddleware, TwistedSleepSpiderMiddleware, NoOpSpiderMiddleware] [NoOpSpiderMiddleware, TwistedSleepSpiderMiddleware, NoOpSpiderMiddleware]

6
tests/test_spidermiddleware_start.py
View File

@ -3,11 +3,11 @@ from scrapy.spidermiddlewares.start import StartSpiderMiddleware
from scrapy.spiders import Spider from scrapy.spiders import Spider
from scrapy.utils.misc import build_from_crawler from scrapy.utils.misc import build_from_crawler
from scrapy.utils.test import get_crawler from scrapy.utils.test import get_crawler
from tests.utils.decorators import deferred_f_from_coro_f from tests.utils.decorators import coroutine_test
class TestMiddleware: class TestMiddleware:
@deferred_f_from_coro_f @coroutine_test
async def test_async(self): async def test_async(self):
crawler = get_crawler(Spider) crawler = get_crawler(Spider)
mw = build_from_crawler(StartSpiderMiddleware, crawler) mw = build_from_crawler(StartSpiderMiddleware, crawler)
@ -24,7 +24,7 @@ class TestMiddleware:
] ]
assert result == [True, True, False, "foo"] assert result == [True, True, False, "foo"]
@deferred_f_from_coro_f @coroutine_test
async def test_sync(self): async def test_sync(self):
crawler = get_crawler(Spider) crawler = get_crawler(Spider)
mw = build_from_crawler(StartSpiderMiddleware, crawler) mw = build_from_crawler(StartSpiderMiddleware, crawler)

6
tests/test_stats.py
View File

@ -12,7 +12,7 @@ from scrapy.spiders import Spider
from scrapy.statscollectors import DummyStatsCollector, StatsCollector from scrapy.statscollectors import DummyStatsCollector, StatsCollector
from scrapy.utils.test import get_crawler from scrapy.utils.test import get_crawler
from tests.spiders import SimpleSpider from tests.spiders import SimpleSpider
from tests.utils.decorators import deferred_f_from_coro_f from tests.utils.decorators import coroutine_test
if TYPE_CHECKING: if TYPE_CHECKING:
from scrapy.crawler import Crawler from scrapy.crawler import Crawler
@ -121,7 +121,7 @@ class TestStatsCollector:
): ):
assert stats.get_stats(spider) == {"test": "value"} assert stats.get_stats(spider) == {"test": "value"}
@deferred_f_from_coro_f @coroutine_test
async def test_deprecated_spider_arg_custom_collector(self) -> None: async def test_deprecated_spider_arg_custom_collector(self) -> None:
class CustomStatsCollector: class CustomStatsCollector:
def __init__(self, crawler): def __init__(self, crawler):
@ -153,7 +153,7 @@ class TestStatsCollector:
): ):
await crawler.crawl_async(url="data:,") await crawler.crawl_async(url="data:,")
@deferred_f_from_coro_f @coroutine_test
async def test_deprecated_spider_arg_custom_collector_subclass(self) -> None: async def test_deprecated_spider_arg_custom_collector_subclass(self) -> None:
class CustomStatsCollector(StatsCollector): class CustomStatsCollector(StatsCollector):
def open_spider(self, spider): # pylint: disable=signature-differs def open_spider(self, spider): # pylint: disable=signature-differs

6
tests/test_utils_asyncgen.py
View File

@ -1,15 +1,15 @@
from scrapy.utils.asyncgen import as_async_generator, collect_asyncgen from scrapy.utils.asyncgen import as_async_generator, collect_asyncgen
from tests.utils.decorators import deferred_f_from_coro_f from tests.utils.decorators import coroutine_test
class TestAsyncgenUtils: class TestAsyncgenUtils:
@deferred_f_from_coro_f @coroutine_test
async def test_as_async_generator(self): async def test_as_async_generator(self):
ag = as_async_generator(range(42)) ag = as_async_generator(range(42))
results = [i async for i in ag] results = [i async for i in ag]
assert results == list(range(42)) assert results == list(range(42))
@deferred_f_from_coro_f @coroutine_test
async def test_collect_asyncgen(self): async def test_collect_asyncgen(self):
ag = as_async_generator(range(42)) ag = as_async_generator(range(42))
results = await collect_asyncgen(ag) results = await collect_asyncgen(ag)

6
tests/test_utils_asyncio.py
View File

@ -14,7 +14,7 @@ from scrapy.utils.asyncio import (
_parallel_asyncio, _parallel_asyncio,
is_asyncio_available, is_asyncio_available,
) )
from tests.utils.decorators import deferred_f_from_coro_f from tests.utils.decorators import coroutine_test
if TYPE_CHECKING: if TYPE_CHECKING:
from collections.abc import AsyncGenerator from collections.abc import AsyncGenerator
@ -67,7 +67,7 @@ class TestParallelAsyncio:
await asyncio.sleep(random.random() / 20) await asyncio.sleep(random.random() / 20)
yield i yield i
@deferred_f_from_coro_f @coroutine_test
async def test_simple(self): async def test_simple(self):
for length in [20, 50, 100]: for length in [20, 50, 100]:
parallel_count = [0] parallel_count = [0]
@ -85,7 +85,7 @@ class TestParallelAsyncio:
assert list(range(length)) == sorted(results) assert list(range(length)) == sorted(results)
assert max_parallel_count[0] <= self.CONCURRENT_ITEMS assert max_parallel_count[0] <= self.CONCURRENT_ITEMS
@deferred_f_from_coro_f @coroutine_test
async def test_delays(self): async def test_delays(self):
for length in [20, 50, 100]: for length in [20, 50, 100]:
parallel_count = [0] parallel_count = [0]

27
tests/test_utils_defer.py
View File

@ -6,8 +6,7 @@ from asyncio import Future
from typing import TYPE_CHECKING, Any from typing import TYPE_CHECKING, Any
import pytest import pytest
from twisted.internet.defer import Deferred, succeed from twisted.internet.defer import Deferred, inlineCallbacks, succeed
from twisted.internet.defer import inlineCallbacks as inlineCallbacks_orig
from scrapy.utils.asyncgen import as_async_generator, collect_asyncgen from scrapy.utils.asyncgen import as_async_generator, collect_asyncgen
from scrapy.utils.defer import ( from scrapy.utils.defer import (
@ -20,7 +19,7 @@ from scrapy.utils.defer import (
mustbe_deferred, mustbe_deferred,
parallel_async, parallel_async,
) )
from tests.utils.decorators import inlineCallbacks from tests.utils.decorators import inline_callbacks_test
if TYPE_CHECKING: if TYPE_CHECKING:
from collections.abc import AsyncGenerator, Awaitable, Callable, Generator from collections.abc import AsyncGenerator, Awaitable, Callable, Generator
@ -29,7 +28,7 @@ if TYPE_CHECKING:
@pytest.mark.requires_reactor @pytest.mark.requires_reactor
@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
class TestMustbeDeferred: class TestMustbeDeferred:
@inlineCallbacks @inline_callbacks_test
def test_success_function(self) -> Generator[Deferred[Any], Any, None]: def test_success_function(self) -> Generator[Deferred[Any], Any, None]:
steps: list[int] = [] steps: list[int] = []
@ -45,7 +44,7 @@ class TestMustbeDeferred:
steps.append(2) # add another value, that should be caught by assertEqual steps.append(2) # add another value, that should be caught by assertEqual
yield dfd yield dfd
@inlineCallbacks @inline_callbacks_test
def test_unfired_deferred(self) -> Generator[Deferred[Any], Any, None]: def test_unfired_deferred(self) -> Generator[Deferred[Any], Any, None]:
steps: list[int] = [] steps: list[int] = []
@ -224,7 +223,7 @@ class TestParallelAsync:
await maybe_deferred_to_future(dfd) await maybe_deferred_to_future(dfd)
yield i yield i
@inlineCallbacks @inline_callbacks_test
def test_simple(self): def test_simple(self):
for length in [20, 50, 100]: for length in [20, 50, 100]:
parallel_count = [0] parallel_count = [0]
@ -244,7 +243,7 @@ class TestParallelAsync:
assert parallel_count[0] == 0 assert parallel_count[0] == 0
assert max_parallel_count[0] <= self.CONCURRENT_ITEMS, max_parallel_count[0] assert max_parallel_count[0] <= self.CONCURRENT_ITEMS, max_parallel_count[0]
@inlineCallbacks @inline_callbacks_test
def test_delays(self): def test_delays(self):
for length in [20, 50, 100]: for length in [20, 50, 100]:
parallel_count = [0] parallel_count = [0]
@ -276,7 +275,7 @@ class TestDeferredFromCoro:
result = deferred_from_coro(42) result = deferred_from_coro(42)
assert result == 42 assert result == 42
@inlineCallbacks @inline_callbacks_test
def test_coroutine(self): def test_coroutine(self):
async def coroutine() -> int: async def coroutine() -> int:
return 42 return 42
@ -287,7 +286,7 @@ class TestDeferredFromCoro:
assert coro_result == 42 assert coro_result == 42
@pytest.mark.only_asyncio @pytest.mark.only_asyncio
@inlineCallbacks @inline_callbacks_test
def test_coroutine_asyncio(self): def test_coroutine_asyncio(self):
async def coroutine() -> int: async def coroutine() -> int:
await asyncio.sleep(0.01) await asyncio.sleep(0.01)
@ -299,7 +298,7 @@ class TestDeferredFromCoro:
assert coro_result == 42 assert coro_result == 42
@pytest.mark.only_asyncio @pytest.mark.only_asyncio
@inlineCallbacks @inline_callbacks_test
def test_future(self): def test_future(self):
future = Future() future = Future()
result = deferred_from_coro(future) result = deferred_from_coro(future)
@ -310,7 +309,7 @@ class TestDeferredFromCoro:
class TestDeferredFFromCoroF: class TestDeferredFFromCoroF:
@inlineCallbacks_orig @inlineCallbacks
def _assert_result( def _assert_result(
self, c_f: Callable[[], Awaitable[int]] self, c_f: Callable[[], Awaitable[int]]
) -> Generator[Deferred[Any], Any, None]: ) -> Generator[Deferred[Any], Any, None]:
@ -320,7 +319,7 @@ class TestDeferredFFromCoroF:
result = yield d result = yield d
assert result == 42 assert result == 42
@inlineCallbacks @inline_callbacks_test
def test_coroutine(self): def test_coroutine(self):
async def c_f() -> int: async def c_f() -> int:
return 42 return 42
@ -328,7 +327,7 @@ class TestDeferredFFromCoroF:
yield self._assert_result(c_f) yield self._assert_result(c_f)
@pytest.mark.only_asyncio @pytest.mark.only_asyncio
@inlineCallbacks @inline_callbacks_test
def test_coroutine_asyncio(self): def test_coroutine_asyncio(self):
async def c_f() -> int: async def c_f() -> int:
await asyncio.sleep(0.01) await asyncio.sleep(0.01)
@ -337,7 +336,7 @@ class TestDeferredFFromCoroF:
yield self._assert_result(c_f) yield self._assert_result(c_f)
@pytest.mark.only_asyncio @pytest.mark.only_asyncio
@inlineCallbacks @inline_callbacks_test
def test_future(self): def test_future(self):
def c_f() -> Future[int]: def c_f() -> Future[int]:
f: Future[int] = Future() f: Future[int] = Future()

6
tests/test_utils_python.py
View File

@ -20,7 +20,7 @@ from scrapy.utils.python import (
to_unicode, to_unicode,
without_none_values, without_none_values,
) )
from tests.utils.decorators import deferred_f_from_coro_f from tests.utils.decorators import coroutine_test
if TYPE_CHECKING: if TYPE_CHECKING:
from collections.abc import Iterable, Mapping from collections.abc import Iterable, Mapping
@ -64,7 +64,7 @@ class TestMutableAsyncChain:
for i in range(5, 7): for i in range(5, 7):
yield i yield i
@deferred_f_from_coro_f @coroutine_test
async def test_mutableasyncchain(self): async def test_mutableasyncchain(self):
m = MutableAsyncChain(self.g1(), as_async_generator(range(3, 7))) m = MutableAsyncChain(self.g1(), as_async_generator(range(3, 7)))
m.extend(self.g2()) m.extend(self.g2())
@ -74,7 +74,7 @@ class TestMutableAsyncChain:
results = await collect_asyncgen(m) results = await collect_asyncgen(m)
assert results == list(range(1, 10)) assert results == list(range(1, 10))
@deferred_f_from_coro_f @coroutine_test
async def test_mutableasyncchain_exc(self): async def test_mutableasyncchain_exc(self):
m = MutableAsyncChain(self.g1()) m = MutableAsyncChain(self.g1())
m.extend(self.g4()) m.extend(self.g4())

4
tests/test_utils_reactor.py
View File

@ -9,7 +9,7 @@ from scrapy.utils.reactor import (
is_asyncio_reactor_installed, is_asyncio_reactor_installed,
set_asyncio_event_loop, set_asyncio_event_loop,
) )
from tests.utils.decorators import deferred_f_from_coro_f from tests.utils.decorators import coroutine_test
class TestAsyncio: class TestAsyncio:
@ -31,7 +31,7 @@ class TestAsyncio:
@pytest.mark.requires_reactor @pytest.mark.requires_reactor
@pytest.mark.only_asyncio @pytest.mark.only_asyncio
@deferred_f_from_coro_f @coroutine_test
async def test_set_asyncio_event_loop(self): async def test_set_asyncio_event_loop(self):
install_reactor(_asyncio_reactor_path) install_reactor(_asyncio_reactor_path)
assert set_asyncio_event_loop(None) is asyncio.get_running_loop() assert set_asyncio_event_loop(None) is asyncio.get_running_loop()

4
tests/test_utils_signal.py
View File

@ -14,14 +14,14 @@ from scrapy.utils.signal import (
send_catch_log_deferred, send_catch_log_deferred,
) )
from scrapy.utils.test import get_from_asyncio_queue from scrapy.utils.test import get_from_asyncio_queue
from tests.utils.decorators import inlineCallbacks from tests.utils.decorators import inline_callbacks_test
class TestSendCatchLog: class TestSendCatchLog:
# whether the function being tested returns exceptions or failures # whether the function being tested returns exceptions or failures
returns_exceptions: bool = False returns_exceptions: bool = False
@inlineCallbacks @inline_callbacks_test
def test_send_catch_log(self): def test_send_catch_log(self):
test_signal = object() test_signal = object()
handlers_called = set() handlers_called = set()

11
tests/utils/decorators.py
View File

@ -4,8 +4,7 @@ from functools import wraps
from typing import TYPE_CHECKING, Any, ParamSpec from typing import TYPE_CHECKING, Any, ParamSpec
import pytest import pytest
from twisted.internet.defer import Deferred from twisted.internet.defer import Deferred, inlineCallbacks
from twisted.internet.defer import inlineCallbacks as inlineCallbacks_orig
from scrapy.utils.defer import deferred_from_coro, deferred_to_future from scrapy.utils.defer import deferred_from_coro, deferred_to_future
from scrapy.utils.reactor import is_reactor_installed from scrapy.utils.reactor import is_reactor_installed
@ -17,7 +16,7 @@ if TYPE_CHECKING:
_P = ParamSpec("_P") _P = ParamSpec("_P")
def inlineCallbacks( def inline_callbacks_test(
f: Callable[_P, Generator[Deferred[Any], Any, None]], f: Callable[_P, Generator[Deferred[Any], Any, None]],
) -> Callable[_P, Awaitable[None]]: ) -> Callable[_P, Awaitable[None]]:
"""Mark a test function written in a :func:`twisted.internet.defer.inlineCallbacks` style. """Mark a test function written in a :func:`twisted.internet.defer.inlineCallbacks` style.
@ -34,12 +33,12 @@ def inlineCallbacks(
@pytest.mark.asyncio @pytest.mark.asyncio
@wraps(f) @wraps(f)
async def wrapper_coro(*args: _P.args, **kwargs: _P.kwargs) -> None: async def wrapper_coro(*args: _P.args, **kwargs: _P.kwargs) -> None:
await deferred_to_future(inlineCallbacks_orig(f)(*args, **kwargs)) await deferred_to_future(inlineCallbacks(f)(*args, **kwargs))
return wrapper_coro return wrapper_coro
@wraps(f) @wraps(f)
@inlineCallbacks_orig @inlineCallbacks
def wrapper_dfd( def wrapper_dfd(
*args: _P.args, **kwargs: _P.kwargs *args: _P.args, **kwargs: _P.kwargs
) -> Generator[Deferred[Any], Any, None]: ) -> Generator[Deferred[Any], Any, None]:
@ -48,7 +47,7 @@ def inlineCallbacks(
return wrapper_dfd return wrapper_dfd
def deferred_f_from_coro_f( def coroutine_test(
coro_f: Callable[_P, Awaitable[None]], coro_f: Callable[_P, Awaitable[None]],
) -> Callable[_P, Awaitable[None]]: ) -> Callable[_P, Awaitable[None]]:
"""Mark a test function that returns a coroutine. """Mark a test function that returns a coroutine.