mirror of https://github.com/scrapy/scrapy.git
Replace most of the @inlineCallbacks test helpers. (#6883)
This commit is contained in:
Andrey Rakhmatullin
GitHub
parent
0390176ecd
commit
ac956f8595
|
|
@ -4,7 +4,7 @@ import json
|
|||
import logging
|
||||
from ipaddress import IPv4Address
|
||||
from socket import gethostbyname
|
||||
from typing import Any
|
||||
from typing import TYPE_CHECKING, Any
|
||||
from urllib.parse import urlparse
|
||||
|
||||
import pytest
|
||||
|
|
@ -14,11 +14,12 @@ from twisted.internet.ssl import Certificate
|
|||
from twisted.python.failure import Failure
|
||||
from twisted.trial.unittest import TestCase
|
||||
|
||||
from scrapy import signals
|
||||
from scrapy import Spider, signals
|
||||
from scrapy.crawler import CrawlerRunner
|
||||
from scrapy.exceptions import CloseSpider, StopDownload
|
||||
from scrapy.http import Request
|
||||
from scrapy.http.response import Response
|
||||
from scrapy.utils.defer import deferred_f_from_coro_f, maybe_deferred_to_future
|
||||
from scrapy.utils.python import to_unicode
|
||||
from scrapy.utils.test import get_crawler, get_reactor_settings
|
||||
from tests import NON_EXISTING_RESOLVABLE
|
||||
|
|
@ -55,8 +56,13 @@ from tests.spiders import (
|
|||
StartItemSpider,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from scrapy.statscollectors import StatsCollector
|
||||
|
||||
|
||||
class TestCrawl(TestCase):
|
||||
mockserver: MockServer
|
||||
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls.mockserver = MockServer()
|
||||
|
|
@ -72,16 +78,17 @@ class TestCrawl(TestCase):
|
|||
yield crawler.crawl(mockserver=self.mockserver)
|
||||
assert len(crawler.spider.urls_visited) == 11 # 10 + start_url
|
||||
|
||||
@inlineCallbacks
|
||||
def test_fixed_delay(self):
|
||||
yield self._test_delay(total=3, delay=0.2)
|
||||
@deferred_f_from_coro_f
|
||||
async def test_fixed_delay(self):
|
||||
await self._test_delay(total=3, delay=0.2)
|
||||
|
||||
@inlineCallbacks
|
||||
def test_randomized_delay(self):
|
||||
yield self._test_delay(total=3, delay=0.1, randomize=True)
|
||||
@deferred_f_from_coro_f
|
||||
async def test_randomized_delay(self):
|
||||
await self._test_delay(total=3, delay=0.1, randomize=True)
|
||||
|
||||
@inlineCallbacks
|
||||
def _test_delay(self, total, delay, randomize=False):
|
||||
async def _test_delay(
|
||||
self, total: int, delay: float, randomize: bool = False
|
||||
) -> None:
|
||||
crawl_kwargs = {
|
||||
"maxlatency": delay * 2,
|
||||
"mockserver": self.mockserver,
|
||||
|
|
@ -91,7 +98,9 @@ class TestCrawl(TestCase):
|
|||
|
||||
settings = {"DOWNLOAD_DELAY": delay, "RANDOMIZE_DOWNLOAD_DELAY": randomize}
|
||||
crawler = get_crawler(FollowAllSpider, settings)
|
||||
yield crawler.crawl(**crawl_kwargs)
|
||||
await maybe_deferred_to_future(crawler.crawl(**crawl_kwargs))
|
||||
assert crawler.spider
|
||||
assert isinstance(crawler.spider, FollowAllSpider)
|
||||
times = crawler.spider.times
|
||||
total_time = times[-1] - times[0]
|
||||
average = total_time / (len(times) - 1)
|
||||
|
|
@ -103,7 +112,9 @@ class TestCrawl(TestCase):
|
|||
# code above to have any meaning.
|
||||
settings["DOWNLOAD_DELAY"] = 0
|
||||
crawler = get_crawler(FollowAllSpider, settings)
|
||||
yield crawler.crawl(**crawl_kwargs)
|
||||
await maybe_deferred_to_future(crawler.crawl(**crawl_kwargs))
|
||||
assert crawler.spider
|
||||
assert isinstance(crawler.spider, FollowAllSpider)
|
||||
times = crawler.spider.times
|
||||
total_time = times[-1] - times[0]
|
||||
average = total_time / (len(times) - 1)
|
||||
|
|
@ -428,8 +439,9 @@ class TestCrawlSpider(TestCase):
|
|||
def tearDownClass(cls):
|
||||
cls.mockserver.__exit__(None, None, None)
|
||||
|
||||
@inlineCallbacks
|
||||
def _run_spider(self, spider_cls):
|
||||
async def _run_spider(
|
||||
self, spider_cls: type[Spider]
|
||||
) -> tuple[LogCapture, list[Any], StatsCollector]:
|
||||
items = []
|
||||
|
||||
def _on_item_scraped(item):
|
||||
|
|
@ -438,9 +450,12 @@ class TestCrawlSpider(TestCase):
|
|||
crawler = get_crawler(spider_cls)
|
||||
crawler.signals.connect(_on_item_scraped, signals.item_scraped)
|
||||
with LogCapture() as log:
|
||||
yield crawler.crawl(
|
||||
self.mockserver.url("/status?n=200"), mockserver=self.mockserver
|
||||
await maybe_deferred_to_future(
|
||||
crawler.crawl(
|
||||
self.mockserver.url("/status?n=200"), mockserver=self.mockserver
|
||||
)
|
||||
)
|
||||
assert crawler.stats
|
||||
return log, items, crawler.stats
|
||||
|
||||
@inlineCallbacks
|
||||
|
|
@ -521,9 +536,9 @@ class TestCrawlSpider(TestCase):
|
|||
assert "Got response 200" in str(log)
|
||||
|
||||
@pytest.mark.only_asyncio
|
||||
@inlineCallbacks
|
||||
def test_async_def_asyncio_parse_items_list(self):
|
||||
log, items, _ = yield self._run_spider(AsyncDefAsyncioReturnSpider)
|
||||
@deferred_f_from_coro_f
|
||||
async def test_async_def_asyncio_parse_items_list(self):
|
||||
log, items, _ = await self._run_spider(AsyncDefAsyncioReturnSpider)
|
||||
assert "Got response 200" in str(log)
|
||||
assert {"id": 1} in items
|
||||
assert {"id": 2} in items
|
||||
|
|
@ -546,17 +561,17 @@ class TestCrawlSpider(TestCase):
|
|||
assert {"foo": 42} in items
|
||||
|
||||
@pytest.mark.only_asyncio
|
||||
@inlineCallbacks
|
||||
def test_async_def_asyncgen_parse(self):
|
||||
log, _, stats = yield self._run_spider(AsyncDefAsyncioGenSpider)
|
||||
@deferred_f_from_coro_f
|
||||
async def test_async_def_asyncgen_parse(self):
|
||||
log, _, stats = await self._run_spider(AsyncDefAsyncioGenSpider)
|
||||
assert "Got response 200" in str(log)
|
||||
itemcount = stats.get_value("item_scraped_count")
|
||||
assert itemcount == 1
|
||||
|
||||
@pytest.mark.only_asyncio
|
||||
@inlineCallbacks
|
||||
def test_async_def_asyncgen_parse_loop(self):
|
||||
log, items, stats = yield self._run_spider(AsyncDefAsyncioGenLoopSpider)
|
||||
@deferred_f_from_coro_f
|
||||
async def test_async_def_asyncgen_parse_loop(self):
|
||||
log, items, stats = await self._run_spider(AsyncDefAsyncioGenLoopSpider)
|
||||
assert "Got response 200" in str(log)
|
||||
itemcount = stats.get_value("item_scraped_count")
|
||||
assert itemcount == 10
|
||||
|
|
@ -564,9 +579,9 @@ class TestCrawlSpider(TestCase):
|
|||
assert {"foo": i} in items
|
||||
|
||||
@pytest.mark.only_asyncio
|
||||
@inlineCallbacks
|
||||
def test_async_def_asyncgen_parse_exc(self):
|
||||
log, items, stats = yield self._run_spider(AsyncDefAsyncioGenExcSpider)
|
||||
@deferred_f_from_coro_f
|
||||
async def test_async_def_asyncgen_parse_exc(self):
|
||||
log, items, stats = await self._run_spider(AsyncDefAsyncioGenExcSpider)
|
||||
log = str(log)
|
||||
assert "Spider error processing" in log
|
||||
assert "ValueError" in log
|
||||
|
|
@ -576,9 +591,9 @@ class TestCrawlSpider(TestCase):
|
|||
assert {"foo": i} in items
|
||||
|
||||
@pytest.mark.only_asyncio
|
||||
@inlineCallbacks
|
||||
def test_async_def_asyncgen_parse_complex(self):
|
||||
_, items, stats = yield self._run_spider(AsyncDefAsyncioGenComplexSpider)
|
||||
@deferred_f_from_coro_f
|
||||
async def test_async_def_asyncgen_parse_complex(self):
|
||||
_, items, stats = await self._run_spider(AsyncDefAsyncioGenComplexSpider)
|
||||
itemcount = stats.get_value("item_scraped_count")
|
||||
assert itemcount == 156
|
||||
# some random items
|
||||
|
|
@ -588,27 +603,27 @@ class TestCrawlSpider(TestCase):
|
|||
assert {"index2": i} in items
|
||||
|
||||
@pytest.mark.only_asyncio
|
||||
@inlineCallbacks
|
||||
def test_async_def_asyncio_parse_reqs_list(self):
|
||||
log, *_ = yield self._run_spider(AsyncDefAsyncioReqsReturnSpider)
|
||||
@deferred_f_from_coro_f
|
||||
async def test_async_def_asyncio_parse_reqs_list(self):
|
||||
log, *_ = await self._run_spider(AsyncDefAsyncioReqsReturnSpider)
|
||||
for req_id in range(3):
|
||||
assert f"Got response 200, req_id {req_id}" in str(log)
|
||||
|
||||
@pytest.mark.only_not_asyncio
|
||||
@inlineCallbacks
|
||||
def test_async_def_deferred_direct(self):
|
||||
_, items, _ = yield self._run_spider(AsyncDefDeferredDirectSpider)
|
||||
@deferred_f_from_coro_f
|
||||
async def test_async_def_deferred_direct(self):
|
||||
_, items, _ = await self._run_spider(AsyncDefDeferredDirectSpider)
|
||||
assert items == [{"code": 200}]
|
||||
|
||||
@pytest.mark.only_asyncio
|
||||
@inlineCallbacks
|
||||
def test_async_def_deferred_wrapped(self):
|
||||
log, items, _ = yield self._run_spider(AsyncDefDeferredWrappedSpider)
|
||||
@deferred_f_from_coro_f
|
||||
async def test_async_def_deferred_wrapped(self):
|
||||
log, items, _ = await self._run_spider(AsyncDefDeferredWrappedSpider)
|
||||
assert items == [{"code": 200}]
|
||||
|
||||
@inlineCallbacks
|
||||
def test_async_def_deferred_maybe_wrapped(self):
|
||||
_, items, _ = yield self._run_spider(AsyncDefDeferredMaybeWrappedSpider)
|
||||
@deferred_f_from_coro_f
|
||||
async def test_async_def_deferred_maybe_wrapped(self):
|
||||
_, items, _ = await self._run_spider(AsyncDefDeferredMaybeWrappedSpider)
|
||||
assert items == [{"code": 200}]
|
||||
|
||||
@inlineCallbacks
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load Diff
|
|
@ -8,7 +8,6 @@ from unittest import mock
|
|||
import pytest
|
||||
from testfixtures import LogCapture
|
||||
from twisted.internet import defer
|
||||
from twisted.internet.defer import inlineCallbacks
|
||||
from twisted.trial.unittest import TestCase
|
||||
|
||||
from scrapy.core.spidermw import SpiderMiddlewareManager
|
||||
|
|
@ -18,7 +17,6 @@ from scrapy.spiders import Spider
|
|||
from scrapy.utils.asyncgen import collect_asyncgen
|
||||
from scrapy.utils.defer import (
|
||||
deferred_f_from_coro_f,
|
||||
deferred_from_coro,
|
||||
maybe_deferred_to_future,
|
||||
)
|
||||
from scrapy.utils.test import get_crawler
|
||||
|
|
@ -130,25 +128,22 @@ class TestBaseAsyncSpiderMiddleware(TestSpiderMiddleware):
|
|||
yield {"foo": 2}
|
||||
yield {"foo": 3}
|
||||
|
||||
@inlineCallbacks
|
||||
def _get_middleware_result(self, *mw_classes, start_index: int | None = None):
|
||||
async def _get_middleware_result(self, *mw_classes, start_index: int | None = None):
|
||||
setting = self._construct_mw_setting(*mw_classes, start_index=start_index)
|
||||
self.crawler = get_crawler(
|
||||
Spider, {"SPIDER_MIDDLEWARES_BASE": {}, "SPIDER_MIDDLEWARES": setting}
|
||||
)
|
||||
self.spider = self.crawler._create_spider("foo")
|
||||
self.mwman = SpiderMiddlewareManager.from_crawler(self.crawler)
|
||||
result = yield self.mwman.scrape_response(
|
||||
return await self.mwman.scrape_response_async(
|
||||
self._scrape_func, self.response, self.request, self.spider
|
||||
)
|
||||
return result
|
||||
|
||||
@inlineCallbacks
|
||||
def _test_simple_base(
|
||||
async def _test_simple_base(
|
||||
self, *mw_classes, downgrade: bool = False, start_index: int | None = None
|
||||
):
|
||||
with LogCapture() as log:
|
||||
result = yield self._get_middleware_result(
|
||||
result = await self._get_middleware_result(
|
||||
*mw_classes, start_index=start_index
|
||||
)
|
||||
assert isinstance(result, Iterable)
|
||||
|
|
@ -160,16 +155,15 @@ class TestBaseAsyncSpiderMiddleware(TestSpiderMiddleware):
|
|||
ProcessSpiderOutputSimpleMiddleware in mw_classes
|
||||
)
|
||||
|
||||
@inlineCallbacks
|
||||
def _test_asyncgen_base(
|
||||
async def _test_asyncgen_base(
|
||||
self, *mw_classes, downgrade: bool = False, start_index: int | None = None
|
||||
):
|
||||
with LogCapture() as log:
|
||||
result = yield self._get_middleware_result(
|
||||
result = await self._get_middleware_result(
|
||||
*mw_classes, start_index=start_index
|
||||
)
|
||||
assert isinstance(result, AsyncIterator)
|
||||
result_list = yield deferred_from_coro(collect_asyncgen(result))
|
||||
result_list = await collect_asyncgen(result)
|
||||
assert len(result_list) == self.RESULT_COUNT
|
||||
assert isinstance(result_list[0], self.ITEM_TYPE)
|
||||
assert ("downgraded to a non-async" in str(log)) == downgrade
|
||||
|
|
@ -222,41 +216,50 @@ class TestProcessSpiderOutputSimple(TestBaseAsyncSpiderMiddleware):
|
|||
MW_ASYNCGEN = ProcessSpiderOutputAsyncGenMiddleware
|
||||
MW_UNIVERSAL = ProcessSpiderOutputUniversalMiddleware
|
||||
|
||||
def test_simple(self):
|
||||
@deferred_f_from_coro_f
|
||||
async def test_simple(self):
|
||||
"""Simple mw"""
|
||||
return self._test_simple_base(self.MW_SIMPLE)
|
||||
await self._test_simple_base(self.MW_SIMPLE)
|
||||
|
||||
def test_asyncgen(self):
|
||||
@deferred_f_from_coro_f
|
||||
async def test_asyncgen(self):
|
||||
"""Asyncgen mw; upgrade"""
|
||||
return self._test_asyncgen_base(self.MW_ASYNCGEN)
|
||||
await self._test_asyncgen_base(self.MW_ASYNCGEN)
|
||||
|
||||
def test_simple_asyncgen(self):
|
||||
@deferred_f_from_coro_f
|
||||
async def test_simple_asyncgen(self):
|
||||
"""Simple mw -> asyncgen mw; upgrade"""
|
||||
return self._test_asyncgen_base(self.MW_ASYNCGEN, self.MW_SIMPLE)
|
||||
await self._test_asyncgen_base(self.MW_ASYNCGEN, self.MW_SIMPLE)
|
||||
|
||||
def test_asyncgen_simple(self):
|
||||
@deferred_f_from_coro_f
|
||||
async def test_asyncgen_simple(self):
|
||||
"""Asyncgen mw -> simple mw; upgrade then downgrade"""
|
||||
return self._test_simple_base(self.MW_SIMPLE, self.MW_ASYNCGEN, downgrade=True)
|
||||
await self._test_simple_base(self.MW_SIMPLE, self.MW_ASYNCGEN, downgrade=True)
|
||||
|
||||
def test_universal(self):
|
||||
@deferred_f_from_coro_f
|
||||
async def test_universal(self):
|
||||
"""Universal mw"""
|
||||
return self._test_simple_base(self.MW_UNIVERSAL)
|
||||
await self._test_simple_base(self.MW_UNIVERSAL)
|
||||
|
||||
def test_universal_simple(self):
|
||||
@deferred_f_from_coro_f
|
||||
async def test_universal_simple(self):
|
||||
"""Universal mw -> simple mw"""
|
||||
return self._test_simple_base(self.MW_SIMPLE, self.MW_UNIVERSAL)
|
||||
await self._test_simple_base(self.MW_SIMPLE, self.MW_UNIVERSAL)
|
||||
|
||||
def test_simple_universal(self):
|
||||
@deferred_f_from_coro_f
|
||||
async def test_simple_universal(self):
|
||||
"""Simple mw -> universal mw"""
|
||||
return self._test_simple_base(self.MW_UNIVERSAL, self.MW_SIMPLE)
|
||||
await self._test_simple_base(self.MW_UNIVERSAL, self.MW_SIMPLE)
|
||||
|
||||
def test_universal_asyncgen(self):
|
||||
@deferred_f_from_coro_f
|
||||
async def test_universal_asyncgen(self):
|
||||
"""Universal mw -> asyncgen mw; upgrade"""
|
||||
return self._test_asyncgen_base(self.MW_ASYNCGEN, self.MW_UNIVERSAL)
|
||||
await self._test_asyncgen_base(self.MW_ASYNCGEN, self.MW_UNIVERSAL)
|
||||
|
||||
def test_asyncgen_universal(self):
|
||||
@deferred_f_from_coro_f
|
||||
async def test_asyncgen_universal(self):
|
||||
"""Asyncgen mw -> universal mw; upgrade"""
|
||||
return self._test_asyncgen_base(self.MW_UNIVERSAL, self.MW_ASYNCGEN)
|
||||
await self._test_asyncgen_base(self.MW_UNIVERSAL, self.MW_ASYNCGEN)
|
||||
|
||||
|
||||
class TestProcessSpiderOutputAsyncGen(TestProcessSpiderOutputSimple):
|
||||
|
|
@ -266,27 +269,30 @@ class TestProcessSpiderOutputAsyncGen(TestProcessSpiderOutputSimple):
|
|||
for item in super()._scrape_func():
|
||||
yield item
|
||||
|
||||
def test_simple(self):
|
||||
@deferred_f_from_coro_f
|
||||
async def test_simple(self):
|
||||
"""Simple mw; downgrade"""
|
||||
return self._test_simple_base(self.MW_SIMPLE, downgrade=True)
|
||||
await self._test_simple_base(self.MW_SIMPLE, downgrade=True)
|
||||
|
||||
def test_simple_asyncgen(self):
|
||||
@deferred_f_from_coro_f
|
||||
async def test_simple_asyncgen(self):
|
||||
"""Simple mw -> asyncgen mw; downgrade then upgrade"""
|
||||
return self._test_asyncgen_base(
|
||||
self.MW_ASYNCGEN, self.MW_SIMPLE, downgrade=True
|
||||
)
|
||||
await self._test_asyncgen_base(self.MW_ASYNCGEN, self.MW_SIMPLE, downgrade=True)
|
||||
|
||||
def test_universal(self):
|
||||
@deferred_f_from_coro_f
|
||||
async def test_universal(self):
|
||||
"""Universal mw"""
|
||||
return self._test_asyncgen_base(self.MW_UNIVERSAL)
|
||||
await self._test_asyncgen_base(self.MW_UNIVERSAL)
|
||||
|
||||
def test_universal_simple(self):
|
||||
@deferred_f_from_coro_f
|
||||
async def test_universal_simple(self):
|
||||
"""Universal mw -> simple mw; downgrade"""
|
||||
return self._test_simple_base(self.MW_SIMPLE, self.MW_UNIVERSAL, downgrade=True)
|
||||
await self._test_simple_base(self.MW_SIMPLE, self.MW_UNIVERSAL, downgrade=True)
|
||||
|
||||
def test_simple_universal(self):
|
||||
@deferred_f_from_coro_f
|
||||
async def test_simple_universal(self):
|
||||
"""Simple mw -> universal mw; downgrade"""
|
||||
return self._test_simple_base(self.MW_UNIVERSAL, self.MW_SIMPLE, downgrade=True)
|
||||
await self._test_simple_base(self.MW_UNIVERSAL, self.MW_SIMPLE, downgrade=True)
|
||||
|
||||
|
||||
class ProcessSpiderOutputNonIterableMiddleware:
|
||||
|
|
@ -300,25 +306,21 @@ class ProcessSpiderOutputCoroutineMiddleware:
|
|||
|
||||
|
||||
class TestProcessSpiderOutputInvalidResult(TestBaseAsyncSpiderMiddleware):
|
||||
@inlineCallbacks
|
||||
def test_non_iterable(self):
|
||||
@deferred_f_from_coro_f
|
||||
async def test_non_iterable(self):
|
||||
with pytest.raises(
|
||||
_InvalidOutput,
|
||||
match=r"\.process_spider_output must return an iterable, got <class 'NoneType'>",
|
||||
):
|
||||
yield self._get_middleware_result(
|
||||
ProcessSpiderOutputNonIterableMiddleware,
|
||||
)
|
||||
await self._get_middleware_result(ProcessSpiderOutputNonIterableMiddleware)
|
||||
|
||||
@inlineCallbacks
|
||||
def test_coroutine(self):
|
||||
@deferred_f_from_coro_f
|
||||
async def test_coroutine(self):
|
||||
with pytest.raises(
|
||||
_InvalidOutput,
|
||||
match=r"\.process_spider_output must be an asynchronous generator",
|
||||
):
|
||||
yield self._get_middleware_result(
|
||||
ProcessSpiderOutputCoroutineMiddleware,
|
||||
)
|
||||
await self._get_middleware_result(ProcessSpiderOutputCoroutineMiddleware)
|
||||
|
||||
|
||||
class ProcessStartSimpleMiddleware:
|
||||
|
|
@ -445,39 +447,44 @@ class TestBuiltinMiddlewareSimple(TestBaseAsyncSpiderMiddleware):
|
|||
MW_ASYNCGEN = ProcessSpiderOutputAsyncGenMiddleware
|
||||
MW_UNIVERSAL = ProcessSpiderOutputUniversalMiddleware
|
||||
|
||||
@inlineCallbacks
|
||||
def _get_middleware_result(self, *mw_classes, start_index: int | None = None):
|
||||
async def _get_middleware_result(self, *mw_classes, start_index: int | None = None):
|
||||
setting = self._construct_mw_setting(*mw_classes, start_index=start_index)
|
||||
self.crawler = get_crawler(Spider, {"SPIDER_MIDDLEWARES": setting})
|
||||
self.spider = self.crawler._create_spider("foo")
|
||||
self.mwman = SpiderMiddlewareManager.from_crawler(self.crawler)
|
||||
result = yield self.mwman.scrape_response(
|
||||
return await self.mwman.scrape_response_async(
|
||||
self._scrape_func, self.response, self.request, self.spider
|
||||
)
|
||||
return result
|
||||
|
||||
def test_just_builtin(self):
|
||||
return self._test_simple_base()
|
||||
@deferred_f_from_coro_f
|
||||
async def test_just_builtin(self):
|
||||
await self._test_simple_base()
|
||||
|
||||
def test_builtin_simple(self):
|
||||
return self._test_simple_base(self.MW_SIMPLE, start_index=1000)
|
||||
@deferred_f_from_coro_f
|
||||
async def test_builtin_simple(self):
|
||||
await self._test_simple_base(self.MW_SIMPLE, start_index=1000)
|
||||
|
||||
def test_builtin_async(self):
|
||||
@deferred_f_from_coro_f
|
||||
async def test_builtin_async(self):
|
||||
"""Upgrade"""
|
||||
return self._test_asyncgen_base(self.MW_ASYNCGEN, start_index=1000)
|
||||
await self._test_asyncgen_base(self.MW_ASYNCGEN, start_index=1000)
|
||||
|
||||
def test_builtin_universal(self):
|
||||
return self._test_simple_base(self.MW_UNIVERSAL, start_index=1000)
|
||||
@deferred_f_from_coro_f
|
||||
async def test_builtin_universal(self):
|
||||
await self._test_simple_base(self.MW_UNIVERSAL, start_index=1000)
|
||||
|
||||
def test_simple_builtin(self):
|
||||
return self._test_simple_base(self.MW_SIMPLE)
|
||||
@deferred_f_from_coro_f
|
||||
async def test_simple_builtin(self):
|
||||
await self._test_simple_base(self.MW_SIMPLE)
|
||||
|
||||
def test_async_builtin(self):
|
||||
@deferred_f_from_coro_f
|
||||
async def test_async_builtin(self):
|
||||
"""Upgrade"""
|
||||
return self._test_asyncgen_base(self.MW_ASYNCGEN)
|
||||
await self._test_asyncgen_base(self.MW_ASYNCGEN)
|
||||
|
||||
def test_universal_builtin(self):
|
||||
return self._test_simple_base(self.MW_UNIVERSAL)
|
||||
@deferred_f_from_coro_f
|
||||
async def test_universal_builtin(self):
|
||||
await self._test_simple_base(self.MW_UNIVERSAL)
|
||||
|
||||
|
||||
class TestBuiltinMiddlewareAsyncGen(TestBuiltinMiddlewareSimple):
|
||||
|
|
@ -485,28 +492,35 @@ class TestBuiltinMiddlewareAsyncGen(TestBuiltinMiddlewareSimple):
|
|||
for item in super()._scrape_func():
|
||||
yield item
|
||||
|
||||
def test_just_builtin(self):
|
||||
return self._test_asyncgen_base()
|
||||
@deferred_f_from_coro_f
|
||||
async def test_just_builtin(self):
|
||||
await self._test_asyncgen_base()
|
||||
|
||||
def test_builtin_simple(self):
|
||||
@deferred_f_from_coro_f
|
||||
async def test_builtin_simple(self):
|
||||
"""Downgrade"""
|
||||
return self._test_simple_base(self.MW_SIMPLE, downgrade=True, start_index=1000)
|
||||
await self._test_simple_base(self.MW_SIMPLE, downgrade=True, start_index=1000)
|
||||
|
||||
def test_builtin_async(self):
|
||||
return self._test_asyncgen_base(self.MW_ASYNCGEN, start_index=1000)
|
||||
@deferred_f_from_coro_f
|
||||
async def test_builtin_async(self):
|
||||
await self._test_asyncgen_base(self.MW_ASYNCGEN, start_index=1000)
|
||||
|
||||
def test_builtin_universal(self):
|
||||
return self._test_asyncgen_base(self.MW_UNIVERSAL, start_index=1000)
|
||||
@deferred_f_from_coro_f
|
||||
async def test_builtin_universal(self):
|
||||
await self._test_asyncgen_base(self.MW_UNIVERSAL, start_index=1000)
|
||||
|
||||
def test_simple_builtin(self):
|
||||
@deferred_f_from_coro_f
|
||||
async def test_simple_builtin(self):
|
||||
"""Downgrade"""
|
||||
return self._test_simple_base(self.MW_SIMPLE, downgrade=True)
|
||||
await self._test_simple_base(self.MW_SIMPLE, downgrade=True)
|
||||
|
||||
def test_async_builtin(self):
|
||||
return self._test_asyncgen_base(self.MW_ASYNCGEN)
|
||||
@deferred_f_from_coro_f
|
||||
async def test_async_builtin(self):
|
||||
await self._test_asyncgen_base(self.MW_ASYNCGEN)
|
||||
|
||||
def test_universal_builtin(self):
|
||||
return self._test_asyncgen_base(self.MW_UNIVERSAL)
|
||||
@deferred_f_from_coro_f
|
||||
async def test_universal_builtin(self):
|
||||
await self._test_asyncgen_base(self.MW_UNIVERSAL)
|
||||
|
||||
|
||||
class TestProcessSpiderException(TestBaseAsyncSpiderMiddleware):
|
||||
|
|
@ -520,33 +534,38 @@ class TestProcessSpiderException(TestBaseAsyncSpiderMiddleware):
|
|||
def _scrape_func(self, *args, **kwargs):
|
||||
1 / 0
|
||||
|
||||
@inlineCallbacks
|
||||
def _test_asyncgen_nodowngrade(self, *mw_classes):
|
||||
async def _test_asyncgen_nodowngrade(self, *mw_classes):
|
||||
with pytest.raises(
|
||||
_InvalidOutput, match="Async iterable returned from .+ cannot be downgraded"
|
||||
):
|
||||
yield self._get_middleware_result(*mw_classes)
|
||||
await self._get_middleware_result(*mw_classes)
|
||||
|
||||
def test_exc_simple(self):
|
||||
@deferred_f_from_coro_f
|
||||
async def test_exc_simple(self):
|
||||
"""Simple exc mw"""
|
||||
return self._test_simple_base(self.MW_EXC_SIMPLE)
|
||||
await self._test_simple_base(self.MW_EXC_SIMPLE)
|
||||
|
||||
def test_exc_async(self):
|
||||
@deferred_f_from_coro_f
|
||||
async def test_exc_async(self):
|
||||
"""Async exc mw"""
|
||||
return self._test_asyncgen_base(self.MW_EXC_ASYNCGEN)
|
||||
await self._test_asyncgen_base(self.MW_EXC_ASYNCGEN)
|
||||
|
||||
def test_exc_simple_simple(self):
|
||||
@deferred_f_from_coro_f
|
||||
async def test_exc_simple_simple(self):
|
||||
"""Simple exc mw -> simple output mw"""
|
||||
return self._test_simple_base(self.MW_SIMPLE, self.MW_EXC_SIMPLE)
|
||||
await self._test_simple_base(self.MW_SIMPLE, self.MW_EXC_SIMPLE)
|
||||
|
||||
def test_exc_async_async(self):
|
||||
@deferred_f_from_coro_f
|
||||
async def test_exc_async_async(self):
|
||||
"""Async exc mw -> async output mw"""
|
||||
return self._test_asyncgen_base(self.MW_ASYNCGEN, self.MW_EXC_ASYNCGEN)
|
||||
await self._test_asyncgen_base(self.MW_ASYNCGEN, self.MW_EXC_ASYNCGEN)
|
||||
|
||||
def test_exc_simple_async(self):
|
||||
@deferred_f_from_coro_f
|
||||
async def test_exc_simple_async(self):
|
||||
"""Simple exc mw -> async output mw; upgrade"""
|
||||
return self._test_asyncgen_base(self.MW_ASYNCGEN, self.MW_EXC_SIMPLE)
|
||||
await self._test_asyncgen_base(self.MW_ASYNCGEN, self.MW_EXC_SIMPLE)
|
||||
|
||||
def test_exc_async_simple(self):
|
||||
@deferred_f_from_coro_f
|
||||
async def test_exc_async_simple(self):
|
||||
"""Async exc mw -> simple output mw; cannot work as downgrading is not supported"""
|
||||
return self._test_asyncgen_nodowngrade(self.MW_SIMPLE, self.MW_EXC_ASYNCGEN)
|
||||
await self._test_asyncgen_nodowngrade(self.MW_SIMPLE, self.MW_EXC_ASYNCGEN)
|
||||
|
|
|
|||
|
|
@ -1,8 +1,8 @@
|
|||
from testfixtures import LogCapture
|
||||
from twisted.internet.defer import inlineCallbacks
|
||||
from twisted.trial.unittest import TestCase
|
||||
|
||||
from scrapy import Request, Spider
|
||||
from scrapy.utils.defer import deferred_f_from_coro_f, maybe_deferred_to_future
|
||||
from scrapy.utils.test import get_crawler
|
||||
from tests.mockserver import MockServer
|
||||
|
||||
|
|
@ -299,6 +299,8 @@ class NotGeneratorOutputChainSpider(Spider):
|
|||
|
||||
# ================================================================================
|
||||
class TestSpiderMiddleware(TestCase):
|
||||
mockserver: MockServer
|
||||
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls.mockserver = MockServer()
|
||||
|
|
@ -308,53 +310,52 @@ class TestSpiderMiddleware(TestCase):
|
|||
def tearDownClass(cls):
|
||||
cls.mockserver.__exit__(None, None, None)
|
||||
|
||||
@inlineCallbacks
|
||||
def crawl_log(self, spider):
|
||||
async def crawl_log(self, spider: type[Spider]) -> LogCapture:
|
||||
crawler = get_crawler(spider)
|
||||
with LogCapture() as log:
|
||||
yield crawler.crawl(mockserver=self.mockserver)
|
||||
await maybe_deferred_to_future(crawler.crawl(mockserver=self.mockserver))
|
||||
return log
|
||||
|
||||
@inlineCallbacks
|
||||
def test_recovery(self):
|
||||
@deferred_f_from_coro_f
|
||||
async def test_recovery(self):
|
||||
"""
|
||||
(0) Recover from an exception in a spider callback. The final item count should be 3
|
||||
(one yielded from the callback method before the exception is raised, one directly
|
||||
from the recovery middleware and one from the spider when processing the request that
|
||||
was enqueued from the recovery middleware)
|
||||
"""
|
||||
log = yield self.crawl_log(RecoverySpider)
|
||||
log = await self.crawl_log(RecoverySpider)
|
||||
assert "Middleware: TabError exception caught" in str(log)
|
||||
assert str(log).count("Middleware: TabError exception caught") == 1
|
||||
assert "'item_scraped_count': 3" in str(log)
|
||||
|
||||
@inlineCallbacks
|
||||
def test_recovery_asyncgen(self):
|
||||
@deferred_f_from_coro_f
|
||||
async def test_recovery_asyncgen(self):
|
||||
"""
|
||||
Same as test_recovery but with an async callback.
|
||||
"""
|
||||
log = yield self.crawl_log(RecoveryAsyncGenSpider)
|
||||
log = await self.crawl_log(RecoveryAsyncGenSpider)
|
||||
assert "Middleware: TabError exception caught" in str(log)
|
||||
assert str(log).count("Middleware: TabError exception caught") == 1
|
||||
assert "'item_scraped_count': 3" in str(log)
|
||||
|
||||
@inlineCallbacks
|
||||
def test_process_spider_input_without_errback(self):
|
||||
@deferred_f_from_coro_f
|
||||
async def test_process_spider_input_without_errback(self):
|
||||
"""
|
||||
(1.1) An exception from the process_spider_input chain should be caught by the
|
||||
process_spider_exception chain from the start if the Request has no errback
|
||||
"""
|
||||
log1 = yield self.crawl_log(ProcessSpiderInputSpiderWithoutErrback)
|
||||
log1 = await self.crawl_log(ProcessSpiderInputSpiderWithoutErrback)
|
||||
assert "Middleware: will raise IndexError" in str(log1)
|
||||
assert "Middleware: IndexError exception caught" in str(log1)
|
||||
|
||||
@inlineCallbacks
|
||||
def test_process_spider_input_with_errback(self):
|
||||
@deferred_f_from_coro_f
|
||||
async def test_process_spider_input_with_errback(self):
|
||||
"""
|
||||
(1.2) An exception from the process_spider_input chain should not be caught by the
|
||||
process_spider_exception chain if the Request has an errback
|
||||
"""
|
||||
log1 = yield self.crawl_log(ProcessSpiderInputSpiderWithErrback)
|
||||
log1 = await self.crawl_log(ProcessSpiderInputSpiderWithErrback)
|
||||
assert "Middleware: IndexError exception caught" not in str(log1)
|
||||
assert "Middleware: will raise IndexError" in str(log1)
|
||||
assert "Got a Failure on the Request errback" in str(log1)
|
||||
|
|
@ -362,60 +363,60 @@ class TestSpiderMiddleware(TestCase):
|
|||
assert "{'from': 'callback'}" not in str(log1)
|
||||
assert "'item_scraped_count': 1" in str(log1)
|
||||
|
||||
@inlineCallbacks
|
||||
def test_generator_callback(self):
|
||||
@deferred_f_from_coro_f
|
||||
async def test_generator_callback(self):
|
||||
"""
|
||||
(2) An exception from a spider callback (returning a generator) should
|
||||
be caught by the process_spider_exception chain. Items yielded before the
|
||||
exception is raised should be processed normally.
|
||||
"""
|
||||
log2 = yield self.crawl_log(GeneratorCallbackSpider)
|
||||
log2 = await self.crawl_log(GeneratorCallbackSpider)
|
||||
assert "Middleware: ImportError exception caught" in str(log2)
|
||||
assert "'item_scraped_count': 2" in str(log2)
|
||||
|
||||
@inlineCallbacks
|
||||
def test_async_generator_callback(self):
|
||||
@deferred_f_from_coro_f
|
||||
async def test_async_generator_callback(self):
|
||||
"""
|
||||
Same as test_generator_callback but with an async callback.
|
||||
"""
|
||||
log2 = yield self.crawl_log(AsyncGeneratorCallbackSpider)
|
||||
log2 = await self.crawl_log(AsyncGeneratorCallbackSpider)
|
||||
assert "Middleware: ImportError exception caught" in str(log2)
|
||||
assert "'item_scraped_count': 2" in str(log2)
|
||||
|
||||
@inlineCallbacks
|
||||
def test_generator_callback_right_after_callback(self):
|
||||
@deferred_f_from_coro_f
|
||||
async def test_generator_callback_right_after_callback(self):
|
||||
"""
|
||||
(2.1) Special case of (2): Exceptions should be caught
|
||||
even if the middleware is placed right after the spider
|
||||
"""
|
||||
log21 = yield self.crawl_log(GeneratorCallbackSpiderMiddlewareRightAfterSpider)
|
||||
log21 = await self.crawl_log(GeneratorCallbackSpiderMiddlewareRightAfterSpider)
|
||||
assert "Middleware: ImportError exception caught" in str(log21)
|
||||
assert "'item_scraped_count': 2" in str(log21)
|
||||
|
||||
@inlineCallbacks
|
||||
def test_not_a_generator_callback(self):
|
||||
@deferred_f_from_coro_f
|
||||
async def test_not_a_generator_callback(self):
|
||||
"""
|
||||
(3) An exception from a spider callback (returning a list) should
|
||||
be caught by the process_spider_exception chain. No items should be processed.
|
||||
"""
|
||||
log3 = yield self.crawl_log(NotGeneratorCallbackSpider)
|
||||
log3 = await self.crawl_log(NotGeneratorCallbackSpider)
|
||||
assert "Middleware: ZeroDivisionError exception caught" in str(log3)
|
||||
assert "item_scraped_count" not in str(log3)
|
||||
|
||||
@inlineCallbacks
|
||||
def test_not_a_generator_callback_right_after_callback(self):
|
||||
@deferred_f_from_coro_f
|
||||
async def test_not_a_generator_callback_right_after_callback(self):
|
||||
"""
|
||||
(3.1) Special case of (3): Exceptions should be caught
|
||||
even if the middleware is placed right after the spider
|
||||
"""
|
||||
log31 = yield self.crawl_log(
|
||||
log31 = await self.crawl_log(
|
||||
NotGeneratorCallbackSpiderMiddlewareRightAfterSpider
|
||||
)
|
||||
assert "Middleware: ZeroDivisionError exception caught" in str(log31)
|
||||
assert "item_scraped_count" not in str(log31)
|
||||
|
||||
@inlineCallbacks
|
||||
def test_generator_output_chain(self):
|
||||
@deferred_f_from_coro_f
|
||||
async def test_generator_output_chain(self):
|
||||
"""
|
||||
(4) An exception from a middleware's process_spider_output method should be sent
|
||||
to the process_spider_exception method from the next middleware in the chain.
|
||||
|
|
@ -424,7 +425,7 @@ class TestSpiderMiddleware(TestCase):
|
|||
The final item count should be 2 (one from the spider callback and one from the
|
||||
process_spider_exception chain)
|
||||
"""
|
||||
log4 = yield self.crawl_log(GeneratorOutputChainSpider)
|
||||
log4 = await self.crawl_log(GeneratorOutputChainSpider)
|
||||
assert "'item_scraped_count': 2" in str(log4)
|
||||
assert (
|
||||
"GeneratorRecoverMiddleware.process_spider_exception: LookupError caught"
|
||||
|
|
@ -461,8 +462,8 @@ class TestSpiderMiddleware(TestCase):
|
|||
assert str(item_recovered) in str(log4)
|
||||
assert "parse-second-item" not in str(log4)
|
||||
|
||||
@inlineCallbacks
|
||||
def test_not_a_generator_output_chain(self):
|
||||
@deferred_f_from_coro_f
|
||||
async def test_not_a_generator_output_chain(self):
|
||||
"""
|
||||
(5) An exception from a middleware's process_spider_output method should be sent
|
||||
to the process_spider_exception method from the next middleware in the chain.
|
||||
|
|
@ -471,7 +472,7 @@ class TestSpiderMiddleware(TestCase):
|
|||
The final item count should be 1 (from the process_spider_exception chain, the items
|
||||
from the spider callback are lost)
|
||||
"""
|
||||
log5 = yield self.crawl_log(NotGeneratorOutputChainSpider)
|
||||
log5 = await self.crawl_log(NotGeneratorOutputChainSpider)
|
||||
assert "'item_scraped_count': 1" in str(log5)
|
||||
assert (
|
||||
"GeneratorRecoverMiddleware.process_spider_exception: ReferenceError caught"
|
||||
|
|
|
|||
Loading…
Reference in New Issue