RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Remove tests for undefined behavior

This commit is contained in:
Adrián Chaves 2025-03-27 00:11:30 +01:00
parent 49b839b270
commit 581c969ead
1 changed files with 8 additions and 253 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

261
tests/test_engine_loop.py
View File

@ -4,7 +4,6 @@ from twisted.internet.defer import Deferred
from twisted.trial.unittest import TestCase
from scrapy import Request, Spider, signals
from scrapy.core.engine import ExecutionEngine
from scrapy.utils.defer import deferred_f_from_coro_f, maybe_deferred_to_future
from scrapy.utils.test import get_crawler
@ -27,7 +26,7 @@ class MainTestCase(TestCase):
the scheduler (returning no requests while also returning True from
the has_pending_requests() method) should cause the spider to miss the
processing of any later requests."""
seconds = ExecutionEngine._SLOT_HEARTBEAT_INTERVAL + 0.01
seconds = 2
class TestSpider(Spider):
name = "test"
@ -109,6 +108,8 @@ class RequestSendOrderTestCase(TestCase):
that can be sent before the first callback requests are scheduled.
"""
seconds = 0.1 # increase if flaky
@classmethod
def setUpClass(cls):
cls.mockserver = MockServer()
@ -116,10 +117,7 @@ class RequestSendOrderTestCase(TestCase):
@classmethod
def tearDownClass(cls):
cls.mockserver.__exit__(None, None, None)
fast_seconds = 0.001
slow_seconds = 2 # increase if flaky
cls.mockserver.__exit__(None, None, None) # increase if flaky
def _request(self, num, response_seconds, download_slots):
url = self.mockserver.url(f"/delay?n={response_seconds}&{num}")
@ -137,7 +135,7 @@ class RequestSendOrderTestCase(TestCase):
start_fn=None,
):
settings = settings or {}
response_seconds = response_seconds or self.slow_seconds
response_seconds = response_seconds or self.seconds
if start_fn is None:
@ -171,249 +169,6 @@ class RequestSendOrderTestCase(TestCase):
expected_nums = sorted(start_nums + cb_nums)
assert actual_nums == expected_nums, f"{actual_nums=} != {expected_nums=}"
@deferred_f_from_coro_f
async def test_default(self):
await maybe_deferred_to_future(
self._test_request_order(
start_nums=[
1,
2,
3,
4,
5,
6,
7,
8,
9,
10,
11,
12,
13,
14,
15,
16,
26,
24,
23,
22,
21,
20,
19,
18,
17,
],
cb_nums=[25],
)
)
@deferred_f_from_coro_f
async def test_conc1(self):
await maybe_deferred_to_future(
self._test_request_order(
start_nums=[1, 4, 2],
cb_nums=[3],
settings={"CONCURRENT_REQUESTS": 1},
)
)
@deferred_f_from_coro_f
async def test_conc2(self):
await maybe_deferred_to_future(
self._test_request_order(
start_nums=[1, 2, 6, 4, 3],
cb_nums=[5],
settings={"CONCURRENT_REQUESTS": 2},
)
)
@deferred_f_from_coro_f
async def test_conc8(self):
await maybe_deferred_to_future(
self._test_request_order(
start_nums=[1, 2, 3, 4, 5, 6, 7, 8, 18, 16, 15, 14, 13, 12, 11, 10, 9],
cb_nums=[17],
settings={"CONCURRENT_REQUESTS": 8},
)
)
@deferred_f_from_coro_f
async def test_conc16(self):
await maybe_deferred_to_future(
self._test_request_order(
start_nums=[
1,
2,
3,
4,
5,
6,
7,
8,
9,
10,
11,
12,
13,
14,
15,
16,
34,
32,
31,
30,
29,
28,
27,
26,
25,
24,
23,
22,
21,
20,
19,
18,
17,
],
cb_nums=[33],
settings={"CONCURRENT_REQUESTS_PER_DOMAIN": 16},
)
)
@deferred_f_from_coro_f
async def test_conc3_ds2(self):
await maybe_deferred_to_future(
self._test_request_order(
start_nums=[1, 2, 3, 8, 6, 5, 4],
cb_nums=[7],
settings={
"CONCURRENT_REQUESTS": 3,
},
download_slots=2,
)
)
@deferred_f_from_coro_f
async def test_tconc3_dconc2(self):
await maybe_deferred_to_future(
self._test_request_order(
start_nums=[1, 2, 3, 7, 5, 4],
cb_nums=[6],
settings={
"CONCURRENT_REQUESTS": 3,
"CONCURRENT_REQUESTS_PER_DOMAIN": 2,
},
)
)
@deferred_f_from_coro_f
async def test_tconc5_dconc3(self):
await maybe_deferred_to_future(
self._test_request_order(
start_nums=[1, 2, 3, 4, 5, 10, 8, 7, 6],
cb_nums=[9],
settings={
"CONCURRENT_REQUESTS": 5,
"CONCURRENT_REQUESTS_PER_DOMAIN": 3,
},
)
)
@deferred_f_from_coro_f
async def test_tconc5_dconc2_ds3(self):
await maybe_deferred_to_future(
self._test_request_order(
start_nums=[1, 2, 3, 4, 5, 12, 10, 9, 8, 7, 6],
cb_nums=[11],
settings={
"CONCURRENT_REQUESTS": 5,
"CONCURRENT_REQUESTS_PER_DOMAIN": 2,
},
download_slots=3,
)
)
@deferred_f_from_coro_f
async def test_tconc5_dconc3_ds2(self):
await maybe_deferred_to_future(
self._test_request_order(
start_nums=[1, 2, 3, 4, 5, 12, 10, 9, 8, 7, 6],
cb_nums=[11],
settings={
"CONCURRENT_REQUESTS": 5,
"CONCURRENT_REQUESTS_PER_DOMAIN": 3,
},
download_slots=2,
)
)
@deferred_f_from_coro_f
async def test_tconc7_dconc2_ds3(self):
await maybe_deferred_to_future(
self._test_request_order(
start_nums=[1, 2, 3, 4, 5, 6, 7, 15, 13, 12, 11, 10, 9, 8],
cb_nums=[14],
settings={
"CONCURRENT_REQUESTS": 7,
"CONCURRENT_REQUESTS_PER_DOMAIN": 2,
},
download_slots=3,
)
)
@deferred_f_from_coro_f
async def test_tconc7_dconc3_ds2(self):
await maybe_deferred_to_future(
self._test_request_order(
start_nums=[1, 2, 3, 4, 5, 6, 7, 15, 13, 12, 11, 10, 9, 8],
cb_nums=[14],
settings={
"CONCURRENT_REQUESTS": 7,
"CONCURRENT_REQUESTS_PER_DOMAIN": 3,
},
download_slots=2,
)
)
@deferred_f_from_coro_f
async def test_fast(self):
"""Very fast responses may increase the number of start requests sent
in reverse order before the first callback request."""
await maybe_deferred_to_future(
self._test_request_order(
start_nums=[1, 3, 2],
cb_nums=[4],
settings={"CONCURRENT_REQUESTS": 1},
response_seconds=self.fast_seconds,
)
)
@deferred_f_from_coro_f
async def test_await(self):
"""Awaiting slow operations in Spider.start() may lower the number of
first start requests sent in order."""
start_nums = [1, 3]
cb_nums = [2]
response_seconds = self.slow_seconds
download_slots = 1
async def start(spider):
assert len(start_nums) > 1
for num in start_nums[:-1]:
yield self._request(num, response_seconds, download_slots)
await sleep(response_seconds * 2)
yield self._request(start_nums[-1], response_seconds, download_slots)
await maybe_deferred_to_future(
self._test_request_order(
start_nums=start_nums,
cb_nums=cb_nums,
settings={"CONCURRENT_REQUESTS": 2},
response_seconds=response_seconds,
start_fn=start,
)
)
# Examples from the “Start requests” section of the documentation about
# spiders.
@ -421,7 +176,7 @@ class RequestSendOrderTestCase(TestCase):
async def test_start_requests_first(self):
start_nums = [1, 3, 2]
cb_nums = [4]
response_seconds = self.slow_seconds
response_seconds = self.seconds
download_slots = 1
async def start(spider):
@ -443,7 +198,7 @@ class RequestSendOrderTestCase(TestCase):
async def test_start_requests_first_sorted(self):
start_nums = [1, 2, 3]
cb_nums = [4]
response_seconds = self.slow_seconds
response_seconds = self.seconds
download_slots = 1
async def start(spider):
@ -467,7 +222,7 @@ class RequestSendOrderTestCase(TestCase):
async def test_lazy(self):
start_nums = [1, 2, 4]
cb_nums = [3]
response_seconds = self.slow_seconds
response_seconds = self.seconds
download_slots = 1
async def start(spider):