mirror of https://github.com/scrapy/scrapy.git
Update test expectations for the default reactor
This commit is contained in:
Adrián Chaves
parent
eddf582ba9
commit
406d2fb100
|
|
@ -1,5 +1,6 @@
|
|||
from collections import deque
|
||||
|
||||
import pytest
|
||||
from twisted.internet.defer import Deferred
|
||||
from twisted.trial.unittest import TestCase
|
||||
|
||||
|
|
@ -88,6 +89,8 @@ class RequestSendOrderTestCase(TestCase):
|
|||
It is a very unintuitive behavior, documented as “undefined” so that we may
|
||||
change it in the future without breaking the contract.
|
||||
|
||||
For the asyncio reactor:
|
||||
|
||||
1. First, the first CONCURRENT_REQUESTS start requests are sent in order.
|
||||
|
||||
Awaiting slow operations in Spider.start() can lower that.
|
||||
|
|
@ -102,6 +105,9 @@ class RequestSendOrderTestCase(TestCase):
|
|||
but only when there are not enough pending requests yielded from
|
||||
callbacks to reach the configured concurrency.
|
||||
|
||||
For the default Twisted reactor, step 1 sends the last CONCURRENT_REQUESTS
|
||||
start requests in reverse order instead.
|
||||
|
||||
The reverse order is because the scheduler uses a LIFO queue by default
|
||||
(SCHEDULER_MEMORY_QUEUE, SCHEDULER_DISK_QUEUE). The order of the first few
|
||||
requests is unnaffected because they are sent as soon as they are
|
||||
|
|
@ -171,8 +177,11 @@ class RequestSendOrderTestCase(TestCase):
|
|||
expected_nums = sorted(start_nums + cb_nums)
|
||||
assert actual_nums == expected_nums, f"{actual_nums=} != {expected_nums=}"
|
||||
|
||||
# Asyncio reactor behavior
|
||||
|
||||
@pytest.mark.only_asyncio
|
||||
@deferred_f_from_coro_f
|
||||
async def test_default(self):
|
||||
async def test_ar_default(self):
|
||||
await maybe_deferred_to_future(
|
||||
self._test_request_order(
|
||||
start_nums=[
|
||||
|
|
@ -206,8 +215,9 @@ class RequestSendOrderTestCase(TestCase):
|
|||
)
|
||||
)
|
||||
|
||||
@pytest.mark.only_asyncio
|
||||
@deferred_f_from_coro_f
|
||||
async def test_conc1(self):
|
||||
async def test_ar_conc1(self):
|
||||
await maybe_deferred_to_future(
|
||||
self._test_request_order(
|
||||
start_nums=[1, 4, 2],
|
||||
|
|
@ -216,8 +226,9 @@ class RequestSendOrderTestCase(TestCase):
|
|||
)
|
||||
)
|
||||
|
||||
@pytest.mark.only_asyncio
|
||||
@deferred_f_from_coro_f
|
||||
async def test_conc2(self):
|
||||
async def test_ar_conc2(self):
|
||||
await maybe_deferred_to_future(
|
||||
self._test_request_order(
|
||||
start_nums=[1, 2, 6, 4, 3],
|
||||
|
|
@ -226,8 +237,9 @@ class RequestSendOrderTestCase(TestCase):
|
|||
)
|
||||
)
|
||||
|
||||
@pytest.mark.only_asyncio
|
||||
@deferred_f_from_coro_f
|
||||
async def test_conc8(self):
|
||||
async def test_ar_conc8(self):
|
||||
await maybe_deferred_to_future(
|
||||
self._test_request_order(
|
||||
start_nums=[1, 2, 3, 4, 5, 6, 7, 8, 18, 16, 15, 14, 13, 12, 11, 10, 9],
|
||||
|
|
@ -236,8 +248,9 @@ class RequestSendOrderTestCase(TestCase):
|
|||
)
|
||||
)
|
||||
|
||||
@pytest.mark.only_asyncio
|
||||
@deferred_f_from_coro_f
|
||||
async def test_conc16(self):
|
||||
async def test_ar_conc16(self):
|
||||
await maybe_deferred_to_future(
|
||||
self._test_request_order(
|
||||
start_nums=[
|
||||
|
|
@ -280,8 +293,9 @@ class RequestSendOrderTestCase(TestCase):
|
|||
)
|
||||
)
|
||||
|
||||
@pytest.mark.only_asyncio
|
||||
@deferred_f_from_coro_f
|
||||
async def test_conc3_ds2(self):
|
||||
async def test_ar_conc3_ds2(self):
|
||||
await maybe_deferred_to_future(
|
||||
self._test_request_order(
|
||||
start_nums=[1, 2, 3, 8, 6, 5, 4],
|
||||
|
|
@ -293,8 +307,9 @@ class RequestSendOrderTestCase(TestCase):
|
|||
)
|
||||
)
|
||||
|
||||
@pytest.mark.only_asyncio
|
||||
@deferred_f_from_coro_f
|
||||
async def test_tconc3_dconc2(self):
|
||||
async def test_ar_tconc3_dconc2(self):
|
||||
await maybe_deferred_to_future(
|
||||
self._test_request_order(
|
||||
start_nums=[1, 2, 3, 7, 5, 4],
|
||||
|
|
@ -306,8 +321,9 @@ class RequestSendOrderTestCase(TestCase):
|
|||
)
|
||||
)
|
||||
|
||||
@pytest.mark.only_asyncio
|
||||
@deferred_f_from_coro_f
|
||||
async def test_tconc5_dconc3(self):
|
||||
async def test_ar_tconc5_dconc3(self):
|
||||
await maybe_deferred_to_future(
|
||||
self._test_request_order(
|
||||
start_nums=[1, 2, 3, 4, 5, 10, 8, 7, 6],
|
||||
|
|
@ -319,8 +335,9 @@ class RequestSendOrderTestCase(TestCase):
|
|||
)
|
||||
)
|
||||
|
||||
@pytest.mark.only_asyncio
|
||||
@deferred_f_from_coro_f
|
||||
async def test_tconc5_dconc2_ds3(self):
|
||||
async def test_ar_tconc5_dconc2_ds3(self):
|
||||
await maybe_deferred_to_future(
|
||||
self._test_request_order(
|
||||
start_nums=[1, 2, 3, 4, 5, 12, 10, 9, 8, 7, 6],
|
||||
|
|
@ -333,8 +350,9 @@ class RequestSendOrderTestCase(TestCase):
|
|||
)
|
||||
)
|
||||
|
||||
@pytest.mark.only_asyncio
|
||||
@deferred_f_from_coro_f
|
||||
async def test_tconc5_dconc3_ds2(self):
|
||||
async def test_ar_tconc5_dconc3_ds2(self):
|
||||
await maybe_deferred_to_future(
|
||||
self._test_request_order(
|
||||
start_nums=[1, 2, 3, 4, 5, 12, 10, 9, 8, 7, 6],
|
||||
|
|
@ -347,8 +365,9 @@ class RequestSendOrderTestCase(TestCase):
|
|||
)
|
||||
)
|
||||
|
||||
@pytest.mark.only_asyncio
|
||||
@deferred_f_from_coro_f
|
||||
async def test_tconc7_dconc2_ds3(self):
|
||||
async def test_ar_tconc7_dconc2_ds3(self):
|
||||
await maybe_deferred_to_future(
|
||||
self._test_request_order(
|
||||
start_nums=[1, 2, 3, 4, 5, 6, 7, 15, 13, 12, 11, 10, 9, 8],
|
||||
|
|
@ -361,8 +380,9 @@ class RequestSendOrderTestCase(TestCase):
|
|||
)
|
||||
)
|
||||
|
||||
@pytest.mark.only_asyncio
|
||||
@deferred_f_from_coro_f
|
||||
async def test_tconc7_dconc3_ds2(self):
|
||||
async def test_ar_tconc7_dconc3_ds2(self):
|
||||
await maybe_deferred_to_future(
|
||||
self._test_request_order(
|
||||
start_nums=[1, 2, 3, 4, 5, 6, 7, 15, 13, 12, 11, 10, 9, 8],
|
||||
|
|
@ -375,8 +395,9 @@ class RequestSendOrderTestCase(TestCase):
|
|||
)
|
||||
)
|
||||
|
||||
@pytest.mark.only_asyncio
|
||||
@deferred_f_from_coro_f
|
||||
async def test_fast(self):
|
||||
async def test_ar_fast(self):
|
||||
"""Very fast responses may increase the number of start requests sent
|
||||
in reverse order before the first callback request."""
|
||||
await maybe_deferred_to_future(
|
||||
|
|
@ -388,6 +409,240 @@ class RequestSendOrderTestCase(TestCase):
|
|||
)
|
||||
)
|
||||
|
||||
# Default reactor behavior
|
||||
|
||||
@pytest.mark.only_not_asyncio
|
||||
@deferred_f_from_coro_f
|
||||
async def test_dr_default(self):
|
||||
await maybe_deferred_to_future(
|
||||
self._test_request_order(
|
||||
start_nums=[
|
||||
26,
|
||||
24,
|
||||
23,
|
||||
22,
|
||||
21,
|
||||
20,
|
||||
19,
|
||||
18,
|
||||
17,
|
||||
16,
|
||||
15,
|
||||
14,
|
||||
13,
|
||||
12,
|
||||
11,
|
||||
10,
|
||||
9,
|
||||
8,
|
||||
7,
|
||||
6,
|
||||
5,
|
||||
4,
|
||||
3,
|
||||
2,
|
||||
1,
|
||||
],
|
||||
cb_nums=[25],
|
||||
)
|
||||
)
|
||||
|
||||
@pytest.mark.only_not_asyncio
|
||||
@deferred_f_from_coro_f
|
||||
async def test_dr_conc1(self):
|
||||
await maybe_deferred_to_future(
|
||||
self._test_request_order(
|
||||
start_nums=[4, 2, 1],
|
||||
cb_nums=[3],
|
||||
settings={"CONCURRENT_REQUESTS": 1},
|
||||
)
|
||||
)
|
||||
|
||||
@pytest.mark.only_not_asyncio
|
||||
@deferred_f_from_coro_f
|
||||
async def test_dr_conc2(self):
|
||||
await maybe_deferred_to_future(
|
||||
self._test_request_order(
|
||||
start_nums=[6, 4, 3, 2, 1],
|
||||
cb_nums=[5],
|
||||
settings={"CONCURRENT_REQUESTS": 2},
|
||||
)
|
||||
)
|
||||
|
||||
@pytest.mark.only_not_asyncio
|
||||
@deferred_f_from_coro_f
|
||||
async def test_dr_conc8(self):
|
||||
await maybe_deferred_to_future(
|
||||
self._test_request_order(
|
||||
start_nums=[18, 16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1],
|
||||
cb_nums=[17],
|
||||
settings={"CONCURRENT_REQUESTS": 8},
|
||||
)
|
||||
)
|
||||
|
||||
@pytest.mark.only_not_asyncio
|
||||
@deferred_f_from_coro_f
|
||||
async def test_dr_conc16(self):
|
||||
await maybe_deferred_to_future(
|
||||
self._test_request_order(
|
||||
start_nums=[
|
||||
34,
|
||||
32,
|
||||
31,
|
||||
30,
|
||||
29,
|
||||
28,
|
||||
27,
|
||||
26,
|
||||
25,
|
||||
24,
|
||||
23,
|
||||
22,
|
||||
21,
|
||||
20,
|
||||
19,
|
||||
18,
|
||||
17,
|
||||
16,
|
||||
15,
|
||||
14,
|
||||
13,
|
||||
12,
|
||||
11,
|
||||
10,
|
||||
9,
|
||||
8,
|
||||
7,
|
||||
6,
|
||||
5,
|
||||
4,
|
||||
3,
|
||||
2,
|
||||
1,
|
||||
],
|
||||
cb_nums=[33],
|
||||
settings={"CONCURRENT_REQUESTS_PER_DOMAIN": 16},
|
||||
)
|
||||
)
|
||||
|
||||
@pytest.mark.only_not_asyncio
|
||||
@deferred_f_from_coro_f
|
||||
async def test_dr_conc3_ds2(self):
|
||||
await maybe_deferred_to_future(
|
||||
self._test_request_order(
|
||||
start_nums=[8, 6, 5, 4, 3, 2, 1],
|
||||
cb_nums=[7],
|
||||
settings={
|
||||
"CONCURRENT_REQUESTS": 3,
|
||||
},
|
||||
download_slots=2,
|
||||
)
|
||||
)
|
||||
|
||||
@pytest.mark.only_not_asyncio
|
||||
@deferred_f_from_coro_f
|
||||
async def test_dr_tconc3_dconc2(self):
|
||||
await maybe_deferred_to_future(
|
||||
self._test_request_order(
|
||||
start_nums=[7, 5, 4, 3, 2, 1],
|
||||
cb_nums=[6],
|
||||
settings={
|
||||
"CONCURRENT_REQUESTS": 3,
|
||||
"CONCURRENT_REQUESTS_PER_DOMAIN": 2,
|
||||
},
|
||||
)
|
||||
)
|
||||
|
||||
@pytest.mark.only_not_asyncio
|
||||
@deferred_f_from_coro_f
|
||||
async def test_dr_tconc5_dconc3(self):
|
||||
await maybe_deferred_to_future(
|
||||
self._test_request_order(
|
||||
start_nums=[10, 8, 7, 6, 5, 4, 3, 2, 1],
|
||||
cb_nums=[9],
|
||||
settings={
|
||||
"CONCURRENT_REQUESTS": 5,
|
||||
"CONCURRENT_REQUESTS_PER_DOMAIN": 3,
|
||||
},
|
||||
)
|
||||
)
|
||||
|
||||
@pytest.mark.only_not_asyncio
|
||||
@deferred_f_from_coro_f
|
||||
async def test_dr_tconc5_dconc2_ds3(self):
|
||||
await maybe_deferred_to_future(
|
||||
self._test_request_order(
|
||||
start_nums=[12, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1],
|
||||
cb_nums=[11],
|
||||
settings={
|
||||
"CONCURRENT_REQUESTS": 5,
|
||||
"CONCURRENT_REQUESTS_PER_DOMAIN": 2,
|
||||
},
|
||||
download_slots=3,
|
||||
)
|
||||
)
|
||||
|
||||
@pytest.mark.only_not_asyncio
|
||||
@deferred_f_from_coro_f
|
||||
async def test_dr_tconc5_dconc3_ds2(self):
|
||||
await maybe_deferred_to_future(
|
||||
self._test_request_order(
|
||||
start_nums=[12, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1],
|
||||
cb_nums=[11],
|
||||
settings={
|
||||
"CONCURRENT_REQUESTS": 5,
|
||||
"CONCURRENT_REQUESTS_PER_DOMAIN": 3,
|
||||
},
|
||||
download_slots=2,
|
||||
)
|
||||
)
|
||||
|
||||
@pytest.mark.only_not_asyncio
|
||||
@deferred_f_from_coro_f
|
||||
async def test_dr_tconc7_dconc2_ds3(self):
|
||||
await maybe_deferred_to_future(
|
||||
self._test_request_order(
|
||||
start_nums=[15, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1],
|
||||
cb_nums=[14],
|
||||
settings={
|
||||
"CONCURRENT_REQUESTS": 7,
|
||||
"CONCURRENT_REQUESTS_PER_DOMAIN": 2,
|
||||
},
|
||||
download_slots=3,
|
||||
)
|
||||
)
|
||||
|
||||
@pytest.mark.only_not_asyncio
|
||||
@deferred_f_from_coro_f
|
||||
async def test_dr_tconc7_dconc3_ds2(self):
|
||||
await maybe_deferred_to_future(
|
||||
self._test_request_order(
|
||||
start_nums=[15, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1],
|
||||
cb_nums=[14],
|
||||
settings={
|
||||
"CONCURRENT_REQUESTS": 7,
|
||||
"CONCURRENT_REQUESTS_PER_DOMAIN": 3,
|
||||
},
|
||||
download_slots=2,
|
||||
)
|
||||
)
|
||||
|
||||
@pytest.mark.only_not_asyncio
|
||||
@deferred_f_from_coro_f
|
||||
async def test_dr_fast(self):
|
||||
"""Very fast responses may increase the number of start requests sent
|
||||
in reverse order before the first callback request."""
|
||||
await maybe_deferred_to_future(
|
||||
self._test_request_order(
|
||||
start_nums=[3, 2, 1],
|
||||
cb_nums=[4],
|
||||
settings={"CONCURRENT_REQUESTS": 1},
|
||||
response_seconds=self.fast_seconds,
|
||||
)
|
||||
)
|
||||
|
||||
# Behavior shared by both reactors
|
||||
|
||||
@deferred_f_from_coro_f
|
||||
async def test_await(self):
|
||||
"""Awaiting slow operations in Spider.start() may lower the number of
|
||||
|
|
|
|||
Loading…
Reference in New Issue