mirror of https://github.com/scrapy/scrapy.git
Give some leeway for PyPy
This commit is contained in:
parent
db6c9cc32c
commit
f200dbeb82
|
|
@ -20,6 +20,10 @@ from tests.utils.decorators import coroutine_test
|
|||
if TYPE_CHECKING:
|
||||
from tests.mockserver.http import MockServer
|
||||
|
||||
# Guards against a hung subprocess. Generous, because starting a script is
|
||||
# slow on PyPy, slower still with coverage measurement on.
|
||||
SCRIPT_TIMEOUT = 60
|
||||
|
||||
|
||||
class ScriptRunnerMixin(ABC):
|
||||
@property
|
||||
|
|
@ -220,7 +224,7 @@ class TestCrawlerProcessSubprocessBase(ScriptRunnerMixin):
|
|||
) -> None:
|
||||
sig = signal.SIGINT if sys.platform != "win32" else signal.SIGBREAK # type: ignore[attr-defined]
|
||||
args = self.get_script_args(script, "3", *extra_args)
|
||||
p = PopenSpawn(args, timeout=5, env=get_script_run_env())
|
||||
p = PopenSpawn(args, timeout=SCRIPT_TIMEOUT, env=get_script_run_env())
|
||||
p.expect_exact("Spider opened")
|
||||
p.expect_exact("Crawled (200)")
|
||||
p.kill(sig)
|
||||
|
|
@ -238,7 +242,7 @@ class TestCrawlerProcessSubprocessBase(ScriptRunnerMixin):
|
|||
async def _test_shutdown_forced(self, script: str = "sleeping.py") -> None:
|
||||
sig = signal.SIGINT if sys.platform != "win32" else signal.SIGBREAK # type: ignore[attr-defined]
|
||||
args = self.get_script_args(script, "10")
|
||||
p = PopenSpawn(args, timeout=5, env=get_script_run_env())
|
||||
p = PopenSpawn(args, timeout=SCRIPT_TIMEOUT, env=get_script_run_env())
|
||||
p.expect_exact("Spider opened")
|
||||
p.expect_exact("Crawled (200)")
|
||||
p.kill(sig)
|
||||
|
|
|
|||
|
|
@ -70,15 +70,14 @@ class TestCrawl:
|
|||
yield crawler.crawl(mockserver=self.mockserver)
|
||||
slots = crawler.engine.downloader.slots
|
||||
times = crawler.spider.times
|
||||
tolerance = 0.3
|
||||
# Downloading and processing a response add a roughly constant amount
|
||||
# of time on top of the configured delay, so the margin is absolute.
|
||||
# It stays well below the 1 second that separates the delays being
|
||||
# compared, so a slot using the delay of another one still fails.
|
||||
tolerance = 0.75
|
||||
|
||||
delays_real = {k: v[1] - v[0] for k, v in times.items()}
|
||||
error_delta = {
|
||||
k: 1 - min(delays_real[k], v.delay) / max(delays_real[k], v.delay)
|
||||
for k, v in slots.items()
|
||||
}
|
||||
|
||||
assert max(list(error_delta.values())) < tolerance
|
||||
for slot, (first, second) in times.items():
|
||||
assert abs((second - first) - slots[slot].delay) < tolerance
|
||||
|
||||
|
||||
@coroutine_test
|
||||
|
|
|
|||
Loading…
Reference in New Issue