From 955b40e415ef54fb3504667751b4ae446bebd741 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 17 Mar 2025 10:13:27 +0100 Subject: [PATCH] Increase test reliability and min lxml --- docs/news.rst | 7 +++++++ pyproject.toml | 2 +- tests/test_engine_loop.py | 8 +++----- tests/upper-constraints.txt | 2 +- tox.ini | 4 ++-- 5 files changed, 14 insertions(+), 9 deletions(-) diff --git a/docs/news.rst b/docs/news.rst index 890b8eab0..e35e8c2f7 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -13,6 +13,13 @@ Highlights: - Replaced ``start_requests`` (sync) with :meth:`~scrapy.Spider.yield_seeds` (async) and changed how it is iterated by default. +Modified requirements +~~~~~~~~~~~~~~~~~~~~~ + +- Minimum versions increased for these dependencies: + + - lxml_: 4.6.0 → 4.9.3 + Backward-incompatible changes ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/pyproject.toml b/pyproject.toml index ed6a6ed46..ddafa2a41 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,7 +21,7 @@ dependencies = [ "itemadapter>=0.1.0", "packaging", "tldextract", - "lxml>=4.6.0", + "lxml>=4.9.3", "defusedxml>=0.7.1", # Platform-specific dependencies 'PyDispatcher>=2.0.5; platform_python_implementation == "CPython"', diff --git a/tests/test_engine_loop.py b/tests/test_engine_loop.py index 4cc0fd7e2..9275f95dc 100644 --- a/tests/test_engine_loop.py +++ b/tests/test_engine_loop.py @@ -63,7 +63,7 @@ class MainTestCase(TestCase): async def yield_seeds(self): yield Request("data:,a") - await sleep(ExecutionEngine._MIN_BACK_IN_SECONDS * 2) + await sleep(ExecutionEngine._MIN_BACK_IN_SECONDS * 2**2) yield Request("data:,c") def parse(self, response): @@ -129,13 +129,11 @@ class MainTestCase(TestCase): """If the seeds raise an unhandled exception, scheduler requests should still be processed.""" - class TestScheduler(MemoryScheduler): - queue = ["data:,b"] - class TestSpider(Spider): name = "test" async def yield_seeds(self): + self.crawler.engine._slot.scheduler.enqueue_request(Request("data:,b")) yield Request("data:,a") raise RuntimeError @@ -147,7 +145,7 @@ class MainTestCase(TestCase): def track_url(request, spider): actual_urls.append(request.url) - settings = {"SCHEDULER": TestScheduler} + settings = {"SCHEDULER": MemoryScheduler} crawler = get_crawler(TestSpider, settings_dict=settings) crawler.signals.connect(track_url, signals.request_reached_downloader) with LogCapture() as log: diff --git a/tests/upper-constraints.txt b/tests/upper-constraints.txt index 2a335e533..0634af44f 100644 --- a/tests/upper-constraints.txt +++ b/tests/upper-constraints.txt @@ -5,7 +5,7 @@ Automat>=0.8.0 botocore>=1.20.30 itemadapter>=0.1.1 itemloaders>=1.0.3 -lxml>=4.6.1 +lxml>=4.9.3 parsel>=1.5.2 Pillow>=8.0.1 pyOpenSSL>=17.5 # mitmproxy 4.0.4 diff --git a/tox.ini b/tox.ini index eb084f0f5..c4afe41c1 100644 --- a/tox.ini +++ b/tox.ini @@ -101,7 +101,7 @@ deps = cryptography==37.0.0 cssselect==0.9.1 itemadapter==0.1.0 - lxml==4.6.0 + lxml==4.9.3 parsel==1.5.0 pyOpenSSL==22.0.0 queuelib==1.4.2 @@ -200,7 +200,7 @@ deps = cryptography==41.0.5 cssselect==0.9.1 itemadapter==0.1.0 - lxml==4.6.0 + lxml==4.9.3 parsel==1.5.0 pyOpenSSL==23.3.0 queuelib==1.4.2