Increase test reliability and min lxml

docs/news.rst

@@ -13,6 +13,13 @@ Highlights:
 -   Replaced ``start_requests`` (sync) with :meth:`~scrapy.Spider.yield_seeds`
     (async) and changed how it is iterated by default.
 
+Modified requirements
+~~~~~~~~~~~~~~~~~~~~~
+
+-   Minimum versions increased for these dependencies:
+
+    -   lxml_: 4.6.0 → 4.9.3
+
 Backward-incompatible changes
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 

pyproject.toml

@@ -21,7 +21,7 @@ dependencies = [
     "itemadapter>=0.1.0",
     "packaging",
     "tldextract",
-    "lxml>=4.6.0",
+    "lxml>=4.9.3",
     "defusedxml>=0.7.1",
     # Platform-specific dependencies
     'PyDispatcher>=2.0.5; platform_python_implementation == "CPython"',

tests/test_engine_loop.py

@@ -63,7 +63,7 @@ class MainTestCase(TestCase):
 
             async def yield_seeds(self):
                 yield Request("data:,a")
-                await sleep(ExecutionEngine._MIN_BACK_IN_SECONDS * 2)
+                await sleep(ExecutionEngine._MIN_BACK_IN_SECONDS * 2**2)
                 yield Request("data:,c")
 
             def parse(self, response):
@@ -129,13 +129,11 @@ class MainTestCase(TestCase):
         """If the seeds raise an unhandled exception, scheduler requests should
         still be processed."""
 
-        class TestScheduler(MemoryScheduler):
-            queue = ["data:,b"]
-
         class TestSpider(Spider):
             name = "test"
 
             async def yield_seeds(self):
+                self.crawler.engine._slot.scheduler.enqueue_request(Request("data:,b"))
                 yield Request("data:,a")
                 raise RuntimeError
 
@@ -147,7 +145,7 @@ class MainTestCase(TestCase):
         def track_url(request, spider):
             actual_urls.append(request.url)
 
-        settings = {"SCHEDULER": TestScheduler}
+        settings = {"SCHEDULER": MemoryScheduler}
         crawler = get_crawler(TestSpider, settings_dict=settings)
         crawler.signals.connect(track_url, signals.request_reached_downloader)
         with LogCapture() as log:

tests/upper-constraints.txt

@@ -5,7 +5,7 @@ Automat>=0.8.0
 botocore>=1.20.30
 itemadapter>=0.1.1
 itemloaders>=1.0.3
-lxml>=4.6.1
+lxml>=4.9.3
 parsel>=1.5.2
 Pillow>=8.0.1
 pyOpenSSL>=17.5  # mitmproxy 4.0.4

tox.ini

@@ -101,7 +101,7 @@ deps =
     cryptography==37.0.0
     cssselect==0.9.1
     itemadapter==0.1.0
-    lxml==4.6.0
+    lxml==4.9.3
     parsel==1.5.0
     pyOpenSSL==22.0.0
     queuelib==1.4.2
@@ -200,7 +200,7 @@ deps =
     cryptography==41.0.5
     cssselect==0.9.1
     itemadapter==0.1.0
-    lxml==4.6.0
+    lxml==4.9.3
     parsel==1.5.0
     pyOpenSSL==23.3.0
     queuelib==1.4.2