Solve warning-related issues

scrapy/utils/test.py

@@ -66,12 +66,14 @@ def get_crawler(
     will be used to populate the crawler settings with a project level
     priority.
     """
-    # When needed, useful settings can be added here, e.g. ones that prevent
-    # deprecation warnings.
     settings: dict[str, Any] = {
         **get_reactor_settings(),
         **(settings_dict or {}),
     }
+    if prevent_warnings:
+        settings.setdefault("CONCURRENT_REQUESTS_PER_DOMAIN", 8)
+        settings.setdefault("DOWNLOAD_DELAY", 0)
+        settings.setdefault("ROBOTSTXT_OBEY", False)
     runner: CrawlerRunnerBase
     if is_reactor_installed():
         runner = CrawlerRunner(settings)

tests/AsyncCrawlerProcess/reactorless_datauri.py

@@ -12,7 +12,14 @@ class DataSpider(Spider):
         return {"data": response.text}
 
 
-process = AsyncCrawlerProcess(settings={"TWISTED_REACTOR_ENABLED": False})
+process = AsyncCrawlerProcess(
+    settings={
+        "TWISTED_REACTOR_ENABLED": False,
+        "CONCURRENT_REQUESTS_PER_DOMAIN": 8,
+        "DOWNLOAD_DELAY": 0,
+        "ROBOTSTXT_OBEY": False,
+    }
+)
 
 process.crawl(DataSpider)
 process.start()

tests/AsyncCrawlerProcess/reactorless_simple.py

@@ -12,7 +12,14 @@ class NoRequestsSpider(scrapy.Spider):
         yield
 
 
-process = AsyncCrawlerProcess(settings={"TWISTED_REACTOR_ENABLED": False})
+process = AsyncCrawlerProcess(
+    settings={
+        "TWISTED_REACTOR_ENABLED": False,
+        "CONCURRENT_REQUESTS_PER_DOMAIN": 8,
+        "DOWNLOAD_DELAY": 0,
+        "ROBOTSTXT_OBEY": False,
+    }
+)
 
 process.crawl(NoRequestsSpider)
 process.start()

tests/AsyncCrawlerRunner/reactorless_datauri.py

@@ -17,7 +17,14 @@ class DataSpider(Spider):
 
 async def main() -> None:
     configure_logging()
-    runner = AsyncCrawlerRunner(settings={"TWISTED_REACTOR_ENABLED": False})
+    runner = AsyncCrawlerRunner(
+        settings={
+            "TWISTED_REACTOR_ENABLED": False,
+            "CONCURRENT_REQUESTS_PER_DOMAIN": 8,
+            "DOWNLOAD_DELAY": 0,
+            "ROBOTSTXT_OBEY": False,
+        }
+    )
     await runner.crawl(DataSpider)
 
 

tests/AsyncCrawlerRunner/reactorless_simple.py

@@ -17,7 +17,14 @@ class NoRequestsSpider(Spider):
 
 async def main() -> None:
     configure_logging()
-    runner = AsyncCrawlerRunner(settings={"TWISTED_REACTOR_ENABLED": False})
+    runner = AsyncCrawlerRunner(
+        settings={
+            "TWISTED_REACTOR_ENABLED": False,
+            "CONCURRENT_REQUESTS_PER_DOMAIN": 8,
+            "DOWNLOAD_DELAY": 0,
+            "ROBOTSTXT_OBEY": False,
+        }
+    )
     await runner.crawl(NoRequestsSpider)
 
 

tests/test_crawler.py

@@ -3,6 +3,7 @@ from __future__ import annotations
 import asyncio
 import logging
 import re
+import warnings
 from pathlib import Path
 from typing import Any, ClassVar