Rerun tests that need real Internet access when they fail (#7985)

conftest.py

@@ -107,6 +107,14 @@ def pytest_configure(config):
         install_reactor_import_hook()
 
 
+def pytest_collection_modifyitems(items):
+    for item in items:
+        if item.get_closest_marker("requires_internet"):
+            # Requests to real websites fail every now and then in CI for
+            # reasons unrelated to the code under test.
+            item.add_marker(pytest.mark.flaky(reruns=2, reruns_delay=5))
+
+
 def pytest_runtest_setup(item):
     # Skip tests based on reactor markers
     reactor = item.config.getoption("--reactor")

tests/utils/bases/download_handlers_http.py

@@ -1513,6 +1513,12 @@ class TestMitmProxyBase(ABC):
         assert "Proxy Authentication Required" in log or "407" in log
 
 
+# Tests below are rerun on failure (see pytest_collection_modifyitems() in the
+# root conftest.py), so an attempt must give up soon enough for a rerun to be
+# cheap.
+REAL_WEBSITE_SETTINGS = {"DOWNLOAD_TIMEOUT": 30}
+
+
 class TestRealWebsiteBase(ABC):
     @property
     @abstractmethod
@@ -1537,7 +1543,9 @@ class TestRealWebsiteBase(ABC):
     async def get_dh(
         self, settings_dict: dict[str, Any] | None = None
     ) -> AsyncGenerator[DownloadHandlerProtocol]:
-        crawler = get_crawler(DefaultSpider, settings_dict)
+        crawler = get_crawler(
+            DefaultSpider, {**REAL_WEBSITE_SETTINGS, **(settings_dict or {})}
+        )
         crawler.spider = crawler._create_spider()
         dh = build_from_crawler(self.download_handler_cls, crawler)
         try:
@@ -1555,7 +1563,9 @@ class TestRealWebsiteBase(ABC):
 
     @coroutine_test
     async def test_download_with_spider(self) -> None:
-        crawler = get_crawler(SingleRequestSpider, self.settings_dict)
+        crawler = get_crawler(
+            SingleRequestSpider, {**REAL_WEBSITE_SETTINGS, **(self.settings_dict or {})}
+        )
         await maybe_deferred_to_future(
             crawler.crawl(seed=Request("https://books.toscrape.com/"))
         )

tox.ini

@@ -44,6 +44,7 @@ deps =
     pygments
     pytest
     pytest-cov >= 7.0.0
+    pytest-rerunfailures
     pytest-timeout
     pytest-xdist
     sybil >= 1.3.0  # https://github.com/cjw296/sybil/issues/20#issuecomment-605433422