Fix tests

tests/__init__.py

@@ -8,6 +8,9 @@ import os
 import socket
 from pathlib import Path
 
+from twisted import version as TWISTED_VERSION
+from twisted.python.versions import Version
+
 # ignore system-wide proxies for tests
 # which would send requests to a totally unsuspecting server
 # (e.g. because urllib does not fully understand the proxy spec)
@@ -30,3 +33,6 @@ except socket.gaierror:
 def get_testdata(*paths: str) -> bytes:
     """Return test data"""
     return Path(tests_datadir, *paths).read_bytes()
+
+
+TWISTED_KEEPS_TRACEBACKS = TWISTED_VERSION >= Version("twisted", 24, 10, 0)

tests/test_cmdline_crawl_with_pipeline/__init__.py

@@ -2,6 +2,8 @@ import sys
 from pathlib import Path
 from subprocess import PIPE, Popen
 
+from .. import TWISTED_KEEPS_TRACEBACKS
+
 
 class TestCmdlineCrawlPipeline:
     def _execute(self, spname):
@@ -17,7 +19,9 @@ class TestCmdlineCrawlPipeline:
 
     def test_exception_at_open_spider_in_pipeline(self):
         returncode, stderr = self._execute("exception")
-        assert (
-            returncode == 0
-        )  # An unhandled exception in a pipeline should not stop the crawl
-        assert b'RuntimeError("exception")' in stderr
+        # An unhandled exception in a pipeline should not stop the crawl
+        assert returncode == 0
+        if TWISTED_KEEPS_TRACEBACKS:
+            assert b'RuntimeError("exception")' in stderr
+        else:
+            assert b"RuntimeError: exception" in stderr

tests/test_engine_loop.py

@@ -122,7 +122,7 @@ class RequestSendOrderTestCase(TestCase):
         cls.mockserver.__exit__(None, None, None)
 
     fast_seconds = 0.001
-    slow_seconds = 0.4  # increase if flaky
+    slow_seconds = 1  # increase if flaky
 
     def _request(self, num, response_seconds, download_slots):
         url = self.mockserver.url(f"/delay?n={response_seconds}&{num}")

tests/test_spider_start.py

@@ -2,9 +2,7 @@ from asyncio import sleep
 
 import pytest
 from testfixtures import LogCapture
-from twisted import version as TWISTED_VERSION
 from twisted.internet.defer import Deferred
-from twisted.python.versions import Version
 from twisted.trial.unittest import TestCase
 
 from scrapy import Spider, signals
@@ -23,8 +21,6 @@ ASYNC_GEN_ERROR_MINIMUM_SECONDS = ExecutionEngine._SLOT_HEARTBEAT_INTERVAL + 0.0
 ITEM_A = {"id": "a"}
 ITEM_B = {"id": "b"}
 
-TWISTED_KEEPS_TRACEBACKS = TWISTED_VERSION >= Version("twisted", 24, 10, 0)
-
 
 def twisted_sleep(seconds):
     from twisted.internet import reactor
@@ -176,9 +172,4 @@ class MainTestCase(TestCase):
         ):
             await self._test_spider(TestSpider, [])
 
-        if TWISTED_KEEPS_TRACEBACKS:
-            assert "in start_requests\n    raise RuntimeError" in str(log), log
-        else:
-            assert "in _process_next_spider_start_yield\n    item_or_request =" in str(
-                log
-            ), log
+        assert "in start_requests\n    raise RuntimeError" in str(log)