Pin older Twisted in tests, update type hints. (#6882)

scrapy/core/downloader/handlers/http11.py

@@ -8,7 +8,7 @@ import re
 from contextlib import suppress
 from io import BytesIO
 from time import time
-from typing import TYPE_CHECKING, Any, TypedDict, TypeVar
+from typing import TYPE_CHECKING, Any, TypedDict, TypeVar, cast
 from urllib.parse import urldefrag, urlparse
 
 from twisted.internet import ssl
@@ -27,7 +27,7 @@ from twisted.web.client import (
 from twisted.web.client import Response as TxResponse
 from twisted.web.http import PotentialDataLoss, _DataLoss
 from twisted.web.http_headers import Headers as TxHeaders
-from twisted.web.iweb import UNKNOWN_LENGTH, IBodyProducer, IPolicyForHTTPS
+from twisted.web.iweb import UNKNOWN_LENGTH, IBodyProducer, IPolicyForHTTPS, IResponse
 from zope.interface import implementer
 
 from scrapy import Request, Spider, signals
@@ -286,11 +286,11 @@ class TunnelingAgent(Agent):
         key: Any,
         endpoint: TCP4ClientEndpoint,
         method: bytes,
-        parsedURI: bytes,
+        parsedURI: URI,
         headers: TxHeaders | None,
         bodyProducer: IBodyProducer | None,
         requestPath: bytes,
-    ) -> Deferred[TxResponse]:
+    ) -> Deferred[IResponse]:
         # proxy host and port are required for HTTP pool `key`
         # otherwise, same remote host connection request could reuse
         # a cached tunneled connection to a different proxy
@@ -329,14 +329,14 @@ class ScrapyProxyAgent(Agent):
         uri: bytes,
         headers: TxHeaders | None = None,
         bodyProducer: IBodyProducer | None = None,
-    ) -> Deferred[TxResponse]:
+    ) -> Deferred[IResponse]:
         """
         Issue a new request via the configured proxy.
         """
         # Cache *all* connections under the same key, since we are only
         # connecting to a single destination, the proxy:
         return self._requestWithEndpoint(
-            key=("http-proxy", self._proxyURI.host, self._proxyURI.port),
+            key=(b"http-proxy", self._proxyURI.host, self._proxyURI.port),
             endpoint=self._getEndpoint(self._proxyURI),
             method=method,
             parsedURI=URI.fromBytes(uri),
@@ -426,8 +426,11 @@ class ScrapyAgent:
             headers.removeHeader(b"Proxy-Authorization")
         bodyproducer = _RequestBodyProducer(request.body) if request.body else None
         start_time = time()
-        d: Deferred[TxResponse] = agent.request(
-            method, to_bytes(url, encoding="ascii"), headers, bodyproducer
+        d: Deferred[IResponse] = agent.request(
+            method,
+            to_bytes(url, encoding="ascii"),
+            headers,
+            cast(IBodyProducer, bodyproducer),
         )
         # set download latency
         d.addCallback(self._cb_latency, request, start_time)

scrapy/utils/test.py

@@ -10,7 +10,7 @@ import warnings
 from importlib import import_module
 from pathlib import Path
 from posixpath import split
-from typing import TYPE_CHECKING, Any, TypeVar
+from typing import TYPE_CHECKING, Any, TypeVar, cast
 from unittest import TestCase, mock
 
 from twisted.trial.unittest import SkipTest
@@ -211,4 +211,4 @@ def get_web_client_agent_req(url: str) -> Deferred[TxResponse]:
     from twisted.web.client import Agent  # imports twisted.internet.reactor
 
     agent = Agent(reactor)
-    return agent.request(b"GET", url.encode("utf-8"))
+    return cast("Deferred[TxResponse]", agent.request(b"GET", url.encode("utf-8")))

tests/test_core_downloader.py

@@ -4,7 +4,7 @@ import shutil
 import warnings
 from pathlib import Path
 from tempfile import mkdtemp
-from typing import Any
+from typing import Any, cast
 
 import OpenSSL.SSL
 import pytest
@@ -14,6 +14,7 @@ from twisted.trial import unittest
 from twisted.web import server, static
 from twisted.web.client import Agent, BrowserLikePolicyForHTTPS, readBody
 from twisted.web.client import Response as TxResponse
+from twisted.web.iweb import IBodyProducer
 
 from scrapy.core.downloader import Slot
 from scrapy.core.downloader.contextfactory import (
@@ -76,8 +77,15 @@ class TestContextFactoryBase(unittest.TestCase):
 
         agent = Agent(reactor, contextFactory=client_context_factory)
         body_producer = _RequestBodyProducer(body.encode()) if body else None
-        response: TxResponse = await maybe_deferred_to_future(
-            agent.request(b"GET", url.encode(), bodyProducer=body_producer)
+        response: TxResponse = cast(
+            TxResponse,
+            await maybe_deferred_to_future(
+                agent.request(
+                    b"GET",
+                    url.encode(),
+                    bodyProducer=cast(IBodyProducer, body_producer),
+                )
+            ),
         )
         with warnings.catch_warnings():
             # https://github.com/twisted/twisted/issues/8227

tox.ini

@@ -20,6 +20,7 @@ deps =
     sybil >= 1.3.0  # https://github.com/cjw296/sybil/issues/20#issuecomment-605433422
     testfixtures
     pywin32; sys_platform == "win32"
+    Twisted < 25.5.0  # https://github.com/twisted/twisted/issues/12467
 
 [testenv]
 deps =