From 24a827c72e9f5b35ddcd12ccce2ce7c6611d2845 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 9 Jun 2025 12:53:42 +0500 Subject: [PATCH] Pin older Twisted in tests, update type hints. (#6882) --- scrapy/core/downloader/handlers/http11.py | 19 +++++++++++-------- scrapy/utils/test.py | 4 ++-- tests/test_core_downloader.py | 14 +++++++++++--- tox.ini | 1 + 4 files changed, 25 insertions(+), 13 deletions(-) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 74a6e54ee..54fef48b6 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -8,7 +8,7 @@ import re from contextlib import suppress from io import BytesIO from time import time -from typing import TYPE_CHECKING, Any, TypedDict, TypeVar +from typing import TYPE_CHECKING, Any, TypedDict, TypeVar, cast from urllib.parse import urldefrag, urlparse from twisted.internet import ssl @@ -27,7 +27,7 @@ from twisted.web.client import ( from twisted.web.client import Response as TxResponse from twisted.web.http import PotentialDataLoss, _DataLoss from twisted.web.http_headers import Headers as TxHeaders -from twisted.web.iweb import UNKNOWN_LENGTH, IBodyProducer, IPolicyForHTTPS +from twisted.web.iweb import UNKNOWN_LENGTH, IBodyProducer, IPolicyForHTTPS, IResponse from zope.interface import implementer from scrapy import Request, Spider, signals @@ -286,11 +286,11 @@ class TunnelingAgent(Agent): key: Any, endpoint: TCP4ClientEndpoint, method: bytes, - parsedURI: bytes, + parsedURI: URI, headers: TxHeaders | None, bodyProducer: IBodyProducer | None, requestPath: bytes, - ) -> Deferred[TxResponse]: + ) -> Deferred[IResponse]: # proxy host and port are required for HTTP pool `key` # otherwise, same remote host connection request could reuse # a cached tunneled connection to a different proxy @@ -329,14 +329,14 @@ class ScrapyProxyAgent(Agent): uri: bytes, headers: TxHeaders | None = None, bodyProducer: IBodyProducer | None = None, - ) -> Deferred[TxResponse]: + ) -> Deferred[IResponse]: """ Issue a new request via the configured proxy. """ # Cache *all* connections under the same key, since we are only # connecting to a single destination, the proxy: return self._requestWithEndpoint( - key=("http-proxy", self._proxyURI.host, self._proxyURI.port), + key=(b"http-proxy", self._proxyURI.host, self._proxyURI.port), endpoint=self._getEndpoint(self._proxyURI), method=method, parsedURI=URI.fromBytes(uri), @@ -426,8 +426,11 @@ class ScrapyAgent: headers.removeHeader(b"Proxy-Authorization") bodyproducer = _RequestBodyProducer(request.body) if request.body else None start_time = time() - d: Deferred[TxResponse] = agent.request( - method, to_bytes(url, encoding="ascii"), headers, bodyproducer + d: Deferred[IResponse] = agent.request( + method, + to_bytes(url, encoding="ascii"), + headers, + cast(IBodyProducer, bodyproducer), ) # set download latency d.addCallback(self._cb_latency, request, start_time) diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py index 4a732bd72..3780ad23e 100644 --- a/scrapy/utils/test.py +++ b/scrapy/utils/test.py @@ -10,7 +10,7 @@ import warnings from importlib import import_module from pathlib import Path from posixpath import split -from typing import TYPE_CHECKING, Any, TypeVar +from typing import TYPE_CHECKING, Any, TypeVar, cast from unittest import TestCase, mock from twisted.trial.unittest import SkipTest @@ -216,4 +216,4 @@ def get_web_client_agent_req(url: str) -> Deferred[TxResponse]: from twisted.web.client import Agent # imports twisted.internet.reactor agent = Agent(reactor) - return agent.request(b"GET", url.encode("utf-8")) + return cast("Deferred[TxResponse]", agent.request(b"GET", url.encode("utf-8"))) diff --git a/tests/test_core_downloader.py b/tests/test_core_downloader.py index 1bffd69ed..668a2cd1b 100644 --- a/tests/test_core_downloader.py +++ b/tests/test_core_downloader.py @@ -4,7 +4,7 @@ import shutil import warnings from pathlib import Path from tempfile import mkdtemp -from typing import Any +from typing import Any, cast import OpenSSL.SSL import pytest @@ -15,6 +15,7 @@ from twisted.trial import unittest from twisted.web import server, static from twisted.web.client import Agent, BrowserLikePolicyForHTTPS, readBody from twisted.web.client import Response as TxResponse +from twisted.web.iweb import IBodyProducer from scrapy.core.downloader import Slot from scrapy.core.downloader.contextfactory import ( @@ -73,8 +74,15 @@ class TestContextFactoryBase(unittest.TestCase): ) -> bytes: agent = Agent(reactor, contextFactory=client_context_factory) body_producer = _RequestBodyProducer(body.encode()) if body else None - response: TxResponse = await maybe_deferred_to_future( - agent.request(b"GET", url.encode(), bodyProducer=body_producer) + response: TxResponse = cast( + TxResponse, + await maybe_deferred_to_future( + agent.request( + b"GET", + url.encode(), + bodyProducer=cast(IBodyProducer, body_producer), + ) + ), ) with warnings.catch_warnings(): # https://github.com/twisted/twisted/issues/8227 diff --git a/tox.ini b/tox.ini index 92cfc3794..5680d98d1 100644 --- a/tox.ini +++ b/tox.ini @@ -20,6 +20,7 @@ deps = sybil >= 1.3.0 # https://github.com/cjw296/sybil/issues/20#issuecomment-605433422 testfixtures pywin32; sys_platform == "win32" + Twisted < 25.5.0 # https://github.com/twisted/twisted/issues/12467 [testenv] deps =