Works
World
", "text/html", ), ) - self.putChild( + put_child( + self, b"enc-gb18030", Data(b"gb18030 encoding
", "text/html; charset=gb18030"), ) - self.putChild(b"redirect", Redirect(b"/redirected")) - self.putChild( - b"redirect-no-meta-refresh", NoMetaRefreshRedirect(b"/redirected") + put_child(self, b"redirect", Redirect(b"/redirected")) + put_child( + self, b"redirect-no-meta-refresh", NoMetaRefreshRedirect(b"/redirected") ) - self.putChild(b"redirected", Data(b"Redirected here", "text/plain")) + put_child(self, b"redirected", Data(b"Redirected here", "text/plain")) numbers = [str(x).encode("utf8") for x in range(2**18)] - self.putChild(b"numbers", Data(b"".join(numbers), "text/plain")) - self.putChild(b"wait", ForeverTakingResource()) - self.putChild(b"hang-after-headers", ForeverTakingResource(write=True)) - self.putChild(b"host", HostHeaderResource()) - self.putChild(b"client-ip", ClientIPResource()) - self.putChild(b"broken", BrokenDownloadResource()) - self.putChild(b"chunked", ChunkedResource()) - self.putChild(b"broken-chunked", BrokenChunkedResource()) - self.putChild(b"contentlength", ContentLengthHeaderResource()) - self.putChild(b"nocontenttype", EmptyContentTypeHeaderResource()) - self.putChild(b"largechunkedfile", LargeChunkedFileResource()) - self.putChild(b"compress", Compress()) - self.putChild(b"duplicate-header", DuplicateHeaderResource()) - self.putChild(b"response-headers", ResponseHeadersResource()) - self.putChild(b"set-cookie", SetCookie()) - self.putChild(b"uri", UriResource()) + put_child(self, b"numbers", Data(b"".join(numbers), "text/plain")) + put_child(self, b"wait", ForeverTakingResource()) + put_child(self, b"hang-after-headers", ForeverTakingResource(write=True)) + put_child(self, b"host", HostHeaderResource()) + put_child(self, b"client-ip", ClientIPResource()) + put_child(self, b"broken", BrokenDownloadResource()) + put_child(self, b"chunked", ChunkedResource()) + put_child(self, b"broken-chunked", BrokenChunkedResource()) + put_child(self, b"contentlength", ContentLengthHeaderResource()) + put_child(self, b"nocontenttype", EmptyContentTypeHeaderResource()) + put_child(self, b"largechunkedfile", LargeChunkedFileResource()) + put_child(self, b"compress", Compress()) + put_child(self, b"duplicate-header", DuplicateHeaderResource()) + put_child(self, b"response-headers", ResponseHeadersResource()) + put_child(self, b"set-cookie", SetCookie()) + put_child(self, b"uri", UriResource()) - def getChild(self, path, request): + def getChild(self, path: bytes, request: Request) -> Root: return self - def render(self, request): + def render(self, request: Request) -> bytes: return b"Scrapy mock HTTP server\n" diff --git a/tests/mockserver/http_base.py b/tests/mockserver/http_base.py index 343c79781..5bc1252d6 100644 --- a/tests/mockserver/http_base.py +++ b/tests/mockserver/http_base.py @@ -17,6 +17,7 @@ from .utils import ssl_context_factory if TYPE_CHECKING: from collections.abc import Callable + from types import TracebackType from twisted.web import resource @@ -60,7 +61,12 @@ class BaseMockServer(ABC): self.https_port = https_parsed.port return self - def __exit__(self, exc_type, exc_value, traceback) -> None: + def __exit__( + self, + exc_type: type[BaseException] | None, + exc_value: BaseException | None, + traceback: TracebackType | None, + ) -> None: if self.proc: self.proc.kill() self.proc.communicate() @@ -135,7 +141,7 @@ def main_factory( context_factory = ssl_context_factory(**context_factory_kw) https_port = reactor.listenSSL(0, factory, context_factory) - def print_listening(): + def print_listening() -> None: if listen_http: http_host = http_port.getHost() http_address = f"http://{http_host.host}:{http_host.port}" diff --git a/tests/mockserver/http_resources.py b/tests/mockserver/http_resources.py index 98ac6cf6a..cb028bc10 100644 --- a/tests/mockserver/http_resources.py +++ b/tests/mockserver/http_resources.py @@ -3,7 +3,7 @@ from __future__ import annotations import gzip import json import random -from typing import TYPE_CHECKING, ParamSpec, TypeVar +from typing import TYPE_CHECKING, Any, ParamSpec, TypeVar from urllib.parse import urlencode from twisted.internet.task import deferLater @@ -14,17 +14,24 @@ from twisted.web.util import Redirect, redirectTo from scrapy.utils.python import to_bytes, to_unicode if TYPE_CHECKING: - from collections.abc import Callable + from collections.abc import Callable, Sequence from twisted.internet.defer import Deferred - from twisted.web.http import Request + from twisted.python.failure import Failure + from twisted.web.http import Request as HTTPRequest + from twisted.web.server import Request _T = TypeVar("_T") _P = ParamSpec("_P") -def getarg(request, name, default=None, type_=None): +def getarg( + request: Request, + name: bytes, + default: Any = None, + type_: Callable[[bytes], Any] | None = None, +) -> Any: if name in request.args: value = request.args[name][0] if type_ is not None: @@ -33,73 +40,91 @@ def getarg(request, name, default=None, type_=None): return default -def close_connection(request): +def close_connection(request: Request) -> None: # We have to force a disconnection for HTTP/1.1 clients. Otherwise # client keeps the connection open waiting for more data. request.channel.loseConnection() request.finish() +def put_child(parent: resource.Resource, path: bytes, child: resource.Resource) -> None: + # zope.interface has no type hints, so mypy cannot tell that Resource + # instances provide the IResource interface that putChild() expects. + parent.putChild(path, child) # type: ignore[arg-type] + + +class BaseResource(resource.Resource): + """Base class for mockserver resources, with type hints.""" + + # Only needed to give subclasses a typed __init__ to call. + def __init__(self) -> None: # pylint: disable=useless-parent-delegation + super().__init__() # type: ignore[no-untyped-call] + + # most of the following resources are copied from twisted.web.test.test_webclient -class ForeverTakingResource(resource.Resource): +class ForeverTakingResource(BaseResource): """ L{ForeverTakingResource} is a resource which never finishes responding to requests. """ - def __init__(self, write=False): - resource.Resource.__init__(self) + def __init__(self, write: bool = False): + super().__init__() self._write = write - def render(self, request): + def render(self, request: Request) -> int: if self._write: request.write(b"some bytes") return server.NOT_DONE_YET -class HostHeaderResource(resource.Resource): +class HostHeaderResource(BaseResource): """ A testing resource which renders itself as the value of the host header from the request. """ - def render(self, request): - return request.requestHeaders.getRawHeaders(b"host")[0] + def render(self, request: Request) -> bytes: + headers = request.requestHeaders.getRawHeaders(b"host") + assert headers + return headers[0] -class ClientIPResource(resource.Resource): +class ClientIPResource(BaseResource): """ A testing resource which renders itself as the request client IP address. """ - def render(self, request): + def render(self, request: Request) -> bytes: client_address = request.getClientAddress() if client_address is None or client_address.host is None: return b"" return to_bytes(client_address.host) -class PayloadResource(resource.Resource): +class PayloadResource(BaseResource): """ A testing resource which renders itself as the contents of the request body as long as the request body is 100 bytes long, otherwise which renders itself as C{"ERROR"}. """ - def render(self, request): - data = request.content.read() - contentLength = request.requestHeaders.getRawHeaders(b"content-length")[0] - if len(data) != 100 or int(contentLength) != 100: + def render(self, request: Request) -> bytes: + assert request.content + data: bytes = request.content.read() + content_length = request.requestHeaders.getRawHeaders(b"content-length") + assert content_length + if len(data) != 100 or int(content_length[0]) != 100: return b"ERROR" return data -class LeafResource(resource.Resource): +class LeafResource(BaseResource): isLeaf = True def deferRequest( self, - request: Request, + request: HTTPRequest, delay: float, f: Callable[_P, _T], *a: _P.args, @@ -107,7 +132,7 @@ class LeafResource(resource.Resource): ) -> Deferred[_T]: from twisted.internet import reactor - def _cancelrequest(_): + def _cancelrequest(_: Failure) -> None: # silence CancelledError d.addErrback(lambda _: None) d.cancel() @@ -118,12 +143,13 @@ class LeafResource(resource.Resource): class Follow(LeafResource): - def render(self, request): + def render(self, request: Request) -> int: total = getarg(request, b"total", 100, type_=int) show = getarg(request, b"show", 1, type_=int) order = getarg(request, b"order", b"desc") maxlatency = getarg(request, b"maxlatency", 0, type_=float) n = getarg(request, b"n", total, type_=int) + nlist: Sequence[int] if order == b"rand": nlist = [random.randint(1, total) for _ in range(show)] else: # order == "desc" @@ -133,7 +159,7 @@ class Follow(LeafResource): self.deferRequest(request, lag, self.renderRequest, request, nlist) return NOT_DONE_YET - def renderRequest(self, request, nlist): + def renderRequest(self, request: Request, nlist: Sequence[int]) -> None: s = """ """ args = request.args.copy() for nl in nlist: @@ -146,45 +172,47 @@ class Follow(LeafResource): class Delay(LeafResource): - def render_GET(self, request): + def render_GET(self, request: Request) -> int: n = getarg(request, b"n", 1, type_=float) b = getarg(request, b"b", 1, type_=int) if b: # send headers now and delay body - request.write("") + request.write(b"") self.deferRequest(request, n, self._delayedRender, request, n) return NOT_DONE_YET - def _delayedRender(self, request, n): + def _delayedRender(self, request: Request, n: float) -> None: request.write(to_bytes(f"Response delayed for {n:.3f} seconds\n")) request.finish() class Status(LeafResource): - def render_GET(self, request): + def render_GET(self, request: Request) -> bytes: n = getarg(request, b"n", 200, type_=int) request.setResponseCode(n) return b"" class Raw(LeafResource): - def render_GET(self, request): + def render_GET(self, request: Request) -> int: request.startedWriting = 1 self.deferRequest(request, 0, self._delayedRender, request) return NOT_DONE_YET render_POST = render_GET - def _delayedRender(self, request): + def _delayedRender(self, request: Request) -> None: raw = getarg(request, b"raw", b"HTTP 1.1 200 OK\n") request.startedWriting = 1 request.write(raw) + assert request.channel.transport is not None request.channel.transport.loseConnection() request.finish() class Echo(LeafResource): - def render_GET(self, request): + def render_GET(self, request: Request) -> bytes: + assert request.content output = { "headers": { to_unicode(k): [to_unicode(v) for v in vs] @@ -198,27 +226,29 @@ class Echo(LeafResource): class RedirectTo(LeafResource): - def render(self, request): + def render(self, request: Request) -> bytes: goto = getarg(request, b"goto", b"/") # we force the body content, otherwise Twisted redirectTo() # returns HTML with int: request.setHeader(b"Content-Length", b"1024") self.deferRequest(request, 0, self._delayedRender, request) return NOT_DONE_YET - def _delayedRender(self, request): + def _delayedRender(self, request: Request) -> None: request.write(b"partial content\n") request.finish() class Drop(Partial): - def _delayedRender(self, request): + def _delayedRender(self, request: Request) -> None: abort = getarg(request, b"abort", 0, type_=int) request.write(b"this connection will be dropped\n") tr = request.channel.transport @@ -233,8 +263,10 @@ class Drop(Partial): class ArbitraryLengthPayloadResource(LeafResource): - def render(self, request): - return request.content.read() + def render(self, request: Request) -> bytes: + assert request.content + data: bytes = request.content.read() + return data class NoMetaRefreshRedirect(Redirect): @@ -245,21 +277,23 @@ class NoMetaRefreshRedirect(Redirect): ) -class ContentLengthHeaderResource(resource.Resource): +class ContentLengthHeaderResource(BaseResource): """ A testing resource which renders itself as the value of the Content-Length header from the request. """ - def render(self, request): - return request.requestHeaders.getRawHeaders(b"content-length")[0] + def render(self, request: Request) -> bytes: + headers = request.requestHeaders.getRawHeaders(b"content-length") + assert headers + return headers[0] -class ChunkedResource(resource.Resource): - def render(self, request): +class ChunkedResource(BaseResource): + def render(self, request: Request) -> int: from twisted.internet import reactor - def response(): + def response() -> None: request.write(b"chunked ") request.write(b"content\n") request.finish() @@ -268,11 +302,11 @@ class ChunkedResource(resource.Resource): return server.NOT_DONE_YET -class BrokenChunkedResource(resource.Resource): - def render(self, request): +class BrokenChunkedResource(BaseResource): + def render(self, request: Request) -> int: from twisted.internet import reactor - def response(): + def response() -> None: request.write(b"chunked ") request.write(b"content\n") # Disable terminating chunk on finish. @@ -283,11 +317,11 @@ class BrokenChunkedResource(resource.Resource): return server.NOT_DONE_YET -class BrokenDownloadResource(resource.Resource): - def render(self, request): +class BrokenDownloadResource(BaseResource): + def render(self, request: Request) -> int: from twisted.internet import reactor - def response(): + def response() -> None: request.setHeader(b"Content-Length", b"20") request.write(b"partial") close_connection(request) @@ -296,22 +330,24 @@ class BrokenDownloadResource(resource.Resource): return server.NOT_DONE_YET -class EmptyContentTypeHeaderResource(resource.Resource): +class EmptyContentTypeHeaderResource(BaseResource): """ A testing resource which renders itself as the value of request body without content-type header in response. """ - def render(self, request): + def render(self, request: Request) -> bytes: + assert request.content request.setHeader("content-type", "") - return request.content.read() + data: bytes = request.content.read() + return data -class LargeChunkedFileResource(resource.Resource): - def render(self, request): +class LargeChunkedFileResource(BaseResource): + def render(self, request: Request) -> int: from twisted.internet import reactor - def response(): + def response() -> None: for _ in range(1024): request.write(b"x" * 1024) request.finish() @@ -320,43 +356,45 @@ class LargeChunkedFileResource(resource.Resource): return server.NOT_DONE_YET -class DuplicateHeaderResource(resource.Resource): - def render(self, request): +class DuplicateHeaderResource(BaseResource): + def render(self, request: Request) -> bytes: request.responseHeaders.setRawHeaders(b"Set-Cookie", [b"a=b", b"c=d"]) return b"" -class UriResource(resource.Resource): +class UriResource(BaseResource): """Return the full uri that was requested""" - def getChild(self, path, request): + def getChild(self, path: bytes, request: Request) -> resource.Resource: return self - def render(self, request): + def render(self, request: Request) -> bytes | int: # Note: this is an ugly hack for CONNECT request timeout test. # Returning some data here fail SSL/TLS handshake # ToDo: implement proper HTTPS proxy tests, not faking them. if request.method != b"CONNECT": return request.uri + assert request.transport is not None request.transport.write(b"HTTP/1.1 200 Connection established\r\n\r\n") return NOT_DONE_YET -class ResponseHeadersResource(resource.Resource): +class ResponseHeadersResource(BaseResource): """Return a response with headers set from the JSON request body""" - def render(self, request): + def render(self, request: Request) -> bytes: + assert request.content body = json.loads(request.content.read().decode()) for header_name, header_value in body.items(): request.responseHeaders.setRawHeaders(header_name, [header_value]) return json.dumps(body).encode("utf-8") -class Compress(resource.Resource): +class Compress(BaseResource): """Compress the data sent in the request url params and set Content-Encoding header""" - def render(self, request): - data = request.args.get(b"data")[0] + def render(self, request: Request) -> bytes: + data = request.args[b"data"][0] accept_encoding_header = request.getHeader(b"accept-encoding") @@ -370,10 +408,10 @@ class Compress(resource.Resource): return b"Did not receive a valid accept-encoding header" -class SetCookie(resource.Resource): +class SetCookie(BaseResource): """Return a response with a Set-Cookie header for each request url parameter""" - def render(self, request): + def render(self, request: Request) -> bytes: for cookie_name, cookie_values in request.args.items(): for cookie_value in cookie_values: cookie = (cookie_name.decode() + "=" + cookie_value.decode()).encode() diff --git a/tests/mockserver/simple_https.py b/tests/mockserver/simple_https.py index fdea666e1..2a6cb6dd8 100644 --- a/tests/mockserver/simple_https.py +++ b/tests/mockserver/simple_https.py @@ -2,18 +2,23 @@ from __future__ import annotations -from twisted.web import resource +from typing import TYPE_CHECKING + from twisted.web.static import Data from .http_base import BaseMockServer, main_factory +from .http_resources import BaseResource, put_child + +if TYPE_CHECKING: + from twisted.web.server import Request -class Root(resource.Resource): - def __init__(self): - resource.Resource.__init__(self) - self.putChild(b"file", Data(b"0123456789", "text/plain")) +class Root(BaseResource): + def __init__(self) -> None: + super().__init__() + put_child(self, b"file", Data(b"0123456789", "text/plain")) - def getChild(self, path, request): + def getChild(self, path: bytes, request: Request) -> Root: return self @@ -29,7 +34,7 @@ class SimpleMockServer(BaseMockServer): cipher_string: str | None = None, tls_min_version: str | None = None, tls_max_version: str | None = None, - ): + ) -> None: super().__init__() self.keyfile = keyfile self.certfile = certfile diff --git a/tests/test_core_downloader.py b/tests/test_core_downloader.py index 3e4139b3e..912c0450b 100644 --- a/tests/test_core_downloader.py +++ b/tests/test_core_downloader.py @@ -14,7 +14,7 @@ from twisted.web import server, static from twisted.web.client import Agent, BrowserLikePolicyForHTTPS, readBody from twisted.web.client import Response as TxResponse -from scrapy import Request +from scrapy import Request, Spider from scrapy.core.downloader import Downloader, Slot, tls from scrapy.core.downloader.contextfactory import ( _load_context_factory_from_settings, @@ -31,14 +31,17 @@ from scrapy.utils.misc import build_from_crawler from scrapy.utils.python import to_bytes from scrapy.utils.spider import DefaultSpider from scrapy.utils.test import get_crawler -from tests.mockserver.http_resources import PayloadResource +from tests.mockserver.http_resources import PayloadResource, put_child from tests.mockserver.utils import ssl_context_factory from tests.utils.decorators import coroutine_test if TYPE_CHECKING: from twisted.internet.defer import Deferred + from twisted.internet.interfaces import IListeningPort from twisted.web.iweb import IBodyProducer + from scrapy.http import Response + class TestSlot: def test_repr(self): @@ -52,7 +55,7 @@ class TestContextFactoryBase: async def server_url(self, tmp_path): (tmp_path / "file").write_bytes(b"0123456789") r = static.File(str(tmp_path)) - r.putChild(b"payload", PayloadResource()) + put_child(r, b"payload", PayloadResource()) site = server.Site(r, timeout=None) port = self._listen(site) portno = port.getHost().port @@ -61,7 +64,7 @@ class TestContextFactoryBase: await port.stopListening() - def _listen(self, site): + def _listen(self, site: server.Site) -> IListeningPort: from twisted.internet import reactor return reactor.listenSSL( @@ -317,7 +320,10 @@ def test_needs_backout(concurrency: int, active: int, expected: bool) -> None: @coroutine_test async def test_fetch_deprecated_spider_arg(): class CustomDownloader(Downloader): - def fetch(self, request, spider): # pylint: disable=signature-differs + # requiring the spider argument is what triggers the deprecation + def fetch( # type: ignore[override] # pylint: disable=signature-differs + self, request: Request, spider: Spider + ) -> Deferred[Response | Request]: return super().fetch(request, spider) crawler = get_crawler(DefaultSpider, {"DOWNLOADER": CustomDownloader}) diff --git a/tests/test_downloader_handler_twisted_ftp.py b/tests/test_downloader_handler_twisted_ftp.py index 489b70e74..14de97b21 100644 --- a/tests/test_downloader_handler_twisted_ftp.py +++ b/tests/test_downloader_handler_twisted_ftp.py @@ -156,14 +156,17 @@ class TestFTP(TestFTPBase): for filename, content in self.test_files: (userdir / filename).write_bytes(content) - def _get_factory(self, root): + def _get_factory(self, root: Path) -> FTPFactory: from twisted.protocols.ftp import FTPFactory, FTPRealm realm = FTPRealm(anonymousRoot=str(root), userHome=str(root)) - p = portal.Portal(realm) + # zope.interface has no type hints, so mypy cannot tell that these + # objects provide the interfaces that Portal expects. + p = portal.Portal(realm) # type: ignore[arg-type] users_checker = checkers.InMemoryUsernamePasswordDatabaseDontUse() - users_checker.addUser(self.username, self.password) - p.registerChecker(users_checker, credentials.IUsernamePassword) + # the FTP protocol authenticates with str credentials + users_checker.addUser(self.username, self.password) # type: ignore[arg-type] + p.registerChecker(users_checker, credentials.IUsernamePassword) # type: ignore[arg-type] return FTPFactory(portal=p) @deferred_f_from_coro_f @@ -192,12 +195,17 @@ class TestAnonymousFTP(TestFTPBase): for filename, content in self.test_files: (root / filename).write_bytes(content) - def _get_factory(self, tmp_path): + def _get_factory(self, tmp_path: Path) -> FTPFactory: from twisted.protocols.ftp import FTPFactory, FTPRealm realm = FTPRealm(anonymousRoot=str(tmp_path)) - p = portal.Portal(realm) - p.registerChecker(checkers.AllowAnonymousAccess(), credentials.IAnonymous) + # zope.interface has no type hints, so mypy cannot tell that these + # objects provide the interfaces that Portal expects. + p = portal.Portal(realm) # type: ignore[arg-type] + p.registerChecker( + checkers.AllowAnonymousAccess(), # type: ignore[arg-type] + credentials.IAnonymous, + ) return FTPFactory(portal=p, userAnonymous=self.username) diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py index 3c1347fd3..b8586d1ca 100644 --- a/tests/test_http2_client_protocol.py +++ b/tests/test_http2_client_protocol.py @@ -30,7 +30,7 @@ from scrapy.utils.defer import ( deferred_from_coro, maybe_deferred_to_future, ) -from tests.mockserver.http_resources import LeafResource, Status +from tests.mockserver.http_resources import LeafResource, Status, put_child from tests.mockserver.utils import ssl_context_factory if TYPE_CHECKING: @@ -199,18 +199,18 @@ class TestHttps2ClientProtocol: @pytest.fixture def site(self, tmp_path): r = File(str(tmp_path)) - r.putChild(b"get-data-html-small", GetDataHtmlSmall()) - r.putChild(b"get-data-html-large", GetDataHtmlLarge()) + put_child(r, b"get-data-html-small", GetDataHtmlSmall()) + put_child(r, b"get-data-html-large", GetDataHtmlLarge()) - r.putChild(b"post-data-json-small", PostDataJsonSmall()) - r.putChild(b"post-data-json-large", PostDataJsonLarge()) + put_child(r, b"post-data-json-small", PostDataJsonSmall()) + put_child(r, b"post-data-json-large", PostDataJsonLarge()) - r.putChild(b"dataloss", Dataloss()) - r.putChild(b"no-content-length-header", NoContentLengthHeader()) - r.putChild(b"status", Status()) - r.putChild(b"query-params", QueryParams()) - r.putChild(b"timeout", TimeoutResponse()) - r.putChild(b"request-headers", RequestHeaders()) + put_child(r, b"dataloss", Dataloss()) + put_child(r, b"no-content-length-header", NoContentLengthHeader()) + put_child(r, b"status", Status()) + put_child(r, b"query-params", QueryParams()) + put_child(r, b"timeout", TimeoutResponse()) + put_child(r, b"request-headers", RequestHeaders()) return Site(r, timeout=None) @async_yield_fixture # type: ignore[untyped-decorator] diff --git a/tox.ini b/tox.ini index edde83356..ac32064a6 100644 --- a/tox.ini +++ b/tox.ini @@ -111,7 +111,8 @@ commands = pre-commit run {posargs:--all-files} [testenv:pylint] -basepython = python3 +# Some checks are Python-version-dependent, so pin the version used in CI. +basepython = python3.14 deps = {[testenv:extra-deps]deps} pylint==4.0.6