RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Type tests.mockserver and other resource-defining files (#7865) 

* Type tests.mockserver and other resource-defining files

* Address pylint issues
This commit is contained in:
Adrian 2026-08-03 20:44:28 +02:00 committed by GitHub
parent 298c9e610e
commit a7385d6e51
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
11 changed files with 260 additions and 154 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
pyproject.toml
View File

@ -118,13 +118,10 @@ allow_incomplete_defs = true # 59 errors
# TODO
[[tool.mypy.overrides]]
module = [
"tests.mockserver.*",
"tests.spiders",
"tests.test_closespider",
"tests.test_cmdline",
"tests.test_contracts",
"tests.test_core_downloader",
"tests.test_downloader_handler_twisted_ftp",
"tests.test_downloaderslotssettings",
"tests.test_dupefilters",
"tests.test_engine_loop",
@ -135,7 +132,6 @@ module = [
"tests.test_feedexport_postprocess",
"tests.test_feedexport_storages",
"tests.test_feedexport_uri_params",
"tests.test_http2_client_protocol",
"tests.test_item",
"tests.test_linkextractors",
"tests.test_loader",

39
tests/mockserver/dns.py
View File

@ -2,6 +2,7 @@ from __future__ import annotations
import sys
from subprocess import PIPE, Popen
from typing import TYPE_CHECKING
from twisted.internet import defer
from twisted.names import dns, error
@ -9,39 +10,63 @@ from twisted.names.server import DNSServerFactory
from tests.utils import get_script_run_env
if TYPE_CHECKING:
from collections.abc import Sequence
from types import TracebackType
from twisted.internet.defer import Deferred
# typing.Self requires Python 3.11
from typing_extensions import Self
_Answers = tuple[list[dns.RRHeader], list[dns.RRHeader], list[dns.RRHeader]]
class MockDNSResolver:
"""
Implements twisted.internet.interfaces.IResolver partially
"""
def _resolve(self, name):
def _resolve(self, name: bytes) -> _Answers:
record = dns.Record_A(address=b"127.0.0.1")
answer = dns.RRHeader(name=name, payload=record)
# zope.interface has no type hints, so mypy cannot tell that Record_A
# provides the IEncodableRecord interface.
answer = dns.RRHeader(name=name, payload=record) # type: ignore[arg-type]
return [answer], [], []
def query(self, query, timeout=None):
def query(
self, query: dns.Query, timeout: Sequence[int] | None = None
) -> Deferred[_Answers]:
if query.type == dns.A:
return defer.succeed(self._resolve(query.name.name))
return defer.fail(error.DomainError())
def lookupAllRecords(self, name, timeout=None):
def lookupAllRecords(
self, name: bytes, timeout: Sequence[int] | None = None
) -> Deferred[_Answers]:
return defer.succeed(self._resolve(name))
class MockDNSServer:
def __enter__(self):
def __enter__(self) -> Self:
self.proc = Popen(
[sys.executable, "-u", "-m", "tests.mockserver.dns"],
stdout=PIPE,
env=get_script_run_env(),
text=True,
)
assert self.proc.stdout is not None
self.host = "127.0.0.1"
self.port = int(self.proc.stdout.readline().strip().split(":")[1])
return self
def __exit__(self, exc_type, exc_value, traceback):
def __exit__(
self,
exc_type: type[BaseException] | None,
exc_value: BaseException | None,
traceback: TracebackType | None,
) -> None:
self.proc.kill()
self.proc.communicate()
@ -54,7 +79,7 @@ def main() -> None:
protocol = dns.DNSDatagramProtocol(controller=factory)
listener = reactor.listenUDP(0, protocol)
def print_listening():
def print_listening() -> None:
host = listener.getHost()
print(f"{host.host}:{host.port}")

20
tests/mockserver/ftp.py
View File

@ -7,6 +7,7 @@ from pathlib import Path
from shutil import rmtree
from subprocess import PIPE, Popen
from tempfile import mkdtemp
from typing import TYPE_CHECKING
from pyftpdlib.authorizers import DummyAuthorizer
from pyftpdlib.handlers import FTPHandler
@ -14,6 +15,12 @@ from pyftpdlib.servers import FTPServer
from tests.utils import get_script_run_env
if TYPE_CHECKING:
from types import TracebackType
# typing.Self requires Python 3.11
from typing_extensions import Self
class MockFTPServer:
"""Creates an FTP server on a random port with a default passwordless user
@ -26,7 +33,7 @@ class MockFTPServer:
self.port: int | None = None
self.path: Path | None = None
def __enter__(self):
def __enter__(self) -> Self:
self.path = Path(mkdtemp())
self.proc = Popen(
[sys.executable, "-u", "-m", "tests.mockserver.ftp", "-d", str(self.path)],
@ -34,6 +41,7 @@ class MockFTPServer:
env=get_script_run_env(),
text=True,
)
assert self.proc.stderr is not None
for line in self.proc.stderr:
if "starting FTP server" in line and (
m := re.search(r"starting FTP server on ([^ :]+):(\d+),", line)
@ -48,12 +56,18 @@ class MockFTPServer:
)
return self
def __exit__(self, exc_type, exc_value, traceback):
def __exit__(
self,
exc_type: type[BaseException] | None,
exc_value: BaseException | None,
traceback: TracebackType | None,
) -> None:
rmtree(str(self.path))
assert self.proc is not None
self.proc.kill()
self.proc.communicate()
def url(self, path):
def url(self, path: str) -> str:
return f"ftp://{self.host}:{self.port}/{path}"

85
tests/mockserver/http.py
View File

@ -1,8 +1,8 @@
from __future__ import annotations
from pathlib import Path
from typing import TYPE_CHECKING
from twisted.web import resource
from twisted.web.static import Data, File
from twisted.web.util import Redirect
@ -11,6 +11,7 @@ from tests import tests_datadir
from .http_base import BaseMockServer, main_factory
from .http_resources import (
ArbitraryLengthPayloadResource,
BaseResource,
BrokenChunkedResource,
BrokenDownloadResource,
ChunkedResource,
@ -35,62 +36,68 @@ from .http_resources import (
SetCookie,
Status,
UriResource,
put_child,
)
if TYPE_CHECKING:
from twisted.web.server import Request
class Root(resource.Resource):
def __init__(self):
class Root(BaseResource):
def __init__(self) -> None:
super().__init__()
self.putChild(b"status", Status())
self.putChild(b"follow", Follow())
self.putChild(b"delay", Delay())
self.putChild(b"partial", Partial())
self.putChild(b"drop", Drop())
self.putChild(b"raw", Raw())
self.putChild(b"echo", Echo())
self.putChild(b"payload", PayloadResource())
self.putChild(b"alpayload", ArbitraryLengthPayloadResource())
self.putChild(b"static", File(str(Path(tests_datadir, "test_site/"))))
self.putChild(b"redirect-to", RedirectTo())
self.putChild(b"text", Data(b"Works", "text/plain"))
self.putChild(
put_child(self, b"status", Status())
put_child(self, b"follow", Follow())
put_child(self, b"delay", Delay())
put_child(self, b"partial", Partial())
put_child(self, b"drop", Drop())
put_child(self, b"raw", Raw())
put_child(self, b"echo", Echo())
put_child(self, b"payload", PayloadResource())
put_child(self, b"alpayload", ArbitraryLengthPayloadResource())
put_child(self, b"static", File(str(Path(tests_datadir, "test_site/"))))
put_child(self, b"redirect-to", RedirectTo())
put_child(self, b"text", Data(b"Works", "text/plain"))
put_child(
self,
b"html",
Data(
b"<body><p class='one'>Works</p><p class='two'>World</p></body>",
"text/html",
),
)
self.putChild(
put_child(
self,
b"enc-gb18030",
Data(b"<p>gb18030 encoding</p>", "text/html; charset=gb18030"),
)
self.putChild(b"redirect", Redirect(b"/redirected"))
self.putChild(
b"redirect-no-meta-refresh", NoMetaRefreshRedirect(b"/redirected")
put_child(self, b"redirect", Redirect(b"/redirected"))
put_child(
self, b"redirect-no-meta-refresh", NoMetaRefreshRedirect(b"/redirected")
)
self.putChild(b"redirected", Data(b"Redirected here", "text/plain"))
put_child(self, b"redirected", Data(b"Redirected here", "text/plain"))
numbers = [str(x).encode("utf8") for x in range(2**18)]
self.putChild(b"numbers", Data(b"".join(numbers), "text/plain"))
self.putChild(b"wait", ForeverTakingResource())
self.putChild(b"hang-after-headers", ForeverTakingResource(write=True))
self.putChild(b"host", HostHeaderResource())
self.putChild(b"client-ip", ClientIPResource())
self.putChild(b"broken", BrokenDownloadResource())
self.putChild(b"chunked", ChunkedResource())
self.putChild(b"broken-chunked", BrokenChunkedResource())
self.putChild(b"contentlength", ContentLengthHeaderResource())
self.putChild(b"nocontenttype", EmptyContentTypeHeaderResource())
self.putChild(b"largechunkedfile", LargeChunkedFileResource())
self.putChild(b"compress", Compress())
self.putChild(b"duplicate-header", DuplicateHeaderResource())
self.putChild(b"response-headers", ResponseHeadersResource())
self.putChild(b"set-cookie", SetCookie())
self.putChild(b"uri", UriResource())
put_child(self, b"numbers", Data(b"".join(numbers), "text/plain"))
put_child(self, b"wait", ForeverTakingResource())
put_child(self, b"hang-after-headers", ForeverTakingResource(write=True))
put_child(self, b"host", HostHeaderResource())
put_child(self, b"client-ip", ClientIPResource())
put_child(self, b"broken", BrokenDownloadResource())
put_child(self, b"chunked", ChunkedResource())
put_child(self, b"broken-chunked", BrokenChunkedResource())
put_child(self, b"contentlength", ContentLengthHeaderResource())
put_child(self, b"nocontenttype", EmptyContentTypeHeaderResource())
put_child(self, b"largechunkedfile", LargeChunkedFileResource())
put_child(self, b"compress", Compress())
put_child(self, b"duplicate-header", DuplicateHeaderResource())
put_child(self, b"response-headers", ResponseHeadersResource())
put_child(self, b"set-cookie", SetCookie())
put_child(self, b"uri", UriResource())
def getChild(self, path, request):
def getChild(self, path: bytes, request: Request) -> Root:
return self
def render(self, request):
def render(self, request: Request) -> bytes:
return b"Scrapy mock HTTP server\n"

10
tests/mockserver/http_base.py
View File

@ -17,6 +17,7 @@ from .utils import ssl_context_factory
if TYPE_CHECKING:
from collections.abc import Callable
from types import TracebackType
from twisted.web import resource
@ -60,7 +61,12 @@ class BaseMockServer(ABC):
self.https_port = https_parsed.port
return self
def __exit__(self, exc_type, exc_value, traceback) -> None:
def __exit__(
self,
exc_type: type[BaseException] | None,
exc_value: BaseException | None,
traceback: TracebackType | None,
) -> None:
if self.proc:
self.proc.kill()
self.proc.communicate()
@ -135,7 +141,7 @@ def main_factory(
context_factory = ssl_context_factory(**context_factory_kw)
https_port = reactor.listenSSL(0, factory, context_factory)
def print_listening():
def print_listening() -> None:
if listen_http:
http_host = http_port.getHost()
http_address = f"http://{http_host.host}:{http_host.port}"

174
tests/mockserver/http_resources.py
View File

@ -3,7 +3,7 @@ from __future__ import annotations
import gzip
import json
import random
from typing import TYPE_CHECKING, ParamSpec, TypeVar
from typing import TYPE_CHECKING, Any, ParamSpec, TypeVar
from urllib.parse import urlencode
from twisted.internet.task import deferLater
@ -14,17 +14,24 @@ from twisted.web.util import Redirect, redirectTo
from scrapy.utils.python import to_bytes, to_unicode
if TYPE_CHECKING:
from collections.abc import Callable
from collections.abc import Callable, Sequence
from twisted.internet.defer import Deferred
from twisted.web.http import Request
from twisted.python.failure import Failure
from twisted.web.http import Request as HTTPRequest
from twisted.web.server import Request
_T = TypeVar("_T")
_P = ParamSpec("_P")
def getarg(request, name, default=None, type_=None):
def getarg(
request: Request,
name: bytes,
default: Any = None,
type_: Callable[[bytes], Any] | None = None,
) -> Any:
if name in request.args:
value = request.args[name][0]
if type_ is not None:
@ -33,73 +40,91 @@ def getarg(request, name, default=None, type_=None):
return default
def close_connection(request):
def close_connection(request: Request) -> None:
# We have to force a disconnection for HTTP/1.1 clients. Otherwise
# client keeps the connection open waiting for more data.
request.channel.loseConnection()
request.finish()
def put_child(parent: resource.Resource, path: bytes, child: resource.Resource) -> None:
# zope.interface has no type hints, so mypy cannot tell that Resource
# instances provide the IResource interface that putChild() expects.
parent.putChild(path, child) # type: ignore[arg-type]
class BaseResource(resource.Resource):
"""Base class for mockserver resources, with type hints."""
# Only needed to give subclasses a typed __init__ to call.
def __init__(self) -> None: # pylint: disable=useless-parent-delegation
super().__init__() # type: ignore[no-untyped-call]
# most of the following resources are copied from twisted.web.test.test_webclient
class ForeverTakingResource(resource.Resource):
class ForeverTakingResource(BaseResource):
"""
L{ForeverTakingResource} is a resource which never finishes responding
to requests.
"""
def __init__(self, write=False):
resource.Resource.__init__(self)
def __init__(self, write: bool = False):
super().__init__()
self._write = write
def render(self, request):
def render(self, request: Request) -> int:
if self._write:
request.write(b"some bytes")
return server.NOT_DONE_YET
class HostHeaderResource(resource.Resource):
class HostHeaderResource(BaseResource):
"""
A testing resource which renders itself as the value of the host header
from the request.
"""
def render(self, request):
return request.requestHeaders.getRawHeaders(b"host")[0]
def render(self, request: Request) -> bytes:
headers = request.requestHeaders.getRawHeaders(b"host")
assert headers
return headers[0]
class ClientIPResource(resource.Resource):
class ClientIPResource(BaseResource):
"""
A testing resource which renders itself as the request client IP address.
"""
def render(self, request):
def render(self, request: Request) -> bytes:
client_address = request.getClientAddress()
if client_address is None or client_address.host is None:
return b""
return to_bytes(client_address.host)
class PayloadResource(resource.Resource):
class PayloadResource(BaseResource):
"""
A testing resource which renders itself as the contents of the request body
as long as the request body is 100 bytes long, otherwise which renders
itself as C{"ERROR"}.
"""
def render(self, request):
data = request.content.read()
contentLength = request.requestHeaders.getRawHeaders(b"content-length")[0]
if len(data) != 100 or int(contentLength) != 100:
def render(self, request: Request) -> bytes:
assert request.content
data: bytes = request.content.read()
content_length = request.requestHeaders.getRawHeaders(b"content-length")
assert content_length
if len(data) != 100 or int(content_length[0]) != 100:
return b"ERROR"
return data
class LeafResource(resource.Resource):
class LeafResource(BaseResource):
isLeaf = True
def deferRequest(
self,
request: Request,
request: HTTPRequest,
delay: float,
f: Callable[_P, _T],
*a: _P.args,
@ -107,7 +132,7 @@ class LeafResource(resource.Resource):
) -> Deferred[_T]:
from twisted.internet import reactor
def _cancelrequest(_):
def _cancelrequest(_: Failure) -> None:
# silence CancelledError
d.addErrback(lambda _: None)
d.cancel()
@ -118,12 +143,13 @@ class LeafResource(resource.Resource):
class Follow(LeafResource):
def render(self, request):
def render(self, request: Request) -> int:
total = getarg(request, b"total", 100, type_=int)
show = getarg(request, b"show", 1, type_=int)
order = getarg(request, b"order", b"desc")
maxlatency = getarg(request, b"maxlatency", 0, type_=float)
n = getarg(request, b"n", total, type_=int)
nlist: Sequence[int]
if order == b"rand":
nlist = [random.randint(1, total) for _ in range(show)]
else: # order == "desc"
@ -133,7 +159,7 @@ class Follow(LeafResource):
self.deferRequest(request, lag, self.renderRequest, request, nlist)
return NOT_DONE_YET
def renderRequest(self, request, nlist):
def renderRequest(self, request: Request, nlist: Sequence[int]) -> None:
s = """<html> <head></head> <body>"""
args = request.args.copy()
for nl in nlist:
@ -146,45 +172,47 @@ class Follow(LeafResource):
class Delay(LeafResource):
def render_GET(self, request):
def render_GET(self, request: Request) -> int:
n = getarg(request, b"n", 1, type_=float)
b = getarg(request, b"b", 1, type_=int)
if b:
# send headers now and delay body
request.write("")
request.write(b"")
self.deferRequest(request, n, self._delayedRender, request, n)
return NOT_DONE_YET
def _delayedRender(self, request, n):
def _delayedRender(self, request: Request, n: float) -> None:
request.write(to_bytes(f"Response delayed for {n:.3f} seconds\n"))
request.finish()
class Status(LeafResource):
def render_GET(self, request):
def render_GET(self, request: Request) -> bytes:
n = getarg(request, b"n", 200, type_=int)
request.setResponseCode(n)
return b""
class Raw(LeafResource):
def render_GET(self, request):
def render_GET(self, request: Request) -> int:
request.startedWriting = 1
self.deferRequest(request, 0, self._delayedRender, request)
return NOT_DONE_YET
render_POST = render_GET
def _delayedRender(self, request):
def _delayedRender(self, request: Request) -> None:
raw = getarg(request, b"raw", b"HTTP 1.1 200 OK\n")
request.startedWriting = 1
request.write(raw)
assert request.channel.transport is not None
request.channel.transport.loseConnection()
request.finish()
class Echo(LeafResource):
def render_GET(self, request):
def render_GET(self, request: Request) -> bytes:
assert request.content
output = {
"headers": {
to_unicode(k): [to_unicode(v) for v in vs]
@ -198,27 +226,29 @@ class Echo(LeafResource):
class RedirectTo(LeafResource):
def render(self, request):
def render(self, request: Request) -> bytes:
goto = getarg(request, b"goto", b"/")
# we force the body content, otherwise Twisted redirectTo()
# returns HTML with <meta http-equiv="refresh"
redirectTo(goto, request)
# zope.interface has no type hints, so mypy cannot tell that Request
# provides the IRequest interface.
redirectTo(goto, request) # type: ignore[arg-type]
return b"redirecting..."
class Partial(LeafResource):
def render_GET(self, request):
def render_GET(self, request: Request) -> int:
request.setHeader(b"Content-Length", b"1024")
self.deferRequest(request, 0, self._delayedRender, request)
return NOT_DONE_YET
def _delayedRender(self, request):
def _delayedRender(self, request: Request) -> None:
request.write(b"partial content\n")
request.finish()
class Drop(Partial):
def _delayedRender(self, request):
def _delayedRender(self, request: Request) -> None:
abort = getarg(request, b"abort", 0, type_=int)
request.write(b"this connection will be dropped\n")
tr = request.channel.transport
@ -233,8 +263,10 @@ class Drop(Partial):
class ArbitraryLengthPayloadResource(LeafResource):
def render(self, request):
return request.content.read()
def render(self, request: Request) -> bytes:
assert request.content
data: bytes = request.content.read()
return data
class NoMetaRefreshRedirect(Redirect):
@ -245,21 +277,23 @@ class NoMetaRefreshRedirect(Redirect):
)
class ContentLengthHeaderResource(resource.Resource):
class ContentLengthHeaderResource(BaseResource):
"""
A testing resource which renders itself as the value of the Content-Length
header from the request.
"""
def render(self, request):
return request.requestHeaders.getRawHeaders(b"content-length")[0]
def render(self, request: Request) -> bytes:
headers = request.requestHeaders.getRawHeaders(b"content-length")
assert headers
return headers[0]
class ChunkedResource(resource.Resource):
def render(self, request):
class ChunkedResource(BaseResource):
def render(self, request: Request) -> int:
from twisted.internet import reactor
def response():
def response() -> None:
request.write(b"chunked ")
request.write(b"content\n")
request.finish()
@ -268,11 +302,11 @@ class ChunkedResource(resource.Resource):
return server.NOT_DONE_YET
class BrokenChunkedResource(resource.Resource):
def render(self, request):
class BrokenChunkedResource(BaseResource):
def render(self, request: Request) -> int:
from twisted.internet import reactor
def response():
def response() -> None:
request.write(b"chunked ")
request.write(b"content\n")
# Disable terminating chunk on finish.
@ -283,11 +317,11 @@ class BrokenChunkedResource(resource.Resource):
return server.NOT_DONE_YET
class BrokenDownloadResource(resource.Resource):
def render(self, request):
class BrokenDownloadResource(BaseResource):
def render(self, request: Request) -> int:
from twisted.internet import reactor
def response():
def response() -> None:
request.setHeader(b"Content-Length", b"20")
request.write(b"partial")
close_connection(request)
@ -296,22 +330,24 @@ class BrokenDownloadResource(resource.Resource):
return server.NOT_DONE_YET
class EmptyContentTypeHeaderResource(resource.Resource):
class EmptyContentTypeHeaderResource(BaseResource):
"""
A testing resource which renders itself as the value of request body
without content-type header in response.
"""
def render(self, request):
def render(self, request: Request) -> bytes:
assert request.content
request.setHeader("content-type", "")
return request.content.read()
data: bytes = request.content.read()
return data
class LargeChunkedFileResource(resource.Resource):
def render(self, request):
class LargeChunkedFileResource(BaseResource):
def render(self, request: Request) -> int:
from twisted.internet import reactor
def response():
def response() -> None:
for _ in range(1024):
request.write(b"x" * 1024)
request.finish()
@ -320,43 +356,45 @@ class LargeChunkedFileResource(resource.Resource):
return server.NOT_DONE_YET
class DuplicateHeaderResource(resource.Resource):
def render(self, request):
class DuplicateHeaderResource(BaseResource):
def render(self, request: Request) -> bytes:
request.responseHeaders.setRawHeaders(b"Set-Cookie", [b"a=b", b"c=d"])
return b""
class UriResource(resource.Resource):
class UriResource(BaseResource):
"""Return the full uri that was requested"""
def getChild(self, path, request):
def getChild(self, path: bytes, request: Request) -> resource.Resource:
return self
def render(self, request):
def render(self, request: Request) -> bytes | int:
# Note: this is an ugly hack for CONNECT request timeout test.
# Returning some data here fail SSL/TLS handshake
# ToDo: implement proper HTTPS proxy tests, not faking them.
if request.method != b"CONNECT":
return request.uri
assert request.transport is not None
request.transport.write(b"HTTP/1.1 200 Connection established\r\n\r\n")
return NOT_DONE_YET
class ResponseHeadersResource(resource.Resource):
class ResponseHeadersResource(BaseResource):
"""Return a response with headers set from the JSON request body"""
def render(self, request):
def render(self, request: Request) -> bytes:
assert request.content
body = json.loads(request.content.read().decode())
for header_name, header_value in body.items():
request.responseHeaders.setRawHeaders(header_name, [header_value])
return json.dumps(body).encode("utf-8")
class Compress(resource.Resource):
class Compress(BaseResource):
"""Compress the data sent in the request url params and set Content-Encoding header"""
def render(self, request):
data = request.args.get(b"data")[0]
def render(self, request: Request) -> bytes:
data = request.args[b"data"][0]
accept_encoding_header = request.getHeader(b"accept-encoding")
@ -370,10 +408,10 @@ class Compress(resource.Resource):
return b"Did not receive a valid accept-encoding header"
class SetCookie(resource.Resource):
class SetCookie(BaseResource):
"""Return a response with a Set-Cookie header for each request url parameter"""
def render(self, request):
def render(self, request: Request) -> bytes:
for cookie_name, cookie_values in request.args.items():
for cookie_value in cookie_values:
cookie = (cookie_name.decode() + "=" + cookie_value.decode()).encode()

19
tests/mockserver/simple_https.py
View File

@ -2,18 +2,23 @@
from __future__ import annotations
from twisted.web import resource
from typing import TYPE_CHECKING
from twisted.web.static import Data
from .http_base import BaseMockServer, main_factory
from .http_resources import BaseResource, put_child
if TYPE_CHECKING:
from twisted.web.server import Request
class Root(resource.Resource):
def __init__(self):
resource.Resource.__init__(self)
self.putChild(b"file", Data(b"0123456789", "text/plain"))
class Root(BaseResource):
def __init__(self) -> None:
super().__init__()
put_child(self, b"file", Data(b"0123456789", "text/plain"))
def getChild(self, path, request):
def getChild(self, path: bytes, request: Request) -> Root:
return self
@ -29,7 +34,7 @@ class SimpleMockServer(BaseMockServer):
cipher_string: str | None = None,
tls_min_version: str | None = None,
tls_max_version: str | None = None,
):
) -> None:
super().__init__()
self.keyfile = keyfile
self.certfile = certfile

16
tests/test_core_downloader.py
View File

@ -14,7 +14,7 @@ from twisted.web import server, static
from twisted.web.client import Agent, BrowserLikePolicyForHTTPS, readBody
from twisted.web.client import Response as TxResponse
from scrapy import Request
from scrapy import Request, Spider
from scrapy.core.downloader import Downloader, Slot, tls
from scrapy.core.downloader.contextfactory import (
_load_context_factory_from_settings,
@ -31,14 +31,17 @@ from scrapy.utils.misc import build_from_crawler
from scrapy.utils.python import to_bytes
from scrapy.utils.spider import DefaultSpider
from scrapy.utils.test import get_crawler
from tests.mockserver.http_resources import PayloadResource
from tests.mockserver.http_resources import PayloadResource, put_child
from tests.mockserver.utils import ssl_context_factory
from tests.utils.decorators import coroutine_test
if TYPE_CHECKING:
from twisted.internet.defer import Deferred
from twisted.internet.interfaces import IListeningPort
from twisted.web.iweb import IBodyProducer
from scrapy.http import Response
class TestSlot:
def test_repr(self):
@ -52,7 +55,7 @@ class TestContextFactoryBase:
async def server_url(self, tmp_path):
(tmp_path / "file").write_bytes(b"0123456789")
r = static.File(str(tmp_path))
r.putChild(b"payload", PayloadResource())
put_child(r, b"payload", PayloadResource())
site = server.Site(r, timeout=None)
port = self._listen(site)
portno = port.getHost().port
@ -61,7 +64,7 @@ class TestContextFactoryBase:
await port.stopListening()
def _listen(self, site):
def _listen(self, site: server.Site) -> IListeningPort:
from twisted.internet import reactor
return reactor.listenSSL(
@ -317,7 +320,10 @@ def test_needs_backout(concurrency: int, active: int, expected: bool) -> None:
@coroutine_test
async def test_fetch_deprecated_spider_arg():
class CustomDownloader(Downloader):
def fetch(self, request, spider): # pylint: disable=signature-differs
# requiring the spider argument is what triggers the deprecation
def fetch( # type: ignore[override] # pylint: disable=signature-differs
self, request: Request, spider: Spider
) -> Deferred[Response | Request]:
return super().fetch(request, spider)
crawler = get_crawler(DefaultSpider, {"DOWNLOADER": CustomDownloader})

22
tests/test_downloader_handler_twisted_ftp.py
View File

@ -156,14 +156,17 @@ class TestFTP(TestFTPBase):
for filename, content in self.test_files:
(userdir / filename).write_bytes(content)
def _get_factory(self, root):
def _get_factory(self, root: Path) -> FTPFactory:
from twisted.protocols.ftp import FTPFactory, FTPRealm
realm = FTPRealm(anonymousRoot=str(root), userHome=str(root))
p = portal.Portal(realm)
# zope.interface has no type hints, so mypy cannot tell that these
# objects provide the interfaces that Portal expects.
p = portal.Portal(realm) # type: ignore[arg-type]
users_checker = checkers.InMemoryUsernamePasswordDatabaseDontUse()
users_checker.addUser(self.username, self.password)
p.registerChecker(users_checker, credentials.IUsernamePassword)
# the FTP protocol authenticates with str credentials
users_checker.addUser(self.username, self.password) # type: ignore[arg-type]
p.registerChecker(users_checker, credentials.IUsernamePassword) # type: ignore[arg-type]
return FTPFactory(portal=p)
@deferred_f_from_coro_f
@ -192,12 +195,17 @@ class TestAnonymousFTP(TestFTPBase):
for filename, content in self.test_files:
(root / filename).write_bytes(content)
def _get_factory(self, tmp_path):
def _get_factory(self, tmp_path: Path) -> FTPFactory:
from twisted.protocols.ftp import FTPFactory, FTPRealm
realm = FTPRealm(anonymousRoot=str(tmp_path))
p = portal.Portal(realm)
p.registerChecker(checkers.AllowAnonymousAccess(), credentials.IAnonymous)
# zope.interface has no type hints, so mypy cannot tell that these
# objects provide the interfaces that Portal expects.
p = portal.Portal(realm) # type: ignore[arg-type]
p.registerChecker(
checkers.AllowAnonymousAccess(), # type: ignore[arg-type]
credentials.IAnonymous,
)
return FTPFactory(portal=p, userAnonymous=self.username)

22
tests/test_http2_client_protocol.py
View File

@ -30,7 +30,7 @@ from scrapy.utils.defer import (
deferred_from_coro,
maybe_deferred_to_future,
)
from tests.mockserver.http_resources import LeafResource, Status
from tests.mockserver.http_resources import LeafResource, Status, put_child
from tests.mockserver.utils import ssl_context_factory
if TYPE_CHECKING:
@ -199,18 +199,18 @@ class TestHttps2ClientProtocol:
@pytest.fixture
def site(self, tmp_path):
r = File(str(tmp_path))
r.putChild(b"get-data-html-small", GetDataHtmlSmall())
r.putChild(b"get-data-html-large", GetDataHtmlLarge())
put_child(r, b"get-data-html-small", GetDataHtmlSmall())
put_child(r, b"get-data-html-large", GetDataHtmlLarge())
r.putChild(b"post-data-json-small", PostDataJsonSmall())
r.putChild(b"post-data-json-large", PostDataJsonLarge())
put_child(r, b"post-data-json-small", PostDataJsonSmall())
put_child(r, b"post-data-json-large", PostDataJsonLarge())
r.putChild(b"dataloss", Dataloss())
r.putChild(b"no-content-length-header", NoContentLengthHeader())
r.putChild(b"status", Status())
r.putChild(b"query-params", QueryParams())
r.putChild(b"timeout", TimeoutResponse())
r.putChild(b"request-headers", RequestHeaders())
put_child(r, b"dataloss", Dataloss())
put_child(r, b"no-content-length-header", NoContentLengthHeader())
put_child(r, b"status", Status())
put_child(r, b"query-params", QueryParams())
put_child(r, b"timeout", TimeoutResponse())
put_child(r, b"request-headers", RequestHeaders())
return Site(r, timeout=None)
@async_yield_fixture # type: ignore[untyped-decorator]

3
tox.ini
View File

@ -111,7 +111,8 @@ commands =
pre-commit run {posargs:--all-files}
[testenv:pylint]
basepython = python3
# Some checks are Python-version-dependent, so pin the version used in CI.
basepython = python3.14
deps =
{[testenv:extra-deps]deps}
pylint==4.0.6