from __future__ import annotations import argparse import json import os import random import sys from pathlib import Path from shutil import rmtree from subprocess import PIPE, Popen from tempfile import mkdtemp from typing import TYPE_CHECKING from urllib.parse import urlencode from OpenSSL import SSL from twisted.internet import defer, reactor, ssl from twisted.internet.task import deferLater from twisted.names import dns, error from twisted.names.server import DNSServerFactory from twisted.web import resource, server from twisted.web.server import NOT_DONE_YET, GzipEncoderFactory, Site from twisted.web.static import File from twisted.web.util import redirectTo from scrapy.utils.python import to_bytes, to_unicode if TYPE_CHECKING: from twisted.internet.protocol import ServerFactory def getarg(request, name, default=None, type=None): if name in request.args: value = request.args[name][0] if type is not None: value = type(value) return value return default def get_mockserver_env() -> dict[str, str]: """Return a OS environment dict suitable to run mockserver processes.""" tests_path = Path(__file__).parent.parent pythonpath = str(tests_path) + os.pathsep + os.environ.get("PYTHONPATH", "") env = os.environ.copy() env["PYTHONPATH"] = pythonpath return env # most of the following resources are copied from twisted.web.test.test_webclient class ForeverTakingResource(resource.Resource): """ L{ForeverTakingResource} is a resource which never finishes responding to requests. """ def __init__(self, write=False): resource.Resource.__init__(self) self._write = write def render(self, request): if self._write: request.write(b"some bytes") return server.NOT_DONE_YET class ErrorResource(resource.Resource): def render(self, request): request.setResponseCode(401) if request.args.get(b"showlength"): request.setHeader(b"content-length", b"0") return b"" class NoLengthResource(resource.Resource): def render(self, request): return b"nolength" class HostHeaderResource(resource.Resource): """ A testing resource which renders itself as the value of the host header from the request. """ def render(self, request): return request.requestHeaders.getRawHeaders(b"host")[0] class PayloadResource(resource.Resource): """ A testing resource which renders itself as the contents of the request body as long as the request body is 100 bytes long, otherwise which renders itself as C{"ERROR"}. """ def render(self, request): data = request.content.read() contentLength = request.requestHeaders.getRawHeaders(b"content-length")[0] if len(data) != 100 or int(contentLength) != 100: return b"ERROR" return data class BrokenDownloadResource(resource.Resource): def render(self, request): # only sends 3 bytes even though it claims to send 5 request.setHeader(b"content-length", b"5") request.write(b"abc") return b"" class LeafResource(resource.Resource): isLeaf = True def deferRequest(self, request, delay, f, *a, **kw): def _cancelrequest(_): # silence CancelledError d.addErrback(lambda _: None) d.cancel() d = deferLater(reactor, delay, f, *a, **kw) request.notifyFinish().addErrback(_cancelrequest) return d class Follow(LeafResource): def render(self, request): total = getarg(request, b"total", 100, type=int) show = getarg(request, b"show", 1, type=int) order = getarg(request, b"order", b"desc") maxlatency = getarg(request, b"maxlatency", 0, type=float) n = getarg(request, b"n", total, type=int) if order == b"rand": nlist = [random.randint(1, total) for _ in range(show)] else: # order == "desc" nlist = range(n, max(n - show, 0), -1) lag = random.random() * maxlatency self.deferRequest(request, lag, self.renderRequest, request, nlist) return NOT_DONE_YET def renderRequest(self, request, nlist): s = """ """ args = request.args.copy() for nl in nlist: args[b"n"] = [to_bytes(str(nl))] argstr = urlencode(args, doseq=True) s += f"follow {nl}
" s += """""" request.write(to_bytes(s)) request.finish() class Delay(LeafResource): def render_GET(self, request): n = getarg(request, b"n", 1, type=float) b = getarg(request, b"b", 1, type=int) if b: # send headers now and delay body request.write("") self.deferRequest(request, n, self._delayedRender, request, n) return NOT_DONE_YET def _delayedRender(self, request, n): request.write(to_bytes(f"Response delayed for {n:.3f} seconds\n")) request.finish() class Status(LeafResource): def render_GET(self, request): n = getarg(request, b"n", 200, type=int) request.setResponseCode(n) return b"" class Raw(LeafResource): def render_GET(self, request): request.startedWriting = 1 self.deferRequest(request, 0, self._delayedRender, request) return NOT_DONE_YET render_POST = render_GET def _delayedRender(self, request): raw = getarg(request, b"raw", b"HTTP 1.1 200 OK\n") request.startedWriting = 1 request.write(raw) request.channel.transport.loseConnection() request.finish() class Echo(LeafResource): def render_GET(self, request): output = { "headers": { to_unicode(k): [to_unicode(v) for v in vs] for k, vs in request.requestHeaders.getAllRawHeaders() }, "body": to_unicode(request.content.read()), } return to_bytes(json.dumps(output)) render_POST = render_GET class RedirectTo(LeafResource): def render(self, request): goto = getarg(request, b"goto", b"/") # we force the body content, otherwise Twisted redirectTo() # returns HTML with