RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/master' into faster-shutdown-2

This commit is contained in:
Adrian Chaves 2026-08-04 10:04:22 +02:00
parent 25f732b5bb 4cdde1c79c
commit 7452277cfe
36 changed files with 626 additions and 457 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.github/workflows/codspeed.yml vendored
View File

@ -53,7 +53,7 @@ jobs:
uv tool install --with tox-uv tox
tox -n -e benchmark
- name: Run benchmarks
uses: CodSpeedHQ/action@f22792bfac16f3e14eb9fbea76f4a48e9cc22b93 # v4.19.1
uses: CodSpeedHQ/action@0ca9cbbf4623b599a6c3ed4fc8a922942705d9f1 # v5.0.2
with:
mode: simulation
run: tox -e benchmark

16
docs/faq.rst
View File

@ -220,21 +220,15 @@ the :ref:`topics-signals-ref` to know which ones.
What does the response status code 999 mean?
--------------------------------------------
999 is a custom response status code used by Yahoo sites to throttle requests.
999 is a custom response status code used by some sites to throttle requests.
Try slowing down the crawling speed by using a download delay of ``2`` (or
higher) in your spider:
higher) for the affected domains, with the :setting:`DOWNLOAD_SLOTS` setting:
.. code-block:: python
from scrapy.spiders import CrawlSpider
class MySpider(CrawlSpider):
name = "myspider"
download_delay = 2
# [ ... rest of the spider code ... ]
DOWNLOAD_SLOTS = {
"example.com": {"delay": 2},
}
Or by setting a global download delay in your project with the
:setting:`DOWNLOAD_DELAY` setting.

3
docs/news.rst
View File

@ -1417,7 +1417,8 @@ Deprecations
- ``download_warnsize`` (use :setting:`DOWNLOAD_WARNSIZE`)
- ``max_concurrent_requests`` (use :setting:`CONCURRENT_REQUESTS`)
- ``max_concurrent_requests`` (use
:setting:`CONCURRENT_REQUESTS_PER_DOMAIN`)
- ``user_agent`` (use :setting:`USER_AGENT`)

7
docs/topics/autothrottle.rst
View File

@ -106,10 +106,9 @@ delay of its download slot:
Request("https://example.com", meta={"autothrottle_dont_adjust_delay": True})
Note, however, that AutoThrottle still determines the starting delay of every
download slot by setting the ``download_delay`` attribute on the running
spider. If you want AutoThrottle not to impact a download slot at all, in
addition to setting this meta key in all requests that use that download slot,
you might want to set a custom value for the ``delay`` attribute of that
download slot. If you want AutoThrottle not to impact a download slot at all,
in addition to setting this meta key in all requests that use that download
slot, you might want to set a custom value for the ``delay`` attribute of that
download slot, e.g. using :setting:`DOWNLOAD_SLOTS`.
Settings

4
docs/topics/settings.rst
View File

@ -953,10 +953,6 @@ desired.
.. _spider-download_delay-attribute:
.. note::
This delay can be set per spider using :attr:`download_delay` spider attribute.
It is possible to change this setting per domain by using
:setting:`DOWNLOAD_SLOTS`.

19
extras/qpsclient.py
View File

@ -16,23 +16,20 @@ class QPSSpider(Spider):
name = "qps"
benchurl = "http://localhost:8880/"
# Max concurrency is limited by global CONCURRENT_REQUESTS setting
max_concurrent_requests = 8
# Requests per second goal
qps = None # same as: 1 / download_delay
download_delay = None
qps = None # same as: 1 / DOWNLOAD_DELAY
# time in seconds to delay server responses
latency = None
# number of slots to create
slots = 1
def __init__(self, *a, **kw):
super().__init__(*a, **kw)
if self.qps is not None:
self.qps = float(self.qps)
self.download_delay = 1 / self.qps
elif self.download_delay is not None:
self.download_delay = float(self.download_delay)
@classmethod
def from_crawler(cls, crawler, *args, **kwargs):
spider = super().from_crawler(crawler, *args, **kwargs)
if spider.qps is not None:
spider.qps = float(spider.qps)
crawler.settings.set("DOWNLOAD_DELAY", 1 / spider.qps, priority="spider")
return spider
async def start(self):
url = self.benchurl

15
pyproject.toml
View File

@ -118,13 +118,10 @@ allow_incomplete_defs = true # 59 errors
# TODO
[[tool.mypy.overrides]]
module = [
"tests.mockserver.*",
"tests.spiders",
"tests.test_closespider",
"tests.test_cmdline",
"tests.test_contracts",
"tests.test_core_downloader",
"tests.test_downloader_handler_twisted_ftp",
"tests.test_downloaderslotssettings",
"tests.test_dupefilters",
"tests.test_engine_loop",
@ -135,12 +132,6 @@ module = [
"tests.test_feedexport_postprocess",
"tests.test_feedexport_storages",
"tests.test_feedexport_uri_params",
"tests.test_http2_client_protocol",
"tests.test_http_headers",
"tests.test_http_request",
"tests.test_http_request_form",
"tests.test_http_response",
"tests.test_http_response_text",
"tests.test_item",
"tests.test_linkextractors",
"tests.test_loader",
@ -152,10 +143,6 @@ module = [
"tests.test_pipeline_media",
"tests.test_pipelines",
"tests.test_pqueues",
"tests.test_request_attribute_binding",
"tests.test_request_cb_kwargs",
"tests.test_request_dict",
"tests.test_request_left",
"tests.test_scheduler_base",
"tests.test_settings",
"tests.test_spider",
@ -166,8 +153,6 @@ module = [
"tests.test_squeues",
"tests.test_squeues_request",
"tests.test_stats",
"tests.utils.bases.http_request",
"tests.utils.bases.http_response",
"tests.utils.bases.spider",
]
check_untyped_defs = false

31
scrapy/core/downloader/__init__.py
View File

@ -27,7 +27,6 @@ from scrapy.utils.defer import (
deferred_from_coro,
maybe_deferred_to_future,
)
from scrapy.utils.deprecate import warn_on_deprecated_spider_attribute
from scrapy.utils.httpobj import urlparse_cached
if TYPE_CHECKING:
@ -80,22 +79,6 @@ class Slot:
)
def _get_concurrency_delay(
concurrency: int, spider: Spider, settings: BaseSettings
) -> tuple[int, float]:
delay: float = settings.getfloat("DOWNLOAD_DELAY")
if hasattr(spider, "download_delay"):
delay = spider.download_delay
if hasattr(spider, "max_concurrent_requests"): # pragma: no cover
warn_on_deprecated_spider_attribute(
"max_concurrent_requests", "CONCURRENT_REQUESTS"
)
concurrency = spider.max_concurrent_requests
return concurrency, delay
class Downloader:
DOWNLOAD_SLOT = "download_slot"
_SLOT_GC_INTERVAL: float = 60.0 # seconds
@ -112,6 +95,9 @@ class Downloader:
"CONCURRENT_REQUESTS_PER_DOMAIN"
)
self.ip_concurrency: int = self.settings.getint("CONCURRENT_REQUESTS_PER_IP")
# Default delay of new slots. AutoThrottle overrides it to apply
# AUTOTHROTTLE_START_DELAY.
self._delay: float = self.settings.getfloat("DOWNLOAD_DELAY")
self.randomize_delay: bool = self.settings.getbool("RANDOMIZE_DOWNLOAD_DELAY")
self.middleware: DownloaderMiddlewareManager = (
DownloaderMiddlewareManager.from_crawler(crawler)
@ -149,16 +135,11 @@ class Downloader:
) -> tuple[str, Slot]:
key = self.get_slot_key(request)
if key not in self.slots:
assert self.crawler.spider
slot_settings = self.per_slot_settings.get(key, {})
conc = self.ip_concurrency or self.domain_concurrency
conc, delay = _get_concurrency_delay(
conc, self.crawler.spider, self.settings
)
conc, delay = (
slot_settings.get("concurrency", conc),
slot_settings.get("delay", delay),
conc = slot_settings.get(
"concurrency", self.ip_concurrency or self.domain_concurrency
)
delay = slot_settings.get("delay", self._delay)
randomize_delay = slot_settings.get("randomize_delay", self.randomize_delay)
new_slot = Slot(conc, delay, randomize_delay)
self.slots[key] = new_slot

28
scrapy/crawler.py
View File

@ -110,6 +110,10 @@ class Crawler:
return
self.addons.load_settings(self.settings)
self._apply_deprecated_spider_attr("download_delay", "DOWNLOAD_DELAY")
self._apply_deprecated_spider_attr(
"max_concurrent_requests", "CONCURRENT_REQUESTS_PER_DOMAIN"
)
self.stats = load_object(self.settings["STATS_CLASS"])(self)
lf_cls: type[LogFormatter] = load_object(self.settings["LOG_FORMATTER"])
@ -165,6 +169,30 @@ class Crawler:
"Overridden settings:\n%(settings)s", {"settings": pprint.pformat(d)}
)
def _apply_deprecated_spider_attr(self, attr: str, setting: str) -> None:
"""Bridge a deprecated spider attribute onto *setting*, warning about
the deprecation (and about being ignored when *setting* is already set
at spider or higher priority)."""
spider = self.spider if self.spider is not None else self.spidercls
if not hasattr(spider, attr):
return
if (self.settings.getpriority(setting) or 0) >= SETTINGS_PRIORITIES["spider"]:
warnings.warn(
f"The {attr!r} spider attribute is deprecated. It is also being "
f"ignored because {setting} is already set at spider or higher "
f"priority. Remove the {attr!r} attribute from your spider.",
category=ScrapyDeprecationWarning,
stacklevel=3,
)
return
warnings.warn(
f"The {attr!r} spider attribute is deprecated. Use the {setting} "
f"setting instead.",
category=ScrapyDeprecationWarning,
stacklevel=3,
)
self.settings.set(setting, getattr(spider, attr), priority="spider")
def _apply_reactorless_default_settings(self) -> None:
"""Change some setting defaults when not using a Twisted reactor.

7
scrapy/downloadermiddlewares/retry.py
View File

@ -14,7 +14,7 @@ from typing import TYPE_CHECKING
from scrapy.exceptions import NotConfigured
from scrapy.utils.decorators import _warn_spider_arg
from scrapy.utils.misc import load_object
from scrapy.utils.misc import _load_objects
from scrapy.utils.python import global_object_name
from scrapy.utils.response import response_status_message
@ -149,10 +149,7 @@ class RetryMiddleware:
self.retry_http_codes = {int(x) for x in settings.getlist("RETRY_HTTP_CODES")}
self.priority_adjust = settings.getint("RETRY_PRIORITY_ADJUST")
self.give_up_log_level = settings["RETRY_GIVE_UP_LOG_LEVEL"]
self.exceptions_to_retry = tuple(
load_object(x) if isinstance(x, str) else x
for x in settings.getlist("RETRY_EXCEPTIONS")
)
self.exceptions_to_retry = _load_objects(settings.getlist("RETRY_EXCEPTIONS"))
@classmethod
def from_crawler(cls, crawler: Crawler) -> Self:

16
scrapy/extensions/throttle.py
View File

@ -43,18 +43,18 @@ class AutoThrottle:
return cls(crawler)
def _spider_opened(self, spider: Spider) -> None:
self.mindelay = self._min_delay(spider)
self.maxdelay = self._max_delay(spider)
spider.download_delay = self._start_delay(spider) # type: ignore[attr-defined]
self.mindelay = self._min_delay()
self.maxdelay = self._max_delay()
assert self.crawler.engine
self.crawler.engine.downloader._delay = self._start_delay()
def _min_delay(self, spider: Spider) -> float:
s = self.crawler.settings
return getattr(spider, "download_delay", s.getfloat("DOWNLOAD_DELAY"))
def _min_delay(self) -> float:
return self.crawler.settings.getfloat("DOWNLOAD_DELAY")
def _max_delay(self, spider: Spider) -> float:
def _max_delay(self) -> float:
return self.crawler.settings.getfloat("AUTOTHROTTLE_MAX_DELAY")
def _start_delay(self, spider: Spider) -> float:
def _start_delay(self) -> float:
return max(
self.mindelay, self.crawler.settings.getfloat("AUTOTHROTTLE_START_DELAY")
)

8
scrapy/http/request/form.py
View File

@ -7,7 +7,7 @@ See documentation in docs/topics/request-response.rst
from __future__ import annotations
from collections.abc import Iterable
from collections.abc import Iterable, Mapping
from typing import TYPE_CHECKING, Any, ClassVar, TypeAlias, cast
from urllib.parse import urlencode, urljoin, urlsplit, urlunsplit
from warnings import warn
@ -34,7 +34,7 @@ if TYPE_CHECKING:
FormdataVType: TypeAlias = str | Iterable[str]
FormdataKVType: TypeAlias = tuple[str, FormdataVType]
FormdataType: TypeAlias = dict[str, FormdataVType] | list[FormdataKVType] | None
FormdataType: TypeAlias = Mapping[str, FormdataVType] | Iterable[FormdataKVType] | None
class FormRequest(Request):
@ -100,7 +100,7 @@ class FormRequest(Request):
super().__init__(*args, **kwargs)
if formdata:
items = formdata.items() if isinstance(formdata, dict) else formdata
items = formdata.items() if isinstance(formdata, Mapping) else formdata
form_query_str = _urlencode(items, self.encoding)
if self.method == "POST":
self.headers.setdefault(
@ -248,7 +248,7 @@ def _get_inputs(
if clickable and clickable[0] not in formdata and clickable[0] is not None:
values.append(clickable)
formdata_items = formdata.items() if isinstance(formdata, dict) else formdata
formdata_items = formdata.items() if isinstance(formdata, Mapping) else formdata
values.extend((k, v) for k, v in formdata_items if v is not None)
return values

5
scrapy/utils/misc.py
View File

@ -90,6 +90,11 @@ def load_object(path: str | Callable[..., Any]) -> Any:
return obj
def _load_objects(objects: Iterable[str | Callable[..., Any]]) -> tuple[Any, ...]:
"""Resolve *objects* (objects or import paths) to a tuple of objects."""
return tuple(load_object(obj) if isinstance(obj, str) else obj for obj in objects)
def walk_modules_iter(path: str) -> Iterable[ModuleType]:
"""Loads a module and all its submodules from the given module path and
returns them. If *any* module throws an exception while importing, that

39
tests/mockserver/dns.py
View File

@ -2,6 +2,7 @@ from __future__ import annotations
import sys
from subprocess import PIPE, Popen
from typing import TYPE_CHECKING
from twisted.internet import defer
from twisted.names import dns, error
@ -9,39 +10,63 @@ from twisted.names.server import DNSServerFactory
from tests.utils import get_script_run_env
if TYPE_CHECKING:
from collections.abc import Sequence
from types import TracebackType
from twisted.internet.defer import Deferred
# typing.Self requires Python 3.11
from typing_extensions import Self
_Answers = tuple[list[dns.RRHeader], list[dns.RRHeader], list[dns.RRHeader]]
class MockDNSResolver:
"""
Implements twisted.internet.interfaces.IResolver partially
"""
def _resolve(self, name):
def _resolve(self, name: bytes) -> _Answers:
record = dns.Record_A(address=b"127.0.0.1")
answer = dns.RRHeader(name=name, payload=record)
# zope.interface has no type hints, so mypy cannot tell that Record_A
# provides the IEncodableRecord interface.
answer = dns.RRHeader(name=name, payload=record) # type: ignore[arg-type]
return [answer], [], []
def query(self, query, timeout=None):
def query(
self, query: dns.Query, timeout: Sequence[int] | None = None
) -> Deferred[_Answers]:
if query.type == dns.A:
return defer.succeed(self._resolve(query.name.name))
return defer.fail(error.DomainError())
def lookupAllRecords(self, name, timeout=None):
def lookupAllRecords(
self, name: bytes, timeout: Sequence[int] | None = None
) -> Deferred[_Answers]:
return defer.succeed(self._resolve(name))
class MockDNSServer:
def __enter__(self):
def __enter__(self) -> Self:
self.proc = Popen(
[sys.executable, "-u", "-m", "tests.mockserver.dns"],
stdout=PIPE,
env=get_script_run_env(),
text=True,
)
assert self.proc.stdout is not None
self.host = "127.0.0.1"
self.port = int(self.proc.stdout.readline().strip().split(":")[1])
return self
def __exit__(self, exc_type, exc_value, traceback):
def __exit__(
self,
exc_type: type[BaseException] | None,
exc_value: BaseException | None,
traceback: TracebackType | None,
) -> None:
self.proc.kill()
self.proc.communicate()
@ -54,7 +79,7 @@ def main() -> None:
protocol = dns.DNSDatagramProtocol(controller=factory)
listener = reactor.listenUDP(0, protocol)
def print_listening():
def print_listening() -> None:
host = listener.getHost()
print(f"{host.host}:{host.port}")

20
tests/mockserver/ftp.py
View File

@ -7,6 +7,7 @@ from pathlib import Path
from shutil import rmtree
from subprocess import PIPE, Popen
from tempfile import mkdtemp
from typing import TYPE_CHECKING
from pyftpdlib.authorizers import DummyAuthorizer
from pyftpdlib.handlers import FTPHandler
@ -14,6 +15,12 @@ from pyftpdlib.servers import FTPServer
from tests.utils import get_script_run_env
if TYPE_CHECKING:
from types import TracebackType
# typing.Self requires Python 3.11
from typing_extensions import Self
class MockFTPServer:
"""Creates an FTP server on a random port with a default passwordless user
@ -26,7 +33,7 @@ class MockFTPServer:
self.port: int | None = None
self.path: Path | None = None
def __enter__(self):
def __enter__(self) -> Self:
self.path = Path(mkdtemp())
self.proc = Popen(
[sys.executable, "-u", "-m", "tests.mockserver.ftp", "-d", str(self.path)],
@ -34,6 +41,7 @@ class MockFTPServer:
env=get_script_run_env(),
text=True,
)
assert self.proc.stderr is not None
for line in self.proc.stderr:
if "starting FTP server" in line and (
m := re.search(r"starting FTP server on ([^ :]+):(\d+),", line)
@ -48,12 +56,18 @@ class MockFTPServer:
)
return self
def __exit__(self, exc_type, exc_value, traceback):
def __exit__(
self,
exc_type: type[BaseException] | None,
exc_value: BaseException | None,
traceback: TracebackType | None,
) -> None:
rmtree(str(self.path))
assert self.proc is not None
self.proc.kill()
self.proc.communicate()
def url(self, path):
def url(self, path: str) -> str:
return f"ftp://{self.host}:{self.port}/{path}"

85
tests/mockserver/http.py
View File

@ -1,8 +1,8 @@
from __future__ import annotations
from pathlib import Path
from typing import TYPE_CHECKING
from twisted.web import resource
from twisted.web.static import Data, File
from twisted.web.util import Redirect
@ -11,6 +11,7 @@ from tests import tests_datadir
from .http_base import BaseMockServer, main_factory
from .http_resources import (
ArbitraryLengthPayloadResource,
BaseResource,
BrokenChunkedResource,
BrokenDownloadResource,
ChunkedResource,
@ -35,62 +36,68 @@ from .http_resources import (
SetCookie,
Status,
UriResource,
put_child,
)
if TYPE_CHECKING:
from twisted.web.server import Request
class Root(resource.Resource):
def __init__(self):
class Root(BaseResource):
def __init__(self) -> None:
super().__init__()
self.putChild(b"status", Status())
self.putChild(b"follow", Follow())
self.putChild(b"delay", Delay())
self.putChild(b"partial", Partial())
self.putChild(b"drop", Drop())
self.putChild(b"raw", Raw())
self.putChild(b"echo", Echo())
self.putChild(b"payload", PayloadResource())
self.putChild(b"alpayload", ArbitraryLengthPayloadResource())
self.putChild(b"static", File(str(Path(tests_datadir, "test_site/"))))
self.putChild(b"redirect-to", RedirectTo())
self.putChild(b"text", Data(b"Works", "text/plain"))
self.putChild(
put_child(self, b"status", Status())
put_child(self, b"follow", Follow())
put_child(self, b"delay", Delay())
put_child(self, b"partial", Partial())
put_child(self, b"drop", Drop())
put_child(self, b"raw", Raw())
put_child(self, b"echo", Echo())
put_child(self, b"payload", PayloadResource())
put_child(self, b"alpayload", ArbitraryLengthPayloadResource())
put_child(self, b"static", File(str(Path(tests_datadir, "test_site/"))))
put_child(self, b"redirect-to", RedirectTo())
put_child(self, b"text", Data(b"Works", "text/plain"))
put_child(
self,
b"html",
Data(
b"<body><p class='one'>Works</p><p class='two'>World</p></body>",
"text/html",
),
)
self.putChild(
put_child(
self,
b"enc-gb18030",
Data(b"<p>gb18030 encoding</p>", "text/html; charset=gb18030"),
)
self.putChild(b"redirect", Redirect(b"/redirected"))
self.putChild(
b"redirect-no-meta-refresh", NoMetaRefreshRedirect(b"/redirected")
put_child(self, b"redirect", Redirect(b"/redirected"))
put_child(
self, b"redirect-no-meta-refresh", NoMetaRefreshRedirect(b"/redirected")
)
self.putChild(b"redirected", Data(b"Redirected here", "text/plain"))
put_child(self, b"redirected", Data(b"Redirected here", "text/plain"))
numbers = [str(x).encode("utf8") for x in range(2**18)]
self.putChild(b"numbers", Data(b"".join(numbers), "text/plain"))
self.putChild(b"wait", ForeverTakingResource())
self.putChild(b"hang-after-headers", ForeverTakingResource(write=True))
self.putChild(b"host", HostHeaderResource())
self.putChild(b"client-ip", ClientIPResource())
self.putChild(b"broken", BrokenDownloadResource())
self.putChild(b"chunked", ChunkedResource())
self.putChild(b"broken-chunked", BrokenChunkedResource())
self.putChild(b"contentlength", ContentLengthHeaderResource())
self.putChild(b"nocontenttype", EmptyContentTypeHeaderResource())
self.putChild(b"largechunkedfile", LargeChunkedFileResource())
self.putChild(b"compress", Compress())
self.putChild(b"duplicate-header", DuplicateHeaderResource())
self.putChild(b"response-headers", ResponseHeadersResource())
self.putChild(b"set-cookie", SetCookie())
self.putChild(b"uri", UriResource())
put_child(self, b"numbers", Data(b"".join(numbers), "text/plain"))
put_child(self, b"wait", ForeverTakingResource())
put_child(self, b"hang-after-headers", ForeverTakingResource(write=True))
put_child(self, b"host", HostHeaderResource())
put_child(self, b"client-ip", ClientIPResource())
put_child(self, b"broken", BrokenDownloadResource())
put_child(self, b"chunked", ChunkedResource())
put_child(self, b"broken-chunked", BrokenChunkedResource())
put_child(self, b"contentlength", ContentLengthHeaderResource())
put_child(self, b"nocontenttype", EmptyContentTypeHeaderResource())
put_child(self, b"largechunkedfile", LargeChunkedFileResource())
put_child(self, b"compress", Compress())
put_child(self, b"duplicate-header", DuplicateHeaderResource())
put_child(self, b"response-headers", ResponseHeadersResource())
put_child(self, b"set-cookie", SetCookie())
put_child(self, b"uri", UriResource())
def getChild(self, path, request):
def getChild(self, path: bytes, request: Request) -> Root:
return self
def render(self, request):
def render(self, request: Request) -> bytes:
return b"Scrapy mock HTTP server\n"

10
tests/mockserver/http_base.py
View File

@ -17,6 +17,7 @@ from .utils import ssl_context_factory
if TYPE_CHECKING:
from collections.abc import Callable
from types import TracebackType
from twisted.web import resource
@ -60,7 +61,12 @@ class BaseMockServer(ABC):
self.https_port = https_parsed.port
return self
def __exit__(self, exc_type, exc_value, traceback) -> None:
def __exit__(
self,
exc_type: type[BaseException] | None,
exc_value: BaseException | None,
traceback: TracebackType | None,
) -> None:
if self.proc:
self.proc.kill()
self.proc.communicate()
@ -135,7 +141,7 @@ def main_factory(
context_factory = ssl_context_factory(**context_factory_kw)
https_port = reactor.listenSSL(0, factory, context_factory)
def print_listening():
def print_listening() -> None:
if listen_http:
http_host = http_port.getHost()
http_address = f"http://{http_host.host}:{http_host.port}"

174
tests/mockserver/http_resources.py
View File

@ -3,7 +3,7 @@ from __future__ import annotations
import gzip
import json
import random
from typing import TYPE_CHECKING, ParamSpec, TypeVar
from typing import TYPE_CHECKING, Any, ParamSpec, TypeVar
from urllib.parse import urlencode
from twisted.internet.task import deferLater
@ -14,17 +14,24 @@ from twisted.web.util import Redirect, redirectTo
from scrapy.utils.python import to_bytes, to_unicode
if TYPE_CHECKING:
from collections.abc import Callable
from collections.abc import Callable, Sequence
from twisted.internet.defer import Deferred
from twisted.web.http import Request
from twisted.python.failure import Failure
from twisted.web.http import Request as HTTPRequest
from twisted.web.server import Request
_T = TypeVar("_T")
_P = ParamSpec("_P")
def getarg(request, name, default=None, type_=None):
def getarg(
request: Request,
name: bytes,
default: Any = None,
type_: Callable[[bytes], Any] | None = None,
) -> Any:
if name in request.args:
value = request.args[name][0]
if type_ is not None:
@ -33,73 +40,91 @@ def getarg(request, name, default=None, type_=None):
return default
def close_connection(request):
def close_connection(request: Request) -> None:
# We have to force a disconnection for HTTP/1.1 clients. Otherwise
# client keeps the connection open waiting for more data.
request.channel.loseConnection()
request.finish()
def put_child(parent: resource.Resource, path: bytes, child: resource.Resource) -> None:
# zope.interface has no type hints, so mypy cannot tell that Resource
# instances provide the IResource interface that putChild() expects.
parent.putChild(path, child) # type: ignore[arg-type]
class BaseResource(resource.Resource):
"""Base class for mockserver resources, with type hints."""
# Only needed to give subclasses a typed __init__ to call.
def __init__(self) -> None: # pylint: disable=useless-parent-delegation
super().__init__() # type: ignore[no-untyped-call]
# most of the following resources are copied from twisted.web.test.test_webclient
class ForeverTakingResource(resource.Resource):
class ForeverTakingResource(BaseResource):
"""
L{ForeverTakingResource} is a resource which never finishes responding
to requests.
"""
def __init__(self, write=False):
resource.Resource.__init__(self)
def __init__(self, write: bool = False):
super().__init__()
self._write = write
def render(self, request):
def render(self, request: Request) -> int:
if self._write:
request.write(b"some bytes")
return server.NOT_DONE_YET
class HostHeaderResource(resource.Resource):
class HostHeaderResource(BaseResource):
"""
A testing resource which renders itself as the value of the host header
from the request.
"""
def render(self, request):
return request.requestHeaders.getRawHeaders(b"host")[0]
def render(self, request: Request) -> bytes:
headers = request.requestHeaders.getRawHeaders(b"host")
assert headers
return headers[0]
class ClientIPResource(resource.Resource):
class ClientIPResource(BaseResource):
"""
A testing resource which renders itself as the request client IP address.
"""
def render(self, request):
def render(self, request: Request) -> bytes:
client_address = request.getClientAddress()
if client_address is None or client_address.host is None:
return b""
return to_bytes(client_address.host)
class PayloadResource(resource.Resource):
class PayloadResource(BaseResource):
"""
A testing resource which renders itself as the contents of the request body
as long as the request body is 100 bytes long, otherwise which renders
itself as C{"ERROR"}.
"""
def render(self, request):
data = request.content.read()
contentLength = request.requestHeaders.getRawHeaders(b"content-length")[0]
if len(data) != 100 or int(contentLength) != 100:
def render(self, request: Request) -> bytes:
assert request.content
data: bytes = request.content.read()
content_length = request.requestHeaders.getRawHeaders(b"content-length")
assert content_length
if len(data) != 100 or int(content_length[0]) != 100:
return b"ERROR"
return data
class LeafResource(resource.Resource):
class LeafResource(BaseResource):
isLeaf = True
def deferRequest(
self,
request: Request,
request: HTTPRequest,
delay: float,
f: Callable[_P, _T],
*a: _P.args,
@ -107,7 +132,7 @@ class LeafResource(resource.Resource):
) -> Deferred[_T]:
from twisted.internet import reactor
def _cancelrequest(_):
def _cancelrequest(_: Failure) -> None:
# silence CancelledError
d.addErrback(lambda _: None)
d.cancel()
@ -118,12 +143,13 @@ class LeafResource(resource.Resource):
class Follow(LeafResource):
def render(self, request):
def render(self, request: Request) -> int:
total = getarg(request, b"total", 100, type_=int)
show = getarg(request, b"show", 1, type_=int)
order = getarg(request, b"order", b"desc")
maxlatency = getarg(request, b"maxlatency", 0, type_=float)
n = getarg(request, b"n", total, type_=int)
nlist: Sequence[int]
if order == b"rand":
nlist = [random.randint(1, total) for _ in range(show)]
else: # order == "desc"
@ -133,7 +159,7 @@ class Follow(LeafResource):
self.deferRequest(request, lag, self.renderRequest, request, nlist)
return NOT_DONE_YET
def renderRequest(self, request, nlist):
def renderRequest(self, request: Request, nlist: Sequence[int]) -> None:
s = """<html> <head></head> <body>"""
args = request.args.copy()
for nl in nlist:
@ -146,45 +172,47 @@ class Follow(LeafResource):
class Delay(LeafResource):
def render_GET(self, request):
def render_GET(self, request: Request) -> int:
n = getarg(request, b"n", 1, type_=float)
b = getarg(request, b"b", 1, type_=int)
if b:
# send headers now and delay body
request.write("")
request.write(b"")
self.deferRequest(request, n, self._delayedRender, request, n)
return NOT_DONE_YET
def _delayedRender(self, request, n):
def _delayedRender(self, request: Request, n: float) -> None:
request.write(to_bytes(f"Response delayed for {n:.3f} seconds\n"))
request.finish()
class Status(LeafResource):
def render_GET(self, request):
def render_GET(self, request: Request) -> bytes:
n = getarg(request, b"n", 200, type_=int)
request.setResponseCode(n)
return b""
class Raw(LeafResource):
def render_GET(self, request):
def render_GET(self, request: Request) -> int:
request.startedWriting = 1
self.deferRequest(request, 0, self._delayedRender, request)
return NOT_DONE_YET
render_POST = render_GET
def _delayedRender(self, request):
def _delayedRender(self, request: Request) -> None:
raw = getarg(request, b"raw", b"HTTP 1.1 200 OK\n")
request.startedWriting = 1
request.write(raw)
assert request.channel.transport is not None
request.channel.transport.loseConnection()
request.finish()
class Echo(LeafResource):
def render_GET(self, request):
def render_GET(self, request: Request) -> bytes:
assert request.content
output = {
"headers": {
to_unicode(k): [to_unicode(v) for v in vs]
@ -198,27 +226,29 @@ class Echo(LeafResource):
class RedirectTo(LeafResource):
def render(self, request):
def render(self, request: Request) -> bytes:
goto = getarg(request, b"goto", b"/")
# we force the body content, otherwise Twisted redirectTo()
# returns HTML with <meta http-equiv="refresh"
redirectTo(goto, request)
# zope.interface has no type hints, so mypy cannot tell that Request
# provides the IRequest interface.
redirectTo(goto, request) # type: ignore[arg-type]
return b"redirecting..."
class Partial(LeafResource):
def render_GET(self, request):
def render_GET(self, request: Request) -> int:
request.setHeader(b"Content-Length", b"1024")
self.deferRequest(request, 0, self._delayedRender, request)
return NOT_DONE_YET
def _delayedRender(self, request):
def _delayedRender(self, request: Request) -> None:
request.write(b"partial content\n")
request.finish()
class Drop(Partial):
def _delayedRender(self, request):
def _delayedRender(self, request: Request) -> None:
abort = getarg(request, b"abort", 0, type_=int)
request.write(b"this connection will be dropped\n")
tr = request.channel.transport
@ -233,8 +263,10 @@ class Drop(Partial):
class ArbitraryLengthPayloadResource(LeafResource):
def render(self, request):
return request.content.read()
def render(self, request: Request) -> bytes:
assert request.content
data: bytes = request.content.read()
return data
class NoMetaRefreshRedirect(Redirect):
@ -245,21 +277,23 @@ class NoMetaRefreshRedirect(Redirect):
)
class ContentLengthHeaderResource(resource.Resource):
class ContentLengthHeaderResource(BaseResource):
"""
A testing resource which renders itself as the value of the Content-Length
header from the request.
"""
def render(self, request):
return request.requestHeaders.getRawHeaders(b"content-length")[0]
def render(self, request: Request) -> bytes:
headers = request.requestHeaders.getRawHeaders(b"content-length")
assert headers
return headers[0]
class ChunkedResource(resource.Resource):
def render(self, request):
class ChunkedResource(BaseResource):
def render(self, request: Request) -> int:
from twisted.internet import reactor
def response():
def response() -> None:
request.write(b"chunked ")
request.write(b"content\n")
request.finish()
@ -268,11 +302,11 @@ class ChunkedResource(resource.Resource):
return server.NOT_DONE_YET
class BrokenChunkedResource(resource.Resource):
def render(self, request):
class BrokenChunkedResource(BaseResource):
def render(self, request: Request) -> int:
from twisted.internet import reactor
def response():
def response() -> None:
request.write(b"chunked ")
request.write(b"content\n")
# Disable terminating chunk on finish.
@ -283,11 +317,11 @@ class BrokenChunkedResource(resource.Resource):
return server.NOT_DONE_YET
class BrokenDownloadResource(resource.Resource):
def render(self, request):
class BrokenDownloadResource(BaseResource):
def render(self, request: Request) -> int:
from twisted.internet import reactor
def response():
def response() -> None:
request.setHeader(b"Content-Length", b"20")
request.write(b"partial")
close_connection(request)
@ -296,22 +330,24 @@ class BrokenDownloadResource(resource.Resource):
return server.NOT_DONE_YET
class EmptyContentTypeHeaderResource(resource.Resource):
class EmptyContentTypeHeaderResource(BaseResource):
"""
A testing resource which renders itself as the value of request body
without content-type header in response.
"""
def render(self, request):
def render(self, request: Request) -> bytes:
assert request.content
request.setHeader("content-type", "")
return request.content.read()
data: bytes = request.content.read()
return data
class LargeChunkedFileResource(resource.Resource):
def render(self, request):
class LargeChunkedFileResource(BaseResource):
def render(self, request: Request) -> int:
from twisted.internet import reactor
def response():
def response() -> None:
for _ in range(1024):
request.write(b"x" * 1024)
request.finish()
@ -320,43 +356,45 @@ class LargeChunkedFileResource(resource.Resource):
return server.NOT_DONE_YET
class DuplicateHeaderResource(resource.Resource):
def render(self, request):
class DuplicateHeaderResource(BaseResource):
def render(self, request: Request) -> bytes:
request.responseHeaders.setRawHeaders(b"Set-Cookie", [b"a=b", b"c=d"])
return b""
class UriResource(resource.Resource):
class UriResource(BaseResource):
"""Return the full uri that was requested"""
def getChild(self, path, request):
def getChild(self, path: bytes, request: Request) -> resource.Resource:
return self
def render(self, request):
def render(self, request: Request) -> bytes | int:
# Note: this is an ugly hack for CONNECT request timeout test.
# Returning some data here fail SSL/TLS handshake
# ToDo: implement proper HTTPS proxy tests, not faking them.
if request.method != b"CONNECT":
return request.uri
assert request.transport is not None
request.transport.write(b"HTTP/1.1 200 Connection established\r\n\r\n")
return NOT_DONE_YET
class ResponseHeadersResource(resource.Resource):
class ResponseHeadersResource(BaseResource):
"""Return a response with headers set from the JSON request body"""
def render(self, request):
def render(self, request: Request) -> bytes:
assert request.content
body = json.loads(request.content.read().decode())
for header_name, header_value in body.items():
request.responseHeaders.setRawHeaders(header_name, [header_value])
return json.dumps(body).encode("utf-8")
class Compress(resource.Resource):
class Compress(BaseResource):
"""Compress the data sent in the request url params and set Content-Encoding header"""
def render(self, request):
data = request.args.get(b"data")[0]
def render(self, request: Request) -> bytes:
data = request.args[b"data"][0]
accept_encoding_header = request.getHeader(b"accept-encoding")
@ -370,10 +408,10 @@ class Compress(resource.Resource):
return b"Did not receive a valid accept-encoding header"
class SetCookie(resource.Resource):
class SetCookie(BaseResource):
"""Return a response with a Set-Cookie header for each request url parameter"""
def render(self, request):
def render(self, request: Request) -> bytes:
for cookie_name, cookie_values in request.args.items():
for cookie_value in cookie_values:
cookie = (cookie_name.decode() + "=" + cookie_value.decode()).encode()

19
tests/mockserver/simple_https.py
View File

@ -2,18 +2,23 @@
from __future__ import annotations
from twisted.web import resource
from typing import TYPE_CHECKING
from twisted.web.static import Data
from .http_base import BaseMockServer, main_factory
from .http_resources import BaseResource, put_child
if TYPE_CHECKING:
from twisted.web.server import Request
class Root(resource.Resource):
def __init__(self):
resource.Resource.__init__(self)
self.putChild(b"file", Data(b"0123456789", "text/plain"))
class Root(BaseResource):
def __init__(self) -> None:
super().__init__()
put_child(self, b"file", Data(b"0123456789", "text/plain"))
def getChild(self, path, request):
def getChild(self, path: bytes, request: Request) -> Root:
return self
@ -29,7 +34,7 @@ class SimpleMockServer(BaseMockServer):
cipher_string: str | None = None,
tls_min_version: str | None = None,
tls_max_version: str | None = None,
):
) -> None:
super().__init__()
self.keyfile = keyfile
self.certfile = certfile

16
tests/test_core_downloader.py
View File

@ -16,7 +16,7 @@ from twisted.web import server, static
from twisted.web.client import Agent, BrowserLikePolicyForHTTPS, readBody
from twisted.web.client import Response as TxResponse
from scrapy import Request
from scrapy import Request, Spider
from scrapy.core.downloader import Downloader, Slot, tls
from scrapy.core.downloader.contextfactory import (
_load_context_factory_from_settings,
@ -33,14 +33,17 @@ from scrapy.utils.misc import build_from_crawler
from scrapy.utils.python import to_bytes
from scrapy.utils.spider import DefaultSpider
from scrapy.utils.test import get_crawler
from tests.mockserver.http_resources import PayloadResource
from tests.mockserver.http_resources import PayloadResource, put_child
from tests.mockserver.utils import ssl_context_factory
from tests.utils.decorators import coroutine_test
if TYPE_CHECKING:
from twisted.internet.interfaces import IListeningPort
from twisted.python.failure import Failure
from twisted.web.iweb import IBodyProducer
from scrapy.http import Response
class TestSlot:
def test_repr(self):
@ -54,7 +57,7 @@ class TestContextFactoryBase:
async def server_url(self, tmp_path):
(tmp_path / "file").write_bytes(b"0123456789")
r = static.File(str(tmp_path))
r.putChild(b"payload", PayloadResource())
put_child(r, b"payload", PayloadResource())
site = server.Site(r, timeout=None)
port = self._listen(site)
portno = port.getHost().port
@ -63,7 +66,7 @@ class TestContextFactoryBase:
await port.stopListening()
def _listen(self, site):
def _listen(self, site: server.Site) -> IListeningPort:
from twisted.internet import reactor
return reactor.listenSSL(
@ -319,7 +322,10 @@ def test_needs_backout(concurrency: int, active: int, expected: bool) -> None:
@coroutine_test
async def test_fetch_deprecated_spider_arg():
class CustomDownloader(Downloader):
def fetch(self, request, spider): # pylint: disable=signature-differs
# requiring the spider argument is what triggers the deprecation
def fetch( # type: ignore[override] # pylint: disable=signature-differs
self, request: Request, spider: Spider
) -> Deferred[Response | Request]:
return super().fetch(request, spider)
crawler = get_crawler(DefaultSpider, {"DOWNLOADER": CustomDownloader})

33
tests/test_crawler.py
View File

@ -74,6 +74,39 @@ class TestCrawler:
assert not settings.frozen
assert crawler.settings.frozen
@pytest.mark.parametrize(
("attr", "setting"),
[
("download_delay", "DOWNLOAD_DELAY"),
("max_concurrent_requests", "CONCURRENT_REQUESTS_PER_DOMAIN"),
],
)
def test_deprecated_spider_attr(self, attr: str, setting: str) -> None:
crawler = get_raw_crawler(type("_Spider", (DefaultSpider,), {attr: 2}))
with pytest.warns(
ScrapyDeprecationWarning,
match=f"The {attr!r} spider attribute is deprecated. Use the {setting} ",
):
crawler._apply_settings()
assert crawler.settings.getint(setting) == 2
@pytest.mark.parametrize(
("attr", "setting"),
[
("download_delay", "DOWNLOAD_DELAY"),
("max_concurrent_requests", "CONCURRENT_REQUESTS_PER_DOMAIN"),
],
)
def test_deprecated_spider_attr_ignored(self, attr: str, setting: str) -> None:
crawler = get_raw_crawler(type("_Spider", (DefaultSpider,), {attr: 2}))
crawler.settings.set(setting, 3, priority="spider")
with pytest.warns(
ScrapyDeprecationWarning,
match=f"The {attr!r} spider attribute is deprecated. It is also being ",
):
crawler._apply_settings()
assert crawler.settings.getint(setting) == 3
def test_crawler_accepts_dict(self) -> None:
crawler = get_crawler(DefaultSpider, {"foo": "bar"})
assert crawler.settings["foo"] == "bar"

22
tests/test_downloader_handler_twisted_ftp.py
View File

@ -156,14 +156,17 @@ class TestFTP(TestFTPBase):
for filename, content in self.test_files:
(userdir / filename).write_bytes(content)
def _get_factory(self, root):
def _get_factory(self, root: Path) -> FTPFactory:
from twisted.protocols.ftp import FTPFactory, FTPRealm
realm = FTPRealm(anonymousRoot=str(root), userHome=str(root))
p = portal.Portal(realm)
# zope.interface has no type hints, so mypy cannot tell that these
# objects provide the interfaces that Portal expects.
p = portal.Portal(realm) # type: ignore[arg-type]
users_checker = checkers.InMemoryUsernamePasswordDatabaseDontUse()
users_checker.addUser(self.username, self.password)
p.registerChecker(users_checker, credentials.IUsernamePassword)
# the FTP protocol authenticates with str credentials
users_checker.addUser(self.username, self.password) # type: ignore[arg-type]
p.registerChecker(users_checker, credentials.IUsernamePassword) # type: ignore[arg-type]
return FTPFactory(portal=p)
@deferred_f_from_coro_f
@ -192,12 +195,17 @@ class TestAnonymousFTP(TestFTPBase):
for filename, content in self.test_files:
(root / filename).write_bytes(content)
def _get_factory(self, tmp_path):
def _get_factory(self, tmp_path: Path) -> FTPFactory:
from twisted.protocols.ftp import FTPFactory, FTPRealm
realm = FTPRealm(anonymousRoot=str(tmp_path))
p = portal.Portal(realm)
p.registerChecker(checkers.AllowAnonymousAccess(), credentials.IAnonymous)
# zope.interface has no type hints, so mypy cannot tell that these
# objects provide the interfaces that Portal expects.
p = portal.Portal(realm) # type: ignore[arg-type]
p.registerChecker(
checkers.AllowAnonymousAccess(), # type: ignore[arg-type]
credentials.IAnonymous,
)
return FTPFactory(portal=p, userAnonymous=self.username)

12
tests/test_engine_download.py
View File

@ -116,6 +116,18 @@ class TestEngineDownloadAsync:
engine._slot.add_request.assert_called_once_with(request)
engine._slot.remove_request.assert_called_once_with(request)
@coroutine_test
async def test_download_async_fetch_needs_spider(self, engine):
engine._downloader_fetch_needs_spider = True
request = Request("http://example.com")
response = Response("http://example.com", body=b"test body")
engine.spider = Mock()
engine.downloader.fetch.return_value = defer.succeed(response)
result = await self._download(engine, request)
assert result == response
engine.downloader.fetch.assert_called_once_with(request, engine.spider)
@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
class TestEngineDownload(TestEngineDownloadAsync):

106
tests/test_extension_throttle.py
View File

@ -3,7 +3,7 @@ from unittest.mock import Mock
import pytest
from scrapy import Request, Spider
from scrapy import Request
from scrapy.exceptions import NotConfigured
from scrapy.extensions.throttle import AutoThrottle
from scrapy.http.response import Response
@ -25,6 +25,13 @@ def get_crawler(settings=None, spidercls=None):
return _get_crawler(settings_dict=settings, spidercls=spidercls)
def _mock_downloader(crawler):
"""Give *crawler* a mock engine, whose downloader AutoThrottle reads."""
crawler.engine = Mock()
crawler.engine.downloader.slots = {}
return crawler.engine.downloader
@pytest.mark.parametrize(
("value", "expected"),
[
@ -60,29 +67,21 @@ def test_target_concurrency_invalid(value):
@pytest.mark.parametrize(
("spider", "setting", "expected"),
("setting", "expected"),
[
(UNSET, UNSET, DOWNLOAD_DELAY),
(1.0, UNSET, 1.0),
(UNSET, 1.0, 1.0),
(1.0, 2.0, 1.0),
(3.0, 2.0, 3.0),
(UNSET, DOWNLOAD_DELAY),
(1.0, 1.0),
],
)
def test_mindelay_definition(spider, setting, expected):
def test_mindelay_definition(setting, expected):
settings = {}
if setting is not UNSET:
settings["DOWNLOAD_DELAY"] = setting
class _TestSpider(Spider):
name = "test"
if spider is not UNSET:
_TestSpider.download_delay = spider
crawler = get_crawler(settings, _TestSpider)
crawler = get_crawler(settings)
at = build_from_crawler(AutoThrottle, crawler)
at._spider_opened(_TestSpider())
_mock_downloader(crawler)
at._spider_opened(DefaultSpider())
assert at.mindelay == expected
@ -99,58 +98,43 @@ def test_maxdelay_definition(value, expected):
settings["AUTOTHROTTLE_MAX_DELAY"] = value
crawler = get_crawler(settings)
at = build_from_crawler(AutoThrottle, crawler)
_mock_downloader(crawler)
at._spider_opened(DefaultSpider())
assert at.maxdelay == expected
@pytest.mark.parametrize(
("min_spider", "min_setting", "start_setting", "expected"),
("min_setting", "start_setting", "expected"),
[
(UNSET, UNSET, UNSET, AUTOTHROTTLE_START_DELAY),
(AUTOTHROTTLE_START_DELAY - 1.0, UNSET, UNSET, AUTOTHROTTLE_START_DELAY),
(AUTOTHROTTLE_START_DELAY + 1.0, UNSET, UNSET, AUTOTHROTTLE_START_DELAY + 1.0),
(UNSET, AUTOTHROTTLE_START_DELAY - 1.0, UNSET, AUTOTHROTTLE_START_DELAY),
(UNSET, AUTOTHROTTLE_START_DELAY + 1.0, UNSET, AUTOTHROTTLE_START_DELAY + 1.0),
(UNSET, UNSET, AUTOTHROTTLE_START_DELAY - 1.0, AUTOTHROTTLE_START_DELAY - 1.0),
(UNSET, UNSET, AUTOTHROTTLE_START_DELAY + 1.0, AUTOTHROTTLE_START_DELAY + 1.0),
(
AUTOTHROTTLE_START_DELAY + 1.0,
AUTOTHROTTLE_START_DELAY + 2.0,
UNSET,
AUTOTHROTTLE_START_DELAY + 1.0,
),
(UNSET, UNSET, AUTOTHROTTLE_START_DELAY),
(AUTOTHROTTLE_START_DELAY - 1.0, UNSET, AUTOTHROTTLE_START_DELAY),
(AUTOTHROTTLE_START_DELAY + 1.0, UNSET, AUTOTHROTTLE_START_DELAY + 1.0),
(UNSET, AUTOTHROTTLE_START_DELAY - 1.0, AUTOTHROTTLE_START_DELAY - 1.0),
(UNSET, AUTOTHROTTLE_START_DELAY + 1.0, AUTOTHROTTLE_START_DELAY + 1.0),
(
AUTOTHROTTLE_START_DELAY + 2.0,
UNSET,
AUTOTHROTTLE_START_DELAY + 1.0,
AUTOTHROTTLE_START_DELAY + 2.0,
),
(
AUTOTHROTTLE_START_DELAY + 1.0,
UNSET,
AUTOTHROTTLE_START_DELAY + 2.0,
AUTOTHROTTLE_START_DELAY + 2.0,
),
],
)
def test_startdelay_definition(min_spider, min_setting, start_setting, expected):
def test_startdelay_definition(min_setting, start_setting, expected):
settings = {}
if min_setting is not UNSET:
settings["DOWNLOAD_DELAY"] = min_setting
if start_setting is not UNSET:
settings["AUTOTHROTTLE_START_DELAY"] = start_setting
class _TestSpider(Spider):
name = "test"
if min_spider is not UNSET:
_TestSpider.download_delay = min_spider
crawler = get_crawler(settings, _TestSpider)
crawler = get_crawler(settings)
at = build_from_crawler(AutoThrottle, crawler)
spider = _TestSpider()
at._spider_opened(spider)
assert spider.download_delay == expected
downloader = _mock_downloader(crawler)
at._spider_opened(DefaultSpider())
assert downloader._delay == expected
@pytest.mark.parametrize(
@ -174,15 +158,13 @@ def test_startdelay_definition(min_spider, min_setting, start_setting, expected)
def test_skipped(meta, slot):
crawler = get_crawler()
at = build_from_crawler(AutoThrottle, crawler)
downloader = _mock_downloader(crawler)
spider = DefaultSpider()
at._spider_opened(spider)
request = Request("https://example.com", meta=meta)
crawler.engine = Mock()
crawler.engine.downloader = Mock()
crawler.engine.downloader.slots = {}
if slot is not None:
crawler.engine.downloader.slots[slot] = object()
downloader.slots[slot] = object()
at._adjust_delay = None # Raise exception if called.
at._response_downloaded(None, request, spider)
@ -204,18 +186,16 @@ def test_adjustment(download_latency, target_concurrency, slot_delay, expected):
settings = {"AUTOTHROTTLE_TARGET_CONCURRENCY": target_concurrency}
crawler = get_crawler(settings)
at = build_from_crawler(AutoThrottle, crawler)
downloader = _mock_downloader(crawler)
spider = DefaultSpider()
at._spider_opened(spider)
meta = {"download_latency": download_latency, "download_slot": "foo"}
request = Request("https://example.com", meta=meta)
response = Response(request.url)
crawler.engine = Mock()
crawler.engine.downloader = Mock()
crawler.engine.downloader.slots = {}
slot = Mock()
slot.delay = slot_delay
crawler.engine.downloader.slots["foo"] = slot
downloader.slots["foo"] = slot
at._response_downloaded(response, request, spider)
@ -240,18 +220,16 @@ def test_adjustment_limits(mindelay, maxdelay, expected):
}
crawler = get_crawler(settings)
at = build_from_crawler(AutoThrottle, crawler)
downloader = _mock_downloader(crawler)
spider = DefaultSpider()
at._spider_opened(spider)
meta = {"download_latency": download_latency, "download_slot": "foo"}
request = Request("https://example.com", meta=meta)
response = Response(request.url)
crawler.engine = Mock()
crawler.engine.downloader = Mock()
crawler.engine.downloader.slots = {}
slot = Mock()
slot.delay = slot_delay
crawler.engine.downloader.slots["foo"] = slot
downloader.slots["foo"] = slot
at._response_downloaded(response, request, spider)
@ -272,18 +250,16 @@ def test_adjustment_bad_response(
settings = {"AUTOTHROTTLE_TARGET_CONCURRENCY": target_concurrency}
crawler = get_crawler(settings)
at = build_from_crawler(AutoThrottle, crawler)
downloader = _mock_downloader(crawler)
spider = DefaultSpider()
at._spider_opened(spider)
meta = {"download_latency": download_latency, "download_slot": "foo"}
request = Request("https://example.com", meta=meta)
response = Response(request.url, status=400)
crawler.engine = Mock()
crawler.engine.downloader = Mock()
crawler.engine.downloader.slots = {}
slot = Mock()
slot.delay = slot_delay
crawler.engine.downloader.slots["foo"] = slot
downloader.slots["foo"] = slot
at._response_downloaded(response, request, spider)
@ -294,19 +270,17 @@ def test_debug(caplog):
settings = {"AUTOTHROTTLE_DEBUG": True}
crawler = get_crawler(settings)
at = build_from_crawler(AutoThrottle, crawler)
downloader = _mock_downloader(crawler)
spider = DefaultSpider()
at._spider_opened(spider)
meta = {"download_latency": 1.0, "download_slot": "foo"}
request = Request("https://example.com", meta=meta)
response = Response(request.url, body=b"foo")
crawler.engine = Mock()
crawler.engine.downloader = Mock()
crawler.engine.downloader.slots = {}
slot = Mock()
slot.delay = 2.0
slot.transferring = (None, None)
crawler.engine.downloader.slots["foo"] = slot
downloader.slots["foo"] = slot
caplog.clear()
with caplog.at_level(INFO):
@ -324,19 +298,17 @@ def test_debug(caplog):
def test_debug_disabled(caplog):
crawler = get_crawler()
at = build_from_crawler(AutoThrottle, crawler)
downloader = _mock_downloader(crawler)
spider = DefaultSpider()
at._spider_opened(spider)
meta = {"download_latency": 1.0, "download_slot": "foo"}
request = Request("https://example.com", meta=meta)
response = Response(request.url, body=b"foo")
crawler.engine = Mock()
crawler.engine.downloader = Mock()
crawler.engine.downloader.slots = {}
slot = Mock()
slot.delay = 2.0
slot.transferring = (None, None)
crawler.engine.downloader.slots["foo"] = slot
downloader.slots["foo"] = slot
caplog.clear()
with caplog.at_level(INFO):

22
tests/test_http2_client_protocol.py
View File

@ -30,7 +30,7 @@ from scrapy.utils.defer import (
deferred_from_coro,
maybe_deferred_to_future,
)
from tests.mockserver.http_resources import LeafResource, Status
from tests.mockserver.http_resources import LeafResource, Status, put_child
from tests.mockserver.utils import ssl_context_factory
if TYPE_CHECKING:
@ -199,18 +199,18 @@ class TestHttps2ClientProtocol:
@pytest.fixture
def site(self, tmp_path):
r = File(str(tmp_path))
r.putChild(b"get-data-html-small", GetDataHtmlSmall())
r.putChild(b"get-data-html-large", GetDataHtmlLarge())
put_child(r, b"get-data-html-small", GetDataHtmlSmall())
put_child(r, b"get-data-html-large", GetDataHtmlLarge())
r.putChild(b"post-data-json-small", PostDataJsonSmall())
r.putChild(b"post-data-json-large", PostDataJsonLarge())
put_child(r, b"post-data-json-small", PostDataJsonSmall())
put_child(r, b"post-data-json-large", PostDataJsonLarge())
r.putChild(b"dataloss", Dataloss())
r.putChild(b"no-content-length-header", NoContentLengthHeader())
r.putChild(b"status", Status())
r.putChild(b"query-params", QueryParams())
r.putChild(b"timeout", TimeoutResponse())
r.putChild(b"request-headers", RequestHeaders())
put_child(r, b"dataloss", Dataloss())
put_child(r, b"no-content-length-header", NoContentLengthHeader())
put_child(r, b"status", Status())
put_child(r, b"query-params", QueryParams())
put_child(r, b"timeout", TimeoutResponse())
put_child(r, b"request-headers", RequestHeaders())
return Site(r, timeout=None)
@async_yield_fixture # type: ignore[untyped-decorator]

29
tests/test_http_headers.py
View File

@ -6,9 +6,6 @@ from scrapy.http import Headers
class TestHeaders:
def assertSortedEqual(self, first, second, msg=None):
assert sorted(first) == sorted(second), msg
def test_basics(self):
h = Headers({"Content-Type": "text/html", "Content-Length": 1234})
assert h["Content-Type"]
@ -39,7 +36,7 @@ class TestHeaders:
assert h["X-Forwarded-For"] == b"ip2"
assert h.get("X-Forwarded-For") == b"ip2"
assert h.getlist("X-Forwarded-For") == [b"ip1", b"ip2"]
assert h.getlist("X-Forwarded-For") is not hlist
assert h.getlist("X-Forwarded-For") is not hlist # type: ignore[comparison-overlap]
def test_multivalue_for_one_header(self):
h = Headers((("a", "b"), ("a", "c")))
@ -49,19 +46,19 @@ class TestHeaders:
def test_encode_utf8(self):
h = Headers({"key": "\xa3"}, encoding="utf-8")
key, val = dict(h).popitem()
key, val = dict(h.items()).popitem()
assert isinstance(key, bytes), key
assert isinstance(val[0], bytes), val[0]
assert val[0] == b"\xc2\xa3"
def test_encode_latin1(self):
h = Headers({"key": "\xa3"}, encoding="latin1")
_, val = dict(h).popitem()
_, val = dict(h.items()).popitem()
assert val[0] == b"\xa3"
def test_encode_multiple(self):
h = Headers({"key": ["\xa3"]}, encoding="utf-8")
_, val = dict(h).popitem()
_, val = dict(h.items()).popitem()
assert val[0] == b"\xc2\xa3"
def test_delete_and_contains(self):
@ -75,7 +72,7 @@ class TestHeaders:
h = Headers()
hlist = ["ip1", "ip2"]
olist = h.setdefault("X-Forwarded-For", hlist)
assert h.getlist("X-Forwarded-For") is not hlist
assert h.getlist("X-Forwarded-For") is not hlist # type: ignore[comparison-overlap]
assert h.getlist("X-Forwarded-For") is olist
h = Headers()
@ -87,16 +84,16 @@ class TestHeaders:
idict = {"Content-Type": "text/html", "X-Forwarded-For": ["ip1", "ip2"]}
h = Headers(idict)
assert dict(h) == {
assert dict(h.items()) == {
b"Content-Type": [b"text/html"],
b"X-Forwarded-For": [b"ip1", b"ip2"],
}
self.assertSortedEqual(h.keys(), [b"X-Forwarded-For", b"Content-Type"])
self.assertSortedEqual(
h.items(),
[(b"X-Forwarded-For", [b"ip1", b"ip2"]), (b"Content-Type", [b"text/html"])],
)
self.assertSortedEqual(h.values(), [b"ip2", b"text/html"])
assert sorted(h.keys()) == [b"Content-Type", b"X-Forwarded-For"]
assert sorted(h.items()) == [
(b"Content-Type", [b"text/html"]),
(b"X-Forwarded-For", [b"ip1", b"ip2"]),
]
assert set(h.values()) == {b"ip2", b"text/html"}
def test_update(self):
h = Headers()
@ -162,4 +159,4 @@ class TestHeaders:
with pytest.raises(TypeError, match="Unsupported value type"):
Headers().setdefault("foo", object())
with pytest.raises(TypeError, match="Unsupported value type"):
Headers().setlist("foo", [object()])
Headers().setlist("foo", [object()]) # type: ignore[list-item]

3
tests/test_http_request.py
View File

@ -1,4 +1,5 @@
import xmlrpc.client
from typing import Any
import pytest
@ -17,7 +18,7 @@ class TestXmlRpcRequest(TestRequestBase):
default_method = "POST"
default_headers = {b"Content-Type": [b"text/xml"]}
def _test_request(self, **kwargs):
def _test_request(self, **kwargs: Any) -> None:
r = self.request_class("http://scrapytest.org/rpc2", **kwargs)
assert r.headers[b"Content-Type"] == b"text/xml"
assert r.body == to_bytes(

75
tests/test_http_request_form.py
View File

@ -2,6 +2,7 @@ from __future__ import annotations
import re
import warnings
from typing import TYPE_CHECKING, Any
from urllib.parse import parse_qs, unquote_to_bytes
import pytest
@ -12,20 +13,32 @@ from scrapy.utils.httpobj import urlparse_cached
from scrapy.utils.python import to_unicode
from tests.utils.bases.http_request import TestRequestBase
if TYPE_CHECKING:
from scrapy import Request
def _buildresponse(body, **kwargs):
def _buildresponse(body: bytes | str, **kwargs: Any) -> HtmlResponse:
kwargs.setdefault("body", body)
kwargs.setdefault("url", "http://example.com")
kwargs.setdefault("encoding", "utf-8")
return HtmlResponse(**kwargs)
def _qs(req, encoding="utf-8", to_unicode=False):
qs = req.body if req.method == "POST" else req.url.partition("?")[2]
uqs = unquote_to_bytes(qs)
if to_unicode:
uqs = uqs.decode(encoding)
return parse_qs(uqs, True)
def _query_string(req: Request) -> bytes:
return req.body if req.method == "POST" else req.url.partition("?")[2].encode()
def _qs(req: Request) -> dict[bytes, list[bytes]]:
return parse_qs(unquote_to_bytes(_query_string(req)), True)
def _qs_unicode(req: Request, encoding: str = "utf-8") -> dict[str, list[str]]:
qs = unquote_to_bytes(_query_string(req)).decode(encoding)
return parse_qs(qs, True)
def _assert_query_equal(first: bytes, second: bytes) -> None:
assert sorted(to_unicode(first).split("&")) == sorted(to_unicode(second).split("&"))
# FormRequest.from_response() is deprecated in favor of form2request, so the
@ -34,11 +47,6 @@ def _qs(req, encoding="utf-8", to_unicode=False):
class TestFormRequest(TestRequestBase):
request_class = FormRequest
def assertQueryEqual(self, first, second, msg=None):
first = to_unicode(first).split("&")
second = to_unicode(second).split("&")
assert sorted(first) == sorted(second), msg
def test_init_not_deprecated(self):
# Building a request directly from form data is not deprecated.
with warnings.catch_warnings():
@ -75,20 +83,22 @@ class TestFormRequest(TestRequestBase):
assert fs[b"b"] == [b"2"]
assert fs.get(b"c") is None
data = {"a": "1", "b": "2"}
mapping = {"a": "1", "b": "2"}
fs = _qs(
self.request_class("http://www.example.com/", method="GET", formdata=data)
self.request_class(
"http://www.example.com/", method="GET", formdata=mapping
)
)
assert fs[b"a"] == [b"1"]
assert fs[b"b"] == [b"2"]
def test_default_encoding_bytes(self):
# using default encoding (utf-8)
data = {b"one": b"two", b"price": b"\xc2\xa3 100"}
data: dict[Any, Any] = {b"one": b"two", b"price": b"\xc2\xa3 100"}
r2 = self.request_class("http://www.example.com", formdata=data)
assert r2.method == "POST"
assert r2.encoding == "utf-8"
self.assertQueryEqual(r2.body, b"price=%C2%A3+100&one=two")
_assert_query_equal(r2.body, b"price=%C2%A3+100&one=two")
assert r2.headers[b"Content-Type"] == b"application/x-www-form-urlencoded"
def test_default_encoding_textual_data(self):
@ -97,26 +107,26 @@ class TestFormRequest(TestRequestBase):
r2 = self.request_class("http://www.example.com", formdata=data)
assert r2.method == "POST"
assert r2.encoding == "utf-8"
self.assertQueryEqual(r2.body, b"price=%C2%A3+100&%C2%B5+one=two")
_assert_query_equal(r2.body, b"price=%C2%A3+100&%C2%B5+one=two")
assert r2.headers[b"Content-Type"] == b"application/x-www-form-urlencoded"
def test_default_encoding_mixed_data(self):
# using default encoding (utf-8)
data = {"\u00b5one": b"two", b"price\xc2\xa3": "\u00a3 100"}
data: dict[Any, Any] = {"\u00b5one": b"two", b"price\xc2\xa3": "\u00a3 100"}
r2 = self.request_class("http://www.example.com", formdata=data)
assert r2.method == "POST"
assert r2.encoding == "utf-8"
self.assertQueryEqual(r2.body, b"%C2%B5one=two&price%C2%A3=%C2%A3+100")
_assert_query_equal(r2.body, b"%C2%B5one=two&price%C2%A3=%C2%A3+100")
assert r2.headers[b"Content-Type"] == b"application/x-www-form-urlencoded"
def test_custom_encoding_bytes(self):
data = {b"\xb5 one": b"two", b"price": b"\xa3 100"}
data: dict[Any, Any] = {b"\xb5 one": b"two", b"price": b"\xa3 100"}
r2 = self.request_class(
"http://www.example.com", formdata=data, encoding="latin1"
)
assert r2.method == "POST"
assert r2.encoding == "latin1"
self.assertQueryEqual(r2.body, b"price=%A3+100&%B5+one=two")
_assert_query_equal(r2.body, b"price=%A3+100&%B5+one=two")
assert r2.headers[b"Content-Type"] == b"application/x-www-form-urlencoded"
def test_custom_encoding_textual_data(self):
@ -131,7 +141,7 @@ class TestFormRequest(TestRequestBase):
# using multiples values for a single key
data = {"price": "\xa3 100", "colours": ["red", "blue", "green"]}
r3 = self.request_class("http://www.example.com", formdata=data)
self.assertQueryEqual(
_assert_query_equal(
r3.body, b"colours=red&colours=blue&colours=green&price=%C2%A3+100"
)
@ -173,7 +183,7 @@ class TestFormRequest(TestRequestBase):
assert req.method == "POST"
assert req.headers[b"Content-type"] == b"application/x-www-form-urlencoded"
assert req.url == "http://www.example.com/this/post.php"
fs = _qs(req, to_unicode=True)
fs = _qs_unicode(req)
assert set(fs["test £"]) == {"val1", "val2"}
assert set(fs["one"]) == {"two", "three"}
assert fs["test2"] == ["xxx µ"]
@ -196,7 +206,7 @@ class TestFormRequest(TestRequestBase):
assert req.method == "POST"
assert req.headers[b"Content-type"] == b"application/x-www-form-urlencoded"
assert req.url == "http://www.example.com/this/post.php"
fs = _qs(req, to_unicode=True, encoding="latin1")
fs = _qs_unicode(req, encoding="latin1")
assert set(fs["test £"]) == {"val1", "val2"}
assert set(fs["one"]) == {"two", "three"}
assert fs["test2"] == ["xxx µ"]
@ -218,7 +228,7 @@ class TestFormRequest(TestRequestBase):
assert req.method == "POST"
assert req.headers[b"Content-type"] == b"application/x-www-form-urlencoded"
assert req.url == "http://www.example.com/this/post.php"
fs = _qs(req, to_unicode=True)
fs = _qs_unicode(req)
assert set(fs["test £"]) == {"val1", "val2"}
assert set(fs["one"]) == {"two", "three"}
assert fs["test2"] == ["xxx µ"]
@ -305,7 +315,10 @@ class TestFormRequest(TestRequestBase):
<input type="hidden" name="two" value="3">
</form>"""
)
req = self.request_class.from_response(response, formdata={"two": None})
req = self.request_class.from_response(
response,
formdata={"two": None}, # type: ignore[arg-type]
)
fs = _qs(req)
assert fs[b"one"] == [b"1"]
assert b"two" not in fs
@ -450,7 +463,7 @@ class TestFormRequest(TestRequestBase):
req = self.request_class.from_response(
response, clickdata={"name": "price in \u00a3"}
)
fs = _qs(req, to_unicode=True)
fs = _qs_unicode(req)
assert fs["price in \u00a3"]
def test_from_response_unicode_clickdata_latin1(self):
@ -466,7 +479,7 @@ class TestFormRequest(TestRequestBase):
req = self.request_class.from_response(
response, clickdata={"name": "price in \u00a5"}
)
fs = _qs(req, to_unicode=True, encoding="latin1")
fs = _qs_unicode(req, encoding="latin1")
assert fs["price in \u00a5"]
def test_from_response_multiple_forms_clickdata(self):
@ -737,7 +750,7 @@ class TestFormRequest(TestRequestBase):
</form>"""
)
req = self.request_class.from_response(res)
fs = _qs(req, to_unicode=True)
fs = _qs_unicode(req)
assert fs == {"i1": ["i1v2"], "i2": ["i2v1"], "i4": ["i4v2", "i4v3"]}
def test_from_response_radio(self):
@ -1022,7 +1035,7 @@ class TestFormRequest(TestRequestBase):
with pytest.raises(
ValueError, match="formdata should be a dict or iterable of tuples"
):
FormRequest.from_response(response, formdata=123)
FormRequest.from_response(response, formdata=123) # type: ignore[arg-type]
def test_form_response_with_custom_invalid_formdata_value_error(self):
"""Test that a ValueError is raised for fault-inducing iterable formdata input"""
@ -1037,7 +1050,7 @@ class TestFormRequest(TestRequestBase):
with pytest.raises(
ValueError, match="formdata should be a dict or iterable of tuples"
):
FormRequest.from_response(response, formdata=("a",))
FormRequest.from_response(response, formdata=("a",)) # type: ignore[arg-type]
def test_get_form_with_xpath_no_form_parent(self):
"""Test that _get_from raised a ValueError when an XPath selects an element

15
tests/test_http_response_text.py
View File

@ -1,6 +1,7 @@
from __future__ import annotations
import codecs
from typing import cast
from unittest import mock
import pytest
@ -14,6 +15,12 @@ from tests.utils.bases.http_response import TestResponseBase
class TestTextResponse(TestResponseBase):
response_class = TextResponse
def _links_response(self) -> TextResponse:
return cast("TextResponse", super()._links_response())
def _links_response_no_href(self) -> TextResponse:
return cast("TextResponse", super()._links_response_no_href())
def test_follow_None_encoding(self):
# unlike the base Response, TextResponse.follow() falls back to the
# response encoding when encoding is None instead of raising
@ -21,7 +28,7 @@ class TestTextResponse(TestResponseBase):
req = r.follow("foo", encoding=None)
assert req.encoding == "cp1252"
def test_replace(self):
def test_replace(self) -> None:
super().test_replace()
r1 = self.response_class(
"http://www.example.com", body="hello", encoding="cp852"
@ -344,7 +351,7 @@ class TestTextResponse(TestResponseBase):
def test_follow_selector_list(self):
resp = self._links_response()
with pytest.raises(ValueError, match="SelectorList"):
resp.follow(resp.css("a"))
resp.follow(resp.css("a")) # type: ignore[arg-type]
def test_follow_selector_invalid(self):
resp = self._links_response()
@ -616,7 +623,7 @@ class CustomResponse(TextResponse):
class TestCustomResponse(TestTextResponse):
response_class = CustomResponse
def test_copy(self):
def test_copy(self) -> None:
super().test_copy()
r1 = self.response_class(
url="https://example.org",
@ -632,7 +639,7 @@ class TestCustomResponse(TestTextResponse):
assert r1.lost == "lost"
assert r2.lost is None
def test_replace(self):
def test_replace(self) -> None:
super().test_replace()
r1 = self.response_class(
url="https://example.org",

5
tests/test_request_attribute_binding.py
View File

@ -85,6 +85,7 @@ class TestCrawl:
url = self.mockserver.url("/status?n=200")
crawler = get_crawler(SingleRequestSpider)
yield crawler.crawl(seed=url, mockserver=self.mockserver)
assert isinstance(crawler.spider, SingleRequestSpider)
response = crawler.spider.meta["responses"][0]
assert response.request.url == url
@ -94,6 +95,7 @@ class TestCrawl:
url = self.mockserver.url(f"/status?n={status}")
crawler = get_crawler(SingleRequestSpider)
yield crawler.crawl(seed=url, mockserver=self.mockserver)
assert isinstance(crawler.spider, SingleRequestSpider)
failure = crawler.spider.meta["failure"]
response = failure.value.response
assert failure.request.url == url
@ -111,6 +113,7 @@ class TestCrawl:
},
)
yield crawler.crawl(seed=url, mockserver=self.mockserver)
assert isinstance(crawler.spider, SingleRequestSpider)
failure = crawler.spider.meta["failure"]
assert failure.request.url == url
assert isinstance(failure.value, ZeroDivisionError)
@ -178,6 +181,7 @@ class TestCrawl:
},
)
yield crawler.crawl(seed=url, mockserver=self.mockserver)
assert isinstance(crawler.spider, SingleRequestSpider)
response = crawler.spider.meta["responses"][0]
assert response.body == b"Caught ZeroDivisionError"
assert response.request.url == OVERRIDDEN_URL
@ -201,6 +205,7 @@ class TestCrawl:
},
)
yield crawler.crawl(seed=url, mockserver=self.mockserver)
assert isinstance(crawler.spider, SingleRequestSpider)
response = crawler.spider.meta["responses"][0]
assert response.body == b"Caught ZeroDivisionError"
assert response.request.url == url

26
tests/test_request_cb_kwargs.py
View File

@ -49,6 +49,7 @@ class InjectArgumentsSpiderMiddleware:
async for element in result:
if (
isinstance(element, Request)
and element.callback
and element.callback.__name__ == "parse_spider_mw_2"
):
element.cb_kwargs["from_process_spider_output"] = True
@ -68,7 +69,12 @@ class KeywordArgumentsSpider(MockServerSpider):
checks: list[bool] = []
def _inc_checks(self, count: int = 1) -> None:
assert self.crawler.stats
self.crawler.stats.inc_value("boolean_checks", count)
async def start(self):
assert self.mockserver
data = {"key": "value", "number": 123, "callback": "some_callback"}
yield Request(self.mockserver.url("/first"), self.parse_first, cb_kwargs=data)
yield Request(
@ -89,9 +95,10 @@ class KeywordArgumentsSpider(MockServerSpider):
yield Request(self.mockserver.url("/spider_mw"), self.parse_spider_mw)
def parse_first(self, response, key, number):
assert self.mockserver
self.checks.append(key == "value")
self.checks.append(number == 123)
self.crawler.stats.inc_value("boolean_checks", 2)
self._inc_checks(2)
yield response.follow(
self.mockserver.url("/two"),
self.parse_second,
@ -100,28 +107,28 @@ class KeywordArgumentsSpider(MockServerSpider):
def parse_second(self, response, new_key):
self.checks.append(new_key == "new_value")
self.crawler.stats.inc_value("boolean_checks")
self._inc_checks()
def parse_general(self, response, **kwargs):
if response.url.endswith("/general_with"):
self.checks.append(kwargs["key"] == "value")
self.checks.append(kwargs["number"] == 123)
self.checks.append(kwargs["callback"] == "some_callback")
self.crawler.stats.inc_value("boolean_checks", 3)
self._inc_checks(3)
elif response.url.endswith("/general_without"):
self.checks.append(kwargs == {})
self.crawler.stats.inc_value("boolean_checks")
self._inc_checks()
def parse_no_kwargs(self, response):
self.checks.append(response.url.endswith("/no_kwargs"))
self.crawler.stats.inc_value("boolean_checks")
self._inc_checks()
def parse_default(self, response, key, number=None, default=99):
self.checks.append(response.url.endswith("/default"))
self.checks.append(key == "value")
self.checks.append(number == 123)
self.checks.append(default == 99)
self.crawler.stats.inc_value("boolean_checks", 4)
self._inc_checks(4)
def parse_takes_less(self, response, key, callback):
"""
@ -140,17 +147,18 @@ class KeywordArgumentsSpider(MockServerSpider):
):
self.checks.append(bool(from_process_request))
self.checks.append(bool(from_process_response))
self.crawler.stats.inc_value("boolean_checks", 2)
self._inc_checks(2)
def parse_spider_mw(self, response, from_process_spider_input, from_process_start):
assert self.mockserver
self.checks.append(bool(from_process_spider_input))
self.checks.append(bool(from_process_start))
self.crawler.stats.inc_value("boolean_checks", 2)
self._inc_checks(2)
return Request(self.mockserver.url("/spider_mw_2"), self.parse_spider_mw_2)
def parse_spider_mw_2(self, response, from_process_spider_output):
self.checks.append(bool(from_process_spider_output))
self.crawler.stats.inc_value("boolean_checks", 1)
self._inc_checks()
class TestCallbackKeywordArguments:

58
tests/test_request_dict.py
View File

@ -1,7 +1,10 @@
from typing import Any
import pytest
from twisted.python.failure import Failure
from scrapy import Request, Spider
from scrapy.http import JsonRequest
from scrapy.http import JsonRequest, Response
from scrapy.utils.request import request_from_dict
@ -10,7 +13,7 @@ class CustomRequest(Request):
class TestRequestSerialization:
def setup_method(self):
def setup_method(self) -> None:
self.spider = MethodsSpider()
def test_basic(self):
@ -42,12 +45,14 @@ class TestRequestSerialization:
r = Request("http://www.example.com", body=b"\xc2\xa3")
self._assert_serializes_ok(r)
def _assert_serializes_ok(self, request, spider=None):
def _assert_serializes_ok(
self, request: Request, spider: Spider | None = None
) -> None:
d = request.to_dict(spider=spider)
request2 = request_from_dict(d, spider=spider)
self._assert_same_request(request, request2)
def _assert_same_request(self, r1, r2):
def _assert_same_request(self, r1: Request, r2: Request) -> None:
assert r1.__class__ == r2.__class__
assert r1.url == r2.url
assert r1.callback == r2.callback
@ -64,6 +69,7 @@ class TestRequestSerialization:
assert r1.dont_filter == r2.dont_filter
assert r1.flags == r2.flags
if isinstance(r1, JsonRequest):
assert isinstance(r2, JsonRequest)
assert r1.dumps_kwargs == r2.dumps_kwargs
def test_request_class(self):
@ -83,8 +89,8 @@ class TestRequestSerialization:
def test_reference_callback_serialization(self):
r = Request(
"http://www.example.com",
callback=self.spider.parse_item_reference,
errback=self.spider.handle_error_reference,
callback=self.spider.parse_item_reference, # type: ignore[arg-type,misc]
errback=self.spider.handle_error_reference, # type: ignore[arg-type,misc]
)
self._assert_serializes_ok(r, spider=self.spider)
request_dict = r.to_dict(spider=self.spider)
@ -94,8 +100,8 @@ class TestRequestSerialization:
def test_private_reference_callback_serialization(self):
r = Request(
"http://www.example.com",
callback=self.spider._MethodsSpider__parse_item_reference,
errback=self.spider._MethodsSpider__handle_error_reference,
callback=self.spider._MethodsSpider__parse_item_reference, # type: ignore[attr-defined]
errback=self.spider._MethodsSpider__handle_error_reference, # type: ignore[attr-defined]
)
self._assert_serializes_ok(r, spider=self.spider)
request_dict = r.to_dict(spider=self.spider)
@ -105,7 +111,7 @@ class TestRequestSerialization:
def test_private_callback_serialization(self):
r = Request(
"http://www.example.com",
callback=self.spider._MethodsSpider__parse_item_private,
callback=self.spider._MethodsSpider__parse_item_private, # type: ignore[attr-defined]
errback=self.spider.handle_error,
)
self._assert_serializes_ok(r, spider=self.spider)
@ -113,7 +119,7 @@ class TestRequestSerialization:
def test_mixin_private_callback_serialization(self):
r = Request(
"http://www.example.com",
callback=self.spider._SpiderMixin__mixin_callback,
callback=self.spider._SpiderMixin__mixin_callback, # type: ignore[attr-defined]
errback=self.spider.handle_error,
)
self._assert_serializes_ok(r, spider=self.spider)
@ -127,7 +133,7 @@ class TestRequestSerialization:
self._assert_serializes_ok(r, spider=self.spider)
def test_unserializable_callback1(self):
r = Request("http://www.example.com", callback=lambda x: x)
r = Request("http://www.example.com", callback=lambda x: x) # type: ignore[misc]
with pytest.raises(
ValueError, match="is not an instance method in: <MethodsSpider"
):
@ -144,12 +150,12 @@ class TestRequestSerialization:
class MySpider(Spider):
name = "my_spider"
def parse(self, response):
def parse(self, response: Response) -> None:
pass
spider = MySpider()
r = Request("http://www.example.com", callback=spider.parse)
spider.parse = None
spider.parse = None # type: ignore[method-assign,assignment]
with pytest.raises(ValueError, match="is not an instance method in: <MySpider"):
r.to_dict(spider=spider)
@ -157,7 +163,7 @@ class TestRequestSerialization:
"""Callback method is not available in the spider passed to from_dict"""
spider = SpiderDelegation()
r = Request("http://www.example.com", callback=spider.delegated_callback)
d = r.to_dict(spider=spider)
d = r.to_dict(spider=spider) # type: ignore[arg-type]
with pytest.raises(
ValueError, match="Method 'delegated_callback' not found in: <Spider"
):
@ -165,28 +171,30 @@ class TestRequestSerialization:
class SpiderMixin:
def __mixin_callback(self, response): # pylint: disable=unused-private-member
def __mixin_callback( # pylint: disable=unused-private-member
self, response: Response
) -> None:
pass
class SpiderDelegation:
def delegated_callback(self, response):
def delegated_callback(self, response: Response) -> None:
pass
def parse_item(response):
def parse_item(response: Response) -> None:
pass
def handle_error(failure):
def handle_error(failure: Failure) -> None:
pass
def private_parse_item(response):
def private_parse_item(response: Response) -> None:
pass
def private_handle_error(failure):
def private_handle_error(failure: Failure) -> None:
pass
@ -197,15 +205,17 @@ class MethodsSpider(Spider, SpiderMixin):
__parse_item_reference = private_parse_item
__handle_error_reference = private_handle_error
def __init__(self, **kwargs):
def __init__(self, **kwargs: Any) -> None:
super().__init__(**kwargs)
self.delegated_callback = SpiderDelegation().delegated_callback
def parse_item(self, response):
def parse_item(self, response: Response) -> None:
pass
def handle_error(self, failure):
def handle_error(self, failure: Failure) -> None:
pass
def __parse_item_private(self, response): # pylint: disable=unused-private-member
def __parse_item_private( # pylint: disable=unused-private-member
self, response: Response
) -> None:
pass

43
tests/test_request_left.py
View File

@ -1,57 +1,62 @@
from __future__ import annotations
from typing import TYPE_CHECKING, Any
from scrapy.signals import request_left_downloader
from scrapy.spiders import Spider
from scrapy.utils.test import get_crawler
from tests.mockserver.http import MockServer
from tests.utils.decorators import inline_callbacks_test
if TYPE_CHECKING:
from scrapy import Request
from scrapy.crawler import Crawler
from tests.mockserver.http import MockServer
class SignalCatcherSpider(Spider):
name = "signal_catcher"
def __init__(self, crawler, url, *args, **kwargs):
def __init__(self, crawler: Crawler, url: str, *args: Any, **kwargs: Any):
super().__init__(*args, **kwargs)
crawler.signals.connect(self.on_request_left, signal=request_left_downloader)
self.caught_times = 0
self.start_urls = [url]
@classmethod
def from_crawler(cls, crawler, *args, **kwargs):
def from_crawler(
cls, crawler: Crawler, *args: Any, **kwargs: Any
) -> SignalCatcherSpider:
return cls(crawler, *args, **kwargs)
def on_request_left(self, request, spider):
def on_request_left(self, request: Request, spider: Spider) -> None:
self.caught_times += 1
class TestCatching:
@classmethod
def setup_class(cls):
cls.mockserver = MockServer()
cls.mockserver.__enter__()
@classmethod
def teardown_class(cls):
cls.mockserver.__exit__(None, None, None)
@inline_callbacks_test
def test_success(self):
def test_success(self, mockserver: MockServer):
crawler = get_crawler(SignalCatcherSpider)
yield crawler.crawl(self.mockserver.url("/status?n=200"))
yield crawler.crawl(mockserver.url("/status?n=200"))
assert isinstance(crawler.spider, SignalCatcherSpider)
assert crawler.spider.caught_times == 1
@inline_callbacks_test
def test_timeout(self):
def test_timeout(self, mockserver: MockServer):
crawler = get_crawler(SignalCatcherSpider, {"DOWNLOAD_TIMEOUT": 0.1})
yield crawler.crawl(self.mockserver.url("/delay?n=0.2"))
yield crawler.crawl(mockserver.url("/delay?n=0.2"))
assert isinstance(crawler.spider, SignalCatcherSpider)
assert crawler.spider.caught_times == 1
@inline_callbacks_test
def test_disconnect(self):
def test_disconnect(self, mockserver: MockServer):
crawler = get_crawler(SignalCatcherSpider)
yield crawler.crawl(self.mockserver.url("/drop"))
yield crawler.crawl(mockserver.url("/drop"))
assert isinstance(crawler.spider, SignalCatcherSpider)
assert crawler.spider.caught_times == 1
@inline_callbacks_test
def test_noconnect(self):
crawler = get_crawler(SignalCatcherSpider)
yield crawler.crawl("http://thereisdefinetelynosuchdomain.com")
assert isinstance(crawler.spider, SignalCatcherSpider)
assert crawler.spider.caught_times == 1

59
tests/utils/bases/http_request.py
View File

@ -3,8 +3,9 @@ from abc import ABC, abstractmethod
from typing import Any
import pytest
from twisted.python.failure import Failure
from scrapy.http import Headers, Request
from scrapy.http import Headers, Request, Response
from scrapy.http.request import NO_CALLBACK
from scrapy.utils.request import request_to_curl
@ -22,15 +23,15 @@ class TestRequestBase(ABC):
def test_init(self):
# Request requires url in the __init__ method
with pytest.raises(TypeError):
self.request_class()
self.request_class() # type: ignore[call-arg]
# url argument must be basestring
with pytest.raises(TypeError):
self.request_class(123)
self.request_class(123) # type: ignore[arg-type]
# priority argument must be an integer
with pytest.raises(TypeError, match="Request priority not an integer"):
self.request_class("http://www.example.com", priority="1")
self.request_class("http://www.example.com", priority="1") # type: ignore[arg-type]
r = self.request_class("http://www.example.com")
assert isinstance(r.url, str)
@ -205,14 +206,17 @@ class TestRequestBase(ABC):
def test_copy(self):
"""Test Request copy"""
def somecallback():
def somecallback(response: Response) -> None:
pass
def someerrback(failure: Failure) -> None:
pass
r1 = self.request_class(
"http://www.example.com",
flags=["f1", "f2"],
callback=somecallback,
errback=somecallback,
errback=someerrback,
)
r1.meta["foo"] = "bar"
r1.cb_kwargs["key"] = "value"
@ -220,7 +224,7 @@ class TestRequestBase(ABC):
# make sure callbaclks are copied
assert r1.callback is somecallback
assert r1.errback is somecallback
assert r1.errback is someerrback
assert r2.callback is r1.callback
assert r2.errback is r1.errback
@ -251,7 +255,7 @@ class TestRequestBase(ABC):
def test_copy_inherited_classes(self):
"""Test Request children copies preserve their class"""
class CustomRequest(self.request_class):
class CustomRequest(self.request_class): # type: ignore[misc,name-defined]
pass
r1 = CustomRequest("http://www.example.com")
@ -283,7 +287,9 @@ class TestRequestBase(ABC):
assert r4.dont_filter is False
# the cls argument allows changing the resulting class
custom_request_cls = type("CustomRequest", (self.request_class,), {})
custom_request_cls: type[Request] = type(
"CustomRequest", (self.request_class,), {}
)
r5 = r1.replace(cls=custom_request_cls)
assert isinstance(r5, custom_request_cls)
assert r5.url == r1.url
@ -295,33 +301,36 @@ class TestRequestBase(ABC):
def test_immutable_attributes(self):
r = self.request_class("http://example.com")
with pytest.raises(AttributeError):
r.url = "http://example2.com"
r.url = "http://example2.com" # type: ignore[misc]
with pytest.raises(AttributeError):
r.body = "xxx"
r.body = "xxx" # type: ignore[misc,assignment]
def test_callback_and_errback(self):
def a_function():
def a_callback(response: Response) -> None:
pass
def an_errback(failure: Failure) -> None:
pass
r1 = self.request_class("http://example.com")
assert r1.callback is None
assert r1.errback is None
r2 = self.request_class("http://example.com", callback=a_function)
assert r2.callback is a_function
r2 = self.request_class("http://example.com", callback=a_callback)
assert r2.callback is a_callback
assert r2.errback is None
r3 = self.request_class("http://example.com", errback=a_function)
r3 = self.request_class("http://example.com", errback=an_errback)
assert r3.callback is None
assert r3.errback is a_function
assert r3.errback is an_errback
r4 = self.request_class(
url="http://example.com",
callback=a_function,
errback=a_function,
callback=a_callback,
errback=an_errback,
)
assert r4.callback is a_function
assert r4.errback is a_function
assert r4.callback is a_callback
assert r4.errback is an_errback
r5 = self.request_class(
url="http://example.com",
@ -329,18 +338,18 @@ class TestRequestBase(ABC):
errback=NO_CALLBACK,
)
assert r5.callback is NO_CALLBACK
assert r5.errback is NO_CALLBACK
assert r5.errback is NO_CALLBACK # type: ignore[comparison-overlap]
def test_callback_and_errback_type(self):
with pytest.raises(TypeError):
self.request_class("http://example.com", callback="a_function")
self.request_class("http://example.com", callback="a_function") # type: ignore[arg-type]
with pytest.raises(TypeError):
self.request_class("http://example.com", errback="a_function")
self.request_class("http://example.com", errback="a_function") # type: ignore[arg-type]
with pytest.raises(TypeError):
self.request_class(
url="http://example.com",
callback="a_function",
errback="a_function",
callback="a_function", # type: ignore[arg-type]
errback="a_function", # type: ignore[arg-type]
)
def test_setters(self):

48
tests/utils/bases/http_response.py
View File

@ -7,7 +7,7 @@ import pytest
from w3lib.encoding import resolve_encoding
from scrapy.exceptions import NotSupported
from scrapy.http import Headers, Request, Response
from scrapy.http import Headers, Request, Response, TextResponse
from scrapy.link import Link
from scrapy.utils._deps_compat import W3LIB_STRIPS_URLS
from tests import get_testdata
@ -15,6 +15,8 @@ from tests import get_testdata
if TYPE_CHECKING:
from collections.abc import Iterable
from parsel import Selector
class TestResponseBase(ABC):
@property
@ -25,14 +27,14 @@ class TestResponseBase(ABC):
def test_init(self):
# Response requires url in the constructor
with pytest.raises(TypeError):
self.response_class()
self.response_class() # type: ignore[call-arg]
assert isinstance(
self.response_class("http://example.com/"), self.response_class
)
with pytest.raises(TypeError):
self.response_class(b"http://example.com")
self.response_class(b"http://example.com") # type: ignore[arg-type]
with pytest.raises(TypeError):
self.response_class(url="http://example.com", body={})
self.response_class(url="http://example.com", body={}) # type: ignore[arg-type]
# body can be str or None
assert isinstance(
self.response_class("http://example.com/", body=b""),
@ -67,12 +69,12 @@ class TestResponseBase(ABC):
r = self.response_class("http://www.example.com", status=301)
assert r.status == 301
r = self.response_class("http://www.example.com", status="301")
r = self.response_class("http://www.example.com", status="301") # type: ignore[arg-type]
assert r.status == 301
with pytest.raises(ValueError, match=r"invalid literal for int\(\)"):
self.response_class("http://example.com", status="lala200")
self.response_class("http://example.com", status="lala200") # type: ignore[arg-type]
def test_copy(self):
def test_copy(self) -> None:
"""Test Response copy"""
r1 = self.response_class("http://www.example.com", body=b"Some body")
@ -121,7 +123,7 @@ class TestResponseBase(ABC):
def test_copy_inherited_classes(self):
"""Test Response children copies preserve their class"""
class CustomResponse(self.response_class):
class CustomResponse(self.response_class): # type: ignore[misc,name-defined]
pass
r1 = CustomResponse("http://www.example.com")
@ -129,7 +131,7 @@ class TestResponseBase(ABC):
assert isinstance(r2, CustomResponse)
def test_replace(self):
def test_replace(self) -> None:
"""Test Response.replace() method"""
hdrs = Headers({"key": "value"})
r1 = self.response_class("http://www.example.com")
@ -146,7 +148,9 @@ class TestResponseBase(ABC):
assert r4.body == b""
assert not r4.flags
def _assert_response_values(self, response, encoding, body):
def _assert_response_values(
self, response: TextResponse, encoding: str, body: str | bytes
) -> None:
if isinstance(body, str):
body_unicode = body
body_bytes = body.encode(encoding)
@ -160,15 +164,15 @@ class TestResponseBase(ABC):
assert response.body == body_bytes
assert response.text == body_unicode
def _assert_response_encoding(self, response, encoding):
def _assert_response_encoding(self, response: TextResponse, encoding: str) -> None:
assert response.encoding == resolve_encoding(encoding)
def test_immutable_attributes(self):
r = self.response_class("http://example.com")
with pytest.raises(AttributeError):
r.url = "http://example2.com"
r.url = "http://example2.com" # type: ignore[misc]
with pytest.raises(AttributeError):
r.body = "xxx"
r.body = "xxx" # type: ignore[misc,assignment]
def test_setter_mutable_lazy_loading(self):
"""Mutable attributes are set internally to None only until they are
@ -256,7 +260,7 @@ class TestResponseBase(ABC):
def test_follow_None_url(self):
r = self.response_class("http://example.com")
with pytest.raises(ValueError, match="url can't be None"):
r.follow(None)
r.follow(None) # type: ignore[arg-type]
def test_follow_None_encoding(self):
r = self.response_class("http://example.com")
@ -325,20 +329,20 @@ class TestResponseBase(ABC):
r = self.response_class("http://example.com")
if self.response_class == Response:
with pytest.raises(TypeError):
list(r.follow_all(urls=None))
list(r.follow_all(urls=None)) # type: ignore[arg-type]
with pytest.raises(TypeError):
list(r.follow_all(urls=12345))
list(r.follow_all(urls=12345)) # type: ignore[arg-type]
with pytest.raises(ValueError, match="url can't be None"):
list(r.follow_all(urls=[None]))
list(r.follow_all(urls=[None])) # type: ignore[list-item]
else:
with pytest.raises(
ValueError, match="Please supply exactly one of the following arguments"
):
list(r.follow_all(urls=None))
list(r.follow_all(urls=None)) # type: ignore[arg-type]
with pytest.raises(TypeError):
list(r.follow_all(urls=12345))
list(r.follow_all(urls=12345)) # type: ignore[arg-type]
with pytest.raises(ValueError, match="url can't be None"):
list(r.follow_all(urls=[None]))
list(r.follow_all(urls=[None])) # type: ignore[list-item]
@pytest.mark.xfail(
not W3LIB_STRIPS_URLS,
@ -384,14 +388,14 @@ class TestResponseBase(ABC):
def _assert_followed_url(
self,
follow_obj: str | Link,
follow_obj: str | Link | Selector,
target_url: str,
response: Response | None = None,
encoding: str | None = None,
) -> None:
if response is None:
response = self._links_response()
req = response.follow(follow_obj)
req = response.follow(follow_obj) # type: ignore[arg-type]
assert req.url == target_url
if encoding is not None:
assert req.encoding == encoding

3
tox.ini
View File

@ -111,7 +111,8 @@ commands =
pre-commit run {posargs:--all-files}
[testenv:pylint]
basepython = python3
# Some checks are Python-version-dependent, so pin the version used in CI.
basepython = python3.14
deps =
{[testenv:extra-deps]deps}
pylint==4.0.6