RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Type downloader middleware tests (#7858)

This commit is contained in:
Adrian 2026-08-03 12:49:15 +02:00 committed by GitHub
parent e83c709574
commit 2b2e18199b
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
13 changed files with 236 additions and 178 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
pyproject.toml
View File

@ -125,17 +125,6 @@ module = [
"tests.test_contracts",
"tests.test_core_downloader",
"tests.test_downloader_handler_twisted_ftp",
"tests.test_downloadermiddleware_cookies",
"tests.test_downloadermiddleware_httpauth",
"tests.test_downloadermiddleware_httpcache",
"tests.test_downloadermiddleware_httpcompression",
"tests.test_downloadermiddleware_httpproxy",
"tests.test_downloadermiddleware_offsite",
"tests.test_downloadermiddleware_redirect",
"tests.test_downloadermiddleware_redirect_base",
"tests.test_downloadermiddleware_redirect_metarefresh",
"tests.test_downloadermiddleware_retry",
"tests.test_downloadermiddleware_robotstxt",
"tests.test_downloaderslotssettings",
"tests.test_dupefilters",
"tests.test_engine_loop",
@ -167,7 +156,6 @@ module = [
"tests.test_request_cb_kwargs",
"tests.test_request_dict",
"tests.test_request_left",
"tests.test_robotstxt_interface",
"tests.test_scheduler_base",
"tests.test_settings",
"tests.test_spider",

4
scrapy/http/request/__init__.py
View File

@ -50,7 +50,9 @@ class VerboseCookie(TypedDict):
secure: NotRequired[bool]
CookiesT: TypeAlias = dict[str | bytes, str | bytes] | list[VerboseCookie]
CookiesT: TypeAlias = (
dict[str | bytes, str | bytes | bool | float | int] | list[VerboseCookie]
)
RequestTypeVar = TypeVar("RequestTypeVar", bound="Request")

66
tests/test_downloadermiddleware_cookies.py
View File

@ -1,5 +1,6 @@
import logging
from collections.abc import Iterable
from typing import Any
import pytest
@ -219,7 +220,7 @@ class TestCookiesMiddleware:
def test_complex_cookies(self):
# merge some cookies into jar
cookies = [
cookies: list[VerboseCookie] = [
{
"name": "C1",
"value": "value1",
@ -483,13 +484,13 @@ class TestCookiesMiddleware:
def _test_cookie_redirect(
self,
source,
target,
source: str | dict[str, Any],
target: str | dict[str, Any],
*,
cookies1,
cookies2,
):
input_cookies = {"a": "b"}
cookies1: bool,
cookies2: bool,
) -> None:
input_cookies: CookiesT = {"a": "b"}
if not isinstance(source, dict):
source = {"url": source}
@ -551,11 +552,11 @@ class TestCookiesMiddleware:
def _test_cookie_header_redirect(
self,
source,
target,
source: str | dict[str, Any],
target: str | dict[str, Any],
*,
cookies2,
):
cookies2: bool,
) -> None:
"""Test the handling of a user-defined Cookie header when building a
redirect follow-up request.
@ -623,14 +624,14 @@ class TestCookiesMiddleware:
def _test_user_set_cookie_domain_followup(
self,
url1,
url2,
domain,
url1: str,
url2: str,
domain: str,
*,
cookies1,
cookies2,
):
input_cookies = [
cookies1: bool,
cookies2: bool,
) -> None:
input_cookies: list[VerboseCookie] = [
{
"name": "a",
"value": "b",
@ -686,16 +687,16 @@ class TestCookiesMiddleware:
def _test_server_set_cookie_domain_followup(
self,
url1,
url2,
domain,
url1: str,
url2: str,
domain: str,
*,
cookies,
):
cookies: bool,
) -> None:
request1 = Request(url1)
self.mw.process_request(request1)
input_cookies = [
input_cookies: list[VerboseCookie] = [
{
"name": "a",
"value": "b",
@ -747,8 +748,14 @@ class TestCookiesMiddleware:
)
def _test_cookie_redirect_scheme_change(
self, secure, from_scheme, to_scheme, cookies1, cookies2, cookies3
):
self,
secure: bool | object,
from_scheme: str,
to_scheme: str,
cookies1: bool,
cookies2: bool,
cookies3: bool,
) -> None:
"""When a redirect causes the URL scheme to change from *from_scheme*
to *to_scheme*, while domain and port remain the same, and given a
cookie on the initial request with its secure attribute set to
@ -756,10 +763,11 @@ class TestCookiesMiddleware:
initial request (*cookies1*), if it should be kept by the redirect
middleware (*cookies2*), and if it should be present on the Cookie
header in the redirected request (*cookie3*)."""
cookie_kwargs = {}
cookie: VerboseCookie = {"name": "a", "value": "b"}
if secure is not UNSET:
cookie_kwargs["secure"] = secure
input_cookies = [{"name": "a", "value": "b", **cookie_kwargs}]
assert isinstance(secure, bool)
cookie["secure"] = secure
input_cookies = [cookie]
request1 = Request(f"{from_scheme}://a.example", cookies=input_cookies)
self.mw.process_request(request1)

8
tests/test_downloadermiddleware_httpauth.py
View File

@ -1,3 +1,5 @@
from typing import Any
import pytest
from w3lib.http import basic_auth_header
@ -10,8 +12,10 @@ from scrapy.utils.test import get_crawler
_DOMAIN_NOT_SET = object()
def make_mw(user="", passwd="", domain=_DOMAIN_NOT_SET):
settings: dict = {
def make_mw(
user: str = "", passwd: str = "", domain: str | object = _DOMAIN_NOT_SET
) -> HttpAuthMiddleware:
settings: dict[str, Any] = {
"HTTPAUTH_USER": user,
"HTTPAUTH_PASS": passwd,
}

31
tests/test_downloadermiddleware_httpcache.py
View File

@ -94,14 +94,14 @@ class TestBase:
finally:
mw.spider_closed(crawler.spider)
def assertEqualResponse(self, response1, response2):
def assertEqualResponse(self, response1: Response, response2: Response) -> None:
assert response1.url == response2.url
assert response1.status == response2.status
assert response1.headers == response2.headers
assert response1.body == response2.body
class StorageTestMixin:
class StorageTestMixin(TestBase):
"""Mixin containing storage-specific test methods."""
def _corrupt_cache_entry(
@ -135,6 +135,8 @@ class StorageTestMixin:
def test_corrupted_cache_entry_is_a_miss(self, caplog):
with self._middleware() as mw:
spider = mw.crawler.spider
assert spider
assert mw.crawler.stats
mw.storage.store_response(spider, self.request, self.response)
self._corrupt_cache_entry(mw.storage, spider, self.request)
@ -155,6 +157,8 @@ class StorageTestMixin:
def test_corrupted_cache_entry_ignore_missing(self):
with self._middleware(HTTPCACHE_IGNORE_MISSING=True) as mw:
spider = mw.crawler.spider
assert spider
assert mw.crawler.stats
mw.storage.store_response(spider, self.request, self.response)
self._corrupt_cache_entry(mw.storage, spider, self.request)
@ -180,7 +184,7 @@ class StorageTestMixin:
self.assertEqualResponse(response, cached_response)
class PolicyTestMixin:
class PolicyTestMixin(TestBase):
"""Mixin containing policy-specific test methods."""
def test_dont_cache(self):
@ -302,6 +306,7 @@ class DummyPolicyTestMixin(PolicyTestMixin):
assert mw.process_request(self.request) is None
fresh_response = self.response.replace(body=b"new body")
response = mw.process_response(self.request, fresh_response)
assert isinstance(response, Response)
self.assertEqualResponse(self.response, response)
assert "cached" in response.flags
assert mw.stats.get_value("httpcache/revalidate") == 1
@ -313,12 +318,12 @@ class RFC2616PolicyTestMixin(PolicyTestMixin):
@staticmethod
def _process_requestresponse(
mw: HttpCacheMiddleware, request: Request, response: Response | None
) -> Response | Request:
result = None
) -> Response:
result: Request | Response | None = None
try:
result = mw.process_request(request)
if result:
assert isinstance(result, (Request, Response))
assert isinstance(result, Response)
return result
assert response is not None
result = mw.process_response(request, response)
@ -346,6 +351,7 @@ class RFC2616PolicyTestMixin(PolicyTestMixin):
res2 = self._process_requestresponse(mw, req0, res0)
assert "cached" not in res2.flags
res3 = mw.process_request(req0)
assert isinstance(res3, Response)
assert "cached" in res3.flags
self.assertEqualResponse(res2, res3)
# request with no-cache directive must not return cached response
@ -634,6 +640,7 @@ class RFC2616PolicyTestMixin(PolicyTestMixin):
assert mw.process_request(req0) is None
res1 = mw.process_exception(req0, e("foo"))
# Use cached response as recovery
assert isinstance(res1, Response)
assert "cached" in res1.flags
self.assertEqualResponse(res0, res1)
# Do not use cached response for unhandled exceptions
@ -684,26 +691,22 @@ class DbmStorageTestMixin(StorageTestMixin):
class TestFilesystemStorageWithDummyPolicy(
TestBase, FilesystemStorageTestMixin, DummyPolicyTestMixin
FilesystemStorageTestMixin, DummyPolicyTestMixin
):
policy_class = "scrapy.extensions.httpcache.DummyPolicy"
class TestFilesystemStorageWithRFC2616Policy(
TestBase, FilesystemStorageTestMixin, RFC2616PolicyTestMixin
FilesystemStorageTestMixin, RFC2616PolicyTestMixin
):
policy_class = "scrapy.extensions.httpcache.RFC2616Policy"
class TestDbmStorageWithDummyPolicy(
TestBase, DbmStorageTestMixin, DummyPolicyTestMixin
):
class TestDbmStorageWithDummyPolicy(DbmStorageTestMixin, DummyPolicyTestMixin):
policy_class = "scrapy.extensions.httpcache.DummyPolicy"
class TestDbmStorageWithRFC2616Policy(
TestBase, DbmStorageTestMixin, RFC2616PolicyTestMixin
):
class TestDbmStorageWithRFC2616Policy(DbmStorageTestMixin, RFC2616PolicyTestMixin):
policy_class = "scrapy.extensions.httpcache.RFC2616Policy"

58
tests/test_downloadermiddleware_httpcompression.py
View File

@ -3,6 +3,7 @@ from importlib.util import find_spec
from io import BytesIO
from logging import WARNING
from pathlib import Path
from typing import Any
import pytest
from w3lib.encoding import resolve_encoding
@ -15,6 +16,7 @@ from scrapy.exceptions import IgnoreRequest, NotConfigured, ScrapyDeprecationWar
from scrapy.http import HtmlResponse, Request, Response
from scrapy.responsetypes import responsetypes
from scrapy.spiders import Spider
from scrapy.utils._compression import _DecompressionMaxSizeExceeded
from scrapy.utils.gz import gunzip
from scrapy.utils.test import get_crawler
from tests import tests_datadir
@ -72,6 +74,7 @@ class TestHttpCompression:
def setup_method(self):
self.crawler = get_crawler(Spider)
self.mw = HttpCompressionMiddleware.from_crawler(self.crawler)
assert self.crawler.stats
self.crawler.stats.open_spider()
def _getresponse(self, coding: str) -> Response:
@ -96,7 +99,8 @@ class TestHttpCompression:
)
return response
def assertStatsEqual(self, key, value):
def assertStatsEqual(self, key: str, value: Any) -> None:
assert self.crawler.stats
assert self.crawler.stats.get_value(key) == value, str(
self.crawler.stats.get_stats()
)
@ -145,6 +149,7 @@ class TestHttpCompression:
def test_process_response_gzip(self):
response = self._getresponse("gzip")
assert response.request
request = response.request
assert response.headers["Content-Encoding"] == b"gzip"
@ -159,6 +164,7 @@ class TestHttpCompression:
_skip_if_no_br()
response = self._getresponse("br")
assert response.request
request = response.request
assert response.headers["Content-Encoding"] == b"br"
newresponse = self.mw.process_response(request, response)
@ -172,6 +178,7 @@ class TestHttpCompression:
if find_spec("brotli") is not None or find_spec("brotlicffi") is not None:
pytest.skip("Requires not having brotli support")
response = self._getresponse("br")
assert response.request
request = response.request
assert response.headers["Content-Encoding"] == b"br"
caplog.clear()
@ -201,6 +208,7 @@ class TestHttpCompression:
if not check_key.startswith("zstd-"):
continue
response = self._getresponse(check_key)
assert response.request
request = response.request
assert response.headers["Content-Encoding"] == b"zstd"
newresponse = self.mw.process_response(request, response)
@ -216,6 +224,7 @@ class TestHttpCompression:
if find_spec("zstandard") is not None:
pytest.skip("Requires not having zstandard support")
response = self._getresponse("zstd-static-content-size")
assert response.request
request = response.request
assert response.headers["Content-Encoding"] == b"zstd"
caplog.clear()
@ -239,6 +248,7 @@ class TestHttpCompression:
def test_process_response_rawdeflate(self):
response = self._getresponse("rawdeflate")
assert response.request
request = response.request
assert response.headers["Content-Encoding"] == b"deflate"
@ -251,6 +261,7 @@ class TestHttpCompression:
def test_process_response_zlibdelate(self):
response = self._getresponse("zlibdeflate")
assert response.request
request = response.request
assert response.headers["Content-Encoding"] == b"deflate"
@ -275,6 +286,7 @@ class TestHttpCompression:
def test_multipleencodings(self):
response = self._getresponse("gzip")
response.headers["Content-Encoding"] = ["uuencode", "gzip"]
assert response.request
request = response.request
newresponse = self.mw.process_response(request, response)
assert newresponse is not response
@ -282,6 +294,7 @@ class TestHttpCompression:
def test_multi_compression_single_header(self):
response = self._getresponse("gzip-deflate")
assert response.request
request = response.request
newresponse = self.mw.process_response(request, response)
assert newresponse is not response
@ -293,6 +306,7 @@ class TestHttpCompression:
) -> None:
response = self._getresponse("gzip-deflate")
response.headers["Content-Encoding"] = [b"gzip, foo, deflate"]
assert response.request
request = response.request
caplog.clear()
with caplog.at_level(
@ -315,6 +329,7 @@ class TestHttpCompression:
def test_multi_compression_multiple_header(self):
response = self._getresponse("gzip-deflate")
response.headers["Content-Encoding"] = ["gzip", "deflate"]
assert response.request
request = response.request
newresponse = self.mw.process_response(request, response)
assert newresponse is not response
@ -324,6 +339,7 @@ class TestHttpCompression:
def test_multi_compression_multiple_header_invalid_compression(self):
response = self._getresponse("gzip-deflate")
response.headers["Content-Encoding"] = ["gzip", "foo", "deflate"]
assert response.request
request = response.request
newresponse = self.mw.process_response(request, response)
assert newresponse is not response
@ -332,6 +348,7 @@ class TestHttpCompression:
def test_multi_compression_single_and_multiple_header(self):
response = self._getresponse("gzip-deflate-gzip")
response.headers["Content-Encoding"] = ["gzip", "deflate, gzip"]
assert response.request
request = response.request
newresponse = self.mw.process_response(request, response)
assert newresponse is not response
@ -341,6 +358,7 @@ class TestHttpCompression:
def test_multi_compression_single_and_multiple_header_invalid_compression(self):
response = self._getresponse("gzip-deflate")
response.headers["Content-Encoding"] = ["gzip", "foo,deflate"]
assert response.request
request = response.request
newresponse = self.mw.process_response(request, response)
assert newresponse is not response
@ -397,9 +415,7 @@ class TestHttpCompression:
self.assertStatsEqual("httpcompression/response_bytes", len(plainbody))
def test_process_response_no_content_type_header(self):
headers = {
"Content-Encoding": "identity",
}
headers = {b"Content-Encoding": b"identity"}
plainbody = (
b"<html><head><title>Some page</title>"
b'<meta http-equiv="Content-Type" content="text/html; charset=gb2312">'
@ -414,6 +430,7 @@ class TestHttpCompression:
newresponse = self.mw.process_response(request, response)
assert isinstance(newresponse, respcls)
assert isinstance(newresponse, HtmlResponse)
assert newresponse.body == plainbody
assert newresponse.encoding == resolve_encoding("gb2312")
self.assertStatsEqual("httpcompression/response_count", 1)
@ -422,6 +439,7 @@ class TestHttpCompression:
def test_process_response_gzipped_contenttype(self):
response = self._getresponse("gzip")
response.headers["Content-Type"] = "application/gzip"
assert response.request
request = response.request
newresponse = self.mw.process_response(request, response)
@ -434,6 +452,7 @@ class TestHttpCompression:
def test_process_response_gzip_app_octetstream_contenttype(self):
response = self._getresponse("gzip")
response.headers["Content-Type"] = "application/octet-stream"
assert response.request
request = response.request
newresponse = self.mw.process_response(request, response)
@ -446,6 +465,7 @@ class TestHttpCompression:
def test_process_response_gzip_binary_octetstream_contenttype(self):
response = self._getresponse("x-gzip")
response.headers["Content-Type"] = "binary/octet-stream"
assert response.request
request = response.request
newresponse = self.mw.process_response(request, response)
@ -504,6 +524,7 @@ class TestHttpCompression:
def test_process_response_head_request_no_decode_required(self):
response = self._getresponse("gzip")
response.headers["Content-Type"] = "application/gzip"
assert response.request
request = response.request
request.method = "HEAD"
response = response.replace(body=None)
@ -513,7 +534,7 @@ class TestHttpCompression:
self.assertStatsEqual("httpcompression/response_count", None)
self.assertStatsEqual("httpcompression/response_bytes", None)
def _test_compression_bomb_setting(self, compression_id):
def _test_compression_bomb_setting(self, compression_id: str) -> None:
settings = {"DOWNLOAD_MAXSIZE": 1_000_000}
crawler = get_crawler(Spider, settings_dict=settings)
spider = crawler._create_spider("scrapytest.org")
@ -521,9 +542,12 @@ class TestHttpCompression:
mw.open_spider(spider)
response = self._getresponse(f"bomb-{compression_id}") # 11_511_612 B
assert response.request
with pytest.raises(IgnoreRequest) as exc_info:
mw.process_response(response.request, response)
assert exc_info.value.__cause__.decompressed_size < 1_100_000
cause = exc_info.value.__cause__
assert isinstance(cause, _DecompressionMaxSizeExceeded)
assert cause.decompressed_size < 1_100_000
def test_compression_bomb_setting_br(self):
_skip_if_no_br()
@ -549,6 +573,7 @@ class TestHttpCompression:
mw.open_spider(spider)
response = self._getresponse("bomb-gzip") # 11_511_612 B
assert response.request
caplog.clear()
with (
caplog.at_level(
@ -565,7 +590,7 @@ class TestHttpCompression:
)
]
def _test_compression_bomb_spider_attr(self, compression_id):
def _test_compression_bomb_spider_attr(self, compression_id: str) -> None:
class DownloadMaxSizeSpider(Spider):
download_maxsize = 1_000_000
@ -575,9 +600,12 @@ class TestHttpCompression:
mw.open_spider(spider)
response = self._getresponse(f"bomb-{compression_id}")
assert response.request
with pytest.raises(IgnoreRequest) as exc_info:
mw.process_response(response.request, response)
assert exc_info.value.__cause__.decompressed_size < 1_100_000
cause = exc_info.value.__cause__
assert isinstance(cause, _DecompressionMaxSizeExceeded)
assert cause.decompressed_size < 1_100_000
@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
def test_compression_bomb_spider_attr_br(self):
@ -599,7 +627,7 @@ class TestHttpCompression:
self._test_compression_bomb_spider_attr("zstd")
def _test_compression_bomb_request_meta(self, compression_id):
def _test_compression_bomb_request_meta(self, compression_id: str) -> None:
crawler = get_crawler(Spider)
spider = crawler._create_spider("scrapytest.org")
mw = HttpCompressionMiddleware.from_crawler(crawler)
@ -607,9 +635,12 @@ class TestHttpCompression:
response = self._getresponse(f"bomb-{compression_id}")
response.meta["download_maxsize"] = 1_000_000
assert response.request
with pytest.raises(IgnoreRequest) as exc_info:
mw.process_response(response.request, response)
assert exc_info.value.__cause__.decompressed_size < 1_100_000
cause = exc_info.value.__cause__
assert isinstance(cause, _DecompressionMaxSizeExceeded)
assert cause.decompressed_size < 1_100_000
def test_compression_bomb_request_meta_br(self):
_skip_if_no_br()
@ -789,7 +820,7 @@ class TestHttpCompression:
self._test_download_warnsize_request_meta(caplog, "zstd")
def _get_truncated_response(self, compression_id):
def _get_truncated_response(self, compression_id: str) -> Response:
crawler = get_crawler(Spider)
spider = crawler._create_spider("scrapytest.org")
mw = HttpCompressionMiddleware.from_crawler(crawler)
@ -797,7 +828,10 @@ class TestHttpCompression:
response = self._getresponse(compression_id)
truncated_body = response.body[: len(response.body) // 2]
response = response.replace(body=truncated_body)
return mw.process_response(response.request, response)
assert response.request
new_response = mw.process_response(response.request, response)
assert isinstance(new_response, Response)
return new_response
def test_process_truncated_response_br(self):
_skip_if_no_br()

6
tests/test_downloadermiddleware_httpproxy.py
View File

@ -14,7 +14,8 @@ class TestHttpProxyMiddleware:
self._oldenv = os.environ.copy()
def teardown_method(self):
os.environ = self._oldenv
os.environ.clear()
os.environ.update(self._oldenv)
def test_not_enabled(self):
crawler = get_crawler(Spider, {"HTTPPROXY_ENABLED": False})
@ -22,7 +23,8 @@ class TestHttpProxyMiddleware:
HttpProxyMiddleware.from_crawler(crawler)
def test_no_environment_proxies(self):
os.environ = {"dummy_proxy": "reset_env_and_do_not_raise"}
os.environ.clear()
os.environ["dummy_proxy"] = "reset_env_and_do_not_raise"
mw = HttpProxyMiddleware()
for url in ("http://e.com", "https://e.com", "file:///tmp/a"):

20
tests/test_downloadermiddleware_offsite.py
View File

@ -1,4 +1,5 @@
import re
from typing import Any
import pytest
@ -53,7 +54,7 @@ def test_process_request_dont_filter(value, filtered):
crawler.spider = crawler._create_spider(name="a", allowed_domains=["a.example"])
mw = OffsiteMiddleware.from_crawler(crawler)
mw.spider_opened(crawler.spider)
kwargs = {}
kwargs: dict[str, Any] = {}
if value is not UNSET:
kwargs["dont_filter"] = value
request = Request("https://b.example", **kwargs)
@ -82,7 +83,7 @@ def test_process_request_allow_offsite(allow_offsite, dont_filter, filtered):
crawler.spider = crawler._create_spider(name="a", allowed_domains=["a.example"])
mw = OffsiteMiddleware.from_crawler(crawler)
mw.spider_opened(crawler.spider)
kwargs = {"meta": {}}
kwargs: dict[str, Any] = {"meta": {}}
if allow_offsite is not UNSET:
kwargs["meta"]["allow_offsite"] = allow_offsite
if dont_filter is not UNSET:
@ -105,7 +106,7 @@ def test_process_request_allow_offsite(allow_offsite, dont_filter, filtered):
)
def test_process_request_no_allowed_domains(value):
crawler = get_crawler(Spider)
kwargs = {}
kwargs: dict[str, Any] = {}
if value is not UNSET:
kwargs["allowed_domains"] = value
crawler.spider = crawler._create_spider(name="a", **kwargs)
@ -152,7 +153,7 @@ def test_request_scheduled_domain_filtering(allowed_domain, url, allowed):
mw.spider_opened(crawler.spider)
request = Request(url)
if allowed:
assert mw.request_scheduled(request, crawler.spider) is None
mw.request_scheduled(request, crawler.spider)
else:
with pytest.raises(IgnoreRequest):
mw.request_scheduled(request, crawler.spider)
@ -172,7 +173,7 @@ def test_request_scheduled_dont_filter(value, filtered):
crawler.spider = crawler._create_spider(name="a", allowed_domains=["a.example"])
mw = OffsiteMiddleware.from_crawler(crawler)
mw.spider_opened(crawler.spider)
kwargs = {}
kwargs: dict[str, Any] = {}
if value is not UNSET:
kwargs["dont_filter"] = value
request = Request("https://b.example", **kwargs)
@ -180,7 +181,7 @@ def test_request_scheduled_dont_filter(value, filtered):
with pytest.raises(IgnoreRequest):
mw.request_scheduled(request, crawler.spider)
else:
assert mw.request_scheduled(request, crawler.spider) is None
mw.request_scheduled(request, crawler.spider)
@pytest.mark.parametrize(
@ -193,14 +194,14 @@ def test_request_scheduled_dont_filter(value, filtered):
)
def test_request_scheduled_no_allowed_domains(value):
crawler = get_crawler(Spider)
kwargs = {}
kwargs: dict[str, Any] = {}
if value is not UNSET:
kwargs["allowed_domains"] = value
crawler.spider = crawler._create_spider(name="a", **kwargs)
mw = OffsiteMiddleware.from_crawler(crawler)
mw.spider_opened(crawler.spider)
request = Request("https://example.com")
assert mw.request_scheduled(request, crawler.spider) is None
mw.request_scheduled(request, crawler.spider)
def test_request_scheduled_invalid_domains():
@ -210,7 +211,7 @@ def test_request_scheduled_invalid_domains():
mw = OffsiteMiddleware.from_crawler(crawler)
mw.spider_opened(crawler.spider)
request = Request("https://a.example")
assert mw.request_scheduled(request, crawler.spider) is None
mw.request_scheduled(request, crawler.spider)
for letter in ("b", "c"):
request = Request(f"https://{letter}.example")
with pytest.raises(IgnoreRequest):
@ -227,6 +228,7 @@ def test_repeated_offsite_domain():
with pytest.raises(IgnoreRequest):
mw.process_request(req1)
assert "other.org" in mw.domains_seen
assert crawler.stats
assert crawler.stats.get_value("offsite/domains") == 1
assert crawler.stats.get_value("offsite/filtered") == 1
with pytest.raises(IgnoreRequest):

20
tests/test_downloadermiddleware_redirect.py
View File

@ -309,7 +309,7 @@ class TestRedirectMiddleware(TestRedirectBase):
url = "http://www.example.com/301"
url2 = "http://www.example.com/redirected"
def _test_passthrough(req):
def _test_passthrough(req: Request) -> None:
rsp = Response(url, headers={"Location": url2}, status=301, request=req)
r = self.mw.process_response(req, rsp)
assert r is rsp
@ -404,15 +404,17 @@ def test_response_referrer_policy(policy, source_url, target_url, expected_refer
status=301,
headers={"Location": target_url, **extra_headers},
)
source_request = redirect_mw.process_response(source_request, response_redirect)
assert isinstance(source_request, Request)
target_request = redirect_mw.process_response(source_request, response_redirect)
assert isinstance(target_request, Request)
assert source_request.headers.get("Referer") == expected_referrer
assert target_request.headers.get("Referer") == expected_referrer
def test_no_warning_when_referer_middleware_present(caplog):
crawler = get_crawler()
crawler.get_spider_middleware = MagicMock(return_value=MagicMock())
crawler.get_spider_middleware = MagicMock( # type: ignore[method-assign]
return_value=MagicMock()
)
mw = build_from_crawler(RedirectMiddleware, crawler)
caplog.clear()
with caplog.at_level(logging.WARNING):
@ -426,7 +428,9 @@ def test_no_warning_when_referer_middleware_present(caplog):
def test_warning_redirect_middleware(caplog):
crawler = get_crawler()
crawler.get_spider_middleware = MagicMock(return_value=None)
crawler.get_spider_middleware = MagicMock( # type: ignore[method-assign]
return_value=None
)
mw = build_from_crawler(RedirectMiddleware, crawler)
with caplog.at_level(logging.WARNING):
mw._engine_started()
@ -449,7 +453,9 @@ def test_warning_subclass(caplog):
pass
crawler = get_crawler()
crawler.get_spider_middleware = MagicMock(return_value=None)
crawler.get_spider_middleware = MagicMock( # type: ignore[method-assign]
return_value=None
)
mw = build_from_crawler(MyRedirectMiddleware, crawler)
with caplog.at_level(logging.WARNING):
mw._engine_started()

16
tests/test_downloadermiddleware_redirect_metarefresh.py
View File

@ -21,7 +21,7 @@ from tests.utils.redirect import (
)
def meta_refresh_body(url, interval=5):
def meta_refresh_body(url: str, interval: int = 5) -> bytes:
html = f"""<html><head><meta http-equiv="refresh" content="{interval};url={url}"/></head></html>"""
return html.encode("utf-8")
@ -34,10 +34,14 @@ class TestMetaRefreshMiddleware(TestRedirectBase):
crawler = get_crawler(Spider)
self.mw = self.mwcls.from_crawler(crawler)
def _body(self, interval=5, url="http://example.org/newpage"):
def _body(
self, interval: int = 5, url: str = "http://example.org/newpage"
) -> bytes:
return meta_refresh_body(url, interval)
def get_response(self, request, location):
def get_response(
self, request: Request, location: str, status: int = 302
) -> Response:
return HtmlResponse(request.url, body=self._body(url=location))
def test_meta_refresh(self):
@ -75,7 +79,7 @@ class TestMetaRefreshMiddleware(TestRedirectBase):
assert "Content-Length" not in req2.headers, (
"Content-Length header must not be present in redirected request"
)
assert not req2.body, f"Redirected body must be empty, not '{req2.body}'"
assert not req2.body, f"Redirected body must be empty, not {req2.body!r}"
def test_ignore_tags_default(self):
req = Request(url="http://example.org")
@ -142,7 +146,9 @@ def test_meta_refresh_schemes(url, location, target):
def test_warning_meta_refresh_middleware(caplog):
crawler = get_crawler()
crawler.get_spider_middleware = MagicMock(return_value=None)
crawler.get_spider_middleware = MagicMock( # type: ignore[method-assign]
return_value=None
)
mw = build_from_crawler(MetaRefreshMiddleware, crawler)
with caplog.at_level(logging.WARNING):
mw._engine_started()

69
tests/test_downloadermiddleware_retry.py
View File

@ -31,6 +31,7 @@ class TestRetry:
req = Request("http://www.scrapytest.org/503")
rsp = Response("http://www.scrapytest.org/503", body=b"", status=503)
req2 = self.mw.process_response(req, rsp)
assert isinstance(req2, Request)
assert req2.priority < req.priority
def test_404(self):
@ -53,9 +54,9 @@ class TestRetry:
rsp = Response("http://www.scrapytest.org/503", body=b"", status=503)
# first retry
req = self.mw.process_response(req, rsp)
assert isinstance(req, Request)
assert req.meta["retry_times"] == 1
req2 = self.mw.process_response(req, rsp)
assert isinstance(req2, Request)
assert req2.meta["retry_times"] == 1
def test_dont_retry_exc(self):
req = Request("http://www.scrapytest.org/503", meta={"dont_retry": True})
@ -68,18 +69,19 @@ class TestRetry:
rsp = Response("http://www.scrapytest.org/503", body=b"", status=503)
# first retry
req = self.mw.process_response(req, rsp)
assert isinstance(req, Request)
assert req.meta["retry_times"] == 1
req2 = self.mw.process_response(req, rsp)
assert isinstance(req2, Request)
assert req2.meta["retry_times"] == 1
# second retry
req = self.mw.process_response(req, rsp)
assert isinstance(req, Request)
assert req.meta["retry_times"] == 2
req3 = self.mw.process_response(req2, rsp)
assert isinstance(req3, Request)
assert req3.meta["retry_times"] == 2
# discard it
assert self.mw.process_response(req, rsp) is rsp
assert self.mw.process_response(req3, rsp) is rsp
assert self.crawler.stats
assert self.crawler.stats.get_value("retry/max_reached") == 1
assert (
self.crawler.stats.get_value("retry/reason_count/503 Service Unavailable")
@ -131,6 +133,7 @@ class TestRetry:
self._test_retry_exception(req, exc("foo"))
stats = self.crawler.stats
assert stats
assert stats.get_value("retry/max_reached") == len(exceptions)
assert stats.get_value("retry/count") == len(exceptions) * 2
assert (
@ -149,29 +152,30 @@ class TestRetry:
req = Request(f"http://www.scrapytest.org/{exc.__name__}")
self._test_retry_exception(req, exc("foo"), mw)
def _test_retry_exception(self, req, exception, mw=None):
def _test_retry_exception(
self, req: Request, exception: Exception, mw: RetryMiddleware | None = None
) -> None:
if mw is None:
mw = self.mw
# first retry
req = mw.process_exception(req, exception)
assert isinstance(req, Request)
assert req.meta["retry_times"] == 1
req2 = mw.process_exception(req, exception)
assert isinstance(req2, Request)
assert req2.meta["retry_times"] == 1
# second retry
req = mw.process_exception(req, exception)
assert isinstance(req, Request)
assert req.meta["retry_times"] == 2
req3 = mw.process_exception(req2, exception)
assert isinstance(req3, Request)
assert req3.meta["retry_times"] == 2
# discard it
req = mw.process_exception(req, exception)
assert req is None
assert mw.process_exception(req3, exception) is None
class TestMaxRetryTimes:
invalid_url = "http://www.scrapytest.org/invalid_url"
def get_middleware(self, settings=None):
def get_middleware(self, settings: dict[str, Any] | None = None) -> RetryMiddleware:
crawler = get_crawler(DefaultSpider, settings or {})
crawler.spider = crawler._create_spider()
return RetryMiddleware.from_crawler(crawler)
@ -275,20 +279,18 @@ class TestMaxRetryTimes:
def _test_retry(
self,
req,
exception,
max_retry_times,
middleware=None,
):
middleware = middleware or self.mw
req: Request,
exception: Exception,
max_retry_times: int,
middleware: RetryMiddleware,
) -> None:
for _ in range(max_retry_times):
req = middleware.process_exception(req, exception)
assert isinstance(req, Request)
result = middleware.process_exception(req, exception)
assert isinstance(result, Request)
req = result
# discard it
req = middleware.process_exception(req, exception)
assert req is None
assert middleware.process_exception(req, exception) is None
class TestGetRetryRequest:
@ -428,7 +430,7 @@ class TestGetRetryRequest:
def test_no_spider(self):
request = Request("https://example.com")
with pytest.raises(TypeError):
get_retry_request(request) # pylint: disable=missing-kwoa
get_retry_request(request) # type: ignore[call-arg] # pylint: disable=missing-kwoa
def test_max_retry_times_setting(self):
max_retry_times = 0
@ -471,6 +473,7 @@ class TestGetRetryRequest:
request,
spider=spider,
)
assert new_request
assert new_request.priority == priority_adjust
def test_priority_adjust_argument(self):
@ -482,6 +485,7 @@ class TestGetRetryRequest:
spider=spider,
priority_adjust=priority_adjust,
)
assert new_request
assert new_request.priority == priority_adjust
def test_log_extra_retry_success(self, caplog: pytest.LogCaptureFixture) -> None:
@ -732,6 +736,7 @@ class TestGetRetryRequest:
reason=expected_reason,
stats_base_key=stats_key,
)
assert spider.crawler.stats
for stat in (
f"{stats_key}/count",
f"{stats_key}/reason_count/{expected_reason}",

32
tests/test_downloadermiddleware_robotstxt.py
View File

@ -1,7 +1,6 @@
from __future__ import annotations
import asyncio
from typing import TYPE_CHECKING
from unittest import mock
import pytest
@ -19,9 +18,6 @@ from scrapy.utils.defer import deferred_from_coro, maybe_deferred_to_future
from tests.utils.decorators import coroutine_test
from tests.utils.robotstxt import rerp_available
if TYPE_CHECKING:
from scrapy.crawler import Crawler
class TestRobotsTxtMiddleware:
def setup_method(self) -> None:
@ -39,7 +35,7 @@ class TestRobotsTxtMiddleware:
with pytest.raises(NotConfigured):
RobotsTxtMiddleware(self.crawler)
def _get_successful_crawler(self) -> Crawler:
def _get_successful_crawler(self) -> mock.MagicMock:
crawler = self.crawler
crawler.settings.set("ROBOTSTXT_OBEY", True)
ROBOTS = """
@ -54,8 +50,8 @@ Disallow: /some/randome/page.html
""".encode()
response = TextResponse("http://site.local/robots.txt", body=ROBOTS)
async def return_response(request):
deferred = Deferred()
async def return_response(request: Request) -> Response:
deferred: Deferred[Response] = Deferred()
call_later(0, deferred.callback, response)
return await maybe_deferred_to_future(deferred)
@ -130,15 +126,15 @@ Disallow: /some/randome/page.html
Request("http://site.local/static/", meta=meta), middleware
)
def _get_garbage_crawler(self) -> Crawler:
def _get_garbage_crawler(self) -> mock.MagicMock:
crawler = self.crawler
crawler.settings.set("ROBOTSTXT_OBEY", True)
response = Response(
"http://site.local/robots.txt", body=b"GIF89a\xd3\x00\xfe\x00\xa2"
)
async def return_response(request):
deferred = Deferred()
async def return_response(request: Request) -> Response:
deferred: Deferred[Response] = Deferred()
call_later(0, deferred.callback, response)
return await maybe_deferred_to_future(deferred)
@ -154,13 +150,13 @@ Disallow: /some/randome/page.html
await self.assertNotIgnored(Request("http://site.local/admin/main"), middleware)
await self.assertNotIgnored(Request("http://site.local/static/"), middleware)
def _get_emptybody_crawler(self) -> Crawler:
def _get_emptybody_crawler(self) -> mock.MagicMock:
crawler = self.crawler
crawler.settings.set("ROBOTSTXT_OBEY", True)
response = Response("http://site.local/robots.txt")
async def return_response(request):
deferred = Deferred()
async def return_response(request: Request) -> Response:
deferred: Deferred[Response] = Deferred()
call_later(0, deferred.callback, response)
return await maybe_deferred_to_future(deferred)
@ -180,8 +176,8 @@ Disallow: /some/randome/page.html
self.crawler.settings.set("ROBOTSTXT_OBEY", True)
err = CannotResolveHostError("Robotstxt address not found")
async def return_failure(request):
deferred = Deferred()
async def return_failure(request: Request) -> Response:
deferred: Deferred[Response] = Deferred()
call_later(0, deferred.errback, failure.Failure(err))
return await maybe_deferred_to_future(deferred)
@ -208,8 +204,8 @@ Disallow: /some/randome/page.html
async def test_ignore_robotstxt_request(self):
self.crawler.settings.set("ROBOTSTXT_OBEY", True)
async def ignore_request(request):
deferred = Deferred()
async def ignore_request(request: Request) -> Response:
deferred: Deferred[Response] = Deferred()
call_later(0, deferred.errback, failure.Failure(IgnoreRequest()))
return await maybe_deferred_to_future(deferred)
@ -236,7 +232,7 @@ Disallow: /some/randome/page.html
@coroutine_test
async def test_robotstxt_local_file(self):
middleware = RobotsTxtMiddleware(self._get_emptybody_crawler())
middleware.process_request_2 = mock.MagicMock()
middleware.process_request_2 = mock.MagicMock() # type: ignore[method-assign]
await middleware.process_request(Request("data:text/plain,Hello World data"))
assert not middleware.process_request_2.called

72
tests/test_robotstxt_interface.py
View File

@ -1,3 +1,7 @@
from __future__ import annotations
from typing import TYPE_CHECKING
import pytest
from scrapy.robotstxt import (
@ -10,22 +14,32 @@ from scrapy.robotstxt import (
from scrapy.utils._deps_compat import STDLIB_IMPROVED_ROBOTFILEPARSER
from tests.utils.robotstxt import rerp_available
if TYPE_CHECKING:
# typing.Self requires Python 3.11
from typing_extensions import Self
from scrapy.crawler import Crawler
class BaseRobotParserTest:
def _setUp(self, parser_cls):
parser_cls: type[RobotParser]
def _setUp(self, parser_cls: type[RobotParser]) -> None:
self.parser_cls = parser_cls
def _parse(self, robotstxt_body: bytes) -> RobotParser:
# The parser backends only use the crawler to get the spider to log with.
return self.parser_cls.from_crawler(None, robotstxt_body) # type: ignore[arg-type]
def test_allowed(self):
robotstxt_robotstxt_body = (
b"User-agent: * \nDisallow: /disallowed \nAllow: /allowed \nCrawl-delay: 10"
)
rp = self.parser_cls.from_crawler(
crawler=None, robotstxt_body=robotstxt_robotstxt_body
)
rp = self._parse(robotstxt_robotstxt_body)
assert rp.allowed("https://www.site.local/allowed", "*")
assert not rp.allowed("https://www.site.local/disallowed", "*")
def test_allowed_wildcards(self):
def test_allowed_wildcards(self) -> None:
robotstxt_robotstxt_body = b"""User-agent: first
Disallow: /disallowed/*/end$
@ -33,9 +47,7 @@ class BaseRobotParserTest:
Allow: /*allowed
Disallow: /
"""
rp = self.parser_cls.from_crawler(
crawler=None, robotstxt_body=robotstxt_robotstxt_body
)
rp = self._parse(robotstxt_robotstxt_body)
assert rp.allowed("https://www.site.local/disallowed", "first")
assert not rp.allowed("https://www.site.local/disallowed/xyz/end", "first")
@ -46,23 +58,19 @@ class BaseRobotParserTest:
assert rp.allowed("https://www.site.local/is_still_allowed", "second")
assert rp.allowed("https://www.site.local/is_allowed_too", "second")
def test_length_based_precedence(self):
def test_length_based_precedence(self) -> None:
robotstxt_robotstxt_body = b"User-agent: * \nDisallow: / \nAllow: /page"
rp = self.parser_cls.from_crawler(
crawler=None, robotstxt_body=robotstxt_robotstxt_body
)
rp = self._parse(robotstxt_robotstxt_body)
assert rp.allowed("https://www.site.local/page", "*")
def test_order_based_precedence(self):
def test_order_based_precedence(self) -> None:
robotstxt_robotstxt_body = b"User-agent: * \nDisallow: / \nAllow: /page"
rp = self.parser_cls.from_crawler(
crawler=None, robotstxt_body=robotstxt_robotstxt_body
)
rp = self._parse(robotstxt_robotstxt_body)
assert not rp.allowed("https://www.site.local/page", "*")
def test_empty_response(self):
"""empty response should equal 'allow all'"""
rp = self.parser_cls.from_crawler(crawler=None, robotstxt_body=b"")
rp = self._parse(b"")
assert rp.allowed("https://site.local/", "*")
assert rp.allowed("https://site.local/", "chrome")
assert rp.allowed("https://site.local/index.html", "*")
@ -71,9 +79,7 @@ class BaseRobotParserTest:
def test_garbage_response(self):
"""garbage response should be discarded, equal 'allow all'"""
robotstxt_robotstxt_body = b"GIF89a\xd3\x00\xfe\x00\xa2"
rp = self.parser_cls.from_crawler(
crawler=None, robotstxt_body=robotstxt_robotstxt_body
)
rp = self._parse(robotstxt_robotstxt_body)
assert rp.allowed("https://site.local/", "*")
assert rp.allowed("https://site.local/", "chrome")
assert rp.allowed("https://site.local/index.html", "*")
@ -81,12 +87,12 @@ class BaseRobotParserTest:
def test_crawl_delay(self):
robotstxt_body = b"User-agent: *\nDisallow: /private\nCrawl-delay: 10\n"
rp = self.parser_cls.from_crawler(crawler=None, robotstxt_body=robotstxt_body)
rp = self._parse(robotstxt_body)
assert rp.crawl_delay("*") == 10.0
def test_crawl_delay_unset(self):
robotstxt_body = b"User-agent: *\nDisallow: /private\n"
rp = self.parser_cls.from_crawler(crawler=None, robotstxt_body=robotstxt_body)
rp = self._parse(robotstxt_body)
assert rp.crawl_delay("*") is None
def test_unicode_url_and_useragent(self):
@ -100,9 +106,7 @@ class BaseRobotParserTest:
User-Agent: UnicödeBöt
Disallow: /some/randome/page.html""".encode()
rp = self.parser_cls.from_crawler(
crawler=None, robotstxt_body=robotstxt_robotstxt_body
)
rp = self._parse(robotstxt_robotstxt_body)
assert rp.allowed("https://site.local/", "*")
assert not rp.allowed("https://site.local/admin/", "*")
assert not rp.allowed("https://site.local/static/", "*")
@ -117,15 +121,13 @@ class TestRobotParser:
def test_crawl_delay_unsupported(self):
class AllowAllRobotParser(RobotParser):
@classmethod
def from_crawler(cls, crawler, robotstxt_body):
def from_crawler(cls, crawler: Crawler, robotstxt_body: bytes) -> Self:
return cls()
def allowed(self, url, user_agent):
def allowed(self, url: str | bytes, user_agent: str | bytes) -> bool:
return True
rp = AllowAllRobotParser.from_crawler(
crawler=None, robotstxt_body=b"User-agent: *\nCrawl-delay: 10\n"
)
rp = AllowAllRobotParser()
assert rp.crawl_delay("*") is None
@ -162,21 +164,21 @@ class TestPythonRobotParser(BaseRobotParserTest):
not STDLIB_IMPROVED_ROBOTFILEPARSER,
reason="RobotFileParser from this Python version does not support length based directives precedence.",
)
def test_length_based_precedence(self):
def test_length_based_precedence(self) -> None:
super().test_length_based_precedence()
@pytest.mark.skipif(
STDLIB_IMPROVED_ROBOTFILEPARSER,
reason="RobotFileParser from this Python version does not support order based directives precedence.",
)
def test_order_based_precedence(self):
def test_order_based_precedence(self) -> None:
super().test_order_based_precedence()
@pytest.mark.skipif(
not STDLIB_IMPROVED_ROBOTFILEPARSER,
reason="RobotFileParser from this Python version does not support wildcards.",
)
def test_allowed_wildcards(self):
def test_allowed_wildcards(self) -> None:
super().test_allowed_wildcards()
@ -185,7 +187,7 @@ class TestRerpRobotParser(BaseRobotParserTest):
def setup_method(self):
super()._setUp(RerpRobotParser)
def test_length_based_precedence(self):
def test_length_based_precedence(self) -> None:
pytest.skip("Rerp does not support length based directives precedence.")
@ -193,5 +195,5 @@ class TestProtegoRobotParser(BaseRobotParserTest):
def setup_method(self):
super()._setUp(ProtegoRobotParser)
def test_order_based_precedence(self):
def test_order_based_precedence(self) -> None:
pytest.skip("Protego does not support order based directives precedence.")