RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/master' into throttling

This commit is contained in:
Adrian Chaves 2026-06-25 13:39:28 +02:00
parent 7fdee93e42 c690eac770
commit 4af625ac1b
18 changed files with 310 additions and 52 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
docs/topics/practices.rst
View File

@ -347,10 +347,10 @@ finishes before starting the next one:
install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor")
react(deferred_f_from_coro_f(crawl))
.. note:: When running multiple spiders in the same process, :ref:`reactor
settings <reactor-settings>` should not have a different value per spider.
Also, :ref:`pre-crawler settings <pre-crawler-settings>` cannot be defined
per spider.
.. note:: When running multiple spiders in the same process, :ref:`logging
settings <logging-settings>` and :ref:`reactor settings <reactor-settings>`
should not have a different value per spider, and :ref:`pre-crawler
settings <pre-crawler-settings>` cannot be defined per spider.
.. seealso:: :ref:`run-from-script`.

64
docs/topics/settings.rst
View File

@ -361,6 +361,31 @@ All of these settings, except for :setting:`ASYNCIO_EVENT_LOOP`, are only used
when the Twisted reactor is used, i.e. when :setting:`TWISTED_REACTOR_ENABLED`
is ``True``.
.. _logging-settings:
Logging settings
----------------
**Logging settings** are settings that configure the global root logging
handler installed by :func:`~scrapy.utils.log.configure_logging`.
These settings can be defined from a spider. However, because only 1 root
logging handler is active per process, these settings cannot use a different
value per spider when :ref:`running multiple spiders in the same process
<run-multiple-spiders>`.
These settings are:
- :setting:`LOG_DATEFORMAT`
- :setting:`LOG_ENABLED`
- :setting:`LOG_ENCODING`
- :setting:`LOG_FILE`
- :setting:`LOG_FILE_APPEND`
- :setting:`LOG_FORMAT`
- :setting:`LOG_LEVEL`
- :setting:`LOG_SHORT_NAMES`
- :setting:`LOG_STDOUT`
.. _topics-settings-ref:
Built-in settings reference
@ -481,6 +506,8 @@ Note that the event loop class must inherit from :class:`asyncio.AbstractEventLo
:func:`asyncio.set_event_loop`, which will set the specified event loop
as the current loop for the current OS thread.
.. note:: This is a :ref:`reactor setting <reactor-settings>`.
.. setting:: BOT_NAME
BOT_NAME
@ -650,6 +677,8 @@ Whether to enable DNS in-memory cache.
:setting:`TWISTED_REACTOR_ENABLED` is ``False``, and may have no effect
either when :setting:`DNS_RESOLVER` is set to a different resolver.
.. note:: This is a :ref:`reactor setting <reactor-settings>`.
.. setting:: DNSCACHE_SIZE
DNSCACHE_SIZE
@ -659,6 +688,8 @@ Default: ``10000``
DNS in-memory cache size, see :setting:`DNSCACHE_ENABLED`.
.. note:: This is a :ref:`reactor setting <reactor-settings>`.
.. setting:: TWISTED_DNS_RESOLVER
TWISTED_DNS_RESOLVER
@ -676,6 +707,8 @@ take the :setting:`DNS_TIMEOUT` setting into account.
.. note::
This setting has no effect when :setting:`TWISTED_REACTOR_ENABLED` is ``False``.
.. note:: This is a :ref:`reactor setting <reactor-settings>`.
.. setting:: DNS_TIMEOUT
DNS_TIMEOUT
@ -691,6 +724,8 @@ Timeout for processing of DNS queries in seconds. Float is supported.
:setting:`TWISTED_REACTOR_ENABLED` is ``False``, and may have no effect
either when :setting:`DNS_RESOLVER` is set to a different resolver.
.. note:: This is a :ref:`reactor setting <reactor-settings>`.
.. setting:: DOWNLOADER
DOWNLOADER
@ -1326,6 +1361,8 @@ Default: ``True``
Whether to enable logging.
.. note:: This is a :ref:`logging setting <logging-settings>`.
.. setting:: LOG_ENCODING
LOG_ENCODING
@ -1335,6 +1372,8 @@ Default: ``'utf-8'``
The encoding to use for logging.
.. note:: This is a :ref:`logging setting <logging-settings>`.
.. setting:: LOG_FILE
LOG_FILE
@ -1344,6 +1383,8 @@ Default: ``None``
File name to use for logging output. If ``None``, standard error will be used.
.. note:: This is a :ref:`logging setting <logging-settings>`.
.. setting:: LOG_FILE_APPEND
LOG_FILE_APPEND
@ -1354,6 +1395,8 @@ Default: ``True``
If ``False``, the log file specified with :setting:`LOG_FILE` will be
overwritten (discarding the output from previous runs, if any).
.. note:: This is a :ref:`logging setting <logging-settings>`.
.. setting:: LOG_FORMAT
LOG_FORMAT
@ -1365,6 +1408,8 @@ String for formatting log messages. Refer to the
:ref:`Python logging documentation <logrecord-attributes>` for the whole
list of available placeholders.
.. note:: This is a :ref:`logging setting <logging-settings>`.
.. setting:: LOG_DATEFORMAT
LOG_DATEFORMAT
@ -1377,6 +1422,8 @@ in :setting:`LOG_FORMAT`. Refer to the
:ref:`Python datetime documentation <strftime-strptime-behavior>` for the
whole list of available directives.
.. note:: This is a :ref:`logging setting <logging-settings>`.
.. setting:: LOG_FORMATTER
LOG_FORMATTER
@ -1396,6 +1443,8 @@ Default: ``'DEBUG'``
Minimum level to log. Available levels are: CRITICAL, ERROR, WARNING,
INFO, DEBUG. For more info see :ref:`topics-logging`.
.. note:: This is a :ref:`logging setting <logging-settings>`.
.. setting:: LOG_STDOUT
LOG_STDOUT
@ -1407,6 +1456,8 @@ If ``True``, all standard output (and error) of your process will be redirected
to the log. For example if you ``print('hello')`` it will appear in the Scrapy
log.
.. note:: This is a :ref:`logging setting <logging-settings>`.
.. setting:: LOG_SHORT_NAMES
LOG_SHORT_NAMES
@ -1417,6 +1468,8 @@ Default: ``False``
If ``True``, the logs will just contain the root path. If it is set to ``False``
then it displays the component responsible for the log output
.. note:: This is a :ref:`logging setting <logging-settings>`.
.. setting:: LOG_VERSIONS
LOG_VERSIONS
@ -1559,6 +1612,8 @@ multi-purpose thread pool used by various Scrapy components. Threaded
DNS Resolver, BlockingFeedStorage, S3FilesStore just to name a few. Increase
this value if you're experiencing problems with insufficient blocking IO.
.. note:: This is a :ref:`reactor setting <reactor-settings>`.
.. setting:: REDIRECT_PRIORITY_ADJUST
REDIRECT_PRIORITY_ADJUST
@ -1785,6 +1840,8 @@ Default: ``'scrapy.spiderloader.SpiderLoader'``
The class that will be used for loading spiders, which must implement the
:ref:`topics-api-spiderloader`.
.. note:: This is a :ref:`pre-crawler setting <pre-crawler-settings>`.
.. setting:: SPIDER_LOADER_WARN_ONLY
SPIDER_LOADER_WARN_ONLY
@ -1797,6 +1854,8 @@ it will fail loudly if there is any ``ImportError`` or ``SyntaxError`` exception
But you can choose to silence this exception and turn it into a simple
warning by setting ``SPIDER_LOADER_WARN_ONLY = True``.
.. note:: This is a :ref:`pre-crawler setting <pre-crawler-settings>`.
.. setting:: SPIDER_MIDDLEWARES
SPIDER_MIDDLEWARES
@ -1842,6 +1901,8 @@ Example:
SPIDER_MODULES = ["mybot.spiders_prod", "mybot.spiders_dev"]
.. note:: This is a :ref:`pre-crawler setting <pre-crawler-settings>`.
.. setting:: STATS_CLASS
STATS_CLASS
@ -1913,7 +1974,7 @@ stopped) will not apply. This mode is currently experimental and may not be
suitable for production use. It may also not be supported by 3rd-party code.
See :ref:`asyncio-without-reactor` for more information about this mode.
.. note:: This setting can't be set :ref:`per-spider <spider-settings>`.
.. note:: This is a :ref:`pre-crawler setting <pre-crawler-settings>`.
.. versionadded:: 2.15.0
@ -2021,6 +2082,7 @@ current platform.
For additional information, see :doc:`core/howto/choosing-reactor`.
.. note:: This is a :ref:`reactor setting <reactor-settings>`.
.. setting:: URLLENGTH_LIMIT

90
scrapy/downloadermiddlewares/httpcompression.py
View File

@ -6,7 +6,7 @@ from logging import getLogger
from typing import TYPE_CHECKING, Any
from scrapy import Request, Spider, signals
from scrapy.exceptions import IgnoreRequest, NotConfigured
from scrapy.exceptions import IgnoreRequest, NotConfigured, ScrapyDeprecationWarning
from scrapy.http import Response, TextResponse
from scrapy.responsetypes import responsetypes
from scrapy.utils._compression import (
@ -70,6 +70,12 @@ class HttpCompressionMiddleware:
crawler: Crawler | None = None,
):
if not crawler:
warnings.warn(
"Instantiating HttpCompressionMiddleware without a 'crawler' "
"argument is deprecated.",
category=ScrapyDeprecationWarning,
stacklevel=2,
)
self.stats = stats
self._max_size = 1073741824
self._warn_size = 33554432
@ -108,49 +114,47 @@ class HttpCompressionMiddleware:
) -> Request | Response:
if request.method == "HEAD":
return response
if isinstance(response, Response):
content_encoding = response.headers.getlist("Content-Encoding")
if content_encoding:
max_size = request.meta.get("download_maxsize", self._max_size)
warn_size = request.meta.get("download_warnsize", self._warn_size)
try:
decoded_body, content_encoding = self._handle_encoding(
response.body, content_encoding, max_size
)
except _DecompressionMaxSizeExceeded as e:
raise IgnoreRequest(
f"Ignored response {response} because its body "
f"({len(response.body)} B compressed, "
f"{e.decompressed_size} B decompressed so far) exceeded "
f"DOWNLOAD_MAXSIZE ({max_size} B) during decompression."
) from e
if len(response.body) < warn_size <= len(decoded_body):
logger.warning(
f"{response} body size after decompression "
f"({len(decoded_body)} B) is larger than the "
f"download warning size ({warn_size} B)."
)
if content_encoding:
self._warn_unknown_encoding(response, content_encoding)
response.headers["Content-Encoding"] = content_encoding
if self.stats:
self.stats.inc_value(
"httpcompression/response_bytes",
len(decoded_body),
)
self.stats.inc_value("httpcompression/response_count")
respcls = responsetypes.from_args(
headers=response.headers, url=response.url, body=decoded_body
content_encoding = response.headers.getlist("Content-Encoding")
if content_encoding:
max_size = request.meta.get("download_maxsize", self._max_size)
warn_size = request.meta.get("download_warnsize", self._warn_size)
try:
decoded_body, content_encoding = self._handle_encoding(
response.body, content_encoding, max_size
)
kwargs: dict[str, Any] = {"body": decoded_body}
if issubclass(respcls, TextResponse):
# force recalculating the encoding until we make sure the
# responsetypes guessing is reliable
kwargs["encoding"] = None
response = response.replace(cls=respcls, **kwargs)
if not content_encoding:
del response.headers["Content-Encoding"]
except _DecompressionMaxSizeExceeded as e:
raise IgnoreRequest(
f"Ignored response {response} because its body "
f"({len(response.body)} B compressed, "
f"{e.decompressed_size} B decompressed so far) exceeded "
f"DOWNLOAD_MAXSIZE ({max_size} B) during decompression."
) from e
if len(response.body) < warn_size <= len(decoded_body):
logger.warning(
f"{response} body size after decompression "
f"({len(decoded_body)} B) is larger than the "
f"download warning size ({warn_size} B)."
)
if content_encoding:
self._warn_unknown_encoding(response, content_encoding)
response.headers["Content-Encoding"] = content_encoding
if self.stats:
self.stats.inc_value(
"httpcompression/response_bytes",
len(decoded_body),
)
self.stats.inc_value("httpcompression/response_count")
respcls = responsetypes.from_args(
headers=response.headers, url=response.url, body=decoded_body
)
kwargs: dict[str, Any] = {"body": decoded_body}
if issubclass(respcls, TextResponse):
# force recalculating the encoding until we make sure the
# responsetypes guessing is reliable
kwargs["encoding"] = None
response = response.replace(cls=respcls, **kwargs)
if not content_encoding:
del response.headers["Content-Encoding"]
return response
def _handle_encoding(

6
scrapy/spidermiddlewares/referer.py
View File

@ -308,6 +308,12 @@ class RefererMiddleware(BaseSpiderMiddleware):
# Reference: https://www.w3.org/TR/referrer-policy/#referrer-policy-empty-string
self.policies[""] = NoReferrerWhenDowngradePolicy
if settings is None:
warn(
"Instantiating RefererMiddleware without a 'settings' argument is "
"deprecated.",
ScrapyDeprecationWarning,
stacklevel=2,
)
return
setting_policies = settings.getdict("REFERRER_POLICIES")
for policy_name, policy_class_import_path in setting_policies.items():

14
tests/test_downloadermiddleware_cookies.py
View File

@ -131,6 +131,20 @@ class TestCookiesMiddleware:
),
)
def test_debug_no_cookies(self):
crawler = get_crawler(settings_dict={"COOKIES_DEBUG": True})
mw = CookiesMiddleware.from_crawler(crawler)
with LogCapture(
"scrapy.downloadermiddlewares.cookies",
propagate=False,
level=logging.DEBUG,
) as log:
req = Request("http://scrapytest.org/")
res = Response("http://scrapytest.org/") # no Set-Cookie header
mw.process_response(req, res)
mw.process_request(req) # no cookies to send either
log.check() # no log output since cl is empty in both cases
def test_setting_disabled_cookies_debug(self):
crawler = get_crawler(settings_dict={"COOKIES_DEBUG": False})
mw = CookiesMiddleware.from_crawler(crawler)

6
tests/test_downloadermiddleware_downloadtimeout.py
View File

@ -35,3 +35,9 @@ class TestDownloadTimeoutMiddleware:
req.meta["download_timeout"] = 1
assert mw.process_request(req) is None
assert req.meta.get("download_timeout") == 1
def test_zero_download_timeout(self):
req, spider, mw = self.get_request_spider_mw({"DOWNLOAD_TIMEOUT": 0})
mw.spider_opened(spider)
assert mw.process_request(req) is None
assert req.meta.get("download_timeout") is None

1
tests/test_downloadermiddleware_httpcache.py
View File

@ -135,6 +135,7 @@ class PolicyTestMixin:
def test_dont_cache(self):
with self._middleware() as mw:
self.request.meta["dont_cache"] = True
assert mw.process_request(self.request) is None
mw.process_response(self.request, self.response)
assert mw.storage.retrieve_response(mw.crawler.spider, self.request) is None

18
tests/test_downloadermiddleware_httpcompression.py
View File

@ -11,7 +11,7 @@ from scrapy.downloadermiddlewares.httpcompression import (
ACCEPTED_ENCODINGS,
HttpCompressionMiddleware,
)
from scrapy.exceptions import IgnoreRequest, NotConfigured
from scrapy.exceptions import IgnoreRequest, NotConfigured, ScrapyDeprecationWarning
from scrapy.http import HtmlResponse, Request, Response
from scrapy.responsetypes import responsetypes
from scrapy.spiders import Spider
@ -124,6 +124,22 @@ class TestHttpCompression:
HttpCompressionMiddleware,
)
def test_no_crawler_constructor(self):
with pytest.warns(ScrapyDeprecationWarning, match="HttpCompressionMiddleware"):
mw = HttpCompressionMiddleware()
buf = BytesIO()
with GzipFile(fileobj=buf, mode="wb") as f:
f.write(b"hello")
body = buf.getvalue()
request = Request("http://scrapytest.org")
response = Response(
"http://scrapytest.org",
body=body,
headers={"Content-Encoding": "gzip"},
)
newresponse = mw.process_response(request, response)
assert newresponse.body == b"hello"
def test_process_request(self):
request = Request("http://scrapytest.org")
assert "Accept-Encoding" not in request.headers

6
tests/test_downloadermiddleware_httpproxy.py
View File

@ -373,6 +373,12 @@ class TestHttpProxyMiddleware:
assert "proxy" not in request.meta
assert b"Proxy-Authorization" not in request.headers
def test_proxy_unparseable_url_clears_meta(self):
middleware = HttpProxyMiddleware()
request = Request("http://example.com", meta={"proxy": "//"})
assert middleware.process_request(request) is None
assert request.meta["proxy"] is None
def test_proxy_authentication_header_disabled_proxy(self):
middleware = HttpProxyMiddleware()
request = Request(

18
tests/test_downloadermiddleware_offsite.py
View File

@ -213,3 +213,21 @@ def test_request_scheduled_invalid_domains():
request = Request(f"https://{letter}.example")
with pytest.raises(IgnoreRequest):
mw.request_scheduled(request, crawler.spider)
def test_repeated_offsite_domain():
crawler = get_crawler(Spider)
crawler.spider = crawler._create_spider(name="a", allowed_domains=["example.com"])
mw = OffsiteMiddleware.from_crawler(crawler)
mw.spider_opened(crawler.spider)
req1 = Request("http://other.org/1")
req2 = Request("http://other.org/2")
with pytest.raises(IgnoreRequest):
mw.process_request(req1)
assert "other.org" in mw.domains_seen
assert crawler.stats.get_value("offsite/domains") == 1
assert crawler.stats.get_value("offsite/filtered") == 1
with pytest.raises(IgnoreRequest):
mw.process_request(req2)
assert crawler.stats.get_value("offsite/domains") == 1 # not incremented again
assert crawler.stats.get_value("offsite/filtered") == 2

17
tests/test_downloadermiddleware_redirect.py
View File

@ -4,6 +4,7 @@ from unittest.mock import MagicMock
import pytest
from scrapy.downloadermiddlewares.redirect import RedirectMiddleware
from scrapy.exceptions import NotConfigured
from scrapy.http import Request, Response
from scrapy.spidermiddlewares.referer import (
POLICY_NO_REFERRER,
@ -265,6 +266,16 @@ class TestRedirectMiddleware(Base.Test):
assert isinstance(req2, Request)
assert req2.url == "http://www.example.com/redirected#frag"
def test_redirect_target_has_fragment(self):
url = "http://www.example.com/302#original"
url2 = "http://www.example.com/redirected#target"
req = Request(url)
rsp = Response(url, headers={"Location": url2}, status=302)
req2 = self.mw.process_response(req, rsp)
assert isinstance(req2, Request)
assert req2.url == "http://www.example.com/redirected#target"
def test_redirect_302_head(self):
url = "http://www.example.com/302"
url2 = "http://www.example.com/redirected2"
@ -458,3 +469,9 @@ def test_warning_subclass(caplog):
assert (
"(if defined in your code base) to override the handle_referer() method"
) in caplog.text
def test_not_configured():
crawler = get_crawler(DefaultSpider, {"REDIRECT_ENABLED": False})
with pytest.raises(NotConfigured):
RedirectMiddleware.from_crawler(crawler)

7
tests/test_downloadermiddleware_redirect_metarefresh.py
View File

@ -7,6 +7,7 @@ from unittest.mock import MagicMock
import pytest
from scrapy.downloadermiddlewares.redirect import MetaRefreshMiddleware
from scrapy.exceptions import NotConfigured
from scrapy.http import HtmlResponse, Request, Response
from scrapy.spiders import Spider
from scrapy.utils.misc import build_from_crawler
@ -157,3 +158,9 @@ def test_warning_meta_refresh_middleware(caplog):
"replace scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware "
"with a subclass that overrides the handle_referer() method"
) in caplog.text
def test_not_configured():
crawler = get_crawler(Spider, {"METAREFRESH_ENABLED": False})
with pytest.raises(NotConfigured):
MetaRefreshMiddleware.from_crawler(crawler)

14
tests/test_downloadermiddleware_stats.py
View File

@ -1,4 +1,7 @@
from scrapy.downloadermiddlewares.stats import DownloaderStats
import pytest
from scrapy.downloadermiddlewares.stats import DownloaderStats, get_header_size
from scrapy.exceptions import NotConfigured
from scrapy.http import Request, Response
from scrapy.spiders import Spider
from scrapy.utils.test import get_crawler
@ -39,5 +42,14 @@ class TestDownloaderStats:
1,
)
def test_from_crawler_not_configured(self):
crawler = get_crawler(Spider, {"DOWNLOADER_STATS": False})
with pytest.raises(NotConfigured):
DownloaderStats.from_crawler(crawler)
def teardown_method(self):
self.crawler.stats.close_spider()
def test_get_header_size_non_list_value():
assert get_header_size({"Content-Type": "text/html"}) == 0

5
tests/test_spidermiddleware_base.py
View File

@ -56,6 +56,11 @@ async def test_processed_request(crawler: Crawler) -> None:
spider_output = [test_req1, {"foo": "bar"}, test_req2, test_req3]
for processed in [
list(mw.process_spider_output(Response("data:,"), spider_output)),
await collect_asyncgen(
mw.process_spider_output_async(
Response("data:,"), as_async_generator(spider_output)
)
),
await collect_asyncgen(mw.process_start(as_async_generator(spider_output))),
]:
assert len(processed) == 3

25
tests/test_spidermiddleware_depth.py
View File

@ -7,6 +7,7 @@ import pytest
from scrapy.http import Request, Response
from scrapy.spidermiddlewares.depth import DepthMiddleware
from scrapy.spiders import Spider
from scrapy.utils.misc import build_from_crawler
from scrapy.utils.test import get_crawler
if TYPE_CHECKING:
@ -55,3 +56,27 @@ def test_process_spider_output(mw: DepthMiddleware, stats: StatsCollector) -> No
rdm = stats.get_value("request_depth_max")
assert rdm == 1
def test_priority_and_non_verbose_stats() -> None:
crawler = get_crawler(
Spider,
{"DEPTH_LIMIT": 0, "DEPTH_STATS_VERBOSE": False, "DEPTH_PRIORITY": 10},
)
assert crawler.stats is not None
crawler.stats.open_spider()
try:
mw = build_from_crawler(DepthMiddleware, crawler)
resp = Response("http://toscrape.com")
resp.request = Request("http://toscrape.com")
resp.request.meta["depth"] = 2
out = list(mw.process_spider_output(resp, [Request("http://toscrape.com")]))
assert len(out) == 1
# priority is decremented by depth * DEPTH_PRIORITY
assert out[0].priority == -30
assert out[0].meta["depth"] == 3
# non-verbose stats don't track per-depth counts but still track the max
assert crawler.stats.get_value("request_depth_count/3") is None
assert crawler.stats.get_value("request_depth_max") == 3
finally:
crawler.stats.close_spider()

43
tests/test_spidermiddleware_referer.py
View File

@ -6,7 +6,7 @@ from urllib.parse import urlparse
import pytest
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
from scrapy.http import Request, Response
from scrapy.settings import Settings
from scrapy.spidermiddlewares.referer import (
@ -1017,6 +1017,14 @@ class TestPolicyMethodResponseParamRename:
response=self.response, resp_or_url=self.response, request=self.request
)
def test_missing_response(self):
with pytest.raises(TypeError, match="Missing required argument: 'response'"):
self.mw.policy(request=self.request)
def test_missing_request(self):
with pytest.raises(TypeError, match="Missing required argument: 'request'"):
self.mw.policy(response=self.response)
@coroutine_test
async def test_response_policy_only_supports_policy_names():
@ -1115,3 +1123,36 @@ async def test_referer_policies_setting():
]
assert len(output) == 1
assert output[0].headers == {b"Referer": [b"https://python.org/"]}
class TestReferrerPolicyHelpers:
def test_origin_referrer_local_scheme(self):
# A local scheme yields no referrer.
assert UnsafeUrlPolicy().origin_referrer("data:,foo") is None
def test_strip_url_empty(self):
assert UnsafeUrlPolicy().strip_url("") is None
def test_potentially_trustworthy_data_scheme(self):
assert UnsafeUrlPolicy().potentially_trustworthy("data:,foo") is False
def test_default_policy():
crawler = get_crawler()
mw = build_from_crawler(RefererMiddleware, crawler)
assert mw.default_policy is DefaultReferrerPolicy
def test_no_settings_constructor():
with pytest.warns(
ScrapyDeprecationWarning,
match="Instantiating RefererMiddleware without a 'settings' argument",
):
mw = RefererMiddleware()
assert mw.default_policy is DefaultReferrerPolicy
def test_not_configured_when_disabled():
crawler = get_crawler(settings_dict={"REFERER_ENABLED": False})
with pytest.raises(NotConfigured):
build_from_crawler(RefererMiddleware, crawler)

12
tests/test_spidermiddleware_start.py
View File

@ -1,4 +1,4 @@
from scrapy.http import Request
from scrapy.http import Request, Response
from scrapy.spidermiddlewares.start import StartSpiderMiddleware
from scrapy.spiders import Spider
from scrapy.utils.misc import build_from_crawler
@ -23,3 +23,13 @@ class TestMiddleware:
async for request in mw.process_start(start())
]
assert result == [True, True, False, "foo"]
def test_spider_output_not_marked(self):
# Requests from a non-None response (spider output) are not flagged.
crawler = get_crawler(Spider)
mw = build_from_crawler(StartSpiderMiddleware, crawler)
response = Response("data:,")
request = Request("data:,1")
out = list(mw.process_spider_output(response, [request]))
assert out == [request]
assert "is_start_request" not in request.meta

8
tests/test_spidermiddleware_urllength.py
View File

@ -5,9 +5,11 @@ from typing import TYPE_CHECKING
import pytest
from scrapy.exceptions import NotConfigured
from scrapy.http import Request, Response
from scrapy.spidermiddlewares.urllength import UrlLengthMiddleware
from scrapy.spiders import Spider
from scrapy.utils.misc import build_from_crawler
from scrapy.utils.test import get_crawler
if TYPE_CHECKING:
@ -46,6 +48,12 @@ def test_middleware_works(mw: UrlLengthMiddleware) -> None:
assert process_spider_output(mw) == [short_url_req]
def test_not_configured_without_limit() -> None:
crawler = get_crawler(Spider, {"URLLENGTH_LIMIT": 0})
with pytest.raises(NotConfigured):
build_from_crawler(UrlLengthMiddleware, crawler)
def test_logging(
stats: StatsCollector, mw: UrlLengthMiddleware, caplog: pytest.LogCaptureFixture
) -> None: