From dd4549e6f9a92f4844b552aad49ab4431f73514f Mon Sep 17 00:00:00 2001 From: Adrian Date: Wed, 24 Jun 2026 22:48:28 +0200 Subject: [PATCH 1/3] Improve test coverage for downloader middlewares (#7655) * Improve test coverage for downloader middlewares * Improve coverage further --- .../downloadermiddlewares/httpcompression.py | 90 ++++++++++--------- tests/test_downloadermiddleware_cookies.py | 14 +++ ...st_downloadermiddleware_downloadtimeout.py | 6 ++ tests/test_downloadermiddleware_httpcache.py | 1 + ...st_downloadermiddleware_httpcompression.py | 18 +++- tests/test_downloadermiddleware_httpproxy.py | 6 ++ tests/test_downloadermiddleware_offsite.py | 18 ++++ tests/test_downloadermiddleware_redirect.py | 17 ++++ ...wnloadermiddleware_redirect_metarefresh.py | 7 ++ tests/test_downloadermiddleware_stats.py | 14 ++- 10 files changed, 146 insertions(+), 45 deletions(-) diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py index 414c3d8a3..2b1721ced 100644 --- a/scrapy/downloadermiddlewares/httpcompression.py +++ b/scrapy/downloadermiddlewares/httpcompression.py @@ -6,7 +6,7 @@ from logging import getLogger from typing import TYPE_CHECKING, Any from scrapy import Request, Spider, signals -from scrapy.exceptions import IgnoreRequest, NotConfigured +from scrapy.exceptions import IgnoreRequest, NotConfigured, ScrapyDeprecationWarning from scrapy.http import Response, TextResponse from scrapy.responsetypes import responsetypes from scrapy.utils._compression import ( @@ -70,6 +70,12 @@ class HttpCompressionMiddleware: crawler: Crawler | None = None, ): if not crawler: + warnings.warn( + "Instantiating HttpCompressionMiddleware without a 'crawler' " + "argument is deprecated.", + category=ScrapyDeprecationWarning, + stacklevel=2, + ) self.stats = stats self._max_size = 1073741824 self._warn_size = 33554432 @@ -108,49 +114,47 @@ class HttpCompressionMiddleware: ) -> Request | Response: if request.method == "HEAD": return response - if isinstance(response, Response): - content_encoding = response.headers.getlist("Content-Encoding") - if content_encoding: - max_size = request.meta.get("download_maxsize", self._max_size) - warn_size = request.meta.get("download_warnsize", self._warn_size) - try: - decoded_body, content_encoding = self._handle_encoding( - response.body, content_encoding, max_size - ) - except _DecompressionMaxSizeExceeded as e: - raise IgnoreRequest( - f"Ignored response {response} because its body " - f"({len(response.body)} B compressed, " - f"{e.decompressed_size} B decompressed so far) exceeded " - f"DOWNLOAD_MAXSIZE ({max_size} B) during decompression." - ) from e - if len(response.body) < warn_size <= len(decoded_body): - logger.warning( - f"{response} body size after decompression " - f"({len(decoded_body)} B) is larger than the " - f"download warning size ({warn_size} B)." - ) - if content_encoding: - self._warn_unknown_encoding(response, content_encoding) - response.headers["Content-Encoding"] = content_encoding - if self.stats: - self.stats.inc_value( - "httpcompression/response_bytes", - len(decoded_body), - ) - self.stats.inc_value("httpcompression/response_count") - respcls = responsetypes.from_args( - headers=response.headers, url=response.url, body=decoded_body + content_encoding = response.headers.getlist("Content-Encoding") + if content_encoding: + max_size = request.meta.get("download_maxsize", self._max_size) + warn_size = request.meta.get("download_warnsize", self._warn_size) + try: + decoded_body, content_encoding = self._handle_encoding( + response.body, content_encoding, max_size ) - kwargs: dict[str, Any] = {"body": decoded_body} - if issubclass(respcls, TextResponse): - # force recalculating the encoding until we make sure the - # responsetypes guessing is reliable - kwargs["encoding"] = None - response = response.replace(cls=respcls, **kwargs) - if not content_encoding: - del response.headers["Content-Encoding"] - + except _DecompressionMaxSizeExceeded as e: + raise IgnoreRequest( + f"Ignored response {response} because its body " + f"({len(response.body)} B compressed, " + f"{e.decompressed_size} B decompressed so far) exceeded " + f"DOWNLOAD_MAXSIZE ({max_size} B) during decompression." + ) from e + if len(response.body) < warn_size <= len(decoded_body): + logger.warning( + f"{response} body size after decompression " + f"({len(decoded_body)} B) is larger than the " + f"download warning size ({warn_size} B)." + ) + if content_encoding: + self._warn_unknown_encoding(response, content_encoding) + response.headers["Content-Encoding"] = content_encoding + if self.stats: + self.stats.inc_value( + "httpcompression/response_bytes", + len(decoded_body), + ) + self.stats.inc_value("httpcompression/response_count") + respcls = responsetypes.from_args( + headers=response.headers, url=response.url, body=decoded_body + ) + kwargs: dict[str, Any] = {"body": decoded_body} + if issubclass(respcls, TextResponse): + # force recalculating the encoding until we make sure the + # responsetypes guessing is reliable + kwargs["encoding"] = None + response = response.replace(cls=respcls, **kwargs) + if not content_encoding: + del response.headers["Content-Encoding"] return response def _handle_encoding( diff --git a/tests/test_downloadermiddleware_cookies.py b/tests/test_downloadermiddleware_cookies.py index 225562644..f79591020 100644 --- a/tests/test_downloadermiddleware_cookies.py +++ b/tests/test_downloadermiddleware_cookies.py @@ -131,6 +131,20 @@ class TestCookiesMiddleware: ), ) + def test_debug_no_cookies(self): + crawler = get_crawler(settings_dict={"COOKIES_DEBUG": True}) + mw = CookiesMiddleware.from_crawler(crawler) + with LogCapture( + "scrapy.downloadermiddlewares.cookies", + propagate=False, + level=logging.DEBUG, + ) as log: + req = Request("http://scrapytest.org/") + res = Response("http://scrapytest.org/") # no Set-Cookie header + mw.process_response(req, res) + mw.process_request(req) # no cookies to send either + log.check() # no log output since cl is empty in both cases + def test_setting_disabled_cookies_debug(self): crawler = get_crawler(settings_dict={"COOKIES_DEBUG": False}) mw = CookiesMiddleware.from_crawler(crawler) diff --git a/tests/test_downloadermiddleware_downloadtimeout.py b/tests/test_downloadermiddleware_downloadtimeout.py index c744d259c..e6b17960e 100644 --- a/tests/test_downloadermiddleware_downloadtimeout.py +++ b/tests/test_downloadermiddleware_downloadtimeout.py @@ -35,3 +35,9 @@ class TestDownloadTimeoutMiddleware: req.meta["download_timeout"] = 1 assert mw.process_request(req) is None assert req.meta.get("download_timeout") == 1 + + def test_zero_download_timeout(self): + req, spider, mw = self.get_request_spider_mw({"DOWNLOAD_TIMEOUT": 0}) + mw.spider_opened(spider) + assert mw.process_request(req) is None + assert req.meta.get("download_timeout") is None diff --git a/tests/test_downloadermiddleware_httpcache.py b/tests/test_downloadermiddleware_httpcache.py index e5d726764..6c86d7adf 100644 --- a/tests/test_downloadermiddleware_httpcache.py +++ b/tests/test_downloadermiddleware_httpcache.py @@ -135,6 +135,7 @@ class PolicyTestMixin: def test_dont_cache(self): with self._middleware() as mw: self.request.meta["dont_cache"] = True + assert mw.process_request(self.request) is None mw.process_response(self.request, self.response) assert mw.storage.retrieve_response(mw.crawler.spider, self.request) is None diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py index 30caa094f..5c4085657 100644 --- a/tests/test_downloadermiddleware_httpcompression.py +++ b/tests/test_downloadermiddleware_httpcompression.py @@ -11,7 +11,7 @@ from scrapy.downloadermiddlewares.httpcompression import ( ACCEPTED_ENCODINGS, HttpCompressionMiddleware, ) -from scrapy.exceptions import IgnoreRequest, NotConfigured +from scrapy.exceptions import IgnoreRequest, NotConfigured, ScrapyDeprecationWarning from scrapy.http import HtmlResponse, Request, Response from scrapy.responsetypes import responsetypes from scrapy.spiders import Spider @@ -124,6 +124,22 @@ class TestHttpCompression: HttpCompressionMiddleware, ) + def test_no_crawler_constructor(self): + with pytest.warns(ScrapyDeprecationWarning, match="HttpCompressionMiddleware"): + mw = HttpCompressionMiddleware() + buf = BytesIO() + with GzipFile(fileobj=buf, mode="wb") as f: + f.write(b"hello") + body = buf.getvalue() + request = Request("http://scrapytest.org") + response = Response( + "http://scrapytest.org", + body=body, + headers={"Content-Encoding": "gzip"}, + ) + newresponse = mw.process_response(request, response) + assert newresponse.body == b"hello" + def test_process_request(self): request = Request("http://scrapytest.org") assert "Accept-Encoding" not in request.headers diff --git a/tests/test_downloadermiddleware_httpproxy.py b/tests/test_downloadermiddleware_httpproxy.py index a2d421e39..7ed848764 100644 --- a/tests/test_downloadermiddleware_httpproxy.py +++ b/tests/test_downloadermiddleware_httpproxy.py @@ -373,6 +373,12 @@ class TestHttpProxyMiddleware: assert "proxy" not in request.meta assert b"Proxy-Authorization" not in request.headers + def test_proxy_unparseable_url_clears_meta(self): + middleware = HttpProxyMiddleware() + request = Request("http://example.com", meta={"proxy": "//"}) + assert middleware.process_request(request) is None + assert request.meta["proxy"] is None + def test_proxy_authentication_header_disabled_proxy(self): middleware = HttpProxyMiddleware() request = Request( diff --git a/tests/test_downloadermiddleware_offsite.py b/tests/test_downloadermiddleware_offsite.py index edfb15d10..c0b8dc4dd 100644 --- a/tests/test_downloadermiddleware_offsite.py +++ b/tests/test_downloadermiddleware_offsite.py @@ -213,3 +213,21 @@ def test_request_scheduled_invalid_domains(): request = Request(f"https://{letter}.example") with pytest.raises(IgnoreRequest): mw.request_scheduled(request, crawler.spider) + + +def test_repeated_offsite_domain(): + crawler = get_crawler(Spider) + crawler.spider = crawler._create_spider(name="a", allowed_domains=["example.com"]) + mw = OffsiteMiddleware.from_crawler(crawler) + mw.spider_opened(crawler.spider) + req1 = Request("http://other.org/1") + req2 = Request("http://other.org/2") + with pytest.raises(IgnoreRequest): + mw.process_request(req1) + assert "other.org" in mw.domains_seen + assert crawler.stats.get_value("offsite/domains") == 1 + assert crawler.stats.get_value("offsite/filtered") == 1 + with pytest.raises(IgnoreRequest): + mw.process_request(req2) + assert crawler.stats.get_value("offsite/domains") == 1 # not incremented again + assert crawler.stats.get_value("offsite/filtered") == 2 diff --git a/tests/test_downloadermiddleware_redirect.py b/tests/test_downloadermiddleware_redirect.py index 42a25cd5b..1da7bbf3e 100644 --- a/tests/test_downloadermiddleware_redirect.py +++ b/tests/test_downloadermiddleware_redirect.py @@ -4,6 +4,7 @@ from unittest.mock import MagicMock import pytest from scrapy.downloadermiddlewares.redirect import RedirectMiddleware +from scrapy.exceptions import NotConfigured from scrapy.http import Request, Response from scrapy.spidermiddlewares.referer import ( POLICY_NO_REFERRER, @@ -265,6 +266,16 @@ class TestRedirectMiddleware(Base.Test): assert isinstance(req2, Request) assert req2.url == "http://www.example.com/redirected#frag" + def test_redirect_target_has_fragment(self): + url = "http://www.example.com/302#original" + url2 = "http://www.example.com/redirected#target" + req = Request(url) + rsp = Response(url, headers={"Location": url2}, status=302) + + req2 = self.mw.process_response(req, rsp) + assert isinstance(req2, Request) + assert req2.url == "http://www.example.com/redirected#target" + def test_redirect_302_head(self): url = "http://www.example.com/302" url2 = "http://www.example.com/redirected2" @@ -458,3 +469,9 @@ def test_warning_subclass(caplog): assert ( "(if defined in your code base) to override the handle_referer() method" ) in caplog.text + + +def test_not_configured(): + crawler = get_crawler(DefaultSpider, {"REDIRECT_ENABLED": False}) + with pytest.raises(NotConfigured): + RedirectMiddleware.from_crawler(crawler) diff --git a/tests/test_downloadermiddleware_redirect_metarefresh.py b/tests/test_downloadermiddleware_redirect_metarefresh.py index 416fbc2aa..d849cc8fb 100644 --- a/tests/test_downloadermiddleware_redirect_metarefresh.py +++ b/tests/test_downloadermiddleware_redirect_metarefresh.py @@ -7,6 +7,7 @@ from unittest.mock import MagicMock import pytest from scrapy.downloadermiddlewares.redirect import MetaRefreshMiddleware +from scrapy.exceptions import NotConfigured from scrapy.http import HtmlResponse, Request, Response from scrapy.spiders import Spider from scrapy.utils.misc import build_from_crawler @@ -157,3 +158,9 @@ def test_warning_meta_refresh_middleware(caplog): "replace scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware " "with a subclass that overrides the handle_referer() method" ) in caplog.text + + +def test_not_configured(): + crawler = get_crawler(Spider, {"METAREFRESH_ENABLED": False}) + with pytest.raises(NotConfigured): + MetaRefreshMiddleware.from_crawler(crawler) diff --git a/tests/test_downloadermiddleware_stats.py b/tests/test_downloadermiddleware_stats.py index cf7b614c4..5609360a7 100644 --- a/tests/test_downloadermiddleware_stats.py +++ b/tests/test_downloadermiddleware_stats.py @@ -1,4 +1,7 @@ -from scrapy.downloadermiddlewares.stats import DownloaderStats +import pytest + +from scrapy.downloadermiddlewares.stats import DownloaderStats, get_header_size +from scrapy.exceptions import NotConfigured from scrapy.http import Request, Response from scrapy.spiders import Spider from scrapy.utils.test import get_crawler @@ -39,5 +42,14 @@ class TestDownloaderStats: 1, ) + def test_from_crawler_not_configured(self): + crawler = get_crawler(Spider, {"DOWNLOADER_STATS": False}) + with pytest.raises(NotConfigured): + DownloaderStats.from_crawler(crawler) + def teardown_method(self): self.crawler.stats.close_spider() + + +def test_get_header_size_non_list_value(): + assert get_header_size({"Content-Type": "text/html"}) == 0 From 65e8954a0689b65ba8ae187a3045e8d4ed00b84f Mon Sep 17 00:00:00 2001 From: Adrian Date: Thu, 25 Jun 2026 12:35:25 +0200 Subject: [PATCH 2/3] Improve test coverage for spider middlewares (#7664) --- scrapy/spidermiddlewares/referer.py | 6 ++++ tests/test_spidermiddleware_base.py | 5 +++ tests/test_spidermiddleware_depth.py | 25 ++++++++++++++ tests/test_spidermiddleware_referer.py | 43 +++++++++++++++++++++++- tests/test_spidermiddleware_start.py | 12 ++++++- tests/test_spidermiddleware_urllength.py | 8 +++++ 6 files changed, 97 insertions(+), 2 deletions(-) diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py index 2c9452174..264f685c1 100644 --- a/scrapy/spidermiddlewares/referer.py +++ b/scrapy/spidermiddlewares/referer.py @@ -308,6 +308,12 @@ class RefererMiddleware(BaseSpiderMiddleware): # Reference: https://www.w3.org/TR/referrer-policy/#referrer-policy-empty-string self.policies[""] = NoReferrerWhenDowngradePolicy if settings is None: + warn( + "Instantiating RefererMiddleware without a 'settings' argument is " + "deprecated.", + ScrapyDeprecationWarning, + stacklevel=2, + ) return setting_policies = settings.getdict("REFERRER_POLICIES") for policy_name, policy_class_import_path in setting_policies.items(): diff --git a/tests/test_spidermiddleware_base.py b/tests/test_spidermiddleware_base.py index 70326f3f1..4de57fda9 100644 --- a/tests/test_spidermiddleware_base.py +++ b/tests/test_spidermiddleware_base.py @@ -56,6 +56,11 @@ async def test_processed_request(crawler: Crawler) -> None: spider_output = [test_req1, {"foo": "bar"}, test_req2, test_req3] for processed in [ list(mw.process_spider_output(Response("data:,"), spider_output)), + await collect_asyncgen( + mw.process_spider_output_async( + Response("data:,"), as_async_generator(spider_output) + ) + ), await collect_asyncgen(mw.process_start(as_async_generator(spider_output))), ]: assert len(processed) == 3 diff --git a/tests/test_spidermiddleware_depth.py b/tests/test_spidermiddleware_depth.py index 00e547305..307ed2440 100644 --- a/tests/test_spidermiddleware_depth.py +++ b/tests/test_spidermiddleware_depth.py @@ -7,6 +7,7 @@ import pytest from scrapy.http import Request, Response from scrapy.spidermiddlewares.depth import DepthMiddleware from scrapy.spiders import Spider +from scrapy.utils.misc import build_from_crawler from scrapy.utils.test import get_crawler if TYPE_CHECKING: @@ -55,3 +56,27 @@ def test_process_spider_output(mw: DepthMiddleware, stats: StatsCollector) -> No rdm = stats.get_value("request_depth_max") assert rdm == 1 + + +def test_priority_and_non_verbose_stats() -> None: + crawler = get_crawler( + Spider, + {"DEPTH_LIMIT": 0, "DEPTH_STATS_VERBOSE": False, "DEPTH_PRIORITY": 10}, + ) + assert crawler.stats is not None + crawler.stats.open_spider() + try: + mw = build_from_crawler(DepthMiddleware, crawler) + resp = Response("http://toscrape.com") + resp.request = Request("http://toscrape.com") + resp.request.meta["depth"] = 2 + out = list(mw.process_spider_output(resp, [Request("http://toscrape.com")])) + assert len(out) == 1 + # priority is decremented by depth * DEPTH_PRIORITY + assert out[0].priority == -30 + assert out[0].meta["depth"] == 3 + # non-verbose stats don't track per-depth counts but still track the max + assert crawler.stats.get_value("request_depth_count/3") is None + assert crawler.stats.get_value("request_depth_max") == 3 + finally: + crawler.stats.close_spider() diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.py index b3a434c3f..32e83f990 100644 --- a/tests/test_spidermiddleware_referer.py +++ b/tests/test_spidermiddleware_referer.py @@ -6,7 +6,7 @@ from urllib.parse import urlparse import pytest -from scrapy.exceptions import ScrapyDeprecationWarning +from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning from scrapy.http import Request, Response from scrapy.settings import Settings from scrapy.spidermiddlewares.referer import ( @@ -1017,6 +1017,14 @@ class TestPolicyMethodResponseParamRename: response=self.response, resp_or_url=self.response, request=self.request ) + def test_missing_response(self): + with pytest.raises(TypeError, match="Missing required argument: 'response'"): + self.mw.policy(request=self.request) + + def test_missing_request(self): + with pytest.raises(TypeError, match="Missing required argument: 'request'"): + self.mw.policy(response=self.response) + @coroutine_test async def test_response_policy_only_supports_policy_names(): @@ -1115,3 +1123,36 @@ async def test_referer_policies_setting(): ] assert len(output) == 1 assert output[0].headers == {b"Referer": [b"https://python.org/"]} + + +class TestReferrerPolicyHelpers: + def test_origin_referrer_local_scheme(self): + # A local scheme yields no referrer. + assert UnsafeUrlPolicy().origin_referrer("data:,foo") is None + + def test_strip_url_empty(self): + assert UnsafeUrlPolicy().strip_url("") is None + + def test_potentially_trustworthy_data_scheme(self): + assert UnsafeUrlPolicy().potentially_trustworthy("data:,foo") is False + + +def test_default_policy(): + crawler = get_crawler() + mw = build_from_crawler(RefererMiddleware, crawler) + assert mw.default_policy is DefaultReferrerPolicy + + +def test_no_settings_constructor(): + with pytest.warns( + ScrapyDeprecationWarning, + match="Instantiating RefererMiddleware without a 'settings' argument", + ): + mw = RefererMiddleware() + assert mw.default_policy is DefaultReferrerPolicy + + +def test_not_configured_when_disabled(): + crawler = get_crawler(settings_dict={"REFERER_ENABLED": False}) + with pytest.raises(NotConfigured): + build_from_crawler(RefererMiddleware, crawler) diff --git a/tests/test_spidermiddleware_start.py b/tests/test_spidermiddleware_start.py index 26397d37c..def3a3df3 100644 --- a/tests/test_spidermiddleware_start.py +++ b/tests/test_spidermiddleware_start.py @@ -1,4 +1,4 @@ -from scrapy.http import Request +from scrapy.http import Request, Response from scrapy.spidermiddlewares.start import StartSpiderMiddleware from scrapy.spiders import Spider from scrapy.utils.misc import build_from_crawler @@ -23,3 +23,13 @@ class TestMiddleware: async for request in mw.process_start(start()) ] assert result == [True, True, False, "foo"] + + def test_spider_output_not_marked(self): + # Requests from a non-None response (spider output) are not flagged. + crawler = get_crawler(Spider) + mw = build_from_crawler(StartSpiderMiddleware, crawler) + response = Response("data:,") + request = Request("data:,1") + out = list(mw.process_spider_output(response, [request])) + assert out == [request] + assert "is_start_request" not in request.meta diff --git a/tests/test_spidermiddleware_urllength.py b/tests/test_spidermiddleware_urllength.py index 750ae3b07..1ed3a5637 100644 --- a/tests/test_spidermiddleware_urllength.py +++ b/tests/test_spidermiddleware_urllength.py @@ -5,9 +5,11 @@ from typing import TYPE_CHECKING import pytest +from scrapy.exceptions import NotConfigured from scrapy.http import Request, Response from scrapy.spidermiddlewares.urllength import UrlLengthMiddleware from scrapy.spiders import Spider +from scrapy.utils.misc import build_from_crawler from scrapy.utils.test import get_crawler if TYPE_CHECKING: @@ -46,6 +48,12 @@ def test_middleware_works(mw: UrlLengthMiddleware) -> None: assert process_spider_output(mw) == [short_url_req] +def test_not_configured_without_limit() -> None: + crawler = get_crawler(Spider, {"URLLENGTH_LIMIT": 0}) + with pytest.raises(NotConfigured): + build_from_crawler(UrlLengthMiddleware, crawler) + + def test_logging( stats: StatsCollector, mw: UrlLengthMiddleware, caplog: pytest.LogCaptureFixture ) -> None: From c690eac770a9fb803cdf0003ac3dc73e409146dc Mon Sep 17 00:00:00 2001 From: Adrian Date: Thu, 25 Jun 2026 13:34:37 +0200 Subject: [PATCH 3/3] =?UTF-8?q?Document=20=E2=80=9Clogging=20settings?= =?UTF-8?q?=E2=80=9D=20as=20special=20settings=20(#7668)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/topics/practices.rst | 8 ++--- docs/topics/settings.rst | 64 ++++++++++++++++++++++++++++++++++++++- 2 files changed, 67 insertions(+), 5 deletions(-) diff --git a/docs/topics/practices.rst b/docs/topics/practices.rst index 8a04f7ada..e7faf48c4 100644 --- a/docs/topics/practices.rst +++ b/docs/topics/practices.rst @@ -347,10 +347,10 @@ finishes before starting the next one: install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor") react(deferred_f_from_coro_f(crawl)) -.. note:: When running multiple spiders in the same process, :ref:`reactor - settings ` should not have a different value per spider. - Also, :ref:`pre-crawler settings ` cannot be defined - per spider. +.. note:: When running multiple spiders in the same process, :ref:`logging + settings ` and :ref:`reactor settings ` + should not have a different value per spider, and :ref:`pre-crawler + settings ` cannot be defined per spider. .. seealso:: :ref:`run-from-script`. diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 455985a56..c452f88d1 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -361,6 +361,31 @@ All of these settings, except for :setting:`ASYNCIO_EVENT_LOOP`, are only used when the Twisted reactor is used, i.e. when :setting:`TWISTED_REACTOR_ENABLED` is ``True``. +.. _logging-settings: + +Logging settings +---------------- + +**Logging settings** are settings that configure the global root logging +handler installed by :func:`~scrapy.utils.log.configure_logging`. + +These settings can be defined from a spider. However, because only 1 root +logging handler is active per process, these settings cannot use a different +value per spider when :ref:`running multiple spiders in the same process +`. + +These settings are: + +- :setting:`LOG_DATEFORMAT` +- :setting:`LOG_ENABLED` +- :setting:`LOG_ENCODING` +- :setting:`LOG_FILE` +- :setting:`LOG_FILE_APPEND` +- :setting:`LOG_FORMAT` +- :setting:`LOG_LEVEL` +- :setting:`LOG_SHORT_NAMES` +- :setting:`LOG_STDOUT` + .. _topics-settings-ref: Built-in settings reference @@ -481,6 +506,8 @@ Note that the event loop class must inherit from :class:`asyncio.AbstractEventLo :func:`asyncio.set_event_loop`, which will set the specified event loop as the current loop for the current OS thread. +.. note:: This is a :ref:`reactor setting `. + .. setting:: BOT_NAME BOT_NAME @@ -662,6 +689,8 @@ Whether to enable DNS in-memory cache. :setting:`TWISTED_REACTOR_ENABLED` is ``False``, and may have no effect either when :setting:`DNS_RESOLVER` is set to a different resolver. +.. note:: This is a :ref:`reactor setting `. + .. setting:: DNSCACHE_SIZE DNSCACHE_SIZE @@ -671,6 +700,8 @@ Default: ``10000`` DNS in-memory cache size, see :setting:`DNSCACHE_ENABLED`. +.. note:: This is a :ref:`reactor setting `. + .. setting:: TWISTED_DNS_RESOLVER TWISTED_DNS_RESOLVER @@ -688,6 +719,8 @@ take the :setting:`DNS_TIMEOUT` setting into account. .. note:: This setting has no effect when :setting:`TWISTED_REACTOR_ENABLED` is ``False``. +.. note:: This is a :ref:`reactor setting `. + .. setting:: DNS_TIMEOUT DNS_TIMEOUT @@ -703,6 +736,8 @@ Timeout for processing of DNS queries in seconds. Float is supported. :setting:`TWISTED_REACTOR_ENABLED` is ``False``, and may have no effect either when :setting:`DNS_RESOLVER` is set to a different resolver. +.. note:: This is a :ref:`reactor setting `. + .. setting:: DOWNLOADER DOWNLOADER @@ -1442,6 +1477,8 @@ Default: ``True`` Whether to enable logging. +.. note:: This is a :ref:`logging setting `. + .. setting:: LOG_ENCODING LOG_ENCODING @@ -1451,6 +1488,8 @@ Default: ``'utf-8'`` The encoding to use for logging. +.. note:: This is a :ref:`logging setting `. + .. setting:: LOG_FILE LOG_FILE @@ -1460,6 +1499,8 @@ Default: ``None`` File name to use for logging output. If ``None``, standard error will be used. +.. note:: This is a :ref:`logging setting `. + .. setting:: LOG_FILE_APPEND LOG_FILE_APPEND @@ -1470,6 +1511,8 @@ Default: ``True`` If ``False``, the log file specified with :setting:`LOG_FILE` will be overwritten (discarding the output from previous runs, if any). +.. note:: This is a :ref:`logging setting `. + .. setting:: LOG_FORMAT LOG_FORMAT @@ -1481,6 +1524,8 @@ String for formatting log messages. Refer to the :ref:`Python logging documentation ` for the whole list of available placeholders. +.. note:: This is a :ref:`logging setting `. + .. setting:: LOG_DATEFORMAT LOG_DATEFORMAT @@ -1493,6 +1538,8 @@ in :setting:`LOG_FORMAT`. Refer to the :ref:`Python datetime documentation ` for the whole list of available directives. +.. note:: This is a :ref:`logging setting `. + .. setting:: LOG_FORMATTER LOG_FORMATTER @@ -1512,6 +1559,8 @@ Default: ``'DEBUG'`` Minimum level to log. Available levels are: CRITICAL, ERROR, WARNING, INFO, DEBUG. For more info see :ref:`topics-logging`. +.. note:: This is a :ref:`logging setting `. + .. setting:: LOG_STDOUT LOG_STDOUT @@ -1523,6 +1572,8 @@ If ``True``, all standard output (and error) of your process will be redirected to the log. For example if you ``print('hello')`` it will appear in the Scrapy log. +.. note:: This is a :ref:`logging setting `. + .. setting:: LOG_SHORT_NAMES LOG_SHORT_NAMES @@ -1533,6 +1584,8 @@ Default: ``False`` If ``True``, the logs will just contain the root path. If it is set to ``False`` then it displays the component responsible for the log output +.. note:: This is a :ref:`logging setting `. + .. setting:: LOG_VERSIONS LOG_VERSIONS @@ -1695,6 +1748,8 @@ multi-purpose thread pool used by various Scrapy components. Threaded DNS Resolver, BlockingFeedStorage, S3FilesStore just to name a few. Increase this value if you're experiencing problems with insufficient blocking IO. +.. note:: This is a :ref:`reactor setting `. + .. setting:: REDIRECT_PRIORITY_ADJUST REDIRECT_PRIORITY_ADJUST @@ -1921,6 +1976,8 @@ Default: ``'scrapy.spiderloader.SpiderLoader'`` The class that will be used for loading spiders, which must implement the :ref:`topics-api-spiderloader`. +.. note:: This is a :ref:`pre-crawler setting `. + .. setting:: SPIDER_LOADER_WARN_ONLY SPIDER_LOADER_WARN_ONLY @@ -1933,6 +1990,8 @@ it will fail loudly if there is any ``ImportError`` or ``SyntaxError`` exception But you can choose to silence this exception and turn it into a simple warning by setting ``SPIDER_LOADER_WARN_ONLY = True``. +.. note:: This is a :ref:`pre-crawler setting `. + .. setting:: SPIDER_MIDDLEWARES SPIDER_MIDDLEWARES @@ -1978,6 +2037,8 @@ Example: SPIDER_MODULES = ["mybot.spiders_prod", "mybot.spiders_dev"] +.. note:: This is a :ref:`pre-crawler setting `. + .. setting:: STATS_CLASS STATS_CLASS @@ -2049,7 +2110,7 @@ stopped) will not apply. This mode is currently experimental and may not be suitable for production use. It may also not be supported by 3rd-party code. See :ref:`asyncio-without-reactor` for more information about this mode. -.. note:: This setting can't be set :ref:`per-spider `. +.. note:: This is a :ref:`pre-crawler setting `. .. versionadded:: 2.15.0 @@ -2156,6 +2217,7 @@ current platform. For additional information, see :doc:`core/howto/choosing-reactor`. +.. note:: This is a :ref:`reactor setting `. .. setting:: URLLENGTH_LIMIT