From 3a3695526122b9e3ccaf477ec32b6ff1ad750a60 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 15 Jun 2026 11:39:58 +0500 Subject: [PATCH 01/47] Assorted test fixes (#7616) --- scrapy/utils/_deps_compat.py | 12 ++++ tests/ignores.txt | 2 - tests/mockserver/simple_https.py | 2 +- tests/spiders.py | 2 +- tests/test_addons.py | 1 + tests/test_cmdline/extensions.py | 12 ---- tests/test_cmdline/settings.py | 2 +- tests/test_command_startproject.py | 2 +- tests/test_commands.py | 4 +- tests/test_contracts.py | 10 +-- tests/test_core_downloader.py | 5 +- tests/test_crawl.py | 4 +- tests/test_crawler.py | 27 ++------ tests/test_downloader_handler_twisted_ftp.py | 6 +- .../test_downloader_handler_twisted_http2.py | 5 ++ tests/test_downloader_handlers.py | 17 +++--- tests/test_downloader_handlers_http_base.py | 2 +- tests/test_downloadermiddleware_httpcache.py | 4 +- ...st_downloadermiddleware_httpcompression.py | 6 +- ...test_downloadermiddleware_redirect_base.py | 2 +- tests/test_downloadermiddleware_stats.py | 2 +- tests/test_downloaderslotssettings.py | 4 -- tests/test_dupefilters.py | 2 +- tests/test_engine.py | 5 +- tests/test_feedexport.py | 4 +- tests/test_feedexport_batch.py | 6 +- tests/test_feedexport_postprocess.py | 4 +- tests/test_http2_client_protocol.py | 6 +- tests/test_http_request.py | 5 +- tests/test_http_response.py | 61 +++++++++++++------ tests/test_http_response_text.py | 14 ++--- tests/test_logformatter.py | 8 +-- tests/test_pipeline_media.py | 4 -- tests/test_robotstxt_interface.py | 12 ++-- tests/test_scheduler_base.py | 2 +- tests/test_spidermiddleware.py | 18 ------ tests/test_spidermiddleware_process_start.py | 9 --- tests/test_spidermiddleware_referer.py | 2 +- tests/test_spiderstate.py | 17 ++++-- tests/test_utils_asyncio.py | 2 + tests/test_utils_deprecate.py | 4 +- tests/test_utils_httpobj.py | 2 +- tests/test_utils_log.py | 17 +++--- tests/test_utils_response.py | 2 +- 44 files changed, 154 insertions(+), 185 deletions(-) diff --git a/scrapy/utils/_deps_compat.py b/scrapy/utils/_deps_compat.py index fad7e6f6b..08e8cad24 100644 --- a/scrapy/utils/_deps_compat.py +++ b/scrapy/utils/_deps_compat.py @@ -1,7 +1,15 @@ +import sys + from OpenSSL import __version__ as PYOPENSSL_VERSION_STRING from packaging.version import Version from twisted import version as TWISTED_VERSION from twisted.python.versions import Version as TxVersion +from w3lib import __version__ as W3LIB_VERSION_STRING + +# improved urllib.robotparser, https://github.com/python/cpython/pull/149374 +STDLIB_IMPROVED_ROBOTFILEPARSER = sys.version_info >= (3, 14, 5) or ( + (3, 13, 14) <= sys.version_info < (3, 14) +) TWISTED_FAILURE_HAS_STACK = TWISTED_VERSION < TxVersion("twisted", 24, 10, 0) # changes to private _sslverify code, https://github.com/twisted/twisted/pull/12506 @@ -14,3 +22,7 @@ PYOPENSSL_VERSION = Version(PYOPENSSL_VERSION_STRING) PYOPENSSL_WANTS_X509_PKEY = PYOPENSSL_VERSION < Version("24.3.0") # SSL.Context.set_cipher_list() creates a temporary connection, making the context immutable PYOPENSSL_SET_CIPHER_LIST_TMP_CONN = PYOPENSSL_VERSION < Version("25.2.0") + +W3LIB_VERSION = Version(W3LIB_VERSION_STRING) +# safe_url_string() strips the input, https://github.com/scrapy/w3lib/pull/207 +W3LIB_STRIPS_URLS = W3LIB_VERSION >= Version("2.1.1") diff --git a/tests/ignores.txt b/tests/ignores.txt index 222288841..94edcf186 100644 --- a/tests/ignores.txt +++ b/tests/ignores.txt @@ -1,3 +1 @@ -scrapy/downloadermiddlewares/cookies.py scrapy/extensions/statsmailer.py -scrapy/extensions/memusage.py diff --git a/tests/mockserver/simple_https.py b/tests/mockserver/simple_https.py index 943775fa5..fdea666e1 100644 --- a/tests/mockserver/simple_https.py +++ b/tests/mockserver/simple_https.py @@ -33,7 +33,7 @@ class SimpleMockServer(BaseMockServer): super().__init__() self.keyfile = keyfile self.certfile = certfile - self.cipher_string = cipher_string or "" + self.cipher_string = cipher_string self.tls_min_version = tls_min_version self.tls_max_version = tls_max_version diff --git a/tests/spiders.py b/tests/spiders.py index 55d1ea365..612dc11c9 100644 --- a/tests/spiders.py +++ b/tests/spiders.py @@ -222,7 +222,7 @@ class AsyncDefDeferredWrappedSpider(SimpleSpider): class AsyncDefDeferredMaybeWrappedSpider(SimpleSpider): - name = "asyncdef_deferred_wrapped" + name = "asyncdef_deferred_maybe_wrapped" async def parse(self, response): await maybe_deferred_to_future(defer.succeed(None)) diff --git a/tests/test_addons.py b/tests/test_addons.py index db0fb2f31..14ebddda8 100644 --- a/tests/test_addons.py +++ b/tests/test_addons.py @@ -161,6 +161,7 @@ class TestAddonManager: settings.set("KEY", 0, priority="default") runner = runner_cls(settings) crawler = runner.create_crawler(Spider) + crawler._apply_settings() assert crawler.settings.getint("KEY") == 20 def test_fallback_workflow(self): diff --git a/tests/test_cmdline/extensions.py b/tests/test_cmdline/extensions.py index 11c821f8d..ef1e50c0c 100644 --- a/tests/test_cmdline/extensions.py +++ b/tests/test_cmdline/extensions.py @@ -1,14 +1,2 @@ -"""A test extension used to check the settings loading order""" - - -class TestExtension: - def __init__(self, settings): - settings.set("TEST1", f"{settings['TEST1']} + started") - - @classmethod - def from_crawler(cls, crawler): - return cls(crawler.settings) - - class DummyExtension: pass diff --git a/tests/test_cmdline/settings.py b/tests/test_cmdline/settings.py index 32b15e191..ec71bba0c 100644 --- a/tests/test_cmdline/settings.py +++ b/tests/test_cmdline/settings.py @@ -1,7 +1,7 @@ from pathlib import Path EXTENSIONS = { - "tests.test_cmdline.extensions.TestExtension": 0, + "tests.test_cmdline.extensions.DummyExtension": 0, } TEST1 = "default" diff --git a/tests/test_command_startproject.py b/tests/test_command_startproject.py index 2a9d0ed57..7ac6c4fb0 100644 --- a/tests/test_command_startproject.py +++ b/tests/test_command_startproject.py @@ -257,7 +257,7 @@ class TestStartprojectTemplates: assert actual_permissions == expected_permissions - def test_startproject_permissions_umask_022(self, tmp_path: Path) -> None: + def test_startproject_permissions_umask_002(self, tmp_path: Path) -> None: """Check that generated files have the right permissions when the system uses a umask value that causes new files to have different permissions than those from the template folder.""" diff --git a/tests/test_commands.py b/tests/test_commands.py index edb03da1b..0657f393c 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -214,9 +214,7 @@ class MySpider(scrapy.Spider): self._append_settings(proj_path / self.project_name, "TWISTED_REACTOR = None\n") self._assert_spider_works(self.NORMAL_MSG, proj_path, "sp") - self._assert_spider_asyncio_fail( - self.NORMAL_MSG, proj_path, "aiosp", "-s", "TWISTED_REACTOR=" - ) + self._assert_spider_asyncio_fail(self.NORMAL_MSG, proj_path, "aiosp") def test_spider_settings_asyncio(self, proj_path: Path) -> None: """The reactor is set via the spider settings to the asyncio value. diff --git a/tests/test_contracts.py b/tests/test_contracts.py index 008e326ec..e80945b93 100644 --- a/tests/test_contracts.py +++ b/tests/test_contracts.py @@ -388,7 +388,7 @@ class TestContractsManager: request = self.conman.from_method(spider.returns_item_meta, self.results) assert request.meta["key"] == "example" response.meta = request.meta - request.callback(ResponseMetaMock) + request.callback(response) assert response.meta["key"] == "example" self.should_succeed() @@ -476,14 +476,14 @@ class TestContractsManager: # invalid regex request = self.conman.from_method(spider.invalid_regex, self.results) - self.should_succeed() + assert request is None # invalid regex with valid contract request = self.conman.from_method( spider.invalid_regex_with_valid_contract, self.results ) - self.should_succeed() request.callback(response) + self.should_succeed() def test_custom_contracts(self): self.conman.from_spider(CustomContractSuccessSpider(), self.results) @@ -578,7 +578,7 @@ class TestCustomContractPrePostProcess: spider = DemoSpider() response = ResponseMock() contract = CustomFailContractPreProcess(spider.returns_request) - conman = ContractsManager([contract]) + conman = ContractsManager([UrlContract, ReturnsContract, contract]) request = conman.from_method(spider.returns_request, self.results) contract.add_pre_hook(request, self.results) @@ -592,7 +592,7 @@ class TestCustomContractPrePostProcess: spider = DemoSpider() response = ResponseMock() contract = CustomFailContractPostProcess(spider.returns_request) - conman = ContractsManager([contract]) + conman = ContractsManager([UrlContract, ReturnsContract, contract]) request = conman.from_method(spider.returns_request, self.results) contract.add_post_hook(request, self.results) diff --git a/tests/test_core_downloader.py b/tests/test_core_downloader.py index abeaa2f65..e348bfb7f 100644 --- a/tests/test_core_downloader.py +++ b/tests/test_core_downloader.py @@ -36,7 +36,6 @@ from tests.utils.decorators import coroutine_test if TYPE_CHECKING: from twisted.internet.defer import Deferred - from twisted.internet.ssl import ContextFactory from twisted.web.iweb import IBodyProducer @@ -48,8 +47,6 @@ class TestSlot: @pytest.mark.requires_reactor # this test is related to the Twisted HTTP code class TestContextFactoryBase: - context_factory: ContextFactory | None = None - @async_yield_fixture async def server_url(self, tmp_path): (tmp_path / "file").write_bytes(b"0123456789") @@ -69,7 +66,7 @@ class TestContextFactoryBase: return reactor.listenSSL( 0, site, - contextFactory=self.context_factory or ssl_context_factory(), + contextFactory=ssl_context_factory(), interface="127.0.0.1", ) diff --git a/tests/test_crawl.py b/tests/test_crawl.py index ada4c31ce..85d98f847 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -342,7 +342,7 @@ with multiples lines assert "responses" in crawler.spider.meta assert "failures" not in crawler.spider.meta # start() doesn't set Referer header - echo0 = json.loads(to_unicode(crawler.spider.meta["responses"][2].body)) + echo0 = json.loads(to_unicode(crawler.spider.meta["responses"][0].body)) assert "Referer" not in echo0["headers"] # following request sets Referer to the source request url echo1 = json.loads(to_unicode(crawler.spider.meta["responses"][1].body)) @@ -390,7 +390,7 @@ with multiples lines est = [x for sublist in est for x in sublist] # flatten est = [x.lstrip().rstrip() for x in est] it = iter(est) - s = dict(zip(it, it, strict=False)) + s = dict(zip(it, it, strict=True)) assert s["engine.spider.name"] == crawler.spider.name assert s["len(engine.scraper.slot.active)"] == "1" diff --git a/tests/test_crawler.py b/tests/test_crawler.py index 82956735b..31d195c4b 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -240,11 +240,7 @@ class TestCrawler(TestBaseCrawler): @classmethod def from_crawler(cls, crawler): - try: - crawler.get_downloader_middleware(DefaultSpider) - except Exception as e: - MySpider.result = e - raise + crawler.get_downloader_middleware(DefaultSpider) crawler = get_raw_crawler(MySpider, BASE_SETTINGS) with pytest.raises(RuntimeError): @@ -322,11 +318,7 @@ class TestCrawler(TestBaseCrawler): @classmethod def from_crawler(cls, crawler): - try: - crawler.get_extension(DefaultSpider) - except Exception as e: - MySpider.result = e - raise + crawler.get_extension(DefaultSpider) crawler = get_raw_crawler(MySpider, BASE_SETTINGS) with pytest.raises(RuntimeError): @@ -404,11 +396,7 @@ class TestCrawler(TestBaseCrawler): @classmethod def from_crawler(cls, crawler): - try: - crawler.get_item_pipeline(DefaultSpider) - except Exception as e: - MySpider.result = e - raise + crawler.get_item_pipeline(DefaultSpider) crawler = get_raw_crawler(MySpider, BASE_SETTINGS) with pytest.raises(RuntimeError): @@ -486,11 +474,7 @@ class TestCrawler(TestBaseCrawler): @classmethod def from_crawler(cls, crawler): - try: - crawler.get_spider_middleware(DefaultSpider) - except Exception as e: - MySpider.result = e - raise + crawler.get_spider_middleware(DefaultSpider) crawler = get_raw_crawler(MySpider, BASE_SETTINGS) with pytest.raises(RuntimeError): @@ -755,11 +739,12 @@ class TestCrawlerRunnerHasSpider: ): await self._crawl(runner, NoRequestsSpider) else: - CrawlerRunner( + runner = CrawlerRunner( settings={ "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", } ) + await self._crawl(runner, NoRequestsSpider) @pytest.mark.only_asyncio diff --git a/tests/test_downloader_handler_twisted_ftp.py b/tests/test_downloader_handler_twisted_ftp.py index 361e91382..489b70e74 100644 --- a/tests/test_downloader_handler_twisted_ftp.py +++ b/tests/test_downloader_handler_twisted_ftp.py @@ -59,7 +59,7 @@ class TestFTPBase(ABC): port = reactor.listenTCP(0, factory, interface="127.0.0.1") portno = port.getHost().port - yield f"https://127.0.0.1:{portno}/" + yield f"ftp://127.0.0.1:{portno}/" await port.stopListening() @@ -142,15 +142,11 @@ class TestFTPBase(ABC): server_url: str, dh: FTPDownloadHandler, ) -> None: - f, local_fname = mkstemp() - local_fname_path = Path(local_fname) - os.close(f) meta = {} meta.update(self.req_meta) request = Request(url=server_url + filename, meta=meta) r = await dh.download_request(request) assert type(r) is response_class # pylint: disable=unidiomatic-typecheck - local_fname_path.unlink() class TestFTP(TestFTPBase): diff --git a/tests/test_downloader_handler_twisted_http2.py b/tests/test_downloader_handler_twisted_http2.py index 5f79a5453..bea97642e 100644 --- a/tests/test_downloader_handler_twisted_http2.py +++ b/tests/test_downloader_handler_twisted_http2.py @@ -24,6 +24,7 @@ from tests.test_downloader_handlers_http_base import ( TestHttpWithCrawlerBase, TestMitmProxyBase, TestRealWebsiteBase, + TestSimpleHttpsBase, ) from tests.utils.decorators import coroutine_test @@ -156,6 +157,10 @@ class TestHttp2(H2DownloadHandlerMixin, TestHttpsBase): await download_handler.download_request(request) +class TestSimpleHttp2(H2DownloadHandlerMixin, TestSimpleHttpsBase): + pass + + class TestHttp2WrongHostname(H2DownloadHandlerMixin, TestHttpsWrongHostnameBase): pass diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index eadb7740e..e685f607a 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -191,17 +191,14 @@ class TestS3: @contextlib.contextmanager def _mocked_date(self, date): - try: - import botocore.auth # noqa: F401,PLC0415 - except ImportError: + import botocore.auth # noqa: F401,PLC0415 + + # We need to mock botocore.auth.formatdate, because otherwise + # botocore overrides Date header with current date and time + # and Authorization header is different each time + with mock.patch("botocore.auth.formatdate") as mock_formatdate: + mock_formatdate.return_value = date yield - else: - # We need to mock botocore.auth.formatdate, because otherwise - # botocore overrides Date header with current date and time - # and Authorization header is different each time - with mock.patch("botocore.auth.formatdate") as mock_formatdate: - mock_formatdate.return_value = date - yield @coroutine_test async def test_request_signing1(self): diff --git a/tests/test_downloader_handlers_http_base.py b/tests/test_downloader_handlers_http_base.py index 0e1ff07c9..223f288bf 100644 --- a/tests/test_downloader_handlers_http_base.py +++ b/tests/test_downloader_handlers_http_base.py @@ -563,7 +563,7 @@ class TestHttpBase(ABC): ) -> None: request = Request(mockserver.url("/text", is_secure=self.is_secure)) - # 10 is minimal size for this request and the limit is only counted on + # 5 is minimal size for this request and the limit is only counted on # response body. (regardless of headers) async with self.get_dh({"DOWNLOAD_MAXSIZE": 5}) as download_handler: response = await download_handler.download_request(request) diff --git a/tests/test_downloadermiddleware_httpcache.py b/tests/test_downloadermiddleware_httpcache.py index 548c0d8ee..e5d726764 100644 --- a/tests/test_downloadermiddleware_httpcache.py +++ b/tests/test_downloadermiddleware_httpcache.py @@ -203,7 +203,7 @@ class DummyPolicyTestMixin(PolicyTestMixin): assert mw.process_request(req) is None # s3 scheme response is cached by default - req, res = Request("s3://bucket/key"), Response("http://bucket/key") + req, res = Request("s3://bucket/key"), Response("s3://bucket/key") with self._middleware() as mw: assert mw.process_request(req) is None mw.process_response(req, res) @@ -214,7 +214,7 @@ class DummyPolicyTestMixin(PolicyTestMixin): assert "cached" in cached.flags # ignore s3 scheme - req, res = Request("s3://bucket/key2"), Response("http://bucket/key2") + req, res = Request("s3://bucket/key2"), Response("s3://bucket/key2") with self._middleware(HTTPCACHE_IGNORE_SCHEMES=["s3"]) as mw: assert mw.process_request(req) is None mw.process_response(req, res) diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py index bb7fcd6c7..30caa094f 100644 --- a/tests/test_downloadermiddleware_httpcompression.py +++ b/tests/test_downloadermiddleware_httpcompression.py @@ -361,7 +361,7 @@ class TestHttpCompression: zf.write(plainbody) zf.close() response = Response( - "http;//www.example.com/", headers=headers, body=f.getvalue() + "http://www.example.com/", headers=headers, body=f.getvalue() ) request = Request("http://www.example.com/") @@ -386,7 +386,7 @@ class TestHttpCompression: zf.write(plainbody) zf.close() response = HtmlResponse( - "http;//www.example.com/page.html", headers=headers, body=f.getvalue() + "http://www.example.com/page.html", headers=headers, body=f.getvalue() ) request = Request("http://www.example.com/") @@ -493,7 +493,7 @@ class TestHttpCompression: gz_resp.close() response = Response( - "http;//www.example.com/", headers=headers, body=r.getvalue() + "http://www.example.com/", headers=headers, body=r.getvalue() ) request = Request("http://www.example.com/") diff --git a/tests/test_downloadermiddleware_redirect_base.py b/tests/test_downloadermiddleware_redirect_base.py index 44ade93b7..32935769f 100644 --- a/tests/test_downloadermiddleware_redirect_base.py +++ b/tests/test_downloadermiddleware_redirect_base.py @@ -122,7 +122,7 @@ class Base: req1 = Request("http://a.example/first") rsp1 = self.get_response(req1, "/redirected") req2 = self.mw.process_response(req1, rsp1) - rsp2 = self.get_response(req1, "/redirected2") + rsp2 = self.get_response(req2, "/redirected2") req3 = self.mw.process_response(req2, rsp2) assert req2.url == "http://a.example/redirected" diff --git a/tests/test_downloadermiddleware_stats.py b/tests/test_downloadermiddleware_stats.py index 67af4264c..cf7b614c4 100644 --- a/tests/test_downloadermiddleware_stats.py +++ b/tests/test_downloadermiddleware_stats.py @@ -16,7 +16,7 @@ class TestDownloaderStats: self.crawler.stats.open_spider() self.req = Request("http://scrapytest.org") - self.res = Response("scrapytest.org", status=400) + self.res = Response("http://scrapytest.org", status=400) def assertStatsEqual(self, key, value): assert self.crawler.stats.get_value(key) == value, str( diff --git a/tests/test_downloaderslotssettings.py b/tests/test_downloaderslotssettings.py index a717b18a6..9d58a6e09 100644 --- a/tests/test_downloaderslotssettings.py +++ b/tests/test_downloaderslotssettings.py @@ -5,7 +5,6 @@ import pytest from scrapy import Request from scrapy.core.downloader import Downloader, Slot -from scrapy.crawler import CrawlerRunner from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.utils.spider import DefaultSpider from scrapy.utils.test import get_crawler @@ -65,9 +64,6 @@ class TestCrawl: def teardown_class(cls): cls.mockserver.__exit__(None, None, None) - def setup_method(self): - self.runner = CrawlerRunner() - @inline_callbacks_test def test_delay(self): crawler = get_crawler(DownloaderSlotsSettingsTestSpider) diff --git a/tests/test_dupefilters.py b/tests/test_dupefilters.py index b38bf9570..412a59fcd 100644 --- a/tests/test_dupefilters.py +++ b/tests/test_dupefilters.py @@ -86,7 +86,7 @@ class TestRFPDupeFilter: df.close("finished") df2 = _get_dupefilter(settings={"JOBDIR": path}, open_=False) - assert df != df2 + assert df is not df2 try: df2.open() assert df2.request_seen(r1) diff --git a/tests/test_engine.py b/tests/test_engine.py index 2cd583721..e51eb4664 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -6,7 +6,6 @@ import subprocess import sys from collections import defaultdict from dataclasses import dataclass -from logging import DEBUG from typing import TYPE_CHECKING, Any, cast from unittest.mock import Mock, call from urllib.parse import urlparse @@ -395,6 +394,7 @@ class TestEngine(TestEngineBase): self._assert_downloaded_responses(run, count=9) self._assert_scraped_items(run) self._assert_signals_caught(run) + self._assert_headers_received(run) self._assert_bytes_received(run) @coroutine_test @@ -606,7 +606,7 @@ class TestEngineDownload(TestEngineDownloadAsync): @coroutine_test -async def test_request_scheduled_signal(caplog): +async def test_request_scheduled_signal(): class TestScheduler(BaseScheduler): def __init__(self): self.enqueued = [] @@ -633,7 +633,6 @@ async def test_request_scheduled_signal(caplog): keep_request = Request("https://keep.example") engine._schedule_request(keep_request) drop_request = Request("https://drop.example") - caplog.set_level(DEBUG) engine._schedule_request(drop_request) assert scheduler.enqueued == [keep_request], ( f"{scheduler.enqueued!r} != [{keep_request!r}]" diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 27e0c6445..c1d6f04eb 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -32,7 +32,6 @@ from scrapy.extensions.feedexport import ( FeedSlot, FileFeedStorage, IFeedStorage, - S3FeedStorage, ) from scrapy.utils.python import to_unicode from scrapy.utils.test import get_crawler @@ -499,8 +498,7 @@ class TestFeedExport(TestFeedExportBase): }, } crawler = get_crawler(ItemSpider, settings) - with mock.patch.object(S3FeedStorage, "store"): - yield crawler.crawl(mockserver=self.mockserver) + yield crawler.crawl(mockserver=self.mockserver) assert "feedexport/success_count/FileFeedStorage" in crawler.stats.get_stats() assert "feedexport/success_count/StdoutFeedStorage" in crawler.stats.get_stats() assert crawler.stats.get_value("feedexport/success_count/FileFeedStorage") == 1 diff --git a/tests/test_feedexport_batch.py b/tests/test_feedexport_batch.py index d855d0f74..0a926479b 100644 --- a/tests/test_feedexport_batch.py +++ b/tests/test_feedexport_batch.py @@ -315,7 +315,7 @@ class TestBatchDeliveries(TestFeedExportBase): } data = await self.exported_data(items, settings) for fmt, expected in formats.items(): - for expected_batch, got_batch in zip(expected, data[fmt], strict=False): + for expected_batch, got_batch in zip(expected, data[fmt], strict=True): assert got_batch == expected_batch @coroutine_test @@ -339,7 +339,7 @@ class TestBatchDeliveries(TestFeedExportBase): } data = await self.exported_data(items, settings) for fmt, expected in formats.items(): - for expected_batch, got_batch in zip(expected, data[fmt], strict=False): + for expected_batch, got_batch in zip(expected, data[fmt], strict=True): assert got_batch == expected_batch @coroutine_test @@ -447,7 +447,7 @@ class TestBatchDeliveries(TestFeedExportBase): yield crawler.crawl() assert len(CustomS3FeedStorage.stubs) == len(items) - for stub in CustomS3FeedStorage.stubs[:-1]: + for stub in CustomS3FeedStorage.stubs: stub.assert_no_pending_responses() assert ( "feedexport/success_count/CustomS3FeedStorage" in crawler.stats.get_stats() diff --git a/tests/test_feedexport_postprocess.py b/tests/test_feedexport_postprocess.py index fa1c0586a..6ebcab152 100644 --- a/tests/test_feedexport_postprocess.py +++ b/tests/test_feedexport_postprocess.py @@ -270,7 +270,7 @@ class TestFeedPostProcessedExports(TestFeedExportBase): self._named_tempfile("check_CHECK_NONE"): lzma.compress( self.expected, check=lzma.CHECK_NONE ), - self._named_tempfile("check_CHECK_CRC256"): lzma.compress( + self._named_tempfile("CHECK_SHA256"): lzma.compress( self.expected, check=lzma.CHECK_SHA256 ), } @@ -282,7 +282,7 @@ class TestFeedPostProcessedExports(TestFeedExportBase): "postprocessing": ["scrapy.extensions.postprocessing.LZMAPlugin"], "lzma_check": lzma.CHECK_NONE, }, - self._named_tempfile("check_CHECK_CRC256"): { + self._named_tempfile("CHECK_SHA256"): { "format": "csv", "postprocessing": ["scrapy.extensions.postprocessing.LZMAPlugin"], "lzma_check": lzma.CHECK_SHA256, diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py index cec5d728b..28f306e31 100644 --- a/tests/test_http2_client_protocol.py +++ b/tests/test_http2_client_protocol.py @@ -141,7 +141,7 @@ class Dataloss(LeafResource): class NoContentLengthHeader(LeafResource): def render_GET(self, request: TxRequest): - request.requestHeaders.removeHeader("Content-Length") + request.responseHeaders.removeHeader("Content-Length") self.deferRequest(request, 0, self._delayed_render, request) return NOT_DONE_YET @@ -460,9 +460,7 @@ class TestHttps2ClientProtocol: def test_invalid_negotiated_protocol( self, server_port: int, client: H2ClientProtocol ) -> Generator[Deferred[Any], Any, None]: - with mock.patch( - "scrapy.core.http2.protocol.PROTOCOL_NAME", return_value=b"not-h2" - ): + with mock.patch("scrapy.core.http2.protocol.PROTOCOL_NAME", new=b"not-h2"): request = Request(url=self.get_url(server_port, "/status?n=200")) with pytest.raises(ResponseFailed): yield make_request_dfd(client, request) diff --git a/tests/test_http_request.py b/tests/test_http_request.py index fed5dbab7..fd494504d 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -23,7 +23,6 @@ class TestRequest: # url argument must be basestring with pytest.raises(TypeError): self.request_class(123) - r = self.request_class("http://www.example.com") r = self.request_class("http://www.example.com") assert isinstance(r.url, str) @@ -211,11 +210,11 @@ class TestRequest: r1.cb_kwargs["key"] = "value" r2 = r1.copy() - # make sure copy does not propagate callbacks + # make sure callbaclks are copied assert r1.callback is somecallback assert r1.errback is somecallback assert r2.callback is r1.callback - assert r2.errback is r2.errback + assert r2.errback is r1.errback # make sure flags list is shallow copied assert r1.flags is not r2.flags, "flags must be a shallow copy, not identical" diff --git a/tests/test_http_response.py b/tests/test_http_response.py index 09c95dc29..079c547c7 100644 --- a/tests/test_http_response.py +++ b/tests/test_http_response.py @@ -1,13 +1,19 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + import pytest -from packaging.version import Version as parse_version -from w3lib import __version__ as w3lib_version from w3lib.encoding import resolve_encoding from scrapy.exceptions import NotSupported from scrapy.http import Headers, Request, Response from scrapy.link import Link +from scrapy.utils._deps_compat import W3LIB_STRIPS_URLS from tests import get_testdata +if TYPE_CHECKING: + from collections.abc import Iterable + class TestResponse: response_class = Response @@ -249,7 +255,7 @@ class TestResponse: r.follow(None) @pytest.mark.xfail( - parse_version(w3lib_version) < parse_version("2.1.1"), + not W3LIB_STRIPS_URLS, reason="https://github.com/scrapy/w3lib/pull/207", strict=True, ) @@ -257,7 +263,7 @@ class TestResponse: self._assert_followed_url("foo ", "http://example.com/foo") @pytest.mark.xfail( - parse_version(w3lib_version) < parse_version("2.1.1"), + not W3LIB_STRIPS_URLS, reason="https://github.com/scrapy/w3lib/pull/207", strict=True, ) @@ -325,16 +331,26 @@ class TestResponse: with pytest.raises(ValueError, match="url can't be None"): list(r.follow_all(urls=[None])) + @pytest.mark.xfail( + not W3LIB_STRIPS_URLS, + reason="https://github.com/scrapy/w3lib/pull/207", + strict=True, + ) def test_follow_all_whitespace(self): relative = ["foo ", "bar ", "foo/bar ", "bar/foo "] absolute = [ - "http://example.com/foo%20", - "http://example.com/bar%20", - "http://example.com/foo/bar%20", - "http://example.com/bar/foo%20", + "http://example.com/foo", + "http://example.com/bar", + "http://example.com/foo/bar", + "http://example.com/bar/foo", ] self._assert_followed_all_urls(relative, absolute) + @pytest.mark.xfail( + not W3LIB_STRIPS_URLS, + reason="https://github.com/scrapy/w3lib/pull/207", + strict=True, + ) def test_follow_all_whitespace_links(self): absolute = [ "http://example.com/foo ", @@ -342,8 +358,8 @@ class TestResponse: "http://example.com/foo/bar ", "http://example.com/bar/foo ", ] - links = map(Link, absolute) - expected = [u.replace(" ", "%20") for u in absolute] + links = [Link(u) for u in absolute] + expected = [u.strip() for u in absolute] self._assert_followed_all_urls(links, expected) def test_follow_all_flags(self): @@ -357,25 +373,36 @@ class TestResponse: for req in fol: assert req.flags == ["cached", "allowed"] - def _assert_followed_url(self, follow_obj, target_url, response=None): + def _assert_followed_url( + self, + follow_obj: str | Link, + target_url: str, + response: Response | None = None, + encoding: str | None = None, + ) -> None: if response is None: response = self._links_response() req = response.follow(follow_obj) assert req.url == target_url - return req + if encoding is not None: + assert req.encoding == encoding - def _assert_followed_all_urls(self, follow_obj, target_urls, response=None): + def _assert_followed_all_urls( + self, + follow_obj: Iterable[str | Link], + target_urls: Iterable[str], + response: Response | None = None, + ) -> None: if response is None: response = self._links_response() followed = response.follow_all(follow_obj) - for req, target in zip(followed, target_urls, strict=False): + for req, target in zip(followed, target_urls, strict=True): assert req.url == target - yield req - def _links_response(self): + def _links_response(self) -> Response: body = get_testdata("link_extractor", "linkextractor.html") return self.response_class("http://example.com/index", body=body) - def _links_response_no_href(self): + def _links_response_no_href(self) -> Response: body = get_testdata("link_extractor", "linkextractor_no_href.html") return self.response_class("http://example.com/index", body=body) diff --git a/tests/test_http_response_text.py b/tests/test_http_response_text.py index c16af52b9..4b3fa2302 100644 --- a/tests/test_http_response_text.py +++ b/tests/test_http_response_text.py @@ -179,7 +179,6 @@ class TestTextResponse(TestResponse): # Inferring encoding from body also cache decoded body as sideeffect, # this test tries to ensure that calling response.encoding and # response.text in indistinct order doesn't affect final - # response.text in indistinct order doesn't affect final # values for encoding and decoded body. url = "http://example.com" body = b"\xef\xbb\xbfWORD" @@ -308,11 +307,12 @@ class TestTextResponse(TestResponse): "http://example.com/sample3.html#foo", "http://www.google.com/something", "http://example.com/innertag.html", + "http://example.com/page%204.html", ] # select elements for sellist in [resp.css("a"), resp.xpath("//a")]: - for sel, url in zip(sellist, urls, strict=False): + for sel, url in zip(sellist, urls, strict=True): self._assert_followed_url(sel, url, response=resp) # select elements @@ -324,7 +324,7 @@ class TestTextResponse(TestResponse): # href attributes should work for sellist in [resp.css("a::attr(href)"), resp.xpath("//a/@href")]: - for sel, url in zip(sellist, urls, strict=False): + for sel, url in zip(sellist, urls, strict=True): self._assert_followed_url(sel, url, response=resp) # non-a elements are not supported @@ -376,12 +376,12 @@ class TestTextResponse(TestResponse): encoding="utf8", body='click me'.encode(), ) - req = self._assert_followed_url( + self._assert_followed_url( resp1.css("a")[0], "http://example.com/foo?%D0%BF%D1%80%D0%B8%D0%B2%D0%B5%D1%82", response=resp1, + encoding="utf8", ) - assert req.encoding == "utf8" resp2 = self.response_class( "http://example.com", @@ -390,12 +390,12 @@ class TestTextResponse(TestResponse): "cp1251" ), ) - req = self._assert_followed_url( + self._assert_followed_url( resp2.css("a")[0], "http://example.com/foo?%EF%F0%E8%E2%E5%F2", response=resp2, + encoding="cp1251", ) - assert req.encoding == "cp1251" def test_follow_flags(self): res = self.response_class("http://example.com/") diff --git a/tests/test_logformatter.py b/tests/test_logformatter.py index 9806315b4..360aa613e 100644 --- a/tests/test_logformatter.py +++ b/tests/test_logformatter.py @@ -28,14 +28,14 @@ class TestLogFormatter: self.spider = Spider("default") self.spider.crawler = get_crawler() - def test_crawled_with_referer(self): + def test_crawled_without_referer(self): req = Request("http://www.example.com") res = Response("http://www.example.com") logkws = self.formatter.crawled(req, res, self.spider) logline = logkws["msg"] % logkws["args"] assert logline == "Crawled (200) (referer: None)" - def test_crawled_without_referer(self): + def test_crawled_with_referer(self): req = Request( "http://www.example.com", headers={"referer": "http://example.com"} ) @@ -198,7 +198,7 @@ class TestLogformatterSubclass(TestLogFormatter): self.spider = Spider("default") self.spider.crawler = get_crawler(Spider) - def test_crawled_with_referer(self): + def test_crawled_without_referer(self): req = Request("http://www.example.com") res = Response("http://www.example.com") logkws = self.formatter.crawled(req, res, self.spider) @@ -207,7 +207,7 @@ class TestLogformatterSubclass(TestLogFormatter): logline == "Crawled (200) (referer: None) []" ) - def test_crawled_without_referer(self): + def test_crawled_with_referer(self): req = Request( "http://www.example.com", headers={"referer": "http://example.com"}, diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py index 44df0bdd4..da1bfa317 100644 --- a/tests/test_pipeline_media.py +++ b/tests/test_pipeline_media.py @@ -175,10 +175,6 @@ class MockedMediaPipeline(UserDefinedPipeline): super().__init__(*args, crawler=crawler, **kwargs) self._mockcalled = [] - def download(self, request, info): - self._mockcalled.append("download") - return super().download(request, info) - def media_to_download(self, request, info, *, item=None): self._mockcalled.append("media_to_download") if "result" in request.meta: diff --git a/tests/test_robotstxt_interface.py b/tests/test_robotstxt_interface.py index 29b23496a..5249736f2 100644 --- a/tests/test_robotstxt_interface.py +++ b/tests/test_robotstxt_interface.py @@ -1,5 +1,3 @@ -import sys - import pytest from scrapy.robotstxt import ( @@ -8,6 +6,7 @@ from scrapy.robotstxt import ( RerpRobotParser, decode_robotstxt, ) +from scrapy.utils._deps_compat import STDLIB_IMPROVED_ROBOTFILEPARSER def rerp_available() -> bool: @@ -139,28 +138,25 @@ class TestDecodeRobotsTxt: class TestPythonRobotParser(BaseRobotParserTest): - # https://github.com/python/cpython/pull/149374 improves it - IMPROVED_ROBOTFILEPARSER = sys.version_info >= (3, 14, 5) - def setup_method(self): super()._setUp(PythonRobotParser) @pytest.mark.skipif( - not IMPROVED_ROBOTFILEPARSER, + not STDLIB_IMPROVED_ROBOTFILEPARSER, reason="RobotFileParser from this Python version does not support length based directives precedence.", ) def test_length_based_precedence(self): super().test_length_based_precedence() @pytest.mark.skipif( - IMPROVED_ROBOTFILEPARSER, + STDLIB_IMPROVED_ROBOTFILEPARSER, reason="RobotFileParser from this Python version does not support order based directives precedence.", ) def test_order_based_precedence(self): super().test_order_based_precedence() @pytest.mark.skipif( - not IMPROVED_ROBOTFILEPARSER, + not STDLIB_IMPROVED_ROBOTFILEPARSER, reason="RobotFileParser from this Python version does not support wildcards.", ) def test_allowed_wildcards(self): diff --git a/tests/test_scheduler_base.py b/tests/test_scheduler_base.py index db023e1f8..08acacae7 100644 --- a/tests/test_scheduler_base.py +++ b/tests/test_scheduler_base.py @@ -104,7 +104,7 @@ class TestMinimalScheduler(InterfaceCheckMixin): for url in URLS: assert self.scheduler.enqueue_request(Request(url)) assert not self.scheduler.enqueue_request(Request(url)) - assert self.scheduler.has_pending_requests + assert self.scheduler.has_pending_requests() dequeued = [] while self.scheduler.has_pending_requests(): diff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.py index a0d296553..e5891474b 100644 --- a/tests/test_spidermiddleware.py +++ b/tests/test_spidermiddleware.py @@ -238,16 +238,6 @@ class TestProcessSpiderOutputAsyncGen(TestProcessSpiderOutputSimple): yield item -class ProcessSpiderOutputNonIterableMiddleware: - def process_spider_output(self, response, result): - return - - -class ProcessSpiderOutputCoroutineMiddleware: - async def process_spider_output(self, response, result): - return result - - class ProcessStartSimpleMiddleware: async def process_start(self, start): async for item_or_request in start: @@ -423,20 +413,12 @@ class TestBuiltinMiddlewareAsyncGen(TestBuiltinMiddlewareSimple): class TestProcessSpiderException(TestBaseAsyncSpiderMiddleware): ITEM_TYPE = dict MW_ASYNCGEN = ProcessSpiderOutputAsyncGenMiddleware - MW_UNIVERSAL = ProcessSpiderOutputUniversalMiddleware MW_EXC_SIMPLE = ProcessSpiderExceptionSimpleIterableMiddleware MW_EXC_ASYNCGEN = ProcessSpiderExceptionAsyncIteratorMiddleware def _callback(self) -> Any: 1 / 0 - async def _test_asyncgen_nodowngrade(self, *mw_classes: type[Any]) -> None: - with pytest.raises( - _InvalidOutput, - match=r"Async iterable returned from .+ cannot be downgraded", - ): - await self._get_middleware_result(*mw_classes) - @coroutine_test async def test_exc_simple(self): """Simple exc mw""" diff --git a/tests/test_spidermiddleware_process_start.py b/tests/test_spidermiddleware_process_start.py index c907c6d73..21df73a65 100644 --- a/tests/test_spidermiddleware_process_start.py +++ b/tests/test_spidermiddleware_process_start.py @@ -14,7 +14,6 @@ from .utils.decorators import coroutine_test ITEM_A = {"id": "a"} ITEM_B = {"id": "b"} ITEM_C = {"id": "c"} -ITEM_D = {"id": "d"} class AsyncioSleepSpiderMiddleware: @@ -47,10 +46,6 @@ class ModernWrapSpider(Spider): yield ITEM_B -class ModernWrapSpiderSubclass(ModernWrapSpider): - name = "test" - - class ModernWrapSpiderMiddleware: async def process_start(self, start): yield ITEM_A @@ -79,10 +74,6 @@ class TestMain: expected_items = expected_items or [ITEM_A, ITEM_B, ITEM_C] await self._test([spider_middleware], spider_cls, expected_items) - async def _test_douple_wrap(self, smw1, smw2, spider_cls, expected_items=None): - expected_items = expected_items or [ITEM_A, ITEM_A, ITEM_B, ITEM_C, ITEM_C] - await self._test([smw1, smw2], spider_cls, expected_items) - @coroutine_test async def test_modern_mw_modern_spider(self): with warnings.catch_warnings(): diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.py index 7431ea6ac..a9089419a 100644 --- a/tests/test_spidermiddleware_referer.py +++ b/tests/test_spidermiddleware_referer.py @@ -836,7 +836,7 @@ class TestRequestMetaSettingFallback: request_meta, policy_class, check_warning, - ) in self.params[3:]: + ) in self.params: mw = RefererMiddleware(Settings(settings)) response = Response(origin, headers=response_headers) diff --git a/tests/test_spiderstate.py b/tests/test_spiderstate.py index 491fc88f7..e44cfca90 100644 --- a/tests/test_spiderstate.py +++ b/tests/test_spiderstate.py @@ -1,4 +1,7 @@ +from __future__ import annotations + from datetime import datetime, timezone +from typing import TYPE_CHECKING import pytest @@ -7,8 +10,11 @@ from scrapy.extensions.spiderstate import SpiderState from scrapy.spiders import Spider from scrapy.utils.test import get_crawler +if TYPE_CHECKING: + from pathlib import Path -def test_store_load(tmp_path): + +def test_store_load(tmp_path: Path) -> None: jobdir = str(tmp_path) spider = Spider(name="default") @@ -16,6 +22,7 @@ def test_store_load(tmp_path): ss = SpiderState(jobdir) ss.spider_opened(spider) + assert hasattr(spider, "state") spider.state["one"] = 1 spider.state["dt"] = dt ss.spider_closed(spider) @@ -23,21 +30,23 @@ def test_store_load(tmp_path): spider2 = Spider(name="default") ss2 = SpiderState(jobdir) ss2.spider_opened(spider2) - assert spider.state == {"one": 1, "dt": dt} + assert hasattr(spider2, "state") + assert spider2.state == {"one": 1, "dt": dt} ss2.spider_closed(spider2) -def test_state_attribute(): +def test_state_attribute() -> None: # state attribute must be present if jobdir is not set, to provide a # consistent interface spider = Spider(name="default") ss = SpiderState() ss.spider_opened(spider) + assert hasattr(spider, "state") assert spider.state == {} ss.spider_closed(spider) -def test_not_configured(): +def test_not_configured() -> None: crawler = get_crawler(Spider) with pytest.raises(NotConfigured): SpiderState.from_crawler(crawler) diff --git a/tests/test_utils_asyncio.py b/tests/test_utils_asyncio.py index a871a282e..5532b4a31 100644 --- a/tests/test_utils_asyncio.py +++ b/tests/test_utils_asyncio.py @@ -83,6 +83,7 @@ class TestParallelAsyncio: max_parallel_count, ) assert list(range(length)) == sorted(results) + assert parallel_count[0] == 0 assert max_parallel_count[0] <= self.CONCURRENT_ITEMS @coroutine_test @@ -101,6 +102,7 @@ class TestParallelAsyncio: max_parallel_count, ) assert list(range(length)) == sorted(results) + assert parallel_count[0] == 0 assert max_parallel_count[0] <= self.CONCURRENT_ITEMS diff --git a/tests/test_utils_deprecate.py b/tests/test_utils_deprecate.py index c5425d99d..5ea6f678e 100644 --- a/tests/test_utils_deprecate.py +++ b/tests/test_utils_deprecate.py @@ -22,9 +22,7 @@ class NewName(SomeBaseClass): class TestWarnWhenSubclassed: - def _mywarnings( - self, w: list[WarningMessage], category: type[Warning] = MyWarning - ) -> list[WarningMessage]: + def _mywarnings(self, w: list[WarningMessage]) -> list[WarningMessage]: return [x for x in w if x.category is MyWarning] def test_no_warning_on_definition(self): diff --git a/tests/test_utils_httpobj.py b/tests/test_utils_httpobj.py index 9bd86f7fb..0eb330461 100644 --- a/tests/test_utils_httpobj.py +++ b/tests/test_utils_httpobj.py @@ -17,4 +17,4 @@ def test_urlparse_cached(): assert req1a == urlp assert req1a is req1b assert req1a is not req2 - assert req1a is not req2 + assert req1b is not req2 diff --git a/tests/test_utils_log.py b/tests/test_utils_log.py index 8e5020022..ee552df64 100644 --- a/tests/test_utils_log.py +++ b/tests/test_utils_log.py @@ -76,15 +76,16 @@ class TestLogCounterHandler: @pytest.fixture def logger(self, crawler: Crawler) -> Generator[logging.Logger]: logger = logging.getLogger("test") - logger.setLevel(logging.NOTSET) + logger.setLevel(logging.DEBUG) logger.propagate = False - handler = LogCounterHandler(crawler) + handler = LogCounterHandler(crawler, level=crawler.settings.get("LOG_LEVEL")) logger.addHandler(handler) - - yield logger - - logger.propagate = True - logger.removeHandler(handler) + try: + yield logger + finally: + logger.propagate = True + logger.setLevel(logging.NOTSET) + logger.removeHandler(handler) def test_init(self, crawler: Crawler, logger: logging.Logger) -> None: assert crawler.stats @@ -102,7 +103,7 @@ class TestLogCounterHandler: def test_filtered_out_level(self, crawler: Crawler, logger: logging.Logger) -> None: logger.debug("test log msg") assert crawler.stats - assert crawler.stats.get_value("log_count/INFO") is None + assert crawler.stats.get_value("log_count/DEBUG") is None class TestStreamLogger: diff --git a/tests/test_utils_response.py b/tests/test_utils_response.py index e02bdfb69..4544cd29e 100644 --- a/tests/test_utils_response.py +++ b/tests/test_utils_response.py @@ -23,7 +23,7 @@ def _read_browser_output(burl: str): def test_open_in_browser(): - url = "http:///www.example.com/some/page.html" + url = "http://www.example.com/some/page.html" body = ( b" test page test body " ) From f63a3aff254f43f06fa7de074159ff1f42546789 Mon Sep 17 00:00:00 2001 From: Nishita Matlani <86043614+Nishieee@users.noreply.github.com> Date: Mon, 15 Jun 2026 02:41:55 -0400 Subject: [PATCH 02/47] Fix strip_url() removing default port from password. (#7605) --- scrapy/utils/url.py | 3 ++- tests/test_utils_url.py | 12 ++++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py index 8f75e2618..4d2bbdda2 100644 --- a/scrapy/utils/url.py +++ b/scrapy/utils/url.py @@ -139,7 +139,8 @@ def strip_url( ("ftp", 21), } ): - netloc = netloc.replace(f":{parsed_url.port}", "") + port_suffix = f":{parsed_url.port}" + netloc = netloc.removesuffix(port_suffix) return urlunparse( ( diff --git a/tests/test_utils_url.py b/tests/test_utils_url.py index 19a31c353..a74b9a41d 100644 --- a/tests/test_utils_url.py +++ b/tests/test_utils_url.py @@ -346,6 +346,18 @@ class TestStripUrl: "ftp://username:password@www.example.com:221/file.txt", "ftp://username:password@www.example.com:221/file.txt", ), + ( + "http://user:80@www.example.com:80/index.html", + "http://user:80@www.example.com/index.html", + ), + ( + "https://user:443@www.example.com:443/index.html", + "https://user:443@www.example.com/index.html", + ), + ( + "ftp://user:21@www.example.com:21/file.txt", + "ftp://user:21@www.example.com/file.txt", + ), ], ) def test_default_ports(self, url: str, expected: str) -> None: From 9893a7fac62a5aa29cf2bbf3282742fbfac3b7c5 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 15 Jun 2026 11:44:33 +0500 Subject: [PATCH 03/47] Fix deprecation warnings with pyOpenSSL 26.3.0. (#7619) --- scrapy/utils/_deps_compat.py | 4 +- scrapy/utils/ssl.py | 72 ++++++++++++++++----- tests/mockserver/utils.py | 4 +- tests/test_downloader_handlers_http_base.py | 16 ++++- tox.ini | 2 +- 5 files changed, 74 insertions(+), 24 deletions(-) diff --git a/scrapy/utils/_deps_compat.py b/scrapy/utils/_deps_compat.py index 08e8cad24..17957aba4 100644 --- a/scrapy/utils/_deps_compat.py +++ b/scrapy/utils/_deps_compat.py @@ -18,8 +18,8 @@ TWISTED_TLS_NEW_IMPL = TWISTED_VERSION >= TxVersion("twisted", 26, 4, 0) TWISTED_TLS_LIMITS_OFFBY1 = TWISTED_VERSION < TxVersion("twisted", 26, 4, 0) PYOPENSSL_VERSION = Version(PYOPENSSL_VERSION_STRING) -# SSL.Context.use_certificate() wants an X509 object, SSL.Context.use_privatekey() wants a PKey object -PYOPENSSL_WANTS_X509_PKEY = PYOPENSSL_VERSION < Version("24.3.0") +# pyOpenSSL X.509 APIs are deprecated and cryptography-based ones are preferred +PYOPENSSL_X509_DEPRECATED = PYOPENSSL_VERSION >= Version("24.3.0") # SSL.Context.set_cipher_list() creates a temporary connection, making the context immutable PYOPENSSL_SET_CIPHER_LIST_TMP_CONN = PYOPENSSL_VERSION < Version("25.2.0") diff --git a/scrapy/utils/ssl.py b/scrapy/utils/ssl.py index 22e9414b8..828a99e23 100644 --- a/scrapy/utils/ssl.py +++ b/scrapy/utils/ssl.py @@ -2,6 +2,7 @@ from __future__ import annotations import logging import ssl +import warnings from typing import TYPE_CHECKING, Any, TypedDict, TypeVar import OpenSSL._util as pyOpenSSLutil @@ -9,7 +10,11 @@ import OpenSSL.SSL import OpenSSL.version from twisted.internet.ssl import CertificateOptions, TLSVersion -from scrapy.utils._deps_compat import TWISTED_TLS_LIMITS_OFFBY1 +from scrapy.exceptions import ScrapyDeprecationWarning +from scrapy.utils._deps_compat import ( + PYOPENSSL_X509_DEPRECATED, + TWISTED_TLS_LIMITS_OFFBY1, +) from scrapy.utils.python import to_unicode if TYPE_CHECKING: @@ -116,23 +121,42 @@ def _log_sslobj_debug_info(sslobj: ssl.SSLObject) -> None: # pyOpenSSL utils -def ffi_buf_to_string(buf: Any) -> str: +def _ffi_buf_to_string(buf: Any) -> str: return to_unicode(pyOpenSSLutil.ffi.string(buf)) -def x509name_to_string(x509name: X509Name) -> str: +def ffi_buf_to_string(buf: Any) -> str: # pragma: no cover + warnings.warn( + "ffi_buf_to_string() is deprecated.", + ScrapyDeprecationWarning, + stacklevel=2, + ) + return ffi_buf_to_string(buf) + + +def _x509name_to_string(x509name: X509Name) -> str: # from OpenSSL.crypto.X509Name.__repr__ + # only used on pyOpenSSL < 24.3.0 result_buffer: Any = pyOpenSSLutil.ffi.new("char[]", 512) pyOpenSSLutil.lib.X509_NAME_oneline( x509name._name, result_buffer, len(result_buffer) ) - return ffi_buf_to_string(result_buffer) + return _ffi_buf_to_string(result_buffer) -def get_temp_key_info(ssl_object: Any) -> str | None: +def x509name_to_string(x509name: X509Name) -> str: # pragma: no cover + warnings.warn( + "x509name_to_string() is deprecated.", + ScrapyDeprecationWarning, + stacklevel=2, + ) + return _x509name_to_string(x509name) + + +def _get_temp_key_info(ssl_object: Any) -> str | None: # adapted from OpenSSL apps/s_cb.c::ssl_print_tmp_key() if not hasattr(pyOpenSSLutil.lib, "SSL_get_server_tmp_key"): - # removed in cryptography 40.0.0 + # removed in cryptography 40.0.0 (required starting from pyOpenSSL 23.1.0) return None temp_key_p = pyOpenSSLutil.ffi.new("EVP_PKEY **") if not pyOpenSSLutil.lib.SSL_get_server_tmp_key(ssl_object, temp_key_p): @@ -157,13 +181,22 @@ def get_temp_key_info(ssl_object: Any) -> str | None: cname = pyOpenSSLutil.lib.EC_curve_nid2nist(nid) if cname == pyOpenSSLutil.ffi.NULL: cname = pyOpenSSLutil.lib.OBJ_nid2sn(nid) - key_info.append(ffi_buf_to_string(cname)) + key_info.append(_ffi_buf_to_string(cname)) else: - key_info.append(ffi_buf_to_string(pyOpenSSLutil.lib.OBJ_nid2sn(key_type))) + key_info.append(_ffi_buf_to_string(pyOpenSSLutil.lib.OBJ_nid2sn(key_type))) key_info.append(f"{pyOpenSSLutil.lib.EVP_PKEY_bits(temp_key)} bits") return ", ".join(key_info) +def get_temp_key_info(ssl_object: Any) -> str | None: # pragma: no cover + warnings.warn( + "get_temp_key_info() is deprecated. It's also a no-op with cryptography 40.0.0+.", + ScrapyDeprecationWarning, + stacklevel=2, + ) + return _get_temp_key_info(ssl_object) + + def get_openssl_version() -> str: system_openssl_bytes = OpenSSL.SSL.SSLeay_version(OpenSSL.SSL.SSLEAY_VERSION) system_openssl = system_openssl_bytes.decode("ascii", errors="replace") @@ -177,14 +210,21 @@ def _log_ssl_conn_debug_info(hostname: str, connection: OpenSSL.SSL.Connection) connection.get_protocol_version_name(), connection.get_cipher_name(), ) - server_cert = connection.get_peer_certificate() - if server_cert: - logger.debug( - 'SSL connection certificate: issuer "%s", subject "%s"', - x509name_to_string(server_cert.get_issuer()), - x509name_to_string(server_cert.get_subject()), - ) - key_info = get_temp_key_info(connection._ssl) + if PYOPENSSL_X509_DEPRECATED: + if server_cert := connection.get_peer_certificate(as_cryptography=True): + logger.debug( + 'SSL connection certificate: issuer "%s", subject "%s"', + server_cert.issuer.rfc4514_string(), + server_cert.subject.rfc4514_string(), + ) + else: # noqa: PLR5501 + if server_cert_pyopenssl := connection.get_peer_certificate(): + logger.debug( + 'SSL connection certificate: issuer "%s", subject "%s"', + _x509name_to_string(server_cert_pyopenssl.get_issuer()), + _x509name_to_string(server_cert_pyopenssl.get_subject()), + ) + key_info = _get_temp_key_info(connection._ssl) if key_info: logger.debug("SSL temp key: %s", key_info) diff --git a/tests/mockserver/utils.py b/tests/mockserver/utils.py index 7aa656780..5c4ca7457 100644 --- a/tests/mockserver/utils.py +++ b/tests/mockserver/utils.py @@ -10,7 +10,7 @@ from OpenSSL.crypto import FILETYPE_PEM, load_certificate, load_privatekey from twisted.internet.ssl import CertificateOptions, ContextFactory from scrapy.core.downloader.tls import _TWISTED_VERSION_MAP -from scrapy.utils._deps_compat import PYOPENSSL_WANTS_X509_PKEY +from scrapy.utils._deps_compat import PYOPENSSL_X509_DEPRECATED from scrapy.utils.python import to_bytes from scrapy.utils.ssl import _get_cert_options_version_kwargs @@ -29,7 +29,7 @@ def ssl_context_factory( keyfile_path = Path(__file__).parent.parent / keyfile certfile_path = Path(__file__).parent.parent / certfile - if not PYOPENSSL_WANTS_X509_PKEY: + if PYOPENSSL_X509_DEPRECATED: cert = load_pem_x509_certificate(certfile_path.read_bytes()) key = load_pem_private_key(keyfile_path.read_bytes(), password=None) else: diff --git a/tests/test_downloader_handlers_http_base.py b/tests/test_downloader_handlers_http_base.py index 223f288bf..25a200817 100644 --- a/tests/test_downloader_handlers_http_base.py +++ b/tests/test_downloader_handlers_http_base.py @@ -33,7 +33,10 @@ from scrapy.exceptions import ( UnsupportedURLSchemeError, ) from scrapy.http import Headers, HtmlResponse, Request, Response, TextResponse -from scrapy.utils._deps_compat import TWISTED_TLS_LIMITS_OFFBY1 +from scrapy.utils._deps_compat import ( + PYOPENSSL_X509_DEPRECATED, + TWISTED_TLS_LIMITS_OFFBY1, +) from scrapy.utils.defer import deferred_from_coro, maybe_deferred_to_future from scrapy.utils.misc import build_from_crawler from scrapy.utils.spider import DefaultSpider @@ -831,8 +834,15 @@ class TestHttpsBase(TestHttpBase): is_secure = True tls_log_message = ( - 'SSL connection certificate: issuer "/C=IE/O=Scrapy/CN=localhost", ' - 'subject "/C=IE/O=Scrapy/CN=localhost"' + ( + 'SSL connection certificate: issuer "CN=localhost,O=Scrapy,C=IE", ' + 'subject "CN=localhost,O=Scrapy,C=IE"' + ) + if PYOPENSSL_X509_DEPRECATED + else ( + 'SSL connection certificate: issuer "/C=IE/O=Scrapy/CN=localhost", ' + 'subject "/C=IE/O=Scrapy/CN=localhost"' + ) ) def test_download_conn_lost(self) -> None: # type: ignore[override] diff --git a/tox.ini b/tox.ini index 50ac7aa86..44e240e03 100644 --- a/tox.ini +++ b/tox.ini @@ -82,7 +82,7 @@ deps = ptpython==3.0.32 # newer ones require newer Python ipython==8.39.0 - pyOpenSSL==26.2.0 + pyOpenSSL==26.3.0 pytest==9.0.3 socksio==1.0.0 types-Pygments==2.20.0.20260508 From 4f241b73be197b2a5aa3858302eb241f98e4f944 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 15 Jun 2026 11:45:26 +0500 Subject: [PATCH 04/47] Fix deprecation warnings with pytest 9.1. (#7621) --- tests/test_downloader_handlers_http_base.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/tests/test_downloader_handlers_http_base.py b/tests/test_downloader_handlers_http_base.py index 25a200817..304ef9f2b 100644 --- a/tests/test_downloader_handlers_http_base.py +++ b/tests/test_downloader_handlers_http_base.py @@ -906,16 +906,17 @@ class TestSimpleHttpsBase(ABC): cipher_string: str | None = None @pytest.fixture(scope="class") - def simple_mockserver(self) -> Generator[SimpleMockServer]: + @classmethod + def simple_mockserver(cls) -> Generator[SimpleMockServer]: with SimpleMockServer( - self.keyfile, self.certfile, cipher_string=self.cipher_string + cls.keyfile, cls.certfile, cipher_string=cls.cipher_string ) as simple_mockserver: yield simple_mockserver @pytest.fixture(scope="class") - def url(self, simple_mockserver: SimpleMockServer) -> str: - # need to use self.host instead of what mockserver returns - return f"https://{self.host}:{simple_mockserver.port(is_secure=True)}/file" + @classmethod + def url(cls, simple_mockserver: SimpleMockServer) -> str: + return f"https://{cls.host}:{simple_mockserver.port(is_secure=True)}/file" @property @abstractmethod From cfef12392a52591bf6bbb1d3c5dc57175913a63b Mon Sep 17 00:00:00 2001 From: Syncrain <71864702+syncrain@users.noreply.github.com> Date: Mon, 15 Jun 2026 12:16:30 +0530 Subject: [PATCH 05/47] Fix request_to_curl handling of verbose cookies dictionaries (#7603) --- scrapy/utils/request.py | 10 ++++++++- tests/test_utils_request.py | 42 +++++++++++++++++++++++++++++++++++++ 2 files changed, 51 insertions(+), 1 deletion(-) diff --git a/scrapy/utils/request.py b/scrapy/utils/request.py index ffb7fae49..398403d90 100644 --- a/scrapy/utils/request.py +++ b/scrapy/utils/request.py @@ -179,6 +179,12 @@ def _get_method(obj: Any, name: Any) -> Any: raise ValueError(f"Method {name!r} not found in: {obj}") from None +def _cookie_value_to_unicode(value: str | bytes | float) -> str: + if isinstance(value, bytes): + return value.decode() + return str(value) + + def request_to_curl(request: Request) -> str: """ Converts a :class:`~scrapy.Request` object to a curl command. @@ -202,7 +208,9 @@ def request_to_curl(request: Request) -> str: cookies = f"--cookie '{cookie}'" elif isinstance(request.cookies, list): cookie = "; ".join( - f"{next(iter(c.keys()))}={next(iter(c.values()))}" + f"{_cookie_value_to_unicode(c['name'])}={_cookie_value_to_unicode(c['value'])}" + if "name" in c and "value" in c + else f"{next(iter(c.keys()))}={next(iter(c.values()))}" for c in request.cookies ) cookies = f"--cookie '{cookie}'" diff --git a/tests/test_utils_request.py b/tests/test_utils_request.py index 55c46059e..e4967d4e7 100644 --- a/tests/test_utils_request.py +++ b/tests/test_utils_request.py @@ -413,3 +413,45 @@ class TestRequestToCurl: " --data-raw '{\"foo\": \"bar\"}' --cookie 'foo=bar'" ) self._test_request(request_object, expected_curl_command) + + def test_cookies_list_verbose(self): + request_object = Request( + "https://www.httpbin.org/post", + method="POST", + cookies=[ + { + "name": b"foo", + "value": b"bar", + "domain": "example.com", + "path": "/", + "secure": True, + } + ], + body=json.dumps({"foo": "bar"}), + ) + expected_curl_command = ( + "curl -X POST https://www.httpbin.org/post" + " --data-raw '{\"foo\": \"bar\"}' --cookie 'foo=bar'" + ) + self._test_request(request_object, expected_curl_command) + + def test_cookies_list_verbose_non_string_value(self): + request_object = Request( + "https://www.httpbin.org/post", + method="POST", + cookies=[ + { + "name": "foo", + "value": 1, + "domain": "example.com", + "path": "/", + "secure": True, + } + ], + body=json.dumps({"foo": "bar"}), + ) + expected_curl_command = ( + "curl -X POST https://www.httpbin.org/post" + " --data-raw '{\"foo\": \"bar\"}' --cookie 'foo=1'" + ) + self._test_request(request_object, expected_curl_command) From e74647572d692598ca943db539d45aefdc4956ac Mon Sep 17 00:00:00 2001 From: Adrian Date: Mon, 15 Jun 2026 11:18:29 +0200 Subject: [PATCH 06/47] Test SignalManager.disconnect_all() (#7625) --- tests/test_signalmanager.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 tests/test_signalmanager.py diff --git a/tests/test_signalmanager.py b/tests/test_signalmanager.py new file mode 100644 index 000000000..ce4d97adb --- /dev/null +++ b/tests/test_signalmanager.py @@ -0,0 +1,21 @@ +from scrapy.signalmanager import SignalManager + + +class TestSignalManager: + def test_disconnect_all(self): + signal = object() + sender = object() + sm = SignalManager(sender) + + calls = [] + + def handler(): + calls.append(1) + + sm.connect(handler, signal) + sm.send_catch_log(signal) + assert calls == [1] + + sm.disconnect_all(signal) + sm.send_catch_log(signal) + assert calls == [1] # handler no longer called after disconnect_all From 0a4a92e84334770a3aca3a3c44985cb08692535c Mon Sep 17 00:00:00 2001 From: Kushal Gupta <98078018+guptakushal03@users.noreply.github.com> Date: Tue, 16 Jun 2026 00:54:44 +0530 Subject: [PATCH 07/47] Fix image store ACL settings for S3 and GCS (#7614) * Fix image store ACL settings for S3 and GCS * Fix typing issues in ImagesPipeline ACL settings * Fix typing issues in ImagesPipeline ACL settings * Call parent _update_stores in ImagesPipeline --- scrapy/pipelines/images.py | 25 ++++++++++++++++++++-- tests/test_pipeline_images.py | 39 +++++++++++++++++++++++++++++++++++ 2 files changed, 62 insertions(+), 2 deletions(-) diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py index 83d04e6ca..762b0fdf1 100644 --- a/scrapy/pipelines/images.py +++ b/scrapy/pipelines/images.py @@ -11,14 +11,20 @@ import hashlib import warnings from contextlib import suppress from io import BytesIO -from typing import TYPE_CHECKING, Any, ClassVar +from typing import TYPE_CHECKING, Any, ClassVar, cast from itemadapter import ItemAdapter from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning from scrapy.http import Request, Response from scrapy.http.request import NO_CALLBACK -from scrapy.pipelines.files import FileException, FilesPipeline, _md5sum +from scrapy.pipelines.files import ( + FileException, + FilesPipeline, + GCSFilesStore, + S3FilesStore, + _md5sum, +) from scrapy.utils.defer import ensure_awaitable from scrapy.utils.python import to_bytes @@ -33,6 +39,7 @@ if TYPE_CHECKING: from scrapy.crawler import Crawler from scrapy.pipelines.media import FileInfoOrError, MediaPipeline + from scrapy.settings import BaseSettings class ImageException(FileException): @@ -126,6 +133,20 @@ class ImagesPipeline(FilesPipeline): ) -> str: return await self.image_downloaded(response, request, info, item=item) + @classmethod + def _update_stores(cls, settings: BaseSettings) -> None: + super()._update_stores(settings) + + s3store: type[S3FilesStore] = cast( + "type[S3FilesStore]", cls.STORE_SCHEMES["s3"] + ) + s3store.POLICY = settings["IMAGES_STORE_S3_ACL"] + + gcs_store: type[GCSFilesStore] = cast( + "type[GCSFilesStore]", cls.STORE_SCHEMES["gs"] + ) + gcs_store.POLICY = settings["IMAGES_STORE_GCS_ACL"] or None + async def image_downloaded( self, response: Response, diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py index 38662348f..1b73dd157 100644 --- a/tests/test_pipeline_images.py +++ b/tests/test_pipeline_images.py @@ -14,6 +14,7 @@ from itemadapter import ItemAdapter from scrapy.http import Request, Response from scrapy.item import Field, Item +from scrapy.pipelines.files import GCSFilesStore, S3FilesStore from scrapy.pipelines.images import ImageException, ImagesPipeline from scrapy.utils.test import get_crawler @@ -541,6 +542,44 @@ class TestImagesPipelineCustomSettings: expected_value = settings.get(settings_attr) assert getattr(pipeline_cls, pipe_attr.lower()) == expected_value + def test_images_store_s3_acl_setting_used(self, tmp_path): + old_policy = S3FilesStore.POLICY + + try: + crawler = get_crawler( + None, + { + "IMAGES_STORE": tmp_path, + "IMAGES_STORE_S3_ACL": "public-read", + "FILES_STORE_S3_ACL": "private", + }, + ) + + ImagesPipeline.from_crawler(crawler) + + assert S3FilesStore.POLICY == "public-read" + finally: + S3FilesStore.POLICY = old_policy + + def test_images_store_gcs_acl_setting_used(self, tmp_path): + old_policy = GCSFilesStore.POLICY + + try: + crawler = get_crawler( + None, + { + "IMAGES_STORE": tmp_path, + "IMAGES_STORE_GCS_ACL": "authenticatedRead", + "FILES_STORE_GCS_ACL": "", + }, + ) + + ImagesPipeline.from_crawler(crawler) + + assert GCSFilesStore.POLICY == "authenticatedRead" + finally: + GCSFilesStore.POLICY = old_policy + def _create_image(format_, *a, **kw): buf = io.BytesIO() From b7824db573ca5a50890024f3e2e63c714a62aa20 Mon Sep 17 00:00:00 2001 From: JSap0914 <116227558+JSap0914@users.noreply.github.com> Date: Wed, 17 Jun 2026 18:48:43 +0900 Subject: [PATCH 08/47] Fix rel_has_nofollow to be case-insensitive per HTML spec (#7632) The HTML specification states that link type keywords like 'nofollow' are ASCII case-insensitive. Sites using rel="NoFollow" or rel="NOFOLLOW" were incorrectly treated as follow links, causing Scrapy to crawl pages it should skip. Fix: add .lower() before the token split so all casing variants of 'nofollow' are correctly recognized. Co-authored-by: JSap0914 --- scrapy/utils/misc.py | 2 +- tests/test_utils_misc/__init__.py | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index 0b67eaa34..47568e656 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -162,7 +162,7 @@ def md5sum(file: IO[bytes]) -> str: def rel_has_nofollow(rel: str | None) -> bool: """Return True if link rel attribute has nofollow type""" - return rel is not None and "nofollow" in rel.replace(",", " ").split() + return rel is not None and "nofollow" in rel.lower().replace(",", " ").split() class SupportsFromCrawler(Protocol[_T_co, _P]): diff --git a/tests/test_utils_misc/__init__.py b/tests/test_utils_misc/__init__.py index a995e38e6..c4c861404 100644 --- a/tests/test_utils_misc/__init__.py +++ b/tests/test_utils_misc/__init__.py @@ -160,3 +160,8 @@ class TestUtilsMisc: assert rel_has_nofollow("nofollowfoo") is False assert rel_has_nofollow("foonofollow") is False assert rel_has_nofollow("ugc, , nofollow") is True + # rel attribute values are ASCII case-insensitive per the HTML spec + assert rel_has_nofollow("NoFollow") is True + assert rel_has_nofollow("NOFOLLOW") is True + assert rel_has_nofollow("UGC NoFollow") is True + assert rel_has_nofollow("ugc,NoFollow") is True From fada8be1db6479e235d7e06afa057632dbc2adf1 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 17 Jun 2026 14:56:22 +0500 Subject: [PATCH 09/47] Fix Proxy-Authorization handling in BaseStreamingDownloadHandler (#7630) --- conftest.py | 37 +---- .../downloader/handlers/_base_streaming.py | 16 +- scrapy/core/downloader/handlers/_httpx.py | 3 +- tests/test_downloader_handlers_http_base.py | 152 +++++++++--------- 4 files changed, 96 insertions(+), 112 deletions(-) diff --git a/conftest.py b/conftest.py index cf0568111..532f83f56 100644 --- a/conftest.py +++ b/conftest.py @@ -74,40 +74,19 @@ def mockserver() -> Generator[MockServer]: @pytest.fixture # function scope because it modifies os.environ -def mitm_proxy_server(monkeypatch: pytest.MonkeyPatch) -> Generator[MitmProxy]: - proxy = MitmProxy() +def proxy_server( + request: pytest.FixtureRequest, monkeypatch: pytest.MonkeyPatch +) -> Generator[str]: + kind = request.param + proxy = MitmProxy(mode="socks5" if kind == "socks5" else None) url = proxy.start() + if kind == "https": + url = url.replace("http://", "https://") monkeypatch.setenv("http_proxy", url) monkeypatch.setenv("https_proxy", url) try: - yield proxy - finally: - proxy.stop() - - -@pytest.fixture # function scope because it modifies os.environ -def mitm_proxy_server_https(monkeypatch: pytest.MonkeyPatch) -> Generator[MitmProxy]: - proxy = MitmProxy() - url = proxy.start().replace("http://", "https://") - monkeypatch.setenv("http_proxy", url) - monkeypatch.setenv("https_proxy", url) - - try: - yield proxy - finally: - proxy.stop() - - -@pytest.fixture # function scope because it modifies os.environ -def socks5_proxy_server(monkeypatch: pytest.MonkeyPatch) -> Generator[MitmProxy]: - proxy = MitmProxy(mode="socks5") - url = proxy.start() - monkeypatch.setenv("http_proxy", url) - monkeypatch.setenv("https_proxy", url) - - try: - yield proxy + yield kind finally: proxy.stop() diff --git a/scrapy/core/downloader/handlers/_base_streaming.py b/scrapy/core/downloader/handlers/_base_streaming.py index 16b655716..5a669c732 100644 --- a/scrapy/core/downloader/handlers/_base_streaming.py +++ b/scrapy/core/downloader/handlers/_base_streaming.py @@ -241,6 +241,16 @@ class BaseStreamingDownloadHandler(BaseHttpDownloadHandler, ABC, Generic[_Respon body=response_body.getvalue(), ) + @staticmethod + def _request_headers(request: Request) -> Headers: + """Get a prepared copy of the request headers. + + This removes the Proxy-Authorization header. + """ + headers = request.headers.copy() + headers.pop(b"Proxy-Authorization", None) + return headers + def _get_bind_address_host(self) -> str | None: """Return the host portion of the bind address. @@ -279,10 +289,8 @@ class BaseStreamingDownloadHandler(BaseHttpDownloadHandler, ABC, Generic[_Respon if not proxy: return None, None proxy = add_http_if_no_scheme(proxy) - auth_header: list[bytes] | None = request.headers.pop( - b"Proxy-Authorization", None - ) - return proxy, auth_header[0].decode("ascii") if auth_header else None + auth_header: bytes | None = request.headers.get(b"Proxy-Authorization") + return proxy, auth_header.decode("ascii") if auth_header else None def _extract_proxy_url_with_creds(self, request: Request) -> str | None: """Return the proxy URL with the userinfo added based on the diff --git a/scrapy/core/downloader/handlers/_httpx.py b/scrapy/core/downloader/handlers/_httpx.py index c960fc152..9b54b44d8 100644 --- a/scrapy/core/downloader/handlers/_httpx.py +++ b/scrapy/core/downloader/handlers/_httpx.py @@ -152,13 +152,14 @@ class HttpxDownloadHandler(_Base): f"SOCKS proxy support in {type(self).__name__} requires the 'httpx[socks]' extra to be installed." ) client = self._get_client(proxy) + headers = self._request_headers(request).to_tuple_list() try: async with client.stream( request.method, request.url, content=request.body, - headers=request.headers.to_tuple_list(), + headers=headers, timeout=timeout, ) as response: yield response diff --git a/tests/test_downloader_handlers_http_base.py b/tests/test_downloader_handlers_http_base.py index 304ef9f2b..2ae9206a0 100644 --- a/tests/test_downloader_handlers_http_base.py +++ b/tests/test_downloader_handlers_http_base.py @@ -1328,6 +1328,9 @@ class TestHttpProxyBase(ABC): assert response.body == self.expected_http_proxy_request_body +PROXY_KINDS = ["http", "https", "socks5"] + + class TestMitmProxyBase(ABC): # whether the handler supports HTTPS proxies with HTTPS destinations handler_supports_tls_in_tls: bool = True @@ -1338,57 +1341,54 @@ class TestMitmProxyBase(ABC): def settings_dict(self) -> dict[str, Any] | None: raise NotImplementedError - @pytest.mark.parametrize( - "https_dest", [False, True], ids=["HTTP dest", "HTTPS dest"] - ) - @pytest.mark.usefixtures("mitm_proxy_server") - @coroutine_test - async def test_http_proxy( - self, caplog: pytest.LogCaptureFixture, mockserver: MockServer, https_dest: bool - ) -> None: - """HTTP proxy, HTTP or HTTPS destination.""" - crawler = get_crawler(SingleRequestSpider, self.settings_dict) - with caplog.at_level(logging.DEBUG): - await crawler.crawl_async( - seed=mockserver.url("/status?n=200", is_secure=https_dest) - ) - assert isinstance(crawler.spider, SingleRequestSpider) - self._assert_got_response_code(200, caplog.text) - self._assert_headers(crawler.spider.meta["responses"][0].headers, https_dest) - - @pytest.mark.parametrize( - "https_dest", [False, True], ids=["HTTP dest", "HTTPS dest"] - ) - @pytest.mark.usefixtures("mitm_proxy_server_https") - @coroutine_test - async def test_https_proxy( - self, caplog: pytest.LogCaptureFixture, mockserver: MockServer, https_dest: bool - ) -> None: - """HTTPS proxy, HTTP or HTTPS destination.""" - if https_dest and not self.handler_supports_tls_in_tls: + def _maybe_skip(self, proxy_kind: str, https_dest: bool) -> None: + if proxy_kind == "socks5" and not self.handler_supports_socks: + pytest.skip("SOCKS proxies are not supported") + if ( + proxy_kind == "https" + and https_dest + and not self.handler_supports_tls_in_tls + ): pytest.skip("HTTPS proxies for HTTPS destinations are not supported") - crawler = get_crawler(SingleRequestSpider, self.settings_dict) - with caplog.at_level(logging.DEBUG): - await crawler.crawl_async( - seed=mockserver.url("/status?n=200", is_secure=https_dest) - ) - assert isinstance(crawler.spider, SingleRequestSpider) - self._assert_got_response_code(200, caplog.text) - self._assert_headers(crawler.spider.meta["responses"][0].headers, https_dest) + @pytest.mark.parametrize("proxy_server", PROXY_KINDS, indirect=True) @pytest.mark.parametrize( "https_dest", [False, True], ids=["HTTP dest", "HTTPS dest"] ) - @pytest.mark.usefixtures("mitm_proxy_server") @coroutine_test - async def test_http_proxy_auth_error( + async def test_proxy( self, caplog: pytest.LogCaptureFixture, - monkeypatch: pytest.MonkeyPatch, + proxy_server: str, mockserver: MockServer, https_dest: bool, ) -> None: - """HTTP proxy, HTTP or HTTPS destination, wrong proxy creds.""" + """HTTP/HTTPS/SOCKS5 proxy, HTTP or HTTPS destination.""" + self._maybe_skip(proxy_server, https_dest) + crawler = get_crawler(SingleRequestSpider, self.settings_dict) + with caplog.at_level(logging.DEBUG): + await crawler.crawl_async( + seed=mockserver.url("/status?n=200", is_secure=https_dest) + ) + assert isinstance(crawler.spider, SingleRequestSpider) + self._assert_got_response_code(200, caplog.text) + self._assert_headers(crawler.spider.meta["responses"][0].headers, https_dest) + + @pytest.mark.parametrize("proxy_server", PROXY_KINDS, indirect=True) + @pytest.mark.parametrize( + "https_dest", [False, True], ids=["HTTP dest", "HTTPS dest"] + ) + @coroutine_test + async def test_proxy_auth_error( + self, + caplog: pytest.LogCaptureFixture, + monkeypatch: pytest.MonkeyPatch, + proxy_server: str, + mockserver: MockServer, + https_dest: bool, + ) -> None: + """HTTP/HTTPS/SOCKS5 proxy, HTTP or HTTPS destination, wrong proxy creds.""" + self._maybe_skip(proxy_server, https_dest) envvar = "https_proxy" if https_dest else "http_proxy" monkeypatch.setenv(envvar, wrong_credentials(os.environ[envvar])) crawler = get_crawler(SimpleSpider, self.settings_dict) @@ -1396,20 +1396,28 @@ class TestMitmProxyBase(ABC): await crawler.crawl_async( mockserver.url("/status?n=200", is_secure=https_dest) ) - # The proxy returns a 407 error code but it does not reach the client; - # it just sees an exception. - self._assert_got_auth_exception(caplog.text) + if proxy_server == "socks5": + assert "DownloadConnectionRefusedError" in caplog.text + else: + # The proxy returns a 407 error code but it does not reach the + # client; it just sees an exception. + self._assert_got_auth_exception(caplog.text) + @pytest.mark.parametrize("proxy_server", PROXY_KINDS, indirect=True) @pytest.mark.parametrize( "https_dest", [False, True], ids=["HTTP dest", "HTTPS dest"] ) - @pytest.mark.usefixtures("mitm_proxy_server") @coroutine_test - async def test_dont_leak_proxy_authorization_header( - self, caplog: pytest.LogCaptureFixture, mockserver: MockServer, https_dest: bool + async def test_proxy_dont_leak_auth_header( + self, + caplog: pytest.LogCaptureFixture, + proxy_server: str, + mockserver: MockServer, + https_dest: bool, ) -> None: - """HTTP proxy, HTTP or HTTPS destination. Check that the auth header - is not sent to the destination.""" + """HTTP/HTTPS/SOCKS5 proxy, HTTP or HTTPS destination. Check that the + auth header is not sent to the destination.""" + self._maybe_skip(proxy_server, https_dest) request = Request(mockserver.url("/echo", is_secure=https_dest)) crawler = get_crawler(SingleRequestSpider, self.settings_dict) with caplog.at_level(logging.DEBUG): @@ -1420,48 +1428,36 @@ class TestMitmProxyBase(ABC): echo = json.loads(crawler.spider.meta["responses"][0].text) assert "Proxy-Authorization" not in echo["headers"] + @pytest.mark.parametrize("proxy_server", PROXY_KINDS, indirect=True) @pytest.mark.parametrize( "https_dest", [False, True], ids=["HTTP dest", "HTTPS dest"] ) - @pytest.mark.usefixtures("socks5_proxy_server") @coroutine_test - async def test_download_with_socks_proxy( - self, caplog: pytest.LogCaptureFixture, mockserver: MockServer, https_dest: bool - ) -> None: - """SOCKS5 proxy, HTTP or HTTPS destination.""" - if not self.handler_supports_socks: - pytest.skip("SOCKS proxies are not supported") - crawler = get_crawler(SingleRequestSpider, self.settings_dict) - with caplog.at_level(logging.DEBUG): - await crawler.crawl_async( - seed=mockserver.url("/status?n=200", is_secure=https_dest) - ) - assert isinstance(crawler.spider, SingleRequestSpider) - self._assert_got_response_code(200, caplog.text) - self._assert_headers(crawler.spider.meta["responses"][0].headers, https_dest) - - @pytest.mark.parametrize( - "https_dest", [False, True], ids=["HTTP dest", "HTTPS dest"] - ) - @pytest.mark.usefixtures("socks5_proxy_server") - @coroutine_test - async def test_socks_proxy_auth_error( + async def test_proxy_redirect( self, caplog: pytest.LogCaptureFixture, - monkeypatch: pytest.MonkeyPatch, + proxy_server: str, mockserver: MockServer, https_dest: bool, ) -> None: - if not self.handler_supports_socks: - pytest.skip("SOCKS proxies are not supported") - envvar = "https_proxy" if https_dest else "http_proxy" - monkeypatch.setenv(envvar, wrong_credentials(os.environ[envvar])) - crawler = get_crawler(SimpleSpider, self.settings_dict) + """HTTP/HTTPS/SOCKS5 proxy, HTTP or HTTPS destination, following a + redirect. Check that the redirected request still goes through the + proxy and doesn't lose the proxy auth. + """ + self._maybe_skip(proxy_server, https_dest) + crawler = get_crawler(SingleRequestSpider, self.settings_dict) with caplog.at_level(logging.DEBUG): await crawler.crawl_async( - mockserver.url("/status?n=200", is_secure=https_dest) + seed=mockserver.url("/redirect", is_secure=https_dest) ) - assert "DownloadConnectionRefusedError" in caplog.text + assert isinstance(crawler.spider, SingleRequestSpider) + assert crawler.spider.meta.get("failure") is None + responses = crawler.spider.meta.get("responses", []) + assert len(responses) == 1 + assert responses[0].status == 200 + assert responses[0].url == mockserver.url("/redirected", is_secure=https_dest) + self._assert_got_response_code(200, caplog.text) + self._assert_headers(responses[0].headers, https_dest) @staticmethod def _assert_headers(headers: Headers, https_dest: bool) -> None: From abbf3b95fcdf814c3926029ea80e4c33bd866c71 Mon Sep 17 00:00:00 2001 From: Lions-1 <128939687+Lions-1@users.noreply.github.com> Date: Fri, 19 Jun 2026 05:41:26 +0100 Subject: [PATCH 10/47] tests: integration coverage for memusage extension (#7017) --- scrapy/extensions/memusage.py | 6 +- tests/test_extension_memusage.py | 115 +++++++++++++++++++++++++++++++ tests/utils/__init__.py | 17 +++++ 3 files changed, 135 insertions(+), 3 deletions(-) create mode 100644 tests/test_extension_memusage.py diff --git a/scrapy/extensions/memusage.py b/scrapy/extensions/memusage.py index 01af02ba3..1444c8941 100644 --- a/scrapy/extensions/memusage.py +++ b/scrapy/extensions/memusage.py @@ -113,7 +113,7 @@ class MemoryUsage: {"memusage": mem}, extra={"crawler": self.crawler}, ) - if self.notify_mails: + if self.notify_mails: # pragma: no cover subj = ( f"{self.crawler.settings['BOT_NAME']} terminated: " f"memory usage exceeded {mem}MiB at {socket.gethostname()}" @@ -146,7 +146,7 @@ class MemoryUsage: {"memusage": mem}, extra={"crawler": self.crawler}, ) - if self.notify_mails: + if self.notify_mails: # pragma: no cover subj = ( f"{self.crawler.settings['BOT_NAME']} warning: " f"memory usage reached {mem}MiB at {socket.gethostname()}" @@ -155,7 +155,7 @@ class MemoryUsage: self.crawler.stats.set_value("memusage/warning_notified", 1) self.warned = True - def _send_report(self, rcpts: list[str], subject: str) -> None: + def _send_report(self, rcpts: list[str], subject: str) -> None: # pragma: no cover """send notification mail with some additional useful info""" assert self.crawler.engine assert self.crawler.stats diff --git a/tests/test_extension_memusage.py b/tests/test_extension_memusage.py new file mode 100644 index 000000000..a474725d8 --- /dev/null +++ b/tests/test_extension_memusage.py @@ -0,0 +1,115 @@ +from __future__ import annotations + +import logging +import sys + +import pytest + +from scrapy import signals +from scrapy.core import engine as engine_mod +from scrapy.exceptions import NotConfigured +from scrapy.extensions import memusage as memusage_mod +from scrapy.extensions.memusage import MemoryUsage +from scrapy.spiders import Spider +from scrapy.utils.test import get_crawler +from tests.utils import OneShotLoop +from tests.utils.decorators import coroutine_test + +# MemoryUsage relies on the stdlib 'resource' module (not available on Windows) +pytestmark = pytest.mark.skipif( + sys.platform.startswith("win"), + reason="MemoryUsage extension not available on Windows", +) + + +MB = 1024 * 1024 + + +class _LoopSpider(Spider): + name = "loop-data-spider" + + def __init__(self, url: str, loops: int = 60, **kw): + super().__init__(**kw) + self.url = url + self.loops = loops + self.start_urls = [url] + + def parse(self, response): + count = response.meta.get("count", 0) + if count + 1 < self.loops: + yield response.follow( + self.url, callback=self.parse, meta={"count": count + 1} + ) + + +def test_memusage_disabled() -> None: + settings = { + "MEMUSAGE_ENABLED": False, + } + with pytest.raises(NotConfigured): + MemoryUsage.from_crawler(get_crawler(settings_dict=settings)) + + +@coroutine_test +async def test_memusage_limit_closes_spider_with_reason_and_error_log( + caplog: pytest.LogCaptureFixture, monkeypatch: pytest.MonkeyPatch +) -> None: + settings = { + "MEMUSAGE_LIMIT_MB": 10, + "MEMUSAGE_CHECK_INTERVAL_SECONDS": 0.01, + "TELNETCONSOLE_ENABLED": False, + "LOG_LEVEL": "INFO", + } + + # Avoid background LoopingCall that can log after the test finishes. + monkeypatch.setattr(memusage_mod, "create_looping_call", OneShotLoop) + # Avoid engine start/stop races (the extension stops the engine in engine_started). + monkeypatch.setattr(engine_mod, "create_looping_call", OneShotLoop) + monkeypatch.setattr(MemoryUsage, "get_virtual_size", lambda _: 250 * MB) + + crawler = get_crawler(spidercls=_LoopSpider, settings_dict=settings) + + with caplog.at_level(logging.ERROR, logger="scrapy.extensions.memusage"): + await crawler.crawl_async(url="data:,", loops=100) + + assert crawler.stats + assert crawler.stats.get_value("memusage/limit_reached") == 1 + assert crawler.stats.get_value("finish_reason") == "memusage_exceeded" + assert any( + "memory usage exceeded" in r.getMessage().lower() for r in caplog.records + ) + + +@coroutine_test +async def test_memusage_warning_logs_but_allows_normal_finish( + caplog: pytest.LogCaptureFixture, monkeypatch: pytest.MonkeyPatch +) -> None: + settings = { + "MEMUSAGE_WARNING_MB": 50, + "MEMUSAGE_LIMIT_MB": 0, # no hard limit + "MEMUSAGE_CHECK_INTERVAL_SECONDS": 0.01, + "TELNETCONSOLE_ENABLED": False, + "LOG_LEVEL": "INFO", + } + + # Avoid background LoopingCall that can log after the test finishes. + monkeypatch.setattr(memusage_mod, "create_looping_call", OneShotLoop) + monkeypatch.setattr(MemoryUsage, "get_virtual_size", lambda self: 75 * MB) + + crawler = get_crawler(spidercls=_LoopSpider, settings_dict=settings) + + warning_signals: list[int] = [] + + def on_warning_reached() -> None: + warning_signals.append(1) + + crawler.signals.connect(on_warning_reached, signal=signals.memusage_warning_reached) + + with caplog.at_level(logging.WARNING, logger="scrapy.extensions.memusage"): + await crawler.crawl_async(url="data:,", loops=60) + + assert warning_signals == [1] + assert crawler.stats + assert crawler.stats.get_value("memusage/warning_reached") == 1 + assert crawler.stats.get_value("finish_reason") == "finished" + assert any("memory usage reached" in r.getMessage().lower() for r in caplog.records) diff --git a/tests/utils/__init__.py b/tests/utils/__init__.py index 5b7848d54..73c0e3776 100644 --- a/tests/utils/__init__.py +++ b/tests/utils/__init__.py @@ -1,5 +1,6 @@ import asyncio import os +from collections.abc import Callable from pathlib import Path from twisted.internet.defer import Deferred @@ -31,3 +32,19 @@ def get_script_run_env() -> dict[str, str]: env = os.environ.copy() env["PYTHONPATH"] = pythonpath return env + + +class OneShotLoop: + """Test stub for create_looping_call: run once immediately, no background task.""" + + def __init__(self, func: Callable[[], None]): + self.func = func + self.running = False + + def start(self, _interval: float, now: bool = True) -> None: + self.running = True + if now: + self.func() + + def stop(self) -> None: + self.running = False From e5e48883b580b2fcfe8d30b301675f1f3ef3d67c Mon Sep 17 00:00:00 2001 From: Vishal Gawade Date: Fri, 19 Jun 2026 00:49:32 -0400 Subject: [PATCH 11/47] Deprecate ScrapyCommand.help() (#7633) Co-authored-by: Vishal Gawade --- scrapy/commands/__init__.py | 19 ++++++++++++---- tests/test_commands.py | 45 +++++++++++++++++++++++++++++++++++++ 2 files changed, 60 insertions(+), 4 deletions(-) diff --git a/scrapy/commands/__init__.py b/scrapy/commands/__init__.py index 19b6f6681..598e8060e 100644 --- a/scrapy/commands/__init__.py +++ b/scrapy/commands/__init__.py @@ -16,6 +16,8 @@ from twisted.python import failure from scrapy.exceptions import ScrapyDeprecationWarning, UsageError from scrapy.utils.conf import arglist_to_dict, feed_process_params_from_cli +from scrapy.utils.deprecate import method_is_overridden +from scrapy.utils.python import global_object_name if TYPE_CHECKING: from collections.abc import Iterable @@ -36,6 +38,14 @@ class ScrapyCommand(ABC): def __init__(self) -> None: self.settings: Settings | None = None # set in scrapy.cmdline + if method_is_overridden(self.__class__, ScrapyCommand, "help"): + warnings.warn( + "The ScrapyCommand.help() method is deprecated and overriding " + f"it, as the {global_object_name(self.__class__)} class does, " + "has no effect; override long_desc() instead.", + ScrapyDeprecationWarning, + stacklevel=2, + ) def set_crawler(self, crawler: Crawler) -> None: # pragma: no cover warnings.warn( @@ -68,10 +78,11 @@ class ScrapyCommand(ABC): return self.short_desc() def help(self) -> str: - """An extensive help for the command. It will be shown when using the - "help" command. It can contain newlines since no post-formatting will - be applied to its contents. - """ + warnings.warn( + "ScrapyCommand.help() is deprecated, use long_desc() instead.", + ScrapyDeprecationWarning, + stacklevel=2, + ) return self.long_desc() def add_options(self, parser: argparse.ArgumentParser) -> None: diff --git a/tests/test_commands.py b/tests/test_commands.py index 0657f393c..d7b7a9ff2 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -12,6 +12,7 @@ import pytest import scrapy from scrapy.cmdline import _pop_command_name, _print_unknown_command_msg from scrapy.commands import ScrapyCommand, ScrapyHelpFormatter, view +from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.settings import Settings from scrapy.utils.reactor import _asyncio_reactor_path from tests.utils.cmdline import call, proc @@ -28,6 +29,50 @@ class EmptyCommand(ScrapyCommand): pass +class TestHelpDeprecation: + def test_calling_help_is_deprecated(self) -> None: + command = EmptyCommand() + with pytest.warns( + ScrapyDeprecationWarning, + match=r"ScrapyCommand\.help\(\) is deprecated, use long_desc\(\) instead\.", + ): + result = command.help() + # help() still delegates to long_desc() for backward compatibility. + assert result == command.long_desc() + + def test_overriding_help_is_deprecated(self) -> None: + class HelpCommand(ScrapyCommand): + def short_desc(self) -> str: + return "" + + def run(self, args: list[str], opts: argparse.Namespace) -> None: + pass + + def help(self) -> str: + return "custom help" + + with pytest.warns( + ScrapyDeprecationWarning, + match=r"The ScrapyCommand\.help\(\) method is deprecated and " + r"overriding it, as the .*HelpCommand class does, has no effect; " + r"override long_desc\(\) instead\.", + ): + HelpCommand() + + def test_not_overriding_help_does_not_warn(self, recwarn) -> None: + # Commands that do not override help() must not emit the + # override-deprecation warning when instantiated, including subclasses + # several levels below ScrapyCommand (as the built-in commands are). + class SubCommand(EmptyCommand): + pass + + EmptyCommand() + SubCommand() + assert not [ + w for w in recwarn.list if issubclass(w.category, ScrapyDeprecationWarning) + ] + + class TestCommandSettings: def setup_method(self): self.command = EmptyCommand() From 3f3cb885eddf34fc6fd76a1acbeff601cbc6f506 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 19 Jun 2026 09:55:02 +0500 Subject: [PATCH 12/47] Address warnings in tests. (#7637) --- scrapy/contracts/__init__.py | 4 ++++ scrapy/utils/misc.py | 9 ++------ tests/ignores.txt | 2 ++ tests/test_crawl.py | 3 +++ tests/test_downloader_handlers_http_base.py | 3 +++ tests/test_extension_statsmailer.py | 23 +++++++++++++------- tests/test_http2_client_protocol.py | 3 +++ tests/test_spider_sitemap.py | 5 +---- tests/test_spider_start.py | 13 +++-------- tests/test_spidermiddleware_process_start.py | 5 +---- tests/test_utils_project.py | 8 ++----- 11 files changed, 39 insertions(+), 39 deletions(-) diff --git a/scrapy/contracts/__init__.py b/scrapy/contracts/__init__.py index c0da7dfa4..ebbdf1b98 100644 --- a/scrapy/contracts/__init__.py +++ b/scrapy/contracts/__init__.py @@ -51,6 +51,8 @@ class Contract: results.addSuccess(self.testcase_pre) cb_result = cb(response, **cb_kwargs) if isinstance(cb_result, (AsyncGenerator, CoroutineType)): + if isinstance(cb_result, CoroutineType): + cb_result.close() raise TypeError("Contracts don't support async callbacks") return list(cast("Iterable[Any]", iterate_spider_output(cb_result))) @@ -67,6 +69,8 @@ class Contract: def wrapper(response: Response, **cb_kwargs: Any) -> list[Any]: cb_result = cb(response, **cb_kwargs) if isinstance(cb_result, (AsyncGenerator, CoroutineType)): + if isinstance(cb_result, CoroutineType): + cb_result.close() raise TypeError("Contracts don't support async callbacks") output = list(cast("Iterable[Any]", iterate_spider_output(cb_result))) try: diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index 47568e656..20e7cb381 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -135,14 +135,9 @@ def walk_modules(path: str) -> list[ModuleType]: # pragma: no cover return list(walk_modules_iter(path)) -def md5sum(file: IO[bytes]) -> str: +def md5sum(file: IO[bytes]) -> str: # pragma: no cover """Calculate the md5 checksum of a file-like object without reading its - whole content in memory. - - >>> from io import BytesIO - >>> md5sum(BytesIO(b'file content to hash')) - '784406af91dd5a54fbb9c84c2236595a' - """ + whole content in memory.""" warnings.warn( ( "The scrapy.utils.misc.md5sum function is deprecated and will be " diff --git a/tests/ignores.txt b/tests/ignores.txt index 94edcf186..3717bbc95 100644 --- a/tests/ignores.txt +++ b/tests/ignores.txt @@ -1 +1,3 @@ +scrapy/core/downloader/handlers/http.py scrapy/extensions/statsmailer.py +scrapy/mail.py diff --git a/tests/test_crawl.py b/tests/test_crawl.py index 85d98f847..66b15bfdd 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -715,6 +715,9 @@ class TestCrawlSpider: assert isinstance(crawler.spider, SingleRequestSpider) assert crawler.spider.meta["responses"][0].certificate is None + @pytest.mark.filterwarnings( + r"ignore:.*You should use cryptography's X\.509 APIs:DeprecationWarning" + ) @pytest.mark.parametrize( "url", [ diff --git a/tests/test_downloader_handlers_http_base.py b/tests/test_downloader_handlers_http_base.py index 2ae9206a0..c4a8193c8 100644 --- a/tests/test_downloader_handlers_http_base.py +++ b/tests/test_downloader_handlers_http_base.py @@ -1148,6 +1148,9 @@ class TestHttpWithCrawlerBase(ABC): reason = crawler.spider.meta["close_reason"] # type: ignore[attr-defined] assert reason == "finished" + @pytest.mark.filterwarnings( + r"ignore:.*You should use cryptography's X\.509 APIs:DeprecationWarning" + ) @coroutine_test async def test_response_ssl_certificate(self, mockserver: MockServer) -> None: if not self.is_secure: diff --git a/tests/test_extension_statsmailer.py b/tests/test_extension_statsmailer.py index 28db389c3..4d208a1ea 100644 --- a/tests/test_extension_statsmailer.py +++ b/tests/test_extension_statsmailer.py @@ -1,20 +1,27 @@ +import warnings from unittest.mock import MagicMock import pytest from scrapy import signals -from scrapy.exceptions import NotConfigured +from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning from scrapy.signalmanager import SignalManager from scrapy.statscollectors import StatsCollector from scrapy.utils.spider import DefaultSpider -pytestmark = pytest.mark.filterwarnings( - "ignore:The scrapy.extensions.statsmailer module is deprecated:scrapy.exceptions.ScrapyDeprecationWarning", - "ignore:The scrapy.mail module is deprecated:scrapy.exceptions.ScrapyDeprecationWarning", -) - -from scrapy.extensions import statsmailer # noqa: E402 -from scrapy.mail import MailSender # noqa: E402 +with warnings.catch_warnings(): + warnings.filterwarnings( + "ignore", + r"The scrapy\.extensions\.statsmailer module is deprecated", + ScrapyDeprecationWarning, + ) + warnings.filterwarnings( + "ignore", + r"The scrapy\.mail module is deprecated", + ScrapyDeprecationWarning, + ) + from scrapy.extensions import statsmailer + from scrapy.mail import MailSender @pytest.fixture diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py index 28f306e31..3c1347fd3 100644 --- a/tests/test_http2_client_protocol.py +++ b/tests/test_http2_client_protocol.py @@ -669,6 +669,9 @@ class TestHttps2ClientProtocol: response = await make_request(client, request) assert response.status == status + @pytest.mark.filterwarnings( + r"ignore:.*You should use cryptography's X\.509 APIs:DeprecationWarning" + ) @deferred_f_from_coro_f async def test_response_has_correct_certificate_ip_address( self, diff --git a/tests/test_spider_sitemap.py b/tests/test_spider_sitemap.py index 57c209615..0af99ab6d 100644 --- a/tests/test_spider_sitemap.py +++ b/tests/test_spider_sitemap.py @@ -2,7 +2,6 @@ from __future__ import annotations import gzip import re -import warnings from datetime import datetime from io import BytesIO from logging import WARNING @@ -429,9 +428,7 @@ Sitemap: /sitemap-relative-url.xml crawler = get_crawler(TestSpider) spider = TestSpider.from_crawler(crawler) - with warnings.catch_warnings(): - warnings.simplefilter("error") - requests = [request async for request in spider.start()] + requests = [request async for request in spider.start()] assert len(requests) == 1 request = requests[0] diff --git a/tests/test_spider_start.py b/tests/test_spider_start.py index aef2093ac..9257e0232 100644 --- a/tests/test_spider_start.py +++ b/tests/test_spider_start.py @@ -1,6 +1,5 @@ from __future__ import annotations -import warnings from asyncio import sleep from typing import Any @@ -45,9 +44,7 @@ class TestMain: async def parse(self, response): yield ITEM_A - with warnings.catch_warnings(): - warnings.simplefilter("error") - await self._test_spider(TestSpider, [ITEM_A]) + await self._test_spider(TestSpider, [ITEM_A]) @coroutine_test async def test_start(self): @@ -57,9 +54,7 @@ class TestMain: async def start(self): yield ITEM_A - with warnings.catch_warnings(): - warnings.simplefilter("error") - await self._test_spider(TestSpider, [ITEM_A]) + await self._test_spider(TestSpider, [ITEM_A]) @coroutine_test async def test_start_subclass(self): @@ -70,9 +65,7 @@ class TestMain: class TestSpider(BaseSpider): name = "test" - with warnings.catch_warnings(): - warnings.simplefilter("error") - await self._test_spider(TestSpider, [ITEM_A]) + await self._test_spider(TestSpider, [ITEM_A]) async def _test_start(self, start_, expected_items=None): class TestSpider(Spider): diff --git a/tests/test_spidermiddleware_process_start.py b/tests/test_spidermiddleware_process_start.py index 21df73a65..67ce3a920 100644 --- a/tests/test_spidermiddleware_process_start.py +++ b/tests/test_spidermiddleware_process_start.py @@ -1,4 +1,3 @@ -import warnings from asyncio import sleep import pytest @@ -76,9 +75,7 @@ class TestMain: @coroutine_test async def test_modern_mw_modern_spider(self): - with warnings.catch_warnings(): - warnings.simplefilter("error") - await self._test_wrap(ModernWrapSpiderMiddleware, ModernWrapSpider) + await self._test_wrap(ModernWrapSpiderMiddleware, ModernWrapSpider) async def _test_sleep(self, spider_middlewares): class TestSpider(Spider): diff --git a/tests/test_utils_project.py b/tests/test_utils_project.py index 20a3d940c..5333a55cb 100644 --- a/tests/test_utils_project.py +++ b/tests/test_utils_project.py @@ -1,5 +1,4 @@ import os -import warnings from pathlib import Path import pytest @@ -41,11 +40,8 @@ class TestGetProjectSettings: envvars = { "SCRAPY_SETTINGS_MODULE": value, } - with warnings.catch_warnings(): - warnings.simplefilter("error") - with set_environ(**envvars): - settings = get_project_settings() - + with set_environ(**envvars): + settings = get_project_settings() assert settings.get("SETTINGS_MODULE") == value def test_invalid_envvar(self): From 699c93f6b2623df87c56ac486cb2cd44717a9cbf Mon Sep 17 00:00:00 2001 From: Adrian Date: Fri, 19 Jun 2026 11:22:03 +0200 Subject: [PATCH 13/47] Complete test coverage for linkextractors (#7639) * Complete test coverage for linkextractors * pylint: disable use-implicit-booleaness-not-comparison --- pyproject.toml | 1 + tests/test_linkextractors.py | 35 ++++++++++++++++++++++++++++++++- tests/test_request_cb_kwargs.py | 4 +--- tests/test_settings/__init__.py | 2 +- tests/test_spider.py | 2 +- tests/test_utils_python.py | 2 +- 6 files changed, 39 insertions(+), 7 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 2d857dc6d..6b0c561f8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -230,6 +230,7 @@ disable = [ "undefined-variable", "unused-argument", "unused-variable", + "use-implicit-booleaness-not-comparison", "useless-import-alias", # used as a hint to mypy "useless-return", # https://github.com/pylint-dev/pylint/issues/6530 "wrong-import-position", diff --git a/tests/test_linkextractors.py b/tests/test_linkextractors.py index 15d358d2a..063b92dac 100644 --- a/tests/test_linkextractors.py +++ b/tests/test_linkextractors.py @@ -9,7 +9,7 @@ from w3lib import __version__ as w3lib_version from scrapy.http import HtmlResponse, XmlResponse from scrapy.link import Link -from scrapy.linkextractors.lxmlhtml import LxmlLinkExtractor +from scrapy.linkextractors.lxmlhtml import LxmlLinkExtractor, LxmlParserLinkExtractor from tests import get_testdata @@ -837,3 +837,36 @@ class TestLxmlLinkExtractor(Base.TestLinkExtractorBase): def test_link_allowed_is_false_with_missing_url_prefix(self): bad_link = Link("should_have_prefix.example") assert not LxmlLinkExtractor()._link_allowed(bad_link) + + +class TestLxmlParserLinkExtractor: + def test_extract_links(self): + html = b'Link' + response = HtmlResponse("http://example.com/", body=html) + lx = LxmlParserLinkExtractor() + assert lx.extract_links(response) == [ + Link(url="http://example.com/page.html", text="Link", nofollow=False), + ] + + def test_strip_false(self): + # With strip=False, trailing whitespace on a relative href survives urljoin + # and is visible to process_value (safe_url_string cleans it up afterward). + # Here process_value rejects URLs that still carry trailing whitespace, + # demonstrating the difference from strip=True. + def reject_trailing_whitespace(url): + return None if url != url.rstrip() else url + + html = b'Link' + response = HtmlResponse("http://example.com/", body=html) + + lx_strip = LxmlParserLinkExtractor( + strip=True, process=reject_trailing_whitespace + ) + assert lx_strip.extract_links(response) == [ + Link(url="http://example.com/page.html", text="Link", nofollow=False), + ] + + lx_no_strip = LxmlParserLinkExtractor( + strip=False, process=reject_trailing_whitespace + ) + assert lx_no_strip.extract_links(response) == [] diff --git a/tests/test_request_cb_kwargs.py b/tests/test_request_cb_kwargs.py index 0d96e1d88..ee9d2ed51 100644 --- a/tests/test_request_cb_kwargs.py +++ b/tests/test_request_cb_kwargs.py @@ -102,9 +102,7 @@ class KeywordArgumentsSpider(MockServerSpider): self.checks.append(kwargs["callback"] == "some_callback") self.crawler.stats.inc_value("boolean_checks", 3) elif response.url.endswith("/general_without"): - self.checks.append( - kwargs == {} # pylint: disable=use-implicit-booleaness-not-comparison - ) + self.checks.append(kwargs == {}) self.crawler.stats.inc_value("boolean_checks") def parse_no_kwargs(self, response): diff --git a/tests/test_settings/__init__.py b/tests/test_settings/__init__.py index 7b3c52d65..9599ed6c9 100644 --- a/tests/test_settings/__init__.py +++ b/tests/test_settings/__init__.py @@ -1,4 +1,4 @@ -# pylint: disable=unsubscriptable-object,unsupported-membership-test,use-implicit-booleaness-not-comparison +# pylint: disable=unsubscriptable-object,unsupported-membership-test # (too many false positives) import logging diff --git a/tests/test_spider.py b/tests/test_spider.py index 23efed77a..526cc8f23 100644 --- a/tests/test_spider.py +++ b/tests/test_spider.py @@ -22,7 +22,7 @@ class TestSpider: def test_base_spider(self): spider = self.spider_class("example.com") assert spider.name == "example.com" - assert spider.start_urls == [] # pylint: disable=use-implicit-booleaness-not-comparison + assert spider.start_urls == [] def test_spider_args(self): """``__init__`` method arguments are assigned to spider attributes""" diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py index e8fe45749..2e8047e2d 100644 --- a/tests/test_utils_python.py +++ b/tests/test_utils_python.py @@ -175,7 +175,7 @@ def test_get_func_args(): assert get_func_args(partial_f2) == ["a", "c"] assert get_func_args(partial_f3) == ["c"] assert get_func_args(cal) == ["a", "b", "c"] - assert get_func_args(object) == [] # pylint: disable=use-implicit-booleaness-not-comparison + assert get_func_args(object) == [] assert get_func_args(str.split, stripself=True) == ["sep", "maxsplit"] assert get_func_args(" ".join, stripself=True) == ["iterable"] From 7b3f88f8abcbf45682140ae35456810b49065ccf Mon Sep 17 00:00:00 2001 From: Adrian Date: Fri, 19 Jun 2026 15:28:32 +0200 Subject: [PATCH 14/47] Improve test coverage for pqueues.py (#7640) --- tests/test_pqueues.py | 52 ++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 51 insertions(+), 1 deletion(-) diff --git a/tests/test_pqueues.py b/tests/test_pqueues.py index 7be9241b9..6c6a6584a 100644 --- a/tests/test_pqueues.py +++ b/tests/test_pqueues.py @@ -8,7 +8,7 @@ from scrapy.core.downloader import Downloader from scrapy.http.request import Request from scrapy.pqueues import DownloaderAwarePriorityQueue, ScrapyPriorityQueue from scrapy.spiders import Spider -from scrapy.squeues import FifoMemoryQueue +from scrapy.squeues import FifoMemoryQueue, PickleFifoDiskQueue from scrapy.utils.misc import build_from_crawler, load_object from scrapy.utils.test import get_crawler from tests.test_scheduler import MockDownloader @@ -76,6 +76,29 @@ class TestPriorityQueue: assert queue.pop().url == req3.url assert not queue.close() + def test_init_prios_with_start_queue(self): + temp_dir = tempfile.mkdtemp() + queue = ScrapyPriorityQueue.from_crawler( + self.crawler, + PickleFifoDiskQueue, + temp_dir, + start_queue_cls=PickleFifoDiskQueue, + ) + req = Request("https://example.org/", meta={"is_start_request": True}) + queue.push(req) + startprios = queue.close() + + queue2 = ScrapyPriorityQueue.from_crawler( + self.crawler, + PickleFifoDiskQueue, + temp_dir, + startprios, + start_queue_cls=PickleFifoDiskQueue, + ) + assert len(queue2) == 1 + assert queue2.pop().url == req.url + queue2.close() + def test_queue_push_pop_priorities(self): temp_dir = tempfile.mkdtemp() queue = ScrapyPriorityQueue.from_crawler( @@ -207,6 +230,33 @@ class TestDownloaderAwarePriorityQueue: assert slots == ["slot-a", "slot-b", "slot-c", "slot-a"] + def test_pop_prefers_slot_with_fewer_active_downloads(self): + downloader = self.queue._downloader_interface.downloader + + req_a = Request("https://example.org/a") + req_a.meta[Downloader.DOWNLOAD_SLOT] = "slot-a" + req_b = Request("https://example.org/b") + req_b.meta[Downloader.DOWNLOAD_SLOT] = "slot-b" + req_c = Request("https://example.org/c") + req_c.meta[Downloader.DOWNLOAD_SLOT] = "slot-c" + + for req in (req_a, req_b, req_c): + self.queue.push(req) + + downloader.increment("slot-a") + downloader.increment("slot-c") + + popped = self.queue.pop() + assert popped.url == req_b.url + + def test_contains(self): + req = Request("https://example.org/") + req.meta[Downloader.DOWNLOAD_SLOT] = "example-slot" + assert "example-slot" not in self.queue + self.queue.push(req) + assert "example-slot" in self.queue + assert "other-slot" not in self.queue + @pytest.mark.parametrize( ("input_", "output"), From d2842a205c7986070f77b1f4033504a0fd95e5fe Mon Sep 17 00:00:00 2001 From: Adrian Date: Fri, 19 Jun 2026 16:59:25 +0200 Subject: [PATCH 15/47] Improve coverage statscollectors (#7641) --- scrapy/__main__.py | 1 + tests/test_stats.py | 9 ++++++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/scrapy/__main__.py b/scrapy/__main__.py index 697b9b1e9..17b1aa538 100644 --- a/scrapy/__main__.py +++ b/scrapy/__main__.py @@ -1,3 +1,4 @@ +# pragma: no file cover from scrapy.cmdline import execute if __name__ == "__main__": diff --git a/tests/test_stats.py b/tests/test_stats.py index 27af18bb1..6e6aa0cc4 100644 --- a/tests/test_stats.py +++ b/tests/test_stats.py @@ -94,8 +94,13 @@ class TestStatsCollector: assert stats.get_value("test2") == 35 stats.min_value("test4", 7) assert stats.get_value("test4") == 7 + stats.set_stats({"replaced": "stats"}) + assert stats.get_stats() == {"replaced": "stats"} + stats.clear_stats() + assert stats.get_stats() == {} - def test_dummy_collector(self, crawler: Crawler) -> None: + def test_dummy_collector(self) -> None: + crawler = get_crawler(Spider, {"STATS_DUMP": False}) stats = DummyStatsCollector(crawler) assert stats.get_stats() == {} assert stats.get_value("anything") is None @@ -104,9 +109,11 @@ class TestStatsCollector: stats.inc_value("v1") stats.max_value("v2", 100) stats.min_value("v3", 100) + stats.set_stats({"key": "val"}) stats.open_spider() stats.set_value("test", "value") assert stats.get_stats() == {} + stats.close_spider() def test_deprecated_spider_arg(self, crawler: Crawler, spider: Spider) -> None: stats = StatsCollector(crawler) From 6393858c7e45ed5393fe98b570505f04855664c1 Mon Sep 17 00:00:00 2001 From: Adrian Date: Fri, 19 Jun 2026 20:26:54 +0200 Subject: [PATCH 16/47] Improve coverage resolver (#7642) * Improve test coverage for resolver.py * Make the Twisted code more readable --- scrapy/resolver.py | 2 +- tests/test_resolver.py | 55 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 56 insertions(+), 1 deletion(-) create mode 100644 tests/test_resolver.py diff --git a/scrapy/resolver.py b/scrapy/resolver.py index f5f00ab0f..270a7fbf5 100644 --- a/scrapy/resolver.py +++ b/scrapy/resolver.py @@ -75,7 +75,7 @@ class HostResolution: def __init__(self, name: str): self.name: str = name - def cancel(self) -> None: + def cancel(self) -> None: # pragma: no cover raise NotImplementedError diff --git a/tests/test_resolver.py b/tests/test_resolver.py new file mode 100644 index 000000000..83fc8693b --- /dev/null +++ b/tests/test_resolver.py @@ -0,0 +1,55 @@ +from __future__ import annotations + +from unittest.mock import Mock + +import pytest + +from scrapy.resolver import CachingHostnameResolver, CachingThreadedResolver, dnscache +from scrapy.utils.defer import maybe_deferred_to_future +from scrapy.utils.test import get_crawler +from tests.utils.decorators import coroutine_test + + +@pytest.fixture(autouse=True) +def reset_dnscache(): + original_limit = dnscache.limit + dnscache.clear() + yield + dnscache.clear() + dnscache.limit = original_limit + + +def test_caching_threaded_resolver_dnscache_disabled(): + crawler = get_crawler(settings_dict={"DNSCACHE_ENABLED": False}) + CachingThreadedResolver.from_crawler(crawler, Mock()) + assert dnscache.limit == 0 + + +@coroutine_test +async def test_caching_threaded_resolver_getHostByName_cache_hit(): + resolver = CachingThreadedResolver(Mock(), cache_size=10, timeout=5.0) + dnscache["example.com"] = "1.2.3.4" + + result = await maybe_deferred_to_future(resolver.getHostByName("example.com")) + assert result == "1.2.3.4" + + +def test_caching_hostname_resolver_dnscache_disabled(): + crawler = get_crawler(settings_dict={"DNSCACHE_ENABLED": False}) + CachingHostnameResolver.from_crawler(crawler, Mock()) + assert dnscache.limit == 0 + + +def test_caching_hostname_resolver_no_addresses_not_cached(): + def fake_resolve(receiver, *_): + receiver.resolutionBegan(Mock()) + receiver.resolutionComplete() + return receiver + + reactor = Mock() + reactor.nameResolver.resolveHostName.side_effect = fake_resolve + + resolver = CachingHostnameResolver(reactor, cache_size=10) + resolver.resolveHostName(Mock(), "example.com") + + assert "example.com" not in dnscache From c9f952c2584f490cd2e5c843980212abc67c2971 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sat, 20 Jun 2026 00:04:34 +0500 Subject: [PATCH 17/47] Refactor and improve catching warnings in tests. (#7643) --- scrapy/spidermiddlewares/referer.py | 6 +- scrapy/spiders/crawl.py | 2 + tests/test_crawler.py | 5 +- tests/test_dupefilters.py | 13 +- tests/test_http_request.py | 6 +- tests/test_http_request_json.py | 41 +++-- tests/test_pipeline_files.py | 23 ++- tests/test_pipeline_media.py | 32 ++-- tests/test_scheduler.py | 7 +- tests/test_scrapy__getattr__.py | 21 +-- tests/test_settings/__init__.py | 14 +- tests/test_spider_crawl.py | 24 +-- tests/test_spiderloader/__init__.py | 105 +++++-------- tests/test_spidermiddleware_referer.py | 64 ++++---- tests/test_utils_curl.py | 8 +- tests/test_utils_datatypes.py | 15 +- tests/test_utils_deprecate.py | 148 ++++++++---------- ...t_return_with_argument_inside_generator.py | 84 +++------- tests/test_utils_sitemap.py | 11 +- 19 files changed, 275 insertions(+), 354 deletions(-) diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py index 6c5acf0de..2c9452174 100644 --- a/scrapy/spidermiddlewares/referer.py +++ b/scrapy/spidermiddlewares/referer.py @@ -11,7 +11,7 @@ from typing import TYPE_CHECKING, cast from urllib.parse import urlparse from warnings import warn -from scrapy.exceptions import NotConfigured +from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning from scrapy.http import Request, Response from scrapy.spidermiddlewares.base import BaseSpiderMiddleware from scrapy.utils.misc import load_object @@ -349,7 +349,7 @@ class RefererMiddleware(BaseSpiderMiddleware): response = kwargs.pop("resp_or_url") warn( "Passing 'resp_or_url' is deprecated, use 'response' instead.", - DeprecationWarning, + ScrapyDeprecationWarning, stacklevel=2, ) if response is None: @@ -360,7 +360,7 @@ class RefererMiddleware(BaseSpiderMiddleware): warn( "Passing a response URL to RefererMiddleware.policy() instead " "of a Response object is deprecated.", - DeprecationWarning, + ScrapyDeprecationWarning, stacklevel=2, ) allow_import_path = True diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.py index d2da31f35..373c0b8b5 100644 --- a/scrapy/spiders/crawl.py +++ b/scrapy/spiders/crawl.py @@ -110,6 +110,7 @@ class CrawlSpider(Spider): "deprecated: it will be removed in future Scrapy releases. " "Please override the CrawlSpider.parse_with_rules method " "instead.", + ScrapyDeprecationWarning, stacklevel=2, ) @@ -199,6 +200,7 @@ class CrawlSpider(Spider): "The CrawlSpider._parse_response method is deprecated: " "it will be removed in future Scrapy releases. " "Please use the CrawlSpider.parse_with_rules method instead.", + ScrapyDeprecationWarning, stacklevel=2, ) return self.parse_with_rules(response, callback, cb_kwargs, follow) diff --git a/tests/test_crawler.py b/tests/test_crawler.py index 31d195c4b..c646aea4b 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -3,7 +3,6 @@ from __future__ import annotations import asyncio import logging import re -import warnings from pathlib import Path from typing import Any, ClassVar @@ -88,9 +87,7 @@ class TestCrawler(TestBaseCrawler): self.assertOptionIsDefault(crawler.settings, "RETRY_ENABLED") def test_crawler_accepts_None(self) -> None: - with warnings.catch_warnings(): - warnings.simplefilter("ignore", ScrapyDeprecationWarning) - crawler = Crawler(DefaultSpider) + crawler = Crawler(DefaultSpider) self.assertOptionIsDefault(crawler.settings, "RETRY_ENABLED") def test_crawler_rejects_spider_objects(self) -> None: diff --git a/tests/test_dupefilters.py b/tests/test_dupefilters.py index 412a59fcd..5d79691b2 100644 --- a/tests/test_dupefilters.py +++ b/tests/test_dupefilters.py @@ -3,8 +3,8 @@ import shutil import sys import tempfile from pathlib import Path -from warnings import catch_warnings +import pytest from testfixtures import LogCapture from scrapy.core.scheduler import Scheduler @@ -260,11 +260,8 @@ class TestBaseDupeFilter: dupefilter = _get_dupefilter( settings={"DUPEFILTER_CLASS": BaseDupeFilter}, ) - with catch_warnings(record=True) as warning_list: + with pytest.warns( + ScrapyDeprecationWarning, + match=r"Calling BaseDupeFilter\.log\(\) is deprecated.", + ): dupefilter.log(None, None) - assert len(warning_list) == 1 - assert ( - str(warning_list[0].message) - == "Calling BaseDupeFilter.log() is deprecated." - ) - assert warning_list[0].category == ScrapyDeprecationWarning diff --git a/tests/test_http_request.py b/tests/test_http_request.py index fd494504d..e22689d49 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -460,11 +460,13 @@ class TestRequest: def test_from_curl_ignore_unknown_options(self): # By default: it works and ignores the unknown options: --foo and -z with warnings.catch_warnings(): # avoid warning when executing tests - warnings.simplefilter("ignore") + warnings.filterwarnings( + "ignore", category=UserWarning, message="Unrecognized options:" + ) r = self.request_class.from_curl( 'curl -X DELETE "http://example.org" --foo -z', ) - assert r.method == "DELETE" + assert r.method == "DELETE" # If `ignore_unknown_options` is set to `False` it raises an error with # the unknown options: --foo and -z diff --git a/tests/test_http_request_json.py b/tests/test_http_request_json.py index fcfe78365..65022afe1 100644 --- a/tests/test_http_request_json.py +++ b/tests/test_http_request_json.py @@ -4,6 +4,8 @@ import json import warnings from unittest import mock +import pytest + from scrapy.http import JsonRequest from scrapy.utils.python import to_bytes from tests.test_http_request import TestRequest @@ -63,40 +65,40 @@ class TestJsonRequest(TestRequest): data = { "name": "value", } - with warnings.catch_warnings(record=True) as _warnings: + with pytest.warns(UserWarning, match="data will be ignored"): r5 = self.request_class(url="http://www.example.com/", body=body, data=data) - assert r5.body == body - assert r5.method == "GET" - assert len(_warnings) == 1 - assert "data will be ignored" in str(_warnings[0].message) + assert r5.body == body + assert r5.method == "GET" def test_empty_body_data(self): """passing any body value and data should result a warning""" data = { "name": "value", } - with warnings.catch_warnings(record=True) as _warnings: + with pytest.warns(UserWarning, match="data will be ignored"): r6 = self.request_class(url="http://www.example.com/", body=b"", data=data) - assert r6.body == b"" - assert r6.method == "GET" - assert len(_warnings) == 1 - assert "data will be ignored" in str(_warnings[0].message) + assert r6.body == b"" + assert r6.method == "GET" def test_body_none_data(self): data = { "name": "value", } - with warnings.catch_warnings(record=True) as _warnings: + with warnings.catch_warnings(): + warnings.filterwarnings( + "error", category=UserWarning, message="Both body and data passed" + ) r7 = self.request_class(url="http://www.example.com/", body=None, data=data) - assert r7.body == to_bytes(json.dumps(data)) - assert r7.method == "POST" - assert len(_warnings) == 0 + assert r7.body == to_bytes(json.dumps(data)) + assert r7.method == "POST" def test_body_data_none(self): - with warnings.catch_warnings(record=True) as _warnings: + with warnings.catch_warnings(): + warnings.filterwarnings( + "error", category=UserWarning, message="Both body and data passed" + ) r8 = self.request_class(url="http://www.example.com/", body=None, data=None) - assert r8.method == "GET" - assert len(_warnings) == 0 + assert r8.method == "GET" def test_dumps_sort_keys(self): """Test that sort_keys=True is passed to json.dumps by default""" @@ -183,8 +185,5 @@ class TestJsonRequest(TestRequest): } r1 = self.request_class(url="http://www.example.com/", data=data1, body=body1) - with warnings.catch_warnings(record=True) as _warnings: + with pytest.warns(UserWarning, match="data will be ignored"): r1.replace(data=data2, body=body2) - assert "Both body and data passed. data will be ignored" in str( - _warnings[0].message - ) diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py index c6a41fa6e..3829a35f5 100644 --- a/tests/test_pipeline_files.py +++ b/tests/test_pipeline_files.py @@ -2,7 +2,6 @@ import dataclasses import os import random import time -import warnings from abc import ABC, abstractmethod from datetime import datetime from ftplib import FTP @@ -786,12 +785,10 @@ class TestBuildFromCrawler: class Pipeline(FilesPipeline): pass - with warnings.catch_warnings(record=True) as w: - pipe = Pipeline.from_crawler(self.crawler) - assert pipe.crawler == self.crawler - assert pipe._fingerprinter - assert len(w) == 0 - assert pipe.store + pipe = Pipeline.from_crawler(self.crawler) + assert pipe.crawler == self.crawler + assert pipe._fingerprinter + assert pipe.store def test_has_from_crawler_and_init(self): class Pipeline(FilesPipeline): @@ -805,13 +802,11 @@ class TestBuildFromCrawler: o._from_crawler_called = True return o - with warnings.catch_warnings(record=True) as w: - pipe = Pipeline.from_crawler(self.crawler) - assert pipe.crawler == self.crawler - assert pipe._fingerprinter - assert len(w) == 0 - assert pipe.store - assert pipe._from_crawler_called + pipe = Pipeline.from_crawler(self.crawler) + assert pipe.crawler == self.crawler + assert pipe._fingerprinter + assert pipe.store + assert pipe._from_crawler_called @pytest.mark.parametrize("store", [None, ""]) diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py index da1bfa317..8ac949da7 100644 --- a/tests/test_pipeline_media.py +++ b/tests/test_pipeline_media.py @@ -1,6 +1,5 @@ from __future__ import annotations -import warnings from unittest.mock import MagicMock import pytest @@ -425,11 +424,9 @@ class TestBuildFromCrawler: class Pipeline(UserDefinedPipeline): pass - with warnings.catch_warnings(record=True) as w: - pipe = Pipeline.from_crawler(self.crawler) - assert pipe.crawler == self.crawler - assert pipe._fingerprinter - assert len(w) == 0 + pipe = Pipeline.from_crawler(self.crawler) + assert pipe.crawler == self.crawler + assert pipe._fingerprinter def test_has_from_crawler_and_init(self): class Pipeline(UserDefinedPipeline): @@ -447,13 +444,11 @@ class TestBuildFromCrawler: o._from_crawler_called = True return o - with warnings.catch_warnings(record=True) as w: - pipe = Pipeline.from_crawler(self.crawler) - assert pipe.crawler == self.crawler - assert pipe._fingerprinter - assert len(w) == 0 - assert pipe._from_crawler_called - assert pipe._init_called + pipe = Pipeline.from_crawler(self.crawler) + assert pipe.crawler == self.crawler + assert pipe._fingerprinter + assert pipe._from_crawler_called + assert pipe._init_called def test_has_from_crawler(self): class Pipeline(UserDefinedPipeline): @@ -467,13 +462,10 @@ class TestBuildFromCrawler: o.store_uri = settings["FILES_STORE"] return o - with warnings.catch_warnings(record=True) as w: - pipe = Pipeline.from_crawler(self.crawler) - # this and the next assert will fail as MediaPipeline.from_crawler() wasn't called - assert pipe.crawler == self.crawler - assert pipe._fingerprinter - assert len(w) == 0 - assert pipe._from_crawler_called + pipe = Pipeline.from_crawler(self.crawler) + assert pipe.crawler == self.crawler + assert pipe._fingerprinter + assert pipe._from_crawler_called class MediaFailedFailurePipeline(MockedMediaPipeline): diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py index eb88baf01..8637de41e 100644 --- a/tests/test_scheduler.py +++ b/tests/test_scheduler.py @@ -12,6 +12,7 @@ import pytest from scrapy.core.downloader import Downloader from scrapy.core.scheduler import BaseScheduler, Scheduler from scrapy.crawler import Crawler +from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import Request from scrapy.spiders import Spider from scrapy.utils.defer import ensure_awaitable @@ -396,7 +397,11 @@ class TestIncompatibility: def test_incompatibility(self): with warnings.catch_warnings(): - warnings.filterwarnings("ignore") + warnings.filterwarnings( + "ignore", + category=ScrapyDeprecationWarning, + message="The CONCURRENT_REQUESTS_PER_IP setting is deprecated", + ) with pytest.raises( ValueError, match="does not support CONCURRENT_REQUESTS_PER_IP" ): diff --git a/tests/test_scrapy__getattr__.py b/tests/test_scrapy__getattr__.py index 4c365859b..4647e1041 100644 --- a/tests/test_scrapy__getattr__.py +++ b/tests/test_scrapy__getattr__.py @@ -1,15 +1,18 @@ -import warnings +from __future__ import annotations + +import pytest + +from scrapy.exceptions import ScrapyDeprecationWarning -def test_deprecated_concurrent_requests_per_ip_attribute(): - with warnings.catch_warnings(record=True) as warns: +def test_deprecated_concurrent_requests_per_ip_attribute() -> None: + with pytest.warns( + ScrapyDeprecationWarning, + match=r"scrapy\.settings\.default_settings\.CONCURRENT_REQUESTS_PER_IP attribute is deprecated", + ): from scrapy.settings.default_settings import ( # noqa: PLC0415 CONCURRENT_REQUESTS_PER_IP, ) - assert CONCURRENT_REQUESTS_PER_IP is not None - assert isinstance(CONCURRENT_REQUESTS_PER_IP, int) - assert ( - "The scrapy.settings.default_settings.CONCURRENT_REQUESTS_PER_IP attribute is deprecated, use scrapy.settings.default_settings.CONCURRENT_REQUESTS_PER_DOMAIN instead." - in warns[0].message.args - ) + assert CONCURRENT_REQUESTS_PER_IP is not None + assert isinstance(CONCURRENT_REQUESTS_PER_IP, int) diff --git a/tests/test_settings/__init__.py b/tests/test_settings/__init__.py index 9599ed6c9..908bb417c 100644 --- a/tests/test_settings/__init__.py +++ b/tests/test_settings/__init__.py @@ -2,12 +2,12 @@ # (too many false positives) import logging -import warnings from unittest import mock import pytest from scrapy.core.downloader.handlers.file import FileDownloadHandler +from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.settings import ( SETTINGS_PRIORITIES, BaseSettings, @@ -711,15 +711,13 @@ def test_remove_from_list(before, name, item, after): def test_deprecated_concurrent_requests_per_ip_setting(): - with warnings.catch_warnings(record=True) as warns: - settings = Settings({"CONCURRENT_REQUESTS_PER_IP": 1}) + settings = Settings({"CONCURRENT_REQUESTS_PER_IP": 1}) + with pytest.warns( + ScrapyDeprecationWarning, + match="The CONCURRENT_REQUESTS_PER_IP setting is deprecated", + ): settings.get("CONCURRENT_REQUESTS_PER_IP") - assert ( - str(warns[0].message) - == "The CONCURRENT_REQUESTS_PER_IP setting is deprecated, use CONCURRENT_REQUESTS_PER_DOMAIN instead." - ) - class Component1: pass diff --git a/tests/test_spider_crawl.py b/tests/test_spider_crawl.py index c97934b74..2eeae110b 100644 --- a/tests/test_spider_crawl.py +++ b/tests/test_spider_crawl.py @@ -8,6 +8,7 @@ import pytest from testfixtures import LogCapture from w3lib.url import safe_url_string +from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import HtmlResponse, Request, TextResponse from scrapy.linkextractors import LinkExtractor from scrapy.spiders import CrawlSpider, Rule, Spider @@ -264,13 +265,16 @@ class TestCrawlSpider(TestSpider): start_urls = "https://www.example.com" _follow_links = False - with warnings.catch_warnings(record=True) as w: + with warnings.catch_warnings(): + warnings.simplefilter("error", category=ScrapyDeprecationWarning) spider = _CrawlSpider() - assert len(w) == 0 + with pytest.warns( + ScrapyDeprecationWarning, + match=r"CrawlSpider\._parse_response method is deprecated", + ): spider._parse_response( TextResponse(spider.start_urls, body=b""), None, None ) - assert len(w) == 1 def test_parse_response_override(self): class _CrawlSpider(CrawlSpider): @@ -281,26 +285,28 @@ class TestCrawlSpider(TestSpider): start_urls = "https://www.example.com" _follow_links = False - with warnings.catch_warnings(record=True) as w: - assert len(w) == 0 + with pytest.warns( + ScrapyDeprecationWarning, + match=r"CrawlSpider\._parse_response method, which the", + ): spider = _CrawlSpider() - assert len(w) == 1 + with warnings.catch_warnings(): + warnings.simplefilter("error", category=ScrapyDeprecationWarning) spider._parse_response( TextResponse(spider.start_urls, body=b""), None, None ) - assert len(w) == 1 def test_parse_with_rules(self): class _CrawlSpider(CrawlSpider): name = "test" start_urls = "https://www.example.com" - with warnings.catch_warnings(record=True) as w: + with warnings.catch_warnings(): + warnings.simplefilter("error", category=ScrapyDeprecationWarning) spider = _CrawlSpider() spider.parse_with_rules( TextResponse(spider.start_urls, body=b""), None, None ) - assert len(w) == 0 class TestDeprecation: diff --git a/tests/test_spiderloader/__init__.py b/tests/test_spiderloader/__init__.py index de27ad519..d85942f90 100644 --- a/tests/test_spiderloader/__init__.py +++ b/tests/test_spiderloader/__init__.py @@ -1,7 +1,6 @@ import contextlib import shutil import sys -import warnings from pathlib import Path from unittest import mock @@ -137,50 +136,38 @@ class TestSpiderLoader: SpiderLoader.from_settings(settings) def test_bad_spider_modules_warning(self): - with warnings.catch_warnings(record=True) as w: - module = "tests.test_spiderloader.test_spiders.doesnotexist" - settings = Settings( - {"SPIDER_MODULES": [module], "SPIDER_LOADER_WARN_ONLY": True} - ) + module = "tests.test_spiderloader.test_spiders.doesnotexist" + settings = Settings( + {"SPIDER_MODULES": [module], "SPIDER_LOADER_WARN_ONLY": True} + ) + with pytest.warns(RuntimeWarning, match="Could not load spiders from module"): spider_loader = SpiderLoader.from_settings(settings) - if str(w[0].message).startswith("_SixMetaPathImporter"): - # needed on 3.10 because of https://github.com/benjaminp/six/issues/349, - # at least until all six versions we can import (including botocore.vendored.six) - # are updated to 1.16.0+ - w.pop(0) - assert "Could not load spiders from module" in str(w[0].message) - spiders = spider_loader.list() - assert not spiders + spiders = spider_loader.list() + assert not spiders def test_syntax_error_exception(self): module = "tests.test_spiderloader.test_spiders.spider1" + settings = Settings({"SPIDER_MODULES": [module]}) with mock.patch.object(SpiderLoader, "_load_spiders") as m: m.side_effect = SyntaxError - settings = Settings({"SPIDER_MODULES": [module]}) with pytest.raises(SyntaxError): SpiderLoader.from_settings(settings) def test_syntax_error_warning(self): - with ( - warnings.catch_warnings(record=True) as w, - mock.patch.object(SpiderLoader, "_load_spiders") as m, - ): + module = "tests.test_spiderloader.test_spiders.spider1" + settings = Settings( + {"SPIDER_MODULES": [module], "SPIDER_LOADER_WARN_ONLY": True} + ) + with mock.patch.object(SpiderLoader, "_load_spiders") as m: m.side_effect = SyntaxError - module = "tests.test_spiderloader.test_spiders.spider1" - settings = Settings( - {"SPIDER_MODULES": [module], "SPIDER_LOADER_WARN_ONLY": True} - ) - spider_loader = SpiderLoader.from_settings(settings) - if str(w[0].message).startswith("_SixMetaPathImporter"): - # needed on 3.10 because of https://github.com/benjaminp/six/issues/349, - # at least until all six versions we can import (including botocore.vendored.six) - # are updated to 1.16.0+ - w.pop(0) - assert "Could not load spiders from module" in str(w[0].message) + with pytest.warns( + RuntimeWarning, match="Could not load spiders from module" + ): + spider_loader = SpiderLoader.from_settings(settings) - spiders = spider_loader.list() - assert not spiders + spiders = spider_loader.list() + assert not spiders class TestDuplicateSpiderNameLoader: @@ -190,21 +177,17 @@ class TestDuplicateSpiderNameLoader: # copy 1 spider module so as to have duplicate spider name shutil.copyfile(spiders_dir / "spider3.py", spiders_dir / "spider3dupe.py") - with warnings.catch_warnings(record=True) as w: + msg = r"""There are several spiders with the same name: + + Spider3 named 'spider3' \(in test_spiders_xxx\.spider3\) + + Spider3 named 'spider3' \(in test_spiders_xxx\.spider3dupe\) + + This can cause unexpected behavior\.""" + with pytest.warns(UserWarning, match=msg): spider_loader = SpiderLoader.from_settings(settings) - - assert len(w) == 1 - msg = str(w[0].message) - assert "several spiders with the same name" in msg - assert "'spider3'" in msg - assert msg.count("'spider3'") == 2 - - assert "'spider1'" not in msg - assert "'spider2'" not in msg - assert "'spider4'" not in msg - - spiders = set(spider_loader.list()) - assert spiders == {"spider1", "spider2", "spider3", "spider4"} + spiders = set(spider_loader.list()) + assert spiders == {"spider1", "spider2", "spider3", "spider4"} def test_multiple_dupename_warning(self, spider_loader_env): settings, spiders_dir = spider_loader_env @@ -213,23 +196,21 @@ class TestDuplicateSpiderNameLoader: shutil.copyfile(spiders_dir / "spider1.py", spiders_dir / "spider1dupe.py") shutil.copyfile(spiders_dir / "spider2.py", spiders_dir / "spider2dupe.py") - with warnings.catch_warnings(record=True) as w: + msg = r"""There are several spiders with the same name: + + Spider1 named 'spider1' \(in test_spiders_xxx\.spider1\) + + Spider1 named 'spider1' \(in test_spiders_xxx\.spider1dupe\) + + Spider2 named 'spider2' \(in test_spiders_xxx\.spider2\) + + Spider2 named 'spider2' \(in test_spiders_xxx\.spider2dupe\) + + This can cause unexpected behavior\.""" + with pytest.warns(UserWarning, match=msg): spider_loader = SpiderLoader.from_settings(settings) - - assert len(w) == 1 - msg = str(w[0].message) - assert "several spiders with the same name" in msg - assert "'spider1'" in msg - assert msg.count("'spider1'") == 2 - - assert "'spider2'" in msg - assert msg.count("'spider2'") == 2 - - assert "'spider3'" not in msg - assert "'spider4'" not in msg - - spiders = set(spider_loader.list()) - assert spiders == {"spider1", "spider2", "spider3", "spider4"} + spiders = set(spider_loader.list()) + assert spiders == {"spider1", "spider2", "spider3", "spider4"} class CustomSpiderLoader(SpiderLoader): diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.py index a9089419a..b3a434c3f 100644 --- a/tests/test_spidermiddleware_referer.py +++ b/tests/test_spidermiddleware_referer.py @@ -6,6 +6,7 @@ from urllib.parse import urlparse import pytest +from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import Request, Response from scrapy.settings import Settings from scrapy.spidermiddlewares.referer import ( @@ -842,13 +843,14 @@ class TestRequestMetaSettingFallback: response = Response(origin, headers=response_headers) request = Request(target, meta=request_meta) - with warnings.catch_warnings(record=True) as w: + if check_warning: + with pytest.warns( + RuntimeWarning, match="Could not load referrer policy" + ): + policy = mw.policy(response, request) + else: policy = mw.policy(response, request) - assert isinstance(policy, policy_class) - - if check_warning: - assert len(w) == 1 - assert w[0].category is RuntimeWarning, w[0].message + assert isinstance(policy, policy_class) class TestSettingsPolicyByName: @@ -973,49 +975,39 @@ class TestPolicyMethodResponseParamRename: self.response = Response("http://www.example.com") def test_pos_string(self): - with warnings.catch_warnings(record=True) as w: + with pytest.warns( + ScrapyDeprecationWarning, + match=r"Passing a response URL to RefererMiddleware\.policy\(\)", + ): self.mw.policy("http://old.com", self.request) - found = False - for warning in w: - if "Passing a response URL" in str(warning.message): - found = True - break - assert found def test_pos_response(self): - with warnings.catch_warnings(record=True) as w: + with warnings.catch_warnings(): + warnings.filterwarnings( + "error", + category=ScrapyDeprecationWarning, + message=r"Passing 'resp_or_url' is deprecated", + ) self.mw.policy(self.response, self.request) - for warning in w: - assert "resp_or_url" not in str(warning.message) def test_key_resp_or_url(self): - with warnings.catch_warnings(record=True) as w: + with pytest.warns( + ScrapyDeprecationWarning, match=r"Passing 'resp_or_url' is deprecated" + ): self.mw.policy(resp_or_url=self.response, request=self.request) - found = False - for warning in w: - if "Passing 'resp_or_url' is deprecated, use 'response' instead" in str( - warning.message - ): - found = True - break - assert found def test_key_response(self): - with warnings.catch_warnings(record=True) as w: + with warnings.catch_warnings(): + warnings.filterwarnings( + "error", + category=ScrapyDeprecationWarning, + message=r"Passing 'resp_or_url' is deprecated", + ) self.mw.policy(response=self.response, request=self.request) - for warning in w: - assert "resp_or_url" not in str(warning.message) def test_key_response_string(self): - with warnings.catch_warnings(record=True) as w: - warnings.simplefilter("always") + with pytest.warns(ScrapyDeprecationWarning, match="Passing a response URL"): self.mw.policy(response="http://old.com", request=self.request) - found = False - for warning in w: - if "Passing a response URL" in str(warning.message): - found = True - break - assert found def test_both_resp_or_url_and_response(self): with pytest.raises( diff --git a/tests/test_utils_curl.py b/tests/test_utils_curl.py index b1532ca77..0627a4b93 100644 --- a/tests/test_utils_curl.py +++ b/tests/test_utils_curl.py @@ -213,10 +213,12 @@ class TestCurlToRequestKwargs: def test_ignore_unknown_options(self): # case 1: ignore_unknown_options=True: + curl_command = "curl --bar --baz http://www.example.com" + expected_result = {"method": "GET", "url": "http://www.example.com"} with warnings.catch_warnings(): # avoid warning when executing tests - warnings.simplefilter("ignore") - curl_command = "curl --bar --baz http://www.example.com" - expected_result = {"method": "GET", "url": "http://www.example.com"} + warnings.filterwarnings( + "ignore", category=UserWarning, message="Unrecognized options:" + ) assert curl_to_request_kwargs(curl_command) == expected_result # case 2: ignore_unknown_options=False (raise exception): diff --git a/tests/test_utils_datatypes.py b/tests/test_utils_datatypes.py index f573993a1..ba6b82503 100644 --- a/tests/test_utils_datatypes.py +++ b/tests/test_utils_datatypes.py @@ -1,5 +1,4 @@ import copy -import warnings from abc import ABC, abstractmethod from collections.abc import Iterator, Mapping, MutableMapping from typing import Any @@ -227,18 +226,12 @@ class TestCaselessDict(TestCaseInsensitiveDictBase): dict_class = CaselessDict def test_deprecation_message(self): - with warnings.catch_warnings(record=True) as caught: - warnings.filterwarnings("always", category=ScrapyDeprecationWarning) + with pytest.warns( + ScrapyDeprecationWarning, + match=r"scrapy.utils.datatypes.CaselessDict is deprecated", + ): self.dict_class({"foo": "bar"}) - assert len(caught) == 1 - assert issubclass(caught[0].category, ScrapyDeprecationWarning) - assert ( - str(caught[0].message) - == "scrapy.utils.datatypes.CaselessDict is deprecated," - " please use scrapy.utils.datatypes.CaseInsensitiveDict instead" - ) - class TestSequenceExclude: def test_list(self): diff --git a/tests/test_utils_deprecate.py b/tests/test_utils_deprecate.py index 5ea6f678e..0706fec99 100644 --- a/tests/test_utils_deprecate.py +++ b/tests/test_utils_deprecate.py @@ -1,7 +1,6 @@ import inspect import warnings from unittest import mock -from warnings import WarningMessage import pytest @@ -22,34 +21,26 @@ class NewName(SomeBaseClass): class TestWarnWhenSubclassed: - def _mywarnings(self, w: list[WarningMessage]) -> list[WarningMessage]: - return [x for x in w if x.category is MyWarning] - def test_no_warning_on_definition(self): - with warnings.catch_warnings(record=True) as w: + with warnings.catch_warnings(): + warnings.simplefilter("error", category=ScrapyDeprecationWarning) create_deprecated_class("Deprecated", NewName) - w = self._mywarnings(w) - assert w == [] - def test_subclassing_warning_message(self): + msg = ( + r"tests\.test_utils_deprecate\.UserClass inherits from " + r"deprecated class tests\.test_utils_deprecate\.Deprecated, " + r"please inherit from tests\.test_utils_deprecate\.NewName." + r" \(warning only on first subclass, there may be others\)" + ) Deprecated = create_deprecated_class( "Deprecated", NewName, warn_category=MyWarning ) - - with warnings.catch_warnings(record=True) as w: + with pytest.warns(MyWarning, match=msg) as w: class UserClass(Deprecated): pass - w = self._mywarnings(w) - assert len(w) == 1 - assert ( - str(w[0].message) == "tests.test_utils_deprecate.UserClass inherits from " - "deprecated class tests.test_utils_deprecate.Deprecated, " - "please inherit from tests.test_utils_deprecate.NewName." - " (warning only on first subclass, there may be others)" - ) assert w[0].lineno == inspect.getsourcelines(UserClass)[1] def test_custom_class_paths(self): @@ -61,62 +52,77 @@ class TestWarnWhenSubclassed: warn_category=MyWarning, ) - with warnings.catch_warnings(record=True) as w: + with pytest.warns( + MyWarning, + match=r"UserClass inherits from deprecated class bar\.OldClass, please inherit from foo\.NewClass", + ): class UserClass(Deprecated): pass + with pytest.warns( + MyWarning, + match=r"bar\.OldClass is deprecated, instantiate foo\.NewClass instead", + ): _ = Deprecated() - w = self._mywarnings(w) - assert len(w) == 2 - assert "foo.NewClass" in str(w[0].message) - assert "bar.OldClass" in str(w[0].message) - assert "foo.NewClass" in str(w[1].message) - assert "bar.OldClass" in str(w[1].message) - def test_subclassing_warns_only_on_direct_children(self): Deprecated = create_deprecated_class( "Deprecated", NewName, warn_once=False, warn_category=MyWarning ) - with warnings.catch_warnings(record=True) as w: + with pytest.warns( + MyWarning, + match="UserClass inherits from deprecated class", + ): class UserClass(Deprecated): pass + with warnings.catch_warnings(): + warnings.simplefilter("error", MyWarning) + class NoWarnOnMe(UserClass): pass - w = self._mywarnings(w) - assert len(w) == 1 - assert "UserClass" in str(w[0].message) - def test_subclassing_warns_once_by_default(self): Deprecated = create_deprecated_class( "Deprecated", NewName, warn_category=MyWarning ) - with warnings.catch_warnings(record=True) as w: + with pytest.warns( + MyWarning, + match="UserClass inherits from deprecated class", + ): class UserClass(Deprecated): pass + with warnings.catch_warnings(): + warnings.simplefilter("error", MyWarning) + class FooClass(Deprecated): pass class BarClass(Deprecated): pass - w = self._mywarnings(w) - assert len(w) == 1 - assert "UserClass" in str(w[0].message) - def test_warning_on_instance(self): Deprecated = create_deprecated_class( "Deprecated", NewName, warn_category=MyWarning ) + with pytest.warns(MyWarning) as w: + _, lineno = Deprecated(), inspect.getlineno(inspect.currentframe()) + + w = [x for x in w if x.category is MyWarning] + assert len(w) == 1 + assert ( + str(w[0].message) == "tests.test_utils_deprecate.Deprecated is deprecated, " + "instantiate tests.test_utils_deprecate.NewName instead." + ) + assert w[0].lineno == lineno + # ignore subclassing warnings with warnings.catch_warnings(): warnings.simplefilter("ignore", MyWarning) @@ -124,29 +130,20 @@ class TestWarnWhenSubclassed: class UserClass(Deprecated): pass - with warnings.catch_warnings(record=True) as w: - _, lineno = Deprecated(), inspect.getlineno(inspect.currentframe()) - _ = UserClass() # subclass instances don't warn - - w = self._mywarnings(w) - assert len(w) == 1 - assert ( - str(w[0].message) == "tests.test_utils_deprecate.Deprecated is deprecated, " - "instantiate tests.test_utils_deprecate.NewName instead." - ) - assert w[0].lineno == lineno + with warnings.catch_warnings(): + warnings.simplefilter("error", MyWarning) + UserClass() # subclass instances don't warn def test_warning_auto_message(self): - with warnings.catch_warnings(record=True) as w: - Deprecated = create_deprecated_class("Deprecated", NewName) + Deprecated = create_deprecated_class("Deprecated", NewName) + with pytest.warns( + ScrapyDeprecationWarning, + match=r"UserClass2 inherits from deprecated class tests\.test_utils_deprecate\.Deprecated, please inherit from tests\.test_utils_deprecate\.NewName", + ): class UserClass2(Deprecated): pass - msg = str(w[0].message) - assert "tests.test_utils_deprecate.NewName" in msg - assert "tests.test_utils_deprecate.Deprecated" in msg - def test_issubclass(self): with warnings.catch_warnings(): warnings.simplefilter("ignore", ScrapyDeprecationWarning) @@ -222,8 +219,8 @@ class TestWarnWhenSubclassed: create_deprecated_class("Deprecated", New) def test_deprecate_subclass_of_deprecated_class(self): - with warnings.catch_warnings(record=True) as w: - warnings.simplefilter("always") + with warnings.catch_warnings(): + warnings.simplefilter("error", MyWarning) Deprecated = create_deprecated_class( "Deprecated", NewName, warn_category=MyWarning ) @@ -234,33 +231,26 @@ class TestWarnWhenSubclassed: warn_category=MyWarning, ) - w = self._mywarnings(w) - assert len(w) == 0, [str(warning) for warning in w] - - with warnings.catch_warnings(record=True) as w: + with pytest.warns( + MyWarning, + match=r"AlsoDeprecated is deprecated, instantiate foo\.Bar instead", + ): AlsoDeprecated() + with pytest.warns( + MyWarning, + match=r"UserClass inherits from deprecated class tests\.test_utils_deprecate\.AlsoDeprecated, please inherit from foo\.Bar", + ): + class UserClass(AlsoDeprecated): pass - w = self._mywarnings(w) - assert len(w) == 2 - assert "AlsoDeprecated" in str(w[0].message) - assert "foo.Bar" in str(w[0].message) - assert "AlsoDeprecated" in str(w[1].message) - assert "foo.Bar" in str(w[1].message) - def test_inspect_stack(self): with ( mock.patch("inspect.stack", side_effect=IndexError), - warnings.catch_warnings(record=True) as w, + pytest.warns(UserWarning, match="Error detecting parent module"), ): - DeprecatedName = create_deprecated_class("DeprecatedName", NewName) - - class SubClass(DeprecatedName): - pass - - assert "Error detecting parent module" in str(w[0].message) + create_deprecated_class("DeprecatedName", NewName) @mock.patch( @@ -272,12 +262,12 @@ class TestWarnWhenSubclassed: ) class TestUpdateClassPath: def test_old_path_gets_fixed(self): - with warnings.catch_warnings(record=True) as w: + with pytest.warns( + ScrapyDeprecationWarning, + match="`scrapy.contrib.debug.Debug` class is deprecated, use `scrapy.extensions.debug.Debug` instead", + ): output = update_classpath("scrapy.contrib.debug.Debug") assert output == "scrapy.extensions.debug.Debug" - assert len(w) == 1 - assert "scrapy.contrib.debug.Debug" in str(w[0].message) - assert "scrapy.extensions.debug.Debug" in str(w[0].message) def test_sorted_replacement(self): with warnings.catch_warnings(): @@ -286,10 +276,10 @@ class TestUpdateClassPath: assert output == "scrapy.pipelines.Pipeline" def test_unmatched_path_stays_the_same(self): - with warnings.catch_warnings(record=True) as w: + with warnings.catch_warnings(): + warnings.simplefilter("error", ScrapyDeprecationWarning) output = update_classpath("scrapy.unmatched.Path") assert output == "scrapy.unmatched.Path" - assert len(w) == 0 def test_returns_nonstring(self): for notastring in [None, True, [1, 2, 3], object()]: diff --git a/tests/test_utils_misc/test_return_with_argument_inside_generator.py b/tests/test_utils_misc/test_return_with_argument_inside_generator.py index 3783416b9..1acc3aac2 100644 --- a/tests/test_utils_misc/test_return_with_argument_inside_generator.py +++ b/tests/test_utils_misc/test_return_with_argument_inside_generator.py @@ -93,29 +93,27 @@ https://example.org assert is_generator_with_return_value(h1) assert is_generator_with_return_value(i1) - with warnings.catch_warnings(record=True) as w: + with pytest.warns( + UserWarning, + match='The "MockSpider.top_level_return_something" method is a generator', + ): warn_on_generator_with_return_value(mock_spider, top_level_return_something) - assert len(w) == 1 - assert ( - 'The "MockSpider.top_level_return_something" method is a generator' - in str(w[0].message) - ) - with warnings.catch_warnings(record=True) as w: + with pytest.warns( + UserWarning, match='The "MockSpider.f1" method is a generator' + ): warn_on_generator_with_return_value(mock_spider, f1) - assert len(w) == 1 - assert 'The "MockSpider.f1" method is a generator' in str(w[0].message) - with warnings.catch_warnings(record=True) as w: + with pytest.warns( + UserWarning, match='The "MockSpider.g1" method is a generator' + ): warn_on_generator_with_return_value(mock_spider, g1) - assert len(w) == 1 - assert 'The "MockSpider.g1" method is a generator' in str(w[0].message) - with warnings.catch_warnings(record=True) as w: + with pytest.warns( + UserWarning, match='The "MockSpider.h1" method is a generator' + ): warn_on_generator_with_return_value(mock_spider, h1) - assert len(w) == 1 - assert 'The "MockSpider.h1" method is a generator' in str(w[0].message) - with warnings.catch_warnings(record=True) as w: + with pytest.warns( + UserWarning, match='The "MockSpider.i1" method is a generator' + ): warn_on_generator_with_return_value(mock_spider, i1) - assert len(w) == 1 - assert 'The "MockSpider.i1" method is a generator' in str(w[0].message) def test_generators_return_none(self, mock_spider): def f2(): @@ -160,32 +158,18 @@ https://example.org assert not is_generator_with_return_value(k2) # not recursive assert not is_generator_with_return_value(l2) - with warnings.catch_warnings(record=True) as w: + with warnings.catch_warnings(): + warnings.simplefilter("error", UserWarning) warn_on_generator_with_return_value(mock_spider, top_level_return_none) - assert len(w) == 0 - with warnings.catch_warnings(record=True) as w: warn_on_generator_with_return_value(mock_spider, f2) - assert len(w) == 0 - with warnings.catch_warnings(record=True) as w: warn_on_generator_with_return_value(mock_spider, g2) - assert len(w) == 0 - with warnings.catch_warnings(record=True) as w: warn_on_generator_with_return_value(mock_spider, h2) - assert len(w) == 0 - with warnings.catch_warnings(record=True) as w: warn_on_generator_with_return_value(mock_spider, i2) - assert len(w) == 0 - with warnings.catch_warnings(record=True) as w: warn_on_generator_with_return_value(mock_spider, j2) - assert len(w) == 0 - with warnings.catch_warnings(record=True) as w: warn_on_generator_with_return_value(mock_spider, k2) - assert len(w) == 0 - with warnings.catch_warnings(record=True) as w: warn_on_generator_with_return_value(mock_spider, l2) - assert len(w) == 0 - def test_generators_return_none_with_decorator(self, mock_spider): # noqa: PLR0915 + def test_generators_return_none_with_decorator(self, mock_spider): def decorator(func): def inner_func(): func() @@ -241,39 +225,23 @@ https://example.org assert not is_generator_with_return_value(k3) # not recursive assert not is_generator_with_return_value(l3) - with warnings.catch_warnings(record=True) as w: + with warnings.catch_warnings(): + warnings.simplefilter("error", UserWarning) warn_on_generator_with_return_value(mock_spider, top_level_return_none) - assert len(w) == 0 - with warnings.catch_warnings(record=True) as w: warn_on_generator_with_return_value(mock_spider, f3) - assert len(w) == 0 - with warnings.catch_warnings(record=True) as w: warn_on_generator_with_return_value(mock_spider, g3) - assert len(w) == 0 - with warnings.catch_warnings(record=True) as w: warn_on_generator_with_return_value(mock_spider, h3) - assert len(w) == 0 - with warnings.catch_warnings(record=True) as w: warn_on_generator_with_return_value(mock_spider, i3) - assert len(w) == 0 - with warnings.catch_warnings(record=True) as w: warn_on_generator_with_return_value(mock_spider, j3) - assert len(w) == 0 - with warnings.catch_warnings(record=True) as w: warn_on_generator_with_return_value(mock_spider, k3) - assert len(w) == 0 - with warnings.catch_warnings(record=True) as w: warn_on_generator_with_return_value(mock_spider, l3) - assert len(w) == 0 @mock.patch( "scrapy.utils.misc.is_generator_with_return_value", new=_indentation_error ) def test_indentation_error(self, mock_spider): - with warnings.catch_warnings(record=True) as w: + with pytest.warns(UserWarning, match="Unable to determine"): warn_on_generator_with_return_value(mock_spider, top_level_return_none) - assert len(w) == 1 - assert "Unable to determine" in str(w[0].message) def test_partial(self): def cb(arg1, arg2): @@ -300,13 +268,11 @@ https://example.org yield 1 return "value" - with warnings.catch_warnings(record=True) as w: + with warnings.catch_warnings(): + warnings.simplefilter("error", UserWarning) warn_on_generator_with_return_value(spider, gen_with_return) - assert len(w) == 0 spider.settings.settings_dict["WARN_ON_GENERATOR_RETURN_VALUE"] = True - with warnings.catch_warnings(record=True) as w: + with pytest.warns(UserWarning, match="is a generator"): warn_on_generator_with_return_value(spider, gen_with_return) - assert len(w) == 1 - assert "is a generator" in str(w[0].message) diff --git a/tests/test_utils_sitemap.py b/tests/test_utils_sitemap.py index f2bcd7541..3599c4824 100644 --- a/tests/test_utils_sitemap.py +++ b/tests/test_utils_sitemap.py @@ -1,5 +1,6 @@ -import warnings +import pytest +from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.utils.sitemap import Sitemap, sitemap_urls_from_robots @@ -204,7 +205,10 @@ Disallow: /forum/search/ Disallow: /forum/active/ """ - with warnings.catch_warnings(record=True) as w: + with pytest.warns( + ScrapyDeprecationWarning, + match="Passing `str` type as `robots_text` is deprecated", + ): assert list( sitemap_urls_from_robots(robots, base_url="http://example.com") ) == [ @@ -213,9 +217,6 @@ Disallow: /forum/active/ "http://example.com/sitemap-uppercase.xml", "http://example.com/sitemap-relative-url.xml", ] - assert "Passing `str` type as `robots_text` is deprecated, use `bytes`" in str( - w[0].message - ) def test_sitemap_blanklines(): From 75f05d4e80b82705f54b2fe6b2948c463c7eb36b Mon Sep 17 00:00:00 2001 From: "Shashank S. Khasare" Date: Mon, 22 Jun 2026 12:20:11 +0530 Subject: [PATCH 18/47] Add test coverage for scrapy.utils.decorators (#7645) --- tests/test_utils_decorators.py | 107 +++++++++++++++++++++++++++++++++ 1 file changed, 107 insertions(+) create mode 100644 tests/test_utils_decorators.py diff --git a/tests/test_utils_decorators.py b/tests/test_utils_decorators.py new file mode 100644 index 000000000..9743e1a50 --- /dev/null +++ b/tests/test_utils_decorators.py @@ -0,0 +1,107 @@ +from __future__ import annotations + +import warnings + +import pytest +from twisted.internet.defer import Deferred + +from scrapy.exceptions import ScrapyDeprecationWarning +from scrapy.utils.decorators import _warn_spider_arg, deprecated, inthread +from scrapy.utils.defer import maybe_deferred_to_future +from tests.utils.decorators import coroutine_test + + +class TestDeprecated: + def test_warns_and_still_calls(self): + @deprecated() + def add(a, b): + return a + b + + with pytest.warns( + ScrapyDeprecationWarning, match=r"Call to deprecated function add\." + ): + result = add(2, 3) + + assert result == 5 + + def test_use_instead_in_message(self): + @deprecated(use_instead="other_function") + def old(): + return None + + with pytest.warns( + ScrapyDeprecationWarning, + match=r"Call to deprecated function old\. Use other_function instead\.", + ): + old() + + def test_applied_without_parentheses(self): + @deprecated + def square(x): + return x * x + + with pytest.warns( + ScrapyDeprecationWarning, match=r"Call to deprecated function square\." + ) as record: + result = square(4) + + assert result == 16 + # No "Use ... instead." part when applied directly to the function. + assert "instead" not in str(record[0].message) + + +class TestInthread: + @coroutine_test + async def test_returns_deferred_with_result(self): + @inthread + def multiply(a, b): + return a * b + + deferred = multiply(6, 7) + assert isinstance(deferred, Deferred) + assert await maybe_deferred_to_future(deferred) == 42 + + +class TestWarnSpiderArg: + def test_sync_warns_with_spider_arg(self): + @_warn_spider_arg + def parse(response, spider=None): + return response + + with pytest.warns( + ScrapyDeprecationWarning, match=r"Passing a 'spider' argument" + ): + assert parse("response", spider="spider") == "response" + + def test_sync_no_warning_without_spider_arg(self): + @_warn_spider_arg + def parse(response, spider=None): + return response + + with warnings.catch_warnings(): + warnings.simplefilter("error", category=ScrapyDeprecationWarning) + assert parse("response") == "response" + + @coroutine_test + async def test_async_warns_with_spider_arg(self): + @_warn_spider_arg + async def parse(response, spider=None): + return response + + with pytest.warns( + ScrapyDeprecationWarning, match=r"Passing a 'spider' argument" + ): + assert await parse("response", spider="spider") == "response" + + @coroutine_test + async def test_asyncgen_warns_with_spider_arg(self): + @_warn_spider_arg + async def parse(response, spider=None): + yield response + + with pytest.warns( + ScrapyDeprecationWarning, match=r"Passing a 'spider' argument" + ): + results = [item async for item in parse("response", spider="spider")] + + assert results == ["response"] From b6596de317acb16c1a0de0a77496640f7542364b Mon Sep 17 00:00:00 2001 From: Adrian Date: Mon, 22 Jun 2026 17:10:10 +0200 Subject: [PATCH 19/47] Do not ignore CrawlerProcess settings (#7647) * Do not ignore CrawlerProcess settings * Update test_crawlerrunner_accepts_crawler --- scrapy/crawler.py | 8 ++++++-- tests/test_crawl.py | 4 ++-- tests/test_crawler.py | 36 ++++++++++++++++++++++++++++++++++++ 3 files changed, 44 insertions(+), 4 deletions(-) diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 8e3ae7879..9828fe6f5 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -363,9 +363,12 @@ class CrawlerRunnerBase(ABC): """ Return a :class:`~scrapy.crawler.Crawler` object. - * If ``crawler_or_spidercls`` is a Crawler, it is returned as-is. + * If ``crawler_or_spidercls`` is a Crawler, the runner's settings are + merged into it as defaults: for each setting, the runner's value + is applied only if the Crawler does not already have that setting at + an equal or higher priority. The Crawler is then returned. * If ``crawler_or_spidercls`` is a Spider subclass, a new Crawler - is constructed for it. + is constructed for it using this runner's settings. * If ``crawler_or_spidercls`` is a string, this function finds a spider with this name in a Scrapy project (using spider loader), then creates a Crawler instance for it. @@ -376,6 +379,7 @@ class CrawlerRunnerBase(ABC): "it must be a spider class (or a Crawler object)" ) if isinstance(crawler_or_spidercls, Crawler): + crawler_or_spidercls.settings.update(self.settings) return crawler_or_spidercls return self._create_crawler(crawler_or_spidercls) diff --git a/tests/test_crawl.py b/tests/test_crawl.py index 66b15bfdd..be6c80429 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -14,7 +14,7 @@ from twisted.internet.ssl import Certificate from twisted.python.failure import Failure from scrapy import Spider, signals -from scrapy.crawler import AsyncCrawlerRunner, CrawlerRunner +from scrapy.crawler import AsyncCrawlerRunner, Crawler, CrawlerRunner from scrapy.exceptions import CloseSpider, ScrapyDeprecationWarning, StopDownload from scrapy.http import Request from scrapy.http.response import Response @@ -432,7 +432,7 @@ with multiples lines async def test_crawlerrunner_accepts_crawler( self, caplog: pytest.LogCaptureFixture, mockserver: MockServer ) -> None: - crawler = get_crawler(SimpleSpider) + crawler = Crawler(SimpleSpider, get_reactor_settings()) runner = CrawlerRunner() with caplog.at_level(logging.DEBUG): await maybe_deferred_to_future( diff --git a/tests/test_crawler.py b/tests/test_crawler.py index c646aea4b..0cddfd0ed 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -651,6 +651,42 @@ class TestAsyncCrawlerProcess(TestBaseCrawler): self.assertOptionIsDefault(runner.settings, "RETRY_ENABLED") +@pytest.mark.parametrize("runner_cls", [AsyncCrawlerRunner, CrawlerRunner]) +def test_runner_settings_applied_to_crawler_instance( + runner_cls: type[CrawlerRunnerBase], +) -> None: + runner = runner_cls({"FOO": "runner"}) + crawler = Crawler(DefaultSpider) + result = runner.create_crawler(crawler) + assert result is crawler + assert result.settings["FOO"] == "runner" + + +@pytest.mark.parametrize("runner_cls", [AsyncCrawlerRunner, CrawlerRunner]) +def test_spider_custom_settings_override_runner( + runner_cls: type[CrawlerRunnerBase], +) -> None: + class MySpider(DefaultSpider): + custom_settings = {"FOO": "spider"} + + runner = runner_cls({"FOO": "runner"}) + crawler = Crawler(MySpider) + runner.create_crawler(crawler) + assert crawler.settings["FOO"] == "spider" + + +def test_create_crawler_instance_consistent_with_spider_class() -> None: + runner = AsyncCrawlerRunner({"FOO": "runner"}) + + crawler_from_class = runner.create_crawler(DefaultSpider) + + pre_built = Crawler(DefaultSpider) + runner.create_crawler(pre_built) + + assert crawler_from_class.settings["FOO"] == "runner" + assert pre_built.settings["FOO"] == "runner" + + class ExceptionSpider(scrapy.Spider): name = "exception" From 7499d17e281cd214a70538ea92c05b14d8459dde Mon Sep 17 00:00:00 2001 From: Adrian Date: Mon, 22 Jun 2026 17:10:38 +0200 Subject: [PATCH 20/47] Improve test coverage for responsetypes.py (#7646) * Improve test coverage for responsetypes.py * Solve typing issues --- scrapy/responsetypes.py | 7 +------ tests/test_responsetypes.py | 3 +++ 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/scrapy/responsetypes.py b/scrapy/responsetypes.py index cd62f02af..29e9b6bb7 100644 --- a/scrapy/responsetypes.py +++ b/scrapy/responsetypes.py @@ -40,18 +40,13 @@ class ResponseTypes: self.classes: dict[str, type[Response]] = {} self.mimetypes: MimeTypes = MimeTypes() mimedata = get_data("scrapy", "mime.types") - if not mimedata: - raise ValueError( - "The mime.types file is not found in the Scrapy installation" - ) + assert mimedata is not None self.mimetypes.readfp(StringIO(mimedata.decode("utf8"))) for mimetype, cls in self.CLASSES.items(): self.classes[mimetype] = load_object(cls) def from_mimetype(self, mimetype: str) -> type[Response]: """Return the most appropriate Response class for the given mimetype""" - if mimetype is None: - return Response if mimetype in self.classes: return self.classes[mimetype] basetype = f"{mimetype.split('/', maxsplit=1)[0]}/*" diff --git a/tests/test_responsetypes.py b/tests/test_responsetypes.py index 5b04c7436..42ab29267 100644 --- a/tests/test_responsetypes.py +++ b/tests/test_responsetypes.py @@ -40,6 +40,9 @@ class TestResponseTypes: retcls = responsetypes.from_content_disposition(source) assert retcls is cls, f"{source} ==> {retcls} != {cls}" + def test_from_content_disposition_no_filename(self): + assert responsetypes.from_content_disposition(b"attachment") is Response + def test_from_content_type(self): mappings = [ ("text/html; charset=UTF-8", HtmlResponse), From f605defefc5cb8e7a969fba7da66c2087868c104 Mon Sep 17 00:00:00 2001 From: Adrian Date: Mon, 22 Jun 2026 17:12:16 +0200 Subject: [PATCH 21/47] Document scrapy-lint, remove start_url check (#7627) * Document scrapy-lint, remove start_url check * Remove the offsite URL check in favor of scrapy-lint --- .pre-commit-config.yaml | 2 +- docs/conf.py | 1 + docs/requirements.in | 2 +- docs/requirements.txt | 2 +- docs/topics/practices.rst | 8 ++++++++ scrapy/downloadermiddlewares/offsite.py | 18 +----------------- scrapy/spiders/__init__.py | 6 ------ tests/test_downloadermiddleware_offsite.py | 10 ++-------- tests/test_spider_crawl.py | 15 --------------- tox.ini | 2 +- 10 files changed, 16 insertions(+), 50 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 6b9ef3c04..754bd6f9c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -27,6 +27,6 @@ repos: hooks: - id: sphinx-lint - repo: https://github.com/scrapy/sphinx-scrapy - rev: 0.8.6 + rev: 0.8.8 hooks: - id: sphinx-scrapy diff --git a/docs/conf.py b/docs/conf.py index 99d5df7da..b950c4ee2 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -158,6 +158,7 @@ scrapy_intersphinx_enable = [ "itemloaders", "parsel", "pytest", + "scrapy-lint", "sphinx", "tox", "twisted", diff --git a/docs/requirements.in b/docs/requirements.in index 140791641..a1f3a7468 100644 --- a/docs/requirements.in +++ b/docs/requirements.in @@ -5,4 +5,4 @@ sphinx sphinx-notfound-page sphinx-rtd-theme sphinx-rtd-dark-mode -sphinx-scrapy @ git+https://github.com/scrapy/sphinx-scrapy.git@0.8.6 +sphinx-scrapy @ git+https://github.com/scrapy/sphinx-scrapy.git@0.8.8 diff --git a/docs/requirements.txt b/docs/requirements.txt index 9c93dacd0..a5cbad302 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -153,7 +153,7 @@ sphinx-rtd-theme==3.1.0 # via # -r docs/requirements.in # sphinx-rtd-dark-mode -sphinx-scrapy @ git+https://github.com/scrapy/sphinx-scrapy.git@b1d55db4d16a5425fc68576d63519bbfe26dd9c0 +sphinx-scrapy @ git+https://github.com/scrapy/sphinx-scrapy.git@c0b2ac815afc3cb8857d575cecb5d55c05e6b737 # via -r docs/requirements.in sphinx-sitemap==2.9.0 # via sphinx-scrapy diff --git a/docs/topics/practices.rst b/docs/topics/practices.rst index b3c58d6d3..8a04f7ada 100644 --- a/docs/topics/practices.rst +++ b/docs/topics/practices.rst @@ -440,6 +440,14 @@ Here are some tips to keep in mind when dealing with these kinds of sites: If you are still unable to prevent your bot getting banned, consider contacting `commercial support`_. +.. _static-analysis: + +Static analysis +=============== + +Consider using :doc:`scrapy-lint `, a linter for Scrapy +projects that detects common mistakes and anti-patterns. + .. _Tor project: https://www.torproject.org/ .. _commercial support: https://www.scrapy.org/companies .. _ProxyMesh: https://proxymesh.com/ diff --git a/scrapy/downloadermiddlewares/offsite.py b/scrapy/downloadermiddlewares/offsite.py index 10f19bacc..e03dc040e 100644 --- a/scrapy/downloadermiddlewares/offsite.py +++ b/scrapy/downloadermiddlewares/offsite.py @@ -2,7 +2,6 @@ from __future__ import annotations import logging import re -import warnings from typing import TYPE_CHECKING from scrapy import Request, Spider, signals @@ -75,25 +74,10 @@ class OffsiteMiddleware: allowed_domains = getattr(spider, "allowed_domains", None) if not allowed_domains: return re.compile("") # allow all by default - url_pattern = re.compile(r"^https?://.*$") - port_pattern = re.compile(r":\d+$") domains = [] for domain in allowed_domains: if domain is None: continue - if url_pattern.match(domain): - message = ( - "allowed_domains accepts only domains, not URLs. " - f"Ignoring URL entry {domain} in allowed_domains." - ) - warnings.warn(message, stacklevel=2) - elif port_pattern.search(domain): - message = ( - "allowed_domains accepts only domains without ports. " - f"Ignoring entry {domain} in allowed_domains." - ) - warnings.warn(message, stacklevel=2) - else: - domains.append(re.escape(domain)) + domains.append(re.escape(domain)) regex = rf"^(.*\.)?({'|'.join(domains)})$" return re.compile(regex) diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.py index 6e8c67adb..de527d04f 100644 --- a/scrapy/spiders/__init__.py +++ b/scrapy/spiders/__init__.py @@ -125,12 +125,6 @@ class Spider(object_ref): .. seealso:: :ref:`start-requests` """ - if not self.start_urls and hasattr(self, "start_url"): - raise AttributeError( - "Crawling could not start: 'start_urls' not found " - "or empty (but found 'start_url' attribute instead, " - "did you miss an 's'?)" - ) for url in self.start_urls: yield Request(url, dont_filter=True) diff --git a/tests/test_downloadermiddleware_offsite.py b/tests/test_downloadermiddleware_offsite.py index dc0a31a76..edfb15d10 100644 --- a/tests/test_downloadermiddleware_offsite.py +++ b/tests/test_downloadermiddleware_offsite.py @@ -1,5 +1,3 @@ -import warnings - import pytest from scrapy import Request, Spider @@ -120,9 +118,7 @@ def test_process_request_invalid_domains(): allowed_domains = ["a.example", None, "http:////b.example", "//c.example"] crawler.spider = crawler._create_spider(name="a", allowed_domains=allowed_domains) mw = OffsiteMiddleware.from_crawler(crawler) - with warnings.catch_warnings(): - warnings.simplefilter("ignore", UserWarning) - mw.spider_opened(crawler.spider) + mw.spider_opened(crawler.spider) request = Request("https://a.example") assert mw.process_request(request) is None for letter in ("b", "c"): @@ -210,9 +206,7 @@ def test_request_scheduled_invalid_domains(): allowed_domains = ["a.example", None, "http:////b.example", "//c.example"] crawler.spider = crawler._create_spider(name="a", allowed_domains=allowed_domains) mw = OffsiteMiddleware.from_crawler(crawler) - with warnings.catch_warnings(): - warnings.simplefilter("ignore", UserWarning) - mw.spider_opened(crawler.spider) + mw.spider_opened(crawler.spider) request = Request("https://a.example") assert mw.request_scheduled(request, crawler.spider) is None for letter in ("b", "c"): diff --git a/tests/test_spider_crawl.py b/tests/test_spider_crawl.py index 2eeae110b..63010e195 100644 --- a/tests/test_spider_crawl.py +++ b/tests/test_spider_crawl.py @@ -2,10 +2,8 @@ from __future__ import annotations import re import warnings -from logging import ERROR import pytest -from testfixtures import LogCapture from w3lib.url import safe_url_string from scrapy.exceptions import ScrapyDeprecationWarning @@ -14,7 +12,6 @@ from scrapy.linkextractors import LinkExtractor from scrapy.spiders import CrawlSpider, Rule, Spider from scrapy.utils.test import get_crawler from tests.test_spider import TestSpider -from tests.utils.decorators import inline_callbacks_test class TestCrawlSpider(TestSpider): @@ -247,18 +244,6 @@ class TestCrawlSpider(TestSpider): assert hasattr(spider, "_follow_links") assert not spider._follow_links - @inline_callbacks_test - def test_start_url(self): - class TestSpider(self.spider_class): - name = "test" - start_url = "https://www.example.com" - - crawler = get_crawler(TestSpider) - with LogCapture("scrapy.core.engine", propagate=False, level=ERROR) as log: - yield crawler.crawl() - assert "Error while reading start items and requests" in str(log) - assert "did you miss an 's'?" in str(log) - def test_parse_response_use(self): class _CrawlSpider(CrawlSpider): name = "test" diff --git a/tox.ini b/tox.ini index 44e240e03..2fa7b455d 100644 --- a/tox.ini +++ b/tox.ini @@ -5,7 +5,7 @@ [tox] requires = - sphinx-scrapy[tox] @ git+https://github.com/scrapy/sphinx-scrapy.git@0.8.6 + sphinx-scrapy[tox] @ git+https://github.com/scrapy/sphinx-scrapy.git@0.8.8 envlist = pre-commit pylint From f5a62a293f82bb3a7d4be1357ae8392d61f56021 Mon Sep 17 00:00:00 2001 From: tanishqtayade Date: Tue, 23 Jun 2026 12:34:01 +0530 Subject: [PATCH 22/47] Fix cell-var-from-loop bug in _send_catch_log_deferred (#7649) --- scrapy/utils/signal.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/scrapy/utils/signal.py b/scrapy/utils/signal.py index ee391a49e..919f67240 100644 --- a/scrapy/utils/signal.py +++ b/scrapy/utils/signal.py @@ -125,13 +125,11 @@ def _send_catch_log_deferred( **named, ) d.addErrback(logerror, receiver) - # TODO https://pylint.readthedocs.io/en/latest/user_guide/messages/warning/cell-var-from-loop.html + d2: Deferred[tuple[TypingAny, TypingAny]] = d.addBoth( - lambda result: ( - receiver, # pylint: disable=cell-var-from-loop # noqa: B023 - result, - ) + lambda result, recv: (recv, result), receiver ) + dfds.append(d2) results = yield DeferredList(dfds) From fb3455304d9e9f5332a97edbe2ea0a77a874e974 Mon Sep 17 00:00:00 2001 From: smellslikeml Date: Tue, 23 Jun 2026 00:04:19 -0700 Subject: [PATCH 23/47] Add content-based image filtering example (#4954) --- docs/topics/media-pipeline.rst | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rst index b542c6c05..c67d2d627 100644 --- a/docs/topics/media-pipeline.rst +++ b/docs/topics/media-pipeline.rst @@ -774,4 +774,28 @@ To enable your custom media pipeline component you must add its class import pat ITEM_PIPELINES = {"myproject.pipelines.MyImagesPipeline": 300} +Content-based image filtering pipeline +-------------------------------------- + +This example overrides ``get_images()`` to filter images using a classifier, +such as a TensorFlow_ model. Override ``is_valid_image()`` with your +classification logic: + +.. code-block:: python + + from scrapy.pipelines.images import ImagesPipeline, ImageException + + + class ImageClassifierPipeline(ImagesPipeline): + def is_valid_image(self, image): + raise NotImplementedError + + def get_images(self, response, request, info, *, item=None): + for path, image, buf in super().get_images(response, request, info, item=item): + if not self.is_valid_image(image): + raise ImageException("Image does not match criteria") + yield path, image, buf + + .. _MD5 hash: https://en.wikipedia.org/wiki/MD5 +.. _TensorFlow: https://tensorflow.org From b78ab3d6c8715d488ac657e0d09d0cf7bc6abc42 Mon Sep 17 00:00:00 2001 From: Adrian Date: Tue, 23 Jun 2026 11:47:21 +0200 Subject: [PATCH 24/47] Improve test coverage for settings/ (#7654) --- tests/test_settings/__init__.py | 59 +++++++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) diff --git a/tests/test_settings/__init__.py b/tests/test_settings/__init__.py index 908bb417c..3282b0591 100644 --- a/tests/test_settings/__init__.py +++ b/tests/test_settings/__init__.py @@ -539,6 +539,56 @@ class TestBaseSettings: msg = caplog.records[0].message assert "tests.test_settings.Component1" in msg + def test_getdictorlist(self): + settings = BaseSettings() + + # No value and no default → {} + assert settings.getdictorlist("MISSING") == {} + + # String: valid JSON dict + settings.set("S_DICT_STR", '{"key": "val"}') + assert settings.getdictorlist("S_DICT_STR") == {"key": "val"} + + # String: valid JSON list + settings.set("S_LIST_STR", '["a", "b"]') + assert settings.getdictorlist("S_LIST_STR") == ["a", "b"] + + # String: invalid JSON → comma-split fallback + settings.set("S_CSV", "a,b,c") + assert settings.getdictorlist("S_CSV") == ["a", "b", "c"] + + # String: valid JSON but not dict or list → ValueError caught → comma-split + settings.set("S_JSON_NUMBER", "123") + assert settings.getdictorlist("S_JSON_NUMBER") == ["123"] + + # Tuple → list + settings.set("S_TUPLE", ("x", "y")) + assert settings.getdictorlist("S_TUPLE") == ["x", "y"] + + # Unsupported type → raises ValueError + settings.set("S_INT", 42) + with pytest.raises(ValueError, match="must be a dict, list, tuple, or string"): + settings.getdictorlist("S_INT") + + # Dict value → deepcopy returned + settings.set("S_DICT", {"key": "val"}) + assert settings.getdictorlist("S_DICT") == {"key": "val"} + + # List value → deepcopy returned + settings.set("S_LIST", ["a", "b"]) + assert settings.getdictorlist("S_LIST") == ["a", "b"] + + def test_repr_pretty_(self): + settings = BaseSettings({"key": "value"}) + mock_p = mock.Mock() + + settings._repr_pretty_(mock_p, cycle=False) + assert mock_p.text.call_count == 1 + + mock_p.reset_mock() + settings._repr_pretty_(mock_p, cycle=True) + mock_p.text.assert_called_once_with(repr(settings)) + def test_getwithbase_invalid_setting_name(self): settings = BaseSettings() with pytest.raises( @@ -710,6 +760,15 @@ def test_remove_from_list(before, name, item, after): assert settings.getpriority(name) == expected_settings.getpriority(name) +def test_deprecated_dns_resolver_setting(): + settings = Settings() + with pytest.warns( + ScrapyDeprecationWarning, + match="The DNS_RESOLVER setting is deprecated", + ): + settings.get("DNS_RESOLVER") + + def test_deprecated_concurrent_requests_per_ip_setting(): settings = Settings({"CONCURRENT_REQUESTS_PER_IP": 1}) with pytest.warns( From dd4549e6f9a92f4844b552aad49ab4431f73514f Mon Sep 17 00:00:00 2001 From: Adrian Date: Wed, 24 Jun 2026 22:48:28 +0200 Subject: [PATCH 25/47] Improve test coverage for downloader middlewares (#7655) * Improve test coverage for downloader middlewares * Improve coverage further --- .../downloadermiddlewares/httpcompression.py | 90 ++++++++++--------- tests/test_downloadermiddleware_cookies.py | 14 +++ ...st_downloadermiddleware_downloadtimeout.py | 6 ++ tests/test_downloadermiddleware_httpcache.py | 1 + ...st_downloadermiddleware_httpcompression.py | 18 +++- tests/test_downloadermiddleware_httpproxy.py | 6 ++ tests/test_downloadermiddleware_offsite.py | 18 ++++ tests/test_downloadermiddleware_redirect.py | 17 ++++ ...wnloadermiddleware_redirect_metarefresh.py | 7 ++ tests/test_downloadermiddleware_stats.py | 14 ++- 10 files changed, 146 insertions(+), 45 deletions(-) diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py index 414c3d8a3..2b1721ced 100644 --- a/scrapy/downloadermiddlewares/httpcompression.py +++ b/scrapy/downloadermiddlewares/httpcompression.py @@ -6,7 +6,7 @@ from logging import getLogger from typing import TYPE_CHECKING, Any from scrapy import Request, Spider, signals -from scrapy.exceptions import IgnoreRequest, NotConfigured +from scrapy.exceptions import IgnoreRequest, NotConfigured, ScrapyDeprecationWarning from scrapy.http import Response, TextResponse from scrapy.responsetypes import responsetypes from scrapy.utils._compression import ( @@ -70,6 +70,12 @@ class HttpCompressionMiddleware: crawler: Crawler | None = None, ): if not crawler: + warnings.warn( + "Instantiating HttpCompressionMiddleware without a 'crawler' " + "argument is deprecated.", + category=ScrapyDeprecationWarning, + stacklevel=2, + ) self.stats = stats self._max_size = 1073741824 self._warn_size = 33554432 @@ -108,49 +114,47 @@ class HttpCompressionMiddleware: ) -> Request | Response: if request.method == "HEAD": return response - if isinstance(response, Response): - content_encoding = response.headers.getlist("Content-Encoding") - if content_encoding: - max_size = request.meta.get("download_maxsize", self._max_size) - warn_size = request.meta.get("download_warnsize", self._warn_size) - try: - decoded_body, content_encoding = self._handle_encoding( - response.body, content_encoding, max_size - ) - except _DecompressionMaxSizeExceeded as e: - raise IgnoreRequest( - f"Ignored response {response} because its body " - f"({len(response.body)} B compressed, " - f"{e.decompressed_size} B decompressed so far) exceeded " - f"DOWNLOAD_MAXSIZE ({max_size} B) during decompression." - ) from e - if len(response.body) < warn_size <= len(decoded_body): - logger.warning( - f"{response} body size after decompression " - f"({len(decoded_body)} B) is larger than the " - f"download warning size ({warn_size} B)." - ) - if content_encoding: - self._warn_unknown_encoding(response, content_encoding) - response.headers["Content-Encoding"] = content_encoding - if self.stats: - self.stats.inc_value( - "httpcompression/response_bytes", - len(decoded_body), - ) - self.stats.inc_value("httpcompression/response_count") - respcls = responsetypes.from_args( - headers=response.headers, url=response.url, body=decoded_body + content_encoding = response.headers.getlist("Content-Encoding") + if content_encoding: + max_size = request.meta.get("download_maxsize", self._max_size) + warn_size = request.meta.get("download_warnsize", self._warn_size) + try: + decoded_body, content_encoding = self._handle_encoding( + response.body, content_encoding, max_size ) - kwargs: dict[str, Any] = {"body": decoded_body} - if issubclass(respcls, TextResponse): - # force recalculating the encoding until we make sure the - # responsetypes guessing is reliable - kwargs["encoding"] = None - response = response.replace(cls=respcls, **kwargs) - if not content_encoding: - del response.headers["Content-Encoding"] - + except _DecompressionMaxSizeExceeded as e: + raise IgnoreRequest( + f"Ignored response {response} because its body " + f"({len(response.body)} B compressed, " + f"{e.decompressed_size} B decompressed so far) exceeded " + f"DOWNLOAD_MAXSIZE ({max_size} B) during decompression." + ) from e + if len(response.body) < warn_size <= len(decoded_body): + logger.warning( + f"{response} body size after decompression " + f"({len(decoded_body)} B) is larger than the " + f"download warning size ({warn_size} B)." + ) + if content_encoding: + self._warn_unknown_encoding(response, content_encoding) + response.headers["Content-Encoding"] = content_encoding + if self.stats: + self.stats.inc_value( + "httpcompression/response_bytes", + len(decoded_body), + ) + self.stats.inc_value("httpcompression/response_count") + respcls = responsetypes.from_args( + headers=response.headers, url=response.url, body=decoded_body + ) + kwargs: dict[str, Any] = {"body": decoded_body} + if issubclass(respcls, TextResponse): + # force recalculating the encoding until we make sure the + # responsetypes guessing is reliable + kwargs["encoding"] = None + response = response.replace(cls=respcls, **kwargs) + if not content_encoding: + del response.headers["Content-Encoding"] return response def _handle_encoding( diff --git a/tests/test_downloadermiddleware_cookies.py b/tests/test_downloadermiddleware_cookies.py index 225562644..f79591020 100644 --- a/tests/test_downloadermiddleware_cookies.py +++ b/tests/test_downloadermiddleware_cookies.py @@ -131,6 +131,20 @@ class TestCookiesMiddleware: ), ) + def test_debug_no_cookies(self): + crawler = get_crawler(settings_dict={"COOKIES_DEBUG": True}) + mw = CookiesMiddleware.from_crawler(crawler) + with LogCapture( + "scrapy.downloadermiddlewares.cookies", + propagate=False, + level=logging.DEBUG, + ) as log: + req = Request("http://scrapytest.org/") + res = Response("http://scrapytest.org/") # no Set-Cookie header + mw.process_response(req, res) + mw.process_request(req) # no cookies to send either + log.check() # no log output since cl is empty in both cases + def test_setting_disabled_cookies_debug(self): crawler = get_crawler(settings_dict={"COOKIES_DEBUG": False}) mw = CookiesMiddleware.from_crawler(crawler) diff --git a/tests/test_downloadermiddleware_downloadtimeout.py b/tests/test_downloadermiddleware_downloadtimeout.py index c744d259c..e6b17960e 100644 --- a/tests/test_downloadermiddleware_downloadtimeout.py +++ b/tests/test_downloadermiddleware_downloadtimeout.py @@ -35,3 +35,9 @@ class TestDownloadTimeoutMiddleware: req.meta["download_timeout"] = 1 assert mw.process_request(req) is None assert req.meta.get("download_timeout") == 1 + + def test_zero_download_timeout(self): + req, spider, mw = self.get_request_spider_mw({"DOWNLOAD_TIMEOUT": 0}) + mw.spider_opened(spider) + assert mw.process_request(req) is None + assert req.meta.get("download_timeout") is None diff --git a/tests/test_downloadermiddleware_httpcache.py b/tests/test_downloadermiddleware_httpcache.py index e5d726764..6c86d7adf 100644 --- a/tests/test_downloadermiddleware_httpcache.py +++ b/tests/test_downloadermiddleware_httpcache.py @@ -135,6 +135,7 @@ class PolicyTestMixin: def test_dont_cache(self): with self._middleware() as mw: self.request.meta["dont_cache"] = True + assert mw.process_request(self.request) is None mw.process_response(self.request, self.response) assert mw.storage.retrieve_response(mw.crawler.spider, self.request) is None diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py index 30caa094f..5c4085657 100644 --- a/tests/test_downloadermiddleware_httpcompression.py +++ b/tests/test_downloadermiddleware_httpcompression.py @@ -11,7 +11,7 @@ from scrapy.downloadermiddlewares.httpcompression import ( ACCEPTED_ENCODINGS, HttpCompressionMiddleware, ) -from scrapy.exceptions import IgnoreRequest, NotConfigured +from scrapy.exceptions import IgnoreRequest, NotConfigured, ScrapyDeprecationWarning from scrapy.http import HtmlResponse, Request, Response from scrapy.responsetypes import responsetypes from scrapy.spiders import Spider @@ -124,6 +124,22 @@ class TestHttpCompression: HttpCompressionMiddleware, ) + def test_no_crawler_constructor(self): + with pytest.warns(ScrapyDeprecationWarning, match="HttpCompressionMiddleware"): + mw = HttpCompressionMiddleware() + buf = BytesIO() + with GzipFile(fileobj=buf, mode="wb") as f: + f.write(b"hello") + body = buf.getvalue() + request = Request("http://scrapytest.org") + response = Response( + "http://scrapytest.org", + body=body, + headers={"Content-Encoding": "gzip"}, + ) + newresponse = mw.process_response(request, response) + assert newresponse.body == b"hello" + def test_process_request(self): request = Request("http://scrapytest.org") assert "Accept-Encoding" not in request.headers diff --git a/tests/test_downloadermiddleware_httpproxy.py b/tests/test_downloadermiddleware_httpproxy.py index a2d421e39..7ed848764 100644 --- a/tests/test_downloadermiddleware_httpproxy.py +++ b/tests/test_downloadermiddleware_httpproxy.py @@ -373,6 +373,12 @@ class TestHttpProxyMiddleware: assert "proxy" not in request.meta assert b"Proxy-Authorization" not in request.headers + def test_proxy_unparseable_url_clears_meta(self): + middleware = HttpProxyMiddleware() + request = Request("http://example.com", meta={"proxy": "//"}) + assert middleware.process_request(request) is None + assert request.meta["proxy"] is None + def test_proxy_authentication_header_disabled_proxy(self): middleware = HttpProxyMiddleware() request = Request( diff --git a/tests/test_downloadermiddleware_offsite.py b/tests/test_downloadermiddleware_offsite.py index edfb15d10..c0b8dc4dd 100644 --- a/tests/test_downloadermiddleware_offsite.py +++ b/tests/test_downloadermiddleware_offsite.py @@ -213,3 +213,21 @@ def test_request_scheduled_invalid_domains(): request = Request(f"https://{letter}.example") with pytest.raises(IgnoreRequest): mw.request_scheduled(request, crawler.spider) + + +def test_repeated_offsite_domain(): + crawler = get_crawler(Spider) + crawler.spider = crawler._create_spider(name="a", allowed_domains=["example.com"]) + mw = OffsiteMiddleware.from_crawler(crawler) + mw.spider_opened(crawler.spider) + req1 = Request("http://other.org/1") + req2 = Request("http://other.org/2") + with pytest.raises(IgnoreRequest): + mw.process_request(req1) + assert "other.org" in mw.domains_seen + assert crawler.stats.get_value("offsite/domains") == 1 + assert crawler.stats.get_value("offsite/filtered") == 1 + with pytest.raises(IgnoreRequest): + mw.process_request(req2) + assert crawler.stats.get_value("offsite/domains") == 1 # not incremented again + assert crawler.stats.get_value("offsite/filtered") == 2 diff --git a/tests/test_downloadermiddleware_redirect.py b/tests/test_downloadermiddleware_redirect.py index 42a25cd5b..1da7bbf3e 100644 --- a/tests/test_downloadermiddleware_redirect.py +++ b/tests/test_downloadermiddleware_redirect.py @@ -4,6 +4,7 @@ from unittest.mock import MagicMock import pytest from scrapy.downloadermiddlewares.redirect import RedirectMiddleware +from scrapy.exceptions import NotConfigured from scrapy.http import Request, Response from scrapy.spidermiddlewares.referer import ( POLICY_NO_REFERRER, @@ -265,6 +266,16 @@ class TestRedirectMiddleware(Base.Test): assert isinstance(req2, Request) assert req2.url == "http://www.example.com/redirected#frag" + def test_redirect_target_has_fragment(self): + url = "http://www.example.com/302#original" + url2 = "http://www.example.com/redirected#target" + req = Request(url) + rsp = Response(url, headers={"Location": url2}, status=302) + + req2 = self.mw.process_response(req, rsp) + assert isinstance(req2, Request) + assert req2.url == "http://www.example.com/redirected#target" + def test_redirect_302_head(self): url = "http://www.example.com/302" url2 = "http://www.example.com/redirected2" @@ -458,3 +469,9 @@ def test_warning_subclass(caplog): assert ( "(if defined in your code base) to override the handle_referer() method" ) in caplog.text + + +def test_not_configured(): + crawler = get_crawler(DefaultSpider, {"REDIRECT_ENABLED": False}) + with pytest.raises(NotConfigured): + RedirectMiddleware.from_crawler(crawler) diff --git a/tests/test_downloadermiddleware_redirect_metarefresh.py b/tests/test_downloadermiddleware_redirect_metarefresh.py index 416fbc2aa..d849cc8fb 100644 --- a/tests/test_downloadermiddleware_redirect_metarefresh.py +++ b/tests/test_downloadermiddleware_redirect_metarefresh.py @@ -7,6 +7,7 @@ from unittest.mock import MagicMock import pytest from scrapy.downloadermiddlewares.redirect import MetaRefreshMiddleware +from scrapy.exceptions import NotConfigured from scrapy.http import HtmlResponse, Request, Response from scrapy.spiders import Spider from scrapy.utils.misc import build_from_crawler @@ -157,3 +158,9 @@ def test_warning_meta_refresh_middleware(caplog): "replace scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware " "with a subclass that overrides the handle_referer() method" ) in caplog.text + + +def test_not_configured(): + crawler = get_crawler(Spider, {"METAREFRESH_ENABLED": False}) + with pytest.raises(NotConfigured): + MetaRefreshMiddleware.from_crawler(crawler) diff --git a/tests/test_downloadermiddleware_stats.py b/tests/test_downloadermiddleware_stats.py index cf7b614c4..5609360a7 100644 --- a/tests/test_downloadermiddleware_stats.py +++ b/tests/test_downloadermiddleware_stats.py @@ -1,4 +1,7 @@ -from scrapy.downloadermiddlewares.stats import DownloaderStats +import pytest + +from scrapy.downloadermiddlewares.stats import DownloaderStats, get_header_size +from scrapy.exceptions import NotConfigured from scrapy.http import Request, Response from scrapy.spiders import Spider from scrapy.utils.test import get_crawler @@ -39,5 +42,14 @@ class TestDownloaderStats: 1, ) + def test_from_crawler_not_configured(self): + crawler = get_crawler(Spider, {"DOWNLOADER_STATS": False}) + with pytest.raises(NotConfigured): + DownloaderStats.from_crawler(crawler) + def teardown_method(self): self.crawler.stats.close_spider() + + +def test_get_header_size_non_list_value(): + assert get_header_size({"Content-Type": "text/html"}) == 0 From 65e8954a0689b65ba8ae187a3045e8d4ed00b84f Mon Sep 17 00:00:00 2001 From: Adrian Date: Thu, 25 Jun 2026 12:35:25 +0200 Subject: [PATCH 26/47] Improve test coverage for spider middlewares (#7664) --- scrapy/spidermiddlewares/referer.py | 6 ++++ tests/test_spidermiddleware_base.py | 5 +++ tests/test_spidermiddleware_depth.py | 25 ++++++++++++++ tests/test_spidermiddleware_referer.py | 43 +++++++++++++++++++++++- tests/test_spidermiddleware_start.py | 12 ++++++- tests/test_spidermiddleware_urllength.py | 8 +++++ 6 files changed, 97 insertions(+), 2 deletions(-) diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py index 2c9452174..264f685c1 100644 --- a/scrapy/spidermiddlewares/referer.py +++ b/scrapy/spidermiddlewares/referer.py @@ -308,6 +308,12 @@ class RefererMiddleware(BaseSpiderMiddleware): # Reference: https://www.w3.org/TR/referrer-policy/#referrer-policy-empty-string self.policies[""] = NoReferrerWhenDowngradePolicy if settings is None: + warn( + "Instantiating RefererMiddleware without a 'settings' argument is " + "deprecated.", + ScrapyDeprecationWarning, + stacklevel=2, + ) return setting_policies = settings.getdict("REFERRER_POLICIES") for policy_name, policy_class_import_path in setting_policies.items(): diff --git a/tests/test_spidermiddleware_base.py b/tests/test_spidermiddleware_base.py index 70326f3f1..4de57fda9 100644 --- a/tests/test_spidermiddleware_base.py +++ b/tests/test_spidermiddleware_base.py @@ -56,6 +56,11 @@ async def test_processed_request(crawler: Crawler) -> None: spider_output = [test_req1, {"foo": "bar"}, test_req2, test_req3] for processed in [ list(mw.process_spider_output(Response("data:,"), spider_output)), + await collect_asyncgen( + mw.process_spider_output_async( + Response("data:,"), as_async_generator(spider_output) + ) + ), await collect_asyncgen(mw.process_start(as_async_generator(spider_output))), ]: assert len(processed) == 3 diff --git a/tests/test_spidermiddleware_depth.py b/tests/test_spidermiddleware_depth.py index 00e547305..307ed2440 100644 --- a/tests/test_spidermiddleware_depth.py +++ b/tests/test_spidermiddleware_depth.py @@ -7,6 +7,7 @@ import pytest from scrapy.http import Request, Response from scrapy.spidermiddlewares.depth import DepthMiddleware from scrapy.spiders import Spider +from scrapy.utils.misc import build_from_crawler from scrapy.utils.test import get_crawler if TYPE_CHECKING: @@ -55,3 +56,27 @@ def test_process_spider_output(mw: DepthMiddleware, stats: StatsCollector) -> No rdm = stats.get_value("request_depth_max") assert rdm == 1 + + +def test_priority_and_non_verbose_stats() -> None: + crawler = get_crawler( + Spider, + {"DEPTH_LIMIT": 0, "DEPTH_STATS_VERBOSE": False, "DEPTH_PRIORITY": 10}, + ) + assert crawler.stats is not None + crawler.stats.open_spider() + try: + mw = build_from_crawler(DepthMiddleware, crawler) + resp = Response("http://toscrape.com") + resp.request = Request("http://toscrape.com") + resp.request.meta["depth"] = 2 + out = list(mw.process_spider_output(resp, [Request("http://toscrape.com")])) + assert len(out) == 1 + # priority is decremented by depth * DEPTH_PRIORITY + assert out[0].priority == -30 + assert out[0].meta["depth"] == 3 + # non-verbose stats don't track per-depth counts but still track the max + assert crawler.stats.get_value("request_depth_count/3") is None + assert crawler.stats.get_value("request_depth_max") == 3 + finally: + crawler.stats.close_spider() diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.py index b3a434c3f..32e83f990 100644 --- a/tests/test_spidermiddleware_referer.py +++ b/tests/test_spidermiddleware_referer.py @@ -6,7 +6,7 @@ from urllib.parse import urlparse import pytest -from scrapy.exceptions import ScrapyDeprecationWarning +from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning from scrapy.http import Request, Response from scrapy.settings import Settings from scrapy.spidermiddlewares.referer import ( @@ -1017,6 +1017,14 @@ class TestPolicyMethodResponseParamRename: response=self.response, resp_or_url=self.response, request=self.request ) + def test_missing_response(self): + with pytest.raises(TypeError, match="Missing required argument: 'response'"): + self.mw.policy(request=self.request) + + def test_missing_request(self): + with pytest.raises(TypeError, match="Missing required argument: 'request'"): + self.mw.policy(response=self.response) + @coroutine_test async def test_response_policy_only_supports_policy_names(): @@ -1115,3 +1123,36 @@ async def test_referer_policies_setting(): ] assert len(output) == 1 assert output[0].headers == {b"Referer": [b"https://python.org/"]} + + +class TestReferrerPolicyHelpers: + def test_origin_referrer_local_scheme(self): + # A local scheme yields no referrer. + assert UnsafeUrlPolicy().origin_referrer("data:,foo") is None + + def test_strip_url_empty(self): + assert UnsafeUrlPolicy().strip_url("") is None + + def test_potentially_trustworthy_data_scheme(self): + assert UnsafeUrlPolicy().potentially_trustworthy("data:,foo") is False + + +def test_default_policy(): + crawler = get_crawler() + mw = build_from_crawler(RefererMiddleware, crawler) + assert mw.default_policy is DefaultReferrerPolicy + + +def test_no_settings_constructor(): + with pytest.warns( + ScrapyDeprecationWarning, + match="Instantiating RefererMiddleware without a 'settings' argument", + ): + mw = RefererMiddleware() + assert mw.default_policy is DefaultReferrerPolicy + + +def test_not_configured_when_disabled(): + crawler = get_crawler(settings_dict={"REFERER_ENABLED": False}) + with pytest.raises(NotConfigured): + build_from_crawler(RefererMiddleware, crawler) diff --git a/tests/test_spidermiddleware_start.py b/tests/test_spidermiddleware_start.py index 26397d37c..def3a3df3 100644 --- a/tests/test_spidermiddleware_start.py +++ b/tests/test_spidermiddleware_start.py @@ -1,4 +1,4 @@ -from scrapy.http import Request +from scrapy.http import Request, Response from scrapy.spidermiddlewares.start import StartSpiderMiddleware from scrapy.spiders import Spider from scrapy.utils.misc import build_from_crawler @@ -23,3 +23,13 @@ class TestMiddleware: async for request in mw.process_start(start()) ] assert result == [True, True, False, "foo"] + + def test_spider_output_not_marked(self): + # Requests from a non-None response (spider output) are not flagged. + crawler = get_crawler(Spider) + mw = build_from_crawler(StartSpiderMiddleware, crawler) + response = Response("data:,") + request = Request("data:,1") + out = list(mw.process_spider_output(response, [request])) + assert out == [request] + assert "is_start_request" not in request.meta diff --git a/tests/test_spidermiddleware_urllength.py b/tests/test_spidermiddleware_urllength.py index 750ae3b07..1ed3a5637 100644 --- a/tests/test_spidermiddleware_urllength.py +++ b/tests/test_spidermiddleware_urllength.py @@ -5,9 +5,11 @@ from typing import TYPE_CHECKING import pytest +from scrapy.exceptions import NotConfigured from scrapy.http import Request, Response from scrapy.spidermiddlewares.urllength import UrlLengthMiddleware from scrapy.spiders import Spider +from scrapy.utils.misc import build_from_crawler from scrapy.utils.test import get_crawler if TYPE_CHECKING: @@ -46,6 +48,12 @@ def test_middleware_works(mw: UrlLengthMiddleware) -> None: assert process_spider_output(mw) == [short_url_req] +def test_not_configured_without_limit() -> None: + crawler = get_crawler(Spider, {"URLLENGTH_LIMIT": 0}) + with pytest.raises(NotConfigured): + build_from_crawler(UrlLengthMiddleware, crawler) + + def test_logging( stats: StatsCollector, mw: UrlLengthMiddleware, caplog: pytest.LogCaptureFixture ) -> None: From c690eac770a9fb803cdf0003ac3dc73e409146dc Mon Sep 17 00:00:00 2001 From: Adrian Date: Thu, 25 Jun 2026 13:34:37 +0200 Subject: [PATCH 27/47] =?UTF-8?q?Document=20=E2=80=9Clogging=20settings?= =?UTF-8?q?=E2=80=9D=20as=20special=20settings=20(#7668)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/topics/practices.rst | 8 ++--- docs/topics/settings.rst | 64 ++++++++++++++++++++++++++++++++++++++- 2 files changed, 67 insertions(+), 5 deletions(-) diff --git a/docs/topics/practices.rst b/docs/topics/practices.rst index 8a04f7ada..e7faf48c4 100644 --- a/docs/topics/practices.rst +++ b/docs/topics/practices.rst @@ -347,10 +347,10 @@ finishes before starting the next one: install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor") react(deferred_f_from_coro_f(crawl)) -.. note:: When running multiple spiders in the same process, :ref:`reactor - settings ` should not have a different value per spider. - Also, :ref:`pre-crawler settings ` cannot be defined - per spider. +.. note:: When running multiple spiders in the same process, :ref:`logging + settings ` and :ref:`reactor settings ` + should not have a different value per spider, and :ref:`pre-crawler + settings ` cannot be defined per spider. .. seealso:: :ref:`run-from-script`. diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 455985a56..c452f88d1 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -361,6 +361,31 @@ All of these settings, except for :setting:`ASYNCIO_EVENT_LOOP`, are only used when the Twisted reactor is used, i.e. when :setting:`TWISTED_REACTOR_ENABLED` is ``True``. +.. _logging-settings: + +Logging settings +---------------- + +**Logging settings** are settings that configure the global root logging +handler installed by :func:`~scrapy.utils.log.configure_logging`. + +These settings can be defined from a spider. However, because only 1 root +logging handler is active per process, these settings cannot use a different +value per spider when :ref:`running multiple spiders in the same process +`. + +These settings are: + +- :setting:`LOG_DATEFORMAT` +- :setting:`LOG_ENABLED` +- :setting:`LOG_ENCODING` +- :setting:`LOG_FILE` +- :setting:`LOG_FILE_APPEND` +- :setting:`LOG_FORMAT` +- :setting:`LOG_LEVEL` +- :setting:`LOG_SHORT_NAMES` +- :setting:`LOG_STDOUT` + .. _topics-settings-ref: Built-in settings reference @@ -481,6 +506,8 @@ Note that the event loop class must inherit from :class:`asyncio.AbstractEventLo :func:`asyncio.set_event_loop`, which will set the specified event loop as the current loop for the current OS thread. +.. note:: This is a :ref:`reactor setting `. + .. setting:: BOT_NAME BOT_NAME @@ -662,6 +689,8 @@ Whether to enable DNS in-memory cache. :setting:`TWISTED_REACTOR_ENABLED` is ``False``, and may have no effect either when :setting:`DNS_RESOLVER` is set to a different resolver. +.. note:: This is a :ref:`reactor setting `. + .. setting:: DNSCACHE_SIZE DNSCACHE_SIZE @@ -671,6 +700,8 @@ Default: ``10000`` DNS in-memory cache size, see :setting:`DNSCACHE_ENABLED`. +.. note:: This is a :ref:`reactor setting `. + .. setting:: TWISTED_DNS_RESOLVER TWISTED_DNS_RESOLVER @@ -688,6 +719,8 @@ take the :setting:`DNS_TIMEOUT` setting into account. .. note:: This setting has no effect when :setting:`TWISTED_REACTOR_ENABLED` is ``False``. +.. note:: This is a :ref:`reactor setting `. + .. setting:: DNS_TIMEOUT DNS_TIMEOUT @@ -703,6 +736,8 @@ Timeout for processing of DNS queries in seconds. Float is supported. :setting:`TWISTED_REACTOR_ENABLED` is ``False``, and may have no effect either when :setting:`DNS_RESOLVER` is set to a different resolver. +.. note:: This is a :ref:`reactor setting `. + .. setting:: DOWNLOADER DOWNLOADER @@ -1442,6 +1477,8 @@ Default: ``True`` Whether to enable logging. +.. note:: This is a :ref:`logging setting `. + .. setting:: LOG_ENCODING LOG_ENCODING @@ -1451,6 +1488,8 @@ Default: ``'utf-8'`` The encoding to use for logging. +.. note:: This is a :ref:`logging setting `. + .. setting:: LOG_FILE LOG_FILE @@ -1460,6 +1499,8 @@ Default: ``None`` File name to use for logging output. If ``None``, standard error will be used. +.. note:: This is a :ref:`logging setting `. + .. setting:: LOG_FILE_APPEND LOG_FILE_APPEND @@ -1470,6 +1511,8 @@ Default: ``True`` If ``False``, the log file specified with :setting:`LOG_FILE` will be overwritten (discarding the output from previous runs, if any). +.. note:: This is a :ref:`logging setting `. + .. setting:: LOG_FORMAT LOG_FORMAT @@ -1481,6 +1524,8 @@ String for formatting log messages. Refer to the :ref:`Python logging documentation ` for the whole list of available placeholders. +.. note:: This is a :ref:`logging setting `. + .. setting:: LOG_DATEFORMAT LOG_DATEFORMAT @@ -1493,6 +1538,8 @@ in :setting:`LOG_FORMAT`. Refer to the :ref:`Python datetime documentation ` for the whole list of available directives. +.. note:: This is a :ref:`logging setting `. + .. setting:: LOG_FORMATTER LOG_FORMATTER @@ -1512,6 +1559,8 @@ Default: ``'DEBUG'`` Minimum level to log. Available levels are: CRITICAL, ERROR, WARNING, INFO, DEBUG. For more info see :ref:`topics-logging`. +.. note:: This is a :ref:`logging setting `. + .. setting:: LOG_STDOUT LOG_STDOUT @@ -1523,6 +1572,8 @@ If ``True``, all standard output (and error) of your process will be redirected to the log. For example if you ``print('hello')`` it will appear in the Scrapy log. +.. note:: This is a :ref:`logging setting `. + .. setting:: LOG_SHORT_NAMES LOG_SHORT_NAMES @@ -1533,6 +1584,8 @@ Default: ``False`` If ``True``, the logs will just contain the root path. If it is set to ``False`` then it displays the component responsible for the log output +.. note:: This is a :ref:`logging setting `. + .. setting:: LOG_VERSIONS LOG_VERSIONS @@ -1695,6 +1748,8 @@ multi-purpose thread pool used by various Scrapy components. Threaded DNS Resolver, BlockingFeedStorage, S3FilesStore just to name a few. Increase this value if you're experiencing problems with insufficient blocking IO. +.. note:: This is a :ref:`reactor setting `. + .. setting:: REDIRECT_PRIORITY_ADJUST REDIRECT_PRIORITY_ADJUST @@ -1921,6 +1976,8 @@ Default: ``'scrapy.spiderloader.SpiderLoader'`` The class that will be used for loading spiders, which must implement the :ref:`topics-api-spiderloader`. +.. note:: This is a :ref:`pre-crawler setting `. + .. setting:: SPIDER_LOADER_WARN_ONLY SPIDER_LOADER_WARN_ONLY @@ -1933,6 +1990,8 @@ it will fail loudly if there is any ``ImportError`` or ``SyntaxError`` exception But you can choose to silence this exception and turn it into a simple warning by setting ``SPIDER_LOADER_WARN_ONLY = True``. +.. note:: This is a :ref:`pre-crawler setting `. + .. setting:: SPIDER_MIDDLEWARES SPIDER_MIDDLEWARES @@ -1978,6 +2037,8 @@ Example: SPIDER_MODULES = ["mybot.spiders_prod", "mybot.spiders_dev"] +.. note:: This is a :ref:`pre-crawler setting `. + .. setting:: STATS_CLASS STATS_CLASS @@ -2049,7 +2110,7 @@ stopped) will not apply. This mode is currently experimental and may not be suitable for production use. It may also not be supported by 3rd-party code. See :ref:`asyncio-without-reactor` for more information about this mode. -.. note:: This setting can't be set :ref:`per-spider `. +.. note:: This is a :ref:`pre-crawler setting `. .. versionadded:: 2.15.0 @@ -2156,6 +2217,7 @@ current platform. For additional information, see :doc:`core/howto/choosing-reactor`. +.. note:: This is a :ref:`reactor setting `. .. setting:: URLLENGTH_LIMIT From 74e6b61071c9fee372da4342604cd35510d03718 Mon Sep 17 00:00:00 2001 From: Adrian Date: Thu, 25 Jun 2026 13:44:51 +0200 Subject: [PATCH 28/47] Make DOWNLOADER_CLIENT_TLS_CIPHERS=None enable Twisted defaults (#7665) * Make DOWNLOADER_CLIENT_TLS_CIPHERS=None enable Twisted defaults * Remove pointless comment * Remove unnecessary mypy comments --- docs/topics/settings.rst | 3 ++ scrapy/core/downloader/contextfactory.py | 11 +++-- scrapy/core/downloader/tls.py | 12 +++--- tests/test_core_downloader.py | 52 +++++++++++++++++++++++- 4 files changed, 66 insertions(+), 12 deletions(-) diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index c452f88d1..b92733ede 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -763,6 +763,9 @@ necessary to access certain HTTPS websites: for example, you may need to use ``'DEFAULT:!DH'`` for a website with weak DH parameters or enable a specific cipher that is not included in ``DEFAULT`` if a website requires it. +Set this setting to ``None`` to use the default ciphers of the underlying TLS +implementation. + .. _OpenSSL cipher list format: https://docs.openssl.org/master/man1/openssl-ciphers/#cipher-list-format .. note:: diff --git a/scrapy/core/downloader/contextfactory.py b/scrapy/core/downloader/contextfactory.py index 40f09ffb2..c1796c723 100644 --- a/scrapy/core/downloader/contextfactory.py +++ b/scrapy/core/downloader/contextfactory.py @@ -17,7 +17,6 @@ from zope.interface.verify import verifyObject from scrapy.core.downloader.tls import ( _TWISTED_VERSION_MAP, - DEFAULT_CIPHERS, _openssl_methods, _ScrapyClientTLSOptions, _ScrapyClientTLSOptions26, @@ -68,11 +67,11 @@ class _ScrapyClientContextFactory(BrowserLikePolicyForHTTPS): self.tls_min_version: TLSVersion | None = tls_min_version self.tls_max_version: TLSVersion | None = tls_max_version self.tls_verbose_logging: bool = tls_verbose_logging # unused - self.tls_ciphers: AcceptableCiphers - if tls_ciphers: - self.tls_ciphers = AcceptableCiphers.fromOpenSSLCipherString(tls_ciphers) - else: - self.tls_ciphers = DEFAULT_CIPHERS + self.tls_ciphers: AcceptableCiphers | None = ( + AcceptableCiphers.fromOpenSSLCipherString(tls_ciphers) + if tls_ciphers + else None + ) self._verify_certificates = verify_certificates @classmethod diff --git a/scrapy/core/downloader/tls.py b/scrapy/core/downloader/tls.py index 3b903b39d..c5cf1156d 100644 --- a/scrapy/core/downloader/tls.py +++ b/scrapy/core/downloader/tls.py @@ -43,6 +43,13 @@ _openssl_methods: dict[str, int] = { def __getattr__(name: str) -> Any: + if name == "DEFAULT_CIPHERS": + warnings.warn( + "scrapy.core.downloader.tls.DEFAULT_CIPHERS is deprecated.", + ScrapyDeprecationWarning, + stacklevel=2, + ) + return AcceptableCiphers.fromOpenSSLCipherString("DEFAULT") deprecated = { "METHOD_TLS": "TLS", "METHOD_TLSv10": "TLSv1.0", @@ -177,8 +184,3 @@ class _ScrapyClientTLSOptions26(ClientTLSOptions): return True return verifyCallback - - -DEFAULT_CIPHERS: AcceptableCiphers = AcceptableCiphers.fromOpenSSLCipherString( - "DEFAULT" -) diff --git a/tests/test_core_downloader.py b/tests/test_core_downloader.py index e348bfb7f..fdd5edc27 100644 --- a/tests/test_core_downloader.py +++ b/tests/test_core_downloader.py @@ -8,7 +8,7 @@ import pytest from pytest_twisted import async_yield_fixture from twisted.internet.protocol import Factory from twisted.internet.protocol import Protocol as TxProtocol -from twisted.internet.ssl import optionsForClientTLS +from twisted.internet.ssl import AcceptableCiphers, optionsForClientTLS from twisted.protocols.tls import TLSMemoryBIOFactory, TLSMemoryBIOProtocol from twisted.web import server, static from twisted.web.client import Agent, BrowserLikePolicyForHTTPS, readBody @@ -180,6 +180,49 @@ class TestContextFactory(TestContextFactoryBase): assert options & 0x4 # OP_LEGACY_SERVER_CONNECT +class TestContextFactoryCiphers(TestContextFactoryBase): + async def _assert_factory_works( + self, server_url: str, client_context_factory: _ScrapyClientContextFactory + ) -> None: + s = "0123456789" * 10 + body = await self.get_page( + server_url + "payload", client_context_factory, body=s + ) + assert body == to_bytes(s) + + def test_default(self) -> None: + """The default 'DEFAULT' value is passed to Twisted as is.""" + crawler = get_crawler() + factory = build_from_crawler(_ScrapyClientContextFactory, crawler) + assert factory.tls_ciphers is not None + # OpenSSLAcceptableCiphers has no __eq__, so compare the parsed ciphers. + assert ( + factory.tls_ciphers._ciphers + == AcceptableCiphers.fromOpenSSLCipherString("DEFAULT")._ciphers + ) + assert factory._get_cert_options_kwargs()["acceptableCiphers"] is not None + + def test_custom(self) -> None: + crawler = get_crawler( + settings_dict={"DOWNLOADER_CLIENT_TLS_CIPHERS": "CAMELLIA256-SHA"} + ) + factory = build_from_crawler(_ScrapyClientContextFactory, crawler) + assert factory.tls_ciphers is not None + assert ( + factory.tls_ciphers._ciphers + == AcceptableCiphers.fromOpenSSLCipherString("CAMELLIA256-SHA")._ciphers + ) + + @coroutine_test + async def test_none(self, server_url: str) -> None: + """A None value enables the Twisted default ciphers.""" + crawler = get_crawler(settings_dict={"DOWNLOADER_CLIENT_TLS_CIPHERS": None}) + factory = build_from_crawler(_ScrapyClientContextFactory, crawler) + assert factory.tls_ciphers is None + assert factory._get_cert_options_kwargs()["acceptableCiphers"] is None + await self._assert_factory_works(server_url, factory) + + class TestContextFactoryTLSMethod(TestContextFactoryBase): async def _assert_factory_works( self, server_url: str, client_context_factory: _ScrapyClientContextFactory @@ -278,3 +321,10 @@ def test_deprecated_tls_module_names() -> None: match="scrapy.core.downloader.tls.openssl_methods is deprecated", ): assert isinstance(tls.openssl_methods, dict) + with pytest.warns( + ScrapyDeprecationWarning, + match="scrapy.core.downloader.tls.DEFAULT_CIPHERS is deprecated", + ): + assert tls.DEFAULT_CIPHERS._ciphers == ( + AcceptableCiphers.fromOpenSSLCipherString("DEFAULT")._ciphers + ) From d8d7de2339b3aeabdc144fa7d0d11cae3ecfd1bb Mon Sep 17 00:00:00 2001 From: Sriniketh24 Date: Fri, 26 Jun 2026 07:56:59 +0530 Subject: [PATCH 29/47] Fix FTPDownloadHandler not closing FTP connection after download (#7667) --- scrapy/core/downloader/handlers/ftp.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/scrapy/core/downloader/handlers/ftp.py b/scrapy/core/downloader/handlers/ftp.py index 6258067c1..9064aa53a 100644 --- a/scrapy/core/downloader/handlers/ftp.py +++ b/scrapy/core/downloader/handlers/ftp.py @@ -119,7 +119,10 @@ class FTPDownloadHandler(BaseDownloadHandler): httpcode = self.CODE_MAPPING.get(ftpcode, self.CODE_MAPPING["default"]) return Response(url=request.url, status=httpcode, body=message.encode()) raise - protocol.close() + finally: + protocol.close() + assert client.transport + client.transport.loseConnection() headers = {"local filename": protocol.filename or b"", "size": protocol.size} body = protocol.filename or protocol.body.read() respcls = responsetypes.from_args(url=request.url, body=body) From 0007676e8d5210f8d0d05ffb12610ca780a3c5ea Mon Sep 17 00:00:00 2001 From: Adrian Date: Fri, 26 Jun 2026 09:21:12 +0200 Subject: [PATCH 30/47] Improve test coverage for http/ (#7672) * Improve test coverage for http/ * Use isinstance() instead of type() --- tests/test_http_cookies.py | 53 +++++++++++++++++++++++++++++++- tests/test_http_headers.py | 1 + tests/test_http_request.py | 10 ++++++ tests/test_http_request_form.py | 3 ++ tests/test_http_response.py | 5 +++ tests/test_http_response_text.py | 7 +++++ 6 files changed, 78 insertions(+), 1 deletion(-) diff --git a/tests/test_http_cookies.py b/tests/test_http_cookies.py index 660b76d08..ce9296764 100644 --- a/tests/test_http_cookies.py +++ b/tests/test_http_cookies.py @@ -1,8 +1,59 @@ +from http.cookiejar import DefaultCookiePolicy + from scrapy.http import Request, Response -from scrapy.http.cookies import WrappedRequest, WrappedResponse +from scrapy.http.cookies import CookieJar, WrappedRequest, WrappedResponse from scrapy.utils.httpobj import urlparse_cached +class TestCookieJar: + def setup_method(self): + self.jar = CookieJar() + self.request = Request("http://example.com/") + self.response = Response( + "http://example.com/", + headers={"Set-Cookie": "name=value; Domain=example.com; Path=/"}, + ) + + def test_extract_cookies(self): + assert len(self.jar) == 0 + self.jar.extract_cookies(self.response, self.request) + assert len(self.jar) == 1 + cookie = next(iter(self.jar)) + assert cookie.name == "name" + assert cookie.value == "value" + assert ".example.com" in self.jar._cookies + + def test_make_cookies_and_set_cookie(self): + cookies = self.jar.make_cookies(self.response, self.request) + assert len(cookies) == 1 + jar = CookieJar() + for cookie in cookies: + jar.set_cookie(cookie) + assert len(jar) == 1 + + def test_clear(self): + self.jar.extract_cookies(self.response, self.request) + assert len(self.jar) == 1 + self.jar.clear() + assert len(self.jar) == 0 + + def test_clear_session_cookies(self): + self.jar.extract_cookies(self.response, self.request) + assert len(self.jar) == 1 + self.jar.clear_session_cookies() + assert len(self.jar) == 0 + + def test_set_policy(self): + policy = DefaultCookiePolicy() + self.jar.set_policy(policy) + assert self.jar.jar._policy is policy + + def test_check_expired_frequency(self): + jar = CookieJar(check_expired_frequency=1) + jar.add_cookie_header(self.request) + assert jar.processed == 1 + + class TestWrappedRequest: def setup_method(self): self.request = Request( diff --git a/tests/test_http_headers.py b/tests/test_http_headers.py index 243aa6afe..aff3562e3 100644 --- a/tests/test_http_headers.py +++ b/tests/test_http_headers.py @@ -140,6 +140,7 @@ class TestHeaders: h1["foo"] = "bar" h1["foo"] = None h1.setdefault("foo", "bar") + assert h1["foo"] is None assert h1.get("foo") is None assert h1.getlist("foo") == [] diff --git a/tests/test_http_request.py b/tests/test_http_request.py index e22689d49..1941b826f 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -24,6 +24,10 @@ class TestRequest: with pytest.raises(TypeError): self.request_class(123) + # priority argument must be an integer + with pytest.raises(TypeError, match="Request priority not an integer"): + self.request_class("http://www.example.com", priority="1") + r = self.request_class("http://www.example.com") assert isinstance(r.url, str) assert r.url == "http://www.example.com" @@ -274,6 +278,12 @@ class TestRequest: assert r4.meta == {} assert r4.dont_filter is False + # the cls argument allows changing the resulting class + custom_request_cls = type("CustomRequest", (self.request_class,), {}) + r5 = r1.replace(cls=custom_request_cls) + assert isinstance(r5, custom_request_cls) + assert r5.url == r1.url + def test_method_always_str(self): r = self.request_class("http://www.example.com", method="POST") assert isinstance(r.method, str) diff --git a/tests/test_http_request_form.py b/tests/test_http_request_form.py index a4f87d50d..c667e48a1 100644 --- a/tests/test_http_request_form.py +++ b/tests/test_http_request_form.py @@ -294,6 +294,9 @@ class TestFormRequest(TestRequest): assert request.method == "GET" request = FormRequest.from_response(response, method="POST") assert request.method == "POST" + # an explicit method=None skips form-method normalization + request = FormRequest.from_response(response, method=None) + assert request.method == "NONE" def test_from_response_override_url(self): response = _buildresponse( diff --git a/tests/test_http_response.py b/tests/test_http_response.py index 079c547c7..a8ea4920e 100644 --- a/tests/test_http_response.py +++ b/tests/test_http_response.py @@ -254,6 +254,11 @@ class TestResponse: with pytest.raises(ValueError, match="url can't be None"): r.follow(None) + def test_follow_None_encoding(self): + r = self.response_class("http://example.com") + with pytest.raises(ValueError, match="encoding can't be None"): + r.follow("foo", encoding=None) + @pytest.mark.xfail( not W3LIB_STRIPS_URLS, reason="https://github.com/scrapy/w3lib/pull/207", diff --git a/tests/test_http_response_text.py b/tests/test_http_response_text.py index 4b3fa2302..507fd1864 100644 --- a/tests/test_http_response_text.py +++ b/tests/test_http_response_text.py @@ -14,6 +14,13 @@ from tests.test_http_response import TestResponse class TestTextResponse(TestResponse): response_class = TextResponse + def test_follow_None_encoding(self): + # unlike the base Response, TextResponse.follow() falls back to the + # response encoding when encoding is None instead of raising + r = self.response_class("http://example.com", body=b"hello", encoding="cp1252") + req = r.follow("foo", encoding=None) + assert req.encoding == "cp1252" + def test_replace(self): super().test_replace() r1 = self.response_class( From 0ccddb4f61b27dd493fd65329d039b88a3333e7b Mon Sep 17 00:00:00 2001 From: Adrian Date: Fri, 26 Jun 2026 11:47:30 +0200 Subject: [PATCH 31/47] Improve test coverage for contracts (#7677) --- tests/test_contracts.py | 180 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 180 insertions(+) diff --git a/tests/test_contracts.py b/tests/test_contracts.py index e80945b93..698044de7 100644 --- a/tests/test_contracts.py +++ b/tests/test_contracts.py @@ -133,6 +133,29 @@ class DemoSpider(Spider): """ return DemoItem(url=response.url) + def returns_request_range_fail(self, response): + """method which returns fewer requests than the expected range + @url http://scrapy.org + @returns requests 2 3 + """ + return Request("http://scrapy.org", callback=self.returns_item) + + def yields_item_and_request(self, response): + """yields one item and one request + @url http://scrapy.org + @returns items 1 1 + @scrapes name url + """ + yield DemoItem(name="test", url=response.url) + yield Request("http://scrapy.org", callback=self.returns_item) + + async def returns_async_gen(self, response): + """async generator callback + @url http://scrapy.org + @returns items 1 1 + """ + yield DemoItem(url=response.url) + def returns_dict_fail(self, response): """method which returns item @url http://scrapy.org @@ -437,6 +460,48 @@ class TestContractsManager: request.callback(response) self.should_error() + def test_returns_invalid_argument_count(self): + spider = DemoSpider() + with pytest.raises(ValueError, match="expected 1, 2 or 3, got 0"): + ReturnsContract(spider.returns_item) + with pytest.raises(ValueError, match="expected 1, 2 or 3, got 4"): + ReturnsContract(spider.returns_item, "items", "1", "2", "3") + + def test_returns_default_bounds(self): + spider = DemoSpider() + contract = ReturnsContract(spider.returns_item, "items") + assert contract.min_bound == 1 + assert contract.max_bound == float("inf") + + def test_returns_range_fail(self): + spider = DemoSpider() + response = ResponseMock() + + request = self.conman.from_method( + spider.returns_request_range_fail, self.results + ) + request.callback(response) + self.should_fail() + assert "expected 2..3" in self.results.failures[-1][-1] + + def test_returns_and_scrapes_ignore_other_types(self): + spider = DemoSpider() + response = ResponseMock() + + # @returns and @scrapes only count matching output objects and skip + # the request that is also yielded. + request = self.conman.from_method(spider.yields_item_and_request, self.results) + request.callback(response) + self.should_succeed() + + def test_testcase_str(self): + spider = DemoSpider() + contract = UrlContract(spider.returns_request, "http://scrapy.org") + assert ( + str(contract.testcase_pre) + == "[demo_spider] returns_request (@url pre-hook)" + ) + def test_scrapes(self): spider = DemoSpider() response = ResponseMock() @@ -570,6 +635,41 @@ class CustomFailContractPostProcess(Contract): raise KeyboardInterrupt("Post-process exception") +class PreProcessSuccessContract(Contract): + name = "pre_success" + + def pre_process(self, response): + return + + +class PreProcessAssertionFailContract(Contract): + name = "pre_assertion_fail" + + def pre_process(self, response): + raise AssertionError("pre-process assertion") + + +class PreProcessErrorContract(Contract): + name = "pre_error" + + def pre_process(self, response): + raise ValueError("pre-process error") + + +class PostProcessSuccessContract(Contract): + name = "post_success" + + def post_process(self, output): + return + + +class PostProcessErrorContract(Contract): + name = "post_error" + + def post_process(self, output): + raise ValueError("post-process error") + + class TestCustomContractPrePostProcess: def setup_method(self): self.results = TextTestResult(stream=None, descriptions=False, verbosity=0) @@ -601,3 +701,83 @@ class TestCustomContractPrePostProcess: assert not self.results.failures assert not self.results.errors + + def test_pre_hook_success(self): + spider = DemoSpider() + response = ResponseMock() + contract = PreProcessSuccessContract(spider.returns_request) + conman = ContractsManager([UrlContract, ReturnsContract, contract]) + + request = conman.from_method(spider.returns_request, self.results) + contract.add_pre_hook(request, self.results) + request.callback(response, **request.cb_kwargs) + + assert not self.results.failures + assert not self.results.errors + + def test_pre_hook_assertion_failure(self): + spider = DemoSpider() + response = ResponseMock() + contract = PreProcessAssertionFailContract(spider.returns_request) + conman = ContractsManager([UrlContract, ReturnsContract, contract]) + + request = conman.from_method(spider.returns_request, self.results) + contract.add_pre_hook(request, self.results) + request.callback(response, **request.cb_kwargs) + + assert self.results.failures + assert not self.results.errors + + def test_pre_hook_error(self): + spider = DemoSpider() + response = ResponseMock() + contract = PreProcessErrorContract(spider.returns_request) + conman = ContractsManager([UrlContract, ReturnsContract, contract]) + + request = conman.from_method(spider.returns_request, self.results) + contract.add_pre_hook(request, self.results) + request.callback(response, **request.cb_kwargs) + + assert self.results.errors + + def test_pre_hook_async_callback(self): + spider = DemoSpider() + response = ResponseMock() + contract = PreProcessSuccessContract(spider.returns_request_async) + request = Request("http://scrapy.org", callback=spider.returns_request_async) + contract.add_pre_hook(request, self.results) + + with pytest.raises(TypeError, match="async callbacks"): + request.callback(response) + + def test_pre_hook_async_generator(self): + spider = DemoSpider() + response = ResponseMock() + contract = PreProcessSuccessContract(spider.returns_async_gen) + request = Request("http://scrapy.org", callback=spider.returns_async_gen) + contract.add_pre_hook(request, self.results) + + with pytest.raises(TypeError, match="async callbacks"): + request.callback(response) + + def test_post_hook_async_generator(self): + spider = DemoSpider() + response = ResponseMock() + contract = PostProcessSuccessContract(spider.returns_async_gen) + request = Request("http://scrapy.org", callback=spider.returns_async_gen) + contract.add_post_hook(request, self.results) + + with pytest.raises(TypeError, match="async callbacks"): + request.callback(response) + + def test_post_hook_error(self): + spider = DemoSpider() + response = ResponseMock() + contract = PostProcessErrorContract(spider.returns_request) + conman = ContractsManager([UrlContract, ReturnsContract, contract]) + + request = conman.from_method(spider.returns_request, self.results) + contract.add_post_hook(request, self.results) + request.callback(response, **request.cb_kwargs) + + assert self.results.errors From 7ab404c72546a9f0ef5bf4a6ee54512d3c1d6851 Mon Sep 17 00:00:00 2001 From: Adrian Date: Fri, 26 Jun 2026 11:58:37 +0200 Subject: [PATCH 32/47] Add a documentation page about security (#7678) --- docs/index.rst | 5 + docs/topics/download-handlers.rst | 4 + docs/topics/downloader-middleware.rst | 2 + docs/topics/security.rst | 207 ++++++++++++++++++++++++++ docs/topics/settings.rst | 16 ++ docs/topics/spider-middleware.rst | 2 + docs/topics/telnetconsole.rst | 4 + 7 files changed, 240 insertions(+) create mode 100644 docs/topics/security.rst diff --git a/docs/index.rst b/docs/index.rst index a46a2ad9f..8e8624a22 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -151,6 +151,7 @@ Solving specific problems topics/debug topics/contracts topics/practices + topics/security topics/broad-crawls topics/developer-tools topics/dynamic-content @@ -175,6 +176,10 @@ Solving specific problems :doc:`topics/practices` Get familiar with some Scrapy common practices. +:doc:`topics/security` + Understand the security implications of Scrapy defaults and how to harden + them. + :doc:`topics/broad-crawls` Tune Scrapy for crawling a lot domains in parallel. diff --git a/docs/topics/download-handlers.rst b/docs/topics/download-handlers.rst index 888bfaf08..c901e01e4 100644 --- a/docs/topics/download-handlers.rst +++ b/docs/topics/download-handlers.rst @@ -39,6 +39,10 @@ for additional schemes and to replace or disable default ones: "sftp": "my.download_handlers.SftpHandler", } +.. seealso:: :ref:`security-unencrypted-protocols` and + :ref:`security-local-resources`, for the security implications of the + default ``http``, ``ftp``, ``file`` and ``data`` handlers. + Replacing HTTP(S) download handlers ----------------------------------- diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 0c1af5276..5649453b1 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -379,6 +379,8 @@ that this risks leaking credentials to unrelated domains. This setting must be explicitly configured whenever :setting:`HTTPAUTH_USER` or :setting:`HTTPAUTH_PASS` is set. +.. seealso:: :ref:`security-credential-leakage` + .. _Basic access authentication: https://en.wikipedia.org/wiki/Basic_access_authentication diff --git a/docs/topics/security.rst b/docs/topics/security.rst new file mode 100644 index 000000000..2ca270045 --- /dev/null +++ b/docs/topics/security.rst @@ -0,0 +1,207 @@ +.. _security: + +======== +Security +======== + +Scrapy defaults are optimized for web scraping, not for the security posture +that you might expect from software that handles untrusted input or runs in a +shared or exposed environment. Some common security practices are unnecessary +for many scraping use cases, and a few can even prevent valid ones (for +example, sites that you must scrape may use misconfigured TLS certificates or +serve content over unencrypted protocols). + +This page highlights the Scrapy defaults that have security implications, so +that you can make an informed decision about whether to keep them, and explains +how to harden them along with the trade-offs involved. + +.. note:: + + None of the options below are silver bullets. Which of them make sense + depends on your threat model: whether the URLs you crawl come from trusted + sources, whether the machine running Scrapy is exposed to a network you do + not control, whether the data you handle is sensitive, and so on. + +.. _security-untrusted-responses: + +Treat responses as untrusted input +================================== + +Regardless of any setting, remember that response data comes from servers you +do not control, even when you trust the site you are crawling, as responses may +be tampered with in transit or the server itself may be compromised. + +Never pass response data to functions that can execute code or otherwise act on +their input in an unsafe way, such as :func:`eval`, :func:`exec`, or +:func:`pickle.loads`, and be careful when writing response data to paths +derived from the response itself. + +TLS connections +=============== + +.. _security-certificate-verification: + +Certificate verification +------------------------ + +By default Scrapy does **not** verify the TLS certificate of HTTPS servers, as +controlled by the :setting:`DOWNLOAD_VERIFY_CERTIFICATES` setting (default: +``False``). + +This default favors reach over security: many sites that are otherwise fine to +scrape have expired, self-signed, or otherwise invalid certificates, and +verifying certificates would make requests to them fail. + +If the integrity of the connection matters to you (for example, to detect +man-in-the-middle attacks), set: + +.. code-block:: python + + DOWNLOAD_VERIFY_CERTIFICATES = True + +* **Pro:** requests to servers with invalid or untrusted certificates fail + instead of silently succeeding, protecting you from some man-in-the-middle + attacks. + +* **Con:** you can no longer scrape sites with misconfigured certificates + without re-disabling verification for them. + +.. _security-tls-protocols-ciphers: + +Protocol versions and ciphers +----------------------------- + +You can restrict the TLS protocol versions that Scrapy accepts through the +:setting:`DOWNLOAD_TLS_MIN_VERSION` and :setting:`DOWNLOAD_TLS_MAX_VERSION` +settings, e.g. to reject obsolete protocol versions. + +By default Scrapy uses the OpenSSL ``DEFAULT`` cipher list +(:setting:`DOWNLOADER_CLIENT_TLS_CIPHERS`), which favors compatibility and still +allows some older, weaker ciphers. Set it to ``None`` to instead use the curated +cipher list of the underlying TLS implementation (Twisted), which excludes weak +ciphers: + +.. code-block:: python + + DOWNLOADER_CLIENT_TLS_CIPHERS = None + +* **Pro:** connections that would negotiate a weak cipher fail instead of + succeeding. + +* **Con:** you can no longer connect to servers that only support the excluded + ciphers. + +.. _security-unencrypted-protocols: + +Unencrypted protocols +===================== + +By default Scrapy enables download handlers for unencrypted protocols, namely +``http://`` and ``ftp://`` (see :setting:`DOWNLOAD_HANDLERS_BASE`). Data sent +and received over these protocols, including any credentials, travels in plain +text and can be read or modified by anyone on the network path. + +If you only crawl over encrypted protocols, you can disable the unencrypted +ones so that no request can accidentally be sent unencrypted: + +.. code-block:: python + + DOWNLOAD_HANDLERS = { + "http": None, + "ftp": None, + } + +* **Pro:** a misconfigured or maliciously-redirected request cannot leak data + over an unencrypted connection, as such requests fail instead. + +* **Con:** you can no longer crawl resources that are only available over those + protocols. + +Note that disabling the ``http`` handler also prevents plain-HTTP requests that +result from following an ``http://`` redirect or link, which is often the point +of disabling it. + +.. _security-local-resources: + +Local and non-network resources +=============================== + +By default Scrapy enables download handlers for the ``file://`` and ``data:`` +schemes (see :setting:`DOWNLOAD_HANDLERS_BASE`). The ``file://`` handler reads +arbitrary files from the local filesystem, limited only by the permissions of +the process running Scrapy. + +This is convenient (for example, to parse a local HTML file), but it is a risk +if any of the URLs you schedule come from an untrusted source: a crafted +``file:///etc/passwd`` URL could read local files. + +If you do not need them, disable these handlers: + +.. code-block:: python + + DOWNLOAD_HANDLERS = { + "file": None, + "data": None, + } + +* **Pro:** crawled URLs cannot be used to read local files or inline data. + +* **Con:** you can no longer fetch ``file://`` or ``data:`` URLs. + +More generally, if you crawl URLs from untrusted sources, consider validating +their schemes (and, where applicable, their hosts) before scheduling requests, +to avoid server-side request forgery (SSRF) and similar issues. + +.. _security-telnet: + +Telnet console +============== + +Scrapy enables the :ref:`telnet console ` by default +(:setting:`TELNETCONSOLE_ENABLED`). The telnet console is a Python shell +running inside the Scrapy process, so anyone who can connect to it can run +arbitrary code in that process. + +By default the console binds to ``127.0.0.1`` (:setting:`TELNETCONSOLE_HOST`) +and is protected by a username (:setting:`TELNETCONSOLE_USERNAME`, default +``scrapy``) and an automatically generated password +(:setting:`TELNETCONSOLE_PASSWORD`), so it is only reachable from the local +machine. + +.. warning:: + + Telnet does not provide any transport-layer security, so the + username/password authentication does not protect the credentials or the + session from anyone able to observe the traffic. Never expose the telnet + console over an untrusted network by changing :setting:`TELNETCONSOLE_HOST` + to a non-local address. + +If you do not use the telnet console, disable it entirely: + +.. code-block:: python + + TELNETCONSOLE_ENABLED = False + +* **Pro:** removes a local code-execution surface and one less listening port. + +* **Con:** you can no longer :ref:`inspect and control a running crawler + ` through it. + +.. _security-credential-leakage: + +Credential leakage across domains +================================= + +Some Scrapy features attach credentials or other sensitive headers to requests, +and a crawl that spans multiple domains can leak them to unintended hosts: + +* HTTP authentication credentials set through + :class:`~scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware` are only + sent to the domain set in :setting:`HTTPAUTH_DOMAIN`. Leave this set to the + intended domain rather than ``None`` so that credentials are not sent to + every domain you crawl. + +* The ``Referer`` header may disclose the URLs you crawl to other sites. The + default :setting:`REFERRER_POLICY` already avoids sending the referrer from + HTTPS to HTTP, but you can tighten it further (for example, to + ``same-origin`` or ``no-referrer``) if needed. diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index b92733ede..11251a92c 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -774,6 +774,8 @@ implementation. handler `, so it's not guaranteed to be supported by all 3rd-party handlers. +.. seealso:: :ref:`security-tls-protocols-ciphers` + .. setting:: DOWNLOAD_TLS_MAX_VERSION DOWNLOAD_TLS_MAX_VERSION @@ -807,6 +809,8 @@ modern environments. by all 3rd-party handlers. Additionally, the set of supported TLS versions depends on the TLS implementation being used by the handler. +.. seealso:: :ref:`security-tls-protocols-ciphers` + .. setting:: DOWNLOAD_TLS_MIN_VERSION DOWNLOAD_TLS_MIN_VERSION @@ -819,6 +823,8 @@ be used by Scrapy. See :setting:`DOWNLOAD_TLS_MAX_VERSION` for the details and limitations. +.. seealso:: :ref:`security-tls-protocols-ciphers` + .. setting:: DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING @@ -979,6 +985,9 @@ enabled in your project. See :setting:`DOWNLOAD_HANDLERS_BASE` for example format. +.. seealso:: :ref:`security-unencrypted-protocols` and + :ref:`security-local-resources` + .. setting:: DOWNLOAD_HANDLERS_BASE DOWNLOAD_HANDLERS_BASE @@ -1026,6 +1035,9 @@ handler (without replacement), place this in your ``settings.py``: "ftp": None, } +.. seealso:: :ref:`security-unencrypted-protocols` and + :ref:`security-local-resources` + .. setting:: DOWNLOAD_SLOTS @@ -1184,6 +1196,8 @@ when making a request and abort the request if the verification fails. certificate problems are logged when this setting is set to ``False``) depends on its implementation. +.. seealso:: :ref:`security-certificate-verification` + .. setting:: DUPEFILTER_CLASS DUPEFILTER_CLASS @@ -2074,6 +2088,8 @@ Default: ``True`` (``False`` when :setting:`TWISTED_REACTOR_ENABLED` is ``False` A boolean which specifies if the :ref:`telnet console ` will be enabled (provided its extension is also enabled). +.. seealso:: :ref:`security-telnet` + .. setting:: TEMPLATES_DIR TEMPLATES_DIR diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index 99bbdf292..e2cd0f986 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -355,6 +355,8 @@ Default: ``'scrapy.spidermiddlewares.referer.DefaultReferrerPolicy'`` using the special ``"referrer_policy"`` :ref:`Request.meta ` key, with the same acceptable values as for the ``REFERRER_POLICY`` setting. +.. seealso:: :ref:`security-credential-leakage` + Acceptable values for REFERRER_POLICY ************************************* diff --git a/docs/topics/telnetconsole.rst b/docs/topics/telnetconsole.rst index e274edc9b..a30258112 100644 --- a/docs/topics/telnetconsole.rst +++ b/docs/topics/telnetconsole.rst @@ -29,6 +29,8 @@ disable it if you want. For more information about the extension itself see .. note:: This feature is not supported when :setting:`TWISTED_REACTOR_ENABLED` is ``False``. +.. seealso:: :ref:`security-telnet` + .. highlight:: none How to access the telnet console @@ -190,6 +192,8 @@ Default: ``'127.0.0.1'`` The interface the telnet console should listen on +.. seealso:: :ref:`security-telnet` + .. setting:: TELNETCONSOLE_USERNAME From 1b940a75ac20401b1e944a65c8901569c06e091b Mon Sep 17 00:00:00 2001 From: Adrian Date: Fri, 26 Jun 2026 13:18:22 +0200 Subject: [PATCH 33/47] Improve the item pipeline docs (#7676) --- docs/topics/item-pipeline.rst | 111 ++++++++++++++++++ .../project/module/pipelines.py.tmpl | 2 +- 2 files changed, 112 insertions(+), 1 deletion(-) diff --git a/docs/topics/item-pipeline.rst b/docs/topics/item-pipeline.rst index 9a53f88fb..f1fa463d8 100644 --- a/docs/topics/item-pipeline.rst +++ b/docs/topics/item-pipeline.rst @@ -57,6 +57,8 @@ Any of these methods may be defined as a coroutine function (``async def``). Item pipeline example ===================== +.. _price-pipeline-example: + Price validation and dropping items with no prices -------------------------------------------------- @@ -246,6 +248,8 @@ returns multiples items with the same id: return item +.. _activating-item-pipeline: + Activating an Item Pipeline component ===================================== @@ -262,3 +266,110 @@ To activate an Item Pipeline component you must add its class to the The integer values you assign to classes in this setting determine the order in which they run: items go through from lower valued to higher valued classes. It's customary to define these numbers in the 0-1000 range. + +A complete example +================== + +The examples above show item pipeline components on their own. In a project, a +pipeline is one of four pieces that work together: the :ref:`item +` your spider produces, the :ref:`spider ` that +yields it, the pipeline that processes it, and the :setting:`ITEM_PIPELINES` +setting that enables the pipeline. + +The following example wires those pieces together to validate the price of +books scraped from `books.toscrape.com`_, reusing the ``PricePipeline`` from +:ref:`price-pipeline-example` above. + +Define the item in ``myproject/items.py``: + +.. code-block:: python + + from dataclasses import dataclass + + + @dataclass + class BookItem: + title: str + price: float + +Yield instances of that item from your spider, e.g. in +``myproject/spiders/books.py``: + +.. skip: next +.. code-block:: python + + import scrapy + + from myproject.items import BookItem + + + class BooksSpider(scrapy.Spider): + name = "books" + start_urls = ["https://books.toscrape.com/"] + + def parse(self, response): + for book in response.css("article.product_pod"): + yield BookItem( + title=book.css("h3 a::attr(title)").get(), + price=float(book.css("p.price_color::text").re_first(r"[\d.]+")), + ) + +Put the ``PricePipeline`` shown earlier in ``myproject/pipelines.py``, and +enable it in ``myproject/settings.py``: + +.. code-block:: python + + ITEM_PIPELINES = { + "myproject.pipelines.PricePipeline": 300, + } + +With these pieces in place, every ``BookItem`` that ``BooksSpider`` yields +passes through ``PricePipeline`` before it reaches the :ref:`feed exports +` or any other output. + +.. _books.toscrape.com: https://books.toscrape.com/ + + +Common pitfalls +=============== + +The pipeline does not run +------------------------- + +A pipeline component only runs if its class is listed in the +:setting:`ITEM_PIPELINES` setting, normally in your project's +:file:`settings.py` file (see :ref:`activating-item-pipeline`). Adding it to +the spider or elsewhere has no effect. + +To confirm that Scrapy loaded your pipeline, look for a line like this near the +start of the crawl log:: + + [scrapy.middleware] INFO: Enabled item pipelines: + ['myproject.pipelines.PricePipeline'] + +If your pipeline is missing from that list, check that its import path matches +the :setting:`ITEM_PIPELINES` entry, and that the setting is not being +overridden, for example by :attr:`~scrapy.Spider.custom_settings` or by a +redefinition of :setting:`ITEM_PIPELINES` in :file:`settings.py`. + +The item is not returned +------------------------ + +:meth:`process_item` must return the item (or raise +:exc:`~scrapy.exceptions.DropItem`). A common mistake is to modify the item but +forget to return it: + +.. code-block:: python + + def process_item(self, item): + ItemAdapter(item)["price"] *= 1.15 + # Bug: returns None, so the next component gets None instead of the item. + +Return the item so that the next component, and the rest of Scrapy, can keep +processing it: + +.. code-block:: python + + def process_item(self, item): + ItemAdapter(item)["price"] *= 1.15 + return item diff --git a/scrapy/templates/project/module/pipelines.py.tmpl b/scrapy/templates/project/module/pipelines.py.tmpl index e845f43e9..a6494b391 100644 --- a/scrapy/templates/project/module/pipelines.py.tmpl +++ b/scrapy/templates/project/module/pipelines.py.tmpl @@ -9,5 +9,5 @@ from itemadapter import ItemAdapter class ${ProjectName}Pipeline: - def process_item(self, item, spider): + def process_item(self, item): return item From edc353c975d2c7fdef480bcaad27c4ff6602682d Mon Sep 17 00:00:00 2001 From: Adrian Date: Fri, 26 Jun 2026 16:40:36 +0200 Subject: [PATCH 34/47] Improve test coverage for shell/ (#7680) * Improve test coverage for shell/ * Address test issues * Make the shell config test more reliable on Windows --- tests/test_command_shell.py | 196 +++++++++++++++++++++++++++++++++++- 1 file changed, 195 insertions(+), 1 deletion(-) diff --git a/tests/test_command_shell.py b/tests/test_command_shell.py index 1585835cc..1c109a333 100644 --- a/tests/test_command_shell.py +++ b/tests/test_command_shell.py @@ -1,19 +1,28 @@ from __future__ import annotations +import importlib.util import os +import signal import sys from io import BytesIO from pathlib import Path -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any +from unittest.mock import AsyncMock, MagicMock, patch import pytest from pexpect.popen_spawn import PopenSpawn +from scrapy import Spider +from scrapy.http import Request, Response +from scrapy.shell import Shell, inspect_response from scrapy.utils.reactor import _asyncio_reactor_path +from scrapy.utils.test import get_crawler from tests import NON_EXISTING_RESOLVABLE, tests_datadir from tests.utils.cmdline import proc +from tests.utils.decorators import coroutine_test if TYPE_CHECKING: + from scrapy.crawler import Crawler from tests.mockserver.http import MockServer @@ -139,6 +148,19 @@ class TestShellCommand: ) assert ret == 0, err + def test_shelp(self) -> None: + ret, out, _ = proc("shell", "-c", "shelp()") + assert ret == 0, out + assert "Available Scrapy objects" in out + + def test_fetch_request_with_callbacks(self, mockserver: MockServer) -> None: + url = mockserver.url("/text") + code = ( + f"fetch(scrapy.Request('{url}', callback=lambda r: r, errback=lambda f: f))" + ) + ret, out, _ = proc("shell", "-c", code) + assert ret == 0, out + class TestInteractiveShell: def test_fetch(self, mockserver: MockServer) -> None: @@ -161,3 +183,175 @@ class TestInteractiveShell: p.wait() # type: ignore[no-untyped-call] logfile.seek(0) assert "Traceback" not in logfile.read().decode() + + @staticmethod + def _isolate_config(env: dict[str, str], config_home: Path) -> None: + """Point every scrapy.cfg location (see + :func:`scrapy.utils.conf.get_sources`) at ``config_home``. + + ``XDG_CONFIG_HOME`` is read by Scrapy on all platforms, while + ``~/.scrapy.cfg`` goes through :func:`os.path.expanduser`, which uses + ``HOME`` on POSIX and ``USERPROFILE`` on Windows. The working directory + stays at the repository root (no scrapy.cfg) so subprocess coverage data + is still collected there. + """ + env.pop("SCRAPY_PYTHON_SHELL", None) + env["HOME"] = str(config_home) + env["USERPROFILE"] = str(config_home) + env["XDG_CONFIG_HOME"] = str(config_home) + + def _run_interactive_shell(self, env: dict[str, str]) -> str: + args = (sys.executable, "-m", "scrapy.cmdline", "shell") + logfile = BytesIO() + p = PopenSpawn(args, env=env, timeout=5) + p.logfile_read = logfile + p.expect_exact("Available Scrapy objects") + p.sendeof() + p.wait() # type: ignore[no-untyped-call] + logfile.seek(0) + return logfile.read().decode() + + @pytest.mark.skipif( + importlib.util.find_spec("IPython") is None, + reason="Without IPython installed, shell=python and the default both " + "select the standard Python shell, so the setting has no observable effect.", + ) + def test_shell_from_cfg(self, tmp_path: Path) -> None: + config_home = tmp_path / "config" + config_home.mkdir() + (config_home / "scrapy.cfg").write_text("[settings]\nshell = python\n") + env = os.environ.copy() + self._isolate_config(env, config_home) + args = (sys.executable, "-m", "scrapy.cmdline", "shell") + logfile = BytesIO() + p = PopenSpawn(args, env=env, timeout=10) + p.logfile_read = logfile + p.expect_exact("Available Scrapy objects") + # The standard Python shell never imports IPython, whereas the IPython + # shell (the default when installed) does; this confirms the configured + # shell=python was honored, regardless of platform-specific prompts. + p.sendline("import sys; print('IPYMODULE', 'IPython' in sys.modules)") + p.expect_exact("IPYMODULE False") + p.sendeof() + p.wait() # type: ignore[no-untyped-call] + logfile.seek(0) + assert "Traceback" not in logfile.read().decode() + + def test_shell_default_shells(self, tmp_path: Path) -> None: + config_home = tmp_path / "config" + config_home.mkdir() + env = os.environ.copy() + self._isolate_config(env, config_home) + assert "Traceback" not in self._run_interactive_shell(env) + + +@pytest.fixture +def restore_sigint(): + """Shell.start() installs SIG_IGN as the SIGINT handler; restore it.""" + handler = signal.getsignal(signal.SIGINT) + try: + yield + finally: + signal.signal(signal.SIGINT, handler) + + +def _no_reactor_crawler(monkeypatch: pytest.MonkeyPatch) -> Crawler: + """Return a crawler that reports ``TWISTED_REACTOR_ENABLED=False``. + + A genuine no-reactor crawler cannot be built while a Twisted reactor is + installed (as it is during the test run), so we build a normal crawler and + make its settings report the reactor as disabled, which is all the shell + code looks at. + """ + crawler = get_crawler() + real_getbool = crawler.settings.getbool + + def fake_getbool(name: str, *args: Any, **kwargs: Any) -> bool: + if name == "TWISTED_REACTOR_ENABLED": + return False + return real_getbool(name, *args, **kwargs) + + monkeypatch.setattr(crawler.settings, "getbool", fake_getbool) + return crawler + + +@pytest.mark.requires_reactor +class TestShell: + """Tests for :class:`~scrapy.shell.Shell` paths with no ``scrapy shell`` + command-line route: those reached through + :func:`scrapy.shell.inspect_response` (called from spider callbacks) or only + through direct API use, hence not covered by the subprocess tests above. + """ + + def test_populate_vars_fetch_not_available(self) -> None: + shell = Shell(get_crawler()) + shell._inthread = False + shell.populate_vars() + assert "fetch" not in shell.vars + + def test_get_help_fetch_not_available(self) -> None: + shell = Shell(get_crawler()) + shell._inthread = False + shell.populate_vars() + help_text = shell.get_help() + assert "fetch(url" not in help_text + assert "shelp()" in help_text + + def test_start_with_request(self, restore_sigint: None) -> None: + shell = Shell(get_crawler(), code="1") + shell.fetch = MagicMock() # type: ignore[method-assign] + request = Request("data:,") + shell.start(request=request) + shell.fetch.assert_called_once_with(request, None) + + def test_start_with_response( + self, restore_sigint: None, capsys: pytest.CaptureFixture[str] + ) -> None: + shell = Shell(get_crawler(), code="response.url") + request = Request("data:,") + response = Response("data:,", request=request) + shell.start(response=response) + assert "data:," in capsys.readouterr().out + assert shell.vars["response"] is response + assert shell.vars["request"] is request + + @patch("scrapy.shell.start_python_console") + def test_inspect_response( + self, mock_console: MagicMock, restore_sigint: None + ) -> None: + crawler = get_crawler() + spider = crawler._create_spider() + response = Response("data:,", request=Request("data:,")) + sigint_handler = signal.getsignal(signal.SIGINT) + inspect_response(response, spider) + mock_console.assert_called_once() + assert signal.getsignal(signal.SIGINT) is sigint_handler + + @coroutine_test + async def test_open_spider_explicit_spider(self) -> None: + crawler = get_crawler() + crawler.engine = MagicMock() + crawler.engine.open_spider_async = AsyncMock() + shell = Shell(crawler) + spider = Spider("test") + await shell._open_spider(spider) + assert shell.spider is spider + assert crawler.spider is spider + crawler.engine.open_spider_async.assert_called_once_with(close_if_idle=False) + + +@pytest.mark.only_asyncio +class TestShellNoReactor: + @coroutine_test + @patch("scrapy.shell.start_python_console") + async def test_inspect_response_no_reactor( + self, + mock_console: MagicMock, + restore_sigint: None, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + crawler = _no_reactor_crawler(monkeypatch) + spider = crawler._create_spider() + response = Response("data:,", request=Request("data:,")) + inspect_response(response, spider) + mock_console.assert_called_once() From cf5607f8bca45fa7c86b053b599a457542641ec1 Mon Sep 17 00:00:00 2001 From: Adrian Date: Fri, 26 Jun 2026 17:35:46 +0200 Subject: [PATCH 35/47] Undeprecated the basic FormRequest API (#7671) --- docs/topics/request-response.rst | 5 +++ scrapy/http/__init__.py | 17 +-------- scrapy/http/request/form.py | 63 ++++++++++++++++++++++++++++---- tests/test_http_request_form.py | 28 +++++++++++++- 4 files changed, 89 insertions(+), 24 deletions(-) diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 700238fe9..5e70a4f98 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -911,6 +911,11 @@ Request subclasses Here is the list of built-in :class:`~scrapy.Request` subclasses. You can also subclass it to implement your own custom functionality. +FormRequest +----------- + +.. autoclass:: scrapy.FormRequest + JsonRequest ----------- diff --git a/scrapy/http/__init__.py b/scrapy/http/__init__.py index e20e894ae..0e5c2b53b 100644 --- a/scrapy/http/__init__.py +++ b/scrapy/http/__init__.py @@ -5,11 +5,9 @@ Use this module (instead of the more specific ones) when importing Headers, Request and Response outside this module. """ -from warnings import catch_warnings, filterwarnings - -from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http.headers import Headers from scrapy.http.request import Request +from scrapy.http.request.form import FormRequest from scrapy.http.request.json_request import JsonRequest from scrapy.http.request.rpc import XmlRpcRequest from scrapy.http.response import Response @@ -17,19 +15,6 @@ from scrapy.http.response.html import HtmlResponse from scrapy.http.response.json import JsonResponse from scrapy.http.response.text import TextResponse from scrapy.http.response.xml import XmlResponse -from scrapy.utils.deprecate import create_deprecated_class - -with catch_warnings(): - filterwarnings("ignore", category=ScrapyDeprecationWarning) - - from scrapy.http.request.form import FormRequest as _FormRequest - - FormRequest = create_deprecated_class( - name="FormRequest", - new_class=_FormRequest, - subclass_warn_message="{cls} inherits from deprecated class {old}, use the form2request library instead.", - instance_warn_message="{cls} is deprecated, use the form2request library instead.", - ) __all__ = [ "FormRequest", diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index 4da595b22..f1a8dbf3b 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -32,19 +32,61 @@ if TYPE_CHECKING: from scrapy.http.response.text import TextResponse -warn( - "The entire scrapy.http.request.form module is deprecated. Use the " - "form2request library instead.", - ScrapyDeprecationWarning, - stacklevel=2, -) - FormdataVType: TypeAlias = str | Iterable[str] FormdataKVType: TypeAlias = tuple[str, FormdataVType] FormdataType: TypeAlias = dict[str, FormdataVType] | list[FormdataKVType] | None class FormRequest(Request): + """A :class:`~scrapy.Request` subclass with a ``formdata`` parameter that + url-encodes the given data and assigns it to the request, which makes it + convenient to send arbitrary form data via HTTP POST or GET without an HTML + ``
`` element to parse. + + .. note:: To build a request from an HTML ```` element found in a + response, use :doc:`form2request ` instead. See + :ref:`form`. + + The remaining arguments are the same as for the :class:`~scrapy.Request` + class and are not documented here. + + :param formdata: a dictionary (or iterable of (key, value) tuples) + containing HTML form data which will be url-encoded. If + :attr:`~scrapy.Request.method` is not given and ``formdata`` is + provided, the method is set to ``"POST"`` and the data is assigned to + the request body; if the method is ``"GET"``, the data is added to the + URL query string instead. + :type formdata: dict or collections.abc.Iterable + + To send data via HTTP POST, simulating an HTML form submission, return a + :class:`~scrapy.FormRequest` object from your spider: + + .. skip: next + .. code-block:: python + + return [ + FormRequest( + url="http://www.example.com/post/action", + formdata={"name": "John Doe", "age": "27"}, + callback=self.after_post, + ) + ] + + To send the data in the URL query string instead, use the ``GET`` method: + + .. skip: next + .. code-block:: python + + return [ + FormRequest( + url="http://www.example.com/search", + method="GET", + formdata={"q": "keyword", "page": "1"}, + callback=self.parse_results, + ) + ] + """ + __slots__ = () valid_form_methods: ClassVar[list[str]] = ["GET", "POST"] @@ -84,6 +126,13 @@ class FormRequest(Request): formcss: str | None = None, **kwargs: Any, ) -> Self: + warn( + "FormRequest.from_response() is deprecated. Use the form2request " + "library instead.", + ScrapyDeprecationWarning, + stacklevel=2, + ) + kwargs.setdefault("encoding", response.encoding) if formcss is not None: diff --git a/tests/test_http_request_form.py b/tests/test_http_request_form.py index c667e48a1..af86b35c0 100644 --- a/tests/test_http_request_form.py +++ b/tests/test_http_request_form.py @@ -1,10 +1,12 @@ from __future__ import annotations import re +import warnings from urllib.parse import parse_qs, unquote_to_bytes import pytest +from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import FormRequest, HtmlResponse from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.python import to_unicode @@ -26,15 +28,39 @@ def _qs(req, encoding="utf-8", to_unicode=False): return parse_qs(uqs, True) +# FormRequest.from_response() is deprecated in favor of form2request, so the +# many tests below that exercise it ignore the resulting deprecation warning. @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") class TestFormRequest(TestRequest): - request_class = FormRequest # type: ignore[assignment] + request_class = FormRequest def assertQueryEqual(self, first, second, msg=None): first = to_unicode(first).split("&") second = to_unicode(second).split("&") assert sorted(first) == sorted(second), msg + def test_init_not_deprecated(self): + # Building a request directly from form data is not deprecated. + with warnings.catch_warnings(): + warnings.simplefilter("error", ScrapyDeprecationWarning) + self.request_class( + "http://www.example.com", formdata={"a": "1"}, method="POST" + ) + self.request_class( + "http://www.example.com", method="GET", formdata={"a": "1"} + ) + + def test_from_response_deprecated(self): + response = _buildresponse( + """ + + """ + ) + with pytest.warns( + ScrapyDeprecationWarning, match=r"FormRequest\.from_response\(\)" + ): + self.request_class.from_response(response) + def test_empty_formdata(self): r1 = self.request_class("http://www.example.com", formdata={}) assert r1.body == b"" From 4b40d2d06a05174b693958145152559c94ebe487 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 26 Jun 2026 23:05:08 +0500 Subject: [PATCH 36/47] Close various garbage collectible resources (mostly in tests) (#7644) * Set TELNETCONSOLE_ENABLED=False in get_crawler(). * Make _get_console_and_portal() a context manager. * Allow PYTHONTRACEMALLOC in tox. * Close the event loop in test_custom_asyncio_loop_enabled_false(). * Close the handler in _uninstall_scrapy_root_handler(). * Close queues in test_squeues.py. * Close empty queues in init_prios(). * Close PopenSpawn stdin and stdout explicitly. * Silence the unclosed socket warning. * Implement more methods in CustomStatsCollector. * Link the Twisted issue. * Close the connection on download_maxsize. * Fix typing. * Add a comment about CustomStatsCollector. * Restore the coverage. * Properly close fixture queues. * More robust file closing in feed storages. * Close the file in TestMarshalItemExporter.test_nonstring_types_item(). * Cleanup temporary file handling in test_exporters.py. * Close the file in test_stats_file_failed(). * Use an explicit TextIOWrapper in XmlItemExporter. --- pyproject.toml | 2 + scrapy/core/downloader/handlers/http11.py | 3 +- scrapy/exporters.py | 11 ++- scrapy/extensions/feedexport.py | 30 +++---- scrapy/pqueues.py | 4 + scrapy/utils/ftp.py | 4 +- scrapy/utils/log.py | 9 ++- scrapy/utils/test.py | 1 + tests/test_command_runspider.py | 1 + tests/test_command_shell.py | 4 + tests/test_crawler_subprocess.py | 8 ++ tests/test_exporters.py | 16 +++- tests/test_extension_telnet.py | 95 ++++++++++++++--------- tests/test_feedexport.py | 8 +- tests/test_squeues.py | 9 +++ tests/test_squeues_request.py | 26 +++++-- tests/test_stats.py | 10 +++ tests/test_utils_asyncio.py | 9 +++ tox.ini | 1 + 19 files changed, 183 insertions(+), 68 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 6b0c561f8..b8d9067f9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -274,6 +274,8 @@ markers = [ ] filterwarnings = [ "ignore::DeprecationWarning:twisted.web.static", + # Twisted doesn't close failed sockets after CannotListenError: https://github.com/twisted/twisted/issues/6108 + "ignore:Exception ignored in. warnsize: diff --git a/scrapy/exporters.py b/scrapy/exporters.py index e18f1e6ed..c3540d724 100644 --- a/scrapy/exporters.py +++ b/scrapy/exporters.py @@ -171,7 +171,15 @@ class XmlItemExporter(BaseItemExporter): super().__init__(**kwargs) if not self.encoding: self.encoding = "utf-8" - self.xg = XMLGenerator(file, encoding=self.encoding) + # copied from xml.sax.saxutils._gettextwriter() + self.stream = TextIOWrapper( + file, + encoding=self.encoding, + errors="xmlcharrefreplace", + newline="\n", + write_through=True, + ) + self.xg = XMLGenerator(self.stream, encoding=self.encoding) def _beautify_newline(self, new_item: bool = False) -> None: if self.indent is not None and (self.indent > 0 or new_item): @@ -199,6 +207,7 @@ class XmlItemExporter(BaseItemExporter): def finish_exporting(self) -> None: self.xg.endElement(self.root_element) self.xg.endDocument() + self.stream.detach() # Avoid closing the wrapped file. def _export_xml_field(self, name: str, serialized_value: Any, depth: int) -> None: self._beautify_indent(depth=depth) diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 8029f85c9..fa708a2e1 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -250,20 +250,22 @@ class S3FeedStorage(BlockingFeedStorage): def _store_in_thread(self, file: IO[bytes]) -> None: file.seek(0) - if self.acl: - self.s3_client.upload_fileobj( - Bucket=self.bucketname, - Key=self.keyname, - Fileobj=file, - ExtraArgs={"ACL": self.acl}, - ) - else: - self.s3_client.upload_fileobj( - Bucket=self.bucketname, - Key=self.keyname, - Fileobj=file, - ) - file.close() + try: + if self.acl: + self.s3_client.upload_fileobj( + Bucket=self.bucketname, + Key=self.keyname, + Fileobj=file, + ExtraArgs={"ACL": self.acl}, + ) + else: + self.s3_client.upload_fileobj( + Bucket=self.bucketname, + Key=self.keyname, + Fileobj=file, + ) + finally: + file.close() class GCSFeedStorage(BlockingFeedStorage): diff --git a/scrapy/pqueues.py b/scrapy/pqueues.py index 0ad0b5d78..2efc43d4b 100644 --- a/scrapy/pqueues.py +++ b/scrapy/pqueues.py @@ -141,10 +141,14 @@ class ScrapyPriorityQueue: q = self.qfactory(priority) if q: self.queues[priority] = q + else: + q.close() if self._start_queue_cls: q = self._sqfactory(priority) if q: self._start_queues[priority] = q + else: + q.close() self.curprio = min(startprios) diff --git a/scrapy/utils/ftp.py b/scrapy/utils/ftp.py index 152f3374e..a3e7a4306 100644 --- a/scrapy/utils/ftp.py +++ b/scrapy/utils/ftp.py @@ -1,4 +1,5 @@ import posixpath +from contextlib import closing from ftplib import FTP, error_perm from posixpath import dirname from typing import IO @@ -32,7 +33,7 @@ def ftp_store_file( """Opens a FTP connection with passed credentials,sets current directory to the directory extracted from given path, then uploads the file to server """ - with FTP() as ftp: + with FTP() as ftp, closing(file): ftp.connect(host, port) ftp.login(username, password) if use_active_mode: @@ -42,4 +43,3 @@ def ftp_store_file( ftp_makedirs_cwd(ftp, dirname) command = "STOR" if overwrite else "APPE" ftp.storbinary(f"{command} {filename}", file) - file.close() diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py index 09b67805d..aa77e692a 100644 --- a/scrapy/utils/log.py +++ b/scrapy/utils/log.py @@ -149,11 +149,12 @@ def install_scrapy_root_handler(settings: Settings) -> None: def _uninstall_scrapy_root_handler() -> None: global _scrapy_root_handler # noqa: PLW0603 - if ( - _scrapy_root_handler is not None - and _scrapy_root_handler in logging.root.handlers - ): + if _scrapy_root_handler is None: + return + + if _scrapy_root_handler in logging.root.handlers: logging.root.removeHandler(_scrapy_root_handler) + _scrapy_root_handler.close() _scrapy_root_handler = None diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py index b4e20c3c6..90ed70262 100644 --- a/scrapy/utils/test.py +++ b/scrapy/utils/test.py @@ -69,6 +69,7 @@ def get_crawler( # When needed, useful settings can be added here, e.g. ones that prevent # deprecation warnings. settings: dict[str, Any] = { + "TELNETCONSOLE_ENABLED": False, **get_reactor_settings(), **(settings_dict or {}), } diff --git a/tests/test_command_runspider.py b/tests/test_command_runspider.py index b1455611e..9dabbc942 100644 --- a/tests/test_command_runspider.py +++ b/tests/test_command_runspider.py @@ -212,6 +212,7 @@ class MySpider(scrapy.Spider): f"Using asyncio event loop: {loop.__module__}.{loop.__class__.__name__}" in log ) + loop.close() def test_no_reactor(self, tmp_path: Path) -> None: log = self.get_log( diff --git a/tests/test_command_shell.py b/tests/test_command_shell.py index 1c109a333..f24200f53 100644 --- a/tests/test_command_shell.py +++ b/tests/test_command_shell.py @@ -181,6 +181,10 @@ class TestInteractiveShell: p.expect_exact("HtmlResponse") p.sendeof() p.wait() # type: ignore[no-untyped-call] + if p.proc.stdin: + p.proc.stdin.close() + if p.proc.stdout: + p.proc.stdout.close() logfile.seek(0) assert "Traceback" not in logfile.read().decode() diff --git a/tests/test_crawler_subprocess.py b/tests/test_crawler_subprocess.py index 146d94ecd..e3f9ac161 100644 --- a/tests/test_crawler_subprocess.py +++ b/tests/test_crawler_subprocess.py @@ -215,6 +215,10 @@ class TestCrawlerProcessSubprocessBase(ScriptRunnerMixin): p.expect_exact("shutting down gracefully") p.expect_exact("Spider closed (shutdown)") p.wait() # type: ignore[no-untyped-call] + if p.proc.stdin: + p.proc.stdin.close() + if p.proc.stdout: + p.proc.stdout.close() def test_shutdown_graceful(self) -> None: self._test_shutdown_graceful() @@ -232,6 +236,10 @@ class TestCrawlerProcessSubprocessBase(ScriptRunnerMixin): p.kill(sig) p.expect_exact("forcing unclean shutdown") p.wait() # type: ignore[no-untyped-call] + if p.proc.stdin: + p.proc.stdin.close() + if p.proc.stdout: + p.proc.stdout.close() @coroutine_test async def test_shutdown_forced(self) -> None: diff --git a/tests/test_exporters.py b/tests/test_exporters.py index 2fded613d..877e0708b 100644 --- a/tests/test_exporters.py +++ b/tests/test_exporters.py @@ -3,7 +3,6 @@ import json import marshal import pickle import re -import tempfile from abc import ABC, abstractmethod from datetime import datetime from io import BytesIO @@ -242,7 +241,6 @@ class TestPickleItemExporterDataclass(TestPickleItemExporter): class TestMarshalItemExporter(TestBaseItemExporter): def _get_exporter(self, **kwargs): - self.output = tempfile.TemporaryFile() return MarshalItemExporter(self.output, **kwargs) def _check_output(self): @@ -252,7 +250,7 @@ class TestMarshalItemExporter(TestBaseItemExporter): def test_nonstring_types_item(self): item = self._get_nonstring_types_item() item.pop("time") # datetime is not marshallable - fp = tempfile.TemporaryFile() + fp = BytesIO() ie = MarshalItemExporter(fp) ie.start_exporting() ie.export_item(item) @@ -260,6 +258,7 @@ class TestMarshalItemExporter(TestBaseItemExporter): del ie # See the first “del self.ie” in this file for context. fp.seek(0) assert marshal.load(fp) == item + fp.close() class TestMarshalItemExporterDataclass(TestMarshalItemExporter): @@ -269,7 +268,11 @@ class TestMarshalItemExporterDataclass(TestMarshalItemExporter): class TestCsvItemExporter(TestBaseItemExporter): def _get_exporter(self, **kwargs): - self.output = tempfile.TemporaryFile() + # We need a fresh instance for each exporter, because + # CsvItemExporter.stream.__del__() closes the underlying file + # (CsvItemExporter.finish_exporting() calls detach() but not all tests + # call it). + self.output = BytesIO() return CsvItemExporter(self.output, **kwargs) def assertCsvEqual(self, first, second, msg=None): @@ -389,6 +392,11 @@ class TestCsvItemExporterDataclass(TestCsvItemExporter): class TestXmlItemExporter(TestBaseItemExporter): def _get_exporter(self, **kwargs): + # We need a fresh instance for each exporter, because + # XmlItemExporter.stream.__del__() closes the underlying file + # (XmlItemExporter.finish_exporting() calls detach() but not all tests + # call it). + self.output = BytesIO() return XmlItemExporter(self.output, **kwargs) def assertXmlEquivalent(self, first, second, msg=None): diff --git a/tests/test_extension_telnet.py b/tests/test_extension_telnet.py index 3f9135867..20c801558 100644 --- a/tests/test_extension_telnet.py +++ b/tests/test_extension_telnet.py @@ -1,61 +1,86 @@ +from __future__ import annotations + +from contextlib import contextmanager +from typing import TYPE_CHECKING, Any + import pytest from twisted.conch.telnet import ITelnetProtocol from twisted.cred import credentials from scrapy.extensions.telnet import TelnetConsole +from scrapy.utils.defer import maybe_deferred_to_future from scrapy.utils.test import get_crawler -from tests.utils.decorators import inline_callbacks_test +from tests.utils.decorators import coroutine_test + +if TYPE_CHECKING: + from collections.abc import Generator + + from scrapy.crawler import Crawler pytestmark = pytest.mark.requires_reactor # TelnetConsole requires a reactor -class TestTelnetExtension: - def _get_console_and_portal(self, settings=None): - crawler = get_crawler(settings_dict=settings) - console = TelnetConsole(crawler) +def _get_crawler(settings_dict: dict[str, Any] | None = None) -> Crawler: + settings = { + "TELNETCONSOLE_ENABLED": True, + **(settings_dict or {}), + } + return get_crawler(settings_dict=settings) - # This function has some side effects we don't need for this test - console._get_telnet_vars = dict - console.start_listening() - protocol = console.protocol() - portal = protocol.protocolArgs[0] +@contextmanager +def _get_console_and_portal( + settings: dict[str, Any] | None = None, +) -> Generator[tuple[TelnetConsole, Any]]: + crawler = _get_crawler(settings_dict=settings) + console = TelnetConsole(crawler) - return console, portal + # This function has some side effects we don't need for this test + console._get_telnet_vars = dict # type: ignore[method-assign] - @inline_callbacks_test - def test_bad_credentials(self): - console, portal = self._get_console_and_portal() + console.start_listening() + protocol = console.protocol() + portal = protocol.protocolArgs[0] + + try: + yield console, portal + finally: + console.stop_listening() + + +@coroutine_test +async def test_bad_credentials() -> None: + with _get_console_and_portal() as (_, portal): creds = credentials.UsernamePassword(b"username", b"password") d = portal.login(creds, None, ITelnetProtocol) with pytest.raises(ValueError, match="Invalid credentials"): - yield d - console.stop_listening() + await maybe_deferred_to_future(d) - @inline_callbacks_test - def test_good_credentials(self): - console, portal = self._get_console_and_portal() + +@coroutine_test +async def test_good_credentials() -> None: + with _get_console_and_portal() as (console, portal): creds = credentials.UsernamePassword( console.username.encode("utf8"), console.password.encode("utf8") ) d = portal.login(creds, None, ITelnetProtocol) - yield d - console.stop_listening() + await maybe_deferred_to_future(d) - @inline_callbacks_test - def test_custom_credentials(self): - settings = { - "TELNETCONSOLE_USERNAME": "user", - "TELNETCONSOLE_PASSWORD": "pass", - } - console, portal = self._get_console_and_portal(settings=settings) + +@coroutine_test +async def test_custom_credentials() -> None: + settings = { + "TELNETCONSOLE_USERNAME": "user", + "TELNETCONSOLE_PASSWORD": "pass", + } + with _get_console_and_portal(settings=settings) as (_, portal): creds = credentials.UsernamePassword(b"user", b"pass") d = portal.login(creds, None, ITelnetProtocol) - yield d - console.stop_listening() + await maybe_deferred_to_future(d) - def test_invalid_reversed_portrange(self): - settings = {"TELNETCONSOLE_PORT": [2, 1]} - console = TelnetConsole(get_crawler(settings_dict=settings)) - with pytest.raises(ValueError, match=r"invalid portrange: \[2, 1\]"): - console.start_listening() + +def test_invalid_reversed_portrange() -> None: + settings = {"TELNETCONSOLE_PORT": [2, 1]} + console = TelnetConsole(_get_crawler(settings_dict=settings)) + with pytest.raises(ValueError, match=r"invalid portrange: \[2, 1\]"): + console.start_listening() diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index c1d6f04eb..d7ea60cc8 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -87,6 +87,7 @@ class DummyBlockingFeedStorage(BlockingFeedStorage): class FailingBlockingFeedStorage(DummyBlockingFeedStorage): def _store_in_thread(self, file): + file.close() raise OSError("Cannot store") @@ -477,9 +478,14 @@ class TestFeedExport(TestFeedExportBase): }, } crawler = get_crawler(ItemSpider, settings) + + def store(file: IO[bytes]) -> None: + file.close() + raise KeyError("foo") + with mock.patch( "scrapy.extensions.feedexport.FileFeedStorage.store", - side_effect=KeyError("foo"), + side_effect=store, ): yield crawler.crawl(mockserver=self.mockserver) assert "feedexport/failed_count/FileFeedStorage" in crawler.stats.get_stats() diff --git a/tests/test_squeues.py b/tests/test_squeues.py index ddc12766c..8544602af 100644 --- a/tests/test_squeues.py +++ b/tests/test_squeues.py @@ -42,6 +42,7 @@ def nonserializable_object_test(self): ValueError, match=r"unmarshallable object|can't pickle Selector objects" ): q.push(sel) + q.close() class FifoDiskQueueTestMixin: @@ -53,6 +54,7 @@ class FifoDiskQueueTestMixin: assert q.pop() == "a" assert q.pop() == 123 assert q.pop() == {"a": "dict"} + q.close() test_nonserializable_object = nonserializable_object_test @@ -93,6 +95,7 @@ class PickleFifoDiskQueueTest(t.FifoDiskQueueTest, FifoDiskQueueTestMixin): i2 = q.pop() assert isinstance(i2, MyItem) assert i == i2 + q.close() def test_serialize_loader(self): q = self.queue() @@ -102,6 +105,7 @@ class PickleFifoDiskQueueTest(t.FifoDiskQueueTest, FifoDiskQueueTestMixin): assert isinstance(loader2, MyLoader) assert loader2.default_item_class is MyItem assert loader2.name_out("x") == "xx" + q.close() def test_serialize_request_recursive(self): q = self.queue() @@ -112,6 +116,7 @@ class PickleFifoDiskQueueTest(t.FifoDiskQueueTest, FifoDiskQueueTestMixin): assert isinstance(r2, Request) assert r.url == r2.url assert r2.meta["request"] is r2 + q.close() def test_non_pickable_object(self): q = self.queue() @@ -158,6 +163,7 @@ class LifoDiskQueueTestMixin: assert q.pop() == {"a": "dict"} assert q.pop() == 123 assert q.pop() == "a" + q.close() test_nonserializable_object = nonserializable_object_test @@ -178,6 +184,7 @@ class PickleLifoDiskQueueTest(t.LifoDiskQueueTest, LifoDiskQueueTestMixin): i2 = q.pop() assert isinstance(i2, MyItem) assert i == i2 + q.close() def test_serialize_loader(self): q = self.queue() @@ -187,6 +194,7 @@ class PickleLifoDiskQueueTest(t.LifoDiskQueueTest, LifoDiskQueueTestMixin): assert isinstance(loader2, MyLoader) assert loader2.default_item_class is MyItem assert loader2.name_out("x") == "xx" + q.close() def test_serialize_request_recursive(self): q = self.queue() @@ -197,3 +205,4 @@ class PickleLifoDiskQueueTest(t.LifoDiskQueueTest, LifoDiskQueueTestMixin): assert isinstance(r2, Request) assert r.url == r2.url assert r2.meta["request"] is r2 + q.close() diff --git a/tests/test_squeues_request.py b/tests/test_squeues_request.py index 847a76ab6..c779b005f 100644 --- a/tests/test_squeues_request.py +++ b/tests/test_squeues_request.py @@ -70,7 +70,6 @@ class TestRequestQueueBase(ABC): if test_peek: assert q.peek() is None assert q.pop() is None - q.close() @pytest.mark.parametrize("test_peek", [True, False]) def test_order(self, q: queuelib.queue.BaseQueue, test_peek: bool): @@ -108,7 +107,6 @@ class TestRequestQueueBase(ABC): if test_peek: assert q.peek() is None assert q.pop() is None - q.close() class TestPickleFifoDiskQueueRequest(TestRequestQueueBase): @@ -116,9 +114,13 @@ class TestPickleFifoDiskQueueRequest(TestRequestQueueBase): @pytest.fixture def q(self, crawler, tmp_path): - return PickleFifoDiskQueue.from_crawler( + queue = PickleFifoDiskQueue.from_crawler( crawler=crawler, key=str(tmp_path / "pickle" / "fifo") ) + try: + yield queue + finally: + queue.close() class TestPickleLifoDiskQueueRequest(TestRequestQueueBase): @@ -126,9 +128,13 @@ class TestPickleLifoDiskQueueRequest(TestRequestQueueBase): @pytest.fixture def q(self, crawler, tmp_path): - return PickleLifoDiskQueue.from_crawler( + queue = PickleLifoDiskQueue.from_crawler( crawler=crawler, key=str(tmp_path / "pickle" / "lifo") ) + try: + yield queue + finally: + queue.close() class TestMarshalFifoDiskQueueRequest(TestRequestQueueBase): @@ -136,9 +142,13 @@ class TestMarshalFifoDiskQueueRequest(TestRequestQueueBase): @pytest.fixture def q(self, crawler, tmp_path): - return MarshalFifoDiskQueue.from_crawler( + queue = MarshalFifoDiskQueue.from_crawler( crawler=crawler, key=str(tmp_path / "marshal" / "fifo") ) + try: + yield queue + finally: + queue.close() class TestMarshalLifoDiskQueueRequest(TestRequestQueueBase): @@ -146,9 +156,13 @@ class TestMarshalLifoDiskQueueRequest(TestRequestQueueBase): @pytest.fixture def q(self, crawler, tmp_path): - return MarshalLifoDiskQueue.from_crawler( + queue = MarshalLifoDiskQueue.from_crawler( crawler=crawler, key=str(tmp_path / "marshal" / "lifo") ) + try: + yield queue + finally: + queue.close() class TestFifoMemoryQueueRequest(TestRequestQueueBase): diff --git a/tests/test_stats.py b/tests/test_stats.py index 6e6aa0cc4..05f609fda 100644 --- a/tests/test_stats.py +++ b/tests/test_stats.py @@ -131,6 +131,7 @@ class TestStatsCollector: @coroutine_test async def test_deprecated_spider_arg_custom_collector(self) -> None: + # the class reimplements many methods because those are called during the test crawl class CustomStatsCollector: def __init__(self, crawler): self._stats = {} @@ -141,10 +142,19 @@ class TestStatsCollector: def get_stats(self, spider=None): return self._stats + def get_value(self, key, default=None, spider=None): + return self._stats.get(key, default) + + def set_value(self, key, value, spider=None): + self._stats[key] = value + def inc_value(self, key, count=1, start=0, spider=None): d = self._stats d[key] = d.setdefault(key, start) + count + def max_value(self, key, value, spider=None) -> None: + self._stats[key] = max(self._stats.setdefault(key, value), value) + def close_spider(self, spider, reason): pass diff --git a/tests/test_utils_asyncio.py b/tests/test_utils_asyncio.py index 5532b4a31..a198d1c09 100644 --- a/tests/test_utils_asyncio.py +++ b/tests/test_utils_asyncio.py @@ -149,3 +149,12 @@ class TestAsyncioLoopingCall: with pytest.raises(TypeError): looping_call.start(0.1) assert not looping_call.running + + @coroutine_test + async def test_looping_function_raises( + self, caplog: pytest.LogCaptureFixture + ) -> None: + looping_call = AsyncioLoopingCall(lambda: 1 / 0) + looping_call.start(0.1) + assert not looping_call.running + assert "Error calling the AsyncioLoopingCall function" in caplog.text diff --git a/tox.ini b/tox.ini index 2fa7b455d..5017d7636 100644 --- a/tox.ini +++ b/tox.ini @@ -53,6 +53,7 @@ deps = {[test-requirements]deps} pytest >= 8.4.1 # https://github.com/pytest-dev/pytest/pull/13502 passenv = + PYTHONTRACEMALLOC PYTEST_ADDOPTS S3_TEST_FILE_URI AWS_ACCESS_KEY_ID From 185d6b9a20b7d0e77f4c60435d17e5072bd4d704 Mon Sep 17 00:00:00 2001 From: greymoth Date: Sat, 27 Jun 2026 04:53:32 +0900 Subject: [PATCH 37/47] Fix request_to_curl() corrupting dict cookies with bytes keys/values (#7675) * Fix request_to_curl() corrupting dict cookies with bytes keys/values PR #7603 made the list-cookie branch of request_to_curl() bytes-safe via _cookie_value_to_unicode(), but left the sibling dict-cookie branch using raw f-string interpolation. A dict cookie with bytes keys/values (a supported and common form, e.g. Request(url, cookies={b"k": b"v"})) was rendered as --cookie 'b'k'=b'v'' instead of --cookie 'k=v', producing a broken curl command. Route the dict branch through the same _cookie_value_to_unicode() helper, mirroring the list branch. Add a regression test. Co-Authored-By: Claude Opus 4.8 (1M context) * fix: apply _cookie_value_to_unicode to list-branch cookie key/value --------- Co-authored-by: Claude Opus 4.8 (1M context) --- scrapy/utils/request.py | 7 +++++-- tests/test_utils_request.py | 26 ++++++++++++++++++++++++++ 2 files changed, 31 insertions(+), 2 deletions(-) diff --git a/scrapy/utils/request.py b/scrapy/utils/request.py index 398403d90..4a85526c0 100644 --- a/scrapy/utils/request.py +++ b/scrapy/utils/request.py @@ -204,13 +204,16 @@ def request_to_curl(request: Request) -> str: cookies = "" if request.cookies: if isinstance(request.cookies, dict): - cookie = "; ".join(f"{k}={v}" for k, v in request.cookies.items()) + cookie = "; ".join( + f"{_cookie_value_to_unicode(k)}={_cookie_value_to_unicode(v)}" + for k, v in request.cookies.items() + ) cookies = f"--cookie '{cookie}'" elif isinstance(request.cookies, list): cookie = "; ".join( f"{_cookie_value_to_unicode(c['name'])}={_cookie_value_to_unicode(c['value'])}" if "name" in c and "value" in c - else f"{next(iter(c.keys()))}={next(iter(c.values()))}" + else f"{_cookie_value_to_unicode(next(iter(c.keys())))}={_cookie_value_to_unicode(next(iter(c.values())))}" for c in request.cookies ) cookies = f"--cookie '{cookie}'" diff --git a/tests/test_utils_request.py b/tests/test_utils_request.py index e4967d4e7..63926e9d2 100644 --- a/tests/test_utils_request.py +++ b/tests/test_utils_request.py @@ -401,6 +401,19 @@ class TestRequestToCurl: ) self._test_request(request_object, expected_curl_command) + def test_cookies_dict_bytes(self): + request_object = Request( + "https://www.httpbin.org/post", + method="POST", + cookies={b"foo": b"bar"}, + body=json.dumps({"foo": "bar"}), + ) + expected_curl_command = ( + "curl -X POST https://www.httpbin.org/post" + " --data-raw '{\"foo\": \"bar\"}' --cookie 'foo=bar'" + ) + self._test_request(request_object, expected_curl_command) + def test_cookies_list(self): request_object = Request( "https://www.httpbin.org/post", @@ -455,3 +468,16 @@ class TestRequestToCurl: " --data-raw '{\"foo\": \"bar\"}' --cookie 'foo=1'" ) self._test_request(request_object, expected_curl_command) + + def test_cookies_list_bytes_nonstandard_key(self): + request_object = Request( + "https://www.httpbin.org/post", + method="POST", + cookies=[{b"foo": b"bar"}], + body=json.dumps({"foo": "bar"}), + ) + expected_curl_command = ( + "curl -X POST https://www.httpbin.org/post" + " --data-raw '{\"foo\": \"bar\"}' --cookie 'foo=bar'" + ) + self._test_request(request_object, expected_curl_command) From 9559cbee1e7344d9746b2e2164d10a9d6575f9d1 Mon Sep 17 00:00:00 2001 From: Adrian Date: Mon, 29 Jun 2026 14:21:20 +0200 Subject: [PATCH 38/47] Solve timing issues with HTTP cache tests? (#7692) --- tests/test_downloadermiddleware_httpcache.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/tests/test_downloadermiddleware_httpcache.py b/tests/test_downloadermiddleware_httpcache.py index 6c86d7adf..9d5d6874e 100644 --- a/tests/test_downloadermiddleware_httpcache.py +++ b/tests/test_downloadermiddleware_httpcache.py @@ -6,6 +6,7 @@ import tempfile import time from contextlib import contextmanager from typing import TYPE_CHECKING, Any +from unittest import mock import pytest @@ -47,7 +48,6 @@ class TestBase: settings = { "HTTPCACHE_ENABLED": True, "HTTPCACHE_DIR": self.tmpdir, - "HTTPCACHE_EXPIRATION_SECS": 1, "HTTPCACHE_IGNORE_HTTP_CODES": [], "HTTPCACHE_POLICY": self.policy_class, "HTTPCACHE_STORAGE": self.storage_class, @@ -94,7 +94,7 @@ class StorageTestMixin: """Mixin containing storage-specific test methods.""" def test_storage(self): - with self._storage() as (storage, crawler): + with self._storage(HTTPCACHE_EXPIRATION_SECS=1) as (storage, crawler): request2 = self.request.copy() assert storage.retrieve_response(crawler.spider, request2) is None @@ -103,15 +103,17 @@ class StorageTestMixin: assert isinstance(response2, HtmlResponse) # content-type header self.assertEqualResponse(self.response, response2) - time.sleep(2) # wait for cache to expire - assert storage.retrieve_response(crawler.spider, request2) is None + expired = time.time() + storage.expiration_secs + 1 + with mock.patch("scrapy.extensions.httpcache.time", return_value=expired): + assert storage.retrieve_response(crawler.spider, request2) is None def test_storage_never_expire(self): with self._storage(HTTPCACHE_EXPIRATION_SECS=0) as (storage, crawler): assert storage.retrieve_response(crawler.spider, self.request) is None storage.store_response(crawler.spider, self.request, self.response) - time.sleep(0.5) # give the chance to expire - assert storage.retrieve_response(crawler.spider, self.request) + future = time.time() + 10**6 + with mock.patch("scrapy.extensions.httpcache.time", return_value=future): + assert storage.retrieve_response(crawler.spider, self.request) def test_storage_no_content_type_header(self): """Test that the response body is used to get the right response class From 4b2b56f3848111d55151ccfefddfc6b3d84d1302 Mon Sep 17 00:00:00 2001 From: Javier <114426455+javidiazz@users.noreply.github.com> Date: Mon, 29 Jun 2026 15:34:12 +0200 Subject: [PATCH 39/47] Update on Request objects (#7286) --- docs/topics/request-response.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 5e70a4f98..f2fcfb0b5 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -63,7 +63,7 @@ Request objects .. invisible-code-block: python - from scrapy.http import Request + from scrapy import Request 1. Using a dict: From 6591cb756c16bdb0ce85425071fea84a7cddaf53 Mon Sep 17 00:00:00 2001 From: tanishqtayade Date: Mon, 29 Jun 2026 22:07:06 +0530 Subject: [PATCH 40/47] Fix to LocalCache with limit=0 still stores items rather than disabling the cache (#7663) * Fix cell-var-from-loop in _send_catch_log_deferred Replace lambda capturing receiver by reference with a default argument to capture it by value, fixing a potential bug where all deferred callbacks could reference the last receiver in the loop instead of their respective receivers. Removes the pylint disable comment and TODO that were suppressing this issue. * chore: trigger CI rerun for mypy network error * Fix mypy error: pass receiver via addBoth args instead of lambda default * style: apply pre-commit ruff formatting * fix: LocalCache with limit=0 incorrectly stores items When limit=0 is passed to LocalCache (e.g. when DNSCACHE_ENABLED=False), the condition 'if self.limit' evaluates to False due to Python's truthiness rules, causing items to be stored despite the cache being disabled. This leads to an unbounded memory leak during long crawls when DNS caching is explicitly disabled. Fix changes the condition to 'if self.limit is not None' and adds an early return when limit=0 to correctly handle the disabled cache case. * test: add resolver-level regression tests for DNSCACHE_ENABLED=False Add two regression tests that verify DNS results are not stored in dnscache when DNSCACHE_ENABLED=False (cache_size=0): - test_caching_hostname_resolver_dnscache_disabled_rejects_storage: verifies _CachingResolutionReceiver does not write to dnscache when CachingHostnameResolver is initialized with cache_size=0 - test_caching_threaded_resolver_dnscache_disabled_rejects_storage: verifies dnscache rejects storage at the LocalCache level when limit=0 * test: drop misleading threaded resolver test it was writing directly to dnscache, not actually going through CachingThreadedResolver at all. the threaded resolver already has its own if dnscache.limit: guard so the bug doesnt affect it anyway. keeping only the hostname resolver test which covers the actual bug path through _CachingResolutionReceiver * test: clean up comments in resolver test * test: remove unnecessary comment --- scrapy/utils/datatypes.py | 4 +++- tests/test_resolver.py | 18 ++++++++++++++++++ tests/test_utils_datatypes.py | 10 ++++++++++ 3 files changed, 31 insertions(+), 1 deletion(-) diff --git a/scrapy/utils/datatypes.py b/scrapy/utils/datatypes.py index 4e65c062e..dd0e062d0 100644 --- a/scrapy/utils/datatypes.py +++ b/scrapy/utils/datatypes.py @@ -152,7 +152,9 @@ class LocalCache(OrderedDict[_KT, _VT]): self.limit: int | None = limit def __setitem__(self, key: _KT, value: _VT) -> None: - if self.limit: + if self.limit is not None: + if self.limit == 0: + return while len(self) >= self.limit: self.popitem(last=False) super().__setitem__(key, value) diff --git a/tests/test_resolver.py b/tests/test_resolver.py index 83fc8693b..7cca45ed1 100644 --- a/tests/test_resolver.py +++ b/tests/test_resolver.py @@ -53,3 +53,21 @@ def test_caching_hostname_resolver_no_addresses_not_cached(): resolver.resolveHostName(Mock(), "example.com") assert "example.com" not in dnscache + + +def test_caching_hostname_resolver_dnscache_disabled_rejects_storage(): + + def fake_resolve(receiver, *_): + receiver.resolutionBegan(Mock()) + receiver.addressResolved(Mock()) + receiver.resolutionComplete() + return receiver + + reactor = Mock() + reactor.nameResolver.resolveHostName.side_effect = fake_resolve + + resolver = CachingHostnameResolver(reactor, cache_size=0) + resolver.resolveHostName(Mock(), "example.com") + + assert "example.com" not in dnscache + assert len(dnscache) == 0 diff --git a/tests/test_utils_datatypes.py b/tests/test_utils_datatypes.py index ba6b82503..fe1f60c7d 100644 --- a/tests/test_utils_datatypes.py +++ b/tests/test_utils_datatypes.py @@ -309,6 +309,16 @@ class TestLocalCache: assert str(x) in cache assert cache[str(x)] == x + def test_cache_with_zero_limit(self): + cache = LocalCache(limit=0) + cache["a"] = 1 + cache["b"] = 2 + cache["c"] = 3 + assert len(cache) == 0 + assert "a" not in cache + assert "b" not in cache + assert "c" not in cache + class TestLocalWeakReferencedCache: def test_cache_with_limit(self): From 52147017b43baff1f9b4668e46ab11341aa8f400 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 29 Jun 2026 21:49:09 +0500 Subject: [PATCH 41/47] Cleanup cookie handling in request_to_curl() (#7684) * Adjust CookiesT. * Drop list of plain cookie dicts support from request_to_curl(). * Extract _decode_cookie(). * Unify logging. * Extract _to_verbose_cookies(). * Sync and type hint _cookie_to_set_cookie_value() in tests. * Add tests for bytes in Request.cookies. * Type hint assertCookieValEqual(). --- scrapy/downloadermiddlewares/cookies.py | 32 ++------- scrapy/http/request/__init__.py | 2 +- scrapy/utils/request.py | 73 +++++++++++++------ tests/test_downloadermiddleware_cookies.py | 81 +++++++++++++--------- tests/test_utils_request.py | 44 +++--------- 5 files changed, 116 insertions(+), 116 deletions(-) diff --git a/scrapy/downloadermiddlewares/cookies.py b/scrapy/downloadermiddlewares/cookies.py index cd8c2abca..34d071fd5 100644 --- a/scrapy/downloadermiddlewares/cookies.py +++ b/scrapy/downloadermiddlewares/cookies.py @@ -12,6 +12,7 @@ from scrapy.http.cookies import CookieJar from scrapy.utils.decorators import _warn_spider_arg from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.python import to_unicode +from scrapy.utils.request import _decode_cookie, _to_verbose_cookies if TYPE_CHECKING: from collections.abc import Iterable, Sequence @@ -134,29 +135,10 @@ class CookiesMiddleware: Given a dict consisting of cookie components, return its string representation. Decode from bytes if necessary. """ - decoded = {} + decoded = _decode_cookie(cookie, request) + if decoded is None: + return None flags = set() - for key in ("name", "value", "path", "domain"): - value = cookie.get(key) - if value is None: - if key in {"name", "value"}: - msg = f"Invalid cookie found in request {request}: {cookie} ('{key}' is missing)" - logger.warning(msg) - return None - continue - if isinstance(value, (bool, float, int, str)): - decoded[key] = str(value) - else: - assert isinstance(value, bytes) - try: - decoded[key] = value.decode("utf8") - except UnicodeDecodeError: - logger.warning( - "Non UTF-8 encoded cookie found in request %s: %s", - request, - cookie, - ) - decoded[key] = value.decode("latin1", errors="replace") for flag in ("secure",): value = cookie.get(flag, _UNSET) if value is _UNSET or not value: @@ -177,11 +159,7 @@ class CookiesMiddleware: """ if not request.cookies: return () - cookies: Iterable[VerboseCookie] - if isinstance(request.cookies, dict): - cookies = tuple({"name": k, "value": v} for k, v in request.cookies.items()) - else: - cookies = request.cookies + cookies: Iterable[VerboseCookie] = _to_verbose_cookies(request.cookies) for cookie in cookies: cookie.setdefault("secure", urlparse_cached(request).scheme == "https") formatted = filter(None, (self._format_cookie(c, request) for c in cookies)) diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py index 7db648a45..73c2e7dd4 100644 --- a/scrapy/http/request/__init__.py +++ b/scrapy/http/request/__init__.py @@ -51,7 +51,7 @@ class VerboseCookie(TypedDict): secure: NotRequired[bool] -CookiesT: TypeAlias = dict[str, str] | list[VerboseCookie] +CookiesT: TypeAlias = dict[str | bytes, str | bytes] | list[VerboseCookie] RequestTypeVar = TypeVar("RequestTypeVar", bound="Request") diff --git a/scrapy/utils/request.py b/scrapy/utils/request.py index 4a85526c0..b2fd9a834 100644 --- a/scrapy/utils/request.py +++ b/scrapy/utils/request.py @@ -7,6 +7,7 @@ from __future__ import annotations import hashlib import json +import logging from typing import TYPE_CHECKING, Any, Protocol from urllib.parse import urlunparse from weakref import WeakKeyDictionary @@ -25,6 +26,9 @@ if TYPE_CHECKING: from typing_extensions import Self from scrapy.crawler import Crawler + from scrapy.http.request import CookiesT, VerboseCookie + +logger = logging.getLogger(__name__) _fingerprint_cache: WeakKeyDictionary[ @@ -179,17 +183,52 @@ def _get_method(obj: Any, name: Any) -> Any: raise ValueError(f"Method {name!r} not found in: {obj}") from None -def _cookie_value_to_unicode(value: str | bytes | float) -> str: - if isinstance(value, bytes): - return value.decode() - return str(value) +def _to_verbose_cookies(cookies: CookiesT) -> list[VerboseCookie]: + """Return a list of verbose cookies from ``request.cookies``. + + The list of dicts form is returned as is, the dict one is converted first. + """ + if isinstance(cookies, dict): + return [{"name": k, "value": v} for k, v in cookies.items()] + return cookies + + +def _decode_cookie(cookie: VerboseCookie, request: Request) -> dict[str, str] | None: + """Return a dict with non-flag verbose cookie values converted to strings. + + ``name``, ``value``, ``path``, ``domain`` are included, ``secure`` isn't. + """ + + decoded = {} + for key in ("name", "value", "path", "domain"): + value = cookie.get(key) + if value is None: + if key in {"name", "value"}: + logger.warning( + f"Invalid cookie found in request {request}:" + f" {cookie} ('{key}' is missing)" + ) + return None + continue + if isinstance(value, (bool, float, int, str)): + decoded[key] = str(value) + else: + assert isinstance(value, bytes) + try: + decoded[key] = value.decode("utf8") + except UnicodeDecodeError: + logger.warning( + f"Non UTF-8 encoded cookie found in request {request}: {cookie}", + ) + decoded[key] = value.decode("latin1", errors="replace") + return decoded def request_to_curl(request: Request) -> str: """ Converts a :class:`~scrapy.Request` object to a curl command. - :param :class:`~scrapy.Request`: Request object to be converted + :param request: Request object to be converted :return: string containing the curl command """ method = request.method @@ -201,22 +240,14 @@ def request_to_curl(request: Request) -> str: ) url = request.url - cookies = "" - if request.cookies: - if isinstance(request.cookies, dict): - cookie = "; ".join( - f"{_cookie_value_to_unicode(k)}={_cookie_value_to_unicode(v)}" - for k, v in request.cookies.items() - ) - cookies = f"--cookie '{cookie}'" - elif isinstance(request.cookies, list): - cookie = "; ".join( - f"{_cookie_value_to_unicode(c['name'])}={_cookie_value_to_unicode(c['value'])}" - if "name" in c and "value" in c - else f"{_cookie_value_to_unicode(next(iter(c.keys())))}={_cookie_value_to_unicode(next(iter(c.values())))}" - for c in request.cookies - ) - cookies = f"--cookie '{cookie}'" + + cookie_list: list[VerboseCookie] = _to_verbose_cookies(request.cookies) + pairs = [ + f"{decoded['name']}={decoded['value']}" + for c in cookie_list + if (decoded := _decode_cookie(c, request)) is not None + ] + cookies = f"--cookie '{'; '.join(pairs)}'" if pairs else "" curl_cmd = f"curl -X {method} {url} {data} {headers} {cookies}".strip() return " ".join(curl_cmd.split()) diff --git a/tests/test_downloadermiddleware_cookies.py b/tests/test_downloadermiddleware_cookies.py index f79591020..b4419dc67 100644 --- a/tests/test_downloadermiddleware_cookies.py +++ b/tests/test_downloadermiddleware_cookies.py @@ -1,4 +1,5 @@ import logging +from collections.abc import Iterable import pytest from testfixtures import LogCapture @@ -8,30 +9,34 @@ from scrapy.downloadermiddlewares.defaultheaders import DefaultHeadersMiddleware from scrapy.downloadermiddlewares.redirect import RedirectMiddleware from scrapy.exceptions import NotConfigured from scrapy.http import Request, Response +from scrapy.http.request import CookiesT, VerboseCookie from scrapy.utils.python import to_bytes +from scrapy.utils.request import _to_verbose_cookies from scrapy.utils.spider import DefaultSpider from scrapy.utils.test import get_crawler UNSET = object() -def _cookie_to_set_cookie_value(cookie): +def _cookie_to_set_cookie_value(cookie: VerboseCookie) -> str | None: """Given a cookie defined as a dictionary with name and value keys, and optional path and domain keys, return the equivalent string that can be associated to a ``Set-Cookie`` header.""" decoded = {} for key in ("name", "value", "path", "domain"): - if cookie.get(key) is None: - if key in ("name", "value"): + value = cookie.get(key) + if value is None: + if key in {"name", "value"}: return None continue - if isinstance(cookie[key], (bool, float, int, str)): - decoded[key] = str(cookie[key]) + if isinstance(value, (bool, float, int, str)): + decoded[key] = str(value) else: + assert isinstance(value, bytes) try: - decoded[key] = cookie[key].decode("utf8") + decoded[key] = value.decode("utf8") except UnicodeDecodeError: - decoded[key] = cookie[key].decode("latin1", errors="replace") + decoded[key] = value.decode("latin1", errors="replace") cookie_str = f"{decoded.pop('name')}={decoded.pop('value')}" for key, value in decoded.items(): # path, domain @@ -39,24 +44,30 @@ def _cookie_to_set_cookie_value(cookie): return cookie_str -def _cookies_to_set_cookie_list(cookies): +def _cookies_to_set_cookie_list(cookies: CookiesT) -> Iterable[str]: """Given a group of cookie defined either as a dictionary or as a list of dictionaries (i.e. in a format supported by the cookies parameter of Request), return the equivalent list of strings that can be associated to a ``Set-Cookie`` header.""" if not cookies: return [] - if isinstance(cookies, dict): - cookies = ({"name": k, "value": v} for k, v in cookies.items()) - return filter(None, (_cookie_to_set_cookie_value(cookie) for cookie in cookies)) + return filter( + None, + ( + _cookie_to_set_cookie_value(cookie) + for cookie in _to_verbose_cookies(cookies) + ), + ) class TestCookiesMiddleware: - def assertCookieValEqual(self, first, second, msg=None): - def split_cookies(cookies): + @staticmethod + def assertCookieValEqual(first: bytes | str | None, second: bytes | str) -> None: + def split_cookies(cookies: bytes | str) -> list[bytes]: return sorted([s.strip() for s in to_bytes(cookies).split(b";")]) - assert split_cookies(first) == split_cookies(second), msg + assert first is not None + assert split_cookies(first) == split_cookies(second) def setup_method(self): crawler = get_crawler(DefaultSpider) @@ -372,21 +383,25 @@ class TestCookiesMiddleware: assert self.mw.process_request(req3) is None self.assertCookieValEqual(req3.headers["Cookie"], "a=new; c=d; e=f") - def test_request_cookies_encoding(self): - # 1) UTF8-encoded bytes - req1 = Request("http://example.org", cookies={"a": "á".encode()}) - assert self.mw.process_request(req1) is None - self.assertCookieValEqual(req1.headers["Cookie"], b"a=\xc3\xa1") - - # 2) Non UTF8-encoded bytes - req2 = Request("http://example.org", cookies={"a": "á".encode("latin1")}) - assert self.mw.process_request(req2) is None - self.assertCookieValEqual(req2.headers["Cookie"], b"a=\xc3\xa1") - - # 3) String - req3 = Request("http://example.org", cookies={"a": "á"}) - assert self.mw.process_request(req3) is None - self.assertCookieValEqual(req3.headers["Cookie"], b"a=\xc3\xa1") + @pytest.mark.parametrize( + "cookies", + [ + # UTF8-encoded bytes + {"a": "á".encode()}, + # non UTF8-encoded bytes + {"a": "á".encode("latin1")}, + # string + {"a": "á"}, + # key as bytes + {b"a": "á"}, + # key and value as bytes + {b"a": "á".encode()}, + ], + ) + def test_request_cookies_encoding(self, cookies: CookiesT) -> None: + req = Request("http://example.org", cookies=cookies) + assert self.mw.process_request(req) is None + self.assertCookieValEqual(req.headers["Cookie"], b"a=\xc3\xa1") @pytest.mark.xfail(reason="Cookie header is not currently being processed") def test_request_headers_cookie_encoding(self): @@ -410,7 +425,7 @@ class TestCookiesMiddleware: Invalid cookies are logged as warnings and discarded """ with LogCapture( - "scrapy.downloadermiddlewares.cookies", + "scrapy.utils.request", propagate=False, level=logging.INFO, ) as lc: @@ -425,19 +440,19 @@ class TestCookiesMiddleware: assert self.mw.process_request(req3) is None lc.check( ( - "scrapy.downloadermiddlewares.cookies", + "scrapy.utils.request", "WARNING", "Invalid cookie found in request :" " {'value': 'bar', 'secure': False} ('name' is missing)", ), ( - "scrapy.downloadermiddlewares.cookies", + "scrapy.utils.request", "WARNING", "Invalid cookie found in request :" " {'name': 'foo', 'secure': False} ('value' is missing)", ), ( - "scrapy.downloadermiddlewares.cookies", + "scrapy.utils.request", "WARNING", "Invalid cookie found in request :" " {'name': 'foo', 'value': None, 'secure': False} ('value' is missing)", diff --git a/tests/test_utils_request.py b/tests/test_utils_request.py index 63926e9d2..1642a932b 100644 --- a/tests/test_utils_request.py +++ b/tests/test_utils_request.py @@ -354,16 +354,18 @@ class TestCustomRequestFingerprinter: class TestRequestToCurl: - def _test_request(self, request_object, expected_curl_command): + def _test_request( + self, request_object: Request, expected_curl_command: str + ) -> None: curl_command = request_to_curl(request_object) assert curl_command == expected_curl_command - def test_get(self): + def test_get(self) -> None: request_object = Request("https://www.example.com") expected_curl_command = "curl -X GET https://www.example.com" self._test_request(request_object, expected_curl_command) - def test_post(self): + def test_post(self) -> None: request_object = Request( "https://www.httpbin.org/post", method="POST", @@ -374,7 +376,7 @@ class TestRequestToCurl: ) self._test_request(request_object, expected_curl_command) - def test_headers(self): + def test_headers(self) -> None: request_object = Request( "https://www.httpbin.org/post", method="POST", @@ -388,7 +390,7 @@ class TestRequestToCurl: ) self._test_request(request_object, expected_curl_command) - def test_cookies_dict(self): + def test_cookies_dict(self) -> None: request_object = Request( "https://www.httpbin.org/post", method="POST", @@ -401,7 +403,7 @@ class TestRequestToCurl: ) self._test_request(request_object, expected_curl_command) - def test_cookies_dict_bytes(self): + def test_cookies_dict_bytes(self) -> None: request_object = Request( "https://www.httpbin.org/post", method="POST", @@ -414,20 +416,7 @@ class TestRequestToCurl: ) self._test_request(request_object, expected_curl_command) - def test_cookies_list(self): - request_object = Request( - "https://www.httpbin.org/post", - method="POST", - cookies=[{"foo": "bar"}], - body=json.dumps({"foo": "bar"}), - ) - expected_curl_command = ( - "curl -X POST https://www.httpbin.org/post" - " --data-raw '{\"foo\": \"bar\"}' --cookie 'foo=bar'" - ) - self._test_request(request_object, expected_curl_command) - - def test_cookies_list_verbose(self): + def test_cookies_list_verbose(self) -> None: request_object = Request( "https://www.httpbin.org/post", method="POST", @@ -448,7 +437,7 @@ class TestRequestToCurl: ) self._test_request(request_object, expected_curl_command) - def test_cookies_list_verbose_non_string_value(self): + def test_cookies_list_verbose_non_string_value(self) -> None: request_object = Request( "https://www.httpbin.org/post", method="POST", @@ -468,16 +457,3 @@ class TestRequestToCurl: " --data-raw '{\"foo\": \"bar\"}' --cookie 'foo=1'" ) self._test_request(request_object, expected_curl_command) - - def test_cookies_list_bytes_nonstandard_key(self): - request_object = Request( - "https://www.httpbin.org/post", - method="POST", - cookies=[{b"foo": b"bar"}], - body=json.dumps({"foo": "bar"}), - ) - expected_curl_command = ( - "curl -X POST https://www.httpbin.org/post" - " --data-raw '{\"foo\": \"bar\"}' --cookie 'foo=bar'" - ) - self._test_request(request_object, expected_curl_command) From 6ad8a043cae3d4e0db906788cb60a4c8a3fd7d91 Mon Sep 17 00:00:00 2001 From: Adrian Date: Tue, 30 Jun 2026 12:05:42 +0200 Subject: [PATCH 42/47] Fix genspider --editor (#7683) --- scrapy/commands/edit.py | 23 +++++++++++++-- scrapy/commands/genspider.py | 12 +++++--- tests/test_command_genspider.py | 50 +++++++++++++++++++++++++++++++++ tests/test_commands.py | 28 ++++++++++++++++++ 4 files changed, 106 insertions(+), 7 deletions(-) diff --git a/scrapy/commands/edit.py b/scrapy/commands/edit.py index cd7c57f28..fa75cb09c 100644 --- a/scrapy/commands/edit.py +++ b/scrapy/commands/edit.py @@ -1,12 +1,29 @@ -import argparse +from __future__ import annotations + import os +import shlex +import subprocess import sys -from typing import Any, ClassVar +from pathlib import Path +from typing import TYPE_CHECKING, Any, ClassVar from scrapy.commands import ScrapyCommand from scrapy.exceptions import UsageError from scrapy.spiderloader import get_spider_loader +if TYPE_CHECKING: + import argparse + + +def _edit_file(editor: str, file_path: str | os.PathLike[str]) -> int: + """Open ``file_path`` with ``editor`` and return the editor exit code. + + ``editor`` may include arguments (e.g. ``"code -w"``); it is split with + :func:`shlex.split` and the file is passed as a separate argument, so no + shell is involved. + """ + return subprocess.call([*shlex.split(editor), os.fspath(file_path)]) # noqa: S603 + class Command(ScrapyCommand): requires_project = True @@ -45,4 +62,4 @@ class Command(ScrapyCommand): sfile = sys.modules[spidercls.__module__].__file__ assert sfile sfile = sfile.replace(".pyc", ".py") - self.exitcode = os.system(f'{editor} "{sfile}"') # noqa: S605 + self.exitcode = _edit_file(editor, Path(sfile)) diff --git a/scrapy/commands/genspider.py b/scrapy/commands/genspider.py index cc8624fa1..4277232c3 100644 --- a/scrapy/commands/genspider.py +++ b/scrapy/commands/genspider.py @@ -1,6 +1,5 @@ from __future__ import annotations -import os import shutil import string from importlib import import_module @@ -10,12 +9,14 @@ from urllib.parse import urlparse import scrapy from scrapy.commands import ScrapyCommand +from scrapy.commands.edit import _edit_file from scrapy.exceptions import UsageError from scrapy.spiderloader import get_spider_loader from scrapy.utils.template import render_templatefile, string_camelcase if TYPE_CHECKING: import argparse + import os def sanitize_module_name(module_name: str) -> str: @@ -118,9 +119,11 @@ class Command(ScrapyCommand): template_file = self._find_template(opts.template) if template_file: - self._genspider(module, name, url, opts.template, template_file) + spider_file = self._genspider( + module, name, url, opts.template, template_file + ) if opts.edit: - self.exitcode = os.system(f'scrapy edit "{name}"') # noqa: S605 + self.exitcode = _edit_file(self.settings["EDITOR"], spider_file) def _generate_template_variables( self, @@ -148,7 +151,7 @@ class Command(ScrapyCommand): url: str, template_name: str, template_file: str | os.PathLike[str], - ) -> None: + ) -> Path: """Generate the spider module, based on the given template""" assert self.settings is not None tvars = self._generate_template_variables(module, name, url, template_name) @@ -168,6 +171,7 @@ class Command(ScrapyCommand): ) if spiders_module: print(f"in module:\n {spiders_module.__name__}.{module}") + return Path(spider_file) def _find_template(self, template: str) -> Path | None: template_file = Path(self.templates_dir, f"{template}.tmpl") diff --git a/tests/test_command_genspider.py b/tests/test_command_genspider.py index 67e3eb50a..94db14c30 100644 --- a/tests/test_command_genspider.py +++ b/tests/test_command_genspider.py @@ -1,6 +1,7 @@ from __future__ import annotations import re +import sys from pathlib import Path import pytest @@ -9,6 +10,13 @@ from tests.test_commands import TestProjectBase from tests.utils.cmdline import call, proc +def write_recording_editor(editor: Path) -> None: + """Create an executable editor script that writes the path it is asked to + open (its last argument) into the file given as its first argument.""" + editor.write_text('#!/bin/sh\nprintf "%s" "$2" > "$1"\n', encoding="utf-8") + editor.chmod(0o755) + + def find_in_file(filename: Path, regex: str) -> re.Match[str] | None: """Find first pattern occurrence in file""" pattern = re.compile(regex) @@ -63,6 +71,28 @@ class TestGenspiderCommand(TestProjectBase): assert call("genspider", "--dump=basic", cwd=proj_path) == 0 assert call("genspider", "-d", "basic", cwd=proj_path) == 0 + @pytest.mark.skipif( + sys.platform == "win32", reason="requires a POSIX shell editor script" + ) + def test_edit(self, proj_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + spider = proj_path / self.project_name / "spiders" / "example2.py" + edited = proj_path / "edited.txt" + editor = proj_path / "fake-editor.sh" + write_recording_editor(editor) + # The extra argument exercises shlex-splitting of the EDITOR value. + monkeypatch.setenv("EDITOR", f"{editor} {edited}") + + returncode, _, err = proc( + "genspider", "--edit", "example2", "example2.com", cwd=proj_path + ) + + assert returncode == 0, err + assert "ModuleNotFoundError" not in err + assert spider.exists() + assert (proj_path / edited.read_text(encoding="utf-8")).resolve() == ( + spider.resolve() + ) + def test_same_name_as_project(self, proj_path: Path) -> None: assert call("genspider", self.project_name, cwd=proj_path) == 2 assert not ( @@ -168,6 +198,26 @@ class TestGenspiderStandaloneCommand: call("genspider", "example", "example.com", cwd=tmp_path) assert Path(tmp_path, "example.py").exists() + @pytest.mark.skipif( + sys.platform == "win32", reason="requires a POSIX shell editor script" + ) + def test_edit(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + spider = tmp_path / "example.py" + edited = tmp_path / "edited.txt" + editor = tmp_path / "fake-editor.sh" + write_recording_editor(editor) + monkeypatch.setenv("EDITOR", f"{editor} {edited}") + + returncode, _, err = proc( + "genspider", "--edit", "example", "example.com", cwd=tmp_path + ) + + assert returncode == 0, err + assert spider.exists() + assert (tmp_path / edited.read_text(encoding="utf-8")).resolve() == ( + spider.resolve() + ) + @pytest.mark.parametrize("force", [True, False]) def test_same_name_as_existing_file(self, force: bool, tmp_path: Path) -> None: file_name = "example" diff --git a/tests/test_commands.py b/tests/test_commands.py index d7b7a9ff2..9f81e8602 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -2,6 +2,7 @@ from __future__ import annotations import argparse import json +import sys from io import StringIO from shutil import copytree from typing import TYPE_CHECKING @@ -420,6 +421,33 @@ class TestViewCommand: assert "URL using the Scrapy downloader and show its" in command.long_desc() +class TestEditCommand(TestProjectBase): + @pytest.mark.skipif( + sys.platform == "win32", reason="requires a POSIX shell editor script" + ) + def test_edit(self, proj_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + spider = proj_path / self.project_name / "spiders" / "example.py" + edited = proj_path / "edited.txt" + editor = proj_path / "fake-editor.sh" + # Records the file it is asked to open ($2) into the file given as $1. + editor.write_text('#!/bin/sh\nprintf "%s" "$2" > "$1"\n', encoding="utf-8") + editor.chmod(0o755) + monkeypatch.setenv("EDITOR", f"{editor} {edited}") + + assert call("genspider", "example", "example.com", cwd=proj_path) == 0 + returncode, _, err = proc("edit", "example", cwd=proj_path) + + assert returncode == 0, err + assert (proj_path / edited.read_text(encoding="utf-8")).resolve() == ( + spider.resolve() + ) + + def test_edit_spider_not_found(self, proj_path: Path) -> None: + returncode, _, err = proc("edit", "nonexistent", cwd=proj_path) + assert returncode == 1 + assert "Spider not found: nonexistent" in err + + class TestHelpMessage(TestProjectBase): @pytest.mark.parametrize( "command", From deb7e2861e616bbfefb96433bd72e0d50055cd0e Mon Sep 17 00:00:00 2001 From: Gaurav Yadav Date: Tue, 30 Jun 2026 18:19:47 +0530 Subject: [PATCH 43/47] Fix _get_tag_name() crash for non-string elem.tag (#7686) (#7687) * Fix _get_tag_name() crash for non-string elem.tag (#7686) * test: improve non-string tag test accuracy and add direct unit test * test: use minimal payload for non-string tag test * test: address Adrian+syncrain PR feedback - remove first docstring line (Adrian: unnecessary) - replace weak isinstance assert with no-op call - keep Cython function mention (Adrian: wording is great) * test: replace silent call with assert results == [] per Adrian * chore: drop accidental pyproject.toml and uv.lock changes --- scrapy/utils/sitemap.py | 9 +++++---- tests/test_utils_sitemap.py | 11 +++++++++++ 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/scrapy/utils/sitemap.py b/scrapy/utils/sitemap.py index 1520a4ff0..03b7bf3b1 100644 --- a/scrapy/utils/sitemap.py +++ b/scrapy/utils/sitemap.py @@ -97,10 +97,11 @@ class Sitemap: @staticmethod def _get_tag_name(elem: lxml.etree._Element) -> str: - if TYPE_CHECKING: - assert isinstance(elem.tag, str) - _, _, localname = elem.tag.partition("}") - return localname or elem.tag + tag = elem.tag + if not isinstance(tag, str): + return "" + _, _, localname = tag.partition("}") + return localname or tag def sitemap_urls_from_robots( diff --git a/tests/test_utils_sitemap.py b/tests/test_utils_sitemap.py index 3599c4824..ac57e1739 100644 --- a/tests/test_utils_sitemap.py +++ b/tests/test_utils_sitemap.py @@ -311,3 +311,14 @@ def test_xml_entity_expansion(): """ ) assert list(s) == [{"loc": "http://127.0.0.1:8000/"}] + + +def test_sitemap_non_string_tag(): + """With recover=True and resolve_entities=False, libxml2 >= 2.14.6 (used + by lxml >= 6.1.1) preserves undeclared entity reference nodes whose + .tag is a non-string ``Cython function`` object instead of a ``str``. + _get_tag_name must handle this gracefully instead of raising + AttributeError. + """ + results = list(Sitemap(b"&k;")) + assert results == [] From 00098cb596d0d3957236ba1ba193b09172696bc5 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 30 Jun 2026 18:27:32 +0500 Subject: [PATCH 44/47] Assorted docstring fixes (#7698) * Smaller fixes. * Add more code blocks in docstrings. * Queue stuff. * Response stuff. * Round 2. * Address feedback. --- scrapy/addons.py | 2 +- scrapy/commands/__init__.py | 2 +- scrapy/core/downloader/contextfactory.py | 9 +++- scrapy/core/downloader/handlers/ftp.py | 6 +-- scrapy/core/downloader/handlers/http11.py | 3 +- scrapy/core/http2/agent.py | 6 +-- scrapy/core/http2/protocol.py | 4 +- scrapy/core/scheduler.py | 12 ++--- scrapy/core/scraper.py | 4 +- scrapy/crawler.py | 12 ++--- .../downloadermiddlewares/httpcompression.py | 2 +- scrapy/downloadermiddlewares/redirect.py | 7 ++- scrapy/downloadermiddlewares/retry.py | 8 ++- scrapy/exceptions.py | 2 +- scrapy/extensions/logstats.py | 2 +- scrapy/extensions/throttle.py | 2 +- scrapy/http/cookies.py | 6 +-- scrapy/http/response/html.py | 5 +- scrapy/http/response/xml.py | 5 +- scrapy/item.py | 2 +- scrapy/link.py | 4 +- scrapy/logformatter.py | 24 +++++---- scrapy/mail.py | 2 - scrapy/pipelines/__init__.py | 2 +- scrapy/pipelines/files.py | 2 +- scrapy/pipelines/images.py | 2 +- scrapy/pqueues.py | 7 +-- scrapy/settings/__init__.py | 11 ++-- scrapy/shell.py | 5 +- scrapy/spidermiddlewares/base.py | 8 +-- scrapy/spidermiddlewares/httperror.py | 2 +- scrapy/spidermiddlewares/referer.py | 2 +- scrapy/spiders/feed.py | 29 ++++++----- scrapy/utils/datatypes.py | 2 - scrapy/utils/defer.py | 52 +++++++++++-------- scrapy/utils/deprecate.py | 15 +++--- scrapy/utils/log.py | 3 +- scrapy/utils/response.py | 2 +- scrapy/utils/signal.py | 7 ++- scrapy/utils/trackref.py | 4 +- scrapy/utils/url.py | 4 +- 41 files changed, 151 insertions(+), 139 deletions(-) diff --git a/scrapy/addons.py b/scrapy/addons.py index 2e12f8c8a..470be47c1 100644 --- a/scrapy/addons.py +++ b/scrapy/addons.py @@ -64,7 +64,7 @@ class AddonManager: :param settings: The :class:`~scrapy.settings.BaseSettings` object from \ which to read the early add-on configuration - :type settings: :class:`~scrapy.settings.Settings` + :type settings: :class:`~scrapy.settings.BaseSettings` """ for clspath in build_component_list(settings["ADDONS"]): addoncls = load_object(clspath) diff --git a/scrapy/commands/__init__.py b/scrapy/commands/__init__.py index 598e8060e..d9c919db9 100644 --- a/scrapy/commands/__init__.py +++ b/scrapy/commands/__init__.py @@ -73,7 +73,7 @@ class ScrapyCommand(ABC): def long_desc(self) -> str: """A long description of the command. Return short description when not available. It cannot contain newlines since contents will be formatted - by optparser which removes newlines and wraps text. + by argparse which removes newlines and wraps text. """ return self.short_desc() diff --git a/scrapy/core/downloader/contextfactory.py b/scrapy/core/downloader/contextfactory.py index c1796c723..a934cbbc7 100644 --- a/scrapy/core/downloader/contextfactory.py +++ b/scrapy/core/downloader/contextfactory.py @@ -47,8 +47,13 @@ class _ScrapyClientContextFactory(BrowserLikePolicyForHTTPS): instance. The purpose of this custom class is to provide a ``creatorForNetloc()`` - method that returns a ``_ScrapyClientTLSOptions`` instance configured based - on TLS settings provided to the factory. + method that returns: + + - a ``_ScrapyClientTLSOptions26`` or ``_ScrapyClientTLSOptions`` instance + configured based on TLS settings provided to the factory (when the + certificate verification is disabled); + - a result of ``optionsForClientTLS()`` called with those TLS settings + (when the certificate verification is enabled). """ def __init__( diff --git a/scrapy/core/downloader/handlers/ftp.py b/scrapy/core/downloader/handlers/ftp.py index 9064aa53a..07ff4a74e 100644 --- a/scrapy/core/downloader/handlers/ftp.py +++ b/scrapy/core/downloader/handlers/ftp.py @@ -2,9 +2,9 @@ An asynchronous FTP file download handler for scrapy which somehow emulates an http response. FTP connection parameters are passed using the request meta field: -- ftp_user (required) -- ftp_password (required) -- ftp_passive (by default, enabled) sets FTP connection passive mode +- ftp_user (optional, falls back to FTP_USER) +- ftp_password (optional, falls back to FTP_PASSWORD) +- ftp_passive (optional, falls back to FTP_PASSIVE_MODE) sets FTP connection passive mode - ftp_local_filename - If not given, file data will come in the response.body, as a normal scrapy Response, which will imply that the entire file will be on memory. diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 1fc504c59..17dca5acb 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -104,7 +104,6 @@ class HTTP11DownloadHandler(BaseHttpDownloadHandler): self._disconnect_timeout: int = 1 async def download_request(self, request: Request) -> Response: - """Return a deferred for the HTTP download""" if hasattr(self._crawler.spider, "download_maxsize"): # pragma: no cover warn_on_deprecated_spider_attribute("download_maxsize", "DOWNLOAD_MAXSIZE") if hasattr(self._crawler.spider, "download_warnsize"): # pragma: no cover @@ -283,7 +282,7 @@ def _tunnel_request_data( class _TunnelingAgent(Agent): - """An agent that uses a L{TunnelingTCP4ClientEndpoint} to make HTTPS + """An agent that uses a ``_TunnelingTCP4ClientEndpoint`` to make HTTPS downloads. It may look strange that we have chosen to subclass Agent and not ProxyAgent but consider that after the tunnel is opened the proxy is transparent to the client; thus the agent should behave like there is no diff --git a/scrapy/core/http2/agent.py b/scrapy/core/http2/agent.py index 7137a0f2b..aa55e29a0 100644 --- a/scrapy/core/http2/agent.py +++ b/scrapy/core/http2/agent.py @@ -114,11 +114,7 @@ class H2ConnectionPool: d.errback(ResponseFailed(errors)) def close_connections(self) -> None: - """Close all the HTTP/2 connections and remove them from pool - - Returns: - Deferred that fires when all connections have been closed - """ + """Close all the HTTP/2 connections and remove them from pool.""" for conn in self._connections.values(): assert conn.transport is not None # typing conn.transport.abortConnection() diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py index 39703f976..7136e829e 100644 --- a/scrapy/core/http2/protocol.py +++ b/scrapy/core/http2/protocol.py @@ -101,7 +101,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin): uri is used to verify that incoming client requests have correct base URL. settings -- Scrapy project settings - conn_lost_deferred -- Deferred fires with the reason: Failure to notify + conn_lost_deferred -- Deferred that fires with the list of underlying exceptions to notify that connection was lost tls_verbose_logging -- Whether to log TLS details """ @@ -375,7 +375,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin): def _handle_events(self, events: list[Event]) -> None: """Private method which acts as a bridge between the events - received from the HTTP/2 data and IH2EventsHandler + received from the HTTP/2 data and the handlers in this class. Arguments: events -- A list of events that the remote peer triggered by sending data diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.py index 7217da942..82e31b90b 100644 --- a/scrapy/core/scheduler.py +++ b/scrapy/core/scheduler.py @@ -131,10 +131,10 @@ class Scheduler(BaseScheduler): (:setting:`SCHEDULER_PRIORITY_QUEUE`) that sort requests by :attr:`~scrapy.http.Request.priority`. - By default, a single, memory-based priority queue is used for all requests. - When using :setting:`JOBDIR`, a disk-based priority queue is also created, + By default, memory-based priority queues are used for all requests. + When using :setting:`JOBDIR`, disk-based priority queues are also created, and only unserializable requests are stored in the memory-based priority - queue. For a given priority value, requests in memory take precedence over + queues. For a given priority value, requests in memory take precedence over requests in disk. Each priority queue stores requests in separate internal queues, one per @@ -209,8 +209,8 @@ class Scheduler(BaseScheduler): ------------------------- While pending requests are below the configured values of - :setting:`CONCURRENT_REQUESTS`, :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` - or :setting:`CONCURRENT_REQUESTS_PER_IP`, those requests are sent + :setting:`CONCURRENT_REQUESTS` or + :setting:`CONCURRENT_REQUESTS_PER_DOMAIN`, those requests are sent concurrently. As a result, the first few requests of a crawl may not follow the desired @@ -342,7 +342,7 @@ class Scheduler(BaseScheduler): def open(self, spider: Spider) -> Deferred[None] | None: """ (1) initialize the memory queue - (2) initialize the disk queue if the ``jobdir`` attribute is a valid directory + (2) initialize the disk queue if the ``jobdir`` argument wasn't empty (3) return the result of the dupefilter's ``open`` method """ self.spider: Spider = spider diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index 466ce656d..58e37ce5e 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -441,7 +441,7 @@ class Scraper: self, output: Any, response: Response | Failure ) -> Deferred[None]: """Process each Request/Item (given in the output parameter) returned - from the given spider. + from the spider. Items are sent to the item pipelines, requests are scheduled. """ @@ -451,7 +451,7 @@ class Scraper: self, output: Any, response: Response | Failure ) -> None: """Process each Request/Item (given in the output parameter) returned - from the given spider. + from the spider. Items are sent to the item pipelines, requests are scheduled. """ diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 9828fe6f5..c72752915 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -531,8 +531,8 @@ class AsyncCrawlerRunner(CrawlerRunnerBase): """ Run a crawler with the provided arguments. - It will call the given Crawler's :meth:`~Crawler.crawl` method, while - keeping track of it so it can be stopped later. + It will call the given Crawler's :meth:`~Crawler.crawl_async` method, + while keeping track of it so it can be stopped later. If ``crawler_or_spidercls`` isn't a :class:`~scrapy.crawler.Crawler` instance, this method will try to create one using this parameter as @@ -773,7 +773,7 @@ class CrawlerProcess(CrawlerProcessBase, CrawlerRunner): """ This method starts a :mod:`~twisted.internet.reactor`, adjusts its pool size to :setting:`REACTOR_THREADPOOL_MAXSIZE`, and installs a DNS - resolver based on :setting:`DNSCACHE_ENABLED`. + resolver based on :setting:`TWISTED_DNS_RESOLVER`. If ``stop_after_crawl`` is True, the reactor will be stopped after all crawlers have finished, using :meth:`join`. @@ -875,10 +875,10 @@ class AsyncCrawlerProcess(CrawlerProcessBase, AsyncCrawlerRunner): When using a reactor it adjusts its pool size to :setting:`REACTOR_THREADPOOL_MAXSIZE` and installs a DNS resolver based - on :setting:`DNSCACHE_ENABLED`. + on :setting:`TWISTED_DNS_RESOLVER`. - If ``stop_after_crawl`` is True, the reactor will be stopped after all - crawlers have finished, using :meth:`join`. + If ``stop_after_crawl`` is True, the reactor/event loop will be stopped + after all crawlers have finished, using :meth:`join`. :param bool stop_after_crawl: stop or not the reactor when all crawlers have finished diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py index 2b1721ced..6ca04a50e 100644 --- a/scrapy/downloadermiddlewares/httpcompression.py +++ b/scrapy/downloadermiddlewares/httpcompression.py @@ -60,7 +60,7 @@ else: class HttpCompressionMiddleware: - """This middleware allows compressed (gzip, deflate) traffic to be + """This middleware allows compressed (gzip, deflate etc.) traffic to be sent/received from websites""" def __init__( diff --git a/scrapy/downloadermiddlewares/redirect.py b/scrapy/downloadermiddlewares/redirect.py index 821f41699..45af5c67a 100644 --- a/scrapy/downloadermiddlewares/redirect.py +++ b/scrapy/downloadermiddlewares/redirect.py @@ -196,10 +196,7 @@ class BaseRedirectMiddleware: class RedirectMiddleware(BaseRedirectMiddleware): - """ - Handle redirection of requests based on response status - and meta-refresh html tag. - """ + """Handle redirection of requests based on response status.""" @_warn_spider_arg def process_response( @@ -251,6 +248,8 @@ class RedirectMiddleware(BaseRedirectMiddleware): class MetaRefreshMiddleware(BaseRedirectMiddleware): + """Handle redirection of requests based on meta-refresh html tag.""" + enabled_setting = "METAREFRESH_ENABLED" def __init__(self, settings: BaseSettings): diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.py index 5f125cae4..a0a0b60a2 100644 --- a/scrapy/downloadermiddlewares/retry.py +++ b/scrapy/downloadermiddlewares/retry.py @@ -5,9 +5,6 @@ problems such as a connection timeout or HTTP 500 error. You can change the behaviour of this middleware by modifying the scraping settings: RETRY_TIMES - how many times to retry a failed page RETRY_HTTP_CODES - which HTTP response codes to retry - -Failed pages are collected on the scraping process and rescheduled at the end, -once the spider has finished crawling all regular (non-failed) pages. """ from __future__ import annotations @@ -70,8 +67,9 @@ def get_retry_request( and :ref:`stats `, and to provide extra logging context (see :func:`logging.debug`). - *reason* is a string or an :class:`Exception` object that indicates the - reason why the request needs to be retried. It is used to name retry stats. + *reason* is a string, an :class:`Exception` subclass or an + :class:`Exception` object that indicates the reason why the request needs + to be retried. It is used to name retry stats. *max_retry_times* is a number that determines the maximum number of times that *request* can be retried. If not specified or ``None``, the number is diff --git a/scrapy/exceptions.py b/scrapy/exceptions.py index 204132973..5330eab48 100644 --- a/scrapy/exceptions.py +++ b/scrapy/exceptions.py @@ -115,7 +115,7 @@ class UsageError(Exception): class ScrapyDeprecationWarning(Warning): """Warning category for deprecated features, since the default - DeprecationWarning is silenced on Python 2.7+ + :exc:`DeprecationWarning` is silenced. """ diff --git a/scrapy/extensions/logstats.py b/scrapy/extensions/logstats.py index 3d7674905..6c94d947e 100644 --- a/scrapy/extensions/logstats.py +++ b/scrapy/extensions/logstats.py @@ -22,7 +22,7 @@ logger = logging.getLogger(__name__) class LogStats: """Log basic scraping stats periodically like: - * RPM - Requests per Minute + * RPM - Responses per Minute * IPM - Items per Minute """ diff --git a/scrapy/extensions/throttle.py b/scrapy/extensions/throttle.py index cdb0671ae..542ff1cdc 100644 --- a/scrapy/extensions/throttle.py +++ b/scrapy/extensions/throttle.py @@ -116,7 +116,7 @@ class AutoThrottle: # It works better with problematic sites. new_delay = max(target_delay, new_delay) - # Make sure self.mindelay <= new_delay <= self.max_delay + # Make sure self.mindelay <= new_delay <= self.maxdelay new_delay = min(max(self.mindelay, new_delay), self.maxdelay) # Dont adjust delay if response status != 200 and new delay is smaller diff --git a/scrapy/http/cookies.py b/scrapy/http/cookies.py index 599f20947..8edeae01c 100644 --- a/scrapy/http/cookies.py +++ b/scrapy/http/cookies.py @@ -136,9 +136,9 @@ class _DummyLock: class WrappedRequest: - """Wraps a scrapy Request class with methods defined by urllib2.Request class to interact with CookieJar class - - see http://docs.python.org/library/urllib2.html#urllib2.Request + """Wraps a :class:`scrapy.Request` class with methods defined by + the :class:`urllib.request.Request` class to interact with + the :class:`http.cookiejar.CookieJar` class. """ def __init__(self, request: Request): diff --git a/scrapy/http/response/html.py b/scrapy/http/response/html.py index 70c08c11d..6d3a9ee12 100644 --- a/scrapy/http/response/html.py +++ b/scrapy/http/response/html.py @@ -1,6 +1,7 @@ """ -This module implements the HtmlResponse class which adds encoding -discovering through HTML encoding declarations to the TextResponse class. +This module implements the :class:`HtmlResponse` class which is used as a +content type marker by :class:`~scrapy.selector.Selector` and can be used in +``isinstance()`` checks. See documentation in docs/topics/request-response.rst """ diff --git a/scrapy/http/response/xml.py b/scrapy/http/response/xml.py index 6d9c4cb73..847fb2b3c 100644 --- a/scrapy/http/response/xml.py +++ b/scrapy/http/response/xml.py @@ -1,6 +1,7 @@ """ -This module implements the XmlResponse class which adds encoding -discovering through XML encoding declarations to the TextResponse class. +This module implements the :class:`XmlResponse` class which is used as a +content type marker by :class:`~scrapy.selector.Selector` and can be used in +``isinstance()`` checks. See documentation in docs/topics/request-response.rst """ diff --git a/scrapy/item.py b/scrapy/item.py index 1cc0ae584..d5adc1efb 100644 --- a/scrapy/item.py +++ b/scrapy/item.py @@ -1,7 +1,7 @@ """ Scrapy Item -See documentation in docs/topics/item.rst +See documentation in docs/topics/items.rst """ from __future__ import annotations diff --git a/scrapy/link.py b/scrapy/link.py index 046630403..8211faccd 100644 --- a/scrapy/link.py +++ b/scrapy/link.py @@ -9,7 +9,9 @@ its documentation in: docs/topics/link-extractors.rst class Link: """Link objects represent an extracted link by the LinkExtractor. - Using the anchor tag sample below to illustrate the parameters:: + Using the anchor tag sample below to illustrate the parameters: + + .. code-block:: html Dont follow this one diff --git a/scrapy/logformatter.py b/scrapy/logformatter.py index a50064e08..bfb2d5dff 100644 --- a/scrapy/logformatter.py +++ b/scrapy/logformatter.py @@ -58,18 +58,20 @@ class LogFormatter: logging an action the method must return ``None``. Here is an example on how to create a custom log formatter to lower the severity level of - the log message when an item is dropped from the pipeline:: + the log message when an item is dropped from the pipeline: - class PoliteLogFormatter(logformatter.LogFormatter): - def dropped(self, item, exception, response, spider): - return { - 'level': logging.INFO, # lowering the level from logging.WARNING - 'msg': "Dropped: %(exception)s" + os.linesep + "%(item)s", - 'args': { - 'exception': exception, - 'item': item, - } - } + .. code-block:: python + + class PoliteLogFormatter(logformatter.LogFormatter): + def dropped(self, item, exception, response, spider): + return { + "level": logging.INFO, # lowering the level from logging.WARNING + "msg": "Dropped: %(exception)s" + os.linesep + "%(item)s", + "args": { + "exception": exception, + "item": item, + }, + } """ def crawled( diff --git a/scrapy/mail.py b/scrapy/mail.py index fbd11ad1d..97123e63c 100644 --- a/scrapy/mail.py +++ b/scrapy/mail.py @@ -1,7 +1,5 @@ """ Mail sending helpers - -See documentation in docs/topics/email.rst """ from __future__ import annotations diff --git a/scrapy/pipelines/__init__.py b/scrapy/pipelines/__init__.py index 84fb5f85e..383c461e6 100644 --- a/scrapy/pipelines/__init__.py +++ b/scrapy/pipelines/__init__.py @@ -1,7 +1,7 @@ """ Item pipeline -See documentation in docs/item-pipeline.rst +See documentation in docs/topics/item-pipeline.rst """ from __future__ import annotations diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index 0066fd38f..44c422430 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -423,7 +423,7 @@ class FTPFilesStore: class FilesPipeline(MediaPipeline): - """Abstract pipeline that implement the file downloading + """Pipeline that implements file downloading. This pipeline tries to minimize network transfers and file processing, doing stat of the files and determining if file is new, up-to-date or diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py index 762b0fdf1..79b6c4f27 100644 --- a/scrapy/pipelines/images.py +++ b/scrapy/pipelines/images.py @@ -47,7 +47,7 @@ class ImageException(FileException): class ImagesPipeline(FilesPipeline): - """Abstract pipeline that implement the image thumbnail generation logic""" + """Pipeline that implements the handling logic specific to images.""" MEDIA_NAME: str = "image" diff --git a/scrapy/pqueues.py b/scrapy/pqueues.py index 2efc43d4b..41411ceaf 100644 --- a/scrapy/pqueues.py +++ b/scrapy/pqueues.py @@ -92,9 +92,10 @@ class ScrapyPriorityQueue: - The :data:`~scrapy.Request.priority` of the request. For each combination of the above seen, this class creates an instance of - *downstream_queue_cls* with *key* set to a subdirectory of the persistence - directory, named as the request priority (e.g. ``1``), with an ``s`` suffix - in case of a start request (e.g. ``1s``). + *downstream_queue_cls* (or *start_queue_cls* for start requests if it was + passed) with *key* set to a subdirectory of the persistence directory, + named as the negated request priority (e.g. ``-1``), with an ``s`` suffix + in case of a start request (e.g. ``-1s``). """ @classmethod diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py index be298e9b1..932463ba9 100644 --- a/scrapy/settings/__init__.py +++ b/scrapy/settings/__init__.py @@ -454,9 +454,10 @@ class BaseSettings(MutableMapping[str, Any]): """ Store a key/value attribute with a given priority. - Settings should be populated *before* configuring the Crawler object - (through the :meth:`~scrapy.crawler.Crawler.configure` method), - otherwise they won't have any effect. + Settings should be populated *before* the Crawler object applies them + (in the :meth:`~scrapy.crawler.Crawler.crawl_async` or + :meth:`~scrapy.crawler.Crawler.crawl` method), otherwise they won't + have any effect. :param name: the setting name :type name: str @@ -613,7 +614,7 @@ class BaseSettings(MutableMapping[str, Any]): """ Make a deep copy of current settings. - This method returns a new instance of the :class:`Settings` class, + This method returns a new instance of this class, populated with the same values and their priorities. Modifications to the new object won't be reflected on the original @@ -658,7 +659,7 @@ class BaseSettings(MutableMapping[str, Any]): Make a copy of current settings and convert to a dict. This method returns a new dict populated with the same values - and their priorities as the current settings. + as the current settings. Modifications to the returned dict won't be reflected on the original settings. diff --git a/scrapy/shell.py b/scrapy/shell.py index 44dcd880e..dfea00c46 100644 --- a/scrapy/shell.py +++ b/scrapy/shell.py @@ -75,7 +75,7 @@ if TYPE_CHECKING: # running event loop. # # Side note: it should be possible to remove _request_deferred() by using -# engine.download_async() instead of engine.schedule(), losing the usual stuff +# engine.download_async() instead of engine.crawl(), losing the usual stuff # like spider middlewares (none of which should be important). # # Other architecture problems: @@ -188,7 +188,8 @@ class Shell: async def _schedule(self, request: Request, spider: Spider | None) -> Response: """Send the request to the engine, wait for the result. - Runs in the reactor thread. + Runs in the reactor thread when using the reactor, or in the asyncio + event loop thread otherwise. """ if not self.spider: await self._open_spider(spider) diff --git a/scrapy/spidermiddlewares/base.py b/scrapy/spidermiddlewares/base.py index e09f2d10e..6c62dccb8 100644 --- a/scrapy/spidermiddlewares/base.py +++ b/scrapy/spidermiddlewares/base.py @@ -75,7 +75,7 @@ class BaseSpiderMiddleware: ) -> Request | None: """Return a processed request from the spider output. - This method is called with a single request from the start seeds or the + This method is called with a single request from ``start()`` or the spider output. It should return the same or a different request, or ``None`` to ignore it. @@ -84,7 +84,7 @@ class BaseSpiderMiddleware: :param response: the response being processed :type response: :class:`~scrapy.http.Response` object or ``None`` for - start seeds + start requests :return: the processed request or ``None`` """ @@ -93,7 +93,7 @@ class BaseSpiderMiddleware: def get_processed_item(self, item: Any, response: Response | None) -> Any: """Return a processed item from the spider output. - This method is called with a single item from the start seeds or the + This method is called with a single item from ``start()`` or the spider output. It should return the same or a different item, or ``None`` to ignore it. @@ -102,7 +102,7 @@ class BaseSpiderMiddleware: :param response: the response being processed :type response: :class:`~scrapy.http.Response` object or ``None`` for - start seeds + start items :return: the processed item or ``None`` """ diff --git a/scrapy/spidermiddlewares/httperror.py b/scrapy/spidermiddlewares/httperror.py index 94b6dfbb5..156b73e7e 100644 --- a/scrapy/spidermiddlewares/httperror.py +++ b/scrapy/spidermiddlewares/httperror.py @@ -28,7 +28,7 @@ logger = logging.getLogger(__name__) class HttpError(IgnoreRequest): - """A non-200 response was filtered""" + """A non-2xx response was filtered""" def __init__(self, response: Response, *args: Any, **kwargs: Any): self.response = response diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py index 264f685c1..1305874d5 100644 --- a/scrapy/spidermiddlewares/referer.py +++ b/scrapy/spidermiddlewares/referer.py @@ -92,7 +92,7 @@ class ReferrerPolicy(ABC): ) def origin(self, url: str) -> str | None: - """Return serialized origin (scheme, host, path) for a request or response URL.""" + """Return serialized origin (scheme, host, port) for a request or response URL.""" return self.strip_url(url, origin_only=True) def potentially_trustworthy(self, url: str) -> bool: diff --git a/scrapy/spiders/feed.py b/scrapy/spiders/feed.py index 395183613..925f31ede 100644 --- a/scrapy/spiders/feed.py +++ b/scrapy/spiders/feed.py @@ -54,17 +54,21 @@ class XMLFeedSpider(Spider): return response def parse_node(self, response: Response, selector: Selector) -> Any: - """This method must be overridden with your custom spider functionality""" + """This method is called for the nodes matching the provided tag name + (itertag). Receives the response and an Selector for each node. + + This method must return either an item, a request, or a list + containing any of them. + + This method must be overridden with your custom spider functionality. + """ if hasattr(self, "parse_item"): # backward compatibility return self.parse_item(response, selector) raise NotImplementedError def parse_nodes(self, response: Response, nodes: Iterable[Selector]) -> Any: """This method is called for the nodes matching the provided tag name - (itertag). Receives the response and an Selector for each node. - Overriding this method is mandatory. Otherwise, you spider won't work. - This method must return either an item, a request, or a list - containing any of them. + (itertag). Receives the response and an iterable of Selectors. """ for selector in nodes: @@ -113,6 +117,9 @@ class CSVFeedSpider(Spider): It receives a CSV file in a response; iterates through each of its rows, and calls parse_row with a dict containing each field's data. + This spider also gives the opportunity to override adapt_response and + process_results methods for pre and post-processing purposes. + You can set some options regarding the CSV file, such as the delimiter, quotechar and the file's headers. """ @@ -136,16 +143,14 @@ class CSVFeedSpider(Spider): return response def parse_row(self, response: Response, row: dict[str, str]) -> Any: - """This method must be overridden with your custom spider functionality""" + """Receives a response and a dict (representing each row) with a key for + each provided (or detected) header of the CSV file. + + This method must be overridden with your custom spider functionality. + """ raise NotImplementedError def parse_rows(self, response: Response) -> Any: - """Receives a response and a dict (representing each row) with a key for - each provided (or detected) header of the CSV file. This spider also - gives the opportunity to override adapt_response and - process_results methods for pre and post-processing purposes. - """ - for row in csviter( response, self.delimiter, self.headers, quotechar=self.quotechar ): diff --git a/scrapy/utils/datatypes.py b/scrapy/utils/datatypes.py index dd0e062d0..c020ff4b9 100644 --- a/scrapy/utils/datatypes.py +++ b/scrapy/utils/datatypes.py @@ -1,8 +1,6 @@ """ This module contains data types used by Scrapy which are not included in the Python Standard Library. - -This module must not depend on any module outside the Standard Library. """ from __future__ import annotations diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py index 29a34d4ef..d0259b634 100644 --- a/scrapy/utils/defer.py +++ b/scrapy/utils/defer.py @@ -103,7 +103,7 @@ async def _defer_sleep_async() -> None: def defer_result(result: Any) -> Deferred[Any]: # pragma: no cover warnings.warn( "scrapy.utils.defer.defer_result() is deprecated, use" - " twisted.internet.defer.success() and twisted.internet.defer.fail()," + " twisted.internet.defer.succeed() and twisted.internet.defer.fail()," " plus an explicit sleep if needed, or explicit reactor.callLater().", category=ScrapyDeprecationWarning, stacklevel=2, @@ -469,22 +469,22 @@ def _maybeDeferred_coro( def deferred_to_future(d: Deferred[_T]) -> Future[_T]: """Return an :class:`asyncio.Future` object that wraps *d*. - This function requires - :class:`~twisted.internet.asyncioreactor.AsyncioSelectorReactor` to be - installed. + This function requires an installed asyncio reactor or a running asyncio + event loop, see :ref:`using-asyncio`. - When :ref:`using the asyncio reactor `, you cannot await - on :class:`~twisted.internet.defer.Deferred` objects from :ref:`Scrapy - callables defined as coroutines `, you can only await on - ``Future`` objects. Wrapping ``Deferred`` objects into ``Future`` objects - allows you to wait on them:: + In this state you cannot await on :class:`~twisted.internet.defer.Deferred` + objects from :ref:`Scrapy callables defined as coroutines + `, you can only await on ``Future`` objects. Wrapping + ``Deferred`` objects into ``Future`` objects allows you to wait on them: + + .. code-block:: python class MySpider(Spider): ... + async def parse(self, response): - additional_request = scrapy.Request('https://example.org/price') - deferred = self.crawler.engine.download(additional_request) - additional_response = await deferred_to_future(deferred) + deferred = some_dfd_helper() + result = await deferred_to_future(deferred) .. versionchanged:: 2.14 This function no longer installs an asyncio loop if called before the @@ -492,7 +492,10 @@ def deferred_to_future(d: Deferred[_T]) -> Future[_T]: in this case. """ if not is_asyncio_available(): - raise RuntimeError("deferred_to_future() requires AsyncioSelectorReactor.") + raise RuntimeError( + "deferred_to_future() requires an installed asyncio reactor" + " or a running asyncio event loop." + ) return d.asFuture(asyncio.get_event_loop()) @@ -501,23 +504,26 @@ def maybe_deferred_to_future(d: Deferred[_T]) -> Deferred[_T] | Future[_T]: defined as a coroutine `. What you can await in Scrapy callables defined as coroutines depends on the - value of :setting:`TWISTED_REACTOR`: + value of :setting:`TWISTED_REACTOR` and :setting:`TWISTED_REACTOR_ENABLED`: - - When :ref:`using the asyncio reactor `, you can only - await on :class:`asyncio.Future` objects. + - When :ref:`using the asyncio reactor `, or :ref:`not + using a reactor at all `, you can only await + on :class:`asyncio.Future` objects. - - When not using the asyncio reactor, you can only await on - :class:`~twisted.internet.defer.Deferred` objects. + - When :ref:`using a non-asyncio reactor `, you can only + await on :class:`~twisted.internet.defer.Deferred` objects. - If you want to write code that uses ``Deferred`` objects but works with any - reactor, use this function on all ``Deferred`` objects:: + If you want to write code that uses ``Deferred`` objects but works in both + of these states, use this function on all ``Deferred`` objects: + + .. code-block:: python class MySpider(Spider): ... + async def parse(self, response): - additional_request = scrapy.Request('https://example.org/price') - deferred = self.crawler.engine.download(additional_request) - additional_response = await maybe_deferred_to_future(deferred) + deferred = some_dfd_helper() + result = await maybe_deferred_to_future(deferred) """ if not is_asyncio_available(): return d diff --git a/scrapy/utils/deprecate.py b/scrapy/utils/deprecate.py index 359f819d7..4fd50fdad 100644 --- a/scrapy/utils/deprecate.py +++ b/scrapy/utils/deprecate.py @@ -43,15 +43,18 @@ def create_deprecated_class( It can be used to rename a base class in a library. For example, if we have - class OldName(SomeClass): - # ... + .. code-block:: python - and we want to rename it to NewName, we can do the following:: + class OldName(SomeClass): ... - class NewName(SomeClass): - # ... + and we want to rename it to NewName, we can do the following: - OldName = create_deprecated_class('OldName', NewName) + .. code-block:: python + + class NewName(SomeClass): ... + + + OldName = create_deprecated_class("OldName", NewName) Then, if user class inherits from OldName, warning is issued. Also, if some code uses ``issubclass(sub, OldName)`` or ``isinstance(sub(), OldName)`` diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py index aa77e692a..7645b235e 100644 --- a/scrapy/utils/log.py +++ b/scrapy/utils/log.py @@ -248,8 +248,7 @@ def logformatter_adapter( ) -> tuple[int, str, dict[str, Any] | tuple[Any, ...]]: """ Helper that takes the dictionary output from the methods in LogFormatter - and adapts it into a tuple of positional arguments for logger.log calls, - handling backward compatibility as well. + and adapts it into a tuple of positional arguments for logger.log calls. """ level = logkws.get("level", logging.INFO) diff --git a/scrapy/utils/response.py b/scrapy/utils/response.py index c068d9b1e..7747a7b9b 100644 --- a/scrapy/utils/response.py +++ b/scrapy/utils/response.py @@ -88,7 +88,7 @@ def open_in_browser( def parse_details(self, response): - if "item name" not in response.body: + if "item name" not in response.text: open_in_browser(response) """ # circular imports diff --git a/scrapy/utils/signal.py b/scrapy/utils/signal.py index 919f67240..eca95e225 100644 --- a/scrapy/utils/signal.py +++ b/scrapy/utils/signal.py @@ -39,7 +39,7 @@ def send_catch_log( *arguments: TypingAny, **named: TypingAny, ) -> list[tuple[TypingAny, TypingAny]]: - """Like ``pydispatcher.robust.sendRobust()`` but it also logs errors and returns + """Like ``pydispatch.robust.sendRobust()`` but it also logs errors and returns Failures instead of exceptions. """ dont_log = named.pop("dont_log", ()) @@ -172,9 +172,8 @@ async def _send_catch_log_asyncio( Returns a coroutine that completes once all signal handlers have finished. - This function requires - :class:`~twisted.internet.asyncioreactor.AsyncioSelectorReactor` to be - installed. + This function requires an installed asyncio reactor or a running asyncio + event loop. .. versionadded:: 2.14 """ diff --git a/scrapy/utils/trackref.py b/scrapy/utils/trackref.py index 87df10a02..22f9eadd0 100644 --- a/scrapy/utils/trackref.py +++ b/scrapy/utils/trackref.py @@ -4,9 +4,7 @@ references to live object instances. If you want live objects for a particular class to be tracked, you only have to subclass from object_ref (instead of object). -About performance: This library has a minimal performance impact when enabled, -and no performance penalty at all when disabled (as object_ref becomes just an -alias to object in that case). +This library has a minimal performance impact. .. note:: PyPy uses a tracing garbage collector, so objects may remain in the ``live_refs`` longer than expected, even after they diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py index 4d2bbdda2..f67853ece 100644 --- a/scrapy/utils/url.py +++ b/scrapy/utils/url.py @@ -117,8 +117,8 @@ def strip_url( - ``strip_credentials`` removes "user:password@" - ``strip_default_port`` removes ":80" (resp. ":443", ":21") from http:// (resp. https://, ftp://) URLs - - ``origin_only`` replaces path component with "/", also dropping - query and fragment components ; it also strips credentials + - ``origin_only`` replaces the path component with "/", also dropping + the query component; it also strips credentials - ``strip_fragment`` drops any #fragment component """ From a6d6a48aa600d1b4c3deb6409c17d0d7f55db312 Mon Sep 17 00:00:00 2001 From: Adrian Date: Tue, 30 Jun 2026 16:26:22 +0200 Subject: [PATCH 45/47] Keep Item fields in definition order (#7694) --- scrapy/item.py | 37 ++++++++++++++++++++++++++++++------- tests/test_feedexport.py | 2 +- tests/test_item.py | 24 ++++++++++++++++++++++++ 3 files changed, 55 insertions(+), 8 deletions(-) diff --git a/scrapy/item.py b/scrapy/item.py index d5adc1efb..4d99ea79d 100644 --- a/scrapy/item.py +++ b/scrapy/item.py @@ -25,6 +25,25 @@ class Field(dict[str, Any]): """Container of field metadata""" +def _ordered_field_names(cls: type) -> list[str]: + """Return the names of the :class:`Field` attributes of *cls* in definition + order. + + Fields declared in base classes come first, ordered from the topmost base + to the most derived class. Within each class, fields keep their definition + order. A field redefined in a subclass keeps the position of its first + definition. + """ + names: list[str] = [] + seen: set[str] = set() + for base in reversed(cls.__mro__): + for name, value in vars(base).items(): + if isinstance(value, Field) and name not in seen: + seen.add(name) + names.append(name) + return names + + class ItemMeta(ABCMeta): """Metaclass_ of :class:`Item` that handles field definitions. @@ -39,13 +58,9 @@ class ItemMeta(ABCMeta): _class = super().__new__(mcs, "x_" + class_name, new_bases, attrs) fields = getattr(_class, "fields", {}) - new_attrs = {} - for n in dir(_class): - v = getattr(_class, n) - if isinstance(v, Field): - fields[n] = v - elif n in attrs: - new_attrs[n] = attrs[n] + for n in _ordered_field_names(_class): + fields[n] = getattr(_class, n) + new_attrs = {n: v for n, v in attrs.items() if not isinstance(v, Field)} new_attrs["fields"] = fields new_attrs["_class"] = _class @@ -80,6 +95,14 @@ class Item(MutableMapping[str, Any], object_ref, metaclass=ItemMeta): #: those populated. The keys are the field names and the values are the #: :class:`Field` objects used in the :ref:`Item declaration #: `. + #: + #: Fields are kept in definition order: fields declared in base classes + #: come first, followed by fields declared in subclasses, and a field + #: redefined in a subclass keeps the position of its first definition. + #: + #: .. versionchanged:: VERSION + #: Fields are now returned in definition order rather than alphabetical + #: order. fields: dict[str, Field] def __init__(self, *args: Any, **kwargs: Any): diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index d7ea60cc8..7d751f188 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -746,7 +746,7 @@ class TestFeedExport(TestFeedExportBase): ] formats = { - "csv": b"baz,egg,foo\r\n,spam1,bar1\r\n", + "csv": b"foo,egg,baz\r\nbar1,spam1,\r\n", "json": b'[\n{"hello": "world2", "foo": "bar2"}\n]', "jsonlines": ( b'{"foo": "bar1", "egg": "spam1"}\n{"hello": "world2", "foo": "bar2"}\n' diff --git a/tests/test_item.py b/tests/test_item.py index 34b054e12..7b4c2e918 100644 --- a/tests/test_item.py +++ b/tests/test_item.py @@ -142,6 +142,30 @@ class TestItem: self.assertSortedEqual(list(item.keys()), ["new"]) self.assertSortedEqual(list(item.values()), ["New"]) + def test_fields_order(self): + class TestItem(Item): + name = Field() + keys = Field() + values = Field() + + assert list(TestItem.fields) == ["name", "keys", "values"] + + def test_fields_order_inheritance(self): + class ParentItem(Item): + name = Field() + keys = Field() + values = Field() + + class TestItem(ParentItem): + extra = Field() + keys = Field(serializer=str) + + # Inherited fields come first, in their definition order, followed by + # the fields newly defined in the subclass. A redefined field keeps the + # position of its first definition while taking the new metadata. + assert list(TestItem.fields) == ["name", "keys", "values", "extra"] + assert TestItem.fields["keys"] == {"serializer": str} + def test_metaclass_inheritance(self): class ParentItem(Item): name = Field() From fc5216f15611e40d795f5ab566acff8f9ca0af1e Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 1 Jul 2026 11:50:32 +0500 Subject: [PATCH 46/47] Clarify/cleanup Selector.type (#7704) --- scrapy/selector/unified.py | 47 ++++++++++++++------------------------ 1 file changed, 17 insertions(+), 30 deletions(-) diff --git a/scrapy/selector/unified.py b/scrapy/selector/unified.py index 99b22aca9..f6334c32c 100644 --- a/scrapy/selector/unified.py +++ b/scrapy/selector/unified.py @@ -1,10 +1,6 @@ -""" -XPath selectors based on lxml -""" - from __future__ import annotations -from typing import Any +from typing import Any, Literal from parsel import Selector as _ParselSelector @@ -18,13 +14,10 @@ __all__ = ["Selector", "SelectorList"] _NOT_SET = object() -def _st(response: TextResponse | None, st: str | None) -> str: - if st is None: - return "xml" if isinstance(response, XmlResponse) else "html" - return st +SelectorType = Literal["html", "xml", "json", "text"] -def _response_from_text(text: str | bytes, st: str | None) -> TextResponse: +def _response_from_text(text: str | bytes, st: SelectorType | None) -> TextResponse: rt: type[TextResponse] = XmlResponse if st == "xml" else HtmlResponse return rt(url="about:blank", encoding="utf-8", body=to_bytes(text, "utf-8")) @@ -49,23 +42,16 @@ class Selector(_ParselSelector, object_ref): ``response`` isn't available. Using ``text`` and ``response`` together is undefined behavior. - ``type`` defines the selector type, it can be ``"html"``, ``"xml"``, ``"json"`` - or ``None`` (default). + ``type`` defines the selector type, it can be ``"html"``, ``"xml"``, + ``"json"``, ``"text"`` or ``None`` (default). It's passed to + :class:`parsel.Selector` and its meaning is defined there. However, when + ``type`` is ``None``, it is set to ``"xml"`` for an + :class:`~scrapy.http.XmlResponse` and to ``"html"`` otherwise before + passing it to :class:`parsel.Selector`. - If ``type`` is ``None``, the selector automatically chooses the best type - based on ``response`` type (see below), or defaults to ``"html"`` in case it - is used together with ``text``. - - If ``type`` is ``None`` and a ``response`` is passed, the selector type is - inferred from the response type as follows: - - * ``"html"`` for :class:`~scrapy.http.HtmlResponse` type - * ``"xml"`` for :class:`~scrapy.http.XmlResponse` type - * ``"json"`` for :class:`~scrapy.http.TextResponse` type - * ``"html"`` for anything else - - Otherwise, if ``type`` is set, the selector type will be forced and no - detection will occur. + .. note:: JSON selector support requires ``parsel`` 1.8.0 or higher. With + older versions setting ``type`` to ``"json"`` or ``"text"`` is not + supported. """ __slots__ = ["response"] @@ -75,7 +61,7 @@ class Selector(_ParselSelector, object_ref): self, response: TextResponse | None = None, text: str | None = None, - type: str | None = None, # noqa: A002 + type: SelectorType | None = None, # noqa: A002 root: Any | None = _NOT_SET, **kwargs: Any, ): @@ -84,10 +70,11 @@ class Selector(_ParselSelector, object_ref): f"{self.__class__.__name__}.__init__() received both response and text" ) - st = _st(response, type) + if type is None: + type = "xml" if isinstance(response, XmlResponse) else "html" # noqa: A001 if text is not None: - response = _response_from_text(text, st) + response = _response_from_text(text, type) if response is not None: text = response.text @@ -98,4 +85,4 @@ class Selector(_ParselSelector, object_ref): if root is not _NOT_SET: kwargs["root"] = root - super().__init__(text=text, type=st, **kwargs) + super().__init__(text=text, type=type, **kwargs) From 361f689df785959a59cf939b9efaefc72079f037 Mon Sep 17 00:00:00 2001 From: Adrian Date: Wed, 1 Jul 2026 08:53:07 +0200 Subject: [PATCH 47/47] Improve test coverage for crawler.py (#7682) * Improve test coverage for crawler.py * Silence mypy warnings * Improve test coverage for crawler.py --- ...yncio_enabled_reactor_same_loop_default.py | 31 ++++ .../dns_resolver_deprecated.py | 31 ++++ .../reactorless_sleeping.py | 2 +- tests/AsyncCrawlerProcess/sleeping.py | 2 +- .../CrawlerProcess/dns_resolver_deprecated.py | 31 ++++ tests/CrawlerProcess/sleeping.py | 2 +- tests/test_crawler.py | 163 +++++++++++++++++- tests/test_crawler_subprocess.py | 30 +++- 8 files changed, 286 insertions(+), 6 deletions(-) create mode 100644 tests/AsyncCrawlerProcess/asyncio_enabled_reactor_same_loop_default.py create mode 100644 tests/AsyncCrawlerProcess/dns_resolver_deprecated.py create mode 100644 tests/CrawlerProcess/dns_resolver_deprecated.py diff --git a/tests/AsyncCrawlerProcess/asyncio_enabled_reactor_same_loop_default.py b/tests/AsyncCrawlerProcess/asyncio_enabled_reactor_same_loop_default.py new file mode 100644 index 000000000..c519e123b --- /dev/null +++ b/tests/AsyncCrawlerProcess/asyncio_enabled_reactor_same_loop_default.py @@ -0,0 +1,31 @@ +import asyncio +import sys + +from twisted.internet import asyncioreactor + +import scrapy +from scrapy.crawler import AsyncCrawlerProcess + +if sys.platform == "win32": + asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) +loop = asyncio.SelectorEventLoop() +asyncio.set_event_loop(loop) +asyncioreactor.install(loop) + + +class NoRequestsSpider(scrapy.Spider): + name = "no_request" + + async def start(self): + return + yield + + +process = AsyncCrawlerProcess( + settings={ + "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", + "ASYNCIO_EVENT_LOOP": "asyncio.SelectorEventLoop", + } +) +process.crawl(NoRequestsSpider) +process.start() diff --git a/tests/AsyncCrawlerProcess/dns_resolver_deprecated.py b/tests/AsyncCrawlerProcess/dns_resolver_deprecated.py new file mode 100644 index 000000000..8c8df96bb --- /dev/null +++ b/tests/AsyncCrawlerProcess/dns_resolver_deprecated.py @@ -0,0 +1,31 @@ +import sys + +import scrapy +from scrapy.crawler import AsyncCrawlerProcess +from scrapy.settings import Settings + + +class NoRequestsSpider(scrapy.Spider): + name = "no_request" + + async def start(self): + return + yield + + +settings = Settings() +# The deprecated DNS_RESOLVER setting, set above its default priority so that +# AsyncCrawlerProcess._setup_reactor() emits the deprecation warning. +settings.set("DNS_RESOLVER", "scrapy.resolver.CachingThreadedResolver", priority=10) +if len(sys.argv) > 1 and sys.argv[1] == "twisted-wins": + # TWISTED_DNS_RESOLVER at a higher priority takes precedence over the + # deprecated DNS_RESOLVER setting. + settings.set( + "TWISTED_DNS_RESOLVER", + "scrapy.resolver.CachingThreadedResolver", + priority=20, + ) + +process = AsyncCrawlerProcess(settings) +process.crawl(NoRequestsSpider) +process.start() diff --git a/tests/AsyncCrawlerProcess/reactorless_sleeping.py b/tests/AsyncCrawlerProcess/reactorless_sleeping.py index 12101d221..0d11a0996 100644 --- a/tests/AsyncCrawlerProcess/reactorless_sleeping.py +++ b/tests/AsyncCrawlerProcess/reactorless_sleeping.py @@ -17,4 +17,4 @@ class SleepingSpider(scrapy.Spider): process = AsyncCrawlerProcess(settings={"TWISTED_REACTOR_ENABLED": False}) process.crawl(SleepingSpider) -process.start() +process.start(stop_after_crawl="--no-stop" not in sys.argv) diff --git a/tests/AsyncCrawlerProcess/sleeping.py b/tests/AsyncCrawlerProcess/sleeping.py index 88caf5032..dad6a3f20 100644 --- a/tests/AsyncCrawlerProcess/sleeping.py +++ b/tests/AsyncCrawlerProcess/sleeping.py @@ -17,4 +17,4 @@ class SleepingSpider(scrapy.Spider): process = AsyncCrawlerProcess(settings={}) process.crawl(SleepingSpider) -process.start() +process.start(stop_after_crawl="--no-stop" not in sys.argv) diff --git a/tests/CrawlerProcess/dns_resolver_deprecated.py b/tests/CrawlerProcess/dns_resolver_deprecated.py new file mode 100644 index 000000000..b8cd24325 --- /dev/null +++ b/tests/CrawlerProcess/dns_resolver_deprecated.py @@ -0,0 +1,31 @@ +import sys + +import scrapy +from scrapy.crawler import CrawlerProcess +from scrapy.settings import Settings + + +class NoRequestsSpider(scrapy.Spider): + name = "no_request" + + async def start(self): + return + yield + + +settings = Settings() +# The deprecated DNS_RESOLVER setting, set above its default priority so that +# CrawlerProcess._setup_reactor() emits the deprecation warning. +settings.set("DNS_RESOLVER", "scrapy.resolver.CachingThreadedResolver", priority=10) +if len(sys.argv) > 1 and sys.argv[1] == "twisted-wins": + # TWISTED_DNS_RESOLVER at a higher priority takes precedence over the + # deprecated DNS_RESOLVER setting. + settings.set( + "TWISTED_DNS_RESOLVER", + "scrapy.resolver.CachingThreadedResolver", + priority=20, + ) + +process = CrawlerProcess(settings) +process.crawl(NoRequestsSpider) +process.start() diff --git a/tests/CrawlerProcess/sleeping.py b/tests/CrawlerProcess/sleeping.py index cb8f869e1..a577b1909 100644 --- a/tests/CrawlerProcess/sleeping.py +++ b/tests/CrawlerProcess/sleeping.py @@ -23,4 +23,4 @@ class SleepingSpider(scrapy.Spider): process = CrawlerProcess(settings={}) process.crawl(SleepingSpider) -process.start() +process.start(stop_after_crawl="--no-stop" not in sys.argv) diff --git a/tests/test_crawler.py b/tests/test_crawler.py index 0cddfd0ed..adac32df1 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -3,8 +3,11 @@ from __future__ import annotations import asyncio import logging import re +import signal +import threading from pathlib import Path -from typing import Any, ClassVar +from typing import TYPE_CHECKING, Any, ClassVar +from unittest.mock import MagicMock import pytest from zope.interface.exceptions import MultipleInvalid @@ -32,6 +35,9 @@ from scrapy.utils.spider import DefaultSpider from scrapy.utils.test import get_crawler, get_reactor_settings from tests.utils.decorators import coroutine_test +if TYPE_CHECKING: + from collections.abc import Callable + BASE_SETTINGS: dict[str, Any] = {} @@ -651,6 +657,145 @@ class TestAsyncCrawlerProcess(TestBaseCrawler): self.assertOptionIsDefault(runner.settings, "RETRY_ENABLED") +class TestAsyncCrawlerProcessReactorlessHelpers: + """Unit tests for the reactorless shutdown helpers of AsyncCrawlerProcess. + + These cover defensive branches that guard against shutdown races and that + are not reachable through a full process run. + """ + + @staticmethod + def _bare_process( + monkeypatch: pytest.MonkeyPatch, + ) -> tuple[AsyncCrawlerProcess, list[Any]]: + # AsyncCrawlerProcess.__init__ has global side effects (it installs a + # reactor import hook and an asyncio event loop), so build a bare + # instance and set only the attributes these helpers read. The shutdown + # handlers installed by these helpers are recorded for assertions + # instead of touching the real process-wide signal handlers. + installed_handlers: list[Any] = [] + monkeypatch.setattr( + "scrapy.crawler.install_shutdown_handlers", + lambda handler, *args, **kwargs: installed_handlers.append(handler), + ) + return AsyncCrawlerProcess.__new__(AsyncCrawlerProcess), installed_handlers + + @staticmethod + def _run_in_thread(target: Callable[[], None]) -> None: + # Run target in a dedicated thread so its event loop is not nested + # inside the event loop that may already be running the test session. + thread = threading.Thread(target=target) + thread.start() + thread.join() + + def test_signal_shutdown_reactorless_without_loop( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + process, installed_handlers = self._bare_process(monkeypatch) + process._reactorless_loop = None + # No loop to schedule the shutdown task on, so it returns early, but it + # must still escalate the handler so a second signal forces a kill. + process._signal_shutdown_reactorless(signal.SIGINT, None) + assert installed_handlers == [process._signal_kill_reactorless] + + def test_signal_kill_reactorless_without_loop( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + process, installed_handlers = self._bare_process(monkeypatch) + process._reactorless_loop = None + process._reactorless_main_task = None + # No loop to cancel the main task on, so it returns early, but it must + # still ignore any further signals. + process._signal_kill_reactorless(signal.SIGINT, None) + assert installed_handlers == [signal.SIG_IGN] + + def test_signal_kill_reactorless_without_main_task( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + process, installed_handlers = self._bare_process(monkeypatch) + loop = MagicMock() + process._reactorless_loop = loop + process._reactorless_main_task = None + # No main task to cancel, so nothing is scheduled on the loop. + process._signal_kill_reactorless(signal.SIGINT, None) + assert installed_handlers == [signal.SIG_IGN] + loop.call_soon_threadsafe.assert_not_called() + + def test_shutdown_graceful_reactorless_main_task_already_done( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + process, _ = self._bare_process(monkeypatch) + process._stop_after_crawl = False + + async def noop() -> None: + return None + + monkeypatch.setattr(process, "stop", noop) + monkeypatch.setattr(process, "join", noop) + + def run() -> None: + loop = asyncio.new_event_loop() + try: + main_task: asyncio.Future[None] = loop.create_future() + main_task.set_result(None) + process._reactorless_main_task = main_task + # The main task is already done, so it is not cancelled. + loop.run_until_complete(process._shutdown_graceful_reactorless()) + assert not main_task.cancelled() + finally: + loop.close() + + self._run_in_thread(run) + + def test_create_shutdown_task_closed_loop( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + process, _ = self._bare_process(monkeypatch) + loop = asyncio.new_event_loop() + loop.close() + process._reactorless_loop = loop + process._stop_after_crawl = True + # create_task() raises RuntimeError on a closed loop; the coroutine + # must be closed instead of leaking. + process._create_shutdown_task() + + def test_cancel_all_tasks_logs_task_exception(self) -> None: + contexts: list[dict[str, Any]] = [] + task_was_cancelled: list[bool] = [] + + def run() -> None: + loop = asyncio.new_event_loop() + loop.set_exception_handler(lambda _loop, context: contexts.append(context)) + + async def fail_on_cancel() -> None: + try: + await asyncio.sleep(10) + except asyncio.CancelledError: + raise RuntimeError("boom") + + try: + task = loop.create_task(fail_on_cancel()) + # Let the task start and suspend on the sleep so the + # cancellation is raised inside its body and turned into a + # RuntimeError rather than cancelling the task cleanly. + loop.run_until_complete(asyncio.sleep(0)) + AsyncCrawlerProcess._cancel_all_tasks(loop) + task_was_cancelled.append(task.cancelled()) + finally: + loop.close() + + self._run_in_thread(run) + + # The task raised instead of being cancelled, so its exception is + # reported to the loop exception handler. + assert task_was_cancelled == [False] + assert any( + context.get("message") + == "unhandled exception during AsyncCrawlerProcess shutdown" + for context in contexts + ) + + @pytest.mark.parametrize("runner_cls", [AsyncCrawlerRunner, CrawlerRunner]) def test_runner_settings_applied_to_crawler_instance( runner_cls: type[CrawlerRunnerBase], @@ -687,6 +832,22 @@ def test_create_crawler_instance_consistent_with_spider_class() -> None: assert pre_built.settings["FOO"] == "runner" +@pytest.mark.parametrize("runner_cls", [AsyncCrawlerRunner, CrawlerRunner]) +def test_create_crawler_rejects_spider_object( + runner_cls: type[CrawlerRunnerBase], +) -> None: + runner = runner_cls() + with pytest.raises(ValueError, match="cannot be a spider object"): + runner.create_crawler(DefaultSpider()) # type: ignore[arg-type] + + +@pytest.mark.parametrize("runner_cls", [AsyncCrawlerRunner, CrawlerRunner]) +def test_crawl_rejects_spider_object(runner_cls: type[CrawlerRunnerBase]) -> None: + runner = runner_cls() + with pytest.raises(ValueError, match="cannot be a spider object"): + runner.crawl(DefaultSpider()) # type: ignore[arg-type] + + class ExceptionSpider(scrapy.Spider): name = "exception" diff --git a/tests/test_crawler_subprocess.py b/tests/test_crawler_subprocess.py index e3f9ac161..018a2b31b 100644 --- a/tests/test_crawler_subprocess.py +++ b/tests/test_crawler_subprocess.py @@ -126,6 +126,16 @@ class TestCrawlerProcessSubprocessBase(ScriptRunnerMixin): assert "TimeoutError" not in log assert "scrapy.exceptions.CannotResolveHostError" not in log + def test_dns_resolver_deprecated(self) -> None: + log = self.run_script("dns_resolver_deprecated.py") + assert "Spider closed (finished)" in log + assert "The DNS_RESOLVER setting is deprecated" in log + + def test_dns_resolver_deprecated_twisted_dns_resolver(self) -> None: + log = self.run_script("dns_resolver_deprecated.py", "twisted-wins") + assert "Spider closed (finished)" in log + assert "The DNS_RESOLVER setting is deprecated" in log + def test_twisted_reactor_asyncio(self) -> None: log = self.run_script("twisted_reactor_asyncio.py") assert "Spider closed (finished)" in log @@ -205,9 +215,11 @@ class TestCrawlerProcessSubprocessBase(ScriptRunnerMixin): assert "Spider closed (finished)" in log assert "The value of FOO is 42" in log - def _test_shutdown_graceful(self, script: str = "sleeping.py") -> None: + def _test_shutdown_graceful( + self, script: str = "sleeping.py", *extra_args: str + ) -> None: sig = signal.SIGINT if sys.platform != "win32" else signal.SIGBREAK # type: ignore[attr-defined] - args = self.get_script_args(script, "3") + args = self.get_script_args(script, "3", *extra_args) p = PopenSpawn(args, timeout=5, env=get_script_run_env()) p.expect_exact("Spider opened") p.expect_exact("Crawled (200)") @@ -245,6 +257,9 @@ class TestCrawlerProcessSubprocessBase(ScriptRunnerMixin): async def test_shutdown_forced(self) -> None: await self._test_shutdown_forced() + def test_shutdown_graceful_no_stop(self) -> None: + self._test_shutdown_graceful("sleeping.py", "--no-stop") + class TestCrawlerProcessSubprocess(TestCrawlerProcessSubprocessBase): @property @@ -429,6 +444,17 @@ class TestAsyncCrawlerProcessSubprocess(TestCrawlerProcessSubprocessBase): async def test_shutdown_forced(self) -> None: await self._test_shutdown_forced("reactorless_sleeping.py") + def test_shutdown_graceful_reactorless_no_stop(self) -> None: + self._test_shutdown_graceful("reactorless_sleeping.py", "--no-stop") + + def test_asyncio_enabled_reactor_same_loop_default(self) -> None: + log = self.run_script("asyncio_enabled_reactor_same_loop_default.py") + assert "Spider closed (finished)" in log + assert ( + "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" + in log + ) + class TestCrawlerRunnerSubprocessBase(ScriptRunnerMixin): """Common tests between CrawlerRunner and AsyncCrawlerRunner,