From 9f4651151d70bda66ccdbd2d0890e2c7e94b8d7c Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 24 Mar 2026 14:36:48 +0500 Subject: [PATCH] Deprecate ScrapyCommand.set_crawler(), remove/fix some dead code. (#7276) --- scrapy/commands/__init__.py | 10 +++++++-- scrapy/core/scraper.py | 2 +- tests/test_downloader_handler_httpx.py | 10 ++++----- tests/test_downloadermiddleware_httpcache.py | 14 ------------- tests/test_engine.py | 5 +++-- tests/test_extension_periodic_log.py | 4 ++-- tests/test_feedexport.py | 2 +- tests/test_loader.py | 11 ---------- tests/test_spidermiddleware_process_start.py | 5 +++++ tests/test_utils_defer.py | 22 +------------------- 10 files changed, 25 insertions(+), 60 deletions(-) diff --git a/scrapy/commands/__init__.py b/scrapy/commands/__init__.py index d5945f6f5..d666ca796 100644 --- a/scrapy/commands/__init__.py +++ b/scrapy/commands/__init__.py @@ -7,13 +7,14 @@ from __future__ import annotations import argparse import builtins import os +import warnings from abc import ABC, abstractmethod from pathlib import Path from typing import TYPE_CHECKING, Any from twisted.python import failure -from scrapy.exceptions import UsageError +from scrapy.exceptions import ScrapyDeprecationWarning, UsageError from scrapy.utils.conf import arglist_to_dict, feed_process_params_from_cli if TYPE_CHECKING: @@ -36,7 +37,12 @@ class ScrapyCommand(ABC): def __init__(self) -> None: self.settings: Settings | None = None # set in scrapy.cmdline - def set_crawler(self, crawler: Crawler) -> None: + def set_crawler(self, crawler: Crawler) -> None: # pragma: no cover + warnings.warn( + "ScrapyCommand.set_crawler() is deprecated", + ScrapyDeprecationWarning, + stacklevel=2, + ) if hasattr(self, "_crawler"): raise RuntimeError("crawler already set") self._crawler: Crawler = crawler diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index 695f6ef3e..701f7cb46 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -66,7 +66,7 @@ class Slot: self.queue: deque[QueueTuple] = deque() self.active: set[Request] = set() self.active_size: int = 0 - self.itemproc_size: int = 0 + self.itemproc_size: int = 0 # just for scrapy.utils.engine.get_engine_status() self.closing: Deferred[Spider] | None = None def add_response_request( diff --git a/tests/test_downloader_handler_httpx.py b/tests/test_downloader_handler_httpx.py index 233bc62fe..8d146b496 100644 --- a/tests/test_downloader_handler_httpx.py +++ b/tests/test_downloader_handler_httpx.py @@ -96,23 +96,21 @@ class TestSimpleHttps(HttpxDownloadHandlerMixin, TestSimpleHttpsBase): pass -class Https11WrongHostnameTestCase( - HttpxDownloadHandlerMixin, TestHttpsWrongHostnameBase -): +class TestHttps11WrongHostname(HttpxDownloadHandlerMixin, TestHttpsWrongHostnameBase): pass -class Https11InvalidDNSId(HttpxDownloadHandlerMixin, TestHttpsInvalidDNSIdBase): +class TestHttps11InvalidDNSId(HttpxDownloadHandlerMixin, TestHttpsInvalidDNSIdBase): pass -class Https11InvalidDNSPattern( +class TestHttps11InvalidDNSPattern( HttpxDownloadHandlerMixin, TestHttpsInvalidDNSPatternBase ): pass -class Https11CustomCiphers(HttpxDownloadHandlerMixin, TestHttpsCustomCiphersBase): +class TestHttps11CustomCiphers(HttpxDownloadHandlerMixin, TestHttpsCustomCiphersBase): pass diff --git a/tests/test_downloadermiddleware_httpcache.py b/tests/test_downloadermiddleware_httpcache.py index e94591038..548c0d8ee 100644 --- a/tests/test_downloadermiddleware_httpcache.py +++ b/tests/test_downloadermiddleware_httpcache.py @@ -89,20 +89,6 @@ class TestBase: assert response1.headers == response2.headers assert response1.body == response2.body - def assertEqualRequest(self, request1, request2): - assert request1.url == request2.url - assert request1.headers == request2.headers - assert request1.body == request2.body - - def assertEqualRequestButWithCacheValidators(self, request1, request2): - assert request1.url == request2.url - assert b"If-None-Match" not in request1.headers - assert b"If-Modified-Since" not in request1.headers - assert any( - h in request2.headers for h in (b"If-None-Match", b"If-Modified-Since") - ) - assert request1.body == request2.body - class StorageTestMixin: """Mixin containing storage-specific test methods.""" diff --git a/tests/test_engine.py b/tests/test_engine.py index 0c0432b3c..131beeead 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -26,6 +26,7 @@ from scrapy.http import Headers, Request, Response from scrapy.item import Field, Item from scrapy.linkextractors import LinkExtractor from scrapy.spiders import Spider +from scrapy.statscollectors import MemoryStatsCollector from scrapy.utils.defer import ( _schedule_coro, deferred_from_coro, @@ -42,7 +43,6 @@ if TYPE_CHECKING: from scrapy.core.scheduler import Scheduler from scrapy.crawler import Crawler - from scrapy.statscollectors import MemoryStatsCollector from tests.mockserver.http import MockServer @@ -741,7 +741,8 @@ class TestEngineCloseSpider: engine = ExecutionEngine(crawler, lambda _: None) crawler.engine = engine await engine.open_spider_async() - del cast("MemoryStatsCollector", crawler.stats).spider_stats + assert isinstance(crawler.stats, MemoryStatsCollector) + del crawler.stats.spider_stats await engine.close_spider_async() assert "Stats close failure" in caplog.text diff --git a/tests/test_extension_periodic_log.py b/tests/test_extension_periodic_log.py index 2cc23c491..86517e17b 100644 --- a/tests/test_extension_periodic_log.py +++ b/tests/test_extension_periodic_log.py @@ -98,7 +98,7 @@ class TestPeriodicLog: ext.spider_opened(spider) ext.set_a() a = ext.log_delta() - ext.set_a() + ext.set_b() b = ext.log_delta() ext.spider_closed(spider, reason="finished") return ext, a, b @@ -164,7 +164,7 @@ class TestPeriodicLog: ext.spider_opened(spider) ext.set_a() a = ext.log_crawler_stats() - ext.set_a() + ext.set_b() b = ext.log_crawler_stats() ext.spider_closed(spider, reason="finished") return ext, a, b diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index ab7341d29..ff9b0a3f9 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -288,7 +288,7 @@ class IsExportingListener: if self.start_without_finish: self.start_without_finish = False else: - self.finish_before_start = True + self.finish_without_start = True class ExceptionJsonItemExporter(JsonItemExporter): diff --git a/tests/test_loader.py b/tests/test_loader.py index 224158e7f..7cfc7ed26 100644 --- a/tests/test_loader.py +++ b/tests/test_loader.py @@ -55,17 +55,6 @@ class ProcessorItemLoader(NameItemLoader): name_in = MapCompose(lambda v: v.title()) -class DefaultedItemLoader(NameItemLoader): - default_input_processor = MapCompose(lambda v: v[:-1]) - - -# test processors -def processor_with_args(value, other=None, loader_context=None): - if "key" in loader_context: - return loader_context["key"] - return value - - class TestBasicItemLoader: def test_add_value_on_unknown_field(self): il = ProcessorItemLoader() diff --git a/tests/test_spidermiddleware_process_start.py b/tests/test_spidermiddleware_process_start.py index bccfc4432..3fbe9c94e 100644 --- a/tests/test_spidermiddleware_process_start.py +++ b/tests/test_spidermiddleware_process_start.py @@ -320,6 +320,11 @@ class TestMain: DeprecatedWrapSpider, ) + @coroutine_test + async def test_universal_mw_uses_process_start(self): + """Test that process_start_requests() isn't used when process_start() exists.""" + await self._test([UniversalSpiderMiddleware], ModernWrapSpider, [ITEM_B]) + async def _test_sleep(self, spider_middlewares): class TestSpider(Spider): name = "test" diff --git a/tests/test_utils_defer.py b/tests/test_utils_defer.py index 296bd6224..1b062d28d 100644 --- a/tests/test_utils_defer.py +++ b/tests/test_utils_defer.py @@ -6,7 +6,7 @@ from asyncio import Future from typing import TYPE_CHECKING, Any import pytest -from twisted.internet.defer import Deferred, inlineCallbacks, succeed +from twisted.internet.defer import Deferred, inlineCallbacks from scrapy.utils.asyncgen import as_async_generator, collect_asyncgen from scrapy.utils.defer import ( @@ -65,26 +65,6 @@ class TestMustbeDeferred: yield dfd -def cb1(value, arg1, arg2): - return f"(cb1 {value} {arg1} {arg2})" - - -def cb2(value, arg1, arg2): - return succeed(f"(cb2 {value} {arg1} {arg2})") - - -def cb3(value, arg1, arg2): - return f"(cb3 {value} {arg1} {arg2})" - - -def cb_fail(value, arg1, arg2): - raise TypeError - - -def eb1(failure, arg1, arg2): - return f"(eb1 {failure.value.__class__.__name__} {arg1} {arg2})" - - class TestIterErrback: def test_iter_errback_good(self): def itergood() -> Generator[int, None, None]: