Deprecate ScrapyCommand.set_crawler(), remove/fix some dead code. (#7276)

scrapy/commands/__init__.py

@@ -7,13 +7,14 @@ from __future__ import annotations
 import argparse
 import builtins
 import os
+import warnings
 from abc import ABC, abstractmethod
 from pathlib import Path
 from typing import TYPE_CHECKING, Any
 
 from twisted.python import failure
 
-from scrapy.exceptions import UsageError
+from scrapy.exceptions import ScrapyDeprecationWarning, UsageError
 from scrapy.utils.conf import arglist_to_dict, feed_process_params_from_cli
 
 if TYPE_CHECKING:
@@ -36,7 +37,12 @@ class ScrapyCommand(ABC):
     def __init__(self) -> None:
         self.settings: Settings | None = None  # set in scrapy.cmdline
 
-    def set_crawler(self, crawler: Crawler) -> None:
+    def set_crawler(self, crawler: Crawler) -> None:  # pragma: no cover
+        warnings.warn(
+            "ScrapyCommand.set_crawler() is deprecated",
+            ScrapyDeprecationWarning,
+            stacklevel=2,
+        )
         if hasattr(self, "_crawler"):
             raise RuntimeError("crawler already set")
         self._crawler: Crawler = crawler

scrapy/core/scraper.py

@@ -66,7 +66,7 @@ class Slot:
         self.queue: deque[QueueTuple] = deque()
         self.active: set[Request] = set()
         self.active_size: int = 0
-        self.itemproc_size: int = 0
+        self.itemproc_size: int = 0  # just for scrapy.utils.engine.get_engine_status()
         self.closing: Deferred[Spider] | None = None
 
     def add_response_request(

tests/test_downloader_handler_httpx.py

@@ -96,23 +96,21 @@ class TestSimpleHttps(HttpxDownloadHandlerMixin, TestSimpleHttpsBase):
     pass
 
 
-class Https11WrongHostnameTestCase(
-    HttpxDownloadHandlerMixin, TestHttpsWrongHostnameBase
-):
+class TestHttps11WrongHostname(HttpxDownloadHandlerMixin, TestHttpsWrongHostnameBase):
     pass
 
 
-class Https11InvalidDNSId(HttpxDownloadHandlerMixin, TestHttpsInvalidDNSIdBase):
+class TestHttps11InvalidDNSId(HttpxDownloadHandlerMixin, TestHttpsInvalidDNSIdBase):
     pass
 
 
-class Https11InvalidDNSPattern(
+class TestHttps11InvalidDNSPattern(
     HttpxDownloadHandlerMixin, TestHttpsInvalidDNSPatternBase
 ):
     pass
 
 
-class Https11CustomCiphers(HttpxDownloadHandlerMixin, TestHttpsCustomCiphersBase):
+class TestHttps11CustomCiphers(HttpxDownloadHandlerMixin, TestHttpsCustomCiphersBase):
     pass
 
 

tests/test_downloadermiddleware_httpcache.py

@@ -89,20 +89,6 @@ class TestBase:
         assert response1.headers == response2.headers
         assert response1.body == response2.body
 
-    def assertEqualRequest(self, request1, request2):
-        assert request1.url == request2.url
-        assert request1.headers == request2.headers
-        assert request1.body == request2.body
-
-    def assertEqualRequestButWithCacheValidators(self, request1, request2):
-        assert request1.url == request2.url
-        assert b"If-None-Match" not in request1.headers
-        assert b"If-Modified-Since" not in request1.headers
-        assert any(
-            h in request2.headers for h in (b"If-None-Match", b"If-Modified-Since")
-        )
-        assert request1.body == request2.body
-
 
 class StorageTestMixin:
     """Mixin containing storage-specific test methods."""

tests/test_engine.py

@@ -26,6 +26,7 @@ from scrapy.http import Headers, Request, Response
 from scrapy.item import Field, Item
 from scrapy.linkextractors import LinkExtractor
 from scrapy.spiders import Spider
+from scrapy.statscollectors import MemoryStatsCollector
 from scrapy.utils.defer import (
     _schedule_coro,
     deferred_from_coro,
@@ -42,7 +43,6 @@ if TYPE_CHECKING:
 
     from scrapy.core.scheduler import Scheduler
     from scrapy.crawler import Crawler
-    from scrapy.statscollectors import MemoryStatsCollector
     from tests.mockserver.http import MockServer
 
 
@@ -741,7 +741,8 @@ class TestEngineCloseSpider:
         engine = ExecutionEngine(crawler, lambda _: None)
         crawler.engine = engine
         await engine.open_spider_async()
-        del cast("MemoryStatsCollector", crawler.stats).spider_stats
+        assert isinstance(crawler.stats, MemoryStatsCollector)
+        del crawler.stats.spider_stats
         await engine.close_spider_async()
         assert "Stats close failure" in caplog.text
 

tests/test_extension_periodic_log.py

@@ -98,7 +98,7 @@ class TestPeriodicLog:
             ext.spider_opened(spider)
             ext.set_a()
             a = ext.log_delta()
-            ext.set_a()
+            ext.set_b()
             b = ext.log_delta()
             ext.spider_closed(spider, reason="finished")
             return ext, a, b
@@ -164,7 +164,7 @@ class TestPeriodicLog:
             ext.spider_opened(spider)
             ext.set_a()
             a = ext.log_crawler_stats()
-            ext.set_a()
+            ext.set_b()
             b = ext.log_crawler_stats()
             ext.spider_closed(spider, reason="finished")
             return ext, a, b

tests/test_feedexport.py

@@ -288,7 +288,7 @@ class IsExportingListener:
             if self.start_without_finish:
                 self.start_without_finish = False
             else:
-                self.finish_before_start = True
+                self.finish_without_start = True
 
 
 class ExceptionJsonItemExporter(JsonItemExporter):

tests/test_loader.py

@@ -55,17 +55,6 @@ class ProcessorItemLoader(NameItemLoader):
     name_in = MapCompose(lambda v: v.title())
 
 
-class DefaultedItemLoader(NameItemLoader):
-    default_input_processor = MapCompose(lambda v: v[:-1])
-
-
-# test processors
-def processor_with_args(value, other=None, loader_context=None):
-    if "key" in loader_context:
-        return loader_context["key"]
-    return value
-
-
 class TestBasicItemLoader:
     def test_add_value_on_unknown_field(self):
         il = ProcessorItemLoader()

tests/test_spidermiddleware_process_start.py

@@ -320,6 +320,11 @@ class TestMain:
                 DeprecatedWrapSpider,
             )
 
+    @coroutine_test
+    async def test_universal_mw_uses_process_start(self):
+        """Test that process_start_requests() isn't used when process_start() exists."""
+        await self._test([UniversalSpiderMiddleware], ModernWrapSpider, [ITEM_B])
+
     async def _test_sleep(self, spider_middlewares):
         class TestSpider(Spider):
             name = "test"

tests/test_utils_defer.py

@@ -6,7 +6,7 @@ from asyncio import Future
 from typing import TYPE_CHECKING, Any
 
 import pytest
-from twisted.internet.defer import Deferred, inlineCallbacks, succeed
+from twisted.internet.defer import Deferred, inlineCallbacks
 
 from scrapy.utils.asyncgen import as_async_generator, collect_asyncgen
 from scrapy.utils.defer import (
@@ -65,26 +65,6 @@ class TestMustbeDeferred:
         yield dfd
 
 
-def cb1(value, arg1, arg2):
-    return f"(cb1 {value} {arg1} {arg2})"
-
-
-def cb2(value, arg1, arg2):
-    return succeed(f"(cb2 {value} {arg1} {arg2})")
-
-
-def cb3(value, arg1, arg2):
-    return f"(cb3 {value} {arg1} {arg2})"
-
-
-def cb_fail(value, arg1, arg2):
-    raise TypeError
-
-
-def eb1(failure, arg1, arg2):
-    return f"(eb1 {failure.value.__class__.__name__} {arg1} {arg2})"
-
-
 class TestIterErrback:
     def test_iter_errback_good(self):
         def itergood() -> Generator[int, None, None]: