Merge 39ae2eb5b2 into ad43bf0c56

scrapy/commands/__init__.py

@@ -6,6 +6,7 @@ from __future__ import annotations
 
 import argparse
 import builtins
+import logging
 import os
 import warnings
 from abc import ABC, abstractmethod
@@ -14,7 +15,9 @@ from typing import TYPE_CHECKING, Any, ClassVar
 
 from twisted.python import failure
 
+from scrapy import signals
 from scrapy.exceptions import ScrapyDeprecationWarning, UsageError
+from scrapy.extensions.feedexport import FeedExporter
 from scrapy.utils.conf import arglist_to_dict, feed_process_params_from_cli
 from scrapy.utils.deprecate import method_is_overridden
 from scrapy.utils.python import global_object_name
@@ -22,10 +25,14 @@ from scrapy.utils.python import global_object_name
 if TYPE_CHECKING:
     from collections.abc import Iterable
 
+    from scrapy import Spider
     from scrapy.crawler import Crawler, CrawlerProcessBase
     from scrapy.settings import Settings
 
 
+logger = logging.getLogger(__name__)
+
+
 class ScrapyCommand(ABC):
     requires_project: bool = False
     requires_crawler_process: bool = True
@@ -206,6 +213,24 @@ class BaseRunSpiderCommand(ScrapyCommand):
             )
             self.settings.set("FEEDS", feeds, priority="cmdline")
 
+    def _create_crawler(self, spidercls: type[Spider] | str) -> Crawler:
+        assert self.crawler_process is not None
+        crawler = self.crawler_process.create_crawler(spidercls)
+        crawler.signals.connect(
+            self._warn_if_feeds_unused, signal=signals.engine_started
+        )
+        return crawler
+
+    def _warn_if_feeds_unused(self, sender: Crawler, **kwargs: Any) -> None:
+        if (
+            sender.settings.getdict("FEEDS")
+            and sender.get_extension(FeedExporter) is None
+        ):
+            logger.warning(
+                "The FeedExporter extension is not enabled, so no item will be "
+                "exported to the configured feeds (FEEDS, -o, -O)."
+            )
+
 
 class ScrapyHelpFormatter(argparse.HelpFormatter):
     """

scrapy/commands/crawl.py

@@ -28,7 +28,7 @@ class Command(BaseRunSpiderCommand):
         spname = args[0]
 
         assert self.crawler_process
-        self.crawler_process.crawl(spname, **opts.spargs)
+        self.crawler_process.crawl(self._create_crawler(spname), **opts.spargs)
         self.crawler_process.start()
         if self.crawler_process.bootstrap_failed:
             self.exitcode = 1

scrapy/commands/parse.py

@@ -266,8 +266,8 @@ class Command(BaseRunSpiderCommand):
     def start_parsing(self, url: str, opts: argparse.Namespace) -> None:
         assert self.crawler_process
         assert self.spidercls
-        self.crawler_process.crawl(self.spidercls, **opts.spargs)
-        self.pcrawler = next(iter(self.crawler_process.crawlers))
+        self.pcrawler = self._create_crawler(self.spidercls)
+        self.crawler_process.crawl(self.pcrawler, **opts.spargs)
         self.crawler_process.start()
 
         if not self.first_response:

scrapy/commands/runspider.py

@@ -59,7 +59,7 @@ class Command(BaseRunSpiderCommand):
         spidercls = spclasses.pop()
 
         assert self.crawler_process
-        self.crawler_process.crawl(spidercls, **opts.spargs)
+        self.crawler_process.crawl(self._create_crawler(spidercls), **opts.spargs)
         self.crawler_process.start()
 
         if self.crawler_process.bootstrap_failed:

tests/test_command_runspider.py

@@ -309,6 +309,55 @@ class MySpider(scrapy.Spider):
         log = self.get_log(tmp_path, spider_code, args=args)
         assert "[myspider] DEBUG: FEEDS: {'stdout:': {'format': 'json'}}" in log
 
+    def test_output_feed_exporter_disabled(self, tmp_path: Path) -> None:
+        spider_code = """
+import scrapy
+
+class MySpider(scrapy.Spider):
+    name = 'myspider'
+
+    custom_settings = {
+        "EXTENSIONS": {"scrapy.extensions.feedexport.FeedExporter": None},
+    }
+
+    start_urls = ["data:,"]
+
+    def parse(self, response):
+        yield {"hello": "world"}
+"""
+        args = ["-o", "example.json"]
+        log = self.get_log(tmp_path, spider_code, args=args)
+        assert "FeedExporter extension is not enabled" in log
+        assert not (tmp_path / "example.json").exists()
+
+    def test_output_feed_exporter_subclass(self, tmp_path: Path) -> None:
+        spider_code = """
+import scrapy
+from scrapy.extensions.feedexport import FeedExporter
+
+class MyFeedExporter(FeedExporter):
+    pass
+
+class MySpider(scrapy.Spider):
+    name = 'myspider'
+
+    custom_settings = {
+        "EXTENSIONS": {
+            "scrapy.extensions.feedexport.FeedExporter": None,
+            MyFeedExporter: 0,
+        },
+    }
+
+    start_urls = ["data:,"]
+
+    def parse(self, response):
+        yield {"hello": "world"}
+"""
+        args = ["-o", "example.json"]
+        log = self.get_log(tmp_path, spider_code, args=args)
+        assert "FeedExporter extension is not enabled" not in log
+        assert (tmp_path / "example.json").exists()
+
     @pytest.mark.parametrize("arg", ["output.json:json", "output.json"])
     def test_absolute_path(self, tmp_path: Path, arg: str) -> None:
         spider_code = """