diff --git a/scrapy/commands/__init__.py b/scrapy/commands/__init__.py index d9c919db9..bb602784b 100644 --- a/scrapy/commands/__init__.py +++ b/scrapy/commands/__init__.py @@ -6,6 +6,7 @@ from __future__ import annotations import argparse import builtins +import logging import os import warnings from abc import ABC, abstractmethod @@ -14,7 +15,9 @@ from typing import TYPE_CHECKING, Any, ClassVar from twisted.python import failure +from scrapy import signals from scrapy.exceptions import ScrapyDeprecationWarning, UsageError +from scrapy.extensions.feedexport import FeedExporter from scrapy.utils.conf import arglist_to_dict, feed_process_params_from_cli from scrapy.utils.deprecate import method_is_overridden from scrapy.utils.python import global_object_name @@ -22,10 +25,14 @@ from scrapy.utils.python import global_object_name if TYPE_CHECKING: from collections.abc import Iterable + from scrapy import Spider from scrapy.crawler import Crawler, CrawlerProcessBase from scrapy.settings import Settings +logger = logging.getLogger(__name__) + + class ScrapyCommand(ABC): requires_project: bool = False requires_crawler_process: bool = True @@ -206,6 +213,24 @@ class BaseRunSpiderCommand(ScrapyCommand): ) self.settings.set("FEEDS", feeds, priority="cmdline") + def _create_crawler(self, spidercls: type[Spider] | str) -> Crawler: + assert self.crawler_process is not None + crawler = self.crawler_process.create_crawler(spidercls) + crawler.signals.connect( + self._warn_if_feeds_unused, signal=signals.engine_started + ) + return crawler + + def _warn_if_feeds_unused(self, sender: Crawler, **kwargs: Any) -> None: + if ( + sender.settings.getdict("FEEDS") + and sender.get_extension(FeedExporter) is None + ): + logger.warning( + "The FeedExporter extension is not enabled, so no item will be " + "exported to the configured feeds (FEEDS, -o, -O)." + ) + class ScrapyHelpFormatter(argparse.HelpFormatter): """ diff --git a/scrapy/commands/crawl.py b/scrapy/commands/crawl.py index 4e086e057..7cc761142 100644 --- a/scrapy/commands/crawl.py +++ b/scrapy/commands/crawl.py @@ -28,7 +28,7 @@ class Command(BaseRunSpiderCommand): spname = args[0] assert self.crawler_process - self.crawler_process.crawl(spname, **opts.spargs) + self.crawler_process.crawl(self._create_crawler(spname), **opts.spargs) self.crawler_process.start() if self.crawler_process.bootstrap_failed: self.exitcode = 1 diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py index b6255ec7e..c266c5004 100644 --- a/scrapy/commands/parse.py +++ b/scrapy/commands/parse.py @@ -266,8 +266,8 @@ class Command(BaseRunSpiderCommand): def start_parsing(self, url: str, opts: argparse.Namespace) -> None: assert self.crawler_process assert self.spidercls - self.crawler_process.crawl(self.spidercls, **opts.spargs) - self.pcrawler = next(iter(self.crawler_process.crawlers)) + self.pcrawler = self._create_crawler(self.spidercls) + self.crawler_process.crawl(self.pcrawler, **opts.spargs) self.crawler_process.start() if not self.first_response: diff --git a/scrapy/commands/runspider.py b/scrapy/commands/runspider.py index 9cdb393ab..09d268f1e 100644 --- a/scrapy/commands/runspider.py +++ b/scrapy/commands/runspider.py @@ -59,7 +59,7 @@ class Command(BaseRunSpiderCommand): spidercls = spclasses.pop() assert self.crawler_process - self.crawler_process.crawl(spidercls, **opts.spargs) + self.crawler_process.crawl(self._create_crawler(spidercls), **opts.spargs) self.crawler_process.start() if self.crawler_process.bootstrap_failed: diff --git a/tests/test_command_runspider.py b/tests/test_command_runspider.py index 11036eaeb..25ba8be39 100644 --- a/tests/test_command_runspider.py +++ b/tests/test_command_runspider.py @@ -309,6 +309,55 @@ class MySpider(scrapy.Spider): log = self.get_log(tmp_path, spider_code, args=args) assert "[myspider] DEBUG: FEEDS: {'stdout:': {'format': 'json'}}" in log + def test_output_feed_exporter_disabled(self, tmp_path: Path) -> None: + spider_code = """ +import scrapy + +class MySpider(scrapy.Spider): + name = 'myspider' + + custom_settings = { + "EXTENSIONS": {"scrapy.extensions.feedexport.FeedExporter": None}, + } + + start_urls = ["data:,"] + + def parse(self, response): + yield {"hello": "world"} +""" + args = ["-o", "example.json"] + log = self.get_log(tmp_path, spider_code, args=args) + assert "FeedExporter extension is not enabled" in log + assert not (tmp_path / "example.json").exists() + + def test_output_feed_exporter_subclass(self, tmp_path: Path) -> None: + spider_code = """ +import scrapy +from scrapy.extensions.feedexport import FeedExporter + +class MyFeedExporter(FeedExporter): + pass + +class MySpider(scrapy.Spider): + name = 'myspider' + + custom_settings = { + "EXTENSIONS": { + "scrapy.extensions.feedexport.FeedExporter": None, + MyFeedExporter: 0, + }, + } + + start_urls = ["data:,"] + + def parse(self, response): + yield {"hello": "world"} +""" + args = ["-o", "example.json"] + log = self.get_log(tmp_path, spider_code, args=args) + assert "FeedExporter extension is not enabled" not in log + assert (tmp_path / "example.json").exists() + @pytest.mark.parametrize("arg", ["output.json:json", "output.json"]) def test_absolute_path(self, tmp_path: Path, arg: str) -> None: spider_code = """