from unittest.mock import MagicMock import pytest from scrapy import signals from scrapy.exceptions import NotConfigured from scrapy.signalmanager import SignalManager from scrapy.statscollectors import StatsCollector from scrapy.utils.spider import DefaultSpider pytestmark = pytest.mark.filterwarnings( "ignore:The scrapy.extensions.statsmailer module is deprecated:scrapy.exceptions.ScrapyDeprecationWarning", "ignore:The scrapy.mail module is deprecated:scrapy.exceptions.ScrapyDeprecationWarning", ) from scrapy.extensions import statsmailer # noqa: E402 from scrapy.mail import MailSender # noqa: E402 @pytest.fixture def dummy_stats(): class DummyStats(StatsCollector): def __init__(self): # pylint: disable=super-init-not-called self._stats = {"global_item_scraped_count": 42} def get_stats(self): return {"item_scraped_count": 10, **self._stats} return DummyStats() def test_from_crawler_without_recipients_raises_notconfigured(): crawler = MagicMock() crawler.settings.getlist.return_value = [] crawler.stats = MagicMock() with pytest.raises(NotConfigured): statsmailer.StatsMailer.from_crawler(crawler) def test_from_crawler_with_recipients_initializes_extension(dummy_stats, monkeypatch): crawler = MagicMock() crawler.settings.getlist.return_value = ["test@example.com"] crawler.stats = dummy_stats crawler.signals = SignalManager(crawler) mailer = MagicMock(spec=MailSender) monkeypatch.setattr(statsmailer.MailSender, "from_crawler", lambda _: mailer) ext = statsmailer.StatsMailer.from_crawler(crawler) assert isinstance(ext, statsmailer.StatsMailer) assert ext.recipients == ["test@example.com"] assert ext.mail is mailer def test_from_crawler_connects_spider_closed_signal(dummy_stats, monkeypatch): crawler = MagicMock() crawler.settings.getlist.return_value = ["test@example.com"] crawler.stats = dummy_stats crawler.signals = SignalManager(crawler) mailer = MagicMock(spec=MailSender) monkeypatch.setattr(statsmailer.MailSender, "from_crawler", lambda _: mailer) statsmailer.StatsMailer.from_crawler(crawler) connected = crawler.signals.send_catch_log( signals.spider_closed, spider=DefaultSpider(name="dummy") ) assert connected is not None def test_spider_closed_sends_email(dummy_stats): recipients = ["test@example.com"] mail = MagicMock(spec=MailSender) ext = statsmailer.StatsMailer(dummy_stats, recipients, mail) spider = DefaultSpider(name="dummy") ext.spider_closed(spider) args, _ = mail.send.call_args to, subject, body = args assert to == recipients assert "Scrapy stats for: dummy" in subject assert "global_item_scraped_count" in body assert "item_scraped_count" in body