From 54a4c3af899b8c05c6d2545fa27af664bc231e0c Mon Sep 17 00:00:00 2001 From: Kaileshwar R K <159027881+Kaileshwar16@users.noreply.github.com> Date: Wed, 25 Mar 2026 02:01:12 +0530 Subject: [PATCH] Deprecate the mail API (#7263) --- docs/index.rst | 4 - docs/topics/coroutines.rst | 23 +--- docs/topics/email.rst | 185 ---------------------------- docs/topics/extensions.rst | 30 +---- docs/topics/settings.rst | 48 ++------ docs/topics/signals.rst | 12 ++ scrapy/extensions/memusage.py | 18 ++- scrapy/extensions/statsmailer.py | 9 +- scrapy/mail.py | 11 +- scrapy/signals.py | 1 + tests/test_extension_statsmailer.py | 10 +- tests/test_mail.py | 12 +- 12 files changed, 83 insertions(+), 280 deletions(-) delete mode 100644 docs/topics/email.rst diff --git a/docs/index.rst b/docs/index.rst index fe61f3513..a46a2ad9f 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -128,7 +128,6 @@ Built-in services topics/logging topics/stats - topics/email topics/telnetconsole :doc:`topics/logging` @@ -137,9 +136,6 @@ Built-in services :doc:`topics/stats` Collect statistics about your scraping crawler. -:doc:`topics/email` - Send email notifications when certain events occur. - :doc:`topics/telnetconsole` Inspect a running crawler using a built-in Python console. diff --git a/docs/topics/coroutines.rst b/docs/topics/coroutines.rst index a2dd67f68..ea3a60b4c 100644 --- a/docs/topics/coroutines.rst +++ b/docs/topics/coroutines.rst @@ -73,12 +73,6 @@ In the future we plan to add support for the ``async def`` syntax to these APIs or replace them with other APIs where changing the existing ones isn't possible. -These APIs don't have a coroutine-based counterpart: - -- :class:`~scrapy.mail.MailSender` - - - :meth:`~scrapy.mail.MailSender.send` - These APIs have a coroutine-based implementation and a Deferred-based one: - :class:`scrapy.crawler.Crawler`: @@ -137,18 +131,11 @@ wrapping a :class:`~twisted.internet.defer.Deferred` object into a :class:`~asyncio.Future` object or vice versa. See :ref:`asyncio-await-dfd` for more information about this. -For example: - -- The :meth:`MailSender.send() ` method returns - a :class:`~twisted.internet.defer.Deferred` object that fires when the - email is sent. You can use this object directly in Deferred-based code or - convert it into a :class:`~asyncio.Future` object with - :func:`~scrapy.utils.defer.maybe_deferred_to_future`. -- A custom scheduler needs to define an ``open()`` method that can return a - :class:`~twisted.internet.defer.Deferred` object. You can write a method - that works with Deferreds and returns one directly, or you can write a - coroutine and convert it into a function that returns a Deferred with - :func:`~scrapy.utils.defer.deferred_f_from_coro_f`. +For example: a custom scheduler needs to define an ``open()`` method that can +return a :class:`~twisted.internet.defer.Deferred` object. You can write a +method that works with Deferreds and returns one directly, or you can write a +coroutine and convert it into a function that returns a Deferred with +:func:`~scrapy.utils.defer.deferred_f_from_coro_f`. General usage diff --git a/docs/topics/email.rst b/docs/topics/email.rst deleted file mode 100644 index 1d7bad787..000000000 --- a/docs/topics/email.rst +++ /dev/null @@ -1,185 +0,0 @@ -.. _topics-email: - -============== -Sending e-mail -============== - -.. module:: scrapy.mail - :synopsis: Email sending facility - -Although Python makes sending e-mails relatively easy via the :mod:`smtplib` -library, Scrapy provides its own facility for sending e-mails which is very -easy to use and it's implemented using :doc:`Twisted non-blocking IO -`, to avoid interfering with the non-blocking -IO of the crawler. It also provides a simple API for sending attachments and -it's very easy to configure, with a few :ref:`settings -`. - -Quick example -============= - -There are two ways to instantiate the mail sender. You can instantiate it using -the standard ``__init__`` method: - -.. code-block:: python - - from scrapy.mail import MailSender - - mailer = MailSender() - -Or you can instantiate it passing a :class:`scrapy.Crawler` instance, which -will respect the :ref:`settings `: - -.. skip: start -.. code-block:: python - - mailer = MailSender.from_crawler(crawler) - -And here is how to use it to send an e-mail (without attachments): - -.. code-block:: python - - mailer.send( - to=["someone@example.com"], - subject="Some subject", - body="Some body", - cc=["another@example.com"], - ) -.. skip: end - -MailSender class reference -========================== - -The MailSender :ref:`components ` is the preferred class to -use for sending emails from Scrapy, as it uses :doc:`Twisted non-blocking IO -`, like the rest of the framework. - -.. class:: MailSender(smtphost=None, mailfrom=None, smtpuser=None, smtppass=None, smtpport=None) - - :param smtphost: the SMTP host to use for sending the emails. If omitted, the - :setting:`MAIL_HOST` setting will be used. - :type smtphost: str - - :param mailfrom: the address used to send emails (in the ``From:`` header). - If omitted, the :setting:`MAIL_FROM` setting will be used. - :type mailfrom: str - - :param smtpuser: the SMTP user. If omitted, the :setting:`MAIL_USER` - setting will be used. If not given, no SMTP authentication will be - performed. - :type smtphost: str or bytes - - :param smtppass: the SMTP pass for authentication. - :type smtppass: str or bytes - - :param smtpport: the SMTP port to connect to - :type smtpport: int - - :param smtptls: enforce using SMTP STARTTLS - :type smtptls: bool - - :param smtpssl: enforce using a secure SSL connection - :type smtpssl: bool - - .. method:: send(to, subject, body, cc=None, attachs=(), mimetype='text/plain', charset=None) - - Send email to the given recipients. - - :param to: the e-mail recipients as a string or as a list of strings - :type to: str or list - - :param subject: the subject of the e-mail - :type subject: str - - :param cc: the e-mails to CC as a string or as a list of strings - :type cc: str or list - - :param body: the e-mail body - :type body: str - - :param attachs: an iterable of tuples ``(attach_name, mimetype, - file_object)`` where ``attach_name`` is a string with the name that will - appear on the e-mail's attachment, ``mimetype`` is the mimetype of the - attachment and ``file_object`` is a readable file object with the - contents of the attachment - :type attachs: collections.abc.Iterable - - :param mimetype: the MIME type of the e-mail - :type mimetype: str - - :param charset: the character encoding to use for the e-mail contents - :type charset: str - - -.. _topics-email-settings: - -Mail settings -============= - -These settings define the default ``__init__`` method values of the :class:`MailSender` -class, and can be used to configure e-mail notifications in your project without -writing any code (for those extensions and code that uses :class:`MailSender`). - -.. setting:: MAIL_FROM - -MAIL_FROM ---------- - -Default: ``'scrapy@localhost'`` - -Sender email to use (``From:`` header) for sending emails. - -.. setting:: MAIL_HOST - -MAIL_HOST ---------- - -Default: ``'localhost'`` - -SMTP host to use for sending emails. - -.. setting:: MAIL_PORT - -MAIL_PORT ---------- - -Default: ``25`` - -SMTP port to use for sending emails. - -.. setting:: MAIL_USER - -MAIL_USER ---------- - -Default: ``None`` - -User to use for SMTP authentication. If disabled no SMTP authentication will be -performed. - -.. setting:: MAIL_PASS - -MAIL_PASS ---------- - -Default: ``None`` - -Password to use for SMTP authentication, along with :setting:`MAIL_USER`. - -.. setting:: MAIL_TLS - -MAIL_TLS --------- - -Default: ``False`` - -Enforce using STARTTLS. STARTTLS is a way to take an existing insecure connection, and upgrade it to a secure connection using SSL/TLS. - -.. setting:: MAIL_SSL - -MAIL_SSL --------- - -Default: ``False`` - -Enforce connecting using an SSL encrypted connection diff --git a/docs/topics/extensions.rst b/docs/topics/extensions.rst index 0fbd22a27..f0a8bb5ed 100644 --- a/docs/topics/extensions.rst +++ b/docs/topics/extensions.rst @@ -175,20 +175,16 @@ Memory usage extension Monitors the memory used by the Scrapy process that runs the spider and: -1. sends a notification e-mail when it exceeds a certain value -2. closes the spider when it exceeds a certain value - -The notification e-mails can be triggered when a certain warning value is -reached (:setting:`MEMUSAGE_WARNING_MB`) and when the maximum value is reached -(:setting:`MEMUSAGE_LIMIT_MB`) which will also cause the spider to be closed -and the Scrapy process to be terminated. +1. sends a :signal:`memusage_warning_reached` signal when it exceeds + :setting:`MEMUSAGE_WARNING_MB` +2. closes the spider with the `"memusage_exceeded"` reason when it exceeds + :setting:`MEMUSAGE_LIMIT_MB` This extension is enabled by the :setting:`MEMUSAGE_ENABLED` setting and can be configured with the following settings: * :setting:`MEMUSAGE_LIMIT_MB` * :setting:`MEMUSAGE_WARNING_MB` -* :setting:`MEMUSAGE_NOTIFY_MAIL` * :setting:`MEMUSAGE_CHECK_INTERVAL_SECONDS` Memory debugger extension @@ -332,24 +328,6 @@ closing the spider. If the spider generates more than that number of errors, it will be closed with the reason ``closespider_errorcount``. If zero (or non set), spiders won't be closed by number of errors. -StatsMailer extension -~~~~~~~~~~~~~~~~~~~~~ - -.. module:: scrapy.extensions.statsmailer - :synopsis: StatsMailer extension - -.. class:: StatsMailer - -This simple extension can be used to send a notification e-mail every time a -domain has finished scraping, including the Scrapy stats collected. The email -will be sent to all recipients specified in the :setting:`STATSMAILER_RCPTS` -setting. - -Emails can be sent using the :class:`~scrapy.mail.MailSender` class. To see a -full list of parameters, including examples on how to instantiate -:class:`~scrapy.mail.MailSender` and use mail settings, see -:ref:`topics-email`. - .. module:: scrapy.extensions.debug :synopsis: Extensions for debugging Scrapy diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 2be319ed2..6e8d0b54f 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -1562,13 +1562,12 @@ MEMUSAGE_ENABLED Default: ``True`` -Scope: ``scrapy.extensions.memusage`` +Scope: ``scrapy.extensions.memusage.MemoryUsage`` Whether to enable the memory usage extension. This extension keeps track of a peak memory used by the process (it writes it to stats). It can also optionally shutdown the Scrapy process when it exceeds a memory limit -(see :setting:`MEMUSAGE_LIMIT_MB`), and notify by email when that happened -(see :setting:`MEMUSAGE_NOTIFY_MAIL`). +(see :setting:`MEMUSAGE_LIMIT_MB`). See :ref:`topics-extensions-ref-memusage`. @@ -1579,10 +1578,11 @@ MEMUSAGE_LIMIT_MB Default: ``0`` -Scope: ``scrapy.extensions.memusage`` +Scope: ``scrapy.extensions.memusage.MemoryUsage`` The maximum amount of memory to allow (in megabytes) before shutting down -Scrapy (if MEMUSAGE_ENABLED is True). If zero, no check will be performed. +Scrapy (if :setting:`MEMUSAGE_ENABLED` is ``True``). If zero, no check will be +performed. See :ref:`topics-extensions-ref-memusage`. @@ -1593,7 +1593,7 @@ MEMUSAGE_CHECK_INTERVAL_SECONDS Default: ``60.0`` -Scope: ``scrapy.extensions.memusage`` +Scope: ``scrapy.extensions.memusage.MemoryUsage`` The :ref:`Memory usage extension ` checks the current memory usage, versus the limits set by @@ -1604,23 +1604,6 @@ This sets the length of these intervals, in seconds. See :ref:`topics-extensions-ref-memusage`. -.. setting:: MEMUSAGE_NOTIFY_MAIL - -MEMUSAGE_NOTIFY_MAIL --------------------- - -Default: ``False`` - -Scope: ``scrapy.extensions.memusage`` - -A list of emails to notify if the memory limit has been reached. - -Example:: - - MEMUSAGE_NOTIFY_MAIL = ['user@example.com'] - -See :ref:`topics-extensions-ref-memusage`. - .. setting:: MEMUSAGE_WARNING_MB MEMUSAGE_WARNING_MB @@ -1628,10 +1611,13 @@ MEMUSAGE_WARNING_MB Default: ``0`` -Scope: ``scrapy.extensions.memusage`` +Scope: ``scrapy.extensions.memusage.MemoryUsage`` -The maximum amount of memory to allow (in megabytes) before sending a warning -email notifying about it. If zero, no warning will be produced. +The maximum amount of memory to allow (in megabytes) before sending a +:signal:`memusage_warning_reached` signal (if :setting:`MEMUSAGE_ENABLED` is +``True``). If zero, no signal will be sent. + +See :ref:`topics-extensions-ref-memusage`. .. setting:: NEWSPIDER_MODULE @@ -1983,16 +1969,6 @@ finishes. For more info see: :ref:`topics-stats`. -.. setting:: STATSMAILER_RCPTS - -STATSMAILER_RCPTS ------------------ - -Default: ``[]`` (empty list) - -Send Scrapy stats after spiders finish scraping. See -:class:`~scrapy.extensions.statsmailer.StatsMailer` for more info. - .. setting:: TELNETCONSOLE_ENABLED TELNETCONSOLE_ENABLED diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst index efe3bb8fc..d13733623 100644 --- a/docs/topics/signals.rst +++ b/docs/topics/signals.rst @@ -356,6 +356,18 @@ feed_exporter_closed This signal supports :ref:`asynchronous handlers `. +memusage_warning_reached +~~~~~~~~~~~~~~~~~~~~~~~~ + +.. signal:: memusage_warning_reached + +.. function:: memusage_warning_reached() + + Sent by the :class:`~scrapy.extensions.memusage.MemoryUsage` extension when the + memory usage reaches the warning threshold (:setting:`MEMUSAGE_WARNING_MB`). + + This signal does not support :ref:`asynchronous handlers `. + Request signals --------------- diff --git a/scrapy/extensions/memusage.py b/scrapy/extensions/memusage.py index 990c144b5..f045e9b14 100644 --- a/scrapy/extensions/memusage.py +++ b/scrapy/extensions/memusage.py @@ -9,13 +9,13 @@ from __future__ import annotations import logging import socket import sys +import warnings from importlib import import_module from pprint import pformat from typing import TYPE_CHECKING from scrapy import signals -from scrapy.exceptions import NotConfigured -from scrapy.mail import MailSender +from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning from scrapy.utils.asyncio import AsyncioLoopingCall, create_looping_call from scrapy.utils.defer import _schedule_coro from scrapy.utils.engine import get_engine_status @@ -45,12 +45,23 @@ class MemoryUsage: self.crawler: Crawler = crawler self.warned: bool = False self.notify_mails: list[str] = crawler.settings.getlist("MEMUSAGE_NOTIFY_MAIL") + if self.notify_mails: # pragma: no cover + from scrapy.mail import MailSender # noqa: PLC0415 + + warnings.warn( + "The 'MEMUSAGE_NOTIFY_MAIL' setting is deprecated and will be removed " + "in a future release. Please use the 'memusage_warning_reached' and 'spider_closed' " + "signals to implement custom notifications.", + category=ScrapyDeprecationWarning, + stacklevel=2, + ) + self.mail = MailSender.from_crawler(crawler) + self.limit: int = crawler.settings.getint("MEMUSAGE_LIMIT_MB") * 1024 * 1024 self.warning: int = crawler.settings.getint("MEMUSAGE_WARNING_MB") * 1024 * 1024 self.check_interval: float = crawler.settings.getfloat( "MEMUSAGE_CHECK_INTERVAL_SECONDS" ) - self.mail: MailSender = MailSender.from_crawler(crawler) crawler.signals.connect(self.engine_started, signal=signals.engine_started) crawler.signals.connect(self.engine_stopped, signal=signals.engine_stopped) @@ -128,6 +139,7 @@ class MemoryUsage: assert self.crawler.stats if self.get_virtual_size() > self.warning: self.crawler.stats.set_value("memusage/warning_reached", 1) + self.crawler.signals.send_catch_log(signal=signals.memusage_warning_reached) mem = self.warning / 1024 / 1024 logger.warning( "Memory usage reached %(memusage)dMiB", diff --git a/scrapy/extensions/statsmailer.py b/scrapy/extensions/statsmailer.py index 9e900308d..47eb4bf93 100644 --- a/scrapy/extensions/statsmailer.py +++ b/scrapy/extensions/statsmailer.py @@ -6,10 +6,11 @@ Use STATSMAILER_RCPTS setting to enable and give the recipient mail address from __future__ import annotations +import warnings from typing import TYPE_CHECKING from scrapy import Spider, signals -from scrapy.exceptions import NotConfigured +from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning from scrapy.mail import MailSender if TYPE_CHECKING: @@ -21,6 +22,12 @@ if TYPE_CHECKING: from scrapy.crawler import Crawler from scrapy.statscollectors import StatsCollector +warnings.warn( + "The scrapy.extensions.statsmailer module is deprecated and will be " + "removed in a future release.", + category=ScrapyDeprecationWarning, +) + class StatsMailer: def __init__(self, stats: StatsCollector, recipients: list[str], mail: MailSender): diff --git a/scrapy/mail.py b/scrapy/mail.py index 84209effa..6ebbe195f 100644 --- a/scrapy/mail.py +++ b/scrapy/mail.py @@ -7,6 +7,7 @@ See documentation in docs/topics/email.rst from __future__ import annotations import logging +import warnings from email import encoders as Encoders from email.mime.base import MIMEBase from email.mime.multipart import MIMEMultipart @@ -19,6 +20,7 @@ from typing import IO, TYPE_CHECKING, Any from twisted.internet import ssl from twisted.internet.defer import Deferred +from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.utils.misc import arg_to_iter from scrapy.utils.python import to_bytes @@ -49,6 +51,13 @@ def _to_bytes_or_none(text: str | bytes | None) -> bytes | None: return to_bytes(text) +warnings.warn( + "The scrapy.mail module is deprecated and will be removed in a future release. " + "Please use a dedicated Python mail library instead.", + category=ScrapyDeprecationWarning, +) + + class MailSender: def __init__( self, @@ -71,7 +80,7 @@ class MailSender: self.debug: bool = debug @classmethod - def from_crawler(cls, crawler: Crawler) -> Self: + def from_crawler(cls, crawler: Crawler) -> Self: # pragma: no cover settings = crawler.settings return cls( smtphost=settings["MAIL_HOST"], diff --git a/scrapy/signals.py b/scrapy/signals.py index bdeec1ba0..972f4fd60 100644 --- a/scrapy/signals.py +++ b/scrapy/signals.py @@ -12,6 +12,7 @@ spider_opened = object() spider_idle = object() spider_closed = object() spider_error = object() +memusage_warning_reached = object() request_scheduled = object() request_dropped = object() request_reached_downloader = object() diff --git a/tests/test_extension_statsmailer.py b/tests/test_extension_statsmailer.py index d2e4d5c00..28db389c3 100644 --- a/tests/test_extension_statsmailer.py +++ b/tests/test_extension_statsmailer.py @@ -4,12 +4,18 @@ import pytest from scrapy import signals from scrapy.exceptions import NotConfigured -from scrapy.extensions import statsmailer -from scrapy.mail import MailSender from scrapy.signalmanager import SignalManager from scrapy.statscollectors import StatsCollector from scrapy.utils.spider import DefaultSpider +pytestmark = pytest.mark.filterwarnings( + "ignore:The scrapy.extensions.statsmailer module is deprecated:scrapy.exceptions.ScrapyDeprecationWarning", + "ignore:The scrapy.mail module is deprecated:scrapy.exceptions.ScrapyDeprecationWarning", +) + +from scrapy.extensions import statsmailer # noqa: E402 +from scrapy.mail import MailSender # noqa: E402 + @pytest.fixture def dummy_stats(): diff --git a/tests/test_mail.py b/tests/test_mail.py index 61136dfbe..e6d65cd10 100644 --- a/tests/test_mail.py +++ b/tests/test_mail.py @@ -5,11 +5,18 @@ import pytest from twisted.internet import defer from twisted.internet._sslverify import ClientTLSOptions -from scrapy.mail import MailSender +pytestmark = pytest.mark.filterwarnings( + "ignore:The scrapy.mail module is deprecated:scrapy.exceptions.ScrapyDeprecationWarning" +) + +from scrapy.mail import MailSender # noqa: E402 @pytest.mark.requires_reactor # MailSender requires a reactor class TestMailSender: + def _catch_mail_sent(self, **kwargs): + self.catched_msg = {**kwargs} + def test_send(self): mailsender = MailSender(debug=True) mailsender.send( @@ -88,9 +95,6 @@ class TestMailSender: assert text.get_charset() == Charset("us-ascii") assert attach.get_payload(decode=True) == b"content" - def _catch_mail_sent(self, **kwargs): - self.catched_msg = {**kwargs} - def test_send_utf8(self): subject = "sübjèçt" body = "bödÿ-àéïöñß"