Add RobotParser.crawl_delay() and a robots_parsed signal (#7830)

* Add RobotParser.crawl_delay() and a robots_parsed signal

* Improve test coverage

docs/topics/signals.rst

@@ -504,6 +504,27 @@ headers_received
     :param spider: the spider associated with the response
     :type spider: :class:`~scrapy.Spider` object
 
+robots_parsed
+~~~~~~~~~~~~~
+
+.. signal:: robots_parsed
+.. function:: robots_parsed(robotparser, request)
+
+    .. versionadded:: VERSION
+
+    Sent by
+    :class:`~scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware` after it
+    downloads and parses a :file:`robots.txt` file, for the host that *request*
+    targets.
+
+    This signal supports :ref:`asynchronous handlers <signal-deferred>`.
+
+    :param robotparser: the parser holding the parsed :file:`robots.txt` contents
+    :type robotparser: :class:`~scrapy.robotstxt.RobotParser` object
+
+    :param request: the request that triggered the :file:`robots.txt` download
+    :type request: :class:`~scrapy.Request` object
+
 
 Response signals
 ----------------

scrapy/downloadermiddlewares/robotstxt.py

@@ -11,6 +11,7 @@ from typing import TYPE_CHECKING
 
 from twisted.internet.defer import Deferred
 
+from scrapy import signals
 from scrapy.exceptions import IgnoreRequest, NotConfigured
 from scrapy.http import Request, Response
 from scrapy.http.request import NO_CALLBACK
@@ -98,7 +99,7 @@ class RobotsTxtMiddleware:
             assert self.crawler.stats
             try:
                 resp = await self.crawler.engine.download_async(robotsreq)
-                self._parse_robots(resp, netloc)
+                await self._parse_robots(resp, netloc, request)
             except Exception as e:
                 if not isinstance(e, IgnoreRequest):
                     logger.error(
@@ -115,13 +116,20 @@ class RobotsTxtMiddleware:
             return await maybe_deferred_to_future(parser)
         return parser
 
-    def _parse_robots(self, response: Response, netloc: str) -> None:
+    async def _parse_robots(
+        self, response: Response, netloc: str, request: Request
+    ) -> None:
         assert self.crawler.stats
         self.crawler.stats.inc_value("robotstxt/response_count")
         self.crawler.stats.inc_value(
             f"robotstxt/response_status_count/{response.status}"
         )
         rp = self._parserimpl.from_crawler(self.crawler, response.body)
+        await self.crawler.signals.send_catch_log_async(
+            signal=signals.robots_parsed,
+            robotparser=rp,
+            request=request,
+        )
         rp_dfd = self._parsers[netloc]
         assert isinstance(rp_dfd, Deferred)
         self._parsers[netloc] = rp

scrapy/robotstxt.py

@@ -67,6 +67,15 @@ class RobotParser(metaclass=ABCMeta):
         :type user_agent: str or bytes
         """
 
+    def crawl_delay(self, user_agent: str | bytes) -> float | None:
+        """Return the ``Crawl-delay`` directive for ``user_agent`` as a number
+        of seconds, or ``None`` if it is not set or the backend does not support
+        it.
+
+        .. versionadded:: VERSION
+        """
+        return None
+
 
 class PythonRobotParser(RobotParser):
     def __init__(self, robotstxt_body: bytes, spider: Spider | None):
@@ -85,6 +94,10 @@ class PythonRobotParser(RobotParser):
         url = to_unicode(url)
         return self.rp.can_fetch(user_agent, url)
 
+    def crawl_delay(self, user_agent: str | bytes) -> float | None:
+        delay = self.rp.crawl_delay(to_unicode(user_agent))
+        return None if delay is None else float(delay)
+
 
 class RerpRobotParser(RobotParser):
     def __init__(self, robotstxt_body: bytes, spider: Spider | None):
@@ -105,6 +118,10 @@ class RerpRobotParser(RobotParser):
         url = to_unicode(url)
         return cast("bool", self.rp.is_allowed(user_agent, url))
 
+    def crawl_delay(self, user_agent: str | bytes) -> float | None:
+        delay = self.rp.get_crawl_delay(to_unicode(user_agent))
+        return None if delay is None else float(delay)
+
 
 class ProtegoRobotParser(RobotParser):
     def __init__(self, robotstxt_body: bytes, spider: Spider | None):
@@ -121,3 +138,7 @@ class ProtegoRobotParser(RobotParser):
         user_agent = to_unicode(user_agent)
         url = to_unicode(url)
         return self.rp.can_fetch(url, user_agent)
+
+    def crawl_delay(self, user_agent: str | bytes) -> float | None:
+        delay = self.rp.crawl_delay(to_unicode(user_agent))
+        return None if delay is None else float(delay)

scrapy/signals.py

@@ -21,6 +21,7 @@ response_received = object()
 response_downloaded = object()
 headers_received = object()
 bytes_received = object()
+robots_parsed = object()
 item_scraped = object()
 item_dropped = object()
 item_error = object()

tests/test_downloadermiddleware_robotstxt.py

@@ -8,6 +8,7 @@ import pytest
 from twisted.internet.defer import Deferred, DeferredList
 from twisted.python import failure
 
+from scrapy import signals
 from scrapy.downloadermiddlewares.robotstxt import RobotsTxtMiddleware
 from scrapy.exceptions import CannotResolveHostError, IgnoreRequest, NotConfigured
 from scrapy.http import Request, Response, TextResponse
@@ -27,6 +28,7 @@ class TestRobotsTxtMiddleware:
         self.crawler: mock.MagicMock = mock.MagicMock()
         self.crawler.settings = Settings()
         self.crawler.engine.download_async = mock.AsyncMock()
+        self.crawler.signals.send_catch_log_async = mock.AsyncMock(return_value=[])
 
     def teardown_method(self):
         del self.crawler
@@ -74,6 +76,21 @@ Disallow: /some/randome/page.html
             Request("http://site.local/wiki/Käyttäjä:"), middleware
         )
 
+    @coroutine_test
+    async def test_robotstxt_emits_robots_parsed_signal(self):
+        crawler = self._get_successful_crawler()
+        middleware = RobotsTxtMiddleware(crawler)
+        request = Request("http://site.local/allowed")
+        await self.assertNotIgnored(request, middleware)
+        calls = [
+            kwargs
+            for _, kwargs in crawler.signals.send_catch_log_async.call_args_list
+            if kwargs.get("signal") is signals.robots_parsed
+        ]
+        assert len(calls) == 1
+        assert calls[0]["request"] is request
+        assert calls[0]["robotparser"] is not None
+
     @coroutine_test
     async def test_robotstxt_multiple_reqs(self) -> None:
         middleware = RobotsTxtMiddleware(self._get_successful_crawler())

tests/test_robotstxt_interface.py

@@ -4,6 +4,7 @@ from scrapy.robotstxt import (
     ProtegoRobotParser,
     PythonRobotParser,
     RerpRobotParser,
+    RobotParser,
     decode_robotstxt,
 )
 from scrapy.utils._deps_compat import STDLIB_IMPROVED_ROBOTFILEPARSER
@@ -78,6 +79,16 @@ class BaseRobotParserTest:
         assert rp.allowed("https://site.local/index.html", "*")
         assert rp.allowed("https://site.local/disallowed", "*")
 
+    def test_crawl_delay(self):
+        robotstxt_body = b"User-agent: *\nDisallow: /private\nCrawl-delay: 10\n"
+        rp = self.parser_cls.from_crawler(crawler=None, robotstxt_body=robotstxt_body)
+        assert rp.crawl_delay("*") == 10.0
+
+    def test_crawl_delay_unset(self):
+        robotstxt_body = b"User-agent: *\nDisallow: /private\n"
+        rp = self.parser_cls.from_crawler(crawler=None, robotstxt_body=robotstxt_body)
+        assert rp.crawl_delay("*") is None
+
     def test_unicode_url_and_useragent(self):
         robotstxt_robotstxt_body = """
         User-Agent: *
@@ -102,6 +113,22 @@ class BaseRobotParserTest:
         assert not rp.allowed("https://site.local/some/randome/page.html", "UnicödeBöt")
 
 
+class TestRobotParser:
+    def test_crawl_delay_unsupported(self):
+        class AllowAllRobotParser(RobotParser):
+            @classmethod
+            def from_crawler(cls, crawler, robotstxt_body):
+                return cls()
+
+            def allowed(self, url, user_agent):
+                return True
+
+        rp = AllowAllRobotParser.from_crawler(
+            crawler=None, robotstxt_body=b"User-agent: *\nCrawl-delay: 10\n"
+        )
+        assert rp.crawl_delay("*") is None
+
+
 class TestDecodeRobotsTxt:
     def test_native_string_conversion(self):
         robotstxt_body = b"User-agent: *\nDisallow: /\n"