diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 1abbc4968..a567ab6a9 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -1019,31 +1019,7 @@ RobotsTxtMiddleware .. module:: scrapy.downloadermiddlewares.robotstxt :synopsis: robots.txt middleware -.. class:: RobotsTxtMiddleware - - This middleware filters out requests forbidden by the robots.txt exclusion - standard. - - To make sure Scrapy respects robots.txt make sure the middleware is enabled - and the :setting:`ROBOTSTXT_OBEY` setting is enabled. - - The :setting:`ROBOTSTXT_USER_AGENT` setting can be used to specify the - user agent string to use for matching in the robots.txt_ file. If it - is ``None``, the User-Agent header you are sending with the request or the - :setting:`USER_AGENT` setting (in that order) will be used for determining - the user agent to use in the robots.txt_ file. - - This middleware has to be combined with a robots.txt_ parser. - - Scrapy ships with support for the following robots.txt_ parsers: - - * :ref:`Protego ` (default) - * :ref:`RobotFileParser ` - * :ref:`Robotexclusionrulesparser ` - * :ref:`Reppy ` (deprecated) - - You can change the robots.txt_ parser with the :setting:`ROBOTSTXT_PARSER` - setting. Or you can also :ref:`implement support for a new parser `. +.. autoclass:: RobotsTxtMiddleware .. reqmeta:: dont_obey_robotstxt diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index 21393c482..18193aff3 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -430,6 +430,9 @@ class ExecutionEngine: - ``memusage_exceeded``: See :class:`~scrapy.extensions.memusage.MemoryUsage`. + + - ``robotstxt_denied``: See + :class:`~scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware`. """ if self.slot is None: raise RuntimeError("Engine slot not assigned") diff --git a/scrapy/downloadermiddlewares/robotstxt.py b/scrapy/downloadermiddlewares/robotstxt.py index 6a0ecb7bf..2f5a8de36 100644 --- a/scrapy/downloadermiddlewares/robotstxt.py +++ b/scrapy/downloadermiddlewares/robotstxt.py @@ -18,6 +18,7 @@ from scrapy.exceptions import IgnoreRequest, NotConfigured from scrapy.http import Request, Response from scrapy.http.request import NO_CALLBACK from scrapy.robotstxt import RobotParser +from scrapy.spidermiddlewares.robotstxt import _start_requests_processed from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.log import failure_to_exc_info from scrapy.utils.misc import load_object @@ -31,9 +32,44 @@ logger = logging.getLogger(__name__) class RobotsTxtMiddleware: + """This middleware filters out requests forbidden by the robots.txt + exclusion standard. + + To make sure Scrapy respects robots.txt make sure the middleware is enabled + and the :setting:`ROBOTSTXT_OBEY` setting is enabled. + + The :setting:`ROBOTSTXT_USER_AGENT` setting can be used to specify the + user agent string to use for matching in the robots.txt_ file. If it + is ``None``, the User-Agent header you are sending with the request or the + :setting:`USER_AGENT` setting (in that order) will be used for determining + the user agent to use in the robots.txt_ file. + + This middleware has to be combined with a robots.txt_ parser. + + Scrapy ships with support for the following robots.txt_ parsers: + + * :ref:`Protego ` (default) + * :ref:`RobotFileParser ` + * :ref:`Robotexclusionrulesparser ` + * :ref:`Reppy ` (deprecated) + + You can change the robots.txt_ parser with the :setting:`ROBOTSTXT_PARSER` + setting. Or you can also :ref:`implement support for a new parser + `. + + If all start requests from a spider are ignored due to robots.txt rules, + the spider close reason becomes ``robotstxt_denied``. + """ + DOWNLOAD_PRIORITY: int = 1000 + @classmethod + def from_crawler(cls, crawler: Crawler) -> Self: + return cls(crawler) + def __init__(self, crawler: Crawler): + self._forbidden_start_request_count = 0 + self._total_start_request_count = 0 if not crawler.settings.getbool("ROBOTSTXT_OBEY"): raise NotConfigured self._default_useragent: str = crawler.settings.get("USER_AGENT", "Scrapy") @@ -49,9 +85,13 @@ class RobotsTxtMiddleware: # check if parser dependencies are met, this should throw an error otherwise. self._parserimpl.from_crawler(self.crawler, b"") - @classmethod - def from_crawler(cls, crawler: Crawler) -> Self: - return cls(crawler) + crawler.signals.connect( + self._start_requests_processed, signal=_start_requests_processed + ) + + def _start_requests_processed(self, count): + self._total_start_request_count = count + self._maybe_close() def process_request(self, request: Request, spider: Spider) -> Optional[Deferred]: if request.meta.get("dont_obey_robotstxt"): @@ -80,6 +120,11 @@ class RobotsTxtMiddleware: ) assert self.crawler.stats self.crawler.stats.inc_value("robotstxt/forbidden") + + if request.meta.get("is_start_request", False): + self._forbidden_start_request_count += 1 + self._maybe_close() + raise IgnoreRequest("Forbidden by robots.txt") def robot_parser( @@ -148,3 +193,15 @@ class RobotsTxtMiddleware: assert isinstance(rp_dfd, Deferred) self._parsers[netloc] = None rp_dfd.callback(None) + + def _maybe_close(self): + if not self._total_start_request_count: + return + if self._forbidden_start_request_count < self._total_start_request_count: + return + logger.error( + "Stopping the spider, all start requests failed because they " + "were rejected based on robots.txt rules. See " + "https://docs.scrapy.org/en/latest/topics/downloader-middleware.html#topics-dlmw-robots" + ) + self.crawler.engine.close_spider(self.crawler.spider, "robotstxt_denied") diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index d6b3585e2..07bc950e0 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -305,6 +305,7 @@ SPIDER_MIDDLEWARES_BASE = { "scrapy.spidermiddlewares.referer.RefererMiddleware": 700, "scrapy.spidermiddlewares.urllength.UrlLengthMiddleware": 800, "scrapy.spidermiddlewares.depth.DepthMiddleware": 900, + "scrapy.spidermiddlewares.robotstxt.RobotsTxtSpiderMiddleware": 1000, # Spider side } diff --git a/scrapy/spidermiddlewares/robotstxt.py b/scrapy/spidermiddlewares/robotstxt.py new file mode 100644 index 000000000..3987c7909 --- /dev/null +++ b/scrapy/spidermiddlewares/robotstxt.py @@ -0,0 +1,20 @@ +_start_requests_processed = object() + + +class RobotsTxtSpiderMiddleware: + @classmethod + def from_crawler(cls, crawler): + return cls(crawler) + + def __init__(self, crawler): + self._send_signal = crawler.signals.send_catch_log + + def process_start_requests(self, start_requests, spider): + # Mark start requests and reports to the downloader middleware the + # number of them once all have been processed. + count = 0 + for request in start_requests: + request.meta["is_start_request"] = True + yield request + count += 1 + self._send_signal(_start_requests_processed, count=count) diff --git a/tests/mockserver.py b/tests/mockserver.py index 647b0682e..60ab92c3b 100644 --- a/tests/mockserver.py +++ b/tests/mockserver.py @@ -21,6 +21,7 @@ from twisted.web.server import NOT_DONE_YET, GzipEncoderFactory, Site from twisted.web.static import File from twisted.web.util import redirectTo +from scrapy.utils.misc import load_object from scrapy.utils.python import to_bytes, to_unicode @@ -271,9 +272,16 @@ class Root(resource.Resource): class MockServer: + def __init__(self, resource=None): + self._args = [] + if resource: + resource_path = f"{resource.__module__}.{resource.__name__}" + self._args.append("--resource") + self._args.append(resource_path) + def __enter__(self): self.proc = Popen( - [sys.executable, "-u", "-m", "tests.mockserver", "-t", "http"], + [sys.executable, "-u", "-m", "tests.mockserver", *self._args, "-t", "http"], stdout=PIPE, env=get_mockserver_env(), ) @@ -378,13 +386,14 @@ if __name__ == "__main__": parser.add_argument( "-t", "--type", type=str, choices=("http", "dns"), default="http" ) + parser.add_argument("--resource", type=str, default="tests.mockserver.Root") args = parser.parse_args() factory: ServerFactory if args.type == "http": - root = Root() - factory = Site(root) + resource = load_object(args.resource)() + factory = Site(resource) httpPort = reactor.listenTCP(0, factory) contextFactory = ssl_context_factory() httpsPort = reactor.listenSSL(0, factory, contextFactory) diff --git a/tests/test_downloadermiddleware_robotstxt.py b/tests/test_downloadermiddleware_robotstxt.py index 26898a6a1..d29d455ac 100644 --- a/tests/test_downloadermiddleware_robotstxt.py +++ b/tests/test_downloadermiddleware_robotstxt.py @@ -1,19 +1,38 @@ from unittest import mock from twisted.internet import error, reactor -from twisted.internet.defer import Deferred, DeferredList, maybeDeferred +from twisted.internet.defer import ( + Deferred, + DeferredList, + inlineCallbacks, + maybeDeferred, +) from twisted.python import failure from twisted.trial import unittest +from twisted.web.resource import Resource +from scrapy import Spider from scrapy.downloadermiddlewares.robotstxt import RobotsTxtMiddleware from scrapy.downloadermiddlewares.robotstxt import logger as mw_module_logger from scrapy.exceptions import IgnoreRequest, NotConfigured from scrapy.http import Request, Response, TextResponse from scrapy.http.request import NO_CALLBACK from scrapy.settings import Settings +from scrapy.utils.test import get_crawler +from tests.mockserver import MockServer from tests.test_robotstxt_interface import reppy_available, rerp_available +class RobotsTxtResource(Resource): + def getChild(self, name, request): + return self + + def render_GET(self, request): + if request.path == b"/robots.txt": + return b"User-agent: *\n" b"Disallow: /deny/\n" + return b"foo" + + class RobotsTxtMiddlewareTest(unittest.TestCase): def setUp(self): self.crawler = mock.MagicMock() @@ -246,6 +265,102 @@ Disallow: /some/randome/page.html self.assertEqual(request.url, f"{base_url}/robots.txt") self.assertEqual(request.callback, NO_CALLBACK) + @inlineCallbacks + def test_forbidden_start_url(self): + class TestSpider(Spider): + name = "test" + + def parse(self, response): + TestSpider.response = response.text + + settings = {"ROBOTSTXT_OBEY": True} + crawler = get_crawler(TestSpider, settings_dict=settings) + + with MockServer(RobotsTxtResource) as server: + TestSpider.start_urls = [server.url("/deny/")] + yield crawler.crawl() + + self.assertEqual(crawler.stats.get_value("finish_reason"), "robotstxt_denied") + + @inlineCallbacks + def test_forbidden_start_urls(self): + class TestSpider(Spider): + name = "test" + + def parse(self, response): + TestSpider.response = response.text + + settings = {"ROBOTSTXT_OBEY": True} + crawler = get_crawler(TestSpider, settings_dict=settings) + + with MockServer(RobotsTxtResource) as server: + TestSpider.start_urls = [ + server.url("/deny/foo"), + server.url("/deny/bar"), + server.url("/deny/baz"), + ] + yield crawler.crawl() + + self.assertEqual(crawler.stats.get_value("finish_reason"), "robotstxt_denied") + + @inlineCallbacks + def test_some_forbidden_start_url(self): + class TestSpider(Spider): + name = "test" + + def parse(self, response): + TestSpider.response = response.text + + settings = {"ROBOTSTXT_OBEY": True} + crawler = get_crawler(TestSpider, settings_dict=settings) + + with MockServer(RobotsTxtResource) as server: + TestSpider.start_urls = [server.url("/deny"), server.url("/allow")] + yield crawler.crawl() + + self.assertEqual(crawler.stats.get_value("finish_reason"), "finished") + + @inlineCallbacks + def test_follow_up_forbidden_url(self): + settings = {"ROBOTSTXT_OBEY": True} + with MockServer(RobotsTxtResource) as server: + + class TestSpider(Spider): + name = "test" + start_urls = [server.url("/allow/")] + + def parse(self, response): + yield response.follow(server.url("/deny/")) + + crawler = get_crawler(TestSpider, settings_dict=settings) + yield crawler.crawl() + + self.assertEqual(crawler.stats.get_value("finish_reason"), "finished") + + @inlineCallbacks + def test_forbidden_with_partial_start_request_consumption(self): + """With concurrency lower than the number of start requests + 1, the + code path followed changes, because ``_total_start_request_count`` is + not set in the downloader middleware until *after* some start requests + have been processed.""" + settings = { + "CONCURRENT_REQUESTS": 1, + "ROBOTSTXT_OBEY": True, + } + with MockServer(RobotsTxtResource) as server: + + class TestSpider(Spider): + name = "test" + start_urls = [server.url("/deny/")] + + def parse(self, response): + yield response.follow(server.url("/deny/")) + + crawler = get_crawler(TestSpider, settings_dict=settings) + yield crawler.crawl() + + self.assertEqual(crawler.stats.get_value("finish_reason"), "robotstxt_denied") + class RobotsTxtMiddlewareWithRerpTest(RobotsTxtMiddlewareTest): if not rerp_available():