From 3054235dc09b1667c5c897f976eafa18845283e1 Mon Sep 17 00:00:00 2001 From: Cj Malone Date: Thu, 26 Jan 2023 16:10:57 +0000 Subject: [PATCH 1/2] Don't check robotstxt for local files --- scrapy/downloadermiddlewares/robotstxt.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scrapy/downloadermiddlewares/robotstxt.py b/scrapy/downloadermiddlewares/robotstxt.py index 326c35290..8e9beeeef 100644 --- a/scrapy/downloadermiddlewares/robotstxt.py +++ b/scrapy/downloadermiddlewares/robotstxt.py @@ -38,6 +38,8 @@ class RobotsTxtMiddleware: def process_request(self, request, spider): if request.meta.get("dont_obey_robotstxt"): return + if request.url.startswith("data:") or request.url.startswith("file:"): + return d = maybeDeferred(self.robot_parser, request, spider) d.addCallback(self.process_request_2, request, spider) return d From 33b85a9e2a379b355398e2daf416130bb840167d Mon Sep 17 00:00:00 2001 From: Cj Malone Date: Thu, 26 Jan 2023 19:51:20 +0000 Subject: [PATCH 2/2] Test local files aren't processed --- tests/test_downloadermiddleware_robotstxt.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tests/test_downloadermiddleware_robotstxt.py b/tests/test_downloadermiddleware_robotstxt.py index ac08c6006..fd27e637d 100644 --- a/tests/test_downloadermiddleware_robotstxt.py +++ b/tests/test_downloadermiddleware_robotstxt.py @@ -214,6 +214,19 @@ Disallow: /some/randome/page.html middleware.process_request_2(rp, Request("http://site.local/allowed"), None) rp.allowed.assert_called_once_with("http://site.local/allowed", "Examplebot") + def test_robotstxt_local_file(self): + middleware = RobotsTxtMiddleware(self._get_emptybody_crawler()) + assert not middleware.process_request( + Request("data:text/plain,Hello World data"), None + ) + assert not middleware.process_request( + Request("file:///tests/sample_data/test_site/nothinghere.html"), None + ) + assert isinstance( + middleware.process_request(Request("http://site.local/allowed"), None), + Deferred, + ) + def assertNotIgnored(self, request, middleware): spider = None # not actually used dfd = maybeDeferred(middleware.process_request, request, spider)