Merge pull request #5807 from Cj-Malone/patch-1

Don't check robots.txt for local files

scrapy/downloadermiddlewares/robotstxt.py

@@ -38,6 +38,8 @@ class RobotsTxtMiddleware:
     def process_request(self, request, spider):
         if request.meta.get("dont_obey_robotstxt"):
             return
+        if request.url.startswith("data:") or request.url.startswith("file:"):
+            return
         d = maybeDeferred(self.robot_parser, request, spider)
         d.addCallback(self.process_request_2, request, spider)
         return d

tests/test_downloadermiddleware_robotstxt.py

@@ -214,6 +214,19 @@ Disallow: /some/randome/page.html
         middleware.process_request_2(rp, Request("http://site.local/allowed"), None)
         rp.allowed.assert_called_once_with("http://site.local/allowed", "Examplebot")
 
+    def test_robotstxt_local_file(self):
+        middleware = RobotsTxtMiddleware(self._get_emptybody_crawler())
+        assert not middleware.process_request(
+            Request("data:text/plain,Hello World data"), None
+        )
+        assert not middleware.process_request(
+            Request("file:///tests/sample_data/test_site/nothinghere.html"), None
+        )
+        assert isinstance(
+            middleware.process_request(Request("http://site.local/allowed"), None),
+            Deferred,
+        )
+
     def assertNotIgnored(self, request, middleware):
         spider = None  # not actually used
         dfd = maybeDeferred(middleware.process_request, request, spider)