Don't check robotstxt for local files

2023-01-26 16:10:57 +00:00 · 2023-01-26 16:10:57 +00:00 · 3054235dc0
parent e71eab6932
commit 3054235dc0
1 changed files with 2 additions and 0 deletions
--- a/scrapy/downloadermiddlewares/robotstxt.py
+++ b/scrapy/downloadermiddlewares/robotstxt.py
@ -38,6 +38,8 @@ class RobotsTxtMiddleware:
    def process_request(self, request, spider):
        if request.meta.get("dont_obey_robotstxt"):
            return
+        if request.url.startswith("data:") or request.url.startswith("file:"):
+            return
        d = maybeDeferred(self.robot_parser, request, spider)
        d.addCallback(self.process_request_2, request, spider)
        return d