From 3054235dc09b1667c5c897f976eafa18845283e1 Mon Sep 17 00:00:00 2001
From: Cj Malone <Cj-Malone@users.noreply.github.com>
Date: Thu, 26 Jan 2023 16:10:57 +0000
Subject: [PATCH] Don't check robotstxt for local files

---
 scrapy/downloadermiddlewares/robotstxt.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/scrapy/downloadermiddlewares/robotstxt.py b/scrapy/downloadermiddlewares/robotstxt.py
index 326c35290..8e9beeeef 100644
--- a/scrapy/downloadermiddlewares/robotstxt.py
+++ b/scrapy/downloadermiddlewares/robotstxt.py
@@ -38,6 +38,8 @@ class RobotsTxtMiddleware:
     def process_request(self, request, spider):
         if request.meta.get("dont_obey_robotstxt"):
             return
+        if request.url.startswith("data:") or request.url.startswith("file:"):
+            return
         d = maybeDeferred(self.robot_parser, request, spider)
         d.addCallback(self.process_request_2, request, spider)
         return d