From 7b215c6578de037aca85d538083905af4a0474ce Mon Sep 17 00:00:00 2001
From: Vasiliy Kiryanov <1378948+vasiliyk@users.noreply.github.com>
Date: Wed, 15 Oct 2025 10:15:07 -0400
Subject: [PATCH] UTF-8 BOM at the beginning of the file ignored (#7095)

---
 scrapy/robotstxt.py               | 2 +-
 tests/test_robotstxt_interface.py | 6 ++++++
 2 files changed, 7 insertions(+), 1 deletion(-)

diff --git a/scrapy/robotstxt.py b/scrapy/robotstxt.py
index e1a12be05..18b622546 100644
--- a/scrapy/robotstxt.py
+++ b/scrapy/robotstxt.py
@@ -28,7 +28,7 @@ def decode_robotstxt(
         if to_native_str_type:
             body_decoded = to_unicode(robotstxt_body)
         else:
-            body_decoded = robotstxt_body.decode("utf-8", errors="ignore")
+            body_decoded = robotstxt_body.decode("utf-8-sig", errors="ignore")
     except UnicodeDecodeError:
         # If we found garbage or robots.txt in an encoding other than UTF-8, disregard it.
         # Switch to 'allow all' state.
diff --git a/tests/test_robotstxt_interface.py b/tests/test_robotstxt_interface.py
index 6a24d2e90..bc79e3a17 100644
--- a/tests/test_robotstxt_interface.py
+++ b/tests/test_robotstxt_interface.py
@@ -129,6 +129,12 @@ class TestDecodeRobotsTxt:
         decoded_content = decode_robotstxt(robotstxt_body, spider=None)
         assert decoded_content == "User-agent: *\nDisallow: /\n"
 
+    # UTF-8 BOM at the beginning of the file ignored
+    def test_decode_utf8_bom(self):
+        robotstxt_body = b"\xef\xbb\xbfUser-agent: *\nDisallow: /\n"
+        decoded_content = decode_robotstxt(robotstxt_body, spider=None)
+        assert decoded_content == "User-agent: *\nDisallow: /\n"
+
 
 class TestPythonRobotParser(BaseRobotParserTest):
     def setup_method(self):