diff --git a/scrapy/robotstxt.py b/scrapy/robotstxt.py index e1a12be05..18b622546 100644 --- a/scrapy/robotstxt.py +++ b/scrapy/robotstxt.py @@ -28,7 +28,7 @@ def decode_robotstxt( if to_native_str_type: body_decoded = to_unicode(robotstxt_body) else: - body_decoded = robotstxt_body.decode("utf-8", errors="ignore") + body_decoded = robotstxt_body.decode("utf-8-sig", errors="ignore") except UnicodeDecodeError: # If we found garbage or robots.txt in an encoding other than UTF-8, disregard it. # Switch to 'allow all' state. diff --git a/tests/test_robotstxt_interface.py b/tests/test_robotstxt_interface.py index 6a24d2e90..bc79e3a17 100644 --- a/tests/test_robotstxt_interface.py +++ b/tests/test_robotstxt_interface.py @@ -129,6 +129,12 @@ class TestDecodeRobotsTxt: decoded_content = decode_robotstxt(robotstxt_body, spider=None) assert decoded_content == "User-agent: *\nDisallow: /\n" + # UTF-8 BOM at the beginning of the file ignored + def test_decode_utf8_bom(self): + robotstxt_body = b"\xef\xbb\xbfUser-agent: *\nDisallow: /\n" + decoded_content = decode_robotstxt(robotstxt_body, spider=None) + assert decoded_content == "User-agent: *\nDisallow: /\n" + class TestPythonRobotParser(BaseRobotParserTest): def setup_method(self):