RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

UTF-8 BOM at the beginning of the file ignored (#7095)

This commit is contained in:
Vasiliy Kiryanov 2025-10-15 10:15:07 -04:00 committed by GitHub
parent 4865a500b6
commit 7b215c6578
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 7 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
scrapy/robotstxt.py
View File

@ -28,7 +28,7 @@ def decode_robotstxt(
if to_native_str_type:
body_decoded = to_unicode(robotstxt_body)
else:
body_decoded = robotstxt_body.decode("utf-8", errors="ignore")
body_decoded = robotstxt_body.decode("utf-8-sig", errors="ignore")
except UnicodeDecodeError:
# If we found garbage or robots.txt in an encoding other than UTF-8, disregard it.
# Switch to 'allow all' state.

6
tests/test_robotstxt_interface.py
View File

@ -129,6 +129,12 @@ class TestDecodeRobotsTxt:
decoded_content = decode_robotstxt(robotstxt_body, spider=None)
assert decoded_content == "User-agent: *\nDisallow: /\n"
# UTF-8 BOM at the beginning of the file ignored
def test_decode_utf8_bom(self):
robotstxt_body = b"\xef\xbb\xbfUser-agent: *\nDisallow: /\n"
decoded_content = decode_robotstxt(robotstxt_body, spider=None)
assert decoded_content == "User-agent: *\nDisallow: /\n"
class TestPythonRobotParser(BaseRobotParserTest):
def setup_method(self):