Handle robots.txt files not UTF-8 encoded

scrapy/robotstxt.py

@@ -23,7 +23,7 @@ def decode_robotstxt(robotstxt_body, spider, to_native_str_type=False):
         if to_native_str_type:
             robotstxt_body = to_unicode(robotstxt_body)
         else:
-            robotstxt_body = robotstxt_body.decode("utf-8")
+            robotstxt_body = robotstxt_body.decode("utf-8", errors="ignore")
     except UnicodeDecodeError:
         # If we found garbage or robots.txt in an encoding other than UTF-8, disregard it.
         # Switch to 'allow all' state.

tests/test_robotstxt_interface.py

@@ -1,5 +1,7 @@
 from twisted.trial import unittest
 
+from scrapy.robotstxt import decode_robotstxt
+
 
 def reppy_available():
     # check if reppy parser is installed
@@ -141,6 +143,25 @@ class BaseRobotParserTest:
         )
 
 
+class DecodeRobotsTxtTest(unittest.TestCase):
+    def test_native_string_conversion(self):
+        robotstxt_body = "User-agent: *\nDisallow: /\n".encode("utf-8")
+        decoded_content = decode_robotstxt(
+            robotstxt_body, spider=None, to_native_str_type=True
+        )
+        self.assertEqual(decoded_content, "User-agent: *\nDisallow: /\n")
+
+    def test_decode_utf8(self):
+        robotstxt_body = "User-agent: *\nDisallow: /\n".encode("utf-8")
+        decoded_content = decode_robotstxt(robotstxt_body, spider=None)
+        self.assertEqual(decoded_content, "User-agent: *\nDisallow: /\n")
+
+    def test_decode_non_utf8(self):
+        robotstxt_body = b"User-agent: *\n\xFFDisallow: /\n"
+        decoded_content = decode_robotstxt(robotstxt_body, spider=None)
+        self.assertEqual(decoded_content, "User-agent: *\nDisallow: /\n")
+
+
 class PythonRobotParserTest(BaseRobotParserTest, unittest.TestCase):
     def setUp(self):
         from scrapy.robotstxt import PythonRobotParser