Parse robots.txt content as native str

Fixes #2373

scrapy/downloadermiddlewares/robotstxt.py

@@ -13,6 +13,7 @@ from scrapy.exceptions import NotConfigured, IgnoreRequest
 from scrapy.http import Request
 from scrapy.utils.httpobj import urlparse_cached
 from scrapy.utils.log import failure_to_exc_info
+from scrapy.utils.python import to_native_str
 
 logger = logging.getLogger(__name__)
 
@@ -94,7 +95,9 @@ class RobotsTxtMiddleware(object):
                 # Running rp.parse() will set rp state from
                 # 'disallow all' to 'allow any'.
                 pass
-        rp.parse(body.splitlines())
+        # stdlib's robotparser expects native 'str' ;
+        # with unicode input, non-ASCII encoded bytes decoding fails in Python2
+        rp.parse(to_native_str(body).splitlines())
 
         rp_dfd = self._parsers[netloc]
         self._parsers[netloc] = rp

tests/test_downloadermiddleware_robotstxt.py

@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 from __future__ import absolute_import
 import re
 from twisted.internet import reactor, error
@@ -30,11 +31,15 @@ class RobotsTxtMiddlewareTest(unittest.TestCase):
     def _get_successful_crawler(self):
         crawler = self.crawler
         crawler.settings.set('ROBOTSTXT_OBEY', True)
-        ROBOTS = re.sub(b'^\s+(?m)', b'', b'''
+        ROBOTS = re.sub(b'^\s+(?m)', b'', u'''
         User-Agent: *
         Disallow: /admin/
         Disallow: /static/
-        ''')
+
+        # taken from https://en.wikipedia.org/robots.txt
+        Disallow: /wiki/K%C3%A4ytt%C3%A4j%C3%A4:
+        Disallow: /wiki/Käyttäjä:
+        '''.encode('utf-8'))
         response = TextResponse('http://site.local/robots.txt', body=ROBOTS)
         def return_response(request, spider):
             deferred = Deferred()
@@ -48,7 +53,9 @@ class RobotsTxtMiddlewareTest(unittest.TestCase):
         return DeferredList([
             self.assertNotIgnored(Request('http://site.local/allowed'), middleware),
             self.assertIgnored(Request('http://site.local/admin/main'), middleware),
-            self.assertIgnored(Request('http://site.local/static/'), middleware)
+            self.assertIgnored(Request('http://site.local/static/'), middleware),
+            self.assertIgnored(Request('http://site.local/wiki/K%C3%A4ytt%C3%A4j%C3%A4:'), middleware),
+            self.assertIgnored(Request(u'http://site.local/wiki/Käyttäjä:'), middleware)
         ], fireOnOneErrback=True)
 
     def test_robotstxt_ready_parser(self):