Pass user-agent as native str when checking URLs against robots.txt

scrapy/downloadermiddlewares/robotstxt.py

@@ -41,7 +41,8 @@ class RobotsTxtMiddleware(object):
         return d
 
     def process_request_2(self, rp, request, spider):
-        if rp is not None and not rp.can_fetch(self._useragent, request.url):
+        if rp is not None and not rp.can_fetch(
+                 to_native_str(self._useragent), request.url):
             logger.debug("Forbidden by robots.txt: %(request)s",
                          {'request': request}, extra={'spider': spider})
             raise IgnoreRequest()

tests/test_downloadermiddleware_robotstxt.py

@@ -39,6 +39,9 @@ class RobotsTxtMiddlewareTest(unittest.TestCase):
         # taken from https://en.wikipedia.org/robots.txt
         Disallow: /wiki/K%C3%A4ytt%C3%A4j%C3%A4:
         Disallow: /wiki/Käyttäjä:
+
+        User-Agent: UnicödeBöt
+        Disallow: /some/randome/page.html
         '''.encode('utf-8'))
         response = TextResponse('http://site.local/robots.txt', body=ROBOTS)
         def return_response(request, spider):