From 6cd35c77da6601f218b5042f4b9a9919e642e15c Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Tue, 15 Nov 2016 17:38:32 +0100
Subject: [PATCH] Pass user-agent as native str when checking URLs against
 robots.txt

---
 scrapy/downloadermiddlewares/robotstxt.py    | 3 ++-
 tests/test_downloadermiddleware_robotstxt.py | 3 +++
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/scrapy/downloadermiddlewares/robotstxt.py b/scrapy/downloadermiddlewares/robotstxt.py
index e26c22a09..c3dfa7819 100644
--- a/scrapy/downloadermiddlewares/robotstxt.py
+++ b/scrapy/downloadermiddlewares/robotstxt.py
@@ -41,7 +41,8 @@ class RobotsTxtMiddleware(object):
         return d
 
     def process_request_2(self, rp, request, spider):
-        if rp is not None and not rp.can_fetch(self._useragent, request.url):
+        if rp is not None and not rp.can_fetch(
+                 to_native_str(self._useragent), request.url):
             logger.debug("Forbidden by robots.txt: %(request)s",
                          {'request': request}, extra={'spider': spider})
             raise IgnoreRequest()
diff --git a/tests/test_downloadermiddleware_robotstxt.py b/tests/test_downloadermiddleware_robotstxt.py
index 95208c41f..60306eacb 100644
--- a/tests/test_downloadermiddleware_robotstxt.py
+++ b/tests/test_downloadermiddleware_robotstxt.py
@@ -39,6 +39,9 @@ class RobotsTxtMiddlewareTest(unittest.TestCase):
         # taken from https://en.wikipedia.org/robots.txt
         Disallow: /wiki/K%C3%A4ytt%C3%A4j%C3%A4:
         Disallow: /wiki/Käyttäjä:
+
+        User-Agent: UnicödeBöt
+        Disallow: /some/randome/page.html
         '''.encode('utf-8'))
         response = TextResponse('http://site.local/robots.txt', body=ROBOTS)
         def return_response(request, spider):