Merge pull request #3966 from anubhavp28/robotstxt_useragent

Adds ROBOTSTXT_USER_AGENT setting

docs/topics/downloader-middleware.rst

@@ -989,6 +989,12 @@ RobotsTxtMiddleware
     To make sure Scrapy respects robots.txt make sure the middleware is enabled
     and the :setting:`ROBOTSTXT_OBEY` setting is enabled.
 
+    The :setting:`ROBOTSTXT_USER_AGENT` setting can be used to specify the
+    user agent string to use for matching in the robots.txt_ file. If it
+    is ``None``, the User-Agent header you are sending with the request or the
+    :setting:`USER_AGENT` setting (in that order) will be used for determining
+    the user agent to use in the robots.txt_ file.
+
     This middleware has to be combined with a robots.txt_ parser.
 
     Scrapy ships with support for the following robots.txt_ parsers:

docs/topics/settings.rst

@@ -1170,6 +1170,18 @@ Default: ``'scrapy.robotstxt.PythonRobotParser'``
 The parser backend to use for parsing ``robots.txt`` files. For more information see
 :ref:`topics-dlmw-robots`.
 
+.. setting:: ROBOTSTXT_USER_AGENT
+
+ROBOTSTXT_USER_AGENT
+^^^^^^^^^^^^^^^^^^^^
+
+Default: ``None``
+
+The user agent string to use for matching in the robots.txt file. If ``None``,
+the User-Agent header you are sending with the request or the
+:setting:`USER_AGENT` setting (in that order) will be used for determining
+the user agent to use in the robots.txt file.
+
 .. setting:: SCHEDULER
 
 SCHEDULER
@@ -1428,7 +1440,10 @@ USER_AGENT
 
 Default: ``"Scrapy/VERSION (+https://scrapy.org)"``
 
-The default User-Agent to use when crawling, unless overridden.
+The default User-Agent to use when crawling, unless overridden. This user agent is
+also used by :class:`~scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware`
+if :setting:`ROBOTSTXT_USER_AGENT` setting is ``None`` and
+there is no overridding User-Agent header specified for the request.
 
 
 Settings documented elsewhere:

scrapy/downloadermiddlewares/robotstxt.py

@@ -26,6 +26,7 @@ class RobotsTxtMiddleware(object):
         if not crawler.settings.getbool('ROBOTSTXT_OBEY'):
             raise NotConfigured
         self._default_useragent = crawler.settings.get('USER_AGENT', 'Scrapy')
+        self._robotstxt_useragent = crawler.settings.get('ROBOTSTXT_USER_AGENT', None)
         self.crawler = crawler
         self._parsers = {}
         self._parserimpl = load_object(crawler.settings.get('ROBOTSTXT_PARSER'))
@@ -47,7 +48,10 @@ class RobotsTxtMiddleware(object):
     def process_request_2(self, rp, request, spider):
         if rp is None:
             return
-        useragent = request.headers.get(b'User-Agent', self._default_useragent)
+
+        useragent = self._robotstxt_useragent
+        if not useragent:
+            useragent = request.headers.get(b'User-Agent', self._default_useragent)
         if not rp.allowed(request.url, useragent):
             logger.debug("Forbidden by robots.txt: %(request)s",
                          {'request': request}, extra={'spider': spider})

scrapy/settings/default_settings.py

@@ -247,6 +247,7 @@ RETRY_PRIORITY_ADJUST = -1
 
 ROBOTSTXT_OBEY = False
 ROBOTSTXT_PARSER = 'scrapy.robotstxt.PythonRobotParser'
+ROBOTSTXT_USER_AGENT = None
 
 SCHEDULER = 'scrapy.core.scheduler.Scheduler'
 SCHEDULER_DISK_QUEUE = 'scrapy.squeues.PickleLifoDiskQueue'

tests/test_downloadermiddleware_robotstxt.py

@@ -164,6 +164,15 @@ Disallow: /some/randome/page.html
         d.addCallback(lambda _: self.assertFalse(mw_module_logger.error.called))
         return d
 
+    def test_robotstxt_user_agent_setting(self):
+        crawler = self._get_successful_crawler()
+        crawler.settings.set('ROBOTSTXT_USER_AGENT', 'Examplebot')
+        crawler.settings.set('USER_AGENT', 'Mozilla/5.0 (X11; Linux x86_64)')
+        middleware = RobotsTxtMiddleware(crawler)
+        rp = mock.MagicMock(return_value=True)
+        middleware.process_request_2(rp, Request('http://site.local/allowed'), None)
+        rp.allowed.assert_called_once_with('http://site.local/allowed', 'Examplebot')
+
     def assertNotIgnored(self, request, middleware):
         spider = None  # not actually used
         dfd = maybeDeferred(middleware.process_request, request, spider)