Adds integration with Protego robots.txt parser (#3935)

docs/topics/downloader-middleware.rst

@@ -996,6 +996,7 @@ RobotsTxtMiddleware
     * :ref:`RobotFileParser <python-robotfileparser>` (default)
     * :ref:`Reppy <reppy-parser>`
     * :ref:`Robotexclusionrulesparser <rerp-parser>`
+    * :ref:`Protego <protego-parser>`
 
     You can change the robots.txt_ parser with the :setting:`ROBOTSTXT_PARSER`
     setting. Or you can also :ref:`implement support for a new parser <support-for-new-robots-parser>`.
@@ -1013,7 +1014,7 @@ RobotFileParser
 ~~~~~~~~~~~~~~~
 
 `RobotFileParser <https://docs.python.org/3.7/library/urllib.robotparser.html>`_ is 
-Python's inbuilt ``robots.txt`` parser. The parser is fully compliant with `Martijn Koster's 
+Python's inbuilt robots.txt_ parser. The parser is fully compliant with `Martijn Koster's 
 1996 draft specification <http://www.robotstxt.org/norobots-rfc.txt>`_. It lacks
 support for wildcard matching. Scrapy uses this parser by default.
 
@@ -1059,6 +1060,23 @@ In order to use this parser:
 * Set :setting:`ROBOTSTXT_PARSER` setting to
   ``scrapy.robotstxt.ReppyRobotParser``
 
+.. _protego-parser:
+
+Protego parser
+~~~~~~~~~~~~~~
+
+`Protego <https://github.com/scrapy/protego>`_ is a pure-Python robots.txt_ parser.
+The parser is fully compliant with `Google's Robots.txt Specification
+<https://developers.google.com/search/reference/robots_txt>`_ hence supports wildcard 
+matching, and uses the length based rule similar to `Reppy <https://github.com/seomoz/reppy/>`_.
+
+In order to use this parser:
+
+* Install `Protego <https://github.com/scrapy/protego>`_ by running ``pip install protego``
+
+* Set :setting:`ROBOTSTXT_PARSER` setting to
+  ``scrapy.robotstxt.ProtegoRobotParser``
+
 .. _support-for-new-robots-parser:
 
 Implementing support for a new parser

scrapy/robotstxt.py

@@ -7,6 +7,21 @@ from scrapy.utils.python import to_native_str, to_unicode
 
 logger = logging.getLogger(__name__)
 
+def decode_robotstxt(robotstxt_body, spider, to_native_str_type=False):
+    try:
+        if to_native_str_type:
+            robotstxt_body = to_native_str(robotstxt_body)
+        else:
+            robotstxt_body = robotstxt_body.decode('utf-8')
+    except UnicodeDecodeError:
+        # If we found garbage or robots.txt in an encoding other than UTF-8, disregard it.
+        # Switch to 'allow all' state.
+        logger.warning("Failure while parsing robots.txt. "
+                       "File either contains garbage or is in an encoding other than UTF-8, treating it as an empty file.",
+                       exc_info=sys.exc_info(),
+                       extra={'spider': spider})
+        robotstxt_body = ''
+    return robotstxt_body
 
 class RobotParser(with_metaclass(ABCMeta)):
     @classmethod
@@ -40,17 +55,7 @@ class PythonRobotParser(RobotParser):
     def __init__(self, robotstxt_body, spider):
         from six.moves.urllib_robotparser import RobotFileParser
         self.spider = spider
-        try:
-            robotstxt_body = to_native_str(robotstxt_body)
-        except UnicodeDecodeError:
-            # If we found garbage or robots.txt in an encoding other than UTF-8, disregard it.
-            # Switch to 'allow all' state.
-            logger.warning("Failure while parsing robots.txt using %(parser)s."
-                           " File either contains garbage or is in an encoding other than UTF-8, treating it as an empty file.",
-                           {'parser': "RobotFileParser"},
-                           exc_info=sys.exc_info(),
-                           extra={'spider': self.spider})
-            robotstxt_body = ''
+        robotstxt_body = decode_robotstxt(robotstxt_body, spider, to_native_str_type=True)
         self.rp = RobotFileParser()
         self.rp.parse(robotstxt_body.splitlines())
 
@@ -87,17 +92,7 @@ class RerpRobotParser(RobotParser):
         from robotexclusionrulesparser import RobotExclusionRulesParser
         self.spider = spider
         self.rp = RobotExclusionRulesParser()
-        try:
-            robotstxt_body = robotstxt_body.decode('utf-8')
-        except UnicodeDecodeError:
-            # If we found garbage or robots.txt in an encoding other than UTF-8, disregard it.
-            # Switch to 'allow all' state.
-            logger.warning("Failure while parsing robots.txt using %(parser)s."
-                           " File either contains garbage or is in an encoding other than UTF-8, treating it as an empty file.",
-                           {'parser': "RobotExclusionRulesParser"},
-                           exc_info=sys.exc_info(),
-                           extra={'spider': self.spider})
-            robotstxt_body = ''
+        robotstxt_body = decode_robotstxt(robotstxt_body, spider)
         self.rp.parse(robotstxt_body)
 
     @classmethod
@@ -110,3 +105,22 @@ class RerpRobotParser(RobotParser):
         user_agent = to_unicode(user_agent)
         url = to_unicode(url)
         return self.rp.is_allowed(user_agent, url)
+
+
+class ProtegoRobotParser(RobotParser):
+    def __init__(self, robotstxt_body, spider):
+        from protego import Protego
+        self.spider = spider
+        robotstxt_body = decode_robotstxt(robotstxt_body, spider)
+        self.rp = Protego.parse(robotstxt_body)
+
+    @classmethod
+    def from_crawler(cls, crawler, robotstxt_body):
+        spider = None if not crawler else crawler.spider
+        o = cls(robotstxt_body, spider)
+        return o
+
+    def allowed(self, url, user_agent):
+        user_agent = to_unicode(user_agent)
+        url = to_unicode(url)
+        return self.rp.can_fetch(url, user_agent)

tests/test_robotstxt_interface.py

@@ -20,6 +20,13 @@ def rerp_available():
         return False
     return True
 
+def protego_available():
+    # check if protego parser is installed
+    try:
+        from protego import Protego
+    except ImportError:
+        return False
+    return True
 
 class BaseRobotParserTest:
     def _setUp(self, parser_cls):
@@ -127,7 +134,7 @@ class ReppyRobotParserTest(BaseRobotParserTest, unittest.TestCase):
         super(ReppyRobotParserTest, self)._setUp(ReppyRobotParser)
 
     def test_order_based_precedence(self):
-        raise unittest.SkipTest("Rerp does not support order based directives precedence.")
+        raise unittest.SkipTest("Reppy does not support order based directives precedence.")
 
 
 class RerpRobotParserTest(BaseRobotParserTest, unittest.TestCase):
@@ -140,3 +147,15 @@ class RerpRobotParserTest(BaseRobotParserTest, unittest.TestCase):
 
     def test_length_based_precedence(self):
         raise unittest.SkipTest("Rerp does not support length based directives precedence.")
+
+
+class ProtegoRobotParserTest(BaseRobotParserTest, unittest.TestCase):
+    if not protego_available():
+        skip = "Protego parser is not installed"
+
+    def setUp(self):
+        from scrapy.robotstxt import ProtegoRobotParser
+        super(ProtegoRobotParserTest, self)._setUp(ProtegoRobotParser)
+
+    def test_order_based_precedence(self):
+        raise unittest.SkipTest("Protego does not support order based directives precedence.")

tox.ini

@@ -125,6 +125,7 @@ deps =
     {[testenv:py35]deps}
     reppy
     robotexclusionrulesparser
+    protego
 
 [testenv:py27-extra-deps]
 basepython = python2.7
@@ -132,3 +133,4 @@ deps =
     {[testenv]deps}
     reppy
     robotexclusionrulesparser
+    protego