diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 616b56101..6aa714fb2 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -996,6 +996,7 @@ RobotsTxtMiddleware * :ref:`RobotFileParser ` (default) * :ref:`Reppy ` * :ref:`Robotexclusionrulesparser ` + * :ref:`Protego ` You can change the robots.txt_ parser with the :setting:`ROBOTSTXT_PARSER` setting. Or you can also :ref:`implement support for a new parser `. @@ -1013,7 +1014,7 @@ RobotFileParser ~~~~~~~~~~~~~~~ `RobotFileParser `_ is -Python's inbuilt ``robots.txt`` parser. The parser is fully compliant with `Martijn Koster's +Python's inbuilt robots.txt_ parser. The parser is fully compliant with `Martijn Koster's 1996 draft specification `_. It lacks support for wildcard matching. Scrapy uses this parser by default. @@ -1059,6 +1060,23 @@ In order to use this parser: * Set :setting:`ROBOTSTXT_PARSER` setting to ``scrapy.robotstxt.ReppyRobotParser`` +.. _protego-parser: + +Protego parser +~~~~~~~~~~~~~~ + +`Protego `_ is a pure-Python robots.txt_ parser. +The parser is fully compliant with `Google's Robots.txt Specification +`_ hence supports wildcard +matching, and uses the length based rule similar to `Reppy `_. + +In order to use this parser: + +* Install `Protego `_ by running ``pip install protego`` + +* Set :setting:`ROBOTSTXT_PARSER` setting to + ``scrapy.robotstxt.ProtegoRobotParser`` + .. _support-for-new-robots-parser: Implementing support for a new parser diff --git a/scrapy/robotstxt.py b/scrapy/robotstxt.py index 4bfb275fd..189f165d1 100644 --- a/scrapy/robotstxt.py +++ b/scrapy/robotstxt.py @@ -7,6 +7,21 @@ from scrapy.utils.python import to_native_str, to_unicode logger = logging.getLogger(__name__) +def decode_robotstxt(robotstxt_body, spider, to_native_str_type=False): + try: + if to_native_str_type: + robotstxt_body = to_native_str(robotstxt_body) + else: + robotstxt_body = robotstxt_body.decode('utf-8') + except UnicodeDecodeError: + # If we found garbage or robots.txt in an encoding other than UTF-8, disregard it. + # Switch to 'allow all' state. + logger.warning("Failure while parsing robots.txt. " + "File either contains garbage or is in an encoding other than UTF-8, treating it as an empty file.", + exc_info=sys.exc_info(), + extra={'spider': spider}) + robotstxt_body = '' + return robotstxt_body class RobotParser(with_metaclass(ABCMeta)): @classmethod @@ -40,17 +55,7 @@ class PythonRobotParser(RobotParser): def __init__(self, robotstxt_body, spider): from six.moves.urllib_robotparser import RobotFileParser self.spider = spider - try: - robotstxt_body = to_native_str(robotstxt_body) - except UnicodeDecodeError: - # If we found garbage or robots.txt in an encoding other than UTF-8, disregard it. - # Switch to 'allow all' state. - logger.warning("Failure while parsing robots.txt using %(parser)s." - " File either contains garbage or is in an encoding other than UTF-8, treating it as an empty file.", - {'parser': "RobotFileParser"}, - exc_info=sys.exc_info(), - extra={'spider': self.spider}) - robotstxt_body = '' + robotstxt_body = decode_robotstxt(robotstxt_body, spider, to_native_str_type=True) self.rp = RobotFileParser() self.rp.parse(robotstxt_body.splitlines()) @@ -87,17 +92,7 @@ class RerpRobotParser(RobotParser): from robotexclusionrulesparser import RobotExclusionRulesParser self.spider = spider self.rp = RobotExclusionRulesParser() - try: - robotstxt_body = robotstxt_body.decode('utf-8') - except UnicodeDecodeError: - # If we found garbage or robots.txt in an encoding other than UTF-8, disregard it. - # Switch to 'allow all' state. - logger.warning("Failure while parsing robots.txt using %(parser)s." - " File either contains garbage or is in an encoding other than UTF-8, treating it as an empty file.", - {'parser': "RobotExclusionRulesParser"}, - exc_info=sys.exc_info(), - extra={'spider': self.spider}) - robotstxt_body = '' + robotstxt_body = decode_robotstxt(robotstxt_body, spider) self.rp.parse(robotstxt_body) @classmethod @@ -110,3 +105,22 @@ class RerpRobotParser(RobotParser): user_agent = to_unicode(user_agent) url = to_unicode(url) return self.rp.is_allowed(user_agent, url) + + +class ProtegoRobotParser(RobotParser): + def __init__(self, robotstxt_body, spider): + from protego import Protego + self.spider = spider + robotstxt_body = decode_robotstxt(robotstxt_body, spider) + self.rp = Protego.parse(robotstxt_body) + + @classmethod + def from_crawler(cls, crawler, robotstxt_body): + spider = None if not crawler else crawler.spider + o = cls(robotstxt_body, spider) + return o + + def allowed(self, url, user_agent): + user_agent = to_unicode(user_agent) + url = to_unicode(url) + return self.rp.can_fetch(url, user_agent) diff --git a/tests/test_robotstxt_interface.py b/tests/test_robotstxt_interface.py index 2819786b5..9aaab560a 100644 --- a/tests/test_robotstxt_interface.py +++ b/tests/test_robotstxt_interface.py @@ -20,6 +20,13 @@ def rerp_available(): return False return True +def protego_available(): + # check if protego parser is installed + try: + from protego import Protego + except ImportError: + return False + return True class BaseRobotParserTest: def _setUp(self, parser_cls): @@ -127,7 +134,7 @@ class ReppyRobotParserTest(BaseRobotParserTest, unittest.TestCase): super(ReppyRobotParserTest, self)._setUp(ReppyRobotParser) def test_order_based_precedence(self): - raise unittest.SkipTest("Rerp does not support order based directives precedence.") + raise unittest.SkipTest("Reppy does not support order based directives precedence.") class RerpRobotParserTest(BaseRobotParserTest, unittest.TestCase): @@ -140,3 +147,15 @@ class RerpRobotParserTest(BaseRobotParserTest, unittest.TestCase): def test_length_based_precedence(self): raise unittest.SkipTest("Rerp does not support length based directives precedence.") + + +class ProtegoRobotParserTest(BaseRobotParserTest, unittest.TestCase): + if not protego_available(): + skip = "Protego parser is not installed" + + def setUp(self): + from scrapy.robotstxt import ProtegoRobotParser + super(ProtegoRobotParserTest, self)._setUp(ProtegoRobotParser) + + def test_order_based_precedence(self): + raise unittest.SkipTest("Protego does not support order based directives precedence.") diff --git a/tox.ini b/tox.ini index c3502c2ca..cc845faf1 100644 --- a/tox.ini +++ b/tox.ini @@ -125,6 +125,7 @@ deps = {[testenv:py35]deps} reppy robotexclusionrulesparser + protego [testenv:py27-extra-deps] basepython = python2.7 @@ -132,3 +133,4 @@ deps = {[testenv]deps} reppy robotexclusionrulesparser + protego \ No newline at end of file