Drop Reppy.

docs/topics/downloader-middleware.rst

@@ -1086,7 +1086,6 @@ RobotsTxtMiddleware
     * :ref:`Protego <protego-parser>` (default)
     * :ref:`RobotFileParser <python-robotfileparser>`
     * :ref:`Robotexclusionrulesparser <rerp-parser>`
-    * :ref:`Reppy <reppy-parser>` (deprecated)
 
     You can change the robots.txt_ parser with the :setting:`ROBOTSTXT_PARSER`
     setting. Or you can also :ref:`implement support for a new parser <support-for-new-robots-parser>`.
@@ -1154,37 +1153,6 @@ In order to use this parser, set:
 
 * :setting:`ROBOTSTXT_PARSER` to ``scrapy.robotstxt.PythonRobotParser``
 
-.. _reppy-parser:
-
-Reppy parser
-~~~~~~~~~~~~
-
-Based on `Reppy <https://github.com/seomoz/reppy/>`_:
-
-* is a Python wrapper around `Robots Exclusion Protocol Parser for C++
-  <https://github.com/seomoz/rep-cpp>`_
-
-* is compliant with `Martijn Koster's 1996 draft specification
-  <https://www.robotstxt.org/norobots-rfc.txt>`_
-
-* supports wildcard matching
-
-* uses the length based rule
-
-Native implementation, provides better speed than Protego.
-
-In order to use this parser:
-
-* Install `Reppy <https://github.com/seomoz/reppy/>`_ by running ``pip install reppy``
-
-    .. warning:: `Upstream issue #122
-        <https://github.com/seomoz/reppy/issues/122>`_ prevents reppy usage in Python 3.9+.
-        Because of this the Reppy parser is deprecated.
-
-* Set :setting:`ROBOTSTXT_PARSER` setting to
-  ``scrapy.robotstxt.ReppyRobotParser``
-
-
 .. _rerp-parser:
 
 Robotexclusionrulesparser

scrapy/robotstxt.py

@@ -4,9 +4,7 @@ import logging
 import sys
 from abc import ABCMeta, abstractmethod
 from typing import TYPE_CHECKING
-from warnings import warn
 
-from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.utils.python import to_unicode
 
 if TYPE_CHECKING:
@@ -90,24 +88,6 @@ class PythonRobotParser(RobotParser):
         return self.rp.can_fetch(user_agent, url)
 
 
-class ReppyRobotParser(RobotParser):
-    def __init__(self, robotstxt_body: bytes, spider: Spider | None):
-        warn("ReppyRobotParser is deprecated.", ScrapyDeprecationWarning, stacklevel=2)
-        from reppy.robots import Robots
-
-        self.spider: Spider | None = spider
-        self.rp = Robots.parse("", robotstxt_body)
-
-    @classmethod
-    def from_crawler(cls, crawler: Crawler, robotstxt_body: bytes) -> Self:
-        spider = None if not crawler else crawler.spider
-        o = cls(robotstxt_body, spider)
-        return o
-
-    def allowed(self, url: str | bytes, user_agent: str | bytes) -> bool:
-        return self.rp.allowed(url, user_agent)
-
-
 class RerpRobotParser(RobotParser):
     def __init__(self, robotstxt_body: bytes, spider: Spider | None):
         from robotexclusionrulesparser import RobotExclusionRulesParser

tests/test_downloadermiddleware_robotstxt.py

@@ -11,7 +11,7 @@ from scrapy.exceptions import IgnoreRequest, NotConfigured
 from scrapy.http import Request, Response, TextResponse
 from scrapy.http.request import NO_CALLBACK
 from scrapy.settings import Settings
-from tests.test_robotstxt_interface import reppy_available, rerp_available
+from tests.test_robotstxt_interface import rerp_available
 
 
 class RobotsTxtMiddlewareTest(unittest.TestCase):
@@ -254,14 +254,3 @@ class RobotsTxtMiddlewareWithRerpTest(RobotsTxtMiddlewareTest):
         self.crawler.settings.set(
             "ROBOTSTXT_PARSER", "scrapy.robotstxt.RerpRobotParser"
         )
-
-
-class RobotsTxtMiddlewareWithReppyTest(RobotsTxtMiddlewareTest):
-    if not reppy_available():
-        skip = "Reppy parser is not installed"
-
-    def setUp(self):
-        super().setUp()
-        self.crawler.settings.set(
-            "ROBOTSTXT_PARSER", "scrapy.robotstxt.ReppyRobotParser"
-        )

tests/test_robotstxt_interface.py

@@ -3,15 +3,6 @@ from twisted.trial import unittest
 from scrapy.robotstxt import decode_robotstxt
 
 
-def reppy_available():
-    # check if reppy parser is installed
-    try:
-        from reppy.robots import Robots  # noqa: F401
-    except ImportError:
-        return False
-    return True
-
-
 def rerp_available():
     # check if robotexclusionrulesparser is installed
     try:
@@ -169,21 +160,6 @@ class PythonRobotParserTest(BaseRobotParserTest, unittest.TestCase):
         raise unittest.SkipTest("RobotFileParser does not support wildcards.")
 
 
-class ReppyRobotParserTest(BaseRobotParserTest, unittest.TestCase):
-    if not reppy_available():
-        skip = "Reppy parser is not installed"
-
-    def setUp(self):
-        from scrapy.robotstxt import ReppyRobotParser
-
-        super()._setUp(ReppyRobotParser)
-
-    def test_order_based_precedence(self):
-        raise unittest.SkipTest(
-            "Reppy does not support order based directives precedence."
-        )
-
-
 class RerpRobotParserTest(BaseRobotParserTest, unittest.TestCase):
     if not rerp_available():
         skip = "Rerp parser is not installed"