From 5759b3f0f2b0a45588e7ae7cd455ee5e7d4f531c Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Thu, 17 Oct 2024 23:41:23 +0500
Subject: [PATCH] Drop Reppy.

---
 docs/topics/downloader-middleware.rst        | 32 --------------------
 scrapy/robotstxt.py                          | 20 ------------
 tests/test_downloadermiddleware_robotstxt.py | 13 +-------
 tests/test_robotstxt_interface.py            | 24 ---------------
 4 files changed, 1 insertion(+), 88 deletions(-)

diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst
index c31f7fe43..13064ccdd 100644
--- a/docs/topics/downloader-middleware.rst
+++ b/docs/topics/downloader-middleware.rst
@@ -1086,7 +1086,6 @@ RobotsTxtMiddleware
     * :ref:`Protego <protego-parser>` (default)
     * :ref:`RobotFileParser <python-robotfileparser>`
     * :ref:`Robotexclusionrulesparser <rerp-parser>`
-    * :ref:`Reppy <reppy-parser>` (deprecated)
 
     You can change the robots.txt_ parser with the :setting:`ROBOTSTXT_PARSER`
     setting. Or you can also :ref:`implement support for a new parser <support-for-new-robots-parser>`.
@@ -1154,37 +1153,6 @@ In order to use this parser, set:
 
 * :setting:`ROBOTSTXT_PARSER` to ``scrapy.robotstxt.PythonRobotParser``
 
-.. _reppy-parser:
-
-Reppy parser
-~~~~~~~~~~~~
-
-Based on `Reppy <https://github.com/seomoz/reppy/>`_:
-
-* is a Python wrapper around `Robots Exclusion Protocol Parser for C++
-  <https://github.com/seomoz/rep-cpp>`_
-
-* is compliant with `Martijn Koster's 1996 draft specification
-  <https://www.robotstxt.org/norobots-rfc.txt>`_
-
-* supports wildcard matching
-
-* uses the length based rule
-
-Native implementation, provides better speed than Protego.
-
-In order to use this parser:
-
-* Install `Reppy <https://github.com/seomoz/reppy/>`_ by running ``pip install reppy``
-
-    .. warning:: `Upstream issue #122
-        <https://github.com/seomoz/reppy/issues/122>`_ prevents reppy usage in Python 3.9+.
-        Because of this the Reppy parser is deprecated.
-
-* Set :setting:`ROBOTSTXT_PARSER` setting to
-  ``scrapy.robotstxt.ReppyRobotParser``
-
-
 .. _rerp-parser:
 
 Robotexclusionrulesparser
diff --git a/scrapy/robotstxt.py b/scrapy/robotstxt.py
index a0e5fc671..f0a6e7467 100644
--- a/scrapy/robotstxt.py
+++ b/scrapy/robotstxt.py
@@ -4,9 +4,7 @@ import logging
 import sys
 from abc import ABCMeta, abstractmethod
 from typing import TYPE_CHECKING
-from warnings import warn
 
-from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.utils.python import to_unicode
 
 if TYPE_CHECKING:
@@ -90,24 +88,6 @@ class PythonRobotParser(RobotParser):
         return self.rp.can_fetch(user_agent, url)
 
 
-class ReppyRobotParser(RobotParser):
-    def __init__(self, robotstxt_body: bytes, spider: Spider | None):
-        warn("ReppyRobotParser is deprecated.", ScrapyDeprecationWarning, stacklevel=2)
-        from reppy.robots import Robots
-
-        self.spider: Spider | None = spider
-        self.rp = Robots.parse("", robotstxt_body)
-
-    @classmethod
-    def from_crawler(cls, crawler: Crawler, robotstxt_body: bytes) -> Self:
-        spider = None if not crawler else crawler.spider
-        o = cls(robotstxt_body, spider)
-        return o
-
-    def allowed(self, url: str | bytes, user_agent: str | bytes) -> bool:
-        return self.rp.allowed(url, user_agent)
-
-
 class RerpRobotParser(RobotParser):
     def __init__(self, robotstxt_body: bytes, spider: Spider | None):
         from robotexclusionrulesparser import RobotExclusionRulesParser
diff --git a/tests/test_downloadermiddleware_robotstxt.py b/tests/test_downloadermiddleware_robotstxt.py
index e166cc000..12b541456 100644
--- a/tests/test_downloadermiddleware_robotstxt.py
+++ b/tests/test_downloadermiddleware_robotstxt.py
@@ -11,7 +11,7 @@ from scrapy.exceptions import IgnoreRequest, NotConfigured
 from scrapy.http import Request, Response, TextResponse
 from scrapy.http.request import NO_CALLBACK
 from scrapy.settings import Settings
-from tests.test_robotstxt_interface import reppy_available, rerp_available
+from tests.test_robotstxt_interface import rerp_available
 
 
 class RobotsTxtMiddlewareTest(unittest.TestCase):
@@ -254,14 +254,3 @@ class RobotsTxtMiddlewareWithRerpTest(RobotsTxtMiddlewareTest):
         self.crawler.settings.set(
             "ROBOTSTXT_PARSER", "scrapy.robotstxt.RerpRobotParser"
         )
-
-
-class RobotsTxtMiddlewareWithReppyTest(RobotsTxtMiddlewareTest):
-    if not reppy_available():
-        skip = "Reppy parser is not installed"
-
-    def setUp(self):
-        super().setUp()
-        self.crawler.settings.set(
-            "ROBOTSTXT_PARSER", "scrapy.robotstxt.ReppyRobotParser"
-        )
diff --git a/tests/test_robotstxt_interface.py b/tests/test_robotstxt_interface.py
index 28ad910a8..541979dcc 100644
--- a/tests/test_robotstxt_interface.py
+++ b/tests/test_robotstxt_interface.py
@@ -3,15 +3,6 @@ from twisted.trial import unittest
 from scrapy.robotstxt import decode_robotstxt
 
 
-def reppy_available():
-    # check if reppy parser is installed
-    try:
-        from reppy.robots import Robots  # noqa: F401
-    except ImportError:
-        return False
-    return True
-
-
 def rerp_available():
     # check if robotexclusionrulesparser is installed
     try:
@@ -169,21 +160,6 @@ class PythonRobotParserTest(BaseRobotParserTest, unittest.TestCase):
         raise unittest.SkipTest("RobotFileParser does not support wildcards.")
 
 
-class ReppyRobotParserTest(BaseRobotParserTest, unittest.TestCase):
-    if not reppy_available():
-        skip = "Reppy parser is not installed"
-
-    def setUp(self):
-        from scrapy.robotstxt import ReppyRobotParser
-
-        super()._setUp(ReppyRobotParser)
-
-    def test_order_based_precedence(self):
-        raise unittest.SkipTest(
-            "Reppy does not support order based directives precedence."
-        )
-
-
 class RerpRobotParserTest(BaseRobotParserTest, unittest.TestCase):
     if not rerp_available():
         skip = "Rerp parser is not installed"