Merge pull request #6499 from wRAR/3.8-cleanup

Additional Python 3.7 and 3.8 cleanup, including Reppy support.

docs/news.rst

@@ -1530,7 +1530,7 @@ Documentation
 -   Provided better context and instructions to disable the
     :setting:`URLLENGTH_LIMIT` setting. (:issue:`5135`, :issue:`5250`)
 
--   Documented that :ref:`reppy-parser` does not support Python 3.9+.
+-   Documented that Reppy parser does not support Python 3.9+.
     (:issue:`5226`, :issue:`5231`)
 
 -   Documented :ref:`the scheduler component <topics-scheduler>`.
@@ -3344,7 +3344,7 @@ New features
 *   A new :setting:`ROBOTSTXT_PARSER` setting allows choosing which robots.txt_
     parser to use. It includes built-in support for
     :ref:`RobotFileParser <python-robotfileparser>`,
-    :ref:`Protego <protego-parser>` (default), :ref:`Reppy <reppy-parser>`, and
+    :ref:`Protego <protego-parser>` (default), Reppy, and
     :ref:`Robotexclusionrulesparser <rerp-parser>`, and allows you to
     :ref:`implement support for additional parsers
     <support-for-new-robots-parser>` (:issue:`754`, :issue:`2669`,

docs/topics/downloader-middleware.rst

@@ -1086,7 +1086,6 @@ RobotsTxtMiddleware
     * :ref:`Protego <protego-parser>` (default)
     * :ref:`RobotFileParser <python-robotfileparser>`
     * :ref:`Robotexclusionrulesparser <rerp-parser>`
-    * :ref:`Reppy <reppy-parser>` (deprecated)
 
     You can change the robots.txt_ parser with the :setting:`ROBOTSTXT_PARSER`
     setting. Or you can also :ref:`implement support for a new parser <support-for-new-robots-parser>`.
@@ -1154,37 +1153,6 @@ In order to use this parser, set:
 
 * :setting:`ROBOTSTXT_PARSER` to ``scrapy.robotstxt.PythonRobotParser``
 
-.. _reppy-parser:
-
-Reppy parser
-~~~~~~~~~~~~
-
-Based on `Reppy <https://github.com/seomoz/reppy/>`_:
-
-* is a Python wrapper around `Robots Exclusion Protocol Parser for C++
-  <https://github.com/seomoz/rep-cpp>`_
-
-* is compliant with `Martijn Koster's 1996 draft specification
-  <https://www.robotstxt.org/norobots-rfc.txt>`_
-
-* supports wildcard matching
-
-* uses the length based rule
-
-Native implementation, provides better speed than Protego.
-
-In order to use this parser:
-
-* Install `Reppy <https://github.com/seomoz/reppy/>`_ by running ``pip install reppy``
-
-    .. warning:: `Upstream issue #122
-        <https://github.com/seomoz/reppy/issues/122>`_ prevents reppy usage in Python 3.9+.
-        Because of this the Reppy parser is deprecated.
-
-* Set :setting:`ROBOTSTXT_PARSER` setting to
-  ``scrapy.robotstxt.ReppyRobotParser``
-
-
 .. _rerp-parser:
 
 Robotexclusionrulesparser

scrapy/robotstxt.py

@@ -4,9 +4,7 @@ import logging
 import sys
 from abc import ABCMeta, abstractmethod
 from typing import TYPE_CHECKING
-from warnings import warn
 
-from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.utils.python import to_unicode
 
 if TYPE_CHECKING:
@@ -90,24 +88,6 @@ class PythonRobotParser(RobotParser):
         return self.rp.can_fetch(user_agent, url)
 
 
-class ReppyRobotParser(RobotParser):
-    def __init__(self, robotstxt_body: bytes, spider: Spider | None):
-        warn("ReppyRobotParser is deprecated.", ScrapyDeprecationWarning, stacklevel=2)
-        from reppy.robots import Robots
-
-        self.spider: Spider | None = spider
-        self.rp = Robots.parse("", robotstxt_body)
-
-    @classmethod
-    def from_crawler(cls, crawler: Crawler, robotstxt_body: bytes) -> Self:
-        spider = None if not crawler else crawler.spider
-        o = cls(robotstxt_body, spider)
-        return o
-
-    def allowed(self, url: str | bytes, user_agent: str | bytes) -> bool:
-        return self.rp.allowed(url, user_agent)
-
-
 class RerpRobotParser(RobotParser):
     def __init__(self, robotstxt_body: bytes, spider: Spider | None):
         from robotexclusionrulesparser import RobotExclusionRulesParser

scrapy/utils/reactor.py

@@ -97,10 +97,8 @@ def get_asyncio_event_loop_policy() -> AbstractEventLoopPolicy:
 
 def _get_asyncio_event_loop_policy() -> AbstractEventLoopPolicy:
     policy = asyncio.get_event_loop_policy()
-    if (
-        sys.version_info >= (3, 8)
-        and sys.platform == "win32"
-        and not isinstance(policy, asyncio.WindowsSelectorEventLoopPolicy)
+    if sys.platform == "win32" and not isinstance(
+        policy, asyncio.WindowsSelectorEventLoopPolicy
     ):
         policy = asyncio.WindowsSelectorEventLoopPolicy()
         asyncio.set_event_loop_policy(policy)

tests/CrawlerProcess/asyncio_enabled_reactor.py

@@ -3,7 +3,7 @@ import sys
 
 from twisted.internet import asyncioreactor
 
-if sys.version_info >= (3, 8) and sys.platform == "win32":
+if sys.platform == "win32":
     asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy())
 asyncioreactor.install(asyncio.get_event_loop())
 

tests/CrawlerProcess/asyncio_enabled_reactor_different_loop.py

@@ -4,7 +4,7 @@ import sys
 from twisted.internet import asyncioreactor
 from twisted.python import log
 
-if sys.version_info >= (3, 8) and sys.platform == "win32":
+if sys.platform == "win32":
     asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy())
 asyncioreactor.install(asyncio.get_event_loop())
 

tests/CrawlerProcess/asyncio_enabled_reactor_same_loop.py

@@ -4,7 +4,7 @@ import sys
 from twisted.internet import asyncioreactor
 from uvloop import Loop
 
-if sys.version_info >= (3, 8) and sys.platform == "win32":
+if sys.platform == "win32":
     asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy())
 asyncio.set_event_loop(Loop())
 asyncioreactor.install(asyncio.get_event_loop())

tests/test_downloadermiddleware_robotstxt.py

@@ -11,7 +11,7 @@ from scrapy.exceptions import IgnoreRequest, NotConfigured
 from scrapy.http import Request, Response, TextResponse
 from scrapy.http.request import NO_CALLBACK
 from scrapy.settings import Settings
-from tests.test_robotstxt_interface import reppy_available, rerp_available
+from tests.test_robotstxt_interface import rerp_available
 
 
 class RobotsTxtMiddlewareTest(unittest.TestCase):
@@ -254,14 +254,3 @@ class RobotsTxtMiddlewareWithRerpTest(RobotsTxtMiddlewareTest):
         self.crawler.settings.set(
             "ROBOTSTXT_PARSER", "scrapy.robotstxt.RerpRobotParser"
         )
-
-
-class RobotsTxtMiddlewareWithReppyTest(RobotsTxtMiddlewareTest):
-    if not reppy_available():
-        skip = "Reppy parser is not installed"
-
-    def setUp(self):
-        super().setUp()
-        self.crawler.settings.set(
-            "ROBOTSTXT_PARSER", "scrapy.robotstxt.ReppyRobotParser"
-        )

tests/test_robotstxt_interface.py

@@ -3,15 +3,6 @@ from twisted.trial import unittest
 from scrapy.robotstxt import decode_robotstxt
 
 
-def reppy_available():
-    # check if reppy parser is installed
-    try:
-        from reppy.robots import Robots  # noqa: F401
-    except ImportError:
-        return False
-    return True
-
-
 def rerp_available():
     # check if robotexclusionrulesparser is installed
     try:
@@ -169,21 +160,6 @@ class PythonRobotParserTest(BaseRobotParserTest, unittest.TestCase):
         raise unittest.SkipTest("RobotFileParser does not support wildcards.")
 
 
-class ReppyRobotParserTest(BaseRobotParserTest, unittest.TestCase):
-    if not reppy_available():
-        skip = "Reppy parser is not installed"
-
-    def setUp(self):
-        from scrapy.robotstxt import ReppyRobotParser
-
-        super()._setUp(ReppyRobotParser)
-
-    def test_order_based_precedence(self):
-        raise unittest.SkipTest(
-            "Reppy does not support order based directives precedence."
-        )
-
-
 class RerpRobotParserTest(BaseRobotParserTest, unittest.TestCase):
     if not rerp_available():
         skip = "Rerp parser is not installed"

tox.ini

@@ -26,9 +26,6 @@ deps =
 
     # mitmproxy does not support PyPy
     mitmproxy; implementation_name != 'pypy'
-    # https://github.com/pallets/werkzeug/pull/2768 breaks flask, required by
-    # mitmproxy.
-    werkzeug < 3; python_version < '3.9' and implementation_name != 'pypy'
 passenv =
     S3_TEST_FILE_URI
     AWS_ACCESS_KEY_ID