From 7e07d48cc5bfb4e07e1319334884ab420a2616c0 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Thu, 17 Oct 2024 23:22:37 +0500 Subject: [PATCH 1/3] Small 3.7 and 3.8 cleanup. --- scrapy/utils/reactor.py | 6 ++---- tests/CrawlerProcess/asyncio_enabled_reactor.py | 2 +- .../asyncio_enabled_reactor_different_loop.py | 2 +- tests/CrawlerProcess/asyncio_enabled_reactor_same_loop.py | 2 +- tox.ini | 3 --- 5 files changed, 5 insertions(+), 10 deletions(-) diff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.py index 18bb583b8..f8904a9aa 100644 --- a/scrapy/utils/reactor.py +++ b/scrapy/utils/reactor.py @@ -97,10 +97,8 @@ def get_asyncio_event_loop_policy() -> AbstractEventLoopPolicy: def _get_asyncio_event_loop_policy() -> AbstractEventLoopPolicy: policy = asyncio.get_event_loop_policy() - if ( - sys.version_info >= (3, 8) - and sys.platform == "win32" - and not isinstance(policy, asyncio.WindowsSelectorEventLoopPolicy) + if sys.platform == "win32" and not isinstance( + policy, asyncio.WindowsSelectorEventLoopPolicy ): policy = asyncio.WindowsSelectorEventLoopPolicy() asyncio.set_event_loop_policy(policy) diff --git a/tests/CrawlerProcess/asyncio_enabled_reactor.py b/tests/CrawlerProcess/asyncio_enabled_reactor.py index 01d23c963..f013eed27 100644 --- a/tests/CrawlerProcess/asyncio_enabled_reactor.py +++ b/tests/CrawlerProcess/asyncio_enabled_reactor.py @@ -3,7 +3,7 @@ import sys from twisted.internet import asyncioreactor -if sys.version_info >= (3, 8) and sys.platform == "win32": +if sys.platform == "win32": asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) asyncioreactor.install(asyncio.get_event_loop()) diff --git a/tests/CrawlerProcess/asyncio_enabled_reactor_different_loop.py b/tests/CrawlerProcess/asyncio_enabled_reactor_different_loop.py index 9dc8ce46b..e9d6d8875 100644 --- a/tests/CrawlerProcess/asyncio_enabled_reactor_different_loop.py +++ b/tests/CrawlerProcess/asyncio_enabled_reactor_different_loop.py @@ -4,7 +4,7 @@ import sys from twisted.internet import asyncioreactor from twisted.python import log -if sys.version_info >= (3, 8) and sys.platform == "win32": +if sys.platform == "win32": asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) asyncioreactor.install(asyncio.get_event_loop()) diff --git a/tests/CrawlerProcess/asyncio_enabled_reactor_same_loop.py b/tests/CrawlerProcess/asyncio_enabled_reactor_same_loop.py index be9c83b95..c72a0a17c 100644 --- a/tests/CrawlerProcess/asyncio_enabled_reactor_same_loop.py +++ b/tests/CrawlerProcess/asyncio_enabled_reactor_same_loop.py @@ -4,7 +4,7 @@ import sys from twisted.internet import asyncioreactor from uvloop import Loop -if sys.version_info >= (3, 8) and sys.platform == "win32": +if sys.platform == "win32": asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) asyncio.set_event_loop(Loop()) asyncioreactor.install(asyncio.get_event_loop()) diff --git a/tox.ini b/tox.ini index 79f72a0f2..fbbce48d4 100644 --- a/tox.ini +++ b/tox.ini @@ -26,9 +26,6 @@ deps = # mitmproxy does not support PyPy mitmproxy; implementation_name != 'pypy' - # https://github.com/pallets/werkzeug/pull/2768 breaks flask, required by - # mitmproxy. - werkzeug < 3; python_version < '3.9' and implementation_name != 'pypy' passenv = S3_TEST_FILE_URI AWS_ACCESS_KEY_ID From 5759b3f0f2b0a45588e7ae7cd455ee5e7d4f531c Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Thu, 17 Oct 2024 23:41:23 +0500 Subject: [PATCH 2/3] Drop Reppy. --- docs/topics/downloader-middleware.rst | 32 -------------------- scrapy/robotstxt.py | 20 ------------ tests/test_downloadermiddleware_robotstxt.py | 13 +------- tests/test_robotstxt_interface.py | 24 --------------- 4 files changed, 1 insertion(+), 88 deletions(-) diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index c31f7fe43..13064ccdd 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -1086,7 +1086,6 @@ RobotsTxtMiddleware * :ref:`Protego ` (default) * :ref:`RobotFileParser ` * :ref:`Robotexclusionrulesparser ` - * :ref:`Reppy ` (deprecated) You can change the robots.txt_ parser with the :setting:`ROBOTSTXT_PARSER` setting. Or you can also :ref:`implement support for a new parser `. @@ -1154,37 +1153,6 @@ In order to use this parser, set: * :setting:`ROBOTSTXT_PARSER` to ``scrapy.robotstxt.PythonRobotParser`` -.. _reppy-parser: - -Reppy parser -~~~~~~~~~~~~ - -Based on `Reppy `_: - -* is a Python wrapper around `Robots Exclusion Protocol Parser for C++ - `_ - -* is compliant with `Martijn Koster's 1996 draft specification - `_ - -* supports wildcard matching - -* uses the length based rule - -Native implementation, provides better speed than Protego. - -In order to use this parser: - -* Install `Reppy `_ by running ``pip install reppy`` - - .. warning:: `Upstream issue #122 - `_ prevents reppy usage in Python 3.9+. - Because of this the Reppy parser is deprecated. - -* Set :setting:`ROBOTSTXT_PARSER` setting to - ``scrapy.robotstxt.ReppyRobotParser`` - - .. _rerp-parser: Robotexclusionrulesparser diff --git a/scrapy/robotstxt.py b/scrapy/robotstxt.py index a0e5fc671..f0a6e7467 100644 --- a/scrapy/robotstxt.py +++ b/scrapy/robotstxt.py @@ -4,9 +4,7 @@ import logging import sys from abc import ABCMeta, abstractmethod from typing import TYPE_CHECKING -from warnings import warn -from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.utils.python import to_unicode if TYPE_CHECKING: @@ -90,24 +88,6 @@ class PythonRobotParser(RobotParser): return self.rp.can_fetch(user_agent, url) -class ReppyRobotParser(RobotParser): - def __init__(self, robotstxt_body: bytes, spider: Spider | None): - warn("ReppyRobotParser is deprecated.", ScrapyDeprecationWarning, stacklevel=2) - from reppy.robots import Robots - - self.spider: Spider | None = spider - self.rp = Robots.parse("", robotstxt_body) - - @classmethod - def from_crawler(cls, crawler: Crawler, robotstxt_body: bytes) -> Self: - spider = None if not crawler else crawler.spider - o = cls(robotstxt_body, spider) - return o - - def allowed(self, url: str | bytes, user_agent: str | bytes) -> bool: - return self.rp.allowed(url, user_agent) - - class RerpRobotParser(RobotParser): def __init__(self, robotstxt_body: bytes, spider: Spider | None): from robotexclusionrulesparser import RobotExclusionRulesParser diff --git a/tests/test_downloadermiddleware_robotstxt.py b/tests/test_downloadermiddleware_robotstxt.py index e166cc000..12b541456 100644 --- a/tests/test_downloadermiddleware_robotstxt.py +++ b/tests/test_downloadermiddleware_robotstxt.py @@ -11,7 +11,7 @@ from scrapy.exceptions import IgnoreRequest, NotConfigured from scrapy.http import Request, Response, TextResponse from scrapy.http.request import NO_CALLBACK from scrapy.settings import Settings -from tests.test_robotstxt_interface import reppy_available, rerp_available +from tests.test_robotstxt_interface import rerp_available class RobotsTxtMiddlewareTest(unittest.TestCase): @@ -254,14 +254,3 @@ class RobotsTxtMiddlewareWithRerpTest(RobotsTxtMiddlewareTest): self.crawler.settings.set( "ROBOTSTXT_PARSER", "scrapy.robotstxt.RerpRobotParser" ) - - -class RobotsTxtMiddlewareWithReppyTest(RobotsTxtMiddlewareTest): - if not reppy_available(): - skip = "Reppy parser is not installed" - - def setUp(self): - super().setUp() - self.crawler.settings.set( - "ROBOTSTXT_PARSER", "scrapy.robotstxt.ReppyRobotParser" - ) diff --git a/tests/test_robotstxt_interface.py b/tests/test_robotstxt_interface.py index 28ad910a8..541979dcc 100644 --- a/tests/test_robotstxt_interface.py +++ b/tests/test_robotstxt_interface.py @@ -3,15 +3,6 @@ from twisted.trial import unittest from scrapy.robotstxt import decode_robotstxt -def reppy_available(): - # check if reppy parser is installed - try: - from reppy.robots import Robots # noqa: F401 - except ImportError: - return False - return True - - def rerp_available(): # check if robotexclusionrulesparser is installed try: @@ -169,21 +160,6 @@ class PythonRobotParserTest(BaseRobotParserTest, unittest.TestCase): raise unittest.SkipTest("RobotFileParser does not support wildcards.") -class ReppyRobotParserTest(BaseRobotParserTest, unittest.TestCase): - if not reppy_available(): - skip = "Reppy parser is not installed" - - def setUp(self): - from scrapy.robotstxt import ReppyRobotParser - - super()._setUp(ReppyRobotParser) - - def test_order_based_precedence(self): - raise unittest.SkipTest( - "Reppy does not support order based directives precedence." - ) - - class RerpRobotParserTest(BaseRobotParserTest, unittest.TestCase): if not rerp_available(): skip = "Rerp parser is not installed" From 677e9772070ec8a92033f66dff45d7c421763203 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 18 Oct 2024 00:03:32 +0500 Subject: [PATCH 3/3] Remove dead links to the Reppy doc from the release notes. --- docs/news.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/news.rst b/docs/news.rst index 58b51c9ea..2bbca77cc 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -1530,7 +1530,7 @@ Documentation - Provided better context and instructions to disable the :setting:`URLLENGTH_LIMIT` setting. (:issue:`5135`, :issue:`5250`) -- Documented that :ref:`reppy-parser` does not support Python 3.9+. +- Documented that Reppy parser does not support Python 3.9+. (:issue:`5226`, :issue:`5231`) - Documented :ref:`the scheduler component `. @@ -3344,7 +3344,7 @@ New features * A new :setting:`ROBOTSTXT_PARSER` setting allows choosing which robots.txt_ parser to use. It includes built-in support for :ref:`RobotFileParser `, - :ref:`Protego ` (default), :ref:`Reppy `, and + :ref:`Protego ` (default), Reppy, and :ref:`Robotexclusionrulesparser `, and allows you to :ref:`implement support for additional parsers ` (:issue:`754`, :issue:`2669`,