From 55ac26228b853aec6d543bfaf3ab85ebbf5ab002 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 14 Jun 2023 16:50:43 +0400 Subject: [PATCH] Remove builtin addons. --- scrapy/{addons/__init__.py => addons.py} | 3 - scrapy/addons/builtins.py | 146 ----------------------- sep/sep-021.rst | 25 ---- tests/test_addons/test_builtins.py | 23 ---- 4 files changed, 197 deletions(-) rename scrapy/{addons/__init__.py => addons.py} (99%) delete mode 100644 scrapy/addons/builtins.py delete mode 100644 tests/test_addons/test_builtins.py diff --git a/scrapy/addons/__init__.py b/scrapy/addons.py similarity index 99% rename from scrapy/addons/__init__.py rename to scrapy/addons.py index aad9ef789..c9b873039 100644 --- a/scrapy/addons/__init__.py +++ b/scrapy/addons.py @@ -456,6 +456,3 @@ class AddonManager(Mapping): """ for name in self: self._call_addon(name, "check_configuration", crawler) - - -from scrapy.addons.builtins import * # noqa diff --git a/scrapy/addons/builtins.py b/scrapy/addons/builtins.py deleted file mode 100644 index ea3afbf99..000000000 --- a/scrapy/addons/builtins.py +++ /dev/null @@ -1,146 +0,0 @@ -import scrapy -from scrapy.addons import Addon - -__all__ = [ - "make_builtin_addon", - "depth", - "httperror", - "offsite", - "referer", - "urllength", - "ajaxcrawl", - "chunked", - "cookies", - "defaultheaders", - "downloadtimeout", - "httpauth", - "httpcache", - "httpcompression", - "httpproxy", - "metarefresh", - "redirect", - "retry", - "robotstxt", - "stats", - "useragent", - "autothrottle", - "corestats", - "closespider", - "debugger", - "feedexport", - "logstats", - "memdebug", - "memusage", - "spiderstate", - "stacktracedump", - "statsmailer", - "telnetconsole", -] - - -def make_builtin_addon(addon_name, addon_default_config=None, addon_version=None): - class ThisAddon(Addon): - name = addon_name - version = addon_version or scrapy.__version__ - default_config = addon_default_config or {} - - return ThisAddon - - -# XXX: Below are CLASSES that have lowercase names. This is in line with the -# original SEP-021 but violates PEP8. -# We might consider prepending all built-in addon names with scrapy_ or similar -# to reduce the chance of name clashes. - -# SPIDER MIDDLEWARES - -depth = make_builtin_addon("depth") - -httperror = make_builtin_addon("httperror") - -offsite = make_builtin_addon("offsite") - -referer = make_builtin_addon("referer") - -urllength = make_builtin_addon("urllength") - - -# DOWNLOADER MIDDLEWARES - -ajaxcrawl = make_builtin_addon("ajaxcrawl", {"enabled": True}) - -chunked = make_builtin_addon("chunked") - -cookies = make_builtin_addon("cookies") - -defaultheaders = make_builtin_addon("defaultheaders") - - -# Assume every config entry is a header -def defaultheaders_export_config(self, config, settings): - conf = self.default_config or {} - conf.update(config) - settings.set("DEFAULT_REQUEST_HEADERS", conf, "addon") - - -defaultheaders.export_config = defaultheaders_export_config - -downloadtimeout = make_builtin_addon("downloadtimeout") -downloadtimeout.config_mapping = { - "timeout": "DOWNLOAD_TIMEOUT", - "download_timeout": "DOWNLOAD_TIMEOUT", -} - -httpauth = make_builtin_addon("httpauth") - -httpcache = make_builtin_addon("httpcache", {"enabled": True}) - -httpcompression = make_builtin_addon("httpcompression") -httpcompression.config_mapping = {"enabled": "COMPRESSION_ENABLED"} - -httpproxy = make_builtin_addon("httpproxy") - -metarefresh = make_builtin_addon("metarefresh") -metarefresh.config_mapping = {"max_times": "REDIRECT_MAX_TIMES"} - -redirect = make_builtin_addon("redirect") - -retry = make_builtin_addon("retry") - -robotstxt = make_builtin_addon("robotstxt", {"obey": True}) - -stats = make_builtin_addon("stats") - -useragent = make_builtin_addon("useragent") -useragent.config_mapping = {"user_agent": "USER_AGENT"} - - -# ITEM PIPELINES - - -# EXTENSIONS - -autothrottle = make_builtin_addon("autothrottle", {"enabled": True}) - -corestats = make_builtin_addon("corestats") - -closespider = make_builtin_addon("closespider") - -debugger = make_builtin_addon("debugger") - -feedexport = make_builtin_addon("feedexport") -feedexport.settings_prefix = "FEED" - -logstats = make_builtin_addon("logstats") - -memdebug = make_builtin_addon("memdebug", {"enabled": True}) - -memusage = make_builtin_addon("memusage", {"enabled": True}) - -spiderstate = make_builtin_addon("spiderstate") - -stacktracedump = make_builtin_addon("stacktracedump") - -statsmailer = make_builtin_addon("statsmailer") - -telnetconsole = make_builtin_addon("telnetconsole") diff --git a/sep/sep-021.rst b/sep/sep-021.rst index cb1701014..47cba004c 100644 --- a/sep/sep-021.rst +++ b/sep/sep-021.rst @@ -322,28 +322,3 @@ with ``scrapy.addons.`` prepended (i.e. pointing to Scrapy's ``addons`` submodule). If the object found has an ``_addon`` attribute, that attribute will be treated as the found add-on. This allows, for example, to change the add-on based on the Python version. - -Updating existing extensions ----------------------------- - -An ``Addon`` class is introduced that add-on developers may or may not subclass -depending on how much of the 'default functionality' they want. Naturally, it -does not provide ``NAME`` and ``VERSION``. Its default ``update_settings()`` -exposes the add-on configuration into the global settings namespace with an -appropriate name, e.g. this section from ``scrapy.cfg``:: - - [httpcache] - dir = /some/dir - -would expose ``HTTPCACHE_DIR``. - -Add-on modules will be written for all built-in extensions and placed in -``scrapy.addons``. For many default Scrapy components, it will be sufficient to -create a subclass of ``Addon`` with minor or no method modifications. The -component code remains where it is (i.e. in ``scrapy.pipelines``, etc.). - -Later, the global settings namespace could be cleaned up in a backwards --incompatible fashion by deprecating support for the global setting names, e.g. -``HTTPCACHE_DIR``, and instead instantiate the components with the add-on -configuration in ``update_settings()``. - diff --git a/tests/test_addons/test_builtins.py b/tests/test_addons/test_builtins.py deleted file mode 100644 index 1050cbbed..000000000 --- a/tests/test_addons/test_builtins.py +++ /dev/null @@ -1,23 +0,0 @@ -import unittest - -import scrapy -import scrapy.addons -from scrapy.addons.builtins import make_builtin_addon -from scrapy.settings import Settings - - -class BuiltinAddonsTest(unittest.TestCase): - def test_make_builtin_addon(self): - httpcache = make_builtin_addon("httpcache", {"enabled": True}) - self.assertEqual(httpcache.name, "httpcache") - self.assertEqual(httpcache.default_config, {"enabled": True}) - self.assertEqual(httpcache.version, scrapy.__version__) - httpcache = make_builtin_addon("httpcache", {"enabled": True}, "99.9") - self.assertEqual(httpcache.version, "99.9") - - def test_defaultheaders_export_config(self): - settings = Settings() - dh = scrapy.addons.defaultheaders() - dh.export_config({"X-Test-Header": "val"}, settings) - self.assertIn("X-Test-Header", settings["DEFAULT_REQUEST_HEADERS"]) - self.assertEqual(settings["DEFAULT_REQUEST_HEADERS"]["X-Test-Header"], "val")