mirror of https://github.com/scrapy/scrapy.git
Remove builtin addons.
This commit is contained in:
Andrey Rakhmatullin
parent
075ad6f196
commit
55ac26228b
|
|
@ -456,6 +456,3 @@ class AddonManager(Mapping):
|
|||
"""
|
||||
for name in self:
|
||||
self._call_addon(name, "check_configuration", crawler)
|
||||
|
||||
|
||||
from scrapy.addons.builtins import * # noqa
|
||||
|
|
@ -1,146 +0,0 @@
|
|||
import scrapy
|
||||
from scrapy.addons import Addon
|
||||
|
||||
__all__ = [
|
||||
"make_builtin_addon",
|
||||
"depth",
|
||||
"httperror",
|
||||
"offsite",
|
||||
"referer",
|
||||
"urllength",
|
||||
"ajaxcrawl",
|
||||
"chunked",
|
||||
"cookies",
|
||||
"defaultheaders",
|
||||
"downloadtimeout",
|
||||
"httpauth",
|
||||
"httpcache",
|
||||
"httpcompression",
|
||||
"httpproxy",
|
||||
"metarefresh",
|
||||
"redirect",
|
||||
"retry",
|
||||
"robotstxt",
|
||||
"stats",
|
||||
"useragent",
|
||||
"autothrottle",
|
||||
"corestats",
|
||||
"closespider",
|
||||
"debugger",
|
||||
"feedexport",
|
||||
"logstats",
|
||||
"memdebug",
|
||||
"memusage",
|
||||
"spiderstate",
|
||||
"stacktracedump",
|
||||
"statsmailer",
|
||||
"telnetconsole",
|
||||
]
|
||||
|
||||
|
||||
def make_builtin_addon(addon_name, addon_default_config=None, addon_version=None):
|
||||
class ThisAddon(Addon):
|
||||
name = addon_name
|
||||
version = addon_version or scrapy.__version__
|
||||
default_config = addon_default_config or {}
|
||||
|
||||
return ThisAddon
|
||||
|
||||
|
||||
# XXX: Below are CLASSES that have lowercase names. This is in line with the
|
||||
# original SEP-021 but violates PEP8.
|
||||
# We might consider prepending all built-in addon names with scrapy_ or similar
|
||||
# to reduce the chance of name clashes.
|
||||
|
||||
# SPIDER MIDDLEWARES
|
||||
|
||||
depth = make_builtin_addon("depth")
|
||||
|
||||
httperror = make_builtin_addon("httperror")
|
||||
|
||||
offsite = make_builtin_addon("offsite")
|
||||
|
||||
referer = make_builtin_addon("referer")
|
||||
|
||||
urllength = make_builtin_addon("urllength")
|
||||
|
||||
|
||||
# DOWNLOADER MIDDLEWARES
|
||||
|
||||
ajaxcrawl = make_builtin_addon("ajaxcrawl", {"enabled": True})
|
||||
|
||||
chunked = make_builtin_addon("chunked")
|
||||
|
||||
cookies = make_builtin_addon("cookies")
|
||||
|
||||
defaultheaders = make_builtin_addon("defaultheaders")
|
||||
|
||||
|
||||
# Assume every config entry is a header
|
||||
def defaultheaders_export_config(self, config, settings):
|
||||
conf = self.default_config or {}
|
||||
conf.update(config)
|
||||
settings.set("DEFAULT_REQUEST_HEADERS", conf, "addon")
|
||||
|
||||
|
||||
defaultheaders.export_config = defaultheaders_export_config
|
||||
|
||||
downloadtimeout = make_builtin_addon("downloadtimeout")
|
||||
downloadtimeout.config_mapping = {
|
||||
"timeout": "DOWNLOAD_TIMEOUT",
|
||||
"download_timeout": "DOWNLOAD_TIMEOUT",
|
||||
}
|
||||
|
||||
httpauth = make_builtin_addon("httpauth")
|
||||
|
||||
httpcache = make_builtin_addon("httpcache", {"enabled": True})
|
||||
|
||||
httpcompression = make_builtin_addon("httpcompression")
|
||||
httpcompression.config_mapping = {"enabled": "COMPRESSION_ENABLED"}
|
||||
|
||||
httpproxy = make_builtin_addon("httpproxy")
|
||||
|
||||
metarefresh = make_builtin_addon("metarefresh")
|
||||
metarefresh.config_mapping = {"max_times": "REDIRECT_MAX_TIMES"}
|
||||
|
||||
redirect = make_builtin_addon("redirect")
|
||||
|
||||
retry = make_builtin_addon("retry")
|
||||
|
||||
robotstxt = make_builtin_addon("robotstxt", {"obey": True})
|
||||
|
||||
stats = make_builtin_addon("stats")
|
||||
|
||||
useragent = make_builtin_addon("useragent")
|
||||
useragent.config_mapping = {"user_agent": "USER_AGENT"}
|
||||
|
||||
|
||||
# ITEM PIPELINES
|
||||
|
||||
|
||||
# EXTENSIONS
|
||||
|
||||
autothrottle = make_builtin_addon("autothrottle", {"enabled": True})
|
||||
|
||||
corestats = make_builtin_addon("corestats")
|
||||
|
||||
closespider = make_builtin_addon("closespider")
|
||||
|
||||
debugger = make_builtin_addon("debugger")
|
||||
|
||||
feedexport = make_builtin_addon("feedexport")
|
||||
feedexport.settings_prefix = "FEED"
|
||||
|
||||
logstats = make_builtin_addon("logstats")
|
||||
|
||||
memdebug = make_builtin_addon("memdebug", {"enabled": True})
|
||||
|
||||
memusage = make_builtin_addon("memusage", {"enabled": True})
|
||||
|
||||
spiderstate = make_builtin_addon("spiderstate")
|
||||
|
||||
stacktracedump = make_builtin_addon("stacktracedump")
|
||||
|
||||
statsmailer = make_builtin_addon("statsmailer")
|
||||
|
||||
telnetconsole = make_builtin_addon("telnetconsole")
|
||||
|
|
@ -322,28 +322,3 @@ with ``scrapy.addons.`` prepended (i.e. pointing to Scrapy's ``addons``
|
|||
submodule). If the object found has an ``_addon`` attribute, that attribute
|
||||
will be treated as the found add-on. This allows, for example, to change the
|
||||
add-on based on the Python version.
|
||||
|
||||
Updating existing extensions
|
||||
----------------------------
|
||||
|
||||
An ``Addon`` class is introduced that add-on developers may or may not subclass
|
||||
depending on how much of the 'default functionality' they want. Naturally, it
|
||||
does not provide ``NAME`` and ``VERSION``. Its default ``update_settings()``
|
||||
exposes the add-on configuration into the global settings namespace with an
|
||||
appropriate name, e.g. this section from ``scrapy.cfg``::
|
||||
|
||||
[httpcache]
|
||||
dir = /some/dir
|
||||
|
||||
would expose ``HTTPCACHE_DIR``.
|
||||
|
||||
Add-on modules will be written for all built-in extensions and placed in
|
||||
``scrapy.addons``. For many default Scrapy components, it will be sufficient to
|
||||
create a subclass of ``Addon`` with minor or no method modifications. The
|
||||
component code remains where it is (i.e. in ``scrapy.pipelines``, etc.).
|
||||
|
||||
Later, the global settings namespace could be cleaned up in a backwards
|
||||
-incompatible fashion by deprecating support for the global setting names, e.g.
|
||||
``HTTPCACHE_DIR``, and instead instantiate the components with the add-on
|
||||
configuration in ``update_settings()``.
|
||||
|
||||
|
|
|
|||
|
|
@ -1,23 +0,0 @@
|
|||
import unittest
|
||||
|
||||
import scrapy
|
||||
import scrapy.addons
|
||||
from scrapy.addons.builtins import make_builtin_addon
|
||||
from scrapy.settings import Settings
|
||||
|
||||
|
||||
class BuiltinAddonsTest(unittest.TestCase):
|
||||
def test_make_builtin_addon(self):
|
||||
httpcache = make_builtin_addon("httpcache", {"enabled": True})
|
||||
self.assertEqual(httpcache.name, "httpcache")
|
||||
self.assertEqual(httpcache.default_config, {"enabled": True})
|
||||
self.assertEqual(httpcache.version, scrapy.__version__)
|
||||
httpcache = make_builtin_addon("httpcache", {"enabled": True}, "99.9")
|
||||
self.assertEqual(httpcache.version, "99.9")
|
||||
|
||||
def test_defaultheaders_export_config(self):
|
||||
settings = Settings()
|
||||
dh = scrapy.addons.defaultheaders()
|
||||
dh.export_config({"X-Test-Header": "val"}, settings)
|
||||
self.assertIn("X-Test-Header", settings["DEFAULT_REQUEST_HEADERS"])
|
||||
self.assertEqual(settings["DEFAULT_REQUEST_HEADERS"]["X-Test-Header"], "val")
|
||||
Loading…
Reference in New Issue