mirror of https://github.com/scrapy/scrapy.git
Remove more code.
This commit is contained in:
Andrey Rakhmatullin
parent
760c0db094
commit
9612ae3e93
|
|
@ -17,40 +17,26 @@ Activating and configuring add-ons
|
|||
Add-ons and their configuration live in Scrapy's
|
||||
:class:`~scrapy.addons.AddonManager`. During a :class:`~scrapy.crawler.Crawler`
|
||||
initialization the add-on manager will read a list of enabled add-ons from your
|
||||
``ADDONS`` setting and their optional configuration from the respective
|
||||
settings.
|
||||
``ADDONS`` setting.
|
||||
|
||||
The ``ADDONS`` setting is a dict in which every key is an addon class or its
|
||||
import path and the value is its priority.
|
||||
|
||||
The configuration of an add-on, if necessary at all, is stored as a dictionary
|
||||
setting whose name is the uppercase add-on name.
|
||||
|
||||
This is an example where two add-ons (in this case with one requiring no
|
||||
configuration) are enabled/configured in a project's ``settings.py``::
|
||||
This is an example where two add-ons are enabled in a project's
|
||||
``settings.py``::
|
||||
|
||||
ADDONS = {
|
||||
'path.to.someaddon': 0,
|
||||
path.to.someaddon2: 1,
|
||||
}
|
||||
|
||||
SOMEADDON = {
|
||||
'some_config': True,
|
||||
}
|
||||
|
||||
|
||||
Writing your own add-ons
|
||||
========================
|
||||
|
||||
Add-ons are (any) Python *objects* that provide Scrapy's *add-on interface*:
|
||||
Add-ons are (any) Python *objects* that include the following method:
|
||||
|
||||
.. attribute:: name
|
||||
|
||||
string with add-on name
|
||||
|
||||
:type: ``str``
|
||||
|
||||
.. method:: update_settings(config, settings)
|
||||
.. method:: update_settings(settings)
|
||||
|
||||
This method is called during the initialization of the
|
||||
:class:`~scrapy.crawler.Crawler`. Here, you should perform dependency checks
|
||||
|
|
@ -58,88 +44,26 @@ Add-ons are (any) Python *objects* that provide Scrapy's *add-on interface*:
|
|||
:class:`~scrapy.settings.Settings` object as wished, e.g. enable components
|
||||
for this add-on or set required configuration of other extensions.
|
||||
|
||||
:param config: Configuration of this add-on
|
||||
:type config: ``dict``
|
||||
|
||||
:param settings: The settings object storing Scrapy/component configuration
|
||||
:type settings: :class:`~scrapy.settings.Settings`
|
||||
|
||||
.. method:: check_configuration(config, crawler)
|
||||
|
||||
This method is called when the :class:`~scrapy.crawler.Crawler` has been
|
||||
fully initialized, immediately before it starts crawling. You can perform
|
||||
additional dependency and configuration checks here.
|
||||
|
||||
:param config: Configuration of this add-on
|
||||
:type config: ``dict``
|
||||
|
||||
:param crawler: Fully initialized Scrapy crawler
|
||||
:type crawler: :class:`~scrapy.crawler.Crawler`
|
||||
|
||||
|
||||
Add-on base class
|
||||
=================
|
||||
|
||||
Scrapy comes with a built-in base class for add-ons which provides some
|
||||
convenience functionality: the add-on configuration can be exposed into
|
||||
Scrapy's settings via :meth:`~scrapy.addons.Addon.export_config`, configurable
|
||||
via :attr:`~scrapy.addons.Addon.default_config` and
|
||||
:attr:`~scrapy.addons.Addon.config_mapping`.
|
||||
|
||||
By default, the base add-on class will expose the add-on configuration into
|
||||
Scrapy's settings namespace, in upper case. It is
|
||||
easy to write your own functionality while still being able to use the
|
||||
convenience functions by overwriting
|
||||
:meth:`~scrapy.addons.Addon.update_settings`.
|
||||
|
||||
.. module:: scrapy.addons
|
||||
:noindex:
|
||||
|
||||
.. autoclass:: Addon
|
||||
:members:
|
||||
|
||||
|
||||
Add-on examples
|
||||
===============
|
||||
|
||||
Set some basic configuration using the :class:`Addon` base class::
|
||||
Set some basic configuration::
|
||||
|
||||
from scrapy.addons import Addon
|
||||
|
||||
class MyAddon(Addon):
|
||||
name = 'myaddon'
|
||||
|
||||
def update_settings(self, config, settings):
|
||||
super().update_settings(settings)
|
||||
class MyAddon:
|
||||
def update_settings(self, settings):
|
||||
settings["ITEM_PIPELINES"]["path.to.mypipeline"] = 200
|
||||
settings["DNSCACHE_ENABLED"] = True
|
||||
|
||||
Check dependencies::
|
||||
|
||||
from scrapy.addons import Addon
|
||||
|
||||
class MyAddon(Addon):
|
||||
name = 'myaddon'
|
||||
|
||||
def update_settings(self, config, settings):
|
||||
class MyAddon:
|
||||
def update_settings(self, settings):
|
||||
try:
|
||||
import boto
|
||||
except ImportError:
|
||||
raise RuntimeError("myaddon requires the boto library")
|
||||
super().update_settings(settings)
|
||||
|
||||
Check configuration of fully initialized crawler (see
|
||||
:ref:`topics-api-crawler`)::
|
||||
|
||||
class MyAddon(object):
|
||||
name = 'myaddon'
|
||||
|
||||
def update_settings(self, config, settings):
|
||||
super().update_settings(settings)
|
||||
settings.set('DNSCACHE_ENABLED', False, priority='addon')
|
||||
|
||||
def check_configuration(self, config, crawler):
|
||||
if crawler.settings.getbool('DNSCACHE_ENABLED'):
|
||||
# The spider, some other add-on, or the user messed with the
|
||||
# DNS cache setting
|
||||
raise ValueError("myaddon is incompatible with DNS cache")
|
||||
raise RuntimeError("MyAddon requires the boto library")
|
||||
...
|
||||
|
|
|
|||
124
scrapy/addons.py
124
scrapy/addons.py
|
|
@ -1,100 +1,16 @@
|
|||
from typing import Any, Dict, Iterator, Mapping, Optional, OrderedDict
|
||||
from typing import Any, List
|
||||
|
||||
from scrapy.utils.conf import build_component_list
|
||||
from scrapy.utils.misc import load_object
|
||||
|
||||
|
||||
class Addon(object):
|
||||
name: str
|
||||
|
||||
default_config = None
|
||||
"""``dict`` with default configuration."""
|
||||
|
||||
config_mapping = None
|
||||
"""``dict`` with mappings from config names to setting names. The given
|
||||
setting names will be taken as given, not uppercased.
|
||||
"""
|
||||
|
||||
def export_config(self, config, settings):
|
||||
"""Export the add-on configuration, all keys in caps, into the settings
|
||||
object.
|
||||
|
||||
For example, the add-on configuration ``{'key': 'value'}`` will export
|
||||
the setting ``KEY`` with a value of ``value``. All settings
|
||||
will be exported with ``addon`` priority (see
|
||||
:ref:`topics-api-settings`).
|
||||
|
||||
:param config: Add-on configuration to be exposed
|
||||
:type config: ``dict``
|
||||
|
||||
:param settings: Settings object into which to export the configuration
|
||||
:type settings: :class:`~scrapy.settings.Settings`
|
||||
"""
|
||||
conf = self.default_config or {}
|
||||
conf.update(config)
|
||||
# Since default exported config is case-insensitive (everything will be
|
||||
# uppercased), make mapped config case-insensitive as well
|
||||
conf_mapping = {k.lower(): v for k, v in (self.config_mapping or {}).items()}
|
||||
for key, val in conf.items():
|
||||
if key.lower() in conf_mapping:
|
||||
key = conf_mapping[key.lower()]
|
||||
else:
|
||||
key = key.upper()
|
||||
settings.set(key, val, "addon")
|
||||
|
||||
def update_settings(self, config, settings):
|
||||
"""Modifiy `settings` to enable and configure required components.
|
||||
|
||||
:param config: Add-on configuration
|
||||
:type config: ``dict``
|
||||
|
||||
:param settings: Crawler settings object
|
||||
:type settings: :class:`~scrapy.settings.Settings`
|
||||
"""
|
||||
self.export_config(config, settings)
|
||||
|
||||
def check_configuration(self, config, crawler):
|
||||
"""Perform post-initialization checks on fully configured `crawler`.
|
||||
|
||||
:param config: Add-on configuration
|
||||
:type config: ``dict``
|
||||
|
||||
:param crawler: the fully-initialized crawler
|
||||
:type crawler: :class:`~scrapy.crawler.Crawler`
|
||||
"""
|
||||
pass
|
||||
|
||||
|
||||
class AddonManager(Mapping[str, Addon]):
|
||||
"""This class facilitates loading and storing :ref:`topics-addons`.
|
||||
|
||||
You can treat it like a read-only dictionary in which keys correspond to
|
||||
add-on names and values correspond to the add-on objects. Add-on
|
||||
configurations are saved in the :attr:`config` dictionary attribute::
|
||||
|
||||
addons = AddonManager()
|
||||
# ... load some add-ons here
|
||||
print(addons.enabled) # prints names of all enabled add-ons
|
||||
print(addons['TestAddon'].version) # prints version of add-on with name
|
||||
# 'TestAddon'
|
||||
print(addons.configs['TestAddon']) # prints configuration of 'TestAddon'
|
||||
|
||||
"""
|
||||
class AddonManager:
|
||||
"""This class facilitates loading and storing :ref:`topics-addons`."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._addons: OrderedDict[str, Addon] = OrderedDict[str, Addon]()
|
||||
self.configs: Dict[str, Dict[str, Any]] = {}
|
||||
self.addons: List[Any] = []
|
||||
|
||||
def __getitem__(self, name: str) -> Addon:
|
||||
return self._addons[name]
|
||||
|
||||
def __iter__(self) -> Iterator[str]:
|
||||
return iter(self._addons)
|
||||
|
||||
def __len__(self) -> int:
|
||||
return len(self._addons)
|
||||
|
||||
def add(self, addon: Any, config: Optional[Dict[str, Any]] = None):
|
||||
def add(self, addon: Any) -> None:
|
||||
"""Store an add-on.
|
||||
|
||||
:param addon: The add-on object (or path) to be stored
|
||||
|
|
@ -107,19 +23,13 @@ class AddonManager(Mapping[str, Addon]):
|
|||
addon = load_object(addon)
|
||||
if isinstance(addon, type):
|
||||
addon = addon()
|
||||
name = addon.name
|
||||
if name in self:
|
||||
raise ValueError(f"Addon '{name}' already loaded")
|
||||
self._addons[name] = addon
|
||||
self.configs[name] = config or {}
|
||||
self.addons.append(addon)
|
||||
|
||||
def load_settings(self, settings):
|
||||
def load_settings(self, settings) -> None:
|
||||
"""Load add-ons and configurations from settings object.
|
||||
|
||||
This will load the addon for every add-on path in the
|
||||
``ADDONS`` setting. For each of these add-ons, the configuration will be
|
||||
read from the dictionary setting whose name matches the uppercase add-on
|
||||
name.
|
||||
``ADDONS`` setting.
|
||||
|
||||
:param settings: The :class:`~scrapy.settings.Settings` object from \
|
||||
which to read the add-on configuration
|
||||
|
|
@ -127,9 +37,8 @@ class AddonManager(Mapping[str, Addon]):
|
|||
"""
|
||||
paths = build_component_list(settings["ADDONS"])
|
||||
addons = [load_object(path) for path in paths]
|
||||
configs = [settings.getdict(addon.name.upper()) for addon in addons]
|
||||
for a, c in zip(addons, configs):
|
||||
self.add(a, c)
|
||||
for a in addons:
|
||||
self.add(a)
|
||||
|
||||
def update_settings(self, settings) -> None:
|
||||
"""Call ``update_settings()`` of all held add-ons.
|
||||
|
|
@ -138,14 +47,5 @@ class AddonManager(Mapping[str, Addon]):
|
|||
updated
|
||||
:type settings: :class:`~scrapy.settings.Settings`
|
||||
"""
|
||||
for name in self:
|
||||
self[name].update_settings(self.configs[name], settings)
|
||||
|
||||
def check_configuration(self, crawler) -> None:
|
||||
"""Call ``check_configuration()`` of all held add-ons.
|
||||
|
||||
:param crawler: the fully-initialized crawler
|
||||
:type crawler: :class:`~scrapy.crawler.Crawler`
|
||||
"""
|
||||
for name in self:
|
||||
self[name].check_configuration(self.configs[name], crawler)
|
||||
for addon in self.addons:
|
||||
addon.update_settings(settings)
|
||||
|
|
|
|||
|
|
@ -1,47 +1,29 @@
|
|||
import unittest
|
||||
from typing import Any, Dict, Optional
|
||||
|
||||
from scrapy.addons import Addon, AddonManager
|
||||
from scrapy.addons import AddonManager
|
||||
from scrapy.settings import BaseSettings
|
||||
|
||||
|
||||
class GoodAddon(object):
|
||||
class GoodAddon:
|
||||
name = "GoodAddon"
|
||||
|
||||
def update_settings(self, config, settings):
|
||||
pass
|
||||
def __init__(self, config: Optional[Dict[str, Any]] = None) -> None:
|
||||
super().__init__()
|
||||
self.config = config or {}
|
||||
|
||||
def check_configuration(self, config, crawler):
|
||||
pass
|
||||
def update_settings(self, settings):
|
||||
settings.update(self.config, "addon")
|
||||
|
||||
|
||||
class AddonTest(unittest.TestCase):
|
||||
def setUp(self):
|
||||
class AddonWithAttributes(Addon):
|
||||
name = "Test"
|
||||
|
||||
self.testaddon = AddonWithAttributes()
|
||||
|
||||
def test_export_config(self):
|
||||
settings = BaseSettings()
|
||||
self.testaddon.config_mapping = {"MAPPED_key": "MAPPING_WORKED"}
|
||||
self.testaddon.default_config = {"key": 55, "defaultkey": 100}
|
||||
self.testaddon.export_config(
|
||||
{"key": 313, "OTHERKEY": True, "mapped_KEY": 99}, settings
|
||||
)
|
||||
self.assertEqual(settings["KEY"], 313)
|
||||
self.assertEqual(settings["DEFAULTKEY"], 100)
|
||||
self.assertEqual(settings["OTHERKEY"], True)
|
||||
self.assertNotIn("MAPPED_key", settings)
|
||||
self.assertNotIn("MAPPED_KEY", settings)
|
||||
self.assertEqual(settings["MAPPING_WORKED"], 99)
|
||||
self.assertEqual(settings.getpriority("KEY"), 15)
|
||||
|
||||
def test_update_settings(self):
|
||||
settings = BaseSettings()
|
||||
settings.set("KEY1", "default", priority="default")
|
||||
settings.set("KEY2", "project", priority="project")
|
||||
addon_config = {"key1": "addon", "key2": "addon", "key3": "addon"}
|
||||
self.testaddon.update_settings(addon_config, settings)
|
||||
addon_config = {"KEY1": "addon", "KEY2": "addon", "KEY3": "addon"}
|
||||
testaddon = GoodAddon(addon_config)
|
||||
testaddon.update_settings(settings)
|
||||
self.assertEqual(settings["KEY1"], "addon")
|
||||
self.assertEqual(settings["KEY2"], "project")
|
||||
self.assertEqual(settings["KEY3"], "addon")
|
||||
|
|
@ -54,8 +36,7 @@ class AddonManagerTest(unittest.TestCase):
|
|||
def test_add(self):
|
||||
manager = AddonManager()
|
||||
manager.add("tests.test_addons.GoodAddon")
|
||||
self.assertCountEqual(manager, ["GoodAddon"])
|
||||
self.assertIsInstance(manager["GoodAddon"], GoodAddon)
|
||||
self.assertIsInstance(manager.addons[0], GoodAddon)
|
||||
|
||||
def test_load_settings(self):
|
||||
settings = BaseSettings()
|
||||
|
|
@ -66,7 +47,4 @@ class AddonManagerTest(unittest.TestCase):
|
|||
settings.set("GOODADDON", {"key": "val2"})
|
||||
manager = AddonManager()
|
||||
manager.load_settings(settings)
|
||||
self.assertCountEqual(manager, ["GoodAddon"])
|
||||
self.assertIsInstance(manager["GoodAddon"], GoodAddon)
|
||||
self.assertCountEqual(manager.configs["GoodAddon"], ["key"])
|
||||
self.assertEqual(manager.configs["GoodAddon"]["key"], "val2")
|
||||
self.assertIsInstance(manager.addons[0], GoodAddon)
|
||||
|
|
|
|||
|
|
@ -13,7 +13,6 @@ from twisted.python.failure import Failure
|
|||
from twisted.trial.unittest import TestCase
|
||||
|
||||
from scrapy import signals
|
||||
from scrapy.addons import Addon, AddonManager
|
||||
from scrapy.crawler import CrawlerRunner
|
||||
from scrapy.exceptions import StopDownload
|
||||
from scrapy.http import Request
|
||||
|
|
@ -408,21 +407,6 @@ with multiples lines
|
|||
self._assert_retried(log)
|
||||
self.assertIn("Got response 200", str(log))
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def test_abort_on_addon_failed_check(self):
|
||||
class FailedCheckAddon(Addon):
|
||||
name = "FailedCheckAddon"
|
||||
|
||||
def check_configuration(self, config, crawler):
|
||||
raise ValueError
|
||||
|
||||
addonmgr = AddonManager()
|
||||
addonmgr.add(FailedCheckAddon())
|
||||
crawler = get_crawler(SimpleSpider)
|
||||
crawler.addons = addonmgr
|
||||
with self.assertRaises(ValueError):
|
||||
yield crawler.crawl()
|
||||
|
||||
|
||||
class CrawlSpiderTestCase(TestCase):
|
||||
def setUp(self):
|
||||
|
|
|
|||
Loading…
Reference in New Issue