mirror of https://github.com/scrapy/scrapy.git
Test, linter etc. fixes.
This commit is contained in:
Andrey Rakhmatullin
parent
898e3045a1
commit
075ad6f196
|
|
@ -350,29 +350,31 @@ Check configuration of fully initialized crawler (see
|
|||
|
||||
Provide add-on interface through a module:
|
||||
|
||||
.. No idea why just using '::' doesn't work for this one
|
||||
.. code-block:: python
|
||||
|
||||
name = 'AddonModule'
|
||||
version = '1.0'
|
||||
name = "AddonModule"
|
||||
version = "1.0"
|
||||
|
||||
|
||||
class MyPipeline(object):
|
||||
# ...
|
||||
...
|
||||
|
||||
|
||||
class MyDownloaderMiddleware(object):
|
||||
# ...
|
||||
...
|
||||
|
||||
|
||||
def update_settings(config, settings):
|
||||
settings.set(
|
||||
'ITEM_PIPELINES',
|
||||
"ITEM_PIPELINES",
|
||||
{MyPipeline(): 200},
|
||||
priority='addon',
|
||||
}
|
||||
priority="addon",
|
||||
)
|
||||
settings.set(
|
||||
'DOWNLOADER_MIDDLEWARES',
|
||||
"DOWNLOADER_MIDDLEWARES",
|
||||
{MyDownloaderMiddleware(): 800},
|
||||
priority='addon',
|
||||
}
|
||||
priority="addon",
|
||||
)
|
||||
|
||||
Forward to other add-ons depending on Python version::
|
||||
|
||||
|
|
|
|||
|
|
@ -1,10 +1,10 @@
|
|||
from collections import defaultdict, Mapping, OrderedDict
|
||||
from inspect import isclass
|
||||
import six
|
||||
import warnings
|
||||
from collections import OrderedDict, defaultdict
|
||||
from collections.abc import Mapping
|
||||
from inspect import isclass
|
||||
|
||||
from pkg_resources import WorkingSet, Distribution, Requirement
|
||||
import zope.interface
|
||||
from pkg_resources import Distribution, Requirement, WorkingSet
|
||||
from zope.interface.verify import verifyObject
|
||||
|
||||
from scrapy.interfaces import IAddon
|
||||
|
|
@ -14,7 +14,6 @@ from scrapy.utils.misc import load_module_or_object
|
|||
|
||||
@zope.interface.implementer(IAddon)
|
||||
class Addon(object):
|
||||
|
||||
basic_settings = None
|
||||
"""``dict`` of settings that will be exported via :meth:`export_basics`."""
|
||||
|
||||
|
|
@ -83,8 +82,8 @@ class Addon(object):
|
|||
else:
|
||||
# e.g. for DOWNLOADER_MIDDLEWARES: {'myclass': 100}
|
||||
k = comp
|
||||
v = config.get('order', self.component_order)
|
||||
settings[self.component_type].update({k: v}, 'addon')
|
||||
v = config.get("order", self.component_order)
|
||||
settings[self.component_type].update({k: v}, "addon")
|
||||
|
||||
def export_basics(self, settings):
|
||||
"""Export the :attr:`basic_settings` attribute into the settings object.
|
||||
|
|
@ -95,8 +94,8 @@ class Addon(object):
|
|||
:param settings: Settings object into which to expose the basic settings
|
||||
:type settings: :class:`~scrapy.settings.Settings`
|
||||
"""
|
||||
for setting, value in six.iteritems(self.basic_settings or {}):
|
||||
settings.set(setting, value, 'addon')
|
||||
for setting, value in (self.basic_settings or {}).items():
|
||||
settings.set(setting, value, "addon")
|
||||
|
||||
def export_config(self, config, settings):
|
||||
"""Export the add-on configuration, all keys in caps and with
|
||||
|
|
@ -121,14 +120,13 @@ class Addon(object):
|
|||
prefix = self.settings_prefix or self.name
|
||||
# Since default exported config is case-insensitive (everything will be
|
||||
# uppercased), make mapped config case-insensitive as well
|
||||
conf_mapping = {k.lower(): v
|
||||
for k, v in six.iteritems(self.config_mapping or {})}
|
||||
for key, val in six.iteritems(conf):
|
||||
conf_mapping = {k.lower(): v for k, v in (self.config_mapping or {}).items()}
|
||||
for key, val in conf.items():
|
||||
if key.lower() in conf_mapping:
|
||||
key = conf_mapping[key.lower()]
|
||||
else:
|
||||
key = (prefix + '_' + key).upper()
|
||||
settings.set(key, val, 'addon')
|
||||
key = (prefix + "_" + key).upper()
|
||||
settings.set(key, val, "addon")
|
||||
|
||||
def update_settings(self, config, settings):
|
||||
"""Export both the basic settings and the add-on configuration. I.e.,
|
||||
|
|
@ -210,11 +208,11 @@ class AddonManager(Mapping):
|
|||
verifyObject(IAddon, addon)
|
||||
name = addon.name
|
||||
if name in self:
|
||||
raise ValueError("Addon '{}' already loaded".format(name))
|
||||
raise ValueError(f"Addon '{name}' already loaded")
|
||||
self._addons[name] = addon
|
||||
self.configs[name] = config or {}
|
||||
if name in self._disable_on_add:
|
||||
self.configs[name]['_enabled'] = False
|
||||
self.configs[name]["_enabled"] = False
|
||||
self._disable_on_add.remove(name)
|
||||
|
||||
def remove(self, addon):
|
||||
|
|
@ -229,7 +227,7 @@ class AddonManager(Mapping):
|
|||
"""
|
||||
if addon in self:
|
||||
del self[addon]
|
||||
elif hasattr(addon, 'name') and addon.name in self:
|
||||
elif hasattr(addon, "name") and addon.name in self:
|
||||
del self[addon.name]
|
||||
else:
|
||||
try:
|
||||
|
|
@ -250,14 +248,14 @@ class AddonManager(Mapping):
|
|||
:param path: Python or file path to an add-on
|
||||
:type path: ``str``
|
||||
"""
|
||||
if isinstance(path, six.string_types):
|
||||
if isinstance(path, str):
|
||||
try:
|
||||
obj = load_module_or_object(path)
|
||||
except NameError:
|
||||
raise NameError("Could not find add-on '%s'" % path)
|
||||
raise NameError(f"Could not find add-on '{path}'")
|
||||
else:
|
||||
obj = path
|
||||
if hasattr(obj, '_addon'):
|
||||
if hasattr(obj, "_addon"):
|
||||
obj = AddonManager.get_addon(obj._addon)
|
||||
return obj
|
||||
|
||||
|
|
@ -284,7 +282,7 @@ class AddonManager(Mapping):
|
|||
and values correspond to their configuration
|
||||
:type addonsdict: ``dict``
|
||||
"""
|
||||
for addonpath, addoncfg in six.iteritems(addonsdict):
|
||||
for addonpath, addoncfg in addonsdict.items():
|
||||
self.add(addonpath, addoncfg)
|
||||
|
||||
def load_settings(self, settings):
|
||||
|
|
@ -299,7 +297,7 @@ class AddonManager(Mapping):
|
|||
which to read the add-on configuration
|
||||
:type settings: :class:`~scrapy.settings.Settings`
|
||||
"""
|
||||
paths = build_component_list(settings['ADDONS'])
|
||||
paths = build_component_list(settings["ADDONS"])
|
||||
addons = [self.get_addon(path) for path in paths]
|
||||
configs = [settings.getdict(addon.name.upper()) for addon in addons]
|
||||
for a, c in zip(addons, configs):
|
||||
|
|
@ -322,20 +320,23 @@ class AddonManager(Mapping):
|
|||
add-on.
|
||||
"""
|
||||
# Collect all active add-ons and the components they provide
|
||||
ws = WorkingSet('')
|
||||
ws = WorkingSet("")
|
||||
|
||||
def add_dist(project_name, version, **kwargs):
|
||||
if project_name in ws.entry_keys.get('scrapy', []):
|
||||
raise ImportError("Component {} provided by multiple add-ons"
|
||||
"".format(project_name))
|
||||
if project_name in ws.entry_keys.get("scrapy", []):
|
||||
raise ImportError(
|
||||
f"Component {project_name} provided by multiple add-ons"
|
||||
)
|
||||
else:
|
||||
dist = Distribution(project_name=project_name, version=version,
|
||||
**kwargs)
|
||||
ws.add(dist, entry='scrapy')
|
||||
dist = Distribution(
|
||||
project_name=project_name, version=version, **kwargs
|
||||
)
|
||||
ws.add(dist, entry="scrapy")
|
||||
|
||||
for name in self:
|
||||
ver = self[name].version
|
||||
add_dist(name, ver)
|
||||
for provides_name in getattr(self[name], 'provides', []):
|
||||
for provides_name in getattr(self[name], "provides", []):
|
||||
add_dist(provides_name, ver)
|
||||
|
||||
# Collect all required and modified components
|
||||
|
|
@ -345,8 +346,9 @@ class AddonManager(Mapping):
|
|||
for entry in getattr(self[name], attribute_name, []):
|
||||
attrs[entry].append(name)
|
||||
return attrs
|
||||
modified = compile_attribute_dict('modifies')
|
||||
required = compile_attribute_dict('requires')
|
||||
|
||||
modified = compile_attribute_dict("modifies")
|
||||
required = compile_attribute_dict("requires")
|
||||
|
||||
req_or_mod = set(required.keys()).union(modified.keys())
|
||||
for reqstr in req_or_mod:
|
||||
|
|
@ -355,15 +357,16 @@ class AddonManager(Mapping):
|
|||
# our own exception or is it helpful enough?
|
||||
if ws.find(req) is None:
|
||||
raise ImportError(
|
||||
"Add-ons {} require or modify missing component {}"
|
||||
"".format(required[reqstr]+modified[reqstr], reqstr)
|
||||
f"Add-ons {required[reqstr] + modified[reqstr]} require"
|
||||
f" or modify missing component {reqstr}"
|
||||
)
|
||||
|
||||
mod_and_req = set(required.keys()).intersection(modified.keys())
|
||||
for conflict in mod_and_req:
|
||||
warnings.warn("Component '{}', required by add-ons {}, is modified "
|
||||
"by add-ons {}".format(conflict, required[conflict],
|
||||
modified[conflict]))
|
||||
warnings.warn(
|
||||
f"Component '{conflict}', required by add-ons {required[conflict]},"
|
||||
f" is modified by add-ons {modified[conflict]}"
|
||||
)
|
||||
|
||||
def disable(self, addon):
|
||||
"""Disable an add-on, i.e. prevent its callbacks from being called.
|
||||
|
|
@ -375,7 +378,7 @@ class AddonManager(Mapping):
|
|||
:type addon: ``str``
|
||||
"""
|
||||
if addon in self:
|
||||
self.configs[addon]['_enabled'] = False
|
||||
self.configs[addon]["_enabled"] = False
|
||||
else:
|
||||
self._disable_on_add.append(addon)
|
||||
|
||||
|
|
@ -389,23 +392,23 @@ class AddonManager(Mapping):
|
|||
:type addon: ``str``
|
||||
"""
|
||||
if addon in self:
|
||||
self.configs[addon]['_enabled'] = True
|
||||
self.configs[addon]["_enabled"] = True
|
||||
elif addon in self._disable_on_add:
|
||||
self._disable_on_add.remove(addon)
|
||||
else:
|
||||
raise ValueError("Add-ons need to be added before they can be "
|
||||
"enabled")
|
||||
raise ValueError("Add-ons need to be added before they can be " "enabled")
|
||||
|
||||
@property
|
||||
def disabled(self):
|
||||
"""Names of disabled add-ons"""
|
||||
return ([a for a in self if not self.configs[a].get('_enabled', True)] +
|
||||
self._disable_on_add)
|
||||
return [
|
||||
a for a in self if not self.configs[a].get("_enabled", True)
|
||||
] + self._disable_on_add
|
||||
|
||||
@property
|
||||
def enabled(self):
|
||||
"""Names of enabled add-ons"""
|
||||
return [a for a in self if self.configs[a].get('_enabled', True)]
|
||||
return [a for a in self if self.configs[a].get("_enabled", True)]
|
||||
|
||||
def _call_if_exists(self, obj, cbname, *args, **kwargs):
|
||||
if obj is None:
|
||||
|
|
@ -418,9 +421,10 @@ class AddonManager(Mapping):
|
|||
cb(*args, **kwargs)
|
||||
|
||||
def _call_addon(self, addonname, cbname, *args, **kwargs):
|
||||
if self.configs[addonname].get('_enabled', True):
|
||||
self._call_if_exists(self[addonname], cbname,
|
||||
self.configs[addonname], *args, **kwargs)
|
||||
if self.configs[addonname].get("_enabled", True):
|
||||
self._call_if_exists(
|
||||
self[addonname], cbname, self.configs[addonname], *args, **kwargs
|
||||
)
|
||||
|
||||
def update_addons(self):
|
||||
"""Call ``update_addons()`` of all held add-ons.
|
||||
|
|
@ -432,7 +436,7 @@ class AddonManager(Mapping):
|
|||
while called_addons != set(self):
|
||||
for name in set(self).difference(called_addons):
|
||||
called_addons.add(name)
|
||||
self._call_addon(name, 'update_addons', self)
|
||||
self._call_addon(name, "update_addons", self)
|
||||
|
||||
def update_settings(self, settings):
|
||||
"""Call ``update_settings()`` of all held add-ons.
|
||||
|
|
@ -442,7 +446,7 @@ class AddonManager(Mapping):
|
|||
:type settings: :class:`~scrapy.settings.Settings`
|
||||
"""
|
||||
for name in self:
|
||||
self._call_addon(name, 'update_settings', settings)
|
||||
self._call_addon(name, "update_settings", settings)
|
||||
|
||||
def check_configuration(self, crawler):
|
||||
"""Call ``check_configuration()`` of all held add-ons.
|
||||
|
|
@ -451,7 +455,7 @@ class AddonManager(Mapping):
|
|||
:type crawler: :class:`~scrapy.crawler.Crawler`
|
||||
"""
|
||||
for name in self:
|
||||
self._call_addon(name, 'check_configuration', crawler)
|
||||
self._call_addon(name, "check_configuration", crawler)
|
||||
|
||||
|
||||
from scrapy.addons.builtins import *
|
||||
from scrapy.addons.builtins import * # noqa
|
||||
|
|
|
|||
|
|
@ -1,23 +1,44 @@
|
|||
import scrapy
|
||||
from scrapy.addons import Addon
|
||||
|
||||
__all__ = ['make_builtin_addon',
|
||||
|
||||
'depth', 'httperror', 'offsite', 'referer', 'urllength',
|
||||
|
||||
'ajaxcrawl', 'chunked', 'cookies', 'defaultheaders',
|
||||
'downloadtimeout', 'httpauth', 'httpcache', 'httpcompression',
|
||||
'httpproxy', 'metarefresh', 'redirect', 'retry', 'robotstxt',
|
||||
'stats', 'useragent',
|
||||
|
||||
'autothrottle', 'corestats', 'closespider', 'debugger', 'feedexport',
|
||||
'logstats', 'memdebug', 'memusage', 'spiderstate', 'stacktracedump',
|
||||
'statsmailer', 'telnetconsole',
|
||||
]
|
||||
__all__ = [
|
||||
"make_builtin_addon",
|
||||
"depth",
|
||||
"httperror",
|
||||
"offsite",
|
||||
"referer",
|
||||
"urllength",
|
||||
"ajaxcrawl",
|
||||
"chunked",
|
||||
"cookies",
|
||||
"defaultheaders",
|
||||
"downloadtimeout",
|
||||
"httpauth",
|
||||
"httpcache",
|
||||
"httpcompression",
|
||||
"httpproxy",
|
||||
"metarefresh",
|
||||
"redirect",
|
||||
"retry",
|
||||
"robotstxt",
|
||||
"stats",
|
||||
"useragent",
|
||||
"autothrottle",
|
||||
"corestats",
|
||||
"closespider",
|
||||
"debugger",
|
||||
"feedexport",
|
||||
"logstats",
|
||||
"memdebug",
|
||||
"memusage",
|
||||
"spiderstate",
|
||||
"stacktracedump",
|
||||
"statsmailer",
|
||||
"telnetconsole",
|
||||
]
|
||||
|
||||
|
||||
def make_builtin_addon(addon_name, addon_default_config=None,
|
||||
addon_version=None):
|
||||
def make_builtin_addon(addon_name, addon_default_config=None, addon_version=None):
|
||||
class ThisAddon(Addon):
|
||||
name = addon_name
|
||||
version = addon_version or scrapy.__version__
|
||||
|
|
@ -33,59 +54,65 @@ def make_builtin_addon(addon_name, addon_default_config=None,
|
|||
|
||||
# SPIDER MIDDLEWARES
|
||||
|
||||
depth = make_builtin_addon('depth')
|
||||
depth = make_builtin_addon("depth")
|
||||
|
||||
httperror = make_builtin_addon('httperror')
|
||||
httperror = make_builtin_addon("httperror")
|
||||
|
||||
offsite = make_builtin_addon('offsite')
|
||||
offsite = make_builtin_addon("offsite")
|
||||
|
||||
referer = make_builtin_addon('referer')
|
||||
referer = make_builtin_addon("referer")
|
||||
|
||||
urllength = make_builtin_addon('urllength')
|
||||
urllength = make_builtin_addon("urllength")
|
||||
|
||||
|
||||
# DOWNLOADER MIDDLEWARES
|
||||
|
||||
ajaxcrawl = make_builtin_addon('ajaxcrawl', {'enabled': True})
|
||||
ajaxcrawl = make_builtin_addon("ajaxcrawl", {"enabled": True})
|
||||
|
||||
chunked = make_builtin_addon('chunked')
|
||||
chunked = make_builtin_addon("chunked")
|
||||
|
||||
cookies = make_builtin_addon("cookies")
|
||||
|
||||
defaultheaders = make_builtin_addon("defaultheaders")
|
||||
|
||||
cookies = make_builtin_addon('cookies')
|
||||
|
||||
defaultheaders = make_builtin_addon('defaultheaders')
|
||||
# Assume every config entry is a header
|
||||
def defaultheaders_export_config(self, config, settings):
|
||||
conf = self.default_config or {}
|
||||
conf.update(config)
|
||||
settings.set('DEFAULT_REQUEST_HEADERS', conf, 'addon')
|
||||
settings.set("DEFAULT_REQUEST_HEADERS", conf, "addon")
|
||||
|
||||
|
||||
defaultheaders.export_config = defaultheaders_export_config
|
||||
|
||||
downloadtimeout = make_builtin_addon('downloadtimeout')
|
||||
downloadtimeout.config_mapping = {'timeout': 'DOWNLOAD_TIMEOUT',
|
||||
'download_timeout': 'DOWNLOAD_TIMEOUT'}
|
||||
downloadtimeout = make_builtin_addon("downloadtimeout")
|
||||
downloadtimeout.config_mapping = {
|
||||
"timeout": "DOWNLOAD_TIMEOUT",
|
||||
"download_timeout": "DOWNLOAD_TIMEOUT",
|
||||
}
|
||||
|
||||
httpauth = make_builtin_addon('httpauth')
|
||||
httpauth = make_builtin_addon("httpauth")
|
||||
|
||||
httpcache = make_builtin_addon('httpcache', {'enabled': True})
|
||||
httpcache = make_builtin_addon("httpcache", {"enabled": True})
|
||||
|
||||
httpcompression = make_builtin_addon('httpcompression')
|
||||
httpcompression.config_mapping = {'enabled': 'COMPRESSION_ENABLED'}
|
||||
httpcompression = make_builtin_addon("httpcompression")
|
||||
httpcompression.config_mapping = {"enabled": "COMPRESSION_ENABLED"}
|
||||
|
||||
httpproxy = make_builtin_addon('httpproxy')
|
||||
httpproxy = make_builtin_addon("httpproxy")
|
||||
|
||||
metarefresh = make_builtin_addon('metarefresh')
|
||||
metarefresh.config_mapping = {'max_times': 'REDIRECT_MAX_TIMES'}
|
||||
metarefresh = make_builtin_addon("metarefresh")
|
||||
metarefresh.config_mapping = {"max_times": "REDIRECT_MAX_TIMES"}
|
||||
|
||||
redirect = make_builtin_addon('redirect')
|
||||
redirect = make_builtin_addon("redirect")
|
||||
|
||||
retry = make_builtin_addon('retry')
|
||||
retry = make_builtin_addon("retry")
|
||||
|
||||
robotstxt = make_builtin_addon('robotstxt', {'obey': True})
|
||||
robotstxt = make_builtin_addon("robotstxt", {"obey": True})
|
||||
|
||||
stats = make_builtin_addon('stats')
|
||||
stats = make_builtin_addon("stats")
|
||||
|
||||
useragent = make_builtin_addon('useragent')
|
||||
useragent.config_mapping = {'user_agent': 'USER_AGENT'}
|
||||
useragent = make_builtin_addon("useragent")
|
||||
useragent.config_mapping = {"user_agent": "USER_AGENT"}
|
||||
|
||||
|
||||
# ITEM PIPELINES
|
||||
|
|
@ -93,27 +120,27 @@ useragent.config_mapping = {'user_agent': 'USER_AGENT'}
|
|||
|
||||
# EXTENSIONS
|
||||
|
||||
autothrottle = make_builtin_addon('autothrottle', {'enabled': True})
|
||||
autothrottle = make_builtin_addon("autothrottle", {"enabled": True})
|
||||
|
||||
corestats = make_builtin_addon('corestats')
|
||||
corestats = make_builtin_addon("corestats")
|
||||
|
||||
closespider = make_builtin_addon('closespider')
|
||||
closespider = make_builtin_addon("closespider")
|
||||
|
||||
debugger = make_builtin_addon('debugger')
|
||||
debugger = make_builtin_addon("debugger")
|
||||
|
||||
feedexport = make_builtin_addon('feedexport')
|
||||
feedexport.settings_prefix = 'FEED'
|
||||
feedexport = make_builtin_addon("feedexport")
|
||||
feedexport.settings_prefix = "FEED"
|
||||
|
||||
logstats = make_builtin_addon('logstats')
|
||||
logstats = make_builtin_addon("logstats")
|
||||
|
||||
memdebug = make_builtin_addon('memdebug', {'enabled': True})
|
||||
memdebug = make_builtin_addon("memdebug", {"enabled": True})
|
||||
|
||||
memusage = make_builtin_addon('memusage', {'enabled': True})
|
||||
memusage = make_builtin_addon("memusage", {"enabled": True})
|
||||
|
||||
spiderstate = make_builtin_addon('spiderstate')
|
||||
spiderstate = make_builtin_addon("spiderstate")
|
||||
|
||||
stacktracedump = make_builtin_addon('stacktracedump')
|
||||
stacktracedump = make_builtin_addon("stacktracedump")
|
||||
|
||||
statsmailer = make_builtin_addon('statsmailer')
|
||||
statsmailer = make_builtin_addon("statsmailer")
|
||||
|
||||
telnetconsole = make_builtin_addon('telnetconsole')
|
||||
telnetconsole = make_builtin_addon("telnetconsole")
|
||||
|
|
|
|||
|
|
@ -85,7 +85,7 @@ def load_module_or_object(path):
|
|||
return load_object(path)
|
||||
except (ValueError, NameError, ImportError):
|
||||
pass
|
||||
raise NameError("Could not load '%s'" % path)
|
||||
raise NameError(f"Could not load '{path}'")
|
||||
|
||||
|
||||
def walk_modules(path):
|
||||
|
|
|
|||
|
|
@ -1,145 +1,143 @@
|
|||
from collections import OrderedDict
|
||||
import itertools
|
||||
import os.path
|
||||
import six
|
||||
from tests import mock
|
||||
import unittest
|
||||
import warnings
|
||||
from collections import OrderedDict
|
||||
from unittest import mock
|
||||
|
||||
from pkg_resources import VersionConflict
|
||||
import zope.interface
|
||||
from zope.interface import directlyProvides
|
||||
from zope.interface.exceptions import BrokenImplementation, MultipleInvalid
|
||||
from zope.interface.verify import verifyObject
|
||||
from zope.interface.exceptions import BrokenImplementation
|
||||
|
||||
from scrapy.addons import Addon, AddonManager
|
||||
from scrapy.crawler import Crawler
|
||||
from scrapy.interfaces import IAddon
|
||||
from scrapy.settings import BaseSettings
|
||||
|
||||
from . import addons
|
||||
from . import addonmod
|
||||
from . import addonmod, addons
|
||||
|
||||
|
||||
class AddonTest(unittest.TestCase):
|
||||
|
||||
def setUp(self):
|
||||
self.rawaddon = Addon()
|
||||
|
||||
class AddonWithAttributes(Addon):
|
||||
name = 'Test'
|
||||
version = '1.0'
|
||||
name = "Test"
|
||||
version = "1.0"
|
||||
|
||||
self.testaddon = AddonWithAttributes()
|
||||
|
||||
def test_interface(self):
|
||||
# Raw Addon should fail exactly b/c name and version are not given
|
||||
self.assertFalse(hasattr(self.rawaddon, 'name'))
|
||||
self.assertFalse(hasattr(self.rawaddon, 'version'))
|
||||
self.assertRaises(BrokenImplementation, verifyObject, IAddon,
|
||||
self.rawaddon)
|
||||
self.assertFalse(hasattr(self.rawaddon, "name"))
|
||||
self.assertFalse(hasattr(self.rawaddon, "version"))
|
||||
self.assertRaises(MultipleInvalid, verifyObject, IAddon, self.rawaddon)
|
||||
verifyObject(IAddon, self.testaddon)
|
||||
|
||||
def test_export_component(self):
|
||||
settings = BaseSettings({'ITEM_PIPELINES': BaseSettings(),
|
||||
'DOWNLOAD_HANDLERS': BaseSettings()},
|
||||
'default')
|
||||
settings = BaseSettings(
|
||||
{"ITEM_PIPELINES": BaseSettings(), "DOWNLOAD_HANDLERS": BaseSettings()},
|
||||
"default",
|
||||
)
|
||||
self.testaddon.component_type = None
|
||||
self.testaddon.export_component({}, settings)
|
||||
self.assertEqual(len(settings['ITEM_PIPELINES']), 0)
|
||||
self.testaddon.component_type = 'ITEM_PIPELINES'
|
||||
self.testaddon.component = 'test.component'
|
||||
self.assertEqual(len(settings["ITEM_PIPELINES"]), 0)
|
||||
self.testaddon.component_type = "ITEM_PIPELINES"
|
||||
self.testaddon.component = "test.component"
|
||||
self.testaddon.export_component({}, settings)
|
||||
six.assertCountEqual(self, settings['ITEM_PIPELINES'],
|
||||
['test.component'])
|
||||
self.assertEqual(settings['ITEM_PIPELINES']['test.component'], 0)
|
||||
self.assertCountEqual(settings["ITEM_PIPELINES"], ["test.component"])
|
||||
self.assertEqual(settings["ITEM_PIPELINES"]["test.component"], 0)
|
||||
self.testaddon.component_order = 313
|
||||
self.testaddon.export_component({}, settings)
|
||||
self.assertEqual(settings['ITEM_PIPELINES']['test.component'], 313)
|
||||
self.testaddon.component_type = 'DOWNLOAD_HANDLERS'
|
||||
self.testaddon.component_key = 'http'
|
||||
self.assertEqual(settings["ITEM_PIPELINES"]["test.component"], 313)
|
||||
self.testaddon.component_type = "DOWNLOAD_HANDLERS"
|
||||
self.testaddon.component_key = "http"
|
||||
self.testaddon.export_component({}, settings)
|
||||
self.assertEqual(settings['DOWNLOAD_HANDLERS']['http'],
|
||||
'test.component')
|
||||
self.assertEqual(settings["DOWNLOAD_HANDLERS"]["http"], "test.component")
|
||||
|
||||
def test_export_basics(self):
|
||||
settings = BaseSettings()
|
||||
self.testaddon.basic_settings = {'TESTKEY': 313, 'OTHERKEY': True}
|
||||
self.testaddon.basic_settings = {"TESTKEY": 313, "OTHERKEY": True}
|
||||
self.testaddon.export_basics(settings)
|
||||
self.assertEqual(settings['TESTKEY'], 313)
|
||||
self.assertEqual(settings['OTHERKEY'], True)
|
||||
self.assertEqual(settings.getpriority('TESTKEY'), 15)
|
||||
self.assertEqual(settings["TESTKEY"], 313)
|
||||
self.assertEqual(settings["OTHERKEY"], True)
|
||||
self.assertEqual(settings.getpriority("TESTKEY"), 15)
|
||||
|
||||
def test_export_config(self):
|
||||
settings = BaseSettings()
|
||||
self.testaddon.settings_prefix = None
|
||||
self.testaddon.config_mapping = {'MAPPED_key': 'MAPPING_WORKED'}
|
||||
self.testaddon.default_config = {'key': 55, 'defaultkey': 100}
|
||||
self.testaddon.export_config({'key': 313, 'OTHERKEY': True,
|
||||
'mapped_KEY': 99}, settings)
|
||||
self.assertEqual(settings['TEST_KEY'], 313)
|
||||
self.assertEqual(settings['TEST_DEFAULTKEY'], 100)
|
||||
self.assertEqual(settings['TEST_OTHERKEY'], True)
|
||||
self.assertNotIn('MAPPED_key', settings)
|
||||
self.assertNotIn('MAPPED_KEY', settings)
|
||||
self.assertEqual(settings['MAPPING_WORKED'], 99)
|
||||
self.assertEqual(settings.getpriority('TEST_KEY'), 15)
|
||||
self.testaddon.config_mapping = {"MAPPED_key": "MAPPING_WORKED"}
|
||||
self.testaddon.default_config = {"key": 55, "defaultkey": 100}
|
||||
self.testaddon.export_config(
|
||||
{"key": 313, "OTHERKEY": True, "mapped_KEY": 99}, settings
|
||||
)
|
||||
self.assertEqual(settings["TEST_KEY"], 313)
|
||||
self.assertEqual(settings["TEST_DEFAULTKEY"], 100)
|
||||
self.assertEqual(settings["TEST_OTHERKEY"], True)
|
||||
self.assertNotIn("MAPPED_key", settings)
|
||||
self.assertNotIn("MAPPED_KEY", settings)
|
||||
self.assertEqual(settings["MAPPING_WORKED"], 99)
|
||||
self.assertEqual(settings.getpriority("TEST_KEY"), 15)
|
||||
|
||||
self.testaddon.settings_prefix = 'PREF'
|
||||
self.testaddon.export_config({'newkey': 99}, settings)
|
||||
self.assertEqual(settings['PREF_NEWKEY'], 99)
|
||||
self.testaddon.settings_prefix = "PREF"
|
||||
self.testaddon.export_config({"newkey": 99}, settings)
|
||||
self.assertEqual(settings["PREF_NEWKEY"], 99)
|
||||
|
||||
with mock.patch.object(settings, 'set') as mock_set:
|
||||
with mock.patch.object(settings, "set") as mock_set:
|
||||
self.testaddon.settings_prefix = False
|
||||
self.testaddon.export_config({'thirdnewkey': 99}, settings)
|
||||
self.testaddon.export_config({"thirdnewkey": 99}, settings)
|
||||
self.assertEqual(mock_set.call_count, 0)
|
||||
|
||||
def test_update_settings(self):
|
||||
settings = BaseSettings()
|
||||
settings.set('TEST_KEY1', 'default', priority='default')
|
||||
settings.set('TEST_KEY2', 'project', priority='project')
|
||||
settings.set("TEST_KEY1", "default", priority="default")
|
||||
settings.set("TEST_KEY2", "project", priority="project")
|
||||
self.testaddon.settings_prefix = None
|
||||
self.testaddon.basic_settings = {'OTHERTEST_KEY': 'addon'}
|
||||
addon_config = {'key1': 'addon', 'key2': 'addon', 'key3': 'addon'}
|
||||
self.testaddon.basic_settings = {"OTHERTEST_KEY": "addon"}
|
||||
addon_config = {"key1": "addon", "key2": "addon", "key3": "addon"}
|
||||
self.testaddon.update_settings(addon_config, settings)
|
||||
self.assertEqual(settings['OTHERTEST_KEY'], 'addon')
|
||||
self.assertEqual(settings['TEST_KEY1'], 'addon')
|
||||
self.assertEqual(settings['TEST_KEY2'], 'project')
|
||||
self.assertEqual(settings['TEST_KEY3'], 'addon')
|
||||
self.assertEqual(settings["OTHERTEST_KEY"], "addon")
|
||||
self.assertEqual(settings["TEST_KEY1"], "addon")
|
||||
self.assertEqual(settings["TEST_KEY2"], "project")
|
||||
self.assertEqual(settings["TEST_KEY3"], "addon")
|
||||
|
||||
|
||||
class AddonManagerTest(unittest.TestCase):
|
||||
|
||||
def setUp(self):
|
||||
self.manager = AddonManager()
|
||||
|
||||
def test_add(self):
|
||||
manager = AddonManager()
|
||||
manager.add(addonmod, {'key': 'val1'})
|
||||
manager.add('tests.test_addons.addons.GoodAddon')
|
||||
six.assertCountEqual(self, manager, ['AddonModule', 'GoodAddon'])
|
||||
self.assertIsInstance(manager['GoodAddon'], addons.GoodAddon)
|
||||
six.assertCountEqual(self, manager.configs['AddonModule'], ['key'])
|
||||
self.assertEqual(manager.configs['AddonModule']['key'], 'val1')
|
||||
manager.add(addonmod, {"key": "val1"})
|
||||
manager.add("tests.test_addons.addons.GoodAddon")
|
||||
self.assertCountEqual(manager, ["AddonModule", "GoodAddon"])
|
||||
self.assertIsInstance(manager["GoodAddon"], addons.GoodAddon)
|
||||
self.assertCountEqual(manager.configs["AddonModule"], ["key"])
|
||||
self.assertEqual(manager.configs["AddonModule"]["key"], "val1")
|
||||
self.assertRaises(ValueError, manager.add, addonmod)
|
||||
|
||||
def test_add_dont_instantiate_providing_classes(self):
|
||||
class ProviderGoodAddon(addons.GoodAddon):
|
||||
pass
|
||||
zope.interface.directlyProvides(ProviderGoodAddon, IAddon)
|
||||
|
||||
directlyProvides(ProviderGoodAddon, IAddon)
|
||||
manager = AddonManager()
|
||||
manager.add(ProviderGoodAddon)
|
||||
self.assertIs(manager['GoodAddon'], ProviderGoodAddon)
|
||||
self.assertIs(manager["GoodAddon"], ProviderGoodAddon)
|
||||
|
||||
def test_add_verifies(self):
|
||||
brokenaddon = self.manager.get_addon(
|
||||
'tests.test_addons.addons.BrokenAddon')
|
||||
self.assertRaises(zope.interface.exceptions.BrokenImplementation,
|
||||
self.manager.add,
|
||||
brokenaddon)
|
||||
brokenaddon = self.manager.get_addon("tests.test_addons.addons.BrokenAddon")
|
||||
self.assertRaises(
|
||||
BrokenImplementation,
|
||||
self.manager.add,
|
||||
brokenaddon,
|
||||
)
|
||||
|
||||
def test_add_adds_missing_interface_declaration(self):
|
||||
class GoodAddonWithoutDeclaration(object):
|
||||
name = 'GoodAddonWithoutDeclaration'
|
||||
version = '1.0'
|
||||
name = "GoodAddonWithoutDeclaration"
|
||||
version = "1.0"
|
||||
|
||||
self.manager.add(GoodAddonWithoutDeclaration)
|
||||
|
||||
def test_remove(self):
|
||||
|
|
@ -147,64 +145,66 @@ class AddonManagerTest(unittest.TestCase):
|
|||
|
||||
def test_gets_removed(removearg):
|
||||
manager.add(addonmod)
|
||||
self.assertIn('AddonModule', manager)
|
||||
self.assertIn("AddonModule", manager)
|
||||
manager.remove(removearg)
|
||||
self.assertNotIn('AddonModule', manager)
|
||||
self.assertNotIn("AddonModule", manager)
|
||||
|
||||
test_gets_removed('AddonModule')
|
||||
test_gets_removed("AddonModule")
|
||||
test_gets_removed(addonmod)
|
||||
test_gets_removed('tests.test_addons.addonmod')
|
||||
self.assertRaises(KeyError, manager.remove, 'nonexistent')
|
||||
test_gets_removed("tests.test_addons.addonmod")
|
||||
self.assertRaises(KeyError, manager.remove, "nonexistent")
|
||||
self.assertRaises(KeyError, manager.remove, addons.GoodAddon())
|
||||
|
||||
def test_get_addon(self):
|
||||
goodaddon = self.manager.get_addon('tests.test_addons.addons.GoodAddon')
|
||||
goodaddon = self.manager.get_addon("tests.test_addons.addons.GoodAddon")
|
||||
self.assertIs(goodaddon, addons.GoodAddon)
|
||||
|
||||
loaded_addonmod = self.manager.get_addon("tests.test_addons.addonmod")
|
||||
self.assertIs(loaded_addonmod, addonmod)
|
||||
|
||||
addonspath = os.path.join(os.path.dirname(__file__), 'addons.py')
|
||||
goodaddon = self.manager.get_addon("tests.test_addons.addons")
|
||||
self.assertIsInstance(goodaddon, addons.GoodAddon)
|
||||
|
||||
self.assertRaises(NameError, self.manager.get_addon, 'xy.n_onexistent')
|
||||
self.assertRaises(NameError, self.manager.get_addon, "xy.n_onexistent")
|
||||
|
||||
def test_get_addon_forward(self):
|
||||
class SomeCls(object):
|
||||
_addon = 'tests.test_addons.addons.GoodAddon'
|
||||
_addon = "tests.test_addons.addons.GoodAddon"
|
||||
|
||||
self.assertIs(self.manager.get_addon(SomeCls()), addons.GoodAddon)
|
||||
|
||||
def test_get_addon_nested(self):
|
||||
x = addons.GoodAddon('outer')
|
||||
x._addon = addons.GoodAddon('middle')
|
||||
x._addon._addon = addons.GoodAddon('inner')
|
||||
x = addons.GoodAddon("outer")
|
||||
x._addon = addons.GoodAddon("middle")
|
||||
x._addon._addon = addons.GoodAddon("inner")
|
||||
self.assertIs(self.manager.get_addon(x), x._addon._addon)
|
||||
|
||||
def test_load_dict_load_settings(self):
|
||||
def _test_load_method(func, *args, **kwargs):
|
||||
manager = AddonManager()
|
||||
getattr(manager, func)(*args, **kwargs)
|
||||
six.assertCountEqual(self, manager, ['GoodAddon', 'AddonModule'])
|
||||
self.assertIsInstance(manager['GoodAddon'], addons.GoodAddon)
|
||||
six.assertCountEqual(self, manager.configs['GoodAddon'], ['key'])
|
||||
self.assertEqual(manager.configs['GoodAddon']['key'], 'val2')
|
||||
self.assertEqual(manager['AddonModule'], addonmod)
|
||||
self.assertIn('key', manager.configs['AddonModule'])
|
||||
self.assertEqual(manager.configs['AddonModule']['key'], 'val1')
|
||||
self.assertCountEqual(manager, ["GoodAddon", "AddonModule"])
|
||||
self.assertIsInstance(manager["GoodAddon"], addons.GoodAddon)
|
||||
self.assertCountEqual(manager.configs["GoodAddon"], ["key"])
|
||||
self.assertEqual(manager.configs["GoodAddon"]["key"], "val2")
|
||||
self.assertEqual(manager["AddonModule"], addonmod)
|
||||
self.assertIn("key", manager.configs["AddonModule"])
|
||||
self.assertEqual(manager.configs["AddonModule"]["key"], "val1")
|
||||
|
||||
addonsdict = {
|
||||
"tests.test_addons.addonmod": {'key': 'val1'},
|
||||
'tests.test_addons.addons.GoodAddon': {'key': 'val2'},
|
||||
}
|
||||
_test_load_method('load_dict', addonsdict)
|
||||
"tests.test_addons.addonmod": {"key": "val1"},
|
||||
"tests.test_addons.addons.GoodAddon": {"key": "val2"},
|
||||
}
|
||||
_test_load_method("load_dict", addonsdict)
|
||||
|
||||
settings = BaseSettings()
|
||||
settings.set('ADDONS', {"tests.test_addons.addonmod": 0,
|
||||
'tests.test_addons.addons.GoodAddon': 0})
|
||||
settings.set('ADDONMODULE', {'key': 'val1'})
|
||||
settings.set('GOODADDON', {'key': 'val2'})
|
||||
_test_load_method('load_settings', settings)
|
||||
settings.set(
|
||||
"ADDONS",
|
||||
{"tests.test_addons.addonmod": 0, "tests.test_addons.addons.GoodAddon": 0},
|
||||
)
|
||||
settings.set("ADDONMODULE", {"key": "val1"})
|
||||
settings.set("GOODADDON", {"key": "val2"})
|
||||
_test_load_method("load_settings", settings)
|
||||
|
||||
def test_load_dict_load_settings_order(self):
|
||||
def _test_load_method(expected_order, func, *args, **kwargs):
|
||||
|
|
@ -218,72 +218,71 @@ class AddonManagerTest(unittest.TestCase):
|
|||
for ordered_addons in itertools.permutations(addonlist):
|
||||
expected_order = [a.name for a in ordered_addons]
|
||||
addonsdict = OrderedDict((a, {}) for a in ordered_addons)
|
||||
_test_load_method(expected_order, 'load_dict', addonsdict)
|
||||
settings = BaseSettings({
|
||||
'ADDONS': {a: i for i, a in enumerate(ordered_addons)}
|
||||
})
|
||||
_test_load_method(expected_order, 'load_settings', settings)
|
||||
_test_load_method(expected_order, "load_dict", addonsdict)
|
||||
settings = BaseSettings(
|
||||
{"ADDONS": {a: i for i, a in enumerate(ordered_addons)}}
|
||||
)
|
||||
_test_load_method(expected_order, "load_settings", settings)
|
||||
|
||||
def test_enabled_disabled(self):
|
||||
manager = AddonManager()
|
||||
manager.add(addons.GoodAddon('FirstAddon'))
|
||||
manager.add(addons.GoodAddon('SecondAddon'))
|
||||
self.assertEqual(set(manager.enabled),
|
||||
set(('FirstAddon', 'SecondAddon')))
|
||||
manager.add(addons.GoodAddon("FirstAddon"))
|
||||
manager.add(addons.GoodAddon("SecondAddon"))
|
||||
self.assertEqual(set(manager.enabled), set(("FirstAddon", "SecondAddon")))
|
||||
self.assertEqual(manager.disabled, [])
|
||||
manager.disable('FirstAddon')
|
||||
self.assertEqual(manager.enabled, ['SecondAddon'])
|
||||
self.assertEqual(manager.disabled, ['FirstAddon'])
|
||||
manager.enable('FirstAddon')
|
||||
self.assertEqual(set(manager.enabled),
|
||||
set(('FirstAddon', 'SecondAddon')))
|
||||
manager.disable("FirstAddon")
|
||||
self.assertEqual(manager.enabled, ["SecondAddon"])
|
||||
self.assertEqual(manager.disabled, ["FirstAddon"])
|
||||
manager.enable("FirstAddon")
|
||||
self.assertEqual(set(manager.enabled), set(("FirstAddon", "SecondAddon")))
|
||||
self.assertEqual(manager.disabled, [])
|
||||
|
||||
def test_enable_before_add(self):
|
||||
manager = AddonManager()
|
||||
self.assertRaises(ValueError, manager.enable, 'FirstAddon')
|
||||
manager.disable('FirstAddon')
|
||||
manager.enable('FirstAddon')
|
||||
manager.add(addons.GoodAddon('FirstAddon'))
|
||||
self.assertIn('FirstAddon', manager.enabled)
|
||||
self.assertRaises(ValueError, manager.enable, "FirstAddon")
|
||||
manager.disable("FirstAddon")
|
||||
manager.enable("FirstAddon")
|
||||
manager.add(addons.GoodAddon("FirstAddon"))
|
||||
self.assertIn("FirstAddon", manager.enabled)
|
||||
|
||||
def test_disable_before_add(self):
|
||||
manager = AddonManager()
|
||||
manager.disable('FirstAddon')
|
||||
manager.add(addons.GoodAddon('FirstAddon'))
|
||||
self.assertEqual(manager.disabled, ['FirstAddon'])
|
||||
manager.disable("FirstAddon")
|
||||
manager.add(addons.GoodAddon("FirstAddon"))
|
||||
self.assertEqual(manager.disabled, ["FirstAddon"])
|
||||
|
||||
def test_callbacks(self):
|
||||
first_addon = addons.GoodAddon('FirstAddon')
|
||||
second_addon = addons.GoodAddon('SecondAddon')
|
||||
first_addon = addons.GoodAddon("FirstAddon")
|
||||
second_addon = addons.GoodAddon("SecondAddon")
|
||||
|
||||
manager = AddonManager()
|
||||
manager.add(first_addon, {'test': 'first'})
|
||||
manager.add(second_addon, {'test': 'second'})
|
||||
manager.add(first_addon, {"test": "first"})
|
||||
manager.add(second_addon, {"test": "second"})
|
||||
crawler = mock.create_autospec(Crawler)
|
||||
settings = BaseSettings()
|
||||
|
||||
with mock.patch.object(first_addon, 'update_addons') as ua_first, \
|
||||
mock.patch.object(second_addon, 'update_addons') as ua_second, \
|
||||
mock.patch.object(first_addon, 'update_settings') as us_first, \
|
||||
mock.patch.object(second_addon, 'update_settings') as us_second, \
|
||||
mock.patch.object(first_addon, 'check_configuration') as cc_first, \
|
||||
mock.patch.object(second_addon, 'check_configuration') as cc_second:
|
||||
with mock.patch.object(
|
||||
first_addon, "update_addons"
|
||||
) as ua_first, mock.patch.object(
|
||||
second_addon, "update_addons"
|
||||
) as ua_second, mock.patch.object(
|
||||
first_addon, "update_settings"
|
||||
) as us_first, mock.patch.object(
|
||||
second_addon, "update_settings"
|
||||
) as us_second, mock.patch.object(
|
||||
first_addon, "check_configuration"
|
||||
) as cc_first, mock.patch.object(
|
||||
second_addon, "check_configuration"
|
||||
) as cc_second:
|
||||
manager.update_addons()
|
||||
ua_first.assert_called_once_with(manager.configs['FirstAddon'],
|
||||
manager)
|
||||
ua_second.assert_called_once_with(manager.configs['SecondAddon'],
|
||||
manager)
|
||||
ua_first.assert_called_once_with(manager.configs["FirstAddon"], manager)
|
||||
ua_second.assert_called_once_with(manager.configs["SecondAddon"], manager)
|
||||
manager.update_settings(settings)
|
||||
us_first.assert_called_once_with(manager.configs['FirstAddon'],
|
||||
settings)
|
||||
us_second.assert_called_once_with(manager.configs['SecondAddon'],
|
||||
settings)
|
||||
us_first.assert_called_once_with(manager.configs["FirstAddon"], settings)
|
||||
us_second.assert_called_once_with(manager.configs["SecondAddon"], settings)
|
||||
manager.check_configuration(crawler)
|
||||
cc_first.assert_called_once_with(manager.configs['FirstAddon'],
|
||||
crawler)
|
||||
cc_second.assert_called_once_with(manager.configs['SecondAddon'],
|
||||
crawler)
|
||||
cc_first.assert_called_once_with(manager.configs["FirstAddon"], crawler)
|
||||
cc_second.assert_called_once_with(manager.configs["SecondAddon"], crawler)
|
||||
self.assertEqual(ua_first.call_count, 1)
|
||||
self.assertEqual(ua_second.call_count, 1)
|
||||
self.assertEqual(us_first.call_count, 1)
|
||||
|
|
@ -291,10 +290,10 @@ class AddonManagerTest(unittest.TestCase):
|
|||
|
||||
us_first.reset_mock()
|
||||
us_second.reset_mock()
|
||||
manager.disable('FirstAddon')
|
||||
manager.disable("FirstAddon")
|
||||
manager.update_settings(settings)
|
||||
self.assertEqual(us_first.call_count, 0)
|
||||
manager.enable('FirstAddon')
|
||||
manager.enable("FirstAddon")
|
||||
manager.update_settings(settings)
|
||||
self.assertEqual(us_first.call_count, 1)
|
||||
self.assertEqual(us_second.call_count, 2)
|
||||
|
|
@ -302,44 +301,42 @@ class AddonManagerTest(unittest.TestCase):
|
|||
# This will become relevant when we let spiders implement the add-on
|
||||
# interface and should be replaced with a test where
|
||||
# AddonManager.spidercls = None then.
|
||||
manager._call_if_exists(None, 'irrelevant')
|
||||
manager._call_if_exists(None, "irrelevant")
|
||||
|
||||
def test_update_addons_last_minute_add(self):
|
||||
class AddedAddon(addons.GoodAddon):
|
||||
name = 'AddedAddon'
|
||||
name = "AddedAddon"
|
||||
|
||||
class FirstAddon(addons.GoodAddon):
|
||||
name = 'FirstAddon'
|
||||
name = "FirstAddon"
|
||||
|
||||
def update_addons(self, config, addons):
|
||||
addons.add(AddedAddon())
|
||||
|
||||
manager = AddonManager()
|
||||
first_addon = FirstAddon()
|
||||
with mock.patch.object(first_addon, 'update_addons',
|
||||
wraps=first_addon.update_addons) as ua_first, \
|
||||
mock.patch.object(AddedAddon, 'update_addons') as ua_added:
|
||||
manager.add(first_addon, {'non-empty': 'dict'})
|
||||
with mock.patch.object(
|
||||
first_addon, "update_addons", wraps=first_addon.update_addons
|
||||
) as ua_first, mock.patch.object(AddedAddon, "update_addons") as ua_added:
|
||||
manager.add(first_addon, {"non-empty": "dict"})
|
||||
manager.update_addons()
|
||||
six.assertCountEqual(self, manager, ['FirstAddon', 'AddedAddon'])
|
||||
ua_first.assert_called_once_with(manager.configs['FirstAddon'],
|
||||
manager)
|
||||
ua_added.assert_called_once_with(manager.configs['AddedAddon'],
|
||||
manager)
|
||||
self.assertCountEqual(manager, ["FirstAddon", "AddedAddon"])
|
||||
ua_first.assert_called_once_with(manager.configs["FirstAddon"], manager)
|
||||
ua_added.assert_called_once_with(manager.configs["AddedAddon"], manager)
|
||||
|
||||
def test_check_dependency_clashes_attributes(self):
|
||||
provides = addons.GoodAddon("ProvidesAddon")
|
||||
provides.provides = ('test', )
|
||||
provides.provides = ("test",)
|
||||
provides2 = addons.GoodAddon("ProvidesAddon2")
|
||||
provides2.provides = ('test', )
|
||||
provides2.provides = ("test",)
|
||||
requires = addons.GoodAddon("RequiresAddon")
|
||||
requires.requires = ('test', )
|
||||
requires.requires = ("test",)
|
||||
requires_name = addons.GoodAddon("RequiresNameAddon")
|
||||
requires_name.requires = ('ProvidesAddon', )
|
||||
requires_name.requires = ("ProvidesAddon",)
|
||||
requires_newer = addons.GoodAddon("RequiresNewerAddon")
|
||||
requires_newer.requires = ('test>=2.0', )
|
||||
requires_newer.requires = ("test>=2.0",)
|
||||
modifies = addons.GoodAddon("ModifiesAddon")
|
||||
modifies.modifies = ('test', )
|
||||
modifies.modifies = ("test",)
|
||||
|
||||
def check_with(*addons):
|
||||
manager = AddonManager()
|
||||
|
|
|
|||
|
|
@ -9,8 +9,10 @@ FROM = "test_addons.addonmod"
|
|||
name = "AddonModule"
|
||||
version = "1.0"
|
||||
|
||||
|
||||
def update_settings(config, settings):
|
||||
pass
|
||||
|
||||
|
||||
def check_configuration(config, crawler):
|
||||
pass
|
||||
|
|
|
|||
|
|
@ -1,18 +1,16 @@
|
|||
import zope.interface
|
||||
|
||||
from scrapy.addons import Addon
|
||||
from scrapy.interfaces import IAddon
|
||||
|
||||
|
||||
class Addon(object):
|
||||
FROM = 'test_addons.addons'
|
||||
FROM = "test_addons.addons"
|
||||
|
||||
|
||||
@zope.interface.declarations.implementer(IAddon)
|
||||
class GoodAddon(object):
|
||||
|
||||
name = 'GoodAddon'
|
||||
version = '1.0'
|
||||
name = "GoodAddon"
|
||||
version = "1.0"
|
||||
|
||||
def __init__(self, name=None, version=None):
|
||||
if name is not None:
|
||||
|
|
@ -32,8 +30,7 @@ class GoodAddon(object):
|
|||
|
||||
@zope.interface.declarations.implementer(IAddon)
|
||||
class BrokenAddon(object):
|
||||
|
||||
name = 'BrokenAddon'
|
||||
name = "BrokenAddon"
|
||||
# No version
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -7,19 +7,17 @@ from scrapy.settings import Settings
|
|||
|
||||
|
||||
class BuiltinAddonsTest(unittest.TestCase):
|
||||
|
||||
def test_make_builtin_addon(self):
|
||||
httpcache = make_builtin_addon('httpcache', {'enabled': True})
|
||||
self.assertEqual(httpcache.name, 'httpcache')
|
||||
self.assertEqual(httpcache.default_config, {'enabled': True})
|
||||
httpcache = make_builtin_addon("httpcache", {"enabled": True})
|
||||
self.assertEqual(httpcache.name, "httpcache")
|
||||
self.assertEqual(httpcache.default_config, {"enabled": True})
|
||||
self.assertEqual(httpcache.version, scrapy.__version__)
|
||||
httpcache = make_builtin_addon('httpcache', {'enabled': True}, '99.9')
|
||||
self.assertEqual(httpcache.version, '99.9')
|
||||
httpcache = make_builtin_addon("httpcache", {"enabled": True}, "99.9")
|
||||
self.assertEqual(httpcache.version, "99.9")
|
||||
|
||||
def test_defaultheaders_export_config(self):
|
||||
settings = Settings()
|
||||
dh = scrapy.addons.defaultheaders()
|
||||
dh.export_config({'X-Test-Header': 'val'}, settings)
|
||||
self.assertIn('X-Test-Header', settings['DEFAULT_REQUEST_HEADERS'])
|
||||
self.assertEqual(settings['DEFAULT_REQUEST_HEADERS']['X-Test-Header'],
|
||||
'val')
|
||||
dh.export_config({"X-Test-Header": "val"}, settings)
|
||||
self.assertIn("X-Test-Header", settings["DEFAULT_REQUEST_HEADERS"])
|
||||
self.assertEqual(settings["DEFAULT_REQUEST_HEADERS"]["X-Test-Header"], "val")
|
||||
|
|
|
|||
|
|
@ -419,7 +419,7 @@ with multiples lines
|
|||
|
||||
addonmgr = AddonManager()
|
||||
addonmgr.add(FailedCheckAddon())
|
||||
crawler = self.runner.create_crawler(SimpleSpider)
|
||||
crawler = get_crawler(SimpleSpider)
|
||||
crawler.addons = addonmgr
|
||||
# Doesn't work in 'precise' test environment:
|
||||
# with self.assertRaises(ValueError):
|
||||
|
|
|
|||
|
|
@ -90,14 +90,12 @@ class MiddlewareManagerTest(unittest.TestCase):
|
|||
|
||||
def test_instances_from_settings(self):
|
||||
settings = Settings()
|
||||
myM3 = M3()
|
||||
|
||||
class InstanceTestMiddlewareManager(MiddlewareManager):
|
||||
@classmethod
|
||||
def _get_mwlist_from_settings(cls, settings):
|
||||
return ["tests.test_middleware.M1", M2, myM3]
|
||||
return ["tests.test_middleware.M1", M2]
|
||||
|
||||
mwman = InstanceTestMiddlewareManager.from_settings(settings)
|
||||
self.assertIsInstance(mwman.middlewares[0], M1)
|
||||
self.assertIsInstance(mwman.middlewares[1], M2)
|
||||
self.assertIs(mwman.middlewares[2], myM3)
|
||||
|
|
|
|||
Loading…
Reference in New Issue