diff --git a/docs/topics/addons.rst b/docs/topics/addons.rst index 4dab15a2a..ba6e839a5 100644 --- a/docs/topics/addons.rst +++ b/docs/topics/addons.rst @@ -350,29 +350,31 @@ Check configuration of fully initialized crawler (see Provide add-on interface through a module: -.. No idea why just using '::' doesn't work for this one .. code-block:: python - name = 'AddonModule' - version = '1.0' + name = "AddonModule" + version = "1.0" + class MyPipeline(object): - # ... + ... + class MyDownloaderMiddleware(object): - # ... + ... + def update_settings(config, settings): settings.set( - 'ITEM_PIPELINES', + "ITEM_PIPELINES", {MyPipeline(): 200}, - priority='addon', - } + priority="addon", + ) settings.set( - 'DOWNLOADER_MIDDLEWARES', + "DOWNLOADER_MIDDLEWARES", {MyDownloaderMiddleware(): 800}, - priority='addon', - } + priority="addon", + ) Forward to other add-ons depending on Python version:: diff --git a/scrapy/addons/__init__.py b/scrapy/addons/__init__.py index 169c79eac..aad9ef789 100644 --- a/scrapy/addons/__init__.py +++ b/scrapy/addons/__init__.py @@ -1,10 +1,10 @@ -from collections import defaultdict, Mapping, OrderedDict -from inspect import isclass -import six import warnings +from collections import OrderedDict, defaultdict +from collections.abc import Mapping +from inspect import isclass -from pkg_resources import WorkingSet, Distribution, Requirement import zope.interface +from pkg_resources import Distribution, Requirement, WorkingSet from zope.interface.verify import verifyObject from scrapy.interfaces import IAddon @@ -14,7 +14,6 @@ from scrapy.utils.misc import load_module_or_object @zope.interface.implementer(IAddon) class Addon(object): - basic_settings = None """``dict`` of settings that will be exported via :meth:`export_basics`.""" @@ -83,8 +82,8 @@ class Addon(object): else: # e.g. for DOWNLOADER_MIDDLEWARES: {'myclass': 100} k = comp - v = config.get('order', self.component_order) - settings[self.component_type].update({k: v}, 'addon') + v = config.get("order", self.component_order) + settings[self.component_type].update({k: v}, "addon") def export_basics(self, settings): """Export the :attr:`basic_settings` attribute into the settings object. @@ -95,8 +94,8 @@ class Addon(object): :param settings: Settings object into which to expose the basic settings :type settings: :class:`~scrapy.settings.Settings` """ - for setting, value in six.iteritems(self.basic_settings or {}): - settings.set(setting, value, 'addon') + for setting, value in (self.basic_settings or {}).items(): + settings.set(setting, value, "addon") def export_config(self, config, settings): """Export the add-on configuration, all keys in caps and with @@ -121,14 +120,13 @@ class Addon(object): prefix = self.settings_prefix or self.name # Since default exported config is case-insensitive (everything will be # uppercased), make mapped config case-insensitive as well - conf_mapping = {k.lower(): v - for k, v in six.iteritems(self.config_mapping or {})} - for key, val in six.iteritems(conf): + conf_mapping = {k.lower(): v for k, v in (self.config_mapping or {}).items()} + for key, val in conf.items(): if key.lower() in conf_mapping: key = conf_mapping[key.lower()] else: - key = (prefix + '_' + key).upper() - settings.set(key, val, 'addon') + key = (prefix + "_" + key).upper() + settings.set(key, val, "addon") def update_settings(self, config, settings): """Export both the basic settings and the add-on configuration. I.e., @@ -210,11 +208,11 @@ class AddonManager(Mapping): verifyObject(IAddon, addon) name = addon.name if name in self: - raise ValueError("Addon '{}' already loaded".format(name)) + raise ValueError(f"Addon '{name}' already loaded") self._addons[name] = addon self.configs[name] = config or {} if name in self._disable_on_add: - self.configs[name]['_enabled'] = False + self.configs[name]["_enabled"] = False self._disable_on_add.remove(name) def remove(self, addon): @@ -229,7 +227,7 @@ class AddonManager(Mapping): """ if addon in self: del self[addon] - elif hasattr(addon, 'name') and addon.name in self: + elif hasattr(addon, "name") and addon.name in self: del self[addon.name] else: try: @@ -250,14 +248,14 @@ class AddonManager(Mapping): :param path: Python or file path to an add-on :type path: ``str`` """ - if isinstance(path, six.string_types): + if isinstance(path, str): try: obj = load_module_or_object(path) except NameError: - raise NameError("Could not find add-on '%s'" % path) + raise NameError(f"Could not find add-on '{path}'") else: obj = path - if hasattr(obj, '_addon'): + if hasattr(obj, "_addon"): obj = AddonManager.get_addon(obj._addon) return obj @@ -284,7 +282,7 @@ class AddonManager(Mapping): and values correspond to their configuration :type addonsdict: ``dict`` """ - for addonpath, addoncfg in six.iteritems(addonsdict): + for addonpath, addoncfg in addonsdict.items(): self.add(addonpath, addoncfg) def load_settings(self, settings): @@ -299,7 +297,7 @@ class AddonManager(Mapping): which to read the add-on configuration :type settings: :class:`~scrapy.settings.Settings` """ - paths = build_component_list(settings['ADDONS']) + paths = build_component_list(settings["ADDONS"]) addons = [self.get_addon(path) for path in paths] configs = [settings.getdict(addon.name.upper()) for addon in addons] for a, c in zip(addons, configs): @@ -322,20 +320,23 @@ class AddonManager(Mapping): add-on. """ # Collect all active add-ons and the components they provide - ws = WorkingSet('') + ws = WorkingSet("") def add_dist(project_name, version, **kwargs): - if project_name in ws.entry_keys.get('scrapy', []): - raise ImportError("Component {} provided by multiple add-ons" - "".format(project_name)) + if project_name in ws.entry_keys.get("scrapy", []): + raise ImportError( + f"Component {project_name} provided by multiple add-ons" + ) else: - dist = Distribution(project_name=project_name, version=version, - **kwargs) - ws.add(dist, entry='scrapy') + dist = Distribution( + project_name=project_name, version=version, **kwargs + ) + ws.add(dist, entry="scrapy") + for name in self: ver = self[name].version add_dist(name, ver) - for provides_name in getattr(self[name], 'provides', []): + for provides_name in getattr(self[name], "provides", []): add_dist(provides_name, ver) # Collect all required and modified components @@ -345,8 +346,9 @@ class AddonManager(Mapping): for entry in getattr(self[name], attribute_name, []): attrs[entry].append(name) return attrs - modified = compile_attribute_dict('modifies') - required = compile_attribute_dict('requires') + + modified = compile_attribute_dict("modifies") + required = compile_attribute_dict("requires") req_or_mod = set(required.keys()).union(modified.keys()) for reqstr in req_or_mod: @@ -355,15 +357,16 @@ class AddonManager(Mapping): # our own exception or is it helpful enough? if ws.find(req) is None: raise ImportError( - "Add-ons {} require or modify missing component {}" - "".format(required[reqstr]+modified[reqstr], reqstr) + f"Add-ons {required[reqstr] + modified[reqstr]} require" + f" or modify missing component {reqstr}" ) mod_and_req = set(required.keys()).intersection(modified.keys()) for conflict in mod_and_req: - warnings.warn("Component '{}', required by add-ons {}, is modified " - "by add-ons {}".format(conflict, required[conflict], - modified[conflict])) + warnings.warn( + f"Component '{conflict}', required by add-ons {required[conflict]}," + f" is modified by add-ons {modified[conflict]}" + ) def disable(self, addon): """Disable an add-on, i.e. prevent its callbacks from being called. @@ -375,7 +378,7 @@ class AddonManager(Mapping): :type addon: ``str`` """ if addon in self: - self.configs[addon]['_enabled'] = False + self.configs[addon]["_enabled"] = False else: self._disable_on_add.append(addon) @@ -389,23 +392,23 @@ class AddonManager(Mapping): :type addon: ``str`` """ if addon in self: - self.configs[addon]['_enabled'] = True + self.configs[addon]["_enabled"] = True elif addon in self._disable_on_add: self._disable_on_add.remove(addon) else: - raise ValueError("Add-ons need to be added before they can be " - "enabled") + raise ValueError("Add-ons need to be added before they can be " "enabled") @property def disabled(self): """Names of disabled add-ons""" - return ([a for a in self if not self.configs[a].get('_enabled', True)] + - self._disable_on_add) + return [ + a for a in self if not self.configs[a].get("_enabled", True) + ] + self._disable_on_add @property def enabled(self): """Names of enabled add-ons""" - return [a for a in self if self.configs[a].get('_enabled', True)] + return [a for a in self if self.configs[a].get("_enabled", True)] def _call_if_exists(self, obj, cbname, *args, **kwargs): if obj is None: @@ -418,9 +421,10 @@ class AddonManager(Mapping): cb(*args, **kwargs) def _call_addon(self, addonname, cbname, *args, **kwargs): - if self.configs[addonname].get('_enabled', True): - self._call_if_exists(self[addonname], cbname, - self.configs[addonname], *args, **kwargs) + if self.configs[addonname].get("_enabled", True): + self._call_if_exists( + self[addonname], cbname, self.configs[addonname], *args, **kwargs + ) def update_addons(self): """Call ``update_addons()`` of all held add-ons. @@ -432,7 +436,7 @@ class AddonManager(Mapping): while called_addons != set(self): for name in set(self).difference(called_addons): called_addons.add(name) - self._call_addon(name, 'update_addons', self) + self._call_addon(name, "update_addons", self) def update_settings(self, settings): """Call ``update_settings()`` of all held add-ons. @@ -442,7 +446,7 @@ class AddonManager(Mapping): :type settings: :class:`~scrapy.settings.Settings` """ for name in self: - self._call_addon(name, 'update_settings', settings) + self._call_addon(name, "update_settings", settings) def check_configuration(self, crawler): """Call ``check_configuration()`` of all held add-ons. @@ -451,7 +455,7 @@ class AddonManager(Mapping): :type crawler: :class:`~scrapy.crawler.Crawler` """ for name in self: - self._call_addon(name, 'check_configuration', crawler) + self._call_addon(name, "check_configuration", crawler) -from scrapy.addons.builtins import * +from scrapy.addons.builtins import * # noqa diff --git a/scrapy/addons/builtins.py b/scrapy/addons/builtins.py index 9babdeb6f..ea3afbf99 100644 --- a/scrapy/addons/builtins.py +++ b/scrapy/addons/builtins.py @@ -1,23 +1,44 @@ import scrapy from scrapy.addons import Addon -__all__ = ['make_builtin_addon', - - 'depth', 'httperror', 'offsite', 'referer', 'urllength', - - 'ajaxcrawl', 'chunked', 'cookies', 'defaultheaders', - 'downloadtimeout', 'httpauth', 'httpcache', 'httpcompression', - 'httpproxy', 'metarefresh', 'redirect', 'retry', 'robotstxt', - 'stats', 'useragent', - - 'autothrottle', 'corestats', 'closespider', 'debugger', 'feedexport', - 'logstats', 'memdebug', 'memusage', 'spiderstate', 'stacktracedump', - 'statsmailer', 'telnetconsole', - ] +__all__ = [ + "make_builtin_addon", + "depth", + "httperror", + "offsite", + "referer", + "urllength", + "ajaxcrawl", + "chunked", + "cookies", + "defaultheaders", + "downloadtimeout", + "httpauth", + "httpcache", + "httpcompression", + "httpproxy", + "metarefresh", + "redirect", + "retry", + "robotstxt", + "stats", + "useragent", + "autothrottle", + "corestats", + "closespider", + "debugger", + "feedexport", + "logstats", + "memdebug", + "memusage", + "spiderstate", + "stacktracedump", + "statsmailer", + "telnetconsole", +] -def make_builtin_addon(addon_name, addon_default_config=None, - addon_version=None): +def make_builtin_addon(addon_name, addon_default_config=None, addon_version=None): class ThisAddon(Addon): name = addon_name version = addon_version or scrapy.__version__ @@ -33,59 +54,65 @@ def make_builtin_addon(addon_name, addon_default_config=None, # SPIDER MIDDLEWARES -depth = make_builtin_addon('depth') +depth = make_builtin_addon("depth") -httperror = make_builtin_addon('httperror') +httperror = make_builtin_addon("httperror") -offsite = make_builtin_addon('offsite') +offsite = make_builtin_addon("offsite") -referer = make_builtin_addon('referer') +referer = make_builtin_addon("referer") -urllength = make_builtin_addon('urllength') +urllength = make_builtin_addon("urllength") # DOWNLOADER MIDDLEWARES -ajaxcrawl = make_builtin_addon('ajaxcrawl', {'enabled': True}) +ajaxcrawl = make_builtin_addon("ajaxcrawl", {"enabled": True}) -chunked = make_builtin_addon('chunked') +chunked = make_builtin_addon("chunked") + +cookies = make_builtin_addon("cookies") + +defaultheaders = make_builtin_addon("defaultheaders") -cookies = make_builtin_addon('cookies') -defaultheaders = make_builtin_addon('defaultheaders') # Assume every config entry is a header def defaultheaders_export_config(self, config, settings): conf = self.default_config or {} conf.update(config) - settings.set('DEFAULT_REQUEST_HEADERS', conf, 'addon') + settings.set("DEFAULT_REQUEST_HEADERS", conf, "addon") + + defaultheaders.export_config = defaultheaders_export_config -downloadtimeout = make_builtin_addon('downloadtimeout') -downloadtimeout.config_mapping = {'timeout': 'DOWNLOAD_TIMEOUT', - 'download_timeout': 'DOWNLOAD_TIMEOUT'} +downloadtimeout = make_builtin_addon("downloadtimeout") +downloadtimeout.config_mapping = { + "timeout": "DOWNLOAD_TIMEOUT", + "download_timeout": "DOWNLOAD_TIMEOUT", +} -httpauth = make_builtin_addon('httpauth') +httpauth = make_builtin_addon("httpauth") -httpcache = make_builtin_addon('httpcache', {'enabled': True}) +httpcache = make_builtin_addon("httpcache", {"enabled": True}) -httpcompression = make_builtin_addon('httpcompression') -httpcompression.config_mapping = {'enabled': 'COMPRESSION_ENABLED'} +httpcompression = make_builtin_addon("httpcompression") +httpcompression.config_mapping = {"enabled": "COMPRESSION_ENABLED"} -httpproxy = make_builtin_addon('httpproxy') +httpproxy = make_builtin_addon("httpproxy") -metarefresh = make_builtin_addon('metarefresh') -metarefresh.config_mapping = {'max_times': 'REDIRECT_MAX_TIMES'} +metarefresh = make_builtin_addon("metarefresh") +metarefresh.config_mapping = {"max_times": "REDIRECT_MAX_TIMES"} -redirect = make_builtin_addon('redirect') +redirect = make_builtin_addon("redirect") -retry = make_builtin_addon('retry') +retry = make_builtin_addon("retry") -robotstxt = make_builtin_addon('robotstxt', {'obey': True}) +robotstxt = make_builtin_addon("robotstxt", {"obey": True}) -stats = make_builtin_addon('stats') +stats = make_builtin_addon("stats") -useragent = make_builtin_addon('useragent') -useragent.config_mapping = {'user_agent': 'USER_AGENT'} +useragent = make_builtin_addon("useragent") +useragent.config_mapping = {"user_agent": "USER_AGENT"} # ITEM PIPELINES @@ -93,27 +120,27 @@ useragent.config_mapping = {'user_agent': 'USER_AGENT'} # EXTENSIONS -autothrottle = make_builtin_addon('autothrottle', {'enabled': True}) +autothrottle = make_builtin_addon("autothrottle", {"enabled": True}) -corestats = make_builtin_addon('corestats') +corestats = make_builtin_addon("corestats") -closespider = make_builtin_addon('closespider') +closespider = make_builtin_addon("closespider") -debugger = make_builtin_addon('debugger') +debugger = make_builtin_addon("debugger") -feedexport = make_builtin_addon('feedexport') -feedexport.settings_prefix = 'FEED' +feedexport = make_builtin_addon("feedexport") +feedexport.settings_prefix = "FEED" -logstats = make_builtin_addon('logstats') +logstats = make_builtin_addon("logstats") -memdebug = make_builtin_addon('memdebug', {'enabled': True}) +memdebug = make_builtin_addon("memdebug", {"enabled": True}) -memusage = make_builtin_addon('memusage', {'enabled': True}) +memusage = make_builtin_addon("memusage", {"enabled": True}) -spiderstate = make_builtin_addon('spiderstate') +spiderstate = make_builtin_addon("spiderstate") -stacktracedump = make_builtin_addon('stacktracedump') +stacktracedump = make_builtin_addon("stacktracedump") -statsmailer = make_builtin_addon('statsmailer') +statsmailer = make_builtin_addon("statsmailer") -telnetconsole = make_builtin_addon('telnetconsole') +telnetconsole = make_builtin_addon("telnetconsole") diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index 4bf7e7e66..8577cce02 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -85,7 +85,7 @@ def load_module_or_object(path): return load_object(path) except (ValueError, NameError, ImportError): pass - raise NameError("Could not load '%s'" % path) + raise NameError(f"Could not load '{path}'") def walk_modules(path): diff --git a/tests/test_addons/__init__.py b/tests/test_addons/__init__.py index a4e278fa5..741dd81cf 100644 --- a/tests/test_addons/__init__.py +++ b/tests/test_addons/__init__.py @@ -1,145 +1,143 @@ -from collections import OrderedDict import itertools -import os.path -import six -from tests import mock import unittest import warnings +from collections import OrderedDict +from unittest import mock from pkg_resources import VersionConflict -import zope.interface +from zope.interface import directlyProvides +from zope.interface.exceptions import BrokenImplementation, MultipleInvalid from zope.interface.verify import verifyObject -from zope.interface.exceptions import BrokenImplementation from scrapy.addons import Addon, AddonManager from scrapy.crawler import Crawler from scrapy.interfaces import IAddon from scrapy.settings import BaseSettings -from . import addons -from . import addonmod +from . import addonmod, addons class AddonTest(unittest.TestCase): - def setUp(self): self.rawaddon = Addon() class AddonWithAttributes(Addon): - name = 'Test' - version = '1.0' + name = "Test" + version = "1.0" + self.testaddon = AddonWithAttributes() def test_interface(self): # Raw Addon should fail exactly b/c name and version are not given - self.assertFalse(hasattr(self.rawaddon, 'name')) - self.assertFalse(hasattr(self.rawaddon, 'version')) - self.assertRaises(BrokenImplementation, verifyObject, IAddon, - self.rawaddon) + self.assertFalse(hasattr(self.rawaddon, "name")) + self.assertFalse(hasattr(self.rawaddon, "version")) + self.assertRaises(MultipleInvalid, verifyObject, IAddon, self.rawaddon) verifyObject(IAddon, self.testaddon) def test_export_component(self): - settings = BaseSettings({'ITEM_PIPELINES': BaseSettings(), - 'DOWNLOAD_HANDLERS': BaseSettings()}, - 'default') + settings = BaseSettings( + {"ITEM_PIPELINES": BaseSettings(), "DOWNLOAD_HANDLERS": BaseSettings()}, + "default", + ) self.testaddon.component_type = None self.testaddon.export_component({}, settings) - self.assertEqual(len(settings['ITEM_PIPELINES']), 0) - self.testaddon.component_type = 'ITEM_PIPELINES' - self.testaddon.component = 'test.component' + self.assertEqual(len(settings["ITEM_PIPELINES"]), 0) + self.testaddon.component_type = "ITEM_PIPELINES" + self.testaddon.component = "test.component" self.testaddon.export_component({}, settings) - six.assertCountEqual(self, settings['ITEM_PIPELINES'], - ['test.component']) - self.assertEqual(settings['ITEM_PIPELINES']['test.component'], 0) + self.assertCountEqual(settings["ITEM_PIPELINES"], ["test.component"]) + self.assertEqual(settings["ITEM_PIPELINES"]["test.component"], 0) self.testaddon.component_order = 313 self.testaddon.export_component({}, settings) - self.assertEqual(settings['ITEM_PIPELINES']['test.component'], 313) - self.testaddon.component_type = 'DOWNLOAD_HANDLERS' - self.testaddon.component_key = 'http' + self.assertEqual(settings["ITEM_PIPELINES"]["test.component"], 313) + self.testaddon.component_type = "DOWNLOAD_HANDLERS" + self.testaddon.component_key = "http" self.testaddon.export_component({}, settings) - self.assertEqual(settings['DOWNLOAD_HANDLERS']['http'], - 'test.component') + self.assertEqual(settings["DOWNLOAD_HANDLERS"]["http"], "test.component") def test_export_basics(self): settings = BaseSettings() - self.testaddon.basic_settings = {'TESTKEY': 313, 'OTHERKEY': True} + self.testaddon.basic_settings = {"TESTKEY": 313, "OTHERKEY": True} self.testaddon.export_basics(settings) - self.assertEqual(settings['TESTKEY'], 313) - self.assertEqual(settings['OTHERKEY'], True) - self.assertEqual(settings.getpriority('TESTKEY'), 15) + self.assertEqual(settings["TESTKEY"], 313) + self.assertEqual(settings["OTHERKEY"], True) + self.assertEqual(settings.getpriority("TESTKEY"), 15) def test_export_config(self): settings = BaseSettings() self.testaddon.settings_prefix = None - self.testaddon.config_mapping = {'MAPPED_key': 'MAPPING_WORKED'} - self.testaddon.default_config = {'key': 55, 'defaultkey': 100} - self.testaddon.export_config({'key': 313, 'OTHERKEY': True, - 'mapped_KEY': 99}, settings) - self.assertEqual(settings['TEST_KEY'], 313) - self.assertEqual(settings['TEST_DEFAULTKEY'], 100) - self.assertEqual(settings['TEST_OTHERKEY'], True) - self.assertNotIn('MAPPED_key', settings) - self.assertNotIn('MAPPED_KEY', settings) - self.assertEqual(settings['MAPPING_WORKED'], 99) - self.assertEqual(settings.getpriority('TEST_KEY'), 15) + self.testaddon.config_mapping = {"MAPPED_key": "MAPPING_WORKED"} + self.testaddon.default_config = {"key": 55, "defaultkey": 100} + self.testaddon.export_config( + {"key": 313, "OTHERKEY": True, "mapped_KEY": 99}, settings + ) + self.assertEqual(settings["TEST_KEY"], 313) + self.assertEqual(settings["TEST_DEFAULTKEY"], 100) + self.assertEqual(settings["TEST_OTHERKEY"], True) + self.assertNotIn("MAPPED_key", settings) + self.assertNotIn("MAPPED_KEY", settings) + self.assertEqual(settings["MAPPING_WORKED"], 99) + self.assertEqual(settings.getpriority("TEST_KEY"), 15) - self.testaddon.settings_prefix = 'PREF' - self.testaddon.export_config({'newkey': 99}, settings) - self.assertEqual(settings['PREF_NEWKEY'], 99) + self.testaddon.settings_prefix = "PREF" + self.testaddon.export_config({"newkey": 99}, settings) + self.assertEqual(settings["PREF_NEWKEY"], 99) - with mock.patch.object(settings, 'set') as mock_set: + with mock.patch.object(settings, "set") as mock_set: self.testaddon.settings_prefix = False - self.testaddon.export_config({'thirdnewkey': 99}, settings) + self.testaddon.export_config({"thirdnewkey": 99}, settings) self.assertEqual(mock_set.call_count, 0) def test_update_settings(self): settings = BaseSettings() - settings.set('TEST_KEY1', 'default', priority='default') - settings.set('TEST_KEY2', 'project', priority='project') + settings.set("TEST_KEY1", "default", priority="default") + settings.set("TEST_KEY2", "project", priority="project") self.testaddon.settings_prefix = None - self.testaddon.basic_settings = {'OTHERTEST_KEY': 'addon'} - addon_config = {'key1': 'addon', 'key2': 'addon', 'key3': 'addon'} + self.testaddon.basic_settings = {"OTHERTEST_KEY": "addon"} + addon_config = {"key1": "addon", "key2": "addon", "key3": "addon"} self.testaddon.update_settings(addon_config, settings) - self.assertEqual(settings['OTHERTEST_KEY'], 'addon') - self.assertEqual(settings['TEST_KEY1'], 'addon') - self.assertEqual(settings['TEST_KEY2'], 'project') - self.assertEqual(settings['TEST_KEY3'], 'addon') + self.assertEqual(settings["OTHERTEST_KEY"], "addon") + self.assertEqual(settings["TEST_KEY1"], "addon") + self.assertEqual(settings["TEST_KEY2"], "project") + self.assertEqual(settings["TEST_KEY3"], "addon") class AddonManagerTest(unittest.TestCase): - def setUp(self): self.manager = AddonManager() def test_add(self): manager = AddonManager() - manager.add(addonmod, {'key': 'val1'}) - manager.add('tests.test_addons.addons.GoodAddon') - six.assertCountEqual(self, manager, ['AddonModule', 'GoodAddon']) - self.assertIsInstance(manager['GoodAddon'], addons.GoodAddon) - six.assertCountEqual(self, manager.configs['AddonModule'], ['key']) - self.assertEqual(manager.configs['AddonModule']['key'], 'val1') + manager.add(addonmod, {"key": "val1"}) + manager.add("tests.test_addons.addons.GoodAddon") + self.assertCountEqual(manager, ["AddonModule", "GoodAddon"]) + self.assertIsInstance(manager["GoodAddon"], addons.GoodAddon) + self.assertCountEqual(manager.configs["AddonModule"], ["key"]) + self.assertEqual(manager.configs["AddonModule"]["key"], "val1") self.assertRaises(ValueError, manager.add, addonmod) def test_add_dont_instantiate_providing_classes(self): class ProviderGoodAddon(addons.GoodAddon): pass - zope.interface.directlyProvides(ProviderGoodAddon, IAddon) + + directlyProvides(ProviderGoodAddon, IAddon) manager = AddonManager() manager.add(ProviderGoodAddon) - self.assertIs(manager['GoodAddon'], ProviderGoodAddon) + self.assertIs(manager["GoodAddon"], ProviderGoodAddon) def test_add_verifies(self): - brokenaddon = self.manager.get_addon( - 'tests.test_addons.addons.BrokenAddon') - self.assertRaises(zope.interface.exceptions.BrokenImplementation, - self.manager.add, - brokenaddon) + brokenaddon = self.manager.get_addon("tests.test_addons.addons.BrokenAddon") + self.assertRaises( + BrokenImplementation, + self.manager.add, + brokenaddon, + ) def test_add_adds_missing_interface_declaration(self): class GoodAddonWithoutDeclaration(object): - name = 'GoodAddonWithoutDeclaration' - version = '1.0' + name = "GoodAddonWithoutDeclaration" + version = "1.0" + self.manager.add(GoodAddonWithoutDeclaration) def test_remove(self): @@ -147,64 +145,66 @@ class AddonManagerTest(unittest.TestCase): def test_gets_removed(removearg): manager.add(addonmod) - self.assertIn('AddonModule', manager) + self.assertIn("AddonModule", manager) manager.remove(removearg) - self.assertNotIn('AddonModule', manager) + self.assertNotIn("AddonModule", manager) - test_gets_removed('AddonModule') + test_gets_removed("AddonModule") test_gets_removed(addonmod) - test_gets_removed('tests.test_addons.addonmod') - self.assertRaises(KeyError, manager.remove, 'nonexistent') + test_gets_removed("tests.test_addons.addonmod") + self.assertRaises(KeyError, manager.remove, "nonexistent") self.assertRaises(KeyError, manager.remove, addons.GoodAddon()) def test_get_addon(self): - goodaddon = self.manager.get_addon('tests.test_addons.addons.GoodAddon') + goodaddon = self.manager.get_addon("tests.test_addons.addons.GoodAddon") self.assertIs(goodaddon, addons.GoodAddon) loaded_addonmod = self.manager.get_addon("tests.test_addons.addonmod") self.assertIs(loaded_addonmod, addonmod) - addonspath = os.path.join(os.path.dirname(__file__), 'addons.py') goodaddon = self.manager.get_addon("tests.test_addons.addons") self.assertIsInstance(goodaddon, addons.GoodAddon) - self.assertRaises(NameError, self.manager.get_addon, 'xy.n_onexistent') + self.assertRaises(NameError, self.manager.get_addon, "xy.n_onexistent") def test_get_addon_forward(self): class SomeCls(object): - _addon = 'tests.test_addons.addons.GoodAddon' + _addon = "tests.test_addons.addons.GoodAddon" + self.assertIs(self.manager.get_addon(SomeCls()), addons.GoodAddon) def test_get_addon_nested(self): - x = addons.GoodAddon('outer') - x._addon = addons.GoodAddon('middle') - x._addon._addon = addons.GoodAddon('inner') + x = addons.GoodAddon("outer") + x._addon = addons.GoodAddon("middle") + x._addon._addon = addons.GoodAddon("inner") self.assertIs(self.manager.get_addon(x), x._addon._addon) def test_load_dict_load_settings(self): def _test_load_method(func, *args, **kwargs): manager = AddonManager() getattr(manager, func)(*args, **kwargs) - six.assertCountEqual(self, manager, ['GoodAddon', 'AddonModule']) - self.assertIsInstance(manager['GoodAddon'], addons.GoodAddon) - six.assertCountEqual(self, manager.configs['GoodAddon'], ['key']) - self.assertEqual(manager.configs['GoodAddon']['key'], 'val2') - self.assertEqual(manager['AddonModule'], addonmod) - self.assertIn('key', manager.configs['AddonModule']) - self.assertEqual(manager.configs['AddonModule']['key'], 'val1') + self.assertCountEqual(manager, ["GoodAddon", "AddonModule"]) + self.assertIsInstance(manager["GoodAddon"], addons.GoodAddon) + self.assertCountEqual(manager.configs["GoodAddon"], ["key"]) + self.assertEqual(manager.configs["GoodAddon"]["key"], "val2") + self.assertEqual(manager["AddonModule"], addonmod) + self.assertIn("key", manager.configs["AddonModule"]) + self.assertEqual(manager.configs["AddonModule"]["key"], "val1") addonsdict = { - "tests.test_addons.addonmod": {'key': 'val1'}, - 'tests.test_addons.addons.GoodAddon': {'key': 'val2'}, - } - _test_load_method('load_dict', addonsdict) + "tests.test_addons.addonmod": {"key": "val1"}, + "tests.test_addons.addons.GoodAddon": {"key": "val2"}, + } + _test_load_method("load_dict", addonsdict) settings = BaseSettings() - settings.set('ADDONS', {"tests.test_addons.addonmod": 0, - 'tests.test_addons.addons.GoodAddon': 0}) - settings.set('ADDONMODULE', {'key': 'val1'}) - settings.set('GOODADDON', {'key': 'val2'}) - _test_load_method('load_settings', settings) + settings.set( + "ADDONS", + {"tests.test_addons.addonmod": 0, "tests.test_addons.addons.GoodAddon": 0}, + ) + settings.set("ADDONMODULE", {"key": "val1"}) + settings.set("GOODADDON", {"key": "val2"}) + _test_load_method("load_settings", settings) def test_load_dict_load_settings_order(self): def _test_load_method(expected_order, func, *args, **kwargs): @@ -218,72 +218,71 @@ class AddonManagerTest(unittest.TestCase): for ordered_addons in itertools.permutations(addonlist): expected_order = [a.name for a in ordered_addons] addonsdict = OrderedDict((a, {}) for a in ordered_addons) - _test_load_method(expected_order, 'load_dict', addonsdict) - settings = BaseSettings({ - 'ADDONS': {a: i for i, a in enumerate(ordered_addons)} - }) - _test_load_method(expected_order, 'load_settings', settings) + _test_load_method(expected_order, "load_dict", addonsdict) + settings = BaseSettings( + {"ADDONS": {a: i for i, a in enumerate(ordered_addons)}} + ) + _test_load_method(expected_order, "load_settings", settings) def test_enabled_disabled(self): manager = AddonManager() - manager.add(addons.GoodAddon('FirstAddon')) - manager.add(addons.GoodAddon('SecondAddon')) - self.assertEqual(set(manager.enabled), - set(('FirstAddon', 'SecondAddon'))) + manager.add(addons.GoodAddon("FirstAddon")) + manager.add(addons.GoodAddon("SecondAddon")) + self.assertEqual(set(manager.enabled), set(("FirstAddon", "SecondAddon"))) self.assertEqual(manager.disabled, []) - manager.disable('FirstAddon') - self.assertEqual(manager.enabled, ['SecondAddon']) - self.assertEqual(manager.disabled, ['FirstAddon']) - manager.enable('FirstAddon') - self.assertEqual(set(manager.enabled), - set(('FirstAddon', 'SecondAddon'))) + manager.disable("FirstAddon") + self.assertEqual(manager.enabled, ["SecondAddon"]) + self.assertEqual(manager.disabled, ["FirstAddon"]) + manager.enable("FirstAddon") + self.assertEqual(set(manager.enabled), set(("FirstAddon", "SecondAddon"))) self.assertEqual(manager.disabled, []) def test_enable_before_add(self): manager = AddonManager() - self.assertRaises(ValueError, manager.enable, 'FirstAddon') - manager.disable('FirstAddon') - manager.enable('FirstAddon') - manager.add(addons.GoodAddon('FirstAddon')) - self.assertIn('FirstAddon', manager.enabled) + self.assertRaises(ValueError, manager.enable, "FirstAddon") + manager.disable("FirstAddon") + manager.enable("FirstAddon") + manager.add(addons.GoodAddon("FirstAddon")) + self.assertIn("FirstAddon", manager.enabled) def test_disable_before_add(self): manager = AddonManager() - manager.disable('FirstAddon') - manager.add(addons.GoodAddon('FirstAddon')) - self.assertEqual(manager.disabled, ['FirstAddon']) + manager.disable("FirstAddon") + manager.add(addons.GoodAddon("FirstAddon")) + self.assertEqual(manager.disabled, ["FirstAddon"]) def test_callbacks(self): - first_addon = addons.GoodAddon('FirstAddon') - second_addon = addons.GoodAddon('SecondAddon') + first_addon = addons.GoodAddon("FirstAddon") + second_addon = addons.GoodAddon("SecondAddon") manager = AddonManager() - manager.add(first_addon, {'test': 'first'}) - manager.add(second_addon, {'test': 'second'}) + manager.add(first_addon, {"test": "first"}) + manager.add(second_addon, {"test": "second"}) crawler = mock.create_autospec(Crawler) settings = BaseSettings() - with mock.patch.object(first_addon, 'update_addons') as ua_first, \ - mock.patch.object(second_addon, 'update_addons') as ua_second, \ - mock.patch.object(first_addon, 'update_settings') as us_first, \ - mock.patch.object(second_addon, 'update_settings') as us_second, \ - mock.patch.object(first_addon, 'check_configuration') as cc_first, \ - mock.patch.object(second_addon, 'check_configuration') as cc_second: + with mock.patch.object( + first_addon, "update_addons" + ) as ua_first, mock.patch.object( + second_addon, "update_addons" + ) as ua_second, mock.patch.object( + first_addon, "update_settings" + ) as us_first, mock.patch.object( + second_addon, "update_settings" + ) as us_second, mock.patch.object( + first_addon, "check_configuration" + ) as cc_first, mock.patch.object( + second_addon, "check_configuration" + ) as cc_second: manager.update_addons() - ua_first.assert_called_once_with(manager.configs['FirstAddon'], - manager) - ua_second.assert_called_once_with(manager.configs['SecondAddon'], - manager) + ua_first.assert_called_once_with(manager.configs["FirstAddon"], manager) + ua_second.assert_called_once_with(manager.configs["SecondAddon"], manager) manager.update_settings(settings) - us_first.assert_called_once_with(manager.configs['FirstAddon'], - settings) - us_second.assert_called_once_with(manager.configs['SecondAddon'], - settings) + us_first.assert_called_once_with(manager.configs["FirstAddon"], settings) + us_second.assert_called_once_with(manager.configs["SecondAddon"], settings) manager.check_configuration(crawler) - cc_first.assert_called_once_with(manager.configs['FirstAddon'], - crawler) - cc_second.assert_called_once_with(manager.configs['SecondAddon'], - crawler) + cc_first.assert_called_once_with(manager.configs["FirstAddon"], crawler) + cc_second.assert_called_once_with(manager.configs["SecondAddon"], crawler) self.assertEqual(ua_first.call_count, 1) self.assertEqual(ua_second.call_count, 1) self.assertEqual(us_first.call_count, 1) @@ -291,10 +290,10 @@ class AddonManagerTest(unittest.TestCase): us_first.reset_mock() us_second.reset_mock() - manager.disable('FirstAddon') + manager.disable("FirstAddon") manager.update_settings(settings) self.assertEqual(us_first.call_count, 0) - manager.enable('FirstAddon') + manager.enable("FirstAddon") manager.update_settings(settings) self.assertEqual(us_first.call_count, 1) self.assertEqual(us_second.call_count, 2) @@ -302,44 +301,42 @@ class AddonManagerTest(unittest.TestCase): # This will become relevant when we let spiders implement the add-on # interface and should be replaced with a test where # AddonManager.spidercls = None then. - manager._call_if_exists(None, 'irrelevant') + manager._call_if_exists(None, "irrelevant") def test_update_addons_last_minute_add(self): class AddedAddon(addons.GoodAddon): - name = 'AddedAddon' + name = "AddedAddon" class FirstAddon(addons.GoodAddon): - name = 'FirstAddon' + name = "FirstAddon" def update_addons(self, config, addons): addons.add(AddedAddon()) manager = AddonManager() first_addon = FirstAddon() - with mock.patch.object(first_addon, 'update_addons', - wraps=first_addon.update_addons) as ua_first, \ - mock.patch.object(AddedAddon, 'update_addons') as ua_added: - manager.add(first_addon, {'non-empty': 'dict'}) + with mock.patch.object( + first_addon, "update_addons", wraps=first_addon.update_addons + ) as ua_first, mock.patch.object(AddedAddon, "update_addons") as ua_added: + manager.add(first_addon, {"non-empty": "dict"}) manager.update_addons() - six.assertCountEqual(self, manager, ['FirstAddon', 'AddedAddon']) - ua_first.assert_called_once_with(manager.configs['FirstAddon'], - manager) - ua_added.assert_called_once_with(manager.configs['AddedAddon'], - manager) + self.assertCountEqual(manager, ["FirstAddon", "AddedAddon"]) + ua_first.assert_called_once_with(manager.configs["FirstAddon"], manager) + ua_added.assert_called_once_with(manager.configs["AddedAddon"], manager) def test_check_dependency_clashes_attributes(self): provides = addons.GoodAddon("ProvidesAddon") - provides.provides = ('test', ) + provides.provides = ("test",) provides2 = addons.GoodAddon("ProvidesAddon2") - provides2.provides = ('test', ) + provides2.provides = ("test",) requires = addons.GoodAddon("RequiresAddon") - requires.requires = ('test', ) + requires.requires = ("test",) requires_name = addons.GoodAddon("RequiresNameAddon") - requires_name.requires = ('ProvidesAddon', ) + requires_name.requires = ("ProvidesAddon",) requires_newer = addons.GoodAddon("RequiresNewerAddon") - requires_newer.requires = ('test>=2.0', ) + requires_newer.requires = ("test>=2.0",) modifies = addons.GoodAddon("ModifiesAddon") - modifies.modifies = ('test', ) + modifies.modifies = ("test",) def check_with(*addons): manager = AddonManager() diff --git a/tests/test_addons/addonmod.py b/tests/test_addons/addonmod.py index 8ecf4b81d..c59f6737f 100644 --- a/tests/test_addons/addonmod.py +++ b/tests/test_addons/addonmod.py @@ -9,8 +9,10 @@ FROM = "test_addons.addonmod" name = "AddonModule" version = "1.0" + def update_settings(config, settings): pass + def check_configuration(config, crawler): pass diff --git a/tests/test_addons/addons.py b/tests/test_addons/addons.py index f3442b192..4adb9fe8f 100644 --- a/tests/test_addons/addons.py +++ b/tests/test_addons/addons.py @@ -1,18 +1,16 @@ import zope.interface -from scrapy.addons import Addon from scrapy.interfaces import IAddon class Addon(object): - FROM = 'test_addons.addons' + FROM = "test_addons.addons" @zope.interface.declarations.implementer(IAddon) class GoodAddon(object): - - name = 'GoodAddon' - version = '1.0' + name = "GoodAddon" + version = "1.0" def __init__(self, name=None, version=None): if name is not None: @@ -32,8 +30,7 @@ class GoodAddon(object): @zope.interface.declarations.implementer(IAddon) class BrokenAddon(object): - - name = 'BrokenAddon' + name = "BrokenAddon" # No version diff --git a/tests/test_addons/test_builtins.py b/tests/test_addons/test_builtins.py index c89876950..1050cbbed 100644 --- a/tests/test_addons/test_builtins.py +++ b/tests/test_addons/test_builtins.py @@ -7,19 +7,17 @@ from scrapy.settings import Settings class BuiltinAddonsTest(unittest.TestCase): - def test_make_builtin_addon(self): - httpcache = make_builtin_addon('httpcache', {'enabled': True}) - self.assertEqual(httpcache.name, 'httpcache') - self.assertEqual(httpcache.default_config, {'enabled': True}) + httpcache = make_builtin_addon("httpcache", {"enabled": True}) + self.assertEqual(httpcache.name, "httpcache") + self.assertEqual(httpcache.default_config, {"enabled": True}) self.assertEqual(httpcache.version, scrapy.__version__) - httpcache = make_builtin_addon('httpcache', {'enabled': True}, '99.9') - self.assertEqual(httpcache.version, '99.9') + httpcache = make_builtin_addon("httpcache", {"enabled": True}, "99.9") + self.assertEqual(httpcache.version, "99.9") def test_defaultheaders_export_config(self): settings = Settings() dh = scrapy.addons.defaultheaders() - dh.export_config({'X-Test-Header': 'val'}, settings) - self.assertIn('X-Test-Header', settings['DEFAULT_REQUEST_HEADERS']) - self.assertEqual(settings['DEFAULT_REQUEST_HEADERS']['X-Test-Header'], - 'val') + dh.export_config({"X-Test-Header": "val"}, settings) + self.assertIn("X-Test-Header", settings["DEFAULT_REQUEST_HEADERS"]) + self.assertEqual(settings["DEFAULT_REQUEST_HEADERS"]["X-Test-Header"], "val") diff --git a/tests/test_crawl.py b/tests/test_crawl.py index e353f80cb..1e97863b0 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -419,7 +419,7 @@ with multiples lines addonmgr = AddonManager() addonmgr.add(FailedCheckAddon()) - crawler = self.runner.create_crawler(SimpleSpider) + crawler = get_crawler(SimpleSpider) crawler.addons = addonmgr # Doesn't work in 'precise' test environment: # with self.assertRaises(ValueError): diff --git a/tests/test_middleware.py b/tests/test_middleware.py index 7ddf952f7..fd57d846e 100644 --- a/tests/test_middleware.py +++ b/tests/test_middleware.py @@ -90,14 +90,12 @@ class MiddlewareManagerTest(unittest.TestCase): def test_instances_from_settings(self): settings = Settings() - myM3 = M3() class InstanceTestMiddlewareManager(MiddlewareManager): @classmethod def _get_mwlist_from_settings(cls, settings): - return ["tests.test_middleware.M1", M2, myM3] + return ["tests.test_middleware.M1", M2] mwman = InstanceTestMiddlewareManager.from_settings(settings) self.assertIsInstance(mwman.middlewares[0], M1) self.assertIsInstance(mwman.middlewares[1], M2) - self.assertIs(mwman.middlewares[2], myM3)