From a65fc0db7d7d728bc8abbb37eff358870ce7a9b5 Mon Sep 17 00:00:00 2001 From: Jakob de Maeyer Date: Thu, 12 Nov 2015 18:37:26 +0100 Subject: [PATCH] Drop support for providing file paths as add-ons --- scrapy/utils/misc.py | 24 --------------------- tests/test_addons/__init__.py | 26 ++++++----------------- tests/test_utils_misc/__init__.py | 19 +---------------- tests/test_utils_misc/testpkg/__init__.py | 1 - tests/test_utils_misc/testpkg/submod.py | 1 - 5 files changed, 8 insertions(+), 63 deletions(-) delete mode 100644 tests/test_utils_misc/testpkg/__init__.py delete mode 100644 tests/test_utils_misc/testpkg/submod.py diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index 9461d93e9..e55edd63e 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -1,8 +1,5 @@ """Helper functions which doesn't fit anywhere else""" -import itertools -import os.path import re -import sys import hashlib from importlib import import_module from pkgutil import iter_modules @@ -72,10 +69,6 @@ def load_module_or_object(path): return load_object(path) except (ValueError, NameError, ImportError): pass - try: - return get_module_from_filepath(path) - except ImportError: - pass raise NameError("Could not load '%s'" % path) @@ -101,23 +94,6 @@ def walk_modules(path): return mods -def get_module_from_filepath(path): - """Load and return a python module/package from a file path""" - path = path.rstrip("/") - if path.endswith('.py'): - path = path.rsplit('.py', 1)[0] - basefolder, modname = os.path.split(path) - # XXX: There are other ways to import modules from a full path which don't - # need to modify PYTHONPATH, see - # https://stackoverflow.com/questions/67631/ - # These methods differ between py2 and py3, and apparently the - # py3 method was deprecated in Python 3.4 - sys.path.insert(0, basefolder) - mod = import_module(modname) - sys.path.pop(0) - return mod - - def extract_regex(regex, text, encoding='utf-8'): """Extract a list of unicode strings from the given text/encoding using the following policies: diff --git a/tests/test_addons/__init__.py b/tests/test_addons/__init__.py index 32ee25ad8..a4e278fa5 100644 --- a/tests/test_addons/__init__.py +++ b/tests/test_addons/__init__.py @@ -108,8 +108,6 @@ class AddonTest(unittest.TestCase): class AddonManagerTest(unittest.TestCase): - ADDONMODPATH = os.path.join(os.path.dirname(__file__), 'addonmod.py') - def setUp(self): self.manager = AddonManager() @@ -156,7 +154,6 @@ class AddonManagerTest(unittest.TestCase): test_gets_removed('AddonModule') test_gets_removed(addonmod) test_gets_removed('tests.test_addons.addonmod') - test_gets_removed(self.ADDONMODPATH) self.assertRaises(KeyError, manager.remove, 'nonexistent') self.assertRaises(KeyError, manager.remove, addons.GoodAddon()) @@ -164,18 +161,12 @@ class AddonManagerTest(unittest.TestCase): goodaddon = self.manager.get_addon('tests.test_addons.addons.GoodAddon') self.assertIs(goodaddon, addons.GoodAddon) - loaded_addonmod = self.manager.get_addon(self.ADDONMODPATH) - # XXX: The module is in fact imported twice under different names into - # sys.modules, is there a good assertion for module equality? - self.assertEqual(loaded_addonmod.name, addonmod.name) + loaded_addonmod = self.manager.get_addon("tests.test_addons.addonmod") + self.assertIs(loaded_addonmod, addonmod) - # Does not provide interface, but has _addon attribute pointing to - # GoodAddon instance addonspath = os.path.join(os.path.dirname(__file__), 'addons.py') - goodaddon = self.manager.get_addon(addonspath) - # XXX: Again, the imported class and addons.GoodAddon are different - # since they are imported twice. How to use isInstance? - self.assertEqual(goodaddon.name, addons.GoodAddon.name) + goodaddon = self.manager.get_addon("tests.test_addons.addons") + self.assertIsInstance(goodaddon, addons.GoodAddon) self.assertRaises(NameError, self.manager.get_addon, 'xy.n_onexistent') @@ -198,21 +189,18 @@ class AddonManagerTest(unittest.TestCase): self.assertIsInstance(manager['GoodAddon'], addons.GoodAddon) six.assertCountEqual(self, manager.configs['GoodAddon'], ['key']) self.assertEqual(manager.configs['GoodAddon']['key'], 'val2') - # XXX: Check module equality, see above - self.assertEqual(manager['AddonModule'].name, addonmod.name) + self.assertEqual(manager['AddonModule'], addonmod) self.assertIn('key', manager.configs['AddonModule']) self.assertEqual(manager.configs['AddonModule']['key'], 'val1') addonsdict = { - self.ADDONMODPATH: { - 'key': 'val1', - }, + "tests.test_addons.addonmod": {'key': 'val1'}, 'tests.test_addons.addons.GoodAddon': {'key': 'val2'}, } _test_load_method('load_dict', addonsdict) settings = BaseSettings() - settings.set('ADDONS', {self.ADDONMODPATH: 0, + settings.set('ADDONS', {"tests.test_addons.addonmod": 0, 'tests.test_addons.addons.GoodAddon': 0}) settings.set('ADDONMODULE', {'key': 'val1'}) settings.set('GOODADDON', {'key': 'val2'}) diff --git a/tests/test_utils_misc/__init__.py b/tests/test_utils_misc/__init__.py index f33562b7d..8ea8786d7 100644 --- a/tests/test_utils_misc/__init__.py +++ b/tests/test_utils_misc/__init__.py @@ -4,7 +4,7 @@ import unittest from scrapy.item import Item, Field from scrapy.utils.misc import (load_object, load_module_or_object, arg_to_iter, - walk_modules, get_module_from_filepath) + walk_modules) __doctests__ = ['scrapy.utils.misc'] @@ -21,9 +21,6 @@ class UtilsMiscTestCase(unittest.TestCase): def test_load_module_or_object(self): testmod = load_module_or_object(__name__ + '.testmod') self.assertTrue(hasattr(testmod, 'TESTVAR')) - testmod = load_module_or_object( - os.path.join(os.path.dirname(__file__), 'testmod.py')) - self.assertTrue(hasattr(testmod, 'TESTVAR')) obj = load_object('scrapy.utils.misc.load_object') self.assertIs(obj, load_object) @@ -67,20 +64,6 @@ class UtilsMiscTestCase(unittest.TestCase): finally: sys.path.remove(egg) - def test_get_module_from_filepath(self): - testmodpath = os.path.join(os.path.dirname(__file__), 'testmod.py') - testmod = get_module_from_filepath(testmodpath) - self.assertTrue(hasattr(testmod, 'TESTVAR')) - - testpkgpath = os.path.join(os.path.dirname(__file__), 'testpkg') - testpkg = get_module_from_filepath(testpkgpath) - self.assertTrue(hasattr(testpkg, 'TESTVAR2')) - # Check submodule access - import testpkg.submod - self.assertTrue(hasattr(testpkg.submod, 'TESTVAR3')) - self.assertIs(testpkg.submod.TESTVAR3, - load_object(testpkg.__name__ + ".submod.TESTVAR3")) - def test_arg_to_iter(self): class TestItem(Item): diff --git a/tests/test_utils_misc/testpkg/__init__.py b/tests/test_utils_misc/testpkg/__init__.py deleted file mode 100644 index 12cc2f6d9..000000000 --- a/tests/test_utils_misc/testpkg/__init__.py +++ /dev/null @@ -1 +0,0 @@ -TESTVAR2 = True diff --git a/tests/test_utils_misc/testpkg/submod.py b/tests/test_utils_misc/testpkg/submod.py deleted file mode 100644 index 8a07e3592..000000000 --- a/tests/test_utils_misc/testpkg/submod.py +++ /dev/null @@ -1 +0,0 @@ -TESTVAR3 = True