Drop support for providing file paths as add-ons

scrapy/utils/misc.py

@@ -1,8 +1,5 @@
 """Helper functions which doesn't fit anywhere else"""
-import itertools
-import os.path
 import re
-import sys
 import hashlib
 from importlib import import_module
 from pkgutil import iter_modules
@@ -72,10 +69,6 @@ def load_module_or_object(path):
         return load_object(path)
     except (ValueError, NameError, ImportError):
         pass
-    try:
-        return get_module_from_filepath(path)
-    except ImportError:
-        pass
     raise NameError("Could not load '%s'" % path)
 
 
@@ -101,23 +94,6 @@ def walk_modules(path):
     return mods
 
 
-def get_module_from_filepath(path):
-    """Load and return a python module/package from a file path"""
-    path = path.rstrip("/")
-    if path.endswith('.py'):
-        path = path.rsplit('.py', 1)[0]
-    basefolder, modname = os.path.split(path)
-    # XXX: There are other ways to import modules from a full path which don't
-    #      need to modify PYTHONPATH, see
-    #          https://stackoverflow.com/questions/67631/
-    #      These methods differ between py2 and py3, and apparently the
-    #      py3 method was deprecated in Python 3.4
-    sys.path.insert(0, basefolder)
-    mod = import_module(modname)
-    sys.path.pop(0)
-    return mod
-
-
 def extract_regex(regex, text, encoding='utf-8'):
     """Extract a list of unicode strings from the given text/encoding using the following policies:
 

tests/test_addons/__init__.py

@@ -108,8 +108,6 @@ class AddonTest(unittest.TestCase):
 
 class AddonManagerTest(unittest.TestCase):
 
-    ADDONMODPATH = os.path.join(os.path.dirname(__file__), 'addonmod.py')
-
     def setUp(self):
         self.manager = AddonManager()
 
@@ -156,7 +154,6 @@ class AddonManagerTest(unittest.TestCase):
         test_gets_removed('AddonModule')
         test_gets_removed(addonmod)
         test_gets_removed('tests.test_addons.addonmod')
-        test_gets_removed(self.ADDONMODPATH)
         self.assertRaises(KeyError, manager.remove, 'nonexistent')
         self.assertRaises(KeyError, manager.remove, addons.GoodAddon())
 
@@ -164,18 +161,12 @@ class AddonManagerTest(unittest.TestCase):
         goodaddon = self.manager.get_addon('tests.test_addons.addons.GoodAddon')
         self.assertIs(goodaddon, addons.GoodAddon)
 
-        loaded_addonmod = self.manager.get_addon(self.ADDONMODPATH)
-        # XXX: The module is in fact imported twice under different names into
-        #      sys.modules, is there a good assertion for module equality?
-        self.assertEqual(loaded_addonmod.name, addonmod.name)
+        loaded_addonmod = self.manager.get_addon("tests.test_addons.addonmod")
+        self.assertIs(loaded_addonmod, addonmod)
 
-        # Does not provide interface, but has _addon attribute pointing to
-        # GoodAddon instance
         addonspath = os.path.join(os.path.dirname(__file__), 'addons.py')
-        goodaddon = self.manager.get_addon(addonspath)
-        # XXX: Again, the imported class and addons.GoodAddon are different
-        #      since they are imported twice. How to use isInstance?
-        self.assertEqual(goodaddon.name, addons.GoodAddon.name)
+        goodaddon = self.manager.get_addon("tests.test_addons.addons")
+        self.assertIsInstance(goodaddon, addons.GoodAddon)
 
         self.assertRaises(NameError, self.manager.get_addon, 'xy.n_onexistent')
 
@@ -198,21 +189,18 @@ class AddonManagerTest(unittest.TestCase):
             self.assertIsInstance(manager['GoodAddon'], addons.GoodAddon)
             six.assertCountEqual(self, manager.configs['GoodAddon'], ['key'])
             self.assertEqual(manager.configs['GoodAddon']['key'], 'val2')
-            # XXX: Check module equality, see above
-            self.assertEqual(manager['AddonModule'].name, addonmod.name)
+            self.assertEqual(manager['AddonModule'], addonmod)
             self.assertIn('key', manager.configs['AddonModule'])
             self.assertEqual(manager.configs['AddonModule']['key'], 'val1')
 
         addonsdict = {
-            self.ADDONMODPATH: {
-                'key': 'val1',
-                },
+            "tests.test_addons.addonmod": {'key': 'val1'},
             'tests.test_addons.addons.GoodAddon': {'key': 'val2'},
             }
         _test_load_method('load_dict', addonsdict)
 
         settings = BaseSettings()
-        settings.set('ADDONS', {self.ADDONMODPATH: 0,
+        settings.set('ADDONS', {"tests.test_addons.addonmod": 0,
                                 'tests.test_addons.addons.GoodAddon': 0})
         settings.set('ADDONMODULE', {'key': 'val1'})
         settings.set('GOODADDON', {'key': 'val2'})

tests/test_utils_misc/__init__.py

@@ -4,7 +4,7 @@ import unittest
 
 from scrapy.item import Item, Field
 from scrapy.utils.misc import (load_object, load_module_or_object, arg_to_iter,
-                               walk_modules, get_module_from_filepath)
+                               walk_modules)
 
 __doctests__ = ['scrapy.utils.misc']
 
@@ -21,9 +21,6 @@ class UtilsMiscTestCase(unittest.TestCase):
     def test_load_module_or_object(self):
         testmod = load_module_or_object(__name__ + '.testmod')
         self.assertTrue(hasattr(testmod, 'TESTVAR'))
-        testmod = load_module_or_object(
-                    os.path.join(os.path.dirname(__file__), 'testmod.py'))
-        self.assertTrue(hasattr(testmod, 'TESTVAR'))
         obj = load_object('scrapy.utils.misc.load_object')
         self.assertIs(obj, load_object)
 
@@ -67,20 +64,6 @@ class UtilsMiscTestCase(unittest.TestCase):
         finally:
             sys.path.remove(egg)
 
-    def test_get_module_from_filepath(self):
-        testmodpath = os.path.join(os.path.dirname(__file__), 'testmod.py')
-        testmod = get_module_from_filepath(testmodpath)
-        self.assertTrue(hasattr(testmod, 'TESTVAR'))
-
-        testpkgpath = os.path.join(os.path.dirname(__file__), 'testpkg')
-        testpkg = get_module_from_filepath(testpkgpath)
-        self.assertTrue(hasattr(testpkg, 'TESTVAR2'))
-        # Check submodule access
-        import testpkg.submod
-        self.assertTrue(hasattr(testpkg.submod, 'TESTVAR3'))
-        self.assertIs(testpkg.submod.TESTVAR3,
-                      load_object(testpkg.__name__ + ".submod.TESTVAR3"))
-
     def test_arg_to_iter(self):
 
         class TestItem(Item):

tests/test_utils_misc/testpkg/__init__.py

@@ -1 +0,0 @@
-TESTVAR2 = True

tests/test_utils_misc/testpkg/submod.py

@@ -1 +0,0 @@
-TESTVAR3 = True