From 6b1760d7a1d8949f14a43aa7ce00726e084b2abe Mon Sep 17 00:00:00 2001 From: Rolando Espinoza La fuente Date: Sun, 27 Oct 2013 18:28:23 -0400 Subject: [PATCH] replaced __import__ by importlib.import_module. Since python 2.7, importlib.import_module is the recommended way to import modules programmatically. From __import__'s doc: Import a module. Because this function is meant for use by the Python interpreter and not for general use it is better to use importlib.import_module() to programmatically import a module. --- scrapy/commands/genspider.py | 4 +++- scrapy/commands/runspider.py | 3 ++- scrapy/contrib/httpcache.py | 3 ++- scrapy/tests/test_dependencies.py | 2 +- scrapy/utils/misc.py | 8 +++++--- scrapy/utils/project.py | 8 +++++--- 6 files changed, 18 insertions(+), 10 deletions(-) diff --git a/scrapy/commands/genspider.py b/scrapy/commands/genspider.py index f8a5e2310..21b5ba3ea 100644 --- a/scrapy/commands/genspider.py +++ b/scrapy/commands/genspider.py @@ -2,6 +2,8 @@ from __future__ import print_function import os import shutil import string + +from importlib import import_module from os.path import join, dirname, abspath, exists, splitext import scrapy @@ -90,7 +92,7 @@ class Command(ScrapyCommand): 'classname': '%sSpider' % ''.join([s.capitalize() \ for s in module.split('_')]) } - spiders_module = __import__(self.settings['NEWSPIDER_MODULE'], {}, {}, ['']) + spiders_module = import_module(self.settings['NEWSPIDER_MODULE']) spiders_dir = abspath(dirname(spiders_module.__file__)) spider_file = "%s.py" % join(spiders_dir, module) shutil.copyfile(template_file, spider_file) diff --git a/scrapy/commands/runspider.py b/scrapy/commands/runspider.py index 3fe439b11..14d67e24b 100644 --- a/scrapy/commands/runspider.py +++ b/scrapy/commands/runspider.py @@ -1,5 +1,6 @@ import sys import os +from importlib import import_module from scrapy.utils.spider import iter_spider_classes from scrapy.command import ScrapyCommand @@ -15,7 +16,7 @@ def _import_file(filepath): if dirname: sys.path = [dirname] + sys.path try: - module = __import__(fname, {}, {}, ['']) + module = import_module(fname) finally: if dirname: sys.path.pop(0) diff --git a/scrapy/contrib/httpcache.py b/scrapy/contrib/httpcache.py index b473c2dc3..5dee898f0 100644 --- a/scrapy/contrib/httpcache.py +++ b/scrapy/contrib/httpcache.py @@ -1,6 +1,7 @@ from __future__ import print_function import os import cPickle as pickle +from importlib import import_module from time import time from weakref import WeakKeyDictionary from email.utils import mktime_tz, parsedate_tz @@ -164,7 +165,7 @@ class DbmCacheStorage(object): def __init__(self, settings): self.cachedir = data_path(settings['HTTPCACHE_DIR'], createdir=True) self.expiration_secs = settings.getint('HTTPCACHE_EXPIRATION_SECS') - self.dbmodule = __import__(settings['HTTPCACHE_DBM_MODULE'], {}, {}, ['']) + self.dbmodule = import_module(settings['HTTPCACHE_DBM_MODULE']) self.db = None def open_spider(self, spider): diff --git a/scrapy/tests/test_dependencies.py b/scrapy/tests/test_dependencies.py index 794db0fe6..51b9c3af3 100644 --- a/scrapy/tests/test_dependencies.py +++ b/scrapy/tests/test_dependencies.py @@ -3,7 +3,7 @@ from twisted.trial import unittest class ScrapyUtilsTest(unittest.TestCase): def test_required_openssl_version(self): try: - module = __import__('OpenSSL', {}, {}, ['']) + module = __import__('OpenSSL') except ImportError as ex: raise unittest.SkipTest("OpenSSL is not available") diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index 018fe031f..a16d42eae 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -2,6 +2,8 @@ import re import hashlib + +from importlib import import_module from pkgutil import iter_modules from w3lib.html import remove_entities @@ -35,7 +37,7 @@ def load_object(path): module, name = path[:dot], path[dot+1:] try: - mod = __import__(module, {}, {}, ['']) + mod = import_module(module) except ImportError as e: raise ImportError("Error loading object '%s': %s" % (path, e)) @@ -55,7 +57,7 @@ def walk_modules(path, load=False): """ mods = [] - mod = __import__(path, {}, {}, ['']) + mod = import_module(path) mods.append(mod) if hasattr(mod, '__path__'): for _, subpath, ispkg in iter_modules(mod.__path__): @@ -63,7 +65,7 @@ def walk_modules(path, load=False): if ispkg: mods += walk_modules(fullpath) else: - submod = __import__(fullpath, {}, {}, ['']) + submod = import_module(fullpath) mods.append(submod) return mods diff --git a/scrapy/utils/project.py b/scrapy/utils/project.py index a2edb7b08..332fc18b8 100644 --- a/scrapy/utils/project.py +++ b/scrapy/utils/project.py @@ -1,8 +1,10 @@ import os -from os.path import join, dirname, abspath, isabs, exists import cPickle as pickle import warnings +from importlib import import_module +from os.path import join, dirname, abspath, isabs, exists + from scrapy.utils.conf import closest_scrapy_cfg, get_config, init_env from scrapy.settings import CrawlerSettings from scrapy.exceptions import NotConfigured @@ -14,7 +16,7 @@ def inside_project(): scrapy_module = os.environ.get('SCRAPY_SETTINGS_MODULE') if scrapy_module is not None: try: - __import__(scrapy_module) + import_module(scrapy_module) except ImportError as exc: warnings.warn("Cannot import scrapy settings module %s: %s" % (scrapy_module, exc)) else: @@ -53,7 +55,7 @@ def get_project_settings(): init_env(project) settings_module_path = os.environ.get(ENVVAR) if settings_module_path: - settings_module = __import__(settings_module_path, {}, {}, ['']) + settings_module = import_module(settings_module_path) else: settings_module = None settings = CrawlerSettings(settings_module)