From 343f997ed6a20241480316c6d24490f071a3b524 Mon Sep 17 00:00:00 2001 From: Rolando Espinoza La fuente Date: Sun, 27 Oct 2013 18:28:23 -0400 Subject: [PATCH 1/2] replaced __import__ by importlib.import_module. Since python 2.7, importlib.import_module is the recommended way to import modules programmatically. From __import__'s doc: Import a module. Because this function is meant for use by the Python interpreter and not for general use it is better to use importlib.import_module() to programmatically import a module. --- scrapy/commands/genspider.py | 4 +++- scrapy/commands/runspider.py | 3 ++- scrapy/contrib/httpcache.py | 3 ++- scrapy/tests/test_dependencies.py | 2 +- scrapy/utils/misc.py | 8 +++++--- scrapy/utils/project.py | 8 +++++--- 6 files changed, 18 insertions(+), 10 deletions(-) diff --git a/scrapy/commands/genspider.py b/scrapy/commands/genspider.py index f8a5e2310..21b5ba3ea 100644 --- a/scrapy/commands/genspider.py +++ b/scrapy/commands/genspider.py @@ -2,6 +2,8 @@ from __future__ import print_function import os import shutil import string + +from importlib import import_module from os.path import join, dirname, abspath, exists, splitext import scrapy @@ -90,7 +92,7 @@ class Command(ScrapyCommand): 'classname': '%sSpider' % ''.join([s.capitalize() \ for s in module.split('_')]) } - spiders_module = __import__(self.settings['NEWSPIDER_MODULE'], {}, {}, ['']) + spiders_module = import_module(self.settings['NEWSPIDER_MODULE']) spiders_dir = abspath(dirname(spiders_module.__file__)) spider_file = "%s.py" % join(spiders_dir, module) shutil.copyfile(template_file, spider_file) diff --git a/scrapy/commands/runspider.py b/scrapy/commands/runspider.py index 3fe439b11..14d67e24b 100644 --- a/scrapy/commands/runspider.py +++ b/scrapy/commands/runspider.py @@ -1,5 +1,6 @@ import sys import os +from importlib import import_module from scrapy.utils.spider import iter_spider_classes from scrapy.command import ScrapyCommand @@ -15,7 +16,7 @@ def _import_file(filepath): if dirname: sys.path = [dirname] + sys.path try: - module = __import__(fname, {}, {}, ['']) + module = import_module(fname) finally: if dirname: sys.path.pop(0) diff --git a/scrapy/contrib/httpcache.py b/scrapy/contrib/httpcache.py index b473c2dc3..5dee898f0 100644 --- a/scrapy/contrib/httpcache.py +++ b/scrapy/contrib/httpcache.py @@ -1,6 +1,7 @@ from __future__ import print_function import os import cPickle as pickle +from importlib import import_module from time import time from weakref import WeakKeyDictionary from email.utils import mktime_tz, parsedate_tz @@ -164,7 +165,7 @@ class DbmCacheStorage(object): def __init__(self, settings): self.cachedir = data_path(settings['HTTPCACHE_DIR'], createdir=True) self.expiration_secs = settings.getint('HTTPCACHE_EXPIRATION_SECS') - self.dbmodule = __import__(settings['HTTPCACHE_DBM_MODULE'], {}, {}, ['']) + self.dbmodule = import_module(settings['HTTPCACHE_DBM_MODULE']) self.db = None def open_spider(self, spider): diff --git a/scrapy/tests/test_dependencies.py b/scrapy/tests/test_dependencies.py index 794db0fe6..51b9c3af3 100644 --- a/scrapy/tests/test_dependencies.py +++ b/scrapy/tests/test_dependencies.py @@ -3,7 +3,7 @@ from twisted.trial import unittest class ScrapyUtilsTest(unittest.TestCase): def test_required_openssl_version(self): try: - module = __import__('OpenSSL', {}, {}, ['']) + module = __import__('OpenSSL') except ImportError as ex: raise unittest.SkipTest("OpenSSL is not available") diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index 018fe031f..a16d42eae 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -2,6 +2,8 @@ import re import hashlib + +from importlib import import_module from pkgutil import iter_modules from w3lib.html import remove_entities @@ -35,7 +37,7 @@ def load_object(path): module, name = path[:dot], path[dot+1:] try: - mod = __import__(module, {}, {}, ['']) + mod = import_module(module) except ImportError as e: raise ImportError("Error loading object '%s': %s" % (path, e)) @@ -55,7 +57,7 @@ def walk_modules(path, load=False): """ mods = [] - mod = __import__(path, {}, {}, ['']) + mod = import_module(path) mods.append(mod) if hasattr(mod, '__path__'): for _, subpath, ispkg in iter_modules(mod.__path__): @@ -63,7 +65,7 @@ def walk_modules(path, load=False): if ispkg: mods += walk_modules(fullpath) else: - submod = __import__(fullpath, {}, {}, ['']) + submod = import_module(fullpath) mods.append(submod) return mods diff --git a/scrapy/utils/project.py b/scrapy/utils/project.py index a2edb7b08..332fc18b8 100644 --- a/scrapy/utils/project.py +++ b/scrapy/utils/project.py @@ -1,8 +1,10 @@ import os -from os.path import join, dirname, abspath, isabs, exists import cPickle as pickle import warnings +from importlib import import_module +from os.path import join, dirname, abspath, isabs, exists + from scrapy.utils.conf import closest_scrapy_cfg, get_config, init_env from scrapy.settings import CrawlerSettings from scrapy.exceptions import NotConfigured @@ -14,7 +16,7 @@ def inside_project(): scrapy_module = os.environ.get('SCRAPY_SETTINGS_MODULE') if scrapy_module is not None: try: - __import__(scrapy_module) + import_module(scrapy_module) except ImportError as exc: warnings.warn("Cannot import scrapy settings module %s: %s" % (scrapy_module, exc)) else: @@ -53,7 +55,7 @@ def get_project_settings(): init_env(project) settings_module_path = os.environ.get(ENVVAR) if settings_module_path: - settings_module = __import__(settings_module_path, {}, {}, ['']) + settings_module = import_module(settings_module_path) else: settings_module = None settings = CrawlerSettings(settings_module) From 10e22aa5fba928f4092b4aec597f4539566dca59 Mon Sep 17 00:00:00 2001 From: Rolando Espinoza La fuente Date: Sun, 27 Oct 2013 19:10:25 -0400 Subject: [PATCH 2/2] Replaced remaning __import__(module) calls. This commit replaces the statements __import__(module) as the previous replaced the statements __import__(module, {}, {}, ['']). At first I thought leaving the single-argument calls, but perhaps it's better to be strict rather than having exceptions to the rule in this case. --- scrapy/contrib/memusage.py | 4 +++- scrapy/settings/default_settings.py | 3 ++- scrapy/tests/test_dependencies.py | 3 ++- scrapy/utils/test.py | 3 ++- 4 files changed, 9 insertions(+), 4 deletions(-) diff --git a/scrapy/contrib/memusage.py b/scrapy/contrib/memusage.py index 4d4aae1c9..98dd15027 100644 --- a/scrapy/contrib/memusage.py +++ b/scrapy/contrib/memusage.py @@ -6,6 +6,7 @@ See documentation in docs/topics/extensions.rst import socket from pprint import pformat +from importlib import import_module from twisted.internet import task @@ -20,7 +21,8 @@ class MemoryUsage(object): if not crawler.settings.getbool('MEMUSAGE_ENABLED'): raise NotConfigured try: - self.resource = __import__('resource') + # stdlib's resource module is only availabe on unix platforms. + self.resource = import_module('resource') except ImportError: raise NotConfigured diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index e7475c3d5..91b8d8b48 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -15,6 +15,7 @@ Scrapy developers, if you add a setting here remember to: import os import sys +from importlib import import_module from os.path import join, abspath, dirname BOT_NAME = 'scrapybot' @@ -229,7 +230,7 @@ TEMPLATES_DIR = abspath(join(dirname(__file__), '..', 'templates')) URLLENGTH_LIMIT = 2083 -USER_AGENT = 'Scrapy/%s (+http://scrapy.org)' % __import__('scrapy').__version__ +USER_AGENT = 'Scrapy/%s (+http://scrapy.org)' % import_module('scrapy').__version__ TELNETCONSOLE_ENABLED = 1 TELNETCONSOLE_PORT = [6023, 6073] diff --git a/scrapy/tests/test_dependencies.py b/scrapy/tests/test_dependencies.py index 51b9c3af3..03bf2ffcf 100644 --- a/scrapy/tests/test_dependencies.py +++ b/scrapy/tests/test_dependencies.py @@ -1,9 +1,10 @@ +from importlib import import_module from twisted.trial import unittest class ScrapyUtilsTest(unittest.TestCase): def test_required_openssl_version(self): try: - module = __import__('OpenSSL') + module = import_module('OpenSSL') except ImportError as ex: raise unittest.SkipTest("OpenSSL is not available") diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py index c2d473e6f..ec9ca1c51 100644 --- a/scrapy/utils/test.py +++ b/scrapy/utils/test.py @@ -4,6 +4,7 @@ This module contains some assorted functions used in tests import os +from importlib import import_module from twisted.trial.unittest import SkipTest @@ -39,7 +40,7 @@ def get_crawler(settings_dict=None): def get_pythonpath(): """Return a PYTHONPATH suitable to use in processes so that they find this installation of Scrapy""" - scrapy_path = __import__('scrapy').__path__[0] + scrapy_path = import_module('scrapy').__path__[0] return os.path.dirname(scrapy_path) + os.pathsep + os.environ.get('PYTHONPATH', '') def get_testenv():