diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.py index 8d2d01a3c..caa80a5a1 100644 --- a/scrapy/utils/conf.py +++ b/scrapy/utils/conf.py @@ -1,8 +1,11 @@ -import sys import os -from six.moves.configparser import SafeConfigParser +import sys from operator import itemgetter +import six +from six.moves.configparser import SafeConfigParser + + def build_component_list(base, custom): """Compose a component list based on a custom and base dict of components (typically middlewares or extensions), unless custom is already a list, in @@ -12,8 +15,9 @@ def build_component_list(base, custom): return custom compdict = base.copy() compdict.update(custom) - return [k for k, v in sorted(compdict.items(), key=itemgetter(1)) \ - if v is not None] + items = (x for x in six.iteritems(compdict) if x[1] is not None) + return [x[0] for x in sorted(items, key=itemgetter(1))] + def arglist_to_dict(arglist): """Convert a list of arguments like ['arg1=val1', 'arg2=val2', ...] to a @@ -21,6 +25,7 @@ def arglist_to_dict(arglist): """ return dict(x.split('=', 1) for x in arglist) + def closest_scrapy_cfg(path='.', prevpath=None): """Return the path to the closest scrapy.cfg file by traversing the current directory and its parents @@ -33,6 +38,7 @@ def closest_scrapy_cfg(path='.', prevpath=None): return cfgfile return closest_scrapy_cfg(os.path.dirname(path), path) + def init_env(project='default', set_syspath=True): """Initialize environment to use command-line tool from inside a project dir. This sets the Scrapy settings module and modifies the Python path to @@ -47,6 +53,7 @@ def init_env(project='default', set_syspath=True): if set_syspath and projdir not in sys.path: sys.path.append(projdir) + def get_config(use_closest=True): """Get Scrapy config file as a SafeConfigParser""" sources = get_sources(use_closest) @@ -54,9 +61,10 @@ def get_config(use_closest=True): cfg.read(sources) return cfg + def get_sources(use_closest=True): - sources = ['/etc/scrapy.cfg', r'c:\scrapy\scrapy.cfg', \ - os.path.expanduser('~/.scrapy.cfg')] + sources = ['/etc/scrapy.cfg', r'c:\scrapy\scrapy.cfg', + os.path.expanduser('~/.scrapy.cfg')] if use_closest: sources.append(closest_scrapy_cfg()) return sources diff --git a/tests/py3-ignores.txt b/tests/py3-ignores.txt index 17970fe4a..868264b85 100644 --- a/tests/py3-ignores.txt +++ b/tests/py3-ignores.txt @@ -63,7 +63,6 @@ tests/test_spider.py tests/test_squeue.py tests/test_stats.py tests/test_toplevel.py -tests/test_utils_conf.py tests/test_utils_defer.py tests/test_utils_gz.py tests/test_utils_httpobj.py