From f249b309ab779b5ab518f54f309d7a4ac6661ec7 Mon Sep 17 00:00:00 2001 From: Jakob de Maeyer Date: Wed, 26 Aug 2015 00:26:06 +0200 Subject: [PATCH] Move scrapy.utils.conf.remove_none_values to s.u.python.without_none_values --- scrapy/commands/crawl.py | 5 +++-- scrapy/commands/runspider.py | 5 +++-- scrapy/core/downloader/handlers/__init__.py | 4 ++-- scrapy/downloadermiddlewares/defaultheaders.py | 4 ++-- scrapy/extensions/feedexport.py | 4 ++-- scrapy/utils/conf.py | 9 ++------- scrapy/utils/python.py | 12 ++++++++++++ tests/test_utils_conf.py | 9 +-------- tests/test_utils_python.py | 10 +++++++++- 9 files changed, 36 insertions(+), 26 deletions(-) diff --git a/scrapy/commands/crawl.py b/scrapy/commands/crawl.py index 9c8a3d4ce..7f5c64c20 100644 --- a/scrapy/commands/crawl.py +++ b/scrapy/commands/crawl.py @@ -1,6 +1,7 @@ import os from scrapy.commands import ScrapyCommand -from scrapy.utils.conf import arglist_to_dict, remove_none_values +from scrapy.utils.conf import arglist_to_dict +from scrapy.utils.python import without_none_values from scrapy.exceptions import UsageError @@ -34,7 +35,7 @@ class Command(ScrapyCommand): self.settings.set('FEED_URI', 'stdout:', priority='cmdline') else: self.settings.set('FEED_URI', opts.output, priority='cmdline') - feed_exporters = remove_none_values(self.settings._getcomposite('FEED_EXPORTERS')) + feed_exporters = without_none_values(self.settings._getcomposite('FEED_EXPORTERS')) valid_output_formats = feed_exporters.keys() if not opts.output_format: opts.output_format = os.path.splitext(opts.output)[1].replace(".", "") diff --git a/scrapy/commands/runspider.py b/scrapy/commands/runspider.py index 7d85984c3..72229bcf5 100644 --- a/scrapy/commands/runspider.py +++ b/scrapy/commands/runspider.py @@ -5,7 +5,8 @@ from importlib import import_module from scrapy.utils.spider import iter_spider_classes from scrapy.commands import ScrapyCommand from scrapy.exceptions import UsageError -from scrapy.utils.conf import arglist_to_dict, remove_none_values +from scrapy.utils.conf import arglist_to_dict +from scrapy.utils.python import without_none_values def _import_file(filepath): @@ -57,7 +58,7 @@ class Command(ScrapyCommand): self.settings.set('FEED_URI', 'stdout:', priority='cmdline') else: self.settings.set('FEED_URI', opts.output, priority='cmdline') - feed_exporters = remove_none_values(self.settings._getcomposite('FEED_EXPORTERS')) + feed_exporters = without_none_values(self.settings._getcomposite('FEED_EXPORTERS')) valid_output_formats = feed_exporters.keys() if not opts.output_format: opts.output_format = os.path.splitext(opts.output)[1].replace(".", "") diff --git a/scrapy/core/downloader/handlers/__init__.py b/scrapy/core/downloader/handlers/__init__.py index 9b118c39b..0e78e04f4 100644 --- a/scrapy/core/downloader/handlers/__init__.py +++ b/scrapy/core/downloader/handlers/__init__.py @@ -4,9 +4,9 @@ import logging from twisted.internet import defer import six from scrapy.exceptions import NotSupported, NotConfigured -from scrapy.utils.conf import remove_none_values from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.misc import load_object +from scrapy.utils.python import without_none_values from scrapy import signals @@ -20,7 +20,7 @@ class DownloadHandlers(object): self._schemes = {} # stores acceptable schemes on instancing self._handlers = {} # stores instanced handlers for schemes self._notconfigured = {} # remembers failed handlers - handlers = remove_none_values(crawler.settings._getcomposite('DOWNLOAD_HANDLERS')) + handlers = without_none_values(crawler.settings._getcomposite('DOWNLOAD_HANDLERS')) for scheme, clspath in six.iteritems(handlers): self._schemes[scheme] = clspath diff --git a/scrapy/downloadermiddlewares/defaultheaders.py b/scrapy/downloadermiddlewares/defaultheaders.py index c8924c04a..93fe97673 100644 --- a/scrapy/downloadermiddlewares/defaultheaders.py +++ b/scrapy/downloadermiddlewares/defaultheaders.py @@ -4,7 +4,7 @@ DefaultHeaders downloader middleware See documentation in docs/topics/downloader-middleware.rst """ -from scrapy.utils.conf import remove_none_values +from scrapy.utils.python import without_none_values class DefaultHeadersMiddleware(object): @@ -14,7 +14,7 @@ class DefaultHeadersMiddleware(object): @classmethod def from_crawler(cls, crawler): - headers = remove_none_values(crawler.settings['DEFAULT_REQUEST_HEADERS']) + headers = without_none_values(crawler.settings['DEFAULT_REQUEST_HEADERS']) return cls(headers.items()) def process_request(self, request, spider): diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index fb07657d6..1e27a1e7e 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -18,11 +18,11 @@ from twisted.internet import defer, threads from w3lib.url import file_uri_to_path from scrapy import signals -from scrapy.utils.conf import remove_none_values from scrapy.utils.ftp import ftp_makedirs_cwd from scrapy.exceptions import NotConfigured from scrapy.utils.misc import load_object from scrapy.utils.log import failure_to_exc_info +from scrapy.utils.python import without_none_values logger = logging.getLogger(__name__) @@ -196,7 +196,7 @@ class FeedExporter(object): return item def _load_components(self, setting_prefix): - conf = remove_none_values(self.settings._getcomposite(setting_prefix)) + conf = without_none_values(self.settings._getcomposite(setting_prefix)) d = {} for k, v in conf.items(): try: diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.py index 80c644657..57f2b6322 100644 --- a/scrapy/utils/conf.py +++ b/scrapy/utils/conf.py @@ -1,6 +1,5 @@ import os import sys -import warnings from operator import itemgetter import six @@ -8,6 +7,7 @@ from six.moves.configparser import SafeConfigParser from scrapy.settings import BaseSettings from scrapy.utils.deprecate import update_classpath +from scrapy.utils.python import without_none_values def build_component_list(compdict, convert=update_classpath): @@ -37,15 +37,10 @@ def build_component_list(compdict, convert=update_classpath): if isinstance(compdict, (list, tuple)): _check_components(compdict) return type(compdict)(convert(c) for c in compdict) - compdict = remove_none_values(_map_keys(compdict)) + compdict = without_none_values(_map_keys(compdict)) return [k for k, v in sorted(six.iteritems(compdict), key=itemgetter(1))] -def remove_none_values(compdict): - """Return dict with all pairs that have value 'None' removed""" - return {k: v for k, v in six.iteritems(compdict) if v is not None} - - def arglist_to_dict(arglist): """Convert a list of arguments like ['arg1=val1', 'arg2=val2', ...] to a dict diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index d566783b2..1f9d02df5 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -324,3 +324,15 @@ def retry_on_eintr(function, *args, **kw): except IOError as e: if e.errno != errno.EINTR: raise + + +def without_none_values(iterable): + """Return a copy of `iterable` with all `None` entries removed. + + If `iterable` is a mapping, return a dictionary where all pairs that have + value `None` have been removed. + """ + try: + return {k: v for k, v in six.iteritems(iterable) if v is not None} + except AttributeError: + return type(iterable)((v for v in iterable if v is not None)) diff --git a/tests/test_utils_conf.py b/tests/test_utils_conf.py index e94ccc49b..af15d3184 100644 --- a/tests/test_utils_conf.py +++ b/tests/test_utils_conf.py @@ -1,8 +1,7 @@ import unittest from scrapy.settings import BaseSettings -from scrapy.utils.conf import (build_component_list, arglist_to_dict, - remove_none_values) +from scrapy.utils.conf import build_component_list, arglist_to_dict class BuildComponentListTest(unittest.TestCase): @@ -53,12 +52,6 @@ class BuildComponentListTest(unittest.TestCase): class UtilsConfTestCase(unittest.TestCase): - def test_remove_none_values(self): - comps = {'one': 1, 'none': None, 'three': 3, 'four': 4} - compscopy = dict(comps) - del compscopy['none'] - self.assertEqual(remove_none_values(comps), compscopy) - def test_arglist_to_dict(self): self.assertEqual(arglist_to_dict(['arg1=val1', 'arg2=val2']), {'arg1': 'val1', 'arg2': 'val2'}) diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py index ca394ebf5..4f0834902 100644 --- a/tests/test_utils_python.py +++ b/tests/test_utils_python.py @@ -6,7 +6,8 @@ import six from scrapy.utils.python import ( memoizemethod_noargs, isbinarytext, equal_attributes, - WeakKeyCache, stringify_dict, get_func_args, to_bytes, to_unicode) + WeakKeyCache, stringify_dict, get_func_args, to_bytes, to_unicode, + without_none_values) __doctests__ = ['scrapy.utils.python'] @@ -212,5 +213,12 @@ class UtilsPythonTestCase(unittest.TestCase): self.assertEqual(get_func_args(" ".join), []) self.assertEqual(get_func_args(operator.itemgetter(2)), []) + def test_without_none_values(self): + self.assertEqual(without_none_values([1, None, 3, 4]), [1, 3, 4]) + self.assertEqual(without_none_values((1, None, 3, 4)), (1, 3, 4)) + self.assertEqual( + without_none_values({'one': 1, 'none': None, 'three': 3, 'four': 4}), + {'one': 1, 'three': 3, 'four': 4}) + if __name__ == "__main__": unittest.main()