Move scrapy.utils.conf.remove_none_values to s.u.python.without_none_values

scrapy/commands/crawl.py

@@ -1,6 +1,7 @@
 import os
 from scrapy.commands import ScrapyCommand
-from scrapy.utils.conf import arglist_to_dict, remove_none_values
+from scrapy.utils.conf import arglist_to_dict
+from scrapy.utils.python import without_none_values
 from scrapy.exceptions import UsageError
 
 
@@ -34,7 +35,7 @@ class Command(ScrapyCommand):
                 self.settings.set('FEED_URI', 'stdout:', priority='cmdline')
             else:
                 self.settings.set('FEED_URI', opts.output, priority='cmdline')
-            feed_exporters = remove_none_values(self.settings._getcomposite('FEED_EXPORTERS'))
+            feed_exporters = without_none_values(self.settings._getcomposite('FEED_EXPORTERS'))
             valid_output_formats = feed_exporters.keys()
             if not opts.output_format:
                 opts.output_format = os.path.splitext(opts.output)[1].replace(".", "")

scrapy/commands/runspider.py

@@ -5,7 +5,8 @@ from importlib import import_module
 from scrapy.utils.spider import iter_spider_classes
 from scrapy.commands import ScrapyCommand
 from scrapy.exceptions import UsageError
-from scrapy.utils.conf import arglist_to_dict, remove_none_values
+from scrapy.utils.conf import arglist_to_dict
+from scrapy.utils.python import without_none_values
 
 
 def _import_file(filepath):
@@ -57,7 +58,7 @@ class Command(ScrapyCommand):
                 self.settings.set('FEED_URI', 'stdout:', priority='cmdline')
             else:
                 self.settings.set('FEED_URI', opts.output, priority='cmdline')
-            feed_exporters = remove_none_values(self.settings._getcomposite('FEED_EXPORTERS'))
+            feed_exporters = without_none_values(self.settings._getcomposite('FEED_EXPORTERS'))
             valid_output_formats = feed_exporters.keys()
             if not opts.output_format:
                 opts.output_format = os.path.splitext(opts.output)[1].replace(".", "")

scrapy/core/downloader/handlers/__init__.py

@@ -4,9 +4,9 @@ import logging
 from twisted.internet import defer
 import six
 from scrapy.exceptions import NotSupported, NotConfigured
-from scrapy.utils.conf import remove_none_values
 from scrapy.utils.httpobj import urlparse_cached
 from scrapy.utils.misc import load_object
+from scrapy.utils.python import without_none_values
 from scrapy import signals
 
 
@@ -20,7 +20,7 @@ class DownloadHandlers(object):
         self._schemes = {}  # stores acceptable schemes on instancing
         self._handlers = {}  # stores instanced handlers for schemes
         self._notconfigured = {}  # remembers failed handlers
-        handlers = remove_none_values(crawler.settings._getcomposite('DOWNLOAD_HANDLERS'))
+        handlers = without_none_values(crawler.settings._getcomposite('DOWNLOAD_HANDLERS'))
         for scheme, clspath in six.iteritems(handlers):
             self._schemes[scheme] = clspath
 

scrapy/downloadermiddlewares/defaultheaders.py

@@ -4,7 +4,7 @@ DefaultHeaders downloader middleware
 See documentation in docs/topics/downloader-middleware.rst
 """
 
-from scrapy.utils.conf import remove_none_values
+from scrapy.utils.python import without_none_values
 
 
 class DefaultHeadersMiddleware(object):
@@ -14,7 +14,7 @@ class DefaultHeadersMiddleware(object):
 
     @classmethod
     def from_crawler(cls, crawler):
-        headers = remove_none_values(crawler.settings['DEFAULT_REQUEST_HEADERS'])
+        headers = without_none_values(crawler.settings['DEFAULT_REQUEST_HEADERS'])
         return cls(headers.items())
 
     def process_request(self, request, spider):

scrapy/extensions/feedexport.py

@@ -18,11 +18,11 @@ from twisted.internet import defer, threads
 from w3lib.url import file_uri_to_path
 
 from scrapy import signals
-from scrapy.utils.conf import remove_none_values
 from scrapy.utils.ftp import ftp_makedirs_cwd
 from scrapy.exceptions import NotConfigured
 from scrapy.utils.misc import load_object
 from scrapy.utils.log import failure_to_exc_info
+from scrapy.utils.python import without_none_values
 
 logger = logging.getLogger(__name__)
 
@@ -196,7 +196,7 @@ class FeedExporter(object):
         return item
 
     def _load_components(self, setting_prefix):
-        conf = remove_none_values(self.settings._getcomposite(setting_prefix))
+        conf = without_none_values(self.settings._getcomposite(setting_prefix))
         d = {}
         for k, v in conf.items():
             try:

scrapy/utils/conf.py

@@ -1,6 +1,5 @@
 import os
 import sys
-import warnings
 from operator import itemgetter
 
 import six
@@ -8,6 +7,7 @@ from six.moves.configparser import SafeConfigParser
 
 from scrapy.settings import BaseSettings
 from scrapy.utils.deprecate import update_classpath
+from scrapy.utils.python import without_none_values
 
 
 def build_component_list(compdict, convert=update_classpath):
@@ -37,15 +37,10 @@ def build_component_list(compdict, convert=update_classpath):
     if isinstance(compdict, (list, tuple)):
         _check_components(compdict)
         return type(compdict)(convert(c) for c in compdict)
-    compdict = remove_none_values(_map_keys(compdict))
+    compdict = without_none_values(_map_keys(compdict))
     return [k for k, v in sorted(six.iteritems(compdict), key=itemgetter(1))]
 
 
-def remove_none_values(compdict):
-    """Return dict with all pairs that have value 'None' removed"""
-    return {k: v for k, v in six.iteritems(compdict) if v is not None}
-
-
 def arglist_to_dict(arglist):
     """Convert a list of arguments like ['arg1=val1', 'arg2=val2', ...] to a
     dict

scrapy/utils/python.py

@@ -324,3 +324,15 @@ def retry_on_eintr(function, *args, **kw):
         except IOError as e:
             if e.errno != errno.EINTR:
                 raise
+
+
+def without_none_values(iterable):
+    """Return a copy of `iterable` with all `None` entries removed.
+
+    If `iterable` is a mapping, return a dictionary where all pairs that have
+    value `None` have been removed.
+    """
+    try:
+        return {k: v for k, v in six.iteritems(iterable) if v is not None}
+    except AttributeError:
+        return type(iterable)((v for v in iterable if v is not None))

tests/test_utils_conf.py

@@ -1,8 +1,7 @@
 import unittest
 
 from scrapy.settings import BaseSettings
-from scrapy.utils.conf import (build_component_list, arglist_to_dict,
-                               remove_none_values)
+from scrapy.utils.conf import build_component_list, arglist_to_dict
 
 
 class BuildComponentListTest(unittest.TestCase):
@@ -53,12 +52,6 @@ class BuildComponentListTest(unittest.TestCase):
 
 class UtilsConfTestCase(unittest.TestCase):
 
-    def test_remove_none_values(self):
-        comps = {'one': 1, 'none': None, 'three': 3, 'four': 4}
-        compscopy = dict(comps)
-        del compscopy['none']
-        self.assertEqual(remove_none_values(comps), compscopy)
-
     def test_arglist_to_dict(self):
         self.assertEqual(arglist_to_dict(['arg1=val1', 'arg2=val2']),
             {'arg1': 'val1', 'arg2': 'val2'})

tests/test_utils_python.py

@@ -6,7 +6,8 @@ import six
 
 from scrapy.utils.python import (
     memoizemethod_noargs, isbinarytext, equal_attributes,
-    WeakKeyCache, stringify_dict, get_func_args, to_bytes, to_unicode)
+    WeakKeyCache, stringify_dict, get_func_args, to_bytes, to_unicode,
+    without_none_values)
 
 __doctests__ = ['scrapy.utils.python']
 
@@ -212,5 +213,12 @@ class UtilsPythonTestCase(unittest.TestCase):
         self.assertEqual(get_func_args(" ".join), [])
         self.assertEqual(get_func_args(operator.itemgetter(2)), [])
 
+    def test_without_none_values(self):
+        self.assertEqual(without_none_values([1, None, 3, 4]), [1, 3, 4])
+        self.assertEqual(without_none_values((1, None, 3, 4)), (1, 3, 4))
+        self.assertEqual(
+            without_none_values({'one': 1, 'none': None, 'three': 3, 'four': 4}),
+            {'one': 1, 'three': 3, 'four': 4})
+
 if __name__ == "__main__":
     unittest.main()