Merge pull request #393 from scrapy/itempipe-dict

Make ITEM_PIPELINE setting a dict

docs/news.rst

@@ -8,6 +8,7 @@ Release notes
 
 - Request/Response url/body attributes are now immutable (modifying them had
   been deprecated for a long time)
+- :setting:`ITEM_PIPELINES` is now defined as a dict (instead of a list)
 
 0.18.2 (released 2013-09-03)
 ----------------------------

docs/topics/images.rst

@@ -88,7 +88,7 @@ Enabling your Images Pipeline
 To enable your images pipeline you must first add it to your project
 :setting:`ITEM_PIPELINES` setting::
 
-    ITEM_PIPELINES = ['scrapy.contrib.pipeline.images.ImagesPipeline']
+    ITEM_PIPELINES = {'scrapy.contrib.pipeline.images.ImagesPipeline': 1}
 
 And set the :setting:`IMAGES_STORE` setting to a valid directory that will be
 used for storing the downloaded images. Otherwise the pipeline will remain

docs/topics/item-pipeline.rst

@@ -132,9 +132,9 @@ Activating an Item Pipeline component
 =====================================
 
 To activate an Item Pipeline component you must add its class to the
-:setting:`ITEM_PIPELINES` list, like in the following example::
+:setting:`ITEM_PIPELINES` setting, like in the following example::
 
-   ITEM_PIPELINES = [
-       'myproject.pipeline.PricePipeline',
-       'myproject.pipeline.JsonWriterPipeline',
-   ]
+   ITEM_PIPELINES = {
+       'myproject.pipeline.PricePipeline': 300,
+       'myproject.pipeline.JsonWriterPipeline': 800,
+   }

docs/topics/settings.rst

@@ -453,16 +453,31 @@ and the :ref:`list of available extensions <topics-extensions-ref>`.
 ITEM_PIPELINES
 --------------
 
-Default: ``[]``
+Default: ``{}``
 
-The item pipelines to use (a list of classes).
+A dict containing the item pipelines to use, and their orders. The dict is
+empty by default order values are arbitrary but it's customary to define them
+in the 0-1000 range.
+
+Lists are supported in :setting:`ITEM_PIPELINES` for backwards compatibility,
+but they are deprecated.
 
 Example::
 
-   ITEM_PIPELINES = [
-       'mybot.pipeline.validate.ValidateMyItem',
-       'mybot.pipeline.validate.StoreMyItem'
-   ]
+   ITEM_PIPELINES = {
+       'mybot.pipeline.validate.ValidateMyItem': 300,
+       'mybot.pipeline.validate.StoreMyItem': 800,
+   }
+
+.. setting:: ITEM_PIPELINES_BASE
+
+ITEM_PIPELINES_BASE
+-------------------
+
+Default: ``{}``
+
+A dict containing the pipelines enabled by default in Scrapy. You should never
+modify this setting in your project, modify :setting:`ITEM_PIPELINES` instead.
 
 .. setting:: LOG_ENABLED
 

scrapy/contrib/pipeline/__init__.py

@@ -5,6 +5,7 @@ See documentation in docs/item-pipeline.rst
 """
 
 from scrapy.middleware import MiddlewareManager
+from scrapy.utils.conf import build_component_list
 
 class ItemPipelineManager(MiddlewareManager):
 
@@ -12,7 +13,16 @@ class ItemPipelineManager(MiddlewareManager):
 
     @classmethod
     def _get_mwlist_from_settings(cls, settings):
-        return settings.getlist('ITEM_PIPELINES')
+        item_pipelines = settings['ITEM_PIPELINES']
+        if isinstance(item_pipelines, (tuple, list)):
+            from scrapy.exceptions import ScrapyDeprecationWarning
+            import warnings
+            warnings.warn('ITEM_PIPELINES defined as a list is deprecated, switch to a dict',
+                category=ScrapyDeprecationWarning, stacklevel=1)
+            # convert old ITEM_PIPELINE list to a dict with order 500
+            item_pipelines = dict(zip(item_pipelines, range(500, 500+len(item_pipelines))))
+        return build_component_list(settings['ITEM_PIPELINES_BASE'],
+            settings['ITEM_PIPELINES'])
 
     def _add_middleware(self, pipe):
         super(ItemPipelineManager, self)._add_middleware(pipe)

scrapy/settings/default_settings.py

@@ -150,8 +150,8 @@ HTTPCACHE_POLICY = 'scrapy.contrib.httpcache.DummyPolicy'
 
 ITEM_PROCESSOR = 'scrapy.contrib.pipeline.ItemPipelineManager'
 
-# Item pipelines are typically set in specific commands settings
-ITEM_PIPELINES = []
+ITEM_PIPELINES = {}
+ITEM_PIPELINES_BASE = {}
 
 LOG_ENABLED = True
 LOG_ENCODING = 'utf-8'

scrapy/tests/test_commands.py

@@ -212,8 +212,8 @@ class MyPipeline(object):
         fname = abspath(join(self.proj_mod_path, 'settings.py'))
         with open(fname, 'a') as f:
             f.write("""
-ITEM_PIPELINES = ['{0}.pipelines.MyPipeline']
-""".format(self.project_name))
+ITEM_PIPELINES = {'%s.pipelines.MyPipeline': 1}
+""" % self.project_name)
 
     def test_spider_arguments(self):
         p = self.proc('parse', '--spider', self.spider_name, '-a', 'test_arg=1',