diff --git a/docs/news.rst b/docs/news.rst index 1a156e2cc..41a99c6ff 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -8,6 +8,7 @@ Release notes - Request/Response url/body attributes are now immutable (modifying them had been deprecated for a long time) +- :setting:`ITEM_PIPELINES` is now defined as a dict (instead of a list) 0.18.2 (released 2013-09-03) ---------------------------- diff --git a/docs/topics/images.rst b/docs/topics/images.rst index 9c1de4dd0..f7f33b687 100644 --- a/docs/topics/images.rst +++ b/docs/topics/images.rst @@ -88,7 +88,7 @@ Enabling your Images Pipeline To enable your images pipeline you must first add it to your project :setting:`ITEM_PIPELINES` setting:: - ITEM_PIPELINES = ['scrapy.contrib.pipeline.images.ImagesPipeline'] + ITEM_PIPELINES = {'scrapy.contrib.pipeline.images.ImagesPipeline': 1} And set the :setting:`IMAGES_STORE` setting to a valid directory that will be used for storing the downloaded images. Otherwise the pipeline will remain diff --git a/docs/topics/item-pipeline.rst b/docs/topics/item-pipeline.rst index f6d30e302..025469528 100644 --- a/docs/topics/item-pipeline.rst +++ b/docs/topics/item-pipeline.rst @@ -132,9 +132,9 @@ Activating an Item Pipeline component ===================================== To activate an Item Pipeline component you must add its class to the -:setting:`ITEM_PIPELINES` list, like in the following example:: +:setting:`ITEM_PIPELINES` setting, like in the following example:: - ITEM_PIPELINES = [ - 'myproject.pipeline.PricePipeline', - 'myproject.pipeline.JsonWriterPipeline', - ] + ITEM_PIPELINES = { + 'myproject.pipeline.PricePipeline': 300, + 'myproject.pipeline.JsonWriterPipeline': 800, + } diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index dce63ff30..6f738da91 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -453,16 +453,31 @@ and the :ref:`list of available extensions `. ITEM_PIPELINES -------------- -Default: ``[]`` +Default: ``{}`` -The item pipelines to use (a list of classes). +A dict containing the item pipelines to use, and their orders. The dict is +empty by default order values are arbitrary but it's customary to define them +in the 0-1000 range. + +Lists are supported in :setting:`ITEM_PIPELINES` for backwards compatibility, +but they are deprecated. Example:: - ITEM_PIPELINES = [ - 'mybot.pipeline.validate.ValidateMyItem', - 'mybot.pipeline.validate.StoreMyItem' - ] + ITEM_PIPELINES = { + 'mybot.pipeline.validate.ValidateMyItem': 300, + 'mybot.pipeline.validate.StoreMyItem': 800, + } + +.. setting:: ITEM_PIPELINES_BASE + +ITEM_PIPELINES_BASE +------------------- + +Default: ``{}`` + +A dict containing the pipelines enabled by default in Scrapy. You should never +modify this setting in your project, modify :setting:`ITEM_PIPELINES` instead. .. setting:: LOG_ENABLED diff --git a/scrapy/contrib/pipeline/__init__.py b/scrapy/contrib/pipeline/__init__.py index c1c7e5a22..5c723cbaa 100644 --- a/scrapy/contrib/pipeline/__init__.py +++ b/scrapy/contrib/pipeline/__init__.py @@ -5,6 +5,7 @@ See documentation in docs/item-pipeline.rst """ from scrapy.middleware import MiddlewareManager +from scrapy.utils.conf import build_component_list class ItemPipelineManager(MiddlewareManager): @@ -12,7 +13,16 @@ class ItemPipelineManager(MiddlewareManager): @classmethod def _get_mwlist_from_settings(cls, settings): - return settings.getlist('ITEM_PIPELINES') + item_pipelines = settings['ITEM_PIPELINES'] + if isinstance(item_pipelines, (tuple, list)): + from scrapy.exceptions import ScrapyDeprecationWarning + import warnings + warnings.warn('ITEM_PIPELINES defined as a list is deprecated, switch to a dict', + category=ScrapyDeprecationWarning, stacklevel=1) + # convert old ITEM_PIPELINE list to a dict with order 500 + item_pipelines = dict(zip(item_pipelines, range(500, 500+len(item_pipelines)))) + return build_component_list(settings['ITEM_PIPELINES_BASE'], + settings['ITEM_PIPELINES']) def _add_middleware(self, pipe): super(ItemPipelineManager, self)._add_middleware(pipe) diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index a16110b52..e7475c3d5 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -150,8 +150,8 @@ HTTPCACHE_POLICY = 'scrapy.contrib.httpcache.DummyPolicy' ITEM_PROCESSOR = 'scrapy.contrib.pipeline.ItemPipelineManager' -# Item pipelines are typically set in specific commands settings -ITEM_PIPELINES = [] +ITEM_PIPELINES = {} +ITEM_PIPELINES_BASE = {} LOG_ENABLED = True LOG_ENCODING = 'utf-8' diff --git a/scrapy/tests/test_commands.py b/scrapy/tests/test_commands.py index 5927a2eb3..a6562de17 100644 --- a/scrapy/tests/test_commands.py +++ b/scrapy/tests/test_commands.py @@ -212,8 +212,8 @@ class MyPipeline(object): fname = abspath(join(self.proj_mod_path, 'settings.py')) with open(fname, 'a') as f: f.write(""" -ITEM_PIPELINES = ['{0}.pipelines.MyPipeline'] -""".format(self.project_name)) +ITEM_PIPELINES = {'%s.pipelines.MyPipeline': 1} +""" % self.project_name) def test_spider_arguments(self): p = self.proc('parse', '--spider', self.spider_name, '-a', 'test_arg=1',