diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rst index 4b88c60fd..c18ed25ae 100644 --- a/docs/topics/media-pipeline.rst +++ b/docs/topics/media-pipeline.rst @@ -4,7 +4,7 @@ Downloading and processing files and images =========================================== -.. currentmodule:: scrapy.contrib.pipeline.images +.. currentmodule:: scrapy.pipelines.images Scrapy provides reusable :doc:`item pipelines ` for downloading fies attached to a particular item (for example, when you scrape @@ -114,11 +114,11 @@ To enable your media pipeline you must first add it to your project For Images Pipeline, use:: - ITEM_PIPELINES = {'scrapy.contrib.pipeline.images.ImagesPipeline': 1} + ITEM_PIPELINES = {'scrapy.pipelines.images.ImagesPipeline': 1} For Files Pipeline, use:: - ITEM_PIPELINES = {'scrapy.contrib.pipeline.files.FilesPipeline': 1} + ITEM_PIPELINES = {'scrapy.pipelines.files.FilesPipeline': 1} .. note:: @@ -258,7 +258,7 @@ By default, there are no size constraints, so all images are processed. Extending the Media Pipelines ============================= -.. module:: scrapy.contrib.pipeline.files +.. module:: scrapy.pipelines.files :synopsis: Files Pipeline See here the methods that you can override in your custom Files Pipeline: @@ -338,7 +338,7 @@ See here the methods that you can override in your custom Files Pipeline: By default, the :meth:`item_completed` method returns the item. -.. module:: scrapy.contrib.pipeline.images +.. module:: scrapy.pipelines.images :synopsis: Images Pipeline See here the methods that you can override in your custom Images Pipeline: @@ -374,7 +374,7 @@ Here is a full example of the Images Pipeline whose methods are examplified above:: import scrapy - from scrapy.contrib.pipeline.images import ImagesPipeline + from scrapy.pipelines.images import ImagesPipeline from scrapy.exceptions import DropItem class MyImagesPipeline(ImagesPipeline): diff --git a/scrapy/contrib/pipeline/__init__.py b/scrapy/pipelines/__init__.py similarity index 100% rename from scrapy/contrib/pipeline/__init__.py rename to scrapy/pipelines/__init__.py diff --git a/scrapy/contrib/pipeline/files.py b/scrapy/pipelines/files.py similarity index 99% rename from scrapy/contrib/pipeline/files.py rename to scrapy/pipelines/files.py index 4809005d1..2b17e0f0a 100644 --- a/scrapy/contrib/pipeline/files.py +++ b/scrapy/pipelines/files.py @@ -21,7 +21,7 @@ except ImportError: from twisted.internet import defer, threads -from scrapy.contrib.pipeline.media import MediaPipeline +from scrapy.pipelines.media import MediaPipeline from scrapy.exceptions import NotConfigured, IgnoreRequest from scrapy.http import Request from scrapy.utils.misc import md5sum diff --git a/scrapy/contrib/pipeline/images.py b/scrapy/pipelines/images.py similarity index 97% rename from scrapy/contrib/pipeline/images.py rename to scrapy/pipelines/images.py index 3cbfed49b..8b3bc2222 100644 --- a/scrapy/contrib/pipeline/images.py +++ b/scrapy/pipelines/images.py @@ -17,8 +17,8 @@ from PIL import Image from scrapy.utils.misc import md5sum from scrapy.http import Request from scrapy.exceptions import DropItem -#TODO: from scrapy.contrib.pipeline.media import MediaPipeline -from scrapy.contrib.pipeline.files import FileException, FilesPipeline +#TODO: from scrapy.pipelines.media import MediaPipeline +from scrapy.pipelines.files import FileException, FilesPipeline class NoimagesDrop(DropItem): diff --git a/scrapy/contrib/pipeline/media.py b/scrapy/pipelines/media.py similarity index 100% rename from scrapy/contrib/pipeline/media.py rename to scrapy/pipelines/media.py diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 17db550a7..f4dcdc1a2 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -159,7 +159,7 @@ HTTPCACHE_DBM_MODULE = 'anydbm' HTTPCACHE_POLICY = 'scrapy.contrib.httpcache.DummyPolicy' HTTPCACHE_GZIP = False -ITEM_PROCESSOR = 'scrapy.contrib.pipeline.ItemPipelineManager' +ITEM_PROCESSOR = 'scrapy.pipelines.ItemPipelineManager' ITEM_PIPELINES = {} ITEM_PIPELINES_BASE = {} diff --git a/tests/py3-ignores.txt b/tests/py3-ignores.txt index 69df2f7a2..a29b419b0 100644 --- a/tests/py3-ignores.txt +++ b/tests/py3-ignores.txt @@ -85,6 +85,8 @@ scrapy/core/downloader/handlers/ftp.py scrapy/core/downloader/webclient.py scrapy/contrib/pipeline/images.py scrapy/contrib/pipeline/files.py +scrapy/pipelines/images.py +scrapy/pipelines/files.py scrapy/contrib/linkextractors/sgml.py scrapy/contrib/linkextractors/regex.py scrapy/contrib/linkextractors/htmlparser.py diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py index 84fe4927d..b12f41174 100644 --- a/tests/test_pipeline_files.py +++ b/tests/test_pipeline_files.py @@ -8,7 +8,7 @@ from shutil import rmtree from twisted.trial import unittest from twisted.internet import defer -from scrapy.contrib.pipeline.files import FilesPipeline, FSFilesStore +from scrapy.pipelines.files import FilesPipeline, FSFilesStore from scrapy.item import Item, Field from scrapy.http import Request, Response from scrapy.settings import Settings diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py index f5750b4fc..04cec4b8e 100644 --- a/tests/test_pipeline_images.py +++ b/tests/test_pipeline_images.py @@ -9,7 +9,7 @@ from twisted.trial import unittest from scrapy.item import Item, Field from scrapy.http import Request, Response from scrapy.settings import Settings -from scrapy.contrib.pipeline.images import ImagesPipeline +from scrapy.pipelines.images import ImagesPipeline skip = False try: diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py index 0e946303b..24ba9d64a 100644 --- a/tests/test_pipeline_media.py +++ b/tests/test_pipeline_media.py @@ -8,7 +8,7 @@ from twisted.internet.defer import Deferred, inlineCallbacks from scrapy.http import Request, Response from scrapy.spider import Spider from scrapy.utils.request import request_fingerprint -from scrapy.contrib.pipeline.media import MediaPipeline +from scrapy.pipelines.media import MediaPipeline from scrapy.utils.signal import disconnect_all from scrapy import signals