From 3fce2842ea43a79ca90e5455ae8a60e69956e90d Mon Sep 17 00:00:00 2001 From: Pawel Miech Date: Fri, 13 May 2016 12:35:35 +0200 Subject: [PATCH] [image_pipeline] bring back uppercase pipeline attributes allow users to have class attributes on image pipelines. This assumes that class attributes are useful if users want to have different pipeline classes inhriting from ImagePipeline. --- scrapy/pipelines/files.py | 9 ++++++--- scrapy/pipelines/images.py | 21 +++++++++++++-------- scrapy/settings/default_settings.py | 7 ------- tests/test_pipeline_images.py | 12 ++++++++---- 4 files changed, 27 insertions(+), 22 deletions(-) diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index c94794173..3e6ad554d 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -214,11 +214,14 @@ class FilesPipeline(MediaPipeline): """ MEDIA_NAME = "file" + EXPIRES = 90 STORE_SCHEMES = { '': FSFilesStore, 'file': FSFilesStore, 's3': S3FilesStore, } + DEFAULT_FILES_URLS_FIELD = 'file_urls' + DEFAULT_FILES_RESULT_FIELD = 'files' def __init__(self, store_uri, download_func=None, settings=None): if not store_uri: @@ -228,9 +231,9 @@ class FilesPipeline(MediaPipeline): settings = Settings(settings) self.store = self._get_store(store_uri) - self.expires = settings.getint('FILES_EXPIRES') - self.files_urls_field = settings.get('FILES_URLS_FIELD') - self.files_result_field = settings.get('FILES_RESULT_FIELD') + self.expires = settings.getint('FILES_EXPIRES', self.EXPIRES) + self.files_urls_field = settings.get('FILES_URLS_FIELD', self.DEFAULT_FILES_URLS_FIELD) + self.files_result_field = settings.get('FILES_RESULT_FIELD', self.DEFAULT_FILES_RESULT_FIELD) super(FilesPipeline, self).__init__(download_func=download_func) diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py index c597b6cca..ac78ee6c0 100644 --- a/scrapy/pipelines/images.py +++ b/scrapy/pipelines/images.py @@ -37,26 +37,31 @@ class ImagesPipeline(FilesPipeline): """ MEDIA_NAME = 'image' + MIN_WIDTH = 0 + MIN_HEIGHT = 0 + EXPIRES = 0 + THUMBS = {} + IMAGES_URLS_FIELD = 'image_urls' + IMAGES_RESULT_FIELD = 'images' def __init__(self, store_uri, download_func=None, settings=None): super(ImagesPipeline, self).__init__(store_uri, settings=settings, download_func=download_func) - + if isinstance(settings, dict) or settings is None: settings = Settings(settings) - self.expires = settings.getint('IMAGES_EXPIRES') - self.images_urls_field = settings.get('IMAGES_URLS_FIELD') - self.images_result_field = settings.get('IMAGES_RESULT_FIELD') - self.min_width = settings.getint('IMAGES_MIN_WIDTH') - self.min_height = settings.getint('IMAGES_MIN_HEIGHT') - self.thumbs = settings.get('IMAGES_THUMBS') + self.expires = settings.getint('IMAGES_EXPIRES', self.EXPIRES) + self.images_urls_field = settings.get('IMAGES_URLS_FIELD', self.IMAGES_URLS_FIELD) + self.images_result_field = settings.get('IMAGES_RESULT_FIELD', self.IMAGES_RESULT_FIELD) + self.min_width = settings.getint('IMAGES_MIN_WIDTH', self.MIN_WIDTH) + self.min_height = settings.getint('IMAGES_MIN_HEIGHT', self.MIN_HEIGHT) + self.thumbs = settings.get('IMAGES_THUMBS', self.THUMBS) @classmethod def from_settings(cls, settings): s3store = cls.STORE_SCHEMES['s3'] s3store.AWS_ACCESS_KEY_ID = settings['AWS_ACCESS_KEY_ID'] s3store.AWS_SECRET_ACCESS_KEY = settings['AWS_SECRET_ACCESS_KEY'] - store_uri = settings['IMAGES_STORE'] return cls(store_uri, settings=settings) diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 8f064f81e..1e447e9e2 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -179,13 +179,6 @@ HTTPCACHE_GZIP = False HTTPPROXY_AUTH_ENCODING = 'latin-1' -IMAGES_MIN_WIDTH = 0 -IMAGES_MIN_HEIGHT = 0 -IMAGES_EXPIRES = 90 -IMAGES_THUMBS = {} -IMAGES_URLS_FIELD = 'image_urls' -IMAGES_RESULT_FIELD = 'images' - ITEM_PROCESSOR = 'scrapy.pipelines.ItemPipelineManager' ITEM_PIPELINES = {} diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py index f48547b0f..3ce138000 100644 --- a/tests/test_pipeline_images.py +++ b/tests/test_pipeline_images.py @@ -224,19 +224,22 @@ class ImagesPipelineTestCaseCustomSettings(unittest.TestCase): def test_images_urls_field(self): another_pipeline = ImagesPipeline.from_settings(Settings({'IMAGES_STORE': self.tempdir, 'IMAGES_URLS_FIELD': 'funny_field'})) - self.assertEqual(self.pipeline.images_urls_field, self.default_settings.get('IMAGES_URLS_FIELD')) + default = self.pipeline.IMAGES_URLS_FIELD + self.assertEqual(self.pipeline.images_urls_field, self.default_settings.get('IMAGES_URLS_FIELD', default)) self.assertEqual(another_pipeline.images_urls_field, 'funny_field') def test_images_result_field(self): another_pipeline = ImagesPipeline.from_settings(Settings({'IMAGES_STORE': self.tempdir, 'IMAGES_RESULT_FIELD': 'funny_field'})) - self.assertEqual(self.pipeline.images_result_field, self.default_settings.get('IMAGES_RESULT_FIELD')) + default = self.pipeline.IMAGES_RESULT_FIELD + self.assertEqual(self.pipeline.images_result_field, self.default_settings.get('IMAGES_RESULT_FIELD', default)) self.assertEqual(another_pipeline.images_result_field, 'funny_field') def test_min_width(self): another_pipeline = ImagesPipeline.from_settings(Settings({'IMAGES_STORE': self.tempdir, 'IMAGES_MIN_WIDTH': 42})) - self.assertEqual(self.pipeline.min_width, self.default_settings.getint('IMAGES_MIN_WIDTH')) + default = self.pipeline.MIN_WIDTH + self.assertEqual(self.pipeline.min_width, self.default_settings.getint('IMAGES_MIN_WIDTH', default)) self.assertEqual(another_pipeline.min_width, 42) def test_min_height(self): @@ -249,7 +252,8 @@ class ImagesPipelineTestCaseCustomSettings(unittest.TestCase): custom_thumbs = {'small': (50, 50), 'big': (270, 270)} another_pipeline = ImagesPipeline.from_settings(Settings({'IMAGES_STORE': self.tempdir, 'IMAGES_THUMBS': custom_thumbs})) - self.assertEqual(self.pipeline.thumbs, self.default_settings.get('IMAGES_THUMBS')) + default = self.pipeline.THUMBS + self.assertEqual(self.pipeline.thumbs, self.default_settings.get('IMAGES_THUMBS', default)) self.assertEqual(another_pipeline.thumbs, custom_thumbs)