diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index c94794173..3e6ad554d 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -214,11 +214,14 @@ class FilesPipeline(MediaPipeline): """ MEDIA_NAME = "file" + EXPIRES = 90 STORE_SCHEMES = { '': FSFilesStore, 'file': FSFilesStore, 's3': S3FilesStore, } + DEFAULT_FILES_URLS_FIELD = 'file_urls' + DEFAULT_FILES_RESULT_FIELD = 'files' def __init__(self, store_uri, download_func=None, settings=None): if not store_uri: @@ -228,9 +231,9 @@ class FilesPipeline(MediaPipeline): settings = Settings(settings) self.store = self._get_store(store_uri) - self.expires = settings.getint('FILES_EXPIRES') - self.files_urls_field = settings.get('FILES_URLS_FIELD') - self.files_result_field = settings.get('FILES_RESULT_FIELD') + self.expires = settings.getint('FILES_EXPIRES', self.EXPIRES) + self.files_urls_field = settings.get('FILES_URLS_FIELD', self.DEFAULT_FILES_URLS_FIELD) + self.files_result_field = settings.get('FILES_RESULT_FIELD', self.DEFAULT_FILES_RESULT_FIELD) super(FilesPipeline, self).__init__(download_func=download_func) diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py index c597b6cca..ac78ee6c0 100644 --- a/scrapy/pipelines/images.py +++ b/scrapy/pipelines/images.py @@ -37,26 +37,31 @@ class ImagesPipeline(FilesPipeline): """ MEDIA_NAME = 'image' + MIN_WIDTH = 0 + MIN_HEIGHT = 0 + EXPIRES = 0 + THUMBS = {} + IMAGES_URLS_FIELD = 'image_urls' + IMAGES_RESULT_FIELD = 'images' def __init__(self, store_uri, download_func=None, settings=None): super(ImagesPipeline, self).__init__(store_uri, settings=settings, download_func=download_func) - + if isinstance(settings, dict) or settings is None: settings = Settings(settings) - self.expires = settings.getint('IMAGES_EXPIRES') - self.images_urls_field = settings.get('IMAGES_URLS_FIELD') - self.images_result_field = settings.get('IMAGES_RESULT_FIELD') - self.min_width = settings.getint('IMAGES_MIN_WIDTH') - self.min_height = settings.getint('IMAGES_MIN_HEIGHT') - self.thumbs = settings.get('IMAGES_THUMBS') + self.expires = settings.getint('IMAGES_EXPIRES', self.EXPIRES) + self.images_urls_field = settings.get('IMAGES_URLS_FIELD', self.IMAGES_URLS_FIELD) + self.images_result_field = settings.get('IMAGES_RESULT_FIELD', self.IMAGES_RESULT_FIELD) + self.min_width = settings.getint('IMAGES_MIN_WIDTH', self.MIN_WIDTH) + self.min_height = settings.getint('IMAGES_MIN_HEIGHT', self.MIN_HEIGHT) + self.thumbs = settings.get('IMAGES_THUMBS', self.THUMBS) @classmethod def from_settings(cls, settings): s3store = cls.STORE_SCHEMES['s3'] s3store.AWS_ACCESS_KEY_ID = settings['AWS_ACCESS_KEY_ID'] s3store.AWS_SECRET_ACCESS_KEY = settings['AWS_SECRET_ACCESS_KEY'] - store_uri = settings['IMAGES_STORE'] return cls(store_uri, settings=settings) diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 8f064f81e..1e447e9e2 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -179,13 +179,6 @@ HTTPCACHE_GZIP = False HTTPPROXY_AUTH_ENCODING = 'latin-1' -IMAGES_MIN_WIDTH = 0 -IMAGES_MIN_HEIGHT = 0 -IMAGES_EXPIRES = 90 -IMAGES_THUMBS = {} -IMAGES_URLS_FIELD = 'image_urls' -IMAGES_RESULT_FIELD = 'images' - ITEM_PROCESSOR = 'scrapy.pipelines.ItemPipelineManager' ITEM_PIPELINES = {} diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py index f48547b0f..3ce138000 100644 --- a/tests/test_pipeline_images.py +++ b/tests/test_pipeline_images.py @@ -224,19 +224,22 @@ class ImagesPipelineTestCaseCustomSettings(unittest.TestCase): def test_images_urls_field(self): another_pipeline = ImagesPipeline.from_settings(Settings({'IMAGES_STORE': self.tempdir, 'IMAGES_URLS_FIELD': 'funny_field'})) - self.assertEqual(self.pipeline.images_urls_field, self.default_settings.get('IMAGES_URLS_FIELD')) + default = self.pipeline.IMAGES_URLS_FIELD + self.assertEqual(self.pipeline.images_urls_field, self.default_settings.get('IMAGES_URLS_FIELD', default)) self.assertEqual(another_pipeline.images_urls_field, 'funny_field') def test_images_result_field(self): another_pipeline = ImagesPipeline.from_settings(Settings({'IMAGES_STORE': self.tempdir, 'IMAGES_RESULT_FIELD': 'funny_field'})) - self.assertEqual(self.pipeline.images_result_field, self.default_settings.get('IMAGES_RESULT_FIELD')) + default = self.pipeline.IMAGES_RESULT_FIELD + self.assertEqual(self.pipeline.images_result_field, self.default_settings.get('IMAGES_RESULT_FIELD', default)) self.assertEqual(another_pipeline.images_result_field, 'funny_field') def test_min_width(self): another_pipeline = ImagesPipeline.from_settings(Settings({'IMAGES_STORE': self.tempdir, 'IMAGES_MIN_WIDTH': 42})) - self.assertEqual(self.pipeline.min_width, self.default_settings.getint('IMAGES_MIN_WIDTH')) + default = self.pipeline.MIN_WIDTH + self.assertEqual(self.pipeline.min_width, self.default_settings.getint('IMAGES_MIN_WIDTH', default)) self.assertEqual(another_pipeline.min_width, 42) def test_min_height(self): @@ -249,7 +252,8 @@ class ImagesPipelineTestCaseCustomSettings(unittest.TestCase): custom_thumbs = {'small': (50, 50), 'big': (270, 270)} another_pipeline = ImagesPipeline.from_settings(Settings({'IMAGES_STORE': self.tempdir, 'IMAGES_THUMBS': custom_thumbs})) - self.assertEqual(self.pipeline.thumbs, self.default_settings.get('IMAGES_THUMBS')) + default = self.pipeline.THUMBS + self.assertEqual(self.pipeline.thumbs, self.default_settings.get('IMAGES_THUMBS', default)) self.assertEqual(another_pipeline.thumbs, custom_thumbs)