RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

[image_pipeline] bring back uppercase pipeline attributes 

allow users to have class attributes on image pipelines. This assumes
that class attributes are useful if users want to have different pipeline
classes inhriting from ImagePipeline.
This commit is contained in:
Pawel Miech 2016-05-13 12:35:35 +02:00 committed by Paul Tremberth
parent 308da48806
commit 3fce2842ea
4 changed files with 27 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
scrapy/pipelines/files.py
View File

@ -214,11 +214,14 @@ class FilesPipeline(MediaPipeline):
"""
MEDIA_NAME = "file"
EXPIRES = 90
STORE_SCHEMES = {
'': FSFilesStore,
'file': FSFilesStore,
's3': S3FilesStore,
}
DEFAULT_FILES_URLS_FIELD = 'file_urls'
DEFAULT_FILES_RESULT_FIELD = 'files'
def __init__(self, store_uri, download_func=None, settings=None):
if not store_uri:
@ -228,9 +231,9 @@ class FilesPipeline(MediaPipeline):
settings = Settings(settings)
self.store = self._get_store(store_uri)
self.expires = settings.getint('FILES_EXPIRES')
self.files_urls_field = settings.get('FILES_URLS_FIELD')
self.files_result_field = settings.get('FILES_RESULT_FIELD')
self.expires = settings.getint('FILES_EXPIRES', self.EXPIRES)
self.files_urls_field = settings.get('FILES_URLS_FIELD', self.DEFAULT_FILES_URLS_FIELD)
self.files_result_field = settings.get('FILES_RESULT_FIELD', self.DEFAULT_FILES_RESULT_FIELD)
super(FilesPipeline, self).__init__(download_func=download_func)

21
scrapy/pipelines/images.py
View File

@ -37,26 +37,31 @@ class ImagesPipeline(FilesPipeline):
"""
MEDIA_NAME = 'image'
MIN_WIDTH = 0
MIN_HEIGHT = 0
EXPIRES = 0
THUMBS = {}
IMAGES_URLS_FIELD = 'image_urls'
IMAGES_RESULT_FIELD = 'images'
def __init__(self, store_uri, download_func=None, settings=None):
super(ImagesPipeline, self).__init__(store_uri, settings=settings, download_func=download_func)
if isinstance(settings, dict) or settings is None:
settings = Settings(settings)
self.expires = settings.getint('IMAGES_EXPIRES')
self.images_urls_field = settings.get('IMAGES_URLS_FIELD')
self.images_result_field = settings.get('IMAGES_RESULT_FIELD')
self.min_width = settings.getint('IMAGES_MIN_WIDTH')
self.min_height = settings.getint('IMAGES_MIN_HEIGHT')
self.thumbs = settings.get('IMAGES_THUMBS')
self.expires = settings.getint('IMAGES_EXPIRES', self.EXPIRES)
self.images_urls_field = settings.get('IMAGES_URLS_FIELD', self.IMAGES_URLS_FIELD)
self.images_result_field = settings.get('IMAGES_RESULT_FIELD', self.IMAGES_RESULT_FIELD)
self.min_width = settings.getint('IMAGES_MIN_WIDTH', self.MIN_WIDTH)
self.min_height = settings.getint('IMAGES_MIN_HEIGHT', self.MIN_HEIGHT)
self.thumbs = settings.get('IMAGES_THUMBS', self.THUMBS)
@classmethod
def from_settings(cls, settings):
s3store = cls.STORE_SCHEMES['s3']
s3store.AWS_ACCESS_KEY_ID = settings['AWS_ACCESS_KEY_ID']
s3store.AWS_SECRET_ACCESS_KEY = settings['AWS_SECRET_ACCESS_KEY']
store_uri = settings['IMAGES_STORE']
return cls(store_uri, settings=settings)

7
scrapy/settings/default_settings.py
View File

@ -179,13 +179,6 @@ HTTPCACHE_GZIP = False
HTTPPROXY_AUTH_ENCODING = 'latin-1'
IMAGES_MIN_WIDTH = 0
IMAGES_MIN_HEIGHT = 0
IMAGES_EXPIRES = 90
IMAGES_THUMBS = {}
IMAGES_URLS_FIELD = 'image_urls'
IMAGES_RESULT_FIELD = 'images'
ITEM_PROCESSOR = 'scrapy.pipelines.ItemPipelineManager'
ITEM_PIPELINES = {}

12
tests/test_pipeline_images.py
View File

@ -224,19 +224,22 @@ class ImagesPipelineTestCaseCustomSettings(unittest.TestCase):
def test_images_urls_field(self):
another_pipeline = ImagesPipeline.from_settings(Settings({'IMAGES_STORE': self.tempdir,
'IMAGES_URLS_FIELD': 'funny_field'}))
self.assertEqual(self.pipeline.images_urls_field, self.default_settings.get('IMAGES_URLS_FIELD'))
default = self.pipeline.IMAGES_URLS_FIELD
self.assertEqual(self.pipeline.images_urls_field, self.default_settings.get('IMAGES_URLS_FIELD', default))
self.assertEqual(another_pipeline.images_urls_field, 'funny_field')
def test_images_result_field(self):
another_pipeline = ImagesPipeline.from_settings(Settings({'IMAGES_STORE': self.tempdir,
'IMAGES_RESULT_FIELD': 'funny_field'}))
self.assertEqual(self.pipeline.images_result_field, self.default_settings.get('IMAGES_RESULT_FIELD'))
default = self.pipeline.IMAGES_RESULT_FIELD
self.assertEqual(self.pipeline.images_result_field, self.default_settings.get('IMAGES_RESULT_FIELD', default))
self.assertEqual(another_pipeline.images_result_field, 'funny_field')
def test_min_width(self):
another_pipeline = ImagesPipeline.from_settings(Settings({'IMAGES_STORE': self.tempdir,
'IMAGES_MIN_WIDTH': 42}))
self.assertEqual(self.pipeline.min_width, self.default_settings.getint('IMAGES_MIN_WIDTH'))
default = self.pipeline.MIN_WIDTH
self.assertEqual(self.pipeline.min_width, self.default_settings.getint('IMAGES_MIN_WIDTH', default))
self.assertEqual(another_pipeline.min_width, 42)
def test_min_height(self):
@ -249,7 +252,8 @@ class ImagesPipelineTestCaseCustomSettings(unittest.TestCase):
custom_thumbs = {'small': (50, 50), 'big': (270, 270)}
another_pipeline = ImagesPipeline.from_settings(Settings({'IMAGES_STORE': self.tempdir,
'IMAGES_THUMBS': custom_thumbs}))
self.assertEqual(self.pipeline.thumbs, self.default_settings.get('IMAGES_THUMBS'))
default = self.pipeline.THUMBS
self.assertEqual(self.pipeline.thumbs, self.default_settings.get('IMAGES_THUMBS', default))
self.assertEqual(another_pipeline.thumbs, custom_thumbs)