From d40de1e013a49449a8d216838cebb83abf70c192 Mon Sep 17 00:00:00 2001 From: Pawel Miech Date: Tue, 12 Jul 2016 13:58:36 +0200 Subject: [PATCH] [image_pipeline] style edits * 80 characters line limit * shortening some code * removed dead code * add doctest for _key_for_pipe function --- docs/topics/media-pipeline.rst | 15 ++++++++------- scrapy/pipelines/files.py | 10 ++++++---- scrapy/pipelines/images.py | 18 ++++++++++-------- scrapy/pipelines/media.py | 17 ++++++++--------- 4 files changed, 32 insertions(+), 28 deletions(-) diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rst index 6cbac913c..f18789ab0 100644 --- a/docs/topics/media-pipeline.rst +++ b/docs/topics/media-pipeline.rst @@ -191,10 +191,11 @@ For the Images Pipeline, set :setting:`IMAGES_URLS_FIELD` and/or If you need something more complex and want to override the custom pipeline behaviour, see :ref:`topics-media-pipeline-override`. -If you have multiple image pipelines inheriting from ImagePipeline and you want to have different settings in different pipelines -you can set setting keys preceded with uppercase name of your pipeline class. E.g. if your pipeline is called -MyPipeline and you want to have custom IMAGES_URLS_FIELD you define setting MYPIPELINE_IMAGES_URLS_FIELD and your custom -settings will be used. +If you have multiple image pipelines inheriting from ImagePipeline and you want +to have different settings in different pipelines you can set setting keys +preceded with uppercase name of your pipeline class. E.g. if your pipeline is +called MyPipeline and you want to have custom IMAGES_URLS_FIELD you define +setting MYPIPELINE_IMAGES_URLS_FIELD and your custom settings will be used. Additional features @@ -219,9 +220,9 @@ specifies the delay in number of days:: The default value for both settings is 90 days. -If you have pipeline that subclasses FilesPipeline and you'd like to have different setting -for it you can set setting keys preceded by uppercase class name. E.g. given pipeline class -called MyPipeline you can set setting key: +If you have pipeline that subclasses FilesPipeline and you'd like to have +different setting for it you can set setting keys preceded by uppercase +class name. E.g. given pipeline class called MyPipeline you can set setting key: MYPIPELINE_FILES_EXPIRES = 180 diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index 73eda5f34..8cdc548f6 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -3,7 +3,7 @@ Files Pipeline See documentation in topics/media-pipeline.rst """ - +import functools import hashlib import os import os.path @@ -232,18 +232,20 @@ class FilesPipeline(MediaPipeline): cls_name = "FilesPipeline" self.store = self._get_store(store_uri) + resolve = functools.partial(self._key_for_pipe, + base_class_name=cls_name) self.expires = settings.getint( - self._key_for_pipe('FILES_EXPIRES', cls_name), self.EXPIRES + resolve('FILES_EXPIRES'), self.EXPIRES ) if not hasattr(self, "FILES_URLS_FIELD"): self.FILES_URLS_FIELD = self.DEFAULT_FILES_URLS_FIELD if not hasattr(self, "FILES_RESULT_FIELD"): self.FILES_RESULT_FIELD = self.DEFAULT_FILES_RESULT_FIELD self.files_urls_field = settings.get( - self._key_for_pipe('FILES_URLS_FIELD', cls_name), self.FILES_URLS_FIELD + resolve('FILES_URLS_FIELD'), self.FILES_URLS_FIELD ) self.files_result_field = settings.get( - self._key_for_pipe('FILES_RESULT_FIELD', cls_name), self.FILES_RESULT_FIELD + resolve('FILES_RESULT_FIELD'), self.FILES_RESULT_FIELD ) super(FilesPipeline, self).__init__(download_func=download_func) diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py index 73377e2c2..0278ec32f 100644 --- a/scrapy/pipelines/images.py +++ b/scrapy/pipelines/images.py @@ -3,7 +3,7 @@ Images Pipeline See documentation in topics/media-pipeline.rst """ - +import functools import hashlib import six @@ -62,22 +62,24 @@ class ImagesPipeline(FilesPipeline): if not hasattr(self, "IMAGES_URLS_FIELD"): self.IMAGES_URLS_FIELD = self.DEFAULT_IMAGES_URLS_FIELD - default_images_urls_field = getattr(self, "IMAGES_URLS_FIELD", "DEFAULT_IMAGES_URLS_FIELD") + resolve = functools.partial(self._key_for_pipe, base_class_name=cls_name) + self.images_urls_field = settings.get( - self._key_for_pipe('IMAGES_URLS_FIELD', cls_name), default_images_urls_field + resolve('IMAGES_URLS_FIELD'), + self.IMAGES_URLS_FIELD ) - default_images_result_field = getattr(self, "IMAGES_RESULT_FIELD", "DEFAULT_IMAGES_RESULT_FIELD") self.images_result_field = settings.get( - self._key_for_pipe('IMAGES_RESULT_FIELD', cls_name), default_images_result_field + resolve('IMAGES_RESULT_FIELD'), + self.IMAGES_RESULT_FIELD ) self.min_width = settings.getint( - self._key_for_pipe('IMAGES_MIN_WIDTH', cls_name), self.MIN_WIDTH + resolve('IMAGES_MIN_WIDTH'), self.MIN_WIDTH ) self.min_height = settings.getint( - self._key_for_pipe('IMAGES_MIN_HEIGHT', cls_name), self.MIN_HEIGHT + resolve('IMAGES_MIN_HEIGHT'), self.MIN_HEIGHT ) self.thumbs = settings.get( - self._key_for_pipe('IMAGES_THUMBS', cls_name), self.THUMBS + resolve('IMAGES_THUMBS'), self.THUMBS ) @classmethod diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.py index 740312f8f..fe59b9f7c 100644 --- a/scrapy/pipelines/media.py +++ b/scrapy/pipelines/media.py @@ -28,18 +28,17 @@ class MediaPipeline(object): self.download_func = download_func - def _key_for_pipe(self, key, base_class_name): + def _key_for_pipe(self, key, base_class_name=None): """ - Allow setting settings for user defined MediaPipelines that inherit from base. - - User can define setting key: - - MYPIPELINENAME_IMAGE_SETTING_NAME = - - and it will override default settings and class attributes. + >>> result = MediaPipeline()._key_for_pipe("IMAGES") + >>> assert result == "IMAGES" + >>> class MyPipe(MediaPipeline): + ... pass + >>> other_key = MyPipe()._key_for_pipe("IMAGES", base_class_name="MediaPipeline") + >>> assert other_key == "MYPIPE_IMAGES" """ class_name = self.__class__.__name__ - if class_name == base_class_name: + if class_name == base_class_name or not base_class_name: return key return "{}_{}".format(class_name.upper(), key)