diff --git a/docs/experimental/images.rst b/docs/experimental/images.rst index 49cc1aa4e..1cec9ec22 100644 --- a/docs/experimental/images.rst +++ b/docs/experimental/images.rst @@ -188,6 +188,38 @@ For example:: IMAGES_STORE = '/path/to/valid/dir' +Images Storage +============== + +File system is currently the only officially supported storage, but there is +also (undocumented) support for `Amazon S3`_. + +.. _Amazon S3: https://s3.amazonaws.com/ + +File system storage +------------------- + +The images are stored in files (one per image), using a `SHA1 hash`_ of their +URLs for the file names. + +For example, the following image URL:: + + http://www.example.com/image.jpg + +Whose `SHA1 hash` is:: + + 3afec3b4765f8f0a07b78f98c07b83f013567a0a + +Will be downloaded and stored in the following file:: + + /full/3afec3b4765f8f0a07b78f98c07b83f013567a0a.jpg + +Where: + +* ```` is the directory defined in :setting:`IMAGES_STORE` setting + +* ``full`` is a sub-directory to separate full images from thumbnails (if + used). For more info see :ref:`topics-images-thumbnails`. Additional features =================== @@ -204,6 +236,8 @@ specifies the delay in number of days:: # 90 days of delay for image expiration IMAGES_EXPIRES = 90 +.. _topics-images-thumbnails: + Thumbnail generation -------------------- @@ -225,24 +259,24 @@ For example:: When you use this feature, the Images Pipeline will create thumbnails of the each specified size with this format:: - /thumbs//.jpg + /thumbs//.jpg Where: +* ```` is the one specified in the :setting:`IMAGES_THUMBS` + dictionary keys (``small``, ``big``, etc) + * ```` is the `SHA1 hash`_ of the image url -* ```` is the one specified in the ``THUMBS`` attribute .. _SHA1 hash: http://en.wikipedia.org/wiki/SHA_hash_functions -Example with using ``50`` and ``110`` thumbnail names:: +Example of image files stored using ``small`` and ``big`` thumbnail names:: - /thumbs/63bbfea82b8880ed33cdb762aa11fab722a90a24/50.jpg - /thumbs/63bbfea82b8880ed33cdb762aa11fab722a90a24/110.jpg + /full/63bbfea82b8880ed33cdb762aa11fab722a90a24.jpg + /thumbs/small/63bbfea82b8880ed33cdb762aa11fab722a90a24.jpg + /thumbs/big/63bbfea82b8880ed33cdb762aa11fab722a90a24.jpg -Example with using ``small`` and ``big`` thumbnail names:: - - /thumbs/63bbfea82b8880ed33cdb762aa11fab722a90a24/small.jpg - /thumbs/63bbfea82b8880ed33cdb762aa11fab722a90a24/big.jpg +The first one is the full image, as downloaded from the site. Filtering out small images -------------------------- @@ -259,3 +293,7 @@ For example:: IMAGES_MIN_HEIGHT = 110 IMAGES_MIN_WIDTH = 110 +Note: this size constraints only doesn't affect thumbnail generation at all. + +By default, there are no size constrains, so all images are precessed. + diff --git a/scrapy/contrib/pipeline/images.py b/scrapy/contrib/pipeline/images.py index d4c886c48..de596a3dd 100644 --- a/scrapy/contrib/pipeline/images.py +++ b/scrapy/contrib/pipeline/images.py @@ -170,7 +170,7 @@ class ImagesPipeline(MediaPipeline): MIN_WIDTH = settings.getint('IMAGES_MIN_WIDTH', 0) MIN_HEIGHT = settings.getint('IMAGES_MIN_HEIGHT', 0) EXPIRES = settings.getint('IMAGES_EXPIRES', 90) - THUMBS = settings.getlist('IMAGES_THUMBS') + THUMBS = settings['IMAGES_THUMBS'] STORE_SCHEMES = { '': FSImagesStore, 'file': FSImagesStore, @@ -278,7 +278,7 @@ class ImagesPipeline(MediaPipeline): image, buf = self.convert_image(orig_image) yield key, image, buf - for thumb_id, size in self.THUMBS or []: + for thumb_id, size in self.THUMBS.iteritems() or []: thumb_key = self.thumb_key(request.url, thumb_id) thumb_image, thumb_buf = self.convert_image(image, size) yield thumb_key, thumb_image, thumb_buf