diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rst index 7dff78390..2513faae2 100644 --- a/docs/topics/media-pipeline.rst +++ b/docs/topics/media-pipeline.rst @@ -656,6 +656,26 @@ See here the methods that you can override in your custom Images Pipeline: .. versionadded:: 2.4 The *item* parameter. + .. method:: ImagesPipeline.thumb_path(self, request, thumb_id, response=None, info=None, *, item=None) + + This method is called for every item of :setting:`IMAGES_THUMBS` per downloaded item. It returns the + thumbnail download path of the image originating from the specified + :class:`response `. + + In addition to ``response``, this method receives the original + :class:`request `, + ``thumb_id``, + :class:`info ` and + :class:`item `. + + You can override this method to customize the thumbnail download path of each image. + You can use the ``item`` to determine the file path based on some item + property. + + By default the :meth:`thumb_path` method returns + ``thumbs//.``. + + .. method:: ImagesPipeline.get_media_requests(item, info) Works the same way as :meth:`FilesPipeline.get_media_requests` method, diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py index 9c99dc69e..6b97190ee 100644 --- a/scrapy/pipelines/images.py +++ b/scrapy/pipelines/images.py @@ -141,7 +141,7 @@ class ImagesPipeline(FilesPipeline): yield path, image, buf for thumb_id, size in self.thumbs.items(): - thumb_path = self.thumb_path(request, thumb_id, response=response, info=info) + thumb_path = self.thumb_path(request, thumb_id, response=response, info=info, item=item) thumb_image, thumb_buf = self.convert_image(image, size) yield thumb_path, thumb_image, thumb_buf @@ -179,6 +179,6 @@ class ImagesPipeline(FilesPipeline): image_guid = hashlib.sha1(to_bytes(request.url)).hexdigest() return f'full/{image_guid}.jpg' - def thumb_path(self, request, thumb_id, response=None, info=None): + def thumb_path(self, request, thumb_id, response=None, info=None, *, item=None): thumb_guid = hashlib.sha1(to_bytes(request.url)).hexdigest() return f'thumbs/{thumb_id}/{thumb_guid}.jpg' diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.py index d1bccf323..430c37227 100644 --- a/scrapy/pipelines/media.py +++ b/scrapy/pipelines/media.py @@ -121,7 +121,7 @@ class MediaPipeline: def _make_compatible(self): """Make overridable methods of MediaPipeline and subclasses backwards compatible""" methods = [ - "file_path", "media_to_download", "media_downloaded", + "file_path", "thumb_path", "media_to_download", "media_downloaded", "file_downloaded", "image_downloaded", "get_images" ] diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py index c69cd0e4a..dd94d296b 100644 --- a/tests/test_pipeline_images.py +++ b/tests/test_pipeline_images.py @@ -93,6 +93,22 @@ class ImagesPipelineTestCase(unittest.TestCase): info=object()), 'thumbs/50/850233df65a5b83361798f532f1fc549cd13cbe9.jpg') + def test_thumbnail_name_from_item(self): + """ + Custom thumbnail name based on item data, overriding default implementation + """ + + class CustomImagesPipeline(ImagesPipeline): + def thumb_path(self, request, thumb_id, response=None, info=None, item=None): + return f"thumb/{thumb_id}/{item.get('path')}" + + thumb_path = CustomImagesPipeline.from_settings(Settings( + {'IMAGES_STORE': self.tempdir} + )).thumb_path + item = dict(path='path-to-store-file') + request = Request("http://example.com") + self.assertEqual(thumb_path(request, 'small', item=item), 'thumb/small/path-to-store-file') + def test_convert_image(self): SIZE = (100, 100) # straigh forward case: RGB and JPEG diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py index 893d43052..a802c7cf1 100644 --- a/tests/test_pipeline_media.py +++ b/tests/test_pipeline_media.py @@ -1,4 +1,5 @@ from typing import Optional +import io from testfixtures import LogCapture from twisted.trial import unittest @@ -355,9 +356,12 @@ class MockedMediaPipelineDeprecatedMethods(ImagesPipeline): def get_media_requests(self, item, info): item_url = item['image_urls'][0] + output_img = io.BytesIO() + img = Image.new('RGB', (60, 30), color='red') + img.save(output_img, format='JPEG') return Request( item_url, - meta={'response': Response(item_url, status=200, body=b'data')} + meta={'response': Response(item_url, status=200, body=output_img.getvalue())} ) def inc_stats(self, *args, **kwargs): @@ -379,9 +383,13 @@ class MockedMediaPipelineDeprecatedMethods(ImagesPipeline): self._mockcalled.append('file_path') return super(MockedMediaPipelineDeprecatedMethods, self).file_path(request, response, info) + def thumb_path(self, request, thumb_id, response=None, info=None): + self._mockcalled.append('thumb_path') + return super(MockedMediaPipelineDeprecatedMethods, self).thumb_path(request, thumb_id, response, info) + def get_images(self, response, request, info): self._mockcalled.append('get_images') - return [] + return super(MockedMediaPipelineDeprecatedMethods, self).get_images(response, request, info) def image_downloaded(self, response, request, info): self._mockcalled.append('image_downloaded') @@ -392,7 +400,11 @@ class MediaPipelineDeprecatedMethodsTestCase(unittest.TestCase): skip = skip_pillow def setUp(self): - self.pipe = MockedMediaPipelineDeprecatedMethods(store_uri='store-uri', download_func=_mocked_download_func) + self.pipe = MockedMediaPipelineDeprecatedMethods( + store_uri='store-uri', + download_func=_mocked_download_func, + settings=Settings({"IMAGES_THUMBS": {'small': (50, 50)}}) + ) self.pipe.open_spider(None) self.item = dict(image_urls=['http://picsum.photos/id/1014/200/300'], images=[]) @@ -444,6 +456,16 @@ class MediaPipelineDeprecatedMethodsTestCase(unittest.TestCase): ) self._assert_method_called_with_warnings('file_path', message, warnings) + @inlineCallbacks + def test_thumb_path_called(self): + yield self.pipe.process_item(self.item, None) + warnings = self.flushWarnings([MediaPipeline._compatible]) + message = ( + 'thumb_path(self, request, thumb_id, response=None, info=None) is deprecated, ' + 'please use thumb_path(self, request, thumb_id, response=None, info=None, *, item=None)' + ) + self._assert_method_called_with_warnings('thumb_path', message, warnings) + @inlineCallbacks def test_get_images_called(self): yield self.pipe.process_item(self.item, None)