diff --git a/scrapy/contrib/pipeline/files.py b/scrapy/contrib/pipeline/files.py index 101d45fea..b38a4c21b 100644 --- a/scrapy/contrib/pipeline/files.py +++ b/scrapy/contrib/pipeline/files.py @@ -229,7 +229,7 @@ class FilesPipeline(MediaPipeline): try: key = self.file_key(request.url) - checksum = self.process_downloaded_media(response, request, info) + checksum = self.file_downloaded(response, request, info) except FileException as exc: whyfmt = 'File (error): Error processing image from %(request)s referred in <%(referer)s>: %(errormsg)s' log.msg(format=whyfmt, level=log.WARNING, spider=info.spider, @@ -255,7 +255,7 @@ class FilesPipeline(MediaPipeline): media_ext = os.path.splitext(url)[1] return 'full/%s%s' % (media_guid, media_ext) - def process_downloaded_media(self, response, request, info): + def file_downloaded(self, response, request, info): key = self.file_key(request.url) buf = StringIO(response.body) self.store.persist_file(key, buf, info) diff --git a/scrapy/contrib/pipeline/images.py b/scrapy/contrib/pipeline/images.py index e15ce1e7c..aaad689a0 100644 --- a/scrapy/contrib/pipeline/images.py +++ b/scrapy/contrib/pipeline/images.py @@ -46,7 +46,13 @@ class ImagesPipeline(FilesPipeline): store_uri = settings['IMAGES_STORE'] return cls(store_uri) - def process_downloaded_media(self, response, request, info): + def file_key(self, url): + return self.image_key(url) + + def file_downloaded(self, response, request, info): + return self.image_downloaded(response, request, info) + + def image_downloaded(self, response, request, info): checksum = None for key, image, buf in self.get_images(response, request, info): if checksum is None: @@ -99,13 +105,11 @@ class ImagesPipeline(FilesPipeline): def get_media_requests(self, item, info): return [Request(x) for x in item.get('image_urls', [])] - def file_key(self, url): + # backwards compatibility + def image_key(self, url): media_guid = hashlib.sha1(url).hexdigest() return 'full/%s.jpg' % (media_guid) - # backwards compatibility - image_key = file_key - def item_completed(self, results, item, info): if 'images' in item.fields: item['images'] = [x for ok, x in results if ok] diff --git a/scrapy/tests/test_pipeline_files.py b/scrapy/tests/test_pipeline_files.py index c1b59c0f0..f0aadfbfa 100644 --- a/scrapy/tests/test_pipeline_files.py +++ b/scrapy/tests/test_pipeline_files.py @@ -56,7 +56,7 @@ class FilesPipelineTestCase(unittest.TestCase): item = _create_item_with_files(item_url) patchers = [ mock.patch.object(FilesPipeline, 'inc_stats', return_value=True), - mock.patch.object(FSFilesStore, 'stat_image', return_value={ + mock.patch.object(FSFilesStore, 'stat_file', return_value={ 'checksum': 'abc', 'last_modified': time.time()}), mock.patch.object(FilesPipeline, 'get_media_requests', return_value=[_prepare_request_object(item_url)]) @@ -73,7 +73,7 @@ class FilesPipelineTestCase(unittest.TestCase): item_url = "http://example.com/file2.pdf" item = _create_item_with_files(item_url) patchers = [ - mock.patch.object(FSFilesStore, 'stat_image', return_value={ + mock.patch.object(FSFilesStore, 'stat_file', return_value={ 'checksum': 'abc', 'last_modified': time.time() - (FilesPipeline.EXPIRES * 60 * 60 * 24 * 2)}), mock.patch.object(FilesPipeline, 'get_media_requests',