From 3286ce4f42c7a663f24adee2a6c57ebba9c68ba0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= Date: Thu, 23 Feb 2012 17:47:44 -0200 Subject: [PATCH] Compute image checksum before persisting images. closes #92 Avoids threading issue accesing buffer --- scrapy/contrib/pipeline/images.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/scrapy/contrib/pipeline/images.py b/scrapy/contrib/pipeline/images.py index 9250dc4f7..c72b606ed 100644 --- a/scrapy/contrib/pipeline/images.py +++ b/scrapy/contrib/pipeline/images.py @@ -249,13 +249,12 @@ class ImagesPipeline(MediaPipeline): return dfd def image_downloaded(self, response, request, info): - first_buf = None + checksum = None for key, image, buf in self.get_images(response, request, info): + if checksum is None: + checksum = md5sum(buf) self.store.persist_image(key, image, buf, info) - if first_buf is None: - first_buf = buf - first_buf.seek(0) - return md5sum(first_buf) + return checksum def get_images(self, response, request, info): key = self.image_key(request.url)