diff --git a/scrapy/contrib/pipeline/images.py b/scrapy/contrib/pipeline/images.py index 406d37522..77e9afb98 100644 --- a/scrapy/contrib/pipeline/images.py +++ b/scrapy/contrib/pipeline/images.py @@ -136,6 +136,7 @@ class BaseImagesPipeline(MediaPipeline): self.store_image(key, image, buf, info) if first_buf is None: first_buf = buf + first_buf.seek(0) return md5sum(first_buf) def get_images(self, response, request, info): diff --git a/scrapy/contrib/pipeline/s3images.py b/scrapy/contrib/pipeline/s3images.py index 7f06233bd..a1018ee4c 100644 --- a/scrapy/contrib/pipeline/s3images.py +++ b/scrapy/contrib/pipeline/s3images.py @@ -3,7 +3,7 @@ import rfc822 from scrapy.http import Request from scrapy.core.engine import scrapyengine from scrapy.core.exceptions import NotConfigured -from scrapy.contrib.pipeline.images import BaseImagesPipeline, md5sum +from scrapy.contrib.pipeline.images import BaseImagesPipeline from scrapy.conf import settings diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index 5c3c4b89c..3fe1e4122 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -64,18 +64,17 @@ def extract_regex(regex, text, encoding): else: return [remove_entities(unicode(s, encoding), keep=['lt', 'amp']) for s in strings] -def md5sum(buffer): - """Calculate the md5 checksum of a file +def md5sum(file): + """Calculate the md5 checksum of a file-like object without reading its + whole content in memory. >>> from StringIO import StringIO >>> md5sum(StringIO('file content to hash')) '784406af91dd5a54fbb9c84c2236595a' - """ m = hashlib.md5() - buffer.seek(0) while 1: - d = buffer.read(8096) + d = file.read(8096) if not d: break m.update(d)