diff --git a/scrapy/contrib/pipeline/files.py b/scrapy/contrib/pipeline/files.py new file mode 100644 index 000000000..f03bf69df --- /dev/null +++ b/scrapy/contrib/pipeline/files.py @@ -0,0 +1,268 @@ +""" +Files Pipeline +""" + +import hashlib +import os +import os.path +import rfc822 +import time +import urlparse +from collections import defaultdict +from cStringIO import StringIO + +from twisted.internet import defer, threads + +from scrapy import log +from scrapy.contrib.pipeline.media import MediaPipeline +from scrapy.exceptions import NotConfigured, IgnoreRequest +from scrapy.http import Request +from scrapy.utils.misc import md5sum + + +class FileException(Exception): + """General media error exception""" + + +class FSFilesStore(object): + + def __init__(self, basedir): + if '://' in basedir: + basedir = basedir.split('://', 1)[1] + self.basedir = basedir + self._mkdir(self.basedir) + self.created_directories = defaultdict(set) + + def persist_file(self, key, buf, info, meta=None, headers=None): + absolute_path = self._get_filesystem_path(key) + self._mkdir(os.path.dirname(absolute_path), info) + with open(absolute_path, 'wb') as f: + f.write(buf.getvalue()) + + def stat_file(self, key, info): + absolute_path = self._get_filesystem_path(key) + try: + last_modified = os.path.getmtime(absolute_path) + except: # FIXME: catching everything! + return {} + + with open(absolute_path, 'rb') as f: + checksum = md5sum(f) + + return {'last_modified': last_modified, 'checksum': checksum} + + def _get_filesystem_path(self, key): + path_comps = key.split('/') + return os.path.join(self.basedir, *path_comps) + + def _mkdir(self, dirname, domain=None): + seen = self.created_directories[domain] if domain else set() + if dirname not in seen: + if not os.path.exists(dirname): + os.makedirs(dirname) + seen.add(dirname) + + +class S3FilesStore(object): + + AWS_ACCESS_KEY_ID = None + AWS_SECRET_ACCESS_KEY = None + + POLICY = 'public-read' + HEADERS = { + 'Cache-Control': 'max-age=172800', + } + + def __init__(self, uri): + assert uri.startswith('s3://') + self.bucket, self.prefix = uri[5:].split('/', 1) + + def stat_file(self, key, info): + def _onsuccess(boto_key): + checksum = boto_key.etag.strip('"') + last_modified = boto_key.last_modified + modified_tuple = rfc822.parsedate_tz(last_modified) + modified_stamp = int(rfc822.mktime_tz(modified_tuple)) + return {'checksum': checksum, 'last_modified': modified_stamp} + + return self._get_boto_key(key).addCallback(_onsuccess) + + def _get_boto_bucket(self): + from boto.s3.connection import S3Connection + # disable ssl (is_secure=False) because of this python bug: + # http://bugs.python.org/issue5103 + c = S3Connection(self.AWS_ACCESS_KEY_ID, self.AWS_SECRET_ACCESS_KEY, is_secure=False) + return c.get_bucket(self.bucket, validate=False) + + def _get_boto_key(self, key): + b = self._get_boto_bucket() + key_name = '%s%s' % (self.prefix, key) + return threads.deferToThread(b.get_key, key_name) + + def persist_file(self, key, buf, info, meta=None, headers=None): + """Upload file to S3 storage""" + b = self._get_boto_bucket() + key_name = '%s%s' % (self.prefix, key) + k = b.new_key(key_name) + if meta: + for metakey, metavalue in meta.iteritems(): + k.set_metadata(metakey, str(metavalue)) + h = self.HEADERS.copy() + if headers: + h.update(headers) + buf.seek(0) + return threads.deferToThread(k.set_contents_from_string, buf.getvalue(), + headers=h, policy=self.POLICY) + + +class FilesPipeline(MediaPipeline): + """Abstract pipeline that implement the file downloading + + This pipeline tries to minimize network transfers and file processing, + doing stat of the files and determining if file is new, uptodate or + expired. + + `new` files are those that pipeline never processed and needs to be + downloaded from supplier site the first time. + + `uptodate` files are the ones that the pipeline processed and are still + valid files. + + `expired` files are those that pipeline already processed but the last + modification was made long time ago, so a reprocessing is recommended to + refresh it in case of change. + + """ + + MEDIA_NAME = "file" + EXPIRES = 90 + STORE_SCHEMES = { + '': FSFilesStore, + 'file': FSFilesStore, + 's3': S3FilesStore, + } + + def __init__(self, store_uri, download_func=None): + if not store_uri: + raise NotConfigured + self.store = self._get_store(store_uri) + super(FilesPipeline, self).__init__(download_func=download_func) + + @classmethod + def from_settings(cls, settings): + s3store = cls.STORE_SCHEMES['s3'] + s3store.AWS_ACCESS_KEY_ID = settings['AWS_ACCESS_KEY_ID'] + s3store.AWS_SECRET_ACCESS_KEY = settings['AWS_SECRET_ACCESS_KEY'] + + cls.EXPIRES = settings.getint('FILES_EXPIRES', 90) + store_uri = settings['FILES_STORE'] + return cls(store_uri) + + def _get_store(self, uri): + if os.path.isabs(uri): # to support win32 paths like: C:\\some\dir + scheme = 'file' + else: + scheme = urlparse.urlparse(uri).scheme + store_cls = self.STORE_SCHEMES[scheme] + return store_cls(uri) + + def media_to_download(self, request, info): + def _onsuccess(result): + if not result: + return # returning None force download + + last_modified = result.get('last_modified', None) + if not last_modified: + return # returning None force download + + age_seconds = time.time() - last_modified + age_days = age_seconds / 60 / 60 / 24 + if age_days > self.EXPIRES: + return # returning None force download + + referer = request.headers.get('Referer') + log.msg(format='File (uptodate): Downloaded %(medianame)s from %(request)s referred in <%(referer)s>', + level=log.DEBUG, spider=info.spider, + medianame=self.MEDIA_NAME, request=request, referer=referer) + self.inc_stats(info.spider, 'uptodate') + + checksum = result.get('checksum', None) + return {'url': request.url, 'path': key, 'checksum': checksum} + + key = self.file_key(request.url) + dfd = defer.maybeDeferred(self.store.stat_file, key, info) + dfd.addCallbacks(_onsuccess, lambda _: None) + dfd.addErrback(log.err, self.__class__.__name__ + '.store.stat_file') + return dfd + + def media_failed(self, failure, request, info): + if not isinstance(failure.value, IgnoreRequest): + referer = request.headers.get('Referer') + log.msg(format='File (unknown-error): Error downloading ' + '%(medianame)s from %(request)s referred in ' + '<%(referer)s>: %(exception)s', + level=log.WARNING, spider=info.spider, exception=failure.value, + medianame=self.MEDIA_NAME, request=request, referer=referer) + + raise FileException + + def media_downloaded(self, response, request, info): + referer = request.headers.get('Referer') + + if response.status != 200: + log.msg(format='File (code: %(status)s): Error downloading image from %(request)s referred in <%(referer)s>', + level=log.WARNING, spider=info.spider, + status=response.status, request=request, referer=referer) + raise FileException('download-error') + + if not response.body: + log.msg(format='File (empty-content): Empty image from %(request)s referred in <%(referer)s>: no-content', + level=log.WARNING, spider=info.spider, + request=request, referer=referer) + raise FileException('empty-content') + + status = 'cached' if 'cached' in response.flags else 'downloaded' + log.msg(format='File (%(status)s): Downloaded image from %(request)s referred in <%(referer)s>', + level=log.DEBUG, spider=info.spider, + status=status, request=request, referer=referer) + self.inc_stats(info.spider, status) + + try: + key = self.file_key(request.url) + checksum = self.file_downloaded(response, request, info) + except FileException as exc: + whyfmt = 'File (error): Error processing image from %(request)s referred in <%(referer)s>: %(errormsg)s' + log.msg(format=whyfmt, level=log.WARNING, spider=info.spider, + request=request, referer=referer, errormsg=str(exc)) + raise + except Exception as exc: + whyfmt = 'File (unknown-error): Error processing image from %(request)s referred in <%(referer)s>' + log.err(None, whyfmt % {'request': request, 'referer': referer}, spider=info.spider) + raise FileException(str(exc)) + + return {'url': request.url, 'path': key, 'checksum': checksum} + + def inc_stats(self, spider, status): + spider.crawler.stats.inc_value('file_count', spider=spider) + spider.crawler.stats.inc_value('file_status_count/%s' % status, spider=spider) + + ### Overridable Interface + def get_media_requests(self, item, info): + return [Request(x) for x in item.get('file_urls', [])] + + def file_key(self, url): + media_guid = hashlib.sha1(url).hexdigest() + media_ext = os.path.splitext(url)[1] + return 'full/%s%s' % (media_guid, media_ext) + + def file_downloaded(self, response, request, info): + key = self.file_key(request.url) + buf = StringIO(response.body) + self.store.persist_file(key, buf, info) + checksum = md5sum(buf) + return checksum + + def item_completed(self, results, item, info): + if 'files' in item.fields: + item['files'] = [x for ok, x in results if ok] + return item diff --git a/scrapy/contrib/pipeline/images.py b/scrapy/contrib/pipeline/images.py index 4ccc8e282..aaad689a0 100644 --- a/scrapy/contrib/pipeline/images.py +++ b/scrapy/contrib/pipeline/images.py @@ -4,155 +4,35 @@ Images Pipeline See documentation in topics/images.rst """ -import os -import time import hashlib -import urlparse -import rfc822 from cStringIO import StringIO -from collections import defaultdict -from twisted.internet import defer, threads from PIL import Image -from scrapy import log from scrapy.utils.misc import md5sum from scrapy.http import Request -from scrapy.exceptions import DropItem, NotConfigured, IgnoreRequest -from scrapy.contrib.pipeline.media import MediaPipeline +from scrapy.exceptions import DropItem +#TODO: from scrapy.contrib.pipeline.media import MediaPipeline +from scrapy.contrib.pipeline.files import FileException, FilesPipeline class NoimagesDrop(DropItem): """Product with no images exception""" -class ImageException(Exception): +class ImageException(FileException): """General image error exception""" -class FSImagesStore(object): - - def __init__(self, basedir): - if '://' in basedir: - basedir = basedir.split('://', 1)[1] - self.basedir = basedir - self._mkdir(self.basedir) - self.created_directories = defaultdict(set) - - def persist_image(self, key, image, buf, info): - absolute_path = self._get_filesystem_path(key) - self._mkdir(os.path.dirname(absolute_path), info) - image.save(absolute_path) - - def stat_image(self, key, info): - absolute_path = self._get_filesystem_path(key) - try: - last_modified = os.path.getmtime(absolute_path) - except: # FIXME: catching everything! - return {} - - with open(absolute_path, 'rb') as imagefile: - checksum = md5sum(imagefile) - - return {'last_modified': last_modified, 'checksum': checksum} - - def _get_filesystem_path(self, key): - path_comps = key.split('/') - return os.path.join(self.basedir, *path_comps) - - def _mkdir(self, dirname, domain=None): - seen = self.created_directories[domain] if domain else set() - if dirname not in seen: - if not os.path.exists(dirname): - os.makedirs(dirname) - seen.add(dirname) - - -class S3ImagesStore(object): - - AWS_ACCESS_KEY_ID = None - AWS_SECRET_ACCESS_KEY = None - - POLICY = 'public-read' - HEADERS = { - 'Cache-Control': 'max-age=172800', - 'Content-Type': 'image/jpeg', - } - - def __init__(self, uri): - assert uri.startswith('s3://') - self.bucket, self.prefix = uri[5:].split('/', 1) - - def stat_image(self, key, info): - def _onsuccess(boto_key): - checksum = boto_key.etag.strip('"') - last_modified = boto_key.last_modified - modified_tuple = rfc822.parsedate_tz(last_modified) - modified_stamp = int(rfc822.mktime_tz(modified_tuple)) - return {'checksum': checksum, 'last_modified': modified_stamp} - - return self._get_boto_key(key).addCallback(_onsuccess) - - def _get_boto_bucket(self): - from boto.s3.connection import S3Connection - # disable ssl (is_secure=False) because of this python bug: - # http://bugs.python.org/issue5103 - c = S3Connection(self.AWS_ACCESS_KEY_ID, self.AWS_SECRET_ACCESS_KEY, is_secure=False) - return c.get_bucket(self.bucket, validate=False) - - def _get_boto_key(self, key): - b = self._get_boto_bucket() - key_name = '%s%s' % (self.prefix, key) - return threads.deferToThread(b.get_key, key_name) - - def persist_image(self, key, image, buf, info): - """Upload image to S3 storage""" - width, height = image.size - b = self._get_boto_bucket() - key_name = '%s%s' % (self.prefix, key) - k = b.new_key(key_name) - k.set_metadata('width', str(width)) - k.set_metadata('height', str(height)) - buf.seek(0) - return threads.deferToThread(k.set_contents_from_file, buf, - headers=self.HEADERS, policy=self.POLICY) - - -class ImagesPipeline(MediaPipeline): - """Abstract pipeline that implement the image downloading and thumbnail generation logic - - This pipeline tries to minimize network transfers and image processing, - doing stat of the images and determining if image is new, uptodate or - expired. - - `new` images are those that pipeline never processed and needs to be - downloaded from supplier site the first time. - - `uptodate` images are the ones that the pipeline processed and are still - valid images. - - `expired` images are those that pipeline already processed but the last - modification was made long time ago, so a reprocessing is recommended to - refresh it in case of change. +class ImagesPipeline(FilesPipeline): + """Abstract pipeline that implement the image thumbnail generation logic """ MEDIA_NAME = 'image' MIN_WIDTH = 0 MIN_HEIGHT = 0 - EXPIRES = 90 THUMBS = {} - STORE_SCHEMES = { - '': FSImagesStore, - 'file': FSImagesStore, - 's3': S3ImagesStore, - } - - def __init__(self, store_uri, download_func=None): - if not store_uri: - raise NotConfigured - self.store = self._get_store(store_uri) - super(ImagesPipeline, self).__init__(download_func=download_func) @classmethod def from_settings(cls, settings): @@ -166,89 +46,11 @@ class ImagesPipeline(MediaPipeline): store_uri = settings['IMAGES_STORE'] return cls(store_uri) - def _get_store(self, uri): - if os.path.isabs(uri): # to support win32 paths like: C:\\some\dir - scheme = 'file' - else: - scheme = urlparse.urlparse(uri).scheme - store_cls = self.STORE_SCHEMES[scheme] - return store_cls(uri) + def file_key(self, url): + return self.image_key(url) - def media_downloaded(self, response, request, info): - referer = request.headers.get('Referer') - - if response.status != 200: - log.msg(format='Image (code: %(status)s): Error downloading image from %(request)s referred in <%(referer)s>', - level=log.WARNING, spider=info.spider, - status=response.status, request=request, referer=referer) - raise ImageException('download-error') - - if not response.body: - log.msg(format='Image (empty-content): Empty image from %(request)s referred in <%(referer)s>: no-content', - level=log.WARNING, spider=info.spider, - request=request, referer=referer) - raise ImageException('empty-content') - - status = 'cached' if 'cached' in response.flags else 'downloaded' - log.msg(format='Image (%(status)s): Downloaded image from %(request)s referred in <%(referer)s>', - level=log.DEBUG, spider=info.spider, - status=status, request=request, referer=referer) - self.inc_stats(info.spider, status) - - try: - key = self.image_key(request.url) - checksum = self.image_downloaded(response, request, info) - except ImageException as exc: - whyfmt = 'Image (error): Error processing image from %(request)s referred in <%(referer)s>: %(errormsg)s' - log.msg(format=whyfmt, level=log.WARNING, spider=info.spider, - request=request, referer=referer, errormsg=str(exc)) - raise - except Exception as exc: - whyfmt = 'Image (unknown-error): Error processing image from %(request)s referred in <%(referer)s>' - log.err(None, whyfmt % {'request': request, 'referer': referer}, spider=info.spider) - raise ImageException(str(exc)) - - return {'url': request.url, 'path': key, 'checksum': checksum} - - def media_failed(self, failure, request, info): - if not isinstance(failure.value, IgnoreRequest): - referer = request.headers.get('Referer') - log.msg(format='Image (unknown-error): Error downloading ' - '%(medianame)s from %(request)s referred in ' - '<%(referer)s>: %(exception)s', - level=log.WARNING, spider=info.spider, exception=failure.value, - medianame=self.MEDIA_NAME, request=request, referer=referer) - - raise ImageException - - def media_to_download(self, request, info): - def _onsuccess(result): - if not result: - return # returning None force download - - last_modified = result.get('last_modified', None) - if not last_modified: - return # returning None force download - - age_seconds = time.time() - last_modified - age_days = age_seconds / 60 / 60 / 24 - if age_days > self.EXPIRES: - return # returning None force download - - referer = request.headers.get('Referer') - log.msg(format='Image (uptodate): Downloaded %(medianame)s from %(request)s referred in <%(referer)s>', - level=log.DEBUG, spider=info.spider, - medianame=self.MEDIA_NAME, request=request, referer=referer) - self.inc_stats(info.spider, 'uptodate') - - checksum = result.get('checksum', None) - return {'url': request.url, 'path': key, 'checksum': checksum} - - key = self.image_key(request.url) - dfd = defer.maybeDeferred(self.store.stat_image, key, info) - dfd.addCallbacks(_onsuccess, lambda _: None) - dfd.addErrback(log.err, self.__class__.__name__ + '.store.stat_image') - return dfd + def file_downloaded(self, response, request, info): + return self.image_downloaded(response, request, info) def image_downloaded(self, response, request, info): checksum = None @@ -256,11 +58,15 @@ class ImagesPipeline(MediaPipeline): if checksum is None: buf.seek(0) checksum = md5sum(buf) - self.store.persist_image(key, image, buf, info) + width, height = image.size + self.store.persist_file( + key, buf, info, + meta={'width': width, 'height': height}, + headers={'Content-Type': 'image/jpeg'}) return checksum def get_images(self, response, request, info): - key = self.image_key(request.url) + key = self.file_key(request.url) orig_image = Image.open(StringIO(response.body)) width, height = orig_image.size @@ -276,10 +82,6 @@ class ImagesPipeline(MediaPipeline): thumb_image, thumb_buf = self.convert_image(image, size) yield thumb_key, thumb_image, thumb_buf - def inc_stats(self, spider, status): - spider.crawler.stats.inc_value('image_count', spider=spider) - spider.crawler.stats.inc_value('image_status_count/%s' % status, spider=spider) - def convert_image(self, image, size=None): if image.format == 'PNG' and image.mode == 'RGBA': background = Image.new('RGBA', image.size, (255, 255, 255)) @@ -296,10 +98,6 @@ class ImagesPipeline(MediaPipeline): image.save(buf, 'JPEG') return image, buf - def image_key(self, url): - image_guid = hashlib.sha1(url).hexdigest() - return 'full/%s.jpg' % (image_guid) - def thumb_key(self, url, thumb_id): image_guid = hashlib.sha1(url).hexdigest() return 'thumbs/%s/%s.jpg' % (thumb_id, image_guid) @@ -307,6 +105,11 @@ class ImagesPipeline(MediaPipeline): def get_media_requests(self, item, info): return [Request(x) for x in item.get('image_urls', [])] + # backwards compatibility + def image_key(self, url): + media_guid = hashlib.sha1(url).hexdigest() + return 'full/%s.jpg' % (media_guid) + def item_completed(self, results, item, info): if 'images' in item.fields: item['images'] = [x for ok, x in results if ok] diff --git a/scrapy/contrib/pipeline/media.py b/scrapy/contrib/pipeline/media.py index 9491836d2..21b09150a 100644 --- a/scrapy/contrib/pipeline/media.py +++ b/scrapy/contrib/pipeline/media.py @@ -7,6 +7,7 @@ from scrapy import log from scrapy.utils.request import request_fingerprint from scrapy.utils.misc import arg_to_iter + class MediaPipeline(object): LOG_FAILED_RESULTS = True @@ -65,7 +66,7 @@ class MediaPipeline(object): dfd.addCallback(self._check_media_to_download, request, info) dfd.addBoth(self._cache_result_and_execute_waiters, fp, info) dfd.addErrback(log.err, spider=info.spider) - return dfd.addBoth(lambda _: wad) # it must return wad at last + return dfd.addBoth(lambda _: wad) # it must return wad at last def _check_media_to_download(self, result, request, info): if result is not None: @@ -91,11 +92,11 @@ class MediaPipeline(object): result.frames = [] result.stack = None info.downloading.remove(fp) - info.downloaded[fp] = result # cache result + info.downloaded[fp] = result # cache result for wad in info.waiting.pop(fp): defer_result(result).chainDeferred(wad) - ### Overradiable Interface + ### Overridable Interface def media_to_download(self, request, info): """Check request before starting download""" pass diff --git a/scrapy/tests/test_pipeline_files.py b/scrapy/tests/test_pipeline_files.py new file mode 100644 index 000000000..f0aadfbfa --- /dev/null +++ b/scrapy/tests/test_pipeline_files.py @@ -0,0 +1,108 @@ +import mock +import os +import time +from tempfile import mkdtemp +from shutil import rmtree + +from twisted.trial import unittest +from twisted.internet import defer + +from scrapy.contrib.pipeline.files import FilesPipeline, FSFilesStore +from scrapy.item import Item, Field +from scrapy.http import Request, Response + + +def _mocked_download_func(request, info): + response = request.meta.get('response') + return response() if callable(response) else response + + +class FilesPipelineTestCase(unittest.TestCase): + + def setUp(self): + self.tempdir = mkdtemp() + self.pipeline = FilesPipeline(self.tempdir, download_func=_mocked_download_func) + self.pipeline.open_spider(None) + + def tearDown(self): + rmtree(self.tempdir) + + def test_file_path(self): + image_path = self.pipeline.file_key + self.assertEqual(image_path("https://dev.mydeco.com/mydeco.pdf"), + 'full/c9b564df929f4bc635bdd19fde4f3d4847c757c5.pdf') + self.assertEqual(image_path("http://www.maddiebrown.co.uk///catalogue-items//image_54642_12175_95307.txt"), + 'full/4ce274dd83db0368bafd7e406f382ae088e39219.txt') + self.assertEqual(image_path("https://dev.mydeco.com/two/dirs/with%20spaces%2Bsigns.doc"), + 'full/94ccc495a17b9ac5d40e3eabf3afcb8c2c9b9e1a.doc') + self.assertEqual(image_path("http://www.dfsonline.co.uk/get_prod_image.php?img=status_0907_mdm.jpg"), + 'full/4507be485f38b0da8a0be9eb2e1dfab8a19223f2.jpg') + self.assertEqual(image_path("http://www.dorma.co.uk/images/product_details/2532/"), + 'full/97ee6f8a46cbbb418ea91502fd24176865cf39b2') + self.assertEqual(image_path("http://www.dorma.co.uk/images/product_details/2532"), + 'full/244e0dd7d96a3b7b01f54eded250c9e272577aa1') + + def test_fs_store(self): + assert isinstance(self.pipeline.store, FSFilesStore) + self.assertEqual(self.pipeline.store.basedir, self.tempdir) + + key = 'some/image/key.jpg' + path = os.path.join(self.tempdir, 'some', 'image', 'key.jpg') + self.assertEqual(self.pipeline.store._get_filesystem_path(key), path) + + @defer.inlineCallbacks + def test_file_not_expired(self): + item_url = "http://example.com/file.pdf" + item = _create_item_with_files(item_url) + patchers = [ + mock.patch.object(FilesPipeline, 'inc_stats', return_value=True), + mock.patch.object(FSFilesStore, 'stat_file', return_value={ + 'checksum': 'abc', 'last_modified': time.time()}), + mock.patch.object(FilesPipeline, 'get_media_requests', + return_value=[_prepare_request_object(item_url)]) + ] + map(lambda p: p.start(), patchers) + + result = yield self.pipeline.process_item(item, None) + self.assertEqual(result['files'][0]['checksum'], 'abc') + + map(lambda p: p.stop(), patchers) + + @defer.inlineCallbacks + def test_file_expired(self): + item_url = "http://example.com/file2.pdf" + item = _create_item_with_files(item_url) + patchers = [ + mock.patch.object(FSFilesStore, 'stat_file', return_value={ + 'checksum': 'abc', + 'last_modified': time.time() - (FilesPipeline.EXPIRES * 60 * 60 * 24 * 2)}), + mock.patch.object(FilesPipeline, 'get_media_requests', + return_value=[_prepare_request_object(item_url)]), + mock.patch.object(FilesPipeline, 'inc_stats', return_value=True) + ] + map(lambda p: p.start(), patchers) + + result = yield self.pipeline.process_item(item, None) + self.assertNotEqual(result['files'][0]['checksum'], 'abc') + + map(lambda p: p.stop(), patchers) + + +class ItemWithFiles(Item): + file_urls = Field() + files = Field() + + +def _create_item_with_files(*files): + item = ItemWithFiles() + item['file_urls'] = files + return item + + +def _prepare_request_object(item_url): + return Request( + item_url, + meta={'response': Response(item_url, status=200, body='data')}) + +if __name__ == "__main__": + unittest.main() diff --git a/scrapy/tests/test_pipeline_images.py b/scrapy/tests/test_pipeline_images.py index 03871383e..692e91afa 100644 --- a/scrapy/tests/test_pipeline_images.py +++ b/scrapy/tests/test_pipeline_images.py @@ -16,6 +16,7 @@ else: if not encoders.issubset(set(Image.core.__dict__)): skip = 'Missing JPEG encoders' + def _mocked_download_func(request, info): response = request.meta.get('response') return response() if callable(response) else response @@ -34,7 +35,7 @@ class ImagesPipelineTestCase(unittest.TestCase): rmtree(self.tempdir) def test_image_path(self): - image_path = self.pipeline.image_key + image_path = self.pipeline.file_key self.assertEqual(image_path("https://dev.mydeco.com/mydeco.gif"), 'full/3fd165099d8e71b8a48b2683946e64dbfad8b52d.jpg') self.assertEqual(image_path("http://www.maddiebrown.co.uk///catalogue-items//image_54642_12175_95307.jpg"), @@ -60,15 +61,6 @@ class ImagesPipelineTestCase(unittest.TestCase): self.assertEqual(thumbnail_name("/tmp/some.name/foo", name), 'thumbs/50/92dac2a6a2072c5695a5dff1f865b3cb70c657bb.jpg') - def test_fs_store(self): - from scrapy.contrib.pipeline.images import FSImagesStore - assert isinstance(self.pipeline.store, FSImagesStore) - self.assertEqual(self.pipeline.store.basedir, self.tempdir) - - key = 'some/image/key.jpg' - path = os.path.join(self.tempdir, 'some', 'image', 'key.jpg') - self.assertEqual(self.pipeline.store._get_filesystem_path(key), path) - def test_convert_image(self): SIZE = (100, 100) # straigh forward case: RGB and JPEG @@ -91,7 +83,6 @@ class ImagesPipelineTestCase(unittest.TestCase): self.assertEquals(converted.getcolors(), [(10000, (205, 230, 255))]) - def _create_image(format, *a, **kw): buf = StringIO() Image.new(*a, **kw).save(buf, format)