mirror of https://github.com/scrapy/scrapy.git
Merge pull request #370 from loucash/master
FilesPipeline extracted and separated from ImagesPipeline
This commit is contained in:
Daniel Graña
commit
f54d5c5896
|
|
@ -0,0 +1,268 @@
|
|||
"""
|
||||
Files Pipeline
|
||||
"""
|
||||
|
||||
import hashlib
|
||||
import os
|
||||
import os.path
|
||||
import rfc822
|
||||
import time
|
||||
import urlparse
|
||||
from collections import defaultdict
|
||||
from cStringIO import StringIO
|
||||
|
||||
from twisted.internet import defer, threads
|
||||
|
||||
from scrapy import log
|
||||
from scrapy.contrib.pipeline.media import MediaPipeline
|
||||
from scrapy.exceptions import NotConfigured, IgnoreRequest
|
||||
from scrapy.http import Request
|
||||
from scrapy.utils.misc import md5sum
|
||||
|
||||
|
||||
class FileException(Exception):
|
||||
"""General media error exception"""
|
||||
|
||||
|
||||
class FSFilesStore(object):
|
||||
|
||||
def __init__(self, basedir):
|
||||
if '://' in basedir:
|
||||
basedir = basedir.split('://', 1)[1]
|
||||
self.basedir = basedir
|
||||
self._mkdir(self.basedir)
|
||||
self.created_directories = defaultdict(set)
|
||||
|
||||
def persist_file(self, key, buf, info, meta=None, headers=None):
|
||||
absolute_path = self._get_filesystem_path(key)
|
||||
self._mkdir(os.path.dirname(absolute_path), info)
|
||||
with open(absolute_path, 'wb') as f:
|
||||
f.write(buf.getvalue())
|
||||
|
||||
def stat_file(self, key, info):
|
||||
absolute_path = self._get_filesystem_path(key)
|
||||
try:
|
||||
last_modified = os.path.getmtime(absolute_path)
|
||||
except: # FIXME: catching everything!
|
||||
return {}
|
||||
|
||||
with open(absolute_path, 'rb') as f:
|
||||
checksum = md5sum(f)
|
||||
|
||||
return {'last_modified': last_modified, 'checksum': checksum}
|
||||
|
||||
def _get_filesystem_path(self, key):
|
||||
path_comps = key.split('/')
|
||||
return os.path.join(self.basedir, *path_comps)
|
||||
|
||||
def _mkdir(self, dirname, domain=None):
|
||||
seen = self.created_directories[domain] if domain else set()
|
||||
if dirname not in seen:
|
||||
if not os.path.exists(dirname):
|
||||
os.makedirs(dirname)
|
||||
seen.add(dirname)
|
||||
|
||||
|
||||
class S3FilesStore(object):
|
||||
|
||||
AWS_ACCESS_KEY_ID = None
|
||||
AWS_SECRET_ACCESS_KEY = None
|
||||
|
||||
POLICY = 'public-read'
|
||||
HEADERS = {
|
||||
'Cache-Control': 'max-age=172800',
|
||||
}
|
||||
|
||||
def __init__(self, uri):
|
||||
assert uri.startswith('s3://')
|
||||
self.bucket, self.prefix = uri[5:].split('/', 1)
|
||||
|
||||
def stat_file(self, key, info):
|
||||
def _onsuccess(boto_key):
|
||||
checksum = boto_key.etag.strip('"')
|
||||
last_modified = boto_key.last_modified
|
||||
modified_tuple = rfc822.parsedate_tz(last_modified)
|
||||
modified_stamp = int(rfc822.mktime_tz(modified_tuple))
|
||||
return {'checksum': checksum, 'last_modified': modified_stamp}
|
||||
|
||||
return self._get_boto_key(key).addCallback(_onsuccess)
|
||||
|
||||
def _get_boto_bucket(self):
|
||||
from boto.s3.connection import S3Connection
|
||||
# disable ssl (is_secure=False) because of this python bug:
|
||||
# http://bugs.python.org/issue5103
|
||||
c = S3Connection(self.AWS_ACCESS_KEY_ID, self.AWS_SECRET_ACCESS_KEY, is_secure=False)
|
||||
return c.get_bucket(self.bucket, validate=False)
|
||||
|
||||
def _get_boto_key(self, key):
|
||||
b = self._get_boto_bucket()
|
||||
key_name = '%s%s' % (self.prefix, key)
|
||||
return threads.deferToThread(b.get_key, key_name)
|
||||
|
||||
def persist_file(self, key, buf, info, meta=None, headers=None):
|
||||
"""Upload file to S3 storage"""
|
||||
b = self._get_boto_bucket()
|
||||
key_name = '%s%s' % (self.prefix, key)
|
||||
k = b.new_key(key_name)
|
||||
if meta:
|
||||
for metakey, metavalue in meta.iteritems():
|
||||
k.set_metadata(metakey, str(metavalue))
|
||||
h = self.HEADERS.copy()
|
||||
if headers:
|
||||
h.update(headers)
|
||||
buf.seek(0)
|
||||
return threads.deferToThread(k.set_contents_from_string, buf.getvalue(),
|
||||
headers=h, policy=self.POLICY)
|
||||
|
||||
|
||||
class FilesPipeline(MediaPipeline):
|
||||
"""Abstract pipeline that implement the file downloading
|
||||
|
||||
This pipeline tries to minimize network transfers and file processing,
|
||||
doing stat of the files and determining if file is new, uptodate or
|
||||
expired.
|
||||
|
||||
`new` files are those that pipeline never processed and needs to be
|
||||
downloaded from supplier site the first time.
|
||||
|
||||
`uptodate` files are the ones that the pipeline processed and are still
|
||||
valid files.
|
||||
|
||||
`expired` files are those that pipeline already processed but the last
|
||||
modification was made long time ago, so a reprocessing is recommended to
|
||||
refresh it in case of change.
|
||||
|
||||
"""
|
||||
|
||||
MEDIA_NAME = "file"
|
||||
EXPIRES = 90
|
||||
STORE_SCHEMES = {
|
||||
'': FSFilesStore,
|
||||
'file': FSFilesStore,
|
||||
's3': S3FilesStore,
|
||||
}
|
||||
|
||||
def __init__(self, store_uri, download_func=None):
|
||||
if not store_uri:
|
||||
raise NotConfigured
|
||||
self.store = self._get_store(store_uri)
|
||||
super(FilesPipeline, self).__init__(download_func=download_func)
|
||||
|
||||
@classmethod
|
||||
def from_settings(cls, settings):
|
||||
s3store = cls.STORE_SCHEMES['s3']
|
||||
s3store.AWS_ACCESS_KEY_ID = settings['AWS_ACCESS_KEY_ID']
|
||||
s3store.AWS_SECRET_ACCESS_KEY = settings['AWS_SECRET_ACCESS_KEY']
|
||||
|
||||
cls.EXPIRES = settings.getint('FILES_EXPIRES', 90)
|
||||
store_uri = settings['FILES_STORE']
|
||||
return cls(store_uri)
|
||||
|
||||
def _get_store(self, uri):
|
||||
if os.path.isabs(uri): # to support win32 paths like: C:\\some\dir
|
||||
scheme = 'file'
|
||||
else:
|
||||
scheme = urlparse.urlparse(uri).scheme
|
||||
store_cls = self.STORE_SCHEMES[scheme]
|
||||
return store_cls(uri)
|
||||
|
||||
def media_to_download(self, request, info):
|
||||
def _onsuccess(result):
|
||||
if not result:
|
||||
return # returning None force download
|
||||
|
||||
last_modified = result.get('last_modified', None)
|
||||
if not last_modified:
|
||||
return # returning None force download
|
||||
|
||||
age_seconds = time.time() - last_modified
|
||||
age_days = age_seconds / 60 / 60 / 24
|
||||
if age_days > self.EXPIRES:
|
||||
return # returning None force download
|
||||
|
||||
referer = request.headers.get('Referer')
|
||||
log.msg(format='File (uptodate): Downloaded %(medianame)s from %(request)s referred in <%(referer)s>',
|
||||
level=log.DEBUG, spider=info.spider,
|
||||
medianame=self.MEDIA_NAME, request=request, referer=referer)
|
||||
self.inc_stats(info.spider, 'uptodate')
|
||||
|
||||
checksum = result.get('checksum', None)
|
||||
return {'url': request.url, 'path': key, 'checksum': checksum}
|
||||
|
||||
key = self.file_key(request.url)
|
||||
dfd = defer.maybeDeferred(self.store.stat_file, key, info)
|
||||
dfd.addCallbacks(_onsuccess, lambda _: None)
|
||||
dfd.addErrback(log.err, self.__class__.__name__ + '.store.stat_file')
|
||||
return dfd
|
||||
|
||||
def media_failed(self, failure, request, info):
|
||||
if not isinstance(failure.value, IgnoreRequest):
|
||||
referer = request.headers.get('Referer')
|
||||
log.msg(format='File (unknown-error): Error downloading '
|
||||
'%(medianame)s from %(request)s referred in '
|
||||
'<%(referer)s>: %(exception)s',
|
||||
level=log.WARNING, spider=info.spider, exception=failure.value,
|
||||
medianame=self.MEDIA_NAME, request=request, referer=referer)
|
||||
|
||||
raise FileException
|
||||
|
||||
def media_downloaded(self, response, request, info):
|
||||
referer = request.headers.get('Referer')
|
||||
|
||||
if response.status != 200:
|
||||
log.msg(format='File (code: %(status)s): Error downloading image from %(request)s referred in <%(referer)s>',
|
||||
level=log.WARNING, spider=info.spider,
|
||||
status=response.status, request=request, referer=referer)
|
||||
raise FileException('download-error')
|
||||
|
||||
if not response.body:
|
||||
log.msg(format='File (empty-content): Empty image from %(request)s referred in <%(referer)s>: no-content',
|
||||
level=log.WARNING, spider=info.spider,
|
||||
request=request, referer=referer)
|
||||
raise FileException('empty-content')
|
||||
|
||||
status = 'cached' if 'cached' in response.flags else 'downloaded'
|
||||
log.msg(format='File (%(status)s): Downloaded image from %(request)s referred in <%(referer)s>',
|
||||
level=log.DEBUG, spider=info.spider,
|
||||
status=status, request=request, referer=referer)
|
||||
self.inc_stats(info.spider, status)
|
||||
|
||||
try:
|
||||
key = self.file_key(request.url)
|
||||
checksum = self.file_downloaded(response, request, info)
|
||||
except FileException as exc:
|
||||
whyfmt = 'File (error): Error processing image from %(request)s referred in <%(referer)s>: %(errormsg)s'
|
||||
log.msg(format=whyfmt, level=log.WARNING, spider=info.spider,
|
||||
request=request, referer=referer, errormsg=str(exc))
|
||||
raise
|
||||
except Exception as exc:
|
||||
whyfmt = 'File (unknown-error): Error processing image from %(request)s referred in <%(referer)s>'
|
||||
log.err(None, whyfmt % {'request': request, 'referer': referer}, spider=info.spider)
|
||||
raise FileException(str(exc))
|
||||
|
||||
return {'url': request.url, 'path': key, 'checksum': checksum}
|
||||
|
||||
def inc_stats(self, spider, status):
|
||||
spider.crawler.stats.inc_value('file_count', spider=spider)
|
||||
spider.crawler.stats.inc_value('file_status_count/%s' % status, spider=spider)
|
||||
|
||||
### Overridable Interface
|
||||
def get_media_requests(self, item, info):
|
||||
return [Request(x) for x in item.get('file_urls', [])]
|
||||
|
||||
def file_key(self, url):
|
||||
media_guid = hashlib.sha1(url).hexdigest()
|
||||
media_ext = os.path.splitext(url)[1]
|
||||
return 'full/%s%s' % (media_guid, media_ext)
|
||||
|
||||
def file_downloaded(self, response, request, info):
|
||||
key = self.file_key(request.url)
|
||||
buf = StringIO(response.body)
|
||||
self.store.persist_file(key, buf, info)
|
||||
checksum = md5sum(buf)
|
||||
return checksum
|
||||
|
||||
def item_completed(self, results, item, info):
|
||||
if 'files' in item.fields:
|
||||
item['files'] = [x for ok, x in results if ok]
|
||||
return item
|
||||
|
|
@ -4,155 +4,35 @@ Images Pipeline
|
|||
See documentation in topics/images.rst
|
||||
"""
|
||||
|
||||
import os
|
||||
import time
|
||||
import hashlib
|
||||
import urlparse
|
||||
import rfc822
|
||||
from cStringIO import StringIO
|
||||
from collections import defaultdict
|
||||
|
||||
from twisted.internet import defer, threads
|
||||
from PIL import Image
|
||||
|
||||
from scrapy import log
|
||||
from scrapy.utils.misc import md5sum
|
||||
from scrapy.http import Request
|
||||
from scrapy.exceptions import DropItem, NotConfigured, IgnoreRequest
|
||||
from scrapy.contrib.pipeline.media import MediaPipeline
|
||||
from scrapy.exceptions import DropItem
|
||||
#TODO: from scrapy.contrib.pipeline.media import MediaPipeline
|
||||
from scrapy.contrib.pipeline.files import FileException, FilesPipeline
|
||||
|
||||
|
||||
class NoimagesDrop(DropItem):
|
||||
"""Product with no images exception"""
|
||||
|
||||
|
||||
class ImageException(Exception):
|
||||
class ImageException(FileException):
|
||||
"""General image error exception"""
|
||||
|
||||
|
||||
class FSImagesStore(object):
|
||||
|
||||
def __init__(self, basedir):
|
||||
if '://' in basedir:
|
||||
basedir = basedir.split('://', 1)[1]
|
||||
self.basedir = basedir
|
||||
self._mkdir(self.basedir)
|
||||
self.created_directories = defaultdict(set)
|
||||
|
||||
def persist_image(self, key, image, buf, info):
|
||||
absolute_path = self._get_filesystem_path(key)
|
||||
self._mkdir(os.path.dirname(absolute_path), info)
|
||||
image.save(absolute_path)
|
||||
|
||||
def stat_image(self, key, info):
|
||||
absolute_path = self._get_filesystem_path(key)
|
||||
try:
|
||||
last_modified = os.path.getmtime(absolute_path)
|
||||
except: # FIXME: catching everything!
|
||||
return {}
|
||||
|
||||
with open(absolute_path, 'rb') as imagefile:
|
||||
checksum = md5sum(imagefile)
|
||||
|
||||
return {'last_modified': last_modified, 'checksum': checksum}
|
||||
|
||||
def _get_filesystem_path(self, key):
|
||||
path_comps = key.split('/')
|
||||
return os.path.join(self.basedir, *path_comps)
|
||||
|
||||
def _mkdir(self, dirname, domain=None):
|
||||
seen = self.created_directories[domain] if domain else set()
|
||||
if dirname not in seen:
|
||||
if not os.path.exists(dirname):
|
||||
os.makedirs(dirname)
|
||||
seen.add(dirname)
|
||||
|
||||
|
||||
class S3ImagesStore(object):
|
||||
|
||||
AWS_ACCESS_KEY_ID = None
|
||||
AWS_SECRET_ACCESS_KEY = None
|
||||
|
||||
POLICY = 'public-read'
|
||||
HEADERS = {
|
||||
'Cache-Control': 'max-age=172800',
|
||||
'Content-Type': 'image/jpeg',
|
||||
}
|
||||
|
||||
def __init__(self, uri):
|
||||
assert uri.startswith('s3://')
|
||||
self.bucket, self.prefix = uri[5:].split('/', 1)
|
||||
|
||||
def stat_image(self, key, info):
|
||||
def _onsuccess(boto_key):
|
||||
checksum = boto_key.etag.strip('"')
|
||||
last_modified = boto_key.last_modified
|
||||
modified_tuple = rfc822.parsedate_tz(last_modified)
|
||||
modified_stamp = int(rfc822.mktime_tz(modified_tuple))
|
||||
return {'checksum': checksum, 'last_modified': modified_stamp}
|
||||
|
||||
return self._get_boto_key(key).addCallback(_onsuccess)
|
||||
|
||||
def _get_boto_bucket(self):
|
||||
from boto.s3.connection import S3Connection
|
||||
# disable ssl (is_secure=False) because of this python bug:
|
||||
# http://bugs.python.org/issue5103
|
||||
c = S3Connection(self.AWS_ACCESS_KEY_ID, self.AWS_SECRET_ACCESS_KEY, is_secure=False)
|
||||
return c.get_bucket(self.bucket, validate=False)
|
||||
|
||||
def _get_boto_key(self, key):
|
||||
b = self._get_boto_bucket()
|
||||
key_name = '%s%s' % (self.prefix, key)
|
||||
return threads.deferToThread(b.get_key, key_name)
|
||||
|
||||
def persist_image(self, key, image, buf, info):
|
||||
"""Upload image to S3 storage"""
|
||||
width, height = image.size
|
||||
b = self._get_boto_bucket()
|
||||
key_name = '%s%s' % (self.prefix, key)
|
||||
k = b.new_key(key_name)
|
||||
k.set_metadata('width', str(width))
|
||||
k.set_metadata('height', str(height))
|
||||
buf.seek(0)
|
||||
return threads.deferToThread(k.set_contents_from_file, buf,
|
||||
headers=self.HEADERS, policy=self.POLICY)
|
||||
|
||||
|
||||
class ImagesPipeline(MediaPipeline):
|
||||
"""Abstract pipeline that implement the image downloading and thumbnail generation logic
|
||||
|
||||
This pipeline tries to minimize network transfers and image processing,
|
||||
doing stat of the images and determining if image is new, uptodate or
|
||||
expired.
|
||||
|
||||
`new` images are those that pipeline never processed and needs to be
|
||||
downloaded from supplier site the first time.
|
||||
|
||||
`uptodate` images are the ones that the pipeline processed and are still
|
||||
valid images.
|
||||
|
||||
`expired` images are those that pipeline already processed but the last
|
||||
modification was made long time ago, so a reprocessing is recommended to
|
||||
refresh it in case of change.
|
||||
class ImagesPipeline(FilesPipeline):
|
||||
"""Abstract pipeline that implement the image thumbnail generation logic
|
||||
|
||||
"""
|
||||
|
||||
MEDIA_NAME = 'image'
|
||||
MIN_WIDTH = 0
|
||||
MIN_HEIGHT = 0
|
||||
EXPIRES = 90
|
||||
THUMBS = {}
|
||||
STORE_SCHEMES = {
|
||||
'': FSImagesStore,
|
||||
'file': FSImagesStore,
|
||||
's3': S3ImagesStore,
|
||||
}
|
||||
|
||||
def __init__(self, store_uri, download_func=None):
|
||||
if not store_uri:
|
||||
raise NotConfigured
|
||||
self.store = self._get_store(store_uri)
|
||||
super(ImagesPipeline, self).__init__(download_func=download_func)
|
||||
|
||||
@classmethod
|
||||
def from_settings(cls, settings):
|
||||
|
|
@ -166,89 +46,11 @@ class ImagesPipeline(MediaPipeline):
|
|||
store_uri = settings['IMAGES_STORE']
|
||||
return cls(store_uri)
|
||||
|
||||
def _get_store(self, uri):
|
||||
if os.path.isabs(uri): # to support win32 paths like: C:\\some\dir
|
||||
scheme = 'file'
|
||||
else:
|
||||
scheme = urlparse.urlparse(uri).scheme
|
||||
store_cls = self.STORE_SCHEMES[scheme]
|
||||
return store_cls(uri)
|
||||
def file_key(self, url):
|
||||
return self.image_key(url)
|
||||
|
||||
def media_downloaded(self, response, request, info):
|
||||
referer = request.headers.get('Referer')
|
||||
|
||||
if response.status != 200:
|
||||
log.msg(format='Image (code: %(status)s): Error downloading image from %(request)s referred in <%(referer)s>',
|
||||
level=log.WARNING, spider=info.spider,
|
||||
status=response.status, request=request, referer=referer)
|
||||
raise ImageException('download-error')
|
||||
|
||||
if not response.body:
|
||||
log.msg(format='Image (empty-content): Empty image from %(request)s referred in <%(referer)s>: no-content',
|
||||
level=log.WARNING, spider=info.spider,
|
||||
request=request, referer=referer)
|
||||
raise ImageException('empty-content')
|
||||
|
||||
status = 'cached' if 'cached' in response.flags else 'downloaded'
|
||||
log.msg(format='Image (%(status)s): Downloaded image from %(request)s referred in <%(referer)s>',
|
||||
level=log.DEBUG, spider=info.spider,
|
||||
status=status, request=request, referer=referer)
|
||||
self.inc_stats(info.spider, status)
|
||||
|
||||
try:
|
||||
key = self.image_key(request.url)
|
||||
checksum = self.image_downloaded(response, request, info)
|
||||
except ImageException as exc:
|
||||
whyfmt = 'Image (error): Error processing image from %(request)s referred in <%(referer)s>: %(errormsg)s'
|
||||
log.msg(format=whyfmt, level=log.WARNING, spider=info.spider,
|
||||
request=request, referer=referer, errormsg=str(exc))
|
||||
raise
|
||||
except Exception as exc:
|
||||
whyfmt = 'Image (unknown-error): Error processing image from %(request)s referred in <%(referer)s>'
|
||||
log.err(None, whyfmt % {'request': request, 'referer': referer}, spider=info.spider)
|
||||
raise ImageException(str(exc))
|
||||
|
||||
return {'url': request.url, 'path': key, 'checksum': checksum}
|
||||
|
||||
def media_failed(self, failure, request, info):
|
||||
if not isinstance(failure.value, IgnoreRequest):
|
||||
referer = request.headers.get('Referer')
|
||||
log.msg(format='Image (unknown-error): Error downloading '
|
||||
'%(medianame)s from %(request)s referred in '
|
||||
'<%(referer)s>: %(exception)s',
|
||||
level=log.WARNING, spider=info.spider, exception=failure.value,
|
||||
medianame=self.MEDIA_NAME, request=request, referer=referer)
|
||||
|
||||
raise ImageException
|
||||
|
||||
def media_to_download(self, request, info):
|
||||
def _onsuccess(result):
|
||||
if not result:
|
||||
return # returning None force download
|
||||
|
||||
last_modified = result.get('last_modified', None)
|
||||
if not last_modified:
|
||||
return # returning None force download
|
||||
|
||||
age_seconds = time.time() - last_modified
|
||||
age_days = age_seconds / 60 / 60 / 24
|
||||
if age_days > self.EXPIRES:
|
||||
return # returning None force download
|
||||
|
||||
referer = request.headers.get('Referer')
|
||||
log.msg(format='Image (uptodate): Downloaded %(medianame)s from %(request)s referred in <%(referer)s>',
|
||||
level=log.DEBUG, spider=info.spider,
|
||||
medianame=self.MEDIA_NAME, request=request, referer=referer)
|
||||
self.inc_stats(info.spider, 'uptodate')
|
||||
|
||||
checksum = result.get('checksum', None)
|
||||
return {'url': request.url, 'path': key, 'checksum': checksum}
|
||||
|
||||
key = self.image_key(request.url)
|
||||
dfd = defer.maybeDeferred(self.store.stat_image, key, info)
|
||||
dfd.addCallbacks(_onsuccess, lambda _: None)
|
||||
dfd.addErrback(log.err, self.__class__.__name__ + '.store.stat_image')
|
||||
return dfd
|
||||
def file_downloaded(self, response, request, info):
|
||||
return self.image_downloaded(response, request, info)
|
||||
|
||||
def image_downloaded(self, response, request, info):
|
||||
checksum = None
|
||||
|
|
@ -256,11 +58,15 @@ class ImagesPipeline(MediaPipeline):
|
|||
if checksum is None:
|
||||
buf.seek(0)
|
||||
checksum = md5sum(buf)
|
||||
self.store.persist_image(key, image, buf, info)
|
||||
width, height = image.size
|
||||
self.store.persist_file(
|
||||
key, buf, info,
|
||||
meta={'width': width, 'height': height},
|
||||
headers={'Content-Type': 'image/jpeg'})
|
||||
return checksum
|
||||
|
||||
def get_images(self, response, request, info):
|
||||
key = self.image_key(request.url)
|
||||
key = self.file_key(request.url)
|
||||
orig_image = Image.open(StringIO(response.body))
|
||||
|
||||
width, height = orig_image.size
|
||||
|
|
@ -276,10 +82,6 @@ class ImagesPipeline(MediaPipeline):
|
|||
thumb_image, thumb_buf = self.convert_image(image, size)
|
||||
yield thumb_key, thumb_image, thumb_buf
|
||||
|
||||
def inc_stats(self, spider, status):
|
||||
spider.crawler.stats.inc_value('image_count', spider=spider)
|
||||
spider.crawler.stats.inc_value('image_status_count/%s' % status, spider=spider)
|
||||
|
||||
def convert_image(self, image, size=None):
|
||||
if image.format == 'PNG' and image.mode == 'RGBA':
|
||||
background = Image.new('RGBA', image.size, (255, 255, 255))
|
||||
|
|
@ -296,10 +98,6 @@ class ImagesPipeline(MediaPipeline):
|
|||
image.save(buf, 'JPEG')
|
||||
return image, buf
|
||||
|
||||
def image_key(self, url):
|
||||
image_guid = hashlib.sha1(url).hexdigest()
|
||||
return 'full/%s.jpg' % (image_guid)
|
||||
|
||||
def thumb_key(self, url, thumb_id):
|
||||
image_guid = hashlib.sha1(url).hexdigest()
|
||||
return 'thumbs/%s/%s.jpg' % (thumb_id, image_guid)
|
||||
|
|
@ -307,6 +105,11 @@ class ImagesPipeline(MediaPipeline):
|
|||
def get_media_requests(self, item, info):
|
||||
return [Request(x) for x in item.get('image_urls', [])]
|
||||
|
||||
# backwards compatibility
|
||||
def image_key(self, url):
|
||||
media_guid = hashlib.sha1(url).hexdigest()
|
||||
return 'full/%s.jpg' % (media_guid)
|
||||
|
||||
def item_completed(self, results, item, info):
|
||||
if 'images' in item.fields:
|
||||
item['images'] = [x for ok, x in results if ok]
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ from scrapy import log
|
|||
from scrapy.utils.request import request_fingerprint
|
||||
from scrapy.utils.misc import arg_to_iter
|
||||
|
||||
|
||||
class MediaPipeline(object):
|
||||
|
||||
LOG_FAILED_RESULTS = True
|
||||
|
|
@ -65,7 +66,7 @@ class MediaPipeline(object):
|
|||
dfd.addCallback(self._check_media_to_download, request, info)
|
||||
dfd.addBoth(self._cache_result_and_execute_waiters, fp, info)
|
||||
dfd.addErrback(log.err, spider=info.spider)
|
||||
return dfd.addBoth(lambda _: wad) # it must return wad at last
|
||||
return dfd.addBoth(lambda _: wad) # it must return wad at last
|
||||
|
||||
def _check_media_to_download(self, result, request, info):
|
||||
if result is not None:
|
||||
|
|
@ -91,11 +92,11 @@ class MediaPipeline(object):
|
|||
result.frames = []
|
||||
result.stack = None
|
||||
info.downloading.remove(fp)
|
||||
info.downloaded[fp] = result # cache result
|
||||
info.downloaded[fp] = result # cache result
|
||||
for wad in info.waiting.pop(fp):
|
||||
defer_result(result).chainDeferred(wad)
|
||||
|
||||
### Overradiable Interface
|
||||
### Overridable Interface
|
||||
def media_to_download(self, request, info):
|
||||
"""Check request before starting download"""
|
||||
pass
|
||||
|
|
|
|||
|
|
@ -0,0 +1,108 @@
|
|||
import mock
|
||||
import os
|
||||
import time
|
||||
from tempfile import mkdtemp
|
||||
from shutil import rmtree
|
||||
|
||||
from twisted.trial import unittest
|
||||
from twisted.internet import defer
|
||||
|
||||
from scrapy.contrib.pipeline.files import FilesPipeline, FSFilesStore
|
||||
from scrapy.item import Item, Field
|
||||
from scrapy.http import Request, Response
|
||||
|
||||
|
||||
def _mocked_download_func(request, info):
|
||||
response = request.meta.get('response')
|
||||
return response() if callable(response) else response
|
||||
|
||||
|
||||
class FilesPipelineTestCase(unittest.TestCase):
|
||||
|
||||
def setUp(self):
|
||||
self.tempdir = mkdtemp()
|
||||
self.pipeline = FilesPipeline(self.tempdir, download_func=_mocked_download_func)
|
||||
self.pipeline.open_spider(None)
|
||||
|
||||
def tearDown(self):
|
||||
rmtree(self.tempdir)
|
||||
|
||||
def test_file_path(self):
|
||||
image_path = self.pipeline.file_key
|
||||
self.assertEqual(image_path("https://dev.mydeco.com/mydeco.pdf"),
|
||||
'full/c9b564df929f4bc635bdd19fde4f3d4847c757c5.pdf')
|
||||
self.assertEqual(image_path("http://www.maddiebrown.co.uk///catalogue-items//image_54642_12175_95307.txt"),
|
||||
'full/4ce274dd83db0368bafd7e406f382ae088e39219.txt')
|
||||
self.assertEqual(image_path("https://dev.mydeco.com/two/dirs/with%20spaces%2Bsigns.doc"),
|
||||
'full/94ccc495a17b9ac5d40e3eabf3afcb8c2c9b9e1a.doc')
|
||||
self.assertEqual(image_path("http://www.dfsonline.co.uk/get_prod_image.php?img=status_0907_mdm.jpg"),
|
||||
'full/4507be485f38b0da8a0be9eb2e1dfab8a19223f2.jpg')
|
||||
self.assertEqual(image_path("http://www.dorma.co.uk/images/product_details/2532/"),
|
||||
'full/97ee6f8a46cbbb418ea91502fd24176865cf39b2')
|
||||
self.assertEqual(image_path("http://www.dorma.co.uk/images/product_details/2532"),
|
||||
'full/244e0dd7d96a3b7b01f54eded250c9e272577aa1')
|
||||
|
||||
def test_fs_store(self):
|
||||
assert isinstance(self.pipeline.store, FSFilesStore)
|
||||
self.assertEqual(self.pipeline.store.basedir, self.tempdir)
|
||||
|
||||
key = 'some/image/key.jpg'
|
||||
path = os.path.join(self.tempdir, 'some', 'image', 'key.jpg')
|
||||
self.assertEqual(self.pipeline.store._get_filesystem_path(key), path)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def test_file_not_expired(self):
|
||||
item_url = "http://example.com/file.pdf"
|
||||
item = _create_item_with_files(item_url)
|
||||
patchers = [
|
||||
mock.patch.object(FilesPipeline, 'inc_stats', return_value=True),
|
||||
mock.patch.object(FSFilesStore, 'stat_file', return_value={
|
||||
'checksum': 'abc', 'last_modified': time.time()}),
|
||||
mock.patch.object(FilesPipeline, 'get_media_requests',
|
||||
return_value=[_prepare_request_object(item_url)])
|
||||
]
|
||||
map(lambda p: p.start(), patchers)
|
||||
|
||||
result = yield self.pipeline.process_item(item, None)
|
||||
self.assertEqual(result['files'][0]['checksum'], 'abc')
|
||||
|
||||
map(lambda p: p.stop(), patchers)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def test_file_expired(self):
|
||||
item_url = "http://example.com/file2.pdf"
|
||||
item = _create_item_with_files(item_url)
|
||||
patchers = [
|
||||
mock.patch.object(FSFilesStore, 'stat_file', return_value={
|
||||
'checksum': 'abc',
|
||||
'last_modified': time.time() - (FilesPipeline.EXPIRES * 60 * 60 * 24 * 2)}),
|
||||
mock.patch.object(FilesPipeline, 'get_media_requests',
|
||||
return_value=[_prepare_request_object(item_url)]),
|
||||
mock.patch.object(FilesPipeline, 'inc_stats', return_value=True)
|
||||
]
|
||||
map(lambda p: p.start(), patchers)
|
||||
|
||||
result = yield self.pipeline.process_item(item, None)
|
||||
self.assertNotEqual(result['files'][0]['checksum'], 'abc')
|
||||
|
||||
map(lambda p: p.stop(), patchers)
|
||||
|
||||
|
||||
class ItemWithFiles(Item):
|
||||
file_urls = Field()
|
||||
files = Field()
|
||||
|
||||
|
||||
def _create_item_with_files(*files):
|
||||
item = ItemWithFiles()
|
||||
item['file_urls'] = files
|
||||
return item
|
||||
|
||||
|
||||
def _prepare_request_object(item_url):
|
||||
return Request(
|
||||
item_url,
|
||||
meta={'response': Response(item_url, status=200, body='data')})
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
|
@ -16,6 +16,7 @@ else:
|
|||
if not encoders.issubset(set(Image.core.__dict__)):
|
||||
skip = 'Missing JPEG encoders'
|
||||
|
||||
|
||||
def _mocked_download_func(request, info):
|
||||
response = request.meta.get('response')
|
||||
return response() if callable(response) else response
|
||||
|
|
@ -34,7 +35,7 @@ class ImagesPipelineTestCase(unittest.TestCase):
|
|||
rmtree(self.tempdir)
|
||||
|
||||
def test_image_path(self):
|
||||
image_path = self.pipeline.image_key
|
||||
image_path = self.pipeline.file_key
|
||||
self.assertEqual(image_path("https://dev.mydeco.com/mydeco.gif"),
|
||||
'full/3fd165099d8e71b8a48b2683946e64dbfad8b52d.jpg')
|
||||
self.assertEqual(image_path("http://www.maddiebrown.co.uk///catalogue-items//image_54642_12175_95307.jpg"),
|
||||
|
|
@ -60,15 +61,6 @@ class ImagesPipelineTestCase(unittest.TestCase):
|
|||
self.assertEqual(thumbnail_name("/tmp/some.name/foo", name),
|
||||
'thumbs/50/92dac2a6a2072c5695a5dff1f865b3cb70c657bb.jpg')
|
||||
|
||||
def test_fs_store(self):
|
||||
from scrapy.contrib.pipeline.images import FSImagesStore
|
||||
assert isinstance(self.pipeline.store, FSImagesStore)
|
||||
self.assertEqual(self.pipeline.store.basedir, self.tempdir)
|
||||
|
||||
key = 'some/image/key.jpg'
|
||||
path = os.path.join(self.tempdir, 'some', 'image', 'key.jpg')
|
||||
self.assertEqual(self.pipeline.store._get_filesystem_path(key), path)
|
||||
|
||||
def test_convert_image(self):
|
||||
SIZE = (100, 100)
|
||||
# straigh forward case: RGB and JPEG
|
||||
|
|
@ -91,7 +83,6 @@ class ImagesPipelineTestCase(unittest.TestCase):
|
|||
self.assertEquals(converted.getcolors(), [(10000, (205, 230, 255))])
|
||||
|
||||
|
||||
|
||||
def _create_image(format, *a, **kw):
|
||||
buf = StringIO()
|
||||
Image.new(*a, **kw).save(buf, format)
|
||||
|
|
|
|||
Loading…
Reference in New Issue