mirror of https://github.com/scrapy/scrapy.git
removed unused imports and minor bug fix to media/image pipeline
--HG-- extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%40305
This commit is contained in:
parent
c74364809e
commit
9c19316d21
|
|
@ -1,9 +1,7 @@
|
|||
from __future__ import with_statement
|
||||
|
||||
import re
|
||||
import os
|
||||
import time
|
||||
import sha
|
||||
import hashlib
|
||||
import urllib
|
||||
import urlparse
|
||||
from cStringIO import StringIO
|
||||
|
|
@ -12,7 +10,6 @@ import Image
|
|||
|
||||
from scrapy import log
|
||||
from scrapy.stats import stats
|
||||
from scrapy.http import Request
|
||||
from scrapy.core.exceptions import DropItem, NotConfigured
|
||||
from scrapy.core.exceptions import HttpException
|
||||
from scrapy.conf import settings
|
||||
|
|
@ -96,6 +93,7 @@ class ImagesPipeline(MediaPipeline):
|
|||
log.msg(str(ex), level=log.WARNING, domain=info.domain)
|
||||
raise ex
|
||||
except Exception, ex:
|
||||
referer = request.headers.get('Referer')
|
||||
msg = 'Image (processing-error): Error thumbnailing %s from %s referred in <%s>: %s' % (mtype, request, referer, ex)
|
||||
log.msg(msg, level=log.WARNING, domain=info.domain)
|
||||
raise ImageException(msg)
|
||||
|
|
@ -143,7 +141,7 @@ def image_path(url):
|
|||
if os.sep != '/':
|
||||
urlpath.replace('/', os.sep)
|
||||
if query:
|
||||
img_path = os.path.join(netloc, sha.sha(url).hexdigest())
|
||||
img_path = os.path.join(netloc, hashlib.sha1(url).hexdigest())
|
||||
else:
|
||||
img_path = os.path.join(netloc, urlpath[1:])
|
||||
return urllib.unquote(img_path)
|
||||
|
|
|
|||
|
|
@ -3,10 +3,7 @@ from twisted.internet import defer
|
|||
from scrapy.utils.defer import mustbe_deferred, defer_result
|
||||
from scrapy import log
|
||||
from scrapy.core.engine import scrapyengine
|
||||
from scrapy.core.exceptions import DropItem, NotConfigured
|
||||
from scrapy.http import Request
|
||||
from scrapy.spider import spiders
|
||||
from scrapy.conf import settings
|
||||
|
||||
|
||||
class DomainInfo(object):
|
||||
|
|
|
|||
Loading…
Reference in New Issue