RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

replaced remaining uses of log.msg() 'domain' argument to use 'spider' instead

This commit is contained in:
Pablo Hoffman 2009-11-06 16:11:37 -02:00
parent fee1b751e5
commit 40646d3cd1
9 changed files with 17 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
scrapy/contrib/downloadermiddleware/httpcache.py
View File

@ -118,7 +118,8 @@ class Cache(object):
if expiration_secs >= 0:
expiration_date = metadata['timestamp'] + datetime.timedelta(seconds=expiration_secs)
if datetime.datetime.utcnow() > expiration_date:
log.msg('dropping old cached response from %s' % metadata['timestamp'], level=log.DEBUG, domain=domain)
log.msg('dropping old cached response from %s' % metadata['timestamp'], \
level=log.DEBUG, domain=domain)
return None
return metadata

4
scrapy/contrib/downloadermiddleware/redirect.py
View File

@ -41,11 +41,11 @@ class RedirectMiddleware(object):
redirected.dont_filter = request.dont_filter
redirected.priority = request.priority + self.priority_adjust
log.msg("Redirecting (%s) to %s from %s" % (reason, redirected, request),
domain=spider.domain_name, level=log.DEBUG)
spider=spider, level=log.DEBUG)
return redirected
else:
log.msg("Discarding %s: max redirections reached" % request,
domain=spider.domain_name, level=log.DEBUG)
spider=spider, level=log.DEBUG)
raise IgnoreRequest
def _redirect_request_using_get(self, request, redirect_url):

4
scrapy/contrib/downloadermiddleware/retry.py
View File

@ -58,7 +58,7 @@ class RetryMiddleware(object):
if retries <= self.max_retry_times:
log.msg("Retrying %s (failed %d times): %s" % (request, retries, reason),
domain=spider.domain_name, level=log.DEBUG)
spider=spider, level=log.DEBUG)
retryreq = request.copy()
retryreq.meta['retry_times'] = retries
retryreq.dont_filter = True
@ -66,5 +66,5 @@ class RetryMiddleware(object):
return retryreq
else:
log.msg("Discarding %s (failed %d times): %s" % (request, retries, reason),
domain=spider.domain_name, level=log.DEBUG)
spider=spider, level=log.DEBUG)

2
scrapy/contrib/spidermanager.py
View File

@ -84,7 +84,7 @@ class TwistedPluginSpiderManager(object):
module_name = spider.__module__
module = sys.modules[module_name]
if hasattr(module, 'SPIDER'):
log.msg("Reloading module %s" % module_name, domain=domain, \
log.msg("Reloading module %s" % module_name, spider=spider, \
level=log.DEBUG)
new_module = rebuild(module, doLog=0)
self._spiders[domain] = new_module.SPIDER

2
scrapy/contrib/spidermiddleware/depth.py
View File

@ -25,7 +25,7 @@ class DepthMiddleware(object):
request.meta['depth'] = depth
if self.maxdepth and depth > self.maxdepth:
log.msg("Ignoring link (depth > %d): %s " % (self.maxdepth, request.url), \
level=log.DEBUG, domain=domain)
level=log.DEBUG, spider=spider)
return False
elif self.stats:
stats.inc_value('request_depth_count/%s' % depth, domain=domain)

2
scrapy/contrib/spidermiddleware/urllength.py
View File

@ -19,7 +19,7 @@ class UrlLengthMiddleware(object):
def _filter(request):
if isinstance(request, Request) and len(request.url) > self.maxlength:
log.msg("Ignoring link (url length > %d): %s " % (self.maxlength, request.url), \
level=log.DEBUG, domain=spider.domain_name)
level=log.DEBUG, spider=spider)
return False
else:
return True

2
scrapy/contrib_exp/downloadermiddleware/decompression.py
View File

@ -76,6 +76,6 @@ class DecompressionMiddleware(object):
new_response = func(response)
if new_response:
log.msg('Decompressed response with format: %s' % \
fmt, log.DEBUG, domain=spider.domain_name)
fmt, log.DEBUG, spider=spider)
return new_response
return response

12
scrapy/core/scraper.py
View File

@ -136,7 +136,7 @@ class Scraper(object):
referer = request.headers.get('Referer', None)
msg = "Spider exception caught while processing <%s> (referer: <%s>): %s" % \
(request.url, referer, _failure)
log.msg(msg, log.ERROR, domain=spider.domain_name)
log.msg(msg, log.ERROR, spider=spider)
stats.inc_value("spider_exceptions/%s" % _failure.value.__class__.__name__, \
domain=spider.domain_name)
@ -161,7 +161,7 @@ class Scraper(object):
self.engine.crawl(request=output, spider=spider)
elif isinstance(output, BaseItem):
log.msg("Scraped %s in <%s>" % (output, request.url), level=log.DEBUG, \
domain=domain)
spider=spider)
send_catch_log(signal=signals.item_scraped, sender=self.__class__, \
item=output, spider=spider, response=response)
self.sites[spider].itemproc_size += 1
@ -176,7 +176,7 @@ class Scraper(object):
pass
else:
log.msg("Spider must return Request, BaseItem or None, got %r in %s" % \
(type(output).__name__, request), log.ERROR, domain=domain)
(type(output).__name__, request), log.ERROR, spider=spider)
def _check_propagated_failure(self, spider_failure, propagated_failure, request, spider):
"""Log and silence the bugs raised outside of spiders, but still allow
@ -200,14 +200,14 @@ class Scraper(object):
if isinstance(output, Failure):
ex = output.value
if isinstance(ex, DropItem):
log.msg("Dropped %s - %s" % (item, str(ex)), level=log.WARNING, domain=domain)
log.msg("Dropped %s - %s" % (item, str(ex)), level=log.WARNING, spider=spider)
send_catch_log(signal=signals.item_dropped, sender=self.__class__, \
item=item, spider=spider, exception=output.value)
else:
log.msg('Error processing %s - %s' % (item, output), \
log.ERROR, domain=domain)
log.ERROR, spider=spider)
else:
log.msg("Passed %s" % item, log.INFO, domain=domain)
log.msg("Passed %s" % item, log.INFO, spider=spider)
send_catch_log(signal=signals.item_passed, sender=self.__class__, \
item=item, spider=spider, output=output)

2
scrapy/spider/models.py
View File

@ -66,7 +66,7 @@ class BaseSpider(object):
"""Log the given messages at the given log level. Always use this
method to send log messages from your spider
"""
log.msg(message, domain=self.domain_name, level=level)
log.msg(message, spider=self, level=level)
def start_requests(self):
reqs = []