From 40646d3cd167e26b4e1eb29666712e73fee86a9b Mon Sep 17 00:00:00 2001 From: Pablo Hoffman Date: Fri, 6 Nov 2009 16:11:37 -0200 Subject: [PATCH] replaced remaining uses of log.msg() 'domain' argument to use 'spider' instead --- scrapy/contrib/downloadermiddleware/httpcache.py | 3 ++- scrapy/contrib/downloadermiddleware/redirect.py | 4 ++-- scrapy/contrib/downloadermiddleware/retry.py | 4 ++-- scrapy/contrib/spidermanager.py | 2 +- scrapy/contrib/spidermiddleware/depth.py | 2 +- scrapy/contrib/spidermiddleware/urllength.py | 2 +- .../downloadermiddleware/decompression.py | 2 +- scrapy/core/scraper.py | 12 ++++++------ scrapy/spider/models.py | 2 +- 9 files changed, 17 insertions(+), 16 deletions(-) diff --git a/scrapy/contrib/downloadermiddleware/httpcache.py b/scrapy/contrib/downloadermiddleware/httpcache.py index 583a11a48..0371deac6 100644 --- a/scrapy/contrib/downloadermiddleware/httpcache.py +++ b/scrapy/contrib/downloadermiddleware/httpcache.py @@ -118,7 +118,8 @@ class Cache(object): if expiration_secs >= 0: expiration_date = metadata['timestamp'] + datetime.timedelta(seconds=expiration_secs) if datetime.datetime.utcnow() > expiration_date: - log.msg('dropping old cached response from %s' % metadata['timestamp'], level=log.DEBUG, domain=domain) + log.msg('dropping old cached response from %s' % metadata['timestamp'], \ + level=log.DEBUG, domain=domain) return None return metadata diff --git a/scrapy/contrib/downloadermiddleware/redirect.py b/scrapy/contrib/downloadermiddleware/redirect.py index 52cbd4d9d..249862824 100644 --- a/scrapy/contrib/downloadermiddleware/redirect.py +++ b/scrapy/contrib/downloadermiddleware/redirect.py @@ -41,11 +41,11 @@ class RedirectMiddleware(object): redirected.dont_filter = request.dont_filter redirected.priority = request.priority + self.priority_adjust log.msg("Redirecting (%s) to %s from %s" % (reason, redirected, request), - domain=spider.domain_name, level=log.DEBUG) + spider=spider, level=log.DEBUG) return redirected else: log.msg("Discarding %s: max redirections reached" % request, - domain=spider.domain_name, level=log.DEBUG) + spider=spider, level=log.DEBUG) raise IgnoreRequest def _redirect_request_using_get(self, request, redirect_url): diff --git a/scrapy/contrib/downloadermiddleware/retry.py b/scrapy/contrib/downloadermiddleware/retry.py index e8f1a1735..bd8d9b866 100644 --- a/scrapy/contrib/downloadermiddleware/retry.py +++ b/scrapy/contrib/downloadermiddleware/retry.py @@ -58,7 +58,7 @@ class RetryMiddleware(object): if retries <= self.max_retry_times: log.msg("Retrying %s (failed %d times): %s" % (request, retries, reason), - domain=spider.domain_name, level=log.DEBUG) + spider=spider, level=log.DEBUG) retryreq = request.copy() retryreq.meta['retry_times'] = retries retryreq.dont_filter = True @@ -66,5 +66,5 @@ class RetryMiddleware(object): return retryreq else: log.msg("Discarding %s (failed %d times): %s" % (request, retries, reason), - domain=spider.domain_name, level=log.DEBUG) + spider=spider, level=log.DEBUG) diff --git a/scrapy/contrib/spidermanager.py b/scrapy/contrib/spidermanager.py index 21a96a128..229301c2c 100644 --- a/scrapy/contrib/spidermanager.py +++ b/scrapy/contrib/spidermanager.py @@ -84,7 +84,7 @@ class TwistedPluginSpiderManager(object): module_name = spider.__module__ module = sys.modules[module_name] if hasattr(module, 'SPIDER'): - log.msg("Reloading module %s" % module_name, domain=domain, \ + log.msg("Reloading module %s" % module_name, spider=spider, \ level=log.DEBUG) new_module = rebuild(module, doLog=0) self._spiders[domain] = new_module.SPIDER diff --git a/scrapy/contrib/spidermiddleware/depth.py b/scrapy/contrib/spidermiddleware/depth.py index 8d550ab61..9e89d2f39 100644 --- a/scrapy/contrib/spidermiddleware/depth.py +++ b/scrapy/contrib/spidermiddleware/depth.py @@ -25,7 +25,7 @@ class DepthMiddleware(object): request.meta['depth'] = depth if self.maxdepth and depth > self.maxdepth: log.msg("Ignoring link (depth > %d): %s " % (self.maxdepth, request.url), \ - level=log.DEBUG, domain=domain) + level=log.DEBUG, spider=spider) return False elif self.stats: stats.inc_value('request_depth_count/%s' % depth, domain=domain) diff --git a/scrapy/contrib/spidermiddleware/urllength.py b/scrapy/contrib/spidermiddleware/urllength.py index 148d3ad84..98e1924cf 100644 --- a/scrapy/contrib/spidermiddleware/urllength.py +++ b/scrapy/contrib/spidermiddleware/urllength.py @@ -19,7 +19,7 @@ class UrlLengthMiddleware(object): def _filter(request): if isinstance(request, Request) and len(request.url) > self.maxlength: log.msg("Ignoring link (url length > %d): %s " % (self.maxlength, request.url), \ - level=log.DEBUG, domain=spider.domain_name) + level=log.DEBUG, spider=spider) return False else: return True diff --git a/scrapy/contrib_exp/downloadermiddleware/decompression.py b/scrapy/contrib_exp/downloadermiddleware/decompression.py index 3c069c611..83bc2bb97 100644 --- a/scrapy/contrib_exp/downloadermiddleware/decompression.py +++ b/scrapy/contrib_exp/downloadermiddleware/decompression.py @@ -76,6 +76,6 @@ class DecompressionMiddleware(object): new_response = func(response) if new_response: log.msg('Decompressed response with format: %s' % \ - fmt, log.DEBUG, domain=spider.domain_name) + fmt, log.DEBUG, spider=spider) return new_response return response diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index b20659cf1..06d9487c7 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -136,7 +136,7 @@ class Scraper(object): referer = request.headers.get('Referer', None) msg = "Spider exception caught while processing <%s> (referer: <%s>): %s" % \ (request.url, referer, _failure) - log.msg(msg, log.ERROR, domain=spider.domain_name) + log.msg(msg, log.ERROR, spider=spider) stats.inc_value("spider_exceptions/%s" % _failure.value.__class__.__name__, \ domain=spider.domain_name) @@ -161,7 +161,7 @@ class Scraper(object): self.engine.crawl(request=output, spider=spider) elif isinstance(output, BaseItem): log.msg("Scraped %s in <%s>" % (output, request.url), level=log.DEBUG, \ - domain=domain) + spider=spider) send_catch_log(signal=signals.item_scraped, sender=self.__class__, \ item=output, spider=spider, response=response) self.sites[spider].itemproc_size += 1 @@ -176,7 +176,7 @@ class Scraper(object): pass else: log.msg("Spider must return Request, BaseItem or None, got %r in %s" % \ - (type(output).__name__, request), log.ERROR, domain=domain) + (type(output).__name__, request), log.ERROR, spider=spider) def _check_propagated_failure(self, spider_failure, propagated_failure, request, spider): """Log and silence the bugs raised outside of spiders, but still allow @@ -200,14 +200,14 @@ class Scraper(object): if isinstance(output, Failure): ex = output.value if isinstance(ex, DropItem): - log.msg("Dropped %s - %s" % (item, str(ex)), level=log.WARNING, domain=domain) + log.msg("Dropped %s - %s" % (item, str(ex)), level=log.WARNING, spider=spider) send_catch_log(signal=signals.item_dropped, sender=self.__class__, \ item=item, spider=spider, exception=output.value) else: log.msg('Error processing %s - %s' % (item, output), \ - log.ERROR, domain=domain) + log.ERROR, spider=spider) else: - log.msg("Passed %s" % item, log.INFO, domain=domain) + log.msg("Passed %s" % item, log.INFO, spider=spider) send_catch_log(signal=signals.item_passed, sender=self.__class__, \ item=item, spider=spider, output=output) diff --git a/scrapy/spider/models.py b/scrapy/spider/models.py index bbec56a6c..1ab8b3d6e 100644 --- a/scrapy/spider/models.py +++ b/scrapy/spider/models.py @@ -66,7 +66,7 @@ class BaseSpider(object): """Log the given messages at the given log level. Always use this method to send log messages from your spider """ - log.msg(message, domain=self.domain_name, level=level) + log.msg(message, spider=self, level=level) def start_requests(self): reqs = []