Removed 'sender' argument when sending signals, as we're not sending it consistently, and it's not being used by receivers either

This commit is contained in:
Pablo Hoffman 2010-08-09 14:41:54 -03:00
parent 3d121898e2
commit 5aa8e63957
4 changed files with 14 additions and 15 deletions

View File

@ -19,10 +19,10 @@ default_timeout = settings.getint('DOWNLOAD_TIMEOUT')
def _create_factory(request, spider):
def _download_signals(response):
send_catch_log(signal=signals.request_uploaded, \
sender='download_http', request=request, spider=spider)
send_catch_log(signal=signals.response_downloaded, \
sender='download_http', response=response, spider=spider)
send_catch_log(signal=signals.request_uploaded, request=request, \
spider=spider)
send_catch_log(signal=signals.response_downloaded, response=response, \
spider=spider)
return response
timeout = getattr(spider, "download_timeout", None) or default_timeout

View File

@ -78,9 +78,9 @@ class DownloaderMiddlewareManager(object):
(method.im_self.__class__.__name__, type(response))
if isinstance(response, Request):
send_catch_log(signal=signals.response_received, \
sender=self.__class__, response=response, spider=spider)
response=response, spider=spider)
return response
send_catch_log(signal=signals.response_received, sender=self.__class__, \
send_catch_log(signal=signals.response_received, \
response=response, spider=spider)
return response

View File

@ -47,7 +47,7 @@ class ExecutionEngine(object):
"""Start the execution engine"""
assert not self.running, "Engine already running"
self.start_time = time()
send_catch_log(signal=signals.engine_started, sender=self.__class__)
send_catch_log(signal=signals.engine_started)
self.running = True
def stop(self):
@ -216,7 +216,7 @@ class ExecutionEngine(object):
self.downloader.open_spider(spider)
self.scraper.open_spider(spider)
stats.open_spider(spider)
send_catch_log(signals.spider_opened, sender=self.__class__, spider=spider)
send_catch_log(signals.spider_opened, spider=spider)
self.next_request(spider)
def _spider_idle(self, spider):
@ -227,7 +227,7 @@ class ExecutionEngine(object):
next loop and this function is guaranteed to be called (at least) once
again for this spider.
"""
res = send_catch_log(signal=signals.spider_idle, sender=self.__class__, \
res = send_catch_log(signal=signals.spider_idle, \
spider=spider, dont_log=DontCloseSpider)
if any(isinstance(x, Failure) and isinstance(x.value, DontCloseSpider) \
for _, x in res):
@ -264,8 +264,7 @@ class ExecutionEngine(object):
def _finish_closing_spider(self, spider):
"""This function is called after the spider has been closed"""
reason = self.closing.pop(spider, 'finished')
send_catch_log(signal=signals.spider_closed, sender=self.__class__, \
spider=spider, reason=reason)
send_catch_log(signal=signals.spider_closed, spider=spider, reason=reason)
call = self._next_request_calls.pop(spider, None)
if call and call.active():
call.cancel()
@ -280,4 +279,4 @@ class ExecutionEngine(object):
return dfd
def _finish_stopping_engine(self):
send_catch_log(signal=signals.engine_stopped, sender=self.__class__)
send_catch_log(signal=signals.engine_stopped)

View File

@ -168,7 +168,7 @@ class Scraper(object):
elif isinstance(output, BaseItem):
log.msg("Scraped %s in <%s>" % (output, request.url), level=log.DEBUG, \
spider=spider)
send_catch_log(signal=signals.item_scraped, sender=self.__class__, \
send_catch_log(signal=signals.item_scraped, \
item=output, spider=spider, response=response)
self.sites[spider].itemproc_size += 1
# FIXME: this can't be called here because the stats spider may be
@ -207,13 +207,13 @@ class Scraper(object):
ex = output.value
if isinstance(ex, DropItem):
log.msg("Dropped %s - %s" % (item, str(ex)), level=log.WARNING, spider=spider)
send_catch_log(signal=signals.item_dropped, sender=self.__class__, \
send_catch_log(signal=signals.item_dropped, \
item=item, spider=spider, exception=output.value)
else:
log.msg('Error processing %s - %s' % (item, output), \
log.ERROR, spider=spider)
else:
log.msg("Passed %s" % item, log.INFO, spider=spider)
send_catch_log(signal=signals.item_passed, sender=self.__class__, \
send_catch_log(signal=signals.item_passed, \
item=item, spider=spider, output=output)