mirror of https://github.com/scrapy/scrapy.git
Removed 'sender' argument when sending signals, as we're not sending it consistently, and it's not being used by receivers either
This commit is contained in:
parent
3d121898e2
commit
5aa8e63957
|
|
@ -19,10 +19,10 @@ default_timeout = settings.getint('DOWNLOAD_TIMEOUT')
|
|||
|
||||
def _create_factory(request, spider):
|
||||
def _download_signals(response):
|
||||
send_catch_log(signal=signals.request_uploaded, \
|
||||
sender='download_http', request=request, spider=spider)
|
||||
send_catch_log(signal=signals.response_downloaded, \
|
||||
sender='download_http', response=response, spider=spider)
|
||||
send_catch_log(signal=signals.request_uploaded, request=request, \
|
||||
spider=spider)
|
||||
send_catch_log(signal=signals.response_downloaded, response=response, \
|
||||
spider=spider)
|
||||
return response
|
||||
|
||||
timeout = getattr(spider, "download_timeout", None) or default_timeout
|
||||
|
|
|
|||
|
|
@ -78,9 +78,9 @@ class DownloaderMiddlewareManager(object):
|
|||
(method.im_self.__class__.__name__, type(response))
|
||||
if isinstance(response, Request):
|
||||
send_catch_log(signal=signals.response_received, \
|
||||
sender=self.__class__, response=response, spider=spider)
|
||||
response=response, spider=spider)
|
||||
return response
|
||||
send_catch_log(signal=signals.response_received, sender=self.__class__, \
|
||||
send_catch_log(signal=signals.response_received, \
|
||||
response=response, spider=spider)
|
||||
return response
|
||||
|
||||
|
|
|
|||
|
|
@ -47,7 +47,7 @@ class ExecutionEngine(object):
|
|||
"""Start the execution engine"""
|
||||
assert not self.running, "Engine already running"
|
||||
self.start_time = time()
|
||||
send_catch_log(signal=signals.engine_started, sender=self.__class__)
|
||||
send_catch_log(signal=signals.engine_started)
|
||||
self.running = True
|
||||
|
||||
def stop(self):
|
||||
|
|
@ -216,7 +216,7 @@ class ExecutionEngine(object):
|
|||
self.downloader.open_spider(spider)
|
||||
self.scraper.open_spider(spider)
|
||||
stats.open_spider(spider)
|
||||
send_catch_log(signals.spider_opened, sender=self.__class__, spider=spider)
|
||||
send_catch_log(signals.spider_opened, spider=spider)
|
||||
self.next_request(spider)
|
||||
|
||||
def _spider_idle(self, spider):
|
||||
|
|
@ -227,7 +227,7 @@ class ExecutionEngine(object):
|
|||
next loop and this function is guaranteed to be called (at least) once
|
||||
again for this spider.
|
||||
"""
|
||||
res = send_catch_log(signal=signals.spider_idle, sender=self.__class__, \
|
||||
res = send_catch_log(signal=signals.spider_idle, \
|
||||
spider=spider, dont_log=DontCloseSpider)
|
||||
if any(isinstance(x, Failure) and isinstance(x.value, DontCloseSpider) \
|
||||
for _, x in res):
|
||||
|
|
@ -264,8 +264,7 @@ class ExecutionEngine(object):
|
|||
def _finish_closing_spider(self, spider):
|
||||
"""This function is called after the spider has been closed"""
|
||||
reason = self.closing.pop(spider, 'finished')
|
||||
send_catch_log(signal=signals.spider_closed, sender=self.__class__, \
|
||||
spider=spider, reason=reason)
|
||||
send_catch_log(signal=signals.spider_closed, spider=spider, reason=reason)
|
||||
call = self._next_request_calls.pop(spider, None)
|
||||
if call and call.active():
|
||||
call.cancel()
|
||||
|
|
@ -280,4 +279,4 @@ class ExecutionEngine(object):
|
|||
return dfd
|
||||
|
||||
def _finish_stopping_engine(self):
|
||||
send_catch_log(signal=signals.engine_stopped, sender=self.__class__)
|
||||
send_catch_log(signal=signals.engine_stopped)
|
||||
|
|
|
|||
|
|
@ -168,7 +168,7 @@ class Scraper(object):
|
|||
elif isinstance(output, BaseItem):
|
||||
log.msg("Scraped %s in <%s>" % (output, request.url), level=log.DEBUG, \
|
||||
spider=spider)
|
||||
send_catch_log(signal=signals.item_scraped, sender=self.__class__, \
|
||||
send_catch_log(signal=signals.item_scraped, \
|
||||
item=output, spider=spider, response=response)
|
||||
self.sites[spider].itemproc_size += 1
|
||||
# FIXME: this can't be called here because the stats spider may be
|
||||
|
|
@ -207,13 +207,13 @@ class Scraper(object):
|
|||
ex = output.value
|
||||
if isinstance(ex, DropItem):
|
||||
log.msg("Dropped %s - %s" % (item, str(ex)), level=log.WARNING, spider=spider)
|
||||
send_catch_log(signal=signals.item_dropped, sender=self.__class__, \
|
||||
send_catch_log(signal=signals.item_dropped, \
|
||||
item=item, spider=spider, exception=output.value)
|
||||
else:
|
||||
log.msg('Error processing %s - %s' % (item, output), \
|
||||
log.ERROR, spider=spider)
|
||||
else:
|
||||
log.msg("Passed %s" % item, log.INFO, spider=spider)
|
||||
send_catch_log(signal=signals.item_passed, sender=self.__class__, \
|
||||
send_catch_log(signal=signals.item_passed, \
|
||||
item=item, spider=spider, output=output)
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue