From cbed0f1ea6bae2447081b900087d398b9783cc96 Mon Sep 17 00:00:00 2001 From: Pablo Hoffman Date: Thu, 20 Aug 2009 16:02:40 -0300 Subject: [PATCH] moved send_catch_log to new scrapy.utils.signal module --- scrapy/core/downloader/handlers.py | 7 ++++-- scrapy/core/downloader/middleware.py | 7 ++++-- scrapy/core/engine.py | 33 +++++++++++++++------------- scrapy/core/scraper.py | 9 ++++---- scrapy/core/signals.py | 12 ---------- scrapy/stats/collector/__init__.py | 7 +++--- scrapy/utils/signal.py | 13 +++++++++++ 7 files changed, 50 insertions(+), 38 deletions(-) create mode 100644 scrapy/utils/signal.py diff --git a/scrapy/core/downloader/handlers.py b/scrapy/core/downloader/handlers.py index b36b1dc11..21cf1ed50 100644 --- a/scrapy/core/downloader/handlers.py +++ b/scrapy/core/downloader/handlers.py @@ -17,6 +17,7 @@ from scrapy.http import Headers from scrapy.core.exceptions import NotSupported from scrapy.utils.defer import defer_succeed from scrapy.utils.httpobj import urlparse_cached +from scrapy.utils.signal import send_catch_log from scrapy.core.downloader.dnscache import DNSCache from scrapy.core.downloader.responsetypes import responsetypes from scrapy.core.downloader.webclient import ScrapyHTTPClientFactory @@ -55,8 +56,10 @@ def create_factory(request, spider): headers = Headers(factory.response_headers) respcls = responsetypes.from_args(headers=headers, url=url) r = respcls(url=request.url, status=status, headers=headers, body=body) - signals.send_catch_log(signal=signals.request_uploaded, sender='download_http', request=request, spider=spider) - signals.send_catch_log(signal=signals.response_downloaded, sender='download_http', response=r, spider=spider) + send_catch_log(signal=signals.request_uploaded, sender='download_http', \ + request=request, spider=spider) + send_catch_log(signal=signals.response_downloaded, sender='download_http', \ + response=r, spider=spider) return r factory.deferred.addCallbacks(_create_response) diff --git a/scrapy/core/downloader/middleware.py b/scrapy/core/downloader/middleware.py index 9822f5c59..785168244 100644 --- a/scrapy/core/downloader/middleware.py +++ b/scrapy/core/downloader/middleware.py @@ -7,6 +7,7 @@ docs/topics/downloader-middleware.rst """ from scrapy.core import signals +from scrapy.utils.signal import send_catch_log from scrapy import log from scrapy.http import Request, Response from scrapy.core.exceptions import NotConfigured @@ -76,9 +77,11 @@ class DownloaderMiddlewareManager(object): 'Middleware %s.process_response must return Response or Request, got %s' % \ (method.im_self.__class__.__name__, type(response)) if isinstance(response, Request): - signals.send_catch_log(signal=signals.response_received, sender=self.__class__, response=response, spider=spider) + send_catch_log(signal=signals.response_received, \ + sender=self.__class__, response=response, spider=spider) return response - signals.send_catch_log(signal=signals.response_received, sender=self.__class__, response=response, spider=spider) + send_catch_log(signal=signals.response_received, sender=self.__class__, \ + response=response, spider=spider) return response def process_exception(_failure): diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index b356c7cc4..3e07acc74 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -15,13 +15,13 @@ from scrapy import log from scrapy.stats import stats from scrapy.conf import settings from scrapy.core import signals -from scrapy.core.scheduler import Scheduler from scrapy.core.downloader import Downloader from scrapy.core.scraper import Scraper from scrapy.core.exceptions import IgnoreRequest, DontCloseDomain from scrapy.http import Response, Request from scrapy.spider import spiders from scrapy.utils.misc import load_object +from scrapy.utils.signal import send_catch_log from scrapy.utils.defer import mustbe_deferred class ExecutionEngine(object): @@ -95,7 +95,7 @@ class ExecutionEngine(object): self.control_reactor = control_reactor reactor.callLater(0, self._mainloop) self.start_time = datetime.utcnow() - signals.send_catch_log(signal=signals.engine_started, sender=self.__class__) + send_catch_log(signal=signals.engine_started, sender=self.__class__) self.addtask(self._mainloop, 5.0) for tsk, interval, now in self.tasks: tsk.start(interval, now) @@ -114,7 +114,7 @@ class ExecutionEngine(object): self.running = False for domain in self.open_domains: spider = spiders.fromdomain(domain) - signals.send_catch_log(signal=signals.domain_closed, sender=self.__class__, \ + send_catch_log(signal=signals.domain_closed, sender=self.__class__, \ domain=domain, spider=spider, reason='shutdown') stats.close_domain(domain, reason='shutdown') for tsk, _, _ in self.tasks: # stop looping calls @@ -125,7 +125,7 @@ class ExecutionEngine(object): p.stopListening() if self.control_reactor and reactor.running: reactor.stop() - signals.send_catch_log(signal=signals.engine_stopped, sender=self.__class__) + send_catch_log(signal=signals.engine_stopped, sender=self.__class__) def pause(self): """Pause the execution engine""" @@ -280,26 +280,29 @@ class ExecutionEngine(object): stats.open_domain(domain) # XXX: sent for backwards compatibility (will be removed in Scrapy 0.8) - signals.send_catch_log(signals.domain_open, sender=self.__class__, domain=domain, spider=spider) + send_catch_log(signals.domain_open, sender=self.__class__, \ + domain=domain, spider=spider) - signals.send_catch_log(signals.domain_opened, sender=self.__class__, domain=domain, spider=spider) + send_catch_log(signals.domain_opened, sender=self.__class__, \ + domain=domain, spider=spider) def _domain_idle(self, domain): - """Called when a domain gets idle. This function is called when there are no - remaining pages to download or schedule. It can be called multiple - times. If some extension raises a DontCloseDomain exception (in the - domain_idle signal handler) the domain is not closed until the next - loop and this function is guaranteed to be called (at least) once again - for this domain. + """Called when a domain gets idle. This function is called when there + are no remaining pages to download or schedule. It can be called + multiple times. If some extension raises a DontCloseDomain exception + (in the domain_idle signal handler) the domain is not closed until the + next loop and this function is guaranteed to be called (at least) once + again for this domain. """ spider = spiders.fromdomain(domain) try: - dispatcher.send(signal=signals.domain_idle, sender=self.__class__, domain=domain, spider=spider) + dispatcher.send(signal=signals.domain_idle, sender=self.__class__, \ + domain=domain, spider=spider) except DontCloseDomain: self.next_request(domain) return except: - log.exc("Exception catched on domain_idle signal dispatch") + log.err(_why="Exception catched on domain_idle signal dispatch") if self.domain_is_idle(domain): self.close_domain(domain, reason='finished') @@ -330,7 +333,7 @@ class ExecutionEngine(object): self.scheduler.close_domain(domain) self.scraper.close_domain(domain) reason = self.closing.pop(domain, 'finished') - signals.send_catch_log(signal=signals.domain_closed, sender=self.__class__, \ + send_catch_log(signal=signals.domain_closed, sender=self.__class__, \ domain=domain, spider=spider, reason=reason) stats.close_domain(domain, reason=reason) log.msg("Domain closed (%s)" % reason, domain=domain) diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index 681195b26..f9bc8e4f2 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -6,6 +6,7 @@ from twisted.internet import defer from scrapy.utils.defer import defer_result, defer_succeed, parallel from scrapy.utils.misc import arg_to_iter, load_object +from scrapy.utils.signal import send_catch_log from scrapy.core.exceptions import IgnoreRequest, DropItem from scrapy.core import signals from scrapy.http import Request, Response @@ -153,13 +154,13 @@ class Scraper(object): if domain in self.engine.closing: return elif isinstance(output, Request): - signals.send_catch_log(signal=signals.request_received, request=output, \ + send_catch_log(signal=signals.request_received, request=output, \ spider=spider) self.engine.crawl(request=output, spider=spider) elif isinstance(output, BaseItem): log.msg("Scraped %s in <%s>" % (output, request.url), level=log.DEBUG, \ domain=domain) - signals.send_catch_log(signal=signals.item_scraped, sender=self.__class__, \ + send_catch_log(signal=signals.item_scraped, sender=self.__class__, \ item=output, spider=spider, response=response) self.sites[domain].itemproc_size += 1 stats.max_value('scraper/max_itemproc_size', \ @@ -196,13 +197,13 @@ class Scraper(object): ex = output.value if isinstance(ex, DropItem): log.msg("Dropped %s - %s" % (item, str(ex)), level=log.WARNING, domain=domain) - signals.send_catch_log(signal=signals.item_dropped, sender=self.__class__, \ + send_catch_log(signal=signals.item_dropped, sender=self.__class__, \ item=item, spider=spider, exception=output.value) else: log.msg('Error processing %s - %s' % (item, output), \ log.ERROR, domain=domain) else: log.msg("Passed %s" % item, log.INFO, domain=domain) - signals.send_catch_log(signal=signals.item_passed, sender=self.__class__, \ + send_catch_log(signal=signals.item_passed, sender=self.__class__, \ item=item, spider=spider, output=output) diff --git a/scrapy/core/signals.py b/scrapy/core/signals.py index 37b24e6e1..32d6eb050 100644 --- a/scrapy/core/signals.py +++ b/scrapy/core/signals.py @@ -5,10 +5,6 @@ These signals are documented in docs/topics/signals.rst. Please don't add new signals here without documenting them there. """ -from scrapy.xlib.pydispatch import dispatcher - -from scrapy import log - engine_started = object() engine_stopped = object() domain_opened = object() @@ -25,11 +21,3 @@ item_dropped = object() # XXX: deprecated signals (will be removed in Scrapy 0.8) domain_open = object() -def send_catch_log(signal, sender=None, **kwargs): - """ - Send a signal and log any exceptions raised by its listeners - """ - try: - dispatcher.send(signal=signal, sender=sender, **kwargs) - except: - log.exc("Exception catched on signal dispatch") diff --git a/scrapy/stats/collector/__init__.py b/scrapy/stats/collector/__init__.py index 3ffa1c06a..2551e35a0 100644 --- a/scrapy/stats/collector/__init__.py +++ b/scrapy/stats/collector/__init__.py @@ -7,6 +7,7 @@ from scrapy.xlib.pydispatch import dispatcher from scrapy.stats.signals import stats_domain_opened, stats_domain_closing, \ stats_domain_closed +from scrapy.utils.signal import send_catch_log from scrapy.core import signals from scrapy import log from scrapy.conf import settings @@ -49,14 +50,14 @@ class StatsCollector(object): def open_domain(self, domain): self._stats[domain] = {} - signals.send_catch_log(stats_domain_opened, domain=domain) + send_catch_log(stats_domain_opened, domain=domain) def close_domain(self, domain, reason): if self._dump: log.msg("Dumping stats:\n" + pprint.pformat(self.get_stats(domain)), \ domain=domain) stats = self._stats.pop(domain) - signals.send_catch_log(stats_domain_closed, domain=domain, reason=reason, \ + send_catch_log(stats_domain_closed, domain=domain, reason=reason, \ domain_stats=stats) def engine_stopped(self): @@ -64,7 +65,7 @@ class StatsCollector(object): log.msg("Dumping global stats:\n" + pprint.pformat(self.get_stats())) def _start_closing_domain(self, domain, reason): - signals.send_catch_log(stats_domain_closing, domain=domain, reason=reason) + send_catch_log(stats_domain_closing, domain=domain, reason=reason) self.close_domain(domain, reason) class MemoryStatsCollector(StatsCollector): diff --git a/scrapy/utils/signal.py b/scrapy/utils/signal.py new file mode 100644 index 000000000..a19449486 --- /dev/null +++ b/scrapy/utils/signal.py @@ -0,0 +1,13 @@ +"""Helper functinos for working with signals""" + +from scrapy.xlib.pydispatch import dispatcher +from scrapy import log + +def send_catch_log(*args, **kwargs): + """Same as dispatcher.send but logs any exceptions raised by the signal + handlers + """ + try: + dispatcher.send(*args, **kwargs) + except: + log.exc("Exception catched on signal dispatch")