RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

moved send_catch_log to new scrapy.utils.signal module

This commit is contained in:
Pablo Hoffman 2009-08-20 16:02:40 -03:00
parent 8fad0a5744
commit cbed0f1ea6
7 changed files with 50 additions and 38 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
scrapy/core/downloader/handlers.py
View File

@ -17,6 +17,7 @@ from scrapy.http import Headers
from scrapy.core.exceptions import NotSupported
from scrapy.utils.defer import defer_succeed
from scrapy.utils.httpobj import urlparse_cached
from scrapy.utils.signal import send_catch_log
from scrapy.core.downloader.dnscache import DNSCache
from scrapy.core.downloader.responsetypes import responsetypes
from scrapy.core.downloader.webclient import ScrapyHTTPClientFactory
@ -55,8 +56,10 @@ def create_factory(request, spider):
headers = Headers(factory.response_headers)
respcls = responsetypes.from_args(headers=headers, url=url)
r = respcls(url=request.url, status=status, headers=headers, body=body)
signals.send_catch_log(signal=signals.request_uploaded, sender='download_http', request=request, spider=spider)
signals.send_catch_log(signal=signals.response_downloaded, sender='download_http', response=r, spider=spider)
send_catch_log(signal=signals.request_uploaded, sender='download_http', \
request=request, spider=spider)
send_catch_log(signal=signals.response_downloaded, sender='download_http', \
response=r, spider=spider)
return r
factory.deferred.addCallbacks(_create_response)

7
scrapy/core/downloader/middleware.py
View File

@ -7,6 +7,7 @@ docs/topics/downloader-middleware.rst
"""
from scrapy.core import signals
from scrapy.utils.signal import send_catch_log
from scrapy import log
from scrapy.http import Request, Response
from scrapy.core.exceptions import NotConfigured
@ -76,9 +77,11 @@ class DownloaderMiddlewareManager(object):
'Middleware %s.process_response must return Response or Request, got %s' % \
(method.im_self.__class__.__name__, type(response))
if isinstance(response, Request):
signals.send_catch_log(signal=signals.response_received, sender=self.__class__, response=response, spider=spider)
send_catch_log(signal=signals.response_received, \
sender=self.__class__, response=response, spider=spider)
return response
signals.send_catch_log(signal=signals.response_received, sender=self.__class__, response=response, spider=spider)
send_catch_log(signal=signals.response_received, sender=self.__class__, \
response=response, spider=spider)
return response
def process_exception(_failure):

33
scrapy/core/engine.py
View File

@ -15,13 +15,13 @@ from scrapy import log
from scrapy.stats import stats
from scrapy.conf import settings
from scrapy.core import signals
from scrapy.core.scheduler import Scheduler
from scrapy.core.downloader import Downloader
from scrapy.core.scraper import Scraper
from scrapy.core.exceptions import IgnoreRequest, DontCloseDomain
from scrapy.http import Response, Request
from scrapy.spider import spiders
from scrapy.utils.misc import load_object
from scrapy.utils.signal import send_catch_log
from scrapy.utils.defer import mustbe_deferred
class ExecutionEngine(object):
@ -95,7 +95,7 @@ class ExecutionEngine(object):
self.control_reactor = control_reactor
reactor.callLater(0, self._mainloop)
self.start_time = datetime.utcnow()
signals.send_catch_log(signal=signals.engine_started, sender=self.__class__)
send_catch_log(signal=signals.engine_started, sender=self.__class__)
self.addtask(self._mainloop, 5.0)
for tsk, interval, now in self.tasks:
tsk.start(interval, now)
@ -114,7 +114,7 @@ class ExecutionEngine(object):
self.running = False
for domain in self.open_domains:
spider = spiders.fromdomain(domain)
signals.send_catch_log(signal=signals.domain_closed, sender=self.__class__, \
send_catch_log(signal=signals.domain_closed, sender=self.__class__, \
domain=domain, spider=spider, reason='shutdown')
stats.close_domain(domain, reason='shutdown')
for tsk, _, _ in self.tasks: # stop looping calls
@ -125,7 +125,7 @@ class ExecutionEngine(object):
p.stopListening()
if self.control_reactor and reactor.running:
reactor.stop()
signals.send_catch_log(signal=signals.engine_stopped, sender=self.__class__)
send_catch_log(signal=signals.engine_stopped, sender=self.__class__)
def pause(self):
"""Pause the execution engine"""
@ -280,26 +280,29 @@ class ExecutionEngine(object):
stats.open_domain(domain)
# XXX: sent for backwards compatibility (will be removed in Scrapy 0.8)
signals.send_catch_log(signals.domain_open, sender=self.__class__, domain=domain, spider=spider)
send_catch_log(signals.domain_open, sender=self.__class__, \
domain=domain, spider=spider)
signals.send_catch_log(signals.domain_opened, sender=self.__class__, domain=domain, spider=spider)
send_catch_log(signals.domain_opened, sender=self.__class__, \
domain=domain, spider=spider)
def _domain_idle(self, domain):
"""Called when a domain gets idle. This function is called when there are no
remaining pages to download or schedule. It can be called multiple
times. If some extension raises a DontCloseDomain exception (in the
domain_idle signal handler) the domain is not closed until the next
loop and this function is guaranteed to be called (at least) once again
for this domain.
"""Called when a domain gets idle. This function is called when there
are no remaining pages to download or schedule. It can be called
multiple times. If some extension raises a DontCloseDomain exception
(in the domain_idle signal handler) the domain is not closed until the
next loop and this function is guaranteed to be called (at least) once
again for this domain.
"""
spider = spiders.fromdomain(domain)
try:
dispatcher.send(signal=signals.domain_idle, sender=self.__class__, domain=domain, spider=spider)
dispatcher.send(signal=signals.domain_idle, sender=self.__class__, \
domain=domain, spider=spider)
except DontCloseDomain:
self.next_request(domain)
return
except:
log.exc("Exception catched on domain_idle signal dispatch")
log.err(_why="Exception catched on domain_idle signal dispatch")
if self.domain_is_idle(domain):
self.close_domain(domain, reason='finished')
@ -330,7 +333,7 @@ class ExecutionEngine(object):
self.scheduler.close_domain(domain)
self.scraper.close_domain(domain)
reason = self.closing.pop(domain, 'finished')
signals.send_catch_log(signal=signals.domain_closed, sender=self.__class__, \
send_catch_log(signal=signals.domain_closed, sender=self.__class__, \
domain=domain, spider=spider, reason=reason)
stats.close_domain(domain, reason=reason)
log.msg("Domain closed (%s)" % reason, domain=domain)

9
scrapy/core/scraper.py
View File

@ -6,6 +6,7 @@ from twisted.internet import defer
from scrapy.utils.defer import defer_result, defer_succeed, parallel
from scrapy.utils.misc import arg_to_iter, load_object
from scrapy.utils.signal import send_catch_log
from scrapy.core.exceptions import IgnoreRequest, DropItem
from scrapy.core import signals
from scrapy.http import Request, Response
@ -153,13 +154,13 @@ class Scraper(object):
if domain in self.engine.closing:
return
elif isinstance(output, Request):
signals.send_catch_log(signal=signals.request_received, request=output, \
send_catch_log(signal=signals.request_received, request=output, \
spider=spider)
self.engine.crawl(request=output, spider=spider)
elif isinstance(output, BaseItem):
log.msg("Scraped %s in <%s>" % (output, request.url), level=log.DEBUG, \
domain=domain)
signals.send_catch_log(signal=signals.item_scraped, sender=self.__class__, \
send_catch_log(signal=signals.item_scraped, sender=self.__class__, \
item=output, spider=spider, response=response)
self.sites[domain].itemproc_size += 1
stats.max_value('scraper/max_itemproc_size', \
@ -196,13 +197,13 @@ class Scraper(object):
ex = output.value
if isinstance(ex, DropItem):
log.msg("Dropped %s - %s" % (item, str(ex)), level=log.WARNING, domain=domain)
signals.send_catch_log(signal=signals.item_dropped, sender=self.__class__, \
send_catch_log(signal=signals.item_dropped, sender=self.__class__, \
item=item, spider=spider, exception=output.value)
else:
log.msg('Error processing %s - %s' % (item, output), \
log.ERROR, domain=domain)
else:
log.msg("Passed %s" % item, log.INFO, domain=domain)
signals.send_catch_log(signal=signals.item_passed, sender=self.__class__, \
send_catch_log(signal=signals.item_passed, sender=self.__class__, \
item=item, spider=spider, output=output)

12
scrapy/core/signals.py
View File

@ -5,10 +5,6 @@ These signals are documented in docs/topics/signals.rst. Please don't add new
signals here without documenting them there.
"""
from scrapy.xlib.pydispatch import dispatcher
from scrapy import log
engine_started = object()
engine_stopped = object()
domain_opened = object()
@ -25,11 +21,3 @@ item_dropped = object()
# XXX: deprecated signals (will be removed in Scrapy 0.8)
domain_open = object()
def send_catch_log(signal, sender=None, **kwargs):
"""
Send a signal and log any exceptions raised by its listeners
"""
try:
dispatcher.send(signal=signal, sender=sender, **kwargs)
except:
log.exc("Exception catched on signal dispatch")

7
scrapy/stats/collector/__init__.py
View File

@ -7,6 +7,7 @@ from scrapy.xlib.pydispatch import dispatcher
from scrapy.stats.signals import stats_domain_opened, stats_domain_closing, \
stats_domain_closed
from scrapy.utils.signal import send_catch_log
from scrapy.core import signals
from scrapy import log
from scrapy.conf import settings
@ -49,14 +50,14 @@ class StatsCollector(object):
def open_domain(self, domain):
self._stats[domain] = {}
signals.send_catch_log(stats_domain_opened, domain=domain)
send_catch_log(stats_domain_opened, domain=domain)
def close_domain(self, domain, reason):
if self._dump:
log.msg("Dumping stats:\n" + pprint.pformat(self.get_stats(domain)), \
domain=domain)
stats = self._stats.pop(domain)
signals.send_catch_log(stats_domain_closed, domain=domain, reason=reason, \
send_catch_log(stats_domain_closed, domain=domain, reason=reason, \
domain_stats=stats)
def engine_stopped(self):
@ -64,7 +65,7 @@ class StatsCollector(object):
log.msg("Dumping global stats:\n" + pprint.pformat(self.get_stats()))
def _start_closing_domain(self, domain, reason):
signals.send_catch_log(stats_domain_closing, domain=domain, reason=reason)
send_catch_log(stats_domain_closing, domain=domain, reason=reason)
self.close_domain(domain, reason)
class MemoryStatsCollector(StatsCollector):

13
scrapy/utils/signal.py Normal file
View File

@ -0,0 +1,13 @@
"""Helper functinos for working with signals"""
from scrapy.xlib.pydispatch import dispatcher
from scrapy import log
def send_catch_log(*args, **kwargs):
"""Same as dispatcher.send but logs any exceptions raised by the signal
handlers
"""
try:
dispatcher.send(*args, **kwargs)
except:
log.exc("Exception catched on signal dispatch")