From e8e760c9740b8a3f7da565467c2290197583ebb2 Mon Sep 17 00:00:00 2001 From: Pablo Hoffman Date: Mon, 31 Aug 2009 12:04:01 -0300 Subject: [PATCH] more simplifications to scrapy engine: removed addtasks method --- scrapy/conf/default_settings.py | 1 - scrapy/contrib/memusage.py | 31 +++++++++------ scrapy/core/engine.py | 67 ++++++++++++--------------------- scrapy/utils/mysql.py | 10 ----- 4 files changed, 44 insertions(+), 65 deletions(-) diff --git a/scrapy/conf/default_settings.py b/scrapy/conf/default_settings.py index 1bd591662..b837c0483 100644 --- a/scrapy/conf/default_settings.py +++ b/scrapy/conf/default_settings.py @@ -119,7 +119,6 @@ MEMUSAGE_REPORT = False MEMUSAGE_WARNING_MB = 0 MYSQL_CONNECTION_SETTINGS = {} -MYSQL_CONNECTION_PING_PERIOD = 0 NEWSPIDER_MODULE = '' diff --git a/scrapy/contrib/memusage.py b/scrapy/contrib/memusage.py index 38fe1e643..df7ef7fea 100644 --- a/scrapy/contrib/memusage.py +++ b/scrapy/contrib/memusage.py @@ -7,8 +7,9 @@ See documentation in docs/topics/extensions.rst import os import socket -from scrapy.xlib.pydispatch import dispatcher +from twisted.internet import task +from scrapy.xlib.pydispatch import dispatcher from scrapy.core import signals from scrapy import log from scrapy.core.manager import scrapymanager @@ -26,24 +27,15 @@ class MemoryUsage(object): raise NotConfigured if not os.path.exists('/proc'): raise NotConfigured - self.warned = False - scrapyengine.addtask(self.update, 60.0, now=True) - self.notify_mails = settings.getlist('MEMUSAGE_NOTIFY') self.limit = settings.getint('MEMUSAGE_LIMIT_MB')*1024*1024 self.warning = settings.getint('MEMUSAGE_WARNING_MB')*1024*1024 self.report = settings.getbool('MEMUSAGE_REPORT') - - if self.limit: - scrapyengine.addtask(self._check_limit, 60.0, now=True) - if self.warning: - scrapyengine.addtask(self._check_warning, 60.0, now=True) - self.mail = MailSender() - dispatcher.connect(self.engine_started, signal=signals.engine_started) + dispatcher.connect(self.engine_stopped, signal=signals.engine_stopped) @property @@ -52,6 +44,23 @@ class MemoryUsage(object): def engine_started(self): stats.set_value('memusage/startup', self.virtual) + self.tasks = [] + tsk = task.LoopingCall(self.update) + self.tasks.append(tsk) + tsk.start(60.0, now=True) + if self.limit: + tsk = task.LoopingCall(self._check_limit) + self.tasks.append(tsk) + tsk.start(60.0, now=True) + if self.warning: + tsk = task.LoopingCall(self._check_warning) + self.tasks.append(tsk) + tsk.start(60.0, now=True) + + def engine_stopped(self): + for tsk in self.tasks: + if tsk.running: + tsk.stop() def update(self): stats.max_value('memusage/max', self.virtual) diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index 4481851cd..70b33dc40 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -7,7 +7,6 @@ For more information see docs/topics/architecture.rst from datetime import datetime from twisted.internet import reactor, task -from twisted.internet.error import CannotListenError from twisted.python.failure import Failure from scrapy.xlib.pydispatch import dispatcher @@ -30,11 +29,11 @@ class ExecutionEngine(object): self.configured = False self.keep_alive = False self.closing = {} # dict (domain -> reason) of spiders being closed - self.tasks = [] self.running = False self.paused = False self.control_reactor = True self._next_request_pending = set() + self._mainloop_task = task.LoopingCall(self._mainloop) def configure(self): """ @@ -46,52 +45,34 @@ class ExecutionEngine(object): self.scraper = Scraper(self) self.configured = True - def addtask(self, function, interval, args=None, kwargs=None, now=False): - """ - Adds a looping task. Use this instead of twisted task.LooopingCall to - make sure the reactor is left in a clean state after the engine is - stopped. - """ - if not args: - args = [] - if not kwargs: - kwargs = {} - tsk = task.LoopingCall(function, *args, **kwargs) - self.tasks.append((tsk, interval, now)) - if self.running: - tsk.start(interval, now) - return tsk - def start(self, control_reactor=True): """Start the execution engine""" - if not self.running: - self.control_reactor = control_reactor - reactor.callLater(0, self._mainloop) - self.start_time = datetime.utcnow() - send_catch_log(signal=signals.engine_started, sender=self.__class__) - self.addtask(self._mainloop, 5.0) - for tsk, interval, now in self.tasks: - tsk.start(interval, now) - self.running = True - if control_reactor: - reactor.run() # blocking call + if self.running: + return + self.control_reactor = control_reactor + self.start_time = datetime.utcnow() + send_catch_log(signal=signals.engine_started, sender=self.__class__) + self._mainloop_task.start(5.0, now=True) + reactor.callWhenRunning(self._mainloop) + self.running = True + if control_reactor: + reactor.run() # blocking call def stop(self): """Stop the execution engine""" - if self.running: - self.running = False - for domain in self.open_domains: - spider = spiders.fromdomain(domain) - send_catch_log(signal=signals.domain_closed, sender=self.__class__, \ - domain=domain, spider=spider, reason='shutdown') - stats.close_domain(domain, reason='shutdown') - for tsk, _, _ in self.tasks: # stop looping calls - if tsk.running: - tsk.stop() - self.tasks = [] - if self.control_reactor and reactor.running: - reactor.stop() - send_catch_log(signal=signals.engine_stopped, sender=self.__class__) + if not self.running: + return + self.running = False + for domain in self.open_domains: + spider = spiders.fromdomain(domain) + send_catch_log(signal=signals.domain_closed, sender=self.__class__, \ + domain=domain, spider=spider, reason='shutdown') + stats.close_domain(domain, reason='shutdown') + if self._mainloop_task.running: + self._mainloop_task.stop() + if self.control_reactor and reactor.running: + reactor.stop() + send_catch_log(signal=signals.engine_stopped, sender=self.__class__) def pause(self): """Pause the execution engine""" diff --git a/scrapy/utils/mysql.py b/scrapy/utils/mysql.py index 15859d2a2..a7081f9ed 100644 --- a/scrapy/utils/mysql.py +++ b/scrapy/utils/mysql.py @@ -4,7 +4,6 @@ import re import MySQLdb -from scrapy.core.engine import scrapyengine from scrapy.conf import settings from scrapy import log @@ -37,13 +36,4 @@ def mysql_connect(db_uri_or_dict, **kwargs): d['host'], d['user']), level=log.DEBUG) conn = MySQLdb.connect(**d) - # connection keep-alive - def conn_ping(): - log.msg("Pinging connection to %s/%s" % (d['host'], d['db']), \ - level=log.DEBUG) - conn.ping() - ping_period = settings.getint("MYSQL_CONNECTION_PING_PERIOD") - if ping_period: - scrapyengine.addtask(conn_ping, ping_period) - return conn