From 8344819a927da9130d5a2b57325d1d8e1aa8c1da Mon Sep 17 00:00:00 2001 From: olveyra Date: Wed, 2 Jul 2008 15:07:29 +0000 Subject: [PATCH] added persistence of pending list in master --HG-- extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%4039 --- .../trunk/scrapy/contrib/pbcluster/master/manager.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/scrapy/trunk/scrapy/contrib/pbcluster/master/manager.py b/scrapy/trunk/scrapy/contrib/pbcluster/master/manager.py index 19ad1a96b..b951209a8 100644 --- a/scrapy/trunk/scrapy/contrib/pbcluster/master/manager.py +++ b/scrapy/trunk/scrapy/contrib/pbcluster/master/manager.py @@ -2,6 +2,7 @@ import urlparse import urllib import bisect import sys +import pickle, socket from pydispatch import dispatcher @@ -86,12 +87,17 @@ class Node: class ClusterMaster(object): def __init__(self): + if not settings.getbool('CLUSTER_MASTER_ENABLED'): raise NotConfigured + try: + self.pending = pickle.load( open("pending_cache_%s" % socket.gethostname(), "r") ) + except IOError: + self.pending = [] self.nodes = {} - self.pending = [] dispatcher.connect(self._engine_started, signal=signals.engine_started) - + dispatcher.connect(self._engine_stopped, signal=signals.engine_stopped) + def load_nodes(self): def _make_callback(_factory, _name, _url): @@ -191,3 +197,5 @@ class ClusterMaster(object): def _engine_started(self): self.load_nodes() scrapyengine.addtask(self.update_nodes, settings.getint('CLUSTER_MASTER_POLL_INTERVAL')) + def _engine_stopped(self): + pickle.dump( self.pending, open("pending_cache_%s" % socket.gethostname(), "w") ) \ No newline at end of file