From e71aa06f978da0a7eb37df799871eb162f311c2d Mon Sep 17 00:00:00 2001 From: olveyra Date: Tue, 15 Jul 2008 12:11:32 +0000 Subject: [PATCH] Check the node will not run a domain that is already running --HG-- extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%4065 --- .../contrib/pbcluster/worker/manager.py | 32 ++++++++++--------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/scrapy/trunk/scrapy/contrib/pbcluster/worker/manager.py b/scrapy/trunk/scrapy/contrib/pbcluster/worker/manager.py index 9d401f3e9..245afc532 100644 --- a/scrapy/trunk/scrapy/contrib/pbcluster/worker/manager.py +++ b/scrapy/trunk/scrapy/contrib/pbcluster/worker/manager.py @@ -82,19 +82,21 @@ class ClusterWorker(pb.Root): def remote_run(self, domain, spider_settings=None): """Spawn process to run the given domain.""" if len(self.running) < self.maxproc: - logfile = os.path.join(self.logdir, domain, time.strftime("%FT%T.log")) - if not os.path.exists(os.path.dirname(logfile)): - os.makedirs(os.path.dirname(logfile)) - scrapy_proc = ScrapyProcessProtocol(self, domain, logfile, spider_settings) - args = [sys.executable, sys.argv[0], 'crawl', domain] - self.running[domain] = scrapy_proc - try: - import pysvn - c=pysvn.Client() - r = c.update(settings["CLUSTER_WORKER_SVNWORKDIR"] or ".") - log.msg("Updated to revision %s." %r[0].number ) - except: - pass - proc = reactor.spawnProcess(scrapy_proc, sys.executable, args=args, env=scrapy_proc.env) - return self.status(0, "Started process %s." % scrapy_proc) + if not domain in self.running: + self.running[domain] = scrapy_proc + logfile = os.path.join(self.logdir, domain, time.strftime("%FT%T.log")) + if not os.path.exists(os.path.dirname(logfile)): + os.makedirs(os.path.dirname(logfile)) + scrapy_proc = ScrapyProcessProtocol(self, domain, logfile, spider_settings) + args = [sys.executable, sys.argv[0], 'crawl', domain] + try: + import pysvn + c=pysvn.Client() + r = c.update(settings["CLUSTER_WORKER_SVNWORKDIR"] or ".") + log.msg("Updated to revision %s." %r[0].number ) + except: + pass + proc = reactor.spawnProcess(scrapy_proc, sys.executable, args=args, env=scrapy_proc.env) + return self.status(0, "Started process %s." % scrapy_proc) + return self.status(2, "Domain %s already running." % domain ) return self.status(1, "No free slot to run another process.")