RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Check the node will not run a domain that is already running 

--HG--
extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%4065
This commit is contained in:
olveyra 2008-07-15 12:11:32 +00:00
parent 2b6189a3f8
commit e71aa06f97
1 changed files with 17 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

32
scrapy/trunk/scrapy/contrib/pbcluster/worker/manager.py
View File

@ -82,19 +82,21 @@ class ClusterWorker(pb.Root):
def remote_run(self, domain, spider_settings=None):
"""Spawn process to run the given domain."""
if len(self.running) < self.maxproc:
logfile = os.path.join(self.logdir, domain, time.strftime("%FT%T.log"))
if not os.path.exists(os.path.dirname(logfile)):
os.makedirs(os.path.dirname(logfile))
scrapy_proc = ScrapyProcessProtocol(self, domain, logfile, spider_settings)
args = [sys.executable, sys.argv[0], 'crawl', domain]
self.running[domain] = scrapy_proc
try:
import pysvn
c=pysvn.Client()
r = c.update(settings["CLUSTER_WORKER_SVNWORKDIR"] or ".")
log.msg("Updated to revision %s." %r[0].number )
except:
pass
proc = reactor.spawnProcess(scrapy_proc, sys.executable, args=args, env=scrapy_proc.env)
return self.status(0, "Started process %s." % scrapy_proc)
if not domain in self.running:
self.running[domain] = scrapy_proc
logfile = os.path.join(self.logdir, domain, time.strftime("%FT%T.log"))
if not os.path.exists(os.path.dirname(logfile)):
os.makedirs(os.path.dirname(logfile))
scrapy_proc = ScrapyProcessProtocol(self, domain, logfile, spider_settings)
args = [sys.executable, sys.argv[0], 'crawl', domain]
try:
import pysvn
c=pysvn.Client()
r = c.update(settings["CLUSTER_WORKER_SVNWORKDIR"] or ".")
log.msg("Updated to revision %s." %r[0].number )
except:
pass
proc = reactor.spawnProcess(scrapy_proc, sys.executable, args=args, env=scrapy_proc.env)
return self.status(0, "Started process %s." % scrapy_proc)
return self.status(2, "Domain %s already running." % domain )
return self.status(1, "No free slot to run another process.")