RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

pass to crawl process the correct python path 

--HG--
extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%4043
This commit is contained in:
olveyra 2008-07-04 18:59:36 +00:00
parent 6bbbb0c1f6
commit 1d876db80b
1 changed files with 7 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
scrapy/trunk/scrapy/contrib/pbcluster/worker/manager.py
View File

@ -25,6 +25,7 @@ class ScrapyProcessProtocol(protocol.ProcessProtocol):
self.scrapy_settings.update({'LOGFILE': self.logfile, 'CLUSTER_WORKER_ENABLED': '0', 'WEBCONSOLE_ENABLED': '0'})
for k in self.scrapy_settings:
self.env["SCRAPY_%s" % k] = str(self.scrapy_settings[k])
self.env["PYTHONPATH"]=":".join(sys.path)
def __str__(self):
return "<ScrapyProcess domain=%s, pid=%s, status=%s>" % (self.domain, self.pid, self.status)
@ -83,6 +84,12 @@ class ClusterWorker(pb.Root):
def remote_run(self, domain, spider_settings=None):
"""Spawn process to run the given domain."""
if len(self.running) < self.maxproc:
logfile = os.path.join(self.logdir, domain, time.strftime("%FT%T.log"))
if not os.path.exists(os.path.dirname(logfile)):
os.makedirs(os.path.dirname(logfile))
scrapy_proc = ScrapyProcessProtocol(self, domain, logfile, spider_settings)
args = [sys.executable, sys.argv[0], 'crawl', domain]
self.running[domain] = scrapy_proc
try:
import pysvn
c=pysvn.Client()
@ -90,12 +97,6 @@ class ClusterWorker(pb.Root):
log.msg("Updated to revision %s." %r[0].number )
except:
pass
logfile = os.path.join(self.logdir, domain, time.strftime("%FT%T.log"))
if not os.path.exists(os.path.dirname(logfile)):
os.makedirs(os.path.dirname(logfile))
scrapy_proc = ScrapyProcessProtocol(self, domain, logfile, spider_settings)
args = [sys.executable, sys.argv[0], 'crawl', domain]
proc = reactor.spawnProcess(scrapy_proc, sys.executable, args=args, env=scrapy_proc.env)
self.running[domain] = scrapy_proc
return self.status(0, "Started process %s." % scrapy_proc)
return self.status(1, "No free slot to run another process.")