RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Now worker pass settings to process via SCRAPY_PICKLED_SETTINGS 

--HG--
extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%4053
This commit is contained in:
olveyra 2008-07-11 16:08:28 +00:00
parent 8ea2dc94d7
commit 0c35b7ed2a
1 changed files with 3 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
scrapy/trunk/scrapy/contrib/pbcluster/worker/manager.py
View File

@ -1,7 +1,4 @@
import sys
import os
import time
import datetime
import sys, os, time, datetime, pickle
from twisted.internet import protocol, reactor
from twisted.spread import pb
@ -23,8 +20,8 @@ class ScrapyProcessProtocol(protocol.ProcessProtocol):
#We conserve original setting format for info purposes (avoid lots of unnecesary "SCRAPY_")
self.scrapy_settings = spider_settings or {}
self.scrapy_settings.update({'LOGFILE': self.logfile, 'CLUSTER_WORKER_ENABLED': '0', 'WEBCONSOLE_ENABLED': '0'})
for k in self.scrapy_settings:
self.env["SCRAPY_%s" % k] = str(self.scrapy_settings[k])
pickled_settings = pickle.dumps(self.scrapy_settings)
self.env["SCRAPY_PICKLED_SETTINGS"] = pickled_settings
self.env["PYTHONPATH"] = ":".join(sys.path)#this is need so this crawl process knows where to locate local_scrapy_settings.
def __str__(self):