some improvements and fixes to scrapy.service

scrapy/service.py

@@ -1,7 +1,7 @@
 import sys, os
 
 from twisted.python import log
-from twisted.internet import reactor, protocol, error
+from twisted.internet import reactor, defer, protocol, error
 from twisted.application.service import Service
 
 from scrapy.utils.py26 import cpu_count
@@ -11,50 +11,58 @@ from scrapy.conf import settings
 class ScrapyService(Service):
 
     def startService(self):
-        reactor.callWhenRunning(self.start_processes)
+        reactor.callWhenRunning(self.spawn_processes)
 
-    def start_processes(self):
-        for i in range(cpu_count()):
-            self.start_process(i+1)
+    def spawn_processes(self):
+        for settings_module, count in settings['PROJECTS'].items():
+            for i in range(count or cpu_count()):
+                self.spawn_process(settings_module, i)
 
-    def start_process(self, id):
-        args = [sys.executable, '-m', 'scrapy.service']
+    def spawn_process(self, settings_module, position):
+        args = [sys.executable, '-m', 'scrapy.cmdline', 'start']
         env = os.environ.copy()
-        self.set_log_file(env, id)
-        pp = ScrapyProcessProtocol(self, id, env.get('SCRAPY_LOG_FILE'))
+        botname = self.get_bot_name(settings_module)
+        logfile = self.get_log_file(settings_module, position)
+        env['SCRAPY_SETTINGS_MODULE'] = settings_module
+        env['SCRAPY_LOG_FILE'] = logfile
+        pp = ScrapyProcessProtocol(self, botname, settings_module, logfile)
         reactor.spawnProcess(pp, sys.executable, args=args, env=env)
+        pp.deferred.addCallback(self.respawn_process, args, env)
 
-    def set_log_file(self, env, suffix):
-        logfile = settings['LOG_FILE']
-        if logfile:
-            file, ext = os.path.splitext(logfile)
-            env['SCRAPY_LOG_FILE'] = "%s-%s%s" % (file, suffix, ext)
+    def respawn_process(self, pp, args, env):
+        reactor.callLater(5, reactor.spawnProcess, pp, sys.executable, \
+            args=args, env=env)
 
+    def get_log_file(self, settings_module, position):
+        botname = self.get_bot_name(settings_module)
+        basename = "%s-%s" % (botname, position) if position else botname
+        return os.path.join(settings['LOG_DIR'], "%s.log" % basename)
+
+    def get_bot_name(self, settings_module):
+        mod = __import__(settings_module, {}, {}, [''], -1)
+        return mod.BOT_NAME
 
 class ScrapyProcessProtocol(protocol.ProcessProtocol):
 
-    def __init__(self, service, id, logfile):
+    def __init__(self, service, botname, settings_module, logfile):
         self.service = service
-        self.id = id
+        self.botname = botname
+        self.settings_module = settings_module
         self.logfile = logfile
         self.pid = None
+        self.deferred = defer.Deferred()
 
     def connectionMade(self):
         self.pid = self.transport.pid
-        log.msg("Process %r started: pid=%r logfile=%r" % (self.id, self.pid, \
-            self.logfile))
+        log.msg("Crawler %r started: pid=%r settings=%r log=%r" % \
+            (self.botname, self.pid, self.settings_module, self.logfile))
 
     def processEnded(self, status):
         if isinstance(status.value, error.ProcessDone):
-            log.msg("Process %r finished: pid=%r logfile=%r" % (self.id, \
-                self.pid, self.logfile))
+            log.msg("Crawler %r finished: pid=%r settings=%r log=%r" % \
+                (self.botname, self.pid, self.settings_module, self.logfile))
         else:
-            log.msg("Process %r died: exitstatus=%r pid=%r logfile=%r" % \
-                (self.id, status.value.exitCode, self.pid, self.logfile))
-        reactor.callLater(5, self.service.start_process, self.id)
-
-
-if __name__ == '__main__':
-    from scrapy.core.manager import scrapymanager
-    scrapymanager.configure()
-    scrapymanager.start(keep_alive=True)
+            log.msg("Crawler %r died: exitstatus=%r pid=%r settings=%r log=%r" % \
+                (self.botname, status.value.exitCode, self.pid, self.settings_module, \
+                self.logfile))
+        self.deferred.callback(self)