added scrapy.service and scrapy.tac for running from twistd

bin/scrapy.tac

@@ -0,0 +1,5 @@
+from twisted.application.service import Application
+from scrapy.service import ScrapyService
+
+application = Application("Scrapy")
+ScrapyService().setServiceParent(application)

scrapy/service.py

@@ -0,0 +1,60 @@
+import sys, os
+
+from twisted.python import log
+from twisted.internet import reactor, protocol, error
+from twisted.application.service import Service
+
+from scrapy.utils.py26 import cpu_count
+from scrapy.conf import settings
+
+
+class ScrapyService(Service):
+
+    def startService(self):
+        reactor.callWhenRunning(self.start_processes)
+
+    def start_processes(self):
+        for i in range(cpu_count()):
+            self.start_process(i+1)
+
+    def start_process(self, id):
+        args = [sys.executable, '-m', 'scrapy.service']
+        env = os.environ.copy()
+        self.set_log_file(env, id)
+        pp = ScrapyProcessProtocol(self, id, env.get('SCRAPY_LOG_FILE'))
+        reactor.spawnProcess(pp, sys.executable, args=args, env=env)
+
+    def set_log_file(self, env, suffix):
+        logfile = settings['LOG_FILE']
+        if logfile:
+            file, ext = os.path.splitext(logfile)
+            env['SCRAPY_LOG_FILE'] = "%s-%s%s" % (file, suffix, ext)
+
+
+class ScrapyProcessProtocol(protocol.ProcessProtocol):
+
+    def __init__(self, service, id, logfile):
+        self.service = service
+        self.id = id
+        self.logfile = logfile
+        self.pid = None
+
+    def connectionMade(self):
+        self.pid = self.transport.pid
+        log.msg("Process %r started: pid=%r logfile=%r" % (self.id, self.pid, \
+            self.logfile))
+
+    def processEnded(self, status):
+        if isinstance(status.value, error.ProcessDone):
+            log.msg("Process %r finished: pid=%r logfile=%r" % (self.id, \
+                self.pid, self.logfile))
+        else:
+            log.msg("Process %r died: exitstatus=%r pid=%r logfile=%r" % \
+                (self.id, status.value.exitCode, self.pid, self.logfile))
+        reactor.callLater(5, self.service.start_process, self.id)
+
+
+if __name__ == '__main__':
+    from scrapy.core.manager import scrapymanager
+    scrapymanager.configure()
+    scrapymanager.start(keep_alive=True)

scrapy/utils/py26.py

@@ -0,0 +1,37 @@
+"""
+This module provides functions added in Python 2.6, which weren't yet available
+in Python 2.5. The Python 2.6 function is used when available.
+"""
+
+import sys, os
+
+try:
+    import multiprocessing
+    cpu_count = multiprocessing.cpu_count
+except ImportError:
+    def cpu_count():
+        '''
+        Returns the number of CPUs in the system
+        '''
+        if sys.platform == 'win32':
+            try:
+                num = int(os.environ['NUMBER_OF_PROCESSORS'])
+            except (ValueError, KeyError):
+                num = 0
+        elif 'bsd' in sys.platform or sys.platform == 'darwin':
+            try:
+                num = int(os.popen('sysctl -n hw.ncpu').read())
+            except ValueError:
+                num = 0
+        else:
+            try:
+                num = os.sysconf('SC_NPROCESSORS_ONLN')
+            except (ValueError, OSError, AttributeError):
+                num = 0
+
+        if num >= 1:
+            return num
+        else:
+            raise NotImplementedError('cannot determine number of cpus')
+
+