diff --git a/bin/scrapy.tac b/bin/scrapy.tac new file mode 100644 index 000000000..35e25c56f --- /dev/null +++ b/bin/scrapy.tac @@ -0,0 +1,5 @@ +from twisted.application.service import Application +from scrapy.service import ScrapyService + +application = Application("Scrapy") +ScrapyService().setServiceParent(application) diff --git a/scrapy/service.py b/scrapy/service.py new file mode 100644 index 000000000..7247fd0ca --- /dev/null +++ b/scrapy/service.py @@ -0,0 +1,60 @@ +import sys, os + +from twisted.python import log +from twisted.internet import reactor, protocol, error +from twisted.application.service import Service + +from scrapy.utils.py26 import cpu_count +from scrapy.conf import settings + + +class ScrapyService(Service): + + def startService(self): + reactor.callWhenRunning(self.start_processes) + + def start_processes(self): + for i in range(cpu_count()): + self.start_process(i+1) + + def start_process(self, id): + args = [sys.executable, '-m', 'scrapy.service'] + env = os.environ.copy() + self.set_log_file(env, id) + pp = ScrapyProcessProtocol(self, id, env.get('SCRAPY_LOG_FILE')) + reactor.spawnProcess(pp, sys.executable, args=args, env=env) + + def set_log_file(self, env, suffix): + logfile = settings['LOG_FILE'] + if logfile: + file, ext = os.path.splitext(logfile) + env['SCRAPY_LOG_FILE'] = "%s-%s%s" % (file, suffix, ext) + + +class ScrapyProcessProtocol(protocol.ProcessProtocol): + + def __init__(self, service, id, logfile): + self.service = service + self.id = id + self.logfile = logfile + self.pid = None + + def connectionMade(self): + self.pid = self.transport.pid + log.msg("Process %r started: pid=%r logfile=%r" % (self.id, self.pid, \ + self.logfile)) + + def processEnded(self, status): + if isinstance(status.value, error.ProcessDone): + log.msg("Process %r finished: pid=%r logfile=%r" % (self.id, \ + self.pid, self.logfile)) + else: + log.msg("Process %r died: exitstatus=%r pid=%r logfile=%r" % \ + (self.id, status.value.exitCode, self.pid, self.logfile)) + reactor.callLater(5, self.service.start_process, self.id) + + +if __name__ == '__main__': + from scrapy.core.manager import scrapymanager + scrapymanager.configure() + scrapymanager.start(keep_alive=True) diff --git a/scrapy/utils/py26.py b/scrapy/utils/py26.py new file mode 100644 index 000000000..44a416a55 --- /dev/null +++ b/scrapy/utils/py26.py @@ -0,0 +1,37 @@ +""" +This module provides functions added in Python 2.6, which weren't yet available +in Python 2.5. The Python 2.6 function is used when available. +""" + +import sys, os + +try: + import multiprocessing + cpu_count = multiprocessing.cpu_count +except ImportError: + def cpu_count(): + ''' + Returns the number of CPUs in the system + ''' + if sys.platform == 'win32': + try: + num = int(os.environ['NUMBER_OF_PROCESSORS']) + except (ValueError, KeyError): + num = 0 + elif 'bsd' in sys.platform or sys.platform == 'darwin': + try: + num = int(os.popen('sysctl -n hw.ncpu').read()) + except ValueError: + num = 0 + else: + try: + num = os.sysconf('SC_NPROCESSORS_ONLN') + except (ValueError, OSError, AttributeError): + num = 0 + + if num >= 1: + return num + else: + raise NotImplementedError('cannot determine number of cpus') + +