From fa644f7a5ed6099e9e88b05ec6be1aef1c5bd3ab Mon Sep 17 00:00:00 2001 From: Pablo Hoffman Date: Mon, 27 Dec 2010 16:22:32 -0200 Subject: [PATCH] Some simplifications to Scrapyd architecture and internals: - launcher no longer knows about egg storage - removed get_spider_list_from_eggifile() file and replaced by simpler get_spider_list() which doesn't receive en egg file as argument - changed "egg runner" name to just "runner" to reflect the fact that it doesn't necesarilly run eggs (though it does in the default case) --HG-- rename : scrapyd/eggrunner.py => scrapyd/runner.py --- docs/topics/scrapyd.rst | 4 ++-- scrapyd/default_scrapyd.conf | 2 +- scrapyd/eggrunner.py | 9 -------- scrapyd/eggutils.py | 22 +----------------- scrapyd/environ.py | 6 ++--- scrapyd/interfaces.py | 6 +---- scrapyd/launcher.py | 42 ++++++++++------------------------ scrapyd/runner.py | 37 ++++++++++++++++++++++++++++++ scrapyd/tests/test_eggutils.py | 14 ------------ scrapyd/tests/test_environ.py | 18 ++------------- scrapyd/tests/test_utils.py | 38 ++++++++++++++++++++++++++++-- scrapyd/utils.py | 15 ++++++++++++ scrapyd/webservice.py | 8 +++---- scrapyd/website.py | 2 +- 14 files changed, 113 insertions(+), 110 deletions(-) delete mode 100644 scrapyd/eggrunner.py create mode 100644 scrapyd/runner.py delete mode 100644 scrapyd/tests/test_eggutils.py diff --git a/docs/topics/scrapyd.rst b/docs/topics/scrapyd.rst index ecff38884..b64f8487e 100644 --- a/docs/topics/scrapyd.rst +++ b/docs/topics/scrapyd.rst @@ -192,8 +192,8 @@ logs_to_keep The number of logs to keep per spider. Defaults to ``5``. -egg_runner ----------- +runner +------ The module that will be used for launching sub-processes. You can customize the Scrapy processes launched from Scrapyd by using your own module. diff --git a/scrapyd/default_scrapyd.conf b/scrapyd/default_scrapyd.conf index b73dcf4be..fa4bd7006 100644 --- a/scrapyd/default_scrapyd.conf +++ b/scrapyd/default_scrapyd.conf @@ -7,5 +7,5 @@ max_proc = 0 max_proc_per_cpu = 4 http_port = 6800 debug = off -egg_runner = scrapyd.eggrunner +runner = scrapyd.runner application = scrapyd.app.application diff --git a/scrapyd/eggrunner.py b/scrapyd/eggrunner.py deleted file mode 100644 index 481c0a2bc..000000000 --- a/scrapyd/eggrunner.py +++ /dev/null @@ -1,9 +0,0 @@ -import os - -from scrapyd.eggutils import activate_egg - -eggpath = os.environ.get('SCRAPY_EGGFILE') -if eggpath: - activate_egg(eggpath) -from scrapy.cmdline import execute -execute() diff --git a/scrapyd/eggutils.py b/scrapyd/eggutils.py index 4a34029f1..a511a0ca5 100644 --- a/scrapyd/eggutils.py +++ b/scrapyd/eggutils.py @@ -1,24 +1,4 @@ -from __future__ import with_statement - -import os, sys, shutil, pkg_resources -from subprocess import Popen, PIPE -from tempfile import NamedTemporaryFile - -def get_spider_list_from_eggfile(eggfile, project, eggrunner='scrapyd.eggrunner'): - with NamedTemporaryFile(suffix='.egg') as f: - shutil.copyfileobj(eggfile, f) - f.flush() - eggfile.seek(0) - pargs = [sys.executable, '-m', eggrunner, 'list'] - env = os.environ.copy() - env['SCRAPY_PROJECT'] = project - env['SCRAPY_EGGFILE'] = f.name - proc = Popen(pargs, stdout=PIPE, stderr=PIPE, env=env) - out, err = proc.communicate() - if proc.returncode: - msg = err or out or 'unknown error' - raise RuntimeError(msg.splitlines()[-1]) - return out.splitlines() +import os, pkg_resources def activate_egg(eggpath): """Activate a Scrapy egg file. This is meant to be used from egg runners diff --git a/scrapyd/environ.py b/scrapyd/environ.py index 5854bcd02..8b208d76e 100644 --- a/scrapyd/environ.py +++ b/scrapyd/environ.py @@ -18,16 +18,14 @@ class Environment(object): self.settings = {} self.initenv = initenv - def get_environment(self, message, slot, eggpath): + def get_environment(self, message, slot): project = message['_project'] env = self.initenv.copy() env['SCRAPY_SLOT'] = str(slot) env['SCRAPY_PROJECT'] = project env['SCRAPY_SPIDER'] = message['_spider'] env['SCRAPY_JOB'] = message['_job'] - if eggpath: - env['SCRAPY_EGGFILE'] = eggpath - elif project in self.settings: + if project in self.settings: env['SCRAPY_SETTINGS_MODULE'] = self.settings[project] dbpath = os.path.join(self.dbs_dir, '%s.db' % project) env['SCRAPY_SQLITE_DB'] = dbpath diff --git a/scrapyd/interfaces.py b/scrapyd/interfaces.py index 686103627..1e9514269 100644 --- a/scrapyd/interfaces.py +++ b/scrapyd/interfaces.py @@ -64,13 +64,9 @@ class ISpiderScheduler(Interface): class IEnvironment(Interface): """A component to generate the environment of crawler processes""" - def get_environment(message, slot, eggpath): + def get_environment(message, slot): """Return the environment variables to use for running the process. `message` is the message received from the IPoller.next() method `slot` is the Launcher slot where the process will be running. - `eggpath` is the path to an eggfile that contains the project code. The - `eggpath` may be `None` if no egg was found for the project, in - which case the project must be on the python path and its settings - defined in scrapyd.conf [settings] section """ diff --git a/scrapyd/launcher.py b/scrapyd/launcher.py index f53b6fa47..6e5fdb6f5 100644 --- a/scrapyd/launcher.py +++ b/scrapyd/launcher.py @@ -1,6 +1,4 @@ import sys, os -from shutil import copyfileobj -from tempfile import mkstemp from datetime import datetime from twisted.internet import reactor, defer, protocol, error @@ -10,7 +8,7 @@ from twisted.python import log from scrapy.utils.py26 import cpu_count from scrapy.utils.python import stringify_dict from scrapyd.utils import get_crawl_args -from .interfaces import IPoller, IEggStorage, IEnvironment +from .interfaces import IPoller, IEnvironment class Launcher(Service): @@ -21,57 +19,41 @@ class Launcher(Service): self.max_proc = config.getint('max_proc', 0) if not self.max_proc: self.max_proc = cpu_count() * config.getint('max_proc_per_cpu', 4) - self.egg_runner = config.get('egg_runner', 'scrapyd.eggrunner') + self.runner = config.get('runner', 'scrapyd.runner') self.app = app def startService(self): for slot in range(self.max_proc): self._wait_for_project(slot) - log.msg("%s started: max_proc=%r, egg_runner=%r" % (self.parent.name, \ - self.max_proc, self.egg_runner), system="Launcher") + log.msg("%s started: max_proc=%r, runner=%r" % (self.parent.name, \ + self.max_proc, self.runner), system="Launcher") def _wait_for_project(self, slot): poller = self.app.getComponent(IPoller) poller.next().addCallback(self._spawn_process, slot) - def _get_eggpath(self, project): - eggstorage = self.app.getComponent(IEggStorage) - version, eggf = eggstorage.get(project) - if eggf is None: - return - prefix = '%s-%s-' % (project, version) - fd, eggpath = mkstemp(prefix=prefix, suffix='.egg') - lf = os.fdopen(fd, 'wb') - copyfileobj(eggf, lf) - lf.close() - return eggpath - def _spawn_process(self, message, slot): msg = stringify_dict(message, keys_only=False) project = msg['_project'] - eggpath = self._get_eggpath(project) - args = [sys.executable, '-m', self.egg_runner, 'crawl'] + args = [sys.executable, '-m', self.runner, 'crawl'] args += get_crawl_args(msg) e = self.app.getComponent(IEnvironment) - env = e.get_environment(msg, slot, eggpath) + env = e.get_environment(msg, slot) env = stringify_dict(env, keys_only=False) - pp = ScrapyProcessProtocol(eggpath, slot, project, msg['_spider'], \ + pp = ScrapyProcessProtocol(slot, project, msg['_spider'], \ msg['_job'], env) - pp.deferred.addBoth(self._process_finished, eggpath, slot) + pp.deferred.addBoth(self._process_finished, slot) reactor.spawnProcess(pp, sys.executable, args=args, env=env) self.processes[slot] = pp - def _process_finished(self, _, eggpath, slot): - if eggpath: - os.remove(eggpath) + def _process_finished(self, _, slot): self.processes.pop(slot) self._wait_for_project(slot) class ScrapyProcessProtocol(protocol.ProcessProtocol): - def __init__(self, eggfile, slot, project, spider, job, env): - self.eggfile = eggfile + def __init__(self, slot, project, spider, job, env): self.slot = slot self.pid = None self.project = project @@ -100,6 +82,6 @@ class ScrapyProcessProtocol(protocol.ProcessProtocol): self.deferred.callback(self) def log(self, msg): - msg += "project=%r spider=%r job=%r pid=%r egg=%r log=%r" % (self.project, \ - self.spider, self.job, self.pid, self.eggfile, self.logfile) + msg += "project=%r spider=%r job=%r pid=%r log=%r" % (self.project, \ + self.spider, self.job, self.pid, self.logfile) log.msg(msg, system="Launcher") diff --git a/scrapyd/runner.py b/scrapyd/runner.py new file mode 100644 index 000000000..8f47a339b --- /dev/null +++ b/scrapyd/runner.py @@ -0,0 +1,37 @@ +import os +import shutil +import tempfile +from contextlib import contextmanager + +from scrapyd import get_application +from scrapyd.interfaces import IEggStorage +from scrapyd.eggutils import activate_egg + +@contextmanager +def project_environment(project): + app = get_application() + eggstorage = app.getComponent(IEggStorage) + version, eggfile = eggstorage.get(project) + if eggfile: + prefix = '%s-%s-' % (project, version) + fd, eggpath = tempfile.mkstemp(prefix=prefix, suffix='.egg') + lf = os.fdopen(fd, 'wb') + shutil.copyfileobj(eggfile, lf) + lf.close() + activate_egg(eggpath) + else: + eggpath = None + try: + yield + finally: + if eggpath: + os.remove(eggpath) + +def main(): + project = os.environ['SCRAPY_PROJECT'] + with project_environment(project): + from scrapy.cmdline import execute + execute() + +if __name__ == '__main__': + main() diff --git a/scrapyd/tests/test_eggutils.py b/scrapyd/tests/test_eggutils.py deleted file mode 100644 index 88346b091..000000000 --- a/scrapyd/tests/test_eggutils.py +++ /dev/null @@ -1,14 +0,0 @@ -import unittest -from cStringIO import StringIO - -from scrapyd.eggutils import get_spider_list_from_eggfile -from scrapy.utils.py26 import get_data - -__package__ = 'scrapyd.tests' # required for compatibility with python 2.5 - -class EggUtilsTest(unittest.TestCase): - - def test_get_spider_list_from_eggfile(self): - eggfile = StringIO(get_data(__package__, 'mybot.egg')) - spiders = get_spider_list_from_eggfile(eggfile, 'mybot') - self.assertEqual(set(spiders), set(['spider1', 'spider2'])) diff --git a/scrapyd/tests/test_environ.py b/scrapyd/tests/test_environ.py index 172c28543..e8230a3e9 100644 --- a/scrapyd/tests/test_environ.py +++ b/scrapyd/tests/test_environ.py @@ -8,7 +8,7 @@ from scrapyd.interfaces import IEnvironment from scrapyd.config import Config from scrapyd.environ import Environment -class EggStorageTest(unittest.TestCase): +class EnvironmentTest(unittest.TestCase): def setUp(self): d = self.mktemp() @@ -24,25 +24,11 @@ class EggStorageTest(unittest.TestCase): def test_get_environment_with_eggfile(self): msg = {'_project': 'mybot', '_spider': 'myspider', '_job': 'ID'} slot = 3 - env = self.environ.get_environment(msg, slot, '/path/to/file.egg') + env = self.environ.get_environment(msg, slot) self.assertEqual(env['SCRAPY_PROJECT'], 'mybot') self.assertEqual(env['SCRAPY_SLOT'], '3') self.assertEqual(env['SCRAPY_SPIDER'], 'myspider') self.assertEqual(env['SCRAPY_JOB'], 'ID') self.assert_(env['SCRAPY_SQLITE_DB'].endswith('mybot.db')) self.assert_(env['SCRAPY_LOG_FILE'].endswith('/mybot/myspider/ID.log')) - self.assert_(env['SCRAPY_EGGFILE'].endswith('/path/to/file.egg')) self.failIf('SCRAPY_SETTINGS_MODULE' in env) - - def test_get_environment_without_eggfile(self): - msg = {'_project': 'newbot', '_spider': 'myspider', '_job': 'ID'} - slot = 3 - env = self.environ.get_environment(msg, slot, None) - self.assertEqual(env['SCRAPY_PROJECT'], 'newbot') - self.assertEqual(env['SCRAPY_SLOT'], '3') - self.assertEqual(env['SCRAPY_SPIDER'], 'myspider') - self.assertEqual(env['SCRAPY_JOB'], 'ID') - self.assert_(env['SCRAPY_SQLITE_DB'].endswith('newbot.db')) - self.assert_(env['SCRAPY_LOG_FILE'].endswith('/newbot/myspider/ID.log')) - self.assertEqual(env['SCRAPY_SETTINGS_MODULE'], 'newbot.settings') - self.failIf('SCRAPY_EGGFILE' in env) diff --git a/scrapyd/tests/test_utils.py b/scrapyd/tests/test_utils.py index c622b8d10..e5b53beb0 100644 --- a/scrapyd/tests/test_utils.py +++ b/scrapyd/tests/test_utils.py @@ -1,6 +1,16 @@ -import unittest +from __future__ import with_statement -from scrapyd.utils import get_crawl_args +import os +from cStringIO import StringIO + +from twisted.trial import unittest + +from scrapy.utils.py26 import get_data +from scrapyd.interfaces import IEggStorage +from scrapyd.utils import get_crawl_args, get_spider_list +from scrapyd import get_application + +__package__ = 'scrapyd.tests' # required for compatibility with python 2.5 class UtilsTest(unittest.TestCase): @@ -11,3 +21,27 @@ class UtilsTest(unittest.TestCase): cargs = get_crawl_args(msg) self.assertEqual(cargs, ['lala', '-a', 'arg1=val1']) assert all(isinstance(x, str) for x in cargs), cargs + +class GetSpiderListTest(unittest.TestCase): + + def test_get_spider_list(self): + path = self.mktemp() + j = os.path.join + eggs_dir = j(path, 'eggs') + os.makedirs(eggs_dir) + dbs_dir = j(path, 'dbs') + os.makedirs(dbs_dir) + logs_dir = j(path, 'logs') + os.makedirs(logs_dir) + os.chdir(path) + with open('scrapyd.conf', 'w') as f: + f.write("[scrapyd]\n") + f.write("eggs_dir = %s\n" % eggs_dir) + f.write("dbs_dir = %s\n" % dbs_dir) + f.write("logs_dir = %s\n" % logs_dir) + app = get_application() + eggstorage = app.getComponent(IEggStorage) + eggfile = StringIO(get_data(__package__, 'mybot.egg')) + eggstorage.put(eggfile, 'mybot', 'r1') + self.assertEqual(sorted(get_spider_list('mybot')), ['spider1', 'spider2']) + diff --git a/scrapyd/utils.py b/scrapyd/utils.py index ee8195413..6a670377c 100644 --- a/scrapyd/utils.py +++ b/scrapyd/utils.py @@ -1,4 +1,6 @@ +import sys import os +from subprocess import Popen, PIPE from ConfigParser import NoSectionError from scrapy.spiderqueue import SqliteSpiderQueue @@ -41,3 +43,16 @@ def get_crawl_args(message): args += ['-a'] args += ['%s=%s' % (k, v)] return args + +def get_spider_list(project, runner='scrapyd.runner'): + """Return the spider list from the given project, using the given runner""" + env = os.environ.copy() + env['SCRAPY_PROJECT'] = project + pargs = [sys.executable, '-m', runner, 'list'] + proc = Popen(pargs, stdout=PIPE, stderr=PIPE, env=env) + out, err = proc.communicate() + if proc.returncode: + msg = err or out or 'unknown error' + raise RuntimeError(msg.splitlines()[-1]) + return out.splitlines() + diff --git a/scrapyd/webservice.py b/scrapyd/webservice.py index 36bda89b7..73af9f15d 100644 --- a/scrapyd/webservice.py +++ b/scrapyd/webservice.py @@ -4,7 +4,7 @@ import uuid from cStringIO import StringIO from scrapy.utils.txweb import JsonResource -from .eggutils import get_spider_list_from_eggfile +from .utils import get_spider_list class WsResource(JsonResource): @@ -42,8 +42,8 @@ class AddVersion(WsResource): project = d['project'][0] version = d['version'][0] eggf = StringIO(d['egg'][0]) - spiders = get_spider_list_from_eggfile(eggf, project) self.root.eggstorage.put(eggf, project, version) + spiders = get_spider_list(project) self.root.update_projects() return {"status": "ok", "project": project, "version": version, \ "spiders": len(spiders)} @@ -65,9 +65,7 @@ class ListSpiders(WsResource): def render_GET(self, txrequest): project = txrequest.args['project'][0] - _, eggf = self.root.eggstorage.get(project) - spiders = get_spider_list_from_eggfile(eggf, project, \ - eggrunner=self.root.egg_runner) + spiders = get_spider_list(project, runner=self.root.runner) return {"status": "ok", "spiders": spiders} class DeleteProject(WsResource): diff --git a/scrapyd/website.py b/scrapyd/website.py index 2e6203b84..4889ad1ca 100644 --- a/scrapyd/website.py +++ b/scrapyd/website.py @@ -11,7 +11,7 @@ class Root(resource.Resource): def __init__(self, config, app): resource.Resource.__init__(self) self.debug = config.getboolean('debug', False) - self.eggrunner = config.get('egg_runner') + self.runner = config.get('runner') logsdir = config.get('logs_dir') self.app = app self.putChild('', Home(self))