diff --git a/scrapyd/environ.py b/scrapyd/environ.py index 70bef2974..bf511a4cf 100644 --- a/scrapyd/environ.py +++ b/scrapyd/environ.py @@ -19,7 +19,7 @@ class Environment(object): self.initenv = initenv def get_environment(self, message, slot, eggpath): - project = message['project'] + project = message['_project'] env = self.initenv.copy() env['SCRAPY_PROJECT'] = project if eggpath: @@ -32,11 +32,11 @@ class Environment(object): return env def _get_log_file(self, message): - logsdir = os.path.join(self.logs_dir, message['project'], \ - message['spider']) + logsdir = os.path.join(self.logs_dir, message['_project'], \ + message['_spider']) if not os.path.exists(logsdir): os.makedirs(logsdir) to_delete = sorted(os.listdir(logsdir), reverse=True)[:-self.logs_to_keep] for x in to_delete: os.remove(os.path.join(logsdir, x)) - return os.path.join(logsdir, "%s.log" % message['_id']) + return os.path.join(logsdir, "%s.log" % message['_job']) diff --git a/scrapyd/interfaces.py b/scrapyd/interfaces.py index c9c6f839e..686103627 100644 --- a/scrapyd/interfaces.py +++ b/scrapyd/interfaces.py @@ -35,9 +35,11 @@ class IPoller(Interface): project that needs to run, or already fired if there was a project waiting to run already. - The message is a dict containing (at least) the name of the project to - be run in the 'project' key. This message will be passed later to - IEnvironment.get_environment(). + The message is a dict containing (at least): + * the name of the project to be run in the '_project' key + * the name of the spider to be run in the '_spider' key + * a unique identifier for this run in the `_job` key + This message will be passed later to IEnvironment.get_environment(). """ def update_projects(): @@ -65,9 +67,7 @@ class IEnvironment(Interface): def get_environment(message, slot, eggpath): """Return the environment variables to use for running the process. - `message` is the message received from the IPoller.next() augmented to - contain (at least) the following keys: project, spider, _id (where - _id is a unique identifier for this run) + `message` is the message received from the IPoller.next() method `slot` is the Launcher slot where the process will be running. `eggpath` is the path to an eggfile that contains the project code. The `eggpath` may be `None` if no egg was found for the project, in diff --git a/scrapyd/launcher.py b/scrapyd/launcher.py index a2eff56b9..4eab512c7 100644 --- a/scrapyd/launcher.py +++ b/scrapyd/launcher.py @@ -48,13 +48,15 @@ class Launcher(Service): def _spawn_process(self, message, slot): msg = stringify_dict(message, keys_only=False) - project = msg['project'] + project = msg['_project'] eggpath = self._get_eggpath(project) args = [sys.executable, '-m', self.egg_runner, 'crawl'] args += get_crawl_args(msg) e = self.app.getComponent(IEnvironment) env = e.get_environment(msg, slot, eggpath) - pp = ScrapyProcessProtocol(eggpath, slot, project, msg['spider'], msg['_id']) + env = stringify_dict(env, keys_only=False) + pp = ScrapyProcessProtocol(eggpath, slot, project, msg['_spider'], \ + msg['_job'], env) pp.deferred.addBoth(self._process_finished, eggpath, slot) reactor.spawnProcess(pp, sys.executable, args=args, env=env) self.processes[slot] = pp @@ -68,7 +70,7 @@ class Launcher(Service): class ScrapyProcessProtocol(protocol.ProcessProtocol): - def __init__(self, eggfile, slot, project, spider, job): + def __init__(self, eggfile, slot, project, spider, job, env): self.eggfile = eggfile self.slot = slot self.pid = None @@ -76,6 +78,8 @@ class ScrapyProcessProtocol(protocol.ProcessProtocol): self.spider = spider self.job = job self.start_time = datetime.now() + self.env = env + self.logfile = env['SCRAPY_LOG_FILE'] self.deferred = defer.Deferred() def outReceived(self, data): diff --git a/scrapyd/poller.py b/scrapyd/poller.py index 05c9feaa3..e828617be 100644 --- a/scrapyd/poller.py +++ b/scrapyd/poller.py @@ -29,6 +29,6 @@ class QueuePoller(object): def _message(self, queue_msg, project): d = queue_msg.copy() - d['project'] = project - d['spider'] = d.pop('name') + d['_project'] = project + d['_spider'] = d.pop('name') return d diff --git a/scrapyd/tests/test_environ.py b/scrapyd/tests/test_environ.py index 2133a5d5b..b7c3d97ef 100644 --- a/scrapyd/tests/test_environ.py +++ b/scrapyd/tests/test_environ.py @@ -22,7 +22,7 @@ class EggStorageTest(unittest.TestCase): verifyObject(IEnvironment, self.environ) def test_get_environment_with_eggfile(self): - msg = {'project': 'mybot', 'spider': 'myspider', '_id': 'ID'} + msg = {'_project': 'mybot', '_spider': 'myspider', '_job': 'ID'} slot = 3 env = self.environ.get_environment(msg, slot, '/path/to/file.egg') self.assertEqual(env['SCRAPY_PROJECT'], 'mybot') @@ -32,7 +32,7 @@ class EggStorageTest(unittest.TestCase): self.failIf('SCRAPY_SETTINGS_MODULE' in env) def test_get_environment_without_eggfile(self): - msg = {'project': 'newbot', 'spider': 'myspider', '_id': 'ID'} + msg = {'_project': 'newbot', '_spider': 'myspider', '_job': 'ID'} slot = 3 env = self.environ.get_environment(msg, slot, None) self.assertEqual(env['SCRAPY_PROJECT'], 'newbot') diff --git a/scrapyd/tests/test_poller.py b/scrapyd/tests/test_poller.py index 0cae9c13c..32291d3e8 100644 --- a/scrapyd/tests/test_poller.py +++ b/scrapyd/tests/test_poller.py @@ -37,5 +37,5 @@ class QueuePollerTest(unittest.TestCase): self.poller.poll() self.queues['mybot1'].pop() self.poller.poll() - self.failUnlessEqual(d1.result, {'project': 'mybot1', 'spider': 'spider1'}) - self.failUnlessEqual(d2.result, {'project': 'mybot2', 'spider': 'spider2'}) + self.failUnlessEqual(d1.result, {'_project': 'mybot1', '_spider': 'spider1'}) + self.failUnlessEqual(d2.result, {'_project': 'mybot2', '_spider': 'spider2'}) diff --git a/scrapyd/tests/test_utils.py b/scrapyd/tests/test_utils.py index 8bc174100..c622b8d10 100644 --- a/scrapyd/tests/test_utils.py +++ b/scrapyd/tests/test_utils.py @@ -5,9 +5,9 @@ from scrapyd.utils import get_crawl_args class UtilsTest(unittest.TestCase): def test_get_crawl_args(self): - msg = {'project': 'lolo', 'spider': 'lala'} + msg = {'_project': 'lolo', '_spider': 'lala'} self.assertEqual(get_crawl_args(msg), ['lala']) - msg = {'project': 'lolo', 'spider': 'lala', 'arg1': u'val1'} + msg = {'_project': 'lolo', '_spider': 'lala', 'arg1': u'val1'} cargs = get_crawl_args(msg) self.assertEqual(cargs, ['lala', '-a', 'arg1=val1']) assert all(isinstance(x, str) for x in cargs), cargs diff --git a/scrapyd/utils.py b/scrapyd/utils.py index 0971f6f0d..ee8195413 100644 --- a/scrapyd/utils.py +++ b/scrapyd/utils.py @@ -35,8 +35,8 @@ def get_crawl_args(message): that will be started for this message """ msg = message.copy() - args = [unicode_to_str(msg['spider'])] - del msg['project'], msg['spider'] + args = [unicode_to_str(msg['_spider'])] + del msg['_project'], msg['_spider'] for k, v in stringify_dict(msg, keys_only=False).items(): args += ['-a'] args += ['%s=%s' % (k, v)] diff --git a/scrapyd/webservice.py b/scrapyd/webservice.py index 6c6fa5b73..36bda89b7 100644 --- a/scrapyd/webservice.py +++ b/scrapyd/webservice.py @@ -28,7 +28,7 @@ class Schedule(WsResource): project = args.pop('project') spider = args.pop('spider') jobid = uuid.uuid1().hex - args['_id'] = jobid + args['_job'] = jobid self.root.scheduler.schedule(project, spider, **args) jobids = {spider: jobid} return {"status": "ok", "jobs": jobids} diff --git a/scrapyd/website.py b/scrapyd/website.py index c53866832..61075f3a9 100644 --- a/scrapyd/website.py +++ b/scrapyd/website.py @@ -57,8 +57,8 @@ class Home(resource.Resource):
| Project | Spider | Job | PID | Runtime | " + s += "Project | Spider | Job | PID | Runtime | Log | " s += "
|---|---|---|---|---|---|---|---|---|---|---|
| %s | " % getattr(p, a) s += "%s | " % (datetime.now() - p.start_time) + s += "Log | " % (p.project, p.spider, p.job) s += "