diff --git a/scrapyd/default_scrapyd.conf b/scrapyd/default_scrapyd.conf index fa4bd7006..5680002cb 100644 --- a/scrapyd/default_scrapyd.conf +++ b/scrapyd/default_scrapyd.conf @@ -5,6 +5,7 @@ logs_to_keep = 5 dbs_dir = dbs max_proc = 0 max_proc_per_cpu = 4 +finished_to_keep = 100 http_port = 6800 debug = off runner = scrapyd.runner diff --git a/scrapyd/launcher.py b/scrapyd/launcher.py index 6e5fdb6f5..e8db16329 100644 --- a/scrapyd/launcher.py +++ b/scrapyd/launcher.py @@ -16,6 +16,8 @@ class Launcher(Service): def __init__(self, config, app): self.processes = {} + self.finished = [] + self.finished_to_keep = config.getint('finished_to_keep', 100) self.max_proc = config.getint('max_proc', 0) if not self.max_proc: self.max_proc = cpu_count() * config.getint('max_proc_per_cpu', 4) @@ -47,7 +49,9 @@ class Launcher(Service): self.processes[slot] = pp def _process_finished(self, _, slot): - self.processes.pop(slot) + process = self.processes.pop(slot) + self.finished.append(process) + del self.finished[:-self.finished_to_keep] # keep last 100 finished jobs self._wait_for_project(slot) diff --git a/scrapyd/webservice.py b/scrapyd/webservice.py index 7909fd0b3..79bc53f64 100644 --- a/scrapyd/webservice.py +++ b/scrapyd/webservice.py @@ -68,14 +68,15 @@ class ListSpiders(WsResource): class ListJobs(WsResource): - def render_POST(self, txrequest): + def render_GET(self, txrequest): project = txrequest.args['project'][0] spiders = self.root.launcher.processes.values() - jlist = list() - for s in spiders: - if project == s.project: - jlist.append({"job": {"id":s.job, "spider": s.spider}}) - return {"status":"ok", "jobs": jlist} + running = [{"id": s.job, "spider": s.spider} for s in spiders if s.project == project] + queue = self.root.poller.queues[project] + pending = [{"id": x["_job"], "spider": x["name"]} for x in queue.list()] + finished = [{"id": s.job, "spider": s.spider} for s in self.root.launcher.finished + if s.project == project] + return {"status":"ok", "pending": pending, "running": running, "finished": finished} class DeleteProject(WsResource):