some some changes to scrapyd listjobs.json api:

* the api is now a GET instead of POST (for consistency)
* the api also returns pending and finished jobs, in addition to running
  ones
* only the last 100 finished jobs are kept (can be changed through the
  finished_to_keep setting)

scrapyd/default_scrapyd.conf

@@ -5,6 +5,7 @@ logs_to_keep = 5
 dbs_dir     = dbs
 max_proc    = 0
 max_proc_per_cpu = 4
+finished_to_keep = 100
 http_port   = 6800
 debug       = off
 runner      = scrapyd.runner

scrapyd/launcher.py

@@ -16,6 +16,8 @@ class Launcher(Service):
 
     def __init__(self, config, app):
         self.processes = {}
+        self.finished = []
+        self.finished_to_keep = config.getint('finished_to_keep', 100)
         self.max_proc = config.getint('max_proc', 0)
         if not self.max_proc:
             self.max_proc = cpu_count() * config.getint('max_proc_per_cpu', 4)
@@ -47,7 +49,9 @@ class Launcher(Service):
         self.processes[slot] = pp
 
     def _process_finished(self, _, slot):
-        self.processes.pop(slot)
+        process = self.processes.pop(slot)
+        self.finished.append(process)
+        del self.finished[:-self.finished_to_keep] # keep last 100 finished jobs
         self._wait_for_project(slot)
 
 

scrapyd/webservice.py

@@ -68,14 +68,15 @@ class ListSpiders(WsResource):
 
 class ListJobs(WsResource):
 
-    def render_POST(self, txrequest):
+    def render_GET(self, txrequest):
         project = txrequest.args['project'][0]
         spiders = self.root.launcher.processes.values()
-        jlist = list()
-        for s in spiders:
-            if project == s.project:
-                jlist.append({"job": {"id":s.job, "spider": s.spider}})
-        return {"status":"ok", "jobs": jlist}
+        running = [{"id": s.job, "spider": s.spider} for s in spiders if s.project == project]
+        queue = self.root.poller.queues[project]
+        pending = [{"id": x["_job"], "spider": x["name"]} for x in queue.list()]
+        finished = [{"id": s.job, "spider": s.spider} for s in self.root.launcher.finished
+            if s.project == project]
+        return {"status":"ok", "pending": pending, "running": running, "finished": finished}
 
 class DeleteProject(WsResource):