From 485bc180dfd1c824de00ff689ca4d44a8f449770 Mon Sep 17 00:00:00 2001 From: Pablo Hoffman Date: Tue, 3 Jan 2012 22:21:26 -0200 Subject: [PATCH] scrapyd: improved web interface to also show pending and finished jobs --- scrapyd/launcher.py | 2 ++ scrapyd/website.py | 29 ++++++++++++++++++++++------- 2 files changed, 24 insertions(+), 7 deletions(-) diff --git a/scrapyd/launcher.py b/scrapyd/launcher.py index 1bc04f4c9..270209ebc 100644 --- a/scrapyd/launcher.py +++ b/scrapyd/launcher.py @@ -50,6 +50,7 @@ class Launcher(Service): def _process_finished(self, _, slot): process = self.processes.pop(slot) + process.end_time = datetime.now() self.finished.append(process) del self.finished[:-self.finished_to_keep] # keep last 100 finished jobs self._wait_for_project(slot) @@ -64,6 +65,7 @@ class ScrapyProcessProtocol(protocol.ProcessProtocol): self.spider = spider self.job = job self.start_time = datetime.now() + self.end_time = None self.env = env self.logfile = env['SCRAPY_LOG_FILE'] self.deferred = defer.Deferred() diff --git a/scrapyd/website.py b/scrapyd/website.py index 7f8c7791b..6fe5db997 100644 --- a/scrapyd/website.py +++ b/scrapyd/website.py @@ -25,7 +25,7 @@ class Root(resource.Resource): self.putChild('delversion.json', webservice.DeleteVersion(self)) self.putChild('listjobs.json', webservice.ListJobs(self)) self.putChild('logs', static.File(logsdir, 'text/plain')) - self.putChild('procmon', ProcessMonitor(self)) + self.putChild('jobs', Jobs(self)) self.update_projects() def update_projects(self): @@ -67,7 +67,7 @@ class Home(resource.Resource):

Scrapyd

Available projects: %(projects)s

@@ -86,7 +86,7 @@ monitoring)

""" % vars -class ProcessMonitor(resource.Resource): +class Jobs(resource.Resource): def __init__(self, root): resource.Resource.__init__(self) @@ -95,12 +95,19 @@ class ProcessMonitor(resource.Resource): def render(self, txrequest): s = "Scrapyd" s += "" - s += "

Process monitor

" + s += "

Jobs

" s += "

Go back

" s += "" - s += "" s += "" - s += "" + s += "" + for project, queue in self.root.poller.queues.items(): + for m in queue.list(): + s += "" + s += "" % project + s += "" % str(m['name']) + s += "" % str(m['_job']) + s += "" + s += "" for p in self.root.launcher.processes.values(): s += "" for a in ['project', 'spider', 'job', 'pid']: @@ -108,8 +115,16 @@ class ProcessMonitor(resource.Resource): s += "" % (datetime.now() - p.start_time) s += "" % (p.project, p.spider, p.job) s += "" + s += "" + for p in self.root.launcher.finished: + s += "" + for a in ['project', 'spider', 'job']: + s += "" % getattr(p, a) + s += "" + s += "" % (p.end_time - p.start_time) + s += "" % (p.project, p.spider, p.job) + s += "" s += "
ProjectSpiderJobPIDRuntimeLog
Pending
%s%s%s
Running
%sLog
Finished
%s%sLog
" s += "" s += "" return s -