RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

scrapyd: improved web interface to also show pending and finished jobs

This commit is contained in:
Pablo Hoffman 2012-01-03 22:21:26 -02:00
parent f07e968a93
commit 485bc180df
2 changed files with 24 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
scrapyd/launcher.py
View File

@ -50,6 +50,7 @@ class Launcher(Service):
def _process_finished(self, _, slot):
process = self.processes.pop(slot)
process.end_time = datetime.now()
self.finished.append(process)
del self.finished[:-self.finished_to_keep] # keep last 100 finished jobs
self._wait_for_project(slot)
@ -64,6 +65,7 @@ class ScrapyProcessProtocol(protocol.ProcessProtocol):
self.spider = spider
self.job = job
self.start_time = datetime.now()
self.end_time = None
self.env = env
self.logfile = env['SCRAPY_LOG_FILE']
self.deferred = defer.Deferred()

29
scrapyd/website.py
View File

@ -25,7 +25,7 @@ class Root(resource.Resource):
self.putChild('delversion.json', webservice.DeleteVersion(self))
self.putChild('listjobs.json', webservice.ListJobs(self))
self.putChild('logs', static.File(logsdir, 'text/plain'))
self.putChild('procmon', ProcessMonitor(self))
self.putChild('jobs', Jobs(self))
self.update_projects()
def update_projects(self):
@ -67,7 +67,7 @@ class Home(resource.Resource):
<h1>Scrapyd</h1>
<p>Available projects: <b>%(projects)s</b></p>
<ul>
<li><a href="/procmon">Process monitor</a></li>
<li><a href="/jobs">Jobs</a></li>
<li><a href="/logs/">Logs</li>
<li><a href="http://doc.scrapy.org/topics/scrapyd.html">Documentation</a></li>
</ul>
@ -86,7 +86,7 @@ monitoring)</p>
""" % vars
class ProcessMonitor(resource.Resource):
class Jobs(resource.Resource):
def __init__(self, root):
resource.Resource.__init__(self)
@ -95,12 +95,19 @@ class ProcessMonitor(resource.Resource):
def render(self, txrequest):
s = "<html><head><title>Scrapyd</title></title>"
s += "<body>"
s += "<h1>Process monitor</h1>"
s += "<h1>Jobs</h1>"
s += "<p><a href='..'>Go back</a></p>"
s += "<table border='1'>"
s += "<tr>"
s += "<th>Project</th><th>Spider</th><th>Job</th><th>PID</th><th>Runtime</th><th>Log</th>"
s += "</tr>"
s += "<tr><th colspan='6' style='background-color: #ddd'>Pending</th></tr>"
for project, queue in self.root.poller.queues.items():
for m in queue.list():
s += "<tr>"
s += "<td>%s</td>" % project
s += "<td>%s</td>" % str(m['name'])
s += "<td>%s</td>" % str(m['_job'])
s += "</tr>"
s += "<tr><th colspan='6' style='background-color: #ddd'>Running</th></tr>"
for p in self.root.launcher.processes.values():
s += "<tr>"
for a in ['project', 'spider', 'job', 'pid']:
@ -108,8 +115,16 @@ class ProcessMonitor(resource.Resource):
s += "<td>%s</td>" % (datetime.now() - p.start_time)
s += "<td><a href='/logs/%s/%s/%s.log'>Log</a></td>" % (p.project, p.spider, p.job)
s += "</tr>"
s += "<tr><th colspan='6' style='background-color: #ddd'>Finished</th></tr>"
for p in self.root.launcher.finished:
s += "<tr>"
for a in ['project', 'spider', 'job']:
s += "<td>%s</td>" % getattr(p, a)
s += "<td></td>"
s += "<td>%s</td>" % (p.end_time - p.start_time)
s += "<td><a href='/logs/%s/%s/%s.log'>Log</a></td>" % (p.project, p.spider, p.job)
s += "</tr>"
s += "</table>"
s += "</body>"
s += "</html>"
return s