RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

fixed spiderstats webconsole extension

This commit is contained in:
Pablo Hoffman 2009-07-29 19:26:12 -03:00
parent 01b79e386f
commit 4a1fc74d69
1 changed files with 10 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
scrapy/contrib/webconsole/livestats.py
View File

@ -49,28 +49,30 @@ class LiveStats(object):
sch = scrapyengine.scheduler
dwl = scrapyengine.downloader
totdomains = totscraped = totcrawled = totscheduled = totactive = totpending = 0
totdomains = totscraped = totcrawled = totscheduled = totactive = totpending = totdqueued = tottransf = 0
s = banner(self)
s += "<table border='1'>\n"
s += "<tr><th>Domain</th><th>Items<br>Scraped</th><th>Pages<br>Crawled</th><th>Scheduler<br>Pending</th><th>Downloader<br/>Pending</th><th>Downloader<br/>Active</th><th>Start time</th><th>Finish time</th><th>Run time</th></tr>\n"
s += "<tr><th>Domain</th><th>Items<br>Scraped</th><th>Pages<br>Crawled</th><th>Scheduler<br>Pending</th><th>Downloader<br/>Queued</th><th>Downloader<br/>Active</th><th>Downloader<br/>Transferring</th><th>Start time</th><th>Finish time</th><th>Run time</th></tr>\n"
for d in sorted(self.domains.keys()):
scheduled = len(sch.pending_requests[d]) if d in sch.pending_requests else 0
active = len(dwl.sites[d].active) if d in dwl.sites else 0
pending = len(dwl.sites[d].queue) if d in dwl.queue else 0
dqueued = len(dwl.sites[d].queue) if d in dwl.sites else 0
transf = len(dwl.sites[d].transferring) if d in dwl.sites else 0
stats = self.domains[d]
runtime = stats.finished - stats.started if stats.finished else datetime.now() - stats.started
s += '<tr><td>%s</td><td align="right">%d</td><td align="right">%d</td><td align="right">%d</td><td align="right">%d</td><td align="right">%d</td><td>%s</td><td>%s</td><td>%s</td></tr>\n' % \
(d, stats.scraped, stats.crawled, scheduled, pending, active, str(stats.started), str(stats.finished), str(runtime))
s += '<tr><td>%s</td><td align="right">%d</td><td align="right">%d</td><td align="right">%d</td><td align="right">%d</td><td align="right">%d</td><td align="right">%d</td><td>%s</td><td>%s</td><td>%s</td></tr>\n' % \
(d, stats.scraped, stats.crawled, scheduled, dqueued, active, transf, str(stats.started), str(stats.finished), str(runtime))
totdomains += 1
totscraped += stats.scraped
totcrawled += stats.crawled
totscheduled += scheduled
totactive += active
totpending += pending
s += '<tr><td><b>%d domains</b></td><td align="right"><b>%d</b></td><td align="right"><b>%d</b></td><td align="right"><b>%d</b></td><td align="right"><b>%d</b></td><td align="right"><b>%d</b></td><td/><td/></tr>\n' % \
(totdomains, totscraped, totcrawled, totscheduled, totactive, totpending)
totdqueued += dqueued
tottransf += transf
s += '<tr><td><b>%d domains</b></td><td align="right"><b>%d</b></td><td align="right"><b>%d</b></td><td align="right"><b>%d</b></td><td align="right"><b>%d</b></td><td align="right"><b>%d</b></td><td align="right"><b>%d</b></td><td/><td/></tr>\n' % \
(totdomains, totscraped, totcrawled, totscheduled, totdqueued, totactive, tottransf)
s += "</table>\n"
s += "</body>\n"