mirror of https://github.com/scrapy/scrapy.git
ported spiderctl web console extensin to work with new core based on spider references
This commit is contained in:
Pablo Hoffman
parent
47aa716630
commit
fcbbb5001e
|
|
@ -16,7 +16,7 @@ class Spiderctl(object):
|
|||
webconsole_name = 'Spider control panel'
|
||||
|
||||
def __init__(self):
|
||||
self.running = set()
|
||||
self.running = {}
|
||||
self.finished = set()
|
||||
dispatcher.connect(self.domain_opened, signal=signals.domain_opened)
|
||||
dispatcher.connect(self.domain_closed, signal=signals.domain_closed)
|
||||
|
|
@ -24,42 +24,41 @@ class Spiderctl(object):
|
|||
from scrapy.management.web import webconsole_discover_module
|
||||
dispatcher.connect(self.webconsole_discover_module, signal=webconsole_discover_module)
|
||||
|
||||
def domain_opened(self, domain, spider):
|
||||
self.running.add(domain)
|
||||
def domain_opened(self, spider):
|
||||
self.running[spider.domain_name] = spider
|
||||
|
||||
def domain_closed(self, domain, spider):
|
||||
self.running.remove(domain)
|
||||
self.finished.add(domain)
|
||||
def domain_closed(self, spider):
|
||||
del self.running[spider.domain_name]
|
||||
self.finished.add(spider.domain_name)
|
||||
|
||||
def webconsole_render(self, wc_request):
|
||||
if wc_request.args:
|
||||
changes = self.webconsole_control(wc_request)
|
||||
|
||||
enabled_domains = spiders.list()
|
||||
self.scheduled = scrapyengine.domain_scheduler.pending_domains
|
||||
self.idle = [d for d in enabled_domains if d not in self.scheduled
|
||||
self.scheduled = [s.domain_name for s in scrapyengine.spider_scheduler._pending_spiders]
|
||||
self.idle = [d for d in self.enabled_domains if d not in self.scheduled
|
||||
and d not in self.running
|
||||
and d not in self.finished]
|
||||
|
||||
s = banner(self)
|
||||
s += '<table border=1">\n'
|
||||
s += "<tr><th>Running (%d/%d)</th><th>Scheduled (%d)</th><th>Finished (%d)</th><th>Idle (%d)</th></tr>\n" % \
|
||||
(len(self.running),
|
||||
s += "<tr><th>Idle (%d)</th><th>Scheduled (%d)</th><th>Running (%d/%d)</th><th>Finished (%d)</th></tr>\n" % \
|
||||
(len(self.idle),
|
||||
len(self.running),
|
||||
settings['CONCURRENT_DOMAINS'],
|
||||
len(self.scheduled),
|
||||
len(self.finished),
|
||||
len(self.idle))
|
||||
len(self.finished))
|
||||
s += "<tr>\n"
|
||||
|
||||
# running
|
||||
# idle
|
||||
s += "<td valign='top'>\n"
|
||||
s += '<form method="post" action=".">\n'
|
||||
s += '<select name="stop_running_domains" multiple="multiple">\n'
|
||||
for domain in sorted(self.running):
|
||||
s += '<select name="add_pending_domains" multiple="multiple">\n'
|
||||
for domain in sorted(self.idle):
|
||||
s += "<option>%s</option>\n" % domain
|
||||
s += '</select><br>\n'
|
||||
s += '<br />'
|
||||
s += '<input type="submit" value="Stop selected">\n'
|
||||
s += '<input type="submit" value="Schedule selected">\n'
|
||||
s += '</form>\n'
|
||||
s += "</td>\n"
|
||||
|
||||
|
|
@ -76,6 +75,18 @@ class Spiderctl(object):
|
|||
|
||||
s += "</td>\n"
|
||||
|
||||
# running
|
||||
s += "<td valign='top'>\n"
|
||||
s += '<form method="post" action=".">\n'
|
||||
s += '<select name="stop_running_domains" multiple="multiple">\n'
|
||||
for domain in sorted(self.running):
|
||||
s += "<option>%s</option>\n" % domain
|
||||
s += '</select><br>\n'
|
||||
s += '<br />'
|
||||
s += '<input type="submit" value="Stop selected">\n'
|
||||
s += '</form>\n'
|
||||
s += "</td>\n"
|
||||
|
||||
# finished
|
||||
s += "<td valign='top'>\n"
|
||||
s += '<form method="post" action=".">\n'
|
||||
|
|
@ -88,45 +99,7 @@ class Spiderctl(object):
|
|||
s += '</form>\n'
|
||||
s += "</td>\n"
|
||||
|
||||
# idle
|
||||
s += "<td valign='top'>\n"
|
||||
s += '<form method="post" action=".">\n'
|
||||
s += '<select name="add_pending_domains" multiple="multiple">\n'
|
||||
for domain in sorted(self.idle):
|
||||
s += "<option>%s</option>\n" % domain
|
||||
s += '</select><br>\n'
|
||||
s += '<br />'
|
||||
s += '<input type="submit" value="Schedule selected">\n'
|
||||
s += '</form>\n'
|
||||
s += "</td>\n"
|
||||
|
||||
s += "</tr>\n"
|
||||
s += "<tr>\n"
|
||||
|
||||
s += "<td> </td>\n"
|
||||
|
||||
s += "<td>\n"
|
||||
s += '<form method="post" action=".">\n'
|
||||
s += "<textarea name='bulk_remove_domains' rows='10' cols='25'></textarea>\n"
|
||||
s += '<br>\n'
|
||||
s += '<input type="submit" value="Bulk remove domains"><br />\n'
|
||||
s += "<span style='font-size: small'>(enter one domain per line)</span>\n"
|
||||
s += '</form>\n'
|
||||
s += '</td>\n'
|
||||
|
||||
s += "<td> </td>\n"
|
||||
|
||||
s += "<td>\n"
|
||||
s += '<form method="post" action=".">\n'
|
||||
s += "<textarea name='bulk_schedule_domains' rows='10' cols='25'></textarea>\n"
|
||||
s += '<br>\n'
|
||||
s += '<input type="submit" value="Bulk schedule domains"><br/>\n'
|
||||
s += "<span style='font-size: small'>(enter one domain per line)</span>\n"
|
||||
s += '</form>\n'
|
||||
s += "</td>\n"
|
||||
|
||||
s += "</tr>\n"
|
||||
|
||||
s += "</table>\n"
|
||||
|
||||
if wc_request.args:
|
||||
|
|
@ -139,36 +112,26 @@ class Spiderctl(object):
|
|||
|
||||
def webconsole_control(self, wc_request):
|
||||
args = wc_request.args
|
||||
enabled_domains = spiders.list()
|
||||
s = "<hr />\n"
|
||||
|
||||
if "stop_running_domains" in args:
|
||||
s += "<p>"
|
||||
s += "Stopped spiders: <ul><li>%s</li></ul>" % "</li><li>".join(args["stop_running_domains"])
|
||||
stopped_domains = []
|
||||
for domain in args["stop_running_domains"]:
|
||||
scrapyengine.close_domain(domain)
|
||||
if domain in self.running:
|
||||
scrapyengine.close_spider(self.running[domain])
|
||||
stopped_domains.append(domain)
|
||||
s += "Stopped spiders: <ul><li>%s</li></ul>" % "</li><li>".join(stopped_domains)
|
||||
s += "</p>"
|
||||
if "remove_pending_domains" in args:
|
||||
removed = []
|
||||
for domain in args["remove_pending_domains"]:
|
||||
if scrapyengine.domain_scheduler.remove_pending_domain(domain):
|
||||
if scrapyengine.spider_scheduler.remove_pending_domain(domain):
|
||||
removed.append(domain)
|
||||
if removed:
|
||||
s += "<p>"
|
||||
s += "Removed scheduled spiders: <ul><li>%s</li></ul>" % "</li><li>".join(args["remove_pending_domains"])
|
||||
s += "</p>"
|
||||
if "bulk_remove_domains" in args:
|
||||
scheduled = []
|
||||
to_remove = set([d.strip() for d in args["bulk_remove_domains"][0].split("\n")])
|
||||
removed = set([d for d in scrapyengine.scheduler.pending_domains if d in to_remove])
|
||||
scrapyengine.scheduler.pending_domains = [d for d in scrapyengine.scheduler.pending_domains if d not in removed]
|
||||
if removed:
|
||||
s += "<p>"
|
||||
s += "Removed: <ul><li>%s</li></ul>" % "</li><li>".join(removed)
|
||||
s += "</p>"
|
||||
s += "<p>"
|
||||
s += "Not removed: <ul><li>%s</li></ul>" % "</li><li>".join(to_remove - removed)
|
||||
s += "</p>"
|
||||
if "add_pending_domains" in args:
|
||||
for domain in args["add_pending_domains"]:
|
||||
if domain not in scrapyengine.scheduler.pending_requests:
|
||||
|
|
@ -176,20 +139,6 @@ class Spiderctl(object):
|
|||
s += "<p>"
|
||||
s += "Scheduled spiders: <ul><li>%s</li></ul>" % "</li><li>".join(args["add_pending_domains"])
|
||||
s += "</p>"
|
||||
if "bulk_schedule_domains" in args:
|
||||
to_schedule = set([d.strip() for d in args["bulk_schedule_domains"][0].split("\n")])
|
||||
scheduled = set()
|
||||
for domain in to_schedule:
|
||||
if domain in enabled_domains and domain not in scrapyengine.scheduler.pending_requests:
|
||||
scrapymanager.crawl(domain)
|
||||
scheduled.add(domain)
|
||||
if scheduled:
|
||||
s += "<p>"
|
||||
s += "Scheduled: <ul><li>%s</li></ul>" % "</li><li>".join(scheduled)
|
||||
s += "</p>"
|
||||
s += "<p>"
|
||||
s += "Not scheduled: <ul><li>%s</li></ul>" % "</li><li>".join(to_schedule - scheduled)
|
||||
s += "</p>"
|
||||
if "rerun_finished_domains" in args:
|
||||
for domain in args["rerun_finished_domains"]:
|
||||
if domain not in scrapyengine.scheduler.pending_requests:
|
||||
|
|
@ -202,4 +151,5 @@ class Spiderctl(object):
|
|||
return s
|
||||
|
||||
def webconsole_discover_module(self):
|
||||
self.enabled_domains = spiders.list()
|
||||
return self
|
||||
|
|
|
|||
Loading…
Reference in New Issue