From fcbbb5001e093d4f2b5a522061f833be7a514370 Mon Sep 17 00:00:00 2001 From: Pablo Hoffman Date: Mon, 14 Sep 2009 20:35:47 -0300 Subject: [PATCH] ported spiderctl web console extensin to work with new core based on spider references --- scrapy/contrib/webconsole/spiderctl.py | 120 ++++++++----------------- 1 file changed, 35 insertions(+), 85 deletions(-) diff --git a/scrapy/contrib/webconsole/spiderctl.py b/scrapy/contrib/webconsole/spiderctl.py index 853479179..bae839e72 100644 --- a/scrapy/contrib/webconsole/spiderctl.py +++ b/scrapy/contrib/webconsole/spiderctl.py @@ -16,7 +16,7 @@ class Spiderctl(object): webconsole_name = 'Spider control panel' def __init__(self): - self.running = set() + self.running = {} self.finished = set() dispatcher.connect(self.domain_opened, signal=signals.domain_opened) dispatcher.connect(self.domain_closed, signal=signals.domain_closed) @@ -24,42 +24,41 @@ class Spiderctl(object): from scrapy.management.web import webconsole_discover_module dispatcher.connect(self.webconsole_discover_module, signal=webconsole_discover_module) - def domain_opened(self, domain, spider): - self.running.add(domain) + def domain_opened(self, spider): + self.running[spider.domain_name] = spider - def domain_closed(self, domain, spider): - self.running.remove(domain) - self.finished.add(domain) + def domain_closed(self, spider): + del self.running[spider.domain_name] + self.finished.add(spider.domain_name) def webconsole_render(self, wc_request): if wc_request.args: changes = self.webconsole_control(wc_request) - enabled_domains = spiders.list() - self.scheduled = scrapyengine.domain_scheduler.pending_domains - self.idle = [d for d in enabled_domains if d not in self.scheduled + self.scheduled = [s.domain_name for s in scrapyengine.spider_scheduler._pending_spiders] + self.idle = [d for d in self.enabled_domains if d not in self.scheduled and d not in self.running and d not in self.finished] s = banner(self) s += '\n' - s += "\n" % \ - (len(self.running), + s += "\n" % \ + (len(self.idle), + len(self.running), settings['CONCURRENT_DOMAINS'], len(self.scheduled), - len(self.finished), - len(self.idle)) + len(self.finished)) s += "\n" - # running + # idle s += "\n" @@ -76,6 +75,18 @@ class Spiderctl(object): s += "\n" + # running + s += "\n" + # finished s += "\n" - # idle - s += "\n" - s += "\n" - s += "\n" - - s += "\n" - - s += "\n' - - s += "\n" - - s += "\n" - - s += "\n" - s += "
Running (%d/%d)Scheduled (%d)Finished (%d)Idle (%d)
Idle (%d)Scheduled (%d)Running (%d/%d)Finished (%d)
\n" s += '
\n' - s += '\n' + for domain in sorted(self.idle): s += "\n" % domain s += '
\n' s += '
' - s += '\n' + s += '\n' s += '
\n' s += "		\n" + s += '
\n' + s += '
\n' + s += '
' + s += '\n' + s += '
\n' + s += "		\n" s += '
\n' @@ -88,45 +99,7 @@ class Spiderctl(object): s += '
\n' s += "		\n" - s += '
\n' - s += '
\n' - s += '
' - s += '\n' - s += '
\n' - s += "
 \n" - s += '
\n' - s += "\n" - s += '
\n' - s += '
\n' - s += "(enter one domain per line)\n" - s += '
\n' - s += '		 \n" - s += '
\n' - s += "\n" - s += '
\n' - s += '
\n' - s += "(enter one domain per line)\n" - s += '
\n' - s += "
\n" if wc_request.args: @@ -139,36 +112,26 @@ class Spiderctl(object): def webconsole_control(self, wc_request): args = wc_request.args - enabled_domains = spiders.list() s = "
\n" if "stop_running_domains" in args: s += "

" - s += "Stopped spiders:

" % "
  • ".join(args["stop_running_domains"]) + stopped_domains = [] for domain in args["stop_running_domains"]: - scrapyengine.close_domain(domain) + if domain in self.running: + scrapyengine.close_spider(self.running[domain]) + stopped_domains.append(domain) + s += "Stopped spiders: " % "
  • ".join(stopped_domains) s += "

    " if "remove_pending_domains" in args: removed = [] for domain in args["remove_pending_domains"]: - if scrapyengine.domain_scheduler.remove_pending_domain(domain): + if scrapyengine.spider_scheduler.remove_pending_domain(domain): removed.append(domain) if removed: s += "

    " s += "Removed scheduled spiders:

    " % "
  • ".join(args["remove_pending_domains"]) s += "

    " - if "bulk_remove_domains" in args: - scheduled = [] - to_remove = set([d.strip() for d in args["bulk_remove_domains"][0].split("\n")]) - removed = set([d for d in scrapyengine.scheduler.pending_domains if d in to_remove]) - scrapyengine.scheduler.pending_domains = [d for d in scrapyengine.scheduler.pending_domains if d not in removed] - if removed: - s += "

    " - s += "Removed:

    " % "
  • ".join(removed) - s += "

    " - s += "

    " - s += "Not removed:

    " % "
  • ".join(to_remove - removed) - s += "

    " if "add_pending_domains" in args: for domain in args["add_pending_domains"]: if domain not in scrapyengine.scheduler.pending_requests: @@ -176,20 +139,6 @@ class Spiderctl(object): s += "

    " s += "Scheduled spiders:

    " % "
  • ".join(args["add_pending_domains"]) s += "

    " - if "bulk_schedule_domains" in args: - to_schedule = set([d.strip() for d in args["bulk_schedule_domains"][0].split("\n")]) - scheduled = set() - for domain in to_schedule: - if domain in enabled_domains and domain not in scrapyengine.scheduler.pending_requests: - scrapymanager.crawl(domain) - scheduled.add(domain) - if scheduled: - s += "

    " - s += "Scheduled:

    " % "
  • ".join(scheduled) - s += "

    " - s += "

    " - s += "Not scheduled:

    " % "
  • ".join(to_schedule - scheduled) - s += "

    " if "rerun_finished_domains" in args: for domain in args["rerun_finished_domains"]: if domain not in scrapyengine.scheduler.pending_requests: @@ -202,4 +151,5 @@ class Spiderctl(object): return s def webconsole_discover_module(self): + self.enabled_domains = spiders.list() return self