From 2fd5cf644e61c55fd62edd7dc17ffcef84af4fb7 Mon Sep 17 00:00:00 2001 From: olveyra Date: Fri, 12 Sep 2008 20:48:48 +0000 Subject: [PATCH] - Support for not output in cluster status - schedule option by default does not have output --HG-- extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%40228 --- .../contrib/pbcluster/master/manager.py | 17 +++++++------ .../scrapy/contrib/pbcluster/master/web.py | 24 ++++++++++--------- .../contrib/pbcluster/master/ws_api.txt | 2 +- .../pbcluster/tools/scrapy-cluster-ctl.py | 11 +++++---- 4 files changed, 31 insertions(+), 23 deletions(-) diff --git a/scrapy/trunk/scrapy/contrib/pbcluster/master/manager.py b/scrapy/trunk/scrapy/contrib/pbcluster/master/manager.py index c4cbb0466..d05dc6c42 100644 --- a/scrapy/trunk/scrapy/contrib/pbcluster/master/manager.py +++ b/scrapy/trunk/scrapy/contrib/pbcluster/master/manager.py @@ -35,17 +35,19 @@ class Broker(pb.Referenceable): else: deferred.addCallbacks(callback=self._set_status, errback=lambda reason: log.msg(reason, log.ERROR)) - def status_as_dict(self, verbosity=0): + def status_as_dict(self, verbosity=1): + if verbosity == 0: + return status = {"alive": self.alive} if self.alive: - if verbosity == 0: + if verbosity == 1: #dont show spider settings status["running"] = [] for proc in self.running: proccopy = proc.copy() del proccopy["settings"] status["running"].append(proccopy) - else: + elif verbosity == 2: status["running"] = self.running status["maxproc"] = self.maxproc status["freeslots"] = self.maxproc - len(self.running) @@ -307,17 +309,18 @@ class ClusterMaster: if domain == p['domain']: return p - def print_pending(self, verbosity=0): - if verbosity == 0: + def print_pending(self, verbosity=1): + if verbosity == 1: pending = [] for p in self.pending: pp = p.copy() del pp["settings"] pending.append(pp) return pending - else: + elif verbosity == 2: return self.pending - + return + def _engine_started(self): self.load_nodes() scrapyengine.addtask(self.update_nodes, settings.getint('CLUSTER_MASTER_POLL_INTERVAL')) diff --git a/scrapy/trunk/scrapy/contrib/pbcluster/master/web.py b/scrapy/trunk/scrapy/contrib/pbcluster/master/web.py index 55b30167b..9fe14632e 100644 --- a/scrapy/trunk/scrapy/contrib/pbcluster/master/web.py +++ b/scrapy/trunk/scrapy/contrib/pbcluster/master/web.py @@ -76,7 +76,7 @@ class ClusterMasterWeb(ClusterMaster): self.schedule(domains, spider_settings, priority) if ws: - return self.ws_status(wc_request) + return self.ws_status(wc_request, verbosity=0) if "stop" in args: if ws: @@ -217,20 +217,22 @@ class ClusterMasterWeb(ClusterMaster): def webconsole_discover_module(self): return self - def ws_status(self, wc_request): + def ws_status(self, wc_request, verbosity=1): format = wc_request.args['format'][0] if 'format' in wc_request.args else 'json' - verbosity = wc_request.args['verbosity'][0] if 'verbosity' in wc_request.args else '0' + verbosity = int(wc_request.args['verbosity'][0]) if 'verbosity' in wc_request.args else verbosity wc_request.setHeader('content-type', 'text/plain') status = {} nodes_status = {} - for d, n in self.nodes.iteritems(): - nodes_status[d] = n.status_as_dict(int(verbosity)) - status["nodes"] = nodes_status - status["pending"] = self.print_pending(int(verbosity)) - status["loading"] = self.loading - content = serialize(status, format) - return content - + if verbosity > 0: + for d, n in self.nodes.iteritems(): + nodes_status[d] = n.status_as_dict(verbosity) + status["nodes"] = nodes_status + status["pending"] = self.print_pending(verbosity) + status["loading"] = self.loading + content = serialize(status, format) + return content + return "" + def ws_statistics(self, wc_request): format = wc_request.args['format'][0] if 'format' in wc_request.args else 'json' content = serialize(self.statistics, format) diff --git a/scrapy/trunk/scrapy/contrib/pbcluster/master/ws_api.txt b/scrapy/trunk/scrapy/contrib/pbcluster/master/ws_api.txt index 0f84c9620..c47626a08 100644 --- a/scrapy/trunk/scrapy/contrib/pbcluster/master/ws_api.txt +++ b/scrapy/trunk/scrapy/contrib/pbcluster/master/ws_api.txt @@ -27,7 +27,7 @@ Query parameters - `enable_node`: revert the state setted by 'disable_node' - - `verbosity`: sets the verbosity level (0 is the default minimal, 1 includes domain settings) + - `verbosity`: sets the output verbosity level (1 is the default minimal, 2 includes domain settings, 0 disables output) - `statistics`: shows the pending/running/scraped/lost statistics diff --git a/scrapy/trunk/scrapy/contrib/pbcluster/tools/scrapy-cluster-ctl.py b/scrapy/trunk/scrapy/contrib/pbcluster/tools/scrapy-cluster-ctl.py index 2125f7143..6c165dd16 100755 --- a/scrapy/trunk/scrapy/contrib/pbcluster/tools/scrapy-cluster-ctl.py +++ b/scrapy/trunk/scrapy/contrib/pbcluster/tools/scrapy-cluster-ctl.py @@ -21,13 +21,15 @@ def main(): parser.add_option("--status", dest="status", action="store_true", help="Print cluster master status and quit.") parser.add_option("--statistics", dest="statistics", action="store_true", help="Print cluster statistics") parser.add_option("--stop", dest="stop", action="store_true", help="Stops a running domain.") - parser.add_option("--verbosity", dest="verbosity", type="int", help="Sets the report status verbosity.", default=0) + parser.add_option("--verbosity", dest="verbosity", type="int", help="Sets the report status verbosity.") (opts, args) = parser.parse_args() output = "" domains = [] urlstring = "http://%s:%s/cluster_master/ws/" % (opts.server, opts.port) - post = {"format":opts.format, "verbosity":opts.verbosity} + post = {"format":opts.format} + if isinstance(opts.verbosity, int): + post["verbosity"] = opts.verbosity if args: domains = ",".join(args) @@ -60,10 +62,11 @@ def main(): else: parser.print_help() return - + f = urllib.urlopen(urlstring, urllib.urlencode(post)) output=f.read() - + if not output: + return if not opts.output: print output else: