diff --git a/scrapy/trunk/scrapy/contrib/pbcluster/master/manager.py b/scrapy/trunk/scrapy/contrib/pbcluster/master/manager.py index b65b56bd6..126e12c56 100644 --- a/scrapy/trunk/scrapy/contrib/pbcluster/master/manager.py +++ b/scrapy/trunk/scrapy/contrib/pbcluster/master/manager.py @@ -35,11 +35,18 @@ class Node: self.master = master self.available = True - @property - def status_as_dict(self): + def status_as_dict(self, verbosity=0): status = {"alive": self.alive} if self.alive: - status["running"] = self.running + if verbosity == 0: + #dont show spider settings + status["running"] = [] + for proc in self.running: + proccopy = proc.copy() + del proccopy["settings"] + status["running"].append(proccopy) + else: + status["running"] = self.running status["maxproc"] = self.maxproc status["available"] = self.available status["starttime"] = self.starttime @@ -271,6 +278,18 @@ class ClusterMaster(pb.Root): if domain == p['domain']: return p + def print_pending(self, verbosity=0): + if verbosity == 0: + pending = [] + for p in self.pending: + pp = p.copy() + del pp["settings"] + pending.append(pp) + return pending + else: + return self.pending + + def _engine_started(self): self.load_nodes() scrapyengine.addtask(self.update_nodes, settings.getint('CLUSTER_MASTER_POLL_INTERVAL')) diff --git a/scrapy/trunk/scrapy/contrib/pbcluster/master/web.py b/scrapy/trunk/scrapy/contrib/pbcluster/master/web.py index 92ac9f83c..bbe8a1d38 100644 --- a/scrapy/trunk/scrapy/contrib/pbcluster/master/web.py +++ b/scrapy/trunk/scrapy/contrib/pbcluster/master/web.py @@ -221,13 +221,14 @@ class ClusterMasterWeb(ClusterMaster): def ws_status(self, wc_request): format = wc_request.args['format'][0] if 'format' in wc_request.args else 'json' + verbosity = wc_request.args['verbosity'][0] if 'verbosity' in wc_request.args else '0' wc_request.setHeader('content-type', 'text/plain') status = {} nodes_status = {} for d, n in self.nodes.iteritems(): - nodes_status[d] = n.status_as_dict + nodes_status[d] = n.status_as_dict(int(verbosity)) status["nodes"] = nodes_status - status["pending"] = self.pending + status["pending"] = self.print_pending(int(verbosity)) status["loading"] = self.loading content = serialize(status, format) return content \ No newline at end of file diff --git a/scrapy/trunk/scrapy/contrib/pbcluster/master/ws_api.txt b/scrapy/trunk/scrapy/contrib/pbcluster/master/ws_api.txt index f85c549e4..7b546a0d5 100644 --- a/scrapy/trunk/scrapy/contrib/pbcluster/master/ws_api.txt +++ b/scrapy/trunk/scrapy/contrib/pbcluster/master/ws_api.txt @@ -26,6 +26,8 @@ Query parameters: 'enable_node': revert the state setted by 'disable_node' +'verbosity': sets the verbosity level (0 is the default minimal, 1 includes domain settings) + Examples: 1) Schedule argos.co.uk, diy.com, littlewoodsdirect.com spiders, with priority=PRIORITY_NOW, and settings UNAVAILABLES_NOTIFY=2 and UNAVAILABLES_DAYS_BACK=3. Answer with pprint format diff --git a/scrapy/trunk/scrapy/contrib/pbcluster/worker/manager.py b/scrapy/trunk/scrapy/contrib/pbcluster/worker/manager.py index 732805a1b..ee3a536ae 100644 --- a/scrapy/trunk/scrapy/contrib/pbcluster/worker/manager.py +++ b/scrapy/trunk/scrapy/contrib/pbcluster/worker/manager.py @@ -83,7 +83,7 @@ class ClusterWorker(pb.Root): """Spawn process to run the given domain.""" if len(self.running) < self.maxproc: if not domain in self.running: - logfile = os.path.join(self.logdir, domain, time.strftime("%FT%T.log")) + logfile = os.path.join(self.logdir, time.strftime("%F"), domain, time.strftime("%FT%T.log")) if not os.path.exists(os.path.dirname(logfile)): os.makedirs(os.path.dirname(logfile)) scrapy_proc = ScrapyProcessProtocol(self, domain, logfile, spider_settings)