From f515942f546f73befcac86f794d9a7746ae3d2d8 Mon Sep 17 00:00:00 2001
From: olveyra <none@none>
Date: Thu, 17 Jul 2008 15:41:15 +0000
Subject: [PATCH] - added verbosity levels - now log paths includes a folder
 named by the date, so it is easier to mantain logs in server

--HG--
extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%4072
---
 .../contrib/pbcluster/master/manager.py       | 25 ++++++++++++++++---
 .../scrapy/contrib/pbcluster/master/web.py    |  5 ++--
 .../contrib/pbcluster/master/ws_api.txt       |  2 ++
 .../contrib/pbcluster/worker/manager.py       |  2 +-
 4 files changed, 28 insertions(+), 6 deletions(-)

diff --git a/scrapy/trunk/scrapy/contrib/pbcluster/master/manager.py b/scrapy/trunk/scrapy/contrib/pbcluster/master/manager.py
index b65b56bd6..126e12c56 100644
--- a/scrapy/trunk/scrapy/contrib/pbcluster/master/manager.py
+++ b/scrapy/trunk/scrapy/contrib/pbcluster/master/manager.py
@@ -35,11 +35,18 @@ class Node:
         self.master = master
         self.available = True
 
-    @property
-    def status_as_dict(self):
+    def status_as_dict(self, verbosity=0):
         status = {"alive": self.alive}
         if self.alive:
-            status["running"] = self.running
+            if verbosity == 0:
+                #dont show spider settings
+                status["running"] = []
+                for proc in self.running:
+                    proccopy = proc.copy()
+                    del proccopy["settings"]
+                    status["running"].append(proccopy)
+            else:
+                status["running"] = self.running
             status["maxproc"] = self.maxproc
             status["available"] = self.available
             status["starttime"] = self.starttime
@@ -271,6 +278,18 @@ class ClusterMaster(pb.Root):
             if domain == p['domain']:
                 return p
 
+    def print_pending(self, verbosity=0):
+        if verbosity == 0:
+            pending = []
+            for p in self.pending:
+                pp = p.copy()
+                del pp["settings"]
+                pending.append(pp)
+            return pending
+        else:
+            return self.pending
+            
+        
     def _engine_started(self):
         self.load_nodes()
         scrapyengine.addtask(self.update_nodes, settings.getint('CLUSTER_MASTER_POLL_INTERVAL'))
diff --git a/scrapy/trunk/scrapy/contrib/pbcluster/master/web.py b/scrapy/trunk/scrapy/contrib/pbcluster/master/web.py
index 92ac9f83c..bbe8a1d38 100644
--- a/scrapy/trunk/scrapy/contrib/pbcluster/master/web.py
+++ b/scrapy/trunk/scrapy/contrib/pbcluster/master/web.py
@@ -221,13 +221,14 @@ class ClusterMasterWeb(ClusterMaster):
 
     def ws_status(self, wc_request):
         format = wc_request.args['format'][0] if 'format' in wc_request.args else 'json'
+        verbosity = wc_request.args['verbosity'][0] if 'verbosity' in wc_request.args else '0'
         wc_request.setHeader('content-type', 'text/plain')
         status = {}
         nodes_status = {}
         for d, n in self.nodes.iteritems():
-            nodes_status[d] = n.status_as_dict
+            nodes_status[d] = n.status_as_dict(int(verbosity))
         status["nodes"] = nodes_status
-        status["pending"] = self.pending
+        status["pending"] = self.print_pending(int(verbosity))
         status["loading"] = self.loading
         content = serialize(status, format)
         return content
\ No newline at end of file
diff --git a/scrapy/trunk/scrapy/contrib/pbcluster/master/ws_api.txt b/scrapy/trunk/scrapy/contrib/pbcluster/master/ws_api.txt
index f85c549e4..7b546a0d5 100644
--- a/scrapy/trunk/scrapy/contrib/pbcluster/master/ws_api.txt
+++ b/scrapy/trunk/scrapy/contrib/pbcluster/master/ws_api.txt
@@ -26,6 +26,8 @@ Query parameters:
 
 'enable_node': revert the state setted by 'disable_node'
 
+'verbosity': sets the verbosity level (0 is the default minimal, 1 includes domain settings)
+
 Examples:
 
 1) Schedule argos.co.uk, diy.com, littlewoodsdirect.com spiders, with priority=PRIORITY_NOW, and settings UNAVAILABLES_NOTIFY=2 and UNAVAILABLES_DAYS_BACK=3. Answer with pprint format
diff --git a/scrapy/trunk/scrapy/contrib/pbcluster/worker/manager.py b/scrapy/trunk/scrapy/contrib/pbcluster/worker/manager.py
index 732805a1b..ee3a536ae 100644
--- a/scrapy/trunk/scrapy/contrib/pbcluster/worker/manager.py
+++ b/scrapy/trunk/scrapy/contrib/pbcluster/worker/manager.py
@@ -83,7 +83,7 @@ class ClusterWorker(pb.Root):
         """Spawn process to run the given domain."""
         if len(self.running) < self.maxproc:
             if not domain in self.running:
-                logfile = os.path.join(self.logdir, domain, time.strftime("%FT%T.log"))
+                logfile = os.path.join(self.logdir, time.strftime("%F"), domain, time.strftime("%FT%T.log"))
                 if not os.path.exists(os.path.dirname(logfile)):
                     os.makedirs(os.path.dirname(logfile))
                 scrapy_proc = ScrapyProcessProtocol(self, domain, logfile, spider_settings)