- added verbosity levels

- now log paths includes a folder named by the date, so it is easier
to mantain logs in server

--HG--
extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%4072

scrapy/trunk/scrapy/contrib/pbcluster/master/manager.py

@@ -35,11 +35,18 @@ class Node:
         self.master = master
         self.available = True
 
-    @property
-    def status_as_dict(self):
+    def status_as_dict(self, verbosity=0):
         status = {"alive": self.alive}
         if self.alive:
-            status["running"] = self.running
+            if verbosity == 0:
+                #dont show spider settings
+                status["running"] = []
+                for proc in self.running:
+                    proccopy = proc.copy()
+                    del proccopy["settings"]
+                    status["running"].append(proccopy)
+            else:
+                status["running"] = self.running
             status["maxproc"] = self.maxproc
             status["available"] = self.available
             status["starttime"] = self.starttime
@@ -271,6 +278,18 @@ class ClusterMaster(pb.Root):
             if domain == p['domain']:
                 return p
 
+    def print_pending(self, verbosity=0):
+        if verbosity == 0:
+            pending = []
+            for p in self.pending:
+                pp = p.copy()
+                del pp["settings"]
+                pending.append(pp)
+            return pending
+        else:
+            return self.pending
+            
+        
     def _engine_started(self):
         self.load_nodes()
         scrapyengine.addtask(self.update_nodes, settings.getint('CLUSTER_MASTER_POLL_INTERVAL'))

scrapy/trunk/scrapy/contrib/pbcluster/master/web.py

@@ -221,13 +221,14 @@ class ClusterMasterWeb(ClusterMaster):
 
     def ws_status(self, wc_request):
         format = wc_request.args['format'][0] if 'format' in wc_request.args else 'json'
+        verbosity = wc_request.args['verbosity'][0] if 'verbosity' in wc_request.args else '0'
         wc_request.setHeader('content-type', 'text/plain')
         status = {}
         nodes_status = {}
         for d, n in self.nodes.iteritems():
-            nodes_status[d] = n.status_as_dict
+            nodes_status[d] = n.status_as_dict(int(verbosity))
         status["nodes"] = nodes_status
-        status["pending"] = self.pending
+        status["pending"] = self.print_pending(int(verbosity))
         status["loading"] = self.loading
         content = serialize(status, format)
         return content

scrapy/trunk/scrapy/contrib/pbcluster/master/ws_api.txt

@@ -26,6 +26,8 @@ Query parameters:
 
 'enable_node': revert the state setted by 'disable_node'
 
+'verbosity': sets the verbosity level (0 is the default minimal, 1 includes domain settings)
+
 Examples:
 
 1) Schedule argos.co.uk, diy.com, littlewoodsdirect.com spiders, with priority=PRIORITY_NOW, and settings UNAVAILABLES_NOTIFY=2 and UNAVAILABLES_DAYS_BACK=3. Answer with pprint format

scrapy/trunk/scrapy/contrib/pbcluster/worker/manager.py

@@ -83,7 +83,7 @@ class ClusterWorker(pb.Root):
         """Spawn process to run the given domain."""
         if len(self.running) < self.maxproc:
             if not domain in self.running:
-                logfile = os.path.join(self.logdir, domain, time.strftime("%FT%T.log"))
+                logfile = os.path.join(self.logdir, time.strftime("%F"), domain, time.strftime("%FT%T.log"))
                 if not os.path.exists(os.path.dirname(logfile)):
                     os.makedirs(os.path.dirname(logfile))
                 scrapy_proc = ScrapyProcessProtocol(self, domain, logfile, spider_settings)