Ok, first functional version of pbcluster

--HG--
extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%4036

scrapy/trunk/scrapy/contrib/pbcluster/master/manager.py

@@ -31,12 +31,12 @@ class Node:
         self.master = master
 
     def _set_status(self, status):
+        self.status_as_dict = status
         if not status:
             self.available = False
         else:
             self.available = True
             self.running = status['running']
-            self.closing = status['closing']
             self.maxproc = status['maxproc']
             self.starttime = status['starttime']
             self.timestamp = status['timestamp']

scrapy/trunk/scrapy/contrib/pbcluster/master/web.py

@@ -4,7 +4,8 @@ from pydispatch import dispatcher
 
 from scrapy.spider import spiders 
 from scrapy.management.web import banner, webconsole_discover_module
-from scrapy.contrib.pbcluster.master.manager import ClusterMaster, priorities
+from scrapy.contrib.pbcluster.master.manager import *
+from scrapy.utils.serialization import serialize
 
 class ClusterMasterWeb(ClusterMaster):
     webconsole_id = 'cluster_master'
@@ -21,6 +22,8 @@ class ClusterMasterWeb(ClusterMaster):
             return self.render_nodes(wc_request)
         elif wc_request.path == '/cluster_master/domains/':
             return self.render_domains(wc_request)
+        elif wc_request.path == '/cluster_master/ws/':
+            return self.webconsole_control(wc_request, ws=True)
         elif wc_request.args:
             changes = self.webconsole_control(wc_request)
 
@@ -44,22 +47,56 @@ class ClusterMasterWeb(ClusterMaster):
 
         return str(s)
 
-    def webconsole_control(self, wc_request):
+    def webconsole_control(self, wc_request, ws=False):
         args = wc_request.args
-
         if "updatenodes" in args:
             self.update_nodes()
+            if ws:
+                return self.ws_status(wc_request)
 
         if "schedule" in args:
-            self.schedule(args["schedule"], priority=eval(args["priority"][0]))
+            if ws:
+                sep = ","
+                domains = args["schedule"][0].split(sep)
+            else:
+                sep = "\r"
+                domains = args["schedule"]
+            priority = eval(args.get("priority", ["PRIORITY_NORMAL"])[0])
+            slist = args.get("settings", [""])[0].split(sep)
+            spider_settings = {}
+            for s in slist:
+                try:
+                    k, v = s.strip().split("=")
+                except ValueError:
+                    pass
+                else:
+                    spider_settings[k] = v
+            self.schedule(domains, spider_settings, priority)
+            if ws:
+                return self.ws_status(wc_request)
 
         if "stop" in args:
-            self.stop(args["stop"])
+            if ws:
+                domains = args["stop"][0].split(",")
+            else:
+                domains=args["stop"]
+            self.stop(domains)
+            if ws:
+                return self.ws_status(wc_request)
 
         if "remove" in args:
-            self.remove(args["remove"])
+            if ws:
+                domains = args["remove"][0].split(",")
+            else:
+                domains=args["remove"]
+            self.remove(domains)
+            if ws:
+                return self.ws_status(wc_request)
 
-        return ""
+        if ws:
+            return self.ws_status(wc_request)
+        else:
+            return ""
 
     def render_nodes(self, wc_request):
         if wc_request.args:
@@ -110,7 +147,7 @@ class ClusterMasterWeb(ClusterMaster):
         for domain in sorted(inactive_domains):
             s += "<option>%s</option>\n" % domain
         s += "</select>\n"
-        s += "</br>\n"
+        s += "<br />\n"
         s += "Priority:<br />\n"
         s += "<select name='priority'>\n"
         for p, pname in priorities.items():
@@ -119,6 +156,12 @@ class ClusterMasterWeb(ClusterMaster):
             else:
                 s += "<option value='%s'>%s</option>" % (p, pname)
         s += "</select>\n"
+        s += "<br />\n"
+        s += "Spider settings:<br />\n"
+        s += "<textarea name='settings' rows='6'>\n"
+        s += "UNAVAILABLES_NOTIFY=2\n"
+        s += "UNAVAILABLES_DAYS_BACK=3\n"
+        s += "</textarea>\n"
         s += "<p><input type='submit' value='Schedule selected domains'></p>\n"
         s += "</form>\n"
 
@@ -161,3 +204,18 @@ class ClusterMasterWeb(ClusterMaster):
 
     def webconsole_discover_module(self):
         return self
+
+    def ws_status(self, wc_request):
+        format = wc_request.args['format'][0] if 'format' in wc_request.args else 'json'
+        wc_request.setHeader('content-type', 'text/plain')
+        status = {}
+        nodes_status = {}
+        running = []
+        for d, n in self.nodes.iteritems():
+            nodes_status[d] = n.status_as_dict
+            running.extend(n.running)
+        status["nodes"] = nodes_status
+        status["pending"] = self.pending
+        status["running"] = running
+        content = serialize(status, format)
+        return content

scrapy/trunk/scrapy/contrib/pbcluster/master/ws_api.txt

@@ -0,0 +1,35 @@
+The webservice API is available at
+
+http://server:port/cluster_master/ws/
+
+Query parameters:
+
+'format': the answer format. By default, format=json. Other formats: pprint, pickle.
+
+'schedule': schedules a comma separated list of domains. Schedule function takes optional parameters:
+
+    'priority': sets the queue priority for the specified domains. You can use a positive integer or the following predefined values:
+
+                    PRIORITY_NORMAL (=20)
+                    PRIORITY_QUICK  (=10)
+                    PRIORITY_NOW    (=0)
+
+                by default, PRIORITY_NORMAL is used.
+
+    'settings': run settings for the specified domains. This is a comma separated list of <setting_name>=<value> pairs.
+
+'remove': removes from pending list a comma separated list of domains.
+
+'stop': stops comma separated list of domains (they have to be running in some node)
+
+Examples:
+
+1) Schedule argos.co.uk, diy.com, littlewoodsdirect.com spiders, with priority=PRIORITY_NOW, and settings UNAVAILABLES_NOTIFY=2 and UNAVAILABLES_DAYS_BACK=3
+
+    http://localhost:8080/cluster_master/ws/?format=pprint&schedule=argos.co.uk,diy.com,littlewoodsdirect.com&priority=PRIORITY_NOW&settings=UNAVAILABLES_NOTIFY=2,UNAVAILABLES_DAYS_BACK=3
+
+2) Get status:
+
+    http://localhost:8080/cluster_master/ws/?format=pprint
+
+3) Remove from pending lists 

scrapy/trunk/scrapy/contrib/pbcluster/worker/manager.py

@@ -88,7 +88,6 @@ class ClusterWorker(pb.Root):
             if not os.path.exists(os.path.dirname(logfile)):
                 os.makedirs(os.path.dirname(logfile))
             scrapy_proc = ScrapyProcessProtocol(self, domain, logfile, spider_settings)
-    
             args = [sys.executable, sys.argv[0], 'crawl', domain]
             proc = reactor.spawnProcess(scrapy_proc, sys.executable, args=args, env=scrapy_proc.env)
             self.running[domain] = scrapy_proc