diff --git a/scrapy/trunk/scrapy/contrib/pbcluster/master/manager.py b/scrapy/trunk/scrapy/contrib/pbcluster/master/manager.py index 8fd805fd7..85299a61d 100644 --- a/scrapy/trunk/scrapy/contrib/pbcluster/master/manager.py +++ b/scrapy/trunk/scrapy/contrib/pbcluster/master/manager.py @@ -31,12 +31,12 @@ class Node: self.master = master def _set_status(self, status): + self.status_as_dict = status if not status: self.available = False else: self.available = True self.running = status['running'] - self.closing = status['closing'] self.maxproc = status['maxproc'] self.starttime = status['starttime'] self.timestamp = status['timestamp'] diff --git a/scrapy/trunk/scrapy/contrib/pbcluster/master/web.py b/scrapy/trunk/scrapy/contrib/pbcluster/master/web.py index 6a54172f3..1a2c9aae1 100644 --- a/scrapy/trunk/scrapy/contrib/pbcluster/master/web.py +++ b/scrapy/trunk/scrapy/contrib/pbcluster/master/web.py @@ -4,7 +4,8 @@ from pydispatch import dispatcher from scrapy.spider import spiders from scrapy.management.web import banner, webconsole_discover_module -from scrapy.contrib.pbcluster.master.manager import ClusterMaster, priorities +from scrapy.contrib.pbcluster.master.manager import * +from scrapy.utils.serialization import serialize class ClusterMasterWeb(ClusterMaster): webconsole_id = 'cluster_master' @@ -21,6 +22,8 @@ class ClusterMasterWeb(ClusterMaster): return self.render_nodes(wc_request) elif wc_request.path == '/cluster_master/domains/': return self.render_domains(wc_request) + elif wc_request.path == '/cluster_master/ws/': + return self.webconsole_control(wc_request, ws=True) elif wc_request.args: changes = self.webconsole_control(wc_request) @@ -44,22 +47,56 @@ class ClusterMasterWeb(ClusterMaster): return str(s) - def webconsole_control(self, wc_request): + def webconsole_control(self, wc_request, ws=False): args = wc_request.args - if "updatenodes" in args: self.update_nodes() + if ws: + return self.ws_status(wc_request) if "schedule" in args: - self.schedule(args["schedule"], priority=eval(args["priority"][0])) + if ws: + sep = "," + domains = args["schedule"][0].split(sep) + else: + sep = "\r" + domains = args["schedule"] + priority = eval(args.get("priority", ["PRIORITY_NORMAL"])[0]) + slist = args.get("settings", [""])[0].split(sep) + spider_settings = {} + for s in slist: + try: + k, v = s.strip().split("=") + except ValueError: + pass + else: + spider_settings[k] = v + self.schedule(domains, spider_settings, priority) + if ws: + return self.ws_status(wc_request) if "stop" in args: - self.stop(args["stop"]) + if ws: + domains = args["stop"][0].split(",") + else: + domains=args["stop"] + self.stop(domains) + if ws: + return self.ws_status(wc_request) if "remove" in args: - self.remove(args["remove"]) + if ws: + domains = args["remove"][0].split(",") + else: + domains=args["remove"] + self.remove(domains) + if ws: + return self.ws_status(wc_request) - return "" + if ws: + return self.ws_status(wc_request) + else: + return "" def render_nodes(self, wc_request): if wc_request.args: @@ -110,7 +147,7 @@ class ClusterMasterWeb(ClusterMaster): for domain in sorted(inactive_domains): s += "\n" % domain s += "\n" - s += "
\n" + s += "
\n" s += "Priority:
\n" s += "\n" + s += "
\n" + s += "Spider settings:
\n" + s += "\n" s += "

\n" s += "\n" @@ -161,3 +204,18 @@ class ClusterMasterWeb(ClusterMaster): def webconsole_discover_module(self): return self + + def ws_status(self, wc_request): + format = wc_request.args['format'][0] if 'format' in wc_request.args else 'json' + wc_request.setHeader('content-type', 'text/plain') + status = {} + nodes_status = {} + running = [] + for d, n in self.nodes.iteritems(): + nodes_status[d] = n.status_as_dict + running.extend(n.running) + status["nodes"] = nodes_status + status["pending"] = self.pending + status["running"] = running + content = serialize(status, format) + return content \ No newline at end of file diff --git a/scrapy/trunk/scrapy/contrib/pbcluster/master/ws_api.txt b/scrapy/trunk/scrapy/contrib/pbcluster/master/ws_api.txt new file mode 100644 index 000000000..cda68cb7e --- /dev/null +++ b/scrapy/trunk/scrapy/contrib/pbcluster/master/ws_api.txt @@ -0,0 +1,35 @@ +The webservice API is available at + +http://server:port/cluster_master/ws/ + +Query parameters: + +'format': the answer format. By default, format=json. Other formats: pprint, pickle. + +'schedule': schedules a comma separated list of domains. Schedule function takes optional parameters: + + 'priority': sets the queue priority for the specified domains. You can use a positive integer or the following predefined values: + + PRIORITY_NORMAL (=20) + PRIORITY_QUICK (=10) + PRIORITY_NOW (=0) + + by default, PRIORITY_NORMAL is used. + + 'settings': run settings for the specified domains. This is a comma separated list of = pairs. + +'remove': removes from pending list a comma separated list of domains. + +'stop': stops comma separated list of domains (they have to be running in some node) + +Examples: + +1) Schedule argos.co.uk, diy.com, littlewoodsdirect.com spiders, with priority=PRIORITY_NOW, and settings UNAVAILABLES_NOTIFY=2 and UNAVAILABLES_DAYS_BACK=3 + + http://localhost:8080/cluster_master/ws/?format=pprint&schedule=argos.co.uk,diy.com,littlewoodsdirect.com&priority=PRIORITY_NOW&settings=UNAVAILABLES_NOTIFY=2,UNAVAILABLES_DAYS_BACK=3 + +2) Get status: + + http://localhost:8080/cluster_master/ws/?format=pprint + +3) Remove from pending lists \ No newline at end of file diff --git a/scrapy/trunk/scrapy/contrib/pbcluster/worker/manager.py b/scrapy/trunk/scrapy/contrib/pbcluster/worker/manager.py index 7005aca60..1641234cf 100644 --- a/scrapy/trunk/scrapy/contrib/pbcluster/worker/manager.py +++ b/scrapy/trunk/scrapy/contrib/pbcluster/worker/manager.py @@ -88,7 +88,6 @@ class ClusterWorker(pb.Root): if not os.path.exists(os.path.dirname(logfile)): os.makedirs(os.path.dirname(logfile)) scrapy_proc = ScrapyProcessProtocol(self, domain, logfile, spider_settings) - args = [sys.executable, sys.argv[0], 'crawl', domain] proc = reactor.spawnProcess(scrapy_proc, sys.executable, args=args, env=scrapy_proc.env) self.running[domain] = scrapy_proc