diff --git a/scrapy/trunk/scrapy/contrib/pbcluster/master/web.py b/scrapy/trunk/scrapy/contrib/pbcluster/master/web.py index be384b18b..b99469520 100644 --- a/scrapy/trunk/scrapy/contrib/pbcluster/master/web.py +++ b/scrapy/trunk/scrapy/contrib/pbcluster/master/web.py @@ -233,6 +233,8 @@ class ClusterMasterWeb(ClusterMaster): wc_request.setHeader('content-type', 'text/plain') status = {} nodes_status = {} + for d, n in self.nodes.iteritems(): + nodes_status[d] = n.status_as_dict status["nodes"] = nodes_status status["pending"] = self.pending content = serialize(status, format) diff --git a/scrapy/trunk/scrapy/contrib/pbcluster/master/ws_api.txt b/scrapy/trunk/scrapy/contrib/pbcluster/master/ws_api.txt index cda68cb7e..525525812 100644 --- a/scrapy/trunk/scrapy/contrib/pbcluster/master/ws_api.txt +++ b/scrapy/trunk/scrapy/contrib/pbcluster/master/ws_api.txt @@ -16,7 +16,9 @@ Query parameters: by default, PRIORITY_NORMAL is used. - 'settings': run settings for the specified domains. This is a comma separated list of = pairs. + 'settings': run settings for the specified domains. This is a comma separated list of = pairs. By default it is empty. + + 'env': additional environment settings passed to the crawl process. This is a comma separated list of = pairs. By default it is empty. 'remove': removes from pending list a comma separated list of domains. @@ -24,12 +26,18 @@ Query parameters: Examples: -1) Schedule argos.co.uk, diy.com, littlewoodsdirect.com spiders, with priority=PRIORITY_NOW, and settings UNAVAILABLES_NOTIFY=2 and UNAVAILABLES_DAYS_BACK=3 +1) Schedule argos.co.uk, diy.com, littlewoodsdirect.com spiders, with priority=PRIORITY_NOW, and settings UNAVAILABLES_NOTIFY=2 and UNAVAILABLES_DAYS_BACK=3. Answer with pprint format - http://localhost:8080/cluster_master/ws/?format=pprint&schedule=argos.co.uk,diy.com,littlewoodsdirect.com&priority=PRIORITY_NOW&settings=UNAVAILABLES_NOTIFY=2,UNAVAILABLES_DAYS_BACK=3 + http://localhost:8080/cluster_master/ws/?format=pprint&schedule=argos.co.uk,diy.com,littlewoodsdirect.com&priority=PRIORITY_NOW&settings=UNAVAILABLES_NOTIFY=2,UNAVAILABLES_DAYS_BACK=3&env=PYTHONPATH=/opt/scraping/conf/feed -2) Get status: +2) Get status with pprint format: http://localhost:8080/cluster_master/ws/?format=pprint -3) Remove from pending lists \ No newline at end of file +3) Remove from pending lists domains argos.co.uk and diy.com. Answer with pprint format: + + http://localhost:8080/cluster_master/ws/?remove=argos.co.uk,diy.com + +4) Stop running domain littlewoodsdirect.com: + + http://localhost:8080/cluster_master/ws/?stop=littlewoodsdirect.com \ No newline at end of file