From 3fe36e77d2216d6a55cc719ffa2ee676154cc45e Mon Sep 17 00:00:00 2001 From: olveyra Date: Sat, 6 Sep 2008 17:03:47 +0000 Subject: [PATCH] Removed PRIORITY constants, added DEFAULT_PRIORITY setting --HG-- extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%40211 --- .../scrapy/contrib/pbcluster/master/manager.py | 11 ++--------- scrapy/trunk/scrapy/contrib/pbcluster/master/web.py | 13 ++++--------- .../scrapy/contrib/pbcluster/master/ws_api.txt | 13 ++++--------- .../contrib/pbcluster/tools/scrapy-cluster-ctl.py | 2 +- 4 files changed, 11 insertions(+), 28 deletions(-) diff --git a/scrapy/trunk/scrapy/contrib/pbcluster/master/manager.py b/scrapy/trunk/scrapy/contrib/pbcluster/master/manager.py index d58465e49..0a5d9462d 100644 --- a/scrapy/trunk/scrapy/contrib/pbcluster/master/manager.py +++ b/scrapy/trunk/scrapy/contrib/pbcluster/master/manager.py @@ -11,14 +11,7 @@ from scrapy.core.engine import scrapyengine from scrapy.core.exceptions import NotConfigured from scrapy.conf import settings -priorities = { 20:'NORMAL', - 10:'QUICK', - 0:'NOW', -} - -# Set priorities module attributes -for val, attr in priorities.items(): - setattr(sys.modules[__name__], "PRIORITY_%s" % attr, val ) +DEFAULT_PRIORITY = settings.getint("DEFAULT_PRIORITY") or 20 def my_import(name): mod = __import__(name) @@ -244,7 +237,7 @@ class ClusterMaster: def remove_node(self, nodename): raise NotImplemented - def schedule(self, domains, spider_settings=None, priority=PRIORITY_NORMAL): + def schedule(self, domains, spider_settings=None, priority=DEFAULT_PRIORITY): i = 0 for p in self.pending: if p['priority'] <= priority: diff --git a/scrapy/trunk/scrapy/contrib/pbcluster/master/web.py b/scrapy/trunk/scrapy/contrib/pbcluster/master/web.py index abf17e4ea..55b30167b 100644 --- a/scrapy/trunk/scrapy/contrib/pbcluster/master/web.py +++ b/scrapy/trunk/scrapy/contrib/pbcluster/master/web.py @@ -4,7 +4,7 @@ from pydispatch import dispatcher from scrapy.spider import spiders from scrapy.management.web import banner, webconsole_discover_module -from scrapy.contrib.pbcluster.master.manager import * +from scrapy.contrib.pbcluster.master.manager import ClusterMaster, DEFAULT_PRIORITY from scrapy.utils.serialization import serialize class ClusterMasterWeb(ClusterMaster): @@ -61,7 +61,7 @@ class ClusterMasterWeb(ClusterMaster): else: sep = "\r" domains = args["schedule"] - priority = int(args.get("priority", ["PRIORITY_NORMAL"])[0]) + priority = int(args.get("priority", [DEFAULT_PRIORITY])[0]) #spider settings slist = args.get("settings", [""])[0].split(sep) @@ -162,14 +162,9 @@ class ClusterMasterWeb(ClusterMaster): s += "\n" % domain s += "\n" s += "
\n" + s += "Priority:
\n" - s += "\n" + s += "%s" % DEFAULT_PRIORITY s += "
\n" #spider settings diff --git a/scrapy/trunk/scrapy/contrib/pbcluster/master/ws_api.txt b/scrapy/trunk/scrapy/contrib/pbcluster/master/ws_api.txt index 52fc95ca4..0f84c9620 100644 --- a/scrapy/trunk/scrapy/contrib/pbcluster/master/ws_api.txt +++ b/scrapy/trunk/scrapy/contrib/pbcluster/master/ws_api.txt @@ -14,13 +14,8 @@ Query parameters - `schedule`: schedules a comma separated list of domains. Schedule function takes optional parameters: - "priority": sets the queue priority for the specified domains. You can use a positive integer or the following predefined values: - - PRIORITY_NORMAL (=20) - PRIORITY_QUICK (=10) - PRIORITY_NOW (=0) - - by default, PRIORITY_NORMAL is used. + "priority": sets the queue priority for the specified domains (an integer). The default is setted by "DEFAULT_PRIORITY" + setting (20 if not given). A lower priority number implies more priority. "settings": run settings for the specified domains. This is a comma separated list of = pairs. By default it is empty. @@ -39,9 +34,9 @@ Query parameters Examples: --------- -1) Schedule argos.co.uk, diy.com, littlewoodsdirect.com spiders, with priority=PRIORITY_NOW, and settings UNAVAILABLES_NOTIFY=2 and UNAVAILABLES_DAYS_BACK=3. Answer with pprint format +1) Schedule argos.co.uk, diy.com, littlewoodsdirect.com spiders, with priority=0, and settings UNAVAILABLES_NOTIFY=2 and UNAVAILABLES_DAYS_BACK=3. Answer with pprint format - http://localhost:8080/cluster_master/ws/?format=pprint&schedule=argos.co.uk,diy.com,littlewoodsdirect.com&priority=PRIORITY_NOW&settings=UNAVAILABLES_NOTIFY=2,UNAVAILABLES_DAYS_BACK=3 + http://localhost:8080/cluster_master/ws/?format=pprint&schedule=argos.co.uk,diy.com,littlewoodsdirect.com&priority=0&settings=UNAVAILABLES_NOTIFY=2,UNAVAILABLES_DAYS_BACK=3 2) Get status with pprint format: diff --git a/scrapy/trunk/scrapy/contrib/pbcluster/tools/scrapy-cluster-ctl.py b/scrapy/trunk/scrapy/contrib/pbcluster/tools/scrapy-cluster-ctl.py index ab25bbff8..2125f7143 100755 --- a/scrapy/trunk/scrapy/contrib/pbcluster/tools/scrapy-cluster-ctl.py +++ b/scrapy/trunk/scrapy/contrib/pbcluster/tools/scrapy-cluster-ctl.py @@ -47,7 +47,7 @@ def main(): elif opts.schedule and domains: post["schedule"] = domains if opts.now: - post["priority"] = "PRIORITY_NOW" + post["priority"] = "0" post["settings"] = "UNAVAILABLES_NOTIFY=2" elif opts.remove and domains: post["remove"] = domains