From e8eab24dc0dc44fb0f4af9d855895898a2423660 Mon Sep 17 00:00:00 2001 From: olveyra Date: Wed, 16 Jul 2008 12:09:42 +0000 Subject: [PATCH] don't update status in run callback, so to avoid lots of bouncing domains. The domains will be loaded softly on each node update --HG-- extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%4069 --- scrapy/trunk/scrapy/contrib/pbcluster/master/manager.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scrapy/trunk/scrapy/contrib/pbcluster/master/manager.py b/scrapy/trunk/scrapy/contrib/pbcluster/master/manager.py index cb65c5435..ae274291d 100644 --- a/scrapy/trunk/scrapy/contrib/pbcluster/master/manager.py +++ b/scrapy/trunk/scrapy/contrib/pbcluster/master/manager.py @@ -52,7 +52,8 @@ class Node: self.logdir = status['logdir'] free_slots = self.maxproc - len(self.running) - to_reschedule = [] + #load domains by one, so to mix up better the domain loading between nodes. The next one in the same node will be loaded + #when there is no loading domain or in the next status update. Th if free_slots > 0 and self.master.pending: pending = self.master.pending.pop(0) #if domain already running in some node, reschedule with same priority (so will be moved to run later) @@ -94,7 +95,6 @@ class Node: #self.master.loading check should avoid this to happen self.master.schedule([pending['domain']], pending['settings'], pending['priority']) log.msg("Domain %s rescheduled: already running in node." % pending['domain'], log.WARNING) - self._set_status(status) try: deferred = self.__remote.callRemote("run", pending["domain"], pending["settings"])