From 338a485bc152c13057c63bcc91dc46cd803365e2 Mon Sep 17 00:00:00 2001 From: olveyra Date: Wed, 16 Jul 2008 11:20:53 +0000 Subject: [PATCH] - load only one domain per node (and load the following when the run callback is executed). This way, we avoid to load lot of domains that will bounce. Also, we mix up better the domains between available nodes. --HG-- extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%4068 --- .../scrapy/contrib/pbcluster/master/manager.py | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/scrapy/trunk/scrapy/contrib/pbcluster/master/manager.py b/scrapy/trunk/scrapy/contrib/pbcluster/master/manager.py index 6fcc6b8f3..cb65c5435 100644 --- a/scrapy/trunk/scrapy/contrib/pbcluster/master/manager.py +++ b/scrapy/trunk/scrapy/contrib/pbcluster/master/manager.py @@ -53,17 +53,14 @@ class Node: free_slots = self.maxproc - len(self.running) to_reschedule = [] - while free_slots > 0 and self.master.pending: + if free_slots > 0 and self.master.pending: pending = self.master.pending.pop(0) #if domain already running in some node, reschedule with same priority (so will be moved to run later) if pending['domain'] in self.master.running or pending['domain'] in self.master.loading: - to_reschedule.append(pending) + self.master.schedule([pending['domain']], pending['settings'], pending['priority']) else: self.run(pending) self.master.loading.append(pending['domain']) - free_slots -= 1 - for pending in to_reschedule: - self.master.schedule([pending['domain']], pending['settings'], pending['priority']) def get_status(self): try: @@ -88,16 +85,16 @@ class Node: def _run_callback(status): self.master.loading.remove(pending['domain']) if status['callresponse'][0] == 1: - #slots are complete. Reschedule in master with priority reduced by one. This is a security issue because offen happens that the slots were completed and not yet notified because of the asynchronous response from worker. + #slots are complete. Reschedule in master with priority reduced by one. + #self.master.loading check should avoid this to happen self.master.schedule([pending['domain']], pending['settings'], pending['priority'] - 1) log.msg("Domain %s rescheduled: no proc space in node." % pending['domain'], log.WARNING) elif status['callresponse'][0] == 2: - #domain already running in node. Reschedule with same priority. Dont know if this could really happen, - #because the first check in self.master.loading should avoid to reach this point. + #domain already running in node. Reschedule with same priority. + #self.master.loading check should avoid this to happen self.master.schedule([pending['domain']], pending['settings'], pending['priority']) log.msg("Domain %s rescheduled: already running in node." % pending['domain'], log.WARNING) - if not self.master.loading: - self._set_status(status) + self._set_status(status) try: deferred = self.__remote.callRemote("run", pending["domain"], pending["settings"])