RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

- load only one domain per node (and load the following when the run 

callback is executed). This way, we avoid to load lot of domains that
will bounce. Also, we mix up better the domains between available nodes.

--HG--
extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%4068
This commit is contained in:
olveyra 2008-07-16 11:20:53 +00:00
parent ace6e3c430
commit 338a485bc1
1 changed files with 7 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
scrapy/trunk/scrapy/contrib/pbcluster/master/manager.py
View File

@ -53,17 +53,14 @@ class Node:
free_slots = self.maxproc - len(self.running)
to_reschedule = []
while free_slots > 0 and self.master.pending:
if free_slots > 0 and self.master.pending:
pending = self.master.pending.pop(0)
#if domain already running in some node, reschedule with same priority (so will be moved to run later)
if pending['domain'] in self.master.running or pending['domain'] in self.master.loading:
to_reschedule.append(pending)
self.master.schedule([pending['domain']], pending['settings'], pending['priority'])
else:
self.run(pending)
self.master.loading.append(pending['domain'])
free_slots -= 1
for pending in to_reschedule:
self.master.schedule([pending['domain']], pending['settings'], pending['priority'])
def get_status(self):
try:
@ -88,16 +85,16 @@ class Node:
def _run_callback(status):
self.master.loading.remove(pending['domain'])
if status['callresponse'][0] == 1:
#slots are complete. Reschedule in master with priority reduced by one. This is a security issue because offen happens that the slots were completed and not yet notified because of the asynchronous response from worker.
#slots are complete. Reschedule in master with priority reduced by one.
#self.master.loading check should avoid this to happen
self.master.schedule([pending['domain']], pending['settings'], pending['priority'] - 1)
log.msg("Domain %s rescheduled: no proc space in node." % pending['domain'], log.WARNING)
elif status['callresponse'][0] == 2:
#domain already running in node. Reschedule with same priority. Dont know if this could really happen,
#because the first check in self.master.loading should avoid to reach this point.
#domain already running in node. Reschedule with same priority.
#self.master.loading check should avoid this to happen
self.master.schedule([pending['domain']], pending['settings'], pending['priority'])
log.msg("Domain %s rescheduled: already running in node." % pending['domain'], log.WARNING)
if not self.master.loading:
self._set_status(status)
self._set_status(status)
try:
deferred = self.__remote.callRemote("run", pending["domain"], pending["settings"])