mirror of https://github.com/scrapy/scrapy.git
don't update status in run callback, so to avoid lots of bouncing
domains. The domains will be loaded softly on each node update --HG-- extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%4069
This commit is contained in:
parent
338a485bc1
commit
e8eab24dc0
|
|
@ -52,7 +52,8 @@ class Node:
|
|||
self.logdir = status['logdir']
|
||||
free_slots = self.maxproc - len(self.running)
|
||||
|
||||
to_reschedule = []
|
||||
#load domains by one, so to mix up better the domain loading between nodes. The next one in the same node will be loaded
|
||||
#when there is no loading domain or in the next status update. Th
|
||||
if free_slots > 0 and self.master.pending:
|
||||
pending = self.master.pending.pop(0)
|
||||
#if domain already running in some node, reschedule with same priority (so will be moved to run later)
|
||||
|
|
@ -94,7 +95,6 @@ class Node:
|
|||
#self.master.loading check should avoid this to happen
|
||||
self.master.schedule([pending['domain']], pending['settings'], pending['priority'])
|
||||
log.msg("Domain %s rescheduled: already running in node." % pending['domain'], log.WARNING)
|
||||
self._set_status(status)
|
||||
|
||||
try:
|
||||
deferred = self.__remote.callRemote("run", pending["domain"], pending["settings"])
|
||||
|
|
|
|||
Loading…
Reference in New Issue