delay next_request check after stoping a spider close to avoid 100% cpu usage loops in some cases

2009-11-06 22:02:12 -02:00 · 2009-11-06 22:02:12 -02:00 · 445d8cd9f0
parent aeab5370cb
commit 445d8cd9f0
1 changed files with 1 additions and 1 deletions
--- a/scrapy/core/engine.py
+++ b/scrapy/core/engine.py
@ -260,7 +260,7 @@ class ExecutionEngine(object):
            dispatcher.send(signal=signals.spider_idle, sender=self.__class__, \
                spider=spider)
        except DontCloseDomain:
-            self.next_request(spider)
+            reactor.callLater(5, self.next_request, spider)
            return
        except:
            log.err("Exception catched on spider_idle signal dispatch")