From de0bf22010f6236614a9ab03b00b124ba6b14bd2 Mon Sep 17 00:00:00 2001
From: Pablo Hoffman <pablo@pablohoffman.com>
Date: Wed, 20 Jul 2011 01:18:00 -0300
Subject: [PATCH] speed up consumption of spider start requests, while the
 engine has capacity to process them

---
 scrapy/core/engine.py | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py
index bd412bc89..e81816a05 100644
--- a/scrapy/core/engine.py
+++ b/scrapy/core/engine.py
@@ -108,14 +108,16 @@ class ExecutionEngine(object):
             if not self._next_request(spider):
                 break
 
-        if self.spider_is_idle(spider):
-            slot = self.slots[spider]
+        slot = self.slots[spider]
+        if slot.requests and not self._needs_backout(spider):
             try:
                 request = slot.requests.next()
                 self.crawl(request, spider)
             except StopIteration:
-                if slot.close_if_idle:
-                    self._spider_idle(spider)
+                slot.requests = None
+
+        if self.spider_is_idle(spider) and slot.close_if_idle:
+            self._spider_idle(spider)
 
     def _needs_backout(self, spider):
         slot = self.slots[spider]