removed request_received and added request_scheduled

docs/topics/signals.rst

@@ -181,20 +181,21 @@ spider_error
     :type spider: :class:`~scrapy.spider.BaseSpider` object
 
 
-request_received
+request_scheduled
 ----------------
 
-.. signal:: request_received
+.. signal:: request_scheduled
-.. function:: request_received(request, spider)
+.. function:: request_scheduled(request, spider)
 
-    Sent when the engine receives a :class:`~scrapy.http.Request` from a spider.
+    Sent when the engine schedule a :class:`~scrapy.http.Request`.
 
+    It will receive the requests generated from middlewares.
     This signal does not support returning deferreds from their handlers.
 
-    :param request: the request received
+    :param request: the request scheduled
     :type request: :class:`~scrapy.http.Request` object
 
-    :param spider: the spider which generated the request
+    :param spider: the spider which generated the original request
     :type spider: :class:`~scrapy.spider.BaseSpider` object
 
 response_received

scrapy/core/engine.py

@@ -174,6 +174,8 @@ class ExecutionEngine(object):
         self.slots[spider].nextcall.schedule()
 
     def schedule(self, request, spider):
+        self.signals.send_catch_log(signal=signals.request_scheduled,
+                request=request, spider=spider)
         return self.slots[spider].scheduler.enqueue_request(request)
 
     def download(self, request, spider):

scrapy/core/scraper.py

@@ -162,8 +162,6 @@ class Scraper(object):
         from the given spider
         """
         if isinstance(output, Request):
-            self.signals.send_catch_log(signal=signals.request_received, request=output, \
-                spider=spider)
             self.crawler.engine.crawl(request=output, spider=spider)
         elif isinstance(output, BaseItem):
             self.slot.itemproc_size += 1

scrapy/signals.py

@@ -11,14 +11,17 @@ spider_opened = object()
 spider_idle = object()
 spider_closed = object()
 spider_error = object()
-request_received = object()
+request_scheduled = object()
 response_received = object()
 response_downloaded = object()
 item_scraped = object()
 item_dropped = object()
 
+# for backwards compatibility
 stats_spider_opened = spider_opened
 stats_spider_closing = spider_closed
 stats_spider_closed = spider_closed
 
-item_passed = item_scraped # for backwards compatibility
+item_passed = item_scraped
+
+request_received = request_scheduled

scrapy/tests/test_engine.py

@@ -95,7 +95,7 @@ class CrawlerRun(object):
         self.crawler.install()
         self.crawler.configure()
         self.crawler.signals.connect(self.item_scraped, signals.item_scraped)
-        self.crawler.signals.connect(self.request_received, signals.request_received)
+        self.crawler.signals.connect(self.request_scheduled, signals.request_scheduled)
         self.crawler.signals.connect(self.response_downloaded, signals.response_downloaded)
         self.crawler.crawl(self.spider)
         self.crawler.start()
@@ -122,7 +122,7 @@ class CrawlerRun(object):
     def item_scraped(self, item, spider, response):
         self.itemresp.append((item, response))
 
-    def request_received(self, request, spider):
+    def request_scheduled(self, request, spider):
         self.reqplug.append((request, spider))
 
     def response_downloaded(self, response, spider):
@@ -143,21 +143,20 @@ class EngineTest(unittest.TestCase):
         self.run = CrawlerRun()
         yield self.run.run()
         self._assert_visited_urls()
-        self._assert_received_requests()
+        self._assert_scheduled_requests()
         self._assert_downloaded_responses()
         self._assert_scraped_items()
         self._assert_signals_catched()
 
     def _assert_visited_urls(self):
-        must_be_visited = ["/", "/redirect", "/redirected", 
+        must_be_visited = ["/", "/redirect", "/redirected",
                            "/item1.html", "/item2.html", "/item999.html"]
         urls_visited = set([rp[0].url for rp in self.run.respplug])
         urls_expected = set([self.run.geturl(p) for p in must_be_visited])
         assert urls_expected <= urls_visited, "URLs not visited: %s" % list(urls_expected - urls_visited)
 
-    def _assert_received_requests(self):
+    def _assert_scheduled_requests(self):
-        # 3 requests should be received from the spider. start_urls and redirects don't count
+        self.assertEqual(6, len(self.run.reqplug))
-        self.assertEqual(3, len(self.run.reqplug))
 
         paths_expected = ['/item999.html', '/item2.html', '/item1.html']