New signal

update signature
documentation for new signal
utilize new signal
correct signal handler signature
emit new signal
test another signal
new signal
rename test file
faster test
rename test case
tests for signal emitting in bad cases

docs/topics/signals.rst

@@ -295,6 +295,23 @@ request_reached_downloader
     :param spider: the spider that yielded the request
     :type spider: :class:`~scrapy.spiders.Spider` object
 
+request_left_downloader
+---------------------------
+
+.. signal:: request_left_downloader
+.. function:: request_left_downloader(request, spider)
+
+    Sent when a :class:`~scrapy.http.Request` left downloader even in case of
+    failure.
+
+    The signal does not support returning deferreds from their handlers.
+
+    :param request: the request that reached downloader
+    :type request: :class:`~scrapy.http.Request` object
+
+    :param spider: the spider that yielded the request
+    :type spider: :class:`~scrapy.spiders.Spider` object
+
 response_received
 -----------------
 

scrapy/core/downloader/__init__.py

@@ -188,6 +188,9 @@ class Downloader(object):
         def finish_transferring(_):
             slot.transferring.remove(request)
             self._process_queue(spider, slot)
+            self.signals.send_catch_log(signal=signals.request_left_downloader,
+                                        request=request,
+                                        spider=spider)
             return _
 
         return dfd.addBoth(finish_transferring)

scrapy/pqueues.py

@@ -7,7 +7,7 @@ from queuelib import PriorityQueue
 
 from scrapy.core.downloader import Downloader
 from scrapy.http import Request
-from scrapy.signals import request_reached_downloader, response_downloaded
+from scrapy.signals import request_reached_downloader, request_left_downloader
 from scrapy.utils.httpobj import urlparse_cached
 
 
@@ -163,7 +163,7 @@ class DownloaderAwarePriorityQueue(object):
 
         self._active_downloads = {slot: 0 for slot in self._slot_pqueues.pqueues}
         crawler.signals.connect(self.on_response_download,
-                                signal=response_downloaded)
+                                signal=request_left_downloader)
         crawler.signals.connect(self.on_request_reached_downloader,
                                 signal=request_reached_downloader)
 
@@ -199,7 +199,7 @@ class DownloaderAwarePriorityQueue(object):
         if slot not in self._active_downloads:
             self._active_downloads[slot] = 0
 
-    def on_response_download(self, response, request, spider):
+    def on_response_download(self, request, spider):
         if not self.check_mark(request):
             return
         self.unmark(request)

scrapy/signals.py

@@ -14,6 +14,7 @@ spider_error = object()
 request_scheduled = object()
 request_dropped = object()
 request_reached_downloader = object()
+request_left_downloader = object()
 response_received = object()
 response_downloaded = object()
 item_scraped = object()

tests/test_request_left.py

@@ -0,0 +1,59 @@
+from twisted.internet import defer
+from twisted.trial.unittest import TestCase
+from scrapy.signals import request_left_downloader
+from scrapy.spiders import Spider
+from scrapy.utils.test import get_crawler
+from tests.mockserver import MockServer
+
+class SignalCatcherSpider(Spider):
+    name = 'signal_catcher'
+
+    def __init__(self, crawler, url, *args, **kwargs):
+        super(SignalCatcherSpider, self).__init__(*args, **kwargs)
+        crawler.signals.connect(self.on_response_download,
+                                signal=request_left_downloader)
+        self.catched_times = 0
+        self.start_urls = [url]
+
+    @classmethod
+    def from_crawler(cls, crawler, *args, **kwargs):
+        spider = cls(crawler, *args, **kwargs)
+        return spider
+
+    def on_response_download(self, request, spider):
+        self.catched_times = self.catched_times + 1
+
+
+class TestCatching(TestCase):
+
+    def setUp(self):
+        self.mockserver = MockServer()
+        self.mockserver.__enter__()
+
+    def tearDown(self):
+        self.mockserver.__exit__(None, None, None)
+
+    @defer.inlineCallbacks
+    def test_success(self):
+        crawler = get_crawler(SignalCatcherSpider)
+        yield crawler.crawl(self.mockserver.url("/status?n=200"))
+        self.assertEqual(crawler.spider.catched_times, 1)
+
+    @defer.inlineCallbacks
+    def test_timeout(self):
+        crawler = get_crawler(SignalCatcherSpider,
+                              {'DOWNLOAD_TIMEOUT': 0.1})
+        yield crawler.crawl(self.mockserver.url("/delay?n=0.2"))
+        self.assertEqual(crawler.spider.catched_times, 1)
+
+    @defer.inlineCallbacks
+    def test_disconnect(self):
+        crawler = get_crawler(SignalCatcherSpider)
+        yield crawler.crawl(self.mockserver.url("/drop"))
+        self.assertEqual(crawler.spider.catched_times, 1)
+
+    @defer.inlineCallbacks
+    def test_noconnect(self):
+        crawler = get_crawler(SignalCatcherSpider)
+        yield crawler.crawl('http://thereisdefinetelynosuchdomain.com')
+        self.assertEqual(crawler.spider.catched_times, 1)

tests/test_scheduler.py

@@ -9,7 +9,7 @@ from scrapy.crawler import Crawler
 from scrapy.core.scheduler import Scheduler
 from scrapy.http import Request
 from scrapy.pqueues import _scheduler_slot_read, _scheduler_slot_write
-from scrapy.signals import request_reached_downloader, response_downloaded
+from scrapy.signals import request_reached_downloader, request_left_downloader
 from scrapy.spiders import Spider
 from scrapy.utils.test import get_crawler
 from tests.mockserver import MockServer
@@ -216,9 +216,8 @@ class TestSchedulerWithDownloaderAwareInMemory(BaseSchedulerInMemoryTester,
 
         for request in requests:
             self.mock_crawler.signals.send_catch_log(
-                    signal=response_downloaded,
+                    signal=request_left_downloader,
                     request=request,
-                    response=None,
                     spider=self.spider
                     )
 
@@ -265,9 +264,8 @@ class TestSchedulerWithDownloaderAwareOnDisk(BaseSchedulerOnDiskTester,
 
         for request in requests:
             self.mock_crawler.signals.send_catch_log(
-                    signal=response_downloaded,
+                    signal=request_left_downloader,
                     request=request,
-                    response=None,
                     spider=self.spider
                     )