From dc8701ea429d4ded2f66d6b7c8fbce0bbcd0041a Mon Sep 17 00:00:00 2001
From: Artur Gaspar <artur.gaspar.00@gmail.com>
Date: Wed, 27 Jan 2016 12:56:42 -0200
Subject: [PATCH] Add test for already failed deferreds when downloading page
 in robots.txt middleware.

---
 tests/test_downloadermiddleware_robotstxt.py | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/tests/test_downloadermiddleware_robotstxt.py b/tests/test_downloadermiddleware_robotstxt.py
index 5f45dcb82..f2e94e171 100644
--- a/tests/test_downloadermiddleware_robotstxt.py
+++ b/tests/test_downloadermiddleware_robotstxt.py
@@ -123,6 +123,18 @@ class RobotsTxtMiddlewareTest(unittest.TestCase):
         deferred.addCallback(lambda _: self.assertTrue(middleware._logerror.called))
         return deferred
 
+    def test_robotstxt_immediate_error(self):
+        self.crawler.settings.set('ROBOTSTXT_OBEY', True)
+        err = error.DNSLookupError('Robotstxt address not found')
+        def immediate_failure(request, spider):
+            deferred = Deferred()
+            deferred.errback(failure.Failure(err))
+            return deferred
+        self.crawler.engine.download.side_effect = immediate_failure
+
+        middleware = RobotsTxtMiddleware(self.crawler)
+        return self.assertNotIgnored(Request('http://site.local'), middleware)
+
     def test_ignore_robotstxt_request(self):
         self.crawler.settings.set('ROBOTSTXT_OBEY', True)
         def ignore_request(request, spider):