From 667864ae94168a616beabc3930938b28a593633d Mon Sep 17 00:00:00 2001 From: "Victoria Terenina (torymur)" Date: Thu, 30 Apr 2015 16:28:57 +0300 Subject: [PATCH] testing robotstxt error --- tests/test_downloadermiddleware_robotstxt.py | 23 +++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/tests/test_downloadermiddleware_robotstxt.py b/tests/test_downloadermiddleware_robotstxt.py index bc245dcec..372a0a2c3 100644 --- a/tests/test_downloadermiddleware_robotstxt.py +++ b/tests/test_downloadermiddleware_robotstxt.py @@ -1,7 +1,8 @@ from __future__ import absolute_import import re -from twisted.internet import reactor +from twisted.internet import reactor, error from twisted.internet.defer import Deferred +from twisted.python import failure from twisted.trial import unittest from scrapy.downloadermiddlewares.robotstxt import RobotsTxtMiddleware from scrapy.exceptions import IgnoreRequest, NotConfigured @@ -41,6 +42,26 @@ class RobotsTxtMiddlewareTest(unittest.TestCase): reactor.callFromThread(deferred.callback, None) return deferred + def test_robotstxt_error(self): + crawler = mock.MagicMock() + crawler.settings = Settings() + crawler.settings.set('ROBOTSTXT_OBEY', True) + crawler.engine.download = mock.MagicMock() + err = error.DNSLookupError('Robotstxt address not found') + def return_failure(request, spider): + deferred = Deferred() + reactor.callFromThread(deferred.errback, failure.Failure(err)) + return deferred + crawler.engine.download.side_effect = return_failure + + middleware = RobotsTxtMiddleware(crawler) + middleware._logerror = mock.MagicMock() + middleware.process_request(Request('http://site.local'), None) + deferred = Deferred() + deferred.addErrback(lambda _: self.assertIsNone(middleware._logerror.assert_any_call())) + reactor.callFromThread(deferred.callback, None) + return deferred + def assertNotIgnored(self, request, middleware): spider = None # not actually used self.assertIsNone(middleware.process_request(request, spider))