From dc8701ea429d4ded2f66d6b7c8fbce0bbcd0041a Mon Sep 17 00:00:00 2001 From: Artur Gaspar Date: Wed, 27 Jan 2016 12:56:42 -0200 Subject: [PATCH 1/3] Add test for already failed deferreds when downloading page in robots.txt middleware. --- tests/test_downloadermiddleware_robotstxt.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tests/test_downloadermiddleware_robotstxt.py b/tests/test_downloadermiddleware_robotstxt.py index 5f45dcb82..f2e94e171 100644 --- a/tests/test_downloadermiddleware_robotstxt.py +++ b/tests/test_downloadermiddleware_robotstxt.py @@ -123,6 +123,18 @@ class RobotsTxtMiddlewareTest(unittest.TestCase): deferred.addCallback(lambda _: self.assertTrue(middleware._logerror.called)) return deferred + def test_robotstxt_immediate_error(self): + self.crawler.settings.set('ROBOTSTXT_OBEY', True) + err = error.DNSLookupError('Robotstxt address not found') + def immediate_failure(request, spider): + deferred = Deferred() + deferred.errback(failure.Failure(err)) + return deferred + self.crawler.engine.download.side_effect = immediate_failure + + middleware = RobotsTxtMiddleware(self.crawler) + return self.assertNotIgnored(Request('http://site.local'), middleware) + def test_ignore_robotstxt_request(self): self.crawler.settings.set('ROBOTSTXT_OBEY', True) def ignore_request(request, spider): From b2beb3e85d2e82977d259eea71402809d00d197e Mon Sep 17 00:00:00 2001 From: Artur Gaspar Date: Wed, 27 Jan 2016 13:09:08 -0200 Subject: [PATCH 2/3] Fix handling of already failed deferreds when downloading page in robots.txt middleware. --- scrapy/downloadermiddlewares/robotstxt.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/scrapy/downloadermiddlewares/robotstxt.py b/scrapy/downloadermiddlewares/robotstxt.py index c061c2407..7f6f0d012 100644 --- a/scrapy/downloadermiddlewares/robotstxt.py +++ b/scrapy/downloadermiddlewares/robotstxt.py @@ -8,7 +8,9 @@ import logging from six.moves.urllib import robotparser +from twisted.internet import reactor from twisted.internet.defer import Deferred, maybeDeferred +from twisted.internet.task import deferLater from scrapy.exceptions import NotConfigured, IgnoreRequest from scrapy.http import Request from scrapy.utils.httpobj import urlparse_cached @@ -57,7 +59,13 @@ class RobotsTxtMiddleware(object): priority=self.DOWNLOAD_PRIORITY, meta={'dont_obey_robotstxt': True} ) - dfd = self.crawler.engine.download(robotsreq, spider) + # engine.download() can return an already-called deferred, e.g. if a + # middleware returns a response in process_request(). Using + # deferLater() ensures that the error callback isn't called + # immediately upon being added, so that it doesn't remove the key + # before we check for it. + dfd = deferLater(reactor, 0, self.crawler.engine.download, + robotsreq, spider) dfd.addCallback(self._parse_robots, netloc) dfd.addErrback(self._logerror, robotsreq, spider) dfd.addErrback(self._robots_error, netloc) From 90e3ae1c580875e4e68c9d7238d0fb4642306bf9 Mon Sep 17 00:00:00 2001 From: Artur Gaspar Date: Wed, 27 Jan 2016 21:00:35 -0200 Subject: [PATCH 3/3] Do not forget failed requests in robots.txt middleware. --- scrapy/downloadermiddlewares/robotstxt.py | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/scrapy/downloadermiddlewares/robotstxt.py b/scrapy/downloadermiddlewares/robotstxt.py index 7f6f0d012..6fdba90cc 100644 --- a/scrapy/downloadermiddlewares/robotstxt.py +++ b/scrapy/downloadermiddlewares/robotstxt.py @@ -8,9 +8,7 @@ import logging from six.moves.urllib import robotparser -from twisted.internet import reactor from twisted.internet.defer import Deferred, maybeDeferred -from twisted.internet.task import deferLater from scrapy.exceptions import NotConfigured, IgnoreRequest from scrapy.http import Request from scrapy.utils.httpobj import urlparse_cached @@ -59,13 +57,7 @@ class RobotsTxtMiddleware(object): priority=self.DOWNLOAD_PRIORITY, meta={'dont_obey_robotstxt': True} ) - # engine.download() can return an already-called deferred, e.g. if a - # middleware returns a response in process_request(). Using - # deferLater() ensures that the error callback isn't called - # immediately upon being added, so that it doesn't remove the key - # before we check for it. - dfd = deferLater(reactor, 0, self.crawler.engine.download, - robotsreq, spider) + dfd = self.crawler.engine.download(robotsreq, spider) dfd.addCallback(self._parse_robots, netloc) dfd.addErrback(self._logerror, robotsreq, spider) dfd.addErrback(self._robots_error, netloc) @@ -109,4 +101,6 @@ class RobotsTxtMiddleware(object): rp_dfd.callback(rp) def _robots_error(self, failure, netloc): - self._parsers.pop(netloc).callback(None) + rp_dfd = self._parsers[netloc] + self._parsers[netloc] = None + rp_dfd.callback(None)