From b2beb3e85d2e82977d259eea71402809d00d197e Mon Sep 17 00:00:00 2001 From: Artur Gaspar Date: Wed, 27 Jan 2016 13:09:08 -0200 Subject: [PATCH] Fix handling of already failed deferreds when downloading page in robots.txt middleware. --- scrapy/downloadermiddlewares/robotstxt.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/scrapy/downloadermiddlewares/robotstxt.py b/scrapy/downloadermiddlewares/robotstxt.py index c061c2407..7f6f0d012 100644 --- a/scrapy/downloadermiddlewares/robotstxt.py +++ b/scrapy/downloadermiddlewares/robotstxt.py @@ -8,7 +8,9 @@ import logging from six.moves.urllib import robotparser +from twisted.internet import reactor from twisted.internet.defer import Deferred, maybeDeferred +from twisted.internet.task import deferLater from scrapy.exceptions import NotConfigured, IgnoreRequest from scrapy.http import Request from scrapy.utils.httpobj import urlparse_cached @@ -57,7 +59,13 @@ class RobotsTxtMiddleware(object): priority=self.DOWNLOAD_PRIORITY, meta={'dont_obey_robotstxt': True} ) - dfd = self.crawler.engine.download(robotsreq, spider) + # engine.download() can return an already-called deferred, e.g. if a + # middleware returns a response in process_request(). Using + # deferLater() ensures that the error callback isn't called + # immediately upon being added, so that it doesn't remove the key + # before we check for it. + dfd = deferLater(reactor, 0, self.crawler.engine.download, + robotsreq, spider) dfd.addCallback(self._parse_robots, netloc) dfd.addErrback(self._logerror, robotsreq, spider) dfd.addErrback(self._robots_error, netloc)