From 90e3ae1c580875e4e68c9d7238d0fb4642306bf9 Mon Sep 17 00:00:00 2001 From: Artur Gaspar Date: Wed, 27 Jan 2016 21:00:35 -0200 Subject: [PATCH] Do not forget failed requests in robots.txt middleware. --- scrapy/downloadermiddlewares/robotstxt.py | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/scrapy/downloadermiddlewares/robotstxt.py b/scrapy/downloadermiddlewares/robotstxt.py index 7f6f0d012..6fdba90cc 100644 --- a/scrapy/downloadermiddlewares/robotstxt.py +++ b/scrapy/downloadermiddlewares/robotstxt.py @@ -8,9 +8,7 @@ import logging from six.moves.urllib import robotparser -from twisted.internet import reactor from twisted.internet.defer import Deferred, maybeDeferred -from twisted.internet.task import deferLater from scrapy.exceptions import NotConfigured, IgnoreRequest from scrapy.http import Request from scrapy.utils.httpobj import urlparse_cached @@ -59,13 +57,7 @@ class RobotsTxtMiddleware(object): priority=self.DOWNLOAD_PRIORITY, meta={'dont_obey_robotstxt': True} ) - # engine.download() can return an already-called deferred, e.g. if a - # middleware returns a response in process_request(). Using - # deferLater() ensures that the error callback isn't called - # immediately upon being added, so that it doesn't remove the key - # before we check for it. - dfd = deferLater(reactor, 0, self.crawler.engine.download, - robotsreq, spider) + dfd = self.crawler.engine.download(robotsreq, spider) dfd.addCallback(self._parse_robots, netloc) dfd.addErrback(self._logerror, robotsreq, spider) dfd.addErrback(self._robots_error, netloc) @@ -109,4 +101,6 @@ class RobotsTxtMiddleware(object): rp_dfd.callback(rp) def _robots_error(self, failure, netloc): - self._parsers.pop(netloc).callback(None) + rp_dfd = self._parsers[netloc] + self._parsers[netloc] = None + rp_dfd.callback(None)