RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Fix handling of already failed deferreds when downloading page in robots.txt middleware.

This commit is contained in:
Artur Gaspar 2016-01-27 13:09:08 -02:00
parent dc8701ea42
commit b2beb3e85d
1 changed files with 9 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
scrapy/downloadermiddlewares/robotstxt.py
View File

@ -8,7 +8,9 @@ import logging
from six.moves.urllib import robotparser from six.moves.urllib import robotparser
from twisted.internet import reactor
from twisted.internet.defer import Deferred, maybeDeferred from twisted.internet.defer import Deferred, maybeDeferred
from twisted.internet.task import deferLater
from scrapy.exceptions import NotConfigured, IgnoreRequest from scrapy.exceptions import NotConfigured, IgnoreRequest
from scrapy.http import Request from scrapy.http import Request
from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.httpobj import urlparse_cached
@ -57,7 +59,13 @@ class RobotsTxtMiddleware(object):
priority=self.DOWNLOAD_PRIORITY, priority=self.DOWNLOAD_PRIORITY,
meta={'dont_obey_robotstxt': True} meta={'dont_obey_robotstxt': True}
) )
dfd = self.crawler.engine.download(robotsreq, spider) # engine.download() can return an already-called deferred, e.g. if a
# middleware returns a response in process_request(). Using
# deferLater() ensures that the error callback isn't called
# immediately upon being added, so that it doesn't remove the key
# before we check for it.
dfd = deferLater(reactor, 0, self.crawler.engine.download,
robotsreq, spider)
dfd.addCallback(self._parse_robots, netloc) dfd.addCallback(self._parse_robots, netloc)
dfd.addErrback(self._logerror, robotsreq, spider) dfd.addErrback(self._logerror, robotsreq, spider)
dfd.addErrback(self._robots_error, netloc) dfd.addErrback(self._robots_error, netloc)