mirror of https://github.com/scrapy/scrapy.git
Update robotstxt.py
Add message to IgnoreRequest exception so that it can be detectedin the errbak method of a spider
This commit is contained in:
parent
fd02fa808c
commit
4d5e5378bd
|
|
@ -45,7 +45,7 @@ class RobotsTxtMiddleware(object):
|
|||
to_native_str(self._useragent), request.url):
|
||||
logger.debug("Forbidden by robots.txt: %(request)s",
|
||||
{'request': request}, extra={'spider': spider})
|
||||
raise IgnoreRequest()
|
||||
raise IgnoreRequest("Forbidden by robots.txt")
|
||||
|
||||
def robot_parser(self, request, spider):
|
||||
url = urlparse_cached(request)
|
||||
|
|
|
|||
Loading…
Reference in New Issue