Update robotstxt.py

Add message to IgnoreRequest exception so that it can be detectedin the errbak method of a spider
This commit is contained in:
Wenbin Zhang 2018-02-07 10:59:32 -05:00 committed by GitHub
parent fd02fa808c
commit 4d5e5378bd
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 1 additions and 1 deletions

View File

@ -45,7 +45,7 @@ class RobotsTxtMiddleware(object):
to_native_str(self._useragent), request.url):
logger.debug("Forbidden by robots.txt: %(request)s",
{'request': request}, extra={'spider': spider})
raise IgnoreRequest()
raise IgnoreRequest("Forbidden by robots.txt")
def robot_parser(self, request, spider):
url = urlparse_cached(request)