From 4d5e5378bd2ccea2879102614536410d9338b3f1 Mon Sep 17 00:00:00 2001
From: Wenbin Zhang <WenbinZhang@users.noreply.github.com>
Date: Wed, 7 Feb 2018 10:59:32 -0500
Subject: [PATCH] Update robotstxt.py

Add message to IgnoreRequest exception so that it can be detectedin the errbak method of a spider
---
 scrapy/downloadermiddlewares/robotstxt.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/downloadermiddlewares/robotstxt.py b/scrapy/downloadermiddlewares/robotstxt.py
index c3dfa7819..4f3b5d64f 100644
--- a/scrapy/downloadermiddlewares/robotstxt.py
+++ b/scrapy/downloadermiddlewares/robotstxt.py
@@ -45,7 +45,7 @@ class RobotsTxtMiddleware(object):
                  to_native_str(self._useragent), request.url):
             logger.debug("Forbidden by robots.txt: %(request)s",
                          {'request': request}, extra={'spider': spider})
-            raise IgnoreRequest()
+            raise IgnoreRequest("Forbidden by robots.txt")
 
     def robot_parser(self, request, spider):
         url = urlparse_cached(request)