RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

changed robots.txt forbidden log level to DEBUG. closes #268

This commit is contained in:
Pablo Hoffman 2010-10-27 09:17:58 -02:00
parent 99096cdacf
commit 6f4be21d4c
1 changed files with 4 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
scrapy/contrib/downloadermiddleware/robotstxt.py
View File

@ -1,5 +1,5 @@
"""
This is a middleware to respect robots.txt policies. To active it you must
This is a middleware to respect robots.txt policies. To activate it you must
enable this middleware and enable the ROBOTSTXT_OBEY setting.
"""
@ -8,7 +8,7 @@ import robotparser
from scrapy.xlib.pydispatch import dispatcher
from scrapy import signals
from scrapy import signals, log
from scrapy.project import crawler
from scrapy.exceptions import NotConfigured, IgnoreRequest
from scrapy.http import Request
@ -25,7 +25,6 @@ class RobotsTxtMiddleware(object):
self._parsers = {}
self._spider_netlocs = {}
self._useragents = {}
self._pending = {}
dispatcher.connect(self.spider_opened, signals.spider_opened)
dispatcher.connect(self.spider_closed, signals.spider_closed)
@ -33,7 +32,8 @@ class RobotsTxtMiddleware(object):
useragent = self._useragents[spider]
rp = self.robot_parser(request, spider)
if rp and not rp.can_fetch(useragent, request.url):
raise IgnoreRequest("URL forbidden by robots.txt: %s" % request.url)
log.msg("Forbidden by robots.txt: %s" % request, log.DEBUG)
raise IgnoreRequest
def robot_parser(self, request, spider):
url = urlparse_cached(request)