RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

fixed bug in robots middleware reported by fencer in #101

This commit is contained in:
Pablo Hoffman 2009-09-04 12:36:29 -03:00
parent dad05957f3
commit 86de5180fc
1 changed files with 3 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
scrapy/contrib/downloadermiddleware/robotstxt.py
View File

@ -31,11 +31,12 @@ class RobotsTxtMiddleware(object):
def process_request(self, request, spider):
useragent = self._useragents[spider]
rp = self.robot_parser(request.url, spider)
rp = self.robot_parser(request, spider)
if rp and not rp.can_fetch(useragent, request.url):
raise IgnoreRequest("URL forbidden by robots.txt: %s" % request.url)
def robot_parser(self, url, spider):
def robot_parser(self, request, spider):
url = urlparse_cached(request)
netloc = url.netloc
if netloc not in self._parsers:
self._parsers[netloc] = None