From 86de5180fcb1be576dcb44ce4ca072e8a719fbdb Mon Sep 17 00:00:00 2001 From: Pablo Hoffman Date: Fri, 4 Sep 2009 12:36:29 -0300 Subject: [PATCH] fixed bug in robots middleware reported by fencer in #101 --- scrapy/contrib/downloadermiddleware/robotstxt.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scrapy/contrib/downloadermiddleware/robotstxt.py b/scrapy/contrib/downloadermiddleware/robotstxt.py index a0c6c98f5..161ae9f34 100644 --- a/scrapy/contrib/downloadermiddleware/robotstxt.py +++ b/scrapy/contrib/downloadermiddleware/robotstxt.py @@ -31,11 +31,12 @@ class RobotsTxtMiddleware(object): def process_request(self, request, spider): useragent = self._useragents[spider] - rp = self.robot_parser(request.url, spider) + rp = self.robot_parser(request, spider) if rp and not rp.can_fetch(useragent, request.url): raise IgnoreRequest("URL forbidden by robots.txt: %s" % request.url) - def robot_parser(self, url, spider): + def robot_parser(self, request, spider): + url = urlparse_cached(request) netloc = url.netloc if netloc not in self._parsers: self._parsers[netloc] = None