RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #419 from nramirezuy/robotstxt-multispider 

robotstxt mid multi spider support removed
This commit is contained in:
Pablo Hoffman 2013-10-09 07:43:11 -07:00
parent 9ad736d7ce 10646f1768
commit 7ec01799d5
1 changed files with 5 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
scrapy/contrib/downloadermiddleware/robotstxt.py
View File

@ -11,6 +11,7 @@ from scrapy.exceptions import NotConfigured, IgnoreRequest
from scrapy.http import Request
from scrapy.utils.httpobj import urlparse_cached
class RobotsTxtMiddleware(object):
DOWNLOAD_PRIORITY = 1000
@ -19,18 +20,16 @@ class RobotsTxtMiddleware(object):
raise NotConfigured
self.crawler = crawler
self._useragent = crawler.settings.get('USER_AGENT')
self._parsers = {}
self._spider_netlocs = {}
self._useragents = {}
crawler.signals.connect(self.spider_opened, signals.spider_opened)
crawler.signals.connect(self.spider_closed, signals.spider_closed)
self._spider_netlocs = set()
@classmethod
def from_crawler(cls, crawler):
return cls(crawler)
def process_request(self, request, spider):
useragent = self._useragents[spider]
useragent = self._useragent
rp = self.robot_parser(request, spider)
if rp and not rp.can_fetch(useragent, request.url):
log.msg(format="Forbidden by robots.txt: %(request)s",
@ -46,20 +45,10 @@ class RobotsTxtMiddleware(object):
robotsreq = Request(robotsurl, priority=self.DOWNLOAD_PRIORITY)
dfd = self.crawler.engine.download(robotsreq, spider)
dfd.addCallback(self._parse_robots)
self._spider_netlocs[spider].add(netloc)
self._spider_netlocs.add(netloc)
return self._parsers[netloc]
def _parse_robots(self, response):
rp = robotparser.RobotFileParser(response.url)
rp.parse(response.body.splitlines())
self._parsers[urlparse_cached(response).netloc] = rp
def spider_opened(self, spider):
self._spider_netlocs[spider] = set()
self._useragents[spider] = spider.settings['USER_AGENT']
def spider_closed(self, spider):
for netloc in self._spider_netlocs[spider]:
del self._parsers[netloc]
del self._spider_netlocs[spider]
del self._useragents[spider]