RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Automated merge after applying ajones patch

This commit is contained in:
Pablo Hoffman 2009-11-28 12:31:05 -02:00
parent 207aae2b04 0dcf47a706
commit de35eee3cd
1 changed files with 2 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
scrapy/contrib/downloadermiddleware/robotstxt.py
View File

@ -58,7 +58,7 @@ class RobotsTxtMiddleware(object):
or settings['USER_AGENT']
def spider_closed(self, spider):
for netloc in self._spider_netlocs[domain]:
for netloc in self._spider_netlocs[spider]:
del self._parsers[netloc]
del self._spider_netlocs[domain]
del self._spider_netlocs[spider]
del self._useragents[spider]