From 0dcf47a7068319258807ef51e81724c1c31591ff Mon Sep 17 00:00:00 2001 From: "ajones1@gmail.com" Date: Wed, 25 Nov 2009 12:59:28 -0800 Subject: [PATCH] fix name errors on robots.txt middleware during spider_close --- scrapy/contrib/downloadermiddleware/robotstxt.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scrapy/contrib/downloadermiddleware/robotstxt.py b/scrapy/contrib/downloadermiddleware/robotstxt.py index eede2a1df..3784ff9d2 100644 --- a/scrapy/contrib/downloadermiddleware/robotstxt.py +++ b/scrapy/contrib/downloadermiddleware/robotstxt.py @@ -58,7 +58,7 @@ class RobotsTxtMiddleware(object): or settings['USER_AGENT'] def spider_closed(self, spider): - for netloc in self._spider_netlocs[domain]: + for netloc in self._spider_netlocs[spider]: del self._parsers[netloc] - del self._spider_netlocs[domain] + del self._spider_netlocs[spider] del self._useragents[spider]