diff --git a/scrapy/contrib/downloadermiddleware/robotstxt.py b/scrapy/contrib/downloadermiddleware/robotstxt.py
index f0dfd3fef..cc9f863f9 100644
--- a/scrapy/contrib/downloadermiddleware/robotstxt.py
+++ b/scrapy/contrib/downloadermiddleware/robotstxt.py
@@ -11,6 +11,7 @@ from scrapy.exceptions import NotConfigured, IgnoreRequest
 from scrapy.http import Request
 from scrapy.utils.httpobj import urlparse_cached
 
+
 class RobotsTxtMiddleware(object):
     DOWNLOAD_PRIORITY = 1000
 
@@ -19,18 +20,16 @@ class RobotsTxtMiddleware(object):
             raise NotConfigured
 
         self.crawler = crawler
+        self._useragent = crawler.settings.get('USER_AGENT')
         self._parsers = {}
-        self._spider_netlocs = {}
-        self._useragents = {}
-        crawler.signals.connect(self.spider_opened, signals.spider_opened)
-        crawler.signals.connect(self.spider_closed, signals.spider_closed)
+        self._spider_netlocs = set()
 
     @classmethod
     def from_crawler(cls, crawler):
         return cls(crawler)
 
     def process_request(self, request, spider):
-        useragent = self._useragents[spider]
+        useragent = self._useragent
         rp = self.robot_parser(request, spider)
         if rp and not rp.can_fetch(useragent, request.url):
             log.msg(format="Forbidden by robots.txt: %(request)s",
@@ -46,20 +45,10 @@ class RobotsTxtMiddleware(object):
             robotsreq = Request(robotsurl, priority=self.DOWNLOAD_PRIORITY)
             dfd = self.crawler.engine.download(robotsreq, spider)
             dfd.addCallback(self._parse_robots)
-            self._spider_netlocs[spider].add(netloc)
+            self._spider_netlocs.add(netloc)
         return self._parsers[netloc]
 
     def _parse_robots(self, response):
         rp = robotparser.RobotFileParser(response.url)
         rp.parse(response.body.splitlines())
         self._parsers[urlparse_cached(response).netloc] = rp
-
-    def spider_opened(self, spider):
-        self._spider_netlocs[spider] = set()
-        self._useragents[spider] = spider.settings['USER_AGENT']
-
-    def spider_closed(self, spider):
-        for netloc in self._spider_netlocs[spider]:
-            del self._parsers[netloc]
-        del self._spider_netlocs[spider]
-        del self._useragents[spider]