Merge pull request #421 from nramirezuy/useragent-multispider

useragent mid multi spider support removed

scrapy/contrib/downloadermiddleware/useragent.py

@@ -1,25 +1,23 @@
 """Set User-Agent header per spider or use a default value from settings"""
 
-from scrapy.utils.python import WeakKeyCache
+from scrapy import signals
 
 
 class UserAgentMiddleware(object):
     """This middleware allows spiders to override the user_agent"""
 
     def __init__(self, user_agent='Scrapy'):
-        self.cache = WeakKeyCache(self._user_agent)
         self.user_agent = user_agent
 
     @classmethod
     def from_crawler(cls, crawler):
-        return cls(crawler.settings['USER_AGENT'])
+        o = cls(crawler.settings['USER_AGENT'])
+        crawler.signals.connect(o.spider_opened, signal=signals.spider_opened)
+        return o
 
-    def _user_agent(self, spider):
-        if hasattr(spider, 'user_agent'):
-            return spider.user_agent
-        return self.user_agent
+    def spider_opened(self, spider):
+        self.user_agent = getattr(spider, 'user_agent', self.user_agent)
 
     def process_request(self, request, spider):
-        ua = self.cache[spider]
-        if ua:
-            request.headers.setdefault('User-Agent', ua)
+        if self.user_agent:
+            request.headers.setdefault('User-Agent', self.user_agent)

scrapy/tests/test_downloadermiddleware_useragent.py

@@ -24,6 +24,7 @@ class UserAgentMiddlewareTest(TestCase):
         # settings UESR_AGENT to None should remove the user agent
         spider, mw = self.get_spider_and_mw('default_useragent')
         spider.user_agent = None
+        mw.spider_opened(spider)
         req = Request('http://scrapytest.org/')
         assert mw.process_request(req, spider) is None
         assert req.headers.get('User-Agent') is None
@@ -31,6 +32,7 @@ class UserAgentMiddlewareTest(TestCase):
     def test_spider_agent(self):
         spider, mw = self.get_spider_and_mw('default_useragent')
         spider.user_agent = 'spider_useragent'
+        mw.spider_opened(spider)
         req = Request('http://scrapytest.org/')
         assert mw.process_request(req, spider) is None
         self.assertEquals(req.headers['User-Agent'], 'spider_useragent')
@@ -38,6 +40,7 @@ class UserAgentMiddlewareTest(TestCase):
     def test_header_agent(self):
         spider, mw = self.get_spider_and_mw('default_useragent')
         spider.user_agent = 'spider_useragent'
+        mw.spider_opened(spider)
         req = Request('http://scrapytest.org/', headers={'User-Agent': 'header_useragent'})
         assert mw.process_request(req, spider) is None
         self.assertEquals(req.headers['User-Agent'], 'header_useragent')
@@ -45,7 +48,7 @@ class UserAgentMiddlewareTest(TestCase):
     def test_no_agent(self):
         spider, mw = self.get_spider_and_mw(None)
         spider.user_agent = None
+        mw.spider_opened(spider)
         req = Request('http://scrapytest.org/')
         assert mw.process_request(req, spider) is None
         assert 'User-Agent' not in req.headers
-