diff --git a/scrapy/contrib/downloadermiddleware/useragent.py b/scrapy/contrib/downloadermiddleware/useragent.py index 8084ab1ad..17793cad7 100644 --- a/scrapy/contrib/downloadermiddleware/useragent.py +++ b/scrapy/contrib/downloadermiddleware/useragent.py @@ -1,25 +1,23 @@ """Set User-Agent header per spider or use a default value from settings""" -from scrapy.utils.python import WeakKeyCache +from scrapy import signals class UserAgentMiddleware(object): """This middleware allows spiders to override the user_agent""" def __init__(self, user_agent='Scrapy'): - self.cache = WeakKeyCache(self._user_agent) self.user_agent = user_agent @classmethod def from_crawler(cls, crawler): - return cls(crawler.settings['USER_AGENT']) + o = cls(crawler.settings['USER_AGENT']) + crawler.signals.connect(o.spider_opened, signal=signals.spider_opened) + return o - def _user_agent(self, spider): - if hasattr(spider, 'user_agent'): - return spider.user_agent - return self.user_agent + def spider_opened(self, spider): + self.user_agent = getattr(spider, 'user_agent', self.user_agent) def process_request(self, request, spider): - ua = self.cache[spider] - if ua: - request.headers.setdefault('User-Agent', ua) + if self.user_agent: + request.headers.setdefault('User-Agent', self.user_agent) diff --git a/scrapy/tests/test_downloadermiddleware_useragent.py b/scrapy/tests/test_downloadermiddleware_useragent.py index eced125eb..ece55f081 100644 --- a/scrapy/tests/test_downloadermiddleware_useragent.py +++ b/scrapy/tests/test_downloadermiddleware_useragent.py @@ -24,6 +24,7 @@ class UserAgentMiddlewareTest(TestCase): # settings UESR_AGENT to None should remove the user agent spider, mw = self.get_spider_and_mw('default_useragent') spider.user_agent = None + mw.spider_opened(spider) req = Request('http://scrapytest.org/') assert mw.process_request(req, spider) is None assert req.headers.get('User-Agent') is None @@ -31,6 +32,7 @@ class UserAgentMiddlewareTest(TestCase): def test_spider_agent(self): spider, mw = self.get_spider_and_mw('default_useragent') spider.user_agent = 'spider_useragent' + mw.spider_opened(spider) req = Request('http://scrapytest.org/') assert mw.process_request(req, spider) is None self.assertEquals(req.headers['User-Agent'], 'spider_useragent') @@ -38,6 +40,7 @@ class UserAgentMiddlewareTest(TestCase): def test_header_agent(self): spider, mw = self.get_spider_and_mw('default_useragent') spider.user_agent = 'spider_useragent' + mw.spider_opened(spider) req = Request('http://scrapytest.org/', headers={'User-Agent': 'header_useragent'}) assert mw.process_request(req, spider) is None self.assertEquals(req.headers['User-Agent'], 'header_useragent') @@ -45,7 +48,7 @@ class UserAgentMiddlewareTest(TestCase): def test_no_agent(self): spider, mw = self.get_spider_and_mw(None) spider.user_agent = None + mw.spider_opened(spider) req = Request('http://scrapytest.org/') assert mw.process_request(req, spider) is None assert 'User-Agent' not in req.headers -