From 560dd3fd8ee470f72bc215f29b0e4e4ab3d77185 Mon Sep 17 00:00:00 2001 From: Daniel Grana Date: Mon, 27 Apr 2009 16:56:16 +0000 Subject: [PATCH] useragent: if spider user_agent attribute is None, use default useragent instead of not including one --HG-- extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%401095 --- .../trunk/scrapy/contrib/downloadermiddleware/useragent.py | 2 +- .../scrapy/tests/test_downloadermiddleware_useragent.py | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/scrapy/trunk/scrapy/contrib/downloadermiddleware/useragent.py b/scrapy/trunk/scrapy/contrib/downloadermiddleware/useragent.py index ddf51cb89..29206b2c8 100644 --- a/scrapy/trunk/scrapy/contrib/downloadermiddleware/useragent.py +++ b/scrapy/trunk/scrapy/contrib/downloadermiddleware/useragent.py @@ -9,6 +9,6 @@ class UserAgentMiddleware(object): default_useragent = settings.get('USER_AGENT') def process_request(self, request, spider): - ua = getattr(spider, 'user_agent', self.default_useragent) + ua = getattr(spider, 'user_agent', None) or self.default_useragent if ua: request.headers.setdefault('User-Agent', ua) diff --git a/scrapy/trunk/scrapy/tests/test_downloadermiddleware_useragent.py b/scrapy/trunk/scrapy/tests/test_downloadermiddleware_useragent.py index 091567ffd..4ebb36d2b 100644 --- a/scrapy/trunk/scrapy/tests/test_downloadermiddleware_useragent.py +++ b/scrapy/trunk/scrapy/tests/test_downloadermiddleware_useragent.py @@ -23,7 +23,14 @@ class CookiesMiddlewareTest(TestCase): assert self.mw.process_request(req, self.spider) is None self.assertEquals(req.headers['User-Agent'], 'default_useragent') + # None or not present user_agent attribute is the same + self.spider.user_agent = None + req = Request('http://scrapytest.org/') + assert self.mw.process_request(req, self.spider) is None + self.assertEquals(req.headers['User-Agent'], 'default_useragent') + def test_spider_agent(self): + self.mw.default_useragent = 'default_useragent' self.spider.user_agent = 'spider_useragent' req = Request('http://scrapytest.org/') assert self.mw.process_request(req, self.spider) is None