mirror of https://github.com/scrapy/scrapy.git
23 lines
720 B
Python
23 lines
720 B
Python
"""Set User-Agent header per spider or use a default value from settings"""
|
|
|
|
from scrapy.utils.python import WeakKeyCache
|
|
from scrapy.utils import deprecate
|
|
|
|
|
|
class UserAgentMiddleware(object):
|
|
"""This middleware allows spiders to override the user_agent"""
|
|
|
|
def __init__(self):
|
|
self.cache = WeakKeyCache(self._user_agent)
|
|
|
|
def _user_agent(self, spider):
|
|
if hasattr(spider, 'user_agent'):
|
|
deprecate.attribute(spider, 'user_agent', 'USER_AGENT')
|
|
return spider.user_agent
|
|
return spider.settings['USER_AGENT']
|
|
|
|
def process_request(self, request, spider):
|
|
ua = self.cache[spider]
|
|
if ua:
|
|
request.headers.setdefault('User-Agent', ua)
|