diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 1ca78ccc6..0ef3fb071 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -681,7 +681,9 @@ HttpProxyMiddleware * ``no_proxy`` You can also set the meta key ``proxy`` per-request, to a value like - ``http://some_proxy_server:port``. + ``http://some_proxy_server:port`` or ``http://username:password@some_proxy_server:port``. + Keep in mind this value will take precedence over ``http_proxy``/``https_proxy`` + environment variables, and it will also ignore ``no_proxy`` environment variable. .. _urllib: https://docs.python.org/2/library/urllib.html .. _urllib2: https://docs.python.org/2/library/urllib2.html @@ -949,8 +951,16 @@ enable it for :ref:`broad crawls `. HttpProxyMiddleware settings ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +.. setting:: HTTPPROXY_ENABLED .. setting:: HTTPPROXY_AUTH_ENCODING +HTTPPROXY_ENABLED +^^^^^^^^^^^^^^^^^ + +Default: ``True`` + +Whether or not to enable the :class:`HttpProxyMiddleware`. + HTTPPROXY_AUTH_ENCODING ^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/scrapy/downloadermiddlewares/httpproxy.py b/scrapy/downloadermiddlewares/httpproxy.py index 98c87aa9c..0d5320bf8 100644 --- a/scrapy/downloadermiddlewares/httpproxy.py +++ b/scrapy/downloadermiddlewares/httpproxy.py @@ -20,23 +20,25 @@ class HttpProxyMiddleware(object): for type, url in getproxies().items(): self.proxies[type] = self._get_proxy(url, type) - if not self.proxies: - raise NotConfigured - @classmethod def from_crawler(cls, crawler): + if not crawler.settings.getbool('HTTPPROXY_ENABLED'): + raise NotConfigured auth_encoding = crawler.settings.get('HTTPPROXY_AUTH_ENCODING') return cls(auth_encoding) + def _basic_auth_header(self, username, password): + user_pass = to_bytes( + '%s:%s' % (unquote(username), unquote(password)), + encoding=self.auth_encoding) + return base64.b64encode(user_pass).strip() + def _get_proxy(self, url, orig_type): proxy_type, user, password, hostport = _parse_proxy(url) proxy_url = urlunparse((proxy_type or orig_type, hostport, '', '', '', '')) if user: - user_pass = to_bytes( - '%s:%s' % (unquote(user), unquote(password)), - encoding=self.auth_encoding) - creds = base64.b64encode(user_pass).strip() + creds = self._basic_auth_header(user, password) else: creds = None @@ -45,6 +47,15 @@ class HttpProxyMiddleware(object): def process_request(self, request, spider): # ignore if proxy is already set if 'proxy' in request.meta: + if request.meta['proxy'] is None: + return + # extract credentials if present + creds, proxy_url = self._get_proxy(request.meta['proxy'], '') + request.meta['proxy'] = proxy_url + if creds and not request.headers.get('Proxy-Authorization'): + request.headers['Proxy-Authorization'] = b'Basic ' + creds + return + elif not self.proxies: return parsed = urlparse_cached(request) diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 24714a7a8..cb88bc2bf 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -174,6 +174,7 @@ HTTPCACHE_DBM_MODULE = 'anydbm' if six.PY2 else 'dbm' HTTPCACHE_POLICY = 'scrapy.extensions.httpcache.DummyPolicy' HTTPCACHE_GZIP = False +HTTPPROXY_ENABLED = True HTTPPROXY_AUTH_ENCODING = 'latin-1' IMAGES_STORE_S3_ACL = 'private' diff --git a/tests/test_downloadermiddleware_httpproxy.py b/tests/test_downloadermiddleware_httpproxy.py index 2b26431a4..c77179ceb 100644 --- a/tests/test_downloadermiddleware_httpproxy.py +++ b/tests/test_downloadermiddleware_httpproxy.py @@ -1,11 +1,14 @@ import os import sys +from functools import partial from twisted.trial.unittest import TestCase, SkipTest from scrapy.downloadermiddlewares.httpproxy import HttpProxyMiddleware from scrapy.exceptions import NotConfigured from scrapy.http import Response, Request from scrapy.spiders import Spider +from scrapy.crawler import Crawler +from scrapy.settings import Settings spider = Spider('foo') @@ -20,9 +23,10 @@ class TestDefaultHeadersMiddleware(TestCase): def tearDown(self): os.environ = self._oldenv - def test_no_proxies(self): - os.environ = {} - self.assertRaises(NotConfigured, HttpProxyMiddleware) + def test_not_enabled(self): + settings = Settings({'HTTPPROXY_ENABLED': False}) + crawler = Crawler(spider, settings) + self.assertRaises(NotConfigured, partial(HttpProxyMiddleware.from_crawler, crawler)) def test_no_enviroment_proxies(self): os.environ = {'dummy_proxy': 'reset_env_and_do_not_raise'} @@ -47,6 +51,13 @@ class TestDefaultHeadersMiddleware(TestCase): self.assertEquals(req.url, url) self.assertEquals(req.meta.get('proxy'), proxy) + def test_proxy_precedence_meta(self): + os.environ['http_proxy'] = 'https://proxy.com' + mw = HttpProxyMiddleware() + req = Request('http://scrapytest.org', meta={'proxy': 'https://new.proxy:3128'}) + assert mw.process_request(req, spider) is None + self.assertEquals(req.meta, {'proxy': 'https://new.proxy:3128'}) + def test_proxy_auth(self): os.environ['http_proxy'] = 'https://user:pass@proxy:3128' mw = HttpProxyMiddleware() @@ -54,6 +65,11 @@ class TestDefaultHeadersMiddleware(TestCase): assert mw.process_request(req, spider) is None self.assertEquals(req.meta, {'proxy': 'https://proxy:3128'}) self.assertEquals(req.headers.get('Proxy-Authorization'), b'Basic dXNlcjpwYXNz') + # proxy from request.meta + req = Request('http://scrapytest.org', meta={'proxy': 'https://username:password@proxy:3128'}) + assert mw.process_request(req, spider) is None + self.assertEquals(req.meta, {'proxy': 'https://proxy:3128'}) + self.assertEquals(req.headers.get('Proxy-Authorization'), b'Basic dXNlcm5hbWU6cGFzc3dvcmQ=') def test_proxy_auth_empty_passwd(self): os.environ['http_proxy'] = 'https://user:@proxy:3128' @@ -62,6 +78,11 @@ class TestDefaultHeadersMiddleware(TestCase): assert mw.process_request(req, spider) is None self.assertEquals(req.meta, {'proxy': 'https://proxy:3128'}) self.assertEquals(req.headers.get('Proxy-Authorization'), b'Basic dXNlcjo=') + # proxy from request.meta + req = Request('http://scrapytest.org', meta={'proxy': 'https://username:@proxy:3128'}) + assert mw.process_request(req, spider) is None + self.assertEquals(req.meta, {'proxy': 'https://proxy:3128'}) + self.assertEquals(req.headers.get('Proxy-Authorization'), b'Basic dXNlcm5hbWU6') def test_proxy_auth_encoding(self): # utf-8 encoding @@ -72,6 +93,12 @@ class TestDefaultHeadersMiddleware(TestCase): self.assertEquals(req.meta, {'proxy': 'https://proxy:3128'}) self.assertEquals(req.headers.get('Proxy-Authorization'), b'Basic bcOhbjpwYXNz') + # proxy from request.meta + req = Request('http://scrapytest.org', meta={'proxy': u'https://\u00FCser:pass@proxy:3128'}) + assert mw.process_request(req, spider) is None + self.assertEquals(req.meta, {'proxy': 'https://proxy:3128'}) + self.assertEquals(req.headers.get('Proxy-Authorization'), b'Basic w7xzZXI6cGFzcw==') + # default latin-1 encoding mw = HttpProxyMiddleware(auth_encoding='latin-1') req = Request('http://scrapytest.org') @@ -79,15 +106,21 @@ class TestDefaultHeadersMiddleware(TestCase): self.assertEquals(req.meta, {'proxy': 'https://proxy:3128'}) self.assertEquals(req.headers.get('Proxy-Authorization'), b'Basic beFuOnBhc3M=') + # proxy from request.meta, latin-1 encoding + req = Request('http://scrapytest.org', meta={'proxy': u'https://\u00FCser:pass@proxy:3128'}) + assert mw.process_request(req, spider) is None + self.assertEquals(req.meta, {'proxy': 'https://proxy:3128'}) + self.assertEquals(req.headers.get('Proxy-Authorization'), b'Basic /HNlcjpwYXNz') + def test_proxy_already_seted(self): - os.environ['http_proxy'] = http_proxy = 'https://proxy.for.http:3128' + os.environ['http_proxy'] = 'https://proxy.for.http:3128' mw = HttpProxyMiddleware() req = Request('http://noproxy.com', meta={'proxy': None}) assert mw.process_request(req, spider) is None assert 'proxy' in req.meta and req.meta['proxy'] is None def test_no_proxy(self): - os.environ['http_proxy'] = http_proxy = 'https://proxy.for.http:3128' + os.environ['http_proxy'] = 'https://proxy.for.http:3128' mw = HttpProxyMiddleware() os.environ['no_proxy'] = '*' @@ -104,3 +137,9 @@ class TestDefaultHeadersMiddleware(TestCase): req = Request('http://noproxy.com') assert mw.process_request(req, spider) is None assert 'proxy' not in req.meta + + # proxy from meta['proxy'] takes precedence + os.environ['no_proxy'] = '*' + req = Request('http://noproxy.com', meta={'proxy': 'http://proxy.com'}) + assert mw.process_request(req, spider) is None + self.assertEquals(req.meta, {'proxy': 'http://proxy.com'})