From 9c0aae724ed821fd954a14db83902a86f7fe7731 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Fri, 3 Feb 2017 10:32:36 -0300 Subject: [PATCH] Use credentials from request.meta['proxy'] if present --- docs/topics/downloader-middleware.rst | 5 ++- scrapy/downloadermiddlewares/httpproxy.py | 24 +++++++---- tests/test_downloadermiddleware_httpproxy.py | 43 +++++++++++++++++--- 3 files changed, 57 insertions(+), 15 deletions(-) diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 1ca78ccc6..f0ff3c77c 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -681,7 +681,10 @@ HttpProxyMiddleware * ``no_proxy`` You can also set the meta key ``proxy`` per-request, to a value like - ``http://some_proxy_server:port``. + ``http://username:password@some_proxy_server:port``. Keep in mind + this value will take precedence over ``http_proxy``/``https_proxy`` + environment variables, and it will also ignore ``no_proxy`` environment + variable. .. _urllib: https://docs.python.org/2/library/urllib.html .. _urllib2: https://docs.python.org/2/library/urllib2.html diff --git a/scrapy/downloadermiddlewares/httpproxy.py b/scrapy/downloadermiddlewares/httpproxy.py index 98c87aa9c..edc1c52ed 100644 --- a/scrapy/downloadermiddlewares/httpproxy.py +++ b/scrapy/downloadermiddlewares/httpproxy.py @@ -8,7 +8,6 @@ except ImportError: from six.moves.urllib.parse import urlunparse from scrapy.utils.httpobj import urlparse_cached -from scrapy.exceptions import NotConfigured from scrapy.utils.python import to_bytes @@ -20,23 +19,23 @@ class HttpProxyMiddleware(object): for type, url in getproxies().items(): self.proxies[type] = self._get_proxy(url, type) - if not self.proxies: - raise NotConfigured - @classmethod def from_crawler(cls, crawler): auth_encoding = crawler.settings.get('HTTPPROXY_AUTH_ENCODING') return cls(auth_encoding) + def _basic_auth_header(self, username, password): + user_pass = to_bytes( + '%s:%s' % (unquote(username), unquote(password)), + encoding=self.auth_encoding) + return base64.b64encode(user_pass).strip() + def _get_proxy(self, url, orig_type): proxy_type, user, password, hostport = _parse_proxy(url) proxy_url = urlunparse((proxy_type or orig_type, hostport, '', '', '', '')) if user: - user_pass = to_bytes( - '%s:%s' % (unquote(user), unquote(password)), - encoding=self.auth_encoding) - creds = base64.b64encode(user_pass).strip() + creds = self._basic_auth_header(user, password) else: creds = None @@ -45,6 +44,15 @@ class HttpProxyMiddleware(object): def process_request(self, request, spider): # ignore if proxy is already set if 'proxy' in request.meta: + if request.meta['proxy'] is None: + return + # extract credentials if present + creds, proxy_url = self._get_proxy(request.meta['proxy'], '') + request.meta['proxy'] = proxy_url + if creds and not request.headers.get('Proxy-Authorization'): + request.headers['Proxy-Authorization'] = b'Basic ' + creds + return + elif not self.proxies: return parsed = urlparse_cached(request) diff --git a/tests/test_downloadermiddleware_httpproxy.py b/tests/test_downloadermiddleware_httpproxy.py index 2b26431a4..dd09e4dd0 100644 --- a/tests/test_downloadermiddleware_httpproxy.py +++ b/tests/test_downloadermiddleware_httpproxy.py @@ -20,10 +20,6 @@ class TestDefaultHeadersMiddleware(TestCase): def tearDown(self): os.environ = self._oldenv - def test_no_proxies(self): - os.environ = {} - self.assertRaises(NotConfigured, HttpProxyMiddleware) - def test_no_enviroment_proxies(self): os.environ = {'dummy_proxy': 'reset_env_and_do_not_raise'} mw = HttpProxyMiddleware() @@ -47,6 +43,13 @@ class TestDefaultHeadersMiddleware(TestCase): self.assertEquals(req.url, url) self.assertEquals(req.meta.get('proxy'), proxy) + def test_proxy_precedence_meta(self): + os.environ['http_proxy'] = 'https://proxy.com' + mw = HttpProxyMiddleware() + req = Request('http://scrapytest.org', meta={'proxy': 'https://new.proxy:3128'}) + assert mw.process_request(req, spider) is None + self.assertEquals(req.meta, {'proxy': 'https://new.proxy:3128'}) + def test_proxy_auth(self): os.environ['http_proxy'] = 'https://user:pass@proxy:3128' mw = HttpProxyMiddleware() @@ -54,6 +57,11 @@ class TestDefaultHeadersMiddleware(TestCase): assert mw.process_request(req, spider) is None self.assertEquals(req.meta, {'proxy': 'https://proxy:3128'}) self.assertEquals(req.headers.get('Proxy-Authorization'), b'Basic dXNlcjpwYXNz') + # proxy from request.meta + req = Request('http://scrapytest.org', meta={'proxy': 'https://username:password@proxy:3128'}) + assert mw.process_request(req, spider) is None + self.assertEquals(req.meta, {'proxy': 'https://proxy:3128'}) + self.assertEquals(req.headers.get('Proxy-Authorization'), b'Basic dXNlcm5hbWU6cGFzc3dvcmQ=') def test_proxy_auth_empty_passwd(self): os.environ['http_proxy'] = 'https://user:@proxy:3128' @@ -62,6 +70,11 @@ class TestDefaultHeadersMiddleware(TestCase): assert mw.process_request(req, spider) is None self.assertEquals(req.meta, {'proxy': 'https://proxy:3128'}) self.assertEquals(req.headers.get('Proxy-Authorization'), b'Basic dXNlcjo=') + # proxy from request.meta + req = Request('http://scrapytest.org', meta={'proxy': 'https://username:@proxy:3128'}) + assert mw.process_request(req, spider) is None + self.assertEquals(req.meta, {'proxy': 'https://proxy:3128'}) + self.assertEquals(req.headers.get('Proxy-Authorization'), b'Basic dXNlcm5hbWU6') def test_proxy_auth_encoding(self): # utf-8 encoding @@ -72,6 +85,12 @@ class TestDefaultHeadersMiddleware(TestCase): self.assertEquals(req.meta, {'proxy': 'https://proxy:3128'}) self.assertEquals(req.headers.get('Proxy-Authorization'), b'Basic bcOhbjpwYXNz') + # proxy from request.meta + req = Request('http://scrapytest.org', meta={'proxy': u'https://\u00FCser:pass@proxy:3128'}) + assert mw.process_request(req, spider) is None + self.assertEquals(req.meta, {'proxy': 'https://proxy:3128'}) + self.assertEquals(req.headers.get('Proxy-Authorization'), b'Basic w7xzZXI6cGFzcw==') + # default latin-1 encoding mw = HttpProxyMiddleware(auth_encoding='latin-1') req = Request('http://scrapytest.org') @@ -79,15 +98,21 @@ class TestDefaultHeadersMiddleware(TestCase): self.assertEquals(req.meta, {'proxy': 'https://proxy:3128'}) self.assertEquals(req.headers.get('Proxy-Authorization'), b'Basic beFuOnBhc3M=') + # proxy from request.meta, latin-1 encoding + req = Request('http://scrapytest.org', meta={'proxy': u'https://\u00FCser:pass@proxy:3128'}) + assert mw.process_request(req, spider) is None + self.assertEquals(req.meta, {'proxy': 'https://proxy:3128'}) + self.assertEquals(req.headers.get('Proxy-Authorization'), b'Basic /HNlcjpwYXNz') + def test_proxy_already_seted(self): - os.environ['http_proxy'] = http_proxy = 'https://proxy.for.http:3128' + os.environ['http_proxy'] = 'https://proxy.for.http:3128' mw = HttpProxyMiddleware() req = Request('http://noproxy.com', meta={'proxy': None}) assert mw.process_request(req, spider) is None assert 'proxy' in req.meta and req.meta['proxy'] is None def test_no_proxy(self): - os.environ['http_proxy'] = http_proxy = 'https://proxy.for.http:3128' + os.environ['http_proxy'] = 'https://proxy.for.http:3128' mw = HttpProxyMiddleware() os.environ['no_proxy'] = '*' @@ -104,3 +129,9 @@ class TestDefaultHeadersMiddleware(TestCase): req = Request('http://noproxy.com') assert mw.process_request(req, spider) is None assert 'proxy' not in req.meta + + # proxy from meta['proxy'] takes precedence + os.environ['no_proxy'] = '*' + req = Request('http://noproxy.com', meta={'proxy': 'http://proxy.com'}) + assert mw.process_request(req, spider) is None + self.assertEquals(req.meta, {'proxy': 'http://proxy.com'})