From 9c0aae724ed821fd954a14db83902a86f7fe7731 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Fri, 3 Feb 2017 10:32:36 -0300 Subject: [PATCH 1/2] Use credentials from request.meta['proxy'] if present --- docs/topics/downloader-middleware.rst | 5 ++- scrapy/downloadermiddlewares/httpproxy.py | 24 +++++++---- tests/test_downloadermiddleware_httpproxy.py | 43 +++++++++++++++++--- 3 files changed, 57 insertions(+), 15 deletions(-) diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 1ca78ccc6..f0ff3c77c 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -681,7 +681,10 @@ HttpProxyMiddleware * ``no_proxy`` You can also set the meta key ``proxy`` per-request, to a value like - ``http://some_proxy_server:port``. + ``http://username:password@some_proxy_server:port``. Keep in mind + this value will take precedence over ``http_proxy``/``https_proxy`` + environment variables, and it will also ignore ``no_proxy`` environment + variable. .. _urllib: https://docs.python.org/2/library/urllib.html .. _urllib2: https://docs.python.org/2/library/urllib2.html diff --git a/scrapy/downloadermiddlewares/httpproxy.py b/scrapy/downloadermiddlewares/httpproxy.py index 98c87aa9c..edc1c52ed 100644 --- a/scrapy/downloadermiddlewares/httpproxy.py +++ b/scrapy/downloadermiddlewares/httpproxy.py @@ -8,7 +8,6 @@ except ImportError: from six.moves.urllib.parse import urlunparse from scrapy.utils.httpobj import urlparse_cached -from scrapy.exceptions import NotConfigured from scrapy.utils.python import to_bytes @@ -20,23 +19,23 @@ class HttpProxyMiddleware(object): for type, url in getproxies().items(): self.proxies[type] = self._get_proxy(url, type) - if not self.proxies: - raise NotConfigured - @classmethod def from_crawler(cls, crawler): auth_encoding = crawler.settings.get('HTTPPROXY_AUTH_ENCODING') return cls(auth_encoding) + def _basic_auth_header(self, username, password): + user_pass = to_bytes( + '%s:%s' % (unquote(username), unquote(password)), + encoding=self.auth_encoding) + return base64.b64encode(user_pass).strip() + def _get_proxy(self, url, orig_type): proxy_type, user, password, hostport = _parse_proxy(url) proxy_url = urlunparse((proxy_type or orig_type, hostport, '', '', '', '')) if user: - user_pass = to_bytes( - '%s:%s' % (unquote(user), unquote(password)), - encoding=self.auth_encoding) - creds = base64.b64encode(user_pass).strip() + creds = self._basic_auth_header(user, password) else: creds = None @@ -45,6 +44,15 @@ class HttpProxyMiddleware(object): def process_request(self, request, spider): # ignore if proxy is already set if 'proxy' in request.meta: + if request.meta['proxy'] is None: + return + # extract credentials if present + creds, proxy_url = self._get_proxy(request.meta['proxy'], '') + request.meta['proxy'] = proxy_url + if creds and not request.headers.get('Proxy-Authorization'): + request.headers['Proxy-Authorization'] = b'Basic ' + creds + return + elif not self.proxies: return parsed = urlparse_cached(request) diff --git a/tests/test_downloadermiddleware_httpproxy.py b/tests/test_downloadermiddleware_httpproxy.py index 2b26431a4..dd09e4dd0 100644 --- a/tests/test_downloadermiddleware_httpproxy.py +++ b/tests/test_downloadermiddleware_httpproxy.py @@ -20,10 +20,6 @@ class TestDefaultHeadersMiddleware(TestCase): def tearDown(self): os.environ = self._oldenv - def test_no_proxies(self): - os.environ = {} - self.assertRaises(NotConfigured, HttpProxyMiddleware) - def test_no_enviroment_proxies(self): os.environ = {'dummy_proxy': 'reset_env_and_do_not_raise'} mw = HttpProxyMiddleware() @@ -47,6 +43,13 @@ class TestDefaultHeadersMiddleware(TestCase): self.assertEquals(req.url, url) self.assertEquals(req.meta.get('proxy'), proxy) + def test_proxy_precedence_meta(self): + os.environ['http_proxy'] = 'https://proxy.com' + mw = HttpProxyMiddleware() + req = Request('http://scrapytest.org', meta={'proxy': 'https://new.proxy:3128'}) + assert mw.process_request(req, spider) is None + self.assertEquals(req.meta, {'proxy': 'https://new.proxy:3128'}) + def test_proxy_auth(self): os.environ['http_proxy'] = 'https://user:pass@proxy:3128' mw = HttpProxyMiddleware() @@ -54,6 +57,11 @@ class TestDefaultHeadersMiddleware(TestCase): assert mw.process_request(req, spider) is None self.assertEquals(req.meta, {'proxy': 'https://proxy:3128'}) self.assertEquals(req.headers.get('Proxy-Authorization'), b'Basic dXNlcjpwYXNz') + # proxy from request.meta + req = Request('http://scrapytest.org', meta={'proxy': 'https://username:password@proxy:3128'}) + assert mw.process_request(req, spider) is None + self.assertEquals(req.meta, {'proxy': 'https://proxy:3128'}) + self.assertEquals(req.headers.get('Proxy-Authorization'), b'Basic dXNlcm5hbWU6cGFzc3dvcmQ=') def test_proxy_auth_empty_passwd(self): os.environ['http_proxy'] = 'https://user:@proxy:3128' @@ -62,6 +70,11 @@ class TestDefaultHeadersMiddleware(TestCase): assert mw.process_request(req, spider) is None self.assertEquals(req.meta, {'proxy': 'https://proxy:3128'}) self.assertEquals(req.headers.get('Proxy-Authorization'), b'Basic dXNlcjo=') + # proxy from request.meta + req = Request('http://scrapytest.org', meta={'proxy': 'https://username:@proxy:3128'}) + assert mw.process_request(req, spider) is None + self.assertEquals(req.meta, {'proxy': 'https://proxy:3128'}) + self.assertEquals(req.headers.get('Proxy-Authorization'), b'Basic dXNlcm5hbWU6') def test_proxy_auth_encoding(self): # utf-8 encoding @@ -72,6 +85,12 @@ class TestDefaultHeadersMiddleware(TestCase): self.assertEquals(req.meta, {'proxy': 'https://proxy:3128'}) self.assertEquals(req.headers.get('Proxy-Authorization'), b'Basic bcOhbjpwYXNz') + # proxy from request.meta + req = Request('http://scrapytest.org', meta={'proxy': u'https://\u00FCser:pass@proxy:3128'}) + assert mw.process_request(req, spider) is None + self.assertEquals(req.meta, {'proxy': 'https://proxy:3128'}) + self.assertEquals(req.headers.get('Proxy-Authorization'), b'Basic w7xzZXI6cGFzcw==') + # default latin-1 encoding mw = HttpProxyMiddleware(auth_encoding='latin-1') req = Request('http://scrapytest.org') @@ -79,15 +98,21 @@ class TestDefaultHeadersMiddleware(TestCase): self.assertEquals(req.meta, {'proxy': 'https://proxy:3128'}) self.assertEquals(req.headers.get('Proxy-Authorization'), b'Basic beFuOnBhc3M=') + # proxy from request.meta, latin-1 encoding + req = Request('http://scrapytest.org', meta={'proxy': u'https://\u00FCser:pass@proxy:3128'}) + assert mw.process_request(req, spider) is None + self.assertEquals(req.meta, {'proxy': 'https://proxy:3128'}) + self.assertEquals(req.headers.get('Proxy-Authorization'), b'Basic /HNlcjpwYXNz') + def test_proxy_already_seted(self): - os.environ['http_proxy'] = http_proxy = 'https://proxy.for.http:3128' + os.environ['http_proxy'] = 'https://proxy.for.http:3128' mw = HttpProxyMiddleware() req = Request('http://noproxy.com', meta={'proxy': None}) assert mw.process_request(req, spider) is None assert 'proxy' in req.meta and req.meta['proxy'] is None def test_no_proxy(self): - os.environ['http_proxy'] = http_proxy = 'https://proxy.for.http:3128' + os.environ['http_proxy'] = 'https://proxy.for.http:3128' mw = HttpProxyMiddleware() os.environ['no_proxy'] = '*' @@ -104,3 +129,9 @@ class TestDefaultHeadersMiddleware(TestCase): req = Request('http://noproxy.com') assert mw.process_request(req, spider) is None assert 'proxy' not in req.meta + + # proxy from meta['proxy'] takes precedence + os.environ['no_proxy'] = '*' + req = Request('http://noproxy.com', meta={'proxy': 'http://proxy.com'}) + assert mw.process_request(req, spider) is None + self.assertEquals(req.meta, {'proxy': 'http://proxy.com'}) From ae0ea31abd4e65f7decb4482d1669d7af17bca0a Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Wed, 8 Feb 2017 13:21:10 -0300 Subject: [PATCH 2/2] Add HTTPPROXY_ENABLED setting (default True) --- docs/topics/downloader-middleware.rst | 15 +++++++++++---- scrapy/downloadermiddlewares/httpproxy.py | 3 +++ scrapy/settings/default_settings.py | 1 + tests/test_downloadermiddleware_httpproxy.py | 8 ++++++++ 4 files changed, 23 insertions(+), 4 deletions(-) diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index f0ff3c77c..0ef3fb071 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -681,10 +681,9 @@ HttpProxyMiddleware * ``no_proxy`` You can also set the meta key ``proxy`` per-request, to a value like - ``http://username:password@some_proxy_server:port``. Keep in mind - this value will take precedence over ``http_proxy``/``https_proxy`` - environment variables, and it will also ignore ``no_proxy`` environment - variable. + ``http://some_proxy_server:port`` or ``http://username:password@some_proxy_server:port``. + Keep in mind this value will take precedence over ``http_proxy``/``https_proxy`` + environment variables, and it will also ignore ``no_proxy`` environment variable. .. _urllib: https://docs.python.org/2/library/urllib.html .. _urllib2: https://docs.python.org/2/library/urllib2.html @@ -952,8 +951,16 @@ enable it for :ref:`broad crawls `. HttpProxyMiddleware settings ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +.. setting:: HTTPPROXY_ENABLED .. setting:: HTTPPROXY_AUTH_ENCODING +HTTPPROXY_ENABLED +^^^^^^^^^^^^^^^^^ + +Default: ``True`` + +Whether or not to enable the :class:`HttpProxyMiddleware`. + HTTPPROXY_AUTH_ENCODING ^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/scrapy/downloadermiddlewares/httpproxy.py b/scrapy/downloadermiddlewares/httpproxy.py index edc1c52ed..0d5320bf8 100644 --- a/scrapy/downloadermiddlewares/httpproxy.py +++ b/scrapy/downloadermiddlewares/httpproxy.py @@ -8,6 +8,7 @@ except ImportError: from six.moves.urllib.parse import urlunparse from scrapy.utils.httpobj import urlparse_cached +from scrapy.exceptions import NotConfigured from scrapy.utils.python import to_bytes @@ -21,6 +22,8 @@ class HttpProxyMiddleware(object): @classmethod def from_crawler(cls, crawler): + if not crawler.settings.getbool('HTTPPROXY_ENABLED'): + raise NotConfigured auth_encoding = crawler.settings.get('HTTPPROXY_AUTH_ENCODING') return cls(auth_encoding) diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 24714a7a8..cb88bc2bf 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -174,6 +174,7 @@ HTTPCACHE_DBM_MODULE = 'anydbm' if six.PY2 else 'dbm' HTTPCACHE_POLICY = 'scrapy.extensions.httpcache.DummyPolicy' HTTPCACHE_GZIP = False +HTTPPROXY_ENABLED = True HTTPPROXY_AUTH_ENCODING = 'latin-1' IMAGES_STORE_S3_ACL = 'private' diff --git a/tests/test_downloadermiddleware_httpproxy.py b/tests/test_downloadermiddleware_httpproxy.py index dd09e4dd0..c77179ceb 100644 --- a/tests/test_downloadermiddleware_httpproxy.py +++ b/tests/test_downloadermiddleware_httpproxy.py @@ -1,11 +1,14 @@ import os import sys +from functools import partial from twisted.trial.unittest import TestCase, SkipTest from scrapy.downloadermiddlewares.httpproxy import HttpProxyMiddleware from scrapy.exceptions import NotConfigured from scrapy.http import Response, Request from scrapy.spiders import Spider +from scrapy.crawler import Crawler +from scrapy.settings import Settings spider = Spider('foo') @@ -20,6 +23,11 @@ class TestDefaultHeadersMiddleware(TestCase): def tearDown(self): os.environ = self._oldenv + def test_not_enabled(self): + settings = Settings({'HTTPPROXY_ENABLED': False}) + crawler = Crawler(spider, settings) + self.assertRaises(NotConfigured, partial(HttpProxyMiddleware.from_crawler, crawler)) + def test_no_enviroment_proxies(self): os.environ = {'dummy_proxy': 'reset_env_and_do_not_raise'} mw = HttpProxyMiddleware()