RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #2530 from elacuesta/proxy_credentials 

[MRG+1] Use credentials from request.meta['proxy']
This commit is contained in:
Mikhail Korobov 2017-02-14 20:58:39 +06:00 committed by GitHub
parent 7dd7646e65 ae0ea31abd
commit afac3fd2c2
4 changed files with 74 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
docs/topics/downloader-middleware.rst
View File

@ -681,7 +681,9 @@ HttpProxyMiddleware
* ``no_proxy``
You can also set the meta key ``proxy`` per-request, to a value like
``http://some_proxy_server:port``.
``http://some_proxy_server:port`` or ``http://username:password@some_proxy_server:port``.
Keep in mind this value will take precedence over ``http_proxy``/``https_proxy``
environment variables, and it will also ignore ``no_proxy`` environment variable.
.. _urllib: https://docs.python.org/2/library/urllib.html
.. _urllib2: https://docs.python.org/2/library/urllib2.html
@ -949,8 +951,16 @@ enable it for :ref:`broad crawls <topics-broad-crawls>`.
HttpProxyMiddleware settings
~~~~~~~~~~~~~~~~~~~~~~~~~~~~
.. setting:: HTTPPROXY_ENABLED
.. setting:: HTTPPROXY_AUTH_ENCODING
HTTPPROXY_ENABLED
^^^^^^^^^^^^^^^^^
Default: ``True``
Whether or not to enable the :class:`HttpProxyMiddleware`.
HTTPPROXY_AUTH_ENCODING
^^^^^^^^^^^^^^^^^^^^^^^

25
scrapy/downloadermiddlewares/httpproxy.py
View File

@ -20,23 +20,25 @@ class HttpProxyMiddleware(object):
for type, url in getproxies().items():
self.proxies[type] = self._get_proxy(url, type)
if not self.proxies:
raise NotConfigured
@classmethod
def from_crawler(cls, crawler):
if not crawler.settings.getbool('HTTPPROXY_ENABLED'):
raise NotConfigured
auth_encoding = crawler.settings.get('HTTPPROXY_AUTH_ENCODING')
return cls(auth_encoding)
def _basic_auth_header(self, username, password):
user_pass = to_bytes(
'%s:%s' % (unquote(username), unquote(password)),
encoding=self.auth_encoding)
return base64.b64encode(user_pass).strip()
def _get_proxy(self, url, orig_type):
proxy_type, user, password, hostport = _parse_proxy(url)
proxy_url = urlunparse((proxy_type or orig_type, hostport, '', '', '', ''))
if user:
user_pass = to_bytes(
'%s:%s' % (unquote(user), unquote(password)),
encoding=self.auth_encoding)
creds = base64.b64encode(user_pass).strip()
creds = self._basic_auth_header(user, password)
else:
creds = None
@ -45,6 +47,15 @@ class HttpProxyMiddleware(object):
def process_request(self, request, spider):
# ignore if proxy is already set
if 'proxy' in request.meta:
if request.meta['proxy'] is None:
return
# extract credentials if present
creds, proxy_url = self._get_proxy(request.meta['proxy'], '')
request.meta['proxy'] = proxy_url
if creds and not request.headers.get('Proxy-Authorization'):
request.headers['Proxy-Authorization'] = b'Basic ' + creds
return
elif not self.proxies:
return
parsed = urlparse_cached(request)

1
scrapy/settings/default_settings.py
View File

@ -174,6 +174,7 @@ HTTPCACHE_DBM_MODULE = 'anydbm' if six.PY2 else 'dbm'
HTTPCACHE_POLICY = 'scrapy.extensions.httpcache.DummyPolicy'
HTTPCACHE_GZIP = False
HTTPPROXY_ENABLED = True
HTTPPROXY_AUTH_ENCODING = 'latin-1'
IMAGES_STORE_S3_ACL = 'private'

49
tests/test_downloadermiddleware_httpproxy.py
View File

@ -1,11 +1,14 @@
import os
import sys
from functools import partial
from twisted.trial.unittest import TestCase, SkipTest
from scrapy.downloadermiddlewares.httpproxy import HttpProxyMiddleware
from scrapy.exceptions import NotConfigured
from scrapy.http import Response, Request
from scrapy.spiders import Spider
from scrapy.crawler import Crawler
from scrapy.settings import Settings
spider = Spider('foo')
@ -20,9 +23,10 @@ class TestDefaultHeadersMiddleware(TestCase):
def tearDown(self):
os.environ = self._oldenv
def test_no_proxies(self):
os.environ = {}
self.assertRaises(NotConfigured, HttpProxyMiddleware)
def test_not_enabled(self):
settings = Settings({'HTTPPROXY_ENABLED': False})
crawler = Crawler(spider, settings)
self.assertRaises(NotConfigured, partial(HttpProxyMiddleware.from_crawler, crawler))
def test_no_enviroment_proxies(self):
os.environ = {'dummy_proxy': 'reset_env_and_do_not_raise'}
@ -47,6 +51,13 @@ class TestDefaultHeadersMiddleware(TestCase):
self.assertEquals(req.url, url)
self.assertEquals(req.meta.get('proxy'), proxy)
def test_proxy_precedence_meta(self):
os.environ['http_proxy'] = 'https://proxy.com'
mw = HttpProxyMiddleware()
req = Request('http://scrapytest.org', meta={'proxy': 'https://new.proxy:3128'})
assert mw.process_request(req, spider) is None
self.assertEquals(req.meta, {'proxy': 'https://new.proxy:3128'})
def test_proxy_auth(self):
os.environ['http_proxy'] = 'https://user:pass@proxy:3128'
mw = HttpProxyMiddleware()
@ -54,6 +65,11 @@ class TestDefaultHeadersMiddleware(TestCase):
assert mw.process_request(req, spider) is None
self.assertEquals(req.meta, {'proxy': 'https://proxy:3128'})
self.assertEquals(req.headers.get('Proxy-Authorization'), b'Basic dXNlcjpwYXNz')
# proxy from request.meta
req = Request('http://scrapytest.org', meta={'proxy': 'https://username:password@proxy:3128'})
assert mw.process_request(req, spider) is None
self.assertEquals(req.meta, {'proxy': 'https://proxy:3128'})
self.assertEquals(req.headers.get('Proxy-Authorization'), b'Basic dXNlcm5hbWU6cGFzc3dvcmQ=')
def test_proxy_auth_empty_passwd(self):
os.environ['http_proxy'] = 'https://user:@proxy:3128'
@ -62,6 +78,11 @@ class TestDefaultHeadersMiddleware(TestCase):
assert mw.process_request(req, spider) is None
self.assertEquals(req.meta, {'proxy': 'https://proxy:3128'})
self.assertEquals(req.headers.get('Proxy-Authorization'), b'Basic dXNlcjo=')
# proxy from request.meta
req = Request('http://scrapytest.org', meta={'proxy': 'https://username:@proxy:3128'})
assert mw.process_request(req, spider) is None
self.assertEquals(req.meta, {'proxy': 'https://proxy:3128'})
self.assertEquals(req.headers.get('Proxy-Authorization'), b'Basic dXNlcm5hbWU6')
def test_proxy_auth_encoding(self):
# utf-8 encoding
@ -72,6 +93,12 @@ class TestDefaultHeadersMiddleware(TestCase):
self.assertEquals(req.meta, {'proxy': 'https://proxy:3128'})
self.assertEquals(req.headers.get('Proxy-Authorization'), b'Basic bcOhbjpwYXNz')
# proxy from request.meta
req = Request('http://scrapytest.org', meta={'proxy': u'https://\u00FCser:pass@proxy:3128'})
assert mw.process_request(req, spider) is None
self.assertEquals(req.meta, {'proxy': 'https://proxy:3128'})
self.assertEquals(req.headers.get('Proxy-Authorization'), b'Basic w7xzZXI6cGFzcw==')
# default latin-1 encoding
mw = HttpProxyMiddleware(auth_encoding='latin-1')
req = Request('http://scrapytest.org')
@ -79,15 +106,21 @@ class TestDefaultHeadersMiddleware(TestCase):
self.assertEquals(req.meta, {'proxy': 'https://proxy:3128'})
self.assertEquals(req.headers.get('Proxy-Authorization'), b'Basic beFuOnBhc3M=')
# proxy from request.meta, latin-1 encoding
req = Request('http://scrapytest.org', meta={'proxy': u'https://\u00FCser:pass@proxy:3128'})
assert mw.process_request(req, spider) is None
self.assertEquals(req.meta, {'proxy': 'https://proxy:3128'})
self.assertEquals(req.headers.get('Proxy-Authorization'), b'Basic /HNlcjpwYXNz')
def test_proxy_already_seted(self):
os.environ['http_proxy'] = http_proxy = 'https://proxy.for.http:3128'
os.environ['http_proxy'] = 'https://proxy.for.http:3128'
mw = HttpProxyMiddleware()
req = Request('http://noproxy.com', meta={'proxy': None})
assert mw.process_request(req, spider) is None
assert 'proxy' in req.meta and req.meta['proxy'] is None
def test_no_proxy(self):
os.environ['http_proxy'] = http_proxy = 'https://proxy.for.http:3128'
os.environ['http_proxy'] = 'https://proxy.for.http:3128'
mw = HttpProxyMiddleware()
os.environ['no_proxy'] = '*'
@ -104,3 +137,9 @@ class TestDefaultHeadersMiddleware(TestCase):
req = Request('http://noproxy.com')
assert mw.process_request(req, spider) is None
assert 'proxy' not in req.meta
# proxy from meta['proxy'] takes precedence
os.environ['no_proxy'] = '*'
req = Request('http://noproxy.com', meta={'proxy': 'http://proxy.com'})
assert mw.process_request(req, spider) is None
self.assertEquals(req.meta, {'proxy': 'http://proxy.com'})