diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index e90cb0223..64693f25b 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -247,6 +247,18 @@ HttpCompressionMiddleware This middleware allows compressed (gzip, deflate) traffic to be sent/received from web sites. +HttpProxyMiddleware +------------------- + +.. module:: scrapy.contrib.downloadermiddleware.httpproxy + :synopsis: Http Proxy Middleware + +.. class:: HttpProxyMiddleware + + This middleware sets proxy to use for requests, it obeys enviroment + variables 'http_proxy', 'https_proxy', and 'no_proxy' + + RedirectMiddleware ------------------- diff --git a/scrapy/conf/default_settings.py b/scrapy/conf/default_settings.py index e31ca2783..d79c1eda2 100644 --- a/scrapy/conf/default_settings.py +++ b/scrapy/conf/default_settings.py @@ -60,6 +60,7 @@ DOWNLOADER_MIDDLEWARES_BASE = { 'scrapy.contrib.downloadermiddleware.defaultheaders.DefaultHeadersMiddleware': 550, 'scrapy.contrib.downloadermiddleware.redirect.RedirectMiddleware': 600, 'scrapy.contrib.downloadermiddleware.cookies.CookiesMiddleware': 700, + 'scrapy.contrib.downloadermiddleware.httpproxy.HttpProxyMiddleware': 750, 'scrapy.contrib.downloadermiddleware.httpcompression.HttpCompressionMiddleware': 800, 'scrapy.contrib.downloadermiddleware.stats.DownloaderStats': 850, 'scrapy.contrib.downloadermiddleware.httpcache.HttpCacheMiddleware': 900, diff --git a/scrapy/contrib/downloadermiddleware/httpproxy.py b/scrapy/contrib/downloadermiddleware/httpproxy.py new file mode 100644 index 000000000..f2b531d28 --- /dev/null +++ b/scrapy/contrib/downloadermiddleware/httpproxy.py @@ -0,0 +1,51 @@ +import base64 +from urllib import getproxies, unquote, proxy_bypass +from urllib2 import _parse_proxy +from urlparse import urlunparse + +from scrapy.utils.httpobj import urlparse_cached +from scrapy.core.exceptions import NotConfigured + + +class HttpProxyMiddleware(object): + + def __init__(self): + self.proxies = {} + for type, url in getproxies().items(): + self.proxies[type] = self._get_proxy(url, type) + + if not self.proxies: + raise NotConfigured + + def _get_proxy(self, url, orig_type): + proxy_type, user, password, hostport = _parse_proxy(url) + proxy_url = urlunparse((proxy_type or orig_type, hostport, '', '', '', '')) + + if user and password: + user_pass = '%s:%s' % (unquote(user), unquote(password)) + creds = base64.b64encode(user_pass).strip() + else: + creds = None + + return creds, proxy_url + + def process_request(self, request, spider): + # ignore if proxy is already seted + if 'proxy' in request.meta: + return + + parsed = urlparse_cached(request) + scheme = parsed.scheme + + # 'no_proxy' is only supported by http schemes + if scheme in ('http', 'https') and proxy_bypass(parsed.hostname): + return + + if scheme in self.proxies: + self._set_proxy(request, scheme) + + def _set_proxy(self, request, scheme): + creds, proxy = self.proxies[scheme] + request.meta['proxy'] = proxy + if creds: + request.headers['Proxy-Authorization'] = 'Basic ' + creds diff --git a/scrapy/core/downloader/webclient.py b/scrapy/core/downloader/webclient.py index 6eaffb5b0..552fc6597 100644 --- a/scrapy/core/downloader/webclient.py +++ b/scrapy/core/downloader/webclient.py @@ -112,6 +112,10 @@ class ScrapyHTTPClientFactory(HTTPClientFactory): def _set_connection_attributes(self, request): parsed = urlparse_cached(request) self.scheme, self.netloc, self.host, self.port, self.path = _parsed_url_args(parsed) + proxy = request.meta.get('proxy') + if proxy: + self.scheme, _, self.host, self.port, _ = _parse(proxy) + self.path = self.url def gotHeaders(self, headers): self.response_headers = headers diff --git a/scrapy/tests/test_downloader_handlers.py b/scrapy/tests/test_downloader_handlers.py index 11fc3bbdf..f5bf63ec2 100644 --- a/scrapy/tests/test_downloader_handlers.py +++ b/scrapy/tests/test_downloader_handlers.py @@ -103,3 +103,47 @@ class HttpTestCase(unittest.TestCase): request = Request(self.getURL('broken')) d = download_http(request, BaseSpider()) return self.assertFailure(d, PartialDownloadError) + + +class UriResource(resource.Resource): + """Return the full uri that was requested""" + + def getChild(self, path, request): + return self + + def render(self, request): + return request.uri + + +class HttpProxyTestCase(unittest.TestCase): + + def setUp(self): + site = server.Site(UriResource(), timeout=None) + wrapper = WrappingFactory(site) + self.port = reactor.listenTCP(0, wrapper, interface='127.0.0.1') + self.portno = self.port.getHost().port + + def tearDown(self): + return self.port.stopListening() + + def getURL(self, path): + return "http://127.0.0.1:%d/%s" % (self.portno, path) + + def test_download_with_proxy(self): + def _test(response): + self.assertEquals(response.status, 200) + self.assertEquals(response.url, request.url) + self.assertEquals(response.body, 'https://example.com') + + http_proxy = self.getURL('') + request = Request('https://example.com', meta={'proxy': http_proxy}) + return download_http(request, BaseSpider()).addCallback(_test) + + def test_download_without_proxy(self): + def _test(response): + self.assertEquals(response.status, 200) + self.assertEquals(response.url, request.url) + self.assertEquals(response.body, '/path/to/resource') + + request = Request(self.getURL('path/to/resource')) + return download_http(request, BaseSpider()).addCallback(_test) diff --git a/scrapy/tests/test_downloadermiddleware_httpproxy.py b/scrapy/tests/test_downloadermiddleware_httpproxy.py new file mode 100644 index 000000000..c2fb46364 --- /dev/null +++ b/scrapy/tests/test_downloadermiddleware_httpproxy.py @@ -0,0 +1,81 @@ +import os +from unittest import TestCase + +from scrapy.contrib.downloadermiddleware.httpproxy import HttpProxyMiddleware +from scrapy.core.exceptions import NotConfigured +from scrapy.http import Response, Request +from scrapy.spider import BaseSpider +from scrapy.conf import settings + +spider = BaseSpider() + +class TestDefaultHeadersMiddleware(TestCase): + + def setUp(self): + self._oldenv = os.environ.copy() + + def tearDown(self): + os.environ = self._oldenv + + def test_no_proxies(self): + os.environ = {} + self.assertRaises(NotConfigured, HttpProxyMiddleware) + + def test_no_enviroment_proxies(self): + os.environ = {'dummy_proxy': 'reset_env_and_do_not_raise'} + mw = HttpProxyMiddleware() + + for url in ('http://e.com', 'https://e.com', 'file:///tmp/a'): + req = Request(url) + assert mw.process_request(req, spider) is None + self.assertEquals(req.url, url) + self.assertEquals(req.meta, {}) + + def test_enviroment_proxies(self): + os.environ['http_proxy'] = http_proxy = 'https://proxy.for.http:3128' + os.environ['https_proxy'] = https_proxy = 'http://proxy.for.https:8080' + os.environ.pop('file_proxy', None) + mw = HttpProxyMiddleware() + + for url, proxy in [('http://e.com', http_proxy), + ('https://e.com', https_proxy), ('file://tmp/a', None)]: + req = Request(url) + assert mw.process_request(req, spider) is None + self.assertEquals(req.url, url) + self.assertEquals(req.meta.get('proxy'), proxy) + + def test_proxy_auth(self): + os.environ['http_proxy'] = 'https://user:pass@proxy:3128' + mw = HttpProxyMiddleware() + req = Request('http://scrapytest.org') + assert mw.process_request(req, spider) is None + self.assertEquals(req.meta, {'proxy': 'https://proxy:3128'}) + self.assertEquals(req.headers.get('Proxy-Authorization'), 'Basic dXNlcjpwYXNz') + + def test_proxy_already_seted(self): + os.environ['http_proxy'] = http_proxy = 'https://proxy.for.http:3128' + mw = HttpProxyMiddleware() + req = Request('http://noproxy.com', meta={'proxy': None}) + assert mw.process_request(req, spider) is None + assert 'proxy' in req.meta and req.meta['proxy'] is None + + + def test_no_proxy(self): + os.environ['http_proxy'] = http_proxy = 'https://proxy.for.http:3128' + mw = HttpProxyMiddleware() + + os.environ['no_proxy'] = '*' + req = Request('http://noproxy.com') + assert mw.process_request(req, spider) is None + assert 'proxy' not in req.meta + + os.environ['no_proxy'] = 'other.com' + req = Request('http://noproxy.com') + assert mw.process_request(req, spider) is None + assert 'proxy' in req.meta + + os.environ['no_proxy'] = 'other.com,noproxy.com' + req = Request('http://noproxy.com') + assert mw.process_request(req, spider) is None + assert 'proxy' not in req.meta +