Add support to set http proxies per request, and obey enviroment variables http_proxy and no_proxy by default.

docs/topics/downloader-middleware.rst

@@ -247,6 +247,18 @@ HttpCompressionMiddleware
    This middleware allows compressed (gzip, deflate) traffic to be
    sent/received from web sites.
 
+HttpProxyMiddleware
+-------------------
+
+.. module:: scrapy.contrib.downloadermiddleware.httpproxy
+   :synopsis: Http Proxy Middleware
+
+.. class:: HttpProxyMiddleware
+
+   This middleware sets proxy to use for requests, it obeys enviroment
+   variables 'http_proxy', 'https_proxy', and 'no_proxy'
+
+
 RedirectMiddleware
 -------------------
 

scrapy/conf/default_settings.py

@@ -60,6 +60,7 @@ DOWNLOADER_MIDDLEWARES_BASE = {
     'scrapy.contrib.downloadermiddleware.defaultheaders.DefaultHeadersMiddleware': 550,
     'scrapy.contrib.downloadermiddleware.redirect.RedirectMiddleware': 600,
     'scrapy.contrib.downloadermiddleware.cookies.CookiesMiddleware': 700,
+    'scrapy.contrib.downloadermiddleware.httpproxy.HttpProxyMiddleware': 750,
     'scrapy.contrib.downloadermiddleware.httpcompression.HttpCompressionMiddleware': 800,
     'scrapy.contrib.downloadermiddleware.stats.DownloaderStats': 850,
     'scrapy.contrib.downloadermiddleware.httpcache.HttpCacheMiddleware': 900,

scrapy/contrib/downloadermiddleware/httpproxy.py

@@ -0,0 +1,51 @@
+import base64
+from urllib import getproxies, unquote, proxy_bypass
+from urllib2 import _parse_proxy
+from urlparse import urlunparse
+
+from scrapy.utils.httpobj import urlparse_cached
+from scrapy.core.exceptions import NotConfigured
+
+
+class HttpProxyMiddleware(object):
+
+    def __init__(self):
+        self.proxies = {}
+        for type, url in getproxies().items():
+            self.proxies[type] = self._get_proxy(url, type)
+
+        if not self.proxies:
+            raise NotConfigured
+
+    def _get_proxy(self, url, orig_type):
+        proxy_type, user, password, hostport = _parse_proxy(url)
+        proxy_url = urlunparse((proxy_type or orig_type, hostport, '', '', '', ''))
+
+        if user and password:
+            user_pass = '%s:%s' % (unquote(user), unquote(password))
+            creds = base64.b64encode(user_pass).strip()
+        else:
+            creds = None
+
+        return creds, proxy_url
+
+    def process_request(self, request, spider):
+        # ignore if proxy is already seted
+        if 'proxy' in request.meta:
+            return
+
+        parsed = urlparse_cached(request)
+        scheme = parsed.scheme
+
+        # 'no_proxy' is only supported by http schemes
+        if scheme in ('http', 'https') and proxy_bypass(parsed.hostname):
+            return
+
+        if scheme in self.proxies:
+            self._set_proxy(request, scheme)
+
+    def _set_proxy(self, request, scheme):
+        creds, proxy = self.proxies[scheme]
+        request.meta['proxy'] = proxy
+        if creds:
+            request.headers['Proxy-Authorization'] = 'Basic ' + creds

scrapy/core/downloader/webclient.py

@@ -112,6 +112,10 @@ class ScrapyHTTPClientFactory(HTTPClientFactory):
     def _set_connection_attributes(self, request):
         parsed = urlparse_cached(request)
         self.scheme, self.netloc, self.host, self.port, self.path = _parsed_url_args(parsed)
+        proxy = request.meta.get('proxy')
+        if proxy:
+            self.scheme, _, self.host, self.port, _ = _parse(proxy)
+            self.path = self.url
 
     def gotHeaders(self, headers):
         self.response_headers = headers

scrapy/tests/test_downloader_handlers.py

@@ -103,3 +103,47 @@ class HttpTestCase(unittest.TestCase):
         request = Request(self.getURL('broken'))
         d = download_http(request, BaseSpider())
         return self.assertFailure(d, PartialDownloadError)
+
+
+class UriResource(resource.Resource):
+    """Return the full uri that was requested"""
+
+    def getChild(self, path, request):
+        return self
+
+    def render(self, request):
+        return request.uri
+
+
+class HttpProxyTestCase(unittest.TestCase):
+
+    def setUp(self):
+        site = server.Site(UriResource(), timeout=None)
+        wrapper = WrappingFactory(site)
+        self.port = reactor.listenTCP(0, wrapper, interface='127.0.0.1')
+        self.portno = self.port.getHost().port
+
+    def tearDown(self):
+        return self.port.stopListening()
+
+    def getURL(self, path):
+        return "http://127.0.0.1:%d/%s" % (self.portno, path)
+
+    def test_download_with_proxy(self):
+        def _test(response):
+            self.assertEquals(response.status, 200)
+            self.assertEquals(response.url, request.url)
+            self.assertEquals(response.body, 'https://example.com')
+
+        http_proxy = self.getURL('')
+        request = Request('https://example.com', meta={'proxy': http_proxy})
+        return download_http(request, BaseSpider()).addCallback(_test)
+
+    def test_download_without_proxy(self):
+        def _test(response):
+            self.assertEquals(response.status, 200)
+            self.assertEquals(response.url, request.url)
+            self.assertEquals(response.body, '/path/to/resource')
+
+        request = Request(self.getURL('path/to/resource'))
+        return download_http(request, BaseSpider()).addCallback(_test)

scrapy/tests/test_downloadermiddleware_httpproxy.py

@@ -0,0 +1,81 @@
+import os
+from unittest import TestCase
+
+from scrapy.contrib.downloadermiddleware.httpproxy import HttpProxyMiddleware
+from scrapy.core.exceptions import NotConfigured
+from scrapy.http import Response, Request
+from scrapy.spider import BaseSpider
+from scrapy.conf import settings
+
+spider = BaseSpider()
+
+class TestDefaultHeadersMiddleware(TestCase):
+
+    def setUp(self):
+        self._oldenv = os.environ.copy()
+
+    def tearDown(self):
+        os.environ = self._oldenv
+
+    def test_no_proxies(self):
+        os.environ = {}
+        self.assertRaises(NotConfigured, HttpProxyMiddleware)
+
+    def test_no_enviroment_proxies(self):
+        os.environ = {'dummy_proxy': 'reset_env_and_do_not_raise'}
+        mw = HttpProxyMiddleware()
+
+        for url in ('http://e.com', 'https://e.com', 'file:///tmp/a'):
+            req = Request(url)
+            assert mw.process_request(req, spider) is None
+            self.assertEquals(req.url, url)
+            self.assertEquals(req.meta, {})
+
+    def test_enviroment_proxies(self):
+        os.environ['http_proxy'] = http_proxy = 'https://proxy.for.http:3128'
+        os.environ['https_proxy'] = https_proxy = 'http://proxy.for.https:8080'
+        os.environ.pop('file_proxy', None)
+        mw = HttpProxyMiddleware()
+
+        for url, proxy in [('http://e.com', http_proxy),
+                ('https://e.com', https_proxy), ('file://tmp/a', None)]:
+            req = Request(url)
+            assert mw.process_request(req, spider) is None
+            self.assertEquals(req.url, url)
+            self.assertEquals(req.meta.get('proxy'), proxy)
+
+    def test_proxy_auth(self):
+        os.environ['http_proxy'] = 'https://user:pass@proxy:3128'
+        mw = HttpProxyMiddleware()
+        req = Request('http://scrapytest.org')
+        assert mw.process_request(req, spider) is None
+        self.assertEquals(req.meta, {'proxy': 'https://proxy:3128'})
+        self.assertEquals(req.headers.get('Proxy-Authorization'), 'Basic dXNlcjpwYXNz')
+
+    def test_proxy_already_seted(self):
+        os.environ['http_proxy'] = http_proxy = 'https://proxy.for.http:3128'
+        mw = HttpProxyMiddleware()
+        req = Request('http://noproxy.com', meta={'proxy': None})
+        assert mw.process_request(req, spider) is None
+        assert 'proxy' in req.meta and req.meta['proxy'] is None
+
+
+    def test_no_proxy(self):
+        os.environ['http_proxy'] = http_proxy = 'https://proxy.for.http:3128'
+        mw = HttpProxyMiddleware()
+
+        os.environ['no_proxy'] = '*'
+        req = Request('http://noproxy.com')
+        assert mw.process_request(req, spider) is None
+        assert 'proxy' not in req.meta
+
+        os.environ['no_proxy'] = 'other.com'
+        req = Request('http://noproxy.com')
+        assert mw.process_request(req, spider) is None
+        assert 'proxy' in req.meta
+
+        os.environ['no_proxy'] = 'other.com,noproxy.com'
+        req = Request('http://noproxy.com')
+        assert mw.process_request(req, spider) is None
+        assert 'proxy' not in req.meta
+