diff --git a/scrapy/downloadermiddlewares/httpproxy.py b/scrapy/downloadermiddlewares/httpproxy.py index 04da11311..d2665b655 100644 --- a/scrapy/downloadermiddlewares/httpproxy.py +++ b/scrapy/downloadermiddlewares/httpproxy.py @@ -13,7 +13,12 @@ class HttpProxyMiddleware: self.auth_encoding = auth_encoding self.proxies = {} for type_, url in getproxies().items(): - self.proxies[type_] = self._get_proxy(url, type_) + try: + self.proxies[type_] = self._get_proxy(url, type_) + # some values such as '/var/run/docker.sock' can't be parsed + # by _parse_proxy and as such should be skipped + except ValueError: + continue @classmethod def from_crawler(cls, crawler): diff --git a/tests/test_downloadermiddleware_httpproxy.py b/tests/test_downloadermiddleware_httpproxy.py index 351631eb8..81d6cc335 100644 --- a/tests/test_downloadermiddleware_httpproxy.py +++ b/tests/test_downloadermiddleware_httpproxy.py @@ -123,7 +123,11 @@ class TestHttpProxyMiddleware(TestCase): def test_no_proxy(self): os.environ['http_proxy'] = 'https://proxy.for.http:3128' + os.environ['no_proxy'] = '/var/run/docker.sock' mw = HttpProxyMiddleware() + # '/var/run/docker.sock' may be used by the user for + # no_proxy value but is not parseable and should be skipped + assert 'no' not in mw.proxies os.environ['no_proxy'] = '*' req = Request('http://noproxy.com')