Merge pull request #4778 from drs-11/unparseable-no-proxy-fix

Check for unparseable no_proxy values

scrapy/downloadermiddlewares/httpproxy.py

@@ -13,7 +13,12 @@ class HttpProxyMiddleware:
         self.auth_encoding = auth_encoding
         self.proxies = {}
         for type_, url in getproxies().items():
-            self.proxies[type_] = self._get_proxy(url, type_)
+            try:
+                self.proxies[type_] = self._get_proxy(url, type_)
+            # some values such as '/var/run/docker.sock' can't be parsed
+            # by _parse_proxy and as such should be skipped
+            except ValueError:
+                continue
 
     @classmethod
     def from_crawler(cls, crawler):

tests/test_downloadermiddleware_httpproxy.py

@@ -145,3 +145,10 @@ class TestHttpProxyMiddleware(TestCase):
         req = Request('http://noproxy.com', meta={'proxy': 'http://proxy.com'})
         assert mw.process_request(req, spider) is None
         self.assertEqual(req.meta, {'proxy': 'http://proxy.com'})
+
+    def test_no_proxy_invalid_values(self):
+        os.environ['no_proxy'] = '/var/run/docker.sock'
+        mw = HttpProxyMiddleware()
+        # '/var/run/docker.sock' may be used by the user for
+        # no_proxy value but is not parseable and should be skipped
+        assert 'no' not in mw.proxies