Fix proxy handling on redirect to different scheme

scrapy/downloadermiddlewares/httpproxy.py

@@ -44,7 +44,7 @@ class HttpProxyMiddleware:
         return creds, proxy_url
 
     def process_request(self, request, spider):
-        creds, proxy_url = None, None
+        creds, proxy_url, scheme = None, None, None
         if "proxy" in request.meta:
             if request.meta["proxy"] is not None:
                 creds, proxy_url = self._get_proxy(request.meta["proxy"], "")
@@ -58,9 +58,11 @@ class HttpProxyMiddleware:
             ) and scheme in self.proxies:
                 creds, proxy_url = self.proxies[scheme]
 
-        self._set_proxy_and_creds(request, proxy_url, creds)
+        self._set_proxy_and_creds(request, proxy_url, creds, scheme)
 
-    def _set_proxy_and_creds(self, request, proxy_url, creds):
+    def _set_proxy_and_creds(self, request, proxy_url, creds, scheme):
+        if scheme:
+            request.meta["_scheme_proxy"] = True
         if proxy_url:
             request.meta["proxy"] = proxy_url
         elif request.meta.get("proxy") is not None:

scrapy/downloadermiddlewares/redirect.py

@@ -17,6 +17,14 @@ def _build_redirect_request(source_request, *, url, **kwargs):
         **kwargs,
         cookies=None,
     )
+    if "_scheme_proxy" in redirect_request.meta:
+        source_request_scheme = urlparse_cached(source_request).scheme
+        redirect_request_scheme = urlparse_cached(redirect_request).scheme
+        if source_request_scheme != redirect_request_scheme:
+            redirect_request.meta.pop("_scheme_proxy")
+            redirect_request.meta.pop("proxy", None)
+            redirect_request.meta.pop("_auth_proxy", None)
+            redirect_request.headers.pop(b"Proxy-Authorization", None)
     has_cookie_header = "Cookie" in redirect_request.headers
     has_authorization_header = "Authorization" in redirect_request.headers
     if has_cookie_header or has_authorization_header:

tests/test_utils_project.py

@@ -6,6 +6,7 @@ import unittest
 import warnings
 from pathlib import Path
 
+from scrapy.utils.misc import set_environ
 from scrapy.utils.project import data_path, get_project_settings
 
 
@@ -38,20 +39,6 @@ class ProjectUtilsTest(unittest.TestCase):
             self.assertEqual(abspath, data_path(abspath))
 
 
-@contextlib.contextmanager
-def set_env(**update):
-    modified = set(update.keys()) & set(os.environ.keys())
-    update_after = {k: os.environ[k] for k in modified}
-    remove_after = frozenset(k for k in update if k not in os.environ)
-    try:
-        os.environ.update(update)
-        yield
-    finally:
-        os.environ.update(update_after)
-        for k in remove_after:
-            os.environ.pop(k)
-
-
 class GetProjectSettingsTestCase(unittest.TestCase):
     def test_valid_envvar(self):
         value = "tests.test_cmdline.settings"
@@ -60,7 +47,7 @@ class GetProjectSettingsTestCase(unittest.TestCase):
         }
         with warnings.catch_warnings():
             warnings.simplefilter("error")
-            with set_env(**envvars):
+            with set_environ(**envvars):
                 settings = get_project_settings()
 
         assert settings.get("SETTINGS_MODULE") == value
@@ -69,7 +56,7 @@ class GetProjectSettingsTestCase(unittest.TestCase):
         envvars = {
             "SCRAPY_FOO": "bar",
         }
-        with set_env(**envvars):
+        with set_environ(**envvars):
             settings = get_project_settings()
 
         assert settings.get("SCRAPY_FOO") is None
@@ -80,7 +67,7 @@ class GetProjectSettingsTestCase(unittest.TestCase):
             "SCRAPY_FOO": "bar",
             "SCRAPY_SETTINGS_MODULE": value,
         }
-        with set_env(**envvars):
+        with set_environ(**envvars):
             settings = get_project_settings()
         assert settings.get("SETTINGS_MODULE") == value
         assert settings.get("SCRAPY_FOO") is None