Fix and re-enable newer mitmproxy usage in tests.

tests/test_proxy_connect.py

@@ -31,6 +31,7 @@ sys.exit(mitmdump())
         self.proc = Popen(
             [
                 sys.executable,
+                "-u",
                 "-c",
                 script,
                 "--listen-host",
@@ -46,7 +47,7 @@ sys.exit(mitmdump())
             stdout=PIPE,
         )
         line = self.proc.stdout.readline().decode("utf-8")
-        host_port = re.search(r"listening at http://([^:]+:\d+)", line).group(1)
+        host_port = re.search(r"listening at (?:http://)?([^:]+:\d+)", line).group(1)
         address = f"http://{self.auth_user}:{self.auth_pass}@{host_port}"
         return address
 

tox.ini

@@ -11,11 +11,7 @@ minversion = 1.7.0
 deps =
     -rtests/requirements.txt
     # mitmproxy does not support PyPy
-    # Python 3.9+ requires mitmproxy >= 5.3.0
-    # mitmproxy >= 5.3.0 requires h2 >= 4.0, Twisted 21.2 requires h2 < 4.0
-    #mitmproxy >= 5.3.0; python_version >= '3.9' and implementation_name != 'pypy'
-    # The tests hang with mitmproxy 8.0.0: https://github.com/scrapy/scrapy/issues/5454
-    mitmproxy >= 4.0.4, < 8; python_version < '3.9' and implementation_name != 'pypy'
+    mitmproxy; implementation_name != 'pypy'
 passenv =
     S3_TEST_FILE_URI
     AWS_ACCESS_KEY_ID
@@ -87,7 +83,7 @@ deps =
     lxml==4.4.1
     -rtests/requirements.txt
 
-    # mitmproxy 4.0.4+ requires upgrading some of the pinned dependencies
+    # mitmproxy 8.0.0 requires upgrading some of the pinned dependencies
     # above, hence we do not install it in pinned environments at the moment
 setenv =
     _SCRAPY_PINNED=true