Merge branch 'tox-pip20.2' into tests-github-actions

.github/workflows/main.yml

@@ -39,11 +39,11 @@ jobs:
           python-version: 3.8
           env:
             TOXENV: py
-        # - os: ubuntu-18.04
-        #   python-version: pypy3
-        #   env:
-        #     TOXENV: pypy3
-        #     PYPY_VERSION: 3.6-v7.3.1
+        - os: ubuntu-18.04
+          python-version: pypy3
+          env:
+            TOXENV: pypy3
+            PYPY_VERSION: 3.6-v7.3.1
 
         # windows tests
         - os: windows-latest
@@ -68,11 +68,11 @@ jobs:
           python-version: 3.6
           env:
             TOXENV: windows-pinned
-        # - os: ubuntu-18.04
-        #   python-version: pypy3
-        #   env:
-        #     TOXENV: pypy3-pinned
-        #     PYPY_VERSION: 3.6-v7.2.0
+        - os: ubuntu-18.04
+          python-version: pypy3
+          env:
+            TOXENV: pypy3-pinned
+            PYPY_VERSION: 3.6-v7.2.0
 
         # extras
         - os: ubuntu-18.04

setup.py

@@ -24,7 +24,6 @@ install_requires = [
     'cssselect>=0.9.1',
     'itemloaders>=1.0.1',
     'parsel>=1.5.0',
-    'PyDispatcher>=2.0.5',
     'pyOpenSSL>=17.5.0',
     'queuelib>=1.4.2',
     'service_identity>=16.0.0',
@@ -34,11 +33,12 @@ install_requires = [
     'itemadapter>=0.1.0',
 ]
 extras_require = {}
-
+cpython_dependencies = [
+    'lxml>=3.5.0',
+    'PyDispatcher>=2.0.5',
+]
 if has_environment_marker_platform_impl_support():
-    extras_require[':platform_python_implementation == "CPython"'] = [
-        'lxml>=3.5.0',
-    ]
+    extras_require[':platform_python_implementation == "CPython"'] = cpython_dependencies
     extras_require[':platform_python_implementation == "PyPy"'] = [
         # Earlier lxml versions are affected by
         # https://foss.heptapod.net/pypy/pypy/-/issues/2498,
@@ -49,7 +49,7 @@ if has_environment_marker_platform_impl_support():
         'PyPyDispatcher>=2.1.0',
     ]
 else:
-    install_requires.append('lxml>=3.5.0')
+    install_requires.extend(cpython_dependencies)
 
 
 setup(

tests/requirements-py3.txt

@@ -1,8 +1,6 @@
 # Tests requirements
 attrs
 dataclasses; python_version == '3.6'
-mitmproxy >= 4.0.4, < 5; python_version >= '3.6' and python_version < '3.7'
-mitmproxy >= 4.0.4; python_version >= '3.7'
 pyftpdlib
 # https://github.com/pytest-dev/pytest-twisted/issues/93
 pytest != 5.4, != 5.4.1

tests/test_dependencies.py

@@ -1,8 +1,14 @@
+import os
+import re
+from configparser import ConfigParser
 from importlib import import_module
+
+from twisted import version as twisted_version
 from twisted.trial import unittest
 
 
 class ScrapyUtilsTest(unittest.TestCase):
+
     def test_required_openssl_version(self):
         try:
             module = import_module('OpenSSL')
@@ -13,6 +19,32 @@ class ScrapyUtilsTest(unittest.TestCase):
             installed_version = [int(x) for x in module.__version__.split('.')[:2]]
             assert installed_version >= [0, 6], "OpenSSL >= 0.6 required"
 
+    def test_pinned_twisted_version(self):
+        """When running tests within a Tox environment with pinned
+        dependencies, make sure that the version of Twisted is the pinned
+        version.
+
+        See https://github.com/scrapy/scrapy/pull/4814#issuecomment-706230011
+        """
+        if not os.environ.get('_SCRAPY_PINNED', None):
+            self.skipTest('Not in a pinned environment')
+
+        tox_config_file_path = os.path.join(
+            os.path.dirname(__file__),
+            '..',
+            'tox.ini',
+        )
+        config_parser = ConfigParser()
+        config_parser.read(tox_config_file_path)
+        pattern = r'Twisted==([\d.]+)'
+        match = re.search(pattern, config_parser['pinned']['deps'])
+        pinned_twisted_version_string = match[1]
+
+        self.assertEqual(
+            twisted_version.short(),
+            pinned_twisted_version_string
+        )
+
 
 if __name__ == "__main__":
     unittest.main()

tests/test_downloadermiddleware.py

@@ -1,11 +1,13 @@
 import asyncio
-from unittest import mock
+from unittest import mock, SkipTest
 
 from pytest import mark
+from twisted import version as twisted_version
 from twisted.internet import defer
 from twisted.internet.defer import Deferred
 from twisted.trial.unittest import TestCase
 from twisted.python.failure import Failure
+from twisted.python.versions import Version
 
 from scrapy.http import Request, Response
 from scrapy.spiders import Spider
@@ -211,10 +213,21 @@ class MiddlewareUsingDeferreds(ManagerTestCase):
         self.assertFalse(download_func.called)
 
 
+@mark.usefixtures('reactor_pytest')
 class MiddlewareUsingCoro(ManagerTestCase):
     """Middlewares using asyncio coroutines should work"""
 
     def test_asyncdef(self):
+        if (
+            self.reactor_pytest == 'asyncio'
+            and twisted_version < Version('twisted', 18, 4, 0)
+        ):
+            raise SkipTest(
+                'Due to https://twistedmatrix.com/trac/ticket/9390, this test '
+                'hangs when using AsyncIO and Twisted versions lower than '
+                '18.4.0'
+            )
+
         resp = Response('http://example.com/index.html')
 
         class CoroMiddleware:
@@ -235,6 +248,12 @@ class MiddlewareUsingCoro(ManagerTestCase):
 
     @mark.only_asyncio()
     def test_asyncdef_asyncio(self):
+        if twisted_version < Version('twisted', 18, 4, 0):
+            raise SkipTest(
+                'Due to https://twistedmatrix.com/trac/ticket/9390, this test '
+                'hangs when using Twisted versions lower than 18.4.0'
+            )
+
         resp = Response('http://example.com/index.html')
 
         class CoroMiddleware:

tests/test_proxy_connect.py

@@ -1,12 +1,9 @@
 import json
 import os
-import platform
 import re
 import sys
 from subprocess import Popen, PIPE
 from urllib.parse import urlsplit, urlunsplit
-from unittest import skipIf
-
 from testfixtures import LogCapture
 from twisted.internet import defer
 from twisted.trial.unittest import TestCase
@@ -57,13 +54,14 @@ def _wrong_credentials(proxy_url):
     return urlunsplit(bad_auth_proxy)
 
 
-@skipIf("pypy" in sys.executable,
-        "mitmproxy does not support PyPy")
-@skipIf(platform.system() == 'Windows' and sys.version_info < (3, 7),
-        "mitmproxy does not support Windows when running Python < 3.7")
 class ProxyConnectTestCase(TestCase):
 
     def setUp(self):
+        try:
+            import mitmproxy  # noqa: F401
+        except ImportError:
+            self.skipTest('mitmproxy is not installed')
+
         self.mockserver = MockServer()
         self.mockserver.__enter__()
         self._oldenv = os.environ.copy()

tests/test_utils_signal.py

@@ -1,11 +1,14 @@
 import asyncio
+from unittest import SkipTest
 
+from pydispatch import dispatcher
 from pytest import mark
 from testfixtures import LogCapture
-from twisted.trial import unittest
-from twisted.python.failure import Failure
+from twisted import version as twisted_version
 from twisted.internet import defer, reactor
-from pydispatch import dispatcher
+from twisted.python.failure import Failure
+from twisted.python.versions import Version
+from twisted.trial import unittest
 
 from scrapy.utils.signal import send_catch_log, send_catch_log_deferred
 from scrapy.utils.test import get_from_asyncio_queue
@@ -68,6 +71,7 @@ class SendCatchLogDeferredTest2(SendCatchLogDeferredTest):
         return d
 
 
+@mark.usefixtures('reactor_pytest')
 class SendCatchLogDeferredAsyncDefTest(SendCatchLogDeferredTest):
 
     async def ok_handler(self, arg, handlers_called):
@@ -76,6 +80,19 @@ class SendCatchLogDeferredAsyncDefTest(SendCatchLogDeferredTest):
         await defer.succeed(42)
         return "OK"
 
+    def test_send_catch_log(self):
+        if (
+            self.reactor_pytest == 'asyncio'
+            and twisted_version < Version('twisted', 18, 4, 0)
+        ):
+            raise SkipTest(
+                'Due to https://twistedmatrix.com/trac/ticket/9390, this test '
+                'fails due to a timeout when using AsyncIO and Twisted '
+                'versions lower than 18.4.0'
+            )
+
+        return super().test_send_catch_log()
+
 
 @mark.only_asyncio()
 class SendCatchLogDeferredAsyncioTest(SendCatchLogDeferredTest):
@@ -86,6 +103,16 @@ class SendCatchLogDeferredAsyncioTest(SendCatchLogDeferredTest):
         await asyncio.sleep(0.2)
         return await get_from_asyncio_queue("OK")
 
+    def test_send_catch_log(self):
+        if twisted_version < Version('twisted', 18, 4, 0):
+            raise SkipTest(
+                'Due to https://twistedmatrix.com/trac/ticket/9390, this test '
+                'fails due to a timeout when using Twisted versions lower '
+                'than 18.4.0'
+            )
+
+        return super().test_send_catch_log()
+
 
 class SendCatchLogTest2(unittest.TestCase):
 

tox.ini

@@ -11,6 +11,10 @@ minversion = 1.7.0
 deps =
     -ctests/constraints.txt
     -rtests/requirements-py3.txt
+    # mitmproxy does not support PyPy
+    # mitmproxy does not support Windows when running Python < 3.7
+    mitmproxy >= 4.0.4; python_version >= '3.7' and implementation_name != 'pypy'
+    mitmproxy >= 4.0.4, < 5; python_version >= '3.6' and python_version < '3.7' and platform_system != 'Windows' and implementation_name != 'pypy'
     # Extras
     botocore>=1.4.87
     Pillow>=4.0.0
@@ -20,6 +24,8 @@ passenv =
     AWS_SECRET_ACCESS_KEY
     GCS_TEST_FILE_URI
     GCS_PROJECT_ID
+#allow tox virtualenv to upgrade pip/wheel/setuptools
+download = true
 commands =
     py.test --cov=scrapy --cov-report= {posargs:--durations=10 docs scrapy tests}
 
@@ -66,7 +72,6 @@ deps =
     itemadapter==0.1.0
     parsel==1.5.0
     Protego==0.1.15
-    PyDispatcher==2.0.5
     pyOpenSSL==17.5.0
     queuelib==1.4.2
     service_identity==16.0.0
@@ -74,15 +79,30 @@ deps =
     w3lib==1.17.0
     zope.interface==4.1.3
     -rtests/requirements-py3.txt
+
+    # mitmproxy 4.0.4+ requires upgrading some of the pinned dependencies
+    # above, hence we do not install it in pinned environments at the moment
+
     # Extras
     botocore==1.4.87
     google-cloud-storage==1.29.0
     Pillow==4.0.0
+install_command =
+    # --use-feature=2020-resolver is required, otherwise the latest verion of
+    # Twisted gets installed.
+    pip install --use-feature=2020-resolver {opts} {packages}
+setenv =
+    _SCRAPY_PINNED=true
 
 [testenv:pinned]
 deps =
     {[pinned]deps}
     lxml==3.5.0
+    PyDispatcher==2.0.5
+install_command =
+    {[pinned]install_command}
+setenv =
+    {[pinned]setenv}
 
 [testenv:windows-pinned]
 basepython = python3
@@ -91,20 +111,33 @@ deps =
     # First lxml version that includes a Windows wheel for Python 3.6, so we do
     # not need to build lxml from sources in a CI Windows job:
     lxml==3.8.0
+    PyDispatcher==2.0.5
+install_command =
+    {[pinned]install_command}
+setenv =
+    {[pinned]setenv}
 
 [testenv:extra-deps]
 deps =
     {[testenv]deps}
     reppy
     robotexclusionrulesparser
+install_command =
+    # Test --use-feature=2020-resolver for the latest version of all
+    # dependencies.
+    pip install --use-feature=2020-resolver {opts} {packages}
 
 [testenv:asyncio]
 commands =
     {[testenv]commands} --reactor=asyncio
 
 [testenv:asyncio-pinned]
-commands = {[testenv:asyncio]commands}
 deps = {[testenv:pinned]deps}
+install_command =
+    {[pinned]install_command}
+commands = {[testenv:asyncio]commands}
+setenv =
+    {[pinned]setenv}
 
 [testenv:pypy3]
 basepython = pypy3
@@ -113,11 +146,15 @@ commands =
 
 [testenv:pypy3-pinned]
 basepython = {[testenv:pypy3]basepython}
-commands = {[testenv:pypy3]commands}
 deps =
     {[pinned]deps}
     lxml==4.0.0
     PyPyDispatcher==2.1.0
+install_command =
+    {[pinned]install_command}
+commands = {[testenv:pypy3]commands}
+setenv =
+    {[pinned]setenv}
 
 [docs]
 changedir = docs