diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index fe194e4fa..6379d1b1b 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -39,11 +39,11 @@ jobs: python-version: 3.8 env: TOXENV: py - # - os: ubuntu-18.04 - # python-version: pypy3 - # env: - # TOXENV: pypy3 - # PYPY_VERSION: 3.6-v7.3.1 + - os: ubuntu-18.04 + python-version: pypy3 + env: + TOXENV: pypy3 + PYPY_VERSION: 3.6-v7.3.1 # windows tests - os: windows-latest @@ -68,11 +68,11 @@ jobs: python-version: 3.6 env: TOXENV: windows-pinned - # - os: ubuntu-18.04 - # python-version: pypy3 - # env: - # TOXENV: pypy3-pinned - # PYPY_VERSION: 3.6-v7.2.0 + - os: ubuntu-18.04 + python-version: pypy3 + env: + TOXENV: pypy3-pinned + PYPY_VERSION: 3.6-v7.2.0 # extras - os: ubuntu-18.04 diff --git a/setup.py b/setup.py index 3956e8020..d1894149b 100644 --- a/setup.py +++ b/setup.py @@ -24,7 +24,6 @@ install_requires = [ 'cssselect>=0.9.1', 'itemloaders>=1.0.1', 'parsel>=1.5.0', - 'PyDispatcher>=2.0.5', 'pyOpenSSL>=17.5.0', 'queuelib>=1.4.2', 'service_identity>=16.0.0', @@ -34,11 +33,12 @@ install_requires = [ 'itemadapter>=0.1.0', ] extras_require = {} - +cpython_dependencies = [ + 'lxml>=3.5.0', + 'PyDispatcher>=2.0.5', +] if has_environment_marker_platform_impl_support(): - extras_require[':platform_python_implementation == "CPython"'] = [ - 'lxml>=3.5.0', - ] + extras_require[':platform_python_implementation == "CPython"'] = cpython_dependencies extras_require[':platform_python_implementation == "PyPy"'] = [ # Earlier lxml versions are affected by # https://foss.heptapod.net/pypy/pypy/-/issues/2498, @@ -49,7 +49,7 @@ if has_environment_marker_platform_impl_support(): 'PyPyDispatcher>=2.1.0', ] else: - install_requires.append('lxml>=3.5.0') + install_requires.extend(cpython_dependencies) setup( diff --git a/tests/requirements-py3.txt b/tests/requirements-py3.txt index f22ce90f3..c9040fd36 100644 --- a/tests/requirements-py3.txt +++ b/tests/requirements-py3.txt @@ -1,8 +1,6 @@ # Tests requirements attrs dataclasses; python_version == '3.6' -mitmproxy >= 4.0.4, < 5; python_version >= '3.6' and python_version < '3.7' -mitmproxy >= 4.0.4; python_version >= '3.7' pyftpdlib # https://github.com/pytest-dev/pytest-twisted/issues/93 pytest != 5.4, != 5.4.1 diff --git a/tests/test_dependencies.py b/tests/test_dependencies.py index 5d0a1d0c9..93e7311d2 100644 --- a/tests/test_dependencies.py +++ b/tests/test_dependencies.py @@ -1,8 +1,14 @@ +import os +import re +from configparser import ConfigParser from importlib import import_module + +from twisted import version as twisted_version from twisted.trial import unittest class ScrapyUtilsTest(unittest.TestCase): + def test_required_openssl_version(self): try: module = import_module('OpenSSL') @@ -13,6 +19,32 @@ class ScrapyUtilsTest(unittest.TestCase): installed_version = [int(x) for x in module.__version__.split('.')[:2]] assert installed_version >= [0, 6], "OpenSSL >= 0.6 required" + def test_pinned_twisted_version(self): + """When running tests within a Tox environment with pinned + dependencies, make sure that the version of Twisted is the pinned + version. + + See https://github.com/scrapy/scrapy/pull/4814#issuecomment-706230011 + """ + if not os.environ.get('_SCRAPY_PINNED', None): + self.skipTest('Not in a pinned environment') + + tox_config_file_path = os.path.join( + os.path.dirname(__file__), + '..', + 'tox.ini', + ) + config_parser = ConfigParser() + config_parser.read(tox_config_file_path) + pattern = r'Twisted==([\d.]+)' + match = re.search(pattern, config_parser['pinned']['deps']) + pinned_twisted_version_string = match[1] + + self.assertEqual( + twisted_version.short(), + pinned_twisted_version_string + ) + if __name__ == "__main__": unittest.main() diff --git a/tests/test_downloadermiddleware.py b/tests/test_downloadermiddleware.py index 79f24c8a1..b538a0ed3 100644 --- a/tests/test_downloadermiddleware.py +++ b/tests/test_downloadermiddleware.py @@ -1,11 +1,13 @@ import asyncio -from unittest import mock +from unittest import mock, SkipTest from pytest import mark +from twisted import version as twisted_version from twisted.internet import defer from twisted.internet.defer import Deferred from twisted.trial.unittest import TestCase from twisted.python.failure import Failure +from twisted.python.versions import Version from scrapy.http import Request, Response from scrapy.spiders import Spider @@ -211,10 +213,21 @@ class MiddlewareUsingDeferreds(ManagerTestCase): self.assertFalse(download_func.called) +@mark.usefixtures('reactor_pytest') class MiddlewareUsingCoro(ManagerTestCase): """Middlewares using asyncio coroutines should work""" def test_asyncdef(self): + if ( + self.reactor_pytest == 'asyncio' + and twisted_version < Version('twisted', 18, 4, 0) + ): + raise SkipTest( + 'Due to https://twistedmatrix.com/trac/ticket/9390, this test ' + 'hangs when using AsyncIO and Twisted versions lower than ' + '18.4.0' + ) + resp = Response('http://example.com/index.html') class CoroMiddleware: @@ -235,6 +248,12 @@ class MiddlewareUsingCoro(ManagerTestCase): @mark.only_asyncio() def test_asyncdef_asyncio(self): + if twisted_version < Version('twisted', 18, 4, 0): + raise SkipTest( + 'Due to https://twistedmatrix.com/trac/ticket/9390, this test ' + 'hangs when using Twisted versions lower than 18.4.0' + ) + resp = Response('http://example.com/index.html') class CoroMiddleware: diff --git a/tests/test_proxy_connect.py b/tests/test_proxy_connect.py index 9eabe6b49..afdfb2578 100644 --- a/tests/test_proxy_connect.py +++ b/tests/test_proxy_connect.py @@ -1,12 +1,9 @@ import json import os -import platform import re import sys from subprocess import Popen, PIPE from urllib.parse import urlsplit, urlunsplit -from unittest import skipIf - from testfixtures import LogCapture from twisted.internet import defer from twisted.trial.unittest import TestCase @@ -57,13 +54,14 @@ def _wrong_credentials(proxy_url): return urlunsplit(bad_auth_proxy) -@skipIf("pypy" in sys.executable, - "mitmproxy does not support PyPy") -@skipIf(platform.system() == 'Windows' and sys.version_info < (3, 7), - "mitmproxy does not support Windows when running Python < 3.7") class ProxyConnectTestCase(TestCase): def setUp(self): + try: + import mitmproxy # noqa: F401 + except ImportError: + self.skipTest('mitmproxy is not installed') + self.mockserver = MockServer() self.mockserver.__enter__() self._oldenv = os.environ.copy() diff --git a/tests/test_utils_signal.py b/tests/test_utils_signal.py index b66588efb..ad7394232 100644 --- a/tests/test_utils_signal.py +++ b/tests/test_utils_signal.py @@ -1,11 +1,14 @@ import asyncio +from unittest import SkipTest +from pydispatch import dispatcher from pytest import mark from testfixtures import LogCapture -from twisted.trial import unittest -from twisted.python.failure import Failure +from twisted import version as twisted_version from twisted.internet import defer, reactor -from pydispatch import dispatcher +from twisted.python.failure import Failure +from twisted.python.versions import Version +from twisted.trial import unittest from scrapy.utils.signal import send_catch_log, send_catch_log_deferred from scrapy.utils.test import get_from_asyncio_queue @@ -68,6 +71,7 @@ class SendCatchLogDeferredTest2(SendCatchLogDeferredTest): return d +@mark.usefixtures('reactor_pytest') class SendCatchLogDeferredAsyncDefTest(SendCatchLogDeferredTest): async def ok_handler(self, arg, handlers_called): @@ -76,6 +80,19 @@ class SendCatchLogDeferredAsyncDefTest(SendCatchLogDeferredTest): await defer.succeed(42) return "OK" + def test_send_catch_log(self): + if ( + self.reactor_pytest == 'asyncio' + and twisted_version < Version('twisted', 18, 4, 0) + ): + raise SkipTest( + 'Due to https://twistedmatrix.com/trac/ticket/9390, this test ' + 'fails due to a timeout when using AsyncIO and Twisted ' + 'versions lower than 18.4.0' + ) + + return super().test_send_catch_log() + @mark.only_asyncio() class SendCatchLogDeferredAsyncioTest(SendCatchLogDeferredTest): @@ -86,6 +103,16 @@ class SendCatchLogDeferredAsyncioTest(SendCatchLogDeferredTest): await asyncio.sleep(0.2) return await get_from_asyncio_queue("OK") + def test_send_catch_log(self): + if twisted_version < Version('twisted', 18, 4, 0): + raise SkipTest( + 'Due to https://twistedmatrix.com/trac/ticket/9390, this test ' + 'fails due to a timeout when using Twisted versions lower ' + 'than 18.4.0' + ) + + return super().test_send_catch_log() + class SendCatchLogTest2(unittest.TestCase): diff --git a/tox.ini b/tox.ini index 33a7388d5..65fac2e4e 100644 --- a/tox.ini +++ b/tox.ini @@ -11,6 +11,10 @@ minversion = 1.7.0 deps = -ctests/constraints.txt -rtests/requirements-py3.txt + # mitmproxy does not support PyPy + # mitmproxy does not support Windows when running Python < 3.7 + mitmproxy >= 4.0.4; python_version >= '3.7' and implementation_name != 'pypy' + mitmproxy >= 4.0.4, < 5; python_version >= '3.6' and python_version < '3.7' and platform_system != 'Windows' and implementation_name != 'pypy' # Extras botocore>=1.4.87 Pillow>=4.0.0 @@ -20,6 +24,8 @@ passenv = AWS_SECRET_ACCESS_KEY GCS_TEST_FILE_URI GCS_PROJECT_ID +#allow tox virtualenv to upgrade pip/wheel/setuptools +download = true commands = py.test --cov=scrapy --cov-report= {posargs:--durations=10 docs scrapy tests} @@ -66,7 +72,6 @@ deps = itemadapter==0.1.0 parsel==1.5.0 Protego==0.1.15 - PyDispatcher==2.0.5 pyOpenSSL==17.5.0 queuelib==1.4.2 service_identity==16.0.0 @@ -74,15 +79,30 @@ deps = w3lib==1.17.0 zope.interface==4.1.3 -rtests/requirements-py3.txt + + # mitmproxy 4.0.4+ requires upgrading some of the pinned dependencies + # above, hence we do not install it in pinned environments at the moment + # Extras botocore==1.4.87 google-cloud-storage==1.29.0 Pillow==4.0.0 +install_command = + # --use-feature=2020-resolver is required, otherwise the latest verion of + # Twisted gets installed. + pip install --use-feature=2020-resolver {opts} {packages} +setenv = + _SCRAPY_PINNED=true [testenv:pinned] deps = {[pinned]deps} lxml==3.5.0 + PyDispatcher==2.0.5 +install_command = + {[pinned]install_command} +setenv = + {[pinned]setenv} [testenv:windows-pinned] basepython = python3 @@ -91,20 +111,33 @@ deps = # First lxml version that includes a Windows wheel for Python 3.6, so we do # not need to build lxml from sources in a CI Windows job: lxml==3.8.0 + PyDispatcher==2.0.5 +install_command = + {[pinned]install_command} +setenv = + {[pinned]setenv} [testenv:extra-deps] deps = {[testenv]deps} reppy robotexclusionrulesparser +install_command = + # Test --use-feature=2020-resolver for the latest version of all + # dependencies. + pip install --use-feature=2020-resolver {opts} {packages} [testenv:asyncio] commands = {[testenv]commands} --reactor=asyncio [testenv:asyncio-pinned] -commands = {[testenv:asyncio]commands} deps = {[testenv:pinned]deps} +install_command = + {[pinned]install_command} +commands = {[testenv:asyncio]commands} +setenv = + {[pinned]setenv} [testenv:pypy3] basepython = pypy3 @@ -113,11 +146,15 @@ commands = [testenv:pypy3-pinned] basepython = {[testenv:pypy3]basepython} -commands = {[testenv:pypy3]commands} deps = {[pinned]deps} lxml==4.0.0 PyPyDispatcher==2.1.0 +install_command = + {[pinned]install_command} +commands = {[testenv:pypy3]commands} +setenv = + {[pinned]setenv} [docs] changedir = docs