From 9186e5a686e04703baf78974824917cf7a5121ec Mon Sep 17 00:00:00 2001 From: dswij Date: Tue, 29 Sep 2020 00:27:39 +0700 Subject: [PATCH 01/28] add pip 20.2 test for tox --- tox.ini | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tox.ini b/tox.ini index 12e40295c..f73eaef36 100644 --- a/tox.ini +++ b/tox.ini @@ -23,6 +23,8 @@ passenv = GCS_PROJECT_ID commands = py.test --cov=scrapy --cov-report= {posargs:--durations=10 docs scrapy tests} +install_command = + pip install --use-feature=2020-resolver {opts} {packages} [testenv:typing] basepython = python3 From f4629fe2cc8469e540b68a48a00fdcbb5ecd976a Mon Sep 17 00:00:00 2001 From: dswij Date: Thu, 1 Oct 2020 14:58:14 +0700 Subject: [PATCH 02/28] Update travis-pip and tox deps conflict for pip20.2 --- .travis.yml | 1 + tox.ini | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index b883c5b78..de5bbe9ab 100644 --- a/.travis.yml +++ b/.travis.yml @@ -49,6 +49,7 @@ install: virtualenv --python="$PYPY_VERSION/bin/pypy3" "$HOME/virtualenvs/$PYPY_VERSION" source "$HOME/virtualenvs/$PYPY_VERSION/bin/activate" fi + - python -m pip install --upgrade pip #force travis to use newest version of pip - pip install -U tox twine wheel codecov script: tox diff --git a/tox.ini b/tox.ini index f73eaef36..9a41fc240 100644 --- a/tox.ini +++ b/tox.ini @@ -64,7 +64,8 @@ commands = [pinned] deps = -ctests/constraints.txt - cryptography==2.0 + #using cryptography-2.1.4 for test jobs to solve dependencies conflict on pip20.2>= + cryptography==2.1.4 cssselect==0.9.1 itemadapter==0.1.0 parsel==1.5.0 From 66201737a0d245a2568f9b17541d38d6134fa03c Mon Sep 17 00:00:00 2001 From: dswij Date: Thu, 1 Oct 2020 15:47:19 +0700 Subject: [PATCH 03/28] fix travis and deps --- .travis.yml | 2 +- tox.ini | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index de5bbe9ab..d53695e01 100644 --- a/.travis.yml +++ b/.travis.yml @@ -48,8 +48,8 @@ install: tar -jxf ${PYPY_VERSION}.tar.bz2 virtualenv --python="$PYPY_VERSION/bin/pypy3" "$HOME/virtualenvs/$PYPY_VERSION" source "$HOME/virtualenvs/$PYPY_VERSION/bin/activate" + python -m pip install -U pip fi - - python -m pip install --upgrade pip #force travis to use newest version of pip - pip install -U tox twine wheel codecov script: tox diff --git a/tox.ini b/tox.ini index 9a41fc240..abbc0752e 100644 --- a/tox.ini +++ b/tox.ini @@ -64,14 +64,15 @@ commands = [pinned] deps = -ctests/constraints.txt - #using cryptography-2.1.4 for test jobs to solve dependencies conflict on pip20.2>= + #using cryptography-2.1.4 to solve dependencies conflict on pip20.2>= cryptography==2.1.4 cssselect==0.9.1 itemadapter==0.1.0 parsel==1.5.0 Protego==0.1.15 PyDispatcher==2.0.5 - pyOpenSSL==16.2.0 + #using pyOpenSSL<18.1 to solve dependencies conflict with mitmproxy on pip20.2>= + pyOpenSSL==18.0.0 queuelib==1.4.2 service_identity==16.0.0 Twisted==17.9.0 From 0ea6ff11360963b7a26f11087cb8f62113070dd5 Mon Sep 17 00:00:00 2001 From: dswij Date: Thu, 1 Oct 2020 16:14:36 +0700 Subject: [PATCH 04/28] travis pip version and deps --- .travis.yml | 2 +- tox.ini | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index d53695e01..1ad5c4e9b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -48,8 +48,8 @@ install: tar -jxf ${PYPY_VERSION}.tar.bz2 virtualenv --python="$PYPY_VERSION/bin/pypy3" "$HOME/virtualenvs/$PYPY_VERSION" source "$HOME/virtualenvs/$PYPY_VERSION/bin/activate" - python -m pip install -U pip fi + - pip install --upgrade pip - pip install -U tox twine wheel codecov script: tox diff --git a/tox.ini b/tox.ini index abbc0752e..e923aeb85 100644 --- a/tox.ini +++ b/tox.ini @@ -64,8 +64,8 @@ commands = [pinned] deps = -ctests/constraints.txt - #using cryptography-2.1.4 to solve dependencies conflict on pip20.2>= - cryptography==2.1.4 + #using cryptography-3.1 to solve dependencies conflict on pip20.2>= + cryptography==3.1 cssselect==0.9.1 itemadapter==0.1.0 parsel==1.5.0 From f7201b1427749cc364c6f76100f01938d581340c Mon Sep 17 00:00:00 2001 From: dswij Date: Thu, 1 Oct 2020 19:59:23 +0700 Subject: [PATCH 05/28] travis and deps --- .travis.yml | 3 ++- tox.ini | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 1ad5c4e9b..4c4bc0ef7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -40,6 +40,8 @@ matrix: - env: TOXENV=asyncio python: 3.8 dist: bionic +before_install: + - pip install -U pip install: - | if [[ ! -z "$PYPY_VERSION" ]]; then @@ -49,7 +51,6 @@ install: virtualenv --python="$PYPY_VERSION/bin/pypy3" "$HOME/virtualenvs/$PYPY_VERSION" source "$HOME/virtualenvs/$PYPY_VERSION/bin/activate" fi - - pip install --upgrade pip - pip install -U tox twine wheel codecov script: tox diff --git a/tox.ini b/tox.ini index e923aeb85..e1e46921e 100644 --- a/tox.ini +++ b/tox.ini @@ -64,8 +64,8 @@ commands = [pinned] deps = -ctests/constraints.txt - #using cryptography-3.1 to solve dependencies conflict on pip20.2>= - cryptography==3.1 + #using cryptography-2.3.1 to solve dependencies conflict on pip20.2>= + cryptography==2.3.1 cssselect==0.9.1 itemadapter==0.1.0 parsel==1.5.0 From 392b489a65b62244fa6cc1e1406c8e2e2d50d966 Mon Sep 17 00:00:00 2001 From: dswij Date: Thu, 1 Oct 2020 20:32:49 +0700 Subject: [PATCH 06/28] travis --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 4c4bc0ef7..da37c85c6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -42,6 +42,7 @@ matrix: dist: bionic before_install: - pip install -U pip + - pip --version install: - | if [[ ! -z "$PYPY_VERSION" ]]; then From c83a16898f35c99b0383736e24293edbeaa5d46f Mon Sep 17 00:00:00 2001 From: dswij Date: Thu, 1 Oct 2020 20:59:05 +0700 Subject: [PATCH 07/28] try removing cache in travis to install pip --- .travis.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index da37c85c6..65fa3fa45 100644 --- a/.travis.yml +++ b/.travis.yml @@ -63,9 +63,6 @@ notifications: skip_join: true channels: - irc.freenode.org#scrapy -cache: - directories: - - $HOME/.cache/pip deploy: provider: pypi distributions: "sdist bdist_wheel" From cc81f9ed06399bd9c4e730a76f54032d9a7e9106 Mon Sep 17 00:00:00 2001 From: dswij Date: Fri, 2 Oct 2020 00:56:59 +0700 Subject: [PATCH 08/28] add download setting for tox --- .travis.yml | 4 +--- tox.ini | 1 + 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 65fa3fa45..5351abf87 100644 --- a/.travis.yml +++ b/.travis.yml @@ -40,9 +40,6 @@ matrix: - env: TOXENV=asyncio python: 3.8 dist: bionic -before_install: - - pip install -U pip - - pip --version install: - | if [[ ! -z "$PYPY_VERSION" ]]; then @@ -63,6 +60,7 @@ notifications: skip_join: true channels: - irc.freenode.org#scrapy + deploy: provider: pypi distributions: "sdist bdist_wheel" diff --git a/tox.ini b/tox.ini index e1e46921e..f51be432d 100644 --- a/tox.ini +++ b/tox.ini @@ -21,6 +21,7 @@ passenv = AWS_SECRET_ACCESS_KEY GCS_TEST_FILE_URI GCS_PROJECT_ID +download = true #allow tox virtualenv to upgarde pip/wheel/setuptools commands = py.test --cov=scrapy --cov-report= {posargs:--durations=10 docs scrapy tests} install_command = From 95b2e94496f86d1a225d573adc2af2facce35401 Mon Sep 17 00:00:00 2001 From: dswij Date: Fri, 2 Oct 2020 01:05:45 +0700 Subject: [PATCH 09/28] fix comment error on tox and re-add cache for travis --- .travis.yml | 4 +++- tox.ini | 3 ++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 5351abf87..b883c5b78 100644 --- a/.travis.yml +++ b/.travis.yml @@ -60,7 +60,9 @@ notifications: skip_join: true channels: - irc.freenode.org#scrapy - +cache: + directories: + - $HOME/.cache/pip deploy: provider: pypi distributions: "sdist bdist_wheel" diff --git a/tox.ini b/tox.ini index f51be432d..8cc522378 100644 --- a/tox.ini +++ b/tox.ini @@ -21,7 +21,8 @@ passenv = AWS_SECRET_ACCESS_KEY GCS_TEST_FILE_URI GCS_PROJECT_ID -download = true #allow tox virtualenv to upgarde pip/wheel/setuptools +#allow tox virtualenv to upgrade pip/wheel/setuptools +download = true commands = py.test --cov=scrapy --cov-report= {posargs:--durations=10 docs scrapy tests} install_command = From 371bb808689bdb6e2bfd1027069e85716c153b51 Mon Sep 17 00:00:00 2001 From: dswij Date: Tue, 6 Oct 2020 19:44:48 +0700 Subject: [PATCH 10/28] Explicitly declare PyDispatcher as dependencies --- .travis.yml | 1 + tox.ini | 6 ++++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index b883c5b78..fb4802070 100644 --- a/.travis.yml +++ b/.travis.yml @@ -48,6 +48,7 @@ install: tar -jxf ${PYPY_VERSION}.tar.bz2 virtualenv --python="$PYPY_VERSION/bin/pypy3" "$HOME/virtualenvs/$PYPY_VERSION" source "$HOME/virtualenvs/$PYPY_VERSION/bin/activate" + $HOME/virtualenvs/$PYPY_VERSION/bin/pypy -m ensurepip --default-pip fi - pip install -U tox twine wheel codecov diff --git a/tox.ini b/tox.ini index 8cc522378..10b144619 100644 --- a/tox.ini +++ b/tox.ini @@ -77,7 +77,7 @@ deps = pyOpenSSL==18.0.0 queuelib==1.4.2 service_identity==16.0.0 - Twisted==17.9.0 + Twisted==20.3.0 w3lib==1.17.0 zope.interface==4.1.3 -rtests/requirements-py3.txt @@ -115,6 +115,9 @@ deps = {[testenv:pinned]deps} [testenv:pypy3] basepython = pypy3 +deps = + {[testenv]deps} + PyDispatcher==2.0.5 commands = py.test {posargs:--durations=10 docs scrapy tests} @@ -124,7 +127,6 @@ commands = {[testenv:pypy3]commands} deps = {[pinned]deps} lxml==4.0.0 - PyPyDispatcher==2.1.0 [docs] changedir = docs From ce6884d517abb75b884ae23ab59e1405daa83187 Mon Sep 17 00:00:00 2001 From: dswij <44697459+dswij@users.noreply.github.com> Date: Tue, 6 Oct 2020 19:51:42 +0700 Subject: [PATCH 11/28] Update tox.ini --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 10b144619..4e6ca3ec9 100644 --- a/tox.ini +++ b/tox.ini @@ -79,7 +79,7 @@ deps = service_identity==16.0.0 Twisted==20.3.0 w3lib==1.17.0 - zope.interface==4.1.3 + zope.interface==5.1.2 -rtests/requirements-py3.txt # Extras botocore==1.4.87 From 9f02df20c51c80739d2362c388930ad93e3de34a Mon Sep 17 00:00:00 2001 From: dswij Date: Wed, 7 Oct 2020 01:10:01 +0700 Subject: [PATCH 12/28] Remove PyDispatcher from general requirements --- setup.py | 2 +- tox.ini | 15 +++++++++++---- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/setup.py b/setup.py index 0c2281400..c046684c0 100644 --- a/setup.py +++ b/setup.py @@ -24,7 +24,6 @@ install_requires = [ 'cssselect>=0.9.1', 'itemloaders>=1.0.1', 'parsel>=1.5.0', - 'PyDispatcher>=2.0.5', 'pyOpenSSL>=16.2.0', 'queuelib>=1.4.2', 'service_identity>=16.0.0', @@ -38,6 +37,7 @@ extras_require = {} if has_environment_marker_platform_impl_support(): extras_require[':platform_python_implementation == "CPython"'] = [ 'lxml>=3.5.0', + 'PyDispatcher>=2.0.5', ] extras_require[':platform_python_implementation == "PyPy"'] = [ # Earlier lxml versions are affected by diff --git a/tox.ini b/tox.ini index 4e6ca3ec9..d29cea0fc 100644 --- a/tox.ini +++ b/tox.ini @@ -72,7 +72,6 @@ deps = itemadapter==0.1.0 parsel==1.5.0 Protego==0.1.15 - PyDispatcher==2.0.5 #using pyOpenSSL<18.1 to solve dependencies conflict with mitmproxy on pip20.2>= pyOpenSSL==18.0.0 queuelib==1.4.2 @@ -89,7 +88,12 @@ deps = [testenv:pinned] deps = {[pinned]deps} - lxml==3.5.0 + PyDispatcher==2.0.5 + +[testenv: pypy-pinned] +deps = + {[pinned]deps} + PyPyDispatcher==2.1.0 [testenv:windows-pinned] basepython = python3 @@ -111,13 +115,16 @@ commands = [testenv:asyncio-pinned] commands = {[testenv:asyncio]commands} -deps = {[testenv:pinned]deps} +deps = + {[testenv:pinned]deps} + lxml==3.5.0 [testenv:pypy3] basepython = pypy3 deps = {[testenv]deps} - PyDispatcher==2.0.5 + PyPyDispatcher>=2.1.0 + lxml==4.0.0 commands = py.test {posargs:--durations=10 docs scrapy tests} From 9461414b14a7444dd52eda517dc5e373f79c2b7e Mon Sep 17 00:00:00 2001 From: dswij Date: Wed, 7 Oct 2020 11:26:53 +0700 Subject: [PATCH 13/28] minor changes to remove unnecessary lines --- .travis.yml | 1 - tox.ini | 19 +++++++------------ 2 files changed, 7 insertions(+), 13 deletions(-) diff --git a/.travis.yml b/.travis.yml index fb4802070..b883c5b78 100644 --- a/.travis.yml +++ b/.travis.yml @@ -48,7 +48,6 @@ install: tar -jxf ${PYPY_VERSION}.tar.bz2 virtualenv --python="$PYPY_VERSION/bin/pypy3" "$HOME/virtualenvs/$PYPY_VERSION" source "$HOME/virtualenvs/$PYPY_VERSION/bin/activate" - $HOME/virtualenvs/$PYPY_VERSION/bin/pypy -m ensurepip --default-pip fi - pip install -U tox twine wheel codecov diff --git a/tox.ini b/tox.ini index d29cea0fc..5dc99621e 100644 --- a/tox.ini +++ b/tox.ini @@ -72,12 +72,14 @@ deps = itemadapter==0.1.0 parsel==1.5.0 Protego==0.1.15 + !pypy3: PyDispatcher==2.0.5 #using pyOpenSSL<18.1 to solve dependencies conflict with mitmproxy on pip20.2>= pyOpenSSL==18.0.0 queuelib==1.4.2 service_identity==16.0.0 Twisted==20.3.0 w3lib==1.17.0 + #zope.interface==5.1.2 to resolve conflict with Twisted==20.3.0 zope.interface==5.1.2 -rtests/requirements-py3.txt # Extras @@ -88,13 +90,8 @@ deps = [testenv:pinned] deps = {[pinned]deps} - PyDispatcher==2.0.5 - -[testenv: pypy-pinned] -deps = - {[pinned]deps} - PyPyDispatcher==2.1.0 - + lxml==3.5.0 + [testenv:windows-pinned] basepython = python3 deps = @@ -114,26 +111,24 @@ commands = {[testenv]commands} --reactor=asyncio [testenv:asyncio-pinned] +deps = {[testenv:pinned]deps} commands = {[testenv:asyncio]commands} -deps = - {[testenv:pinned]deps} - lxml==3.5.0 [testenv:pypy3] basepython = pypy3 deps = {[testenv]deps} - PyPyDispatcher>=2.1.0 lxml==4.0.0 commands = py.test {posargs:--durations=10 docs scrapy tests} [testenv:pypy3-pinned] basepython = {[testenv:pypy3]basepython} -commands = {[testenv:pypy3]commands} deps = {[pinned]deps} lxml==4.0.0 + PyPyDispatcher==2.1.0 +commands = {[testenv:pypy3]commands} [docs] changedir = docs From 7187247c01d3629000b8b460eea26064043c5595 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 30 Oct 2020 10:23:59 +0100 Subject: [PATCH 14/28] Add PyDispatcher>=2.0.5 back to dependencies for old pip --- setup.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/setup.py b/setup.py index c046684c0..aa75c7a34 100644 --- a/setup.py +++ b/setup.py @@ -33,12 +33,12 @@ install_requires = [ 'itemadapter>=0.1.0', ] extras_require = {} - +cpython_dependencies = [ + 'lxml>=3.5.0', + 'PyDispatcher>=2.0.5', +] if has_environment_marker_platform_impl_support(): - extras_require[':platform_python_implementation == "CPython"'] = [ - 'lxml>=3.5.0', - 'PyDispatcher>=2.0.5', - ] + extras_require[':platform_python_implementation == "CPython"'] = cpython_dependencies extras_require[':platform_python_implementation == "PyPy"'] = [ # Earlier lxml versions are affected by # https://foss.heptapod.net/pypy/pypy/-/issues/2498, @@ -49,7 +49,7 @@ if has_environment_marker_platform_impl_support(): 'PyPyDispatcher>=2.1.0', ] else: - install_requires.append('lxml>=3.5.0') + install_requires.extend(cpython_dependencies) setup( From 13bcdc9f881727f5a140d4b8671c81c7de3bc572 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 30 Oct 2020 10:28:42 +0100 Subject: [PATCH 15/28] Restore pinned dependencies in tox.ini --- tox.ini | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/tox.ini b/tox.ini index 5dc99621e..b90ed867f 100644 --- a/tox.ini +++ b/tox.ini @@ -66,21 +66,17 @@ commands = [pinned] deps = -ctests/constraints.txt - #using cryptography-2.3.1 to solve dependencies conflict on pip20.2>= - cryptography==2.3.1 + cryptography==2.0 cssselect==0.9.1 itemadapter==0.1.0 parsel==1.5.0 Protego==0.1.15 - !pypy3: PyDispatcher==2.0.5 - #using pyOpenSSL<18.1 to solve dependencies conflict with mitmproxy on pip20.2>= - pyOpenSSL==18.0.0 + pyOpenSSL==16.2.0 queuelib==1.4.2 service_identity==16.0.0 - Twisted==20.3.0 + Twisted==17.9.0 w3lib==1.17.0 - #zope.interface==5.1.2 to resolve conflict with Twisted==20.3.0 - zope.interface==5.1.2 + zope.interface==4.1.3 -rtests/requirements-py3.txt # Extras botocore==1.4.87 @@ -91,6 +87,7 @@ deps = deps = {[pinned]deps} lxml==3.5.0 + PyDispatcher==2.0.5 [testenv:windows-pinned] basepython = python3 @@ -99,6 +96,7 @@ deps = # First lxml version that includes a Windows wheel for Python 3.6, so we do # not need to build lxml from sources in a CI Windows job: lxml==3.8.0 + PyDispatcher==2.0.5 [testenv:extra-deps] deps = From 3e5bc7773732aa7e9839934cc45cac8252a36aa6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 30 Oct 2020 10:31:33 +0100 Subject: [PATCH 16/28] Remove pinning from the PyPy job --- tox.ini | 3 --- 1 file changed, 3 deletions(-) diff --git a/tox.ini b/tox.ini index b90ed867f..878395e6f 100644 --- a/tox.ini +++ b/tox.ini @@ -114,9 +114,6 @@ commands = {[testenv:asyncio]commands} [testenv:pypy3] basepython = pypy3 -deps = - {[testenv]deps} - lxml==4.0.0 commands = py.test {posargs:--durations=10 docs scrapy tests} From 7327145bf3d45b83b110a46ff79d2e337e36b520 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 30 Oct 2020 21:34:15 +0100 Subject: [PATCH 17/28] Remove mitmproxy from pinned environments --- tests/requirements-py3.txt | 2 -- tests/test_proxy_connect.py | 11 +++++------ tox.ini | 10 +++++++++- 3 files changed, 14 insertions(+), 9 deletions(-) diff --git a/tests/requirements-py3.txt b/tests/requirements-py3.txt index 2247ed917..d44632d8b 100644 --- a/tests/requirements-py3.txt +++ b/tests/requirements-py3.txt @@ -1,8 +1,6 @@ # Tests requirements attrs dataclasses; python_version == '3.6' -mitmproxy; python_version >= '3.7' -mitmproxy >= 4.0.4, < 5; python_version >= '3.6' and python_version < '3.7' pyftpdlib # https://github.com/pytest-dev/pytest-twisted/issues/93 pytest != 5.4, != 5.4.1 diff --git a/tests/test_proxy_connect.py b/tests/test_proxy_connect.py index 9eabe6b49..0db148910 100644 --- a/tests/test_proxy_connect.py +++ b/tests/test_proxy_connect.py @@ -5,8 +5,6 @@ import re import sys from subprocess import Popen, PIPE from urllib.parse import urlsplit, urlunsplit -from unittest import skipIf - from testfixtures import LogCapture from twisted.internet import defer from twisted.trial.unittest import TestCase @@ -57,13 +55,14 @@ def _wrong_credentials(proxy_url): return urlunsplit(bad_auth_proxy) -@skipIf("pypy" in sys.executable, - "mitmproxy does not support PyPy") -@skipIf(platform.system() == 'Windows' and sys.version_info < (3, 7), - "mitmproxy does not support Windows when running Python < 3.7") class ProxyConnectTestCase(TestCase): def setUp(self): + try: + import mitmproxy + except ImportError: + self.skipTest('mitmproxy is not installed') + self.mockserver = MockServer() self.mockserver.__enter__() self._oldenv = os.environ.copy() diff --git a/tox.ini b/tox.ini index 878395e6f..6149a9c58 100644 --- a/tox.ini +++ b/tox.ini @@ -11,6 +11,10 @@ minversion = 1.7.0 deps = -ctests/constraints.txt -rtests/requirements-py3.txt + # mitmproxy does not support PyPy + # mitmproxy does not support Windows when running Python < 3.7 + mitmproxy; python_version >= '3.7' and implementation_name != 'pypy' + mitmproxy >= 4.0.4, < 5; python_version >= '3.6' and python_version < '3.7' and platform_system != 'Windows' and implementation_name != 'pypy' # Extras boto3>=1.13.0 botocore>=1.4.87 @@ -26,7 +30,7 @@ download = true commands = py.test --cov=scrapy --cov-report= {posargs:--durations=10 docs scrapy tests} install_command = - pip install --use-feature=2020-resolver {opts} {packages} + pip install --use-feature=2020-resolver {opts} {packages} [testenv:typing] basepython = python3 @@ -78,6 +82,10 @@ deps = w3lib==1.17.0 zope.interface==4.1.3 -rtests/requirements-py3.txt + + # mitmproxy 4.0.4+ requires upgrading some of the pinned dependencies + # above, hence we do not install it in pinned environments at the moment + # Extras botocore==1.4.87 google-cloud-storage==1.29.0 From 8e7b756727cbd8c207acce3083ae29c85a6c6d51 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 4 Nov 2020 21:26:55 +0100 Subject: [PATCH 18/28] Solve Flake8-reported issues --- tests/test_proxy_connect.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/test_proxy_connect.py b/tests/test_proxy_connect.py index 0db148910..afdfb2578 100644 --- a/tests/test_proxy_connect.py +++ b/tests/test_proxy_connect.py @@ -1,6 +1,5 @@ import json import os -import platform import re import sys from subprocess import Popen, PIPE @@ -59,7 +58,7 @@ class ProxyConnectTestCase(TestCase): def setUp(self): try: - import mitmproxy + import mitmproxy # noqa: F401 except ImportError: self.skipTest('mitmproxy is not installed') From 906626cf0befe332cb61b9cdda7108d61afec776 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 4 Nov 2020 21:50:12 +0100 Subject: [PATCH 19/28] Skip MiddlewareUsingCoro::test_asyncdef on asyncio and old Twisted --- tests/test_downloadermiddleware.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/tests/test_downloadermiddleware.py b/tests/test_downloadermiddleware.py index 79f24c8a1..55af4c240 100644 --- a/tests/test_downloadermiddleware.py +++ b/tests/test_downloadermiddleware.py @@ -1,11 +1,12 @@ import asyncio -from unittest import mock +from unittest import mock, SkipTest from pytest import mark from twisted.internet import defer from twisted.internet.defer import Deferred from twisted.trial.unittest import TestCase from twisted.python.failure import Failure +from twisted.python.versions import Version from scrapy.http import Request, Response from scrapy.spiders import Spider @@ -211,10 +212,24 @@ class MiddlewareUsingDeferreds(ManagerTestCase): self.assertFalse(download_func.called) +@mark.usefixtures('reactor_pytest') class MiddlewareUsingCoro(ManagerTestCase): """Middlewares using asyncio coroutines should work""" def test_asyncdef(self): + import twisted + if ( + self.reactor_pytest == 'asyncio' + and twisted.version < Version('twisted', 18, 4, 0) + ): + raise SkipTest( + 'Due to https://twistedmatrix.com/trac/ticket/9390, this test ' + 'hangs when using AsyncIO and Twisted versions lower than ' + '18.4.0' + ) + + from twisted.python.versions import Version + resp = Response('http://example.com/index.html') class CoroMiddleware: From 6eaf0c5cc99204daed601369c9cea6f7c5e2f878 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 4 Nov 2020 21:54:00 +0100 Subject: [PATCH 20/28] Use Ubuntu Bionic for PyPy tests to try to get a newer OpenSSL version recognized --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index b883c5b78..f0eafd137 100644 --- a/.travis.yml +++ b/.travis.yml @@ -27,6 +27,7 @@ matrix: - env: TOXENV=py python: 3.6 - env: TOXENV=pypy3 PYPY_VERSION=3.6-v7.3.1 + dist: bionic - env: TOXENV=py python: 3.7 From ea851b910e580e840653274182548314e15b4312 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 6 Nov 2020 12:34:29 +0100 Subject: [PATCH 21/28] Clean up Twisted version check --- tests/test_downloadermiddleware.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tests/test_downloadermiddleware.py b/tests/test_downloadermiddleware.py index 55af4c240..6168e0176 100644 --- a/tests/test_downloadermiddleware.py +++ b/tests/test_downloadermiddleware.py @@ -2,6 +2,7 @@ import asyncio from unittest import mock, SkipTest from pytest import mark +from twisted import version as twisted_version from twisted.internet import defer from twisted.internet.defer import Deferred from twisted.trial.unittest import TestCase @@ -217,10 +218,9 @@ class MiddlewareUsingCoro(ManagerTestCase): """Middlewares using asyncio coroutines should work""" def test_asyncdef(self): - import twisted if ( self.reactor_pytest == 'asyncio' - and twisted.version < Version('twisted', 18, 4, 0) + and twisted_version < Version('twisted', 18, 4, 0) ): raise SkipTest( 'Due to https://twistedmatrix.com/trac/ticket/9390, this test ' @@ -228,8 +228,6 @@ class MiddlewareUsingCoro(ManagerTestCase): '18.4.0' ) - from twisted.python.versions import Version - resp = Response('http://example.com/index.html') class CoroMiddleware: From fea5a118993855808183b32809c2c304db28c364 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 6 Nov 2020 12:59:46 +0100 Subject: [PATCH 22/28] Also skip test_asyncdef_asyncio on old Twisted versions --- tests/test_downloadermiddleware.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/test_downloadermiddleware.py b/tests/test_downloadermiddleware.py index 6168e0176..b538a0ed3 100644 --- a/tests/test_downloadermiddleware.py +++ b/tests/test_downloadermiddleware.py @@ -248,6 +248,12 @@ class MiddlewareUsingCoro(ManagerTestCase): @mark.only_asyncio() def test_asyncdef_asyncio(self): + if twisted_version < Version('twisted', 18, 4, 0): + raise SkipTest( + 'Due to https://twistedmatrix.com/trac/ticket/9390, this test ' + 'hangs when using Twisted versions lower than 18.4.0' + ) + resp = Response('http://example.com/index.html') class CoroMiddleware: From 1941f607ca54694d4822933eba973665baa1b45b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 6 Nov 2020 16:25:56 +0100 Subject: [PATCH 23/28] Skip 2 additional tests with older Twisted versions --- tests/test_utils_signal.py | 33 ++++++++++++++++++++++++++++++--- 1 file changed, 30 insertions(+), 3 deletions(-) diff --git a/tests/test_utils_signal.py b/tests/test_utils_signal.py index b66588efb..18a8fb49c 100644 --- a/tests/test_utils_signal.py +++ b/tests/test_utils_signal.py @@ -1,11 +1,14 @@ import asyncio +from unittest import SkipTest +from pydispatch import dispatcher from pytest import mark from testfixtures import LogCapture -from twisted.trial import unittest -from twisted.python.failure import Failure +from twisted import version as twisted_version from twisted.internet import defer, reactor -from pydispatch import dispatcher +from twisted.python.failure import Failure +from twisted.python.versions import Version +from twisted.trial import unittest from scrapy.utils.signal import send_catch_log, send_catch_log_deferred from scrapy.utils.test import get_from_asyncio_queue @@ -68,6 +71,7 @@ class SendCatchLogDeferredTest2(SendCatchLogDeferredTest): return d +@mark.usefixtures('reactor_pytest') class SendCatchLogDeferredAsyncDefTest(SendCatchLogDeferredTest): async def ok_handler(self, arg, handlers_called): @@ -76,6 +80,19 @@ class SendCatchLogDeferredAsyncDefTest(SendCatchLogDeferredTest): await defer.succeed(42) return "OK" + def test_send_catch_log(self): + if ( + self.reactor_pytest == 'asyncio' + and twisted_version < Version('twisted', 18, 4, 0) + ): + raise SkipTest( + 'Due to https://twistedmatrix.com/trac/ticket/9390, this test ' + 'fails due to a timeout when using AsyncIO and Twisted ' + 'versions lower than 18.4.0' + ) + + return super().test_send_catch_log() + @mark.only_asyncio() class SendCatchLogDeferredAsyncioTest(SendCatchLogDeferredTest): @@ -86,6 +103,16 @@ class SendCatchLogDeferredAsyncioTest(SendCatchLogDeferredTest): await asyncio.sleep(0.2) return await get_from_asyncio_queue("OK") + def test_send_catch_log(self): + if (twisted_version < Version('twisted', 18, 4, 0): + raise SkipTest( + 'Due to https://twistedmatrix.com/trac/ticket/9390, this test ' + 'fails due to a timeout when using Twisted versions lower ' + 'than 18.4.0' + ) + + return super().test_send_catch_log() + class SendCatchLogTest2(unittest.TestCase): From 4b28da433384f67d0e6c7d4d557f5bcf268ae846 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 6 Nov 2020 16:46:22 +0100 Subject: [PATCH 24/28] Fix syntax error --- tests/test_utils_signal.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_utils_signal.py b/tests/test_utils_signal.py index 18a8fb49c..ad7394232 100644 --- a/tests/test_utils_signal.py +++ b/tests/test_utils_signal.py @@ -104,7 +104,7 @@ class SendCatchLogDeferredAsyncioTest(SendCatchLogDeferredTest): return await get_from_asyncio_queue("OK") def test_send_catch_log(self): - if (twisted_version < Version('twisted', 18, 4, 0): + if twisted_version < Version('twisted', 18, 4, 0): raise SkipTest( 'Due to https://twistedmatrix.com/trac/ticket/9390, this test ' 'fails due to a timeout when using Twisted versions lower ' From 034d61e6cb8391f045eb9b20824964e951549e01 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 11 Nov 2020 16:46:03 +0100 Subject: [PATCH 25/28] =?UTF-8?q?Restrict=20pip=E2=80=99s=20--use-feature?= =?UTF-8?q?=3D2020-resolver=20to=20the=20extra-deps=20environment?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tox.ini | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tox.ini b/tox.ini index 6149a9c58..f35466d05 100644 --- a/tox.ini +++ b/tox.ini @@ -29,8 +29,6 @@ passenv = download = true commands = py.test --cov=scrapy --cov-report= {posargs:--durations=10 docs scrapy tests} -install_command = - pip install --use-feature=2020-resolver {opts} {packages} [testenv:typing] basepython = python3 @@ -111,6 +109,8 @@ deps = {[testenv]deps} reppy robotexclusionrulesparser +install_command = + pip install --use-feature=2020-resolver {opts} {packages} [testenv:asyncio] commands = From 6ef3dc2029fea1f692aa3054ea47d799772151d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Tue, 17 Nov 2020 22:28:20 +0100 Subject: [PATCH 26/28] Use the new pip resolver for Tox environments with pinned dependencies --- tox.ini | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/tox.ini b/tox.ini index f35466d05..56e736fbf 100644 --- a/tox.ini +++ b/tox.ini @@ -88,12 +88,18 @@ deps = botocore==1.4.87 google-cloud-storage==1.29.0 Pillow==4.0.0 +install_command = + # --use-feature=2020-resolver is required, otherwise the latest verion of + # Twisted gets installed. + pip install --use-feature=2020-resolver {opts} {packages} [testenv:pinned] deps = {[pinned]deps} lxml==3.5.0 PyDispatcher==2.0.5 +install_command = + {[pinned]install_command} [testenv:windows-pinned] basepython = python3 @@ -103,6 +109,8 @@ deps = # not need to build lxml from sources in a CI Windows job: lxml==3.8.0 PyDispatcher==2.0.5 +install_command = + {[pinned]install_command} [testenv:extra-deps] deps = @@ -110,6 +118,8 @@ deps = reppy robotexclusionrulesparser install_command = + # Test --use-feature=2020-resolver for the latest version of all + # dependencies. pip install --use-feature=2020-resolver {opts} {packages} [testenv:asyncio] @@ -118,6 +128,8 @@ commands = [testenv:asyncio-pinned] deps = {[testenv:pinned]deps} +install_command = + {[pinned]install_command} commands = {[testenv:asyncio]commands} [testenv:pypy3] @@ -131,6 +143,8 @@ deps = {[pinned]deps} lxml==4.0.0 PyPyDispatcher==2.1.0 +install_command = + {[pinned]install_command} commands = {[testenv:pypy3]commands} [docs] From 0dc3e6350c230028addd8e73833c93341bab4b42 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 23 Nov 2020 22:10:45 +0100 Subject: [PATCH 27/28] Add a test to check the Twisted version in pinned environments --- tests/test_dependencies.py | 32 ++++++++++++++++++++++++++++++++ tox.ini | 10 ++++++++++ 2 files changed, 42 insertions(+) diff --git a/tests/test_dependencies.py b/tests/test_dependencies.py index 5d0a1d0c9..4e4f190ab 100644 --- a/tests/test_dependencies.py +++ b/tests/test_dependencies.py @@ -1,8 +1,14 @@ +import os +import re +from configparser import ConfigParser from importlib import import_module + +from twisted import version as twisted_version from twisted.trial import unittest class ScrapyUtilsTest(unittest.TestCase): + def test_required_openssl_version(self): try: module = import_module('OpenSSL') @@ -13,6 +19,32 @@ class ScrapyUtilsTest(unittest.TestCase): installed_version = [int(x) for x in module.__version__.split('.')[:2]] assert installed_version >= [0, 6], "OpenSSL >= 0.6 required" + def test_pinned_twisted_version(self): + """When running tests within a Tox environment with pinned + dependencies, make sure that the version of Twisted is the pinned + version. + + See https://github.com/scrapy/scrapy/pull/4814#issuecomment-706230011 + """ + if not os.environ.get('SCRAPY_PINNED', None): + self.skipTest('Not in a pinned environment') + + tox_config_file_path = os.path.join( + os.path.dirname(__file__), + '..', + 'tox.ini', + ) + config_parser = ConfigParser() + config_parser.read(tox_config_file_path) + pattern = r'Twisted==([\d.]+)' + match = re.search(pattern, config_parser['pinned']['deps']) + pinned_twisted_version_string = match[1] + + self.assertEqual( + twisted_version.short(), + pinned_twisted_version_string + ) + if __name__ == "__main__": unittest.main() diff --git a/tox.ini b/tox.ini index 56e736fbf..ea71a2476 100644 --- a/tox.ini +++ b/tox.ini @@ -92,6 +92,8 @@ install_command = # --use-feature=2020-resolver is required, otherwise the latest verion of # Twisted gets installed. pip install --use-feature=2020-resolver {opts} {packages} +setenv = + SCRAPY_PINNED=true [testenv:pinned] deps = @@ -100,6 +102,8 @@ deps = PyDispatcher==2.0.5 install_command = {[pinned]install_command} +setenv = + {[pinned]setenv} [testenv:windows-pinned] basepython = python3 @@ -111,6 +115,8 @@ deps = PyDispatcher==2.0.5 install_command = {[pinned]install_command} +setenv = + {[pinned]setenv} [testenv:extra-deps] deps = @@ -131,6 +137,8 @@ deps = {[testenv:pinned]deps} install_command = {[pinned]install_command} commands = {[testenv:asyncio]commands} +setenv = + {[pinned]setenv} [testenv:pypy3] basepython = pypy3 @@ -146,6 +154,8 @@ deps = install_command = {[pinned]install_command} commands = {[testenv:pypy3]commands} +setenv = + {[pinned]setenv} [docs] changedir = docs From f6879c681ebf94b22ee777f3249dfe59a99559cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 23 Nov 2020 23:53:03 +0100 Subject: [PATCH 28/28] =?UTF-8?q?SCRAPY=5FPINNED=20=E2=86=92=20=5FSCRAPY?= =?UTF-8?q?=5FPINNED?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tests/test_dependencies.py | 2 +- tox.ini | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_dependencies.py b/tests/test_dependencies.py index 4e4f190ab..93e7311d2 100644 --- a/tests/test_dependencies.py +++ b/tests/test_dependencies.py @@ -26,7 +26,7 @@ class ScrapyUtilsTest(unittest.TestCase): See https://github.com/scrapy/scrapy/pull/4814#issuecomment-706230011 """ - if not os.environ.get('SCRAPY_PINNED', None): + if not os.environ.get('_SCRAPY_PINNED', None): self.skipTest('Not in a pinned environment') tox_config_file_path = os.path.join( diff --git a/tox.ini b/tox.ini index ea71a2476..66866301c 100644 --- a/tox.ini +++ b/tox.ini @@ -93,7 +93,7 @@ install_command = # Twisted gets installed. pip install --use-feature=2020-resolver {opts} {packages} setenv = - SCRAPY_PINNED=true + _SCRAPY_PINNED=true [testenv:pinned] deps =