From 2023a6bdcb5791c514d4392d1266c2381c460c42 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 17 Nov 2023 10:12:29 +0100 Subject: [PATCH] Fix the CI (#6149) --- .github/workflows/tests.yml | 20 +++++++++---------- docs/topics/media-pipeline.rst | 2 ++ requirements-py2.txt | 1 - setup.py | 8 ++++---- tests/constraints.txt | 1 - tox.ini | 36 ++++++++++++++++++++-------------- 6 files changed, 37 insertions(+), 31 deletions(-) delete mode 100644 tests/constraints.txt diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index dd43ca9ee..1bd9908c4 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -10,27 +10,23 @@ jobs: include: - python-version: 2.7 env: - TOXENV: py27 + TOXENV: py27-pinned - python-version: 2.7 env: - TOXENV: py27-pinned + TOXENV: py27 - python-version: 2.7 env: TOXENV: py27-extra-deps - python-version: pypy3 env: - TOXENV: pypy + TOXENV: pypy2 PYPY_VERSION: '2.7-v7.3.5' - - python-version: pypy3 - env: - TOXENV: pypy3 - PYPY_VERSION: '3.7-v7.3.5' - - python-version: 3.5 - env: - TOXENV: py35 - python-version: 3.5 env: TOXENV: py35-pinned + - python-version: 3.5 + env: + TOXENV: py35 - python-version: 3.6 env: TOXENV: py36 @@ -43,6 +39,10 @@ jobs: - python-version: 3.8 env: TOXENV: py38-extra-deps + - python-version: pypy3 + env: + TOXENV: pypy3 + PYPY_VERSION: '3.7-v7.3.5' steps: - uses: actions/checkout@v2 diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rst index 0ce431ff5..715637170 100644 --- a/docs/topics/media-pipeline.rst +++ b/docs/topics/media-pipeline.rst @@ -77,6 +77,8 @@ PIL. .. _Pillow: https://github.com/python-pillow/Pillow .. _Python Imaging Library: http://www.pythonware.com/products/pil/ +.. warning:: Pillow 10.0.0 and higher break compatibility with Scrapy 1.8.x. + .. _topics-media-pipeline-enabling: diff --git a/requirements-py2.txt b/requirements-py2.txt index 1f641f7e8..1b2cfbdb8 100644 --- a/requirements-py2.txt +++ b/requirements-py2.txt @@ -15,5 +15,4 @@ cssselect>=0.9.1 lxml>=3.5.0 service_identity>=16.0.0 six>=1.10.0 -Twisted>=16.0.0,<21.2.0 # >=21.2.0 causes https://github.com/scrapy/scrapy/issues/5020 zope.interface>=4.1.3,<5 # >=5 requires https://github.com/scrapy/scrapy/commit/532cd1d93ed58f038346ca6b753462563c85b489 diff --git a/setup.py b/setup.py index 8cad511d1..71a30bc19 100644 --- a/setup.py +++ b/setup.py @@ -67,16 +67,16 @@ setup( install_requires=[ 'Twisted>=16.0.0;python_version=="2.7"', 'Twisted>=17.9.0;python_version>="3.5"', - 'cryptography>=2.0', + 'cryptography>=2.0,<39.0.0', # https://stackoverflow.com/a/75053968 'cssselect>=0.9.1', 'lxml>=3.5.0', - 'parsel>=1.5.0', + 'parsel>=1.5.0,<1.8.0', # https://github.com/scrapy/scrapy/issues/4961 'PyDispatcher>=2.0.5', - 'pyOpenSSL>=16.2.0', + 'pyOpenSSL>=16.2.0,<22.1.0', # https://github.com/scrapy/scrapy/issues/5635 'queuelib>=1.4.2', 'service_identity>=16.0.0', 'six>=1.10.0', - 'w3lib>=1.17.0', + 'w3lib>=1.17.0,<2.0.0', # w3lib.form, used by scrapy/utils/multipart.py, was removed 'zope.interface>=4.1.3', 'protego>=0.1.15', 'tldextract', diff --git a/tests/constraints.txt b/tests/constraints.txt deleted file mode 100644 index 5655ac2d3..000000000 --- a/tests/constraints.txt +++ /dev/null @@ -1 +0,0 @@ -Twisted!=18.4.0 \ No newline at end of file diff --git a/tox.ini b/tox.ini index fbd061cdc..ad9f801cd 100644 --- a/tox.ini +++ b/tox.ini @@ -4,13 +4,12 @@ # and then run "tox" from this directory. [tox] -envlist = py27-pinned,py27,py27-extra-deps,pypy2,py35,py35-pinned,py36,py37,py38,py38-extra-deps,pypy3 +envlist = py27-pinned,py27,py27-extra-deps,pypy2,py35-pinned,py35,py36,py37,py38,py38-extra-deps,pypy3 # https://tox.wiki/en/4.11.3/faq.html#testing-end-of-life-python-versions requires = virtualenv<20.22.0 -[testenv] +[py27] deps = - -ctests/constraints.txt -rrequirements-py2.txt # Extras botocore>=1.3.23 @@ -18,6 +17,12 @@ deps = leveldb Pillow>=3.4.2 -rtests/requirements-py2.txt + +[testenv] +deps = + {[py27]deps} + # >=21.2.0 causes https://github.com/scrapy/scrapy/issues/5020 + Twisted>=16.0.0,<21.2.0 passenv = S3_TEST_FILE_URI AWS_ACCESS_KEY_ID @@ -30,7 +35,6 @@ commands = [testenv:py27-pinned] basepython = python2.7 deps = - -ctests/constraints.txt cryptography==2.0 cssselect==0.9.1 lxml==3.5.0 @@ -49,18 +53,29 @@ deps = botocore==1.3.23 Pillow==3.4.2 -[testenv:pypy] +[testenv:py27-extra-deps] +basepython = python2.7 +deps = + {[testenv]deps} + reppy==0.3.4 + robotexclusionrulesparser + +[testenv:pypy2] basepython = pypy2 commands = py.test {posargs:scrapy tests} +deps = + {[py27]deps} + # Latest version that won’t fail due to the use of the typing module. + Twisted==16.4.1 [testenv:py35] basepython = python3.5 deps = - -ctests/constraints.txt -rrequirements-py3.txt -rtests/requirements-py3.txt mock + # Extras botocore>=1.3.23 Pillow>=3.4.2 @@ -68,7 +83,6 @@ deps = [testenv:py35-pinned] basepython = python3.5 deps = - -ctests/constraints.txt cryptography==2.0 cssselect==0.9.1 lxml==3.5.0 @@ -134,12 +148,4 @@ commands = basepython = python3.8 deps = {[testenv:py35]deps} - reppy - robotexclusionrulesparser - -[testenv:py27-extra-deps] -basepython = python2.7 -deps = - {[testenv]deps} - reppy robotexclusionrulesparser