From 5dbeece8da6686709eb00af9ffb91dd30a832a2e Mon Sep 17 00:00:00 2001 From: elacuesta Date: Wed, 7 Aug 2019 04:36:52 -0300 Subject: [PATCH] [MRG+1] Drop py34 support - Update CI envs (#3892) * Drop py34 support * Travis experiments * More Travis experiments * Bump Twisted version for py35+ (stretch) * Remove Debian build * Remove pinned lxml for Py34 * Fix merge error * Remove unused tox env * Add environment with pinned versions for py36 * Bump minimum Twisted version in py27; Envs with pinned versions for py27 and py35 * Add botocore as extra dep for py27 tests * Update requirements-py2.txt * Add botocore and Pillow as extra dependencies --- .travis.yml | 50 ++++++++++---------- README.rst | 2 +- docs/faq.rst | 2 +- docs/intro/install.rst | 2 +- requirements-py2.txt | 25 ++++++---- requirements-py3.txt | 26 +++++++---- setup.py | 26 +++++------ tests/requirements-py2.txt | 7 +-- tests/requirements-py3.txt | 9 ++-- tox.ini | 94 +++++++++++++++++++------------------- 10 files changed, 128 insertions(+), 115 deletions(-) diff --git a/.travis.yml b/.travis.yml index 138e81c64..0190a7f4d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,5 +1,5 @@ language: python -dist: trusty +dist: xenial branches: only: - master @@ -7,32 +7,28 @@ branches: - /^\d\.\d+\.\d+(rc\d+|\.dev\d+)?$/ matrix: include: - - python: 2.7 - env: TOXENV=py27 - - python: 2.7 - env: TOXENV=jessie - - python: 2.7 - env: TOXENV=pypy - - python: 2.7 - env: TOXENV=pypy3 - - python: 3.4 - env: TOXENV=py34 - - python: 3.5 - env: TOXENV=py35 - - python: 3.6 - env: TOXENV=py36 - - python: 3.7 - env: TOXENV=py37 - dist: xenial - sudo: true - - python: 3.6 - env: TOXENV=docs - - python: 3.7 - env: TOXENV=py37-extra-deps - dist: xenial - sudo: true - - python: 2.7 - env: TOXENV=py27-extra-deps + - env: TOXENV=py27 + python: 2.7 + - env: TOXENV=py27-pinned + python: 2.7 + - env: TOXENV=py27-extra-deps + python: 2.7 + - env: TOXENV=pypy + python: 2.7 + - env: TOXENV=pypy3 + python: 3.5 + - env: TOXENV=py35 + python: 3.5 + - env: TOXENV=py35-pinned + python: 3.5 + - env: TOXENV=py36 + python: 3.6 + - env: TOXENV=py37 + python: 3.7 + - env: TOXENV=py37-extra-deps + python: 3.7 + - env: TOXENV=docs + python: 3.6 install: - | if [ "$TOXENV" = "pypy" ]; then diff --git a/README.rst b/README.rst index c28d217ff..bd82bff06 100644 --- a/README.rst +++ b/README.rst @@ -40,7 +40,7 @@ https://scrapy.org Requirements ============ -* Python 2.7 or Python 3.4+ +* Python 2.7 or Python 3.5+ * Works on Linux, Windows, Mac OSX, BSD Install diff --git a/docs/faq.rst b/docs/faq.rst index 44f2b97bd..9733471bf 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -69,7 +69,7 @@ Here's an example spider using BeautifulSoup API, with ``lxml`` as the HTML pars What Python versions does Scrapy support? ----------------------------------------- -Scrapy is supported under Python 2.7 and Python 3.4+ +Scrapy is supported under Python 2.7 and Python 3.5+ under CPython (default Python implementation) and PyPy (starting with PyPy 5.9). Python 2.6 support was dropped starting at Scrapy 0.20. Python 3 support was added in Scrapy 1.1. diff --git a/docs/intro/install.rst b/docs/intro/install.rst index daec7fcb7..2bf98dbdc 100644 --- a/docs/intro/install.rst +++ b/docs/intro/install.rst @@ -7,7 +7,7 @@ Installation guide Installing Scrapy ================= -Scrapy runs on Python 2.7 and Python 3.4 or above +Scrapy runs on Python 2.7 and Python 3.5 or above under CPython (default Python implementation) and PyPy (starting with PyPy 5.9). If you're using `Anaconda`_ or `Miniconda`_, you can install the package from diff --git a/requirements-py2.txt b/requirements-py2.txt index 0771aae3a..9e6944240 100644 --- a/requirements-py2.txt +++ b/requirements-py2.txt @@ -1,10 +1,17 @@ -Twisted>=13.1.0 -lxml -pyOpenSSL -cssselect>=0.9 -queuelib -w3lib>=1.17.0 -six>=1.5.2 +parsel>=1.5.0 PyDispatcher>=2.0.5 -parsel>=1.5 -service_identity +w3lib>=1.17.0 + +pyOpenSSL>=16.2.0 # Earlier versions fail with "AttributeError: module 'lib' has no attribute 'SSL_ST_INIT'" +queuelib>=1.4.2 # Earlier versions fail with "AttributeError: '...QueueTest' object has no attribute 'qpath'" +cryptography>=2.0 # Earlier versions would fail to install + +# Reference versions taken from +# https://packages.ubuntu.com/xenial/python/ +# https://packages.ubuntu.com/xenial/zope/ +cssselect>=0.9.1 +lxml>=3.5.0 +service_identity>=16.0.0 +six>=1.10.0 +Twisted>=16.0.0 +zope.interface>=4.1.3 diff --git a/requirements-py3.txt b/requirements-py3.txt index 478ed0010..cd183a525 100644 --- a/requirements-py3.txt +++ b/requirements-py3.txt @@ -1,11 +1,17 @@ -Twisted>=17.9.0 -lxml;python_version!="3.4" -lxml<=4.3.5;python_version=="3.4" -pyOpenSSL>=0.13.1 -cssselect>=0.9 -queuelib>=1.1.1 -w3lib>=1.17.0 -six>=1.5.2 +parsel>=1.5.0 PyDispatcher>=2.0.5 -parsel>=1.5 -service_identity +Twisted>=17.9.0 +w3lib>=1.17.0 + +pyOpenSSL>=16.2.0 # Earlier versions fail with "AttributeError: module 'lib' has no attribute 'SSL_ST_INIT'" +queuelib>=1.4.2 # Earlier versions fail with "AttributeError: '...QueueTest' object has no attribute 'qpath'" +cryptography>=2.0 # Earlier versions would fail to install + +# Reference versions taken from +# https://packages.ubuntu.com/xenial/python/ +# https://packages.ubuntu.com/xenial/zope/ +cssselect>=0.9.1 +lxml>=3.5.0 +service_identity>=16.0.0 +six>=1.10.0 +zope.interface>=4.1.3 diff --git a/setup.py b/setup.py index ee0aaabf0..37892cfbf 100644 --- a/setup.py +++ b/setup.py @@ -53,7 +53,6 @@ setup( 'Programming Language :: Python :: 2', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', @@ -63,20 +62,21 @@ setup( 'Topic :: Software Development :: Libraries :: Application Frameworks', 'Topic :: Software Development :: Libraries :: Python Modules', ], - python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*', + python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*', install_requires=[ - 'Twisted>=13.1.0;python_version!="3.4"', - 'Twisted>=13.1.0,<=19.2.0;python_version=="3.4"', - 'w3lib>=1.17.0', - 'queuelib', - 'lxml;python_version!="3.4"', - 'lxml<=4.3.5;python_version=="3.4"', - 'pyOpenSSL', - 'cssselect>=0.9', - 'six>=1.5.2', - 'parsel>=1.5', + 'Twisted>=16.0.0;python_version=="2.7"', + 'Twisted>=17.9.0;python_version>="3.5"', + 'cryptography>=2.0', + 'cssselect>=0.9.1', + 'lxml>=3.5.0', + 'parsel>=1.5.0', 'PyDispatcher>=2.0.5', - 'service_identity', + 'pyOpenSSL>=16.2.0', + 'queuelib>=1.4.2', + 'service_identity>=16.0.0', + 'six>=1.10.0', + 'w3lib>=1.17.0', + 'zope.interface>=4.1.3', ], extras_require=extras_require, ) diff --git a/tests/requirements-py2.txt b/tests/requirements-py2.txt index be809b151..f621eb4eb 100644 --- a/tests/requirements-py2.txt +++ b/tests/requirements-py2.txt @@ -1,14 +1,15 @@ # Tests requirements -mock +brotlipy +jmespath mitmproxy==0.10.1 +mock netlib==0.10.1 pytest pytest-cov pytest-twisted pytest-xdist -jmespath -brotlipy testfixtures + # optional for shell wrapper tests bpython ipython<6.0 diff --git a/tests/requirements-py3.txt b/tests/requirements-py3.txt index ed7bf0be0..cb67bc40e 100644 --- a/tests/requirements-py3.txt +++ b/tests/requirements-py3.txt @@ -1,13 +1,14 @@ +# Tests requirements +jmespath +leveldb; sys_platform != "win32" pytest pytest-cov pytest-twisted pytest-xdist testfixtures -jmespath -leveldb; sys_platform != "win32" -botocore + # optional for shell wrapper tests bpython -ipython brotlipy +ipython pywin32; sys_platform == "win32" diff --git a/tox.ini b/tox.ini index c918731f4..c3502c2ca 100644 --- a/tox.ini +++ b/tox.ini @@ -11,10 +11,10 @@ deps = -ctests/constraints.txt -rrequirements-py2.txt # Extras - botocore + botocore>=1.3.23 google-cloud-storage - Pillow != 3.0.0 leveldb + Pillow>=3.4.2 -rtests/requirements-py2.txt passenv = S3_TEST_FILE_URI @@ -25,72 +25,74 @@ passenv = commands = py.test --cov=scrapy --cov-report= {posargs:scrapy tests} -[testenv:trusty] +[testenv:py27-pinned] basepython = python2.7 deps = - pyOpenSSL==0.13 - lxml==3.3.3 - Twisted==13.2.0 - boto==2.20.1 - Pillow==2.3.0 + -ctests/constraints.txt + cryptography==2.0 cssselect==0.9.1 - zope.interface==4.0.5 + lxml==3.5.0 + parsel==1.5.0 + PyDispatcher==2.0.5 + pyOpenSSL==16.2.0 + queuelib==1.4.2 + service_identity==16.0.0 + six==1.10.0 + Twisted==16.0.0 + w3lib==1.17.0 + zope.interface==4.1.3 -rtests/requirements-py2.txt - -[testenv:jessie] -# https://packages.debian.org/en/jessie/python/ -# https://packages.debian.org/en/jessie/zope/ -basepython = python2.7 -deps = - cryptography==0.6.1 - pyOpenSSL==0.14 - lxml==3.4.0 - Twisted==14.0.2 - boto==2.34.0 - Pillow==2.6.1 - cssselect==0.9.1 - zope.interface==4.1.1 - -rtests/requirements-py2.txt -# Not used directly but allows boto GCE plugins to load. -# https://github.com/GoogleCloudPlatform/compute-image-packages/issues/262 - google-compute-engine==2.8.12 - -[testenv:trunk] -basepython = python2.7 -commands = - pip install -U https://github.com/scrapy/w3lib/archive/master.zip#egg=w3lib - pip install -U https://github.com/scrapy/queuelib/archive/master.zip#egg=queuelib - py.test --cov=scrapy --cov-report= {posargs:scrapy tests} + # Extras + botocore==1.3.23 + Pillow==3.4.2 [testenv:pypy] basepython = pypy commands = py.test {posargs:scrapy tests} -[testenv:py34] -basepython = python3.4 +[testenv:py35] +basepython = python3.5 deps = -ctests/constraints.txt -rrequirements-py3.txt - # Extras - Pillow -rtests/requirements-py3.txt + # Extras + botocore>=1.3.23 + Pillow>=3.4.2 -[testenv:py35] +[testenv:py35-pinned] basepython = python3.5 -deps = {[testenv:py34]deps} +deps = + -ctests/constraints.txt + cryptography==2.0 + cssselect==0.9.1 + lxml==3.5.0 + parsel==1.5.0 + PyDispatcher==2.0.5 + pyOpenSSL==16.2.0 + queuelib==1.4.2 + service_identity==16.0.0 + six==1.10.0 + Twisted==17.9.0 + w3lib==1.17.0 + zope.interface==4.1.3 + -rtests/requirements-py3.txt + # Extras + botocore==1.3.23 + Pillow==3.4.2 [testenv:py36] basepython = python3.6 -deps = {[testenv:py34]deps} +deps = {[testenv:py35]deps} [testenv:py37] basepython = python3.7 -deps = {[testenv:py34]deps} +deps = {[testenv:py35]deps} [testenv:pypy3] basepython = pypy3 -deps = {[testenv:py34]deps} +deps = {[testenv:py35]deps} commands = py.test {posargs:scrapy tests} @@ -119,14 +121,14 @@ commands = [testenv:py37-extra-deps] basepython = python3.7 -deps = - {[testenv:py34]deps} +deps = + {[testenv:py35]deps} reppy robotexclusionrulesparser [testenv:py27-extra-deps] basepython = python2.7 -deps = +deps = {[testenv]deps} reppy robotexclusionrulesparser