[MRG+1] Drop py34 support - Update CI envs (#3892)

* Drop py34 support * Travis experiments * More Travis experiments * Bump Twisted version for py35+ (stretch) * Remove Debian build * Remove pinned lxml for Py34 * Fix merge error * Remove unused tox env * Add environment with pinned versions for py36 * Bump minimum Twisted version in py27; Envs with pinned versions for py27 and py35 * Add botocore as extra dep for py27 tests * Update requirements-py2.txt * Add botocore and Pillow as extra dependencies
2019-08-07 04:36:52 -03:00 · 2019-08-07 04:36:52 -03:00 · 5dbeece8da
parent eef1732374
commit 5dbeece8da
10 changed files with 128 additions and 115 deletions
--- a/.travis.yml
+++ b/.travis.yml
@ -1,5 +1,5 @@
 language: python
-dist: trusty
+dist: xenial
 branches:
  only:
    - master
@ -7,32 +7,28 @@ branches:
    - /^\d\.\d+\.\d+(rc\d+|\.dev\d+)?$/
 matrix:
  include:
-    - python: 2.7
-      env: TOXENV=py27
-    - python: 2.7
-      env: TOXENV=jessie
-    - python: 2.7
-      env: TOXENV=pypy
-    - python: 2.7
-      env: TOXENV=pypy3
-    - python: 3.4
-      env: TOXENV=py34
-    - python: 3.5
-      env: TOXENV=py35
-    - python: 3.6
-      env: TOXENV=py36
-    - python: 3.7
-      env: TOXENV=py37
-      dist: xenial
-      sudo: true
-    - python: 3.6
-      env: TOXENV=docs
-    - python: 3.7
-      env: TOXENV=py37-extra-deps
-      dist: xenial
-      sudo: true
-    - python: 2.7
-      env: TOXENV=py27-extra-deps
+    - env: TOXENV=py27
+      python: 2.7
+    - env: TOXENV=py27-pinned
+      python: 2.7
+    - env: TOXENV=py27-extra-deps
+      python: 2.7
+    - env: TOXENV=pypy
+      python: 2.7
+    - env: TOXENV=pypy3
+      python: 3.5
+    - env: TOXENV=py35
+      python: 3.5
+    - env: TOXENV=py35-pinned
+      python: 3.5
+    - env: TOXENV=py36
+      python: 3.6
+    - env: TOXENV=py37
+      python: 3.7
+    - env: TOXENV=py37-extra-deps
+      python: 3.7
+    - env: TOXENV=docs
+      python: 3.6
 install:
  - |
      if [ "$TOXENV" = "pypy" ]; then
--- a/README.rst
+++ b/README.rst
@ -40,7 +40,7 @@ https://scrapy.org
 Requirements
 ============

-* Python 2.7 or Python 3.4+
+* Python 2.7 or Python 3.5+
 * Works on Linux, Windows, Mac OSX, BSD

 Install
--- a/docs/faq.rst
+++ b/docs/faq.rst
@ -69,7 +69,7 @@ Here's an example spider using BeautifulSoup API, with ``lxml`` as the HTML pars
 What Python versions does Scrapy support?
 -----------------------------------------

-Scrapy is supported under Python 2.7 and Python 3.4+
+Scrapy is supported under Python 2.7 and Python 3.5+
 under CPython (default Python implementation) and PyPy (starting with PyPy 5.9).
 Python 2.6 support was dropped starting at Scrapy 0.20.
 Python 3 support was added in Scrapy 1.1.
--- a/docs/intro/install.rst
+++ b/docs/intro/install.rst
@ -7,7 +7,7 @@ Installation guide
 Installing Scrapy
 =================

-Scrapy runs on Python 2.7 and Python 3.4 or above
+Scrapy runs on Python 2.7 and Python 3.5 or above
 under CPython (default Python implementation) and PyPy (starting with PyPy 5.9).

 If you're using `Anaconda`_ or `Miniconda`_, you can install the package from
--- a/requirements-py2.txt
+++ b/requirements-py2.txt
@ -1,10 +1,17 @@
-Twisted>=13.1.0
-lxml
-pyOpenSSL
-cssselect>=0.9
-queuelib
-w3lib>=1.17.0
-six>=1.5.2
+parsel>=1.5.0
 PyDispatcher>=2.0.5
-parsel>=1.5
-service_identity
+w3lib>=1.17.0
+
+pyOpenSSL>=16.2.0  # Earlier versions fail with "AttributeError: module 'lib' has no attribute 'SSL_ST_INIT'"
+queuelib>=1.4.2    # Earlier versions fail with "AttributeError: '...QueueTest' object has no attribute 'qpath'"
+cryptography>=2.0  # Earlier versions would fail to install
+
+# Reference versions taken from
+# https://packages.ubuntu.com/xenial/python/
+# https://packages.ubuntu.com/xenial/zope/
+cssselect>=0.9.1
+lxml>=3.5.0
+service_identity>=16.0.0
+six>=1.10.0
+Twisted>=16.0.0
+zope.interface>=4.1.3
--- a/requirements-py3.txt
+++ b/requirements-py3.txt
@ -1,11 +1,17 @@
-Twisted>=17.9.0
-lxml;python_version!="3.4"
-lxml<=4.3.5;python_version=="3.4"
-pyOpenSSL>=0.13.1
-cssselect>=0.9
-queuelib>=1.1.1
-w3lib>=1.17.0
-six>=1.5.2
+parsel>=1.5.0
 PyDispatcher>=2.0.5
-parsel>=1.5
-service_identity
+Twisted>=17.9.0
+w3lib>=1.17.0
+
+pyOpenSSL>=16.2.0  # Earlier versions fail with "AttributeError: module 'lib' has no attribute 'SSL_ST_INIT'"
+queuelib>=1.4.2    # Earlier versions fail with "AttributeError: '...QueueTest' object has no attribute 'qpath'"
+cryptography>=2.0  # Earlier versions would fail to install
+
+# Reference versions taken from
+# https://packages.ubuntu.com/xenial/python/
+# https://packages.ubuntu.com/xenial/zope/
+cssselect>=0.9.1
+lxml>=3.5.0
+service_identity>=16.0.0
+six>=1.10.0
+zope.interface>=4.1.3
--- a/setup.py
+++ b/setup.py
@ -53,7 +53,6 @@ setup(
        'Programming Language :: Python :: 2',
        'Programming Language :: Python :: 2.7',
        'Programming Language :: Python :: 3',
-        'Programming Language :: Python :: 3.4',
        'Programming Language :: Python :: 3.5',
        'Programming Language :: Python :: 3.6',
        'Programming Language :: Python :: 3.7',
@ -63,20 +62,21 @@ setup(
        'Topic :: Software Development :: Libraries :: Application Frameworks',
        'Topic :: Software Development :: Libraries :: Python Modules',
    ],
-    python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*',
+    python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*',
    install_requires=[
-        'Twisted>=13.1.0;python_version!="3.4"',
-        'Twisted>=13.1.0,<=19.2.0;python_version=="3.4"',
-        'w3lib>=1.17.0',
-        'queuelib',
-        'lxml;python_version!="3.4"',
-        'lxml<=4.3.5;python_version=="3.4"',
-        'pyOpenSSL',
-        'cssselect>=0.9',
-        'six>=1.5.2',
-        'parsel>=1.5',
+        'Twisted>=16.0.0;python_version=="2.7"',
+        'Twisted>=17.9.0;python_version>="3.5"',
+        'cryptography>=2.0',
+        'cssselect>=0.9.1',
+        'lxml>=3.5.0',
+        'parsel>=1.5.0',
        'PyDispatcher>=2.0.5',
-        'service_identity',
+        'pyOpenSSL>=16.2.0',
+        'queuelib>=1.4.2',
+        'service_identity>=16.0.0',
+        'six>=1.10.0',
+        'w3lib>=1.17.0',
+        'zope.interface>=4.1.3',
    ],
    extras_require=extras_require,
 )
--- a/tests/requirements-py2.txt
+++ b/tests/requirements-py2.txt
@ -1,14 +1,15 @@
 # Tests requirements
-mock
+brotlipy
+jmespath
 mitmproxy==0.10.1
+mock
 netlib==0.10.1
 pytest
 pytest-cov
 pytest-twisted
 pytest-xdist
-jmespath
-brotlipy
 testfixtures
+
 # optional for shell wrapper tests
 bpython
 ipython<6.0
--- a/tests/requirements-py3.txt
+++ b/tests/requirements-py3.txt
@ -1,13 +1,14 @@
+# Tests requirements
+jmespath
+leveldb; sys_platform != "win32"
 pytest
 pytest-cov
 pytest-twisted
 pytest-xdist
 testfixtures
-jmespath
-leveldb; sys_platform != "win32"
-botocore
+
 # optional for shell wrapper tests
 bpython
-ipython
 brotlipy
+ipython
 pywin32; sys_platform == "win32"
--- a/tox.ini
+++ b/tox.ini
@ -11,10 +11,10 @@ deps =
    -ctests/constraints.txt
    -rrequirements-py2.txt
    # Extras
-    botocore
+    botocore>=1.3.23
    google-cloud-storage
-    Pillow != 3.0.0
    leveldb
+    Pillow>=3.4.2
    -rtests/requirements-py2.txt
 passenv =
    S3_TEST_FILE_URI
@ -25,72 +25,74 @@ passenv =
 commands =
    py.test --cov=scrapy --cov-report= {posargs:scrapy tests}

-[testenv:trusty]
+[testenv:py27-pinned]
 basepython = python2.7
 deps =
-    pyOpenSSL==0.13
-    lxml==3.3.3
-    Twisted==13.2.0
-    boto==2.20.1
-    Pillow==2.3.0
+    -ctests/constraints.txt
+    cryptography==2.0
    cssselect==0.9.1
-    zope.interface==4.0.5
+    lxml==3.5.0
+    parsel==1.5.0
+    PyDispatcher==2.0.5
+    pyOpenSSL==16.2.0
+    queuelib==1.4.2
+    service_identity==16.0.0
+    six==1.10.0
+    Twisted==16.0.0
+    w3lib==1.17.0
+    zope.interface==4.1.3
    -rtests/requirements-py2.txt
-
-[testenv:jessie]
-# https://packages.debian.org/en/jessie/python/
-# https://packages.debian.org/en/jessie/zope/
-basepython = python2.7
-deps =
-    cryptography==0.6.1
-    pyOpenSSL==0.14
-    lxml==3.4.0
-    Twisted==14.0.2
-    boto==2.34.0
-    Pillow==2.6.1
-    cssselect==0.9.1
-    zope.interface==4.1.1
-    -rtests/requirements-py2.txt
-# Not used directly but allows boto GCE plugins to load.
-# https://github.com/GoogleCloudPlatform/compute-image-packages/issues/262
-    google-compute-engine==2.8.12
-
-[testenv:trunk]
-basepython = python2.7
-commands =
-    pip install -U https://github.com/scrapy/w3lib/archive/master.zip#egg=w3lib
-    pip install -U https://github.com/scrapy/queuelib/archive/master.zip#egg=queuelib
-    py.test --cov=scrapy --cov-report= {posargs:scrapy tests}
+    # Extras
+    botocore==1.3.23
+    Pillow==3.4.2

 [testenv:pypy]
 basepython = pypy
 commands =
    py.test {posargs:scrapy tests}

-[testenv:py34]
-basepython = python3.4
+[testenv:py35]
+basepython = python3.5
 deps =
    -ctests/constraints.txt
    -rrequirements-py3.txt
-    # Extras
-    Pillow
    -rtests/requirements-py3.txt
+    # Extras
+    botocore>=1.3.23
+    Pillow>=3.4.2

-[testenv:py35]
+[testenv:py35-pinned]
 basepython = python3.5
-deps = {[testenv:py34]deps}
+deps =
+    -ctests/constraints.txt
+    cryptography==2.0
+    cssselect==0.9.1
+    lxml==3.5.0
+    parsel==1.5.0
+    PyDispatcher==2.0.5
+    pyOpenSSL==16.2.0
+    queuelib==1.4.2
+    service_identity==16.0.0
+    six==1.10.0
+    Twisted==17.9.0
+    w3lib==1.17.0
+    zope.interface==4.1.3
+    -rtests/requirements-py3.txt
+    # Extras
+    botocore==1.3.23
+    Pillow==3.4.2

 [testenv:py36]
 basepython = python3.6
-deps = {[testenv:py34]deps}
+deps = {[testenv:py35]deps}

 [testenv:py37]
 basepython = python3.7
-deps = {[testenv:py34]deps}
+deps = {[testenv:py35]deps}

 [testenv:pypy3]
 basepython = pypy3
-deps = {[testenv:py34]deps}
+deps = {[testenv:py35]deps}
 commands =
    py.test {posargs:scrapy tests}

@ -119,14 +121,14 @@ commands =

 [testenv:py37-extra-deps]
 basepython = python3.7
-deps = 
-    {[testenv:py34]deps}
+deps =
+    {[testenv:py35]deps}
    reppy
    robotexclusionrulesparser

 [testenv:py27-extra-deps]
 basepython = python2.7
-deps = 
+deps =
    {[testenv]deps}
    reppy
    robotexclusionrulesparser