mirror of https://github.com/scrapy/scrapy.git
Drop Python 3.6 support (#5514)
* chore: Drop Python 3.6 support * Attend PR comments * Tweak versions * Update dependencies version * fix: Ubuntu workflow * fix windows workflow * chore: Remove comment * update `install_requires` dependencies versions * move lxml to main pinned requirements * Attend code-review comments * remove non-pinned 3.7 from windows workflow * simplify condition * lint * remove paragraph * refactor * remove leftover
This commit is contained in:
Laerte Pereira
GitHub
parent
c5627af15b
commit
407562b38b
|
|
@ -19,7 +19,7 @@ jobs:
|
|||
- python-version: 3.8
|
||||
env:
|
||||
TOXENV: pylint
|
||||
- python-version: 3.6
|
||||
- python-version: 3.7
|
||||
env:
|
||||
TOXENV: typing
|
||||
- python-version: "3.10" # Keep in sync with .readthedocs.yml
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@ jobs:
|
|||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
python-version: ["3.6", "3.7", "3.8", "3.9", "3.10"]
|
||||
python-version: ["3.7", "3.8", "3.9", "3.10"]
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
|
|
|
|||
|
|
@ -8,9 +8,6 @@ jobs:
|
|||
fail-fast: false
|
||||
matrix:
|
||||
include:
|
||||
- python-version: 3.7
|
||||
env:
|
||||
TOXENV: py
|
||||
- python-version: 3.8
|
||||
env:
|
||||
TOXENV: py
|
||||
|
|
@ -26,19 +23,19 @@ jobs:
|
|||
- python-version: pypy3
|
||||
env:
|
||||
TOXENV: pypy3
|
||||
PYPY_VERSION: 3.6-v7.3.3
|
||||
PYPY_VERSION: 3.9-v7.3.9
|
||||
|
||||
# pinned deps
|
||||
- python-version: 3.6.12
|
||||
- python-version: 3.7.13
|
||||
env:
|
||||
TOXENV: pinned
|
||||
- python-version: 3.6.12
|
||||
- python-version: 3.7.13
|
||||
env:
|
||||
TOXENV: asyncio-pinned
|
||||
- python-version: pypy3
|
||||
env:
|
||||
TOXENV: pypy3-pinned
|
||||
PYPY_VERSION: 3.6-v7.2.0
|
||||
PYPY_VERSION: 3.7-v7.3.5
|
||||
|
||||
# extras
|
||||
# extra-deps includes reppy, which does not support Python 3.9
|
||||
|
|
|
|||
|
|
@ -8,12 +8,9 @@ jobs:
|
|||
fail-fast: false
|
||||
matrix:
|
||||
include:
|
||||
- python-version: 3.6
|
||||
env:
|
||||
TOXENV: windows-pinned
|
||||
- python-version: 3.7
|
||||
env:
|
||||
TOXENV: py
|
||||
TOXENV: windows-pinned
|
||||
- python-version: 3.8
|
||||
env:
|
||||
TOXENV: py
|
||||
|
|
|
|||
|
|
@ -57,7 +57,7 @@ including a list of features.
|
|||
Requirements
|
||||
============
|
||||
|
||||
* Python 3.6+
|
||||
* Python 3.7+
|
||||
* Works on Linux, Windows, macOS, BSD
|
||||
|
||||
Install
|
||||
|
|
|
|||
|
|
@ -232,15 +232,15 @@ To run a specific test (say ``tests/test_loader.py``) use:
|
|||
|
||||
To run the tests on a specific :doc:`tox <tox:index>` environment, use
|
||||
``-e <name>`` with an environment name from ``tox.ini``. For example, to run
|
||||
the tests with Python 3.6 use::
|
||||
the tests with Python 3.7 use::
|
||||
|
||||
tox -e py36
|
||||
tox -e py37
|
||||
|
||||
You can also specify a comma-separated list of environments, and use :ref:`tox’s
|
||||
parallel mode <tox:parallel_mode>` to run the tests on multiple environments in
|
||||
parallel::
|
||||
|
||||
tox -e py36,py38 -p auto
|
||||
tox -e py37,py38 -p auto
|
||||
|
||||
To pass command-line options to :doc:`pytest <pytest:index>`, add them after
|
||||
``--`` in your call to :doc:`tox <tox:index>`. Using ``--`` overrides the
|
||||
|
|
@ -250,9 +250,9 @@ default positional arguments (``scrapy tests``) after ``--`` as well::
|
|||
tox -- scrapy tests -x # stop after first failure
|
||||
|
||||
You can also use the `pytest-xdist`_ plugin. For example, to run all tests on
|
||||
the Python 3.6 :doc:`tox <tox:index>` environment using all your CPU cores::
|
||||
the Python 3.7 :doc:`tox <tox:index>` environment using all your CPU cores::
|
||||
|
||||
tox -e py36 -- scrapy tests -n auto
|
||||
tox -e py37 -- scrapy tests -n auto
|
||||
|
||||
To see coverage report install :doc:`coverage <coverage:index>`
|
||||
(``pip install coverage``) and run:
|
||||
|
|
|
|||
|
|
@ -9,8 +9,8 @@ Installation guide
|
|||
Supported Python versions
|
||||
=========================
|
||||
|
||||
Scrapy requires Python 3.6+, either the CPython implementation (default) or
|
||||
the PyPy 7.2.0+ implementation (see :ref:`python:implementations`).
|
||||
Scrapy requires Python 3.7+, either the CPython implementation (default) or
|
||||
the PyPy 7.3.5+ implementation (see :ref:`python:implementations`).
|
||||
|
||||
.. _intro-install-scrapy:
|
||||
|
||||
|
|
|
|||
|
|
@ -102,11 +102,6 @@ Additionally, ``dataclass`` items also allow to:
|
|||
* define custom field metadata through :func:`dataclasses.field`, which can be used to
|
||||
:ref:`customize serialization <topics-exporters-field-serialization>`.
|
||||
|
||||
They work natively in Python 3.7 or later, or using the `dataclasses
|
||||
backport`_ in Python 3.6.
|
||||
|
||||
.. _dataclasses backport: https://pypi.org/project/dataclasses/
|
||||
|
||||
Example::
|
||||
|
||||
from dataclasses import dataclass
|
||||
|
|
|
|||
|
|
@ -70,7 +70,7 @@ The advantage of using the :class:`ImagesPipeline` for image files is that you
|
|||
can configure some extra functions like generating thumbnails and filtering
|
||||
the images based on their size.
|
||||
|
||||
The Images Pipeline requires Pillow_ 4.0.0 or greater. It is used for
|
||||
The Images Pipeline requires Pillow_ 7.1.0 or greater. It is used for
|
||||
thumbnailing and normalizing images to JPEG/RGB format.
|
||||
|
||||
.. _Pillow: https://github.com/python-pillow/Pillow
|
||||
|
|
|
|||
|
|
@ -28,8 +28,8 @@ twisted_version = (_txv.major, _txv.minor, _txv.micro)
|
|||
|
||||
|
||||
# Check minimum required Python version
|
||||
if sys.version_info < (3, 6):
|
||||
print(f"Scrapy {__version__} requires Python 3.6+")
|
||||
if sys.version_info < (3, 7):
|
||||
print(f"Scrapy {__version__} requires Python 3.7+")
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -1,11 +0,0 @@
|
|||
import warnings
|
||||
|
||||
from scrapy.exceptions import ScrapyDeprecationWarning
|
||||
from scrapy.utils.asyncgen import collect_asyncgen # noqa: F401
|
||||
|
||||
|
||||
warnings.warn(
|
||||
"Module `scrapy.utils.py36` is deprecated, please import from `scrapy.utils.asyncgen` instead.",
|
||||
category=ScrapyDeprecationWarning,
|
||||
stacklevel=2,
|
||||
)
|
||||
19
setup.py
19
setup.py
|
|
@ -19,35 +19,29 @@ def has_environment_marker_platform_impl_support():
|
|||
|
||||
|
||||
install_requires = [
|
||||
'Twisted>=17.9.0',
|
||||
'cryptography>=2.0',
|
||||
'Twisted>=18.9.0',
|
||||
'cryptography>=2.8',
|
||||
'cssselect>=0.9.1',
|
||||
'itemloaders>=1.0.1',
|
||||
'parsel>=1.5.0',
|
||||
'pyOpenSSL>=16.2.0',
|
||||
'pyOpenSSL>=19.1.0',
|
||||
'queuelib>=1.4.2',
|
||||
'service_identity>=16.0.0',
|
||||
'w3lib>=1.17.0',
|
||||
'zope.interface>=4.1.3',
|
||||
'zope.interface>=5.1.0',
|
||||
'protego>=0.1.15',
|
||||
'itemadapter>=0.1.0',
|
||||
'setuptools',
|
||||
'tldextract',
|
||||
'lxml>=4.3.0',
|
||||
]
|
||||
extras_require = {}
|
||||
cpython_dependencies = [
|
||||
'lxml>=3.5.0',
|
||||
'PyDispatcher>=2.0.5',
|
||||
]
|
||||
if has_environment_marker_platform_impl_support():
|
||||
extras_require[':platform_python_implementation == "CPython"'] = cpython_dependencies
|
||||
extras_require[':platform_python_implementation == "PyPy"'] = [
|
||||
# Earlier lxml versions are affected by
|
||||
# https://foss.heptapod.net/pypy/pypy/-/issues/2498,
|
||||
# which was fixed in Cython 0.26, released on 2017-06-19, and used to
|
||||
# generate the C headers of lxml release tarballs published since then, the
|
||||
# first of which was:
|
||||
'lxml>=4.0.0',
|
||||
'PyPyDispatcher>=2.1.0',
|
||||
]
|
||||
else:
|
||||
|
|
@ -84,7 +78,6 @@ setup(
|
|||
'Operating System :: OS Independent',
|
||||
'Programming Language :: Python',
|
||||
'Programming Language :: Python :: 3',
|
||||
'Programming Language :: Python :: 3.6',
|
||||
'Programming Language :: Python :: 3.7',
|
||||
'Programming Language :: Python :: 3.8',
|
||||
'Programming Language :: Python :: 3.9',
|
||||
|
|
@ -95,7 +88,7 @@ setup(
|
|||
'Topic :: Software Development :: Libraries :: Application Frameworks',
|
||||
'Topic :: Software Development :: Libraries :: Python Modules',
|
||||
],
|
||||
python_requires='>=3.6',
|
||||
python_requires='>=3.7',
|
||||
install_requires=install_requires,
|
||||
extras_require=extras_require,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -1,14 +1,12 @@
|
|||
# Tests requirements
|
||||
attrs
|
||||
dataclasses; python_version == '3.6'
|
||||
pyftpdlib
|
||||
pytest
|
||||
pytest-cov==3.0.0
|
||||
pytest-xdist
|
||||
sybil >= 1.3.0 # https://github.com/cjw296/sybil/issues/20#issuecomment-605433422
|
||||
testfixtures
|
||||
uvloop < 0.15.0; platform_system != "Windows" and python_version == '3.6'
|
||||
uvloop; platform_system != "Windows" and python_version > '3.6'
|
||||
uvloop; platform_system != "Windows"
|
||||
|
||||
# optional for shell wrapper tests
|
||||
bpython
|
||||
|
|
|
|||
|
|
@ -3,7 +3,6 @@ import gc
|
|||
import operator
|
||||
import platform
|
||||
import unittest
|
||||
from datetime import datetime
|
||||
from itertools import count
|
||||
from warnings import catch_warnings, filterwarnings
|
||||
|
||||
|
|
@ -224,12 +223,7 @@ class UtilsPythonTestCase(unittest.TestCase):
|
|||
elif platform.python_implementation() == 'PyPy':
|
||||
self.assertEqual(get_func_args(str.split, stripself=True), ['sep', 'maxsplit'])
|
||||
self.assertEqual(get_func_args(operator.itemgetter(2), stripself=True), ['obj'])
|
||||
|
||||
build_date = datetime.strptime(platform.python_build()[1], '%b %d %Y')
|
||||
if build_date >= datetime(2020, 4, 7): # PyPy 3.6-v7.3.1
|
||||
self.assertEqual(get_func_args(" ".join, stripself=True), ['iterable'])
|
||||
else:
|
||||
self.assertEqual(get_func_args(" ".join, stripself=True), ['list'])
|
||||
self.assertEqual(get_func_args(" ".join, stripself=True), ['iterable'])
|
||||
|
||||
def test_without_none_values(self):
|
||||
self.assertEqual(without_none_values([1, None, 3, 4]), [1, 3, 4])
|
||||
|
|
|
|||
23
tox.ini
23
tox.ini
|
|
@ -11,15 +11,13 @@ minversion = 1.7.0
|
|||
deps =
|
||||
-rtests/requirements.txt
|
||||
# mitmproxy does not support PyPy
|
||||
# mitmproxy does not support Windows when running Python < 3.7
|
||||
# Python 3.9+ requires mitmproxy >= 5.3.0
|
||||
# mitmproxy >= 5.3.0 requires h2 >= 4.0, Twisted 21.2 requires h2 < 4.0
|
||||
#mitmproxy >= 5.3.0; python_version >= '3.9' and implementation_name != 'pypy'
|
||||
# The tests hang with mitmproxy 8.0.0: https://github.com/scrapy/scrapy/issues/5454
|
||||
mitmproxy >= 4.0.4, < 8; python_version >= '3.7' and python_version < '3.9' and implementation_name != 'pypy'
|
||||
mitmproxy >= 4.0.4, < 5; python_version >= '3.6' and python_version < '3.7' and platform_system != 'Windows' and implementation_name != 'pypy'
|
||||
mitmproxy >= 4.0.4, < 8; python_version < '3.9' and implementation_name != 'pypy'
|
||||
# newer markupsafe is incompatible with deps of old mitmproxy (which we get on Python 3.7 and lower)
|
||||
markupsafe < 2.1.0; python_version >= '3.6' and python_version < '3.8' and implementation_name != 'pypy'
|
||||
markupsafe < 2.1.0; python_version < '3.8' and implementation_name != 'pypy'
|
||||
# Extras
|
||||
botocore>=1.4.87
|
||||
passenv =
|
||||
|
|
@ -44,7 +42,6 @@ deps =
|
|||
types-pyOpenSSL==20.0.3
|
||||
types-setuptools==57.0.0
|
||||
commands =
|
||||
pip install types-dataclasses # remove once py36 support is dropped
|
||||
mypy --show-error-codes {posargs: scrapy tests}
|
||||
|
||||
[testenv:security]
|
||||
|
|
@ -75,18 +72,19 @@ commands =
|
|||
|
||||
[pinned]
|
||||
deps =
|
||||
cryptography==2.0
|
||||
cryptography==2.8
|
||||
cssselect==0.9.1
|
||||
h2==3.0
|
||||
itemadapter==0.1.0
|
||||
parsel==1.5.0
|
||||
Protego==0.1.15
|
||||
pyOpenSSL==16.2.0
|
||||
pyOpenSSL==19.1.0
|
||||
queuelib==1.4.2
|
||||
service_identity==16.0.0
|
||||
Twisted[http2]==17.9.0
|
||||
Twisted[http2]==18.9.0
|
||||
w3lib==1.17.0
|
||||
zope.interface==4.1.3
|
||||
zope.interface==5.1.0
|
||||
lxml==4.3.0
|
||||
-rtests/requirements.txt
|
||||
|
||||
# mitmproxy 4.0.4+ requires upgrading some of the pinned dependencies
|
||||
|
|
@ -95,7 +93,7 @@ deps =
|
|||
# Extras
|
||||
botocore==1.4.87
|
||||
google-cloud-storage==1.29.0
|
||||
Pillow==4.0.0
|
||||
Pillow==7.1.0
|
||||
setenv =
|
||||
_SCRAPY_PINNED=true
|
||||
install_command =
|
||||
|
|
@ -104,7 +102,6 @@ install_command =
|
|||
[testenv:pinned]
|
||||
deps =
|
||||
{[pinned]deps}
|
||||
lxml==3.5.0
|
||||
PyDispatcher==2.0.5
|
||||
install_command = {[pinned]install_command}
|
||||
setenv =
|
||||
|
|
@ -114,9 +111,6 @@ setenv =
|
|||
basepython = python3
|
||||
deps =
|
||||
{[pinned]deps}
|
||||
# First lxml version that includes a Windows wheel for Python 3.6, so we do
|
||||
# not need to build lxml from sources in a CI Windows job:
|
||||
lxml==3.8.0
|
||||
PyDispatcher==2.0.5
|
||||
install_command = {[pinned]install_command}
|
||||
setenv =
|
||||
|
|
@ -155,7 +149,6 @@ commands =
|
|||
basepython = {[testenv:pypy3]basepython}
|
||||
deps =
|
||||
{[pinned]deps}
|
||||
lxml==4.0.0
|
||||
PyPyDispatcher==2.1.0
|
||||
commands = {[testenv:pypy3]commands}
|
||||
install_command = {[pinned]install_command}
|
||||
|
|
|
|||
Loading…
Reference in New Issue