From 2122278d4bb7177a550bc0b04dd96d1fc1fad1a0 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sun, 18 Jun 2023 18:37:50 +0400 Subject: [PATCH 1/3] Drop Python 3.7 support. --- .github/workflows/tests-macos.yml | 2 +- .github/workflows/tests-ubuntu.yml | 13 +++++-------- .github/workflows/tests-windows.yml | 5 +---- README.rst | 2 +- docs/contributing.rst | 12 ++++++------ docs/intro/install.rst | 2 +- scrapy/__init__.py | 4 ++-- setup.py | 3 +-- tox.ini | 8 +++----- 9 files changed, 21 insertions(+), 30 deletions(-) diff --git a/.github/workflows/tests-macos.yml b/.github/workflows/tests-macos.yml index 174d245ca..3044a1af3 100644 --- a/.github/workflows/tests-macos.yml +++ b/.github/workflows/tests-macos.yml @@ -7,7 +7,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"] + python-version: ["3.8", "3.9", "3.10", "3.11"] steps: - uses: actions/checkout@v3 diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml index 96b26a1f8..39e3b0af7 100644 --- a/.github/workflows/tests-ubuntu.yml +++ b/.github/workflows/tests-ubuntu.yml @@ -8,9 +8,6 @@ jobs: fail-fast: false matrix: include: - - python-version: 3.8 - env: - TOXENV: py - python-version: 3.9 env: TOXENV: py @@ -28,19 +25,19 @@ jobs: TOXENV: pypy3 # pinned deps - - python-version: 3.7.13 + - python-version: 3.8.17 env: TOXENV: pinned - - python-version: 3.7.13 + - python-version: 3.8.17 env: TOXENV: asyncio-pinned - - python-version: pypy3.7 + - python-version: pypy3.8 env: TOXENV: pypy3-pinned - - python-version: 3.7.13 + - python-version: 3.8.17 env: TOXENV: extra-deps-pinned - - python-version: 3.7.13 + - python-version: 3.8.17 env: TOXENV: botocore-pinned diff --git a/.github/workflows/tests-windows.yml b/.github/workflows/tests-windows.yml index f60c48841..5bcf74d5e 100644 --- a/.github/workflows/tests-windows.yml +++ b/.github/workflows/tests-windows.yml @@ -8,12 +8,9 @@ jobs: fail-fast: false matrix: include: - - python-version: 3.7 - env: - TOXENV: windows-pinned - python-version: 3.8 env: - TOXENV: py + TOXENV: windows-pinned - python-version: 3.9 env: TOXENV: py diff --git a/README.rst b/README.rst index 970bf2c35..1918850d6 100644 --- a/README.rst +++ b/README.rst @@ -58,7 +58,7 @@ including a list of features. Requirements ============ -* Python 3.7+ +* Python 3.8+ * Works on Linux, Windows, macOS, BSD Install diff --git a/docs/contributing.rst b/docs/contributing.rst index eef92e148..2b3249601 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -265,15 +265,15 @@ To run a specific test (say ``tests/test_loader.py``) use: To run the tests on a specific :doc:`tox ` environment, use ``-e `` with an environment name from ``tox.ini``. For example, to run -the tests with Python 3.7 use:: +the tests with Python 3.10 use:: - tox -e py37 + tox -e py310 You can also specify a comma-separated list of environments, and use :ref:`tox’s parallel mode ` to run the tests on multiple environments in parallel:: - tox -e py37,py38 -p auto + tox -e py39,py310 -p auto To pass command-line options to :doc:`pytest `, add them after ``--`` in your call to :doc:`tox `. Using ``--`` overrides the @@ -283,9 +283,9 @@ default positional arguments (``scrapy tests``) after ``--`` as well:: tox -- scrapy tests -x # stop after first failure You can also use the `pytest-xdist`_ plugin. For example, to run all tests on -the Python 3.7 :doc:`tox ` environment using all your CPU cores:: +the Python 3.10 :doc:`tox ` environment using all your CPU cores:: - tox -e py37 -- scrapy tests -n auto + tox -e py310 -- scrapy tests -n auto To see coverage report install :doc:`coverage ` (``pip install coverage``) and run: @@ -322,4 +322,4 @@ And their unit-tests are in:: .. _pytest-xdist: https://github.com/pytest-dev/pytest-xdist .. _good first issues: https://github.com/scrapy/scrapy/issues?q=is%3Aissue+is%3Aopen+label%3A%22good+first+issue%22 .. _help wanted issues: https://github.com/scrapy/scrapy/issues?q=is%3Aissue+is%3Aopen+label%3A%22help+wanted%22 -.. _test coverage: https://app.codecov.io/gh/scrapy/scrapy \ No newline at end of file +.. _test coverage: https://app.codecov.io/gh/scrapy/scrapy diff --git a/docs/intro/install.rst b/docs/intro/install.rst index 2c2079f68..c90c1d2bf 100644 --- a/docs/intro/install.rst +++ b/docs/intro/install.rst @@ -9,7 +9,7 @@ Installation guide Supported Python versions ========================= -Scrapy requires Python 3.7+, either the CPython implementation (default) or +Scrapy requires Python 3.8+, either the CPython implementation (default) or the PyPy implementation (see :ref:`python:implementations`). .. _intro-install-scrapy: diff --git a/scrapy/__init__.py b/scrapy/__init__.py index a757a9290..cc0e539c4 100644 --- a/scrapy/__init__.py +++ b/scrapy/__init__.py @@ -34,8 +34,8 @@ twisted_version = (_txv.major, _txv.minor, _txv.micro) # Check minimum required Python version -if sys.version_info < (3, 7): - print(f"Scrapy {__version__} requires Python 3.7+") +if sys.version_info < (3, 8): + print(f"Scrapy {__version__} requires Python 3.8+") sys.exit(1) diff --git a/setup.py b/setup.py index c6bcf2439..f1cd4c5e2 100644 --- a/setup.py +++ b/setup.py @@ -80,7 +80,6 @@ setup( "Operating System :: OS Independent", "Programming Language :: Python", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", @@ -91,7 +90,7 @@ setup( "Topic :: Software Development :: Libraries :: Application Frameworks", "Topic :: Software Development :: Libraries :: Python Modules", ], - python_requires=">=3.7", + python_requires=">=3.8", install_requires=install_requires, extras_require=extras_require, ) diff --git a/tox.ini b/tox.ini index 5f8bf85f2..d5b6118f5 100644 --- a/tox.ini +++ b/tox.ini @@ -16,8 +16,6 @@ deps = #mitmproxy >= 5.3.0; python_version >= '3.9' and implementation_name != 'pypy' # The tests hang with mitmproxy 8.0.0: https://github.com/scrapy/scrapy/issues/5454 mitmproxy >= 4.0.4, < 8; python_version < '3.9' and implementation_name != 'pypy' - # newer markupsafe is incompatible with deps of old mitmproxy (which we get on Python 3.7 and lower) - markupsafe < 2.1.0; python_version < '3.8' and implementation_name != 'pypy' passenv = S3_TEST_FILE_URI AWS_ACCESS_KEY_ID @@ -96,7 +94,7 @@ commands = pytest --cov=scrapy --cov-report=xml --cov-report= {posargs:--durations=10 scrapy tests} [testenv:pinned] -basepython = python3.7 +basepython = python3.8 deps = {[pinned]deps} PyDispatcher==2.0.5 @@ -129,7 +127,7 @@ deps = Twisted[http2] [testenv:extra-deps-pinned] -basepython = python3.7 +basepython = python3.8 deps = {[pinned]deps} boto3==1.20.0 @@ -211,7 +209,7 @@ commands = pytest --cov=scrapy --cov-report=xml --cov-report= {posargs:tests -k s3} [testenv:botocore-pinned] -basepython = python3.7 +basepython = python3.8 deps = {[pinned]deps} botocore==1.4.87 From 1b2c9a3e0ae9766623a06ce7e739a3dfda399159 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sun, 18 Jun 2023 18:38:56 +0400 Subject: [PATCH 2/3] Bump isort and flake8 versions. --- .pre-commit-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index faf8808f2..31e9ed1ad 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -5,7 +5,7 @@ repos: - id: bandit args: [-r, -c, .bandit.yml] - repo: https://github.com/PyCQA/flake8 - rev: 5.0.4 # 6.0.0 drops Python 3.7 support + rev: 6.0.0 hooks: - id: flake8 - repo: https://github.com/psf/black.git @@ -13,7 +13,7 @@ repos: hooks: - id: black - repo: https://github.com/pycqa/isort - rev: 5.11.5 # 5.12 drops Python 3.7 support + rev: 5.12.0 hooks: - id: isort - repo: https://github.com/adamchainz/blacken-docs From 075b89eab5e201246a5bddc4a6ce9c7921de082b Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sun, 18 Jun 2023 19:08:41 +0400 Subject: [PATCH 3/3] Bump lxml and cryptography to versions with 3.8 wheels available. --- setup.py | 4 ++-- tox.ini | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/setup.py b/setup.py index f1cd4c5e2..ccfe20ae5 100644 --- a/setup.py +++ b/setup.py @@ -20,7 +20,7 @@ def has_environment_marker_platform_impl_support(): install_requires = [ "Twisted>=18.9.0", - "cryptography>=3.4.6", + "cryptography>=36.0.0", "cssselect>=0.9.1", "itemloaders>=1.0.1", "parsel>=1.5.0", @@ -34,7 +34,7 @@ install_requires = [ "setuptools", "packaging", "tldextract", - "lxml>=4.3.0", + "lxml>=4.4.1", ] extras_require = {} cpython_dependencies = [ diff --git a/tox.ini b/tox.ini index d5b6118f5..ec3a59366 100644 --- a/tox.ini +++ b/tox.ini @@ -69,7 +69,7 @@ commands = [pinned] deps = - cryptography==3.4.6 + cryptography==36.0.0 cssselect==0.9.1 h2==3.0 itemadapter==0.1.0 @@ -81,7 +81,7 @@ deps = Twisted[http2]==18.9.0 w3lib==1.17.0 zope.interface==5.1.0 - lxml==4.3.0 + lxml==4.4.1 -rtests/requirements.txt # mitmproxy 4.0.4+ requires upgrading some of the pinned dependencies