From 0097b4c0bb4de6e651e8b9d064aae140e11698d5 Mon Sep 17 00:00:00 2001 From: Laerte Pereira Date: Tue, 20 Jun 2023 16:40:38 -0300 Subject: [PATCH 1/6] cleanup: Remove `pkg_resources` usage --- docs/topics/components.rst | 2 +- scrapy/cmdline.py | 5 ++--- setup.py | 2 +- tests/test_crawler.py | 2 +- 4 files changed, 5 insertions(+), 6 deletions(-) diff --git a/docs/topics/components.rst b/docs/topics/components.rst index 1ed55f000..478dd9647 100644 --- a/docs/topics/components.rst +++ b/docs/topics/components.rst @@ -70,7 +70,7 @@ If your requirement is a minimum Scrapy version, you may use .. code-block:: python - from pkg_resources import parse_version + from packaging.version import parse as parse_version import scrapy diff --git a/scrapy/cmdline.py b/scrapy/cmdline.py index 730e55350..cfa771104 100644 --- a/scrapy/cmdline.py +++ b/scrapy/cmdline.py @@ -3,8 +3,7 @@ import cProfile import inspect import os import sys - -import pkg_resources +from importlib.metadata import entry_points import scrapy from scrapy.commands import BaseRunSpiderCommand, ScrapyCommand, ScrapyHelpFormatter @@ -49,7 +48,7 @@ def _get_commands_from_module(module, inproject): def _get_commands_from_entry_points(inproject, group="scrapy.commands"): cmds = {} - for entry_point in pkg_resources.iter_entry_points(group): + for entry_point in entry_points(group): obj = entry_point.load() if inspect.isclass(obj): cmds[entry_point.name] = obj() diff --git a/setup.py b/setup.py index ccfe20ae5..f918db09e 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,6 @@ from pathlib import Path -from pkg_resources import parse_version +from packaging.version import parse as parse_version from setuptools import __version__ as setuptools_version from setuptools import find_packages, setup diff --git a/tests/test_crawler.py b/tests/test_crawler.py index ecb9c9b62..d54a2cb7e 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -6,7 +6,7 @@ import sys import warnings from pathlib import Path -from pkg_resources import parse_version +from packaging.version import parse as parse_version from pytest import mark, raises from twisted import version as twisted_version from twisted.internet import defer From 6afb31b82b5a0a5d2f37962c250fbf34c21d8580 Mon Sep 17 00:00:00 2001 From: Laerte Pereira Date: Tue, 20 Jun 2023 16:48:07 -0300 Subject: [PATCH 2/6] chore: Add `packaging` to tests deps --- tests/requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/requirements.txt b/tests/requirements.txt index 618949795..72350b216 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -14,3 +14,4 @@ brotli # optional for HTTP compress downloader middleware tests zstandard; implementation_name != 'pypy' # optional for HTTP compress downloader middleware tests ipython pywin32; sys_platform == "win32" +packaging \ No newline at end of file From 6e1af20ac4dd537a4643df5c022f948cf07d05ec Mon Sep 17 00:00:00 2001 From: Laerte Pereira Date: Tue, 20 Jun 2023 17:00:01 -0300 Subject: [PATCH 3/6] fix: add `build-system` --- tox.ini | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tox.ini b/tox.ini index ec3a59366..79d692599 100644 --- a/tox.ini +++ b/tox.ini @@ -218,3 +218,6 @@ setenv = {[pinned]setenv} commands = pytest --cov=scrapy --cov-report=xml --cov-report= {posargs:tests -k s3} + +[build-system] +build-backend = 'setuptools.build_meta' \ No newline at end of file From a93a63c208af1d13d5ea84623d160337c7fec6c5 Mon Sep 17 00:00:00 2001 From: Laerte Pereira Date: Tue, 20 Jun 2023 17:05:49 -0300 Subject: [PATCH 4/6] fix: move import to inside function --- setup.py | 3 ++- tests/requirements.txt | 1 - tox.ini | 3 --- 3 files changed, 2 insertions(+), 5 deletions(-) diff --git a/setup.py b/setup.py index f918db09e..dfe5b80ec 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,5 @@ from pathlib import Path -from packaging.version import parse as parse_version from setuptools import __version__ as setuptools_version from setuptools import find_packages, setup @@ -15,6 +14,8 @@ def has_environment_marker_platform_impl_support(): it is 18.5, see: https://setuptools.readthedocs.io/en/latest/history.html#id235 """ + from packaging.version import parse as parse_version + return parse_version(setuptools_version) >= parse_version("18.5") diff --git a/tests/requirements.txt b/tests/requirements.txt index 72350b216..618949795 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -14,4 +14,3 @@ brotli # optional for HTTP compress downloader middleware tests zstandard; implementation_name != 'pypy' # optional for HTTP compress downloader middleware tests ipython pywin32; sys_platform == "win32" -packaging \ No newline at end of file diff --git a/tox.ini b/tox.ini index 79d692599..ec3a59366 100644 --- a/tox.ini +++ b/tox.ini @@ -218,6 +218,3 @@ setenv = {[pinned]setenv} commands = pytest --cov=scrapy --cov-report=xml --cov-report= {posargs:tests -k s3} - -[build-system] -build-backend = 'setuptools.build_meta' \ No newline at end of file From 0b1da44a05cc64970aa11ccc4d7a4a3bec143443 Mon Sep 17 00:00:00 2001 From: Laerte Pereira Date: Tue, 20 Jun 2023 17:14:21 -0300 Subject: [PATCH 5/6] chore: Remove deprecated code --- setup.py | 31 ++++--------------------------- 1 file changed, 4 insertions(+), 27 deletions(-) diff --git a/setup.py b/setup.py index dfe5b80ec..1f214571b 100644 --- a/setup.py +++ b/setup.py @@ -1,24 +1,10 @@ from pathlib import Path -from setuptools import __version__ as setuptools_version from setuptools import find_packages, setup version = (Path(__file__).parent / "scrapy/VERSION").read_text("ascii").strip() -def has_environment_marker_platform_impl_support(): - """Code extracted from 'pytest/setup.py' - https://github.com/pytest-dev/pytest/blob/7538680c/setup.py#L31 - - The first known release to support environment marker with range operators - it is 18.5, see: - https://setuptools.readthedocs.io/en/latest/history.html#id235 - """ - from packaging.version import parse as parse_version - - return parse_version(setuptools_version) >= parse_version("18.5") - - install_requires = [ "Twisted>=18.9.0", "cryptography>=36.0.0", @@ -37,19 +23,10 @@ install_requires = [ "tldextract", "lxml>=4.4.1", ] -extras_require = {} -cpython_dependencies = [ - "PyDispatcher>=2.0.5", -] -if has_environment_marker_platform_impl_support(): - extras_require[ - ':platform_python_implementation == "CPython"' - ] = cpython_dependencies - extras_require[':platform_python_implementation == "PyPy"'] = [ - "PyPyDispatcher>=2.1.0", - ] -else: - install_requires.extend(cpython_dependencies) +extras_require = { + ':platform_python_implementation == "CPython"': ["PyDispatcher>=2.0.5"], + ':platform_python_implementation == "PyPy"': ["PyPyDispatcher>=2.1.0"], +} setup( From 82cf00bbc931723320c9aca3cf0305372027d0a0 Mon Sep 17 00:00:00 2001 From: Laerte Pereira Date: Tue, 20 Jun 2023 18:27:03 -0300 Subject: [PATCH 6/6] fix: default value --- scrapy/cmdline.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/cmdline.py b/scrapy/cmdline.py index cfa771104..efc9b36ea 100644 --- a/scrapy/cmdline.py +++ b/scrapy/cmdline.py @@ -48,7 +48,7 @@ def _get_commands_from_module(module, inproject): def _get_commands_from_entry_points(inproject, group="scrapy.commands"): cmds = {} - for entry_point in entry_points(group): + for entry_point in entry_points().get(group, {}): obj = entry_point.load() if inspect.isclass(obj): cmds[entry_point.name] = obj()