From 8c23da943c5e892515f4fa2eb57229839802010a Mon Sep 17 00:00:00 2001 From: Swayam Gupta <78016781+swayam0322@users.noreply.github.com> Date: Tue, 19 Nov 2024 19:51:15 +0530 Subject: [PATCH] Integrating configs into pyproject.toml (#6547) --- .bandit.yml | 7 - .bumpversion.cfg | 11 -- .coveragerc | 12 -- .isort.cfg | 2 - .pre-commit-config.yaml | 3 +- MANIFEST.in | 1 - pylintrc | 73 --------- pyproject.toml | 235 ++++++++++++++++++++++++++++ pytest.ini | 26 --- setup.cfg | 24 --- setup.py | 75 --------- tests/test_crawler.py | 2 +- tests/test_spiderloader/__init__.py | 7 +- tox.ini | 2 +- 14 files changed, 243 insertions(+), 237 deletions(-) delete mode 100644 .bandit.yml delete mode 100644 .bumpversion.cfg delete mode 100644 .coveragerc delete mode 100644 .isort.cfg delete mode 100644 pylintrc create mode 100644 pyproject.toml delete mode 100644 pytest.ini delete mode 100644 setup.cfg delete mode 100644 setup.py diff --git a/.bandit.yml b/.bandit.yml deleted file mode 100644 index b7f1817e0..000000000 --- a/.bandit.yml +++ /dev/null @@ -1,7 +0,0 @@ -skips: -- B101 # assert_used, needed for mypy -- B321 # ftplib, https://github.com/scrapy/scrapy/issues/4180 -- B402 # import_ftplib, https://github.com/scrapy/scrapy/issues/4180 -- B411 # import_xmlrpclib, https://github.com/PyCQA/bandit/issues/1082 -- B503 # ssl_with_bad_defaults -exclude_dirs: ['tests'] diff --git a/.bumpversion.cfg b/.bumpversion.cfg deleted file mode 100644 index f83e3e890..000000000 --- a/.bumpversion.cfg +++ /dev/null @@ -1,11 +0,0 @@ -[bumpversion] -current_version = 2.12.0 -commit = True -tag = True -tag_name = {new_version} - -[bumpversion:file:scrapy/VERSION] - -[bumpversion:file:SECURITY.md] -parse = (?P\d+)\.(?P\d+)\.x -serialize = {major}.{minor}.x diff --git a/.coveragerc b/.coveragerc deleted file mode 100644 index f9ad353d5..000000000 --- a/.coveragerc +++ /dev/null @@ -1,12 +0,0 @@ -[run] -branch = true -include = scrapy/* -omit = - tests/* -disable_warnings = include-ignored - -[report] -# https://github.com/nedbat/coveragepy/issues/831#issuecomment-517778185 -exclude_lines = - pragma: no cover - if TYPE_CHECKING: diff --git a/.isort.cfg b/.isort.cfg deleted file mode 100644 index f238bf7ea..000000000 --- a/.isort.cfg +++ /dev/null @@ -1,2 +0,0 @@ -[settings] -profile = black diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index fbd710f6f..b411f4927 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -3,7 +3,8 @@ repos: rev: 1.7.9 hooks: - id: bandit - args: [-r, -c, .bandit.yml] + args: ["-c", "pyproject.toml"] + additional_dependencies: ["bandit[toml]"] - repo: https://github.com/PyCQA/flake8 rev: 7.1.0 hooks: diff --git a/MANIFEST.in b/MANIFEST.in index 06971e39c..7700ae7bd 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -10,7 +10,6 @@ include scrapy/py.typed include codecov.yml include conftest.py -include pytest.ini include tox.ini recursive-include scrapy/templates * diff --git a/pylintrc b/pylintrc deleted file mode 100644 index e927b903c..000000000 --- a/pylintrc +++ /dev/null @@ -1,73 +0,0 @@ -[MASTER] -persistent=no -jobs=1 # >1 hides results - -[MESSAGES CONTROL] -disable=abstract-method, - arguments-differ, - arguments-renamed, - attribute-defined-outside-init, - bad-classmethod-argument, - bare-except, - broad-except, - broad-exception-raised, - c-extension-no-member, - consider-using-with, - cyclic-import, - dangerous-default-value, - disallowed-name, - duplicate-code, # https://github.com/PyCQA/pylint/issues/214 - eval-used, - fixme, - function-redefined, - global-statement, - implicit-str-concat, - import-error, - import-outside-toplevel, - inherit-non-class, - invalid-name, - invalid-overridden-method, - isinstance-second-argument-not-valid-type, - keyword-arg-before-vararg, - line-too-long, - logging-format-interpolation, - logging-fstring-interpolation, - logging-not-lazy, - lost-exception, - missing-docstring, - no-member, - no-method-argument, - no-name-in-module, - no-self-argument, - no-value-for-parameter, # https://github.com/pylint-dev/pylint/issues/3268 - not-callable, - pointless-statement, - pointless-string-statement, - protected-access, - raise-missing-from, - redefined-builtin, - redefined-outer-name, - reimported, - signature-differs, - too-few-public-methods, - too-many-ancestors, - too-many-arguments, - too-many-branches, - too-many-format-args, - too-many-function-args, - too-many-instance-attributes, - too-many-lines, - too-many-locals, - too-many-public-methods, - too-many-return-statements, - unbalanced-tuple-unpacking, - unnecessary-dunder-call, - unnecessary-pass, - unreachable, - unused-argument, - unused-import, - unused-variable, - used-before-assignment, - useless-return, - wildcard-import, - wrong-import-position diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 000000000..f25715e76 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,235 @@ +[build-system] +requires = ["setuptools >= 61.0"] +build-backend = "setuptools.build_meta" + +[project] +name = "Scrapy" +dynamic = ["version"] +description = "A high-level Web Crawling and Web Scraping framework" +dependencies = [ + "Twisted>=21.7.0", + "cryptography>=37.0.0", + "cssselect>=0.9.1", + "itemloaders>=1.0.1", + "parsel>=1.5.0", + "pyOpenSSL>=22.0.0", + "queuelib>=1.4.2", + "service_identity>=18.1.0", + "w3lib>=1.17.0", + "zope.interface>=5.1.0", + "protego>=0.1.15", + "itemadapter>=0.1.0", + "packaging", + "tldextract", + "lxml>=4.6.0", + "defusedxml>=0.7.1", + # Platform-specific dependencies + 'PyDispatcher>=2.0.5; platform_python_implementation == "CPython"', + 'PyPyDispatcher>=2.1.0; platform_python_implementation == "PyPy"', +] +classifiers = [ + "Framework :: Scrapy", + "Development Status :: 5 - Production/Stable", + "Environment :: Console", + "Intended Audience :: Developers", + "License :: OSI Approved :: BSD License", + "Operating System :: OS Independent", + "Programming Language :: Python", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: Implementation :: CPython", + "Programming Language :: Python :: Implementation :: PyPy", + "Topic :: Internet :: WWW/HTTP", + "Topic :: Software Development :: Libraries :: Application Frameworks", + "Topic :: Software Development :: Libraries :: Python Modules", +] +readme = "README.rst" +requires-python = ">=3.9" +authors = [{ name = "Scrapy developers", email = "pablo@pablohoffman.com" }] +maintainers = [{ name = "Pablo Hoffman", email = "pablo@pablohoffman.com" }] + +[project.urls] +Homepage = "https://scrapy.org/" +Documentation = "https://docs.scrapy.org/" +Source = "https://github.com/scrapy/scrapy" +Tracker = "https://github.com/scrapy/scrapy/issues" +Changelog = "https://github.com/scrapy/scrapy/commits/master/" +releasenotes = "https://docs.scrapy.org/en/latest/news.html" + +[project.scripts] +scrapy = "scrapy.cmdline:execute" + +[tool.setuptools.packages.find] +where = ["."] +include = ["scrapy", "scrapy.*",] + +[tool.setuptools.dynamic] +version = {file = "./scrapy/VERSION"} + +[tool.mypy] +ignore_missing_imports = true + +# Interface classes are hard to support + +[[tool.mypy.overrides]] +module = "twisted.internet.interfaces" +follow_imports = "skip" + +[[tool.mypy.overrides]] +module = "scrapy.interfaces" +ignore_errors = true + +[[tool.mypy.overrides]] +module = "twisted.internet.reactor" +follow_imports = "skip" + +# FIXME: remove the following section once the issues are solved +[[tool.mypy.overrides]] +module = "scrapy.settings.default_settings" +ignore_errors = true + +[tool.bandit] +skips = [ + "B101", # assert_used, needed for mypy + "B321", # ftplib, https://github.com/scrapy/scrapy/issues/4180 + "B402", # import_ftplib, https://github.com/scrapy/scrapy/issues/4180 + "B411", # import_xmlrpclib, https://github.com/PyCQA/bandit/issues/1082 + "B503", # ssl_with_bad_defaults +] +exclude_dirs = ["tests"] + +[tool.bumpversion] +current_version = "2.12.0" +commit = true +tag = true +tag_name = "{new_version}" + +[[tool.bumpversion.files]] +filename = "scrapy/VERSION" + +[[tool.bumpversion.files]] +filename = "SECURITY.md" +parse = """(?P0|[1-9]\\d*)\\.(?P0|[1-9]\\d*)""" +serialize = ["{major}.{minor}"] + +[tool.coverage.run] +branch = true +include = ["scrapy/*"] +omit = ["tests/*"] +disable_warnings = ["include-ignored"] + +[tool.coverage.report] +# https://github.com/nedbat/coveragepy/issues/831#issuecomment-517778185 +exclude_lines = ["pragma: no cover", "if TYPE_CHECKING:"] + +[tool.isort] +profile = "black" + +[tool.pylint.MASTER] +persistent = "no" +jobs = 1 # >1 hides results + +[tool.pylint."MESSAGES CONTROL"] +disable = [ + "abstract-method", + "arguments-differ", + "arguments-renamed", + "attribute-defined-outside-init", + "bad-classmethod-argument", + "bare-except", + "broad-except", + "broad-exception-raised", + "c-extension-no-member", + "consider-using-with", + "cyclic-import", + "dangerous-default-value", + "disallowed-name", + "duplicate-code", # https://github.com/PyCQA/pylint/issues/214 + "eval-used", + "fixme", + "function-redefined", + "global-statement", + "implicit-str-concat", + "import-error", + "import-outside-toplevel", + "inherit-non-class", + "invalid-name", + "invalid-overridden-method", + "isinstance-second-argument-not-valid-type", + "keyword-arg-before-vararg", + "line-too-long", + "logging-format-interpolation", + "logging-fstring-interpolation", + "logging-not-lazy", + "lost-exception", + "missing-docstring", + "no-member", + "no-method-argument", + "no-name-in-module", + "no-self-argument", + "no-value-for-parameter", # https://github.com/pylint-dev/pylint/issues/3268 + "not-callable", + "pointless-statement", + "pointless-string-statement", + "protected-access", + "raise-missing-from", + "redefined-builtin", + "redefined-outer-name", + "reimported", + "signature-differs", + "too-few-public-methods", + "too-many-ancestors", + "too-many-arguments", + "too-many-branches", + "too-many-format-args", + "too-many-function-args", + "too-many-instance-attributes", + "too-many-lines", + "too-many-locals", + "too-many-public-methods", + "too-many-return-statements", + "unbalanced-tuple-unpacking", + "unnecessary-dunder-call", + "unnecessary-pass", + "unreachable", + "unused-argument", + "unused-import", + "unused-variable", + "used-before-assignment", + "useless-return", + "wildcard-import", + "wrong-import-position", +] + +[tool.pytest.ini_options] +xfail_strict = true +usefixtures = "chdir" +python_files = ["test_*.py", "__init__.py"] +python_classes = [] +addopts = [ + "--assert=plain", + "--ignore=docs/_ext", + "--ignore=docs/conf.py", + "--ignore=docs/news.rst", + "--ignore=docs/topics/dynamic-content.rst", + "--ignore=docs/topics/items.rst", + "--ignore=docs/topics/leaks.rst", + "--ignore=docs/topics/loaders.rst", + "--ignore=docs/topics/selectors.rst", + "--ignore=docs/topics/shell.rst", + "--ignore=docs/topics/stats.rst", + "--ignore=docs/topics/telnetconsole.rst", + "--ignore=docs/utils", +] +markers = [ + "only_asyncio: marks tests as only enabled when --reactor=asyncio is passed", + "only_not_asyncio: marks tests as only enabled when --reactor=asyncio is not passed", + "requires_uvloop: marks tests as only enabled when uvloop is known to be working", + "requires_botocore: marks tests that need botocore (but not boto3)", + "requires_boto3: marks tests that need botocore and boto3", +] +filterwarnings = [] \ No newline at end of file diff --git a/pytest.ini b/pytest.ini deleted file mode 100644 index 824c0e9e9..000000000 --- a/pytest.ini +++ /dev/null @@ -1,26 +0,0 @@ -[pytest] -xfail_strict = true -usefixtures = chdir -python_files=test_*.py __init__.py -python_classes= -addopts = - --assert=plain - --ignore=docs/_ext - --ignore=docs/conf.py - --ignore=docs/news.rst - --ignore=docs/topics/dynamic-content.rst - --ignore=docs/topics/items.rst - --ignore=docs/topics/leaks.rst - --ignore=docs/topics/loaders.rst - --ignore=docs/topics/selectors.rst - --ignore=docs/topics/shell.rst - --ignore=docs/topics/stats.rst - --ignore=docs/topics/telnetconsole.rst - --ignore=docs/utils -markers = - only_asyncio: marks tests as only enabled when --reactor=asyncio is passed - only_not_asyncio: marks tests as only enabled when --reactor=asyncio is not passed - requires_uvloop: marks tests as only enabled when uvloop is known to be working - requires_botocore: marks tests that need botocore (but not boto3) - requires_boto3: marks tests that need botocore and boto3 -filterwarnings = diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index 151e784c6..000000000 --- a/setup.cfg +++ /dev/null @@ -1,24 +0,0 @@ -[bdist_rpm] -doc_files = docs AUTHORS INSTALL LICENSE README.rst - -[bdist_wheel] -universal=1 - -[mypy] -ignore_missing_imports = true - -# Interface classes are hard to support - -[mypy-twisted.internet.interfaces] -follow_imports = skip - -[mypy-scrapy.interfaces] -ignore_errors = True - -[mypy-twisted.internet.reactor] -follow_imports = skip - -# FIXME: remove the following sections once the issues are solved - -[mypy-scrapy.settings.default_settings] -ignore_errors = True diff --git a/setup.py b/setup.py deleted file mode 100644 index 6cc1150a5..000000000 --- a/setup.py +++ /dev/null @@ -1,75 +0,0 @@ -from pathlib import Path - -from setuptools import find_packages, setup - -version = (Path(__file__).parent / "scrapy/VERSION").read_text("ascii").strip() - - -install_requires = [ - "Twisted>=21.7.0", - "cryptography>=37.0.0", - "cssselect>=0.9.1", - "itemloaders>=1.0.1", - "parsel>=1.5.0", - "pyOpenSSL>=22.0.0", - "queuelib>=1.4.2", - "service_identity>=18.1.0", - "w3lib>=1.17.0", - "zope.interface>=5.1.0", - "protego>=0.1.15", - "itemadapter>=0.1.0", - "packaging", - "tldextract", - "lxml>=4.6.0", - "defusedxml>=0.7.1", -] -extras_require = { - ':platform_python_implementation == "CPython"': ["PyDispatcher>=2.0.5"], - ':platform_python_implementation == "PyPy"': ["PyPyDispatcher>=2.1.0"], -} - - -setup( - name="Scrapy", - version=version, - url="https://scrapy.org", - project_urls={ - "Documentation": "https://docs.scrapy.org/", - "Source": "https://github.com/scrapy/scrapy", - "Tracker": "https://github.com/scrapy/scrapy/issues", - }, - description="A high-level Web Crawling and Web Scraping framework", - long_description=open("README.rst", encoding="utf-8").read(), - author="Scrapy developers", - author_email="pablo@pablohoffman.com", - maintainer="Pablo Hoffman", - maintainer_email="pablo@pablohoffman.com", - license="BSD", - packages=find_packages(exclude=("tests", "tests.*")), - include_package_data=True, - zip_safe=False, - entry_points={"console_scripts": ["scrapy = scrapy.cmdline:execute"]}, - classifiers=[ - "Framework :: Scrapy", - "Development Status :: 5 - Production/Stable", - "Environment :: Console", - "Intended Audience :: Developers", - "License :: OSI Approved :: BSD License", - "Operating System :: OS Independent", - "Programming Language :: Python", - "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.9", - "Programming Language :: Python :: 3.10", - "Programming Language :: Python :: 3.11", - "Programming Language :: Python :: 3.12", - "Programming Language :: Python :: 3.13", - "Programming Language :: Python :: Implementation :: CPython", - "Programming Language :: Python :: Implementation :: PyPy", - "Topic :: Internet :: WWW/HTTP", - "Topic :: Software Development :: Libraries :: Application Frameworks", - "Topic :: Software Development :: Libraries :: Python Modules", - ], - python_requires=">=3.9", - install_requires=install_requires, - extras_require=extras_require, -) diff --git a/tests/test_crawler.py b/tests/test_crawler.py index 853acf2de..a77531f62 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -899,7 +899,7 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): p.expect_exact("shutting down gracefully") # sending the second signal too fast often causes problems d = Deferred() - reactor.callLater(0.1, d.callback, None) + reactor.callLater(0.01, d.callback, None) yield d p.kill(sig) p.expect_exact("forcing unclean shutdown") diff --git a/tests/test_spiderloader/__init__.py b/tests/test_spiderloader/__init__.py index d2ff9ba48..9b53b9b96 100644 --- a/tests/test_spiderloader/__init__.py +++ b/tests/test_spiderloader/__init__.py @@ -144,9 +144,10 @@ class SpiderLoaderTest(unittest.TestCase): self.assertRaises(SyntaxError, SpiderLoader.from_settings, settings) def test_syntax_error_warning(self): - with warnings.catch_warnings(record=True) as w, mock.patch.object( - SpiderLoader, "_load_spiders" - ) as m: + with ( + warnings.catch_warnings(record=True) as w, + mock.patch.object(SpiderLoader, "_load_spiders") as m, + ): m.side_effect = SyntaxError module = "tests.test_spiderloader.test_spiders.spider1" settings = Settings( diff --git a/tox.ini b/tox.ini index 5783a0e61..4e1a99473 100644 --- a/tox.ini +++ b/tox.ini @@ -79,7 +79,7 @@ deps = {[testenv:extra-deps]deps} pylint==3.2.5 commands = - pylint conftest.py docs extras scrapy setup.py tests + pylint conftest.py docs extras scrapy tests [testenv:twinecheck] basepython = python3