Integrating configs into pyproject.toml (#6547)

.bandit.yml

@@ -1,7 +0,0 @@
-skips:
-- B101  # assert_used, needed for mypy
-- B321  # ftplib, https://github.com/scrapy/scrapy/issues/4180
-- B402  # import_ftplib, https://github.com/scrapy/scrapy/issues/4180
-- B411  # import_xmlrpclib, https://github.com/PyCQA/bandit/issues/1082
-- B503  # ssl_with_bad_defaults
-exclude_dirs: ['tests']

.bumpversion.cfg

@@ -1,11 +0,0 @@
-[bumpversion]
-current_version = 2.12.0
-commit = True
-tag = True
-tag_name = {new_version}
-
-[bumpversion:file:scrapy/VERSION]
-
-[bumpversion:file:SECURITY.md]
-parse = (?P<major>\d+)\.(?P<minor>\d+)\.x
-serialize = {major}.{minor}.x

.coveragerc

@@ -1,12 +0,0 @@
-[run]
-branch = true
-include = scrapy/*
-omit =
-  tests/*
-disable_warnings = include-ignored
-
-[report]
-# https://github.com/nedbat/coveragepy/issues/831#issuecomment-517778185
-exclude_lines =
-    pragma: no cover
-    if TYPE_CHECKING:

.isort.cfg

@@ -1,2 +0,0 @@
-[settings]
-profile = black

.pre-commit-config.yaml

@@ -3,7 +3,8 @@ repos:
   rev: 1.7.9
   hooks:
   - id: bandit
-    args: [-r, -c, .bandit.yml]
+    args: ["-c", "pyproject.toml"]
+    additional_dependencies: ["bandit[toml]"]
 - repo: https://github.com/PyCQA/flake8
   rev: 7.1.0
   hooks:

MANIFEST.in

@@ -10,7 +10,6 @@ include scrapy/py.typed
 
 include codecov.yml
 include conftest.py
-include pytest.ini
 include tox.ini
 
 recursive-include scrapy/templates *

pylintrc

@@ -1,73 +0,0 @@
-[MASTER]
-persistent=no
-jobs=1  # >1 hides results
-
-[MESSAGES CONTROL]
-disable=abstract-method,
-        arguments-differ,
-        arguments-renamed,
-        attribute-defined-outside-init,
-        bad-classmethod-argument,
-        bare-except,
-        broad-except,
-        broad-exception-raised,
-        c-extension-no-member,
-        consider-using-with,
-        cyclic-import,
-        dangerous-default-value,
-        disallowed-name,
-        duplicate-code,  # https://github.com/PyCQA/pylint/issues/214
-        eval-used,
-        fixme,
-        function-redefined,
-        global-statement,
-        implicit-str-concat,
-        import-error,
-        import-outside-toplevel,
-        inherit-non-class,
-        invalid-name,
-        invalid-overridden-method,
-        isinstance-second-argument-not-valid-type,
-        keyword-arg-before-vararg,
-        line-too-long,
-        logging-format-interpolation,
-        logging-fstring-interpolation,
-        logging-not-lazy,
-        lost-exception,
-        missing-docstring,
-        no-member,
-        no-method-argument,
-        no-name-in-module,
-        no-self-argument,
-        no-value-for-parameter,  # https://github.com/pylint-dev/pylint/issues/3268
-        not-callable,
-        pointless-statement,
-        pointless-string-statement,
-        protected-access,
-        raise-missing-from,
-        redefined-builtin,
-        redefined-outer-name,
-        reimported,
-        signature-differs,
-        too-few-public-methods,
-        too-many-ancestors,
-        too-many-arguments,
-        too-many-branches,
-        too-many-format-args,
-        too-many-function-args,
-        too-many-instance-attributes,
-        too-many-lines,
-        too-many-locals,
-        too-many-public-methods,
-        too-many-return-statements,
-        unbalanced-tuple-unpacking,
-        unnecessary-dunder-call,
-        unnecessary-pass,
-        unreachable,
-        unused-argument,
-        unused-import,
-        unused-variable,
-        used-before-assignment,
-        useless-return,
-        wildcard-import,
-        wrong-import-position

pyproject.toml

@@ -0,0 +1,235 @@
+[build-system]
+requires = ["setuptools >= 61.0"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "Scrapy"
+dynamic = ["version"]
+description = "A high-level Web Crawling and Web Scraping framework"
+dependencies = [
+    "Twisted>=21.7.0",
+    "cryptography>=37.0.0",
+    "cssselect>=0.9.1",
+    "itemloaders>=1.0.1",
+    "parsel>=1.5.0",
+    "pyOpenSSL>=22.0.0",
+    "queuelib>=1.4.2",
+    "service_identity>=18.1.0",
+    "w3lib>=1.17.0",
+    "zope.interface>=5.1.0",
+    "protego>=0.1.15",
+    "itemadapter>=0.1.0",
+    "packaging",
+    "tldextract",
+    "lxml>=4.6.0",
+    "defusedxml>=0.7.1",
+    # Platform-specific dependencies
+    'PyDispatcher>=2.0.5; platform_python_implementation == "CPython"',
+    'PyPyDispatcher>=2.1.0; platform_python_implementation == "PyPy"',
+]
+classifiers = [
+    "Framework :: Scrapy",
+    "Development Status :: 5 - Production/Stable",
+    "Environment :: Console",
+    "Intended Audience :: Developers",
+    "License :: OSI Approved :: BSD License",
+    "Operating System :: OS Independent",
+    "Programming Language :: Python",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.9",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
+    "Programming Language :: Python :: Implementation :: CPython",
+    "Programming Language :: Python :: Implementation :: PyPy",
+    "Topic :: Internet :: WWW/HTTP",
+    "Topic :: Software Development :: Libraries :: Application Frameworks",
+    "Topic :: Software Development :: Libraries :: Python Modules",
+]
+readme = "README.rst"
+requires-python = ">=3.9"
+authors = [{ name = "Scrapy developers", email = "pablo@pablohoffman.com" }]
+maintainers = [{ name = "Pablo Hoffman", email = "pablo@pablohoffman.com" }]
+
+[project.urls]
+Homepage = "https://scrapy.org/"
+Documentation = "https://docs.scrapy.org/"
+Source = "https://github.com/scrapy/scrapy"
+Tracker = "https://github.com/scrapy/scrapy/issues"
+Changelog = "https://github.com/scrapy/scrapy/commits/master/"
+releasenotes = "https://docs.scrapy.org/en/latest/news.html"
+
+[project.scripts]
+scrapy = "scrapy.cmdline:execute"
+
+[tool.setuptools.packages.find]
+where = ["."]
+include = ["scrapy", "scrapy.*",]
+
+[tool.setuptools.dynamic]
+version = {file = "./scrapy/VERSION"}
+
+[tool.mypy]
+ignore_missing_imports = true
+
+# Interface classes are hard to support
+
+[[tool.mypy.overrides]]
+module = "twisted.internet.interfaces"
+follow_imports = "skip"
+
+[[tool.mypy.overrides]]
+module = "scrapy.interfaces"
+ignore_errors = true
+
+[[tool.mypy.overrides]]
+module = "twisted.internet.reactor"
+follow_imports = "skip"
+
+# FIXME: remove the following section once the issues are solved
+[[tool.mypy.overrides]]
+module = "scrapy.settings.default_settings"
+ignore_errors = true
+
+[tool.bandit]
+skips = [
+    "B101", # assert_used, needed for mypy
+    "B321", # ftplib, https://github.com/scrapy/scrapy/issues/4180
+    "B402", # import_ftplib, https://github.com/scrapy/scrapy/issues/4180
+    "B411", # import_xmlrpclib, https://github.com/PyCQA/bandit/issues/1082
+    "B503", # ssl_with_bad_defaults
+]
+exclude_dirs = ["tests"]
+
+[tool.bumpversion]
+current_version = "2.12.0"
+commit = true
+tag = true
+tag_name = "{new_version}"
+
+[[tool.bumpversion.files]]
+filename = "scrapy/VERSION"
+
+[[tool.bumpversion.files]]
+filename = "SECURITY.md"
+parse = """(?P<major>0|[1-9]\\d*)\\.(?P<minor>0|[1-9]\\d*)"""
+serialize = ["{major}.{minor}"]
+
+[tool.coverage.run]
+branch = true
+include = ["scrapy/*"]
+omit = ["tests/*"]
+disable_warnings = ["include-ignored"]
+
+[tool.coverage.report]
+# https://github.com/nedbat/coveragepy/issues/831#issuecomment-517778185
+exclude_lines = ["pragma: no cover", "if TYPE_CHECKING:"]
+
+[tool.isort]
+profile = "black"
+
+[tool.pylint.MASTER]
+persistent = "no"
+jobs = 1          # >1 hides results
+
+[tool.pylint."MESSAGES CONTROL"]
+disable = [
+    "abstract-method",
+    "arguments-differ",
+    "arguments-renamed",
+    "attribute-defined-outside-init",
+    "bad-classmethod-argument",
+    "bare-except",
+    "broad-except",
+    "broad-exception-raised",
+    "c-extension-no-member",
+    "consider-using-with",
+    "cyclic-import",
+    "dangerous-default-value",
+    "disallowed-name",
+    "duplicate-code",                            # https://github.com/PyCQA/pylint/issues/214
+    "eval-used",
+    "fixme",
+    "function-redefined",
+    "global-statement",
+    "implicit-str-concat",
+    "import-error",
+    "import-outside-toplevel",
+    "inherit-non-class",
+    "invalid-name",
+    "invalid-overridden-method",
+    "isinstance-second-argument-not-valid-type",
+    "keyword-arg-before-vararg",
+    "line-too-long",
+    "logging-format-interpolation",
+    "logging-fstring-interpolation",
+    "logging-not-lazy",
+    "lost-exception",
+    "missing-docstring",
+    "no-member",
+    "no-method-argument",
+    "no-name-in-module",
+    "no-self-argument",
+    "no-value-for-parameter",                    # https://github.com/pylint-dev/pylint/issues/3268
+    "not-callable",
+    "pointless-statement",
+    "pointless-string-statement",
+    "protected-access",
+    "raise-missing-from",
+    "redefined-builtin",
+    "redefined-outer-name",
+    "reimported",
+    "signature-differs",
+    "too-few-public-methods",
+    "too-many-ancestors",
+    "too-many-arguments",
+    "too-many-branches",
+    "too-many-format-args",
+    "too-many-function-args",
+    "too-many-instance-attributes",
+    "too-many-lines",
+    "too-many-locals",
+    "too-many-public-methods",
+    "too-many-return-statements",
+    "unbalanced-tuple-unpacking",
+    "unnecessary-dunder-call",
+    "unnecessary-pass",
+    "unreachable",
+    "unused-argument",
+    "unused-import",
+    "unused-variable",
+    "used-before-assignment",
+    "useless-return",
+    "wildcard-import",
+    "wrong-import-position",
+]
+
+[tool.pytest.ini_options]
+xfail_strict = true
+usefixtures = "chdir"
+python_files = ["test_*.py", "__init__.py"]
+python_classes = []
+addopts = [
+    "--assert=plain",
+    "--ignore=docs/_ext",
+    "--ignore=docs/conf.py",
+    "--ignore=docs/news.rst",
+    "--ignore=docs/topics/dynamic-content.rst",
+    "--ignore=docs/topics/items.rst",
+    "--ignore=docs/topics/leaks.rst",
+    "--ignore=docs/topics/loaders.rst",
+    "--ignore=docs/topics/selectors.rst",
+    "--ignore=docs/topics/shell.rst",
+    "--ignore=docs/topics/stats.rst",
+    "--ignore=docs/topics/telnetconsole.rst",
+    "--ignore=docs/utils",
+]
+markers = [
+    "only_asyncio: marks tests as only enabled when --reactor=asyncio is passed",
+    "only_not_asyncio: marks tests as only enabled when --reactor=asyncio is not passed",
+    "requires_uvloop: marks tests as only enabled when uvloop is known to be working",
+    "requires_botocore: marks tests that need botocore (but not boto3)",
+    "requires_boto3: marks tests that need botocore and boto3",
+]
+filterwarnings = []

pytest.ini

@@ -1,26 +0,0 @@
-[pytest]
-xfail_strict = true
-usefixtures = chdir
-python_files=test_*.py __init__.py
-python_classes=
-addopts =
-    --assert=plain
-    --ignore=docs/_ext
-    --ignore=docs/conf.py
-    --ignore=docs/news.rst
-    --ignore=docs/topics/dynamic-content.rst
-    --ignore=docs/topics/items.rst
-    --ignore=docs/topics/leaks.rst
-    --ignore=docs/topics/loaders.rst
-    --ignore=docs/topics/selectors.rst
-    --ignore=docs/topics/shell.rst
-    --ignore=docs/topics/stats.rst
-    --ignore=docs/topics/telnetconsole.rst
-    --ignore=docs/utils
-markers =
-    only_asyncio: marks tests as only enabled when --reactor=asyncio is passed
-    only_not_asyncio: marks tests as only enabled when --reactor=asyncio is not passed
-    requires_uvloop: marks tests as only enabled when uvloop is known to be working
-    requires_botocore: marks tests that need botocore (but not boto3)
-    requires_boto3: marks tests that need botocore and boto3
-filterwarnings =

setup.cfg

@@ -1,24 +0,0 @@
-[bdist_rpm]
-doc_files = docs AUTHORS INSTALL LICENSE README.rst
-
-[bdist_wheel]
-universal=1
-
-[mypy]
-ignore_missing_imports = true
-
-# Interface classes are hard to support
-
-[mypy-twisted.internet.interfaces]
-follow_imports = skip
-
-[mypy-scrapy.interfaces]
-ignore_errors = True
-
-[mypy-twisted.internet.reactor]
-follow_imports = skip
-
-# FIXME: remove the following sections once the issues are solved
-
-[mypy-scrapy.settings.default_settings]
-ignore_errors = True

setup.py

@@ -1,75 +0,0 @@
-from pathlib import Path
-
-from setuptools import find_packages, setup
-
-version = (Path(__file__).parent / "scrapy/VERSION").read_text("ascii").strip()
-
-
-install_requires = [
-    "Twisted>=21.7.0",
-    "cryptography>=37.0.0",
-    "cssselect>=0.9.1",
-    "itemloaders>=1.0.1",
-    "parsel>=1.5.0",
-    "pyOpenSSL>=22.0.0",
-    "queuelib>=1.4.2",
-    "service_identity>=18.1.0",
-    "w3lib>=1.17.0",
-    "zope.interface>=5.1.0",
-    "protego>=0.1.15",
-    "itemadapter>=0.1.0",
-    "packaging",
-    "tldextract",
-    "lxml>=4.6.0",
-    "defusedxml>=0.7.1",
-]
-extras_require = {
-    ':platform_python_implementation == "CPython"': ["PyDispatcher>=2.0.5"],
-    ':platform_python_implementation == "PyPy"': ["PyPyDispatcher>=2.1.0"],
-}
-
-
-setup(
-    name="Scrapy",
-    version=version,
-    url="https://scrapy.org",
-    project_urls={
-        "Documentation": "https://docs.scrapy.org/",
-        "Source": "https://github.com/scrapy/scrapy",
-        "Tracker": "https://github.com/scrapy/scrapy/issues",
-    },
-    description="A high-level Web Crawling and Web Scraping framework",
-    long_description=open("README.rst", encoding="utf-8").read(),
-    author="Scrapy developers",
-    author_email="pablo@pablohoffman.com",
-    maintainer="Pablo Hoffman",
-    maintainer_email="pablo@pablohoffman.com",
-    license="BSD",
-    packages=find_packages(exclude=("tests", "tests.*")),
-    include_package_data=True,
-    zip_safe=False,
-    entry_points={"console_scripts": ["scrapy = scrapy.cmdline:execute"]},
-    classifiers=[
-        "Framework :: Scrapy",
-        "Development Status :: 5 - Production/Stable",
-        "Environment :: Console",
-        "Intended Audience :: Developers",
-        "License :: OSI Approved :: BSD License",
-        "Operating System :: OS Independent",
-        "Programming Language :: Python",
-        "Programming Language :: Python :: 3",
-        "Programming Language :: Python :: 3.9",
-        "Programming Language :: Python :: 3.10",
-        "Programming Language :: Python :: 3.11",
-        "Programming Language :: Python :: 3.12",
-        "Programming Language :: Python :: 3.13",
-        "Programming Language :: Python :: Implementation :: CPython",
-        "Programming Language :: Python :: Implementation :: PyPy",
-        "Topic :: Internet :: WWW/HTTP",
-        "Topic :: Software Development :: Libraries :: Application Frameworks",
-        "Topic :: Software Development :: Libraries :: Python Modules",
-    ],
-    python_requires=">=3.9",
-    install_requires=install_requires,
-    extras_require=extras_require,
-)

tests/test_crawler.py

@@ -899,7 +899,7 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase):
         p.expect_exact("shutting down gracefully")
         # sending the second signal too fast often causes problems
         d = Deferred()
-        reactor.callLater(0.1, d.callback, None)
+        reactor.callLater(0.01, d.callback, None)
         yield d
         p.kill(sig)
         p.expect_exact("forcing unclean shutdown")

tests/test_spiderloader/__init__.py

@@ -144,9 +144,10 @@ class SpiderLoaderTest(unittest.TestCase):
             self.assertRaises(SyntaxError, SpiderLoader.from_settings, settings)
 
     def test_syntax_error_warning(self):
-        with warnings.catch_warnings(record=True) as w, mock.patch.object(
+        with (
-            SpiderLoader, "_load_spiders"
+            warnings.catch_warnings(record=True) as w,
-        ) as m:
+            mock.patch.object(SpiderLoader, "_load_spiders") as m,
+        ):
             m.side_effect = SyntaxError
             module = "tests.test_spiderloader.test_spiders.spider1"
             settings = Settings(

tox.ini

@@ -79,7 +79,7 @@ deps =
     {[testenv:extra-deps]deps}
     pylint==3.2.5
 commands =
-    pylint conftest.py docs extras scrapy setup.py tests
+    pylint conftest.py docs extras scrapy tests
 
 [testenv:twinecheck]
 basepython = python3