diff --git a/.bandit.yml b/.bandit.yml deleted file mode 100644 index b7f1817e0..000000000 --- a/.bandit.yml +++ /dev/null @@ -1,7 +0,0 @@ -skips: -- B101 # assert_used, needed for mypy -- B321 # ftplib, https://github.com/scrapy/scrapy/issues/4180 -- B402 # import_ftplib, https://github.com/scrapy/scrapy/issues/4180 -- B411 # import_xmlrpclib, https://github.com/PyCQA/bandit/issues/1082 -- B503 # ssl_with_bad_defaults -exclude_dirs: ['tests'] diff --git a/.bumpversion.cfg b/.bumpversion.cfg deleted file mode 100644 index f83e3e890..000000000 --- a/.bumpversion.cfg +++ /dev/null @@ -1,11 +0,0 @@ -[bumpversion] -current_version = 2.12.0 -commit = True -tag = True -tag_name = {new_version} - -[bumpversion:file:scrapy/VERSION] - -[bumpversion:file:SECURITY.md] -parse = (?P\d+)\.(?P\d+)\.x -serialize = {major}.{minor}.x diff --git a/.coveragerc b/.coveragerc deleted file mode 100644 index f9ad353d5..000000000 --- a/.coveragerc +++ /dev/null @@ -1,12 +0,0 @@ -[run] -branch = true -include = scrapy/* -omit = - tests/* -disable_warnings = include-ignored - -[report] -# https://github.com/nedbat/coveragepy/issues/831#issuecomment-517778185 -exclude_lines = - pragma: no cover - if TYPE_CHECKING: diff --git a/.isort.cfg b/.isort.cfg deleted file mode 100644 index f238bf7ea..000000000 --- a/.isort.cfg +++ /dev/null @@ -1,2 +0,0 @@ -[settings] -profile = black diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index fbd710f6f..b411f4927 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -3,7 +3,8 @@ repos: rev: 1.7.9 hooks: - id: bandit - args: [-r, -c, .bandit.yml] + args: ["-c", "pyproject.toml"] + additional_dependencies: ["bandit[toml]"] - repo: https://github.com/PyCQA/flake8 rev: 7.1.0 hooks: diff --git a/MANIFEST.in b/MANIFEST.in index 06971e39c..7700ae7bd 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -10,7 +10,6 @@ include scrapy/py.typed include codecov.yml include conftest.py -include pytest.ini include tox.ini recursive-include scrapy/templates * diff --git a/pylintrc b/pylintrc deleted file mode 100644 index e927b903c..000000000 --- a/pylintrc +++ /dev/null @@ -1,73 +0,0 @@ -[MASTER] -persistent=no -jobs=1 # >1 hides results - -[MESSAGES CONTROL] -disable=abstract-method, - arguments-differ, - arguments-renamed, - attribute-defined-outside-init, - bad-classmethod-argument, - bare-except, - broad-except, - broad-exception-raised, - c-extension-no-member, - consider-using-with, - cyclic-import, - dangerous-default-value, - disallowed-name, - duplicate-code, # https://github.com/PyCQA/pylint/issues/214 - eval-used, - fixme, - function-redefined, - global-statement, - implicit-str-concat, - import-error, - import-outside-toplevel, - inherit-non-class, - invalid-name, - invalid-overridden-method, - isinstance-second-argument-not-valid-type, - keyword-arg-before-vararg, - line-too-long, - logging-format-interpolation, - logging-fstring-interpolation, - logging-not-lazy, - lost-exception, - missing-docstring, - no-member, - no-method-argument, - no-name-in-module, - no-self-argument, - no-value-for-parameter, # https://github.com/pylint-dev/pylint/issues/3268 - not-callable, - pointless-statement, - pointless-string-statement, - protected-access, - raise-missing-from, - redefined-builtin, - redefined-outer-name, - reimported, - signature-differs, - too-few-public-methods, - too-many-ancestors, - too-many-arguments, - too-many-branches, - too-many-format-args, - too-many-function-args, - too-many-instance-attributes, - too-many-lines, - too-many-locals, - too-many-public-methods, - too-many-return-statements, - unbalanced-tuple-unpacking, - unnecessary-dunder-call, - unnecessary-pass, - unreachable, - unused-argument, - unused-import, - unused-variable, - used-before-assignment, - useless-return, - wildcard-import, - wrong-import-position diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 000000000..f25715e76 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,235 @@ +[build-system] +requires = ["setuptools >= 61.0"] +build-backend = "setuptools.build_meta" + +[project] +name = "Scrapy" +dynamic = ["version"] +description = "A high-level Web Crawling and Web Scraping framework" +dependencies = [ + "Twisted>=21.7.0", + "cryptography>=37.0.0", + "cssselect>=0.9.1", + "itemloaders>=1.0.1", + "parsel>=1.5.0", + "pyOpenSSL>=22.0.0", + "queuelib>=1.4.2", + "service_identity>=18.1.0", + "w3lib>=1.17.0", + "zope.interface>=5.1.0", + "protego>=0.1.15", + "itemadapter>=0.1.0", + "packaging", + "tldextract", + "lxml>=4.6.0", + "defusedxml>=0.7.1", + # Platform-specific dependencies + 'PyDispatcher>=2.0.5; platform_python_implementation == "CPython"', + 'PyPyDispatcher>=2.1.0; platform_python_implementation == "PyPy"', +] +classifiers = [ + "Framework :: Scrapy", + "Development Status :: 5 - Production/Stable", + "Environment :: Console", + "Intended Audience :: Developers", + "License :: OSI Approved :: BSD License", + "Operating System :: OS Independent", + "Programming Language :: Python", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: Implementation :: CPython", + "Programming Language :: Python :: Implementation :: PyPy", + "Topic :: Internet :: WWW/HTTP", + "Topic :: Software Development :: Libraries :: Application Frameworks", + "Topic :: Software Development :: Libraries :: Python Modules", +] +readme = "README.rst" +requires-python = ">=3.9" +authors = [{ name = "Scrapy developers", email = "pablo@pablohoffman.com" }] +maintainers = [{ name = "Pablo Hoffman", email = "pablo@pablohoffman.com" }] + +[project.urls] +Homepage = "https://scrapy.org/" +Documentation = "https://docs.scrapy.org/" +Source = "https://github.com/scrapy/scrapy" +Tracker = "https://github.com/scrapy/scrapy/issues" +Changelog = "https://github.com/scrapy/scrapy/commits/master/" +releasenotes = "https://docs.scrapy.org/en/latest/news.html" + +[project.scripts] +scrapy = "scrapy.cmdline:execute" + +[tool.setuptools.packages.find] +where = ["."] +include = ["scrapy", "scrapy.*",] + +[tool.setuptools.dynamic] +version = {file = "./scrapy/VERSION"} + +[tool.mypy] +ignore_missing_imports = true + +# Interface classes are hard to support + +[[tool.mypy.overrides]] +module = "twisted.internet.interfaces" +follow_imports = "skip" + +[[tool.mypy.overrides]] +module = "scrapy.interfaces" +ignore_errors = true + +[[tool.mypy.overrides]] +module = "twisted.internet.reactor" +follow_imports = "skip" + +# FIXME: remove the following section once the issues are solved +[[tool.mypy.overrides]] +module = "scrapy.settings.default_settings" +ignore_errors = true + +[tool.bandit] +skips = [ + "B101", # assert_used, needed for mypy + "B321", # ftplib, https://github.com/scrapy/scrapy/issues/4180 + "B402", # import_ftplib, https://github.com/scrapy/scrapy/issues/4180 + "B411", # import_xmlrpclib, https://github.com/PyCQA/bandit/issues/1082 + "B503", # ssl_with_bad_defaults +] +exclude_dirs = ["tests"] + +[tool.bumpversion] +current_version = "2.12.0" +commit = true +tag = true +tag_name = "{new_version}" + +[[tool.bumpversion.files]] +filename = "scrapy/VERSION" + +[[tool.bumpversion.files]] +filename = "SECURITY.md" +parse = """(?P0|[1-9]\\d*)\\.(?P0|[1-9]\\d*)""" +serialize = ["{major}.{minor}"] + +[tool.coverage.run] +branch = true +include = ["scrapy/*"] +omit = ["tests/*"] +disable_warnings = ["include-ignored"] + +[tool.coverage.report] +# https://github.com/nedbat/coveragepy/issues/831#issuecomment-517778185 +exclude_lines = ["pragma: no cover", "if TYPE_CHECKING:"] + +[tool.isort] +profile = "black" + +[tool.pylint.MASTER] +persistent = "no" +jobs = 1 # >1 hides results + +[tool.pylint."MESSAGES CONTROL"] +disable = [ + "abstract-method", + "arguments-differ", + "arguments-renamed", + "attribute-defined-outside-init", + "bad-classmethod-argument", + "bare-except", + "broad-except", + "broad-exception-raised", + "c-extension-no-member", + "consider-using-with", + "cyclic-import", + "dangerous-default-value", + "disallowed-name", + "duplicate-code", # https://github.com/PyCQA/pylint/issues/214 + "eval-used", + "fixme", + "function-redefined", + "global-statement", + "implicit-str-concat", + "import-error", + "import-outside-toplevel", + "inherit-non-class", + "invalid-name", + "invalid-overridden-method", + "isinstance-second-argument-not-valid-type", + "keyword-arg-before-vararg", + "line-too-long", + "logging-format-interpolation", + "logging-fstring-interpolation", + "logging-not-lazy", + "lost-exception", + "missing-docstring", + "no-member", + "no-method-argument", + "no-name-in-module", + "no-self-argument", + "no-value-for-parameter", # https://github.com/pylint-dev/pylint/issues/3268 + "not-callable", + "pointless-statement", + "pointless-string-statement", + "protected-access", + "raise-missing-from", + "redefined-builtin", + "redefined-outer-name", + "reimported", + "signature-differs", + "too-few-public-methods", + "too-many-ancestors", + "too-many-arguments", + "too-many-branches", + "too-many-format-args", + "too-many-function-args", + "too-many-instance-attributes", + "too-many-lines", + "too-many-locals", + "too-many-public-methods", + "too-many-return-statements", + "unbalanced-tuple-unpacking", + "unnecessary-dunder-call", + "unnecessary-pass", + "unreachable", + "unused-argument", + "unused-import", + "unused-variable", + "used-before-assignment", + "useless-return", + "wildcard-import", + "wrong-import-position", +] + +[tool.pytest.ini_options] +xfail_strict = true +usefixtures = "chdir" +python_files = ["test_*.py", "__init__.py"] +python_classes = [] +addopts = [ + "--assert=plain", + "--ignore=docs/_ext", + "--ignore=docs/conf.py", + "--ignore=docs/news.rst", + "--ignore=docs/topics/dynamic-content.rst", + "--ignore=docs/topics/items.rst", + "--ignore=docs/topics/leaks.rst", + "--ignore=docs/topics/loaders.rst", + "--ignore=docs/topics/selectors.rst", + "--ignore=docs/topics/shell.rst", + "--ignore=docs/topics/stats.rst", + "--ignore=docs/topics/telnetconsole.rst", + "--ignore=docs/utils", +] +markers = [ + "only_asyncio: marks tests as only enabled when --reactor=asyncio is passed", + "only_not_asyncio: marks tests as only enabled when --reactor=asyncio is not passed", + "requires_uvloop: marks tests as only enabled when uvloop is known to be working", + "requires_botocore: marks tests that need botocore (but not boto3)", + "requires_boto3: marks tests that need botocore and boto3", +] +filterwarnings = [] \ No newline at end of file diff --git a/pytest.ini b/pytest.ini deleted file mode 100644 index 824c0e9e9..000000000 --- a/pytest.ini +++ /dev/null @@ -1,26 +0,0 @@ -[pytest] -xfail_strict = true -usefixtures = chdir -python_files=test_*.py __init__.py -python_classes= -addopts = - --assert=plain - --ignore=docs/_ext - --ignore=docs/conf.py - --ignore=docs/news.rst - --ignore=docs/topics/dynamic-content.rst - --ignore=docs/topics/items.rst - --ignore=docs/topics/leaks.rst - --ignore=docs/topics/loaders.rst - --ignore=docs/topics/selectors.rst - --ignore=docs/topics/shell.rst - --ignore=docs/topics/stats.rst - --ignore=docs/topics/telnetconsole.rst - --ignore=docs/utils -markers = - only_asyncio: marks tests as only enabled when --reactor=asyncio is passed - only_not_asyncio: marks tests as only enabled when --reactor=asyncio is not passed - requires_uvloop: marks tests as only enabled when uvloop is known to be working - requires_botocore: marks tests that need botocore (but not boto3) - requires_boto3: marks tests that need botocore and boto3 -filterwarnings = diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index 151e784c6..000000000 --- a/setup.cfg +++ /dev/null @@ -1,24 +0,0 @@ -[bdist_rpm] -doc_files = docs AUTHORS INSTALL LICENSE README.rst - -[bdist_wheel] -universal=1 - -[mypy] -ignore_missing_imports = true - -# Interface classes are hard to support - -[mypy-twisted.internet.interfaces] -follow_imports = skip - -[mypy-scrapy.interfaces] -ignore_errors = True - -[mypy-twisted.internet.reactor] -follow_imports = skip - -# FIXME: remove the following sections once the issues are solved - -[mypy-scrapy.settings.default_settings] -ignore_errors = True diff --git a/setup.py b/setup.py deleted file mode 100644 index 6cc1150a5..000000000 --- a/setup.py +++ /dev/null @@ -1,75 +0,0 @@ -from pathlib import Path - -from setuptools import find_packages, setup - -version = (Path(__file__).parent / "scrapy/VERSION").read_text("ascii").strip() - - -install_requires = [ - "Twisted>=21.7.0", - "cryptography>=37.0.0", - "cssselect>=0.9.1", - "itemloaders>=1.0.1", - "parsel>=1.5.0", - "pyOpenSSL>=22.0.0", - "queuelib>=1.4.2", - "service_identity>=18.1.0", - "w3lib>=1.17.0", - "zope.interface>=5.1.0", - "protego>=0.1.15", - "itemadapter>=0.1.0", - "packaging", - "tldextract", - "lxml>=4.6.0", - "defusedxml>=0.7.1", -] -extras_require = { - ':platform_python_implementation == "CPython"': ["PyDispatcher>=2.0.5"], - ':platform_python_implementation == "PyPy"': ["PyPyDispatcher>=2.1.0"], -} - - -setup( - name="Scrapy", - version=version, - url="https://scrapy.org", - project_urls={ - "Documentation": "https://docs.scrapy.org/", - "Source": "https://github.com/scrapy/scrapy", - "Tracker": "https://github.com/scrapy/scrapy/issues", - }, - description="A high-level Web Crawling and Web Scraping framework", - long_description=open("README.rst", encoding="utf-8").read(), - author="Scrapy developers", - author_email="pablo@pablohoffman.com", - maintainer="Pablo Hoffman", - maintainer_email="pablo@pablohoffman.com", - license="BSD", - packages=find_packages(exclude=("tests", "tests.*")), - include_package_data=True, - zip_safe=False, - entry_points={"console_scripts": ["scrapy = scrapy.cmdline:execute"]}, - classifiers=[ - "Framework :: Scrapy", - "Development Status :: 5 - Production/Stable", - "Environment :: Console", - "Intended Audience :: Developers", - "License :: OSI Approved :: BSD License", - "Operating System :: OS Independent", - "Programming Language :: Python", - "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.9", - "Programming Language :: Python :: 3.10", - "Programming Language :: Python :: 3.11", - "Programming Language :: Python :: 3.12", - "Programming Language :: Python :: 3.13", - "Programming Language :: Python :: Implementation :: CPython", - "Programming Language :: Python :: Implementation :: PyPy", - "Topic :: Internet :: WWW/HTTP", - "Topic :: Software Development :: Libraries :: Application Frameworks", - "Topic :: Software Development :: Libraries :: Python Modules", - ], - python_requires=">=3.9", - install_requires=install_requires, - extras_require=extras_require, -) diff --git a/tests/test_crawler.py b/tests/test_crawler.py index 853acf2de..a77531f62 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -899,7 +899,7 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): p.expect_exact("shutting down gracefully") # sending the second signal too fast often causes problems d = Deferred() - reactor.callLater(0.1, d.callback, None) + reactor.callLater(0.01, d.callback, None) yield d p.kill(sig) p.expect_exact("forcing unclean shutdown") diff --git a/tests/test_spiderloader/__init__.py b/tests/test_spiderloader/__init__.py index d2ff9ba48..9b53b9b96 100644 --- a/tests/test_spiderloader/__init__.py +++ b/tests/test_spiderloader/__init__.py @@ -144,9 +144,10 @@ class SpiderLoaderTest(unittest.TestCase): self.assertRaises(SyntaxError, SpiderLoader.from_settings, settings) def test_syntax_error_warning(self): - with warnings.catch_warnings(record=True) as w, mock.patch.object( - SpiderLoader, "_load_spiders" - ) as m: + with ( + warnings.catch_warnings(record=True) as w, + mock.patch.object(SpiderLoader, "_load_spiders") as m, + ): m.side_effect = SyntaxError module = "tests.test_spiderloader.test_spiders.spider1" settings = Settings( diff --git a/tox.ini b/tox.ini index 5783a0e61..4e1a99473 100644 --- a/tox.ini +++ b/tox.ini @@ -79,7 +79,7 @@ deps = {[testenv:extra-deps]deps} pylint==3.2.5 commands = - pylint conftest.py docs extras scrapy setup.py tests + pylint conftest.py docs extras scrapy tests [testenv:twinecheck] basepython = python3