[build-system]
requires = ["hatchling>=1.27.0"]
build-backend = "hatchling.build"

[project]
name = "Scrapy"
dynamic = ["version"]
description = "A high-level Web Crawling and Web Scraping framework"
dependencies = [
    "Twisted>=21.7.0",
    "cryptography>=37.0.0",
    "cssselect>=0.9.1",
    "defusedxml>=0.7.1",
    "itemadapter>=0.1.0",
    "itemloaders>=1.0.1",
    "lxml>=4.6.4",
    "packaging",
    "parsel>=1.5.0",
    "protego>=0.1.15",
    "pyOpenSSL>=22.0.0",
    "queuelib>=1.4.2",
    "service_identity>=23.1.0",
    "tldextract",
    "w3lib>=1.17.0",
    "zope.interface>=5.1.0",
    # Platform-specific dependencies
    'PyDispatcher>=2.0.5; platform_python_implementation == "CPython"',
    'PyPyDispatcher>=2.1.0; platform_python_implementation == "PyPy"',
]
classifiers = [
    "Development Status :: 5 - Production/Stable",
    "Environment :: Console",
    "Framework :: Scrapy",
    "Intended Audience :: Developers",
    "Operating System :: OS Independent",
    "Programming Language :: Python",
    "Programming Language :: Python :: 3",
    "Programming Language :: Python :: 3.10",
    "Programming Language :: Python :: 3.11",
    "Programming Language :: Python :: 3.12",
    "Programming Language :: Python :: 3.13",
    "Programming Language :: Python :: 3.14",
    "Programming Language :: Python :: Implementation :: CPython",
    "Programming Language :: Python :: Implementation :: PyPy",
    "Topic :: Internet :: WWW/HTTP",
    "Topic :: Software Development :: Libraries :: Application Frameworks",
    "Topic :: Software Development :: Libraries :: Python Modules",
]
license = "BSD-3-Clause"
license-files = ["LICENSE", "AUTHORS"]
readme = "README.rst"
requires-python = ">=3.10"
authors = [{ name = "Scrapy developers", email = "pablo@pablohoffman.com" }]
maintainers = [{ name = "Pablo Hoffman", email = "pablo@pablohoffman.com" }]

[project.urls]
Homepage = "https://scrapy.org/"
Documentation = "https://docs.scrapy.org/"
Source = "https://github.com/scrapy/scrapy"
Tracker = "https://github.com/scrapy/scrapy/issues"
"Release notes" = "https://docs.scrapy.org/en/latest/news.html"

[project.scripts]
scrapy = "scrapy.cmdline:execute"

[tool.hatch.build.targets.sdist]
include = [
  "/docs",
  "/extras",
  "/scrapy",
  "/tests",
  "/tests_typing",
  "/CODE_OF_CONDUCT.md",
  "/CONTRIBUTING.md",
  "/INSTALL.md",
  "/NEWS",
  "/SECURITY.md",
  "/codecov.yml",
  "/conftest.py",
  "/tox.ini",
]

[tool.hatch.version]
path = "scrapy/VERSION"
pattern = "^(?P<version>.+)$"

[tool.mypy]
strict = true
extra_checks = false # weird addErrback() errors
untyped_calls_exclude = [
    "twisted",
]

[[tool.mypy.overrides]]
module = "tests.*"
allow_untyped_defs = true
allow_incomplete_defs = true # 59 errors

# TODO
[[tool.mypy.overrides]]
module = [
    "tests.mockserver.*",
    "tests.spiders",
    "tests.test_closespider",
    "tests.test_cmdline",
    "tests.test_contracts",
    "tests.test_core_downloader",
    "tests.test_downloader_handler_twisted_ftp",
    "tests.test_downloadermiddleware_cookies",
    "tests.test_downloadermiddleware_httpauth",
    "tests.test_downloadermiddleware_httpcache",
    "tests.test_downloadermiddleware_httpcompression",
    "tests.test_downloadermiddleware_httpproxy",
    "tests.test_downloadermiddleware_offsite",
    "tests.test_downloadermiddleware_redirect",
    "tests.test_downloadermiddleware_redirect_base",
    "tests.test_downloadermiddleware_redirect_metarefresh",
    "tests.test_downloadermiddleware_retry",
    "tests.test_downloadermiddleware_robotstxt",
    "tests.test_downloadermiddleware_stats",
    "tests.test_downloaderslotssettings",
    "tests.test_dupefilters",
    "tests.test_engine_loop",
    "tests.test_exporters",
    "tests.test_extension_statsmailer",
    "tests.test_extension_throttle",
    "tests.test_feedexport",
    "tests.test_feedexport_postprocess",
    "tests.test_feedexport_storages",
    "tests.test_feedexport_uri_params",
    "tests.test_http2_client_protocol",
    "tests.test_http_headers",
    "tests.test_http_request",
    "tests.test_http_request_form",
    "tests.test_http_response",
    "tests.test_http_response_text",
    "tests.test_item",
    "tests.test_link",
    "tests.test_linkextractors",
    "tests.test_loader",
    "tests.test_logformatter",
    "tests.test_logstats",
    "tests.test_mail",
    "tests.test_pipeline_crawl",
    "tests.test_pipeline_files",
    "tests.test_pipeline_images",
    "tests.test_pipeline_media",
    "tests.test_pipelines",
    "tests.test_pqueues",
    "tests.test_request_attribute_binding",
    "tests.test_request_cb_kwargs",
    "tests.test_request_dict",
    "tests.test_request_left",
    "tests.test_robotstxt_interface",
    "tests.test_scheduler_base",
    "tests.test_settings",
    "tests.test_spider",
    "tests.test_spider_crawl",
    "tests.test_spidermiddleware_output_chain",
    "tests.test_spidermiddleware_process_start",
    "tests.test_spider_sitemap",
    "tests.test_squeues",
    "tests.test_squeues_request",
    "tests.test_stats",
    "tests.test_utils_datatypes",
    "tests.test_utils_decorators",
    "tests.test_utils_defer",
    "tests.test_utils_deprecate",
    "tests.test_utils_misc.test_return_with_argument_inside_generator",
    "tests.test_utils_python",
    "tests.test_utils_request",
]
check_untyped_defs = false

# Interface classes are hard to support
[[tool.mypy.overrides]]
module = "twisted.internet.interfaces"
follow_imports = "skip"

[[tool.mypy.overrides]]
module = "scrapy.interfaces"
ignore_errors = true

[[tool.mypy.overrides]]
module = "twisted.internet.reactor"
follow_imports = "skip"

# just for twisted.version
[[tool.mypy.overrides]]
module = "twisted"
implicit_reexport = true

# TODO
[[tool.mypy.overrides]]
module = "scrapy.settings.default_settings"
ignore_errors = true

# usually no type hints
[[tool.mypy.overrides]]
module = [
    "bpython",
    "brotli",
    "brotlicffi",
    "google.*",
    "pydispatch.*",
    "pyftpdlib.*",
    "pytest_twisted",
    "robotexclusionrulesparser",
    "testfixtures",
    "zope.interface.*",
]
ignore_missing_imports = true

[tool.bumpversion]
current_version = "2.17.0"
commit = true
tag = true
tag_name = "{new_version}"

[[tool.bumpversion.files]]
filename = "docs/news.rst"
search = "\\(unreleased\\)$"
replace = "({now:%Y-%m-%d})"
regex = true

[[tool.bumpversion.files]]
filename = "scrapy/VERSION"

[[tool.bumpversion.files]]
filename = "SECURITY.md"
parse = """(?P<major>0|[1-9]\\d*)\\.(?P<minor>0|[1-9]\\d*)"""
serialize = ["{major}.{minor}"]

[tool.coverage.run]
# sysmon, default on 3.14, is too slow: https://github.com/coveragepy/coveragepy/issues/2172
core = "ctrace"
branch = true
include = ["scrapy/*"]
omit = ["tests/*"]
disable_warnings = ["include-ignored"]
patch = [
    "subprocess",
]

[tool.coverage.paths]
source = [
    "scrapy",
    ".tox/**/site-packages/scrapy"
]

[tool.coverage.report]
exclude_also = [
    "@(abc\\.)?abstractmethod",
    '\A(?s:.*# pragma: no file cover.*)\Z',
]

[tool.pylint.MASTER]
persistent = "no"
jobs = 1          # >1 hides results
extension-pkg-allow-list=[
    "lxml",
]
load-plugins = ["pylint_per_file_ignores"]

[tool.pylint."MESSAGES CONTROL"]
enable = [
    "useless-suppression",
]
# Make INFO checks like useless-suppression also cause pylint to return a
# non-zero exit code.
fail-on = "I"
disable = [
    # Ones we want to ignore
    "attribute-defined-outside-init",
    "broad-exception-caught",
    "consider-using-with",
    "cyclic-import",
    "disallowed-name",
    "duplicate-code",  # https://github.com/pylint-dev/pylint/issues/214
    "fixme",
    "inherit-non-class",  # false positives with create_deprecated_class()
    "invalid-name",
    "invalid-overridden-method",
    "isinstance-second-argument-not-valid-type",  # false positives with create_deprecated_class()
    "line-too-long",
    "logging-format-interpolation",
    "logging-fstring-interpolation",
    "logging-not-lazy",
    "missing-docstring",
    "no-member",
    "no-value-for-parameter",  # https://github.com/pylint-dev/pylint/issues/3268
    "not-callable",
    "protected-access",
    "redefined-outer-name",
    "too-few-public-methods",
    "too-many-ancestors",
    "too-many-arguments",
    "too-many-function-args",
    "too-many-instance-attributes",
    "too-many-lines",
    "too-many-locals",
    "too-many-positional-arguments",
    "too-many-public-methods",
    "too-many-return-statements",
    "undefined-variable",
    "unused-argument",
    "unused-variable",
    "use-implicit-booleaness-not-comparison",
    "useless-import-alias",  # used as a hint to mypy
    "useless-return",  # https://github.com/pylint-dev/pylint/issues/6530
    "wrong-import-position",

    # Ones that are implemented in ruff and need to be disabled for some lines (listed here to avoid two disable comments)
    "bare-except",
    "eval-used",
    "global-statement",
    "import-outside-toplevel",
    "import-self",
    "inconsistent-return-statements",
    "redefined-builtin",
    "too-many-branches",
    "unused-import",

    # Ones that we may want to address (fix, ignore per-line or move to "don't want to fix")
    "arguments-differ",
    "keyword-arg-before-vararg",
]
# requires `pylint_per_file_ignores` plugin
per-file-ignores = [
    # Extended list of ones that we may want to address, only for tests
    "./tests/*:abstract-method,arguments-renamed,dangerous-default-value,pointless-statement,raise-missing-from,unnecessary-dunder-call,used-before-assignment",
]

[tool.pytest.ini_options]
addopts = [
    "--reactor=asyncio",
]
xfail_strict = true
python_files = ["test_*.py", "test_*/__init__.py"]
markers = [
    "only_asyncio: marks tests that require the asyncio loop to be used",
    "only_not_asyncio: marks tests that require the asyncio loop to not be used",
    "requires_reactor: marks tests that require a reactor",
    "requires_uvloop: marks tests as only enabled when uvloop is known to be working",
    "requires_botocore: marks tests that need botocore (but not boto3)",
    "requires_boto3: marks tests that need botocore and boto3",
    "requires_mitmproxy: marks tests that need a mitmdump executable",
    "requires_internet: marks tests that need real Internet access",
]
filterwarnings = [
    "ignore::DeprecationWarning:twisted.web.static",
    # Twisted doesn't close failed sockets after CannotListenError: https://github.com/twisted/twisted/issues/6108
    "ignore:Exception ignored in. <socket\\.socket.*laddr=..0\\.0\\.0\\.0., 0.:pytest.PytestUnraisableExceptionWarning",
]

[tool.ruff.lint]
extend-select = [
    # flake8-builtins
    "A",
    # flake8-async
    "ASYNC",
    # flake8-bugbear
    "B",
    # flake8-comprehensions
    "C4",
    # flake8-commas
    "COM",
    # pydocstyle
    "D",
    # flake8-future-annotations
    "FA",
    # flynt
    "FLY",
    # refurb
    "FURB",
    # isort
    "I",
    # flake8-implicit-str-concat
    "ISC",
    # flake8-logging
    "LOG",
    # Perflint
    "PERF",
    # pygrep-hooks
    "PGH",
    # flake8-pie
    "PIE",
    # pylint
    "PL",
    # flake8-pytest-style
    "PT",
    # flake8-use-pathlib
    "PTH",
    # flake8-pyi
    "PYI",
    # flake8-quotes
    "Q",
    # flake8-return
    "RET",
    # flake8-raise
    "RSE",
    # Ruff-specific rules
    "RUF",
    # flake8-bandit
    "S",
    # flake8-simplify
    "SIM",
    # flake8-slots
    "SLOT",
    # flake8-debugger
    "T10",
    # flake8-type-checking
    "TC",
    # flake8-tidy-imports
    "TID",
    # pyupgrade
    "UP",
    # pycodestyle warnings
    "W",
    # flake8-2020
    "YTT",
]
ignore = [
    # Ones we want to ignore

    # Trailing comma missing
    "COM812",
    # Missing docstring in public module
    "D100",
    # Missing docstring in public class
    "D101",
    # Missing docstring in public method
    "D102",
    # Missing docstring in public function
    "D103",
    # Missing docstring in public package
    "D104",
    # Missing docstring in magic method
    "D105",
    # Missing docstring in public nested class
    "D106",
    # Missing docstring in __init__
    "D107",
    # One-line docstring should fit on one line with quotes
    "D200",
    # No blank lines allowed after function docstring
    "D202",
    # 1 blank line required between summary line and description
    "D205",
    # Multi-line docstring closing quotes should be on a separate line
    "D209",
    # First line should end with a period
    "D400",
    # First line should be in imperative mood; try rephrasing
    "D401",
    # First line should not be the function's "signature"
    "D402",
    # First word of the first line should be properly capitalized
    "D403",
    # `try`-`except` within a loop incurs performance overhead
    "PERF203",
    # Too many return statements
    "PLR0911",
    # Too many arguments in function definition
    "PLR0913",
    # Magic value used in comparison
    "PLR2004",
    # String contains ambiguous {}.
    "RUF001",
    # Docstring contains ambiguous {}.
    "RUF002",
    # Comment contains ambiguous {}.
    "RUF003",
    # Use of `assert` detected; needed for mypy
    "S101",
    # FTP-related functions are being called; https://github.com/scrapy/scrapy/issues/4180
    "S321",
    # Use a context manager for opening files
    "SIM115",
    # Yoda condition detected
    "SIM300",
]

[tool.ruff.lint.flake8-tidy-imports]
banned-module-level-imports = [
    "twisted.internet.reactor",
    # indirectly imports twisted.conch.insults.helper which imports twisted.internet.reactor
    "twisted.conch.manhole",
    # directly imports twisted.internet.reactor
    "twisted.protocols.ftp",
]

[tool.ruff.lint.isort]
split-on-trailing-comma = false

[tool.ruff.lint.per-file-ignores]
# Circular import workarounds
"scrapy/linkextractors/__init__.py" = ["E402"]
"scrapy/spiders/__init__.py" = ["E402"]

"tests/**" = [
    # Skip bandit and allow blocking file I/O in tests
    "ASYNC240",
    "S",
    # Ones that we may want to address (fix, ignore per-line or move to "don't want to fix")
    # Assigning to `os.environ` doesn't clear the environment.
    "B003",
    # Do not use mutable data structures for argument defaults.
    "B006",
    # Found useless expression.
    "B018",
    # No explicit stacklevel argument found.
    "B028",
    # Within an `except` clause, raise exceptions with `raise ... from`
    "B904",
    # `for` loop variable overwritten by assignment target
    "PLW2901",
    # Mutable class attributes should be annotated with `typing.ClassVar`
    "RUF012",
    # Use capitalized environment variable
    "SIM112",
]

# Issues pending a review:
"docs/conf.py" = ["E402"]
"scrapy/utils/url.py" = ["F403", "F405"]
"tests/test_loader.py" = ["E741"]

[tool.ruff.lint.pydocstyle]
convention = "pep257"

[tool.sphinx-scrapy]
python-version = "3.14"  # Keep in sync with .github/workflows/checks.yml.