diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index d3715caf2..cb05784fa 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -26,9 +26,13 @@ jobs: - python-version: "3.10" env: TOXENV: typing-tests - - python-version: "3.13" # Keep in sync with .readthedocs.yml + # Keep in sync with pyproject.toml tool.sphinx-scrapy.python-version. + - python-version: "3.13" env: TOXENV: docs + - python-version: "3.13" + env: + TOXENV: docs-tests - python-version: "3.13" env: TOXENV: twinecheck diff --git a/.gitignore b/.gitignore index 4100bcd97..5e52ecf1e 100644 --- a/.gitignore +++ b/.gitignore @@ -3,7 +3,7 @@ *.pyc _trial_temp* dropin.cache -docs/build +docs/_build *egg-info .tox/ venv/ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a2bf1be0c..b86681b21 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -26,3 +26,7 @@ repos: rev: v1.0.2 hooks: - id: sphinx-lint +- repo: https://github.com/scrapy/sphinx-scrapy + rev: 0.7.0 + hooks: + - id: sphinx-scrapy diff --git a/.readthedocs.yml b/.readthedocs.yml index 23e4cabea..dc3827869 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -1,17 +1,9 @@ version: 2 -formats: all -sphinx: - configuration: docs/conf.py - fail_on_warning: true - build: os: ubuntu-24.04 tools: - # For available versions, see: - # https://docs.readthedocs.io/en/stable/config-file/v2.html#build-tools-python - python: "3.13" # Keep in sync with .github/workflows/checks.yml - -python: - install: - - requirements: docs/requirements.txt - - path: . + python: "3.13" + commands: + - pip install tox + - tox -e docs + - cp -a docs/_build/all/. $READTHEDOCS_OUTPUT/html/ diff --git a/docs/README.rst b/docs/README.rst index 36dd5aea4..4d2236b53 100644 --- a/docs/README.rst +++ b/docs/README.rst @@ -65,4 +65,4 @@ To compile the documentation to HTML run the following command:: tox -e docs -Documentation will be generated (in HTML format) inside the ``.tox/docs/tmp/html`` dir. +Documentation will be generated inside the ``docs/_build/all`` dir. diff --git a/docs/_ext/scrapydocs.py b/docs/_ext/scrapydocs.py index 4e232967d..edb91bfb9 100644 --- a/docs/_ext/scrapydocs.py +++ b/docs/_ext/scrapydocs.py @@ -77,6 +77,25 @@ def make_setting_element( return item +def make_setting_markdown_item( + setting_data: SettingData, app: Sphinx, fromdocname: str +) -> str: + uri = app.builder.get_relative_uri(fromdocname, setting_data["docname"]) + if uri.startswith("#"): + target = f"#{setting_data['refid']}" + else: + target = f"{uri}#{setting_data['refid']}" + return f"* [{setting_data['setting_name']}]({target})" + + +def _iter_sorted_settings(env: Any, fromdocname: str) -> list[SettingData]: + return [ + d + for d in sorted(env.scrapy_all_settings, key=itemgetter("setting_name")) # type: ignore[attr-defined] + if fromdocname != d["docname"] + ] + + def replace_settingslist_nodes( app: Sphinx, doctree: document, fromdocname: str ) -> None: @@ -87,13 +106,29 @@ def replace_settingslist_nodes( settings_list.extend( [ make_setting_element(d, app, fromdocname) - for d in sorted(env.scrapy_all_settings, key=itemgetter("setting_name")) # type: ignore[attr-defined] - if fromdocname != d["docname"] + for d in _iter_sorted_settings(env, fromdocname) ] ) node.replace_self(settings_list) +def visit_settingslist_node_markdown(translator: Any, _node: Node) -> None: + builder = translator.builder + env = builder.env + fromdocname = getattr(builder, "current_doc_name", env.docname) + lines = [ + make_setting_markdown_item(setting_data, builder.app, fromdocname) + for setting_data in _iter_sorted_settings(env, fromdocname) + ] + if lines: + translator.add("\n".join(lines), prefix_eol=2, suffix_eol=2) + raise nodes.SkipNode + + +def depart_settingslist_node_markdown(_translator: Any, _node: Node) -> None: + return None + + def source_role( name, rawtext, text: str, lineno, inliner, options=None, content=None ) -> tuple[list[Any], list[Any]]: @@ -127,32 +162,19 @@ def rev_role( def setup(app: Sphinx) -> dict[str, Any]: - app.add_crossref_type( - directivename="setting", - rolename="setting", - indextemplate="pair: %s; setting", - ) - app.add_crossref_type( - directivename="signal", - rolename="signal", - indextemplate="pair: %s; signal", - ) - app.add_crossref_type( - directivename="command", - rolename="command", - indextemplate="pair: %s; command", - ) - app.add_crossref_type( - directivename="reqmeta", - rolename="reqmeta", - indextemplate="pair: %s; reqmeta", - ) app.add_role("source", source_role) app.add_role("commit", commit_role) app.add_role("issue", issue_role) app.add_role("rev", rev_role) - app.add_node(SettingslistNode) + app.add_node( + SettingslistNode, + markdown=(visit_settingslist_node_markdown, depart_settingslist_node_markdown), + singlemarkdown=( + visit_settingslist_node_markdown, + depart_settingslist_node_markdown, + ), + ) app.add_directive("settingslist", SettingsListDirective) app.connect("doctree-read", collect_scrapy_settings_refs) diff --git a/docs/conf.py b/docs/conf.py index 9b1db9bb7..e3e618860 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -28,11 +28,9 @@ author = "Scrapy developers" extensions = [ "notfound.extension", "scrapydocs", - "sphinx.ext.autodoc", + "sphinx_scrapy", "scrapyfixautodoc", # Must be after "sphinx.ext.autodoc" "sphinx.ext.coverage", - "sphinx.ext.intersphinx", - "sphinx.ext.viewcode", "sphinx_rtd_dark_mode", ] @@ -147,22 +145,24 @@ coverage_ignore_pyobjects = [ # -- Options for the InterSphinx extension ----------------------------------- # https://www.sphinx-doc.org/en/master/usage/extensions/intersphinx.html#configuration -intersphinx_mapping = { - "attrs": ("https://www.attrs.org/en/stable/", None), - "coverage": ("https://coverage.readthedocs.io/en/latest", None), - "cryptography": ("https://cryptography.io/en/latest/", None), - "cssselect": ("https://cssselect.readthedocs.io/en/latest", None), - "itemloaders": ("https://itemloaders.readthedocs.io/en/latest/", None), - "parsel": ("https://parsel.readthedocs.io/en/latest/", None), - "pytest": ("https://docs.pytest.org/en/latest", None), - "python": ("https://docs.python.org/3", None), - "sphinx": ("https://www.sphinx-doc.org/en/master", None), - "tox": ("https://tox.wiki/en/latest/", None), - "twisted": ("https://docs.twisted.org/en/stable/", None), - "twistedapi": ("https://docs.twisted.org/en/stable/api/", None), - "w3lib": ("https://w3lib.readthedocs.io/en/latest", None), -} intersphinx_disabled_reftypes: Sequence[str] = [] +# sphinx-scrapy --------------------------------------------------------------- + +scrapy_intersphinx_enable = [ + "attrs", + "coverage", + "cryptography", + "cssselect", + "itemloaders", + "parsel", + "pytest", + "sphinx", + "tox", + "twisted", + "twistedapi", + "w3lib", +] + # -- Other options ------------------------------------------------------------ default_dark_mode = False diff --git a/docs/requirements.in b/docs/requirements.in new file mode 100644 index 000000000..d0ef2455e --- /dev/null +++ b/docs/requirements.in @@ -0,0 +1,8 @@ +h2 +pydantic +scrapy-spider-metadata +sphinx +sphinx-notfound-page +sphinx-rtd-theme +sphinx-rtd-dark-mode +sphinx-scrapy @ git+https://github.com/scrapy/sphinx-scrapy.git@0.7.0 diff --git a/docs/requirements.txt b/docs/requirements.txt index 500db7a7e..354c670c7 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,7 +1,192 @@ +# This file was autogenerated by uv via the following command: +# uv pip compile requirements.in -o requirements.txt +alabaster==1.0.0 + # via sphinx +annotated-types==0.7.0 + # via pydantic +attrs==26.1.0 + # via + # service-identity + # twisted +automat==25.4.16 + # via twisted +babel==2.18.0 + # via sphinx +certifi==2026.2.25 + # via requests +cffi==2.0.0 + # via cryptography +charset-normalizer==3.4.6 + # via requests +constantly==23.10.4 + # via twisted +cryptography==46.0.6 + # via + # pyopenssl + # scrapy + # service-identity +cssselect==1.4.0 + # via + # parsel + # scrapy +defusedxml==0.7.1 + # via scrapy +docutils==0.22.4 + # via + # sphinx + # sphinx-markdown-builder + # sphinx-rtd-theme +filelock==3.25.2 + # via tldextract h2==4.3.0 -pydantic==2.12.3 + # via -r requirements.in +hpack==4.1.0 + # via h2 +hyperframe==6.1.0 + # via h2 +hyperlink==21.0.0 + # via twisted +idna==3.11 + # via + # hyperlink + # requests + # tldextract +imagesize==2.0.0 + # via sphinx +incremental==24.11.0 + # via twisted +itemadapter==0.13.1 + # via + # itemloaders + # scrapy +itemloaders==1.4.0 + # via scrapy +jinja2==3.1.6 + # via sphinx +jmespath==1.1.0 + # via + # itemloaders + # parsel +lxml==6.0.2 + # via + # parsel + # scrapy +markupsafe==3.0.3 + # via jinja2 +packaging==26.0 + # via + # incremental + # parsel + # scrapy + # scrapy-spider-metadata + # sphinx + # sphinx-scrapy +parsel==1.11.0 + # via + # itemloaders + # scrapy +protego==0.6.0 + # via scrapy +pyasn1==0.6.3 + # via + # pyasn1-modules + # service-identity +pyasn1-modules==0.4.2 + # via service-identity +pycparser==3.0 + # via cffi +pydantic==2.12.5 + # via + # -r requirements.in + # scrapy-spider-metadata +pydantic-core==2.41.5 + # via pydantic +pydispatcher==2.0.7 + # via scrapy +pygments==2.19.2 + # via sphinx +pyopenssl==26.0.0 + # via scrapy +queuelib==1.9.0 + # via scrapy +requests==2.33.0 + # via + # requests-file + # sphinx + # tldextract +requests-file==3.0.1 + # via tldextract +roman-numerals==4.1.0 + # via sphinx +scrapy==2.14.2 + # via scrapy-spider-metadata scrapy-spider-metadata==0.2.0 -sphinx==8.1.3 -sphinx-notfound-page==1.0.4 -sphinx-rtd-theme==3.0.2 + # via -r requirements.in +service-identity==24.2.0 + # via scrapy +snowballstemmer==3.0.1 + # via sphinx +sphinx==9.1.0 + # via + # -r requirements.in + # sphinx-copybutton + # sphinx-llms-txt + # sphinx-markdown-builder + # sphinx-notfound-page + # sphinx-rtd-theme + # sphinx-scrapy + # sphinxcontrib-jquery +sphinx-copybutton==0.5.2 + # via sphinx-scrapy +sphinx-llms-txt @ git+https://github.com/zytedata/sphinx-llms-txt.git@5e8866cb0cc249aa2017ad9050b3b83a7ca16f69 + # via sphinx-scrapy +sphinx-markdown-builder @ git+https://github.com/zytedata/sphinx-markdown-builder.git@ac9f8babfe622e4300099ab44b96d9d9228e742e + # via sphinx-scrapy +sphinx-notfound-page==1.1.0 + # via -r requirements.in sphinx-rtd-dark-mode==1.3.0 + # via -r requirements.in +sphinx-rtd-theme==3.1.0 + # via + # -r requirements.in + # sphinx-rtd-dark-mode +sphinx-scrapy @ git+https://github.com/scrapy/sphinx-scrapy.git@46c52fc3a6d5ee632a8ef6bccf4daa1ff53e96fd + # via -r requirements.in +sphinxcontrib-applehelp==2.0.0 + # via sphinx +sphinxcontrib-devhelp==2.0.0 + # via sphinx +sphinxcontrib-htmlhelp==2.1.0 + # via sphinx +sphinxcontrib-jquery==4.1 + # via sphinx-rtd-theme +sphinxcontrib-jsmath==1.0.1 + # via sphinx +sphinxcontrib-qthelp==2.0.0 + # via sphinx +sphinxcontrib-serializinghtml==2.0.0 + # via sphinx +tabulate==0.10.0 + # via sphinx-markdown-builder +tldextract==5.3.1 + # via scrapy +twisted==25.5.0 + # via scrapy +typing-extensions==4.15.0 + # via + # pydantic + # pydantic-core + # twisted + # typing-inspection +typing-inspection==0.4.2 + # via pydantic +urllib3==2.6.3 + # via requests +w3lib==2.4.1 + # via + # parsel + # scrapy +zope-interface==8.2 + # via + # scrapy + # twisted diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index 98374b6b4..806af509f 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -198,7 +198,7 @@ scrapy.Spider The ``parse`` method is in charge of processing the response and returning scraped data and/or more URLs to follow. Other Requests callbacks have - the same requirements as the :class:`Spider` class. + the same requirements as the :class:`~scrapy.Spider` class. This method, as well as any other Request callback, must return a :class:`~scrapy.Request` object, an :ref:`item object `, an diff --git a/pyproject.toml b/pyproject.toml index 117edce0f..70ef73f26 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -464,3 +464,6 @@ split-on-trailing-comma = false [tool.ruff.lint.pydocstyle] convention = "pep257" + +[tool.sphinx-scrapy] +python-version = "3.13" # Keep in sync with .github/workflows/checks.yml. diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.py index b6dba5773..d38b4b9db 100644 --- a/scrapy/downloadermiddlewares/retry.py +++ b/scrapy/downloadermiddlewares/retry.py @@ -25,11 +25,11 @@ if TYPE_CHECKING: # typing.Self requires Python 3.11 from typing_extensions import Self + import scrapy from scrapy.crawler import Crawler from scrapy.http import Response from scrapy.http.request import Request from scrapy.settings import BaseSettings - from scrapy.spiders import Spider retry_logger = getLogger(__name__) @@ -38,7 +38,7 @@ retry_logger = getLogger(__name__) def get_retry_request( request: Request, *, - spider: Spider, + spider: scrapy.Spider, reason: str | Exception | type[Exception] = "unspecified", max_retry_times: int | None = None, priority_adjust: int | None = None, @@ -145,7 +145,10 @@ class RetryMiddleware: @_warn_spider_arg def process_response( - self, request: Request, response: Response, spider: Spider | None = None + self, + request: Request, + response: Response, + spider: scrapy.Spider | None = None, ) -> Request | Response: if request.meta.get("dont_retry", False): return response @@ -156,7 +159,10 @@ class RetryMiddleware: @_warn_spider_arg def process_exception( - self, request: Request, exception: Exception, spider: Spider | None = None + self, + request: Request, + exception: Exception, + spider: scrapy.Spider | None = None, ) -> Request | Response | None: if isinstance(exception, self.exceptions_to_retry) and not request.meta.get( "dont_retry", False diff --git a/tox.ini b/tox.ini index d35b0406c..0e86f964a 100644 --- a/tox.ini +++ b/tox.ini @@ -4,7 +4,9 @@ # and then run "tox" from this directory. [tox] -envlist = pre-commit,pylint,typing,py +requires = + sphinx-scrapy @ git+https://github.com/scrapy/sphinx-scrapy.git@0.7.0 +envlist = pre-commit,pylint,typing,py,docs minversion = 1.7.0 [test-requirements] @@ -238,38 +240,25 @@ commands = setenv = {[pinned]setenv} -[docs] +[testenv:docs-tests] changedir = docs -deps = - -rdocs/requirements.txt -setenv = - READTHEDOCS_PROJECT=scrapy - READTHEDOCS_VERSION=master - -[testenv:docs] -basepython = python3 -changedir = {[docs]changedir} deps = {[test-requirements]deps} - {[docs]deps} -setenv = {[docs]setenv} + -rdocs/requirements.txt commands = - sphinx-build -W -b html . {envtmpdir}/html pytest [testenv:docs-coverage] -basepython = python3 -changedir = {[docs]changedir} -deps = {[docs]deps} -setenv = {[docs]setenv} +changedir = docs +deps = + -rdocs/requirements.txt commands = sphinx-build -b coverage . {envtmpdir}/coverage [testenv:docs-links] -basepython = python3 -changedir = {[docs]changedir} -deps = {[docs]deps} -setenv = {[docs]setenv} +changedir = docs +deps = + -rdocs/requirements.txt commands = sphinx-build -W -b linkcheck . {envtmpdir}/linkcheck