Add llms.txt and llms-full.txt generation (#7380)

.github/workflows/checks.yml

@@ -26,9 +26,13 @@ jobs:
         - python-version: "3.10"
           env:
             TOXENV: typing-tests
-        - python-version: "3.13"  # Keep in sync with .readthedocs.yml
+        # Keep in sync with pyproject.toml tool.sphinx-scrapy.python-version.
+        - python-version: "3.13"
           env:
             TOXENV: docs
+        - python-version: "3.13"
+          env:
+            TOXENV: docs-tests
         - python-version: "3.13"
           env:
             TOXENV: twinecheck

.gitignore

@@ -3,7 +3,7 @@
 *.pyc
 _trial_temp*
 dropin.cache
-docs/build
+docs/_build
 *egg-info
 .tox/
 venv/

.pre-commit-config.yaml

@@ -26,3 +26,7 @@ repos:
   rev: v1.0.2
   hooks:
     - id: sphinx-lint
+- repo: https://github.com/scrapy/sphinx-scrapy
+  rev: 0.7.0
+  hooks:
+    - id: sphinx-scrapy

.readthedocs.yml

@@ -1,17 +1,9 @@
 version: 2
-formats: all
-sphinx:
-  configuration: docs/conf.py
-  fail_on_warning: true
-
 build:
   os: ubuntu-24.04
   tools:
-    # For available versions, see:
-    # https://docs.readthedocs.io/en/stable/config-file/v2.html#build-tools-python
-    python: "3.13"  # Keep in sync with .github/workflows/checks.yml
-
-python:
-  install:
-    - requirements: docs/requirements.txt
-    - path: .
+    python: "3.13"
+  commands:
+    - pip install tox
+    - tox -e docs
+    - cp -a docs/_build/all/. $READTHEDOCS_OUTPUT/html/

docs/README.rst

@@ -65,4 +65,4 @@ To compile the documentation to HTML run the following command::
 
     tox -e docs
 
-Documentation will be generated (in HTML format) inside the ``.tox/docs/tmp/html`` dir.
+Documentation will be generated inside the ``docs/_build/all`` dir.

docs/_ext/scrapydocs.py

@@ -77,6 +77,25 @@ def make_setting_element(
     return item
 
 
+def make_setting_markdown_item(
+    setting_data: SettingData, app: Sphinx, fromdocname: str
+) -> str:
+    uri = app.builder.get_relative_uri(fromdocname, setting_data["docname"])
+    if uri.startswith("#"):
+        target = f"#{setting_data['refid']}"
+    else:
+        target = f"{uri}#{setting_data['refid']}"
+    return f"* [{setting_data['setting_name']}]({target})"
+
+
+def _iter_sorted_settings(env: Any, fromdocname: str) -> list[SettingData]:
+    return [
+        d
+        for d in sorted(env.scrapy_all_settings, key=itemgetter("setting_name"))  # type: ignore[attr-defined]
+        if fromdocname != d["docname"]
+    ]
+
+
 def replace_settingslist_nodes(
     app: Sphinx, doctree: document, fromdocname: str
 ) -> None:
@@ -87,13 +106,29 @@ def replace_settingslist_nodes(
         settings_list.extend(
             [
                 make_setting_element(d, app, fromdocname)
-                for d in sorted(env.scrapy_all_settings, key=itemgetter("setting_name"))  # type: ignore[attr-defined]
-                if fromdocname != d["docname"]
+                for d in _iter_sorted_settings(env, fromdocname)
             ]
         )
         node.replace_self(settings_list)
 
 
+def visit_settingslist_node_markdown(translator: Any, _node: Node) -> None:
+    builder = translator.builder
+    env = builder.env
+    fromdocname = getattr(builder, "current_doc_name", env.docname)
+    lines = [
+        make_setting_markdown_item(setting_data, builder.app, fromdocname)
+        for setting_data in _iter_sorted_settings(env, fromdocname)
+    ]
+    if lines:
+        translator.add("\n".join(lines), prefix_eol=2, suffix_eol=2)
+    raise nodes.SkipNode
+
+
+def depart_settingslist_node_markdown(_translator: Any, _node: Node) -> None:
+    return None
+
+
 def source_role(
     name, rawtext, text: str, lineno, inliner, options=None, content=None
 ) -> tuple[list[Any], list[Any]]:
@@ -127,32 +162,19 @@ def rev_role(
 
 
 def setup(app: Sphinx) -> dict[str, Any]:
-    app.add_crossref_type(
-        directivename="setting",
-        rolename="setting",
-        indextemplate="pair: %s; setting",
-    )
-    app.add_crossref_type(
-        directivename="signal",
-        rolename="signal",
-        indextemplate="pair: %s; signal",
-    )
-    app.add_crossref_type(
-        directivename="command",
-        rolename="command",
-        indextemplate="pair: %s; command",
-    )
-    app.add_crossref_type(
-        directivename="reqmeta",
-        rolename="reqmeta",
-        indextemplate="pair: %s; reqmeta",
-    )
     app.add_role("source", source_role)
     app.add_role("commit", commit_role)
     app.add_role("issue", issue_role)
     app.add_role("rev", rev_role)
 
-    app.add_node(SettingslistNode)
+    app.add_node(
+        SettingslistNode,
+        markdown=(visit_settingslist_node_markdown, depart_settingslist_node_markdown),
+        singlemarkdown=(
+            visit_settingslist_node_markdown,
+            depart_settingslist_node_markdown,
+        ),
+    )
     app.add_directive("settingslist", SettingsListDirective)
 
     app.connect("doctree-read", collect_scrapy_settings_refs)

docs/conf.py

@@ -28,11 +28,9 @@ author = "Scrapy developers"
 extensions = [
     "notfound.extension",
     "scrapydocs",
-    "sphinx.ext.autodoc",
+    "sphinx_scrapy",
     "scrapyfixautodoc",  # Must be after "sphinx.ext.autodoc"
     "sphinx.ext.coverage",
-    "sphinx.ext.intersphinx",
-    "sphinx.ext.viewcode",
     "sphinx_rtd_dark_mode",
 ]
 
@@ -147,22 +145,24 @@ coverage_ignore_pyobjects = [
 # -- Options for the InterSphinx extension -----------------------------------
 # https://www.sphinx-doc.org/en/master/usage/extensions/intersphinx.html#configuration
 
-intersphinx_mapping = {
-    "attrs": ("https://www.attrs.org/en/stable/", None),
-    "coverage": ("https://coverage.readthedocs.io/en/latest", None),
-    "cryptography": ("https://cryptography.io/en/latest/", None),
-    "cssselect": ("https://cssselect.readthedocs.io/en/latest", None),
-    "itemloaders": ("https://itemloaders.readthedocs.io/en/latest/", None),
-    "parsel": ("https://parsel.readthedocs.io/en/latest/", None),
-    "pytest": ("https://docs.pytest.org/en/latest", None),
-    "python": ("https://docs.python.org/3", None),
-    "sphinx": ("https://www.sphinx-doc.org/en/master", None),
-    "tox": ("https://tox.wiki/en/latest/", None),
-    "twisted": ("https://docs.twisted.org/en/stable/", None),
-    "twistedapi": ("https://docs.twisted.org/en/stable/api/", None),
-    "w3lib": ("https://w3lib.readthedocs.io/en/latest", None),
-}
 intersphinx_disabled_reftypes: Sequence[str] = []
 
+# sphinx-scrapy ---------------------------------------------------------------
+
+scrapy_intersphinx_enable = [
+    "attrs",
+    "coverage",
+    "cryptography",
+    "cssselect",
+    "itemloaders",
+    "parsel",
+    "pytest",
+    "sphinx",
+    "tox",
+    "twisted",
+    "twistedapi",
+    "w3lib",
+]
+
 # -- Other options ------------------------------------------------------------
 default_dark_mode = False

docs/requirements.in

@@ -0,0 +1,8 @@
+h2
+pydantic
+scrapy-spider-metadata
+sphinx
+sphinx-notfound-page
+sphinx-rtd-theme
+sphinx-rtd-dark-mode
+sphinx-scrapy @ git+https://github.com/scrapy/sphinx-scrapy.git@0.7.0

docs/requirements.txt

@@ -1,7 +1,192 @@
+# This file was autogenerated by uv via the following command:
+#    uv pip compile requirements.in -o requirements.txt
+alabaster==1.0.0
+    # via sphinx
+annotated-types==0.7.0
+    # via pydantic
+attrs==26.1.0
+    # via
+    #   service-identity
+    #   twisted
+automat==25.4.16
+    # via twisted
+babel==2.18.0
+    # via sphinx
+certifi==2026.2.25
+    # via requests
+cffi==2.0.0
+    # via cryptography
+charset-normalizer==3.4.6
+    # via requests
+constantly==23.10.4
+    # via twisted
+cryptography==46.0.6
+    # via
+    #   pyopenssl
+    #   scrapy
+    #   service-identity
+cssselect==1.4.0
+    # via
+    #   parsel
+    #   scrapy
+defusedxml==0.7.1
+    # via scrapy
+docutils==0.22.4
+    # via
+    #   sphinx
+    #   sphinx-markdown-builder
+    #   sphinx-rtd-theme
+filelock==3.25.2
+    # via tldextract
 h2==4.3.0
-pydantic==2.12.3
+    # via -r requirements.in
+hpack==4.1.0
+    # via h2
+hyperframe==6.1.0
+    # via h2
+hyperlink==21.0.0
+    # via twisted
+idna==3.11
+    # via
+    #   hyperlink
+    #   requests
+    #   tldextract
+imagesize==2.0.0
+    # via sphinx
+incremental==24.11.0
+    # via twisted
+itemadapter==0.13.1
+    # via
+    #   itemloaders
+    #   scrapy
+itemloaders==1.4.0
+    # via scrapy
+jinja2==3.1.6
+    # via sphinx
+jmespath==1.1.0
+    # via
+    #   itemloaders
+    #   parsel
+lxml==6.0.2
+    # via
+    #   parsel
+    #   scrapy
+markupsafe==3.0.3
+    # via jinja2
+packaging==26.0
+    # via
+    #   incremental
+    #   parsel
+    #   scrapy
+    #   scrapy-spider-metadata
+    #   sphinx
+    #   sphinx-scrapy
+parsel==1.11.0
+    # via
+    #   itemloaders
+    #   scrapy
+protego==0.6.0
+    # via scrapy
+pyasn1==0.6.3
+    # via
+    #   pyasn1-modules
+    #   service-identity
+pyasn1-modules==0.4.2
+    # via service-identity
+pycparser==3.0
+    # via cffi
+pydantic==2.12.5
+    # via
+    #   -r requirements.in
+    #   scrapy-spider-metadata
+pydantic-core==2.41.5
+    # via pydantic
+pydispatcher==2.0.7
+    # via scrapy
+pygments==2.19.2
+    # via sphinx
+pyopenssl==26.0.0
+    # via scrapy
+queuelib==1.9.0
+    # via scrapy
+requests==2.33.0
+    # via
+    #   requests-file
+    #   sphinx
+    #   tldextract
+requests-file==3.0.1
+    # via tldextract
+roman-numerals==4.1.0
+    # via sphinx
+scrapy==2.14.2
+    # via scrapy-spider-metadata
 scrapy-spider-metadata==0.2.0
-sphinx==8.1.3
-sphinx-notfound-page==1.0.4
-sphinx-rtd-theme==3.0.2
+    # via -r requirements.in
+service-identity==24.2.0
+    # via scrapy
+snowballstemmer==3.0.1
+    # via sphinx
+sphinx==9.1.0
+    # via
+    #   -r requirements.in
+    #   sphinx-copybutton
+    #   sphinx-llms-txt
+    #   sphinx-markdown-builder
+    #   sphinx-notfound-page
+    #   sphinx-rtd-theme
+    #   sphinx-scrapy
+    #   sphinxcontrib-jquery
+sphinx-copybutton==0.5.2
+    # via sphinx-scrapy
+sphinx-llms-txt @ git+https://github.com/zytedata/sphinx-llms-txt.git@5e8866cb0cc249aa2017ad9050b3b83a7ca16f69
+    # via sphinx-scrapy
+sphinx-markdown-builder @ git+https://github.com/zytedata/sphinx-markdown-builder.git@ac9f8babfe622e4300099ab44b96d9d9228e742e
+    # via sphinx-scrapy
+sphinx-notfound-page==1.1.0
+    # via -r requirements.in
 sphinx-rtd-dark-mode==1.3.0
+    # via -r requirements.in
+sphinx-rtd-theme==3.1.0
+    # via
+    #   -r requirements.in
+    #   sphinx-rtd-dark-mode
+sphinx-scrapy @ git+https://github.com/scrapy/sphinx-scrapy.git@46c52fc3a6d5ee632a8ef6bccf4daa1ff53e96fd
+    # via -r requirements.in
+sphinxcontrib-applehelp==2.0.0
+    # via sphinx
+sphinxcontrib-devhelp==2.0.0
+    # via sphinx
+sphinxcontrib-htmlhelp==2.1.0
+    # via sphinx
+sphinxcontrib-jquery==4.1
+    # via sphinx-rtd-theme
+sphinxcontrib-jsmath==1.0.1
+    # via sphinx
+sphinxcontrib-qthelp==2.0.0
+    # via sphinx
+sphinxcontrib-serializinghtml==2.0.0
+    # via sphinx
+tabulate==0.10.0
+    # via sphinx-markdown-builder
+tldextract==5.3.1
+    # via scrapy
+twisted==25.5.0
+    # via scrapy
+typing-extensions==4.15.0
+    # via
+    #   pydantic
+    #   pydantic-core
+    #   twisted
+    #   typing-inspection
+typing-inspection==0.4.2
+    # via pydantic
+urllib3==2.6.3
+    # via requests
+w3lib==2.4.1
+    # via
+    #   parsel
+    #   scrapy
+zope-interface==8.2
+    # via
+    #   scrapy
+    #   twisted

docs/topics/spiders.rst

@@ -198,7 +198,7 @@ scrapy.Spider
 
        The ``parse`` method is in charge of processing the response and returning
        scraped data and/or more URLs to follow. Other Requests callbacks have
-       the same requirements as the :class:`Spider` class.
+       the same requirements as the :class:`~scrapy.Spider` class.
 
        This method, as well as any other Request callback, must return a
        :class:`~scrapy.Request` object, an :ref:`item object <topics-items>`, an

pyproject.toml

@@ -464,3 +464,6 @@ split-on-trailing-comma = false
 
 [tool.ruff.lint.pydocstyle]
 convention = "pep257"
+
+[tool.sphinx-scrapy]
+python-version = "3.13"  # Keep in sync with .github/workflows/checks.yml.

scrapy/downloadermiddlewares/retry.py

@@ -25,11 +25,11 @@ if TYPE_CHECKING:
     # typing.Self requires Python 3.11
     from typing_extensions import Self
 
+    import scrapy
     from scrapy.crawler import Crawler
     from scrapy.http import Response
     from scrapy.http.request import Request
     from scrapy.settings import BaseSettings
-    from scrapy.spiders import Spider
 
 
 retry_logger = getLogger(__name__)
@@ -38,7 +38,7 @@ retry_logger = getLogger(__name__)
 def get_retry_request(
     request: Request,
     *,
-    spider: Spider,
+    spider: scrapy.Spider,
     reason: str | Exception | type[Exception] = "unspecified",
     max_retry_times: int | None = None,
     priority_adjust: int | None = None,
@@ -145,7 +145,10 @@ class RetryMiddleware:
 
     @_warn_spider_arg
     def process_response(
-        self, request: Request, response: Response, spider: Spider | None = None
+        self,
+        request: Request,
+        response: Response,
+        spider: scrapy.Spider | None = None,
     ) -> Request | Response:
         if request.meta.get("dont_retry", False):
             return response
@@ -156,7 +159,10 @@ class RetryMiddleware:
 
     @_warn_spider_arg
     def process_exception(
-        self, request: Request, exception: Exception, spider: Spider | None = None
+        self,
+        request: Request,
+        exception: Exception,
+        spider: scrapy.Spider | None = None,
     ) -> Request | Response | None:
         if isinstance(exception, self.exceptions_to_retry) and not request.meta.get(
             "dont_retry", False

tox.ini

@@ -4,7 +4,9 @@
 # and then run "tox" from this directory.
 
 [tox]
-envlist = pre-commit,pylint,typing,py
+requires =
+    sphinx-scrapy @ git+https://github.com/scrapy/sphinx-scrapy.git@0.7.0
+envlist = pre-commit,pylint,typing,py,docs
 minversion = 1.7.0
 
 [test-requirements]
@@ -238,38 +240,25 @@ commands =
 setenv =
     {[pinned]setenv}
 
-[docs]
+[testenv:docs-tests]
 changedir = docs
-deps =
-    -rdocs/requirements.txt
-setenv =
-    READTHEDOCS_PROJECT=scrapy
-    READTHEDOCS_VERSION=master
-
-[testenv:docs]
-basepython = python3
-changedir = {[docs]changedir}
 deps =
     {[test-requirements]deps}
-    {[docs]deps}
-setenv = {[docs]setenv}
+    -rdocs/requirements.txt
 commands =
-    sphinx-build -W -b html . {envtmpdir}/html
     pytest
 
 [testenv:docs-coverage]
-basepython = python3
-changedir = {[docs]changedir}
-deps = {[docs]deps}
-setenv = {[docs]setenv}
+changedir = docs
+deps =
+    -rdocs/requirements.txt
 commands =
     sphinx-build -b coverage . {envtmpdir}/coverage
 
 [testenv:docs-links]
-basepython = python3
-changedir = {[docs]changedir}
-deps = {[docs]deps}
-setenv = {[docs]setenv}
+changedir = docs
+deps =
+    -rdocs/requirements.txt
 commands =
     sphinx-build -W -b linkcheck . {envtmpdir}/linkcheck