diff --git a/docs/_ext/scrapydocs.py b/docs/_ext/scrapydocs.py index 1fa1c93d6..f0827f2b1 100644 --- a/docs/_ext/scrapydocs.py +++ b/docs/_ext/scrapydocs.py @@ -1,5 +1,82 @@ from docutils.parsers.rst.roles import set_classes from docutils import nodes +from sphinx.util.compat import Directive +from sphinx.util.nodes import make_refnode +from operator import itemgetter + + +class settingslist_node(nodes.General, nodes.Element): + pass + + +class SettingsListDirective(Directive): + def run(self): + return [settingslist_node('')] + + +def is_setting_index(node): + if node.tagname == 'index': + # index entries for setting directives look like: + # [(u'pair', u'SETTING_NAME; setting', u'std:setting-SETTING_NAME', '')] + entry_type, info, refid, _ = node['entries'][0] + return entry_type == 'pair' and info.endswith('; setting') + return False + + +def get_setting_target(node): + # target nodes are placed next to the node in the doc tree + return node.parent[node.parent.index(node) + 1] + + +def get_setting_name_and_refid(node): + """Extract setting name from directive index node""" + entry_type, info, refid, _ = node['entries'][0] + return info.replace('; setting', ''), refid + + +def collect_scrapy_settings_refs(app, doctree): + env = app.builder.env + + if not hasattr(env, 'scrapy_all_settings'): + env.scrapy_all_settings = [] + + for node in doctree.traverse(is_setting_index): + targetnode = get_setting_target(node) + assert isinstance(targetnode, nodes.target), "Next node is not a target" + + setting_name, refid = get_setting_name_and_refid(node) + + env.scrapy_all_settings.append({ + 'docname': env.docname, + 'setting_name': setting_name, + 'refid': refid, + }) + + +def make_setting_element(setting_data, app, fromdocname): + refnode = make_refnode(app.builder, fromdocname, + todocname=setting_data['docname'], + targetid=setting_data['refid'], + child=nodes.Text(setting_data['setting_name'])) + p = nodes.paragraph() + p += refnode + + item = nodes.list_item() + item += p + return item + + +def replace_settingslist_nodes(app, doctree, fromdocname): + env = app.builder.env + + for node in doctree.traverse(settingslist_node): + settings_list = nodes.bullet_list() + settings_list.extend([make_setting_element(d, app, fromdocname) + for d in sorted(env.scrapy_all_settings, + key=itemgetter('setting_name')) + if fromdocname != d['docname']]) + node.replace_self(settings_list) + def setup(app): app.add_crossref_type( @@ -27,6 +104,12 @@ def setup(app): app.add_role('issue', issue_role) app.add_role('rev', rev_role) + app.add_node(settingslist_node) + app.add_directive('settingslist', SettingsListDirective) + + app.connect('doctree-read', collect_scrapy_settings_refs) + app.connect('doctree-resolved', replace_settingslist_nodes) + def source_role(name, rawtext, text, lineno, inliner, options={}, content=[]): ref = 'https://github.com/scrapy/scrapy/blob/master/' + text set_classes(options) diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 26a6d762d..a72a991e5 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -1015,6 +1015,16 @@ Default: ``"Scrapy/VERSION (+http://scrapy.org)"`` The default User-Agent to use when crawling, unless overridden. + +Settings documented elsewhere: +------------------------------ + +The following settings are documented elsewhere, please check each specific +case to see how to enable and use them. + +.. settingslist:: + + .. _Amazon web services: http://aws.amazon.com/ .. _breadth-first order: http://en.wikipedia.org/wiki/Breadth-first_search .. _depth-first order: http://en.wikipedia.org/wiki/Depth-first_search