Omit pre-built man pages from sdist (#81395)

Since man pages aren't accessible to users after a `pip install`, there's no need to include them in the sdist.
This change makes it trivial to build man pages from source, which makes them much easier to iterate on.
It also simplifies creation and testing of the sdist, since it no longer requires building man pages.

The new `packaging/cli-doc/build.py` script can generate both man pages and RST documentation.
This supports inclusion on the docs site without a dependency on `ansible-core` internals.
Having a single implementation for both simplifies keeping the two formats in sync.

3 files changed, 570 insertions, 0 deletions

diff --git a/packaging/cli-doc/build.py b/packaging/cli-doc/build.py
new file mode 100755
index 0000000000..878ba8eabf
--- /dev/null
+++ b/packaging/cli-doc/build.py
@@ -0,0 +1,279 @@
+#!/usr/bin/env python
+# PYTHON_ARGCOMPLETE_OK
+"""Build documentation for ansible-core CLI programs."""
+
+from __future__ import annotations
+
+import argparse
+import dataclasses
+import importlib
+import inspect
+import io
+import itertools
+import json
+import pathlib
+import sys
+import typing as t
+import warnings
+
+import jinja2
+
+if t.TYPE_CHECKING:
+    from ansible.cli import CLI  # pragma: nocover
+
+SCRIPT_DIR = pathlib.Path(__file__).resolve().parent
+SOURCE_DIR = SCRIPT_DIR.parent.parent
+
+
+def main() -> None:
+    """Main program entry point."""
+    parser = argparse.ArgumentParser(description=__doc__)
+    subparsers = parser.add_subparsers(required=True, metavar='command')
+
+    man_parser = subparsers.add_parser('man', description=build_man.__doc__, help=build_man.__doc__)
+    man_parser.add_argument('--output-dir', required=True, type=pathlib.Path, metavar='DIR', help='output directory')
+    man_parser.add_argument('--template-file', default=SCRIPT_DIR / 'man.j2', type=pathlib.Path, metavar='FILE', help='template file')
+    man_parser.set_defaults(func=build_man)
+
+    rst_parser = subparsers.add_parser('rst', description=build_rst.__doc__, help=build_rst.__doc__)
+    rst_parser.add_argument('--output-dir', required=True, type=pathlib.Path, metavar='DIR', help='output directory')
+    rst_parser.add_argument('--template-file', default=SCRIPT_DIR / 'rst.j2', type=pathlib.Path, metavar='FILE', help='template file')
+    rst_parser.set_defaults(func=build_rst)
+
+    json_parser = subparsers.add_parser('json', description=build_json.__doc__, help=build_json.__doc__)
+    json_parser.add_argument('--output-file', required=True, type=pathlib.Path, metavar='FILE', help='output file')
+    json_parser.set_defaults(func=build_json)
+
+    try:
+        # noinspection PyUnresolvedReferences
+        import argcomplete
+    except ImportError:
+        pass
+    else:
+        argcomplete.autocomplete(parser)
+
+    args = parser.parse_args()
+    kwargs = {name: getattr(args, name) for name in inspect.signature(args.func).parameters}
+
+    sys.path.insert(0, str(SOURCE_DIR / 'lib'))
+
+    args.func(**kwargs)
+
+
+def build_man(output_dir: pathlib.Path, template_file: pathlib.Path) -> None:
+    """Build man pages for ansible-core CLI programs."""
+    if not template_file.resolve().is_relative_to(SCRIPT_DIR):
+        warnings.warn("Custom templates are intended for debugging purposes only. The data model may change in future releases without notice.")
+
+    import docutils.core
+    import docutils.writers.manpage
+
+    output_dir.mkdir(exist_ok=True, parents=True)
+
+    for cli_name, source in generate_rst(template_file).items():
+        with io.StringIO(source) as source_file:
+            docutils.core.publish_file(
+                source=source_file,
+                destination_path=output_dir / f'{cli_name}.1',
+                writer=docutils.writers.manpage.Writer(),
+            )
+
+
+def build_rst(output_dir: pathlib.Path, template_file: pathlib.Path) -> None:
+    """Build RST documentation for ansible-core CLI programs."""
+    if not template_file.resolve().is_relative_to(SCRIPT_DIR):
+        warnings.warn("Custom templates are intended for debugging purposes only. The data model may change in future releases without notice.")
+
+    output_dir.mkdir(exist_ok=True, parents=True)
+
+    for cli_name, source in generate_rst(template_file).items():
+        (output_dir / f'{cli_name}.rst').write_text(source)
+
+
+def build_json(output_file: pathlib.Path) -> None:
+    """Build JSON documentation for ansible-core CLI programs."""
+    warnings.warn("JSON output is intended for debugging purposes only. The data model may change in future releases without notice.")
+
+    output_file.parent.mkdir(exist_ok=True, parents=True)
+    output_file.write_text(json.dumps(collect_programs(), indent=4))
+
+
+def generate_rst(template_file: pathlib.Path) -> dict[str, str]:
+    """Generate RST pages using the provided template."""
+    results: dict[str, str] = {}
+
+    for cli_name, template_vars in collect_programs().items():
+        env = jinja2.Environment(loader=jinja2.FileSystemLoader(template_file.parent))
+        template = env.get_template(template_file.name)
+        results[cli_name] = template.render(template_vars)
+
+    return results
+
+
+def collect_programs() -> dict[str, dict[str, t.Any]]:
+    """Return information about CLI programs."""
+    programs: list[tuple[str, dict[str, t.Any]]] = []
+    cli_bin_name_list: list[str] = []
+
+    for source_file in (SOURCE_DIR / 'lib/ansible/cli').glob('*.py'):
+        if source_file.name != '__init__.py':
+            programs.append(generate_options_docs(source_file, cli_bin_name_list))
+
+    return dict(programs)
+
+
+def generate_options_docs(source_file: pathlib.Path, cli_bin_name_list: list[str]) -> tuple[str, dict[str, t.Any]]:
+    """Generate doc structure from CLI module options."""
+    import ansible.release
+
+    if str(source_file).endswith('/lib/ansible/cli/adhoc.py'):
+        cli_name = 'ansible'
+        cli_class_name = 'AdHocCLI'
+        cli_module_fqn = 'ansible.cli.adhoc'
+    else:
+        cli_module_name = source_file.with_suffix('').name
+        cli_name = f'ansible-{cli_module_name}'
+        cli_class_name = f'{cli_module_name.capitalize()}CLI'
+        cli_module_fqn = f'ansible.cli.{cli_module_name}'
+
+    cli_bin_name_list.append(cli_name)
+
+    cli_module = importlib.import_module(cli_module_fqn)
+    cli_class: type[CLI] = getattr(cli_module, cli_class_name)
+
+    cli = cli_class([cli_name])
+    cli.init_parser()
+
+    parser: argparse.ArgumentParser = cli.parser
+    long_desc = cli.__doc__
+    arguments: dict[str, str] | None = getattr(cli, 'ARGUMENTS', None)
+
+    action_docs = get_action_docs(parser)
+    option_names: tuple[str, ...] = tuple(itertools.chain.from_iterable(opt.options for opt in action_docs))
+    actions: dict[str, dict[str, t.Any]] = {}
+
+    content_depth = populate_subparser_actions(parser, option_names, actions)
+
+    docs = dict(
+        version=ansible.release.__version__,
+        source=str(source_file.relative_to(SOURCE_DIR)),
+        cli_name=cli_name,
+        usage=parser.format_usage(),
+        short_desc=parser.description,
+        long_desc=trim_docstring(long_desc),
+        actions=actions,
+        options=[item.__dict__ for item in action_docs],
+        arguments=arguments,
+        option_names=option_names,
+        cli_bin_name_list=cli_bin_name_list,
+        content_depth=content_depth,
+        inventory='-i' in option_names,
+        library='-M' in option_names,
+    )
+
+    return cli_name, docs
+
+
+def populate_subparser_actions(parser: argparse.ArgumentParser, shared_option_names: tuple[str, ...], actions: dict[str, dict[str, t.Any]]) -> int:
+    """Generate doc structure from CLI module subparser options."""
+    try:
+        # noinspection PyProtectedMember
+        subparsers: dict[str, argparse.ArgumentParser] = parser._subparsers._group_actions[0].choices  # type: ignore
+    except AttributeError:
+        subparsers = {}
+
+    depth = 0
+
+    for subparser_action, subparser in subparsers.items():
+        subparser_option_names: set[str] = set()
+        subparser_action_docs: set[ActionDoc] = set()
+        subparser_actions: dict[str, dict[str, t.Any]] = {}
+
+        for action_doc in get_action_docs(subparser):
+            for option_alias in action_doc.options:
+                if option_alias in shared_option_names:
+                    continue
+
+                subparser_option_names.add(option_alias)
+                subparser_action_docs.add(action_doc)
+
+        depth = populate_subparser_actions(subparser, shared_option_names, subparser_actions)
+
+        actions[subparser_action] = dict(
+            option_names=list(subparser_option_names),
+            options=[item.__dict__ for item in subparser_action_docs],
+            actions=subparser_actions,
+            name=subparser_action,
+            desc=trim_docstring(subparser.get_default("func").__doc__),
+        )
+
+    return depth + 1
+
+
+@dataclasses.dataclass(frozen=True)
+class ActionDoc:
+    """Documentation for an action."""
+    desc: str | None
+    options: tuple[str, ...]
+    arg: str | None
+
+
+def get_action_docs(parser: argparse.ArgumentParser) -> list[ActionDoc]:
+    """Get action documentation from the given argument parser."""
+    action_docs = []
+
+    # noinspection PyProtectedMember
+    for action in parser._actions:
+        if action.help == argparse.SUPPRESS:
+            continue
+
+        # noinspection PyProtectedMember, PyUnresolvedReferences
+        args = action.dest.upper() if isinstance(action, argparse._StoreAction) else None
+
+        if args or action.option_strings:
+            action_docs.append(ActionDoc(
+                desc=action.help,
+                options=tuple(action.option_strings),
+                arg=args,
+            ))
+
+    return action_docs
+
+
+def trim_docstring(docstring: str | None) -> str:
+    """Trim and return the given docstring using the implementation from https://peps.python.org/pep-0257/#handling-docstring-indentation."""
+    if not docstring:
+        return ''  # pragma: nocover
+
+    # Convert tabs to spaces (following the normal Python rules) and split into a list of lines
+    lines = docstring.expandtabs().splitlines()
+
+    # Determine minimum indentation (first line doesn't count)
+    indent = sys.maxsize
+
+    for line in lines[1:]:
+        stripped = line.lstrip()
+
+        if stripped:
+            indent = min(indent, len(line) - len(stripped))
+
+    # Remove indentation (first line is special)
+    trimmed = [lines[0].strip()]
+
+    if indent < sys.maxsize:
+        for line in lines[1:]:
+            trimmed.append(line[indent:].rstrip())
+
+    # Strip off trailing and leading blank lines
+    while trimmed and not trimmed[-1]:
+        trimmed.pop()
+
+    while trimmed and not trimmed[0]:
+        trimmed.pop(0)
+
+    # Return a single string
+    return '\n'.join(trimmed)
+
+
+if __name__ == '__main__':
+    main()
diff --git a/packaging/cli-doc/man.j2 b/packaging/cli-doc/man.j2
new file mode 100644
index 0000000000..adb8093695
--- /dev/null
+++ b/packaging/cli-doc/man.j2
@@ -0,0 +1,139 @@
+{% macro render_action(parent, action, action_docs) -%}
+**{{ parent + action }}**
+  {{ (action_docs['desc']|default(' ')) |replace('\n', ' ')}}
+
+{% if action_docs['options'] %}
+{% for option in action_docs['options']|sort(attribute='options') %}
+{% for switch in option['options'] if switch in action_docs['option_names'] %}  **{{switch}}**{% if option['arg'] %} '{{option['arg']}}'{% endif %}{% if not loop.last %}, {% endif %}{% endfor %}
+
+        {{ (option['desc']) }}
+{% endfor %}
+{% endif %}
+
+{% set nested_actions = action_docs['actions'] %}
+{% if nested_actions %}
+{% for nested_action in nested_actions %}
+{{ render_action(parent + action + ' ', nested_action, nested_actions[nested_action]) }}
+{% endfor %}
+{% endif %}
+
+{%- endmacro %}
+
+{{ cli_name }}
+{{ '=' * ( cli_name|length|int ) }}
+
+{{ '-' * ( short_desc|default('')|string|length|int ) }}
+{{short_desc|default('')}}
+{{ '-' * ( short_desc|default('')|string|length|int ) }}
+
+:Version:        Ansible {{ version }}
+:Manual section: 1
+:Manual group:   System administration commands
+
+
+
+SYNOPSIS
+--------
+{{ usage|replace('%prog', cli_name) }}
+
+
+DESCRIPTION
+-----------
+{{ long_desc|default('', True)|wordwrap }}
+
+{% if options %}
+COMMON OPTIONS
+--------------
+{% for option in options|sort(attribute='options') %}
+{% for switch in option['options'] %}**{{switch}}**{% if option['arg'] %} '{{option['arg']}}'{% endif %}{% if not loop.last %}, {% endif %}{% endfor %}
+
+  {{ option['desc'] }}
+{% endfor %}
+{% endif %}
+
+{% if arguments %}
+ARGUMENTS
+---------
+
+{% for arg in arguments %}
+{{ arg }}
+
+{{ (arguments[arg]|default(' '))|wordwrap }}
+
+{% endfor %}
+{% endif %}
+
+{% if actions %}
+ACTIONS
+-------
+{% for action in actions %}
+{{ render_action('', action, actions[action]) }}
+{% endfor %}
+{% endif %}
+
+
+{% if inventory %}
+INVENTORY
+---------
+
+Ansible stores the hosts it can potentially operate on in an inventory.
+This can be an YAML file, ini-like file, a script, directory, list, etc.
+For additional options, see the documentation on https://docs.ansible.com/.
+
+{% endif %}
+ENVIRONMENT
+-----------
+
+The following environment variables may be specified.
+
+{% if inventory %}
+ANSIBLE_INVENTORY  -- Override the default ansible inventory sources
+
+{% endif %}
+{% if library %}
+ANSIBLE_LIBRARY -- Override the default ansible module library path
+
+{% endif %}
+ANSIBLE_CONFIG -- Specify override location for the ansible config file
+
+Many more are available for most options in ansible.cfg
+
+For a full list check https://docs.ansible.com/. or use the `ansible-config` command.
+
+FILES
+-----
+
+{% if inventory %}
+/etc/ansible/hosts -- Default inventory file
+
+{% endif %}
+/etc/ansible/ansible.cfg -- Config file, used if present
+
+~/.ansible.cfg -- User config file, overrides the default config if present
+
+./ansible.cfg -- Local config file (in current working directory) assumed to be 'project specific' and overrides the rest if present.
+
+As mentioned above, the ANSIBLE_CONFIG environment variable will override all others.
+
+AUTHOR
+------
+
+Ansible was originally written by Michael DeHaan.
+
+
+COPYRIGHT
+---------
+
+Copyright © 2018 Red Hat, Inc | Ansible.
+Ansible is released under the terms of the GPLv3 license.
+
+
+SEE ALSO
+--------
+
+{% for other in cli_bin_name_list|sort %}{% if other != cli_name %}**{{ other }}** (1){% if not loop.last %}, {% endif %}{% endif %}{% endfor %}
+
+Extensive documentation is available in the documentation site:
+<https://docs.ansible.com>.
+IRC and mailing list info can be found in file CONTRIBUTING.md,
+available in: <https://github.com/ansible/ansible>
diff --git a/packaging/cli-doc/rst.j2 b/packaging/cli-doc/rst.j2
new file mode 100644
index 0000000000..4a25653ab1
--- /dev/null
+++ b/packaging/cli-doc/rst.j2
@@ -0,0 +1,152 @@
+{%- set heading = ['-', '+', '#', '*', '^', '"', "'"] -%}
+{% macro render_action(parent, action, action_docs) %}
+
+.. program:: {{cli_name}} {{parent + action}}
+.. _{{cli_name|replace('-','_')}}_{{parent|replace(' ','_')}}{{action}}:
+
+{{ parent + action }}
+{{ heading[parent.count(' ')] * (parent + action)|length }}
+
+{{ (action_docs['desc']|default(' ')) }}
+
+{% if action_docs['options'] %}
+
+
+{% for option in action_docs['options']|sort(attribute='options') %}
+.. option:: {% for switch in option['options'] if switch in action_docs['option_names'] %}{{switch}} {% if option['arg'] %} <{{option['arg']}}>{% endif %}{% if not loop.last %}, {% endif %}{% endfor %}
+
+   {{ (option['desc']) }}
+{% endfor %}
+{% endif %}
+{%- set nested_actions = action_docs['actions'] -%}
+{% if nested_actions %}
+
+{% for nested_action in nested_actions %}
+{{ render_action(parent + action + ' ', nested_action, nested_actions[nested_action]) }}
+
+{% endfor %}
+{%- endif %}
+{%- endmacro -%}
+:source: {{ source }}
+
+{% set name = cli_name -%}
+{% set name_slug = cli_name -%}
+
+.. _{{name}}:
+
+{% set name_len = name|length + 0-%}
+{{ '=' * name_len }}
+{{name}}
+{{ '=' * name_len }}
+
+
+:strong:`{{short_desc|default('')}}`
+
+
+.. contents::
+   :local:
+   :depth: {{content_depth}}
+
+
+.. program:: {{cli_name}}
+
+Synopsis
+========
+
+.. code-block:: bash
+
+   {{ usage|replace('%prog', cli_name) }}
+
+
+Description
+===========
+
+
+{{ long_desc|default('', True)  }}
+
+{% if options %}
+Common Options
+==============
+
+
+{% for option in options|sort(attribute='options') if option.options %}
+
+.. option:: {% for switch in option['options'] %}{{switch}}{% if option['arg'] %} <{{option['arg']}}>{% endif %}{% if not loop.last %}, {% endif %}{% endfor %}
+
+   {{ option['desc'] }}
+{% endfor %}
+{% endif %}
+
+{% if arguments %}
+ARGUMENTS
+=========
+
+.. program:: {{cli_name}}
+
+{% for arg in arguments %}
+.. option:: {{ arg }}
+
+   {{ (arguments[arg]|default(' '))}}
+
+{% endfor %}
+{% endif %}
+
+{% if actions %}
+Actions
+=======
+
+{% for action in actions %}
+{{- render_action('', action, actions[action]) }}
+
+
+
+{% endfor %}
+.. program:: {{cli_name}}
+{% endif %}
+
+Environment
+===========
+
+The following environment variables may be specified.
+
+{% if inventory %}
+:envvar:`ANSIBLE_INVENTORY`  -- Override the default ansible inventory file
+
+{% endif %}
+{% if library %}
+:envvar:`ANSIBLE_LIBRARY` -- Override the default ansible module library path
+
+{% endif %}
+:envvar:`ANSIBLE_CONFIG` -- Override the default ansible config file
+
+Many more are available for most options in ansible.cfg
+
+
+Files
+=====
+
+{% if inventory %}
+:file:`/etc/ansible/hosts` -- Default inventory file
+
+{% endif %}
+:file:`/etc/ansible/ansible.cfg` -- Config file, used if present
+
+:file:`~/.ansible.cfg` -- User config file, overrides the default config if present
+
+Author
+======
+
+Ansible was originally written by Michael DeHaan.
+
+See the `AUTHORS` file for a complete list of contributors.
+
+
+License
+=======
+
+Ansible is released under the terms of the GPLv3+ License.
+
+See also
+========
+
+{% for other in cli_bin_name_list|sort %}{% if other != cli_name %}:manpage:`{{other}}(1)`{% if not loop.last %}, {% endif %}{% endif %}{% endfor %}