[PATCH v4 08/19] tools/docs: sphinx-build-wrapper: add a wrapper for sphinx-build

[PATCH v4 08/19] tools/docs: sphinx-build-wrapper: add a wrapper for sphinx-build

[Mauro Carvalho Chehab]

There are too much magic inside docs Makefile to properly run
sphinx-build. Create an ancillary script that contains all
kernel-related sphinx-build call logic currently at Makefile.

Such script is designed to work both as an standalone command
and as part of a Makefile. As such, it properly handles POSIX
jobserver used by GNU make.

On a side note, there was a line number increase due to the
conversion:

 Documentation/Makefile          |  131 +++----------
 tools/docs/sphinx-build-wrapper |  293 +++++++++++++++++++++++++++++++
 2 files changed, 323 insertions(+), 101 deletions(-)

This is because some things are more verbosed on Python and because
it requires reading env vars from Makefile. Besides it, this script
has some extra features that don't exist at the Makefile:

- It can be called directly from command line;
- It properly return PDF build errors.

When running the script alone, it will only take handle sphinx-build
targets. On other words, it won't runn make rustdoc after building
htmlfiles, nor it will run the extra check scripts.

Signed-off-by: Mauro Carvalho Chehab <mchehab+huawei@kernel.org>
---
 Documentation/Makefile          | 131 ++++----------
 tools/docs/sphinx-build-wrapper | 293 ++++++++++++++++++++++++++++++++
 2 files changed, 323 insertions(+), 101 deletions(-)
 create mode 100755 tools/docs/sphinx-build-wrapper

diff --git a/Documentation/Makefile b/Documentation/Makefile
index deb2029228ed..4736f02b6c9e 100644
--- a/Documentation/Makefile
+++ b/Documentation/Makefile
@@ -23,21 +23,22 @@ SPHINXOPTS    =
 SPHINXDIRS    = .
 DOCS_THEME    =
 DOCS_CSS      =
-_SPHINXDIRS   = $(sort $(patsubst $(srctree)/Documentation/%/index.rst,%,$(wildcard $(srctree)/Documentation/*/index.rst)))
 SPHINX_CONF   = conf.py
 PAPER         =
 BUILDDIR      = $(obj)/output
 PDFLATEX      = xelatex
 LATEXOPTS     = -interaction=batchmode -no-shell-escape
 
+PYTHONPYCACHEPREFIX ?= $(abspath $(BUILDDIR)/__pycache__)
+
+# Wrapper for sphinx-build
+
+BUILD_WRAPPER = $(srctree)/tools/docs/sphinx-build-wrapper
+
 # For denylisting "variable font" files
 # Can be overridden by setting as an env variable
 FONTS_CONF_DENY_VF ?= $(HOME)/deny-vf
 
-ifeq ($(findstring 1, $(KBUILD_VERBOSE)),)
-SPHINXOPTS    += "-q"
-endif
-
 # User-friendly check for sphinx-build
 HAVE_SPHINX := $(shell if which $(SPHINXBUILD) >/dev/null 2>&1; then echo 1; else echo 0; fi)
 
@@ -51,63 +52,31 @@ ifeq ($(HAVE_SPHINX),0)
 
 else # HAVE_SPHINX
 
-# User-friendly check for pdflatex and latexmk
-HAVE_PDFLATEX := $(shell if which $(PDFLATEX) >/dev/null 2>&1; then echo 1; else echo 0; fi)
-HAVE_LATEXMK := $(shell if which latexmk >/dev/null 2>&1; then echo 1; else echo 0; fi)
+# Common documentation targets
+infodocs texinfodocs latexdocs epubdocs xmldocs pdfdocs linkcheckdocs:
+	$(Q)@$(srctree)/tools/docs/sphinx-pre-install --version-check
+	+$(Q)$(PYTHON3) $(BUILD_WRAPPER) $@ \
+		--sphinxdirs="$(SPHINXDIRS)" --conf="$(SPHINX_CONF)" \
+		--builddir="$(BUILDDIR)" \
+		--theme=$(DOCS_THEME) --css=$(DOCS_CSS) --paper=$(PAPER)
 
-ifeq ($(HAVE_LATEXMK),1)
-	PDFLATEX := latexmk -$(PDFLATEX)
-endif #HAVE_LATEXMK
-
-# Internal variables.
-PAPEROPT_a4     = -D latex_elements.papersize=a4paper
-PAPEROPT_letter = -D latex_elements.papersize=letterpaper
-ALLSPHINXOPTS   = -D kerneldoc_srctree=$(srctree) -D kerneldoc_bin=$(KERNELDOC)
-ALLSPHINXOPTS   += $(PAPEROPT_$(PAPER)) $(SPHINXOPTS)
-ifneq ($(wildcard $(srctree)/.config),)
-ifeq ($(CONFIG_RUST),y)
-	# Let Sphinx know we will include rustdoc
-	ALLSPHINXOPTS   +=  -t rustdoc
-endif
+# Special handling for pdfdocs
+ifeq ($(shell which $(PDFLATEX) >/dev/null 2>&1; echo $$?),0)
+pdfdocs: DENY_VF = XDG_CONFIG_HOME=$(FONTS_CONF_DENY_VF)
+else
+pdfdocs:
+	$(warning The '$(PDFLATEX)' command was not found. Make sure you have it installed and in PATH to produce PDF output.)
+	@echo "  SKIP    Sphinx $@ target."
 endif
-# the i18n builder cannot share the environment and doctrees with the others
-I18NSPHINXOPTS  = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) .
-
-# commands; the 'cmd' from scripts/Kbuild.include is not *loopable*
-loop_cmd = $(echo-cmd) $(cmd_$(1)) || exit;
-
-# $2 sphinx builder e.g. "html"
-# $3 name of the build subfolder / e.g. "userspace-api/media", used as:
-#    * dest folder relative to $(BUILDDIR) and
-#    * cache folder relative to $(BUILDDIR)/.doctrees
-# $4 dest subfolder e.g. "man" for man pages at userspace-api/media/man
-# $5 reST source folder relative to $(src),
-#    e.g. "userspace-api/media" for the linux-tv book-set at ./Documentation/userspace-api/media
-
-PYTHONPYCACHEPREFIX ?= $(abspath $(BUILDDIR)/__pycache__)
-
-quiet_cmd_sphinx = SPHINX  $@ --> file://$(abspath $(BUILDDIR)/$3/$4)
-      cmd_sphinx = \
-	PYTHONPYCACHEPREFIX="$(PYTHONPYCACHEPREFIX)" \
-	BUILDDIR=$(abspath $(BUILDDIR)) SPHINX_CONF=$(abspath $(src)/$5/$(SPHINX_CONF)) \
-	$(PYTHON3) $(srctree)/scripts/jobserver-exec \
-	$(CONFIG_SHELL) $(srctree)/Documentation/sphinx/parallel-wrapper.sh \
-	$(SPHINXBUILD) \
-	-b $2 \
-	-c $(abspath $(src)) \
-	-d $(abspath $(BUILDDIR)/.doctrees/$3) \
-	-D version=$(KERNELVERSION) -D release=$(KERNELRELEASE) \
-	$(ALLSPHINXOPTS) \
-	$(abspath $(src)/$5) \
-	$(abspath $(BUILDDIR)/$3/$4) && \
-	if [ "x$(DOCS_CSS)" != "x" ]; then \
-		cp $(if $(patsubst /%,,$(DOCS_CSS)),$(abspath $(srctree)/$(DOCS_CSS)),$(DOCS_CSS)) $(BUILDDIR)/$3/_static/; \
-	fi
 
+# HTML main logic is identical to other targets. However, if rust is enabled,
+# an extra step at the end is required to generate rustdoc.
 htmldocs:
-	@$(srctree)/tools/docs/sphinx-pre-install --version-check
-	@+$(foreach var,$(SPHINXDIRS),$(call loop_cmd,sphinx,html,$(var),,$(var)))
-
+	$(Q)@$(srctree)/tools/docs/sphinx-pre-install --version-check
+	+$(Q)$(PYTHON3) $(BUILD_WRAPPER) $@ \
+		--sphinxdirs="$(SPHINXDIRS)" --conf="$(SPHINX_CONF)" \
+		--builddir="$(BUILDDIR)" \
+		--theme=$(DOCS_THEME) --css=$(DOCS_CSS) --paper=$(PAPER)
 # If Rust support is available and .config exists, add rustdoc generated contents.
 # If there are any, the errors from this make rustdoc will be displayed but
 # won't stop the execution of htmldocs
@@ -118,49 +87,6 @@ ifeq ($(CONFIG_RUST),y)
 endif
 endif
 
-texinfodocs:
-	@$(srctree)/tools/docs/sphinx-pre-install --version-check
-	@+$(foreach var,$(SPHINXDIRS),$(call loop_cmd,sphinx,texinfo,$(var),texinfo,$(var)))
-
-# Note: the 'info' Make target is generated by sphinx itself when
-# running the texinfodocs target define above.
-infodocs: texinfodocs
-	$(MAKE) -C $(BUILDDIR)/texinfo info
-
-linkcheckdocs:
-	@$(foreach var,$(SPHINXDIRS),$(call loop_cmd,sphinx,linkcheck,$(var),,$(var)))
-
-latexdocs:
-	@$(srctree)/tools/docs/sphinx-pre-install --version-check
-	@+$(foreach var,$(SPHINXDIRS),$(call loop_cmd,sphinx,latex,$(var),latex,$(var)))
-
-ifeq ($(HAVE_PDFLATEX),0)
-
-pdfdocs:
-	$(warning The '$(PDFLATEX)' command was not found. Make sure you have it installed and in PATH to produce PDF output.)
-	@echo "  SKIP    Sphinx $@ target."
-
-else # HAVE_PDFLATEX
-
-pdfdocs: DENY_VF = XDG_CONFIG_HOME=$(FONTS_CONF_DENY_VF)
-pdfdocs: latexdocs
-	@$(srctree)/tools/docs/sphinx-pre-install --version-check
-	$(foreach var,$(SPHINXDIRS), \
-	   $(MAKE) PDFLATEX="$(PDFLATEX)" LATEXOPTS="$(LATEXOPTS)" $(DENY_VF) -C $(BUILDDIR)/$(var)/latex || sh $(srctree)/scripts/check-variable-fonts.sh || exit; \
-	   mkdir -p $(BUILDDIR)/$(var)/pdf; \
-	   mv $(subst .tex,.pdf,$(wildcard $(BUILDDIR)/$(var)/latex/*.tex)) $(BUILDDIR)/$(var)/pdf/; \
-	)
-
-endif # HAVE_PDFLATEX
-
-epubdocs:
-	@$(srctree)/tools/docs/sphinx-pre-install --version-check
-	@+$(foreach var,$(SPHINXDIRS),$(call loop_cmd,sphinx,epub,$(var),epub,$(var)))
-
-xmldocs:
-	@$(srctree)/tools/docs/sphinx-pre-install --version-check
-	@+$(foreach var,$(SPHINXDIRS),$(call loop_cmd,sphinx,xml,$(var),xml,$(var)))
-
 endif # HAVE_SPHINX
 
 # The following targets are independent of HAVE_SPHINX, and the rules should
@@ -172,6 +98,9 @@ refcheckdocs:
 cleandocs:
 	$(Q)rm -rf $(BUILDDIR)
 
+# Used only on help
+_SPHINXDIRS   = $(sort $(patsubst $(srctree)/Documentation/%/index.rst,%,$(wildcard $(srctree)/Documentation/*/index.rst)))
+
 dochelp:
 	@echo  ' Linux kernel internal documentation in different formats from ReST:'
 	@echo  '  htmldocs        - HTML'
diff --git a/tools/docs/sphinx-build-wrapper b/tools/docs/sphinx-build-wrapper
new file mode 100755
index 000000000000..3256418d8dc5
--- /dev/null
+++ b/tools/docs/sphinx-build-wrapper
@@ -0,0 +1,293 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: GPL-2.0
+import argparse
+import os
+import shlex
+import shutil
+import subprocess
+import sys
+from lib.python_version import PythonVersion
+
+LIB_DIR = "../../scripts/lib"
+SRC_DIR = os.path.dirname(os.path.realpath(__file__))
+sys.path.insert(0, os.path.join(SRC_DIR, LIB_DIR))
+
+from jobserver import JobserverExec
+
+MIN_PYTHON_VERSION = PythonVersion("3.7").version
+PAPER = ["", "a4", "letter"]
+TARGETS = {
+    "cleandocs":     { "builder": "clean" },
+    "linkcheckdocs": { "builder": "linkcheck" },
+    "htmldocs":      { "builder": "html" },
+    "epubdocs":      { "builder": "epub",    "out_dir": "epub" },
+    "texinfodocs":   { "builder": "texinfo", "out_dir": "texinfo" },
+    "infodocs":      { "builder": "texinfo", "out_dir": "texinfo" },
+    "latexdocs":     { "builder": "latex",   "out_dir": "latex" },
+    "pdfdocs":       { "builder": "latex",   "out_dir": "latex" },
+    "xmldocs":       { "builder": "xml",     "out_dir": "xml" },
+}
+
+class SphinxBuilder:
+    def is_rust_enabled(self):
+        config_path = os.path.join(self.srctree, ".config")
+        if os.path.isfile(config_path):
+            with open(config_path, "r", encoding="utf-8") as f:
+                return "CONFIG_RUST=y" in f.read()
+        return False
+
+    def get_path(self, path, use_cwd=False, abs_path=False):
+        path = os.path.expanduser(path)
+        if not path.startswith("/"):
+            if use_cwd:
+                base = os.getcwd()
+            else:
+                base = self.srctree
+            path = os.path.join(base, path)
+        if abs_path:
+            return os.path.abspath(path)
+        return path
+
+    def __init__(self, builddir, verbose=False, n_jobs=None):
+        self.verbose = None
+        self.kernelversion = os.environ.get("KERNELVERSION", "unknown")
+        self.kernelrelease = os.environ.get("KERNELRELEASE", "unknown")
+        self.pdflatex = os.environ.get("PDFLATEX", "xelatex")
+        self.latexopts = os.environ.get("LATEXOPTS", "-interaction=batchmode -no-shell-escape")
+        if not verbose:
+            verbose = bool(os.environ.get("KBUILD_VERBOSE", "") != "")
+        if verbose is not None:
+            self.verbose = verbose
+        parser = argparse.ArgumentParser()
+        parser.add_argument('-j', '--jobs', type=int)
+        parser.add_argument('-q', '--quiet', type=int)
+        sphinxopts = shlex.split(os.environ.get("SPHINXOPTS", ""))
+        sphinx_args, self.sphinxopts = parser.parse_known_args(sphinxopts)
+        if sphinx_args.quiet is True:
+            self.verbose = False
+        if sphinx_args.jobs:
+            self.n_jobs = sphinx_args.jobs
+        self.n_jobs = n_jobs
+        self.srctree = os.environ.get("srctree")
+        if not self.srctree:
+            self.srctree = "."
+            os.environ["srctree"] = self.srctree
+        self.sphinxbuild = os.environ.get("SPHINXBUILD", "sphinx-build")
+        self.kerneldoc = self.get_path(os.environ.get("KERNELDOC",
+                                                      "scripts/kernel-doc.py"))
+        self.builddir = self.get_path(builddir, use_cwd=True, abs_path=True)
+
+        self.config_rust = self.is_rust_enabled()
+
+        self.pdflatex_cmd = shutil.which(self.pdflatex)
+        self.latexmk_cmd = shutil.which("latexmk")
+
+        self.env = os.environ.copy()
+
+    def run_sphinx(self, sphinx_build, build_args, *args, **pwargs):
+        with JobserverExec() as jobserver:
+            if jobserver.claim:
+                n_jobs = str(jobserver.claim)
+            else:
+                n_jobs = "auto" # Supported since Sphinx 1.7
+            cmd = []
+            cmd.append(sys.executable)
+            cmd.append(sphinx_build)
+            if self.n_jobs:
+                n_jobs = str(self.n_jobs)
+
+            if n_jobs:
+                cmd += [f"-j{n_jobs}"]
+
+            if not self.verbose:
+                cmd.append("-q")
+            cmd += self.sphinxopts
+            cmd += build_args
+            if self.verbose:
+                print(" ".join(cmd))
+            return subprocess.call(cmd, *args, **pwargs)
+
+    def handle_html(self, css, output_dir):
+        if not css:
+            return
+        css = os.path.expanduser(css)
+        if not css.startswith("/"):
+            css = os.path.join(self.srctree, css)
+        static_dir = os.path.join(output_dir, "_static")
+        os.makedirs(static_dir, exist_ok=True)
+        try:
+            shutil.copy2(css, static_dir)
+        except (OSError, IOError) as e:
+            print(f"Warning: Failed to copy CSS: {e}", file=sys.stderr)
+
+    def handle_pdf(self, output_dirs):
+        builds = {}
+        max_len = 0
+        for from_dir in output_dirs:
+            pdf_dir = os.path.join(from_dir, "../pdf")
+            os.makedirs(pdf_dir, exist_ok=True)
+            if self.latexmk_cmd:
+                latex_cmd = [self.latexmk_cmd, f"-{self.pdflatex}"]
+            else:
+                latex_cmd = [self.pdflatex]
+            latex_cmd.extend(shlex.split(self.latexopts))
+            tex_suffix = ".tex"
+            has_tex = False
+            build_failed = False
+            with os.scandir(from_dir) as it:
+                for entry in it:
+                    if not entry.name.endswith(tex_suffix):
+                        continue
+                    name = entry.name[:-len(tex_suffix)]
+                    has_tex = True
+                    try:
+                        subprocess.run(latex_cmd + [entry.path],
+                                       cwd=from_dir, check=True)
+                    except subprocess.CalledProcessError:
+                        pass
+                    pdf_name = name + ".pdf"
+                    pdf_from = os.path.join(from_dir, pdf_name)
+                    pdf_to = os.path.join(pdf_dir, pdf_name)
+                    if os.path.exists(pdf_from):
+                        os.rename(pdf_from, pdf_to)
+                        builds[name] = os.path.relpath(pdf_to, self.builddir)
+                    else:
+                        builds[name] = "FAILED"
+                        build_failed = True
+                    name = entry.name.removesuffix(".tex")
+                    max_len = max(max_len, len(name))
+
+            if not has_tex:
+                name = os.path.basename(from_dir)
+                max_len = max(max_len, len(name))
+                builds[name] = "FAILED (no .tex)"
+                build_failed = True
+        msg = "Summary"
+        msg += "\n" + "=" * len(msg)
+        print()
+        print(msg)
+        for pdf_name, pdf_file in builds.items():
+            print(f"{pdf_name:<{max_len}}: {pdf_file}")
+        print()
+        if build_failed:
+            sys.exit("PDF build failed: not all PDF files were created.")
+        else:
+            print("All PDF files were built.")
+
+    def handle_info(self, output_dirs):
+        for output_dir in output_dirs:
+            try:
+                subprocess.run(["make", "info"], cwd=output_dir, check=True)
+            except subprocess.CalledProcessError as e:
+                sys.exit(f"Error generating info docs: {e}")
+
+    def cleandocs(self, builder):
+        shutil.rmtree(self.builddir, ignore_errors=True)
+
+    def build(self, target, sphinxdirs=None, conf="conf.py",
+              theme=None, css=None, paper=None):
+        builder = TARGETS[target]["builder"]
+        out_dir = TARGETS[target].get("out_dir", "")
+        if target == "cleandocs":
+            self.cleandocs(builder)
+            return
+        if theme:
+                os.environ["DOCS_THEME"] = theme
+        sphinxbuild = shutil.which(self.sphinxbuild, path=self.env["PATH"])
+        if not sphinxbuild:
+            sys.exit(f"Error: {self.sphinxbuild} not found in PATH.\n")
+        if builder == "latex":
+            if not self.pdflatex_cmd and not self.latexmk_cmd:
+                sys.exit("Error: pdflatex or latexmk required for PDF generation")
+        docs_dir = os.path.abspath(os.path.join(self.srctree, "Documentation"))
+        kerneldoc = self.kerneldoc
+        if kerneldoc.startswith(self.srctree):
+            kerneldoc = os.path.relpath(kerneldoc, self.srctree)
+        args = [ "-b", builder, "-c", docs_dir ]
+        if builder == "latex":
+            if not paper:
+                paper = PAPER[1]
+            args.extend(["-D", f"latex_elements.papersize={paper}paper"])
+        if self.config_rust:
+            args.extend(["-t", "rustdoc"])
+        if conf:
+            self.env["SPHINX_CONF"] = self.get_path(conf, abs_path=True)
+        if not sphinxdirs:
+            sphinxdirs = os.environ.get("SPHINXDIRS", ".")
+        sphinxdirs_list = []
+        for sphinxdir in sphinxdirs:
+            if isinstance(sphinxdir, list):
+                sphinxdirs_list += sphinxdir
+            else:
+                for name in sphinxdir.split(" "):
+                    sphinxdirs_list.append(name)
+        output_dirs = []
+        for sphinxdir in sphinxdirs_list:
+            src_dir = os.path.join(docs_dir, sphinxdir)
+            doctree_dir = os.path.join(self.builddir, ".doctrees")
+            output_dir = os.path.join(self.builddir, sphinxdir, out_dir)
+            src_dir = os.path.normpath(src_dir)
+            doctree_dir = os.path.normpath(doctree_dir)
+            output_dir = os.path.normpath(output_dir)
+            os.makedirs(doctree_dir, exist_ok=True)
+            os.makedirs(output_dir, exist_ok=True)
+            output_dirs.append(output_dir)
+            build_args = args + [
+                "-d", doctree_dir,
+                "-D", f"kerneldoc_bin={kerneldoc}",
+                "-D", f"version={self.kernelversion}",
+                "-D", f"release={self.kernelrelease}",
+                "-D", f"kerneldoc_srctree={self.srctree}",
+                src_dir,
+                output_dir,
+            ]
+            try:
+                self.run_sphinx(sphinxbuild, build_args, env=self.env)
+            except (OSError, ValueError, subprocess.SubprocessError) as e:
+                sys.exit(f"Build failed: {repr(e)}")
+            if target in ["htmldocs", "epubdocs"]:
+                self.handle_html(css, output_dir)
+        if target == "pdfdocs":
+            self.handle_pdf(output_dirs)
+        elif target == "infodocs":
+            self.handle_info(output_dirs)
+
+def jobs_type(value):
+    if value is None:
+        return None
+    if value.lower() == 'auto':
+        return value.lower()
+    try:
+        if int(value) >= 1:
+            return value
+        raise argparse.ArgumentTypeError(f"Minimum jobs is 1, got {value}")
+    except ValueError:
+        raise argparse.ArgumentTypeError(f"Must be 'auto' or positive integer, got {value}")
+
+def main():
+    parser = argparse.ArgumentParser(description="Kernel documentation builder")
+    parser.add_argument("target", choices=list(TARGETS.keys()),
+                        help="Documentation target to build")
+    parser.add_argument("--sphinxdirs", nargs="+",
+                        help="Specific directories to build")
+    parser.add_argument("--conf", default="conf.py",
+                        help="Sphinx configuration file")
+    parser.add_argument("--builddir", default="output",
+                        help="Sphinx configuration file")
+    parser.add_argument("--theme", help="Sphinx theme to use")
+    parser.add_argument("--css", help="Custom CSS file for HTML/EPUB")
+    parser.add_argument("--paper", choices=PAPER, default=PAPER[0],
+                        help="Paper size for LaTeX/PDF output")
+    parser.add_argument("-v", "--verbose", action='store_true',
+                        help="place build in verbose mode")
+    parser.add_argument('-j', '--jobs', type=jobs_type,
+                        help="Sets number of jobs to use with sphinx-build")
+    args = parser.parse_args()
+    PythonVersion.check_python(MIN_PYTHON_VERSION)
+    builder = SphinxBuilder(builddir=args.builddir,
+                            verbose=args.verbose, n_jobs=args.jobs)
+    builder.build(args.target, sphinxdirs=args.sphinxdirs, conf=args.conf,
+                  theme=args.theme, css=args.css, paper=args.paper)
+
+if __name__ == "__main__":
+    main()
-- 
2.51.0

Re: [PATCH v4 08/19] tools/docs: sphinx-build-wrapper: add a wrapper for sphinx-build

[Jonathan Corbet]

Finally beginning to look at this.  I'm working from the pulled version,
rather than the commentless patch (please don't do that again :).  A nit
from SphinxBuilder::__init__():

> #
>         # As we handle number of jobs and quiet in separate, we need to pick
>         # both the same way as sphinx-build would pick, optionally accepts
>         # whitespaces or not. So let's use argparse to handle argument expansion
>         #
>         parser = argparse.ArgumentParser()
>         parser.add_argument('-j', '--jobs', type=int)
>         parser.add_argument('-q', '--quiet', type=int)
> 
>         #
>         # Other sphinx-build arguments go as-is, so place them
>         # at self.sphinxopts, using shell parser
>         #
>         sphinxopts = shlex.split(os.environ.get("SPHINXOPTS", "))
> 
>         #
>         # Build a list of sphinx args
>         #
>         sphinx_args, self.sphinxopts = parser.parse_known_args(sphinxopts)
>         if sphinx_args.quiet is True:
>             self.verbose = False
> 
>         if sphinx_args.jobs:
>             self.n_jobs = sphinx_args.jobs
> 
>         #
>         # If the command line argument "-j" is used override SPHINXOPTS
>         #
> 
>         self.n_jobs = n_jobs

First of all, I do wish you would isolate this sort of concern into its
own function.  But, beyond that, you go to all that effort to parse the
--jobs flag, but that last line just throws it all away.  What was the
real purpose here?

Thanks,

jon

Re: [PATCH v4 08/19] tools/docs: sphinx-build-wrapper: add a wrapper for sphinx-build

[Mauro Carvalho Chehab]

On Tue, Sep 09, 2025 at 08:53:50AM -0600, Jonathan Corbet wrote:
> Finally beginning to look at this.  I'm working from the pulled version,
> rather than the commentless patch (please don't do that again :).

Heh, when I had to rebase it, I noticed it was a bad idea to split ;-)

I'll merge the commentless patch at the next respin.

>  A nit
> from SphinxBuilder::__init__():
> 
> > #
> >         # As we handle number of jobs and quiet in separate, we need to pick
> >         # both the same way as sphinx-build would pick, optionally accepts
> >         # whitespaces or not. So let's use argparse to handle argument expansion
> >         #
> >         parser = argparse.ArgumentParser()
> >         parser.add_argument('-j', '--jobs', type=int)
> >         parser.add_argument('-q', '--quiet', type=int)
> > 
> >         #
> >         # Other sphinx-build arguments go as-is, so place them
> >         # at self.sphinxopts, using shell parser
> >         #
> >         sphinxopts = shlex.split(os.environ.get("SPHINXOPTS", "))
> > 
> >         #
> >         # Build a list of sphinx args
> >         #
> >         sphinx_args, self.sphinxopts = parser.parse_known_args(sphinxopts)
> >         if sphinx_args.quiet is True:
> >             self.verbose = False
> > 
> >         if sphinx_args.jobs:
> >             self.n_jobs = sphinx_args.jobs
> > 
> >         #
> >         # If the command line argument "-j" is used override SPHINXOPTS
> >         #
> > 
> >         self.n_jobs = n_jobs
> 
> First of all, I do wish you would isolate this sort of concern into its
> own function.

Ok.

>  But, beyond that, you go to all that effort to parse the
> --jobs flag, but that last line just throws it all away.  What was the
> real purpose here?

heh, it sounds to be something that got lost during a rebase.
This should be, instead:

    if n_jobs:
        self.n_jobs = n_jobs   # this is parser.parse_args().n_jobs from main()

-

Basically, what happens is that the number of jobs can be on
different places:

1) if called via Makefile, no job arguments are passed at
   command line, but SPHINXOPTS may contain "-j"  on it.

   The code shall use jobserver to get it by default, with:

        # Clain all remaining jobs from make jobserver pool
        with JobserverExec() as jobserver:
            if jobserver.claim:
                n_jobs = str(jobserver.claim)
            else:
                n_jobs = "auto" 

            # some logic to call sphinx-build with a parallel flag

        # After with, claim is returned back to the
        # jobserver, to allow other jobs to be executed
        # in parallel, if any.

  this basically claims all remaining make jobs from GNU jobserver.
  So, if the  build started with "-j8" and make was called with
  other args, the number of available slots could be, for
  instance "4".

  The above logic will have jobserver.claim = 4, and run:

    sphinx-build -j4 <other args>

  This is the normal behavior when one does, for instance:

    make -j8 drivers/media htmldocs

2) if called with SPHINXOPTS="-j8", it shall ignore jobserver
   and call sphinx-build with -j8;

both cases (1) and (2) are handler inside a function

-

Now, when sphinx-build-wrapper is called from command line,
there's no GNU jobserver. So:

3) by default, it uses "-jauto". This can be problematic on
   machines with a large number of CPUs but without too much
   free memory (with Sphinx 7.x, one needs a really huge amount
   of RAM to run sphinx with -j - like 128GB or more with -j24)

4) if "-j" parameter is specified, pass it as-is to sphinx-build;

    tools/docs/sphinx-build-wrapper -j16 htmldocs

   this calls sphinx-build with -j16.

5) one might still use:

    SPHINXOPTS=-j8 tools/docs/sphinx-build-wrapper htmldocs

   or, even weirder:

    SPHINXOPTS=-j8 tools/docs/sphinx-build-wrapper -j16 htmldocs

The above logic you reviewed is handling (4) and (5). There:

   - n_jobs comes from command line;

   - this comes from SPHINXOPTS var:
	sphinxopts = shlex.split(os.environ.get("SPHINXOPTS", ""))

if both SPHINXOPTS and -j are specified like:

    SPHINXOPTS=-j8 tools/docs/sphinx-build-wrapper -j16 htmldocs                                                                                                                                                                 
IMO it shall pick the latest one (-j16). 

Yet, perhaps I should have written the code on a different way,
e.g., like:

    if n_jobs:
        # Command line argument takes precedence
        self.n_jobs = n_jobs
    elif sphinx_args.jobs:
        # Otherwise, use what it was specified at SPHINXOPTS if
        # any
        self.n_jobs = sphinx_args.jobs

I'll change it at the next spin and re-test it for all 5 scenarios.

Regards,
Mauro

Re: [PATCH v4 08/19] tools/docs: sphinx-build-wrapper: add a wrapper for sphinx-build

[Jonathan Corbet]

Mauro Carvalho Chehab <mchehab+huawei@kernel.org> writes:

> Basically, what happens is that the number of jobs can be on
> different places:

There is a lot of complexity there, and spread out between __init__(),
run_sphinx(), and handle_pdf().  Is there any way to create a single
figure_out_how_many_damn_jobs() and coalesce that logic there?  That
would help make that part of the system a bit more comprehensible.

That said, I've been unable to make this change break in my testing.  I
guess I'm not seeing a lot of impediments to applying the next version
at this point.

Thanks,

jon

Re: [PATCH v4 08/19] tools/docs: sphinx-build-wrapper: add a wrapper for sphinx-build

[Mauro Carvalho Chehab]

Em Tue, 09 Sep 2025 12:56:17 -0600
Jonathan Corbet <corbet@lwn.net> escreveu:

> Mauro Carvalho Chehab <mchehab+huawei@kernel.org> writes:
> 
> > Basically, what happens is that the number of jobs can be on
> > different places:  
> 
> There is a lot of complexity there, and spread out between __init__(),
> run_sphinx(), and handle_pdf().  Is there any way to create a single
> figure_out_how_many_damn_jobs() and coalesce that logic there?  That
> would help make that part of the system a bit more comprehensible.

I'll try to better organize it, but run_sphinx() does something
different than handle_pdf():

- run_sphinx: claims all tokens;
- handle_pdf: use future.concurrent and handle parallelism inside it.

Perhaps I can move the future.concurrent parallelism to jobserver library
to simplify the code a little bit while offering an interface somewhat similar
to run_sphinx logic. Let's see if I can find a way to do it while keeping
the code generic (*).

Will take a look on it probably on Thursday of Friday.

(*) I did one similar attempt at devel time adding a subprocess call
    wrapper there, but didn't like much the solution, but this was
    before the need to use futures.concurrent.

> That said, I've been unable to make this change break in my testing.  I
> guess I'm not seeing a lot of impediments to applying the next version
> at this point.

Great! I'll probably be respinning the next (hopefully final) version
by the end of this week, if I don't get sidetracked with other things.

Thanks,
Mauro

Re: [PATCH v4 08/19] tools/docs: sphinx-build-wrapper: add a wrapper for sphinx-build

[Jonathan Corbet]

Another nit:

>         # sphinxdirs can be a list or a whitespace-separated string
>         #
>         sphinxdirs_list = []
>         for sphinxdir in sphinxdirs:
>             if isinstance(sphinxdir, list):
>                 sphinxdirs_list += sphinxdir
>             else:
>                 for name in sphinxdir.split(" "):
>                     sphinxdirs_list.append(name)

That inner loop just seems like a complicated way of saying:

	sphinxdirs_list += sphinxdir.split()

?

Thanks,

jon

Re: [PATCH v4 08/19] tools/docs: sphinx-build-wrapper: add a wrapper for sphinx-build

[Mauro Carvalho Chehab]

On Tue, Sep 09, 2025 at 09:21:35AM -0600, Jonathan Corbet wrote:
> Another nit:
> 
> >         # sphinxdirs can be a list or a whitespace-separated string
> >         #
> >         sphinxdirs_list = []
> >         for sphinxdir in sphinxdirs:
> >             if isinstance(sphinxdir, list):
> >                 sphinxdirs_list += sphinxdir
> >             else:
> >                 for name in sphinxdir.split(" "):
> >                     sphinxdirs_list.append(name)
> 
> That inner loop just seems like a complicated way of saying:
> 
> 	sphinxdirs_list += sphinxdir.split()

Yeah, it sounds so ;-)

At the development code version, I had some prints there to be sure
all cases were picked, so I ended coding it as a loop. I forgot to
return it to the much nicer "+=" syntax after finishing debugging it.

-- 
Thanks,
Mauro

Re: [PATCH v4 08/19] tools/docs: sphinx-build-wrapper: add a wrapper for sphinx-build

[Jani Nikula]

On Thu, 04 Sep 2025, Mauro Carvalho Chehab <mchehab+huawei@kernel.org> wrote:
> There are too much magic inside docs Makefile to properly run
> sphinx-build. Create an ancillary script that contains all
> kernel-related sphinx-build call logic currently at Makefile.
>
> Such script is designed to work both as an standalone command
> and as part of a Makefile. As such, it properly handles POSIX
> jobserver used by GNU make.
>
> On a side note, there was a line number increase due to the
> conversion:
>
>  Documentation/Makefile          |  131 +++----------
>  tools/docs/sphinx-build-wrapper |  293 +++++++++++++++++++++++++++++++
>  2 files changed, 323 insertions(+), 101 deletions(-)
>
> This is because some things are more verbosed on Python and because
> it requires reading env vars from Makefile. Besides it, this script
> has some extra features that don't exist at the Makefile:
>
> - It can be called directly from command line;
> - It properly return PDF build errors.
>
> When running the script alone, it will only take handle sphinx-build
> targets. On other words, it won't runn make rustdoc after building
> htmlfiles, nor it will run the extra check scripts.

I've always strongly believed we should aim to make it possible to build
the documentation by running sphinx-build directly on the
command-line. Not that it would be the common way to run it, but to not
accumulate things in the Makefile that need to happen before or
after. To promote handling the documentation build in Sphinx. To be able
to debug issues and try new Sphinx versions without all the hacks.

This patch moves a bunch of that logic into a Python wrapper, and I feel
like it complicates matters. You can no longer rely on 'make V=1' to get
the build commands, for instance.

Newer Sphinx versions have the -M option for "make mode". The Makefiles
produced by sphinx-quickstart only have one build target:

# Catch-all target: route all unknown targets to Sphinx using the new
# "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
%: Makefile
        @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)

That's all.

The proposed wrapper duplicates loads of code that's supposed to be
handled by sphinx-build directly. Including the target/builder names.

Seems to me the goal should be to figure out *generic* wrappers for
handling parallelism, not Sphinx aware/specific.


BR,
Jani.

>
> Signed-off-by: Mauro Carvalho Chehab <mchehab+huawei@kernel.org>
> ---
>  Documentation/Makefile          | 131 ++++----------
>  tools/docs/sphinx-build-wrapper | 293 ++++++++++++++++++++++++++++++++
>  2 files changed, 323 insertions(+), 101 deletions(-)
>  create mode 100755 tools/docs/sphinx-build-wrapper
>
> diff --git a/Documentation/Makefile b/Documentation/Makefile
> index deb2029228ed..4736f02b6c9e 100644
> --- a/Documentation/Makefile
> +++ b/Documentation/Makefile
> @@ -23,21 +23,22 @@ SPHINXOPTS    =
>  SPHINXDIRS    = .
>  DOCS_THEME    =
>  DOCS_CSS      =
> -_SPHINXDIRS   = $(sort $(patsubst $(srctree)/Documentation/%/index.rst,%,$(wildcard $(srctree)/Documentation/*/index.rst)))
>  SPHINX_CONF   = conf.py
>  PAPER         =
>  BUILDDIR      = $(obj)/output
>  PDFLATEX      = xelatex
>  LATEXOPTS     = -interaction=batchmode -no-shell-escape
>  
> +PYTHONPYCACHEPREFIX ?= $(abspath $(BUILDDIR)/__pycache__)
> +
> +# Wrapper for sphinx-build
> +
> +BUILD_WRAPPER = $(srctree)/tools/docs/sphinx-build-wrapper
> +
>  # For denylisting "variable font" files
>  # Can be overridden by setting as an env variable
>  FONTS_CONF_DENY_VF ?= $(HOME)/deny-vf
>  
> -ifeq ($(findstring 1, $(KBUILD_VERBOSE)),)
> -SPHINXOPTS    += "-q"
> -endif
> -
>  # User-friendly check for sphinx-build
>  HAVE_SPHINX := $(shell if which $(SPHINXBUILD) >/dev/null 2>&1; then echo 1; else echo 0; fi)
>  
> @@ -51,63 +52,31 @@ ifeq ($(HAVE_SPHINX),0)
>  
>  else # HAVE_SPHINX
>  
> -# User-friendly check for pdflatex and latexmk
> -HAVE_PDFLATEX := $(shell if which $(PDFLATEX) >/dev/null 2>&1; then echo 1; else echo 0; fi)
> -HAVE_LATEXMK := $(shell if which latexmk >/dev/null 2>&1; then echo 1; else echo 0; fi)
> +# Common documentation targets
> +infodocs texinfodocs latexdocs epubdocs xmldocs pdfdocs linkcheckdocs:
> +	$(Q)@$(srctree)/tools/docs/sphinx-pre-install --version-check
> +	+$(Q)$(PYTHON3) $(BUILD_WRAPPER) $@ \
> +		--sphinxdirs="$(SPHINXDIRS)" --conf="$(SPHINX_CONF)" \
> +		--builddir="$(BUILDDIR)" \
> +		--theme=$(DOCS_THEME) --css=$(DOCS_CSS) --paper=$(PAPER)
>  
> -ifeq ($(HAVE_LATEXMK),1)
> -	PDFLATEX := latexmk -$(PDFLATEX)
> -endif #HAVE_LATEXMK
> -
> -# Internal variables.
> -PAPEROPT_a4     = -D latex_elements.papersize=a4paper
> -PAPEROPT_letter = -D latex_elements.papersize=letterpaper
> -ALLSPHINXOPTS   = -D kerneldoc_srctree=$(srctree) -D kerneldoc_bin=$(KERNELDOC)
> -ALLSPHINXOPTS   += $(PAPEROPT_$(PAPER)) $(SPHINXOPTS)
> -ifneq ($(wildcard $(srctree)/.config),)
> -ifeq ($(CONFIG_RUST),y)
> -	# Let Sphinx know we will include rustdoc
> -	ALLSPHINXOPTS   +=  -t rustdoc
> -endif
> +# Special handling for pdfdocs
> +ifeq ($(shell which $(PDFLATEX) >/dev/null 2>&1; echo $$?),0)
> +pdfdocs: DENY_VF = XDG_CONFIG_HOME=$(FONTS_CONF_DENY_VF)
> +else
> +pdfdocs:
> +	$(warning The '$(PDFLATEX)' command was not found. Make sure you have it installed and in PATH to produce PDF output.)
> +	@echo "  SKIP    Sphinx $@ target."
>  endif
> -# the i18n builder cannot share the environment and doctrees with the others
> -I18NSPHINXOPTS  = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) .
> -
> -# commands; the 'cmd' from scripts/Kbuild.include is not *loopable*
> -loop_cmd = $(echo-cmd) $(cmd_$(1)) || exit;
> -
> -# $2 sphinx builder e.g. "html"
> -# $3 name of the build subfolder / e.g. "userspace-api/media", used as:
> -#    * dest folder relative to $(BUILDDIR) and
> -#    * cache folder relative to $(BUILDDIR)/.doctrees
> -# $4 dest subfolder e.g. "man" for man pages at userspace-api/media/man
> -# $5 reST source folder relative to $(src),
> -#    e.g. "userspace-api/media" for the linux-tv book-set at ./Documentation/userspace-api/media
> -
> -PYTHONPYCACHEPREFIX ?= $(abspath $(BUILDDIR)/__pycache__)
> -
> -quiet_cmd_sphinx = SPHINX  $@ --> file://$(abspath $(BUILDDIR)/$3/$4)
> -      cmd_sphinx = \
> -	PYTHONPYCACHEPREFIX="$(PYTHONPYCACHEPREFIX)" \
> -	BUILDDIR=$(abspath $(BUILDDIR)) SPHINX_CONF=$(abspath $(src)/$5/$(SPHINX_CONF)) \
> -	$(PYTHON3) $(srctree)/scripts/jobserver-exec \
> -	$(CONFIG_SHELL) $(srctree)/Documentation/sphinx/parallel-wrapper.sh \
> -	$(SPHINXBUILD) \
> -	-b $2 \
> -	-c $(abspath $(src)) \
> -	-d $(abspath $(BUILDDIR)/.doctrees/$3) \
> -	-D version=$(KERNELVERSION) -D release=$(KERNELRELEASE) \
> -	$(ALLSPHINXOPTS) \
> -	$(abspath $(src)/$5) \
> -	$(abspath $(BUILDDIR)/$3/$4) && \
> -	if [ "x$(DOCS_CSS)" != "x" ]; then \
> -		cp $(if $(patsubst /%,,$(DOCS_CSS)),$(abspath $(srctree)/$(DOCS_CSS)),$(DOCS_CSS)) $(BUILDDIR)/$3/_static/; \
> -	fi
>  
> +# HTML main logic is identical to other targets. However, if rust is enabled,
> +# an extra step at the end is required to generate rustdoc.
>  htmldocs:
> -	@$(srctree)/tools/docs/sphinx-pre-install --version-check
> -	@+$(foreach var,$(SPHINXDIRS),$(call loop_cmd,sphinx,html,$(var),,$(var)))
> -
> +	$(Q)@$(srctree)/tools/docs/sphinx-pre-install --version-check
> +	+$(Q)$(PYTHON3) $(BUILD_WRAPPER) $@ \
> +		--sphinxdirs="$(SPHINXDIRS)" --conf="$(SPHINX_CONF)" \
> +		--builddir="$(BUILDDIR)" \
> +		--theme=$(DOCS_THEME) --css=$(DOCS_CSS) --paper=$(PAPER)
>  # If Rust support is available and .config exists, add rustdoc generated contents.
>  # If there are any, the errors from this make rustdoc will be displayed but
>  # won't stop the execution of htmldocs
> @@ -118,49 +87,6 @@ ifeq ($(CONFIG_RUST),y)
>  endif
>  endif
>  
> -texinfodocs:
> -	@$(srctree)/tools/docs/sphinx-pre-install --version-check
> -	@+$(foreach var,$(SPHINXDIRS),$(call loop_cmd,sphinx,texinfo,$(var),texinfo,$(var)))
> -
> -# Note: the 'info' Make target is generated by sphinx itself when
> -# running the texinfodocs target define above.
> -infodocs: texinfodocs
> -	$(MAKE) -C $(BUILDDIR)/texinfo info
> -
> -linkcheckdocs:
> -	@$(foreach var,$(SPHINXDIRS),$(call loop_cmd,sphinx,linkcheck,$(var),,$(var)))
> -
> -latexdocs:
> -	@$(srctree)/tools/docs/sphinx-pre-install --version-check
> -	@+$(foreach var,$(SPHINXDIRS),$(call loop_cmd,sphinx,latex,$(var),latex,$(var)))
> -
> -ifeq ($(HAVE_PDFLATEX),0)
> -
> -pdfdocs:
> -	$(warning The '$(PDFLATEX)' command was not found. Make sure you have it installed and in PATH to produce PDF output.)
> -	@echo "  SKIP    Sphinx $@ target."
> -
> -else # HAVE_PDFLATEX
> -
> -pdfdocs: DENY_VF = XDG_CONFIG_HOME=$(FONTS_CONF_DENY_VF)
> -pdfdocs: latexdocs
> -	@$(srctree)/tools/docs/sphinx-pre-install --version-check
> -	$(foreach var,$(SPHINXDIRS), \
> -	   $(MAKE) PDFLATEX="$(PDFLATEX)" LATEXOPTS="$(LATEXOPTS)" $(DENY_VF) -C $(BUILDDIR)/$(var)/latex || sh $(srctree)/scripts/check-variable-fonts.sh || exit; \
> -	   mkdir -p $(BUILDDIR)/$(var)/pdf; \
> -	   mv $(subst .tex,.pdf,$(wildcard $(BUILDDIR)/$(var)/latex/*.tex)) $(BUILDDIR)/$(var)/pdf/; \
> -	)
> -
> -endif # HAVE_PDFLATEX
> -
> -epubdocs:
> -	@$(srctree)/tools/docs/sphinx-pre-install --version-check
> -	@+$(foreach var,$(SPHINXDIRS),$(call loop_cmd,sphinx,epub,$(var),epub,$(var)))
> -
> -xmldocs:
> -	@$(srctree)/tools/docs/sphinx-pre-install --version-check
> -	@+$(foreach var,$(SPHINXDIRS),$(call loop_cmd,sphinx,xml,$(var),xml,$(var)))
> -
>  endif # HAVE_SPHINX
>  
>  # The following targets are independent of HAVE_SPHINX, and the rules should
> @@ -172,6 +98,9 @@ refcheckdocs:
>  cleandocs:
>  	$(Q)rm -rf $(BUILDDIR)
>  
> +# Used only on help
> +_SPHINXDIRS   = $(sort $(patsubst $(srctree)/Documentation/%/index.rst,%,$(wildcard $(srctree)/Documentation/*/index.rst)))
> +
>  dochelp:
>  	@echo  ' Linux kernel internal documentation in different formats from ReST:'
>  	@echo  '  htmldocs        - HTML'
> diff --git a/tools/docs/sphinx-build-wrapper b/tools/docs/sphinx-build-wrapper
> new file mode 100755
> index 000000000000..3256418d8dc5
> --- /dev/null
> +++ b/tools/docs/sphinx-build-wrapper
> @@ -0,0 +1,293 @@
> +#!/usr/bin/env python3
> +# SPDX-License-Identifier: GPL-2.0
> +import argparse
> +import os
> +import shlex
> +import shutil
> +import subprocess
> +import sys
> +from lib.python_version import PythonVersion
> +
> +LIB_DIR = "../../scripts/lib"
> +SRC_DIR = os.path.dirname(os.path.realpath(__file__))
> +sys.path.insert(0, os.path.join(SRC_DIR, LIB_DIR))
> +
> +from jobserver import JobserverExec
> +
> +MIN_PYTHON_VERSION = PythonVersion("3.7").version
> +PAPER = ["", "a4", "letter"]
> +TARGETS = {
> +    "cleandocs":     { "builder": "clean" },
> +    "linkcheckdocs": { "builder": "linkcheck" },
> +    "htmldocs":      { "builder": "html" },
> +    "epubdocs":      { "builder": "epub",    "out_dir": "epub" },
> +    "texinfodocs":   { "builder": "texinfo", "out_dir": "texinfo" },
> +    "infodocs":      { "builder": "texinfo", "out_dir": "texinfo" },
> +    "latexdocs":     { "builder": "latex",   "out_dir": "latex" },
> +    "pdfdocs":       { "builder": "latex",   "out_dir": "latex" },
> +    "xmldocs":       { "builder": "xml",     "out_dir": "xml" },
> +}
> +
> +class SphinxBuilder:
> +    def is_rust_enabled(self):
> +        config_path = os.path.join(self.srctree, ".config")
> +        if os.path.isfile(config_path):
> +            with open(config_path, "r", encoding="utf-8") as f:
> +                return "CONFIG_RUST=y" in f.read()
> +        return False
> +
> +    def get_path(self, path, use_cwd=False, abs_path=False):
> +        path = os.path.expanduser(path)
> +        if not path.startswith("/"):
> +            if use_cwd:
> +                base = os.getcwd()
> +            else:
> +                base = self.srctree
> +            path = os.path.join(base, path)
> +        if abs_path:
> +            return os.path.abspath(path)
> +        return path
> +
> +    def __init__(self, builddir, verbose=False, n_jobs=None):
> +        self.verbose = None
> +        self.kernelversion = os.environ.get("KERNELVERSION", "unknown")
> +        self.kernelrelease = os.environ.get("KERNELRELEASE", "unknown")
> +        self.pdflatex = os.environ.get("PDFLATEX", "xelatex")
> +        self.latexopts = os.environ.get("LATEXOPTS", "-interaction=batchmode -no-shell-escape")
> +        if not verbose:
> +            verbose = bool(os.environ.get("KBUILD_VERBOSE", "") != "")
> +        if verbose is not None:
> +            self.verbose = verbose
> +        parser = argparse.ArgumentParser()
> +        parser.add_argument('-j', '--jobs', type=int)
> +        parser.add_argument('-q', '--quiet', type=int)
> +        sphinxopts = shlex.split(os.environ.get("SPHINXOPTS", ""))
> +        sphinx_args, self.sphinxopts = parser.parse_known_args(sphinxopts)
> +        if sphinx_args.quiet is True:
> +            self.verbose = False
> +        if sphinx_args.jobs:
> +            self.n_jobs = sphinx_args.jobs
> +        self.n_jobs = n_jobs
> +        self.srctree = os.environ.get("srctree")
> +        if not self.srctree:
> +            self.srctree = "."
> +            os.environ["srctree"] = self.srctree
> +        self.sphinxbuild = os.environ.get("SPHINXBUILD", "sphinx-build")
> +        self.kerneldoc = self.get_path(os.environ.get("KERNELDOC",
> +                                                      "scripts/kernel-doc.py"))
> +        self.builddir = self.get_path(builddir, use_cwd=True, abs_path=True)
> +
> +        self.config_rust = self.is_rust_enabled()
> +
> +        self.pdflatex_cmd = shutil.which(self.pdflatex)
> +        self.latexmk_cmd = shutil.which("latexmk")
> +
> +        self.env = os.environ.copy()
> +
> +    def run_sphinx(self, sphinx_build, build_args, *args, **pwargs):
> +        with JobserverExec() as jobserver:
> +            if jobserver.claim:
> +                n_jobs = str(jobserver.claim)
> +            else:
> +                n_jobs = "auto" # Supported since Sphinx 1.7
> +            cmd = []
> +            cmd.append(sys.executable)
> +            cmd.append(sphinx_build)
> +            if self.n_jobs:
> +                n_jobs = str(self.n_jobs)
> +
> +            if n_jobs:
> +                cmd += [f"-j{n_jobs}"]
> +
> +            if not self.verbose:
> +                cmd.append("-q")
> +            cmd += self.sphinxopts
> +            cmd += build_args
> +            if self.verbose:
> +                print(" ".join(cmd))
> +            return subprocess.call(cmd, *args, **pwargs)
> +
> +    def handle_html(self, css, output_dir):
> +        if not css:
> +            return
> +        css = os.path.expanduser(css)
> +        if not css.startswith("/"):
> +            css = os.path.join(self.srctree, css)
> +        static_dir = os.path.join(output_dir, "_static")
> +        os.makedirs(static_dir, exist_ok=True)
> +        try:
> +            shutil.copy2(css, static_dir)
> +        except (OSError, IOError) as e:
> +            print(f"Warning: Failed to copy CSS: {e}", file=sys.stderr)
> +
> +    def handle_pdf(self, output_dirs):
> +        builds = {}
> +        max_len = 0
> +        for from_dir in output_dirs:
> +            pdf_dir = os.path.join(from_dir, "../pdf")
> +            os.makedirs(pdf_dir, exist_ok=True)
> +            if self.latexmk_cmd:
> +                latex_cmd = [self.latexmk_cmd, f"-{self.pdflatex}"]
> +            else:
> +                latex_cmd = [self.pdflatex]
> +            latex_cmd.extend(shlex.split(self.latexopts))
> +            tex_suffix = ".tex"
> +            has_tex = False
> +            build_failed = False
> +            with os.scandir(from_dir) as it:
> +                for entry in it:
> +                    if not entry.name.endswith(tex_suffix):
> +                        continue
> +                    name = entry.name[:-len(tex_suffix)]
> +                    has_tex = True
> +                    try:
> +                        subprocess.run(latex_cmd + [entry.path],
> +                                       cwd=from_dir, check=True)
> +                    except subprocess.CalledProcessError:
> +                        pass
> +                    pdf_name = name + ".pdf"
> +                    pdf_from = os.path.join(from_dir, pdf_name)
> +                    pdf_to = os.path.join(pdf_dir, pdf_name)
> +                    if os.path.exists(pdf_from):
> +                        os.rename(pdf_from, pdf_to)
> +                        builds[name] = os.path.relpath(pdf_to, self.builddir)
> +                    else:
> +                        builds[name] = "FAILED"
> +                        build_failed = True
> +                    name = entry.name.removesuffix(".tex")
> +                    max_len = max(max_len, len(name))
> +
> +            if not has_tex:
> +                name = os.path.basename(from_dir)
> +                max_len = max(max_len, len(name))
> +                builds[name] = "FAILED (no .tex)"
> +                build_failed = True
> +        msg = "Summary"
> +        msg += "\n" + "=" * len(msg)
> +        print()
> +        print(msg)
> +        for pdf_name, pdf_file in builds.items():
> +            print(f"{pdf_name:<{max_len}}: {pdf_file}")
> +        print()
> +        if build_failed:
> +            sys.exit("PDF build failed: not all PDF files were created.")
> +        else:
> +            print("All PDF files were built.")
> +
> +    def handle_info(self, output_dirs):
> +        for output_dir in output_dirs:
> +            try:
> +                subprocess.run(["make", "info"], cwd=output_dir, check=True)
> +            except subprocess.CalledProcessError as e:
> +                sys.exit(f"Error generating info docs: {e}")
> +
> +    def cleandocs(self, builder):
> +        shutil.rmtree(self.builddir, ignore_errors=True)
> +
> +    def build(self, target, sphinxdirs=None, conf="conf.py",
> +              theme=None, css=None, paper=None):
> +        builder = TARGETS[target]["builder"]
> +        out_dir = TARGETS[target].get("out_dir", "")
> +        if target == "cleandocs":
> +            self.cleandocs(builder)
> +            return
> +        if theme:
> +                os.environ["DOCS_THEME"] = theme
> +        sphinxbuild = shutil.which(self.sphinxbuild, path=self.env["PATH"])
> +        if not sphinxbuild:
> +            sys.exit(f"Error: {self.sphinxbuild} not found in PATH.\n")
> +        if builder == "latex":
> +            if not self.pdflatex_cmd and not self.latexmk_cmd:
> +                sys.exit("Error: pdflatex or latexmk required for PDF generation")
> +        docs_dir = os.path.abspath(os.path.join(self.srctree, "Documentation"))
> +        kerneldoc = self.kerneldoc
> +        if kerneldoc.startswith(self.srctree):
> +            kerneldoc = os.path.relpath(kerneldoc, self.srctree)
> +        args = [ "-b", builder, "-c", docs_dir ]
> +        if builder == "latex":
> +            if not paper:
> +                paper = PAPER[1]
> +            args.extend(["-D", f"latex_elements.papersize={paper}paper"])
> +        if self.config_rust:
> +            args.extend(["-t", "rustdoc"])
> +        if conf:
> +            self.env["SPHINX_CONF"] = self.get_path(conf, abs_path=True)
> +        if not sphinxdirs:
> +            sphinxdirs = os.environ.get("SPHINXDIRS", ".")
> +        sphinxdirs_list = []
> +        for sphinxdir in sphinxdirs:
> +            if isinstance(sphinxdir, list):
> +                sphinxdirs_list += sphinxdir
> +            else:
> +                for name in sphinxdir.split(" "):
> +                    sphinxdirs_list.append(name)
> +        output_dirs = []
> +        for sphinxdir in sphinxdirs_list:
> +            src_dir = os.path.join(docs_dir, sphinxdir)
> +            doctree_dir = os.path.join(self.builddir, ".doctrees")
> +            output_dir = os.path.join(self.builddir, sphinxdir, out_dir)
> +            src_dir = os.path.normpath(src_dir)
> +            doctree_dir = os.path.normpath(doctree_dir)
> +            output_dir = os.path.normpath(output_dir)
> +            os.makedirs(doctree_dir, exist_ok=True)
> +            os.makedirs(output_dir, exist_ok=True)
> +            output_dirs.append(output_dir)
> +            build_args = args + [
> +                "-d", doctree_dir,
> +                "-D", f"kerneldoc_bin={kerneldoc}",
> +                "-D", f"version={self.kernelversion}",
> +                "-D", f"release={self.kernelrelease}",
> +                "-D", f"kerneldoc_srctree={self.srctree}",
> +                src_dir,
> +                output_dir,
> +            ]
> +            try:
> +                self.run_sphinx(sphinxbuild, build_args, env=self.env)
> +            except (OSError, ValueError, subprocess.SubprocessError) as e:
> +                sys.exit(f"Build failed: {repr(e)}")
> +            if target in ["htmldocs", "epubdocs"]:
> +                self.handle_html(css, output_dir)
> +        if target == "pdfdocs":
> +            self.handle_pdf(output_dirs)
> +        elif target == "infodocs":
> +            self.handle_info(output_dirs)
> +
> +def jobs_type(value):
> +    if value is None:
> +        return None
> +    if value.lower() == 'auto':
> +        return value.lower()
> +    try:
> +        if int(value) >= 1:
> +            return value
> +        raise argparse.ArgumentTypeError(f"Minimum jobs is 1, got {value}")
> +    except ValueError:
> +        raise argparse.ArgumentTypeError(f"Must be 'auto' or positive integer, got {value}")
> +
> +def main():
> +    parser = argparse.ArgumentParser(description="Kernel documentation builder")
> +    parser.add_argument("target", choices=list(TARGETS.keys()),
> +                        help="Documentation target to build")
> +    parser.add_argument("--sphinxdirs", nargs="+",
> +                        help="Specific directories to build")
> +    parser.add_argument("--conf", default="conf.py",
> +                        help="Sphinx configuration file")
> +    parser.add_argument("--builddir", default="output",
> +                        help="Sphinx configuration file")
> +    parser.add_argument("--theme", help="Sphinx theme to use")
> +    parser.add_argument("--css", help="Custom CSS file for HTML/EPUB")
> +    parser.add_argument("--paper", choices=PAPER, default=PAPER[0],
> +                        help="Paper size for LaTeX/PDF output")
> +    parser.add_argument("-v", "--verbose", action='store_true',
> +                        help="place build in verbose mode")
> +    parser.add_argument('-j', '--jobs', type=jobs_type,
> +                        help="Sets number of jobs to use with sphinx-build")
> +    args = parser.parse_args()
> +    PythonVersion.check_python(MIN_PYTHON_VERSION)
> +    builder = SphinxBuilder(builddir=args.builddir,
> +                            verbose=args.verbose, n_jobs=args.jobs)
> +    builder.build(args.target, sphinxdirs=args.sphinxdirs, conf=args.conf,
> +                  theme=args.theme, css=args.css, paper=args.paper)
> +
> +if __name__ == "__main__":
> +    main()

-- 
Jani Nikula, Intel

Re: [PATCH v4 08/19] tools/docs: sphinx-build-wrapper: add a wrapper for sphinx-build

[Mauro Carvalho Chehab]

Em Wed, 10 Sep 2025 13:46:17 +0300
Jani Nikula <jani.nikula@linux.intel.com> escreveu:

> On Thu, 04 Sep 2025, Mauro Carvalho Chehab <mchehab+huawei@kernel.org> wrote:
> > There are too much magic inside docs Makefile to properly run
> > sphinx-build. Create an ancillary script that contains all
> > kernel-related sphinx-build call logic currently at Makefile.
> >
> > Such script is designed to work both as an standalone command
> > and as part of a Makefile. As such, it properly handles POSIX
> > jobserver used by GNU make.
> >
> > On a side note, there was a line number increase due to the
> > conversion:
> >
> >  Documentation/Makefile          |  131 +++----------
> >  tools/docs/sphinx-build-wrapper |  293 +++++++++++++++++++++++++++++++
> >  2 files changed, 323 insertions(+), 101 deletions(-)
> >
> > This is because some things are more verbosed on Python and because
> > it requires reading env vars from Makefile. Besides it, this script
> > has some extra features that don't exist at the Makefile:
> >
> > - It can be called directly from command line;
> > - It properly return PDF build errors.
> >
> > When running the script alone, it will only take handle sphinx-build
> > targets. On other words, it won't runn make rustdoc after building
> > htmlfiles, nor it will run the extra check scripts.  
> 
> I've always strongly believed we should aim to make it possible to build
> the documentation by running sphinx-build directly on the
> command-line. Not that it would be the common way to run it, but to not
> accumulate things in the Makefile that need to happen before or
> after. To promote handling the documentation build in Sphinx. To be able
> to debug issues and try new Sphinx versions without all the hacks.

That would be the better, but, unfortunately, this is not possible, for 
several reasons:

1. SPHINXDIRS. It needs a lot of magic to work, both before running
   sphinx-build and after (inside conf.py);
2. Several extensions require kernel-specific environment variables to
   work. Calling sphinx-build directly breaks them;
3. Sphinx itself doesn't build several targets alone. Instead, they create
   a Makefile, and an extra step is needed to finish the build. That's 
   the case for pdf and texinfo, for instance;
4. Man pages generation. Sphinx support to generate it is very poor;
5. Rust integration adds more complexity to the table;

I'm not seeing sphinx-build supporting the above needs anytime soon,
and, even if we push our needs to Sphinx and it gets accepted there,
we'll still need to wait for quite a while until LTS distros merge
them.

> This patch moves a bunch of that logic into a Python wrapper, and I feel
> like it complicates matters. You can no longer rely on 'make V=1' to get
> the build commands, for instance.

Quite the opposite. if you try using "make V=1", it won't show the
command line used to call sphinx-build anymore.

This series restore it.

See, if you build with this series with V=1, you will see exactly
what commands are used on the build:

	$ make V=1 htmldocs
	...
	python3 ./tools/docs/sphinx-build-wrapper htmldocs \
	        --sphinxdirs="." --conf="conf.py" \
        	--builddir="Documentation/output" \
	        --theme= --css= --paper=
	python3 /new_devel/docs/sphinx_latest/bin/sphinx-build -j25 -b html -c /new_devel/docs/Documentation -d /new_devel/docs/Documentation/output/.doctrees -D kerneldoc_bin=scripts/kernel-doc.py -D version=6.17.0-rc1 -D release=6.17.0-rc1+ -D kerneldoc_srctree=. /new_devel/docs/Documentation /new_devel/docs/Documentation/output
	...



> Newer Sphinx versions have the -M option for "make mode". The Makefiles
> produced by sphinx-quickstart only have one build target:
> 
> # Catch-all target: route all unknown targets to Sphinx using the new
> # "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).

I didn't know about this, but from [1] it sounds it covers just two
targets: "latexpdf" and "info".

The most complex scenario is still not covered: SPHINXDIRS.

[1] https://www.sphinx-doc.org/en/master/man/sphinx-build.html

> %: Makefile
>         @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
> 
> That's all.

Try doing such change on your makefile. it will break:

	- SPHINXDIRS;
	- V=1;
	- rustdoc

and will still be dependent on variables that are passed via
env from Kernel makefile. So, stil you can't run from command
line. Also, if you call sphinx-build from command line:

	$ sphinx-build -j25 -b html Documentation Documentation/output
	...
	      File "<frozen os>", line 717, in __getitem__
	    KeyError: 'srctree'

It won't work, as several parameters that are required by conf.py and by
Sphinx extensions would be missing (the most important one is srctree, but
there are others in the line too).

> The proposed wrapper duplicates loads of code that's supposed to be
> handled by sphinx-build directly.

Once we get the wrapper, we can work to simplify it, but still I
can't see how to get rid of it.

> Including the target/builder names.

True, but this was a design decision taken lots of years ago: instead
of:
	make html

we're using:

	make htmldocs

This series doesn't change that: either makefile or the script need
to tho the namespace conversion.

> Seems to me the goal should be to figure out *generic* wrappers for
> handling parallelism, not Sphinx aware/specific.
> 
> 
> BR,
> Jani.
> 
> >
> > Signed-off-by: Mauro Carvalho Chehab <mchehab+huawei@kernel.org>
> > ---
> >  Documentation/Makefile          | 131 ++++----------
> >  tools/docs/sphinx-build-wrapper | 293 ++++++++++++++++++++++++++++++++
> >  2 files changed, 323 insertions(+), 101 deletions(-)
> >  create mode 100755 tools/docs/sphinx-build-wrapper
> >
> > diff --git a/Documentation/Makefile b/Documentation/Makefile
> > index deb2029228ed..4736f02b6c9e 100644
> > --- a/Documentation/Makefile
> > +++ b/Documentation/Makefile
> > @@ -23,21 +23,22 @@ SPHINXOPTS    =
> >  SPHINXDIRS    = .
> >  DOCS_THEME    =
> >  DOCS_CSS      =
> > -_SPHINXDIRS   = $(sort $(patsubst $(srctree)/Documentation/%/index.rst,%,$(wildcard $(srctree)/Documentation/*/index.rst)))
> >  SPHINX_CONF   = conf.py
> >  PAPER         =
> >  BUILDDIR      = $(obj)/output
> >  PDFLATEX      = xelatex
> >  LATEXOPTS     = -interaction=batchmode -no-shell-escape
> >  
> > +PYTHONPYCACHEPREFIX ?= $(abspath $(BUILDDIR)/__pycache__)
> > +
> > +# Wrapper for sphinx-build
> > +
> > +BUILD_WRAPPER = $(srctree)/tools/docs/sphinx-build-wrapper
> > +
> >  # For denylisting "variable font" files
> >  # Can be overridden by setting as an env variable
> >  FONTS_CONF_DENY_VF ?= $(HOME)/deny-vf
> >  
> > -ifeq ($(findstring 1, $(KBUILD_VERBOSE)),)
> > -SPHINXOPTS    += "-q"
> > -endif
> > -
> >  # User-friendly check for sphinx-build
> >  HAVE_SPHINX := $(shell if which $(SPHINXBUILD) >/dev/null 2>&1; then echo 1; else echo 0; fi)
> >  
> > @@ -51,63 +52,31 @@ ifeq ($(HAVE_SPHINX),0)
> >  
> >  else # HAVE_SPHINX
> >  
> > -# User-friendly check for pdflatex and latexmk
> > -HAVE_PDFLATEX := $(shell if which $(PDFLATEX) >/dev/null 2>&1; then echo 1; else echo 0; fi)
> > -HAVE_LATEXMK := $(shell if which latexmk >/dev/null 2>&1; then echo 1; else echo 0; fi)
> > +# Common documentation targets
> > +infodocs texinfodocs latexdocs epubdocs xmldocs pdfdocs linkcheckdocs:
> > +	$(Q)@$(srctree)/tools/docs/sphinx-pre-install --version-check
> > +	+$(Q)$(PYTHON3) $(BUILD_WRAPPER) $@ \
> > +		--sphinxdirs="$(SPHINXDIRS)" --conf="$(SPHINX_CONF)" \
> > +		--builddir="$(BUILDDIR)" \
> > +		--theme=$(DOCS_THEME) --css=$(DOCS_CSS) --paper=$(PAPER)
> >  
> > -ifeq ($(HAVE_LATEXMK),1)
> > -	PDFLATEX := latexmk -$(PDFLATEX)
> > -endif #HAVE_LATEXMK
> > -
> > -# Internal variables.
> > -PAPEROPT_a4     = -D latex_elements.papersize=a4paper
> > -PAPEROPT_letter = -D latex_elements.papersize=letterpaper
> > -ALLSPHINXOPTS   = -D kerneldoc_srctree=$(srctree) -D kerneldoc_bin=$(KERNELDOC)
> > -ALLSPHINXOPTS   += $(PAPEROPT_$(PAPER)) $(SPHINXOPTS)
> > -ifneq ($(wildcard $(srctree)/.config),)
> > -ifeq ($(CONFIG_RUST),y)
> > -	# Let Sphinx know we will include rustdoc
> > -	ALLSPHINXOPTS   +=  -t rustdoc
> > -endif
> > +# Special handling for pdfdocs
> > +ifeq ($(shell which $(PDFLATEX) >/dev/null 2>&1; echo $$?),0)
> > +pdfdocs: DENY_VF = XDG_CONFIG_HOME=$(FONTS_CONF_DENY_VF)
> > +else
> > +pdfdocs:
> > +	$(warning The '$(PDFLATEX)' command was not found. Make sure you have it installed and in PATH to produce PDF output.)
> > +	@echo "  SKIP    Sphinx $@ target."
> >  endif
> > -# the i18n builder cannot share the environment and doctrees with the others
> > -I18NSPHINXOPTS  = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) .
> > -
> > -# commands; the 'cmd' from scripts/Kbuild.include is not *loopable*
> > -loop_cmd = $(echo-cmd) $(cmd_$(1)) || exit;
> > -
> > -# $2 sphinx builder e.g. "html"
> > -# $3 name of the build subfolder / e.g. "userspace-api/media", used as:
> > -#    * dest folder relative to $(BUILDDIR) and
> > -#    * cache folder relative to $(BUILDDIR)/.doctrees
> > -# $4 dest subfolder e.g. "man" for man pages at userspace-api/media/man
> > -# $5 reST source folder relative to $(src),
> > -#    e.g. "userspace-api/media" for the linux-tv book-set at ./Documentation/userspace-api/media
> > -
> > -PYTHONPYCACHEPREFIX ?= $(abspath $(BUILDDIR)/__pycache__)
> > -
> > -quiet_cmd_sphinx = SPHINX  $@ --> file://$(abspath $(BUILDDIR)/$3/$4)
> > -      cmd_sphinx = \
> > -	PYTHONPYCACHEPREFIX="$(PYTHONPYCACHEPREFIX)" \
> > -	BUILDDIR=$(abspath $(BUILDDIR)) SPHINX_CONF=$(abspath $(src)/$5/$(SPHINX_CONF)) \
> > -	$(PYTHON3) $(srctree)/scripts/jobserver-exec \
> > -	$(CONFIG_SHELL) $(srctree)/Documentation/sphinx/parallel-wrapper.sh \
> > -	$(SPHINXBUILD) \
> > -	-b $2 \
> > -	-c $(abspath $(src)) \
> > -	-d $(abspath $(BUILDDIR)/.doctrees/$3) \
> > -	-D version=$(KERNELVERSION) -D release=$(KERNELRELEASE) \
> > -	$(ALLSPHINXOPTS) \
> > -	$(abspath $(src)/$5) \
> > -	$(abspath $(BUILDDIR)/$3/$4) && \
> > -	if [ "x$(DOCS_CSS)" != "x" ]; then \
> > -		cp $(if $(patsubst /%,,$(DOCS_CSS)),$(abspath $(srctree)/$(DOCS_CSS)),$(DOCS_CSS)) $(BUILDDIR)/$3/_static/; \
> > -	fi
> >  
> > +# HTML main logic is identical to other targets. However, if rust is enabled,
> > +# an extra step at the end is required to generate rustdoc.
> >  htmldocs:
> > -	@$(srctree)/tools/docs/sphinx-pre-install --version-check
> > -	@+$(foreach var,$(SPHINXDIRS),$(call loop_cmd,sphinx,html,$(var),,$(var)))
> > -
> > +	$(Q)@$(srctree)/tools/docs/sphinx-pre-install --version-check
> > +	+$(Q)$(PYTHON3) $(BUILD_WRAPPER) $@ \
> > +		--sphinxdirs="$(SPHINXDIRS)" --conf="$(SPHINX_CONF)" \
> > +		--builddir="$(BUILDDIR)" \
> > +		--theme=$(DOCS_THEME) --css=$(DOCS_CSS) --paper=$(PAPER)
> >  # If Rust support is available and .config exists, add rustdoc generated contents.
> >  # If there are any, the errors from this make rustdoc will be displayed but
> >  # won't stop the execution of htmldocs
> > @@ -118,49 +87,6 @@ ifeq ($(CONFIG_RUST),y)
> >  endif
> >  endif
> >  
> > -texinfodocs:
> > -	@$(srctree)/tools/docs/sphinx-pre-install --version-check
> > -	@+$(foreach var,$(SPHINXDIRS),$(call loop_cmd,sphinx,texinfo,$(var),texinfo,$(var)))
> > -
> > -# Note: the 'info' Make target is generated by sphinx itself when
> > -# running the texinfodocs target define above.
> > -infodocs: texinfodocs
> > -	$(MAKE) -C $(BUILDDIR)/texinfo info
> > -
> > -linkcheckdocs:
> > -	@$(foreach var,$(SPHINXDIRS),$(call loop_cmd,sphinx,linkcheck,$(var),,$(var)))
> > -
> > -latexdocs:
> > -	@$(srctree)/tools/docs/sphinx-pre-install --version-check
> > -	@+$(foreach var,$(SPHINXDIRS),$(call loop_cmd,sphinx,latex,$(var),latex,$(var)))
> > -
> > -ifeq ($(HAVE_PDFLATEX),0)
> > -
> > -pdfdocs:
> > -	$(warning The '$(PDFLATEX)' command was not found. Make sure you have it installed and in PATH to produce PDF output.)
> > -	@echo "  SKIP    Sphinx $@ target."
> > -
> > -else # HAVE_PDFLATEX
> > -
> > -pdfdocs: DENY_VF = XDG_CONFIG_HOME=$(FONTS_CONF_DENY_VF)
> > -pdfdocs: latexdocs
> > -	@$(srctree)/tools/docs/sphinx-pre-install --version-check
> > -	$(foreach var,$(SPHINXDIRS), \
> > -	   $(MAKE) PDFLATEX="$(PDFLATEX)" LATEXOPTS="$(LATEXOPTS)" $(DENY_VF) -C $(BUILDDIR)/$(var)/latex || sh $(srctree)/scripts/check-variable-fonts.sh || exit; \
> > -	   mkdir -p $(BUILDDIR)/$(var)/pdf; \
> > -	   mv $(subst .tex,.pdf,$(wildcard $(BUILDDIR)/$(var)/latex/*.tex)) $(BUILDDIR)/$(var)/pdf/; \
> > -	)
> > -
> > -endif # HAVE_PDFLATEX
> > -
> > -epubdocs:
> > -	@$(srctree)/tools/docs/sphinx-pre-install --version-check
> > -	@+$(foreach var,$(SPHINXDIRS),$(call loop_cmd,sphinx,epub,$(var),epub,$(var)))
> > -
> > -xmldocs:
> > -	@$(srctree)/tools/docs/sphinx-pre-install --version-check
> > -	@+$(foreach var,$(SPHINXDIRS),$(call loop_cmd,sphinx,xml,$(var),xml,$(var)))
> > -
> >  endif # HAVE_SPHINX
> >  
> >  # The following targets are independent of HAVE_SPHINX, and the rules should
> > @@ -172,6 +98,9 @@ refcheckdocs:
> >  cleandocs:
> >  	$(Q)rm -rf $(BUILDDIR)
> >  
> > +# Used only on help
> > +_SPHINXDIRS   = $(sort $(patsubst $(srctree)/Documentation/%/index.rst,%,$(wildcard $(srctree)/Documentation/*/index.rst)))
> > +
> >  dochelp:
> >  	@echo  ' Linux kernel internal documentation in different formats from ReST:'
> >  	@echo  '  htmldocs        - HTML'
> > diff --git a/tools/docs/sphinx-build-wrapper b/tools/docs/sphinx-build-wrapper
> > new file mode 100755
> > index 000000000000..3256418d8dc5
> > --- /dev/null
> > +++ b/tools/docs/sphinx-build-wrapper
> > @@ -0,0 +1,293 @@
> > +#!/usr/bin/env python3
> > +# SPDX-License-Identifier: GPL-2.0
> > +import argparse
> > +import os
> > +import shlex
> > +import shutil
> > +import subprocess
> > +import sys
> > +from lib.python_version import PythonVersion
> > +
> > +LIB_DIR = "../../scripts/lib"
> > +SRC_DIR = os.path.dirname(os.path.realpath(__file__))
> > +sys.path.insert(0, os.path.join(SRC_DIR, LIB_DIR))
> > +
> > +from jobserver import JobserverExec
> > +
> > +MIN_PYTHON_VERSION = PythonVersion("3.7").version
> > +PAPER = ["", "a4", "letter"]
> > +TARGETS = {
> > +    "cleandocs":     { "builder": "clean" },
> > +    "linkcheckdocs": { "builder": "linkcheck" },
> > +    "htmldocs":      { "builder": "html" },
> > +    "epubdocs":      { "builder": "epub",    "out_dir": "epub" },
> > +    "texinfodocs":   { "builder": "texinfo", "out_dir": "texinfo" },
> > +    "infodocs":      { "builder": "texinfo", "out_dir": "texinfo" },
> > +    "latexdocs":     { "builder": "latex",   "out_dir": "latex" },
> > +    "pdfdocs":       { "builder": "latex",   "out_dir": "latex" },
> > +    "xmldocs":       { "builder": "xml",     "out_dir": "xml" },
> > +}
> > +
> > +class SphinxBuilder:
> > +    def is_rust_enabled(self):
> > +        config_path = os.path.join(self.srctree, ".config")
> > +        if os.path.isfile(config_path):
> > +            with open(config_path, "r", encoding="utf-8") as f:
> > +                return "CONFIG_RUST=y" in f.read()
> > +        return False
> > +
> > +    def get_path(self, path, use_cwd=False, abs_path=False):
> > +        path = os.path.expanduser(path)
> > +        if not path.startswith("/"):
> > +            if use_cwd:
> > +                base = os.getcwd()
> > +            else:
> > +                base = self.srctree
> > +            path = os.path.join(base, path)
> > +        if abs_path:
> > +            return os.path.abspath(path)
> > +        return path
> > +
> > +    def __init__(self, builddir, verbose=False, n_jobs=None):
> > +        self.verbose = None
> > +        self.kernelversion = os.environ.get("KERNELVERSION", "unknown")
> > +        self.kernelrelease = os.environ.get("KERNELRELEASE", "unknown")
> > +        self.pdflatex = os.environ.get("PDFLATEX", "xelatex")
> > +        self.latexopts = os.environ.get("LATEXOPTS", "-interaction=batchmode -no-shell-escape")
> > +        if not verbose:
> > +            verbose = bool(os.environ.get("KBUILD_VERBOSE", "") != "")
> > +        if verbose is not None:
> > +            self.verbose = verbose
> > +        parser = argparse.ArgumentParser()
> > +        parser.add_argument('-j', '--jobs', type=int)
> > +        parser.add_argument('-q', '--quiet', type=int)
> > +        sphinxopts = shlex.split(os.environ.get("SPHINXOPTS", ""))
> > +        sphinx_args, self.sphinxopts = parser.parse_known_args(sphinxopts)
> > +        if sphinx_args.quiet is True:
> > +            self.verbose = False
> > +        if sphinx_args.jobs:
> > +            self.n_jobs = sphinx_args.jobs
> > +        self.n_jobs = n_jobs
> > +        self.srctree = os.environ.get("srctree")
> > +        if not self.srctree:
> > +            self.srctree = "."
> > +            os.environ["srctree"] = self.srctree
> > +        self.sphinxbuild = os.environ.get("SPHINXBUILD", "sphinx-build")
> > +        self.kerneldoc = self.get_path(os.environ.get("KERNELDOC",
> > +                                                      "scripts/kernel-doc.py"))
> > +        self.builddir = self.get_path(builddir, use_cwd=True, abs_path=True)
> > +
> > +        self.config_rust = self.is_rust_enabled()
> > +
> > +        self.pdflatex_cmd = shutil.which(self.pdflatex)
> > +        self.latexmk_cmd = shutil.which("latexmk")
> > +
> > +        self.env = os.environ.copy()
> > +
> > +    def run_sphinx(self, sphinx_build, build_args, *args, **pwargs):
> > +        with JobserverExec() as jobserver:
> > +            if jobserver.claim:
> > +                n_jobs = str(jobserver.claim)
> > +            else:
> > +                n_jobs = "auto" # Supported since Sphinx 1.7
> > +            cmd = []
> > +            cmd.append(sys.executable)
> > +            cmd.append(sphinx_build)
> > +            if self.n_jobs:
> > +                n_jobs = str(self.n_jobs)
> > +
> > +            if n_jobs:
> > +                cmd += [f"-j{n_jobs}"]
> > +
> > +            if not self.verbose:
> > +                cmd.append("-q")
> > +            cmd += self.sphinxopts
> > +            cmd += build_args
> > +            if self.verbose:
> > +                print(" ".join(cmd))
> > +            return subprocess.call(cmd, *args, **pwargs)
> > +
> > +    def handle_html(self, css, output_dir):
> > +        if not css:
> > +            return
> > +        css = os.path.expanduser(css)
> > +        if not css.startswith("/"):
> > +            css = os.path.join(self.srctree, css)
> > +        static_dir = os.path.join(output_dir, "_static")
> > +        os.makedirs(static_dir, exist_ok=True)
> > +        try:
> > +            shutil.copy2(css, static_dir)
> > +        except (OSError, IOError) as e:
> > +            print(f"Warning: Failed to copy CSS: {e}", file=sys.stderr)
> > +
> > +    def handle_pdf(self, output_dirs):
> > +        builds = {}
> > +        max_len = 0
> > +        for from_dir in output_dirs:
> > +            pdf_dir = os.path.join(from_dir, "../pdf")
> > +            os.makedirs(pdf_dir, exist_ok=True)
> > +            if self.latexmk_cmd:
> > +                latex_cmd = [self.latexmk_cmd, f"-{self.pdflatex}"]
> > +            else:
> > +                latex_cmd = [self.pdflatex]
> > +            latex_cmd.extend(shlex.split(self.latexopts))
> > +            tex_suffix = ".tex"
> > +            has_tex = False
> > +            build_failed = False
> > +            with os.scandir(from_dir) as it:
> > +                for entry in it:
> > +                    if not entry.name.endswith(tex_suffix):
> > +                        continue
> > +                    name = entry.name[:-len(tex_suffix)]
> > +                    has_tex = True
> > +                    try:
> > +                        subprocess.run(latex_cmd + [entry.path],
> > +                                       cwd=from_dir, check=True)
> > +                    except subprocess.CalledProcessError:
> > +                        pass
> > +                    pdf_name = name + ".pdf"
> > +                    pdf_from = os.path.join(from_dir, pdf_name)
> > +                    pdf_to = os.path.join(pdf_dir, pdf_name)
> > +                    if os.path.exists(pdf_from):
> > +                        os.rename(pdf_from, pdf_to)
> > +                        builds[name] = os.path.relpath(pdf_to, self.builddir)
> > +                    else:
> > +                        builds[name] = "FAILED"
> > +                        build_failed = True
> > +                    name = entry.name.removesuffix(".tex")
> > +                    max_len = max(max_len, len(name))
> > +
> > +            if not has_tex:
> > +                name = os.path.basename(from_dir)
> > +                max_len = max(max_len, len(name))
> > +                builds[name] = "FAILED (no .tex)"
> > +                build_failed = True
> > +        msg = "Summary"
> > +        msg += "\n" + "=" * len(msg)
> > +        print()
> > +        print(msg)
> > +        for pdf_name, pdf_file in builds.items():
> > +            print(f"{pdf_name:<{max_len}}: {pdf_file}")
> > +        print()
> > +        if build_failed:
> > +            sys.exit("PDF build failed: not all PDF files were created.")
> > +        else:
> > +            print("All PDF files were built.")
> > +
> > +    def handle_info(self, output_dirs):
> > +        for output_dir in output_dirs:
> > +            try:
> > +                subprocess.run(["make", "info"], cwd=output_dir, check=True)
> > +            except subprocess.CalledProcessError as e:
> > +                sys.exit(f"Error generating info docs: {e}")
> > +
> > +    def cleandocs(self, builder):
> > +        shutil.rmtree(self.builddir, ignore_errors=True)
> > +
> > +    def build(self, target, sphinxdirs=None, conf="conf.py",
> > +              theme=None, css=None, paper=None):
> > +        builder = TARGETS[target]["builder"]
> > +        out_dir = TARGETS[target].get("out_dir", "")
> > +        if target == "cleandocs":
> > +            self.cleandocs(builder)
> > +            return
> > +        if theme:
> > +                os.environ["DOCS_THEME"] = theme
> > +        sphinxbuild = shutil.which(self.sphinxbuild, path=self.env["PATH"])
> > +        if not sphinxbuild:
> > +            sys.exit(f"Error: {self.sphinxbuild} not found in PATH.\n")
> > +        if builder == "latex":
> > +            if not self.pdflatex_cmd and not self.latexmk_cmd:
> > +                sys.exit("Error: pdflatex or latexmk required for PDF generation")
> > +        docs_dir = os.path.abspath(os.path.join(self.srctree, "Documentation"))
> > +        kerneldoc = self.kerneldoc
> > +        if kerneldoc.startswith(self.srctree):
> > +            kerneldoc = os.path.relpath(kerneldoc, self.srctree)
> > +        args = [ "-b", builder, "-c", docs_dir ]
> > +        if builder == "latex":
> > +            if not paper:
> > +                paper = PAPER[1]
> > +            args.extend(["-D", f"latex_elements.papersize={paper}paper"])
> > +        if self.config_rust:
> > +            args.extend(["-t", "rustdoc"])
> > +        if conf:
> > +            self.env["SPHINX_CONF"] = self.get_path(conf, abs_path=True)
> > +        if not sphinxdirs:
> > +            sphinxdirs = os.environ.get("SPHINXDIRS", ".")
> > +        sphinxdirs_list = []
> > +        for sphinxdir in sphinxdirs:
> > +            if isinstance(sphinxdir, list):
> > +                sphinxdirs_list += sphinxdir
> > +            else:
> > +                for name in sphinxdir.split(" "):
> > +                    sphinxdirs_list.append(name)
> > +        output_dirs = []
> > +        for sphinxdir in sphinxdirs_list:
> > +            src_dir = os.path.join(docs_dir, sphinxdir)
> > +            doctree_dir = os.path.join(self.builddir, ".doctrees")
> > +            output_dir = os.path.join(self.builddir, sphinxdir, out_dir)
> > +            src_dir = os.path.normpath(src_dir)
> > +            doctree_dir = os.path.normpath(doctree_dir)
> > +            output_dir = os.path.normpath(output_dir)
> > +            os.makedirs(doctree_dir, exist_ok=True)
> > +            os.makedirs(output_dir, exist_ok=True)
> > +            output_dirs.append(output_dir)
> > +            build_args = args + [
> > +                "-d", doctree_dir,
> > +                "-D", f"kerneldoc_bin={kerneldoc}",
> > +                "-D", f"version={self.kernelversion}",
> > +                "-D", f"release={self.kernelrelease}",
> > +                "-D", f"kerneldoc_srctree={self.srctree}",
> > +                src_dir,
> > +                output_dir,
> > +            ]
> > +            try:
> > +                self.run_sphinx(sphinxbuild, build_args, env=self.env)
> > +            except (OSError, ValueError, subprocess.SubprocessError) as e:
> > +                sys.exit(f"Build failed: {repr(e)}")
> > +            if target in ["htmldocs", "epubdocs"]:
> > +                self.handle_html(css, output_dir)
> > +        if target == "pdfdocs":
> > +            self.handle_pdf(output_dirs)
> > +        elif target == "infodocs":
> > +            self.handle_info(output_dirs)
> > +
> > +def jobs_type(value):
> > +    if value is None:
> > +        return None
> > +    if value.lower() == 'auto':
> > +        return value.lower()
> > +    try:
> > +        if int(value) >= 1:
> > +            return value
> > +        raise argparse.ArgumentTypeError(f"Minimum jobs is 1, got {value}")
> > +    except ValueError:
> > +        raise argparse.ArgumentTypeError(f"Must be 'auto' or positive integer, got {value}")
> > +
> > +def main():
> > +    parser = argparse.ArgumentParser(description="Kernel documentation builder")
> > +    parser.add_argument("target", choices=list(TARGETS.keys()),
> > +                        help="Documentation target to build")
> > +    parser.add_argument("--sphinxdirs", nargs="+",
> > +                        help="Specific directories to build")
> > +    parser.add_argument("--conf", default="conf.py",
> > +                        help="Sphinx configuration file")
> > +    parser.add_argument("--builddir", default="output",
> > +                        help="Sphinx configuration file")
> > +    parser.add_argument("--theme", help="Sphinx theme to use")
> > +    parser.add_argument("--css", help="Custom CSS file for HTML/EPUB")
> > +    parser.add_argument("--paper", choices=PAPER, default=PAPER[0],
> > +                        help="Paper size for LaTeX/PDF output")
> > +    parser.add_argument("-v", "--verbose", action='store_true',
> > +                        help="place build in verbose mode")
> > +    parser.add_argument('-j', '--jobs', type=jobs_type,
> > +                        help="Sets number of jobs to use with sphinx-build")
> > +    args = parser.parse_args()
> > +    PythonVersion.check_python(MIN_PYTHON_VERSION)
> > +    builder = SphinxBuilder(builddir=args.builddir,
> > +                            verbose=args.verbose, n_jobs=args.jobs)
> > +    builder.build(args.target, sphinxdirs=args.sphinxdirs, conf=args.conf,
> > +                  theme=args.theme, css=args.css, paper=args.paper)
> > +
> > +if __name__ == "__main__":
> > +    main()  
> 



Thanks,
Mauro

Re: [PATCH v4 08/19] tools/docs: sphinx-build-wrapper: add a wrapper for sphinx-build

[Mauro Carvalho Chehab]

Em Wed, 10 Sep 2025 14:59:26 +0200
Mauro Carvalho Chehab <mchehab+huawei@kernel.org> escreveu:

> Em Wed, 10 Sep 2025 13:46:17 +0300
> Jani Nikula <jani.nikula@linux.intel.com> escreveu:
> 
> > On Thu, 04 Sep 2025, Mauro Carvalho Chehab <mchehab+huawei@kernel.org> wrote:  
> > > There are too much magic inside docs Makefile to properly run
> > > sphinx-build. Create an ancillary script that contains all
> > > kernel-related sphinx-build call logic currently at Makefile.
> > >
> > > Such script is designed to work both as an standalone command
> > > and as part of a Makefile. As such, it properly handles POSIX
> > > jobserver used by GNU make.
> > >
> > > On a side note, there was a line number increase due to the
> > > conversion:
> > >
> > >  Documentation/Makefile          |  131 +++----------
> > >  tools/docs/sphinx-build-wrapper |  293 +++++++++++++++++++++++++++++++
> > >  2 files changed, 323 insertions(+), 101 deletions(-)
> > >
> > > This is because some things are more verbosed on Python and because
> > > it requires reading env vars from Makefile. Besides it, this script
> > > has some extra features that don't exist at the Makefile:
> > >
> > > - It can be called directly from command line;
> > > - It properly return PDF build errors.
> > >
> > > When running the script alone, it will only take handle sphinx-build
> > > targets. On other words, it won't runn make rustdoc after building
> > > htmlfiles, nor it will run the extra check scripts.    
> > 
> > I've always strongly believed we should aim to make it possible to build
> > the documentation by running sphinx-build directly on the
> > command-line. Not that it would be the common way to run it, but to not
> > accumulate things in the Makefile that need to happen before or
> > after. To promote handling the documentation build in Sphinx. To be able
> > to debug issues and try new Sphinx versions without all the hacks.  
> 
> That would be the better, but, unfortunately, this is not possible, for 
> several reasons:
> 
> 1. SPHINXDIRS. It needs a lot of magic to work, both before running
>    sphinx-build and after (inside conf.py);
> 2. Several extensions require kernel-specific environment variables to
>    work. Calling sphinx-build directly breaks them;
> 3. Sphinx itself doesn't build several targets alone. Instead, they create
>    a Makefile, and an extra step is needed to finish the build. That's 
>    the case for pdf and texinfo, for instance;
> 4. Man pages generation. Sphinx support to generate it is very poor;
> 5. Rust integration adds more complexity to the table;

Heh, I ended forgetting what motivated me to do this work: the lack
of a native pdf builder (or an external one that actually works).

The current approach of using LaTeX for PDF is dirty:

- Sphinx can't produce a LaTeX file from the Kernel trees without
  hundreds of warnings;
- latexmk hides some of them, but even it just one warning is reported,
  the return status is not zero.

So, at the end of a PDF build via latex builder and the extra 
makefile, there's no way to know if all PDF files were built or not,
except by having a somewhat complex logic that verifies all files
one by one.

We needed that to check if all PDF files were generated at the set of
test platforms we want docs build to work.

As the logic to check is complex, I would need to either add an
extra magic inside the already too complex Documentation/Makefile,
or to add one more hackish script.

Instead, I opted to reduce the number of scripts required during
PDF builds. So, this series:

- dropped the need of running jobserver;
- dropped the parallel jobs shell script;
- dropped the extra script to generate man pages;
- can later be integrated with sphinx-pre-install, dropping one
  more script;
- didn't add an extra script to fix the return code for PDF;
- now summarizes what PDF files were actually generated and what
  files weren't produced.

> 
> I'm not seeing sphinx-build supporting the above needs anytime soon,
> and, even if we push our needs to Sphinx and it gets accepted there,
> we'll still need to wait for quite a while until LTS distros merge
> them.
> 
> > This patch moves a bunch of that logic into a Python wrapper, and I feel
> > like it complicates matters. You can no longer rely on 'make V=1' to get
> > the build commands, for instance.  
> 
> Quite the opposite. if you try using "make V=1", it won't show the
> command line used to call sphinx-build anymore.
> 
> This series restore it.
> 
> See, if you build with this series with V=1, you will see exactly
> what commands are used on the build:
> 
> 	$ make V=1 htmldocs
> 	...
> 	python3 ./tools/docs/sphinx-build-wrapper htmldocs \
> 	        --sphinxdirs="." --conf="conf.py" \
>         	--builddir="Documentation/output" \
> 	        --theme= --css= --paper=
> 	python3 /new_devel/docs/sphinx_latest/bin/sphinx-build -j25 -b html -c /new_devel/docs/Documentation -d /new_devel/docs/Documentation/output/.doctrees -D kerneldoc_bin=scripts/kernel-doc.py -D version=6.17.0-rc1 -D release=6.17.0-rc1+ -D kerneldoc_srctree=. /new_devel/docs/Documentation /new_devel/docs/Documentation/output
> 	...
> 
> 
> 
> > Newer Sphinx versions have the -M option for "make mode". The Makefiles
> > produced by sphinx-quickstart only have one build target:
> > 
> > # Catch-all target: route all unknown targets to Sphinx using the new
> > # "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).  
> 
> I didn't know about this, but from [1] it sounds it covers just two
> targets: "latexpdf" and "info".
> 
> The most complex scenario is still not covered: SPHINXDIRS.
> 
> [1] https://www.sphinx-doc.org/en/master/man/sphinx-build.html
> 
> > %: Makefile
> >         @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
> > 
> > That's all.  
> 
> Try doing such change on your makefile. it will break:
> 
> 	- SPHINXDIRS;
> 	- V=1;
> 	- rustdoc
> 
> and will still be dependent on variables that are passed via
> env from Kernel makefile. So, stil you can't run from command
> line. Also, if you call sphinx-build from command line:
> 
> 	$ sphinx-build -j25 -b html Documentation Documentation/output
> 	...
> 	      File "<frozen os>", line 717, in __getitem__
> 	    KeyError: 'srctree'
> 
> It won't work, as several parameters that are required by conf.py and by
> Sphinx extensions would be missing (the most important one is srctree, but
> there are others in the line too).
> 
> > The proposed wrapper duplicates loads of code that's supposed to be
> > handled by sphinx-build directly.  
> 
> Once we get the wrapper, we can work to simplify it, but still I
> can't see how to get rid of it.
> 
> > Including the target/builder names.  
> 
> True, but this was a design decision taken lots of years ago: instead
> of:
> 	make html
> 
> we're using:
> 
> 	make htmldocs
> 
> This series doesn't change that: either makefile or the script need
> to tho the namespace conversion.
> 
> > Seems to me the goal should be to figure out *generic* wrappers for
> > handling parallelism, not Sphinx aware/specific.
> > 
> > 
> > BR,
> > Jani.
> >   
> > >
> > > Signed-off-by: Mauro Carvalho Chehab <mchehab+huawei@kernel.org>
> > > ---
> > >  Documentation/Makefile          | 131 ++++----------
> > >  tools/docs/sphinx-build-wrapper | 293 ++++++++++++++++++++++++++++++++
> > >  2 files changed, 323 insertions(+), 101 deletions(-)
> > >  create mode 100755 tools/docs/sphinx-build-wrapper
> > >
> > > diff --git a/Documentation/Makefile b/Documentation/Makefile
> > > index deb2029228ed..4736f02b6c9e 100644
> > > --- a/Documentation/Makefile
> > > +++ b/Documentation/Makefile
> > > @@ -23,21 +23,22 @@ SPHINXOPTS    =
> > >  SPHINXDIRS    = .
> > >  DOCS_THEME    =
> > >  DOCS_CSS      =
> > > -_SPHINXDIRS   = $(sort $(patsubst $(srctree)/Documentation/%/index.rst,%,$(wildcard $(srctree)/Documentation/*/index.rst)))
> > >  SPHINX_CONF   = conf.py
> > >  PAPER         =
> > >  BUILDDIR      = $(obj)/output
> > >  PDFLATEX      = xelatex
> > >  LATEXOPTS     = -interaction=batchmode -no-shell-escape
> > >  
> > > +PYTHONPYCACHEPREFIX ?= $(abspath $(BUILDDIR)/__pycache__)
> > > +
> > > +# Wrapper for sphinx-build
> > > +
> > > +BUILD_WRAPPER = $(srctree)/tools/docs/sphinx-build-wrapper
> > > +
> > >  # For denylisting "variable font" files
> > >  # Can be overridden by setting as an env variable
> > >  FONTS_CONF_DENY_VF ?= $(HOME)/deny-vf
> > >  
> > > -ifeq ($(findstring 1, $(KBUILD_VERBOSE)),)
> > > -SPHINXOPTS    += "-q"
> > > -endif
> > > -
> > >  # User-friendly check for sphinx-build
> > >  HAVE_SPHINX := $(shell if which $(SPHINXBUILD) >/dev/null 2>&1; then echo 1; else echo 0; fi)
> > >  
> > > @@ -51,63 +52,31 @@ ifeq ($(HAVE_SPHINX),0)
> > >  
> > >  else # HAVE_SPHINX
> > >  
> > > -# User-friendly check for pdflatex and latexmk
> > > -HAVE_PDFLATEX := $(shell if which $(PDFLATEX) >/dev/null 2>&1; then echo 1; else echo 0; fi)
> > > -HAVE_LATEXMK := $(shell if which latexmk >/dev/null 2>&1; then echo 1; else echo 0; fi)
> > > +# Common documentation targets
> > > +infodocs texinfodocs latexdocs epubdocs xmldocs pdfdocs linkcheckdocs:
> > > +	$(Q)@$(srctree)/tools/docs/sphinx-pre-install --version-check
> > > +	+$(Q)$(PYTHON3) $(BUILD_WRAPPER) $@ \
> > > +		--sphinxdirs="$(SPHINXDIRS)" --conf="$(SPHINX_CONF)" \
> > > +		--builddir="$(BUILDDIR)" \
> > > +		--theme=$(DOCS_THEME) --css=$(DOCS_CSS) --paper=$(PAPER)
> > >  
> > > -ifeq ($(HAVE_LATEXMK),1)
> > > -	PDFLATEX := latexmk -$(PDFLATEX)
> > > -endif #HAVE_LATEXMK
> > > -
> > > -# Internal variables.
> > > -PAPEROPT_a4     = -D latex_elements.papersize=a4paper
> > > -PAPEROPT_letter = -D latex_elements.papersize=letterpaper
> > > -ALLSPHINXOPTS   = -D kerneldoc_srctree=$(srctree) -D kerneldoc_bin=$(KERNELDOC)
> > > -ALLSPHINXOPTS   += $(PAPEROPT_$(PAPER)) $(SPHINXOPTS)
> > > -ifneq ($(wildcard $(srctree)/.config),)
> > > -ifeq ($(CONFIG_RUST),y)
> > > -	# Let Sphinx know we will include rustdoc
> > > -	ALLSPHINXOPTS   +=  -t rustdoc
> > > -endif
> > > +# Special handling for pdfdocs
> > > +ifeq ($(shell which $(PDFLATEX) >/dev/null 2>&1; echo $$?),0)
> > > +pdfdocs: DENY_VF = XDG_CONFIG_HOME=$(FONTS_CONF_DENY_VF)
> > > +else
> > > +pdfdocs:
> > > +	$(warning The '$(PDFLATEX)' command was not found. Make sure you have it installed and in PATH to produce PDF output.)
> > > +	@echo "  SKIP    Sphinx $@ target."
> > >  endif
> > > -# the i18n builder cannot share the environment and doctrees with the others
> > > -I18NSPHINXOPTS  = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) .
> > > -
> > > -# commands; the 'cmd' from scripts/Kbuild.include is not *loopable*
> > > -loop_cmd = $(echo-cmd) $(cmd_$(1)) || exit;
> > > -
> > > -# $2 sphinx builder e.g. "html"
> > > -# $3 name of the build subfolder / e.g. "userspace-api/media", used as:
> > > -#    * dest folder relative to $(BUILDDIR) and
> > > -#    * cache folder relative to $(BUILDDIR)/.doctrees
> > > -# $4 dest subfolder e.g. "man" for man pages at userspace-api/media/man
> > > -# $5 reST source folder relative to $(src),
> > > -#    e.g. "userspace-api/media" for the linux-tv book-set at ./Documentation/userspace-api/media
> > > -
> > > -PYTHONPYCACHEPREFIX ?= $(abspath $(BUILDDIR)/__pycache__)
> > > -
> > > -quiet_cmd_sphinx = SPHINX  $@ --> file://$(abspath $(BUILDDIR)/$3/$4)
> > > -      cmd_sphinx = \
> > > -	PYTHONPYCACHEPREFIX="$(PYTHONPYCACHEPREFIX)" \
> > > -	BUILDDIR=$(abspath $(BUILDDIR)) SPHINX_CONF=$(abspath $(src)/$5/$(SPHINX_CONF)) \
> > > -	$(PYTHON3) $(srctree)/scripts/jobserver-exec \
> > > -	$(CONFIG_SHELL) $(srctree)/Documentation/sphinx/parallel-wrapper.sh \
> > > -	$(SPHINXBUILD) \
> > > -	-b $2 \
> > > -	-c $(abspath $(src)) \
> > > -	-d $(abspath $(BUILDDIR)/.doctrees/$3) \
> > > -	-D version=$(KERNELVERSION) -D release=$(KERNELRELEASE) \
> > > -	$(ALLSPHINXOPTS) \
> > > -	$(abspath $(src)/$5) \
> > > -	$(abspath $(BUILDDIR)/$3/$4) && \
> > > -	if [ "x$(DOCS_CSS)" != "x" ]; then \
> > > -		cp $(if $(patsubst /%,,$(DOCS_CSS)),$(abspath $(srctree)/$(DOCS_CSS)),$(DOCS_CSS)) $(BUILDDIR)/$3/_static/; \
> > > -	fi
> > >  
> > > +# HTML main logic is identical to other targets. However, if rust is enabled,
> > > +# an extra step at the end is required to generate rustdoc.
> > >  htmldocs:
> > > -	@$(srctree)/tools/docs/sphinx-pre-install --version-check
> > > -	@+$(foreach var,$(SPHINXDIRS),$(call loop_cmd,sphinx,html,$(var),,$(var)))
> > > -
> > > +	$(Q)@$(srctree)/tools/docs/sphinx-pre-install --version-check
> > > +	+$(Q)$(PYTHON3) $(BUILD_WRAPPER) $@ \
> > > +		--sphinxdirs="$(SPHINXDIRS)" --conf="$(SPHINX_CONF)" \
> > > +		--builddir="$(BUILDDIR)" \
> > > +		--theme=$(DOCS_THEME) --css=$(DOCS_CSS) --paper=$(PAPER)
> > >  # If Rust support is available and .config exists, add rustdoc generated contents.
> > >  # If there are any, the errors from this make rustdoc will be displayed but
> > >  # won't stop the execution of htmldocs
> > > @@ -118,49 +87,6 @@ ifeq ($(CONFIG_RUST),y)
> > >  endif
> > >  endif
> > >  
> > > -texinfodocs:
> > > -	@$(srctree)/tools/docs/sphinx-pre-install --version-check
> > > -	@+$(foreach var,$(SPHINXDIRS),$(call loop_cmd,sphinx,texinfo,$(var),texinfo,$(var)))
> > > -
> > > -# Note: the 'info' Make target is generated by sphinx itself when
> > > -# running the texinfodocs target define above.
> > > -infodocs: texinfodocs
> > > -	$(MAKE) -C $(BUILDDIR)/texinfo info
> > > -
> > > -linkcheckdocs:
> > > -	@$(foreach var,$(SPHINXDIRS),$(call loop_cmd,sphinx,linkcheck,$(var),,$(var)))
> > > -
> > > -latexdocs:
> > > -	@$(srctree)/tools/docs/sphinx-pre-install --version-check
> > > -	@+$(foreach var,$(SPHINXDIRS),$(call loop_cmd,sphinx,latex,$(var),latex,$(var)))
> > > -
> > > -ifeq ($(HAVE_PDFLATEX),0)
> > > -
> > > -pdfdocs:
> > > -	$(warning The '$(PDFLATEX)' command was not found. Make sure you have it installed and in PATH to produce PDF output.)
> > > -	@echo "  SKIP    Sphinx $@ target."
> > > -
> > > -else # HAVE_PDFLATEX
> > > -
> > > -pdfdocs: DENY_VF = XDG_CONFIG_HOME=$(FONTS_CONF_DENY_VF)
> > > -pdfdocs: latexdocs
> > > -	@$(srctree)/tools/docs/sphinx-pre-install --version-check
> > > -	$(foreach var,$(SPHINXDIRS), \
> > > -	   $(MAKE) PDFLATEX="$(PDFLATEX)" LATEXOPTS="$(LATEXOPTS)" $(DENY_VF) -C $(BUILDDIR)/$(var)/latex || sh $(srctree)/scripts/check-variable-fonts.sh || exit; \
> > > -	   mkdir -p $(BUILDDIR)/$(var)/pdf; \
> > > -	   mv $(subst .tex,.pdf,$(wildcard $(BUILDDIR)/$(var)/latex/*.tex)) $(BUILDDIR)/$(var)/pdf/; \
> > > -	)
> > > -
> > > -endif # HAVE_PDFLATEX
> > > -
> > > -epubdocs:
> > > -	@$(srctree)/tools/docs/sphinx-pre-install --version-check
> > > -	@+$(foreach var,$(SPHINXDIRS),$(call loop_cmd,sphinx,epub,$(var),epub,$(var)))
> > > -
> > > -xmldocs:
> > > -	@$(srctree)/tools/docs/sphinx-pre-install --version-check
> > > -	@+$(foreach var,$(SPHINXDIRS),$(call loop_cmd,sphinx,xml,$(var),xml,$(var)))
> > > -
> > >  endif # HAVE_SPHINX
> > >  
> > >  # The following targets are independent of HAVE_SPHINX, and the rules should
> > > @@ -172,6 +98,9 @@ refcheckdocs:
> > >  cleandocs:
> > >  	$(Q)rm -rf $(BUILDDIR)
> > >  
> > > +# Used only on help
> > > +_SPHINXDIRS   = $(sort $(patsubst $(srctree)/Documentation/%/index.rst,%,$(wildcard $(srctree)/Documentation/*/index.rst)))
> > > +
> > >  dochelp:
> > >  	@echo  ' Linux kernel internal documentation in different formats from ReST:'
> > >  	@echo  '  htmldocs        - HTML'
> > > diff --git a/tools/docs/sphinx-build-wrapper b/tools/docs/sphinx-build-wrapper
> > > new file mode 100755
> > > index 000000000000..3256418d8dc5
> > > --- /dev/null
> > > +++ b/tools/docs/sphinx-build-wrapper
> > > @@ -0,0 +1,293 @@
> > > +#!/usr/bin/env python3
> > > +# SPDX-License-Identifier: GPL-2.0
> > > +import argparse
> > > +import os
> > > +import shlex
> > > +import shutil
> > > +import subprocess
> > > +import sys
> > > +from lib.python_version import PythonVersion
> > > +
> > > +LIB_DIR = "../../scripts/lib"
> > > +SRC_DIR = os.path.dirname(os.path.realpath(__file__))
> > > +sys.path.insert(0, os.path.join(SRC_DIR, LIB_DIR))
> > > +
> > > +from jobserver import JobserverExec
> > > +
> > > +MIN_PYTHON_VERSION = PythonVersion("3.7").version
> > > +PAPER = ["", "a4", "letter"]
> > > +TARGETS = {
> > > +    "cleandocs":     { "builder": "clean" },
> > > +    "linkcheckdocs": { "builder": "linkcheck" },
> > > +    "htmldocs":      { "builder": "html" },
> > > +    "epubdocs":      { "builder": "epub",    "out_dir": "epub" },
> > > +    "texinfodocs":   { "builder": "texinfo", "out_dir": "texinfo" },
> > > +    "infodocs":      { "builder": "texinfo", "out_dir": "texinfo" },
> > > +    "latexdocs":     { "builder": "latex",   "out_dir": "latex" },
> > > +    "pdfdocs":       { "builder": "latex",   "out_dir": "latex" },
> > > +    "xmldocs":       { "builder": "xml",     "out_dir": "xml" },
> > > +}
> > > +
> > > +class SphinxBuilder:
> > > +    def is_rust_enabled(self):
> > > +        config_path = os.path.join(self.srctree, ".config")
> > > +        if os.path.isfile(config_path):
> > > +            with open(config_path, "r", encoding="utf-8") as f:
> > > +                return "CONFIG_RUST=y" in f.read()
> > > +        return False
> > > +
> > > +    def get_path(self, path, use_cwd=False, abs_path=False):
> > > +        path = os.path.expanduser(path)
> > > +        if not path.startswith("/"):
> > > +            if use_cwd:
> > > +                base = os.getcwd()
> > > +            else:
> > > +                base = self.srctree
> > > +            path = os.path.join(base, path)
> > > +        if abs_path:
> > > +            return os.path.abspath(path)
> > > +        return path
> > > +
> > > +    def __init__(self, builddir, verbose=False, n_jobs=None):
> > > +        self.verbose = None
> > > +        self.kernelversion = os.environ.get("KERNELVERSION", "unknown")
> > > +        self.kernelrelease = os.environ.get("KERNELRELEASE", "unknown")
> > > +        self.pdflatex = os.environ.get("PDFLATEX", "xelatex")
> > > +        self.latexopts = os.environ.get("LATEXOPTS", "-interaction=batchmode -no-shell-escape")
> > > +        if not verbose:
> > > +            verbose = bool(os.environ.get("KBUILD_VERBOSE", "") != "")
> > > +        if verbose is not None:
> > > +            self.verbose = verbose
> > > +        parser = argparse.ArgumentParser()
> > > +        parser.add_argument('-j', '--jobs', type=int)
> > > +        parser.add_argument('-q', '--quiet', type=int)
> > > +        sphinxopts = shlex.split(os.environ.get("SPHINXOPTS", ""))
> > > +        sphinx_args, self.sphinxopts = parser.parse_known_args(sphinxopts)
> > > +        if sphinx_args.quiet is True:
> > > +            self.verbose = False
> > > +        if sphinx_args.jobs:
> > > +            self.n_jobs = sphinx_args.jobs
> > > +        self.n_jobs = n_jobs
> > > +        self.srctree = os.environ.get("srctree")
> > > +        if not self.srctree:
> > > +            self.srctree = "."
> > > +            os.environ["srctree"] = self.srctree
> > > +        self.sphinxbuild = os.environ.get("SPHINXBUILD", "sphinx-build")
> > > +        self.kerneldoc = self.get_path(os.environ.get("KERNELDOC",
> > > +                                                      "scripts/kernel-doc.py"))
> > > +        self.builddir = self.get_path(builddir, use_cwd=True, abs_path=True)
> > > +
> > > +        self.config_rust = self.is_rust_enabled()
> > > +
> > > +        self.pdflatex_cmd = shutil.which(self.pdflatex)
> > > +        self.latexmk_cmd = shutil.which("latexmk")
> > > +
> > > +        self.env = os.environ.copy()
> > > +
> > > +    def run_sphinx(self, sphinx_build, build_args, *args, **pwargs):
> > > +        with JobserverExec() as jobserver:
> > > +            if jobserver.claim:
> > > +                n_jobs = str(jobserver.claim)
> > > +            else:
> > > +                n_jobs = "auto" # Supported since Sphinx 1.7
> > > +            cmd = []
> > > +            cmd.append(sys.executable)
> > > +            cmd.append(sphinx_build)
> > > +            if self.n_jobs:
> > > +                n_jobs = str(self.n_jobs)
> > > +
> > > +            if n_jobs:
> > > +                cmd += [f"-j{n_jobs}"]
> > > +
> > > +            if not self.verbose:
> > > +                cmd.append("-q")
> > > +            cmd += self.sphinxopts
> > > +            cmd += build_args
> > > +            if self.verbose:
> > > +                print(" ".join(cmd))
> > > +            return subprocess.call(cmd, *args, **pwargs)
> > > +
> > > +    def handle_html(self, css, output_dir):
> > > +        if not css:
> > > +            return
> > > +        css = os.path.expanduser(css)
> > > +        if not css.startswith("/"):
> > > +            css = os.path.join(self.srctree, css)
> > > +        static_dir = os.path.join(output_dir, "_static")
> > > +        os.makedirs(static_dir, exist_ok=True)
> > > +        try:
> > > +            shutil.copy2(css, static_dir)
> > > +        except (OSError, IOError) as e:
> > > +            print(f"Warning: Failed to copy CSS: {e}", file=sys.stderr)
> > > +
> > > +    def handle_pdf(self, output_dirs):
> > > +        builds = {}
> > > +        max_len = 0
> > > +        for from_dir in output_dirs:
> > > +            pdf_dir = os.path.join(from_dir, "../pdf")
> > > +            os.makedirs(pdf_dir, exist_ok=True)
> > > +            if self.latexmk_cmd:
> > > +                latex_cmd = [self.latexmk_cmd, f"-{self.pdflatex}"]
> > > +            else:
> > > +                latex_cmd = [self.pdflatex]
> > > +            latex_cmd.extend(shlex.split(self.latexopts))
> > > +            tex_suffix = ".tex"
> > > +            has_tex = False
> > > +            build_failed = False
> > > +            with os.scandir(from_dir) as it:
> > > +                for entry in it:
> > > +                    if not entry.name.endswith(tex_suffix):
> > > +                        continue
> > > +                    name = entry.name[:-len(tex_suffix)]
> > > +                    has_tex = True
> > > +                    try:
> > > +                        subprocess.run(latex_cmd + [entry.path],
> > > +                                       cwd=from_dir, check=True)
> > > +                    except subprocess.CalledProcessError:
> > > +                        pass
> > > +                    pdf_name = name + ".pdf"
> > > +                    pdf_from = os.path.join(from_dir, pdf_name)
> > > +                    pdf_to = os.path.join(pdf_dir, pdf_name)
> > > +                    if os.path.exists(pdf_from):
> > > +                        os.rename(pdf_from, pdf_to)
> > > +                        builds[name] = os.path.relpath(pdf_to, self.builddir)
> > > +                    else:
> > > +                        builds[name] = "FAILED"
> > > +                        build_failed = True
> > > +                    name = entry.name.removesuffix(".tex")
> > > +                    max_len = max(max_len, len(name))
> > > +
> > > +            if not has_tex:
> > > +                name = os.path.basename(from_dir)
> > > +                max_len = max(max_len, len(name))
> > > +                builds[name] = "FAILED (no .tex)"
> > > +                build_failed = True
> > > +        msg = "Summary"
> > > +        msg += "\n" + "=" * len(msg)
> > > +        print()
> > > +        print(msg)
> > > +        for pdf_name, pdf_file in builds.items():
> > > +            print(f"{pdf_name:<{max_len}}: {pdf_file}")
> > > +        print()
> > > +        if build_failed:
> > > +            sys.exit("PDF build failed: not all PDF files were created.")
> > > +        else:
> > > +            print("All PDF files were built.")
> > > +
> > > +    def handle_info(self, output_dirs):
> > > +        for output_dir in output_dirs:
> > > +            try:
> > > +                subprocess.run(["make", "info"], cwd=output_dir, check=True)
> > > +            except subprocess.CalledProcessError as e:
> > > +                sys.exit(f"Error generating info docs: {e}")
> > > +
> > > +    def cleandocs(self, builder):
> > > +        shutil.rmtree(self.builddir, ignore_errors=True)
> > > +
> > > +    def build(self, target, sphinxdirs=None, conf="conf.py",
> > > +              theme=None, css=None, paper=None):
> > > +        builder = TARGETS[target]["builder"]
> > > +        out_dir = TARGETS[target].get("out_dir", "")
> > > +        if target == "cleandocs":
> > > +            self.cleandocs(builder)
> > > +            return
> > > +        if theme:
> > > +                os.environ["DOCS_THEME"] = theme
> > > +        sphinxbuild = shutil.which(self.sphinxbuild, path=self.env["PATH"])
> > > +        if not sphinxbuild:
> > > +            sys.exit(f"Error: {self.sphinxbuild} not found in PATH.\n")
> > > +        if builder == "latex":
> > > +            if not self.pdflatex_cmd and not self.latexmk_cmd:
> > > +                sys.exit("Error: pdflatex or latexmk required for PDF generation")
> > > +        docs_dir = os.path.abspath(os.path.join(self.srctree, "Documentation"))
> > > +        kerneldoc = self.kerneldoc
> > > +        if kerneldoc.startswith(self.srctree):
> > > +            kerneldoc = os.path.relpath(kerneldoc, self.srctree)
> > > +        args = [ "-b", builder, "-c", docs_dir ]
> > > +        if builder == "latex":
> > > +            if not paper:
> > > +                paper = PAPER[1]
> > > +            args.extend(["-D", f"latex_elements.papersize={paper}paper"])
> > > +        if self.config_rust:
> > > +            args.extend(["-t", "rustdoc"])
> > > +        if conf:
> > > +            self.env["SPHINX_CONF"] = self.get_path(conf, abs_path=True)
> > > +        if not sphinxdirs:
> > > +            sphinxdirs = os.environ.get("SPHINXDIRS", ".")
> > > +        sphinxdirs_list = []
> > > +        for sphinxdir in sphinxdirs:
> > > +            if isinstance(sphinxdir, list):
> > > +                sphinxdirs_list += sphinxdir
> > > +            else:
> > > +                for name in sphinxdir.split(" "):
> > > +                    sphinxdirs_list.append(name)
> > > +        output_dirs = []
> > > +        for sphinxdir in sphinxdirs_list:
> > > +            src_dir = os.path.join(docs_dir, sphinxdir)
> > > +            doctree_dir = os.path.join(self.builddir, ".doctrees")
> > > +            output_dir = os.path.join(self.builddir, sphinxdir, out_dir)
> > > +            src_dir = os.path.normpath(src_dir)
> > > +            doctree_dir = os.path.normpath(doctree_dir)
> > > +            output_dir = os.path.normpath(output_dir)
> > > +            os.makedirs(doctree_dir, exist_ok=True)
> > > +            os.makedirs(output_dir, exist_ok=True)
> > > +            output_dirs.append(output_dir)
> > > +            build_args = args + [
> > > +                "-d", doctree_dir,
> > > +                "-D", f"kerneldoc_bin={kerneldoc}",
> > > +                "-D", f"version={self.kernelversion}",
> > > +                "-D", f"release={self.kernelrelease}",
> > > +                "-D", f"kerneldoc_srctree={self.srctree}",
> > > +                src_dir,
> > > +                output_dir,
> > > +            ]
> > > +            try:
> > > +                self.run_sphinx(sphinxbuild, build_args, env=self.env)
> > > +            except (OSError, ValueError, subprocess.SubprocessError) as e:
> > > +                sys.exit(f"Build failed: {repr(e)}")
> > > +            if target in ["htmldocs", "epubdocs"]:
> > > +                self.handle_html(css, output_dir)
> > > +        if target == "pdfdocs":
> > > +            self.handle_pdf(output_dirs)
> > > +        elif target == "infodocs":
> > > +            self.handle_info(output_dirs)
> > > +
> > > +def jobs_type(value):
> > > +    if value is None:
> > > +        return None
> > > +    if value.lower() == 'auto':
> > > +        return value.lower()
> > > +    try:
> > > +        if int(value) >= 1:
> > > +            return value
> > > +        raise argparse.ArgumentTypeError(f"Minimum jobs is 1, got {value}")
> > > +    except ValueError:
> > > +        raise argparse.ArgumentTypeError(f"Must be 'auto' or positive integer, got {value}")
> > > +
> > > +def main():
> > > +    parser = argparse.ArgumentParser(description="Kernel documentation builder")
> > > +    parser.add_argument("target", choices=list(TARGETS.keys()),
> > > +                        help="Documentation target to build")
> > > +    parser.add_argument("--sphinxdirs", nargs="+",
> > > +                        help="Specific directories to build")
> > > +    parser.add_argument("--conf", default="conf.py",
> > > +                        help="Sphinx configuration file")
> > > +    parser.add_argument("--builddir", default="output",
> > > +                        help="Sphinx configuration file")
> > > +    parser.add_argument("--theme", help="Sphinx theme to use")
> > > +    parser.add_argument("--css", help="Custom CSS file for HTML/EPUB")
> > > +    parser.add_argument("--paper", choices=PAPER, default=PAPER[0],
> > > +                        help="Paper size for LaTeX/PDF output")
> > > +    parser.add_argument("-v", "--verbose", action='store_true',
> > > +                        help="place build in verbose mode")
> > > +    parser.add_argument('-j', '--jobs', type=jobs_type,
> > > +                        help="Sets number of jobs to use with sphinx-build")
> > > +    args = parser.parse_args()
> > > +    PythonVersion.check_python(MIN_PYTHON_VERSION)
> > > +    builder = SphinxBuilder(builddir=args.builddir,
> > > +                            verbose=args.verbose, n_jobs=args.jobs)
> > > +    builder.build(args.target, sphinxdirs=args.sphinxdirs, conf=args.conf,
> > > +                  theme=args.theme, css=args.css, paper=args.paper)
> > > +
> > > +if __name__ == "__main__":
> > > +    main()    
> >   
> 
> 
> 
> Thanks,
> Mauro



Thanks,
Mauro

Re: [PATCH v4 08/19] tools/docs: sphinx-build-wrapper: add a wrapper for sphinx-build

[Jani Nikula]

On Wed, 10 Sep 2025, Mauro Carvalho Chehab <mchehab+huawei@kernel.org> wrote:
> Em Wed, 10 Sep 2025 13:46:17 +0300
> Jani Nikula <jani.nikula@linux.intel.com> escreveu:
>
>> On Thu, 04 Sep 2025, Mauro Carvalho Chehab <mchehab+huawei@kernel.org> wrote:
>> > There are too much magic inside docs Makefile to properly run
>> > sphinx-build. Create an ancillary script that contains all
>> > kernel-related sphinx-build call logic currently at Makefile.
>> >
>> > Such script is designed to work both as an standalone command
>> > and as part of a Makefile. As such, it properly handles POSIX
>> > jobserver used by GNU make.
>> >
>> > On a side note, there was a line number increase due to the
>> > conversion:
>> >
>> >  Documentation/Makefile          |  131 +++----------
>> >  tools/docs/sphinx-build-wrapper |  293 +++++++++++++++++++++++++++++++
>> >  2 files changed, 323 insertions(+), 101 deletions(-)
>> >
>> > This is because some things are more verbosed on Python and because
>> > it requires reading env vars from Makefile. Besides it, this script
>> > has some extra features that don't exist at the Makefile:
>> >
>> > - It can be called directly from command line;
>> > - It properly return PDF build errors.
>> >
>> > When running the script alone, it will only take handle sphinx-build
>> > targets. On other words, it won't runn make rustdoc after building
>> > htmlfiles, nor it will run the extra check scripts.  
>> 
>> I've always strongly believed we should aim to make it possible to build
>> the documentation by running sphinx-build directly on the
>> command-line. Not that it would be the common way to run it, but to not
>> accumulate things in the Makefile that need to happen before or
>> after. To promote handling the documentation build in Sphinx. To be able
>> to debug issues and try new Sphinx versions without all the hacks.
>
> That would be the better, but, unfortunately, this is not possible, for 
> several reasons:
>
> 1. SPHINXDIRS. It needs a lot of magic to work, both before running
>    sphinx-build and after (inside conf.py);

Makes you wonder if that's the right solution to the original
problem. It was added as a kind of hack, and it stuck.

> 2. Several extensions require kernel-specific environment variables to
>    work. Calling sphinx-build directly breaks them;

The extensions shouldn't be using environment variables for
configuration anyway. Add config options and set them in conf.py like
everything else?

> 3. Sphinx itself doesn't build several targets alone. Instead, they create
>    a Makefile, and an extra step is needed to finish the build. That's 
>    the case for pdf and texinfo, for instance;

That's not true for the Makefile currently generated by
sphinx-quickstart. Granted, I haven't used Sphinx much for pdf output.

> 4. Man pages generation. Sphinx support to generate it is very poor;

In what way?

> 5. Rust integration adds more complexity to the table;
>
> I'm not seeing sphinx-build supporting the above needs anytime soon,
> and, even if we push our needs to Sphinx and it gets accepted there,
> we'll still need to wait for quite a while until LTS distros merge
> them.

I'm not suggesting to add anything to Sphinx upstream.

>> This patch moves a bunch of that logic into a Python wrapper, and I feel
>> like it complicates matters. You can no longer rely on 'make V=1' to get
>> the build commands, for instance.
>
> Quite the opposite. if you try using "make V=1", it won't show the
> command line used to call sphinx-build anymore.
>
> This series restore it.
>
> See, if you build with this series with V=1, you will see exactly
> what commands are used on the build:
>
> 	$ make V=1 htmldocs
> 	...
> 	python3 ./tools/docs/sphinx-build-wrapper htmldocs \
> 	        --sphinxdirs="." --conf="conf.py" \
>         	--builddir="Documentation/output" \
> 	        --theme= --css= --paper=
> 	python3 /new_devel/docs/sphinx_latest/bin/sphinx-build -j25 -b html -c /new_devel/docs/Documentation -d /new_devel/docs/Documentation/output/.doctrees -D kerneldoc_bin=scripts/kernel-doc.py -D version=6.17.0-rc1 -D release=6.17.0-rc1+ -D kerneldoc_srctree=. /new_devel/docs/Documentation /new_devel/docs/Documentation/output
> 	...
>
>
>
>> Newer Sphinx versions have the -M option for "make mode". The Makefiles
>> produced by sphinx-quickstart only have one build target:
>> 
>> # Catch-all target: route all unknown targets to Sphinx using the new
>> # "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
>
> I didn't know about this, but from [1] it sounds it covers just two
> targets: "latexpdf" and "info".

sphinx-build -M help gives a list of 24 targets.

> The most complex scenario is still not covered: SPHINXDIRS.
>
> [1] https://www.sphinx-doc.org/en/master/man/sphinx-build.html
>
>> %: Makefile
>>         @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
>> 
>> That's all.
>
> Try doing such change on your makefile. it will break:
>
> 	- SPHINXDIRS;
> 	- V=1;
> 	- rustdoc

I know it does. That's the problem.

> and will still be dependent on variables that are passed via
> env from Kernel makefile. So, stil you can't run from command
> line. Also, if you call sphinx-build from command line:
>
> 	$ sphinx-build -j25 -b html Documentation Documentation/output
> 	...
> 	      File "<frozen os>", line 717, in __getitem__
> 	    KeyError: 'srctree'
>
> It won't work, as several parameters that are required by conf.py and by
> Sphinx extensions would be missing (the most important one is srctree, but
> there are others in the line too).
>
>> The proposed wrapper duplicates loads of code that's supposed to be
>> handled by sphinx-build directly.
>
> Once we get the wrapper, we can work to simplify it, but still I
> can't see how to get rid of it.

I just don't understand the mentality of first adding something complex,
and then working to simplify it.

Don't make it a Rube Goldberg machine in the first place.

>> Including the target/builder names.
>
> True, but this was a design decision taken lots of years ago: instead
> of:
> 	make html
>
> we're using:
>
> 	make htmldocs
>
> This series doesn't change that: either makefile or the script need
> to tho the namespace conversion.

In the above Makefile snippet that conversion would be $(@:docs=)

The clean Makefile way of checking for having Sphinx and the required
versions of Python and dependencies etc. would be a .PHONY target that
just checks, and doesn't do *anything* else. It shouldn't be part of the
sphinx-build rules.

PHONY += check-versions
check-versions:
	sphinx-pre-install --version-check

htmldocs: check-versions
	...

Or something like that.

>> Seems to me the goal should be to figure out *generic* wrappers for
>> handling parallelism, not Sphinx aware/specific.
>> 
>> 
>> BR,
>> Jani.

-- 
Jani Nikula, Intel

Re: [PATCH v4 08/19] tools/docs: sphinx-build-wrapper: add a wrapper for sphinx-build

[Mauro Carvalho Chehab]

On Thu, Sep 11, 2025 at 01:23:55PM +0300, Jani Nikula wrote:
> On Wed, 10 Sep 2025, Mauro Carvalho Chehab <mchehab+huawei@kernel.org> wrote:
> > Em Wed, 10 Sep 2025 13:46:17 +0300
> > Jani Nikula <jani.nikula@linux.intel.com> escreveu:
> >
> >> On Thu, 04 Sep 2025, Mauro Carvalho Chehab <mchehab+huawei@kernel.org> wrote:
> >> > There are too much magic inside docs Makefile to properly run
> >> > sphinx-build. Create an ancillary script that contains all
> >> > kernel-related sphinx-build call logic currently at Makefile.
> >> >
> >> > Such script is designed to work both as an standalone command
> >> > and as part of a Makefile. As such, it properly handles POSIX
> >> > jobserver used by GNU make.
> >> >
> >> > On a side note, there was a line number increase due to the
> >> > conversion:
> >> >
> >> >  Documentation/Makefile          |  131 +++----------
> >> >  tools/docs/sphinx-build-wrapper |  293 +++++++++++++++++++++++++++++++
> >> >  2 files changed, 323 insertions(+), 101 deletions(-)
> >> >
> >> > This is because some things are more verbosed on Python and because
> >> > it requires reading env vars from Makefile. Besides it, this script
> >> > has some extra features that don't exist at the Makefile:
> >> >
> >> > - It can be called directly from command line;
> >> > - It properly return PDF build errors.
> >> >
> >> > When running the script alone, it will only take handle sphinx-build
> >> > targets. On other words, it won't runn make rustdoc after building
> >> > htmlfiles, nor it will run the extra check scripts.  
> >> 
> >> I've always strongly believed we should aim to make it possible to build
> >> the documentation by running sphinx-build directly on the
> >> command-line. Not that it would be the common way to run it, but to not
> >> accumulate things in the Makefile that need to happen before or
> >> after. To promote handling the documentation build in Sphinx. To be able
> >> to debug issues and try new Sphinx versions without all the hacks.
> >
> > That would be the better, but, unfortunately, this is not possible, for 
> > several reasons:
> >
> > 1. SPHINXDIRS. It needs a lot of magic to work, both before running
> >    sphinx-build and after (inside conf.py);
> 
> Makes you wonder if that's the right solution to the original
> problem. It was added as a kind of hack, and it stuck.

The problem is, IMHO, due to the lack of flexibility of sphinx-build:
It should have a way on it to do partial documentation builds.

On our specific case, given that building docs takes so much time,
SPHINXDIRS is needed, as it allows to quickly check how a change is
output.

I use it a lot at devel time.

Also, media documentation builds depend on it:

    https://linuxtv.org/downloads/v4l-dvb-apis-new/

And probably other subsystems do the same, confining docs into
a subsystem-specific document.

> 
> > 2. Several extensions require kernel-specific environment variables to
> >    work. Calling sphinx-build directly breaks them;
> 
> The extensions shouldn't be using environment variables for
> configuration anyway. Add config options and set them in conf.py like
> everything else?

Agreed, but that's a separate problem, and should be addressed outside
this path changeset.

This one give one step on such direction by passing some parameters
via -D instead of env, but still conf.py and scripts are handling
them on a very particular way.

> > 3. Sphinx itself doesn't build several targets alone. Instead, they create
> >    a Makefile, and an extra step is needed to finish the build. That's 
> >    the case for pdf and texinfo, for instance;
> 
> That's not true for the Makefile currently generated by
> sphinx-quickstart. Granted, I haven't used Sphinx much for pdf output.

At the beginning, we were relying on the auto-generated pdf makefiles
only, but this had several issues (I can't recall them anymore). So,
we ended to a more complex proccess.

Yet, still broken, as, no matter if using sphinx quickstart way
or not, one needs to run (directly or indirectly) the produced
Makefile inside Documentation/output/.../latex to generate files,
and it will still always return non-zero, even if all PDFs are
built.

The real fix for it is outside our hands: someone needs to change
the way PDF is produced with a proper PDF builder instead of latex
and integrate at Sphinx tree.

There are some OOT docutils and/or sphinx pdf builders. We tried for
a while rst2pdf, but never worked for the Kernel source.

> 
> > 4. Man pages generation. Sphinx support to generate it is very poor;
> 
> In what way?

Have you ever tried to use it?

    "This builder produces manual pages in the groff format. 
     You have to specify which documents are to be included in 
     which manual pages via the man_pages configuration value."
    (https://www.sphinx-doc.org/en/master/usage/builders/index.html#sphinx.builders.manpage.ManualPageBuilder)

I tried when we were migrating to Sphinx. In summary:
    - Each man page requires an entry on a list;

        man_pages = [
            ('man/func', 'func', 'Func Documentation', [authors], 1),
            ...
        ]

    - Each man page requires a .rst file;
    - Each man page .rst file requires an specific format, like this:

        func(9)
        =======

        NAME
        ----
        func - ...

        SYNOPSIS
        --------
...

        DESCRIPTION
        -----------
...

      E.g. it is basically a troff source "converted" to .rst.

> > 5. Rust integration adds more complexity to the table;
> >
> > I'm not seeing sphinx-build supporting the above needs anytime soon,
> > and, even if we push our needs to Sphinx and it gets accepted there,
> > we'll still need to wait for quite a while until LTS distros merge
> > them.
> 
> I'm not suggesting to add anything to Sphinx upstream.

Without Sphinx upstream changes, I can't see how we'll get rid of
sphinx-build pre/post processing.

> >> This patch moves a bunch of that logic into a Python wrapper, and I feel
> >> like it complicates matters. You can no longer rely on 'make V=1' to get
> >> the build commands, for instance.
> >
> > Quite the opposite. if you try using "make V=1", it won't show the
> > command line used to call sphinx-build anymore.
> >
> > This series restore it.
> >
> > See, if you build with this series with V=1, you will see exactly
> > what commands are used on the build:
> >
> > 	$ make V=1 htmldocs
> > 	...
> > 	python3 ./tools/docs/sphinx-build-wrapper htmldocs \
> > 	        --sphinxdirs="." --conf="conf.py" \
> >         	--builddir="Documentation/output" \
> > 	        --theme= --css= --paper=
> > 	python3 /new_devel/docs/sphinx_latest/bin/sphinx-build -j25 -b html -c /new_devel/docs/Documentation -d /new_devel/docs/Documentation/output/.doctrees -D kerneldoc_bin=scripts/kernel-doc.py -D version=6.17.0-rc1 -D release=6.17.0-rc1+ -D kerneldoc_srctree=. /new_devel/docs/Documentation /new_devel/docs/Documentation/output
> > 	...
> >
> >
> >
> >> Newer Sphinx versions have the -M option for "make mode". The Makefiles
> >> produced by sphinx-quickstart only have one build target:
> >> 
> >> # Catch-all target: route all unknown targets to Sphinx using the new
> >> # "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
> >
> > I didn't know about this, but from [1] it sounds it covers just two
> > targets: "latexpdf" and "info".
> 
> sphinx-build -M help gives a list of 24 targets.

Ok, but those are the extra ones. Btw, I'm almost sure we tried it for
latexpdf in the early days. Didn't work well. I guess the problem as
related to returned error codes that are always causing make pdfdocs
to return errors.
 
> > The most complex scenario is still not covered: SPHINXDIRS.
> >
> > [1] https://www.sphinx-doc.org/en/master/man/sphinx-build.html
> >
> >> %: Makefile
> >>         @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
> >> 
> >> That's all.
> >
> > Try doing such change on your makefile. it will break:
> >
> > 	- SPHINXDIRS;
> > 	- V=1;
> > 	- rustdoc
> 
> I know it does. That's the problem.

The wrapper fixes it by handling the pre/post sphinx-build logic
at the right way.

See, this series doesn't change the need or the implementation logic
for pre/post sphinx-build steps(*). It just moves them to a wrapper,
while getting rid of all small hacky scripts that are needed to make
them work.

We can and should still pursue the goal of simplifying pre/post
steps.

(*) The only change is for PDF: after the series, the return code
    is now zero if all PDF files build, non-zero otherwise.

    This makes easier for humans and for CI to be sure that there 
    are nothing at the .rst files causing PDF builds to break.

> 
> > and will still be dependent on variables that are passed via
> > env from Kernel makefile. So, stil you can't run from command
> > line. Also, if you call sphinx-build from command line:
> >
> > 	$ sphinx-build -j25 -b html Documentation Documentation/output
> > 	...
> > 	      File "<frozen os>", line 717, in __getitem__
> > 	    KeyError: 'srctree'
> >
> > It won't work, as several parameters that are required by conf.py and by
> > Sphinx extensions would be missing (the most important one is srctree, but
> > there are others in the line too).
> >
> >> The proposed wrapper duplicates loads of code that's supposed to be
> >> handled by sphinx-build directly.
> >
> > Once we get the wrapper, we can work to simplify it, but still I
> > can't see how to get rid of it.
> 
> I just don't understand the mentality of first adding something complex,
> and then working to simplify it.

It doesn't make it more complex. Quite the opposite:

- it ports what we have as-is to a script;
- it drops hacky glues that were added over time on 4 different files
  to handle sphinxdirs, jobserver, parallelism, second build steps,
  man pages generation;
- it fixes the return code issue with pdf builds.

As all pre/post steps are now in a single place, it makes it easier to
maintain.

> Don't make it a Rube Goldberg machine in the first place.
> 
> >> Including the target/builder names.
> >
> > True, but this was a design decision taken lots of years ago: instead
> > of:
> > 	make html
> >
> > we're using:
> >
> > 	make htmldocs
> >
> > This series doesn't change that: either makefile or the script need
> > to tho the namespace conversion.
> 
> In the above Makefile snippet that conversion would be $(@:docs=)

The same could be done at python. Yet, I opted there to use a dict
mainly because:

- we're not consistent about where files will be stored:
  with sphinx-quickstart, html files goes into _build/html. On our
  build system, we dropped "/html";
- we have some specific rules about where the final PDF files will
  be stored;
- we need to map what builder is used, because of the second step.

So, instead of having code checking for those specifics, I opted to
place on a dict, as it makes clearer and easier to maintain.

> The clean Makefile way of checking for having Sphinx and the required
> versions of Python and dependencies etc. would be a .PHONY target that
> just checks, and doesn't do *anything* else. It shouldn't be part of the
> sphinx-build rules.
> 
> PHONY += check-versions
> check-versions:
> 	sphinx-pre-install --version-check
> 
> htmldocs: check-versions
> 	...
> 
> Or something like that.

The problem with that is that we shouldn't run sphinx-pre-install for
cleandocs or non-doc targets.

Anyway, this series doesn't touch sphinx-pre-install call. It is still
inside docs Makefile.

What we can do in the future is to convert sphinx-pre-install code into
a library and then call a check_versions() method from it at the wrapper
script, like:

    SphinxPreInstal().check_versions()     # or something equivalent

And then dropping it from the build system. Yet, this is out of the scope
of this series.
 
-- 
Thanks,
Mauro

Re: [PATCH v4 08/19] tools/docs: sphinx-build-wrapper: add a wrapper for sphinx-build

[Jonathan Corbet]

Mauro Carvalho Chehab <mchehab+huawei@kernel.org> writes:

> On Thu, Sep 11, 2025 at 01:23:55PM +0300, Jani Nikula wrote:
>> > 1. SPHINXDIRS. It needs a lot of magic to work, both before running
>> >    sphinx-build and after (inside conf.py);
>> 
>> Makes you wonder if that's the right solution to the original
>> problem. It was added as a kind of hack, and it stuck.
>
> The problem is, IMHO, due to the lack of flexibility of sphinx-build:
> It should have a way on it to do partial documentation builds.

A couple of times I have looked into using intersphinx, making each book
into an actually separate book.  The thing I always run into is that
doing a complete docs build, with working references, would require
building everything twice.  This is probably worth another attempt one
of these years...

jon

Re: [PATCH v4 08/19] tools/docs: sphinx-build-wrapper: add a wrapper for sphinx-build

[Jani Nikula]

On Thu, 11 Sep 2025, Jonathan Corbet <corbet@lwn.net> wrote:
> Mauro Carvalho Chehab <mchehab+huawei@kernel.org> writes:
>
>> On Thu, Sep 11, 2025 at 01:23:55PM +0300, Jani Nikula wrote:
>>> > 1. SPHINXDIRS. It needs a lot of magic to work, both before running
>>> >    sphinx-build and after (inside conf.py);
>>> 
>>> Makes you wonder if that's the right solution to the original
>>> problem. It was added as a kind of hack, and it stuck.
>>
>> The problem is, IMHO, due to the lack of flexibility of sphinx-build:
>> It should have a way on it to do partial documentation builds.
>
> A couple of times I have looked into using intersphinx, making each book
> into an actually separate book.  The thing I always run into is that
> doing a complete docs build, with working references, would require
> building everything twice.  This is probably worth another attempt one
> of these years...

I think the main factor in that should be whether it makes sense from
overall documentation standpoint, not the technical details.

Having several books might make sense. It might even be helpful in
organizing the documentation by audiences. But having the granularity of
SPHINXDIRS with that would be overkill. And there needs to be a book to
bring them together, and link to the other books, acting as the landing
page.

I believe it should be possible to generate the intersphinx inventory
without generating the full html or pdf documentation. So I don't think
it's actually two complete docs builds. It might speed things up to have
a number of independent documentation builds.

As to the working references, IIUC partial builds with SPHINXDIRS
doesn't get that part right if there are references outside of the
designated dirs, leading to warnings.


BR,
Jani.






-- 
Jani Nikula, Intel

Re: [PATCH v4 08/19] tools/docs: sphinx-build-wrapper: add a wrapper for sphinx-build

[Jonathan Corbet]

Jani Nikula <jani.nikula@linux.intel.com> writes:

> On Thu, 11 Sep 2025, Jonathan Corbet <corbet@lwn.net> wrote:
>> A couple of times I have looked into using intersphinx, making each book
>> into an actually separate book.  The thing I always run into is that
>> doing a complete docs build, with working references, would require
>> building everything twice.  This is probably worth another attempt one
>> of these years...
>
> I think the main factor in that should be whether it makes sense from
> overall documentation standpoint, not the technical details.
>
> Having several books might make sense. It might even be helpful in
> organizing the documentation by audiences. But having the granularity of
> SPHINXDIRS with that would be overkill. And there needs to be a book to
> bring them together, and link to the other books, acting as the landing
> page.

Well, I think that the number of existing directories needs to be
reduced rather further.  I made progress in that direction by coalescing
all the arch docs under Documentation/arch/.  I would like to do
something similar with all the device-specific docs, creating
Documentation/devices/.  Then we start to get to a reasonable number of
books.

> I believe it should be possible to generate the intersphinx inventory
> without generating the full html or pdf documentation. So I don't think
> it's actually two complete docs builds. It might speed things up to have
> a number of independent documentation builds.

That's a good point, I hadn't looked into that part.  The builder phase
takes a lot of the time, if that could be cut out things would go
faster. 

> As to the working references, IIUC partial builds with SPHINXDIRS
> doesn't get that part right if there are references outside of the
> designated dirs, leading to warnings.

That is true.  My point though is that, to get the references right with
a *full* build, a two-pass approach is needed though, as you suggest,
perhaps the first pass could be faster.

Thanks,

jon

Re: [PATCH v4 08/19] tools/docs: sphinx-build-wrapper: add a wrapper for sphinx-build

[Mauro Carvalho Chehab]

Em Thu, 11 Sep 2025 13:47:54 -0600
Jonathan Corbet <corbet@lwn.net> escreveu:

> Jani Nikula <jani.nikula@linux.intel.com> writes:
> 
> > On Thu, 11 Sep 2025, Jonathan Corbet <corbet@lwn.net> wrote:  
> >> A couple of times I have looked into using intersphinx, making each book
> >> into an actually separate book.  The thing I always run into is that
> >> doing a complete docs build, with working references, would require
> >> building everything twice.  This is probably worth another attempt one
> >> of these years...  

There are a couple of different usecase scenarios for building docs.

1) The first and most important one is to produce book(s) for people
   to use. This is usually done by some automation, and the result is
   placed on places like:
	- https://docs.kernel.org/

   and on subsystem-specific places like:
	- https://linuxtv.org/downloads/v4l-dvb-apis-new/

for scenario (1), taking twice the time to build is not an issue, as
nobody will be sitting on a chair waiting for the build to finish.

On such scenario, SPHINXDIRS is important on subsystem-specific docs.
For instance, on media, we use SPHINXDIRS to pick parts of 3 different
books:

	- Documentation/admin-guide/media/
	- Documentation/driver-api/media/
	- Documentation/userspace-api/media/

What media automation does, once per day, is:

	# Non-essencial parts of index.rst dropped
	cat <<END >Documentation/media/index.rst
	================================
	Linux Kernel Media Documentation
	================================

	.. toctree::

	        admin-guide/index
        	driver-api/index
	        userspace-api/index
	END

	rsync -uAXEHlaSx -W --inplace --delete Documentation/admin-guide/media/ Documentation/media/admin-guide
	rsync -uAXEHlaSx -W --inplace --delete Documentation/driver-api/media/ Documentation/media/driver-api
	rsync -uAXEHlaSx -W --inplace --delete Documentation/userspace-api/media/ Documentation/media/userspace-api

	make SPHINXDIRS='media' CSS='$CSS' DOCS_THEME='$DOCS_THEME' htmldocs
	make SPHINXDIRS='media' pdfdocs
	make SPHINXDIRS='media' epubdocs

2) CI tests. Here, taking more time usually is not a problem, except
   when CI is used before pushing stuff, and the developer has to wait
   it to finish before pushing.

For scenario (2), a build time increase is problematic, as, if it now
takes twice the time, a change like that will require twice the
resources for the build with may increase costs.

3) developers who touched docs. They want a way to quickly build and
   verify the output for their changes.

Here, any time increase is problematic, and SPHINXDIRS play an important 
hole by allowing them to build only the touched documents.

For instance, when I was developing Netlink yaml plugin, I had to use
dozens of times:

	make SPINXDRS=Documentation/netlink/specs/ htmldocs

If I had to build the entire documentation every time, the development
time would increase from days to weeks.

Looking on those three scenarios, the only one where intersphinx is
useful is (1).

From my PoV, we should support intersphinx, but this should be optional.
Also, one has to point from where intersphinx will point unsolved
symbols. So, we would need something like:

	make SPHINXREFMAP=intersphinx_mapping.py htmldocs

where intersphinx_mapping.py would be a file containing intersphinx
configuration. We would add a default map at Documentation/, while
letting it to be overridden if some subsystem has different requirements
or is using a different CSS tamplate or not using alabaster.

> > I think the main factor in that should be whether it makes sense from
> > overall documentation standpoint, not the technical details.

Agreed.

> > Having several books might make sense. It might even be helpful in
> > organizing the documentation by audiences. But having the granularity of
> > SPHINXDIRS with that would be overkill. 

On the contrary. SPHINXDIRS granuarity is very important for scenario (3).

> > And there needs to be a book to
> > bring them together, and link to the other books, acting as the landing
> > page.  
> 
> Well, I think that the number of existing directories needs to be
> reduced rather further.  I made progress in that direction by coalescing
> all the arch docs under Documentation/arch/.  I would like to do
> something similar with all the device-specific docs, creating
> Documentation/devices/.  Then we start to get to a reasonable number of
> books.

I don't think reducing the number of books should be the goal, but,
instead, to have them with a clear and coherent organization with focus
on the audience that will be actually using them.

After reorg, we may have less books. That's fine. But it is also fine
if we end with more books.

I lost the battle years ago, but I still believe that, at least for
some subsystems like media, i2c, DRM, security and others, a 
subsystem-specific book could be better. After all, the audience for
such subsystems is very specialized.

> > I believe it should be possible to generate the intersphinx inventory
> > without generating the full html or pdf documentation. So I don't think
> > it's actually two complete docs builds. It might speed things up to have
> > a number of independent documentation builds.  
> 
> That's a good point, I hadn't looked into that part.  The builder phase
> takes a lot of the time, if that could be cut out things would go
> faster. 

Indeed, but we need to double check if .doctree cache expiration will
happen the right way for all books affected by a partial build.

During this merge window, I sent a RFC patch in the middle of a comment
with a conf.py logic to detect Sphinx cache expiration. I remember I
added a comment asking if we should upstream it or not, but, as nobody
answered, I ended forgetting about it.

If we're willing to experiment with that, I recommend looking on such
patch and add a variant of it, enabled via V=1 or via some debug
parameter.

The goal would be to check if a change on a file will ensure that all
books using it will have cache expiration and be rebuilt.

> > As to the working references, IIUC partial builds with SPHINXDIRS
> > doesn't get that part right if there are references outside of the
> > designated dirs, leading to warnings.  
> 
> That is true.  My point though is that, to get the references right with
> a *full* build, a two-pass approach is needed though, as you suggest,
> perhaps the first pass could be faster.

How fast? during development time, SPHINXDIRS means a couple of seconds:

	$ make clean; time make SPHINXDIRS="peci" htmldocs
	...
	real    0m1,373s
	user    0m1,348s

Even more complex builds, even when picking more than one book, like this:

	$ make clean; time make SPHINXDIRS="driver-api/media/ userspace-api/media/" htmldocs
	...
	real    0m11,801s
	user    0m31,381s
	sys     0m6,880s

it still fits at the seconds range. Can interphinx first pass have a
similar build time?

Thanks,
Mauro

Re: [PATCH v4 08/19] tools/docs: sphinx-build-wrapper: add a wrapper for sphinx-build

[Jani Nikula]

On Fri, 12 Sep 2025, Mauro Carvalho Chehab <mchehab+huawei@kernel.org> wrote:
> Em Thu, 11 Sep 2025 13:47:54 -0600
> Jonathan Corbet <corbet@lwn.net> escreveu:
>
>> Jani Nikula <jani.nikula@linux.intel.com> writes:
>> 
>> > On Thu, 11 Sep 2025, Jonathan Corbet <corbet@lwn.net> wrote:  
>> >> A couple of times I have looked into using intersphinx, making each book
>> >> into an actually separate book.  The thing I always run into is that
>> >> doing a complete docs build, with working references, would require
>> >> building everything twice.  This is probably worth another attempt one
>> >> of these years...  
>
> There are a couple of different usecase scenarios for building docs.
>
> 1) The first and most important one is to produce book(s) for people
>    to use. This is usually done by some automation, and the result is
>    placed on places like:
> 	- https://docs.kernel.org/
>
>    and on subsystem-specific places like:
> 	- https://linuxtv.org/downloads/v4l-dvb-apis-new/
>
> for scenario (1), taking twice the time to build is not an issue, as
> nobody will be sitting on a chair waiting for the build to finish.
>
> On such scenario, SPHINXDIRS is important on subsystem-specific docs.
> For instance, on media, we use SPHINXDIRS to pick parts of 3 different
> books:
>
> 	- Documentation/admin-guide/media/
> 	- Documentation/driver-api/media/
> 	- Documentation/userspace-api/media/
>
> What media automation does, once per day, is:
>
> 	# Non-essencial parts of index.rst dropped
> 	cat <<END >Documentation/media/index.rst
> 	================================
> 	Linux Kernel Media Documentation
> 	================================
>
> 	.. toctree::
>
> 	        admin-guide/index
>         	driver-api/index
> 	        userspace-api/index
> 	END
>
> 	rsync -uAXEHlaSx -W --inplace --delete Documentation/admin-guide/media/ Documentation/media/admin-guide
> 	rsync -uAXEHlaSx -W --inplace --delete Documentation/driver-api/media/ Documentation/media/driver-api
> 	rsync -uAXEHlaSx -W --inplace --delete Documentation/userspace-api/media/ Documentation/media/userspace-api
>
> 	make SPHINXDIRS='media' CSS='$CSS' DOCS_THEME='$DOCS_THEME' htmldocs
> 	make SPHINXDIRS='media' pdfdocs
> 	make SPHINXDIRS='media' epubdocs

I was actually wondering how [1] was built. So it's not a complete build
of anything upstream, but rather something cobbled together downstream.

So your scenario (1) above is actually *two* wildly different scenarios.

And if upstream needs to cater for pretty much random subsets of
documentation being built, cherry-picking documentation from here and
there, I don't know what hope there is in radically refactoring how
documentation gets built upstream.

I presume you have one or more of a) get bunch of broken link warnings
at build, b) get broken links in the output, c) avoid links outside of
your subset altogether.

[1] https://linuxtv.org/downloads/v4l-dvb-apis-new/

> 2) CI tests. Here, taking more time usually is not a problem, except
>    when CI is used before pushing stuff, and the developer has to wait
>    it to finish before pushing.
>
> For scenario (2), a build time increase is problematic, as, if it now
> takes twice the time, a change like that will require twice the
> resources for the build with may increase costs.
>
> 3) developers who touched docs. They want a way to quickly build and
>    verify the output for their changes.
>
> Here, any time increase is problematic, and SPHINXDIRS play an important 
> hole by allowing them to build only the touched documents.

This is actually problematic, because the SPHINXDIRS partial builds will
give you warnings for unresolved references that are just fine if the
entire documentation gets built.

> For instance, when I was developing Netlink yaml plugin, I had to use
> dozens of times:
>
> 	make SPINXDRS=Documentation/netlink/specs/ htmldocs
>
> If I had to build the entire documentation every time, the development
> time would increase from days to weeks.
>
> Looking on those three scenarios, the only one where intersphinx is
> useful is (1).

It's also helpful for 3, and it could be helpful for 2 if CI only checks
some parts of the documentation.

> From my PoV, we should support intersphinx, but this should be optional.

Per my understanding making this somehow optional is not easily
achieved. And you end up with a bunch of extra complexity.

> Also, one has to point from where intersphinx will point unsolved
> symbols. So, we would need something like:
>
> 	make SPHINXREFMAP=intersphinx_mapping.py htmldocs
>
> where intersphinx_mapping.py would be a file containing intersphinx
> configuration. We would add a default map at Documentation/, while
> letting it to be overridden if some subsystem has different requirements
> or is using a different CSS tamplate or not using alabaster.
>
>> > I think the main factor in that should be whether it makes sense from
>> > overall documentation standpoint, not the technical details.
>
> Agreed.
>
>> > Having several books might make sense. It might even be helpful in
>> > organizing the documentation by audiences. But having the granularity of
>> > SPHINXDIRS with that would be overkill. 
>
> On the contrary. SPHINXDIRS granuarity is very important for scenario (3).

Sphinx does support incremental builds, and it's only the very first
build that's slow. IMO a handful of books that you can actually build
without warnings (unlike SPHINXDIRS) with incremental builds is a good
compromise.

>> > And there needs to be a book to
>> > bring them together, and link to the other books, acting as the landing
>> > page.  
>> 
>> Well, I think that the number of existing directories needs to be
>> reduced rather further.  I made progress in that direction by coalescing
>> all the arch docs under Documentation/arch/.  I would like to do
>> something similar with all the device-specific docs, creating
>> Documentation/devices/.  Then we start to get to a reasonable number of
>> books.
>
> I don't think reducing the number of books should be the goal, but,
> instead, to have them with a clear and coherent organization with focus
> on the audience that will be actually using them.
>
> After reorg, we may have less books. That's fine. But it is also fine
> if we end with more books.
>
> I lost the battle years ago, but I still believe that, at least for
> some subsystems like media, i2c, DRM, security and others, a 
> subsystem-specific book could be better. After all, the audience for
> such subsystems is very specialized.
>
>> > I believe it should be possible to generate the intersphinx inventory
>> > without generating the full html or pdf documentation. So I don't think
>> > it's actually two complete docs builds. It might speed things up to have
>> > a number of independent documentation builds.  
>> 
>> That's a good point, I hadn't looked into that part.  The builder phase
>> takes a lot of the time, if that could be cut out things would go
>> faster. 
>
> Indeed, but we need to double check if .doctree cache expiration will
> happen the right way for all books affected by a partial build.
>
> During this merge window, I sent a RFC patch in the middle of a comment
> with a conf.py logic to detect Sphinx cache expiration. I remember I
> added a comment asking if we should upstream it or not, but, as nobody
> answered, I ended forgetting about it.
>
> If we're willing to experiment with that, I recommend looking on such
> patch and add a variant of it, enabled via V=1 or via some debug
> parameter.
>
> The goal would be to check if a change on a file will ensure that all
> books using it will have cache expiration and be rebuilt.
>
>> > As to the working references, IIUC partial builds with SPHINXDIRS
>> > doesn't get that part right if there are references outside of the
>> > designated dirs, leading to warnings.  
>> 
>> That is true.  My point though is that, to get the references right with
>> a *full* build, a two-pass approach is needed though, as you suggest,
>> perhaps the first pass could be faster.
>
> How fast? during development time, SPHINXDIRS means a couple of seconds:
>
> 	$ make clean; time make SPHINXDIRS="peci" htmldocs
> 	...
> 	real    0m1,373s
> 	user    0m1,348s
>
> Even more complex builds, even when picking more than one book, like this:
>
> 	$ make clean; time make SPHINXDIRS="driver-api/media/ userspace-api/media/" htmldocs
> 	...
> 	real    0m11,801s
> 	user    0m31,381s
> 	sys     0m6,880s
>
> it still fits at the seconds range. Can interphinx first pass have a
> similar build time?

Probably not. Can you add links from media to non-media documentation
without warnings? Probably not also.


BR,
Jani.


-- 
Jani Nikula, Intel

Re: [PATCH v4 08/19] tools/docs: sphinx-build-wrapper: add a wrapper for sphinx-build

[Vegard Nossum]

On 12/09/2025 12:16, Jani Nikula wrote:
>> Here, any time increase is problematic, and SPHINXDIRS play an important
>> hole by allowing them to build only the touched documents.
> This is actually problematic, because the SPHINXDIRS partial builds will
> give you warnings for unresolved references that are just fine if the
> entire documentation gets built.

I admit I don't have a full overview of all the problems that are being
solved here (in existing and proposed code), but how hard would it be to
convert the whole SPHINXDIRS thing into a Sphinx plugin that runs early
and discards documents outside of what the user wants to build? By
"discards" I mean in some useful way that reduces runtime compared to a
full build while retaining some benefits of a full build (reference
checking)?


Vegard

Re: [PATCH v4 08/19] tools/docs: sphinx-build-wrapper: add a wrapper for sphinx-build

[Mauro Carvalho Chehab]

Em Fri, 12 Sep 2025 13:34:17 +0200
Vegard Nossum <vegard.nossum@oracle.com> escreveu:

> On 12/09/2025 12:16, Jani Nikula wrote:
> >> Here, any time increase is problematic, and SPHINXDIRS play an important
> >> hole by allowing them to build only the touched documents.  
> > This is actually problematic, because the SPHINXDIRS partial builds will
> > give you warnings for unresolved references that are just fine if the
> > entire documentation gets built.  
> 
> I admit I don't have a full overview of all the problems that are being
> solved here (in existing and proposed code), but how hard would it be to
> convert the whole SPHINXDIRS thing into a Sphinx plugin that runs early
> and discards documents outside of what the user wants to build? By
> "discards" I mean in some useful way that reduces runtime compared to a
> full build while retaining some benefits of a full build (reference
> checking)?

That's not a bad idea, but I guess it is not too easy to implement - at
least inside a Sphinx plugin.

The good news is that conf.py has already a logic to ignore patterns that
could be tweaked and/or placed on a plugin.

The bad news is that existing index.rst files will now reference
non-existing docs. No idea how to "process" them to filter out
such docs. It is probably doable. See, SPHINXDIRS supports pinpointing
any directory, not just Documentation child directories. So, eventually,
such plugin would also need to "fake" the main index.rst. Now, the
question is, if we pick for instance:

	SPHINXDIRS="netlink/spec networking"

What would be the main title that would be at the main index.rst?
I suspect that, for such cases, the title of the index would need
to be manually set at the command line interface.

Another aspect is that SPHINXDIRS affect latex document lists, 
which can be problematic.


Thanks,
Mauro

Re: [PATCH v4 08/19] tools/docs: sphinx-build-wrapper: add a wrapper for sphinx-build

[Mauro Carvalho Chehab]

Em Fri, 12 Sep 2025 13:16:12 +0300
Jani Nikula <jani.nikula@linux.intel.com> escreveu:

> On Fri, 12 Sep 2025, Mauro Carvalho Chehab <mchehab+huawei@kernel.org> wrote:
> > Em Thu, 11 Sep 2025 13:47:54 -0600
> > Jonathan Corbet <corbet@lwn.net> escreveu:
> >  
> >> Jani Nikula <jani.nikula@linux.intel.com> writes:
> >>   
> >> > On Thu, 11 Sep 2025, Jonathan Corbet <corbet@lwn.net> wrote:    
> >> >> A couple of times I have looked into using intersphinx, making each book
> >> >> into an actually separate book.  The thing I always run into is that
> >> >> doing a complete docs build, with working references, would require
> >> >> building everything twice.  This is probably worth another attempt one
> >> >> of these years...    
> >
> > There are a couple of different usecase scenarios for building docs.
> >
> > 1) The first and most important one is to produce book(s) for people
> >    to use. This is usually done by some automation, and the result is
> >    placed on places like:
> > 	- https://docs.kernel.org/
> >
> >    and on subsystem-specific places like:
> > 	- https://linuxtv.org/downloads/v4l-dvb-apis-new/
> >
> > for scenario (1), taking twice the time to build is not an issue, as
> > nobody will be sitting on a chair waiting for the build to finish.
> >
> > On such scenario, SPHINXDIRS is important on subsystem-specific docs.
> > For instance, on media, we use SPHINXDIRS to pick parts of 3 different
> > books:
> >
> > 	- Documentation/admin-guide/media/
> > 	- Documentation/driver-api/media/
> > 	- Documentation/userspace-api/media/
> >
> > What media automation does, once per day, is:
> >
> > 	# Non-essencial parts of index.rst dropped
> > 	cat <<END >Documentation/media/index.rst
> > 	================================
> > 	Linux Kernel Media Documentation
> > 	================================
> >
> > 	.. toctree::
> >
> > 	        admin-guide/index
> >         	driver-api/index
> > 	        userspace-api/index
> > 	END
> >
> > 	rsync -uAXEHlaSx -W --inplace --delete Documentation/admin-guide/media/ Documentation/media/admin-guide
> > 	rsync -uAXEHlaSx -W --inplace --delete Documentation/driver-api/media/ Documentation/media/driver-api
> > 	rsync -uAXEHlaSx -W --inplace --delete Documentation/userspace-api/media/ Documentation/media/userspace-api
> >
> > 	make SPHINXDIRS='media' CSS='$CSS' DOCS_THEME='$DOCS_THEME' htmldocs
> > 	make SPHINXDIRS='media' pdfdocs
> > 	make SPHINXDIRS='media' epubdocs  
> 
> I was actually wondering how [1] was built. So it's not a complete build
> of anything upstream, but rather something cobbled together downstream.

It used to be a direct build from upstream. I had to do this hack when
we decided to split subsystem docs on 3 separate books.

> So your scenario (1) above is actually *two* wildly different scenarios.
> 
> And if upstream needs to cater for pretty much random subsets of
> documentation being built, cherry-picking documentation from here and
> there, I don't know what hope there is in radically refactoring how
> documentation gets built upstream.
> 
> I presume you have one or more of a) get bunch of broken link warnings
> at build, b) get broken links in the output, c) avoid links outside of
> your subset altogether.

There aren't many broken links, and this is not due to (c): almost all
cross-references we have are between media kAPI and media uAPI. Those
were solved when we artificially joined two books and used SPHINXDIRS
feature to produce the docs.

If we had intersphinx support, I would be building the docs in
separate using the standard SPHINXDIRS logic to create such
cross references, pointing to linuxtv.org for media docs and to
docs.kernel.org for other ones.

> 
> [1] https://linuxtv.org/downloads/v4l-dvb-apis-new/
> 
> > 2) CI tests. Here, taking more time usually is not a problem, except
> >    when CI is used before pushing stuff, and the developer has to wait
> >    it to finish before pushing.
> >
> > For scenario (2), a build time increase is problematic, as, if it now
> > takes twice the time, a change like that will require twice the
> > resources for the build with may increase costs.
> >
> > 3) developers who touched docs. They want a way to quickly build and
> >    verify the output for their changes.
> >
> > Here, any time increase is problematic, and SPHINXDIRS play an important 
> > hole by allowing them to build only the touched documents.  
> 
> This is actually problematic, because the SPHINXDIRS partial builds will
> give you warnings for unresolved references that are just fine if the
> entire documentation gets built.

True, but if you pick them before/after a chanseset, you can notice
if the warning was introduced or not by the changeset. Only if it was
introduced by the patchset you need to wait 3 minutes for the full build.

> > For instance, when I was developing Netlink yaml plugin, I had to use
> > dozens of times:
> >
> > 	make SPINXDRS=Documentation/netlink/specs/ htmldocs
> >
> > If I had to build the entire documentation every time, the development
> > time would increase from days to weeks.
> >
> > Looking on those three scenarios, the only one where intersphinx is
> > useful is (1).  
> 
> It's also helpful for 3, and it could be helpful for 2 if CI only checks
> some parts of the documentation.

I'm not arguing against intersphinx. I do think having it is something
we need to aim for.

The question is: does it replace SPHINXDIRS by providing quick builds
if only some of the books were changed?

> > From my PoV, we should support intersphinx, but this should be optional.  
> 
> Per my understanding making this somehow optional is not easily
> achieved. And you end up with a bunch of extra complexity.

True, but I guess extra complexity is unavoidable: intersphinx
requires a list of books with reference locations, with is not the
same for everyone.

This is what expect once we have intersphinx in place:

Use linuxtv.org URLs for all references from:
	- Documentation/admin-guide/media/
	- Documentation/driver-api/media/
	- Documentation/userspace-api/media/

everything else: from kernel.org.

As they were generated from media next branch.

If implement it for DRM, in a way to track what DRM next branches
have, and if you have kapi, uapi and per-driver apis on different
books, you will probably want to solve intersphinx dependencies with 
a FDO specific "search" order, like:

	- xe and i915 books: from intel next branches;
	- amd books: from amd next branches;
	- drm core: from drm-next;
	- everything else: from kernel.org.

So, it is not just making it optional: you also need to provide a
way to allow it to be adjusted were it is needed. IMO, the easiest
way would be to have a separate .py file with intersphinx specifics:

	make SPHINXREFMAP=intersphinx_mapping.py htmldocs

This way, I could create a media_mapping.py file that would include
intersphinx_mapping.py and replace some defaults to do my own mapping.

> > Also, one has to point from where intersphinx will point unsolved
> > symbols. So, we would need something like:
> >
> > 	make SPHINXREFMAP=intersphinx_mapping.py htmldocs
> >
> > where intersphinx_mapping.py would be a file containing intersphinx
> > configuration. We would add a default map at Documentation/, while
> > letting it to be overridden if some subsystem has different requirements
> > or is using a different CSS tamplate or not using alabaster.
> >  
> >> > I think the main factor in that should be whether it makes sense from
> >> > overall documentation standpoint, not the technical details.  
> >
> > Agreed.
> >  
> >> > Having several books might make sense. It might even be helpful in
> >> > organizing the documentation by audiences. But having the granularity of
> >> > SPHINXDIRS with that would be overkill.   
> >
> > On the contrary. SPHINXDIRS granuarity is very important for scenario (3).  
> 
> Sphinx does support incremental builds, and it's only the very first
> build that's slow. IMO a handful of books that you can actually build
> without warnings (unlike SPHINXDIRS) with incremental builds is a good
> compromise.

That's not quite true: when Sphinx detects a missing file, it expires
the caches related to it and don't do incremental builds anymore. I had
to write a patch during the last development cycle due to that, as -rc1
came up with a broken reference because of a file rename. This was only
solved 3 months after the fact.

> >> > And there needs to be a book to
> >> > bring them together, and link to the other books, acting as the landing
> >> > page.    
> >> 
> >> Well, I think that the number of existing directories needs to be
> >> reduced rather further.  I made progress in that direction by coalescing
> >> all the arch docs under Documentation/arch/.  I would like to do
> >> something similar with all the device-specific docs, creating
> >> Documentation/devices/.  Then we start to get to a reasonable number of
> >> books.  
> >
> > I don't think reducing the number of books should be the goal, but,
> > instead, to have them with a clear and coherent organization with focus
> > on the audience that will be actually using them.
> >
> > After reorg, we may have less books. That's fine. But it is also fine
> > if we end with more books.
> >
> > I lost the battle years ago, but I still believe that, at least for
> > some subsystems like media, i2c, DRM, security and others, a 
> > subsystem-specific book could be better. After all, the audience for
> > such subsystems is very specialized.
> >  
> >> > I believe it should be possible to generate the intersphinx inventory
> >> > without generating the full html or pdf documentation. So I don't think
> >> > it's actually two complete docs builds. It might speed things up to have
> >> > a number of independent documentation builds.    
> >> 
> >> That's a good point, I hadn't looked into that part.  The builder phase
> >> takes a lot of the time, if that could be cut out things would go
> >> faster.   
> >
> > Indeed, but we need to double check if .doctree cache expiration will
> > happen the right way for all books affected by a partial build.
> >
> > During this merge window, I sent a RFC patch in the middle of a comment
> > with a conf.py logic to detect Sphinx cache expiration. I remember I
> > added a comment asking if we should upstream it or not, but, as nobody
> > answered, I ended forgetting about it.
> >
> > If we're willing to experiment with that, I recommend looking on such
> > patch and add a variant of it, enabled via V=1 or via some debug
> > parameter.
> >
> > The goal would be to check if a change on a file will ensure that all
> > books using it will have cache expiration and be rebuilt.
> >  
> >> > As to the working references, IIUC partial builds with SPHINXDIRS
> >> > doesn't get that part right if there are references outside of the
> >> > designated dirs, leading to warnings.    
> >> 
> >> That is true.  My point though is that, to get the references right with
> >> a *full* build, a two-pass approach is needed though, as you suggest,
> >> perhaps the first pass could be faster.  
> >
> > How fast? during development time, SPHINXDIRS means a couple of seconds:
> >
> > 	$ make clean; time make SPHINXDIRS="peci" htmldocs
> > 	...
> > 	real    0m1,373s
> > 	user    0m1,348s
> >
> > Even more complex builds, even when picking more than one book, like this:
> >
> > 	$ make clean; time make SPHINXDIRS="driver-api/media/ userspace-api/media/" htmldocs
> > 	...
> > 	real    0m11,801s
> > 	user    0m31,381s
> > 	sys     0m6,880s
> >
> > it still fits at the seconds range. Can interphinx first pass have a
> > similar build time?  
> 
> Probably not. Can you add links from media to non-media documentation
> without warnings? Probably not also.

No, but I can count on my fingers the number of times I made such
changes: 99.9% of the time, doc changes aren't on the few docs that
have subsystem interdependencies. Even the number of dependencies
between media kapi and uapi are not high.

Thanks,
Mauro

Re: [PATCH v4 08/19] tools/docs: sphinx-build-wrapper: add a wrapper for sphinx-build

[Mauro Carvalho Chehab]

Em Thu, 11 Sep 2025 07:38:56 -0600
Jonathan Corbet <corbet@lwn.net> escreveu:

> Mauro Carvalho Chehab <mchehab+huawei@kernel.org> writes:
> 
> > On Thu, Sep 11, 2025 at 01:23:55PM +0300, Jani Nikula wrote:  
> >> > 1. SPHINXDIRS. It needs a lot of magic to work, both before running
> >> >    sphinx-build and after (inside conf.py);  
> >> 
> >> Makes you wonder if that's the right solution to the original
> >> problem. It was added as a kind of hack, and it stuck.  
> >
> > The problem is, IMHO, due to the lack of flexibility of sphinx-build:
> > It should have a way on it to do partial documentation builds.  
> 
> A couple of times I have looked into using intersphinx, making each book
> into an actually separate book.  The thing I always run into is that
> doing a complete docs build, with working references, would require
> building everything twice.  This is probably worth another attempt one
> of these years...

The big advantage of intersphinx is for PDF and LaTeX output, as
this is the only way to have cross-references there.

It is also good for subsystem-specific books (or "sub-"books) like:

	- Documentation/admin-guide/media/
	- Documentation/driver-api/media/
	- Documentation/userspace-api/media/

Right now, we create a single book with all those tree, but I would
prefer to build each of them as separate units, as they are for separated
audiences, but only if cross-references will be solved in a way that
html and pdf docs will point to the other books stored at linuxtv.org.

For html, this won't be any different, in practice, from what we have,
but for PDF and ePub, this would mean smaller books.

Thanks,
Mauro

[PATCH v4 09/19] tools/docs: sphinx-build-wrapper: add comments and blank lines

[Mauro Carvalho Chehab]

To help seing the actual size of the script when it was added,
I opted to strip out all comments from the original script.

Add them here:

 tools/docs/sphinx-build-wrapper | 261 +++++++++++++++++++++++++++++++-
 1 file changed, 257 insertions(+), 4 deletion(-)

As the code from the script has 288 lines of code, it means that
about half of the script are comments.

Also ensure pylint won't report any warnings.

No functional changes.

Signed-off-by: Mauro Carvalho Chehab <mchehab+huawei@kernel.org>
---
 tools/docs/sphinx-build-wrapper | 262 +++++++++++++++++++++++++++++++-
 1 file changed, 258 insertions(+), 4 deletions(-)

diff --git a/tools/docs/sphinx-build-wrapper b/tools/docs/sphinx-build-wrapper
index 3256418d8dc5..ea9f8e17b0bc 100755
--- a/tools/docs/sphinx-build-wrapper
+++ b/tools/docs/sphinx-build-wrapper
@@ -1,21 +1,71 @@
 #!/usr/bin/env python3
 # SPDX-License-Identifier: GPL-2.0
+# Copyright (C) 2025 Mauro Carvalho Chehab <mchehab+huawei@kernel.org>
+#
+# pylint: disable=R0902, R0912, R0913, R0914, R0915, R0917, C0103
+#
+# Converted from docs Makefile and parallel-wrapper.sh, both under
+# GPLv2, copyrighted since 2008 by the following authors:
+#
+#    Akira Yokosawa <akiyks@gmail.com>
+#    Arnd Bergmann <arnd@arndb.de>
+#    Breno Leitao <leitao@debian.org>
+#    Carlos Bilbao <carlos.bilbao@amd.com>
+#    Dave Young <dyoung@redhat.com>
+#    Donald Hunter <donald.hunter@gmail.com>
+#    Geert Uytterhoeven <geert+renesas@glider.be>
+#    Jani Nikula <jani.nikula@intel.com>
+#    Jan Stancek <jstancek@redhat.com>
+#    Jonathan Corbet <corbet@lwn.net>
+#    Joshua Clayton <stillcompiling@gmail.com>
+#    Kees Cook <keescook@chromium.org>
+#    Linus Torvalds <torvalds@linux-foundation.org>
+#    Magnus Damm <damm+renesas@opensource.se>
+#    Masahiro Yamada <masahiroy@kernel.org>
+#    Mauro Carvalho Chehab <mchehab+huawei@kernel.org>
+#    Maxim Cournoyer <maxim.cournoyer@gmail.com>
+#    Peter Foley <pefoley2@pefoley.com>
+#    Randy Dunlap <rdunlap@infradead.org>
+#    Rob Herring <robh@kernel.org>
+#    Shuah Khan <shuahkh@osg.samsung.com>
+#    Thorsten Blum <thorsten.blum@toblux.com>
+#    Tomas Winkler <tomas.winkler@intel.com>
+
+
+"""
+Sphinx build wrapper that handles Kernel-specific business rules:
+
+- it gets the Kernel build environment vars;
+- it determines what's the best parallelism;
+- it handles SPHINXDIRS
+
+This tool ensures that MIN_PYTHON_VERSION is satisfied. If version is
+below that, it seeks for a new Python version. If found, it re-runs using
+the newer version.
+"""
+
 import argparse
 import os
 import shlex
 import shutil
 import subprocess
 import sys
+
 from lib.python_version import PythonVersion
 
 LIB_DIR = "../../scripts/lib"
 SRC_DIR = os.path.dirname(os.path.realpath(__file__))
+
 sys.path.insert(0, os.path.join(SRC_DIR, LIB_DIR))
 
-from jobserver import JobserverExec
+from jobserver import JobserverExec         # pylint: disable=C0413,C0411,E0401
 
+#
+#  Some constants
+#
 MIN_PYTHON_VERSION = PythonVersion("3.7").version
 PAPER = ["", "a4", "letter"]
+
 TARGETS = {
     "cleandocs":     { "builder": "clean" },
     "linkcheckdocs": { "builder": "linkcheck" },
@@ -28,8 +78,19 @@ TARGETS = {
     "xmldocs":       { "builder": "xml",     "out_dir": "xml" },
 }
 
+
+#
+# SphinxBuilder class
+#
+
 class SphinxBuilder:
+    """
+    Handles a sphinx-build target, adding needed arguments to build
+    with the Kernel.
+    """
+
     def is_rust_enabled(self):
+        """Check if rust is enabled at .config"""
         config_path = os.path.join(self.srctree, ".config")
         if os.path.isfile(config_path):
             with open(config_path, "r", encoding="utf-8") as f:
@@ -37,41 +98,88 @@ class SphinxBuilder:
         return False
 
     def get_path(self, path, use_cwd=False, abs_path=False):
+        """
+        Ancillary routine to handle patches the right way, as shell does.
+
+        It first expands "~" and "~user". Then, if patch is not absolute,
+        join self.srctree. Finally, if requested, convert to abspath.
+        """
+
         path = os.path.expanduser(path)
         if not path.startswith("/"):
             if use_cwd:
                 base = os.getcwd()
             else:
                 base = self.srctree
+
             path = os.path.join(base, path)
+
         if abs_path:
             return os.path.abspath(path)
+
         return path
 
     def __init__(self, builddir, verbose=False, n_jobs=None):
+        """Initialize internal variables"""
         self.verbose = None
+
+        #
+        # Normal variables passed from Kernel's makefile
+        #
         self.kernelversion = os.environ.get("KERNELVERSION", "unknown")
         self.kernelrelease = os.environ.get("KERNELRELEASE", "unknown")
         self.pdflatex = os.environ.get("PDFLATEX", "xelatex")
         self.latexopts = os.environ.get("LATEXOPTS", "-interaction=batchmode -no-shell-escape")
+
         if not verbose:
             verbose = bool(os.environ.get("KBUILD_VERBOSE", "") != "")
+
         if verbose is not None:
             self.verbose = verbose
+
+        #
+        # As we handle number of jobs and quiet in separate, we need to pick
+        # both the same way as sphinx-build would pick, optionally accepts
+        # whitespaces or not. So let's use argparse to handle argument expansion
+        #
         parser = argparse.ArgumentParser()
         parser.add_argument('-j', '--jobs', type=int)
         parser.add_argument('-q', '--quiet', type=int)
+
+        #
+        # Other sphinx-build arguments go as-is, so place them
+        # at self.sphinxopts, using shell parser
+        #
         sphinxopts = shlex.split(os.environ.get("SPHINXOPTS", ""))
+
+        #
+        # Build a list of sphinx args
+        #
         sphinx_args, self.sphinxopts = parser.parse_known_args(sphinxopts)
         if sphinx_args.quiet is True:
             self.verbose = False
+
         if sphinx_args.jobs:
             self.n_jobs = sphinx_args.jobs
+
+        #
+        # If the command line argument "-j" is used override SPHINXOPTS
+        #
+
         self.n_jobs = n_jobs
+
+        #
+        # Source tree directory. This needs to be at os.environ, as
+        # Sphinx extensions use it
+        #
         self.srctree = os.environ.get("srctree")
         if not self.srctree:
             self.srctree = "."
             os.environ["srctree"] = self.srctree
+
+        #
+        # Now that we can expand srctree, get other directories as well
+        #
         self.sphinxbuild = os.environ.get("SPHINXBUILD", "sphinx-build")
         self.kerneldoc = self.get_path(os.environ.get("KERNELDOC",
                                                       "scripts/kernel-doc.py"))
@@ -79,20 +187,36 @@ class SphinxBuilder:
 
         self.config_rust = self.is_rust_enabled()
 
+        #
+        # Get directory locations for LaTeX build toolchain
+        #
         self.pdflatex_cmd = shutil.which(self.pdflatex)
         self.latexmk_cmd = shutil.which("latexmk")
 
         self.env = os.environ.copy()
 
     def run_sphinx(self, sphinx_build, build_args, *args, **pwargs):
+        """
+        Executes sphinx-build using current python3 command and setting
+        -j parameter if possible to run the build in parallel.
+        """
+
         with JobserverExec() as jobserver:
             if jobserver.claim:
                 n_jobs = str(jobserver.claim)
             else:
                 n_jobs = "auto" # Supported since Sphinx 1.7
+
             cmd = []
+
             cmd.append(sys.executable)
+
             cmd.append(sphinx_build)
+
+            #
+            # Override auto setting, if explicitly passed from command line
+            # or via SPHINXOPTS
+            #
             if self.n_jobs:
                 n_jobs = str(self.n_jobs)
 
@@ -101,59 +225,100 @@ class SphinxBuilder:
 
             if not self.verbose:
                 cmd.append("-q")
+
             cmd += self.sphinxopts
             cmd += build_args
+
             if self.verbose:
                 print(" ".join(cmd))
             return subprocess.call(cmd, *args, **pwargs)
 
     def handle_html(self, css, output_dir):
+        """
+        Extra steps for HTML and epub output.
+
+        For such targets, we need to ensure that CSS will be properly
+        copied to the output _static directory
+        """
+
         if not css:
             return
+
         css = os.path.expanduser(css)
         if not css.startswith("/"):
             css = os.path.join(self.srctree, css)
+
         static_dir = os.path.join(output_dir, "_static")
         os.makedirs(static_dir, exist_ok=True)
+
         try:
             shutil.copy2(css, static_dir)
         except (OSError, IOError) as e:
             print(f"Warning: Failed to copy CSS: {e}", file=sys.stderr)
 
     def handle_pdf(self, output_dirs):
+        """
+        Extra steps for PDF output.
+
+        As PDF is handled via a LaTeX output, after building the .tex file,
+        a new build is needed to create the PDF output from the latex
+        directory.
+        """
         builds = {}
         max_len = 0
+
         for from_dir in output_dirs:
             pdf_dir = os.path.join(from_dir, "../pdf")
             os.makedirs(pdf_dir, exist_ok=True)
+
             if self.latexmk_cmd:
                 latex_cmd = [self.latexmk_cmd, f"-{self.pdflatex}"]
             else:
                 latex_cmd = [self.pdflatex]
+
             latex_cmd.extend(shlex.split(self.latexopts))
+
             tex_suffix = ".tex"
+
+            #
+            # Process each .tex file
+            #
+
             has_tex = False
             build_failed = False
             with os.scandir(from_dir) as it:
                 for entry in it:
                     if not entry.name.endswith(tex_suffix):
                         continue
+
                     name = entry.name[:-len(tex_suffix)]
                     has_tex = True
+
+                    #
+                    # LaTeX PDF error code is almost useless for us:
+                    # any warning makes it non-zero. For kernel doc builds it
+                    # always return non-zero even when build succeeds.
+                    # So, let's do the best next thing: check if all PDF
+                    # files were built. If they're, print a summary and
+                    # return 0 at the end of this function
+                    #
                     try:
                         subprocess.run(latex_cmd + [entry.path],
                                        cwd=from_dir, check=True)
                     except subprocess.CalledProcessError:
                         pass
+
                     pdf_name = name + ".pdf"
                     pdf_from = os.path.join(from_dir, pdf_name)
                     pdf_to = os.path.join(pdf_dir, pdf_name)
+
                     if os.path.exists(pdf_from):
                         os.rename(pdf_from, pdf_to)
                         builds[name] = os.path.relpath(pdf_to, self.builddir)
                     else:
                         builds[name] = "FAILED"
                         build_failed = True
+
                     name = entry.name.removesuffix(".tex")
                     max_len = max(max_len, len(name))
 
@@ -162,58 +327,100 @@ class SphinxBuilder:
                 max_len = max(max_len, len(name))
                 builds[name] = "FAILED (no .tex)"
                 build_failed = True
+
         msg = "Summary"
         msg += "\n" + "=" * len(msg)
         print()
         print(msg)
+
         for pdf_name, pdf_file in builds.items():
             print(f"{pdf_name:<{max_len}}: {pdf_file}")
+
         print()
+
         if build_failed:
             sys.exit("PDF build failed: not all PDF files were created.")
         else:
             print("All PDF files were built.")
 
     def handle_info(self, output_dirs):
+        """
+        Extra steps for Info output.
+
+        For texinfo generation, an additional make is needed from the
+        texinfo directory.
+        """
+
         for output_dir in output_dirs:
             try:
                 subprocess.run(["make", "info"], cwd=output_dir, check=True)
             except subprocess.CalledProcessError as e:
                 sys.exit(f"Error generating info docs: {e}")
 
-    def cleandocs(self, builder):
+    def cleandocs(self, builder):           # pylint: disable=W0613
+        """Remove documentation output directory"""
         shutil.rmtree(self.builddir, ignore_errors=True)
 
     def build(self, target, sphinxdirs=None, conf="conf.py",
               theme=None, css=None, paper=None):
+        """
+        Build documentation using Sphinx. This is the core function of this
+        module. It prepares all arguments required by sphinx-build.
+        """
+
         builder = TARGETS[target]["builder"]
         out_dir = TARGETS[target].get("out_dir", "")
+
+        #
+        # Cleandocs doesn't require sphinx-build
+        #
         if target == "cleandocs":
             self.cleandocs(builder)
             return
+
         if theme:
-                os.environ["DOCS_THEME"] = theme
+            os.environ["DOCS_THEME"] = theme
+
+        #
+        # Other targets require sphinx-build, so check if it exists
+        #
         sphinxbuild = shutil.which(self.sphinxbuild, path=self.env["PATH"])
         if not sphinxbuild:
             sys.exit(f"Error: {self.sphinxbuild} not found in PATH.\n")
+
         if builder == "latex":
             if not self.pdflatex_cmd and not self.latexmk_cmd:
                 sys.exit("Error: pdflatex or latexmk required for PDF generation")
+
         docs_dir = os.path.abspath(os.path.join(self.srctree, "Documentation"))
+
+        #
+        # Fill in base arguments for Sphinx build
+        #
         kerneldoc = self.kerneldoc
         if kerneldoc.startswith(self.srctree):
             kerneldoc = os.path.relpath(kerneldoc, self.srctree)
+
         args = [ "-b", builder, "-c", docs_dir ]
+
         if builder == "latex":
             if not paper:
                 paper = PAPER[1]
+
             args.extend(["-D", f"latex_elements.papersize={paper}paper"])
+
         if self.config_rust:
             args.extend(["-t", "rustdoc"])
+
         if conf:
             self.env["SPHINX_CONF"] = self.get_path(conf, abs_path=True)
+
         if not sphinxdirs:
             sphinxdirs = os.environ.get("SPHINXDIRS", ".")
+
+        #
+        # sphinxdirs can be a list or a whitespace-separated string
+        #
         sphinxdirs_list = []
         for sphinxdir in sphinxdirs:
             if isinstance(sphinxdir, list):
@@ -221,17 +428,32 @@ class SphinxBuilder:
             else:
                 for name in sphinxdir.split(" "):
                     sphinxdirs_list.append(name)
+
+        #
+        # Step 1:  Build each directory in separate.
+        #
+        # This is not the best way of handling it, as cross-references between
+        # them will be broken, but this is what we've been doing since
+        # the beginning.
+        #
         output_dirs = []
         for sphinxdir in sphinxdirs_list:
             src_dir = os.path.join(docs_dir, sphinxdir)
             doctree_dir = os.path.join(self.builddir, ".doctrees")
             output_dir = os.path.join(self.builddir, sphinxdir, out_dir)
+
+            #
+            # Make directory names canonical
+            #
             src_dir = os.path.normpath(src_dir)
             doctree_dir = os.path.normpath(doctree_dir)
             output_dir = os.path.normpath(output_dir)
+
             os.makedirs(doctree_dir, exist_ok=True)
             os.makedirs(output_dir, exist_ok=True)
+
             output_dirs.append(output_dir)
+
             build_args = args + [
                 "-d", doctree_dir,
                 "-D", f"kerneldoc_bin={kerneldoc}",
@@ -241,31 +463,54 @@ class SphinxBuilder:
                 src_dir,
                 output_dir,
             ]
+
             try:
                 self.run_sphinx(sphinxbuild, build_args, env=self.env)
             except (OSError, ValueError, subprocess.SubprocessError) as e:
                 sys.exit(f"Build failed: {repr(e)}")
+
+            #
+            # Ensure that each html/epub output will have needed static files
+            #
             if target in ["htmldocs", "epubdocs"]:
                 self.handle_html(css, output_dir)
+
+        #
+        # Step 2: Some targets (PDF and info) require an extra step once
+        #         sphinx-build finishes
+        #
         if target == "pdfdocs":
             self.handle_pdf(output_dirs)
         elif target == "infodocs":
             self.handle_info(output_dirs)
 
 def jobs_type(value):
+    """
+    Handle valid values for -j. Accepts Sphinx "-jauto", plus a number
+    equal or bigger than one.
+    """
     if value is None:
         return None
+
     if value.lower() == 'auto':
         return value.lower()
+
     try:
         if int(value) >= 1:
             return value
+
         raise argparse.ArgumentTypeError(f"Minimum jobs is 1, got {value}")
     except ValueError:
-        raise argparse.ArgumentTypeError(f"Must be 'auto' or positive integer, got {value}")
+        raise argparse.ArgumentTypeError(f"Must be 'auto' or positive integer, got {value}")  # pylint: disable=W0707
 
 def main():
+    """
+    Main function. The only mandatory argument is the target. If not
+    specified, the other arguments will use default values if not
+    specified at os.environ.
+    """
     parser = argparse.ArgumentParser(description="Kernel documentation builder")
+
     parser.add_argument("target", choices=list(TARGETS.keys()),
                         help="Documentation target to build")
     parser.add_argument("--sphinxdirs", nargs="+",
@@ -274,18 +519,27 @@ def main():
                         help="Sphinx configuration file")
     parser.add_argument("--builddir", default="output",
                         help="Sphinx configuration file")
+
     parser.add_argument("--theme", help="Sphinx theme to use")
+
     parser.add_argument("--css", help="Custom CSS file for HTML/EPUB")
+
     parser.add_argument("--paper", choices=PAPER, default=PAPER[0],
                         help="Paper size for LaTeX/PDF output")
+
     parser.add_argument("-v", "--verbose", action='store_true',
                         help="place build in verbose mode")
+
     parser.add_argument('-j', '--jobs', type=jobs_type,
                         help="Sets number of jobs to use with sphinx-build")
+
     args = parser.parse_args()
+
     PythonVersion.check_python(MIN_PYTHON_VERSION)
+
     builder = SphinxBuilder(builddir=args.builddir,
                             verbose=args.verbose, n_jobs=args.jobs)
+
     builder.build(args.target, sphinxdirs=args.sphinxdirs, conf=args.conf,
                   theme=args.theme, css=args.css, paper=args.paper)
 
-- 
2.51.0

[PATCH v4 19/19] tools/docs: sphinx-* break documentation bulds on openSUSE

[Mauro Carvalho Chehab]

Before this patch, building htmldocs on opensuseLEAP works
fine:

    # make htmldocs
    Available Python versions:
      /usr/bin/python3.11

    Python 3.6.15 not supported. Changing to /usr/bin/python3.11
    Python 3.6.15 not supported. Changing to /usr/bin/python3.11
    Using alabaster theme
    Using Python kernel-doc

    ...

As the logic detects that Python 3.6 is too old and recommends
intalling python311-Sphinx. If installed, documentation builds
work like a charm.

Yet, some develpers complained that running python3.11 instead
of python3 should not happen. So, let's break the build to make
them happier:

    $ make htmldocs
    Python 3.6.15 not supported. Bailing out
    You could run, instead:
      /usr/bin/python3.11 tools/docs/sphinx-build-wrapper htmldocs \
        --sphinxdirs=. --conf=conf.py --builddir=Documentation/output --theme= --css= \
        --paper=

    Python 3.6.15 not supported. Bailing out
    make[2]: *** [Documentation/Makefile:76: htmldocs] Error 1
    make[1]: *** [Makefile:1806: htmldocs] Error 2
    make: *** [Makefile:248: __sub-make] Error 2

It should be noticed that:

1. after this change, sphinx-pre-install needs to be called
   by hand:

    $ /usr/bin/python3.11 tools/docs/sphinx-pre-install
    Detected OS: openSUSE Leap 15.6.
    Sphinx version: 7.2.6

    All optional dependencies are met.
    Needed package dependencies are met.

2. sphinx-build-wrapper will auto-detect python3.11 and
   suggest a way to build the docs using the parameters passed
   via make variables. In this specific example:

   /usr/bin/python3.11 tools/docs/sphinx-build-wrapper htmldocs --sphinxdirs=. --conf=conf.py --theme= --css= --paper=

3. As this needs to be executed outside docs Makefile, it won't run
   the validation check scripts nor build Rust documentation if
   enabled, as the extra scripts are part of the docs Makefile.

Signed-off-by: Mauro Carvalho Chehab <mchehab+huawei@kernel.org>
---
 tools/docs/lib/python_version.py | 28 ++++++++++++++++++++++++----
 tools/docs/sphinx-build-wrapper  |  3 ++-
 tools/docs/sphinx-pre-install    |  3 ++-
 3 files changed, 28 insertions(+), 6 deletions(-)

diff --git a/tools/docs/lib/python_version.py b/tools/docs/lib/python_version.py
index a9fda2470a26..4fde1b882164 100644
--- a/tools/docs/lib/python_version.py
+++ b/tools/docs/lib/python_version.py
@@ -20,9 +20,11 @@ Python version if present.
 import os
 import re
 import subprocess
+import shlex
 import sys
 
 from glob import glob
+from textwrap import indent
 
 class PythonVersion:
     """
@@ -44,6 +46,25 @@ class PythonVersion:
         """Returns a version tuple as major.minor.patch"""
         return ".".join([str(x) for x in version])
 
+    @staticmethod
+    def cmd_print(cmd, max_len=80):
+        cmd_line = []
+
+        for w in cmd:
+            w = shlex.quote(w)
+
+            if cmd_line:
+                if not max_len or len(cmd_line[-1]) + len(w) < max_len:
+                    cmd_line[-1] += " " + w
+                    continue
+                else:
+                    cmd_line[-1] += " \\"
+                    cmd_line.append(w)
+            else:
+                cmd_line.append(w)
+
+        return "\n  ".join(cmd_line)
+
     def __str__(self):
         """Returns a version tuple as major.minor.patch from self.version"""
         return self.ver_str(self.version)
@@ -130,14 +151,13 @@ class PythonVersion:
         else:
             new_python_cmd = None
 
-        if show_alternatives:
+        if show_alternatives and available_versions:
             print("You could run, instead:")
             for _, cmd in available_versions:
                 args = [cmd, script_path] + sys.argv[1:]
 
-                cmd_str = " ".join(args)
-                print(f"  {cmd_str}")
-            print()
+                cmd_str = indent(PythonVersion.cmd_print(args), "  ")
+                print(f"{cmd_str}\n")
 
         if bail_out:
             msg = f"Python {python_ver} not supported. Bailing out"
diff --git a/tools/docs/sphinx-build-wrapper b/tools/docs/sphinx-build-wrapper
index 183717eb6d87..897c3512d1c7 100755
--- a/tools/docs/sphinx-build-wrapper
+++ b/tools/docs/sphinx-build-wrapper
@@ -725,7 +725,8 @@ def main():
 
     args = parser.parse_args()
 
-    PythonVersion.check_python(MIN_PYTHON_VERSION)
+    PythonVersion.check_python(MIN_PYTHON_VERSION, show_alternatives=True,
+                               bail_out=True)
 
     builder = SphinxBuilder(builddir=args.builddir, venv=args.venv,
                             verbose=args.verbose, n_jobs=args.jobs,
diff --git a/tools/docs/sphinx-pre-install b/tools/docs/sphinx-pre-install
index 663d4e2a3f57..698989584b6a 100755
--- a/tools/docs/sphinx-pre-install
+++ b/tools/docs/sphinx-pre-install
@@ -1531,7 +1531,8 @@ def main():
 
     checker = SphinxDependencyChecker(args)
 
-    PythonVersion.check_python(MIN_PYTHON_VERSION)
+    PythonVersion.check_python(MIN_PYTHON_VERSION,
+                               bail_out=True, success_on_error=True)
     checker.check_needs()
 
 # Call main if not used as module
-- 
2.51.0