[PATCH v4 1/2] scripts: generate_rust_analyzer.py: add versioning infrastructure

[Jesung Yang via B4 Relay <devnull+y.j3ms.n.gmail.com@kernel.org>]

From: Jesung Yang <y.j3ms.n@gmail.com>

Introduce multi-version support for rust-analyzer. The script now
executes `rust-analyzer --version` to query the version string and
generates a `rust-project.json` file compatible with the detected
version.

This is a preparatory patch to address inherent method resolution
failures for primitive types occurring in rust-analyzer v0.3.2693
(2025-11-24) or later when used with our current `rust-project.json`
generation logic. Since the actual fix requires using the `sysroot_src`
field with a feature only available in rust-analyzer v0.3.2727
(2025-12-22) or later, this infrastructure is necessary to maintain
compatibility with older rust-analyzer releases.

Signed-off-by: Jesung Yang <y.j3ms.n@gmail.com>
---
 scripts/generate_rust_analyzer.py | 205 ++++++++++++++++++++++++++++++++++----
 1 file changed, 183 insertions(+), 22 deletions(-)

diff --git a/scripts/generate_rust_analyzer.py b/scripts/generate_rust_analyzer.py
index b4a55344688d..21832763c5be 100755
--- a/scripts/generate_rust_analyzer.py
+++ b/scripts/generate_rust_analyzer.py
@@ -4,10 +4,14 @@
 """
 
 import argparse
+from dataclasses import dataclass
+from datetime import datetime, date
+import enum
 import json
 import logging
 import os
 import pathlib
+import re
 import subprocess
 import sys
 from typing import Dict, Iterable, List, Literal, Optional, TypedDict
@@ -36,6 +40,7 @@ class Crate(TypedDict):
     is_workspace_member: bool
     deps: List[Dependency]
     cfg: List[str]
+    crate_attrs: List[str]
     edition: str
     env: Dict[str, str]
 
@@ -49,7 +54,19 @@ class CrateWithGenerated(Crate):
     source: Source
 
 
+class RustProject(TypedDict):
+    crates: List[Crate]
+    sysroot: str
+
+
+@dataclass(frozen=True)
+class RaVersionCtx:
+    manual_sysroot_crates: bool
+    use_crate_attrs: bool
+
+
 def generate_crates(
+    ctx: RaVersionCtx,
     srctree: pathlib.Path,
     objtree: pathlib.Path,
     sysroot_src: pathlib.Path,
@@ -75,10 +92,14 @@ def generate_crates(
         deps: List[Dependency],
         *,
         cfg: Optional[List[str]],
+        crate_attrs: Optional[List[str]],
         is_workspace_member: Optional[bool],
         edition: Optional[str],
     ) -> Crate:
         cfg = cfg if cfg is not None else crates_cfgs.get(display_name, [])
+        crate_attrs = (
+            crate_attrs if ctx.use_crate_attrs and crate_attrs is not None else []
+        )
         is_workspace_member = (
             is_workspace_member if is_workspace_member is not None else True
         )
@@ -89,6 +110,7 @@ def generate_crates(
             "is_workspace_member": is_workspace_member,
             "deps": deps,
             "cfg": cfg,
+            "crate_attrs": crate_attrs,
             "edition": edition,
             "env": {
                 "RUST_MODFILE": "This is only for rust-analyzer"
@@ -109,6 +131,7 @@ def generate_crates(
             root_module,
             deps,
             cfg=cfg,
+            crate_attrs=None,
             is_workspace_member=is_workspace_member,
             edition=edition,
         )
@@ -147,6 +170,7 @@ def generate_crates(
         deps: List[Dependency],
         *,
         cfg: Optional[List[str]] = None,
+        crate_attrs: Optional[List[str]] = None,
         is_workspace_member: Optional[bool] = None,
         edition: Optional[str] = None,
     ) -> Dependency:
@@ -156,6 +180,7 @@ def generate_crates(
                 root_module,
                 deps,
                 cfg=cfg,
+                crate_attrs=crate_attrs,
                 is_workspace_member=is_workspace_member,
                 edition=edition,
             )
@@ -166,7 +191,9 @@ def generate_crates(
         deps: List[Dependency],
         *,
         cfg: Optional[List[str]] = None,
-    ) -> Dependency:
+    ) -> Optional[Dependency]:
+        if not ctx.manual_sysroot_crates:
+            return None
         return append_crate(
             display_name,
             sysroot_src / display_name / "src" / "lib.rs",
@@ -200,67 +227,70 @@ def generate_crates(
             edition=core_edition,
         )
 
+    def sysroot_deps(*deps: Optional[Dependency]) -> List[Dependency]:
+        return [dep for dep in deps if dep is not None]
+
     # NB: sysroot crates reexport items from one another so setting up our transitive dependencies
     # here is important for ensuring that rust-analyzer can resolve symbols. The sources of truth
     # for this dependency graph are `(sysroot_src / crate / "Cargo.toml" for crate in crates)`.
     core = append_sysroot_crate("core", [])
-    alloc = append_sysroot_crate("alloc", [core])
-    std = append_sysroot_crate("std", [alloc, core])
-    proc_macro = append_sysroot_crate("proc_macro", [core, std])
+    alloc = append_sysroot_crate("alloc", sysroot_deps(core))
+    std = append_sysroot_crate("std", sysroot_deps(alloc, core))
+    proc_macro = append_sysroot_crate("proc_macro", sysroot_deps(core, std))
 
     compiler_builtins = append_crate(
         "compiler_builtins",
         srctree / "rust" / "compiler_builtins.rs",
-        [core],
+        sysroot_deps(core),
     )
 
     proc_macro2 = append_crate(
         "proc_macro2",
         srctree / "rust" / "proc-macro2" / "lib.rs",
-        [core, alloc, std, proc_macro],
+        sysroot_deps(core, alloc, std, proc_macro),
     )
 
     quote = append_crate(
         "quote",
         srctree / "rust" / "quote" / "lib.rs",
-        [core, alloc, std, proc_macro, proc_macro2],
+        sysroot_deps(core, alloc, std, proc_macro) + [proc_macro2],
         edition="2018",
     )
 
     syn = append_crate(
         "syn",
         srctree / "rust" / "syn" / "lib.rs",
-        [std, proc_macro, proc_macro2, quote],
+        sysroot_deps(std, proc_macro) + [proc_macro2, quote],
     )
 
     macros = append_proc_macro_crate(
         "macros",
         srctree / "rust" / "macros" / "lib.rs",
-        [std, proc_macro, proc_macro2, quote, syn],
+        sysroot_deps(std, proc_macro) + [proc_macro2, quote, syn],
     )
 
     build_error = append_crate(
         "build_error",
         srctree / "rust" / "build_error.rs",
-        [core, compiler_builtins],
+        sysroot_deps(core) + [compiler_builtins],
     )
 
     pin_init_internal = append_proc_macro_crate(
         "pin_init_internal",
         srctree / "rust" / "pin-init" / "internal" / "src" / "lib.rs",
-        [std, proc_macro, proc_macro2, quote, syn],
+        sysroot_deps(std, proc_macro) + [proc_macro2, quote, syn],
     )
 
     pin_init = append_crate(
         "pin_init",
         srctree / "rust" / "pin-init" / "src" / "lib.rs",
-        [core, compiler_builtins, pin_init_internal, macros],
+        sysroot_deps(core) + [compiler_builtins, pin_init_internal, macros],
     )
 
     ffi = append_crate(
         "ffi",
         srctree / "rust" / "ffi.rs",
-        [core, compiler_builtins],
+        sysroot_deps(core) + [compiler_builtins],
     )
 
     def append_crate_with_generated(
@@ -272,6 +302,7 @@ def generate_crates(
             srctree / "rust"/ display_name / "lib.rs",
             deps,
             cfg=generated_cfg,
+            crate_attrs=None,
             is_workspace_member=True,
             edition=None,
         )
@@ -288,10 +319,14 @@ def generate_crates(
         }
         return register_crate(crate_with_generated)
 
-    bindings = append_crate_with_generated("bindings", [core, ffi, pin_init])
-    uapi = append_crate_with_generated("uapi", [core, ffi, pin_init])
+    bindings = append_crate_with_generated(
+        "bindings", sysroot_deps(core) + [ffi, pin_init]
+    )
+    uapi = append_crate_with_generated(
+        "uapi", sysroot_deps(core) + [ffi, pin_init]
+    )
     kernel = append_crate_with_generated(
-        "kernel", [core, macros, build_error, pin_init, ffi, bindings, uapi]
+        "kernel", sysroot_deps(core) + [macros, build_error, pin_init, ffi, bindings, uapi]
     )
 
     scripts = srctree / "scripts"
@@ -303,7 +338,7 @@ def generate_crates(
         append_crate(
             name,
             path,
-            [std],
+            sysroot_deps(std),
         )
 
     def is_root_crate(build_file: pathlib.Path, target: str) -> bool:
@@ -335,12 +370,120 @@ def generate_crates(
             append_crate(
                 name,
                 path,
-                [core, kernel, pin_init],
+                sysroot_deps(core) + [kernel, pin_init],
                 cfg=generated_cfg,
             )
 
     return crates
 
+
+Version = tuple[int, int, int]
+
+
+@enum.unique
+class RaVersionInfo(enum.Enum):
+    """
+    Represents rust-analyzer compatibility baselines. Concrete versions are mapped to the most
+    recent baseline they have reached. Must be in release order.
+    """
+
+    # v0.3.1877, released on 2024-03-11; shipped with the rustup 1.78 toolchain.
+    DEFAULT = (
+        datetime.strptime("2024-03-11", "%Y-%m-%d"),
+        (0, 3, 1877),
+        (1, 78, 0),
+    )
+
+    def __init__(
+        self, release_date: date, ra_version: Version, rust_version: Version
+    ) -> None:
+        self.release_date = release_date
+        self.ra_version = ra_version
+        self.rust_version = rust_version
+
+
+def generate_rust_project(
+    version_info: RaVersionInfo,
+    srctree: pathlib.Path,
+    objtree: pathlib.Path,
+    sysroot: pathlib.Path,
+    sysroot_src: pathlib.Path,
+    external_src: Optional[pathlib.Path],
+    cfgs: List[str],
+    core_edition: str,
+) -> RustProject:
+    from typing import NoReturn
+
+    # TODO: Switch to `typing.assert_never` when Python 3.11 is adopted.
+    def assert_never(arg: NoReturn, /) -> NoReturn:
+        # Adapted from:
+        # https://github.com/python/cpython/blob/1b118353bb0a/Lib/typing.py#L2629-L2651
+        value = repr(arg)
+        raise AssertionError(f"Expected code to be unreachable, but got: {value}")
+
+    if version_info == RaVersionInfo.DEFAULT:
+        ctx = RaVersionCtx(
+            use_crate_attrs=False,
+            manual_sysroot_crates=True,
+        )
+        return {
+            "crates": generate_crates(
+                ctx, srctree, objtree, sysroot_src, external_src, cfgs, core_edition
+            ),
+            "sysroot": str(sysroot),
+        }
+    else:
+        assert_never(version_info)
+
+def query_ra_version() -> Optional[str]:
+    try:
+        # Use the rust-analyzer binary found in $PATH.
+        ra_version_output = (
+            subprocess.check_output(
+                ["rust-analyzer", "--version"],
+                stdin=subprocess.DEVNULL,
+            )
+            .decode("utf-8")
+            .strip()
+        )
+        return ra_version_output
+    except FileNotFoundError:
+        logging.warning("Failed to find rust-analyzer in $PATH")
+        return None
+
+def map_ra_version_baseline(ra_version_output: str) -> RaVersionInfo:
+    baselines = reversed(RaVersionInfo)
+
+    version_match = re.search(r"\d+\.\d+\.\d+", ra_version_output)
+    if version_match:
+        version_string = version_match.group()
+        found_version = tuple(map(int, version_string.split(".")))
+
+        # `rust-analyzer --version` shows different version string depending on how the binary
+        # is built: it may print either the Rust version or the rust-analyzer version itself.
+        # To distinguish between them, we leverage rust-analyzer's versioning convention.
+        #
+        # See:
+        # - https://github.com/rust-lang/rust-analyzer/blob/fad5c3d2d642/xtask/src/dist.rs#L19-L21
+        is_ra_version = version_string.startswith(("0.3", "0.4", "0.5"))
+        if is_ra_version:
+            for info in baselines:
+                if found_version >= info.ra_version:
+                    return info
+        else:
+            for info in baselines:
+                if found_version >= info.rust_version:
+                    return info
+
+    date_match = re.search(r"\d{4}-\d{2}-\d{2}", ra_version_output)
+    if date_match:
+        found_date = datetime.strptime(date_match.group(), "%Y-%m-%d")
+        for info in baselines:
+            if found_date >= info.release_date:
+                return info
+
+    return RaVersionInfo.DEFAULT
+
 def main() -> None:
     parser = argparse.ArgumentParser()
     parser.add_argument('--verbose', '-v', action='store_true')
@@ -369,10 +512,28 @@ def main() -> None:
         level=logging.INFO if args.verbose else logging.WARNING
     )
 
-    rust_project = {
-        "crates": generate_crates(args.srctree, args.objtree, args.sysroot_src, args.exttree, args.cfgs, args.core_edition),
-        "sysroot": str(args.sysroot),
-    }
+    ra_version_output = query_ra_version()
+    if ra_version_output:
+        compatible_ra_version = map_ra_version_baseline(ra_version_output)
+    else:
+        logging.warning(
+            "Falling back to `rust-project.json` for rust-analyzer %s, %s (shipped with Rust %s)",
+            ".".join(map(str, RaVersionInfo.DEFAULT.ra_version)),
+            datetime.strftime(RaVersionInfo.DEFAULT.release_date, "%Y-%m-%d"),
+            ".".join(map(str, RaVersionInfo.DEFAULT.rust_version)),
+        )
+        compatible_ra_version = RaVersionInfo.DEFAULT
+
+    rust_project = generate_rust_project(
+        compatible_ra_version,
+        args.srctree,
+        args.objtree,
+        args.sysroot,
+        args.sysroot_src,
+        args.exttree,
+        args.cfgs,
+        args.core_edition,
+    )
 
     json.dump(rust_project, sys.stdout, sort_keys=True, indent=4)
 

-- 
2.52.0