[PATCH 05/20] btf: stop resolving cyclic type references

[Kris Van Hees <kris.van.hees@oracle.com>]

test: expand BTF cyclic type resolution coverage
---
 libdtrace/dt_btf.c                     |   8 +-
 test/internals/btf/tst.cyclic-types.sh |  18 ++++
 test/triggers/Build                    |   8 +-
 test/triggers/btf-cyclic-types.c       | 113 +++++++++++++++++++++++++
 4 files changed, 145 insertions(+), 2 deletions(-)
 create mode 100755 test/internals/btf/tst.cyclic-types.sh
 create mode 100644 test/triggers/btf-cyclic-types.c

diff --git a/libdtrace/dt_btf.c b/libdtrace/dt_btf.c
index 194a251c..039742d1 100644
--- a/libdtrace/dt_btf.c
+++ b/libdtrace/dt_btf.c
@@ -100,8 +100,14 @@ static btf_type_t *
 dt_btf_real_type_by_id(const dt_btf_t *btf, int32_t id)
 {
 	btf_type_t	*type = dt_btf_type_by_id(btf, id);
+	uint64_t	remaining = 1 + (uint64_t)btf->type_cnt;
 
-	while (type != NULL) {
+	if (btf->parent != NULL)
+		remaining += btf->parent->type_cnt;
+
+	/* A longer chain must revisit a type, so reject cyclic BTF references. */
+	while (type != NULL && remaining > 0) {
+		remaining--;
 		switch (BTF_INFO_KIND(type->info)) {
 		case BTF_KIND_CONST:
 		case BTF_KIND_FUNC:
diff --git a/test/internals/btf/tst.cyclic-types.sh b/test/internals/btf/tst.cyclic-types.sh
new file mode 100755
index 00000000..88d092d6
--- /dev/null
+++ b/test/internals/btf/tst.cyclic-types.sh
@@ -0,0 +1,18 @@
+#!/bin/bash
+#
+# Copyright (c) 2026, Oracle and/or its affiliates. All rights reserved.
+# Licensed under the Universal Permissive License v 1.0 as shown at
+# http://oss.oracle.com/licenses/upl.
+#
+# ASSERTION: Cyclic BTF type references fail resolution, while a finite
+#            chain still resolves to its function prototype.
+#
+
+test/triggers/btf-cyclic-types
+rc=$?
+
+echo "Result: $rc"
+
+[[ $rc -eq 0 ]] || exit 1
+
+exit 0
diff --git a/test/triggers/Build b/test/triggers/Build
index 947341c6..5480f423 100644
--- a/test/triggers/Build
+++ b/test/triggers/Build
@@ -25,7 +25,7 @@ EXTERNAL_TRIGGERS = $(EXTERNAL_64BIT_TRIGGERS) $(if $(NATIVE_BITNESS_ONLY),,$(EX
 INTERNAL_64BIT_TRIGGERS = libproc-pldd libproc-consistency libproc-sleeper \
     libproc-sleeper-pie libproc-dlmadopen libproc-lookup-by-name \
     libproc-lookup-victim libproc-execing-bkpts libproc-execing-bkpts-victim \
-    module-discovery shared-ctf
+    btf-cyclic-types shared-ctf module-discovery
 INTERNAL_32BIT_TRIGGERS := libproc-sleeper-32 libproc-sleeper-pie-32
 INTERNAL_TRIGGERS = $(INTERNAL_64BIT_TRIGGERS) $(if $(NATIVE_BITNESS_ONLY),,$(INTERNAL_32BIT_TRIGGERS))
 
@@ -126,6 +126,12 @@ visible-constructor-static_LDFLAGS := -static -s
 visible-constructor_LDFLAGS := -s
 visible-constructor-32_LDFLAGS := -s
 
+# Include the private BTF resolver and discard its unrelated functions, which
+# otherwise require the rest of libdtrace at link time.
+btf-cyclic-types_CFLAGS := -I. -Ilibdtrace -Ilibproc -Ilibcommon \
+    -ffunction-sections -fdata-sections -Wno-pedantic
+btf-cyclic-types_LDFLAGS := -Wl,--gc-sections
+
 # We are using machinery from libproc directly, but libproc calls into
 # nonpublic parts of libdtrace for various shared machinery (dt_list
 # and so on).  Thus, we must link directly to the build libraries, since
diff --git a/test/triggers/btf-cyclic-types.c b/test/triggers/btf-cyclic-types.c
new file mode 100644
index 00000000..cb5358ca
--- /dev/null
+++ b/test/triggers/btf-cyclic-types.c
@@ -0,0 +1,113 @@
+/*
+ * Copyright (c) 2026, Oracle and/or its affiliates. All rights reserved.
+ * Licensed under the Universal Permissive License v 1.0 as shown at
+ * http://oss.oracle.com/licenses/upl.
+ */
+
+#include "libdtrace/dt_btf.c"
+
+static int
+check(const dt_btf_t *btf, int argc, int is_void, int traceable)
+{
+	return dt_btf_func_argc(btf, 1) == argc &&
+	       dt_btf_func_is_void(btf, 1) == is_void &&
+	       dt_btf_func_is_traceable(btf, 1) == traceable;
+}
+
+int
+main(void)
+{
+	btf_type_t	func = { .info = BTF_KIND_FUNC << 24, .type = 1 };
+	btf_type_t	qualifier = { .info = BTF_KIND_CONST << 24, .type = 1 };
+	btf_type_t	proto = { .info = BTF_KIND_FUNC_PROTO << 24 };
+	btf_type_t	*types[] = { NULL, &func, &qualifier, &proto };
+	dt_btf_t	btf = { .type_cnt = 4, .types = types };
+
+	/* A function that refers directly to itself. */
+	if (!check(&btf, -1, 0, 0))
+		return 1;
+
+	/* A function and qualifier that refer to each other. */
+	func.type = 2;
+	if (!check(&btf, -1, 0, 0))
+		return 2;
+
+	/* The same chain must resolve when it ends at a prototype. */
+	qualifier.type = 3;
+	if (!check(&btf, 0, 1, 1))
+		return 3;
+
+	/* A cyclic return type makes an otherwise valid prototype untraceable. */
+	func.type = 3;
+	proto.type = 2;
+	qualifier.type = 2;
+	if (!check(&btf, 0, 0, 0))
+		return 4;
+
+	/* Follow references through a parent and child BTF, then close a loop. */
+	{
+		btf_type_t *parent_types[] = { NULL, &func, &proto };
+		btf_type_t *child_types[] = { NULL, &qualifier };
+		dt_btf_t parent = { .type_cnt = 3, .types = parent_types };
+		dt_btf_t child = { .parent = &parent, .type_cnt = 2,
+				   .types = child_types };
+
+		func.type = 3;
+		proto.type = 0;
+		qualifier.type = 2;
+		if (!check(&child, 0, 1, 1))
+			return 5;
+
+		qualifier.type = 1;
+		if (!check(&child, -1, 0, 0))
+			return 6;
+	}
+
+	/* A cyclic argument makes a valid prototype untraceable. */
+	{
+		struct {
+			btf_type_t proto;
+			btf_param_t arg;
+		} arg_proto = {
+			.proto = { .info = (BTF_KIND_FUNC_PROTO << 24) | 1 },
+			.arg = { .type = 2 }
+		};
+		btf_type_t argument = { .info = BTF_KIND_PTR << 24 };
+		btf_type_t *arg_types[] = { NULL, &func, &argument,
+					   &arg_proto.proto };
+		dt_btf_t arg_btf = { .type_cnt = 4, .types = arg_types };
+
+		func.type = 3;
+		if (!check(&arg_btf, 1, 1, 1))
+			return 7;
+
+		argument.info = BTF_KIND_CONST << 24;
+		argument.type = 2;
+		if (!check(&arg_btf, 1, 1, 0))
+			return 8;
+	}
+
+	/* A long finite qualifier chain still reaches its prototype. */
+	{
+		enum { CHAIN_LEN = 256 };
+		btf_type_t qualifiers[CHAIN_LEN];
+		btf_type_t *chain_types[CHAIN_LEN + 3] = { NULL, &func };
+		dt_btf_t chain_btf = { .type_cnt = CHAIN_LEN + 3,
+				       .types = chain_types };
+		int i;
+
+		func.type = 2;
+		proto.type = 0;
+		for (i = 0; i < CHAIN_LEN; i++) {
+			qualifiers[i] = (btf_type_t) {
+				.info = BTF_KIND_CONST << 24, .type = i + 3
+			};
+			chain_types[i + 2] = &qualifiers[i];
+		}
+		chain_types[CHAIN_LEN + 2] = &proto;
+		if (!check(&chain_btf, 0, 1, 1))
+			return 9;
+	}
+
+	return 0;
+}
-- 
2.52.0