* [PATCH v2 1/2] cgroup: add BPF kfuncs to read a cpu cgroup's stats
2026-08-18 0:24 [PATCH v2 0/2] cgroup: expose cpu.stat to BPF Ziyang Men
@ 2026-08-18 0:24 ` Ziyang Men
2026-08-18 1:28 ` bot+bpf-ci
2026-08-18 17:16 ` Tejun Heo
2026-08-18 0:24 ` [PATCH v2 2/2] selftests/bpf: add cgroup_iter_cpu test for cpu cgroup kfuncs Ziyang Men
1 sibling, 2 replies; 6+ messages in thread
From: Ziyang Men @ 2026-08-18 0:24 UTC (permalink / raw)
To: Tejun Heo, Johannes Weiner, Michal Koutný,
Alexei Starovoitov, Daniel Borkmann, Andrii Nakryiko,
Eduard Zingerman, Kumar Kartikeya Dwivedi, Shuah Khan,
kernel-team
Cc: Ingo Molnar, Peter Zijlstra, Vincent Guittot, Ben Segall,
Dietmar Eggemann, Martin KaFai Lau, Song Liu, Yonghong Song,
Jiri Olsa, Emil Tsalapatis, Roman Gushchin, Shakeel Butt,
JP Kobryn, Ziyang Men, bpf, cgroups, linux-kselftest,
linux-kernel
Collecting cgroup statistics is expensive: the existing
method is to open and parse a cgroup file. memcg already has an
efficient alternative through BPF; this series extends that idea to cpu.
Expose the CPU controller's per-cgroup statistics to BPF, following
the memory controller kfuncs in mm/bpf_memcontrol.c.
Design:
- Add bpf_css_flush_rstat() and bpf_cgroup_base_stat() to cgroup rstat.
The second kfunc returns the raw cgroup_base_stat after the
same cputime adjustment used by cpu.stat.
- Leave reading the CFS bandwidth counters to the BPF program. They are
plain fields of tg->cfs_bandwidth, so they need no kernel code.
- Add bpf_css_to_task_group(), which returns a checked RCU pointer of
the task_group*. This is usefule to compute the throttled time in bpf
side since the bpf_per_cpu_ptr() requires a verifier-known struct
task_group * and a pointer carrying the MEM_PERCPU property.
This use the convention that task_group embeds its css at offset zero,
so no scheduler helper or scheduler source change is needed.
Suggested-by: Shakeel Butt <shakeel.butt@linux.dev>
Suggested-by: Tejun Heo <tj@kernel.org>
Assisted-by: Claude:claude-opus-5
Signed-off-by: Ziyang Men <ziyang.meme@gmail.com>
---
kernel/cgroup/Makefile | 2 ++
kernel/cgroup/bpf_cpu.c | 61 +++++++++++++++++++++++++++++++++++++++++
kernel/cgroup/rstat.c | 59 +++++++++++++++++++++++++++++++++++++--
3 files changed, 120 insertions(+), 2 deletions(-)
create mode 100644 kernel/cgroup/bpf_cpu.c
diff --git a/kernel/cgroup/Makefile b/kernel/cgroup/Makefile
index ede31601a363..0ba59b7eef48 100644
--- a/kernel/cgroup/Makefile
+++ b/kernel/cgroup/Makefile
@@ -1,6 +1,8 @@
# SPDX-License-Identifier: GPL-2.0
obj-y := cgroup.o rstat.o namespace.o cgroup-v1.o freezer.o
+obj-$(CONFIG_BPF_SYSCALL) += bpf_cpu.o
+
obj-$(CONFIG_CGROUP_FREEZER) += legacy_freezer.o
obj-$(CONFIG_CGROUP_PIDS) += pids.o
obj-$(CONFIG_CGROUP_RDMA) += rdma.o
diff --git a/kernel/cgroup/bpf_cpu.c b/kernel/cgroup/bpf_cpu.c
new file mode 100644
index 000000000000..ac165d0b79ef
--- /dev/null
+++ b/kernel/cgroup/bpf_cpu.c
@@ -0,0 +1,61 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * CPU controller BPF kfuncs
+ *
+ * Author: Ziyang Men <ziyang.meme@gmail.com>
+ */
+
+#include <linux/bpf.h>
+#include <linux/btf_ids.h>
+#include <linux/cgroup.h>
+
+#ifdef CONFIG_CGROUP_SCHED
+struct task_group;
+
+__bpf_kfunc_start_defs();
+
+/**
+ * bpf_css_to_task_group - Cast a CPU controller css to its task group
+ * @css: CPU controller css
+ *
+ * Must be called under RCU.
+ * A C cast does not give the verifier a task_group pointer. This kfunc
+ * preserves the task_group and per-CPU types needed to read cfs_rq.
+ *
+ * Return: The task group, or NULL if @css belongs to another controller.
+ */
+__bpf_kfunc struct task_group *
+bpf_css_to_task_group(struct cgroup_subsys_state *css)
+{
+ if (css->ss != &cpu_cgrp_subsys)
+ return NULL;
+
+ /* task_group embeds css at offset zero. */
+ return (struct task_group *)css;
+}
+
+__bpf_kfunc_end_defs();
+
+BTF_KFUNCS_START(bpf_cpu_cgroup_kfunc_ids)
+BTF_ID_FLAGS(func, bpf_css_to_task_group,
+ KF_RCU | KF_RCU_PROTECTED | KF_RET_NULL)
+BTF_KFUNCS_END(bpf_cpu_cgroup_kfunc_ids)
+
+static const struct btf_kfunc_id_set bpf_cpu_cgroup_kfunc_set = {
+ .owner = THIS_MODULE,
+ .set = &bpf_cpu_cgroup_kfunc_ids,
+};
+
+static int __init bpf_cpu_cgroup_kfunc_init(void)
+{
+ int err;
+
+ err = register_btf_kfunc_id_set(BPF_PROG_TYPE_UNSPEC,
+ &bpf_cpu_cgroup_kfunc_set);
+ if (err)
+ pr_warn("error while registering cpu cgroup kfuncs: %d\n", err);
+
+ return err;
+}
+late_initcall(bpf_cpu_cgroup_kfunc_init);
+#endif /* CONFIG_CGROUP_SCHED */
diff --git a/kernel/cgroup/rstat.c b/kernel/cgroup/rstat.c
index de816a43db9f..46c322c4858b 100644
--- a/kernel/cgroup/rstat.c
+++ b/kernel/cgroup/rstat.c
@@ -752,6 +752,54 @@ void cgroup_base_stat_cputime_show(struct seq_file *seq)
cgroup_force_idle_show(seq, &bstat);
}
+#ifdef CONFIG_BPF_SYSCALL
+
+__bpf_kfunc_start_defs();
+
+/**
+ * bpf_css_flush_rstat - Flush a cgroup subsystem's rstat data
+ * @css: cgroup subsystem state to flush
+ */
+__bpf_kfunc void bpf_css_flush_rstat(struct cgroup_subsys_state *css)
+{
+ css_rstat_flush(css);
+}
+
+/**
+ * bpf_cgroup_base_stat - Read a cgroup's base statistics
+ * @cgrp: cgroup to read from
+ * @out: zero-initialized output in nanoseconds
+ *
+ * CPU time is adjusted as for cpu.stat.
+ */
+__bpf_kfunc void bpf_cgroup_base_stat(struct cgroup *cgrp,
+ struct cgroup_base_stat *out)
+{
+ if (cgroup_parent(cgrp)) {
+ __css_rstat_lock(&cgrp->self, -1);
+ *out = cgrp->bstat;
+ cputime_adjust(&cgrp->bstat.cputime, &cgrp->prev_cputime,
+ &out->cputime.utime, &out->cputime.stime);
+ __css_rstat_unlock(&cgrp->self, -1);
+ } else {
+ root_cgroup_cputime(out);
+ }
+}
+
+__bpf_kfunc_end_defs();
+
+BTF_KFUNCS_START(bpf_rstat_common_kfunc_ids)
+BTF_ID_FLAGS(func, bpf_css_flush_rstat, KF_SLEEPABLE)
+BTF_ID_FLAGS(func, bpf_cgroup_base_stat, KF_SLEEPABLE)
+BTF_KFUNCS_END(bpf_rstat_common_kfunc_ids)
+
+static const struct btf_kfunc_id_set bpf_rstat_common_kfunc_set = {
+ .owner = THIS_MODULE,
+ .set = &bpf_rstat_common_kfunc_ids,
+};
+
+#endif /* CONFIG_BPF_SYSCALL */
+
/* Add bpf kfuncs for css_rstat_updated() and css_rstat_flush() */
BTF_KFUNCS_START(bpf_rstat_kfunc_ids)
BTF_ID_FLAGS(func, css_rstat_updated)
@@ -765,7 +813,14 @@ static const struct btf_kfunc_id_set bpf_rstat_kfunc_set = {
static int __init bpf_rstat_kfunc_init(void)
{
- return register_btf_kfunc_id_set(BPF_PROG_TYPE_TRACING,
- &bpf_rstat_kfunc_set);
+ int ret;
+
+ ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_TRACING,
+ &bpf_rstat_kfunc_set);
+#ifdef CONFIG_BPF_SYSCALL
+ ret = ret ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_UNSPEC,
+ &bpf_rstat_common_kfunc_set);
+#endif
+ return ret;
}
late_initcall(bpf_rstat_kfunc_init);
--
2.53.0-Meta
^ permalink raw reply related [flat|nested] 6+ messages in thread* [PATCH v2 2/2] selftests/bpf: add cgroup_iter_cpu test for cpu cgroup kfuncs
2026-08-18 0:24 [PATCH v2 0/2] cgroup: expose cpu.stat to BPF Ziyang Men
2026-08-18 0:24 ` [PATCH v2 1/2] cgroup: add BPF kfuncs to read a cpu cgroup's stats Ziyang Men
@ 2026-08-18 0:24 ` Ziyang Men
2026-08-18 1:28 ` bot+bpf-ci
1 sibling, 1 reply; 6+ messages in thread
From: Ziyang Men @ 2026-08-18 0:24 UTC (permalink / raw)
To: Tejun Heo, Johannes Weiner, Michal Koutný,
Alexei Starovoitov, Daniel Borkmann, Andrii Nakryiko,
Eduard Zingerman, Kumar Kartikeya Dwivedi, Shuah Khan,
kernel-team
Cc: Ingo Molnar, Peter Zijlstra, Vincent Guittot, Ben Segall,
Dietmar Eggemann, Martin KaFai Lau, Song Liu, Yonghong Song,
Jiri Olsa, Emil Tsalapatis, Roman Gushchin, Shakeel Butt,
JP Kobryn, Ziyang Men, bpf, cgroups, linux-kselftest,
linux-kernel
Add cgroup_iter_cpu, a selftest for the CPU controller BPF kfuncs.
The userspace side runs a CPU hog in a test cgroup with cpu.max settled
then:
- checks the CPU-time and throttling counters are nonzero,
- compares whether all values the program read are same as those
reading from cgroup file.
Enable CONFIG_CGROUP_SCHED, CONFIG_FAIR_GROUP_SCHED and
CONFIG_CFS_BANDWIDTH in the selftest config.
Tested on VM with v7.2-rc5.
Suggested-by: Shakeel Butt <shakeel.butt@linux.dev>
Assisted-by: Claude:claude-opus-5
Signed-off-by: Ziyang Men <ziyang.meme@gmail.com>
---
tools/testing/selftests/bpf/cgroup_iter_cpu.h | 22 ++
tools/testing/selftests/bpf/config | 3 +
.../bpf/prog_tests/cgroup_iter_cpu.c | 259 ++++++++++++++++++
.../selftests/bpf/progs/cgroup_iter_cpu.c | 113 ++++++++
4 files changed, 397 insertions(+)
create mode 100644 tools/testing/selftests/bpf/cgroup_iter_cpu.h
create mode 100644 tools/testing/selftests/bpf/prog_tests/cgroup_iter_cpu.c
create mode 100644 tools/testing/selftests/bpf/progs/cgroup_iter_cpu.c
diff --git a/tools/testing/selftests/bpf/cgroup_iter_cpu.h b/tools/testing/selftests/bpf/cgroup_iter_cpu.h
new file mode 100644
index 000000000000..74599a5c0e4d
--- /dev/null
+++ b/tools/testing/selftests/bpf/cgroup_iter_cpu.h
@@ -0,0 +1,22 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/* Copyright (c) 2025 Meta Platforms, Inc. and affiliates. */
+#ifndef __CGROUP_ITER_CPU_H
+#define __CGROUP_ITER_CPU_H
+
+struct cpu_query {
+ /* base cpu time, from cpu.stat */
+ __u64 usage_usec;
+ __u64 user_usec;
+ __u64 system_usec;
+ __u64 nice_usec;
+ __u64 forceidle_usec;
+ /* CFS bandwidth throttling, from cpu.stat and cpu.stat.local */
+ __u64 nr_periods;
+ __u64 nr_throttled;
+ __u64 throttled_usec;
+ __u64 nr_bursts;
+ __u64 burst_usec;
+ __u64 throttled_self_usec;
+};
+
+#endif /* __CGROUP_ITER_CPU_H */
diff --git a/tools/testing/selftests/bpf/config b/tools/testing/selftests/bpf/config
index ea7044f30adc..482b40dde2f9 100644
--- a/tools/testing/selftests/bpf/config
+++ b/tools/testing/selftests/bpf/config
@@ -11,6 +11,9 @@ CONFIG_BPF_STREAM_PARSER=y
CONFIG_BPF_SYSCALL=y
# CONFIG_BPF_UNPRIV_DEFAULT_OFF is not set
CONFIG_CGROUP_BPF=y
+CONFIG_CGROUP_SCHED=y
+CONFIG_FAIR_GROUP_SCHED=y
+CONFIG_CFS_BANDWIDTH=y
CONFIG_CRYPTO_HMAC=y
CONFIG_CRYPTO_SHA256=y
CONFIG_CRYPTO_USER_API=y
diff --git a/tools/testing/selftests/bpf/prog_tests/cgroup_iter_cpu.c b/tools/testing/selftests/bpf/prog_tests/cgroup_iter_cpu.c
new file mode 100644
index 000000000000..cd7e92ababfb
--- /dev/null
+++ b/tools/testing/selftests/bpf/prog_tests/cgroup_iter_cpu.c
@@ -0,0 +1,259 @@
+// SPDX-License-Identifier: GPL-2.0
+/* Copyright (c) 2025 Meta Platforms, Inc. and affiliates. */
+#include <test_progs.h>
+#include <bpf/libbpf.h>
+#include <fcntl.h>
+#include <signal.h>
+#include <sys/prctl.h>
+#include <sys/wait.h>
+#include <unistd.h>
+#include "cgroup_helpers.h"
+#include "cgroup_iter_cpu.h"
+#include "cgroup_iter_cpu.skel.h"
+
+static int read_stats(struct bpf_link *link)
+{
+ int fd, ret = 0;
+ ssize_t bytes;
+
+ fd = bpf_iter_create(bpf_link__fd(link));
+ if (!ASSERT_OK_FD(fd, "bpf_iter_create"))
+ return 1;
+
+ bytes = read(fd, NULL, 0);
+ if (!ASSERT_EQ(bytes, 0, "read fd"))
+ ret = 1;
+
+ close(fd);
+ return ret;
+}
+
+/* Read cgroup file @name into @buf. */
+static int read_cgroup_file(int cgroup_fd, const char *name, char *buf,
+ size_t size)
+{
+ ssize_t n;
+ int fd;
+
+ fd = openat(cgroup_fd, name, O_RDONLY);
+ if (fd < 0)
+ return -1;
+ n = read(fd, buf, size - 1);
+ close(fd);
+ if (n <= 0)
+ return -1;
+ buf[n] = '\0';
+ return 0;
+}
+
+/* Parse the "cpu.stat" file into @out. */
+static int parse_cpu_stat(int cgroup_fd, struct cpu_query *out)
+{
+ char buf[4096], *line, *sp;
+ unsigned long long v;
+
+ if (read_cgroup_file(cgroup_fd, "cpu.stat", buf, sizeof(buf)))
+ return -1;
+
+ for (line = strtok_r(buf, "\n", &sp); line;
+ line = strtok_r(NULL, "\n", &sp)) {
+ if (sscanf(line, "usage_usec %llu", &v) == 1)
+ out->usage_usec = v;
+ else if (sscanf(line, "user_usec %llu", &v) == 1)
+ out->user_usec = v;
+ else if (sscanf(line, "system_usec %llu", &v) == 1)
+ out->system_usec = v;
+ else if (sscanf(line, "nice_usec %llu", &v) == 1)
+ out->nice_usec = v;
+ else if (sscanf(line, "core_sched.force_idle_usec %llu", &v) == 1)
+ out->forceidle_usec = v;
+ else if (sscanf(line, "nr_periods %llu", &v) == 1)
+ out->nr_periods = v;
+ else if (sscanf(line, "nr_throttled %llu", &v) == 1)
+ out->nr_throttled = v;
+ else if (sscanf(line, "throttled_usec %llu", &v) == 1)
+ out->throttled_usec = v;
+ else if (sscanf(line, "nr_bursts %llu", &v) == 1)
+ out->nr_bursts = v;
+ else if (sscanf(line, "burst_usec %llu", &v) == 1)
+ out->burst_usec = v;
+ }
+ return 0;
+}
+
+/*
+ * Parse the "cpu.stat.local" file into @out.
+ */
+static int parse_cpu_stat_local(int cgroup_fd, struct cpu_query *out)
+{
+ unsigned long long v;
+ char buf[256];
+
+ if (read_cgroup_file(cgroup_fd, "cpu.stat.local", buf, sizeof(buf)))
+ return -1;
+ if (sscanf(buf, "throttled_usec %llu", &v) != 1)
+ return -1;
+ out->throttled_self_usec = v;
+ return 0;
+}
+
+/* Read file value the bpf program reads. */
+static int parse_stats(int cgroup_fd, struct cpu_query *out, bool have_bw)
+{
+ if (parse_cpu_stat(cgroup_fd, out))
+ return -1;
+ if (have_bw && parse_cpu_stat_local(cgroup_fd, out))
+ return -1;
+ return 0;
+}
+
+/*
+ * Check whether this kernel accounts CFS bandwidth.
+ */
+static bool cgroup_has_bw_stat(int cgroup_fd)
+{
+ char buf[4096];
+
+ if (read_cgroup_file(cgroup_fd, "cpu.stat", buf, sizeof(buf)))
+ return false;
+ return strstr(buf, "nr_periods ");
+}
+
+/* Fork a child that spins in the current cgroup, kill it if the test exits. */
+static pid_t spawn_cpu_hog(void)
+{
+ pid_t pid = fork();
+
+ if (pid == 0) {
+ prctl(PR_SET_PDEATHSIG, SIGKILL);
+ while (1)
+ ;
+ }
+ return pid;
+}
+
+void test_cgroup_iter_cpu(void)
+{
+ char *cgroup_rel_path = "/cgroup_iter_cpu_test";
+ struct cgroup_iter_cpu *skel;
+ struct cpu_query *q;
+ struct bpf_link *link;
+ bool wrote_max, have_bw;
+ int cgroup_fd;
+ pid_t hog;
+
+ cgroup_fd = cgroup_setup_and_join(cgroup_rel_path);
+ if (!ASSERT_OK_FD(cgroup_fd, "cgroup_setup_and_join"))
+ return;
+
+ wrote_max = !write_cgroup_file(cgroup_rel_path, "cpu.max", "10000 100000");
+
+ skel = cgroup_iter_cpu__open_and_load();
+ if (!ASSERT_OK_PTR(skel, "cgroup_iter_cpu__open_and_load"))
+ goto cleanup_cgroup_fd;
+
+ DECLARE_LIBBPF_OPTS(bpf_iter_attach_opts, opts);
+ union bpf_iter_link_info linfo = {
+ .cgroup.cgroup_fd = cgroup_fd,
+ .cgroup.order = BPF_CGROUP_ITER_SELF_ONLY,
+ };
+ opts.link_info = &linfo;
+ opts.link_info_len = sizeof(linfo);
+
+ link = bpf_program__attach_iter(skel->progs.cgroup_cpu_query, &opts);
+ if (!ASSERT_OK_PTR(link, "bpf_program__attach_iter"))
+ goto cleanup_skel;
+
+ q = &skel->data_query->cpu_query;
+
+ hog = spawn_cpu_hog();
+ if (!ASSERT_GT(hog, 0, "spawn_cpu_hog"))
+ goto cleanup_link;
+
+ sleep(1);
+
+ /* Run the bpf program before anything here reads cpu.stat. */
+ if (!ASSERT_OK(read_stats(link), "read stats"))
+ goto cleanup_hog;
+
+ have_bw = wrote_max && cgroup_has_bw_stat(cgroup_fd);
+
+ if (test__start_subtest("cgroup_iter_cpu__cputime")) {
+ ASSERT_GT(q->usage_usec, 0, "usage_usec");
+ ASSERT_GT(q->user_usec + q->system_usec, 0, "user+system_usec");
+ }
+ if (test__start_subtest("cgroup_iter_cpu__throttling")) {
+ if (!have_bw) {
+ test__skip();
+ } else {
+ ASSERT_GT(q->nr_periods, 0, "nr_periods");
+ ASSERT_GT(q->nr_throttled, 0, "nr_throttled");
+ ASSERT_GT(q->throttled_usec, 0, "throttled_usec");
+ ASSERT_GT(q->throttled_self_usec, 0, "throttled_self_usec");
+ }
+ }
+
+ /*
+ * cpu.stat cputime grows on every tick a task in the cgroup runs, so
+ * stop them all before comparing
+ */
+ if (test__start_subtest("cgroup_iter_cpu__match")) {
+ struct cpu_query filev = {};
+ int i, stable = 0;
+
+ kill(hog, SIGSTOP);
+ waitpid(hog, NULL, WUNTRACED);
+ if (!ASSERT_OK(join_root_cgroup(), "join_root_cgroup"))
+ goto cleanup_hog;
+
+ /*
+ * The period timer keeps adding to nr_periods for a while
+ * after the hog stops
+ */
+ for (i = 0; i < 20; i++) {
+ struct cpu_query before = {}, after = {};
+
+ if (!ASSERT_OK(parse_stats(cgroup_fd, &before, have_bw), "cpu.stat") ||
+ !ASSERT_OK(read_stats(link), "read stats") ||
+ !ASSERT_OK(parse_stats(cgroup_fd, &after, have_bw), "cpu.stat"))
+ goto cleanup_hog;
+
+ if (!memcmp(&before, &after, sizeof(before))) {
+ filev = before;
+ stable = 1;
+ break;
+ }
+ usleep(100000);
+ }
+
+ if (!ASSERT_TRUE(stable, "cpu.stat stable"))
+ goto cleanup_hog;
+
+ ASSERT_EQ(q->usage_usec, filev.usage_usec, "usage_usec");
+ ASSERT_EQ(q->user_usec, filev.user_usec, "user_usec");
+ ASSERT_EQ(q->system_usec, filev.system_usec, "system_usec");
+ ASSERT_EQ(q->nice_usec, filev.nice_usec, "nice_usec");
+ ASSERT_EQ(q->forceidle_usec, filev.forceidle_usec, "forceidle_usec");
+
+ if (have_bw) {
+ ASSERT_EQ(q->nr_periods, filev.nr_periods, "nr_periods");
+ ASSERT_EQ(q->nr_throttled, filev.nr_throttled, "nr_throttled");
+ ASSERT_EQ(q->throttled_usec, filev.throttled_usec, "throttled_usec");
+ ASSERT_EQ(q->nr_bursts, filev.nr_bursts, "nr_bursts");
+ ASSERT_EQ(q->burst_usec, filev.burst_usec, "burst_usec");
+ ASSERT_EQ(q->throttled_self_usec, filev.throttled_self_usec,
+ "throttled_self_usec");
+ }
+ }
+
+cleanup_hog:
+ kill(hog, SIGKILL);
+ waitpid(hog, NULL, 0);
+cleanup_link:
+ bpf_link__destroy(link);
+cleanup_skel:
+ cgroup_iter_cpu__destroy(skel);
+cleanup_cgroup_fd:
+ close(cgroup_fd);
+ cleanup_cgroup_environment();
+}
diff --git a/tools/testing/selftests/bpf/progs/cgroup_iter_cpu.c b/tools/testing/selftests/bpf/progs/cgroup_iter_cpu.c
new file mode 100644
index 000000000000..22b9bb62d910
--- /dev/null
+++ b/tools/testing/selftests/bpf/progs/cgroup_iter_cpu.c
@@ -0,0 +1,113 @@
+// SPDX-License-Identifier: GPL-2.0
+/* Copyright (c) 2025 Meta Platforms, Inc. and affiliates. */
+#include <vmlinux.h>
+#include <bpf/bpf_helpers.h>
+#include <bpf/bpf_core_read.h>
+#include "cgroup_iter_cpu.h"
+
+char _license[] SEC("license") = "GPL";
+
+struct cpu_query cpu_query SEC(".data.query");
+
+extern const void __cpu_possible_mask __ksym;
+
+struct cgroup_base_stat___local {
+ struct task_cputime cputime;
+ __u64 forceidle_sum;
+ __u64 ntime;
+} __attribute__((preserve_access_index));
+
+static __always_inline __u64 read_throttled_self(struct task_group *tg, __u32 cpu)
+{
+ struct cfs_rq *cfs_rq;
+
+ cfs_rq = bpf_per_cpu_ptr(tg->cfs_rq, cpu);
+ if (!cfs_rq)
+ return 0;
+
+ return BPF_CORE_READ(cfs_rq, throttled_clock_self_time);
+}
+
+SEC("iter.s/cgroup")
+int cgroup_cpu_query(struct bpf_iter__cgroup *ctx)
+{
+ struct cgroup_base_stat___local bstat = {};
+ struct cgroup *cgrp = ctx->cgroup;
+ struct cgroup_subsys_state *css;
+ struct task_group *tg;
+ __u64 throttled_self = 0;
+ int ssid;
+
+ if (!cgrp)
+ return 1;
+
+ bpf_css_flush_rstat(&cgrp->self);
+ bpf_cgroup_base_stat(cgrp, (struct cgroup_base_stat *)&bstat);
+
+ cpu_query.usage_usec = bstat.cputime.sum_exec_runtime / 1000;
+ cpu_query.user_usec = bstat.cputime.utime / 1000;
+ cpu_query.system_usec = bstat.cputime.stime / 1000;
+ cpu_query.nice_usec = bstat.ntime / 1000;
+ cpu_query.forceidle_usec = 0;
+ if (bpf_core_field_exists(bstat.forceidle_sum))
+ cpu_query.forceidle_usec = bstat.forceidle_sum / 1000;
+
+ bpf_rcu_read_lock();
+ if (!bpf_core_enum_value_exists(enum cgroup_subsys_id, cpu_cgrp_id) ||
+ !bpf_ksym_exists(bpf_css_to_task_group))
+ goto unlock;
+
+ ssid = bpf_core_enum_value(enum cgroup_subsys_id, cpu_cgrp_id);
+ css = cgrp->subsys[ssid];
+ if (!css)
+ goto unlock;
+
+ tg = bpf_css_to_task_group(css);
+ if (tg && bpf_core_field_exists(tg->cfs_bandwidth.nr_periods)) {
+ cpu_query.nr_periods =
+ (__u32)BPF_CORE_READ(tg, cfs_bandwidth.nr_periods);
+ cpu_query.nr_throttled =
+ (__u32)BPF_CORE_READ(tg, cfs_bandwidth.nr_throttled);
+ cpu_query.throttled_usec =
+ BPF_CORE_READ(tg, cfs_bandwidth.throttled_time) / 1000;
+ cpu_query.nr_bursts =
+ (__u32)BPF_CORE_READ(tg, cfs_bandwidth.nr_burst);
+ cpu_query.burst_usec =
+ BPF_CORE_READ(tg, cfs_bandwidth.burst_time) / 1000;
+ }
+
+ if (tg && bpf_core_field_exists(tg->cfs_rq) &&
+ bpf_core_field_exists(struct cfs_rq, throttled_clock_self_time)) {
+ __u32 mask_bytes = bpf_core_type_size(struct cpumask);
+ __u32 full_words = mask_bytes / sizeof(__u64);
+ int *cpu;
+
+ if (full_words)
+ bpf_for_each(bits, cpu,
+ (const __u64 *)&__cpu_possible_mask,
+ full_words)
+ throttled_self += read_throttled_self(tg, *cpu);
+
+ if (mask_bytes & (sizeof(__u64) - 1)) {
+ __u32 tail = 0;
+ const void *src = (const char *)&__cpu_possible_mask +
+ full_words * sizeof(__u64);
+ int bit;
+
+ if (!bpf_probe_read_kernel(&tail, sizeof(tail), src))
+ bpf_for(bit, 0, 32)
+ if (tail & (1U << bit)) {
+ __u32 tail_cpu = full_words * 64 + bit;
+
+ throttled_self +=
+ read_throttled_self(tg, tail_cpu);
+ }
+ }
+ }
+
+unlock:
+ bpf_rcu_read_unlock();
+ cpu_query.throttled_self_usec = throttled_self / 1000;
+
+ return 0;
+}
--
2.53.0-Meta
^ permalink raw reply related [flat|nested] 6+ messages in thread