[PATCH v3 1/2] mm/damon/core: optimize kdamond_apply_schemes() by inverting scheme and region loops

Josh Law
2026-03-22T21:43:32Z

Currently, kdamond_apply_schemes() iterates over all targets, then over all
regions, and finally calls damon_do_apply_schemes() which iterates over
all schemes. This nested structure causes scheme-level invariants (such as
time intervals, activation status, and quota limits) to be evaluated inside
the innermost loop for every single region.

If a scheme is inactive, has not reached its apply interval, or has already
fulfilled its quota (quota->charged_sz >= quota->esz), the kernel still
needlessly iterates through thousands of regions only to repeatedly
evaluate these same scheme-level conditions and continue.

This patch inlines damon_do_apply_schemes() into kdamond_apply_schemes()
and inverts the loop ordering. It now iterates over schemes on the outside,
and targets/regions on the inside.

This allows the code to evaluate scheme-level limits once per scheme.
If a scheme's quota is met or it is inactive, we completely bypass the
O(Targets * Regions) inner loop for that scheme. This drastically reduces
unnecessary branching, cache thrashing, and CPU overhead in the kdamond
hot path.

Signed-off-by: Josh Law <objecting@objecting.org>
---
mm/damon/core.c | 72 +++++++++++++++++++++----------------------------
1 file changed, 30 insertions(+), 42 deletions(-)

diff --git a/mm/damon/core.c b/mm/damon/core.c
index c884bb31c9b8..a9cfbd6ce3d4 100644
--- a/mm/damon/core.c
+++ b/mm/damon/core.c
@@ -2112,40 +2112,6 @@ static void damos_apply_scheme(struct damon_ctx *c, struct damon_target *t,
damos_update_stat(s, sz, sz_applied, sz_ops_filter_passed);
}

-static void damon_do_apply_schemes(struct damon_ctx *c,
- struct damon_target *t,
- struct damon_region *r)
-{
- struct damos *s;
-
- damon_for_each_scheme(s, c) {
- struct damos_quota *quota = &s->quota;
-
- if (time_before(c->passed_sample_intervals, s->next_apply_sis))
- continue;
-
- if (!s->wmarks.activated)
- continue;
-
- /* Check the quota */
- if (quota->esz && quota->charged_sz >= quota->esz)
- continue;
-
- if (damos_skip_charged_region(t, r, s, c->min_region_sz))
- continue;
-
- if (s->max_nr_snapshots &&
- s->max_nr_snapshots <= s->stat.nr_snapshots)
- continue;
-
- if (damos_valid_target(c, r, s))
- damos_apply_scheme(c, t, r, s);
-
- if (damon_is_last_region(r, t))
- s->stat.nr_snapshots++;
- }
-}
-
/*
* damon_feed_loop_next_input() - get next input to achieve a target score.
* @last_input The last input.
@@ -2494,17 +2460,39 @@ static void kdamond_apply_schemes(struct damon_ctx *c)
return;

mutex_lock(&c->walk_control_lock);
- damon_for_each_target(t, c) {
- if (c->ops.target_valid && c->ops.target_valid(t) == false)
- continue;
-
- damon_for_each_region(r, t)
- damon_do_apply_schemes(c, t, r);
- }
-
damon_for_each_scheme(s, c) {
+ struct damos_quota *quota = &s->quota;
+
if (time_before(c->passed_sample_intervals, s->next_apply_sis))
continue;
+
+ if (!s->wmarks.activated)
+ continue;
+
+ damon_for_each_target(t, c) {
+ if (c->ops.target_valid && c->ops.target_valid(t) == false)
+ continue;
+
+ damon_for_each_region(r, t) {
+ /* Check the quota */
+ if (quota->esz && quota->charged_sz >= quota->esz)
+ goto next_scheme;
+
+ if (s->max_nr_snapshots &&
+ s->max_nr_snapshots <= s->stat.nr_snapshots)
+ goto next_scheme;
+
+ if (damos_skip_charged_region(t, r, s, c->min_region_sz))
+ continue;
+
+ if (damos_valid_target(c, r, s))
+ damos_apply_scheme(c, t, r, s);
+
+ if (damon_is_last_region(r, t))
+ s->stat.nr_snapshots++;
+ }
+ }
+next_scheme:
damos_walk_complete(c, s);
damos_set_next_apply_sis(s, c);
s->last_applied = NULL;
--
2.34.1