From mboxrd@z Thu Jan 1 00:00:00 1970 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-1.web.codeaurora.org [10.30.226.201]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 499533A9605 for ; Tue, 24 Feb 2026 16:38:36 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=10.30.226.201 ARC-Seal:i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1771951116; cv=none; b=ajbtXaROeaRxbqzzrNWiKWns6s6BbdiXXgj6+i8RIR/luD/MQTyq57qGCPO0x/euQIirnXid9sVXuY47lVkomAKaePkuQm5EVEM2TR1UQeDh5t8VvErF8o9Mgd82oP8XmLSiLjmDnh3AVzZZXxndrVLuZWYEfog9DL/WHWUa9Zc= ARC-Message-Signature:i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1771951116; c=relaxed/simple; bh=sjCiGHHw3GKDHbj+ZS6NFRyqQGela1LvA8s4PzLn/V8=; h=Date:Message-ID:From:To:Cc:Subject:References:MIME-Version: Content-Type; b=I14nHZrzuQv4Kv3yWOOEVAYBnvrnJDkdjjBDE4RA0YRhg4JpZDXSChrVkVNRvsCeUe25fRLncEzpiJ/qbXkKpuDssBLYkOQMTY/f8mKa68xsXlr3DvOQrfratpxnbii8fR0fYR1b84TdAjMTVaf8U5YFg3/+nH6KXlY4ifZ41es= ARC-Authentication-Results:i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=KIirvjmC; arc=none smtp.client-ip=10.30.226.201 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="KIirvjmC" Received: by smtp.kernel.org (Postfix) with ESMTPSA id AC62DC116D0; Tue, 24 Feb 2026 16:38:35 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=kernel.org; s=k20201202; t=1771951116; bh=sjCiGHHw3GKDHbj+ZS6NFRyqQGela1LvA8s4PzLn/V8=; h=Date:From:To:Cc:Subject:References:From; b=KIirvjmC9nQDYsj8SuehyjFxBUlQ5ZqrJyNUYP1GdvcO8w6Rl0yuqul6ZNPUMrVpS 56EjLMySG75ibCmDcAgwz4hiD1I3WqA54xGWyROqSPSVGn0jdw+cO/8V02bH3bRU+G zdTZk468J2T3F+M6THUBvJWlXmuX0QigWosQ0N3Qp3Jjo5Ju8hy4WwptpMEjNG6vp7 uPC37tEDJA+jjZqeHcmLo1eJ22tmV/2CPSP7UYl8BCvHY2HX1eQNmZrQ00uOES4q9M 80DgA0GmS861NSF9CCgu2fJNEz9I/oZM1zgkWOrl84VdoULZRINSCfQzO3BbK5heK0 jogIFecYRaPmA== Date: Tue, 24 Feb 2026 17:38:33 +0100 Message-ID: <20260224163431.468186893@kernel.org> User-Agent: quilt/0.68 From: Thomas Gleixner To: LKML Cc: Anna-Maria Behnsen , John Stultz , Stephen Boyd , Daniel Lezcano , Juri Lelli , Vincent Guittot , Dietmar Eggemann , Steven Rostedt , Ben Segall , Mel Gorman , Valentin Schneider , x86@kernel.org, Peter Zijlstra , Frederic Weisbecker , Eric Dumazet Subject: [patch 41/48] hrtimer: Rework next event evaluation References: <20260224163022.795809588@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 The per clock base cached expiry time allows to do a more efficient evaluation of the next expiry on a CPU. Separate the reprogramming evaluation from the NOHZ idle evaluation which needs to exclude the NOHZ timer to keep the reprogramming path lean and clean. Signed-off-by: Thomas Gleixner --- kernel/time/hrtimer.c | 120 ++++++++++++++++++++++++++++---------------------- 1 file changed, 69 insertions(+), 51 deletions(-) --- a/kernel/time/hrtimer.c +++ b/kernel/time/hrtimer.c @@ -546,49 +546,67 @@ static struct hrtimer_clock_base * #define for_each_active_base(base, cpu_base, active) \ while ((base = __next_base((cpu_base), &(active)))) -static ktime_t __hrtimer_next_event_base(struct hrtimer_cpu_base *cpu_base, - const struct hrtimer *exclude, - unsigned int active, ktime_t expires_next) +#if defined(CONFIG_NO_HZ_COMMON) +/* + * Same as hrtimer_bases_next_event() below, but skips the excluded timer and + * does not update cpu_base->next_timer/expires. + */ +static ktime_t hrtimer_bases_next_event_without(struct hrtimer_cpu_base *cpu_base, + const struct hrtimer *exclude, + unsigned int active, ktime_t expires_next) { struct hrtimer_clock_base *base; ktime_t expires; + lockdep_assert_held(&cpu_base->lock); + for_each_active_base(base, cpu_base, active) { - struct timerqueue_node *next; - struct hrtimer *timer; + expires = ktime_sub(base->expires_next, base->offset); + if (expires >= expires_next) + continue; - next = timerqueue_getnext(&base->active); - timer = container_of(next, struct hrtimer, node); - if (timer == exclude) { - /* Get to the next timer in the queue. */ - next = timerqueue_iterate_next(next); - if (!next) - continue; + /* + * If the excluded timer is the first on this base evaluate the + * next timer. + */ + struct timerqueue_node *node = timerqueue_getnext(&base->active); - timer = container_of(next, struct hrtimer, node); + if (unlikely(&exclude->node == node)) { + node = timerqueue_iterate_next(node); + if (!node) + continue; + expires = ktime_sub(node->expires, base->offset); + if (expires >= expires_next) + continue; } - expires = ktime_sub(hrtimer_get_expires(timer), base->offset); - if (expires < expires_next) { - expires_next = expires; + expires_next = expires; + } + /* If base->offset changed, the result might be negative */ + return max(expires_next, 0); +} +#endif - /* Skip cpu_base update if a timer is being excluded. */ - if (exclude) - continue; +static __always_inline struct hrtimer *clock_base_next_timer(struct hrtimer_clock_base *base) +{ + struct timerqueue_node *next = timerqueue_getnext(&base->active); + + return container_of(next, struct hrtimer, node); +} - if (timer->is_soft) - cpu_base->softirq_next_timer = timer; - else - cpu_base->next_timer = timer; +/* Find the base with the earliest expiry */ +static void hrtimer_bases_first(struct hrtimer_cpu_base *cpu_base,unsigned int active, + ktime_t *expires_next, struct hrtimer **next_timer) +{ + struct hrtimer_clock_base *base; + ktime_t expires; + + for_each_active_base(base, cpu_base, active) { + expires = ktime_sub(base->expires_next, base->offset); + if (expires < *expires_next) { + *expires_next = expires; + *next_timer = clock_base_next_timer(base); } } - /* - * clock_was_set() might have changed base->offset of any of - * the clock bases so the result might be negative. Fix it up - * to prevent a false positive in clockevents_program_event(). - */ - if (expires_next < 0) - expires_next = 0; - return expires_next; } /* @@ -617,19 +635,22 @@ static ktime_t __hrtimer_get_next_event( ktime_t expires_next = KTIME_MAX; unsigned int active; + lockdep_assert_held(&cpu_base->lock); + if (!cpu_base->softirq_activated && (active_mask & HRTIMER_ACTIVE_SOFT)) { active = cpu_base->active_bases & HRTIMER_ACTIVE_SOFT; - cpu_base->softirq_next_timer = NULL; - expires_next = __hrtimer_next_event_base(cpu_base, NULL, active, KTIME_MAX); - next_timer = cpu_base->softirq_next_timer; + if (active) + hrtimer_bases_first(cpu_base, active, &expires_next, &next_timer); + cpu_base->softirq_next_timer = next_timer; } if (active_mask & HRTIMER_ACTIVE_HARD) { active = cpu_base->active_bases & HRTIMER_ACTIVE_HARD; + if (active) + hrtimer_bases_first(cpu_base, active, &expires_next, &next_timer); cpu_base->next_timer = next_timer; - expires_next = __hrtimer_next_event_base(cpu_base, NULL, active, expires_next); } - return expires_next; + return max(expires_next, 0); } static ktime_t hrtimer_update_next_event(struct hrtimer_cpu_base *cpu_base) @@ -724,11 +745,7 @@ static void __hrtimer_reprogram(struct h hrtimer_rearm_event(expires_next, false); } -/* - * Reprogram the event source with checking both queues for the - * next event - * Called with interrupts disabled and base->lock held - */ +/* Reprogram the event source with a evaluation of all clock bases */ static void hrtimer_force_reprogram(struct hrtimer_cpu_base *cpu_base, bool skip_equal) { ktime_t expires_next = hrtimer_update_next_event(cpu_base); @@ -1662,19 +1679,20 @@ u64 hrtimer_next_event_without(const str { struct hrtimer_cpu_base *cpu_base = this_cpu_ptr(&hrtimer_bases); u64 expires = KTIME_MAX; + unsigned int active; guard(raw_spinlock_irqsave)(&cpu_base->lock); - if (hrtimer_hres_active(cpu_base)) { - unsigned int active; + if (!hrtimer_hres_active(cpu_base)) + return expires; - if (!cpu_base->softirq_activated) { - active = cpu_base->active_bases & HRTIMER_ACTIVE_SOFT; - expires = __hrtimer_next_event_base(cpu_base, exclude, active, KTIME_MAX); - } - active = cpu_base->active_bases & HRTIMER_ACTIVE_HARD; - expires = __hrtimer_next_event_base(cpu_base, exclude, active, expires); - } - return expires; + active = cpu_base->active_bases & HRTIMER_ACTIVE_SOFT; + if (active && !cpu_base->softirq_activated) + expires = hrtimer_bases_next_event_without(cpu_base, exclude, active, KTIME_MAX); + + active = cpu_base->active_bases & HRTIMER_ACTIVE_HARD; + if (!active) + return expires; + return hrtimer_bases_next_event_without(cpu_base, exclude, active, expires); } #endif