[PATCH v3 0/3] locking/lockdep: Disable KASAN instrumentation of lockdep.c

[PATCH v3 0/3] locking/lockdep: Disable KASAN instrumentation of lockdep.c

[Waiman Long]

 v3: 
  - Add another patch to insert lock events into lockdep.c.
  - Rerun all the tests with the simpler defconfig kernel build and do
    further analysis of the of the performance difference between the
    the RT and non-RT debug kernels.

It is found that disabling KASAN instrumentation when compiling
lockdep.c can significantly improve the performance of RT debug kernel
while the performance benefit of non-RT debug kernel is relatively
modest.

This series also include patches to add locking events to the rtmutex
slow paths and the lockdep code for better analysis of the different
performance behavior between RT and non-RT debug kernels.

Waiman Long (3):
  locking/lock_events: Add locking events for rtmutex slow paths
  locking/lock_events: Add locking events for lockdep
  locking/lockdep: Disable KASAN instrumentation of lockdep.c

 kernel/locking/Makefile           |  3 ++-
 kernel/locking/lock_events_list.h | 28 ++++++++++++++++++++++++++++
 kernel/locking/lockdep.c          |  8 +++++++-
 kernel/locking/rtmutex.c          | 29 ++++++++++++++++++++++++-----
 4 files changed, 61 insertions(+), 7 deletions(-)

-- 
2.48.1

[PATCH v3 1/3] locking/lock_events: Add locking events for rtmutex slow paths

[Waiman Long]

Add locking events for rtlock_slowlock() and rt_mutex_slowlock() for
profiling the slow path behavior of rt_spin_lock() and rt_mutex_lock().

Signed-off-by: Waiman Long <longman@redhat.com>
---
 kernel/locking/lock_events_list.h | 21 +++++++++++++++++++++
 kernel/locking/rtmutex.c          | 29 ++++++++++++++++++++++++-----
 2 files changed, 45 insertions(+), 5 deletions(-)

diff --git a/kernel/locking/lock_events_list.h b/kernel/locking/lock_events_list.h
index 97fb6f3f840a..80b11f194c9f 100644
--- a/kernel/locking/lock_events_list.h
+++ b/kernel/locking/lock_events_list.h
@@ -67,3 +67,24 @@ LOCK_EVENT(rwsem_rlock_handoff)	/* # of read lock handoffs		*/
 LOCK_EVENT(rwsem_wlock)		/* # of write locks acquired		*/
 LOCK_EVENT(rwsem_wlock_fail)	/* # of failed write lock acquisitions	*/
 LOCK_EVENT(rwsem_wlock_handoff)	/* # of write lock handoffs		*/
+
+/*
+ * Locking events for rtlock_slowlock()
+ */
+LOCK_EVENT(rtlock_slowlock)	/* # of rtlock_slowlock() calls		*/
+LOCK_EVENT(rtlock_slow_acq1)	/* # of locks acquired after wait_lock	*/
+LOCK_EVENT(rtlock_slow_acq2)	/* # of locks acquired in for loop	*/
+LOCK_EVENT(rtlock_slow_sleep)	/* # of sleeps				*/
+LOCK_EVENT(rtlock_slow_wake)	/* # of wakeup's			*/
+
+/*
+ * Locking events for rt_mutex_slowlock()
+ */
+LOCK_EVENT(rtmutex_slowlock)	/* # of rt_mutex_slowlock() calls	*/
+LOCK_EVENT(rtmutex_slow_block)	/* # of rt_mutex_slowlock_block() calls	*/
+LOCK_EVENT(rtmutex_slow_acq1)	/* # of locks acquired after wait_lock	*/
+LOCK_EVENT(rtmutex_slow_acq2)	/* # of locks acquired at the end	*/
+LOCK_EVENT(rtmutex_slow_acq3)	/* # of locks acquired in *block()	*/
+LOCK_EVENT(rtmutex_slow_sleep)	/* # of sleeps				*/
+LOCK_EVENT(rtmutex_slow_wake)	/* # of wakeup's			*/
+LOCK_EVENT(rtmutex_deadlock)	/* # of rt_mutex_handle_deadlock()'s	*/
diff --git a/kernel/locking/rtmutex.c b/kernel/locking/rtmutex.c
index 4a8df1800cbb..c80902eacd79 100644
--- a/kernel/locking/rtmutex.c
+++ b/kernel/locking/rtmutex.c
@@ -27,6 +27,7 @@
 #include <trace/events/lock.h>
 
 #include "rtmutex_common.h"
+#include "lock_events.h"
 
 #ifndef WW_RT
 # define build_ww_mutex()	(false)
@@ -1612,10 +1613,13 @@ static int __sched rt_mutex_slowlock_block(struct rt_mutex_base *lock,
 	struct task_struct *owner;
 	int ret = 0;
 
+	lockevent_inc(rtmutex_slow_block);
 	for (;;) {
 		/* Try to acquire the lock: */
-		if (try_to_take_rt_mutex(lock, current, waiter))
+		if (try_to_take_rt_mutex(lock, current, waiter)) {
+			lockevent_inc(rtmutex_slow_acq3);
 			break;
+		}
 
 		if (timeout && !timeout->task) {
 			ret = -ETIMEDOUT;
@@ -1638,8 +1642,10 @@ static int __sched rt_mutex_slowlock_block(struct rt_mutex_base *lock,
 			owner = NULL;
 		raw_spin_unlock_irq_wake(&lock->wait_lock, wake_q);
 
-		if (!owner || !rtmutex_spin_on_owner(lock, waiter, owner))
+		if (!owner || !rtmutex_spin_on_owner(lock, waiter, owner)) {
+			lockevent_inc(rtmutex_slow_sleep);
 			rt_mutex_schedule();
+		}
 
 		raw_spin_lock_irq(&lock->wait_lock);
 		set_current_state(state);
@@ -1694,6 +1700,7 @@ static int __sched __rt_mutex_slowlock(struct rt_mutex_base *lock,
 	int ret;
 
 	lockdep_assert_held(&lock->wait_lock);
+	lockevent_inc(rtmutex_slowlock);
 
 	/* Try to acquire the lock again: */
 	if (try_to_take_rt_mutex(lock, current, NULL)) {
@@ -1701,6 +1708,7 @@ static int __sched __rt_mutex_slowlock(struct rt_mutex_base *lock,
 			__ww_mutex_check_waiters(rtm, ww_ctx, wake_q);
 			ww_mutex_lock_acquired(ww, ww_ctx);
 		}
+		lockevent_inc(rtmutex_slow_acq1);
 		return 0;
 	}
 
@@ -1719,10 +1727,12 @@ static int __sched __rt_mutex_slowlock(struct rt_mutex_base *lock,
 				__ww_mutex_check_waiters(rtm, ww_ctx, wake_q);
 			ww_mutex_lock_acquired(ww, ww_ctx);
 		}
+		lockevent_inc(rtmutex_slow_acq2);
 	} else {
 		__set_current_state(TASK_RUNNING);
 		remove_waiter(lock, waiter);
 		rt_mutex_handle_deadlock(ret, chwalk, lock, waiter);
+		lockevent_inc(rtmutex_deadlock);
 	}
 
 	/*
@@ -1751,6 +1761,7 @@ static inline int __rt_mutex_slowlock_locked(struct rt_mutex_base *lock,
 				  &waiter, wake_q);
 
 	debug_rt_mutex_free_waiter(&waiter);
+	lockevent_cond_inc(rtmutex_slow_wake, !wake_q_empty(wake_q));
 	return ret;
 }
 
@@ -1823,9 +1834,12 @@ static void __sched rtlock_slowlock_locked(struct rt_mutex_base *lock,
 	struct task_struct *owner;
 
 	lockdep_assert_held(&lock->wait_lock);
+	lockevent_inc(rtlock_slowlock);
 
-	if (try_to_take_rt_mutex(lock, current, NULL))
+	if (try_to_take_rt_mutex(lock, current, NULL)) {
+		lockevent_inc(rtlock_slow_acq1);
 		return;
+	}
 
 	rt_mutex_init_rtlock_waiter(&waiter);
 
@@ -1838,8 +1852,10 @@ static void __sched rtlock_slowlock_locked(struct rt_mutex_base *lock,
 
 	for (;;) {
 		/* Try to acquire the lock again */
-		if (try_to_take_rt_mutex(lock, current, &waiter))
+		if (try_to_take_rt_mutex(lock, current, &waiter)) {
+			lockevent_inc(rtlock_slow_acq2);
 			break;
+		}
 
 		if (&waiter == rt_mutex_top_waiter(lock))
 			owner = rt_mutex_owner(lock);
@@ -1847,8 +1863,10 @@ static void __sched rtlock_slowlock_locked(struct rt_mutex_base *lock,
 			owner = NULL;
 		raw_spin_unlock_irq_wake(&lock->wait_lock, wake_q);
 
-		if (!owner || !rtmutex_spin_on_owner(lock, &waiter, owner))
+		if (!owner || !rtmutex_spin_on_owner(lock, &waiter, owner)) {
+			lockevent_inc(rtlock_slow_sleep);
 			schedule_rtlock();
+		}
 
 		raw_spin_lock_irq(&lock->wait_lock);
 		set_current_state(TASK_RTLOCK_WAIT);
@@ -1865,6 +1883,7 @@ static void __sched rtlock_slowlock_locked(struct rt_mutex_base *lock,
 	debug_rt_mutex_free_waiter(&waiter);
 
 	trace_contention_end(lock, 0);
+	lockevent_cond_inc(rtlock_slow_wake, !wake_q_empty(wake_q));
 }
 
 static __always_inline void __sched rtlock_slowlock(struct rt_mutex_base *lock)
-- 
2.48.1

[PATCH v3 2/3] locking/lock_events: Add locking events for lockdep

[Waiman Long]

Add some lock events to the lockdep for profiling its behavior.

Signed-off-by: Waiman Long <longman@redhat.com>
---
 kernel/locking/lock_events_list.h | 7 +++++++
 kernel/locking/lockdep.c          | 8 +++++++-
 2 files changed, 14 insertions(+), 1 deletion(-)

diff --git a/kernel/locking/lock_events_list.h b/kernel/locking/lock_events_list.h
index 80b11f194c9f..9ef9850aeebe 100644
--- a/kernel/locking/lock_events_list.h
+++ b/kernel/locking/lock_events_list.h
@@ -88,3 +88,10 @@ LOCK_EVENT(rtmutex_slow_acq3)	/* # of locks acquired in *block()	*/
 LOCK_EVENT(rtmutex_slow_sleep)	/* # of sleeps				*/
 LOCK_EVENT(rtmutex_slow_wake)	/* # of wakeup's			*/
 LOCK_EVENT(rtmutex_deadlock)	/* # of rt_mutex_handle_deadlock()'s	*/
+
+/*
+ * Locking events for lockdep
+ */
+LOCK_EVENT(lockdep_acquire)
+LOCK_EVENT(lockdep_lock)
+LOCK_EVENT(lockdep_nocheck)
diff --git a/kernel/locking/lockdep.c b/kernel/locking/lockdep.c
index 4470680f0226..8436f017c74d 100644
--- a/kernel/locking/lockdep.c
+++ b/kernel/locking/lockdep.c
@@ -61,6 +61,7 @@
 #include <asm/sections.h>
 
 #include "lockdep_internals.h"
+#include "lock_events.h"
 
 #include <trace/events/lock.h>
 
@@ -170,6 +171,7 @@ static struct task_struct *lockdep_selftest_task_struct;
 static int graph_lock(void)
 {
 	lockdep_lock();
+	lockevent_inc(lockdep_lock);
 	/*
 	 * Make sure that if another CPU detected a bug while
 	 * walking the graph we dont change it (while the other
@@ -5091,8 +5093,12 @@ static int __lock_acquire(struct lockdep_map *lock, unsigned int subclass,
 	if (unlikely(lock->key == &__lockdep_no_track__))
 		return 0;
 
-	if (!prove_locking || lock->key == &__lockdep_no_validate__)
+	lockevent_inc(lockdep_acquire);
+
+	if (!prove_locking || lock->key == &__lockdep_no_validate__) {
 		check = 0;
+		lockevent_inc(lockdep_nocheck);
+	}
 
 	if (subclass < NR_LOCKDEP_CACHING_CLASSES)
 		class = lock->class_cache[subclass];
-- 
2.48.1

[PATCH v3 3/3] locking/lockdep: Disable KASAN instrumentation of lockdep.c

[Waiman Long]

Both KASAN and LOCKDEP are commonly enabled in building a debug kernel.
Each of them can significantly slow down the speed of a debug kernel.
Enabling KASAN instrumentation of the LOCKDEP code will further slow
thing down.

Since LOCKDEP is a high overhead debugging tool, it will never get
enabled in a production kernel. The LOCKDEP code is also pretty mature
and is unlikely to get major changes. There is also a possibility of
recursion similar to KCSAN.

To evaluate the performance impact of disabling KASAN instrumentation
of lockdep.c, the time to do a parallel build of the Linux defconfig
kernel was used as the benchmark. Two x86-64 systems (Skylake & Zen 2)
and an arm64 system were used as test beds. Two sets of non-RT and RT
kernels with similar configurations except mainly CONFIG_PREEMPT_RT
were used for evaulation.

For the Skylake system:

  Kernel			Run time	    Sys time
  ------			--------	    --------
  Non-debug kernel (baseline)	0m47.642s	      4m19.811s
  Debug kernel			2m11.108s (x2.8)     38m20.467s (x8.9)
  Debug kernel (patched)	1m49.602s (x2.3)     31m28.501s (x7.3)
  Debug kernel
  (patched + mitigations=off) 	1m30.988s (x1.9)     26m41.993s (x6.2)

  RT kernel (baseline)		0m54.871s	      7m15.340s
  RT debug kernel		6m07.151s (x6.7)    135m47.428s (x18.7)
  RT debug kernel (patched)	3m42.434s (x4.1)     74m51.636s (x10.3)
  RT debug kernel
  (patched + mitigations=off) 	2m40.383s (x2.9)     57m54.369s (x8.0)

For the Zen 2 system:

  Kernel			Run time	    Sys time
  ------			--------	    --------
  Non-debug kernel (baseline)	1m42.806s	     39m48.714s
  Debug kernel			4m04.524s (x2.4)    125m35.904s (x3.2)
  Debug kernel (patched)	3m56.241s (x2.3)    127m22.378s (x3.2)
  Debug kernel
  (patched + mitigations=off) 	2m38.157s (x1.5)     92m35.680s (x2.3)

  RT kernel (baseline)		 1m51.500s	     14m56.322s
  RT debug kernel		16m04.962s (x8.7)   244m36.463s (x16.4)
  RT debug kernel (patched)	 9m09.073s (x4.9)   129m28.439s (x8.7)
  RT debug kernel
  (patched + mitigations=off) 	 3m31.662s (x1.9)    51m01.391s (x3.4)

For the arm64 system:

  Kernel			Run time	    Sys time
  ------			--------	    --------
  Non-debug kernel (baseline)	1m56.844s	      8m47.150s
  Debug kernel			3m54.774s (x2.0)     92m30.098s (x10.5)
  Debug kernel (patched)	3m32.429s (x1.8)     77m40.779s (x8.8)

  RT kernel (baseline)		 4m01.641s	     18m16.777s
  RT debug kernel		19m32.977s (x4.9)   304m23.965s (x16.7)
  RT debug kernel (patched)	16m28.354s (x4.1)   234m18.149s (x12.8)

Turning the mitigations off doesn't seems to have any noticeable impact
on the performance of the arm64 system. So the mitigation=off entries
aren't included.

For the x86 CPUs, cpu mitigations has a much bigger impact on
performance, especially the RT debug kernel. The SRSO mitigation in
Zen 2 has an especially big impact on the debug kernel. It is also the
majority of the slowdown with mitigations on. It is because the patched
ret instruction slows down function returns. A lot of helper functions
that are normally compiled out or inlined may become real function
calls in the debug kernel. The KASAN instrumentation inserts a lot
of __asan_loadX*() and __kasan_check_read() function calls to memory
access portion of the code. The lockdep's __lock_acquire() function,
for instance, has 66 __asan_loadX*() and 6 __kasan_check_read() calls
added with KASAN instrumentation. Of course, the actual numbers may vary
depending on the compiler used and the exact version of the lockdep code.

With the newly added rtmutex and lockdep lock events, the relevant
event counts for the test runs with the Skylake system were:

  Event type		Debug kernel	RT debug kernel
  ----------		------------	---------------
  lockdep_acquire	1,968,663,277	5,425,313,953
  rtlock_slowlock	     -		  401,701,156
  rtmutex_slowlock	     -		      139,672

The __lock_acquire() calls in the RT debug kernel are x2.8 times of the
non-RT debug kernel with the same workload. Since the __lock_acquire()
function is a big hitter in term of performance slowdown, this makes
the RT debug kernel much slower than the non-RT one. The average lock
nesting depth is likely to be higher in the RT debug kernel too leading
to longer execution time in the __lock_acquire() function.

As the small advantage of enabling KASAN instrumentation to catch
potential memory access error in the lockdep debugging tool is probably
not worth the drawback of further slowing down a debug kernel, disable
KASAN instrumentation in the lockdep code to allow the debug kernels
to regain some performance back, especially for the RT debug kernels.

Signed-off-by: Waiman Long <longman@redhat.com>
---
 kernel/locking/Makefile | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/kernel/locking/Makefile b/kernel/locking/Makefile
index 0db4093d17b8..a114949eeed5 100644
--- a/kernel/locking/Makefile
+++ b/kernel/locking/Makefile
@@ -5,7 +5,8 @@ KCOV_INSTRUMENT		:= n
 
 obj-y += mutex.o semaphore.o rwsem.o percpu-rwsem.o
 
-# Avoid recursion lockdep -> sanitizer -> ... -> lockdep.
+# Avoid recursion lockdep -> sanitizer -> ... -> lockdep & improve performance.
+KASAN_SANITIZE_lockdep.o := n
 KCSAN_SANITIZE_lockdep.o := n
 
 ifdef CONFIG_FUNCTION_TRACER
-- 
2.48.1

Re: [PATCH v3 3/3] locking/lockdep: Disable KASAN instrumentation of lockdep.c

[Boqun Feng]

[Cc KASAN]

A Reviewed-by or Acked-by from KASAN would be nice, thanks!

Regards,
Boqun

On Sun, Feb 09, 2025 at 11:26:12PM -0500, Waiman Long wrote:
> Both KASAN and LOCKDEP are commonly enabled in building a debug kernel.
> Each of them can significantly slow down the speed of a debug kernel.
> Enabling KASAN instrumentation of the LOCKDEP code will further slow
> thing down.
> 
> Since LOCKDEP is a high overhead debugging tool, it will never get
> enabled in a production kernel. The LOCKDEP code is also pretty mature
> and is unlikely to get major changes. There is also a possibility of
> recursion similar to KCSAN.
> 
> To evaluate the performance impact of disabling KASAN instrumentation
> of lockdep.c, the time to do a parallel build of the Linux defconfig
> kernel was used as the benchmark. Two x86-64 systems (Skylake & Zen 2)
> and an arm64 system were used as test beds. Two sets of non-RT and RT
> kernels with similar configurations except mainly CONFIG_PREEMPT_RT
> were used for evaulation.
> 
> For the Skylake system:
> 
>   Kernel			Run time	    Sys time
>   ------			--------	    --------
>   Non-debug kernel (baseline)	0m47.642s	      4m19.811s
>   Debug kernel			2m11.108s (x2.8)     38m20.467s (x8.9)
>   Debug kernel (patched)	1m49.602s (x2.3)     31m28.501s (x7.3)
>   Debug kernel
>   (patched + mitigations=off) 	1m30.988s (x1.9)     26m41.993s (x6.2)
> 
>   RT kernel (baseline)		0m54.871s	      7m15.340s
>   RT debug kernel		6m07.151s (x6.7)    135m47.428s (x18.7)
>   RT debug kernel (patched)	3m42.434s (x4.1)     74m51.636s (x10.3)
>   RT debug kernel
>   (patched + mitigations=off) 	2m40.383s (x2.9)     57m54.369s (x8.0)
> 
> For the Zen 2 system:
> 
>   Kernel			Run time	    Sys time
>   ------			--------	    --------
>   Non-debug kernel (baseline)	1m42.806s	     39m48.714s
>   Debug kernel			4m04.524s (x2.4)    125m35.904s (x3.2)
>   Debug kernel (patched)	3m56.241s (x2.3)    127m22.378s (x3.2)
>   Debug kernel
>   (patched + mitigations=off) 	2m38.157s (x1.5)     92m35.680s (x2.3)
> 
>   RT kernel (baseline)		 1m51.500s	     14m56.322s
>   RT debug kernel		16m04.962s (x8.7)   244m36.463s (x16.4)
>   RT debug kernel (patched)	 9m09.073s (x4.9)   129m28.439s (x8.7)
>   RT debug kernel
>   (patched + mitigations=off) 	 3m31.662s (x1.9)    51m01.391s (x3.4)
> 
> For the arm64 system:
> 
>   Kernel			Run time	    Sys time
>   ------			--------	    --------
>   Non-debug kernel (baseline)	1m56.844s	      8m47.150s
>   Debug kernel			3m54.774s (x2.0)     92m30.098s (x10.5)
>   Debug kernel (patched)	3m32.429s (x1.8)     77m40.779s (x8.8)
> 
>   RT kernel (baseline)		 4m01.641s	     18m16.777s
>   RT debug kernel		19m32.977s (x4.9)   304m23.965s (x16.7)
>   RT debug kernel (patched)	16m28.354s (x4.1)   234m18.149s (x12.8)
> 
> Turning the mitigations off doesn't seems to have any noticeable impact
> on the performance of the arm64 system. So the mitigation=off entries
> aren't included.
> 
> For the x86 CPUs, cpu mitigations has a much bigger impact on
> performance, especially the RT debug kernel. The SRSO mitigation in
> Zen 2 has an especially big impact on the debug kernel. It is also the
> majority of the slowdown with mitigations on. It is because the patched
> ret instruction slows down function returns. A lot of helper functions
> that are normally compiled out or inlined may become real function
> calls in the debug kernel. The KASAN instrumentation inserts a lot
> of __asan_loadX*() and __kasan_check_read() function calls to memory
> access portion of the code. The lockdep's __lock_acquire() function,
> for instance, has 66 __asan_loadX*() and 6 __kasan_check_read() calls
> added with KASAN instrumentation. Of course, the actual numbers may vary
> depending on the compiler used and the exact version of the lockdep code.
> 
> With the newly added rtmutex and lockdep lock events, the relevant
> event counts for the test runs with the Skylake system were:
> 
>   Event type		Debug kernel	RT debug kernel
>   ----------		------------	---------------
>   lockdep_acquire	1,968,663,277	5,425,313,953
>   rtlock_slowlock	     -		  401,701,156
>   rtmutex_slowlock	     -		      139,672
> 
> The __lock_acquire() calls in the RT debug kernel are x2.8 times of the
> non-RT debug kernel with the same workload. Since the __lock_acquire()
> function is a big hitter in term of performance slowdown, this makes
> the RT debug kernel much slower than the non-RT one. The average lock
> nesting depth is likely to be higher in the RT debug kernel too leading
> to longer execution time in the __lock_acquire() function.
> 
> As the small advantage of enabling KASAN instrumentation to catch
> potential memory access error in the lockdep debugging tool is probably
> not worth the drawback of further slowing down a debug kernel, disable
> KASAN instrumentation in the lockdep code to allow the debug kernels
> to regain some performance back, especially for the RT debug kernels.
> 
> Signed-off-by: Waiman Long <longman@redhat.com>
> ---
>  kernel/locking/Makefile | 3 ++-
>  1 file changed, 2 insertions(+), 1 deletion(-)
> 
> diff --git a/kernel/locking/Makefile b/kernel/locking/Makefile
> index 0db4093d17b8..a114949eeed5 100644
> --- a/kernel/locking/Makefile
> +++ b/kernel/locking/Makefile
> @@ -5,7 +5,8 @@ KCOV_INSTRUMENT		:= n
>  
>  obj-y += mutex.o semaphore.o rwsem.o percpu-rwsem.o
>  
> -# Avoid recursion lockdep -> sanitizer -> ... -> lockdep.
> +# Avoid recursion lockdep -> sanitizer -> ... -> lockdep & improve performance.
> +KASAN_SANITIZE_lockdep.o := n
>  KCSAN_SANITIZE_lockdep.o := n
>  
>  ifdef CONFIG_FUNCTION_TRACER
> -- 
> 2.48.1
> 

Re: [PATCH v3 3/3] locking/lockdep: Disable KASAN instrumentation of lockdep.c

[Marco Elver]

On Wed, 12 Feb 2025 at 06:57, Boqun Feng <boqun.feng@gmail.com> wrote:
>
> [Cc KASAN]
>
> A Reviewed-by or Acked-by from KASAN would be nice, thanks!
>
> Regards,
> Boqun
>
> On Sun, Feb 09, 2025 at 11:26:12PM -0500, Waiman Long wrote:
> > Both KASAN and LOCKDEP are commonly enabled in building a debug kernel.
> > Each of them can significantly slow down the speed of a debug kernel.
> > Enabling KASAN instrumentation of the LOCKDEP code will further slow
> > thing down.
> >
> > Since LOCKDEP is a high overhead debugging tool, it will never get
> > enabled in a production kernel. The LOCKDEP code is also pretty mature
> > and is unlikely to get major changes. There is also a possibility of
> > recursion similar to KCSAN.
> >
> > To evaluate the performance impact of disabling KASAN instrumentation
> > of lockdep.c, the time to do a parallel build of the Linux defconfig
> > kernel was used as the benchmark. Two x86-64 systems (Skylake & Zen 2)
> > and an arm64 system were used as test beds. Two sets of non-RT and RT
> > kernels with similar configurations except mainly CONFIG_PREEMPT_RT
> > were used for evaulation.
> >
> > For the Skylake system:
> >
> >   Kernel                      Run time            Sys time
> >   ------                      --------            --------
> >   Non-debug kernel (baseline) 0m47.642s             4m19.811s
> >   Debug kernel                        2m11.108s (x2.8)     38m20.467s (x8.9)
> >   Debug kernel (patched)      1m49.602s (x2.3)     31m28.501s (x7.3)
> >   Debug kernel
> >   (patched + mitigations=off)         1m30.988s (x1.9)     26m41.993s (x6.2)
> >
> >   RT kernel (baseline)                0m54.871s             7m15.340s
> >   RT debug kernel             6m07.151s (x6.7)    135m47.428s (x18.7)
> >   RT debug kernel (patched)   3m42.434s (x4.1)     74m51.636s (x10.3)
> >   RT debug kernel
> >   (patched + mitigations=off)         2m40.383s (x2.9)     57m54.369s (x8.0)
> >
> > For the Zen 2 system:
> >
> >   Kernel                      Run time            Sys time
> >   ------                      --------            --------
> >   Non-debug kernel (baseline) 1m42.806s            39m48.714s
> >   Debug kernel                        4m04.524s (x2.4)    125m35.904s (x3.2)
> >   Debug kernel (patched)      3m56.241s (x2.3)    127m22.378s (x3.2)
> >   Debug kernel
> >   (patched + mitigations=off)         2m38.157s (x1.5)     92m35.680s (x2.3)
> >
> >   RT kernel (baseline)                 1m51.500s           14m56.322s
> >   RT debug kernel             16m04.962s (x8.7)   244m36.463s (x16.4)
> >   RT debug kernel (patched)    9m09.073s (x4.9)   129m28.439s (x8.7)
> >   RT debug kernel
> >   (patched + mitigations=off)          3m31.662s (x1.9)    51m01.391s (x3.4)
> >
> > For the arm64 system:
> >
> >   Kernel                      Run time            Sys time
> >   ------                      --------            --------
> >   Non-debug kernel (baseline) 1m56.844s             8m47.150s
> >   Debug kernel                        3m54.774s (x2.0)     92m30.098s (x10.5)
> >   Debug kernel (patched)      3m32.429s (x1.8)     77m40.779s (x8.8)
> >
> >   RT kernel (baseline)                 4m01.641s           18m16.777s
> >   RT debug kernel             19m32.977s (x4.9)   304m23.965s (x16.7)
> >   RT debug kernel (patched)   16m28.354s (x4.1)   234m18.149s (x12.8)
> >
> > Turning the mitigations off doesn't seems to have any noticeable impact
> > on the performance of the arm64 system. So the mitigation=off entries
> > aren't included.
> >
> > For the x86 CPUs, cpu mitigations has a much bigger impact on
> > performance, especially the RT debug kernel. The SRSO mitigation in
> > Zen 2 has an especially big impact on the debug kernel. It is also the
> > majority of the slowdown with mitigations on. It is because the patched
> > ret instruction slows down function returns. A lot of helper functions
> > that are normally compiled out or inlined may become real function
> > calls in the debug kernel. The KASAN instrumentation inserts a lot
> > of __asan_loadX*() and __kasan_check_read() function calls to memory
> > access portion of the code. The lockdep's __lock_acquire() function,
> > for instance, has 66 __asan_loadX*() and 6 __kasan_check_read() calls
> > added with KASAN instrumentation. Of course, the actual numbers may vary
> > depending on the compiler used and the exact version of the lockdep code.

For completeness-sake, we'd also have to compare with
CONFIG_KASAN_INLINE=y, which gets rid of the __asan_ calls (not the
explicit __kasan_ checks). But I leave it up to you - I'm aware it
results in slow-downs, too. ;-)

> > With the newly added rtmutex and lockdep lock events, the relevant
> > event counts for the test runs with the Skylake system were:
> >
> >   Event type          Debug kernel    RT debug kernel
> >   ----------          ------------    ---------------
> >   lockdep_acquire     1,968,663,277   5,425,313,953
> >   rtlock_slowlock          -            401,701,156
> >   rtmutex_slowlock         -                139,672
> >
> > The __lock_acquire() calls in the RT debug kernel are x2.8 times of the
> > non-RT debug kernel with the same workload. Since the __lock_acquire()
> > function is a big hitter in term of performance slowdown, this makes
> > the RT debug kernel much slower than the non-RT one. The average lock
> > nesting depth is likely to be higher in the RT debug kernel too leading
> > to longer execution time in the __lock_acquire() function.
> >
> > As the small advantage of enabling KASAN instrumentation to catch
> > potential memory access error in the lockdep debugging tool is probably
> > not worth the drawback of further slowing down a debug kernel, disable
> > KASAN instrumentation in the lockdep code to allow the debug kernels
> > to regain some performance back, especially for the RT debug kernels.

It's not about catching a bug in the lockdep code, but rather guard
against bugs in code that allocated the storage for some
synchronization object. Since lockdep state is embedded in each
synchronization object, lockdep checking code may be passed a
reference to garbage data, e.g. on use-after-free (or even
out-of-bounds if there's an array of sync objects). In that case, all
bets are off and lockdep may produce random false reports. Sure the
system is already in a bad state at that point, but it's going to make
debugging much harder.

Our approach has always been to ensure that as soon as there's an
error state detected it's reported as soon as we can, before it
results in random failure as execution continues (e.g. bad lock
reports).

To guard against that, I would propose adding carefully placed
kasan_check_byte() in lockdep code.

Re: [PATCH v3 3/3] locking/lockdep: Disable KASAN instrumentation of lockdep.c

[Waiman Long]

On 2/12/25 6:30 AM, Marco Elver wrote:
> On Wed, 12 Feb 2025 at 06:57, Boqun Feng <boqun.feng@gmail.com> wrote:
>> [Cc KASAN]
>>
>> A Reviewed-by or Acked-by from KASAN would be nice, thanks!
>>
>> Regards,
>> Boqun
>>
>> On Sun, Feb 09, 2025 at 11:26:12PM -0500, Waiman Long wrote:
>>> Both KASAN and LOCKDEP are commonly enabled in building a debug kernel.
>>> Each of them can significantly slow down the speed of a debug kernel.
>>> Enabling KASAN instrumentation of the LOCKDEP code will further slow
>>> thing down.
>>>
>>> Since LOCKDEP is a high overhead debugging tool, it will never get
>>> enabled in a production kernel. The LOCKDEP code is also pretty mature
>>> and is unlikely to get major changes. There is also a possibility of
>>> recursion similar to KCSAN.
>>>
>>> To evaluate the performance impact of disabling KASAN instrumentation
>>> of lockdep.c, the time to do a parallel build of the Linux defconfig
>>> kernel was used as the benchmark. Two x86-64 systems (Skylake & Zen 2)
>>> and an arm64 system were used as test beds. Two sets of non-RT and RT
>>> kernels with similar configurations except mainly CONFIG_PREEMPT_RT
>>> were used for evaulation.
>>>
>>> For the Skylake system:
>>>
>>>    Kernel                      Run time            Sys time
>>>    ------                      --------            --------
>>>    Non-debug kernel (baseline) 0m47.642s             4m19.811s
>>>    Debug kernel                        2m11.108s (x2.8)     38m20.467s (x8.9)
>>>    Debug kernel (patched)      1m49.602s (x2.3)     31m28.501s (x7.3)
>>>    Debug kernel
>>>    (patched + mitigations=off)         1m30.988s (x1.9)     26m41.993s (x6.2)
>>>
>>>    RT kernel (baseline)                0m54.871s             7m15.340s
>>>    RT debug kernel             6m07.151s (x6.7)    135m47.428s (x18.7)
>>>    RT debug kernel (patched)   3m42.434s (x4.1)     74m51.636s (x10.3)
>>>    RT debug kernel
>>>    (patched + mitigations=off)         2m40.383s (x2.9)     57m54.369s (x8.0)
>>>
>>> For the Zen 2 system:
>>>
>>>    Kernel                      Run time            Sys time
>>>    ------                      --------            --------
>>>    Non-debug kernel (baseline) 1m42.806s            39m48.714s
>>>    Debug kernel                        4m04.524s (x2.4)    125m35.904s (x3.2)
>>>    Debug kernel (patched)      3m56.241s (x2.3)    127m22.378s (x3.2)
>>>    Debug kernel
>>>    (patched + mitigations=off)         2m38.157s (x1.5)     92m35.680s (x2.3)
>>>
>>>    RT kernel (baseline)                 1m51.500s           14m56.322s
>>>    RT debug kernel             16m04.962s (x8.7)   244m36.463s (x16.4)
>>>    RT debug kernel (patched)    9m09.073s (x4.9)   129m28.439s (x8.7)
>>>    RT debug kernel
>>>    (patched + mitigations=off)          3m31.662s (x1.9)    51m01.391s (x3.4)
>>>
>>> For the arm64 system:
>>>
>>>    Kernel                      Run time            Sys time
>>>    ------                      --------            --------
>>>    Non-debug kernel (baseline) 1m56.844s             8m47.150s
>>>    Debug kernel                        3m54.774s (x2.0)     92m30.098s (x10.5)
>>>    Debug kernel (patched)      3m32.429s (x1.8)     77m40.779s (x8.8)
>>>
>>>    RT kernel (baseline)                 4m01.641s           18m16.777s
>>>    RT debug kernel             19m32.977s (x4.9)   304m23.965s (x16.7)
>>>    RT debug kernel (patched)   16m28.354s (x4.1)   234m18.149s (x12.8)
>>>
>>> Turning the mitigations off doesn't seems to have any noticeable impact
>>> on the performance of the arm64 system. So the mitigation=off entries
>>> aren't included.
>>>
>>> For the x86 CPUs, cpu mitigations has a much bigger impact on
>>> performance, especially the RT debug kernel. The SRSO mitigation in
>>> Zen 2 has an especially big impact on the debug kernel. It is also the
>>> majority of the slowdown with mitigations on. It is because the patched
>>> ret instruction slows down function returns. A lot of helper functions
>>> that are normally compiled out or inlined may become real function
>>> calls in the debug kernel. The KASAN instrumentation inserts a lot
>>> of __asan_loadX*() and __kasan_check_read() function calls to memory
>>> access portion of the code. The lockdep's __lock_acquire() function,
>>> for instance, has 66 __asan_loadX*() and 6 __kasan_check_read() calls
>>> added with KASAN instrumentation. Of course, the actual numbers may vary
>>> depending on the compiler used and the exact version of the lockdep code.
> For completeness-sake, we'd also have to compare with
> CONFIG_KASAN_INLINE=y, which gets rid of the __asan_ calls (not the
> explicit __kasan_ checks). But I leave it up to you - I'm aware it
> results in slow-downs, too. ;-)
I just realize that my config file for non-RT debug kernel does have 
CONFIG_KASAN_INLINE=y set, though the RT debug kernel does not have 
this. For the non-RT debug kernel, the _asan_report_load* functions are 
still being called because lockdep.c is very big (> 6k lines of code). 
So "call_threshold := 10000" in scripts/Makefile.kasan is probably not 
enough for lockdep.c.
>
>>> With the newly added rtmutex and lockdep lock events, the relevant
>>> event counts for the test runs with the Skylake system were:
>>>
>>>    Event type          Debug kernel    RT debug kernel
>>>    ----------          ------------    ---------------
>>>    lockdep_acquire     1,968,663,277   5,425,313,953
>>>    rtlock_slowlock          -            401,701,156
>>>    rtmutex_slowlock         -                139,672
>>>
>>> The __lock_acquire() calls in the RT debug kernel are x2.8 times of the
>>> non-RT debug kernel with the same workload. Since the __lock_acquire()
>>> function is a big hitter in term of performance slowdown, this makes
>>> the RT debug kernel much slower than the non-RT one. The average lock
>>> nesting depth is likely to be higher in the RT debug kernel too leading
>>> to longer execution time in the __lock_acquire() function.
>>>
>>> As the small advantage of enabling KASAN instrumentation to catch
>>> potential memory access error in the lockdep debugging tool is probably
>>> not worth the drawback of further slowing down a debug kernel, disable
>>> KASAN instrumentation in the lockdep code to allow the debug kernels
>>> to regain some performance back, especially for the RT debug kernels.
> It's not about catching a bug in the lockdep code, but rather guard
> against bugs in code that allocated the storage for some
> synchronization object. Since lockdep state is embedded in each
> synchronization object, lockdep checking code may be passed a
> reference to garbage data, e.g. on use-after-free (or even
> out-of-bounds if there's an array of sync objects). In that case, all
> bets are off and lockdep may produce random false reports. Sure the
> system is already in a bad state at that point, but it's going to make
> debugging much harder.
>
> Our approach has always been to ensure that as soon as there's an
> error state detected it's reported as soon as we can, before it
> results in random failure as execution continues (e.g. bad lock
> reports).
>
> To guard against that, I would propose adding carefully placed
> kasan_check_byte() in lockdep code.

Will take a look at that.

Cheers,
Longman

Re: [PATCH v3 3/3] locking/lockdep: Disable KASAN instrumentation of lockdep.c

[Waiman Long]

On 2/12/25 11:57 AM, Waiman Long wrote:
> On 2/12/25 6:30 AM, Marco Elver wrote:
>> On Wed, 12 Feb 2025 at 06:57, Boqun Feng <boqun.feng@gmail.com> wrote:
>>> [Cc KASAN]
>>>
>>> A Reviewed-by or Acked-by from KASAN would be nice, thanks!
>>>
>>> Regards,
>>> Boqun
>>>
>>> On Sun, Feb 09, 2025 at 11:26:12PM -0500, Waiman Long wrote:
>>>> Both KASAN and LOCKDEP are commonly enabled in building a debug 
>>>> kernel.
>>>> Each of them can significantly slow down the speed of a debug kernel.
>>>> Enabling KASAN instrumentation of the LOCKDEP code will further slow
>>>> thing down.
>>>>
>>>> Since LOCKDEP is a high overhead debugging tool, it will never get
>>>> enabled in a production kernel. The LOCKDEP code is also pretty mature
>>>> and is unlikely to get major changes. There is also a possibility of
>>>> recursion similar to KCSAN.
>>>>
>>>> To evaluate the performance impact of disabling KASAN instrumentation
>>>> of lockdep.c, the time to do a parallel build of the Linux defconfig
>>>> kernel was used as the benchmark. Two x86-64 systems (Skylake & Zen 2)
>>>> and an arm64 system were used as test beds. Two sets of non-RT and RT
>>>> kernels with similar configurations except mainly CONFIG_PREEMPT_RT
>>>> were used for evaulation.
>>>>
>>>> For the Skylake system:
>>>>
>>>>    Kernel                      Run time            Sys time
>>>>    ------                      --------            --------
>>>>    Non-debug kernel (baseline) 0m47.642s 4m19.811s
>>>>    Debug kernel                        2m11.108s (x2.8) 38m20.467s 
>>>> (x8.9)
>>>>    Debug kernel (patched)      1m49.602s (x2.3) 31m28.501s (x7.3)
>>>>    Debug kernel
>>>>    (patched + mitigations=off)         1m30.988s (x1.9) 26m41.993s 
>>>> (x6.2)
>>>>
>>>>    RT kernel (baseline)                0m54.871s 7m15.340s
>>>>    RT debug kernel             6m07.151s (x6.7) 135m47.428s (x18.7)
>>>>    RT debug kernel (patched)   3m42.434s (x4.1) 74m51.636s (x10.3)
>>>>    RT debug kernel
>>>>    (patched + mitigations=off)         2m40.383s (x2.9) 57m54.369s 
>>>> (x8.0)
>>>>
>>>> For the Zen 2 system:
>>>>
>>>>    Kernel                      Run time            Sys time
>>>>    ------                      --------            --------
>>>>    Non-debug kernel (baseline) 1m42.806s 39m48.714s
>>>>    Debug kernel                        4m04.524s (x2.4) 125m35.904s 
>>>> (x3.2)
>>>>    Debug kernel (patched)      3m56.241s (x2.3) 127m22.378s (x3.2)
>>>>    Debug kernel
>>>>    (patched + mitigations=off)         2m38.157s (x1.5) 92m35.680s 
>>>> (x2.3)
>>>>
>>>>    RT kernel (baseline)                 1m51.500s 14m56.322s
>>>>    RT debug kernel             16m04.962s (x8.7) 244m36.463s (x16.4)
>>>>    RT debug kernel (patched)    9m09.073s (x4.9) 129m28.439s (x8.7)
>>>>    RT debug kernel
>>>>    (patched + mitigations=off)          3m31.662s (x1.9) 51m01.391s 
>>>> (x3.4)
>>>>
>>>> For the arm64 system:
>>>>
>>>>    Kernel                      Run time            Sys time
>>>>    ------                      --------            --------
>>>>    Non-debug kernel (baseline) 1m56.844s 8m47.150s
>>>>    Debug kernel                        3m54.774s (x2.0) 92m30.098s 
>>>> (x10.5)
>>>>    Debug kernel (patched)      3m32.429s (x1.8) 77m40.779s (x8.8)
>>>>
>>>>    RT kernel (baseline)                 4m01.641s 18m16.777s
>>>>    RT debug kernel             19m32.977s (x4.9) 304m23.965s (x16.7)
>>>>    RT debug kernel (patched)   16m28.354s (x4.1) 234m18.149s (x12.8)
>>>>
>>>> Turning the mitigations off doesn't seems to have any noticeable 
>>>> impact
>>>> on the performance of the arm64 system. So the mitigation=off entries
>>>> aren't included.
>>>>
>>>> For the x86 CPUs, cpu mitigations has a much bigger impact on
>>>> performance, especially the RT debug kernel. The SRSO mitigation in
>>>> Zen 2 has an especially big impact on the debug kernel. It is also the
>>>> majority of the slowdown with mitigations on. It is because the 
>>>> patched
>>>> ret instruction slows down function returns. A lot of helper functions
>>>> that are normally compiled out or inlined may become real function
>>>> calls in the debug kernel. The KASAN instrumentation inserts a lot
>>>> of __asan_loadX*() and __kasan_check_read() function calls to memory
>>>> access portion of the code. The lockdep's __lock_acquire() function,
>>>> for instance, has 66 __asan_loadX*() and 6 __kasan_check_read() calls
>>>> added with KASAN instrumentation. Of course, the actual numbers may 
>>>> vary
>>>> depending on the compiler used and the exact version of the lockdep 
>>>> code.
>> For completeness-sake, we'd also have to compare with
>> CONFIG_KASAN_INLINE=y, which gets rid of the __asan_ calls (not the
>> explicit __kasan_ checks). But I leave it up to you - I'm aware it
>> results in slow-downs, too. ;-)

That is not correct. Setting CONFIG_KASAN_INLINE=y does have an effect 
in lockdep.c to reduce the number of __asan_* calls. I have posted the 
v4 series with the updated test results. I have also added a new patch 
to KASAN checking in lock_acquire().

Cheers,
Longman