* [PATCH] locking/lockdep: Invalidate stale class_cache entries for zapped classes
@ 2026-08-24 15:51 Eric Dumazet
2026-08-25 0:29 ` Hillf Danton
0 siblings, 1 reply; 13+ messages in thread
From: Eric Dumazet @ 2026-08-24 15:51 UTC (permalink / raw)
To: Peter Zijlstra, Ingo Molnar, Will Deacon, Boqun Feng, Waiman Long
Cc: linux-kernel, netdev, Eric Dumazet, syzbot+2d770620059281e225a4
syzbot reported a lockdep splat hitting DEBUG_LOCKS_WARN_ON(1) in
hlock_class() due to an invalid class_idx:
WARNING: kernel/locking/lockdep.c:238 at __lock_acquire+0x382/0x2cf0 kernel/locking/lockdep.c:5203
Workqueue: wg-crypt-wg0 wg_packet_tx_worker
RIP: 0010:hlock_class kernel/locking/lockdep.c:238 [inline]
RIP: 0010:check_wait_context kernel/locking/lockdep.c:4870 [inline]
RIP: 0010:__lock_acquire+0x389/0x2cf0 kernel/locking/lockdep.c:5203
Call Trace:
<IRQ>
lock_acquire+0x106/0x350 kernel/locking/lockdep.c:5886
_raw_spin_lock+0x2e/0x40 kernel/locking/spinlock.c:173
tcp_tsq_handler+0x29/0x200 net/ipv4/tcp_output.c:1291
tcp_tsq_workfn+0x384/0x410 net/ipv4/tcp_output.c:1325
...
When a lock class is zapped (e.g. during module unload or key
unregistration), zap_class() clears the class's bit in
lock_classes_in_use and removes it from the class hash table.
However, existing lockdep_map instances embedded in data structures
may still retain a pointer to the zapped class in their class_cache[]
array.
When __lock_acquire() subsequently runs on such a lock, it finds
lock->class_cache[subclass] != NULL, skipping register_lock_class() and
assigning hlock->class_idx to the index of the zapped class. When
check_wait_context() or hlock_class() inspects the held_lock, it finds
!test_bit(class_idx, lock_classes_in_use) and warns. Furthermore, if
the zapped slot is subsequently re-allocated to an unrelated lock key,
the stale class_cache entry would erroneously match the unrelated class
(ABA issue).
Add lock_class_cache_is_valid() to validate that the cached class is
within lock_classes bounds, still allocated in lock_classes_in_use
(using uninstrumented arch_test_bit() in __always_inline context so it
is safe in noinstr contexts like match_held_lock()), and that class->key
matches the expected subkey (taking lockdep_set_subclass() overrides into
account). Also use READ_ONCE()/WRITE_ONCE() when accessing class_cache[].
If the entry is invalid or stale, fall back to register_lock_class() /
look_up_lock_class().
Fixes: a0b0fd53e1e6 ("locking/lockdep: Free lock classes that are no longer in use")
Reported-by: syzbot+2d770620059281e225a4@syzkaller.appspotmail.com
Closes: https://lore.kernel.org/netdev/6a8c66dc.4d75e56a.c9a88.0050.GAE@google.com/T/#u
Assisted-by: Gemini:gemini-3.1-pro
Signed-off-by: Eric Dumazet <edumazet@google.com>
---
kernel/locking/lockdep.c | 50 +++++++++++++++++++++++++++++++++-------
1 file changed, 42 insertions(+), 8 deletions(-)
diff --git a/kernel/locking/lockdep.c b/kernel/locking/lockdep.c
index 25d77d4a1061..763f79806bd8 100644
--- a/kernel/locking/lockdep.c
+++ b/kernel/locking/lockdep.c
@@ -963,6 +963,34 @@ look_up_lock_class(const struct lockdep_map *lock, unsigned int subclass)
return NULL;
}
+static __always_inline bool lock_class_cache_is_valid(const struct lockdep_map *lock,
+ const struct lock_class *class,
+ unsigned int subclass)
+{
+ unsigned int class_subclass;
+
+ if (!class)
+ return false;
+
+ if (unlikely(class < lock_classes || class >= lock_classes + MAX_LOCKDEP_KEYS))
+ return false;
+
+ if (unlikely(!arch_test_bit(class - lock_classes, lock_classes_in_use)))
+ return false;
+
+ if (unlikely(!lock->key))
+ return false;
+
+ class_subclass = subclass ? subclass : class->subclass;
+ if (unlikely(class_subclass >= MAX_LOCKDEP_SUBCLASSES))
+ return false;
+
+ if (unlikely(READ_ONCE(class->key) != lock->key->subkeys + class_subclass))
+ return false;
+
+ return true;
+}
+
/*
* Static locks do not have their class-keys yet - for them the key is
* the lock object itself. If the lock is in the per cpu area, the
@@ -1395,9 +1423,9 @@ register_lock_class(struct lockdep_map *lock, unsigned int subclass, int force)
out_set_class_cache:
if (!subclass || force)
- lock->class_cache[0] = class;
+ WRITE_ONCE(lock->class_cache[0], class);
else if (subclass < NR_LOCKDEP_CACHING_CLASSES)
- lock->class_cache[subclass] = class;
+ WRITE_ONCE(lock->class_cache[subclass], class);
/*
* Hash collision, did we smoke some? We found a class with a matching
@@ -4957,7 +4985,7 @@ void lockdep_init_map_type(struct lockdep_map *lock, const char *name,
int i;
for (i = 0; i < NR_LOCKDEP_CACHING_CLASSES; i++)
- lock->class_cache[i] = NULL;
+ WRITE_ONCE(lock->class_cache[i], NULL);
#ifdef CONFIG_LOCK_STAT
lock->cpu = raw_smp_processor_id();
@@ -5022,12 +5050,15 @@ EXPORT_SYMBOL_GPL(__lockdep_no_track__);
void lockdep_set_lock_cmp_fn(struct lockdep_map *lock, lock_cmp_fn cmp_fn,
lock_print_fn print_fn)
{
- struct lock_class *class = lock->class_cache[0];
+ struct lock_class *class = READ_ONCE(lock->class_cache[0]);
unsigned long flags;
raw_local_irq_save(flags);
lockdep_recursion_inc();
+ if (!lock_class_cache_is_valid(lock, class, 0))
+ class = NULL;
+
if (!class)
class = register_lock_class(lock, 0, 0);
@@ -5119,8 +5150,11 @@ static int __lock_acquire(struct lockdep_map *lock, unsigned int subclass,
if (DEBUG_LOCKS_WARN_ON(subclass >= MAX_LOCKDEP_SUBCLASSES))
return 0;
- if (subclass < NR_LOCKDEP_CACHING_CLASSES)
- class = lock->class_cache[subclass];
+ if (subclass < NR_LOCKDEP_CACHING_CLASSES) {
+ class = READ_ONCE(lock->class_cache[subclass]);
+ if (!lock_class_cache_is_valid(lock, class, subclass))
+ class = NULL;
+ }
/*
* Not cached?
*/
@@ -5323,9 +5357,9 @@ static noinstr int match_held_lock(const struct held_lock *hlock,
return 1;
if (hlock->references) {
- const struct lock_class *class = lock->class_cache[0];
+ const struct lock_class *class = READ_ONCE(lock->class_cache[0]);
- if (!class)
+ if (!lock_class_cache_is_valid(lock, class, 0))
class = look_up_lock_class(lock, 0);
/*
base-commit: 0a0d1d55dad570724bf8c7ea83409639cfb4be9b
--
2.55.0.766.g2966f0265a-goog
^ permalink raw reply related [flat|nested] 13+ messages in thread* Re: [PATCH] locking/lockdep: Invalidate stale class_cache entries for zapped classes 2026-08-24 15:51 [PATCH] locking/lockdep: Invalidate stale class_cache entries for zapped classes Eric Dumazet @ 2026-08-25 0:29 ` Hillf Danton 2026-08-25 0:59 ` [syzbot] [net?] WARNING: locking bug in tcp_tsq_handler syzbot 0 siblings, 1 reply; 13+ messages in thread From: Hillf Danton @ 2026-08-25 0:29 UTC (permalink / raw) To: syzbot+2d770620059281e225a4 Cc: Peter Zijlstra, Boqun Feng, linux-kernel, netdev, Eric Dumazet #syz test syzbot reported a lockdep splat hitting DEBUG_LOCKS_WARN_ON(1) in hlock_class() due to an invalid class_idx: WARNING: kernel/locking/lockdep.c:238 at __lock_acquire+0x382/0x2cf0 kernel/locking/lockdep.c:5203 Workqueue: wg-crypt-wg0 wg_packet_tx_worker RIP: 0010:hlock_class kernel/locking/lockdep.c:238 [inline] RIP: 0010:check_wait_context kernel/locking/lockdep.c:4870 [inline] RIP: 0010:__lock_acquire+0x389/0x2cf0 kernel/locking/lockdep.c:5203 Call Trace: <IRQ> lock_acquire+0x106/0x350 kernel/locking/lockdep.c:5886 _raw_spin_lock+0x2e/0x40 kernel/locking/spinlock.c:173 tcp_tsq_handler+0x29/0x200 net/ipv4/tcp_output.c:1291 tcp_tsq_workfn+0x384/0x410 net/ipv4/tcp_output.c:1325 ... When a lock class is zapped (e.g. during module unload or key unregistration), zap_class() clears the class's bit in lock_classes_in_use and removes it from the class hash table. However, existing lockdep_map instances embedded in data structures may still retain a pointer to the zapped class in their class_cache[] array. When __lock_acquire() subsequently runs on such a lock, it finds lock->class_cache[subclass] != NULL, skipping register_lock_class() and assigning hlock->class_idx to the index of the zapped class. When check_wait_context() or hlock_class() inspects the held_lock, it finds !test_bit(class_idx, lock_classes_in_use) and warns. Furthermore, if the zapped slot is subsequently re-allocated to an unrelated lock key, the stale class_cache entry would erroneously match the unrelated class (ABA issue). Add lock_class_cache_is_valid() to validate that the cached class is within lock_classes bounds, still allocated in lock_classes_in_use (using uninstrumented arch_test_bit() in __always_inline context so it is safe in noinstr contexts like match_held_lock()), and that class->key matches the expected subkey (taking lockdep_set_subclass() overrides into account). Also use READ_ONCE()/WRITE_ONCE() when accessing class_cache[]. If the entry is invalid or stale, fall back to register_lock_class() / look_up_lock_class(). Fixes: a0b0fd53e1e6 ("locking/lockdep: Free lock classes that are no longer in use") Reported-by: syzbot+2d770620059281e225a4@syzkaller.appspotmail.com Closes: https://lore.kernel.org/netdev/6a8c66dc.4d75e56a.c9a88.0050.GAE@google.com/T/#u Assisted-by: Gemini:gemini-3.1-pro Signed-off-by: Eric Dumazet <edumazet@google.com> --- kernel/locking/lockdep.c | 50 +++++++++++++++++++++++++++++++++------- 1 file changed, 42 insertions(+), 8 deletions(-) diff --git a/kernel/locking/lockdep.c b/kernel/locking/lockdep.c index 25d77d4a1061..763f79806bd8 100644 --- a/kernel/locking/lockdep.c +++ b/kernel/locking/lockdep.c @@ -963,6 +963,34 @@ look_up_lock_class(const struct lockdep_map *lock, unsigned int subclass) return NULL; } +static __always_inline bool lock_class_cache_is_valid(const struct lockdep_map *lock, + const struct lock_class *class, + unsigned int subclass) +{ + unsigned int class_subclass; + + if (!class) + return false; + + if (unlikely(class < lock_classes || class >= lock_classes + MAX_LOCKDEP_KEYS)) + return false; + + if (unlikely(!arch_test_bit(class - lock_classes, lock_classes_in_use))) + return false; + + if (unlikely(!lock->key)) + return false; + + class_subclass = subclass ? subclass : class->subclass; + if (unlikely(class_subclass >= MAX_LOCKDEP_SUBCLASSES)) + return false; + + if (unlikely(READ_ONCE(class->key) != lock->key->subkeys + class_subclass)) + return false; + + return true; +} + /* * Static locks do not have their class-keys yet - for them the key is * the lock object itself. If the lock is in the per cpu area, the @@ -1395,9 +1423,9 @@ register_lock_class(struct lockdep_map *lock, unsigned int subclass, int force) out_set_class_cache: if (!subclass || force) - lock->class_cache[0] = class; + WRITE_ONCE(lock->class_cache[0], class); else if (subclass < NR_LOCKDEP_CACHING_CLASSES) - lock->class_cache[subclass] = class; + WRITE_ONCE(lock->class_cache[subclass], class); /* * Hash collision, did we smoke some? We found a class with a matching @@ -4957,7 +4985,7 @@ void lockdep_init_map_type(struct lockdep_map *lock, const char *name, int i; for (i = 0; i < NR_LOCKDEP_CACHING_CLASSES; i++) - lock->class_cache[i] = NULL; + WRITE_ONCE(lock->class_cache[i], NULL); #ifdef CONFIG_LOCK_STAT lock->cpu = raw_smp_processor_id(); @@ -5022,12 +5050,15 @@ EXPORT_SYMBOL_GPL(__lockdep_no_track__); void lockdep_set_lock_cmp_fn(struct lockdep_map *lock, lock_cmp_fn cmp_fn, lock_print_fn print_fn) { - struct lock_class *class = lock->class_cache[0]; + struct lock_class *class = READ_ONCE(lock->class_cache[0]); unsigned long flags; raw_local_irq_save(flags); lockdep_recursion_inc(); + if (!lock_class_cache_is_valid(lock, class, 0)) + class = NULL; + if (!class) class = register_lock_class(lock, 0, 0); @@ -5119,8 +5150,11 @@ static int __lock_acquire(struct lockdep_map *lock, unsigned int subclass, if (DEBUG_LOCKS_WARN_ON(subclass >= MAX_LOCKDEP_SUBCLASSES)) return 0; - if (subclass < NR_LOCKDEP_CACHING_CLASSES) - class = lock->class_cache[subclass]; + if (subclass < NR_LOCKDEP_CACHING_CLASSES) { + class = READ_ONCE(lock->class_cache[subclass]); + if (!lock_class_cache_is_valid(lock, class, subclass)) + class = NULL; + } /* * Not cached? */ @@ -5323,9 +5357,9 @@ static noinstr int match_held_lock(const struct held_lock *hlock, return 1; if (hlock->references) { - const struct lock_class *class = lock->class_cache[0]; + const struct lock_class *class = READ_ONCE(lock->class_cache[0]); - if (!class) + if (!lock_class_cache_is_valid(lock, class, 0)) class = look_up_lock_class(lock, 0); /* base-commit: 0a0d1d55dad570724bf8c7ea83409639cfb4be9b -- 2.55.0.766.g2966f0265a-goog ^ permalink raw reply related [flat|nested] 13+ messages in thread
* Re: [syzbot] [net?] WARNING: locking bug in tcp_tsq_handler 2026-08-25 0:29 ` Hillf Danton @ 2026-08-25 0:59 ` syzbot 2026-08-25 1:25 ` Hillf Danton 2026-08-25 1:27 ` Eric Dumazet 0 siblings, 2 replies; 13+ messages in thread From: syzbot @ 2026-08-25 0:59 UTC (permalink / raw) To: boqun, edumazet, hdanton, linux-kernel, netdev, peterz, syzkaller-bugs Hello, syzbot has tested the proposed patch but the reproducer is still triggering an issue: WARNING in tcp_tsq_handler ------------[ cut here ]------------ !lockdep_sock_is_held(sk) && debug_locks WARNING: ./include/net/sock.h:1799 at sock_owned_by_me include/net/sock.h:1799 [inline], CPU#1: kworker/1:0/24 WARNING: ./include/net/sock.h:1799 at sock_owned_by_user include/net/sock.h:1812 [inline], CPU#1: kworker/1:0/24 WARNING: ./include/net/sock.h:1799 at tcp_tsq_handler+0x1a6/0x200 net/ipv4/tcp_output.c:1292, CPU#1: kworker/1:0/24 Modules linked in: CPU: 1 UID: 0 PID: 24 Comm: kworker/1:0 Not tainted syzkaller #0 PREEMPT(full) Hardware name: Google Google Compute Engine/Google Compute Engine, BIOS Google 07/24/2026 Workqueue: wg-crypt-wg0 wg_packet_tx_worker RIP: 0010:sock_owned_by_me include/net/sock.h:1799 [inline] RIP: 0010:sock_owned_by_user include/net/sock.h:1812 [inline] RIP: 0010:tcp_tsq_handler+0x1a6/0x200 net/ipv4/tcp_output.c:1292 Code: 30 96 96 01 e8 7b b3 7a f7 be 02 00 00 00 eb 0a e8 6f b3 7a f7 be 01 00 00 00 4c 89 ff e8 82 3e 8f fa eb d0 e8 5b b3 7a f7 90 <0f> 0b 90 e9 fd fe ff ff 44 89 f9 80 e1 07 80 c1 03 38 c1 0f 8c 07 RSP: 0018:ffffc90000a08bb0 EFLAGS: 00010246 RAX: ffffffff8a4c8ba5 RBX: ffff88802db742d0 RCX: ffff88801e6e5dc0 RDX: 8000000000000100 RSI: 0000000000000100 RDI: 0000000000000000 RBP: 0000000000000001 R08: 0000000000000100 R09: 0000000000000004 R10: dffffc0000000000 R11: fffff52000141164 R12: dffffc0000000000 R13: ffffc90000a08c20 R14: ffff88802db74100 R15: ffff88802db74100 FS: 0000000000000000(0000) GS:ffff888124df1000(0000) knlGS:0000000000000000 CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 CR2: 00005635f5ecdbd8 CR3: 0000000072ba6000 CR4: 00000000003526f0 Call Trace: <IRQ> tcp_tsq_workfn+0x384/0x410 net/ipv4/tcp_output.c:1325 process_one_work kernel/workqueue.c:3387 [inline] process_scheduled_works+0xc3d/0x1630 kernel/workqueue.c:3470 bh_worker+0x451/0x870 kernel/workqueue.c:3773 tasklet_action+0xc/0x70 kernel/softirq.c:997 handle_softirqs+0x226/0x860 kernel/softirq.c:645 do_softirq+0x77/0xd0 kernel/softirq.c:546 </IRQ> <TASK> __local_bh_enable_ip+0x100/0x140 kernel/softirq.c:473 wg_socket_send_skb_to_peer+0x16b/0x1d0 drivers/net/wireguard/socket.c:183 wg_packet_create_data_done drivers/net/wireguard/send.c:251 [inline] wg_packet_tx_worker+0x1c8/0x7e0 drivers/net/wireguard/send.c:276 process_one_work kernel/workqueue.c:3387 [inline] process_scheduled_works+0xc3d/0x1630 kernel/workqueue.c:3470 worker_thread+0xa47/0xfb0 kernel/workqueue.c:3551 kthread+0x38b/0x480 kernel/kthread.c:436 ret_from_fork+0x514/0xb70 arch/x86/kernel/process.c:158 ret_from_fork_asm+0x1a/0x30 arch/x86/entry/entry_64.S:245 </TASK> Tested on: commit: 66498c75 Merge tag 'dmaengine-7.3-rc1' of git://git.ke.. git tree: upstream console output: https://syzkaller.appspot.com/x/log.txt?x=10adc979580000 kernel config: https://syzkaller.appspot.com/x/.config?x=e9ce1d694820ba2b dashboard link: https://syzkaller.appspot.com/bug?extid=2d770620059281e225a4 compiler: Debian clang version 22.1.8 (++20260613092233+e80beda6e255-1~exp1~20260613092250.77), Debian LLD 22.1.8 patch: https://syzkaller.appspot.com/x/patch.diff?x=1497b549580000 ^ permalink raw reply [flat|nested] 13+ messages in thread
* Re: [syzbot] [net?] WARNING: locking bug in tcp_tsq_handler 2026-08-25 0:59 ` [syzbot] [net?] WARNING: locking bug in tcp_tsq_handler syzbot @ 2026-08-25 1:25 ` Hillf Danton 2026-08-25 1:27 ` Eric Dumazet 1 sibling, 0 replies; 13+ messages in thread From: Hillf Danton @ 2026-08-25 1:25 UTC (permalink / raw) To: syzbot; +Cc: boqun, edumazet, linux-kernel, netdev, peterz, syzkaller-bugs > Date: Mon, 24 Aug 2026 17:59:01 -0700 [thread overview] > Hello, > > syzbot has tested the proposed patch but the reproducer is still triggering an issue: > WARNING in tcp_tsq_handler > Fine, another case of HBC, half baked croissant, thanks to syzbot. > ------------[ cut here ]------------ > !lockdep_sock_is_held(sk) && debug_locks > WARNING: ./include/net/sock.h:1799 at sock_owned_by_me include/net/sock.h:1799 [inline], CPU#1: kworker/1:0/24 > WARNING: ./include/net/sock.h:1799 at sock_owned_by_user include/net/sock.h:1812 [inline], CPU#1: kworker/1:0/24 > WARNING: ./include/net/sock.h:1799 at tcp_tsq_handler+0x1a6/0x200 net/ipv4/tcp_output.c:1292, CPU#1: kworker/1:0/24 > Modules linked in: > CPU: 1 UID: 0 PID: 24 Comm: kworker/1:0 Not tainted syzkaller #0 PREEMPT(full) > Hardware name: Google Google Compute Engine/Google Compute Engine, BIOS Google 07/24/2026 > Workqueue: wg-crypt-wg0 wg_packet_tx_worker > RIP: 0010:sock_owned_by_me include/net/sock.h:1799 [inline] > RIP: 0010:sock_owned_by_user include/net/sock.h:1812 [inline] > RIP: 0010:tcp_tsq_handler+0x1a6/0x200 net/ipv4/tcp_output.c:1292 > Code: 30 96 96 01 e8 7b b3 7a f7 be 02 00 00 00 eb 0a e8 6f b3 7a f7 be 01 00 00 00 4c 89 ff e8 82 3e 8f fa eb d0 e8 5b b3 7a f7 90 <0f> 0b 90 e9 fd fe ff ff 44 89 f9 80 e1 07 80 c1 03 38 c1 0f 8c 07 > RSP: 0018:ffffc90000a08bb0 EFLAGS: 00010246 > RAX: ffffffff8a4c8ba5 RBX: ffff88802db742d0 RCX: ffff88801e6e5dc0 > RDX: 8000000000000100 RSI: 0000000000000100 RDI: 0000000000000000 > RBP: 0000000000000001 R08: 0000000000000100 R09: 0000000000000004 > R10: dffffc0000000000 R11: fffff52000141164 R12: dffffc0000000000 > R13: ffffc90000a08c20 R14: ffff88802db74100 R15: ffff88802db74100 > FS: 0000000000000000(0000) GS:ffff888124df1000(0000) knlGS:0000000000000000 > CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 > CR2: 00005635f5ecdbd8 CR3: 0000000072ba6000 CR4: 00000000003526f0 > Call Trace: > <IRQ> > tcp_tsq_workfn+0x384/0x410 net/ipv4/tcp_output.c:1325 > process_one_work kernel/workqueue.c:3387 [inline] > process_scheduled_works+0xc3d/0x1630 kernel/workqueue.c:3470 > bh_worker+0x451/0x870 kernel/workqueue.c:3773 > tasklet_action+0xc/0x70 kernel/softirq.c:997 > handle_softirqs+0x226/0x860 kernel/softirq.c:645 > do_softirq+0x77/0xd0 kernel/softirq.c:546 > </IRQ> > <TASK> > __local_bh_enable_ip+0x100/0x140 kernel/softirq.c:473 > wg_socket_send_skb_to_peer+0x16b/0x1d0 drivers/net/wireguard/socket.c:183 > wg_packet_create_data_done drivers/net/wireguard/send.c:251 [inline] > wg_packet_tx_worker+0x1c8/0x7e0 drivers/net/wireguard/send.c:276 > process_one_work kernel/workqueue.c:3387 [inline] > process_scheduled_works+0xc3d/0x1630 kernel/workqueue.c:3470 > worker_thread+0xa47/0xfb0 kernel/workqueue.c:3551 > kthread+0x38b/0x480 kernel/kthread.c:436 > ret_from_fork+0x514/0xb70 arch/x86/kernel/process.c:158 > ret_from_fork_asm+0x1a/0x30 arch/x86/entry/entry_64.S:245 > </TASK> > > > Tested on: > > commit: 66498c75 Merge tag 'dmaengine-7.3-rc1' of git://git.ke.. > git tree: upstream > console output: https://syzkaller.appspot.com/x/log.txt?x=10adc979580000 > kernel config: https://syzkaller.appspot.com/x/.config?x=e9ce1d694820ba2b > dashboard link: https://syzkaller.appspot.com/bug?extid=2d770620059281e225a4 > compiler: Debian clang version 22.1.8 (++20260613092233+e80beda6e255-1~exp1~20260613092250.77), Debian LLD 22.1.8 > patch: https://syzkaller.appspot.com/x/patch.diff?x=1497b549580000 ^ permalink raw reply [flat|nested] 13+ messages in thread
* Re: [syzbot] [net?] WARNING: locking bug in tcp_tsq_handler 2026-08-25 0:59 ` [syzbot] [net?] WARNING: locking bug in tcp_tsq_handler syzbot 2026-08-25 1:25 ` Hillf Danton @ 2026-08-25 1:27 ` Eric Dumazet 2026-08-25 1:50 ` Eric Dumazet 1 sibling, 1 reply; 13+ messages in thread From: Eric Dumazet @ 2026-08-25 1:27 UTC (permalink / raw) To: syzbot, Shin'ichiro Kawasaki, Nilay Shroff, Keith Busch Cc: boqun, hdanton, linux-kernel, netdev, peterz, syzkaller-bugs On Tue, Aug 25, 2026 at 2:59 AM syzbot <syzbot+2d770620059281e225a4@syzkaller.appspotmail.com> wrote: > > Hello, > > syzbot has tested the proposed patch but the reproducer is still triggering an issue: > WARNING in tcp_tsq_handler > > ------------[ cut here ]------------ > !lockdep_sock_is_held(sk) && debug_locks > WARNING: ./include/net/sock.h:1799 at sock_owned_by_me include/net/sock.h:1799 [inline], CPU#1: kworker/1:0/24 > WARNING: ./include/net/sock.h:1799 at sock_owned_by_user include/net/sock.h:1812 [inline], CPU#1: kworker/1:0/24 > WARNING: ./include/net/sock.h:1799 at tcp_tsq_handler+0x1a6/0x200 net/ipv4/tcp_output.c:1292, CPU#1: kworker/1:0/24 Note this is a different warning. This one might have been added in nvme-tcp in commit commit 19bdb70c77d3b24239a453291299b64040bdba86 Author: Shin'ichiro Kawasaki <shinichiro.kawasaki@wdc.com> Date: Thu Jun 4 11:32:08 2026 +0900 nvme-tcp: lockdep: use dynamic lockdep keys per socket instance The authors assumed that __fput_sync(queue->sock->file) in nvme_tcp_free_queue() synchronously destroys the socket, so they unregistered the keys immediately: __fput_sync(queue->sock->file); queue->sock = NULL; ... #ifdef CONFIG_DEBUG_LOCK_ALLOC lockdep_unregister_key(&queue->nvme_tcp_sk_key); lockdep_unregister_key(&queue->nvme_tcp_slock_key); #endif However, a TCP socket's lifetime is asynchronous: in-flight skbs (e.g. buffered in a qdisc or device ring) hold references on sk->sk_wmem_alloc. When those packets are freed later, tcp_wfree() puts sk on TSQ and tcp_tsq_handler(sk) acquires bh_lock_sock(sk) on a socket whose lockdep key has already been unregistered and zapped. All other kernel storage/networking clients (sunrpc, nbd, cifs, iscsi_tcp, rxe, siw) use static lockdep keys without issue. > Modules linked in: > CPU: 1 UID: 0 PID: 24 Comm: kworker/1:0 Not tainted syzkaller #0 PREEMPT(full) > Hardware name: Google Google Compute Engine/Google Compute Engine, BIOS Google 07/24/2026 > Workqueue: wg-crypt-wg0 wg_packet_tx_worker > RIP: 0010:sock_owned_by_me include/net/sock.h:1799 [inline] > RIP: 0010:sock_owned_by_user include/net/sock.h:1812 [inline] > RIP: 0010:tcp_tsq_handler+0x1a6/0x200 net/ipv4/tcp_output.c:1292 > Code: 30 96 96 01 e8 7b b3 7a f7 be 02 00 00 00 eb 0a e8 6f b3 7a f7 be 01 00 00 00 4c 89 ff e8 82 3e 8f fa eb d0 e8 5b b3 7a f7 90 <0f> 0b 90 e9 fd fe ff ff 44 89 f9 80 e1 07 80 c1 03 38 c1 0f 8c 07 > RSP: 0018:ffffc90000a08bb0 EFLAGS: 00010246 > RAX: ffffffff8a4c8ba5 RBX: ffff88802db742d0 RCX: ffff88801e6e5dc0 > RDX: 8000000000000100 RSI: 0000000000000100 RDI: 0000000000000000 > RBP: 0000000000000001 R08: 0000000000000100 R09: 0000000000000004 > R10: dffffc0000000000 R11: fffff52000141164 R12: dffffc0000000000 > R13: ffffc90000a08c20 R14: ffff88802db74100 R15: ffff88802db74100 > FS: 0000000000000000(0000) GS:ffff888124df1000(0000) knlGS:0000000000000000 > CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 > CR2: 00005635f5ecdbd8 CR3: 0000000072ba6000 CR4: 00000000003526f0 > Call Trace: > <IRQ> > tcp_tsq_workfn+0x384/0x410 net/ipv4/tcp_output.c:1325 > process_one_work kernel/workqueue.c:3387 [inline] > process_scheduled_works+0xc3d/0x1630 kernel/workqueue.c:3470 > bh_worker+0x451/0x870 kernel/workqueue.c:3773 > tasklet_action+0xc/0x70 kernel/softirq.c:997 > handle_softirqs+0x226/0x860 kernel/softirq.c:645 > do_softirq+0x77/0xd0 kernel/softirq.c:546 > </IRQ> > <TASK> > __local_bh_enable_ip+0x100/0x140 kernel/softirq.c:473 > wg_socket_send_skb_to_peer+0x16b/0x1d0 drivers/net/wireguard/socket.c:183 > wg_packet_create_data_done drivers/net/wireguard/send.c:251 [inline] > wg_packet_tx_worker+0x1c8/0x7e0 drivers/net/wireguard/send.c:276 > process_one_work kernel/workqueue.c:3387 [inline] > process_scheduled_works+0xc3d/0x1630 kernel/workqueue.c:3470 > worker_thread+0xa47/0xfb0 kernel/workqueue.c:3551 > kthread+0x38b/0x480 kernel/kthread.c:436 > ret_from_fork+0x514/0xb70 arch/x86/kernel/process.c:158 > ret_from_fork_asm+0x1a/0x30 arch/x86/entry/entry_64.S:245 > </TASK> > > > Tested on: > > commit: 66498c75 Merge tag 'dmaengine-7.3-rc1' of git://git.ke.. > git tree: upstream > console output: https://syzkaller.appspot.com/x/log.txt?x=10adc979580000 > kernel config: https://syzkaller.appspot.com/x/.config?x=e9ce1d694820ba2b > dashboard link: https://syzkaller.appspot.com/bug?extid=2d770620059281e225a4 > compiler: Debian clang version 22.1.8 (++20260613092233+e80beda6e255-1~exp1~20260613092250.77), Debian LLD 22.1.8 > patch: https://syzkaller.appspot.com/x/patch.diff?x=1497b549580000 > ^ permalink raw reply [flat|nested] 13+ messages in thread
* Re: [syzbot] [net?] WARNING: locking bug in tcp_tsq_handler 2026-08-25 1:27 ` Eric Dumazet @ 2026-08-25 1:50 ` Eric Dumazet 2026-08-25 5:44 ` Shin'ichiro Kawasaki 0 siblings, 1 reply; 13+ messages in thread From: Eric Dumazet @ 2026-08-25 1:50 UTC (permalink / raw) To: syzbot, Shin'ichiro Kawasaki, Nilay Shroff, Keith Busch Cc: boqun, hdanton, linux-kernel, netdev, peterz, syzkaller-bugs On Tue, Aug 25, 2026 at 3:27 AM Eric Dumazet <edumazet@google.com> wrote: > > On Tue, Aug 25, 2026 at 2:59 AM syzbot > <syzbot+2d770620059281e225a4@syzkaller.appspotmail.com> wrote: > > > > Hello, > > > > syzbot has tested the proposed patch but the reproducer is still triggering an issue: > > WARNING in tcp_tsq_handler > > > > ------------[ cut here ]------------ > > !lockdep_sock_is_held(sk) && debug_locks > > WARNING: ./include/net/sock.h:1799 at sock_owned_by_me include/net/sock.h:1799 [inline], CPU#1: kworker/1:0/24 > > WARNING: ./include/net/sock.h:1799 at sock_owned_by_user include/net/sock.h:1812 [inline], CPU#1: kworker/1:0/24 > > WARNING: ./include/net/sock.h:1799 at tcp_tsq_handler+0x1a6/0x200 net/ipv4/tcp_output.c:1292, CPU#1: kworker/1:0/24 > > Note this is a different warning. > > This one might have been added in nvme-tcp in commit > > commit 19bdb70c77d3b24239a453291299b64040bdba86 > Author: Shin'ichiro Kawasaki <shinichiro.kawasaki@wdc.com> > Date: Thu Jun 4 11:32:08 2026 +0900 > > nvme-tcp: lockdep: use dynamic lockdep keys per socket instance > > The authors assumed that __fput_sync(queue->sock->file) in > nvme_tcp_free_queue() synchronously destroys the socket, so they > unregistered the keys immediately: > > __fput_sync(queue->sock->file); > queue->sock = NULL; > ... > #ifdef CONFIG_DEBUG_LOCK_ALLOC > lockdep_unregister_key(&queue->nvme_tcp_sk_key); > lockdep_unregister_key(&queue->nvme_tcp_slock_key); > #endif > > However, a TCP socket's lifetime is asynchronous: > in-flight skbs (e.g. buffered in a qdisc or device ring) hold > references on sk->sk_wmem_alloc. > > When those packets are freed later, tcp_wfree() puts sk on TSQ and > tcp_tsq_handler(sk) > acquires bh_lock_sock(sk) on a socket whose lockdep key has already > been unregistered and zapped. > > All other kernel storage/networking clients (sunrpc, nbd, cifs, > iscsi_tcp, rxe, siw) use static lockdep keys without issue. > I think 19bdb70c77d3 should be reverted. We can change TCP to use sk_gfp_mask(sk, GFP_ATOMIC) instead of gfp_any() in tcp_disconnect() This ensures tcp_disconnect() respects sk->sk_allocation = GFP_ATOMIC and never acquires fs_reclaim under sk_lock. WDYT? ^ permalink raw reply [flat|nested] 13+ messages in thread
* Re: [syzbot] [net?] WARNING: locking bug in tcp_tsq_handler 2026-08-25 1:50 ` Eric Dumazet @ 2026-08-25 5:44 ` Shin'ichiro Kawasaki 2026-08-25 6:25 ` Hillf Danton ` (2 more replies) 0 siblings, 3 replies; 13+ messages in thread From: Shin'ichiro Kawasaki @ 2026-08-25 5:44 UTC (permalink / raw) To: Eric Dumazet Cc: syzbot, Nilay Shroff, Keith Busch, boqun, hdanton, linux-kernel, netdev, peterz, syzkaller-bugs On Aug 25, 2026 / 03:50, Eric Dumazet wrote: > On Tue, Aug 25, 2026 at 3:27 AM Eric Dumazet <edumazet@google.com> wrote: > > > > On Tue, Aug 25, 2026 at 2:59 AM syzbot > > <syzbot+2d770620059281e225a4@syzkaller.appspotmail.com> wrote: > > > > > > Hello, > > > > > > syzbot has tested the proposed patch but the reproducer is still triggering an issue: > > > WARNING in tcp_tsq_handler > > > > > > ------------[ cut here ]------------ > > > !lockdep_sock_is_held(sk) && debug_locks > > > WARNING: ./include/net/sock.h:1799 at sock_owned_by_me include/net/sock.h:1799 [inline], CPU#1: kworker/1:0/24 > > > WARNING: ./include/net/sock.h:1799 at sock_owned_by_user include/net/sock.h:1812 [inline], CPU#1: kworker/1:0/24 > > > WARNING: ./include/net/sock.h:1799 at tcp_tsq_handler+0x1a6/0x200 net/ipv4/tcp_output.c:1292, CPU#1: kworker/1:0/24 > > > > Note this is a different warning. > > > > This one might have been added in nvme-tcp in commit > > > > commit 19bdb70c77d3b24239a453291299b64040bdba86 > > Author: Shin'ichiro Kawasaki <shinichiro.kawasaki@wdc.com> > > Date: Thu Jun 4 11:32:08 2026 +0900 > > > > nvme-tcp: lockdep: use dynamic lockdep keys per socket instance > > > > The authors assumed that __fput_sync(queue->sock->file) in > > nvme_tcp_free_queue() synchronously destroys the socket, so they > > unregistered the keys immediately: > > > > __fput_sync(queue->sock->file); > > queue->sock = NULL; > > ... > > #ifdef CONFIG_DEBUG_LOCK_ALLOC > > lockdep_unregister_key(&queue->nvme_tcp_sk_key); > > lockdep_unregister_key(&queue->nvme_tcp_slock_key); > > #endif > > > > However, a TCP socket's lifetime is asynchronous: > > in-flight skbs (e.g. buffered in a qdisc or device ring) hold > > references on sk->sk_wmem_alloc. > > > > When those packets are freed later, tcp_wfree() puts sk on TSQ and > > tcp_tsq_handler(sk) > > acquires bh_lock_sock(sk) on a socket whose lockdep key has already > > been unregistered and zapped. > > > > All other kernel storage/networking clients (sunrpc, nbd, cifs, > > iscsi_tcp, rxe, siw) use static lockdep keys without issue. > > > > I think 19bdb70c77d3 should be reverted. Just reverting the commit will reintroduce the other lockdep WARN that the commit addressed. I hope to have another fix to avoid the WARN. > > We can change TCP to use sk_gfp_mask(sk, GFP_ATOMIC) instead of > gfp_any() in tcp_disconnect() > > This ensures tcp_disconnect() respects sk->sk_allocation = GFP_ATOMIC > and never acquires fs_reclaim under sk_lock. > > WDYT? Thanks for the idea. I did a quick trial with the idea. Step 1: I reverted the commit 19bdb70c77d3 from v7.2 kernel, and confirmed that the blktests test case nvme/005 for tcp transport recreates the lockdep WARN that includes fs_reclaim in its lock chain. Step 2: I created a patch to replace gfp_any() in tcp_disconnect() with GFP_ATOMIC [1]. I applied this patch to the v7.2 based kernel that I used in the step 1. I ran the test case nvme/005 on this kernel, and observed it still fails with the lockdep WARN: fs_reclaim was still included in the lock chain. I think this is expected, since fs_reclaim dependency comes from CPU hotplug bring-up context. Based on this observation, I'm afraid that using GFP_ATOMIC in tcp_disconnect() won't work, unfortunately. Another approach I can think of is to use sk->sk_destruct hook to unregister keys, so that the unregistraion happens after the all in-flight skbs complete. I will try this approach. [1] fix trial patch diff --git a/net/ipv4/tcp.c b/net/ipv4/tcp.c index 455441f1b694..861ce399eee3 100644 --- a/net/ipv4/tcp.c +++ b/net/ipv4/tcp.c @@ -3376,14 +3376,14 @@ int tcp_disconnect(struct sock *sk, int flags) } else if (unlikely(tp->repair)) { WRITE_ONCE(sk->sk_err, ECONNABORTED); } else if (tcp_need_reset(old_state)) { - tcp_send_active_reset(sk, gfp_any(), SK_RST_REASON_TCP_STATE); + tcp_send_active_reset(sk, GFP_ATOMIC, SK_RST_REASON_TCP_STATE); WRITE_ONCE(sk->sk_err, ECONNRESET); } else if (tp->snd_nxt != tp->write_seq && (1 << old_state) & (TCPF_CLOSING | TCPF_LAST_ACK)) { /* The last check adjusts for discrepancy of Linux wrt. RFC * states */ - tcp_send_active_reset(sk, gfp_any(), + tcp_send_active_reset(sk, GFP_ATOMIC, SK_RST_REASON_TCP_DISCONNECT_WITH_DATA); WRITE_ONCE(sk->sk_err, ECONNRESET); } else if (old_state == TCP_SYN_SENT) [2] Lockdep WARN observed at the step 2 [ 86.005477] [ T995] run blktests nvme/005 at 2026-08-25 13:52:07 [ 86.164051] [ T1069] loop0: detected capacity change from 0 to 2097152 [ 86.204542] [ T1073] nvmet: adding nsid 1 to subsystem blktests-subsystem-1 [ 86.274053] [ T1079] nvmet_tcp: enabling port 0 (127.0.0.1:4420) [ 86.483498] [ T104] nvmet: Created nvm controller 1 for subsystem blktests-subsystem-1 for NQN nqn.2014-08.org.nvmexpress:uuid:0f01fb42-9f7f-4856-b0b3-51e60b8de349. [ 86.493643] [ T1089] nvme nvme5: creating 4 I/O queues. [ 86.503743] [ T1089] nvme nvme5: mapped 4/0/0 default/read/poll queues. [ 86.510474] [ T1089] nvme nvme5: new ctrl: NQN "blktests-subsystem-1", addr 127.0.0.1:4420, hostnqn: nqn.2014-08.org.nvmexpress:uuid:0f01fb42-9f7f-4856-b0b3-51e60b8de349 [ 87.138148] [ T103] nvmet: Created nvm controller 2 for subsystem blktests-subsystem-1 for NQN nqn.2014-08.org.nvmexpress:uuid:0f01fb42-9f7f-4856-b0b3-51e60b8de349. [ 87.145904] [ T83] nvme nvme5: creating 4 I/O queues. [ 87.167642] [ T83] nvme nvme5: mapped 4/0/0 default/read/poll queues. [ 87.274245] [ T1137] nvme nvme5: Removing ctrl: NQN "blktests-subsystem-1" [ 87.291121] [ T1137] ====================================================== [ 87.292008] [ T1137] WARNING: possible circular locking dependency detected [ 87.292880] [ T1137] 7.2.0+ #669 Not tainted [ 87.293454] [ T1137] ------------------------------------------------------ [ 87.294356] [ T1137] nvme/1137 is trying to acquire lock: [ 87.295070] [ T1137] ffff88812e989518 (set->srcu){.+.+}-{0:0}, at: __synchronize_srcu+0xc1/0x2f0 [ 87.296192] [ T1137] but task is already holding lock: [ 87.297124] [ T1137] ffff88813bebaac8 (&q->elevator_lock){+.+.}-{4:4}, at: elevator_change+0x197/0x500 [ 87.298299] [ T1137] which lock already depends on the new lock. [ 87.299570] [ T1137] the existing dependency chain (in reverse order) is: [ 87.300685] [ T1137] -> #5 (&q->elevator_lock){+.+.}-{4:4}: [ 87.301657] [ T1137] __mutex_lock+0x1ae/0x2500 [ 87.302338] [ T1137] elevator_change+0x197/0x500 [ 87.303047] [ T1137] elv_iosched_store+0x38f/0x430 [ 87.303719] [ T1137] queue_attr_store+0x25f/0x3e0 [ 87.304377] [ T1137] kernfs_fop_write_iter+0x3d6/0x5e0 [ 87.305121] [ T1137] vfs_write+0x4b3/0xf40 [ 87.305747] [ T1137] ksys_write+0x112/0x250 [ 87.306821] [ T1137] do_syscall_64+0xdf/0x790 [ 87.307851] [ T1137] entry_SYSCALL_64_after_hwframe+0x76/0x7e [ 87.309110] [ T1137] -> #4 (&q->q_usage_counter(io)){++++}-{0:0}: [ 87.310932] [ T1137] blk_alloc_queue+0x605/0x7a0 [ 87.312034] [ T1137] blk_mq_alloc_queue+0x168/0x270 [ 87.313138] [ T1137] scsi_alloc_sdev+0x8df/0xd10 [ 87.314229] [ T1137] scsi_probe_and_add_lun+0x5bd/0xbf0 [ 87.315381] [ T1137] __scsi_add_device+0x233/0x280 [ 87.316480] [ T1137] ata_scsi_scan_host+0x137/0x3a0 [ 87.317560] [ T1137] async_run_entry_fn+0x93/0x550 [ 87.318616] [ T1137] process_one_work+0x8b2/0x15e0 [ 87.319657] [ T1137] worker_thread+0x5fd/0xfe0 [ 87.320653] [ T1137] kthread+0x367/0x460 [ 87.321592] [ T1137] ret_from_fork+0x655/0x9d0 [ 87.322577] [ T1137] ret_from_fork_asm+0x1a/0x30 [ 87.323571] [ T1137] -> #3 (fs_reclaim){+.+.}-{0:0}: [ 87.325134] [ T1137] fs_reclaim_acquire+0xd5/0x120 [ 87.326174] [ T1137] __kmalloc_cache_node_noprof+0x67/0x6f0 [ 87.327258] [ T1137] create_worker+0x117/0x790 [ 87.328234] [ T1137] workqueue_prepare_cpu+0x93/0xf0 [ 87.329219] [ T1137] cpuhp_invoke_callback+0x2c5/0x11f0 [ 87.330236] [ T1137] __cpuhp_invoke_callback_range+0xb6/0x1e0 [ 87.331303] [ T1137] _cpu_up+0x2eb/0x6d0 [ 87.332204] [ T1137] cpu_up+0x111/0x190 [ 87.333062] [ T1137] cpuhp_bringup_mask+0xd3/0x110 [ 87.334049] [ T1137] bringup_nonboot_cpus+0x139/0x170 [ 87.335075] [ T1137] smp_init+0x27/0xe0 [ 87.335914] [ T1137] kernel_init_freeable+0x442/0x710 [ 87.336905] [ T1137] kernel_init+0x18/0x150 [ 87.337769] [ T1137] ret_from_fork+0x655/0x9d0 [ 87.338655] [ T1137] ret_from_fork_asm+0x1a/0x30 [ 87.339554] [ T1137] -> #2 (cpu_hotplug_lock){++++}-{0:0}: [ 87.340978] [ T1137] cpus_read_lock+0x3c/0xe0 [ 87.341886] [ T1137] static_key_disable+0x12/0x30 [ 87.342790] [ T1137] __inet_hash_connect+0xf7f/0x1a60 [ 87.343727] [ T1137] tcp_v4_connect+0xcb7/0x1970 [ 87.344619] [ T1137] __inet_stream_connect+0x399/0xfb0 [ 87.345570] [ T1137] inet_stream_connect+0x55/0xb0 [ 87.346474] [ T1137] kernel_connect+0x103/0x170 [ 87.347385] [ T1137] nvme_tcp_alloc_queue+0xa48/0x1ba0 [nvme_tcp] [ 87.348457] [ T1137] nvme_tcp_alloc_admin_queue+0xff/0x440 [nvme_tcp] [ 87.349569] [ T1137] nvme_tcp_setup_ctrl+0xad/0x8a0 [nvme_tcp] [ 87.350603] [ T1137] nvme_tcp_create_ctrl+0x874/0xc20 [nvme_tcp] [ 87.351646] [ T1137] nvmf_dev_write+0x40b/0x830 [nvme_fabrics] [ 87.352660] [ T1137] vfs_write+0x1cc/0xf40 [ 87.353468] [ T1137] ksys_write+0x112/0x250 [ 87.354310] [ T1137] do_syscall_64+0xdf/0x790 [ 87.355175] [ T1137] entry_SYSCALL_64_after_hwframe+0x76/0x7e [ 87.356196] [ T1137] -> #1 (sk_lock-AF_INET-NVME){+.+.}-{0:0}: [ 87.357635] [ T1137] lock_sock_nested+0x32/0xf0 [ 87.358491] [ T1137] tcp_sendmsg+0x1c/0x50 [ 87.359322] [ T1137] sock_sendmsg+0x31c/0x3f0 [ 87.360180] [ T1137] nvme_tcp_try_send_cmd_pdu+0x60e/0xcc0 [nvme_tcp] [ 87.361282] [ T1137] nvme_tcp_try_send+0x1ef/0xa60 [nvme_tcp] [ 87.362299] [ T1137] nvme_tcp_queue_rq+0xfa3/0x19e0 [nvme_tcp] [ 87.363334] [ T1137] blk_mq_dispatch_rq_list+0x3e0/0x2400 [ 87.364316] [ T1137] __blk_mq_sched_dispatch_requests+0x20a/0x15d0 [ 87.365378] [ T1137] blk_mq_sched_dispatch_requests+0xa7/0x140 [ 87.366411] [ T1137] blk_mq_run_work_fn+0x135/0x2e0 [ 87.367331] [ T1137] process_one_work+0x8b2/0x15e0 [ 87.368243] [ T1137] worker_thread+0x5fd/0xfe0 [ 87.369107] [ T1137] kthread+0x367/0x460 [ 87.369924] [ T1137] ret_from_fork+0x655/0x9d0 [ 87.370776] [ T1137] ret_from_fork_asm+0x1a/0x30 [ 87.371645] [ T1137] -> #0 (set->srcu){.+.+}-{0:0}: [ 87.372966] [ T1137] __lock_acquire+0xe20/0x2440 [ 87.373847] [ T1137] lock_sync+0xbf/0x120 [ 87.374646] [ T1137] __synchronize_srcu+0xe1/0x2f0 [ 87.375530] [ T1137] elevator_switch+0x2bd/0x670 [ 87.376421] [ T1137] elevator_change+0x2e7/0x500 [ 87.377311] [ T1137] elevator_set_none+0xaa/0xf0 [ 87.378203] [ T1137] blk_unregister_queue+0x15e/0x2e0 [ 87.379139] [ T1137] __del_gendisk+0x28b/0xaa0 [ 87.380009] [ T1137] del_gendisk+0x11a/0x1c0 [ 87.380844] [ T1137] nvme_ns_remove+0x331/0x9e0 [nvme_core] [ 87.381851] [ T1137] nvme_remove_namespaces+0x289/0x3f0 [nvme_core] [ 87.382970] [ T1137] nvme_do_delete_ctrl+0xf6/0x160 [nvme_core] [ 87.384028] [ T1137] nvme_delete_ctrl_sync.cold+0x8/0xd [nvme_core] [ 87.385124] [ T1137] nvme_sysfs_delete+0xb7/0xe0 [nvme_core] [ 87.386153] [ T1137] kernfs_fop_write_iter+0x3d6/0x5e0 [ 87.387104] [ T1137] vfs_write+0x4b3/0xf40 [ 87.387935] [ T1137] ksys_write+0x112/0x250 [ 87.388756] [ T1137] do_syscall_64+0xdf/0x790 [ 87.389596] [ T1137] entry_SYSCALL_64_after_hwframe+0x76/0x7e [ 87.390592] [ T1137] other info that might help us debug this: [ 87.392533] [ T1137] Chain exists of: set->srcu --> &q->q_usage_counter(io) --> &q->elevator_lock [ 87.394872] [ T1137] Possible unsafe locking scenario: [ 87.396268] [ T1137] CPU0 CPU1 [ 87.397169] [ T1137] ---- ---- [ 87.398062] [ T1137] lock(&q->elevator_lock); [ 87.398873] [ T1137] lock(&q->q_usage_counter(io)); [ 87.400042] [ T1137] lock(&q->elevator_lock); [ 87.401144] [ T1137] sync(set->srcu); [ 87.401864] [ T1137] *** DEADLOCK *** [ 87.403562] [ T1137] 5 locks held by nvme/1137: [ 87.404365] [ T1137] #0: ffff8881134a0450 (sb_writers#4){.+.+}-{0:0}, at: ksys_write+0x112/0x250 [ 87.405677] [ T1137] #1: ffff888137b45080 (&of->mutex#2){+.+.}-{4:4}, at: kernfs_fop_write_iter+0x257/0x5e0 [ 87.407118] [ T1137] #2: ffff8881475f82d8 (kn->active#141){++++}-{0:0}, at: sysfs_remove_file_self+0x61/0xb0 [ 87.408549] [ T1137] #3: ffff88812d8641c8 (&set->update_nr_hwq_lock){++++}-{4:4}, at: del_gendisk+0x112/0x1c0 [ 87.410012] [ T1137] #4: ffff88813bebaac8 (&q->elevator_lock){+.+.}-{4:4}, at: elevator_change+0x197/0x500 [ 87.411425] [ T1137] stack backtrace: [ 87.412674] [ T1137] CPU: 3 UID: 0 PID: 1137 Comm: nvme Not tainted 7.2.0+ #669 PREEMPT(full) [ 87.412679] [ T1137] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS 1.17.0-10.fc44 06/10/2025 [ 87.412684] [ T1137] Call Trace: [ 87.412689] [ T1137] <TASK> [ 87.412691] [ T1137] dump_stack_lvl+0x6a/0x90 [ 87.412697] [ T1137] print_circular_bug.cold+0x189/0x1eb [ 87.412704] [ T1137] check_noncircular+0x173/0x1a0 [ 87.412708] [ T1137] __lock_acquire+0xe20/0x2440 [ 87.412712] [ T1137] lock_sync+0xbf/0x120 [ 87.412714] [ T1137] ? __synchronize_srcu+0xc1/0x2f0 [ 87.412717] [ T1137] ? __synchronize_srcu+0xc1/0x2f0 [ 87.412720] [ T1137] __synchronize_srcu+0xe1/0x2f0 [ 87.412723] [ T1137] ? __pfx___synchronize_srcu+0x10/0x10 [ 87.412727] [ T1137] ? ktime_get_mono_fast_ns+0x1f5/0x6a0 [ 87.412733] [ T1137] ? _raw_spin_unlock_irqrestore+0x35/0x60 [ 87.412737] [ T1137] elevator_switch+0x2bd/0x670 [ 87.412740] [ T1137] ? elevator_change+0x197/0x500 [ 87.412742] [ T1137] elevator_change+0x2e7/0x500 [ 87.412744] [ T1137] ? kernfs_put.part.0+0x2ca/0x7d0 [ 87.412747] [ T1137] elevator_set_none+0xaa/0xf0 [ 87.412749] [ T1137] ? __pfx_elevator_set_none+0x10/0x10 [ 87.412751] [ T1137] ? kernfs_put.part.0+0x2f3/0x7d0 [ 87.412754] [ T1137] ? kobject_put+0x62/0x530 [ 87.412759] [ T1137] blk_unregister_queue+0x15e/0x2e0 [ 87.412762] [ T1137] __del_gendisk+0x28b/0xaa0 [ 87.412765] [ T1137] ? down_read+0xbd/0x530 [ 87.412767] [ T1137] ? down_read+0x148/0x530 [ 87.412770] [ T1137] ? __pfx___del_gendisk+0x10/0x10 [ 87.412771] [ T1137] ? __pfx_down_read+0x10/0x10 [ 87.412774] [ T1137] ? up_write+0x24c/0x760 [ 87.412776] [ T1137] ? up_write+0x2f8/0x760 [ 87.412779] [ T1137] del_gendisk+0x11a/0x1c0 [ 87.412782] [ T1137] nvme_ns_remove+0x331/0x9e0 [nvme_core] [ 87.412802] [ T1137] ? _raw_spin_unlock_irqrestore+0x35/0x60 [ 87.412805] [ T1137] nvme_remove_namespaces+0x289/0x3f0 [nvme_core] [ 87.412824] [ T1137] ? __pfx_nvme_remove_namespaces+0x10/0x10 [nvme_core] [ 87.412843] [ T1137] nvme_do_delete_ctrl+0xf6/0x160 [nvme_core] [ 87.412863] [ T1137] nvme_delete_ctrl_sync.cold+0x8/0xd [nvme_core] [ 87.412882] [ T1137] nvme_sysfs_delete+0xb7/0xe0 [nvme_core] [ 87.412901] [ T1137] ? __pfx_sysfs_kf_write+0x10/0x10 [ 87.412903] [ T1137] kernfs_fop_write_iter+0x3d6/0x5e0 [ 87.412907] [ T1137] ? __pfx_kernfs_fop_write_iter+0x10/0x10 [ 87.412909] [ T1137] vfs_write+0x4b3/0xf40 [ 87.412913] [ T1137] ? __pfx_vfs_write+0x10/0x10 [ 87.412915] [ T1137] ? __x64_sys_openat+0x10a/0x210 [ 87.412917] [ T1137] ? __pfx___x64_sys_openat+0x10/0x10 [ 87.412920] [ T1137] ? do_syscall_64+0x1ec/0x790 [ 87.412923] [ T1137] ? trace_hardirqs_on_prepare+0x139/0x180 [ 87.412931] [ T1137] ? lockdep_hardirqs_on+0x8c/0x130 [ 87.412933] [ T1137] ? entry_SYSCALL_64_after_hwframe+0x76/0x7e [ 87.412935] [ T1137] ? do_syscall_64+0x20a/0x790 [ 87.412939] [ T1137] ksys_write+0x112/0x250 [ 87.412941] [ T1137] ? __pfx_ksys_write+0x10/0x10 [ 87.412945] [ T1137] do_syscall_64+0xdf/0x790 [ 87.412949] [ T1137] ? fput_close_sync+0xda/0x1b0 [ 87.412953] [ T1137] ? __pfx_fput_close_sync+0x10/0x10 [ 87.412955] [ T1137] ? do_raw_spin_unlock+0x55/0x230 [ 87.412958] [ T1137] ? rcu_is_watching+0x11/0xb0 [ 87.412967] [ T1137] ? do_syscall_64+0x1ec/0x790 [ 87.412969] [ T1137] ? trace_hardirqs_on_prepare+0x139/0x180 [ 87.412971] [ T1137] ? lockdep_hardirqs_on+0x8c/0x130 [ 87.412973] [ T1137] ? entry_SYSCALL_64_after_hwframe+0x76/0x7e [ 87.412975] [ T1137] ? do_syscall_64+0x20a/0x790 [ 87.412977] [ T1137] ? __pfx_ksys_read+0x10/0x10 [ 87.412979] [ T1137] ? do_sys_openat2+0xff/0x170 [ 87.412981] [ T1137] ? __pfx_fput_close_sync+0x10/0x10 [ 87.412983] [ T1137] ? rcu_is_watching+0x11/0xb0 [ 87.412986] [ T1137] ? do_syscall_64+0x1ec/0x790 [ 87.412988] [ T1137] ? trace_hardirqs_on_prepare+0x139/0x180 [ 87.412990] [ T1137] ? lockdep_hardirqs_on+0x8c/0x130 [ 87.412992] [ T1137] ? entry_SYSCALL_64_after_hwframe+0x76/0x7e [ 87.412994] [ T1137] ? do_syscall_64+0x20a/0x790 [ 87.412996] [ T1137] ? rcu_is_watching+0x11/0xb0 [ 87.412999] [ T1137] ? __x64_sys_openat+0x10a/0x210 [ 87.413002] [ T1137] ? __pfx___x64_sys_openat+0x10/0x10 [ 87.413004] [ T1137] ? entry_SYSCALL_64_after_hwframe+0x76/0x7e [ 87.413006] [ T1137] ? rcu_is_watching+0x11/0xb0 [ 87.413009] [ T1137] ? do_syscall_64+0x1ec/0x790 [ 87.413011] [ T1137] ? trace_hardirqs_on_prepare+0x139/0x180 [ 87.413013] [ T1137] ? lockdep_hardirqs_on+0x8c/0x130 [ 87.413015] [ T1137] ? do_syscall_64+0x20a/0x790 [ 87.413017] [ T1137] ? entry_SYSCALL_64_after_hwframe+0x76/0x7e [ 87.413019] [ T1137] ? rcu_is_watching+0x11/0xb0 [ 87.413022] [ T1137] ? trace_hardirqs_on+0x14/0x190 [ 87.413023] [ T1137] ? preempt_count_add+0x7f/0x190 [ 87.413028] [ T1137] ? do_syscall_64+0x5d/0x790 [ 87.413030] [ T1137] ? do_syscall_64+0x8d/0x790 [ 87.413032] [ T1137] ? irqentry_exit+0xfc/0x810 [ 87.413035] [ T1137] entry_SYSCALL_64_after_hwframe+0x76/0x7e [ 87.413037] [ T1137] RIP: 0033:0x7fa87169008e [ 87.413040] [ T1137] Code: 4d 89 d8 e8 94 bd 00 00 4c 8b 5d f8 41 8b 93 08 03 00 00 59 5e 48 83 f8 fc 74 11 c9 c3 0f 1f 80 00 00 00 00 48 8b 45 10 0f 05 <c9> c3 83 e2 39 83 fa 08 75 e7 e8 03 ff ff ff 0f 1f 00 f3 0f 1e fa [ 87.413042] [ T1137] RSP: 002b:00007ffee438c910 EFLAGS: 00000202 ORIG_RAX: 0000000000000001 [ 87.413049] [ T1137] RAX: ffffffffffffffda RBX: 00007fa871865006 RCX: 00007fa87169008e [ 87.413051] [ T1137] RDX: 0000000000000001 RSI: 00007fa871865006 RDI: 0000000000000003 [ 87.413053] [ T1137] RBP: 00007ffee438c920 R08: 0000000000000000 R09: 0000000000000000 [ 87.413054] [ T1137] R10: 0000000000000000 R11: 0000000000000202 R12: 0000000010ade770 [ 87.413055] [ T1137] R13: 0000000000000000 R14: 0000000000000000 R15: 0000000010adef80 [ 87.413059] [ T1137] </TASK> ^ permalink raw reply related [flat|nested] 13+ messages in thread
* Re: [syzbot] [net?] WARNING: locking bug in tcp_tsq_handler 2026-08-25 5:44 ` Shin'ichiro Kawasaki @ 2026-08-25 6:25 ` Hillf Danton 2026-08-25 6:45 ` Hillf Danton 2026-08-25 13:08 ` Shin'ichiro Kawasaki 2 siblings, 0 replies; 13+ messages in thread From: Hillf Danton @ 2026-08-25 6:25 UTC (permalink / raw) To: Shin'ichiro Kawasaki Cc: Eric Dumazet, syzbot, Nilay Shroff, Keith Busch, boqun, linux-kernel, netdev, peterz, syzkaller-bugs On Tue, 25 Aug 2026 14:44:35 +0900 Shin'ichiro Kawasaki wrote: > On Aug 25, 2026 / 03:50, Eric Dumazet wrote: > > > > I think 19bdb70c77d3 should be reverted. > > Just reverting the commit will reintroduce the other lockdep WARN that the > commit addressed. I hope to have another fix to avoid the WARN. > I do not think you know the root cause of that lockdep warn, so reverting 19bdb70c77d3 is the right thing to do because of [11, 12]. [11] ffa1e7ada456 ("block: Make request_queue lockdep splats show up earlier") [12] Subject: Re: [PATCH] nbd: don't warn when reclassifying a busy socket lock Subject: Re: [PATCH] nbd: don't warn when reclassifying a busy socket lock ^ permalink raw reply [flat|nested] 13+ messages in thread
* Re: [syzbot] [net?] WARNING: locking bug in tcp_tsq_handler 2026-08-25 5:44 ` Shin'ichiro Kawasaki 2026-08-25 6:25 ` Hillf Danton @ 2026-08-25 6:45 ` Hillf Danton 2026-08-25 13:26 ` Shin'ichiro Kawasaki 2026-08-25 13:08 ` Shin'ichiro Kawasaki 2 siblings, 1 reply; 13+ messages in thread From: Hillf Danton @ 2026-08-25 6:45 UTC (permalink / raw) To: Shin'ichiro Kawasaki Cc: Eric Dumazet, syzbot, Nilay Shroff, Keith Busch, boqun, linux-kernel, netdev, peterz, syzkaller-bugs On Tue, 25 Aug 2026 14:44:35 +0900 Shin'ichiro Kawasaki wrote: > On Aug 25, 2026 / 03:50, Eric Dumazet wrote: > > > > I think 19bdb70c77d3 should be reverted. > > Just reverting the commit will reintroduce the other lockdep WARN that the > commit addressed. I hope to have another fix to avoid the WARN. > I do not think you know the root cause of that lockdep warn, so reverting 19bdb70c77d3 is the right thing to do because of [11, 12]. [11] ffa1e7ada456 ("block: Make request_queue lockdep splats show up earlier") [12] Subject: Re: [PATCH] nbd: don't warn when reclassifying a busy socket lock https://lore.kernel.org/lkml/20260816014906.1149-1-hdanton@sina.com/ ^ permalink raw reply [flat|nested] 13+ messages in thread
* Re: [syzbot] [net?] WARNING: locking bug in tcp_tsq_handler 2026-08-25 6:45 ` Hillf Danton @ 2026-08-25 13:26 ` Shin'ichiro Kawasaki 2026-08-25 13:53 ` Eric Dumazet 0 siblings, 1 reply; 13+ messages in thread From: Shin'ichiro Kawasaki @ 2026-08-25 13:26 UTC (permalink / raw) To: Hillf Danton Cc: Eric Dumazet, syzbot, Nilay Shroff, Keith Busch, boqun, linux-kernel, netdev, peterz, syzkaller-bugs On Aug 25, 2026 / 14:45, Hillf Danton wrote: > On Tue, 25 Aug 2026 14:44:35 +0900 Shin'ichiro Kawasaki wrote: > > On Aug 25, 2026 / 03:50, Eric Dumazet wrote: > > > > > > I think 19bdb70c77d3 should be reverted. > > > > Just reverting the commit will reintroduce the other lockdep WARN that the > > commit addressed. I hope to have another fix to avoid the WARN. > > > I do not think you know the root cause of that lockdep warn, so reverting > 19bdb70c77d3 is the right thing to do because of [11, 12]. > > [11] ffa1e7ada456 ("block: Make request_queue lockdep splats show up earlier") > [12] Subject: Re: [PATCH] nbd: don't warn when reclassifying a busy socket lock > https://lore.kernel.org/lkml/20260816014906.1149-1-hdanton@sina.com/ I took a look in the commit [11] and the discussion [12], but couldn't follow why the commit 19bdb70c77d3 should be reverted. Could you elaborate a bit more about the reason you suggest the revert? ^ permalink raw reply [flat|nested] 13+ messages in thread
* Re: [syzbot] [net?] WARNING: locking bug in tcp_tsq_handler 2026-08-25 13:26 ` Shin'ichiro Kawasaki @ 2026-08-25 13:53 ` Eric Dumazet 2026-08-25 14:27 ` Eric Dumazet 0 siblings, 1 reply; 13+ messages in thread From: Eric Dumazet @ 2026-08-25 13:53 UTC (permalink / raw) To: Shin'ichiro Kawasaki Cc: Hillf Danton, syzbot, Nilay Shroff, Keith Busch, boqun, linux-kernel, netdev, peterz, syzkaller-bugs On Tue, Aug 25, 2026 at 3:26 PM Shin'ichiro Kawasaki <shinichiro.kawasaki@wdc.com> wrote: > > On Aug 25, 2026 / 14:45, Hillf Danton wrote: > > On Tue, 25 Aug 2026 14:44:35 +0900 Shin'ichiro Kawasaki wrote: > > > On Aug 25, 2026 / 03:50, Eric Dumazet wrote: > > > > > > > > I think 19bdb70c77d3 should be reverted. > > > > > > Just reverting the commit will reintroduce the other lockdep WARN that the > > > commit addressed. I hope to have another fix to avoid the WARN. > > > > > I do not think you know the root cause of that lockdep warn, so reverting > > 19bdb70c77d3 is the right thing to do because of [11, 12]. > > > > [11] ffa1e7ada456 ("block: Make request_queue lockdep splats show up earlier") > > [12] Subject: Re: [PATCH] nbd: don't warn when reclassifying a busy socket lock > > https://lore.kernel.org/lkml/20260816014906.1149-1-hdanton@sina.com/ > > I took a look in the commit [11] and the discussion [12], but couldn't follow > why the commit 19bdb70c77d3 should be reverted. Could you elaborate a bit more > about the reason you suggest the revert? I think you missed one problem in lib/once.c Revert commit e8eef69a99f1 (or restore once_disable_jump(once_key, mod) inside __do_once_sleepable_done()) so static_branch_disable() is never called synchronously under caller locks. Previously, once_disable_jump() deferred static_branch_disable() to a worker thread via schedule_work(). Commit e8eef69a99f1 made it run synchronously in the caller's context. Because __inet_hash_connect() is called under lock_sock(sk), calling static_branch_disable() directly inside DO_ONCE_SLEEPABLE() takes cpus_read_lock() (cpu_hotplug_lock) while holding lock_sock(sk). This created the bogus lock dependency sk_lock -> cpu_hotplug_lock. Every other storage/networking client (sunrpc, nbd, cifs, iscsi_tcp, rxe, siw) uses static lockdep keys (static struct lock_class_key ...[2]) without issue. Use my pending TCP patch, which is a no brainer : https://lore.kernel.org/netdev/20260825023614.1228551-1-edumazet@google.com/ Revert 19bdb70c77d3 in drivers/nvme/host/tcp.c : Restore static lockdep keys nvme_tcp_sk_key[2] and nvme_tcp_slock_key[2]. And we should be good. If you think each nvme-tcp socket MUST have its own LOCKDEP class, please elaborate, because I have no idea why this would be needed. ^ permalink raw reply [flat|nested] 13+ messages in thread
* Re: [syzbot] [net?] WARNING: locking bug in tcp_tsq_handler 2026-08-25 13:53 ` Eric Dumazet @ 2026-08-25 14:27 ` Eric Dumazet 0 siblings, 0 replies; 13+ messages in thread From: Eric Dumazet @ 2026-08-25 14:27 UTC (permalink / raw) To: Shin'ichiro Kawasaki Cc: Hillf Danton, syzbot, Nilay Shroff, Keith Busch, boqun, linux-kernel, netdev, peterz, syzkaller-bugs On Tue, Aug 25, 2026 at 3:53 PM Eric Dumazet <edumazet@google.com> wrote: > > On Tue, Aug 25, 2026 at 3:26 PM Shin'ichiro Kawasaki > <shinichiro.kawasaki@wdc.com> wrote: > > > > On Aug 25, 2026 / 14:45, Hillf Danton wrote: > > > On Tue, 25 Aug 2026 14:44:35 +0900 Shin'ichiro Kawasaki wrote: > > > > On Aug 25, 2026 / 03:50, Eric Dumazet wrote: > > > > > > > > > > I think 19bdb70c77d3 should be reverted. > > > > > > > > Just reverting the commit will reintroduce the other lockdep WARN that the > > > > commit addressed. I hope to have another fix to avoid the WARN. > > > > > > > I do not think you know the root cause of that lockdep warn, so reverting > > > 19bdb70c77d3 is the right thing to do because of [11, 12]. > > > > > > [11] ffa1e7ada456 ("block: Make request_queue lockdep splats show up earlier") > > > [12] Subject: Re: [PATCH] nbd: don't warn when reclassifying a busy socket lock > > > https://lore.kernel.org/lkml/20260816014906.1149-1-hdanton@sina.com/ > > > > I took a look in the commit [11] and the discussion [12], but couldn't follow > > why the commit 19bdb70c77d3 should be reverted. Could you elaborate a bit more > > about the reason you suggest the revert? > > I think you missed one problem in lib/once.c > > Revert commit e8eef69a99f1 (or restore once_disable_jump(once_key, > mod) inside __do_once_sleepable_done()) > so static_branch_disable() is never called synchronously under caller locks. > > Previously, once_disable_jump() deferred static_branch_disable() to a > worker thread via schedule_work(). > > Commit e8eef69a99f1 made it run synchronously in the caller's context. > Because __inet_hash_connect() is called under lock_sock(sk), calling > static_branch_disable() directly inside DO_ONCE_SLEEPABLE() takes > cpus_read_lock() (cpu_hotplug_lock) while holding lock_sock(sk). > > This created the bogus lock dependency sk_lock -> cpu_hotplug_lock. > > Every other storage/networking client (sunrpc, nbd, cifs, iscsi_tcp, > rxe, siw) uses static lockdep keys (static struct lock_class_key > ...[2]) without issue. I sent the revert request for review : https://lore.kernel.org/lkml/20260825142515.1965654-1-edumazet@google.com/T/#u > > Use my pending TCP patch, which is a no brainer : > https://lore.kernel.org/netdev/20260825023614.1228551-1-edumazet@google.com/ > > Revert 19bdb70c77d3 in drivers/nvme/host/tcp.c : Restore static > lockdep keys nvme_tcp_sk_key[2] and nvme_tcp_slock_key[2]. > > And we should be good. > > If you think each nvme-tcp socket MUST have its own LOCKDEP class, > please elaborate, because I have no idea > why this would be needed. ^ permalink raw reply [flat|nested] 13+ messages in thread
* Re: [syzbot] [net?] WARNING: locking bug in tcp_tsq_handler 2026-08-25 5:44 ` Shin'ichiro Kawasaki 2026-08-25 6:25 ` Hillf Danton 2026-08-25 6:45 ` Hillf Danton @ 2026-08-25 13:08 ` Shin'ichiro Kawasaki 2 siblings, 0 replies; 13+ messages in thread From: Shin'ichiro Kawasaki @ 2026-08-25 13:08 UTC (permalink / raw) To: Eric Dumazet Cc: syzbot, Nilay Shroff, Keith Busch, boqun, hdanton, linux-kernel, netdev, peterz, syzkaller-bugs [-- Attachment #1: Type: text/plain, Size: 2077 bytes --] On Aug 25, 2026 / 14:44, Shin'ichiro Kawasaki wrote: > On Aug 25, 2026 / 03:50, Eric Dumazet wrote: [...] > > I think 19bdb70c77d3 should be reverted. > > Just reverting the commit will reintroduce the other lockdep WARN that the > commit addressed. I hope to have another fix to avoid the WARN. > > > > > We can change TCP to use sk_gfp_mask(sk, GFP_ATOMIC) instead of > > gfp_any() in tcp_disconnect() > > > > This ensures tcp_disconnect() respects sk->sk_allocation = GFP_ATOMIC > > and never acquires fs_reclaim under sk_lock. > > > > WDYT? > > Thanks for the idea. I did a quick trial with the idea. > > Step 1: > I reverted the commit 19bdb70c77d3 from v7.2 kernel, and confirmed that > the blktests test case nvme/005 for tcp transport recreates the lockdep > WARN that includes fs_reclaim in its lock chain. > > Step 2: > I created a patch to replace gfp_any() in tcp_disconnect() with GFP_ATOMIC > [1]. I applied this patch to the v7.2 based kernel that I used in the step 1. > I ran the test case nvme/005 on this kernel, and observed it still fails > with the lockdep WARN: fs_reclaim was still included in the lock chain. > > I think this is expected, since fs_reclaim dependency comes from CPU hotplug > bring-up context. > > Based on this observation, I'm afraid that using GFP_ATOMIC in tcp_disconnect() > won't work, unfortunately. > > Another approach I can think of is to use sk->sk_destruct hook to unregister > keys, so that the unregistraion happens after the all in-flight skbs complete. > I will try this approach. I created a patch that delay the lockdep key unregstration until sk desctruct, and attached it to this e-mail. It applies to the recent Linus master branch tip (git hash 818bebeb63dd). Eric, may I ask your comment on the patch and this fix approach? I think this approach will avoid the lockdep that syzbot reported. But I don't know how to confirm it. Could you do the confimration ? (or let me know how to do it). This approach adds some complexity. If anyone has simpler solution, it will be great. [-- Attachment #2: 0001-nvme-tcp-unregister-lockdep-keys-at-socket-destructi.patch --] [-- Type: text/plain, Size: 7235 bytes --] From 3691427f5e9df338bc21068dfb12f4af00a371e7 Mon Sep 17 00:00:00 2001 From: Shin'ichiro Kawasaki <shinichiro.kawasaki@wdc.com> Date: Tue, 25 Aug 2026 17:26:53 +0900 Subject: [PATCH] nvme-tcp: unregister lockdep keys at socket destruction Commit 19bdb70c77d3 ("nvme-tcp: lockdep: use dynamic lockdep keys per socket instance") introduced dynamic lockdep keys for nvme-tcp socket instances. The lockdep keys are unregistered in nvme_tcp_free_queue(), just after __fput_sync(queue->sock->file) call. However, at this point still in-flight skbs are there. When the skbs are freed, the socket and the unregistered lockdep keys can be referenced, which resutls in WARNs [1]. To avoid the WARN, keep the lockdep keys alive until the socket is destroyed. Allocate the keys separately from struct nvme_tcp_queue and replace the socket's sk_destruct callback with an NVMe/TCP wrapper. The wrapper invokes the original destructor and queues work to unregister and free the keys in process context, since socket destruction can run in softirq context. Hold an explicit module reference until the deferred work completes so that both the destructor and work callback remain valid. [1] https://lore.kernel.org/netdev/CANn89i+wnTLC==UnXCpjsS4YxvEfhe5oK0N7fttbqr1zKyqdug@mail.gmail.com/ Fixes: 19bdb70c77d3 ("nvme-tcp: lockdep: use dynamic lockdep keys per socket instance") Signed-off-by: Shin'ichiro Kawasaki <shinichiro.kawasaki@wdc.com> --- drivers/nvme/host/tcp.c | 117 +++++++++++++++++++++++++++++----------- 1 file changed, 86 insertions(+), 31 deletions(-) diff --git a/drivers/nvme/host/tcp.c b/drivers/nvme/host/tcp.c index 5fda9661bdb7..645913edf1f4 100644 --- a/drivers/nvme/host/tcp.c +++ b/drivers/nvme/host/tcp.c @@ -144,11 +144,6 @@ struct nvme_tcp_queue { void (*state_change)(struct sock *); void (*data_ready)(struct sock *); void (*write_space)(struct sock *); - -#ifdef CONFIG_DEBUG_LOCK_ALLOC - struct lock_class_key nvme_tcp_sk_key; - struct lock_class_key nvme_tcp_slock_key; -#endif }; static DEFINE_MUTEX(nvme_tcp_ctrl_mutex); @@ -179,35 +174,93 @@ static const struct blk_mq_ops nvme_tcp_admin_mq_ops; static int nvme_tcp_try_send(struct nvme_tcp_queue *queue); #ifdef CONFIG_DEBUG_LOCK_ALLOC +struct nvme_tcp_lockdep_keys { + struct lock_class_key sk_key; + struct lock_class_key slock_key; + void (*sk_destruct)(struct sock *sk); + struct work_struct free_work; +}; + +static inline struct nvme_tcp_lockdep_keys *nvme_tcp_sock_to_lockdep_keys(struct sock *sk) +{ + struct nvme_tcp_lockdep_keys *keys = container_of( + sk->sk_lock.dep_map.key, struct nvme_tcp_lockdep_keys, sk_key); + + return keys; +} + +static void nvme_tcp_free_lockdep_keys(struct work_struct *work) +{ + struct nvme_tcp_lockdep_keys *keys = container_of(work, + struct nvme_tcp_lockdep_keys, free_work); + + lockdep_unregister_key(&keys->sk_key); + lockdep_unregister_key(&keys->slock_key); + kfree(keys); + module_put(THIS_MODULE); +} + +static void nvme_tcp_sk_destruct(struct sock *sk) +{ + struct nvme_tcp_lockdep_keys *keys = nvme_tcp_sock_to_lockdep_keys(sk); + + if (keys->sk_destruct) + keys->sk_destruct(sk); + + /* + * sk_destruct may run in softirq context. Do cleanup in process + * context. + */ + queue_work(nvme_tcp_wq, &keys->free_work); +} + +static void nvme_tcp_set_sk_destruct(struct sock *sk) +{ + struct nvme_tcp_lockdep_keys *keys = nvme_tcp_sock_to_lockdep_keys(sk); + + keys->sk_destruct = sk->sk_destruct; + + /* keep nvme_tcp loaded until the lockdep key cleanup work completes */ + __module_get(THIS_MODULE); + sk->sk_destruct = nvme_tcp_sk_destruct; +} + /* lockdep can detect a circular dependency of the form * sk_lock -> mmap_lock (page fault) -> fs locks -> sk_lock * because dependencies are tracked for both nvme-tcp and user contexts. Using * a separate class prevents lockdep from conflating nvme-tcp socket use with * user-space socket API use. */ -static void nvme_tcp_reclassify_socket(struct nvme_tcp_queue *queue) +static int nvme_tcp_reclassify_socket(struct nvme_tcp_queue *queue) { + struct nvme_tcp_lockdep_keys *keys; struct sock *sk = queue->sock->sk; if (WARN_ON_ONCE(!sock_allow_reclassification(sk))) - return; + return -EINVAL; + if (WARN_ON_ONCE(sk->sk_family != AF_INET && sk->sk_family != AF_INET6)) + return -EAFNOSUPPORT; + + keys = kzalloc_obj(*keys); + if (!keys) + return -ENOMEM; + + lockdep_register_key(&keys->sk_key); + lockdep_register_key(&keys->slock_key); + INIT_WORK(&keys->free_work, nvme_tcp_free_lockdep_keys); - switch (sk->sk_family) { - case AF_INET: + if (sk->sk_family == AF_INET) sock_lock_init_class_and_name(sk, "slock-AF_INET-NVME", - &queue->nvme_tcp_slock_key, + &keys->slock_key, "sk_lock-AF_INET-NVME", - &queue->nvme_tcp_sk_key); - break; - case AF_INET6: + &keys->sk_key); + else sock_lock_init_class_and_name(sk, "slock-AF_INET6-NVME", - &queue->nvme_tcp_slock_key, + &keys->slock_key, "sk_lock-AF_INET6-NVME", - &queue->nvme_tcp_sk_key); - break; - default: - WARN_ON_ONCE(1); - } + &keys->sk_key); + + return 0; } #endif @@ -1511,11 +1564,6 @@ static void nvme_tcp_free_queue(struct nvme_ctrl *nctrl, int qid) mutex_destroy(&queue->send_mutex); mutex_destroy(&queue->queue_lock); mutex_destroy(&queue->pf_cache_lock); - -#ifdef CONFIG_DEBUG_LOCK_ALLOC - lockdep_unregister_key(&queue->nvme_tcp_sk_key); - lockdep_unregister_key(&queue->nvme_tcp_slock_key); -#endif } static int nvme_tcp_init_connection(struct nvme_tcp_queue *queue) @@ -1831,6 +1879,9 @@ static int nvme_tcp_alloc_queue(struct nvme_ctrl *nctrl, int qid, struct nvme_tcp_queue *queue = &ctrl->queues[qid]; int ret, rcv_pdu_size; struct file *sock_file; +#ifdef CONFIG_DEBUG_LOCK_ALLOC + bool reclassified = false; +#endif mutex_init(&queue->queue_lock); queue->ctrl = ctrl; @@ -1864,9 +1915,10 @@ static int nvme_tcp_alloc_queue(struct nvme_ctrl *nctrl, int qid, sk_net_refcnt_upgrade(queue->sock->sk); #ifdef CONFIG_DEBUG_LOCK_ALLOC - lockdep_register_key(&queue->nvme_tcp_sk_key); - lockdep_register_key(&queue->nvme_tcp_slock_key); - nvme_tcp_reclassify_socket(queue); + ret = nvme_tcp_reclassify_socket(queue); + if (ret) + goto err_sock; + reclassified = true; #endif /* Single syn retry */ @@ -1960,6 +2012,9 @@ static int nvme_tcp_alloc_queue(struct nvme_ctrl *nctrl, int qid, if (ret) goto err_init_connect; +#ifdef CONFIG_DEBUG_LOCK_ALLOC + nvme_tcp_set_sk_destruct(queue->sock->sk); +#endif set_bit(NVME_TCP_Q_ALLOCATED, &queue->flags); return 0; @@ -1969,13 +2024,13 @@ static int nvme_tcp_alloc_queue(struct nvme_ctrl *nctrl, int qid, err_rcv_pdu: kfree(queue->pdu); err_sock: +#ifdef CONFIG_DEBUG_LOCK_ALLOC + if (reclassified) + nvme_tcp_set_sk_destruct(queue->sock->sk); +#endif /* Use sync variant - see nvme_tcp_free_queue() for explanation */ __fput_sync(queue->sock->file); queue->sock = NULL; -#ifdef CONFIG_DEBUG_LOCK_ALLOC - lockdep_unregister_key(&queue->nvme_tcp_sk_key); - lockdep_unregister_key(&queue->nvme_tcp_slock_key); -#endif err_destroy_mutex: mutex_destroy(&queue->send_mutex); mutex_destroy(&queue->queue_lock); -- 2.54.0 ^ permalink raw reply related [flat|nested] 13+ messages in thread
end of thread, other threads:[~2026-08-25 14:27 UTC | newest] Thread overview: 13+ messages (download: mbox.gz follow: Atom feed -- links below jump to the message on this page -- 2026-08-24 15:51 [PATCH] locking/lockdep: Invalidate stale class_cache entries for zapped classes Eric Dumazet 2026-08-25 0:29 ` Hillf Danton 2026-08-25 0:59 ` [syzbot] [net?] WARNING: locking bug in tcp_tsq_handler syzbot 2026-08-25 1:25 ` Hillf Danton 2026-08-25 1:27 ` Eric Dumazet 2026-08-25 1:50 ` Eric Dumazet 2026-08-25 5:44 ` Shin'ichiro Kawasaki 2026-08-25 6:25 ` Hillf Danton 2026-08-25 6:45 ` Hillf Danton 2026-08-25 13:26 ` Shin'ichiro Kawasaki 2026-08-25 13:53 ` Eric Dumazet 2026-08-25 14:27 ` Eric Dumazet 2026-08-25 13:08 ` Shin'ichiro Kawasaki
This is a public inbox, see mirroring instructions for how to clone and mirror all data and code used for this inbox