[PATCH] Bluetooth: l2cap: defer conn param update to avoid conn->lock/hdev->lock inversion

[PATCH] Bluetooth: l2cap: defer conn param update to avoid conn->lock/hdev->lock inversion

[Mikhail Gavrilov]

When a BLE peripheral sends an L2CAP Connection Parameter Update Request
the processing path is:

  process_pending_rx()          [takes conn->lock]
    l2cap_le_sig_channel()
      l2cap_conn_param_update_req()
        hci_le_conn_update()    [takes hdev->lock]

Meanwhile other code paths take the locks in the opposite order:

  l2cap_chan_connect()          [takes hdev->lock]
    ...
      mutex_lock(&conn->lock)

  l2cap_conn_ready()            [hdev->lock via hci_cb_list_lock]
    ...
      mutex_lock(&conn->lock)

This is a classic AB/BA deadlock which lockdep reports as a circular
locking dependency when connecting a BLE MIDI keyboard (Carry-On FC-49).

Fix this by deferring the hci_le_conn_update() and mgmt_new_conn_param()
calls to the hci_cmd_sync workqueue via hci_cmd_sync_queue(), which runs
outside any of the involved locks.

Fixes: f044eb0524a0 ("Bluetooth: Store latency and supervision timeout in connection params")
Signed-off-by: Mikhail Gavrilov <mikhail.v.gavrilov@gmail.com>
---
 net/bluetooth/l2cap_core.c | 76 ++++++++++++++++++++++++++++++++++----
 1 file changed, 69 insertions(+), 7 deletions(-)

diff --git a/net/bluetooth/l2cap_core.c b/net/bluetooth/l2cap_core.c
index 95c65fece39b..e59d3af250ef 100644
--- a/net/bluetooth/l2cap_core.c
+++ b/net/bluetooth/l2cap_core.c
@@ -4670,6 +4670,59 @@ static inline int l2cap_information_rsp(struct l2cap_conn *conn,
 	return 0;
 }
 
+struct conn_param_update_data {
+	u16	handle;
+	bdaddr_t dst;
+	u8	dst_type;
+	u16	min;
+	u16	max;
+	u16	latency;
+	u16	to_multiplier;
+};
+
+static int l2cap_conn_param_update_sync(struct hci_dev *hdev, void *data)
+{
+	struct conn_param_update_data *d = data;
+	struct hci_conn_params *params;
+	struct hci_cp_le_conn_update cp;
+	u8 store_hint = 0x00;
+
+	hci_dev_lock(hdev);
+
+	params = hci_conn_params_lookup(hdev, &d->dst, d->dst_type);
+	if (params) {
+		params->conn_min_interval = d->min;
+		params->conn_max_interval = d->max;
+		params->conn_latency = d->latency;
+		params->supervision_timeout = d->to_multiplier;
+		store_hint = 0x01;
+	}
+
+	hci_dev_unlock(hdev);
+
+	memset(&cp, 0, sizeof(cp));
+	cp.handle		= cpu_to_le16(d->handle);
+	cp.conn_interval_min	= cpu_to_le16(d->min);
+	cp.conn_interval_max	= cpu_to_le16(d->max);
+	cp.conn_latency		= cpu_to_le16(d->latency);
+	cp.supervision_timeout	= cpu_to_le16(d->to_multiplier);
+	cp.min_ce_len		= cpu_to_le16(0x0000);
+	cp.max_ce_len		= cpu_to_le16(0x0000);
+
+	hci_send_cmd(hdev, HCI_OP_LE_CONN_UPDATE, sizeof(cp), &cp);
+
+	mgmt_new_conn_param(hdev, &d->dst, d->dst_type, store_hint,
+			    d->min, d->max, d->latency, d->to_multiplier);
+
+	return 0;
+}
+
+static void l2cap_conn_param_update_destroy(struct hci_dev *hdev, void *data,
+					    int err)
+{
+	kfree(data);
+}
+
 static inline int l2cap_conn_param_update_req(struct l2cap_conn *conn,
 					      struct l2cap_cmd_hdr *cmd,
 					      u16 cmd_len, u8 *data)
@@ -4677,6 +4730,7 @@ static inline int l2cap_conn_param_update_req(struct l2cap_conn *conn,
 	struct hci_conn *hcon = conn->hcon;
 	struct l2cap_conn_param_update_req *req;
 	struct l2cap_conn_param_update_rsp rsp;
+	struct conn_param_update_data *d;
 	u16 min, max, latency, to_multiplier;
 	int err;
 
@@ -4707,14 +4761,22 @@ static inline int l2cap_conn_param_update_req(struct l2cap_conn *conn,
 		       sizeof(rsp), &rsp);
 
 	if (!err) {
-		u8 store_hint;
-
-		store_hint = hci_le_conn_update(hcon, min, max, latency,
-						to_multiplier);
-		mgmt_new_conn_param(hcon->hdev, &hcon->dst, hcon->dst_type,
-				    store_hint, min, max, latency,
-				    to_multiplier);
+		d = kmalloc(sizeof(*d), GFP_KERNEL);
+		if (!d)
+			return 0;
 
+		d->handle = hcon->handle;
+		bacpy(&d->dst, &hcon->dst);
+		d->dst_type = hcon->dst_type;
+		d->min = min;
+		d->max = max;
+		d->latency = latency;
+		d->to_multiplier = to_multiplier;
+
+		if (hci_cmd_sync_queue(hcon->hdev,
+				       l2cap_conn_param_update_sync, d,
+				       l2cap_conn_param_update_destroy) < 0)
+			kfree(d);
 	}
 
 	return 0;
-- 
2.53.0

RE: Bluetooth: l2cap: defer conn param update to avoid conn->lock/hdev->lock inversion

[bluez.test.bot]

[-- Attachment #1: Type: text/plain, Size: 7218 bytes --]

This is automated email and please do not reply to this email!

Dear submitter,

Thank you for submitting the patches to the linux bluetooth mailing list.
This is a CI test results with your patch series:
PW Link:https://patchwork.kernel.org/project/bluetooth/list/?series=1079450

---Test result---

Test Summary:
CheckPatch                    PENDING   0.63 seconds
GitLint                       PENDING   0.31 seconds
SubjectPrefix                 PASS      0.06 seconds
BuildKernel                   PASS      27.42 seconds
CheckAllWarning               PASS      30.20 seconds
CheckSparse                   PASS      28.65 seconds
BuildKernel32                 PASS      26.67 seconds
TestRunnerSetup               PASS      580.34 seconds
TestRunner_l2cap-tester       FAIL      30.12 seconds
TestRunner_iso-tester         PASS      40.08 seconds
TestRunner_bnep-tester        PASS      6.35 seconds
TestRunner_mgmt-tester        FAIL      115.69 seconds
TestRunner_rfcomm-tester      PASS      9.65 seconds
TestRunner_sco-tester         FAIL      14.67 seconds
TestRunner_ioctl-tester       PASS      10.15 seconds
TestRunner_mesh-tester        FAIL      12.53 seconds
TestRunner_smp-tester         PASS      8.66 seconds
TestRunner_userchan-tester    PASS      7.05 seconds
TestRunner_6lowpan-tester     FAIL      8.82 seconds
IncrementalBuild              PENDING   0.34 seconds

Details
##############################
Test: CheckPatch - PENDING
Desc: Run checkpatch.pl script
Output:

##############################
Test: GitLint - PENDING
Desc: Run gitlint
Output:

##############################
Test: TestRunner_l2cap-tester - FAIL
Desc: Run l2cap-tester with test-runner
Output:
Total: 96, Passed: 94 (97.9%), Failed: 2, Not Run: 0

Failed Test Cases
L2CAP BR/EDR Server - Set PHY 1M                     Failed       0.120 seconds
L2CAP BR/EDR Server - Set PHY 3M                     Failed       0.120 seconds
##############################
Test: TestRunner_mgmt-tester - FAIL
Desc: Run mgmt-tester with test-runner
Output:
Total: 494, Passed: 489 (99.0%), Failed: 1, Not Run: 4

Failed Test Cases
Read Exp Feature - Success                           Failed       0.115 seconds
##############################
Test: TestRunner_sco-tester - FAIL
Desc: Run sco-tester with test-runner
Output:
WARNING: possible circular locking dependency detected
7.0.0-rc2-g2b76e604820d #1 Not tainted
------------------------------------------------------
kworker/u5:2/117 is trying to acquire lock:
ffff888001946240 (sk_lock-AF_BLUETOOTH-BTPROTO_SCO){+.+.}-{0:0}, at: sco_connect_cfm+0x358/0x8d0

but task is already holding lock:
ffff8880023c6220 (&conn->lock){+.+.}-{3:3}, at: sco_connect_cfm+0x22d/0x8d0

which lock already depends on the new lock.


the existing dependency chain (in reverse order) is:

-> #1 (&conn->lock){+.+.}-{3:3}:
       lock_acquire+0xf7/0x2c0
       _raw_spin_lock+0x2a/0x40
       sco_sock_connect+0x4d7/0x1280
       __sys_connect+0x1a3/0x260
       __x64_sys_connect+0x6e/0xb0
       do_syscall_64+0xa0/0x570
       entry_SYSCALL_64_after_hwframe+0x74/0x7c

-> #0 (sk_lock-AF_BLUETOOTH-BTPROTO_SCO){+.+.}-{0:0}:
       check_prev_add+0xe9/0xc70
       __lock_acquire+0x1457/0x1df0
       lock_acquire+0xf7/0x2c0
       lock_sock_nested+0x36/0xd0
       sco_connect_cfm+0x358/0x8d0
       hci_sync_conn_complete_evt+0x3d3/0x8e0
       hci_event_packet+0x74f/0xb10
       hci_rx_work+0x398/0xd00
       process_scheduled_works+0xb16/0x1ac0
       worker_thread+0x4ff/0xba0
       kthread+0x368/0x490
       ret_from_fork+0x498/0x7e0
       ret_from_fork_asm+0x19/0x30

other info that might help us debug this:

...
BUG: sleeping function called from invalid context at net/core/sock.c:3782
in_atomic(): 1, irqs_disabled(): 0, non_block: 0, pid: 117, name: kworker/u5:2
preempt_count: 1, expected: 0
RCU nest depth: 0, expected: 0
INFO: lockdep is turned off.
CPU: 0 UID: 0 PID: 117 Comm: kworker/u5:2 Not tainted 7.0.0-rc2-g2b76e604820d #1 PREEMPT(lazy) 
Hardware name: QEMU Standard PC (Q35 + ICH9, 2009), BIOS 1.13.0-1ubuntu1.1 04/01/2014
Workqueue: hci0 hci_rx_work
Call Trace:
 <TASK>
 dump_stack_lvl+0x49/0x60
 __might_resched+0x2ea/0x500
 lock_sock_nested+0x47/0xd0
 ? sco_connect_cfm+0x358/0x8d0
 sco_connect_cfm+0x358/0x8d0
 ? hci_debugfs_create_conn+0x190/0x210
 ? __pfx_sco_connect_cfm+0x10/0x10
 hci_sync_conn_complete_evt+0x3d3/0x8e0
 hci_event_packet+0x74f/0xb10
 ? __pfx_hci_sync_conn_complete_evt+0x10/0x10
 ? __pfx_hci_event_packet+0x10/0x10
 ? mark_held_locks+0x49/0x80
 ? lockdep_hardirqs_on_prepare+0xd4/0x180
 ? _raw_spin_unlock_irqrestore+0x2c/0x50
 hci_rx_work+0x398/0xd00
 process_scheduled_works+0xb16/0x1ac0
 ? __pfx_process_scheduled_works+0x10/0x10
 ? lock_acquire+0xf7/0x2c0
 ? lock_is_held_type+0x9b/0x110
 ? __pfx_hci_rx_work+0x10/0x10
 worker_thread+0x4ff/0xba0
 ? _raw_spin_unlock_irqrestore+0x2c/0x50
 ? __pfx_worker_thread+0x10/0x10
 kthread+0x368/0x490
 ? _raw_spin_unlock_irq+0x23/0x40
 ? __pfx_kthread+0x10/0x10
 ret_from_fork+0x498/0x7e0
 ? __pfx_ret_from_fork+0x10/0x10
 ? __switch_to+0x9e4/0xe50
 ? __switch_to_asm+0x32/0x60
...
Total: 30, Passed: 30 (100.0%), Failed: 0, Not Run: 0
##############################
Test: TestRunner_mesh-tester - FAIL
Desc: Run mesh-tester with test-runner
Output:
Total: 10, Passed: 8 (80.0%), Failed: 2, Not Run: 0

Failed Test Cases
Mesh - Send cancel - 1                               Timed out    2.653 seconds
Mesh - Send cancel - 2                               Timed out    1.992 seconds
##############################
Test: TestRunner_6lowpan-tester - FAIL
Desc: Run 6lowpan-tester with test-runner
Output:
WARNING: possible circular locking dependency detected
7.0.0-rc2-g2b76e604820d #1 Not tainted
------------------------------------------------------
kworker/0:1/11 is trying to acquire lock:
ffff8880026e8940 ((wq_completion)hci0#2){+.+.}-{0:0}, at: touch_wq_lockdep_map+0x75/0x180

but task is already holding lock:
ffffffff9864d720 (rtnl_mutex){+.+.}-{4:4}, at: lowpan_unregister_netdev+0xd/0x30

which lock already depends on the new lock.


the existing dependency chain (in reverse order) is:

-> #4 (rtnl_mutex){+.+.}-{4:4}:
       lock_acquire+0xf7/0x2c0
       __mutex_lock+0x16b/0x1fc0
       lowpan_register_netdev+0x11/0x30
       chan_ready_cb+0x836/0xd00
       l2cap_recv_frame+0x6bfc/0x8b20
       l2cap_recv_acldata+0x790/0xdf0
       hci_rx_work+0x500/0xd00
       process_scheduled_works+0xb16/0x1ac0
       worker_thread+0x4ff/0xba0
       kthread+0x368/0x490
       ret_from_fork+0x498/0x7e0
       ret_from_fork_asm+0x19/0x30

-> #3 (&chan->lock#3/1){+.+.}-{4:4}:
       lock_acquire+0xf7/0x2c0
       __mutex_lock+0x16b/0x1fc0
       l2cap_chan_connect+0x74e/0x1980
       lowpan_control_write+0x523/0x660
       full_proxy_write+0x10b/0x190
       vfs_write+0x1c0/0xf60
       ksys_write+0xf1/0x1d0
       do_syscall_64+0xa0/0x570
       entry_SYSCALL_64_after_hwframe+0x74/0x7c

-> #2 (&conn->lock){+.+.}-{4:4}:
...
Total: 8, Passed: 8 (100.0%), Failed: 0, Not Run: 0
##############################
Test: IncrementalBuild - PENDING
Desc: Incremental build with the patches in the series
Output:



https://github.com/bluez/bluetooth-next/pull/58

---
Regards,
Linux Bluetooth

Re: [PATCH] Bluetooth: l2cap: defer conn param update to avoid conn->lock/hdev->lock inversion

[Paul Menzel]

Dear Mikhail,


Thank you for your patch.

Am 10.04.26 um 00:21 schrieb Mikhail Gavrilov:
> When a BLE peripheral sends an L2CAP Connection Parameter Update Request
> the processing path is:
> 
>    process_pending_rx()          [takes conn->lock]
>      l2cap_le_sig_channel()
>        l2cap_conn_param_update_req()
>          hci_le_conn_update()    [takes hdev->lock]
> 
> Meanwhile other code paths take the locks in the opposite order:
> 
>    l2cap_chan_connect()          [takes hdev->lock]
>      ...
>        mutex_lock(&conn->lock)
> 
>    l2cap_conn_ready()            [hdev->lock via hci_cb_list_lock]
>      ...
>        mutex_lock(&conn->lock)
> 
> This is a classic AB/BA deadlock which lockdep reports as a circular
> locking dependency when connecting a BLE MIDI keyboard (Carry-On FC-49).

Thank you for the great explanation and documenting the test environment.

> Fix this by deferring the hci_le_conn_update() and mgmt_new_conn_param()
> calls to the hci_cmd_sync workqueue via hci_cmd_sync_queue(), which runs
> outside any of the involved locks.

For me, going into more depth about the implementation, for example 
introducing l2cap_conn_param_update_sync(), would be nice, but I guess 
the experts don’t need it.

> Fixes: f044eb0524a0 ("Bluetooth: Store latency and supervision timeout in connection params")
> Signed-off-by: Mikhail Gavrilov <mikhail.v.gavrilov@gmail.com>
> ---
>   net/bluetooth/l2cap_core.c | 76 ++++++++++++++++++++++++++++++++++----
>   1 file changed, 69 insertions(+), 7 deletions(-)
> 
> diff --git a/net/bluetooth/l2cap_core.c b/net/bluetooth/l2cap_core.c
> index 95c65fece39b..e59d3af250ef 100644
> --- a/net/bluetooth/l2cap_core.c
> +++ b/net/bluetooth/l2cap_core.c
> @@ -4670,6 +4670,59 @@ static inline int l2cap_information_rsp(struct l2cap_conn *conn,
>   	return 0;
>   }
>   
> +struct conn_param_update_data {
> +	u16	handle;
> +	bdaddr_t dst;
> +	u8	dst_type;
> +	u16	min;
> +	u16	max;
> +	u16	latency;
> +	u16	to_multiplier;
> +};
> +
> +static int l2cap_conn_param_update_sync(struct hci_dev *hdev, void *data)
> +{
> +	struct conn_param_update_data *d = data;
> +	struct hci_conn_params *params;
> +	struct hci_cp_le_conn_update cp;
> +	u8 store_hint = 0x00;
> +
> +	hci_dev_lock(hdev);
> +
> +	params = hci_conn_params_lookup(hdev, &d->dst, d->dst_type);
> +	if (params) {
> +		params->conn_min_interval = d->min;
> +		params->conn_max_interval = d->max;
> +		params->conn_latency = d->latency;
> +		params->supervision_timeout = d->to_multiplier;
> +		store_hint = 0x01;
> +	}
> +
> +	hci_dev_unlock(hdev);
> +
> +	memset(&cp, 0, sizeof(cp));
> +	cp.handle		= cpu_to_le16(d->handle);
> +	cp.conn_interval_min	= cpu_to_le16(d->min);
> +	cp.conn_interval_max	= cpu_to_le16(d->max);
> +	cp.conn_latency		= cpu_to_le16(d->latency);
> +	cp.supervision_timeout	= cpu_to_le16(d->to_multiplier);
> +	cp.min_ce_len		= cpu_to_le16(0x0000);
> +	cp.max_ce_len		= cpu_to_le16(0x0000);
> +
> +	hci_send_cmd(hdev, HCI_OP_LE_CONN_UPDATE, sizeof(cp), &cp);
> +
> +	mgmt_new_conn_param(hdev, &d->dst, d->dst_type, store_hint,
> +			    d->min, d->max, d->latency, d->to_multiplier);
> +
> +	return 0;
> +}
> +
> +static void l2cap_conn_param_update_destroy(struct hci_dev *hdev, void *data,
> +					    int err)
> +{
> +	kfree(data);
> +}
> +
>   static inline int l2cap_conn_param_update_req(struct l2cap_conn *conn,
>   					      struct l2cap_cmd_hdr *cmd,
>   					      u16 cmd_len, u8 *data)
> @@ -4677,6 +4730,7 @@ static inline int l2cap_conn_param_update_req(struct l2cap_conn *conn,
>   	struct hci_conn *hcon = conn->hcon;
>   	struct l2cap_conn_param_update_req *req;
>   	struct l2cap_conn_param_update_rsp rsp;
> +	struct conn_param_update_data *d;
>   	u16 min, max, latency, to_multiplier;
>   	int err;
>   
> @@ -4707,14 +4761,22 @@ static inline int l2cap_conn_param_update_req(struct l2cap_conn *conn,
>   		       sizeof(rsp), &rsp);
>   
>   	if (!err) {
> -		u8 store_hint;
> -
> -		store_hint = hci_le_conn_update(hcon, min, max, latency,
> -						to_multiplier);
> -		mgmt_new_conn_param(hcon->hdev, &hcon->dst, hcon->dst_type,
> -				    store_hint, min, max, latency,
> -				    to_multiplier);
> +		d = kmalloc(sizeof(*d), GFP_KERNEL);
> +		if (!d)
> +			return 0;
>   
> +		d->handle = hcon->handle;
> +		bacpy(&d->dst, &hcon->dst);
> +		d->dst_type = hcon->dst_type;
> +		d->min = min;
> +		d->max = max;
> +		d->latency = latency;
> +		d->to_multiplier = to_multiplier;
> +
> +		if (hci_cmd_sync_queue(hcon->hdev,
> +				       l2cap_conn_param_update_sync, d,
> +				       l2cap_conn_param_update_destroy) < 0)
> +			kfree(d);
>   	}
>   
>   	return 0;

gemini/gemini-3.1-pro-preview made some comments [1]. At a first glance 
they look valid, but I am no expert.


Kind regards,

Paul


[1]: 
https://sashiko.dev/#/patchset/20260409222122.21394-1-mikhail.v.gavrilov%40gmail.com