From: John Blackwood <john.blackwood@ccur.com>
To: linux-rt-users@vger.kernel.org
Cc: general@lists.openfabrics.org, linux-kernel@vger.kernel.org,
Sven-Thorsten Dietrich <sdietrich@novell.com>
Subject: [PATCH] [WORKAROUND] CONFIG_PREEMPT_RT and ib_umad_close() issue
Date: Mon, 17 Sep 2007 11:22:52 -0400 [thread overview]
Message-ID: <46EE9BCC.1040301@ccur.com> (raw)
When using OFED-1.2.5 based infiniband kernel modules on 2.6.22 based
kernels with the Ingo Molnar CONFIG_PREEMPT_RT applied, then commands
such as ibnetdiscvoer, smpquery, sminfo, etc. will hang. The problem is
with the downgrade_write() rw semaphore usage in the ib_umad_close()
routine.
This patch is a temporary work-around that gets around this
issue by changing the ib_umad_port mutex from a rw_semaphore to a
compat_rw_semaphore.
This is admittedly only a temporary solution.
An example of the BUG console message output and work around
patch are shown below.
bowser> ------------[ cut here ]------------
kernel BUG at kernel/rt.c:352!
invalid opcode: 0000 [#1]
PREEMPT SMP
last sysfs file: /class/infiniband_mad/umad0/port
Modules linked in: rdma_ucm(F) rds(F) ib_ucm(F) ib_srp(F) ib_sdp(F)
rdma_cm(F) iw_cm(F) ib_addr(F) ib_ipoib(F) ib_cm(F) ib_sa(F)
ib_uverbs(F) ib_umad(F) ib_mthca(F) ib_mad(F) ib_core(F)
CPU: 1
EIP: 0060:[<c014d160>] Tainted: GF N VLI
EFLAGS: 00210282 (2.6.22.6-rt_shield_trace #1)
EIP is at rt_downgrade_write+0x0/0x10
eax: f705df7c ebx: 00000008 ecx: f7171014 edx: f705df9c
esi: f7170ffc edi: f7171000 ebp: f7171004 esp: f5fcdef0
ds: 007b es: 007b fs: 00d8 gs: 0000 ss: 0068 preempt:00000001
Process ibnetdiscover (pid: 10842, ti=f5fcc000 task=f6cd00f0
task.ti=f5fcc000)
Stack: f883a4a8 f705df40 00000000 00000008 f6ca1680 f63adf20 f734e7dc
c018c240
00000000 00000000 f734e7dc c2d442c0 f63adf20 f6ca1680 f71d06c0
00000000
00000001 c018a2ec 00000000 00000001 00000003 f44af440 c012bd20
f71d06c0
Call Trace:
[<f883a4a8>] ib_umad_close+0x98/0xf0 [ib_umad]
[<c018c240>] __fput+0x170/0x1a0
[<c018a2ec>] filp_close+0x3c/0x80
[<c012bd20>] close_files+0x50/0x60
[<c012bd98>] put_files_struct+0x28/0x80
[<c012c996>] do_exit+0x1c6/0x570
[<c018b1ca>] sys_write+0x6a/0xf0
[<c012cd96>] do_group_exit+0x26/0x70
[<c01054ff>] sysenter_past_esp+0x68/0x99
=======================
---------------------------
| preempt count: 00000001 ]
| 1-level deep critical section nesting:
----------------------------------------
.. [<c06653b9>] .... __spin_lock_irqsave+0x19/0x50
.....[<00000000>] .. ( <= 0x0)
Code: 5b e9 45 7a 51 00 90 8d 74 26 00 8b 53 1c 85 d2 74 e3 89 d8 89 ca
e8 c0 84 51 00 ff 4b 1c 5b c3 8d 74 26 00 8d bc 27 00 00 00 00 <0f> 0b
eb fe 8d b6 00 00 00 00 8d bf 00 00 00 00 e8 db 79 51 00
EIP: [<c014d160>] rt_downgrade_write+0x0/0x10 SS:ESP 0068:f5fcdef0
BUG: sleeping function called from invalid context ibnetdiscover(10842)
at kernel/rtmutex.c:636
in_atomic():1 [00000001], irqs_disabled():1
[<c0126661>] __might_sleep+0xe1/0x100
[<c036850b>] set_palette+0x2b/0x60
[<c0664f66>] __rt_spin_lock+0x36/0x50
[<c01241be>] __wake_up+0x1e/0x70
[<c012a44b>] wake_up_klogd+0x3b/0x40
[<c0106f76>] die+0x166/0x240
[<c0665c31>] do_trap+0x1b1/0x260
[<c01388a7>] raw_notifier_call_chain+0x17/0x20
[<c0144040>] notify_die+0x30/0x40
[<c01071e0>] do_invalid_op+0x0/0x90
[<c0107263>] do_invalid_op+0x83/0x90
[<c014d160>] rt_downgrade_write+0x0/0x10
[<c01575c2>] add_preempt_count+0x12/0xe0
[<c01575c2>] add_preempt_count+0x12/0xe0
[<c0664f66>] __rt_spin_lock+0x36/0x50
[<c030e345>] lock_list_del_init+0x55/0x80
[<c018c66d>] file_kill+0x18d/0x1a0
[<c06658c2>] error_code+0x72/0x80
[<c015007b>] load_module+0x33b/0xe10
[<c014d160>] rt_downgrade_write+0x0/0x10
[<f883a4a8>] ib_umad_close+0x98/0xf0 [ib_umad]
[<c018c240>] __fput+0x170/0x1a0
[<c018a2ec>] filp_close+0x3c/0x80
[<c012bd20>] close_files+0x50/0x60
[<c012bd98>] put_files_struct+0x28/0x80
[<c012c996>] do_exit+0x1c6/0x570
[<c018b1ca>] sys_write+0x6a/0xf0
[<c012cd96>] do_group_exit+0x26/0x70
[<c01054ff>] sysenter_past_esp+0x68/0x99
=======================
---------------------------
| preempt count: 00000001 ]
| 1-level deep critical section nesting:
----------------------------------------
.. [<c06653b9>] .... __spin_lock_irqsave+0x19/0x50
.....[<00000000>] .. ( <= 0x0)
Fixing recursive fault but reboot is needed!
--- linux-2.6.22/drivers/infiniband/core/user_mad.c 2007-09-17
09:48:45.000000000 -0400
+++ new/drivers/infiniband/core/user_mad.c 2007-09-17 09:50:41.000000000
-0400
@@ -93,7 +93,7 @@ struct ib_umad_port {
struct class_device *sm_class_dev;
struct semaphore sm_sem;
- struct rw_semaphore mutex;
+ struct compat_rw_semaphore mutex;
struct list_head file_list;
struct ib_device *ib_dev;
@@ -159,7 +159,7 @@ static int queue_packet(struct ib_umad_f
{
int ret = 1;
- down_read(&file->port->mutex);
+ compat_down_read(&file->port->mutex);
for (packet->mad.hdr.id = 0;
packet->mad.hdr.id < IB_UMAD_MAX_AGENTS;
@@ -173,7 +173,7 @@ static int queue_packet(struct ib_umad_f
break;
}
- up_read(&file->port->mutex);
+ compat_up_read(&file->port->mutex);
return ret;
}
@@ -461,7 +461,7 @@ static ssize_t ib_umad_write(struct file
goto err;
}
- down_read(&file->port->mutex);
+ compat_down_read(&file->port->mutex);
agent = __get_agent(file, packet->mad.hdr.id);
if (!agent) {
@@ -558,7 +558,7 @@ static ssize_t ib_umad_write(struct file
if (ret)
goto err_send;
- up_read(&file->port->mutex);
+ compat_up_read(&file->port->mutex);
return count;
err_send:
@@ -568,7 +568,7 @@ err_msg:
err_ah:
ib_destroy_ah(ah);
err_up:
- up_read(&file->port->mutex);
+ compat_up_read(&file->port->mutex);
err:
kfree(packet);
return ret;
@@ -597,7 +597,7 @@ static int ib_umad_reg_agent(struct ib_u
int agent_id;
int ret;
- down_write(&file->port->mutex);
+ compat_down_write(&file->port->mutex);
if (!file->port->ib_dev) {
ret = -EPIPE;
@@ -650,7 +650,7 @@ found:
ret = 0;
out:
- up_write(&file->port->mutex);
+ compat_up_write(&file->port->mutex);
return ret;
}
@@ -663,7 +663,7 @@ static int ib_umad_unreg_agent(struct ib
if (get_user(id, (u32 __user *) arg))
return -EFAULT;
- down_write(&file->port->mutex);
+ compat_down_write(&file->port->mutex);
if (id < 0 || id >= IB_UMAD_MAX_AGENTS || !__get_agent(file, id)) {
ret = -EINVAL;
@@ -674,7 +674,7 @@ static int ib_umad_unreg_agent(struct ib
file->agent[id] = NULL;
out:
- up_write(&file->port->mutex);
+ compat_up_write(&file->port->mutex);
if (agent)
ib_unregister_mad_agent(agent);
@@ -710,7 +710,7 @@ static int ib_umad_open(struct inode *in
if (!port)
return -ENXIO;
- down_write(&port->mutex);
+ compat_down_write(&port->mutex);
if (!port->ib_dev) {
ret = -ENXIO;
@@ -736,7 +736,7 @@ static int ib_umad_open(struct inode *in
list_add_tail(&file->port_list, &port->file_list);
out:
- up_write(&port->mutex);
+ compat_up_write(&port->mutex);
return ret;
}
@@ -748,7 +748,7 @@ static int ib_umad_close(struct inode *i
int already_dead;
int i;
- down_write(&file->port->mutex);
+ compat_down_write(&file->port->mutex);
already_dead = file->agents_dead;
file->agents_dead = 1;
@@ -761,14 +761,14 @@ static int ib_umad_close(struct inode *i
list_del(&file->port_list);
- downgrade_write(&file->port->mutex);
+ compat_downgrade_write(&file->port->mutex);
if (!already_dead)
for (i = 0; i < IB_UMAD_MAX_AGENTS; ++i)
if (file->agent[i])
ib_unregister_mad_agent(file->agent[i]);
- up_read(&file->port->mutex);
+ compat_up_read(&file->port->mutex);
kfree(file);
kref_put(&dev->ref, ib_umad_release_dev);
@@ -839,10 +839,10 @@ static int ib_umad_sm_close(struct inode
};
int ret = 0;
- down_write(&port->mutex);
+ compat_down_write(&port->mutex);
if (port->ib_dev)
ret = ib_modify_port(port->ib_dev, port->port_num, 0, &props);
- up_write(&port->mutex);
+ compat_up_write(&port->mutex);
up(&port->sm_sem);
@@ -906,7 +906,7 @@ static int ib_umad_init_port(struct ib_d
port->ib_dev = device;
port->port_num = port_num;
init_MUTEX(&port->sm_sem);
- init_rwsem(&port->mutex);
+ compat_init_rwsem(&port->mutex);
INIT_LIST_HEAD(&port->file_list);
port->dev = cdev_alloc();
@@ -992,7 +992,7 @@ static void ib_umad_kill_port(struct ib_
umad_port[port->dev_num] = NULL;
spin_unlock(&port_lock);
- down_write(&port->mutex);
+ compat_down_write(&port->mutex);
port->ib_dev = NULL;
@@ -1017,17 +1017,17 @@ static void ib_umad_kill_port(struct ib_
file->agents_dead = 1;
list_del_init(&file->port_list);
- downgrade_write(&port->mutex);
+ compat_downgrade_write(&port->mutex);
for (id = 0; id < IB_UMAD_MAX_AGENTS; ++id)
if (file->agent[id])
ib_unregister_mad_agent(file->agent[id]);
- up_read(&port->mutex);
- down_write(&port->mutex);
+ compat_up_read(&port->mutex);
+ compat_down_write(&port->mutex);
}
- up_write(&port->mutex);
+ compat_up_write(&port->mutex);
clear_bit(port->dev_num, dev_map);
}
next reply other threads:[~2007-09-17 15:23 UTC|newest]
Thread overview: 4+ messages / expand[flat|nested] mbox.gz Atom feed top
2007-09-17 15:22 John Blackwood [this message]
2007-09-17 15:56 ` [ofa-general] [PATCH] [WORKAROUND] CONFIG_PREEMPT_RT and ib_umad_close() issue Roland Dreier
2007-09-17 15:56 ` Roland Dreier
2007-09-17 17:07 ` Daniel Walker
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=46EE9BCC.1040301@ccur.com \
--to=john.blackwood@ccur.com \
--cc=general@lists.openfabrics.org \
--cc=linux-kernel@vger.kernel.org \
--cc=linux-rt-users@vger.kernel.org \
--cc=sdietrich@novell.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.