* [PATCH 1/4] futex,plist: Pass the real head of the priority list to plist_del()
2011-03-12 3:14 [PATCH 0/4] futex,plist: updates Steven Rostedt
@ 2011-03-12 3:14 ` Steven Rostedt
2011-03-12 3:14 ` [PATCH 2/4] futex,plist: Remove debug lock assignment from plist_node Steven Rostedt
` (3 subsequent siblings)
4 siblings, 0 replies; 6+ messages in thread
From: Steven Rostedt @ 2011-03-12 3:14 UTC (permalink / raw)
To: linux-kernel
Cc: Ingo Molnar, Andrew Morton, Thomas Gleixner, Peter Zijlstra,
Lai Jiangshan, Darren Hart
[-- Attachment #1: 0001-futex-plist-Pass-the-real-head-of-the-priority-list-.patch --]
[-- Type: text/plain, Size: 3262 bytes --]
From: Lai Jiangshan <laijs@cn.fujitsu.com>
Some plist_del()s in kernel/futex.c are passed a faked head of the
priority list.
It does not fail because the current code does not require the real head
in plist_del(). The current code of plist_del() just uses the head for checking,
so it will not cause a bad result even when we use a faked head.
But it is undocumented usage:
/**
* plist_del - Remove a @node from plist.
*
* @node: &struct plist_node pointer - entry to be removed
* @head: &struct plist_head pointer - list head
*/
The document says that the @head is the "list head" head of the priority list.
In futex code, several places use "plist_del(&q->list, &q->list.plist);",
they pass a fake head. We need to fix them all.
Thanks to Darren Hart for many suggestions.
Acked-by: Darren Hart <dvhart@linux.intel.com>
Signed-off-by: Lai Jiangshan <laijs@cn.fujitsu.com>
LKML-Reference: <4D11984A.5030203@cn.fujitsu.com>
Signed-off-by: Steven Rostedt <rostedt@goodmis.org>
---
kernel/futex.c | 31 +++++++++++++++++++++++--------
1 files changed, 23 insertions(+), 8 deletions(-)
diff --git a/kernel/futex.c b/kernel/futex.c
index b766d28..6feeea4 100644
--- a/kernel/futex.c
+++ b/kernel/futex.c
@@ -775,6 +775,24 @@ retry:
return ret;
}
+/**
+ * __unqueue_futex() - Remove the futex_q from its futex_hash_bucket
+ * @q: The futex_q to unqueue
+ *
+ * The q->lock_ptr must not be NULL and must be held by the caller.
+ */
+static void __unqueue_futex(struct futex_q *q)
+{
+ struct futex_hash_bucket *hb;
+
+ if (WARN_ON(!q->lock_ptr || !spin_is_locked(q->lock_ptr)
+ || plist_node_empty(&q->list)))
+ return;
+
+ hb = container_of(q->lock_ptr, struct futex_hash_bucket, lock);
+ plist_del(&q->list, &hb->chain);
+}
+
/*
* The hash bucket lock must be held when this is called.
* Afterwards, the futex_q must not be accessed.
@@ -792,7 +810,7 @@ static void wake_futex(struct futex_q *q)
*/
get_task_struct(p);
- plist_del(&q->list, &q->list.plist);
+ __unqueue_futex(q);
/*
* The waiting task can free the futex_q as soon as
* q->lock_ptr = NULL is written, without taking any locks. A
@@ -1100,8 +1118,7 @@ void requeue_pi_wake_futex(struct futex_q *q, union futex_key *key,
get_futex_key_refs(key);
q->key = *key;
- WARN_ON(plist_node_empty(&q->list));
- plist_del(&q->list, &q->list.plist);
+ __unqueue_futex(q);
WARN_ON(!q->rt_waiter);
q->rt_waiter = NULL;
@@ -1504,8 +1521,7 @@ retry:
spin_unlock(lock_ptr);
goto retry;
}
- WARN_ON(plist_node_empty(&q->list));
- plist_del(&q->list, &q->list.plist);
+ __unqueue_futex(q);
BUG_ON(q->pi_state);
@@ -1525,8 +1541,7 @@ retry:
static void unqueue_me_pi(struct futex_q *q)
__releases(q->lock_ptr)
{
- WARN_ON(plist_node_empty(&q->list));
- plist_del(&q->list, &q->list.plist);
+ __unqueue_futex(q);
BUG_ON(!q->pi_state);
free_pi_state(q->pi_state);
@@ -2167,7 +2182,7 @@ int handle_early_requeue_pi_wakeup(struct futex_hash_bucket *hb,
* We were woken prior to requeue by a timeout or a signal.
* Unqueue the futex_q and determine which it was.
*/
- plist_del(&q->list, &q->list.plist);
+ plist_del(&q->list, &hb->chain);
/* Handle spurious wakeups gracefully */
ret = -EWOULDBLOCK;
--
1.7.2.3
^ permalink raw reply related [flat|nested] 6+ messages in thread* [PATCH 2/4] futex,plist: Remove debug lock assignment from plist_node
2011-03-12 3:14 [PATCH 0/4] futex,plist: updates Steven Rostedt
2011-03-12 3:14 ` [PATCH 1/4] futex,plist: Pass the real head of the priority list to plist_del() Steven Rostedt
@ 2011-03-12 3:14 ` Steven Rostedt
2011-03-12 3:14 ` [PATCH 3/4] plist: Shrink struct plist_head Steven Rostedt
` (2 subsequent siblings)
4 siblings, 0 replies; 6+ messages in thread
From: Steven Rostedt @ 2011-03-12 3:14 UTC (permalink / raw)
To: linux-kernel
Cc: Ingo Molnar, Andrew Morton, Thomas Gleixner, Peter Zijlstra,
Lai Jiangshan, Darren Hart
[-- Attachment #1: 0002-futex-plist-Remove-debug-lock-assignment-from-plist_.patch --]
[-- Type: text/plain, Size: 1693 bytes --]
From: Lai Jiangshan <laijs@cn.fujitsu.com>
The original code uses &plist_node->plist as the fake head of
the priority list for plist_del(), these debug locks in
the fake head are needed for CONFIG_DEBUG_PI_LIST.
But now we always pass the real head to plist_del(), the debug locks
in plist_node will not be used, so we remove these assignments.
Acked-by: Darren Hart <dvhart@linux.intel.com>
Signed-off-by: Lai Jiangshan <laijs@cn.fujitsu.com>
LKML-Reference: <4D10797E.7040803@cn.fujitsu.com>
Signed-off-by: Steven Rostedt <rostedt@goodmis.org>
---
kernel/futex.c | 9 ---------
1 files changed, 0 insertions(+), 9 deletions(-)
diff --git a/kernel/futex.c b/kernel/futex.c
index 6feeea4..9fe9131 100644
--- a/kernel/futex.c
+++ b/kernel/futex.c
@@ -1089,9 +1089,6 @@ void requeue_futex(struct futex_q *q, struct futex_hash_bucket *hb1,
plist_del(&q->list, &hb1->chain);
plist_add(&q->list, &hb2->chain);
q->lock_ptr = &hb2->lock;
-#ifdef CONFIG_DEBUG_PI_LIST
- q->list.plist.spinlock = &hb2->lock;
-#endif
}
get_futex_key_refs(key2);
q->key = *key2;
@@ -1124,9 +1121,6 @@ void requeue_pi_wake_futex(struct futex_q *q, union futex_key *key,
q->rt_waiter = NULL;
q->lock_ptr = &hb->lock;
-#ifdef CONFIG_DEBUG_PI_LIST
- q->list.plist.spinlock = &hb->lock;
-#endif
wake_up_state(q->task, TASK_NORMAL);
}
@@ -1474,9 +1468,6 @@ static inline void queue_me(struct futex_q *q, struct futex_hash_bucket *hb)
prio = min(current->normal_prio, MAX_RT_PRIO);
plist_node_init(&q->list, prio);
-#ifdef CONFIG_DEBUG_PI_LIST
- q->list.plist.spinlock = &hb->lock;
-#endif
plist_add(&q->list, &hb->chain);
q->task = current;
spin_unlock(&hb->lock);
--
1.7.2.3
^ permalink raw reply related [flat|nested] 6+ messages in thread* [PATCH 3/4] plist: Shrink struct plist_head
2011-03-12 3:14 [PATCH 0/4] futex,plist: updates Steven Rostedt
2011-03-12 3:14 ` [PATCH 1/4] futex,plist: Pass the real head of the priority list to plist_del() Steven Rostedt
2011-03-12 3:14 ` [PATCH 2/4] futex,plist: Remove debug lock assignment from plist_node Steven Rostedt
@ 2011-03-12 3:14 ` Steven Rostedt
2011-03-12 3:14 ` [PATCH 4/4] plist: Add priority list test Steven Rostedt
2011-03-12 3:23 ` [PATCH 0/4] futex,plist: updates Steven Rostedt
4 siblings, 0 replies; 6+ messages in thread
From: Steven Rostedt @ 2011-03-12 3:14 UTC (permalink / raw)
To: linux-kernel
Cc: Ingo Molnar, Andrew Morton, Thomas Gleixner, Peter Zijlstra,
Lai Jiangshan
[-- Attachment #1: 0003-plist-Shrink-struct-plist_head.patch --]
[-- Type: text/plain, Size: 8916 bytes --]
From: Lai Jiangshan <laijs@cn.fujitsu.com>
struct plist_head is used in struct task_struct as well as struct
rtmutex. If we can make it smaller, it will also make these structures
smaller as well.
The field prio_list in struct plist_head is seldom used and we can get
its information from the plist_nodes. Removing this field will decrease
the size of plist_head by half.
Signed-off-by: Lai Jiangshan <laijs@cn.fujitsu.com>
LKML-Reference: <4D107982.9090700@cn.fujitsu.com>
Signed-off-by: Steven Rostedt <rostedt@goodmis.org>
---
include/linux/plist.h | 47 +++++++++++++++++++++--------------------
lib/plist.c | 54 +++++++++++++++++++++++++++++++-----------------
2 files changed, 59 insertions(+), 42 deletions(-)
diff --git a/include/linux/plist.h b/include/linux/plist.h
index 7254eda..c9b9f32 100644
--- a/include/linux/plist.h
+++ b/include/linux/plist.h
@@ -31,15 +31,17 @@
*
* Simple ASCII art explanation:
*
- * |HEAD |
- * | |
- * |prio_list.prev|<------------------------------------|
- * |prio_list.next|<->|pl|<->|pl|<--------------->|pl|<-|
- * |10 | |10| |21| |21| |21| |40| (prio)
- * | | | | | | | | | | | |
- * | | | | | | | | | | | |
- * |node_list.next|<->|nl|<->|nl|<->|nl|<->|nl|<->|nl|<-|
- * |node_list.prev|<------------------------------------|
+ * pl:prio_list (only for plist_node)
+ * nl:node_list
+ * HEAD| NODE(S)
+ * |
+ * ||------------------------------------|
+ * ||->|pl|<->|pl|<--------------->|pl|<-|
+ * | |10| |21| |21| |21| |40| (prio)
+ * | | | | | | | | | | |
+ * | | | | | | | | | | |
+ * |->|nl|<->|nl|<->|nl|<->|nl|<->|nl|<->|nl|<-|
+ * |-------------------------------------------|
*
* The nodes on the prio_list list are sorted by priority to simplify
* the insertion of new nodes. There are no nodes with duplicate
@@ -78,7 +80,6 @@
#include <linux/spinlock_types.h>
struct plist_head {
- struct list_head prio_list;
struct list_head node_list;
#ifdef CONFIG_DEBUG_PI_LIST
raw_spinlock_t *rawlock;
@@ -88,7 +89,8 @@ struct plist_head {
struct plist_node {
int prio;
- struct plist_head plist;
+ struct list_head prio_list;
+ struct list_head node_list;
};
#ifdef CONFIG_DEBUG_PI_LIST
@@ -100,7 +102,6 @@ struct plist_node {
#endif
#define _PLIST_HEAD_INIT(head) \
- .prio_list = LIST_HEAD_INIT((head).prio_list), \
.node_list = LIST_HEAD_INIT((head).node_list)
/**
@@ -133,7 +134,8 @@ struct plist_node {
#define PLIST_NODE_INIT(node, __prio) \
{ \
.prio = (__prio), \
- .plist = { _PLIST_HEAD_INIT((node).plist) }, \
+ .prio_list = LIST_HEAD_INIT((node).prio_list), \
+ .node_list = LIST_HEAD_INIT((node).node_list), \
}
/**
@@ -144,7 +146,6 @@ struct plist_node {
static inline void
plist_head_init(struct plist_head *head, spinlock_t *lock)
{
- INIT_LIST_HEAD(&head->prio_list);
INIT_LIST_HEAD(&head->node_list);
#ifdef CONFIG_DEBUG_PI_LIST
head->spinlock = lock;
@@ -160,7 +161,6 @@ plist_head_init(struct plist_head *head, spinlock_t *lock)
static inline void
plist_head_init_raw(struct plist_head *head, raw_spinlock_t *lock)
{
- INIT_LIST_HEAD(&head->prio_list);
INIT_LIST_HEAD(&head->node_list);
#ifdef CONFIG_DEBUG_PI_LIST
head->rawlock = lock;
@@ -176,7 +176,8 @@ plist_head_init_raw(struct plist_head *head, raw_spinlock_t *lock)
static inline void plist_node_init(struct plist_node *node, int prio)
{
node->prio = prio;
- plist_head_init(&node->plist, NULL);
+ INIT_LIST_HEAD(&node->prio_list);
+ INIT_LIST_HEAD(&node->node_list);
}
extern void plist_add(struct plist_node *node, struct plist_head *head);
@@ -188,7 +189,7 @@ extern void plist_del(struct plist_node *node, struct plist_head *head);
* @head: the head for your list
*/
#define plist_for_each(pos, head) \
- list_for_each_entry(pos, &(head)->node_list, plist.node_list)
+ list_for_each_entry(pos, &(head)->node_list, node_list)
/**
* plist_for_each_safe - iterate safely over a plist of given type
@@ -199,7 +200,7 @@ extern void plist_del(struct plist_node *node, struct plist_head *head);
* Iterate over a plist of given type, safe against removal of list entry.
*/
#define plist_for_each_safe(pos, n, head) \
- list_for_each_entry_safe(pos, n, &(head)->node_list, plist.node_list)
+ list_for_each_entry_safe(pos, n, &(head)->node_list, node_list)
/**
* plist_for_each_entry - iterate over list of given type
@@ -208,7 +209,7 @@ extern void plist_del(struct plist_node *node, struct plist_head *head);
* @mem: the name of the list_struct within the struct
*/
#define plist_for_each_entry(pos, head, mem) \
- list_for_each_entry(pos, &(head)->node_list, mem.plist.node_list)
+ list_for_each_entry(pos, &(head)->node_list, mem.node_list)
/**
* plist_for_each_entry_safe - iterate safely over list of given type
@@ -220,7 +221,7 @@ extern void plist_del(struct plist_node *node, struct plist_head *head);
* Iterate over list of given type, safe against removal of list entry.
*/
#define plist_for_each_entry_safe(pos, n, head, m) \
- list_for_each_entry_safe(pos, n, &(head)->node_list, m.plist.node_list)
+ list_for_each_entry_safe(pos, n, &(head)->node_list, m.node_list)
/**
* plist_head_empty - return !0 if a plist_head is empty
@@ -237,7 +238,7 @@ static inline int plist_head_empty(const struct plist_head *head)
*/
static inline int plist_node_empty(const struct plist_node *node)
{
- return plist_head_empty(&node->plist);
+ return list_empty(&node->node_list);
}
/* All functions below assume the plist_head is not empty. */
@@ -285,7 +286,7 @@ static inline int plist_node_empty(const struct plist_node *node)
static inline struct plist_node *plist_first(const struct plist_head *head)
{
return list_entry(head->node_list.next,
- struct plist_node, plist.node_list);
+ struct plist_node, node_list);
}
/**
@@ -297,7 +298,7 @@ static inline struct plist_node *plist_first(const struct plist_head *head)
static inline struct plist_node *plist_last(const struct plist_head *head)
{
return list_entry(head->node_list.prev,
- struct plist_node, plist.node_list);
+ struct plist_node, node_list);
}
#endif
diff --git a/lib/plist.c b/lib/plist.c
index 1471988..8c614d0 100644
--- a/lib/plist.c
+++ b/lib/plist.c
@@ -59,7 +59,8 @@ static void plist_check_head(struct plist_head *head)
WARN_ON_SMP(!raw_spin_is_locked(head->rawlock));
if (head->spinlock)
WARN_ON_SMP(!spin_is_locked(head->spinlock));
- plist_check_list(&head->prio_list);
+ if (!plist_head_empty(head))
+ plist_check_list(&plist_first(head)->prio_list);
plist_check_list(&head->node_list);
}
@@ -75,25 +76,33 @@ static void plist_check_head(struct plist_head *head)
*/
void plist_add(struct plist_node *node, struct plist_head *head)
{
- struct plist_node *iter;
+ struct plist_node *first, *iter, *prev = NULL;
+ struct list_head *node_next = &head->node_list;
plist_check_head(head);
WARN_ON(!plist_node_empty(node));
+ WARN_ON(!list_empty(&node->prio_list));
- list_for_each_entry(iter, &head->prio_list, plist.prio_list) {
- if (node->prio < iter->prio)
- goto lt_prio;
- else if (node->prio == iter->prio) {
- iter = list_entry(iter->plist.prio_list.next,
- struct plist_node, plist.prio_list);
- goto eq_prio;
+ if (plist_head_empty(head))
+ goto ins_node;
+
+ first = iter = plist_first(head);
+
+ do {
+ if (node->prio < iter->prio) {
+ node_next = &iter->node_list;
+ break;
}
- }
-lt_prio:
- list_add_tail(&node->plist.prio_list, &iter->plist.prio_list);
-eq_prio:
- list_add_tail(&node->plist.node_list, &iter->plist.node_list);
+ prev = iter;
+ iter = list_entry(iter->prio_list.next,
+ struct plist_node, prio_list);
+ } while (iter != first);
+
+ if (!prev || prev->prio != node->prio)
+ list_add_tail(&node->prio_list, &iter->prio_list);
+ins_node:
+ list_add_tail(&node->node_list, node_next);
plist_check_head(head);
}
@@ -108,14 +117,21 @@ void plist_del(struct plist_node *node, struct plist_head *head)
{
plist_check_head(head);
- if (!list_empty(&node->plist.prio_list)) {
- struct plist_node *next = plist_first(&node->plist);
+ if (!list_empty(&node->prio_list)) {
+ if (node->node_list.next != &head->node_list) {
+ struct plist_node *next;
+
+ next = list_entry(node->node_list.next,
+ struct plist_node, node_list);
- list_move_tail(&next->plist.prio_list, &node->plist.prio_list);
- list_del_init(&node->plist.prio_list);
+ /* add the next plist_node into prio_list */
+ if (list_empty(&next->prio_list))
+ list_add(&next->prio_list, &node->prio_list);
+ }
+ list_del_init(&node->prio_list);
}
- list_del_init(&node->plist.node_list);
+ list_del_init(&node->node_list);
plist_check_head(head);
}
--
1.7.2.3
^ permalink raw reply related [flat|nested] 6+ messages in thread* [PATCH 4/4] plist: Add priority list test
2011-03-12 3:14 [PATCH 0/4] futex,plist: updates Steven Rostedt
` (2 preceding siblings ...)
2011-03-12 3:14 ` [PATCH 3/4] plist: Shrink struct plist_head Steven Rostedt
@ 2011-03-12 3:14 ` Steven Rostedt
2011-03-12 3:23 ` [PATCH 0/4] futex,plist: updates Steven Rostedt
4 siblings, 0 replies; 6+ messages in thread
From: Steven Rostedt @ 2011-03-12 3:14 UTC (permalink / raw)
To: linux-kernel
Cc: Ingo Molnar, Andrew Morton, Thomas Gleixner, Peter Zijlstra,
Lai Jiangshan
[-- Attachment #1: 0004-plist-Add-priority-list-test.patch --]
[-- Type: text/plain, Size: 2958 bytes --]
From: Lai Jiangshan <laijs@cn.fujitsu.com>
Add test code for checking plist when the kernel is booting.
Signed-off-by: Lai Jiangshan <laijs@cn.fujitsu.com>
LKML-Reference: <4D107986.1010302@cn.fujitsu.com>
Signed-off-by: Steven Rostedt <rostedt@goodmis.org>
---
lib/plist.c | 81 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++-
1 files changed, 80 insertions(+), 1 deletions(-)
diff --git a/lib/plist.c b/lib/plist.c
index 8c614d0..0ae7e64 100644
--- a/lib/plist.c
+++ b/lib/plist.c
@@ -28,6 +28,8 @@
#ifdef CONFIG_DEBUG_PI_LIST
+static struct plist_head test_head;
+
static void plist_check_prev_next(struct list_head *t, struct list_head *p,
struct list_head *n)
{
@@ -54,7 +56,7 @@ static void plist_check_list(struct list_head *top)
static void plist_check_head(struct plist_head *head)
{
- WARN_ON(!head->rawlock && !head->spinlock);
+ WARN_ON(head != &test_head && !head->rawlock && !head->spinlock);
if (head->rawlock)
WARN_ON_SMP(!raw_spin_is_locked(head->rawlock));
if (head->spinlock)
@@ -135,3 +137,80 @@ void plist_del(struct plist_node *node, struct plist_head *head)
plist_check_head(head);
}
+
+#ifdef CONFIG_DEBUG_PI_LIST
+#include <linux/sched.h>
+#include <linux/module.h>
+#include <linux/init.h>
+
+static struct plist_node __initdata test_node[241];
+
+static void __init plist_test_check(int nr_expect)
+{
+ struct plist_node *first, *prio_pos, *node_pos;
+
+ if (plist_head_empty(&test_head)) {
+ BUG_ON(nr_expect != 0);
+ return;
+ }
+
+ prio_pos = first = plist_first(&test_head);
+ plist_for_each(node_pos, &test_head) {
+ if (nr_expect-- < 0)
+ break;
+ if (node_pos == first)
+ continue;
+ if (node_pos->prio == prio_pos->prio) {
+ BUG_ON(!list_empty(&node_pos->prio_list));
+ continue;
+ }
+
+ BUG_ON(prio_pos->prio > node_pos->prio);
+ BUG_ON(prio_pos->prio_list.next != &node_pos->prio_list);
+ prio_pos = node_pos;
+ }
+
+ BUG_ON(nr_expect != 0);
+ BUG_ON(prio_pos->prio_list.next != &first->prio_list);
+}
+
+static int __init plist_test(void)
+{
+ int nr_expect = 0, i, loop;
+ unsigned int r = local_clock();
+
+ printk(KERN_INFO "start plist test\n");
+ plist_head_init(&test_head, NULL);
+ for (i = 0; i < ARRAY_SIZE(test_node); i++)
+ plist_node_init(test_node + i, 0);
+
+ for (loop = 0; loop < 1000; loop++) {
+ r = r * 193939 % 47629;
+ i = r % ARRAY_SIZE(test_node);
+ if (plist_node_empty(test_node + i)) {
+ r = r * 193939 % 47629;
+ test_node[i].prio = r % 99;
+ plist_add(test_node + i, &test_head);
+ nr_expect++;
+ } else {
+ plist_del(test_node + i, &test_head);
+ nr_expect--;
+ }
+ plist_test_check(nr_expect);
+ }
+
+ for (i = 0; i < ARRAY_SIZE(test_node); i++) {
+ if (plist_node_empty(test_node + i))
+ continue;
+ plist_del(test_node + i, &test_head);
+ nr_expect--;
+ plist_test_check(nr_expect);
+ }
+
+ printk(KERN_INFO "end plist test\n");
+ return 0;
+}
+
+module_init(plist_test);
+
+#endif
--
1.7.2.3
^ permalink raw reply related [flat|nested] 6+ messages in thread