[PATCH -mm -v2 3/5] llist, Move cpu_relax after cmpxchg

Huang Ying
2011-09-08T06:01:03Z

If the first cmpxchg calling succeeds, it is not necessary to use
cpu_relax before cmpxchg. So cpu_relax in a busy loop involving
cmpxchg should go after cmpxchg instead of before that. This patch
fixes this in llist.

Signed-off-by: Huang Ying <ying.huang@intel.com>
Acked-by: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
Cc: Peter Zijlstra <peterz@infradead.org>
---
include/linux/llist.h | 21 +++++++++++++++------
1 file changed, 15 insertions(+), 6 deletions(-)

--- a/include/linux/llist.h
+++ b/include/linux/llist.h
@@ -156,11 +156,14 @@ static inline void llist_add(struct llis
CHECK_NMI_SAFE_CMPXCHG();

entry = head->first;
- do {
+ for (;;) {
old_entry = entry;
new->next = entry;
+ entry = cmpxchg(&head->first, old_entry, new);
+ if (entry == old_entry)
+ break;
cpu_relax();
- } while ((entry = cmpxchg(&head->first, old_entry, new)) != old_entry);
+ }
}

/**
@@ -178,11 +181,14 @@ static inline void llist_add_batch(struc
CHECK_NMI_SAFE_CMPXCHG();

entry = head->first;
- do {
+ for (;;) {
old_entry = entry;
new_last->next = entry;
+ entry = cmpxchg(&head->first, old_entry, new_first);
+ if (entry == old_entry)
+ break;
cpu_relax();
- } while ((entry = cmpxchg(&head->first, old_entry, new_first)) != old_entry);
+ }
}

/**
@@ -206,13 +212,16 @@ static inline struct llist_node *llist_d
CHECK_NMI_SAFE_CMPXCHG();

entry = head->first;
- do {
+ for (;;) {
if (entry == NULL)
return NULL;
old_entry = entry;
next = entry->next;
+ entry = cmpxchg(&head->first, old_entry, next);
+ if (entry == old_entry)
+ break;
cpu_relax();
- } while ((entry = cmpxchg(&head->first, old_entry, next)) != old_entry);
+ }

return entry;
}