[LIST] Add missing rcu_dereference on first element

[LIST] Add missing rcu_dereference on first element

[Herbert Xu]

[-- Attachment #1: Type: text/plain, Size: 1081 bytes --]

Hi:

It seems that all the list_*_rcu primitives are missing a memory barrier
on the very first dereference.  For example,

#define list_for_each_rcu(pos, head) \
	for (pos = (head)->next; prefetch(pos->next), pos != (head); \
		pos = rcu_dereference(pos->next))

It will go something like:

	pos = (head)->next

	prefetch(pos->next)

	pos != (head)

	do stuff

We're missing a barrier here.

	pos = rcu_dereference(pos->next)

		fetch pos->next

		barrier given by rcu_dereference(pos->next)

		store pos

Without the missing barrier, the pos->next value may turn out to be
stale.  In fact, if "do stuff" were also dereferencing pos and relying
on list_for_each_rcu to provide the barrier then it may also break.

So here is a patch to make sure that we have a barrier for the first
element in the list.

Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>

Cheers,
-- 
Visit Openswan at http://www.openswan.org/
Email: Herbert Xu ~{PmV>HI~} <herbert@gondor.apana.org.au>
Home Page: http://gondor.apana.org.au/~herbert/
PGP Key: http://gondor.apana.org.au/~herbert/pubkey.txt

[-- Attachment #2: p --]
[-- Type: text/plain, Size: 3393 bytes --]

diff --git a/include/linux/list.h b/include/linux/list.h
--- a/include/linux/list.h
+++ b/include/linux/list.h
@@ -442,12 +442,15 @@ static inline void list_splice_init(stru
  * as long as the traversal is guarded by rcu_read_lock().
  */
 #define list_for_each_rcu(pos, head) \
-	for (pos = (head)->next; prefetch(pos->next), pos != (head); \
-        	pos = rcu_dereference(pos->next))
+	for (pos = (head)->next; \
+		pos = rcu_dereference(pos), \
+			prefetch(pos->next), pos != (head); \
+        	pos = pos->next)
 
 #define __list_for_each_rcu(pos, head) \
-	for (pos = (head)->next; pos != (head); \
-        	pos = rcu_dereference(pos->next))
+	for (pos = (head)->next; \
+		rcu_dereference(pos) != (head); \
+        	pos = pos->next)
 
 /**
  * list_for_each_safe_rcu	-	iterate over an rcu-protected list safe
@@ -461,8 +464,9 @@ static inline void list_splice_init(stru
  * as long as the traversal is guarded by rcu_read_lock().
  */
 #define list_for_each_safe_rcu(pos, n, head) \
-	for (pos = (head)->next, n = pos->next; pos != (head); \
-		pos = rcu_dereference(n), n = pos->next)
+	for (pos = (head)->next; \
+		n = rcu_dereference(pos)->next, pos != (head); \
+		pos = n)
 
 /**
  * list_for_each_entry_rcu	-	iterate over rcu list of given type
@@ -474,11 +478,11 @@ static inline void list_splice_init(stru
  * the _rcu list-mutation primitives such as list_add_rcu()
  * as long as the traversal is guarded by rcu_read_lock().
  */
-#define list_for_each_entry_rcu(pos, head, member)			\
-	for (pos = list_entry((head)->next, typeof(*pos), member);	\
-	     prefetch(pos->member.next), &pos->member != (head); 	\
-	     pos = rcu_dereference(list_entry(pos->member.next, 	\
-					typeof(*pos), member)))
+#define list_for_each_entry_rcu(pos, head, member) \
+	for (pos = list_entry((head)->next, typeof(*pos), member); \
+		pos = rcu_dereference(pos), \
+			prefetch(pos->member.next), &pos->member != (head); \
+		pos = list_entry(pos->member.next, typeof(*pos), member))
 
 
 /**
@@ -492,8 +496,10 @@ static inline void list_splice_init(stru
  * as long as the traversal is guarded by rcu_read_lock().
  */
 #define list_for_each_continue_rcu(pos, head) \
-	for ((pos) = (pos)->next; prefetch((pos)->next), (pos) != (head); \
-        	(pos) = rcu_dereference((pos)->next))
+	for ((pos) = (pos)->next; \
+		(pos) = rcu_dereference((pos)), \
+			prefetch((pos)->next), (pos) != (head); \
+        	(pos) = (pos)->next)
 
 /*
  * Double linked lists with a single pointer list head.
@@ -696,8 +702,9 @@ static inline void hlist_add_after_rcu(s
 	     pos = n)
 
 #define hlist_for_each_rcu(pos, head) \
-	for ((pos) = (head)->first; pos && ({ prefetch((pos)->next); 1; }); \
-		(pos) = rcu_dereference((pos)->next))
+	for ((pos) = (head)->first; \
+		rcu_dereference((pos)) && ({ prefetch((pos)->next); 1; }); \
+		(pos) = (pos)->next)
 
 /**
  * hlist_for_each_entry	- iterate over list of given type
@@ -762,9 +769,9 @@ static inline void hlist_add_after_rcu(s
  */
 #define hlist_for_each_entry_rcu(tpos, pos, head, member)		 \
 	for (pos = (head)->first;					 \
-	     pos && ({ prefetch(pos->next); 1;}) &&			 \
+	     rcu_dereference(pos) && ({ prefetch(pos->next); 1;}) &&	 \
 		({ tpos = hlist_entry(pos, typeof(*tpos), member); 1;}); \
-	     pos = rcu_dereference(pos->next))
+	     pos = pos->next)
 
 #else
 #warning "don't include kernel headers in userspace"

Re: [LIST] Add missing rcu_dereference on first element

[Paul E. McKenney]

On Sat, Oct 15, 2005 at 10:26:49AM +1000, Herbert Xu wrote:
> Hi:
> 
> It seems that all the list_*_rcu primitives are missing a memory barrier
> on the very first dereference.  For example,
> 
> #define list_for_each_rcu(pos, head) \
> 	for (pos = (head)->next; prefetch(pos->next), pos != (head); \
> 		pos = rcu_dereference(pos->next))
> 
> It will go something like:
> 
> 	pos = (head)->next
> 
> 	prefetch(pos->next)
> 
> 	pos != (head)
> 
> 	do stuff
> 
> We're missing a barrier here.
> 
> 	pos = rcu_dereference(pos->next)
> 
> 		fetch pos->next
> 
> 		barrier given by rcu_dereference(pos->next)
> 
> 		store pos
> 
> Without the missing barrier, the pos->next value may turn out to be
> stale.  In fact, if "do stuff" were also dereferencing pos and relying
> on list_for_each_rcu to provide the barrier then it may also break.
> 
> So here is a patch to make sure that we have a barrier for the first
> element in the list.

Good catch!!!  I wonder if Alpha SMP Linux machines have noticed...

Some comments interspersed.

> Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>
> 
> Cheers,
> -- 
> Visit Openswan at http://www.openswan.org/
> Email: Herbert Xu ~{PmV>HI~} <herbert@gondor.apana.org.au>
> Home Page: http://gondor.apana.org.au/~herbert/
> PGP Key: http://gondor.apana.org.au/~herbert/pubkey.txt

> diff --git a/include/linux/list.h b/include/linux/list.h
> --- a/include/linux/list.h
> +++ b/include/linux/list.h
> @@ -442,12 +442,15 @@ static inline void list_splice_init(stru
>   * as long as the traversal is guarded by rcu_read_lock().
>   */
>  #define list_for_each_rcu(pos, head) \
> -	for (pos = (head)->next; prefetch(pos->next), pos != (head); \
> -        	pos = rcu_dereference(pos->next))
> +	for (pos = (head)->next; \
> +		pos = rcu_dereference(pos), \
> +			prefetch(pos->next), pos != (head); \
> +        	pos = pos->next)

Why not something like the following?  Seems a bit simpler to me.

#define list_for_each_rcu(pos, head) \
	for (pos = rcu_dereference((head)->next); \
		prefetch(pos->next), pos != (head); \
        	pos = rcu_dereference(pos->next))

>  #define __list_for_each_rcu(pos, head) \
> -	for (pos = (head)->next; pos != (head); \
> -        	pos = rcu_dereference(pos->next))
> +	for (pos = (head)->next; \
> +		rcu_dereference(pos) != (head); \
> +        	pos = pos->next)
>  
>  /**
>   * list_for_each_safe_rcu	-	iterate over an rcu-protected list safe
> @@ -461,8 +464,9 @@ static inline void list_splice_init(stru
>   * as long as the traversal is guarded by rcu_read_lock().
>   */
>  #define list_for_each_safe_rcu(pos, n, head) \
> -	for (pos = (head)->next, n = pos->next; pos != (head); \
> -		pos = rcu_dereference(n), n = pos->next)
> +	for (pos = (head)->next; \
> +		n = rcu_dereference(pos)->next, pos != (head); \
> +		pos = n)
>  
>  /**
>   * list_for_each_entry_rcu	-	iterate over rcu list of given type
> @@ -474,11 +478,11 @@ static inline void list_splice_init(stru
>   * the _rcu list-mutation primitives such as list_add_rcu()
>   * as long as the traversal is guarded by rcu_read_lock().
>   */
> -#define list_for_each_entry_rcu(pos, head, member)			\
> -	for (pos = list_entry((head)->next, typeof(*pos), member);	\
> -	     prefetch(pos->member.next), &pos->member != (head); 	\
> -	     pos = rcu_dereference(list_entry(pos->member.next, 	\
> -					typeof(*pos), member)))
> +#define list_for_each_entry_rcu(pos, head, member) \
> +	for (pos = list_entry((head)->next, typeof(*pos), member); \
> +		pos = rcu_dereference(pos), \
> +			prefetch(pos->member.next), &pos->member != (head); \
> +		pos = list_entry(pos->member.next, typeof(*pos), member))
>  
>  
>  /**
> @@ -492,8 +496,10 @@ static inline void list_splice_init(stru
>   * as long as the traversal is guarded by rcu_read_lock().
>   */
>  #define list_for_each_continue_rcu(pos, head) \
> -	for ((pos) = (pos)->next; prefetch((pos)->next), (pos) != (head); \
> -        	(pos) = rcu_dereference((pos)->next))
> +	for ((pos) = (pos)->next; \
> +		(pos) = rcu_dereference((pos)), \
> +			prefetch((pos)->next), (pos) != (head); \
> +        	(pos) = (pos)->next)

The above hurts my head -- childhood trauma due to having to use a
FORTRAN compiler that required "I=I" at odd intervals in order to
generate correct code...  How about the following?

#define list_for_each_continue_rcu(pos, head) \
	for ((pos) = (pos)->next; \
		prefetch(rcu_dereference(pos)->next), (pos) != (head); \
        	(pos) = (pos)->next)

>  /*
>   * Double linked lists with a single pointer list head.
> @@ -696,8 +702,9 @@ static inline void hlist_add_after_rcu(s
>  	     pos = n)
>  
>  #define hlist_for_each_rcu(pos, head) \
> -	for ((pos) = (head)->first; pos && ({ prefetch((pos)->next); 1; }); \
> -		(pos) = rcu_dereference((pos)->next))
> +	for ((pos) = (head)->first; \
> +		rcu_dereference((pos)) && ({ prefetch((pos)->next); 1; }); \
> +		(pos) = (pos)->next)
>  
>  /**
>   * hlist_for_each_entry	- iterate over list of given type
> @@ -762,9 +769,9 @@ static inline void hlist_add_after_rcu(s
>   */
>  #define hlist_for_each_entry_rcu(tpos, pos, head, member)		 \
>  	for (pos = (head)->first;					 \
> -	     pos && ({ prefetch(pos->next); 1;}) &&			 \
> +	     rcu_dereference(pos) && ({ prefetch(pos->next); 1;}) &&	 \
>  		({ tpos = hlist_entry(pos, typeof(*tpos), member); 1;}); \
> -	     pos = rcu_dereference(pos->next))
> +	     pos = pos->next)
>  
>  #else
>  #warning "don't include kernel headers in userspace"

Re: [LIST] Add missing rcu_dereference on first element

[Herbert Xu]

On Fri, Oct 14, 2005 at 07:03:25PM -0700, Paul E. McKenney wrote:
> 
> > diff --git a/include/linux/list.h b/include/linux/list.h
> > --- a/include/linux/list.h
> > +++ b/include/linux/list.h
> > @@ -442,12 +442,15 @@ static inline void list_splice_init(stru
> >   * as long as the traversal is guarded by rcu_read_lock().
> >   */
> >  #define list_for_each_rcu(pos, head) \
> > -	for (pos = (head)->next; prefetch(pos->next), pos != (head); \
> > -        	pos = rcu_dereference(pos->next))
> > +	for (pos = (head)->next; \
> > +		pos = rcu_dereference(pos), \
> > +			prefetch(pos->next), pos != (head); \
> > +        	pos = pos->next)
> 
> Why not something like the following?  Seems a bit simpler to me.
> 
> #define list_for_each_rcu(pos, head) \
> 	for (pos = rcu_dereference((head)->next); \
> 		prefetch(pos->next), pos != (head); \
>         	pos = rcu_dereference(pos->next))

In this case your version is indeed more concise.  However, in most of
the other for_each macros having it in the loop conditional looks more
natural.

So in order to be consistent throughout list.h, I'd like to keep the
rcu_dereference in the loop conditional.

> > @@ -492,8 +496,10 @@ static inline void list_splice_init(stru
> >   * as long as the traversal is guarded by rcu_read_lock().
> >   */
> >  #define list_for_each_continue_rcu(pos, head) \
> > -	for ((pos) = (pos)->next; prefetch((pos)->next), (pos) != (head); \
> > -        	(pos) = rcu_dereference((pos)->next))
> > +	for ((pos) = (pos)->next; \
> > +		(pos) = rcu_dereference((pos)), \
> > +			prefetch((pos)->next), (pos) != (head); \
> > +        	(pos) = (pos)->next)
> 
> The above hurts my head -- childhood trauma due to having to use a
> FORTRAN compiler that required "I=I" at odd intervals in order to
> generate correct code...  How about the following?
> 
> #define list_for_each_continue_rcu(pos, head) \
> 	for ((pos) = (pos)->next; \
> 		prefetch(rcu_dereference(pos)->next), (pos) != (head); \
>         	(pos) = (pos)->next)

I chose to keep it out of prefetch because normally the argument to
prefetch does not have any side-effects.  Even though today's prefetch
is an inline function which does respect side-effects, there is always
a possibility that someone somewhere might decide to implement prefetch
as a macro.

Besides, the expression

i = foo(i)

where foo has side-effects is pretty normal.

Cheers,
-- 
Visit Openswan at http://www.openswan.org/
Email: Herbert Xu ~{PmV>HI~} <herbert@gondor.apana.org.au>
Home Page: http://gondor.apana.org.au/~herbert/
PGP Key: http://gondor.apana.org.au/~herbert/pubkey.txt

Re: [LIST] Add missing rcu_dereference on first element

[Herbert Xu]

[-- Attachment #1: Type: text/plain, Size: 788 bytes --]

Hi Paul:

On Sat, Oct 15, 2005 at 12:39:18PM +1000, herbert wrote:
> 
> Besides, the expression
> 
> i = foo(i)
> 
> where foo has side-effects is pretty normal.

Actually I've changed my mind on this.  I think your version is
better because the side-effect of rcu_dereference will cause the
above assignment to occur twice when i refers to a memory-backed
variable.

Since all current prefetch implementations are safe as far as
side-effects are concerned, here is an updated version that
doesn't do i = foo(i).

Andrew, please replace the previous version with this.

Thanks,
-- 
Visit Openswan at http://www.openswan.org/
Email: Herbert Xu ~{PmV>HI~} <herbert@gondor.apana.org.au>
Home Page: http://gondor.apana.org.au/~herbert/
PGP Key: http://gondor.apana.org.au/~herbert/pubkey.txt

[-- Attachment #2: p --]
[-- Type: text/plain, Size: 3343 bytes --]

diff --git a/include/linux/list.h b/include/linux/list.h
--- a/include/linux/list.h
+++ b/include/linux/list.h
@@ -442,12 +442,14 @@ static inline void list_splice_init(stru
  * as long as the traversal is guarded by rcu_read_lock().
  */
 #define list_for_each_rcu(pos, head) \
-	for (pos = (head)->next; prefetch(pos->next), pos != (head); \
-        	pos = rcu_dereference(pos->next))
+	for (pos = (head)->next; \
+		prefetch(rcu_dereference(pos)->next), pos != (head); \
+        	pos = pos->next)
 
 #define __list_for_each_rcu(pos, head) \
-	for (pos = (head)->next; pos != (head); \
-        	pos = rcu_dereference(pos->next))
+	for (pos = (head)->next; \
+		rcu_dereference(pos) != (head); \
+        	pos = pos->next)
 
 /**
  * list_for_each_safe_rcu	-	iterate over an rcu-protected list safe
@@ -461,8 +463,9 @@ static inline void list_splice_init(stru
  * as long as the traversal is guarded by rcu_read_lock().
  */
 #define list_for_each_safe_rcu(pos, n, head) \
-	for (pos = (head)->next, n = pos->next; pos != (head); \
-		pos = rcu_dereference(n), n = pos->next)
+	for (pos = (head)->next; \
+		n = rcu_dereference(pos)->next, pos != (head); \
+		pos = n)
 
 /**
  * list_for_each_entry_rcu	-	iterate over rcu list of given type
@@ -474,11 +477,11 @@ static inline void list_splice_init(stru
  * the _rcu list-mutation primitives such as list_add_rcu()
  * as long as the traversal is guarded by rcu_read_lock().
  */
-#define list_for_each_entry_rcu(pos, head, member)			\
-	for (pos = list_entry((head)->next, typeof(*pos), member);	\
-	     prefetch(pos->member.next), &pos->member != (head); 	\
-	     pos = rcu_dereference(list_entry(pos->member.next, 	\
-					typeof(*pos), member)))
+#define list_for_each_entry_rcu(pos, head, member) \
+	for (pos = list_entry((head)->next, typeof(*pos), member); \
+		prefetch(rcu_dereference(pos)->member.next), \
+			&pos->member != (head); \
+		pos = list_entry(pos->member.next, typeof(*pos), member))
 
 
 /**
@@ -492,8 +495,9 @@ static inline void list_splice_init(stru
  * as long as the traversal is guarded by rcu_read_lock().
  */
 #define list_for_each_continue_rcu(pos, head) \
-	for ((pos) = (pos)->next; prefetch((pos)->next), (pos) != (head); \
-        	(pos) = rcu_dereference((pos)->next))
+	for ((pos) = (pos)->next; \
+		prefetch(rcu_dereference((pos))->next), (pos) != (head); \
+        	(pos) = (pos)->next)
 
 /*
  * Double linked lists with a single pointer list head.
@@ -696,8 +700,9 @@ static inline void hlist_add_after_rcu(s
 	     pos = n)
 
 #define hlist_for_each_rcu(pos, head) \
-	for ((pos) = (head)->first; pos && ({ prefetch((pos)->next); 1; }); \
-		(pos) = rcu_dereference((pos)->next))
+	for ((pos) = (head)->first; \
+		rcu_dereference((pos)) && ({ prefetch((pos)->next); 1; }); \
+		(pos) = (pos)->next)
 
 /**
  * hlist_for_each_entry	- iterate over list of given type
@@ -762,9 +767,9 @@ static inline void hlist_add_after_rcu(s
  */
 #define hlist_for_each_entry_rcu(tpos, pos, head, member)		 \
 	for (pos = (head)->first;					 \
-	     pos && ({ prefetch(pos->next); 1;}) &&			 \
+	     rcu_dereference(pos) && ({ prefetch(pos->next); 1;}) &&	 \
 		({ tpos = hlist_entry(pos, typeof(*tpos), member); 1;}); \
-	     pos = rcu_dereference(pos->next))
+	     pos = pos->next)
 
 #else
 #warning "don't include kernel headers in userspace"

Re: [LIST] Add missing rcu_dereference on first element

[Paul E. McKenney]

On Sat, Oct 15, 2005 at 01:22:41PM +1000, Herbert Xu wrote:
> Hi Paul:
> 
> On Sat, Oct 15, 2005 at 12:39:18PM +1000, herbert wrote:
> > 
> > Besides, the expression
> > 
> > i = foo(i)
> > 
> > where foo has side-effects is pretty normal.
> 
> Actually I've changed my mind on this.  I think your version is
> better because the side-effect of rcu_dereference will cause the
> above assignment to occur twice when i refers to a memory-backed
> variable.
> 
> Since all current prefetch implementations are safe as far as
> side-effects are concerned, here is an updated version that
> doesn't do i = foo(i).
> 
> Andrew, please replace the previous version with this.

Looks great to me!

						Thanx, Paul

Acked-by: <paulmck@us.ibm.com>

> Thanks,
> -- 
> Visit Openswan at http://www.openswan.org/
> Email: Herbert Xu ~{PmV>HI~} <herbert@gondor.apana.org.au>
> Home Page: http://gondor.apana.org.au/~herbert/
> PGP Key: http://gondor.apana.org.au/~herbert/pubkey.txt

> diff --git a/include/linux/list.h b/include/linux/list.h
> --- a/include/linux/list.h
> +++ b/include/linux/list.h
> @@ -442,12 +442,14 @@ static inline void list_splice_init(stru
>   * as long as the traversal is guarded by rcu_read_lock().
>   */
>  #define list_for_each_rcu(pos, head) \
> -	for (pos = (head)->next; prefetch(pos->next), pos != (head); \
> -        	pos = rcu_dereference(pos->next))
> +	for (pos = (head)->next; \
> +		prefetch(rcu_dereference(pos)->next), pos != (head); \
> +        	pos = pos->next)
>  
>  #define __list_for_each_rcu(pos, head) \
> -	for (pos = (head)->next; pos != (head); \
> -        	pos = rcu_dereference(pos->next))
> +	for (pos = (head)->next; \
> +		rcu_dereference(pos) != (head); \
> +        	pos = pos->next)
>  
>  /**
>   * list_for_each_safe_rcu	-	iterate over an rcu-protected list safe
> @@ -461,8 +463,9 @@ static inline void list_splice_init(stru
>   * as long as the traversal is guarded by rcu_read_lock().
>   */
>  #define list_for_each_safe_rcu(pos, n, head) \
> -	for (pos = (head)->next, n = pos->next; pos != (head); \
> -		pos = rcu_dereference(n), n = pos->next)
> +	for (pos = (head)->next; \
> +		n = rcu_dereference(pos)->next, pos != (head); \
> +		pos = n)
>  
>  /**
>   * list_for_each_entry_rcu	-	iterate over rcu list of given type
> @@ -474,11 +477,11 @@ static inline void list_splice_init(stru
>   * the _rcu list-mutation primitives such as list_add_rcu()
>   * as long as the traversal is guarded by rcu_read_lock().
>   */
> -#define list_for_each_entry_rcu(pos, head, member)			\
> -	for (pos = list_entry((head)->next, typeof(*pos), member);	\
> -	     prefetch(pos->member.next), &pos->member != (head); 	\
> -	     pos = rcu_dereference(list_entry(pos->member.next, 	\
> -					typeof(*pos), member)))
> +#define list_for_each_entry_rcu(pos, head, member) \
> +	for (pos = list_entry((head)->next, typeof(*pos), member); \
> +		prefetch(rcu_dereference(pos)->member.next), \
> +			&pos->member != (head); \
> +		pos = list_entry(pos->member.next, typeof(*pos), member))
>  
>  
>  /**
> @@ -492,8 +495,9 @@ static inline void list_splice_init(stru
>   * as long as the traversal is guarded by rcu_read_lock().
>   */
>  #define list_for_each_continue_rcu(pos, head) \
> -	for ((pos) = (pos)->next; prefetch((pos)->next), (pos) != (head); \
> -        	(pos) = rcu_dereference((pos)->next))
> +	for ((pos) = (pos)->next; \
> +		prefetch(rcu_dereference((pos))->next), (pos) != (head); \
> +        	(pos) = (pos)->next)
>  
>  /*
>   * Double linked lists with a single pointer list head.
> @@ -696,8 +700,9 @@ static inline void hlist_add_after_rcu(s
>  	     pos = n)
>  
>  #define hlist_for_each_rcu(pos, head) \
> -	for ((pos) = (head)->first; pos && ({ prefetch((pos)->next); 1; }); \
> -		(pos) = rcu_dereference((pos)->next))
> +	for ((pos) = (head)->first; \
> +		rcu_dereference((pos)) && ({ prefetch((pos)->next); 1; }); \
> +		(pos) = (pos)->next)
>  
>  /**
>   * hlist_for_each_entry	- iterate over list of given type
> @@ -762,9 +767,9 @@ static inline void hlist_add_after_rcu(s
>   */
>  #define hlist_for_each_entry_rcu(tpos, pos, head, member)		 \
>  	for (pos = (head)->first;					 \
> -	     pos && ({ prefetch(pos->next); 1;}) &&			 \
> +	     rcu_dereference(pos) && ({ prefetch(pos->next); 1;}) &&	 \
>  		({ tpos = hlist_entry(pos, typeof(*tpos), member); 1;}); \
> -	     pos = rcu_dereference(pos->next))
> +	     pos = pos->next)
>  
>  #else
>  #warning "don't include kernel headers in userspace"