[PATCH RT v2] mm: perform lru_add_drain_all() remotely

[PATCH RT v2] mm: perform lru_add_drain_all() remotely

[Luiz Capitulino]

lru_add_drain_all() works by scheduling lru_add_drain_cpu() to run
on all CPUs that have non-empty LRU pagevecs and then waiting for
the scheduled work to complete. However, workqueue threads may never
have the chance to run on a CPU that's running a SCHED_FIFO task.
This causes lru_add_drain_all() to block forever.

This commit solves this problem by changing lru_add_drain_all()
to drain the LRU pagevecs of remote CPUs. This is done by grabbing
swapvec_lock and calling lru_add_drain_cpu().

PS: This is based on an idea and initial implementation by
    Rik van Riel.

Signed-off-by: Rik van Riel <riel@redhat.com>
Signed-off-by: Luiz Capitulino <lcapitulino@redhat.com>
---

Sebastian,

It's not clear to me from our last discussion what's the
best plan for this patch. I'm sending v2 with the changes
you suggested.

Changelog

v2

 - Simple renames
   local_lock_other_cpu() -> local_lock_on()
   local_lock_irqsave_other_cpu() -> local_lock_irq_on()
   (unlock follows same renaiming)

 include/linux/locallock.h | 31 +++++++++++++++++++++++++++++++
 mm/swap.c                 | 35 +++++++++++++++++++++++++----------
 2 files changed, 56 insertions(+), 10 deletions(-)

diff --git a/include/linux/locallock.h b/include/linux/locallock.h
index 6fe5928..f7f94cc 100644
--- a/include/linux/locallock.h
+++ b/include/linux/locallock.h
@@ -104,6 +104,17 @@ static inline void __local_unlock(struct local_irq_lock *lv)
 		put_local_var(lvar);				\
 	} while (0)
 
+#define local_lock_on(lvar, cpu)                         \
+	do {                                                    \
+		__local_lock(&per_cpu(lvar, cpu));              \
+	} while (0)
+
+#define local_unlock_on(lvar, cpu)                       \
+	do {                                                    \
+		__local_unlock(&per_cpu(lvar, cpu));            \
+	} while (0)
+
+
 static inline void __local_lock_irq(struct local_irq_lock *lv)
 {
 	spin_lock_irqsave(&lv->lock, lv->flags);
@@ -163,6 +174,22 @@ static inline int __local_lock_irqsave(struct local_irq_lock *lv)
 		_flags = per_cpu(lvar, cpu).flags;			\
 	} while (0)
 
+#define local_lock_irq_on(lvar, _flags, cpu)			\
+	do {								\
+		if (cpu == smp_processor_id())				\
+			local_lock_irqsave(lvar, _flags);		\
+		else							\
+			local_lock_on(lvar, cpu);		\
+	} while (0)
+
+#define local_unlock_irq_on(lvar, _flags, cpu)	        \
+	do {								\
+		if (cpu == smp_processor_id())				\
+			local_unlock_irqrestore(lvar, _flags);		\
+		else							\
+			local_unlock_on(lvar, cpu);		\
+	} while (0)
+
 static inline int __local_unlock_irqrestore(struct local_irq_lock *lv,
 					    unsigned long flags)
 {
@@ -250,6 +277,10 @@ static inline void local_irq_lock_init(int lvar) { }
 #define local_unlock_irq(lvar)			local_irq_enable()
 #define local_lock_irqsave(lvar, flags)		local_irq_save(flags)
 #define local_unlock_irqrestore(lvar, flags)	local_irq_restore(flags)
+#define local_lock_irq_on(lvar, flags, cpu) \
+	local_irq_save(flags)
+#define local_unlock_irq_on(lvar, flags, cpu) \
+	local_irq_restore(flags)
 
 #define local_spin_trylock_irq(lvar, lock)	spin_trylock_irq(lock)
 #define local_spin_lock_irq(lvar, lock)		spin_lock_irq(lock)
diff --git a/mm/swap.c b/mm/swap.c
index ca194ae..b953718 100644
--- a/mm/swap.c
+++ b/mm/swap.c
@@ -821,9 +821,9 @@ void lru_add_drain_cpu(int cpu)
 		unsigned long flags;
 
 		/* No harm done if a racing interrupt already did this */
-		local_lock_irqsave(rotate_lock, flags);
+		local_lock_irq_on(rotate_lock, flags, cpu);
 		pagevec_move_tail(pvec);
-		local_unlock_irqrestore(rotate_lock, flags);
+		local_unlock_irq_on(rotate_lock, flags, cpu);
 	}
 
 	pvec = &per_cpu(lru_deactivate_file_pvecs, cpu);
@@ -866,12 +866,32 @@ void lru_add_drain(void)
 	local_unlock_cpu(swapvec_lock);
 }
 
+static DEFINE_PER_CPU(struct work_struct, lru_add_drain_work);
+
+#ifdef CONFIG_PREEMPT_RT_BASE
+static inline void remote_lru_add_drain(int cpu, struct cpumask *has_work)
+{
+	local_lock_on(swapvec_lock, cpu);
+	lru_add_drain_cpu(cpu);
+	local_unlock_on(swapvec_lock, cpu);
+}
+#else
 static void lru_add_drain_per_cpu(struct work_struct *dummy)
 {
 	lru_add_drain();
 }
 
-static DEFINE_PER_CPU(struct work_struct, lru_add_drain_work);
+static inline void remote_lru_add_drain(int cpu, struct cpumask *has_work)
+{
+		struct work_struct *work = &per_cpu(lru_add_drain_work, cpu);
+
+		INIT_WORK(work, lru_add_drain_per_cpu);
+		schedule_work_on(cpu, work);
+		cpumask_set_cpu(cpu, has_work);
+
+}
+#endif
+
 
 void lru_add_drain_all(void)
 {
@@ -884,16 +904,11 @@ void lru_add_drain_all(void)
 	cpumask_clear(&has_work);
 
 	for_each_online_cpu(cpu) {
-		struct work_struct *work = &per_cpu(lru_add_drain_work, cpu);

Re: [PATCH RT v2] mm: perform lru_add_drain_all() remotely

[Sebastian Andrzej Siewior]

* Luiz Capitulino | 2016-05-24 15:15:51 [-0400]:

>Sebastian,
>
>It's not clear to me from our last discussion what's the
>best plan for this patch. I'm sending v2 with the changes
>you suggested.

The patch at the bottom is what I intend to take into the next v4.6-RT.
local_lock_irqsave_on() was already part of the first v4.6 release. I
don't like the part where local_lock_irqsave_on() is a simple
local_irq_safe() on !RT but that is another story.

diff --git a/include/linux/locallock.h b/include/linux/locallock.h
index 493e801e0c9b..845c77f1a5ca 100644
--- a/include/linux/locallock.h
+++ b/include/linux/locallock.h
@@ -66,6 +66,9 @@ static inline void __local_lock(struct local_irq_lock *lv)
 #define local_lock(lvar)					\
 	do { __local_lock(&get_local_var(lvar)); } while (0)
 
+#define local_lock_on(lvar, cpu)				\
+	do { __local_lock(&per_cpu(lvar, cpu)); } while (0)
+
 static inline int __local_trylock(struct local_irq_lock *lv)
 {
 	if (lv->owner != current && spin_trylock_local(&lv->lock)) {
@@ -104,6 +107,9 @@ static inline void __local_unlock(struct local_irq_lock *lv)
 		put_local_var(lvar);				\
 	} while (0)
 
+#define local_unlock_on(lvar, cpu)                       \
+	do { __local_unlock(&per_cpu(lvar, cpu)); } while (0)
+
 static inline void __local_lock_irq(struct local_irq_lock *lv)
 {
 	spin_lock_irqsave(&lv->lock, lv->flags);
diff --git a/mm/swap.c b/mm/swap.c
index 892747266c7e..35fab668a782 100644
--- a/mm/swap.c
+++ b/mm/swap.c
@@ -596,9 +596,9 @@ void lru_add_drain_cpu(int cpu)
 		unsigned long flags;
 
 		/* No harm done if a racing interrupt already did this */
-		local_lock_irqsave(rotate_lock, flags);
+		local_lock_irqsave_on(rotate_lock, flags, cpu);
 		pagevec_move_tail(pvec);
-		local_unlock_irqrestore(rotate_lock, flags);
+		local_unlock_irqrestore_on(rotate_lock, flags, cpu);
 	}
 
 	pvec = &per_cpu(lru_deactivate_file_pvecs, cpu);
@@ -666,12 +666,32 @@ void lru_add_drain(void)
 	local_unlock_cpu(swapvec_lock);
 }
 
+
+#ifdef CONFIG_PREEMPT_RT_BASE
+static inline void remote_lru_add_drain(int cpu, struct cpumask *has_work)
+{
+	local_lock_on(swapvec_lock, cpu);
+	lru_add_drain_cpu(cpu);
+	local_unlock_on(swapvec_lock, cpu);
+}
+
+#else
+
 static void lru_add_drain_per_cpu(struct work_struct *dummy)
 {
 	lru_add_drain();
 }
 
 static DEFINE_PER_CPU(struct work_struct, lru_add_drain_work);
+static inline void remote_lru_add_drain(int cpu, struct cpumask *has_work)
+{
+	struct work_struct *work = &per_cpu(lru_add_drain_work, cpu);
+
+	INIT_WORK(work, lru_add_drain_per_cpu);
+	schedule_work_on(cpu, work);
+	cpumask_set_cpu(cpu, has_work);
+}
+#endif
 
 void lru_add_drain_all(void)
 {
@@ -684,21 +704,18 @@ void lru_add_drain_all(void)
 	cpumask_clear(&has_work);
 
 	for_each_online_cpu(cpu) {
-		struct work_struct *work = &per_cpu(lru_add_drain_work, cpu);

Re: [PATCH RT v2] mm: perform lru_add_drain_all() remotely

[Luiz Capitulino]

On Fri, 27 May 2016 15:10:24 +0200
Sebastian Andrzej Siewior <bigeasy@linutronix.de> wrote:

> * Luiz Capitulino | 2016-05-24 15:15:51 [-0400]:
> 
> >Sebastian,
> >
> >It's not clear to me from our last discussion what's the
> >best plan for this patch. I'm sending v2 with the changes
> >you suggested.  
> 
> The patch at the bottom is what I intend to take into the next v4.6-RT.
> local_lock_irqsave_on() was already part of the first v4.6 release. I
> don't like the part where local_lock_irqsave_on() is a simple
> local_irq_safe() on !RT but that is another story.

Looks good to me.

> 
> diff --git a/include/linux/locallock.h b/include/linux/locallock.h
> index 493e801e0c9b..845c77f1a5ca 100644
> --- a/include/linux/locallock.h
> +++ b/include/linux/locallock.h
> @@ -66,6 +66,9 @@ static inline void __local_lock(struct local_irq_lock *lv)
>  #define local_lock(lvar)					\
>  	do { __local_lock(&get_local_var(lvar)); } while (0)
>  
> +#define local_lock_on(lvar, cpu)				\
> +	do { __local_lock(&per_cpu(lvar, cpu)); } while (0)
> +
>  static inline int __local_trylock(struct local_irq_lock *lv)
>  {
>  	if (lv->owner != current && spin_trylock_local(&lv->lock)) {
> @@ -104,6 +107,9 @@ static inline void __local_unlock(struct local_irq_lock *lv)
>  		put_local_var(lvar);				\
>  	} while (0)
>  
> +#define local_unlock_on(lvar, cpu)                       \
> +	do { __local_unlock(&per_cpu(lvar, cpu)); } while (0)
> +
>  static inline void __local_lock_irq(struct local_irq_lock *lv)
>  {
>  	spin_lock_irqsave(&lv->lock, lv->flags);
> diff --git a/mm/swap.c b/mm/swap.c
> index 892747266c7e..35fab668a782 100644
> --- a/mm/swap.c
> +++ b/mm/swap.c
> @@ -596,9 +596,9 @@ void lru_add_drain_cpu(int cpu)
>  		unsigned long flags;
>  
>  		/* No harm done if a racing interrupt already did this */
> -		local_lock_irqsave(rotate_lock, flags);
> +		local_lock_irqsave_on(rotate_lock, flags, cpu);
>  		pagevec_move_tail(pvec);
> -		local_unlock_irqrestore(rotate_lock, flags);
> +		local_unlock_irqrestore_on(rotate_lock, flags, cpu);
>  	}
>  
>  	pvec = &per_cpu(lru_deactivate_file_pvecs, cpu);
> @@ -666,12 +666,32 @@ void lru_add_drain(void)
>  	local_unlock_cpu(swapvec_lock);
>  }
>  
> +
> +#ifdef CONFIG_PREEMPT_RT_BASE
> +static inline void remote_lru_add_drain(int cpu, struct cpumask *has_work)
> +{
> +	local_lock_on(swapvec_lock, cpu);
> +	lru_add_drain_cpu(cpu);
> +	local_unlock_on(swapvec_lock, cpu);
> +}
> +
> +#else
> +
>  static void lru_add_drain_per_cpu(struct work_struct *dummy)
>  {
>  	lru_add_drain();
>  }
>  
>  static DEFINE_PER_CPU(struct work_struct, lru_add_drain_work);
> +static inline void remote_lru_add_drain(int cpu, struct cpumask *has_work)
> +{
> +	struct work_struct *work = &per_cpu(lru_add_drain_work, cpu);
> +
> +	INIT_WORK(work, lru_add_drain_per_cpu);
> +	schedule_work_on(cpu, work);
> +	cpumask_set_cpu(cpu, has_work);
> +}
> +#endif
>  
>  void lru_add_drain_all(void)
>  {
> @@ -684,21 +704,18 @@ void lru_add_drain_all(void)
>  	cpumask_clear(&has_work);
>  
>  	for_each_online_cpu(cpu) {
> -		struct work_struct *work = &per_cpu(lru_add_drain_work, cpu);
> -
>  		if (pagevec_count(&per_cpu(lru_add_pvec, cpu)) ||
>  		    pagevec_count(&per_cpu(lru_rotate_pvecs, cpu)) ||
>  		    pagevec_count(&per_cpu(lru_deactivate_file_pvecs, cpu)) ||
>  		    pagevec_count(&per_cpu(lru_deactivate_pvecs, cpu)) ||
> -		    need_activate_page_drain(cpu)) {
> -			INIT_WORK(work, lru_add_drain_per_cpu);
> -			schedule_work_on(cpu, work);
> -			cpumask_set_cpu(cpu, &has_work);
> -		}
> +		    need_activate_page_drain(cpu))
> +			remote_lru_add_drain(cpu, &has_work);
>  	}
>  
> +#ifndef CONFIG_PREEMPT_RT_BASE
>  	for_each_cpu(cpu, &has_work)
>  		flush_work(&per_cpu(lru_add_drain_work, cpu));
> +#endif
>  
>  	put_online_cpus();
>  	mutex_unlock(&lock);