[PATCH 2 of 5] credit2: Flexible cpu-to-schedule-spinlock mappings

[George Dunlap <george.dunlap@eu.citrix.com>]

4 files changed, 40 insertions(+), 19 deletions(-)
xen/arch/ia64/vmx/vmmu.c   |    6 +++---
xen/common/sched_credit.c  |    8 ++++----
xen/common/schedule.c      |   18 ++++++++++--------
xen/include/xen/sched-if.h |   27 +++++++++++++++++++++++----


Credit2 shares a runqueue between several cpus.  Rather than have
double locking and dealing with the cpu-to-runqueue races, allow
the scheduler to redefine the sched_lock-to-cpu mapping.

Signed-off-by: George Dunlap <george.dunlap@eu.citrix.com>

diff -r 2631707c54b3 -r 21d0f640b0c0 xen/arch/ia64/vmx/vmmu.c
--- a/xen/arch/ia64/vmx/vmmu.c	Wed Apr 14 11:16:58 2010 +0100
+++ b/xen/arch/ia64/vmx/vmmu.c	Wed Apr 14 11:16:58 2010 +0100
@@ -394,7 +394,7 @@
     if (cpu != current->processor)
         return;
     local_irq_save(flags);
-    if (!spin_trylock(&per_cpu(schedule_data, cpu).schedule_lock))
+    if (!spin_trylock(per_cpu(schedule_data, cpu).schedule_lock))
         goto bail2;
     if (v->processor != cpu)
         goto bail1;
@@ -416,7 +416,7 @@
     ia64_dv_serialize_data();
     args->vcpu = NULL;
 bail1:
-    spin_unlock(&per_cpu(schedule_data, cpu).schedule_lock);
+    spin_unlock(per_cpu(schedule_data, cpu).schedule_lock);
 bail2:
     local_irq_restore(flags);
 }
@@ -446,7 +446,7 @@
         do {
             cpu = v->processor;
             if (cpu != current->processor) {
-                spin_barrier(&per_cpu(schedule_data, cpu).schedule_lock);
+                spin_barrier(per_cpu(schedule_data, cpu).schedule_lock);
                 /* Flush VHPT on remote processors. */
                 smp_call_function_single(cpu, &ptc_ga_remote_func, &args, 1);
             } else {
diff -r 2631707c54b3 -r 21d0f640b0c0 xen/common/sched_credit.c
--- a/xen/common/sched_credit.c	Wed Apr 14 11:16:58 2010 +0100
+++ b/xen/common/sched_credit.c	Wed Apr 14 11:16:58 2010 +0100
@@ -789,7 +789,7 @@
 
     spc->runq_sort_last = sort_epoch;
 
-    spin_lock_irqsave(&per_cpu(schedule_data, cpu).schedule_lock, flags);
+    spin_lock_irqsave(per_cpu(schedule_data, cpu).schedule_lock, flags);
 
     runq = &spc->runq;
     elem = runq->next;
@@ -814,7 +814,7 @@
         elem = next;
     }
 
-    spin_unlock_irqrestore(&per_cpu(schedule_data, cpu).schedule_lock, flags);
+    spin_unlock_irqrestore(per_cpu(schedule_data, cpu).schedule_lock, flags);
 }
 
 static void
@@ -1130,7 +1130,7 @@
          * cause a deadlock if the peer CPU is also load balancing and trying
          * to lock this CPU.
          */
-        if ( !spin_trylock(&per_cpu(schedule_data, peer_cpu).schedule_lock) )
+        if ( !spin_trylock(per_cpu(schedule_data, peer_cpu).schedule_lock) )
         {
             CSCHED_STAT_CRANK(steal_trylock_failed);
             continue;
@@ -1140,7 +1140,7 @@
          * Any work over there to steal?
          */
         speer = csched_runq_steal(peer_cpu, cpu, snext->pri);
-        spin_unlock(&per_cpu(schedule_data, peer_cpu).schedule_lock);
+        spin_unlock(per_cpu(schedule_data, peer_cpu).schedule_lock);
         if ( speer != NULL )
             return speer;
     }
diff -r 2631707c54b3 -r 21d0f640b0c0 xen/common/schedule.c
--- a/xen/common/schedule.c	Wed Apr 14 11:16:58 2010 +0100
+++ b/xen/common/schedule.c	Wed Apr 14 11:16:58 2010 +0100
@@ -131,7 +131,7 @@
     s_time_t delta;
 
     ASSERT(v->runstate.state != new_state);
-    ASSERT(spin_is_locked(&per_cpu(schedule_data,v->processor).schedule_lock));
+    ASSERT(spin_is_locked(per_cpu(schedule_data,v->processor).schedule_lock));
 
     vcpu_urgent_count_update(v);
 
@@ -340,7 +340,7 @@
     /* Switch to new CPU, then unlock old CPU. */
     v->processor = new_cpu;
     spin_unlock_irqrestore(
-        &per_cpu(schedule_data, old_cpu).schedule_lock, flags);
+        per_cpu(schedule_data, old_cpu).schedule_lock, flags);
 
     /* Wake on new CPU. */
     vcpu_wake(v);
@@ -846,7 +846,7 @@
 
     sd = &this_cpu(schedule_data);
 
-    spin_lock_irq(&sd->schedule_lock);
+    spin_lock_irq(sd->schedule_lock);
 
     stop_timer(&sd->s_timer);
     
@@ -862,7 +862,7 @@
 
     if ( unlikely(prev == next) )
     {
-        spin_unlock_irq(&sd->schedule_lock);
+        spin_unlock_irq(sd->schedule_lock);
         trace_continue_running(next);
         return continue_running(prev);
     }
@@ -900,7 +900,7 @@
     ASSERT(!next->is_running);
     next->is_running = 1;
 
-    spin_unlock_irq(&sd->schedule_lock);
+    spin_unlock_irq(sd->schedule_lock);
 
     perfc_incr(sched_ctx);
 
@@ -968,7 +968,9 @@
 
     for_each_possible_cpu ( i )
     {
-        spin_lock_init(&per_cpu(schedule_data, i).schedule_lock);
+        spin_lock_init(&per_cpu(schedule_data, i)._lock);
+        per_cpu(schedule_data, i).schedule_lock
+            = &per_cpu(schedule_data, i)._lock;
         init_timer(&per_cpu(schedule_data, i).s_timer, s_timer_fn, NULL, i);
     }
 
@@ -1005,10 +1007,10 @@
 
     for_each_online_cpu ( i )
     {
-        spin_lock(&per_cpu(schedule_data, i).schedule_lock);
+        spin_lock(per_cpu(schedule_data, i).schedule_lock);
         printk("CPU[%02d] ", i);
         SCHED_OP(dump_cpu_state, i);
-        spin_unlock(&per_cpu(schedule_data, i).schedule_lock);
+        spin_unlock(per_cpu(schedule_data, i).schedule_lock);
     }
 
     local_irq_restore(flags);
diff -r 2631707c54b3 -r 21d0f640b0c0 xen/include/xen/sched-if.h
--- a/xen/include/xen/sched-if.h	Wed Apr 14 11:16:58 2010 +0100
+++ b/xen/include/xen/sched-if.h	Wed Apr 14 11:16:58 2010 +0100
@@ -10,8 +10,19 @@
 
 #include <xen/percpu.h>
 
+/*
+ * In order to allow a scheduler to remap the lock->cpu mapping,
+ * we have a per-cpu pointer, along with a pre-allocated set of
+ * locks.  The generic schedule init code will point each schedule lock
+ * pointer to the schedule lock; if the scheduler wants to remap them,
+ * it can simply modify the schedule locks.
+ * 
+ * For cache betterness, keep the actual lock in the same cache area
+ * as the rest of the struct.  Just have the scheduler point to the
+ * one it wants (This may be the one right in front of it).*/
 struct schedule_data {
-    spinlock_t          schedule_lock;  /* spinlock protecting curr        */
+    spinlock_t         *schedule_lock,
+                       _lock;
     struct vcpu        *curr;           /* current task                    */
     struct vcpu        *idle;           /* idle task for this cpu          */
     void               *sched_priv;
@@ -27,11 +38,19 @@
 
     for ( ; ; )
     {
+        /* NB: For schedulers with multiple cores per runqueue,
+         * a vcpu may change processor w/o changing runqueues;
+         * so we may release a lock only to grab it again.
+         *
+         * If that is measured to be an issue, then the check
+         * should be changed to checking if the locks pointed to
+         * by cpu and v->processor are still the same.
+         */
         cpu = v->processor;
-        spin_lock(&per_cpu(schedule_data, cpu).schedule_lock);
+        spin_lock(per_cpu(schedule_data, cpu).schedule_lock);
         if ( likely(v->processor == cpu) )
             break;
-        spin_unlock(&per_cpu(schedule_data, cpu).schedule_lock);
+        spin_unlock(per_cpu(schedule_data, cpu).schedule_lock);
     }
 }
 
@@ -42,7 +61,7 @@
 
 static inline void vcpu_schedule_unlock(struct vcpu *v)
 {
-    spin_unlock(&per_cpu(schedule_data, v->processor).schedule_lock);
+    spin_unlock(per_cpu(schedule_data, v->processor).schedule_lock);
 }
 
 #define vcpu_schedule_unlock_irq(v) \