Re: [PATCH][RESUBMIT] add dom0 vcpu hotplug control

[Ryan Harper <ryanh@us.ibm.com>]

* Christian Limpach <christian.limpach@gmail.com> [2005-06-21 09:11]:
> On 6/14/05, Ryan Harper <ryanh@us.ibm.com> wrote:
> > * Ryan Harper <ryanh@us.ibm.com> [2005-06-07 17:05]:
> > > This patch adds new control messages for vcpu hotplug events.  Via the
> > > xm vcpu_hotplug sub-program, VCPUS in domains can be enabled/disabled
> > > when CONFIG_HOTPLUG_CPU is enabled in the target domain's kernel.
> > >
> > 
> > Updated this patch to use vcpu_to_cpu up/down info to control whether
> > state changes are sent when hotplugging vcpus.
> 
> I've applied both patches.  I think there's a flaw in the way how
> vcpu_to_cpu gets passed up to python, the -1 -> # hack is kind of
> gross and it won't work for cpus > 9?

I updated the vcpu_to_cpu string creation to include a field separator,
which gets rid of the -1 -> # hack and works for cpus > 9.

I ran into some issues with stale vcpu_to_cpu lists when running the
hotplug subprogram.  I would take a vcpu offline, and then issue the
command to bring it back and the vcpu_to_cpu list would not have changed
to indicate the the vcpu actually went down.  If I injected a xm list -v
(which always showed the correct mapping) then subsequent hotplug
commands would see the state change and fire off the hotplug request.  I
don't know that not sending the event when not changing state saves that
much work so I took the state check out and now just send the hotplug
event directly.

> Also the whole hotplug stuff is still missing interrupt re-routing
> when a vcpu is taken down.  To do this, we need an evtchn operation to
> change the vcpu affinity of a port by changing notify_vcpu_id.

I don't fully understand all of the mappings that are happening, so this
part of the patch might be way off.  In any case, I've added a new
evtchn op to set the notify_vcpu_id field of a channel.  I updated the
HOTPLUG_CPU code to use the new routines when bringing cpus up and down.
When taking down a cpu, I route the IPI irq channels to CPU 0, and when
the cpu comes up, it re-routes the channels back to the awakened CPU.

-- 
Ryan Harper
Software Engineer; Linux Technology Center
IBM Corp., Austin, Tx
(512) 838-9253   T/L: 678-9253
ryanh@us.ibm.com


diffstat output:
 linux-2.6.11-xen-sparse/arch/xen/i386/kernel/smpboot.c |   19 +++++++---
 linux-2.6.11-xen-sparse/arch/xen/kernel/evtchn.c       |   32 +++++++++++++++++
 tools/python/xen/xend/XendDomainInfo.py                |    6 ++-
 tools/python/xen/xm/main.py                            |   18 +--------
 xen/common/event_channel.c                             |   29 +++++++++++++++
 xen/include/public/event_channel.h                     |   10 ++++-
 6 files changed, 91 insertions(+), 23 deletions(-)

Signed-off-by: Ryan Harper <ryanh@us.ibm.com>
---
diff -urN b/linux-2.6.11-xen-sparse/arch/xen/i386/kernel/smpboot.c c/linux-2.6.11-xen-sparse/arch/xen/i386/kernel/smpboot.c
--- b/linux-2.6.11-xen-sparse/arch/xen/i386/kernel/smpboot.c	2005-06-21 22:10:55.000000000 -0500
+++ c/linux-2.6.11-xen-sparse/arch/xen/i386/kernel/smpboot.c	2005-06-22 14:31:48.765503988 -0500
@@ -103,6 +103,11 @@
 DEFINE_PER_CPU(int, cpu_state) = { 0 };
 #endif
 
+static DEFINE_PER_CPU(int, resched_irq);
+static DEFINE_PER_CPU(int, callfunc_irq);
+static char resched_name[NR_CPUS][15];
+static char callfunc_name[NR_CPUS][15];
+
 #if 0
 /*
  * Currently trivial. Write the real->protected mode
@@ -1328,6 +1333,10 @@
 	while (!cpu_online(cpu))
 		cpu_relax();
 
+   /* re-route bound IRQs 0 to cpu */
+   rebind_evtchn_from_irq(0, cpu,  per_cpu(resched_irq, cpu));
+   rebind_evtchn_from_irq(0, cpu, per_cpu(callfunc_irq, cpu));
+
 	fixup_irqs(cpu_online_map);
 	/* counter the disable in fixup_irqs() */
 	local_irq_enable();
@@ -1357,6 +1366,11 @@
 
 	cpu_clear(cpu, map);
 	fixup_irqs(map);
+
+   /* re-route IRQs from dead vcpu to another */
+   rebind_evtchn_from_irq(cpu, 0,  per_cpu(resched_irq, cpu));
+   rebind_evtchn_from_irq(cpu, 0, per_cpu(callfunc_irq, cpu));
+
 	/* It's now safe to remove this processor from the online map */
 	cpu_clear(cpu, cpu_online_map);
 
@@ -1514,11 +1528,6 @@
 extern irqreturn_t smp_reschedule_interrupt(int, void *, struct pt_regs *);
 extern irqreturn_t smp_call_function_interrupt(int, void *, struct pt_regs *);
 
-static DEFINE_PER_CPU(int, resched_irq);
-static DEFINE_PER_CPU(int, callfunc_irq);
-static char resched_name[NR_CPUS][15];
-static char callfunc_name[NR_CPUS][15];
-
 void __init smp_intr_init(void)
 {
 	int cpu = smp_processor_id();
diff -urN b/linux-2.6.11-xen-sparse/arch/xen/kernel/evtchn.c c/linux-2.6.11-xen-sparse/arch/xen/kernel/evtchn.c
--- b/linux-2.6.11-xen-sparse/arch/xen/kernel/evtchn.c	2005-06-21 22:10:53.000000000 -0500
+++ c/linux-2.6.11-xen-sparse/arch/xen/kernel/evtchn.c	2005-06-22 14:31:48.767503708 -0500
@@ -271,6 +271,38 @@
     return irq;
 }
 
+void rebind_evtchn_from_ipi(int cpu, int newcpu, int ipi)
+{
+    evtchn_op_t op;
+    int evtchn = per_cpu(ipi_to_evtchn, cpu)[ipi];
+
+    spin_lock(&irq_mapping_update_lock);
+
+    op.cmd          = EVTCHNOP_rebind;
+    op.u.rebind.port = evtchn;
+    op.u.rebind.vcpu = newcpu;
+    if ( HYPERVISOR_event_channel_op(&op) != 0 )
+       printk(KERN_INFO "Failed to rebind IPI%d to CPU%d\n",ipi,newcpu);
+
+    spin_unlock(&irq_mapping_update_lock);
+}
+
+void rebind_evtchn_from_irq(int cpu, int newcpu, int irq)
+{
+    evtchn_op_t op;
+    int evtchn = irq_to_evtchn[irq];
+
+    spin_lock(&irq_mapping_update_lock);
+
+    op.cmd          = EVTCHNOP_rebind;
+    op.u.rebind.port = evtchn;
+    op.u.rebind.vcpu = newcpu;
+    if ( HYPERVISOR_event_channel_op(&op) != 0 )
+       printk(KERN_INFO "Failed to rebind IRQ%d to CPU%d\n",irq,newcpu);
+
+    spin_unlock(&irq_mapping_update_lock);
+}
+
 void unbind_ipi_on_cpu_from_irq(int cpu, int ipi)
 {
     evtchn_op_t op;
diff -urN b/tools/python/xen/xend/XendDomainInfo.py c/tools/python/xen/xend/XendDomainInfo.py
--- b/tools/python/xen/xend/XendDomainInfo.py	2005-06-21 22:10:54.000000000 -0500
+++ c/tools/python/xen/xend/XendDomainInfo.py	2005-06-22 14:31:48.000000000 -0500
@@ -423,8 +423,10 @@
             sxpr.append(['cpu_time', self.info['cpu_time']/1e9])    
             sxpr.append(['vcpus', self.info['vcpus']])
             sxpr.append(['cpumap', self.info['cpumap']])
-            sxpr.append(['vcpu_to_cpu', ''.join(map(lambda x: str(x),
-                        self.info['vcpu_to_cpu'][0:self.info['vcpus']]))])
+            # build a string, using '|' to seperate items, show only up
+            # to number of vcpus in domain, and trim the trailing '|'
+            sxpr.append(['vcpu_to_cpu', ''.join(map(lambda x: str(x)+'|',
+                        self.info['vcpu_to_cpu'][0:self.info['vcpus']]))[:-1]])
             
         if self.start_time:
             up_time =  time.time() - self.start_time  
diff -urN b/tools/python/xen/xm/main.py c/tools/python/xen/xm/main.py
--- b/tools/python/xen/xm/main.py	2005-06-21 22:10:53.000000000 -0500
+++ c/tools/python/xen/xm/main.py	2005-06-22 14:54:50.810258798 -0500
@@ -410,8 +410,7 @@
         print 'Name              Id  VCPU  CPU  CPUMAP'
         for dom in doms:
             info = server.xend_domain(dom)
-            # XXX this is quite broken for cpu's > 9
-            vcpu_to_cpu = sxp.child_value(info, 'vcpu_to_cpu', '?').replace('-1','#')
+            vcpu_to_cpu = sxp.child_value(info, 'vcpu_to_cpu', '-1').split('|')
             cpumap = sxp.child_value(info, 'cpumap', [])
             mask = ((int(sxp.child_value(info, 'vcpus', '0')))**2) - 1
             count = 0
@@ -420,10 +419,7 @@
                 d['name']   = sxp.child_value(info, 'name', '??')
                 d['dom']    = int(sxp.child_value(info, 'id', '-1'))
                 d['vcpu']   = int(count)
-                if cpu == "#":
-                    d['cpu']    = int("-1")
-                else:
-                    d['cpu']    = int(cpu)
+                d['cpu']    = int(cpu)
                 d['cpumap'] = int(cpumap[count])&mask
                 count = count + 1
                 print ("%(name)-16s %(dom)3d  %(vcpu)4d  %(cpu)3d  0x%(cpumap)x" % d)
@@ -593,15 +589,7 @@
         state = int(args[3])
         dom = server.xend_domain(name)
         id = sxp.child_value(dom, 'id')
-        vcpu_to_cpu = sxp.child_value(dom, 'vcpu_to_cpu', '-1')
-        # only send state change if states differ 
-        try:
-            # (down going up) or (up going down)
-            if (vcpu_to_cpu[vcpu] == "-1" and state == 1) or \
-               (vcpu_to_cpu[vcpu] != "-1" and state == 0):
-                server.xend_domain_vcpu_hotplug(id, vcpu, state)
-        except IndexError:
-            print "Invalid VCPU(%d)"%(vcpu)
+        server.xend_domain_vcpu_hotplug(id, vcpu, state)
 
 xm.prog(ProgVcpuhotplug)
 
diff -urN b/xen/common/event_channel.c c/xen/common/event_channel.c
--- b/xen/common/event_channel.c	2005-06-21 22:10:55.000000000 -0500
+++ c/xen/common/event_channel.c	2005-06-22 14:55:20.936237917 -0500
@@ -579,6 +579,29 @@
     return rc;
 }
 
+static long evtchn_rebind(evtchn_rebind_t *bind) 
+{
+    struct domain *d    = current->domain;
+    int            port = bind->port;
+    int            vcpu = bind->vcpu;
+    struct evtchn *chn;
+    long             rc = 0;
+
+    spin_lock(&d->evtchn_lock);
+
+    if ( !port_is_valid(d, port) )
+    {
+        rc = -EINVAL;
+        goto out;
+    }
+
+    chn = evtchn_from_port(d, port);
+    chn->notify_vcpu_id = vcpu;
+
+ out:
+    spin_unlock(&d->evtchn_lock);
+    return rc;
+}
 
 long do_event_channel_op(evtchn_op_t *uop)
 {
@@ -637,6 +660,12 @@
             rc = -EFAULT;
         break;
 
+    case EVTCHNOP_rebind:
+        rc = evtchn_rebind(&op.u.rebind);
+        if ( (rc == 0) && (copy_to_user(uop, &op, sizeof(op)) != 0) )
+            rc = -EFAULT;
+        break;
+
     default:
         rc = -ENOSYS;
         break;
diff -urN b/xen/include/public/event_channel.h c/xen/include/public/event_channel.h
--- b/xen/include/public/event_channel.h	2005-06-21 22:10:55.000000000 -0500
+++ c/xen/include/public/event_channel.h	2005-06-22 14:31:48.801498950 -0500
@@ -162,6 +162,13 @@
     } PACKED u;
 } PACKED evtchn_status_t; /* 20 bytes */
 
+#define EVTCHNOP_rebind        8
+typedef struct {
+    /* IN parameters. */
+    u32 port;                         /*  0 */
+    u32 vcpu;                         /*  4 */
+} PACKED evtchn_rebind_t; /* 8 bytes */
+
 typedef struct {
     u32 cmd; /* EVTCHNOP_* */         /*  0 */
     u32 __reserved;                   /*  4 */
@@ -174,7 +181,8 @@
         evtchn_close_t            close;
         evtchn_send_t             send;
         evtchn_status_t           status;
-        u8                        __dummy[24];
+        evtchn_rebind_t           rebind;
+        u8                        __dummy[16];
     } PACKED u;
 } PACKED evtchn_op_t; /* 32 bytes */