* [PATCH] add dom0 vcpu hotplug control
@ 2005-06-07 22:02 Ryan Harper
2005-06-14 16:37 ` [PATCH][RESUBMIT] " Ryan Harper
0 siblings, 1 reply; 5+ messages in thread
From: Ryan Harper @ 2005-06-07 22:02 UTC (permalink / raw)
To: xen-devel
This patch adds new control messages for vcpu hotplug events. Via the
xm vcpu_hotplug sub-program, VCPUS in domains can be enabled/disabled
when CONFIG_HOTPLUG_CPU is enabled in the target domain's kernel.
Currently there is nothing that tracks whether a VCPU is up or down.
My previous [1]patch added a new per-VCPU flag (VCPUF_down) which could
be used to keep track of which VCPUS are up and down. Right now, there
isn't a hypercall that returns the per-VCPU flag status (something
equivalent to the per-domain flags in get_dominfo ). Have we thought
about a get_vcpu_info hypercall? I'd like to get that implemented so I
can report VCPU state in the xm list --vcpus output. That would also
make it easier to skip sending control messages that don't change the
VCPU's state (e.g. sending a down message to a vcpu that is already
down).
Please apply.
1. http://lists.xensource.com/archives/html/xen-devel/2005-06/msg00192.html
--
Ryan Harper
Software Engineer; Linux Technology Center
IBM Corp., Austin, Tx
(512) 838-9253 T/L: 678-9253
ryanh@us.ibm.com
diffstat output:
linux-2.6.11-xen-sparse/arch/xen/i386/kernel/smpboot.c | 81 +++++++++++++++++
tools/python/xen/lowlevel/xu/xu.c | 16 +++
tools/python/xen/xend/XendClient.py | 6 +
tools/python/xen/xend/XendDomain.py | 4
tools/python/xen/xend/XendDomainInfo.py | 12 ++
tools/python/xen/xend/server/SrvDomain.py | 8 +
tools/python/xen/xend/server/messages.py | 18 +++
tools/python/xen/xm/main.py | 21 ++++
xen/include/public/io/domain_controller.h | 20 ++++
9 files changed, 186 insertions(+)
Signed-off-by: Ryan Harper <ryanh@us.ibm.com>
---
diff -urN a/linux-2.6.11-xen-sparse/arch/xen/i386/kernel/smpboot.c b/linux-2.6.11-xen-sparse/arch/xen/i386/kernel/smpboot.c
--- a/linux-2.6.11-xen-sparse/arch/xen/i386/kernel/smpboot.c 2005-06-06 22:05:33.000000000 -0500
+++ b/linux-2.6.11-xen-sparse/arch/xen/i386/kernel/smpboot.c 2005-06-07 16:53:49.362987126 -0500
@@ -85,6 +85,13 @@
/* Set when the idlers are all forked */
int smp_threads_ready;
+#ifdef CONFIG_HOTPLUG_CPU
+struct vcpu_hotplug_handler_t {
+ void (*fn)();
+ u32 vcpu;
+};
+#endif
+
#if 0
/*
* Trampoline 80x86 program as an array.
@@ -1297,6 +1304,9 @@
}
#ifdef CONFIG_HOTPLUG_CPU
+#include <asm-xen/ctrl_if.h>
+/* hotplug down/up funtion pointer and target vcpu */
+struct vcpu_hotplug_handler_t vcpu_hotplug_handler;
/* must be called with the cpucontrol mutex held */
static int __devinit cpu_enable(unsigned int cpu)
@@ -1357,6 +1367,77 @@
}
printk(KERN_ERR "CPU %u didn't die...\n", cpu);
}
+
+static int vcpu_hotplug_cpu_process(void *unused)
+{
+ struct vcpu_hotplug_handler_t *handler = &vcpu_hotplug_handler;
+
+ if ( handler->fn ) {
+ (*(handler->fn))(handler->vcpu);
+ handler->fn = NULL;
+ }
+ return 0;
+}
+
+static void __vcpu_hotplug_handler(void *unused)
+{
+ int err;
+
+ err = kernel_thread(vcpu_hotplug_cpu_process,
+ NULL, CLONE_FS | CLONE_FILES);
+ if ( err < 0 )
+ printk(KERN_ALERT "Error creating hotplug_cpu process!\n");
+
+}
+
+static void vcpu_hotplug_event_handler(ctrl_msg_t *msg, unsigned long id)
+{
+ static DECLARE_WORK(vcpu_hotplug_work, __vcpu_hotplug_handler, NULL);
+ vcpu_hotplug_t *req = (vcpu_hotplug_t *)&msg->msg[0];
+ struct vcpu_hotplug_handler_t *handler = &vcpu_hotplug_handler;
+ ssize_t ret;
+
+ if ( msg->length != sizeof(vcpu_hotplug_t) )
+ goto parse_error;
+
+ /* grab target vcpu from msg */
+ handler->vcpu = req->vcpu;
+
+ /* determine which function to call based on msg subtype */
+ switch ( msg->subtype ) {
+ case CMSG_VCPU_HOTPLUG_OFF:
+ handler->fn = (void *)&cpu_down;
+ ret = schedule_work(&vcpu_hotplug_work);
+ req->status = (u32) ret;
+ break;
+ case CMSG_VCPU_HOTPLUG_ON:
+ handler->fn = (void *)&cpu_up;
+ ret = schedule_work(&vcpu_hotplug_work);
+ req->status = (u32) ret;
+ break;
+ default:
+ goto parse_error;
+ }
+
+ ctrl_if_send_response(msg);
+ return;
+ parse_error:
+ msg->length = 0;
+ ctrl_if_send_response(msg);
+}
+
+static int __init setup_vcpu_hotplug_event(void)
+{
+ struct vcpu_hotplug_handler_t *handler = &vcpu_hotplug_handler;
+
+ handler->fn = NULL;
+ ctrl_if_register_receiver(CMSG_VCPU_HOTPLUG, vcpu_hotplug_event_handler, 0);
+
+ return 0;
+}
+
+__initcall(setup_vcpu_hotplug_event);
+
#else /* ... !CONFIG_HOTPLUG_CPU */
int __cpu_disable(void)
{
diff -urN a/tools/python/xen/lowlevel/xu/xu.c b/tools/python/xen/lowlevel/xu/xu.c
--- a/tools/python/xen/lowlevel/xu/xu.c 2005-06-06 22:05:28.000000000 -0500
+++ b/tools/python/xen/lowlevel/xu/xu.c 2005-06-07 16:51:43.084342691 -0500
@@ -744,6 +744,14 @@
C2P(mem_request_t, target, Int, Long);
C2P(mem_request_t, status, Int, Long);
return dict;
+ case TYPE(CMSG_VCPU_HOTPLUG, CMSG_VCPU_HOTPLUG_OFF):
+ C2P(vcpu_hotplug_t, vcpu, Int, Long);
+ C2P(vcpu_hotplug_t, status, Int, Long);
+ return dict;
+ case TYPE(CMSG_VCPU_HOTPLUG, CMSG_VCPU_HOTPLUG_ON):
+ C2P(vcpu_hotplug_t, vcpu, Int, Long);
+ C2P(vcpu_hotplug_t, status, Int, Long);
+ return dict;
}
return PyString_FromStringAndSize((char *)xum->msg.msg, xum->msg.length);
@@ -910,6 +918,14 @@
P2C(mem_request_t, target, u32);
P2C(mem_request_t, status, u32);
break;
+ case TYPE(CMSG_VCPU_HOTPLUG, CMSG_VCPU_HOTPLUG_OFF):
+ P2C(vcpu_hotplug_t, vcpu, u32);
+ P2C(vcpu_hotplug_t, status, u32);
+ break;
+ case TYPE(CMSG_VCPU_HOTPLUG, CMSG_VCPU_HOTPLUG_ON):
+ P2C(vcpu_hotplug_t, vcpu, u32);
+ P2C(vcpu_hotplug_t, status, u32);
+ break;
case TYPE(CMSG_USBIF_FE, CMSG_USBIF_FE_INTERFACE_STATUS_CHANGED):
P2C(usbif_fe_interface_status_changed_t, status, u32);
P2C(usbif_fe_interface_status_changed_t, evtchn, u16);
diff -urN a/tools/python/xen/xend/XendClient.py b/tools/python/xen/xend/XendClient.py
--- a/tools/python/xen/xend/XendClient.py 2005-06-06 22:05:27.000000000 -0500
+++ b/tools/python/xen/xend/XendClient.py 2005-06-07 16:51:43.086342400 -0500
@@ -271,6 +271,12 @@
'target' : mem_target })
return val
+ def xend_domain_vcpu_hotplug(self, id, vcpu, state):
+ return self.xendPost(self.domainurl(id),
+ {'op' : 'vcpu_hotplug',
+ 'vcpu' : vcpu,
+ 'state' : state })
+
def xend_domain_vif_limit(self, id, vif, credit, period):
return self.xendPost(self.domainurl(id),
{ 'op' : 'vif_limit_set',
diff -urN a/tools/python/xen/xend/XendDomain.py b/tools/python/xen/xend/XendDomain.py
--- a/tools/python/xen/xend/XendDomain.py 2005-06-06 22:05:32.000000000 -0500
+++ b/tools/python/xen/xend/XendDomain.py 2005-06-07 16:51:43.090341819 -0500
@@ -703,6 +703,10 @@
dominfo = self.domain_lookup(id)
return dominfo.mem_target_set(target)
+ def domain_vcpu_hotplug(self, id, vcpu, state):
+ dominfo = self.domain_lookup(id)
+ return dominfo.vcpu_hotplug(vcpu, state)
+
def instance():
diff -urN a/tools/python/xen/xend/XendDomainInfo.py b/tools/python/xen/xend/XendDomainInfo.py
--- a/tools/python/xen/xend/XendDomainInfo.py 2005-06-06 22:05:33.000000000 -0500
+++ b/tools/python/xen/xend/XendDomainInfo.py 2005-06-07 16:51:43.092341528 -0500
@@ -1071,6 +1071,18 @@
msg = messages.packMsg('mem_request_t', { 'target' : target * (1 << 8)} )
self.channel.writeRequest(msg)
+ def vcpu_hotplug(self, vcpu, state):
+ """Disable or enable VCPU in domain.
+ """
+ if self.channel:
+ if int(state) == 0:
+ msg = messages.packMsg('vcpu_hotplug_off_t', { 'vcpu' : vcpu} )
+ else:
+ msg = messages.packMsg('vcpu_hotplug_on_t', { 'vcpu' : vcpu} )
+
+ self.channel.writeRequest(msg)
+
+
def shutdown(self, reason, key=0):
msgtype = shutdown_messages.get(reason)
if not msgtype:
diff -urN a/tools/python/xen/xend/server/SrvDomain.py b/tools/python/xen/xend/server/SrvDomain.py
--- a/tools/python/xen/xend/server/SrvDomain.py 2005-06-06 22:05:31.000000000 -0500
+++ b/tools/python/xen/xend/server/SrvDomain.py 2005-06-07 16:51:43.095341092 -0500
@@ -180,6 +180,14 @@
val = fn(req.args, {'dom': self.dom.id})
return val
+ def op_vcpu_hotplug(self, op, req):
+ fn = FormFn(self.xd.domain_vcpu_hotplug,
+ [['dom', 'str'],
+ ['vcpu', 'int'],
+ ['state', 'int']])
+ val = fn(req.args, {'dom': self.dom.id})
+ return val
+
def render_POST(self, req):
return self.perform(req)
diff -urN a/tools/python/xen/xend/server/messages.py b/tools/python/xen/xend/server/messages.py
--- a/tools/python/xen/xend/server/messages.py 2005-06-06 22:05:28.000000000 -0500
+++ b/tools/python/xen/xend/server/messages.py 2005-06-07 16:51:43.125336730 -0500
@@ -309,6 +309,24 @@
msg_formats.update(mem_request_formats)
#============================================================================
+# Domain vcpu hotplug message.
+#============================================================================
+
+CMSG_VCPU_HOTPLUG = 10
+CMSG_VCPU_HOTPLUG_OFF = 0
+CMSG_VCPU_HOTPLUG_ON = 1
+
+vcpu_hotplug_formats = {
+ 'vcpu_hotplug_off_t':
+ (CMSG_VCPU_HOTPLUG, CMSG_VCPU_HOTPLUG_OFF),
+
+ 'vcpu_hotplug_on_t':
+ (CMSG_VCPU_HOTPLUG, CMSG_VCPU_HOTPLUG_ON)
+ }
+
+msg_formats.update(vcpu_hotplug_formats)
+
+#============================================================================
class Msg:
pass
diff -urN a/tools/python/xen/xm/main.py b/tools/python/xen/xm/main.py
--- a/tools/python/xen/xm/main.py 2005-06-06 22:05:31.000000000 -0500
+++ b/tools/python/xen/xm/main.py 2005-06-07 16:51:43.127336439 -0500
@@ -568,6 +568,27 @@
xm.prog(ProgBalloon)
+
+class ProgVcpuhotplug(Prog):
+ group = 'domain'
+ name = 'vcpu_hotplug'
+ info = """Enable or disable a VCPU in a domain."""
+
+ def help(self, args):
+ print args[0], "DOM VCPU [0|1]"
+ print """\nRequest virtual processor VCPU to be disabled or enabled in
+domain DOM"""
+
+ def main(self, args):
+ if len(args) != 4: self.err("%s: Invalid arguments(s)" % args[0])
+ dom = args[1]
+ vcpu = args[2]
+ state = args[3]
+ server.xend_domain_vcpu_hotplug(dom, vcpu, state)
+
+xm.prog(ProgVcpuhotplug)
+
+
class ProgDomid(Prog):
group = 'domain'
name = 'domid'
diff -urN a/xen/include/public/io/domain_controller.h b/xen/include/public/io/domain_controller.h
--- a/xen/include/public/io/domain_controller.h 2005-06-06 22:05:31.000000000 -0500
+++ b/xen/include/public/io/domain_controller.h 2005-06-07 16:51:43.139334694 -0500
@@ -61,6 +61,7 @@
#define CMSG_MEM_REQUEST 7 /* Memory reservation reqs */
#define CMSG_USBIF_BE 8 /* USB controller backend */
#define CMSG_USBIF_FE 9 /* USB controller frontend */
+#define CMSG_VCPU_HOTPLUG 10 /* Hotplug VCPU messages */
/******************************************************************************
* CONSOLE DEFINITIONS
@@ -758,6 +759,25 @@
} PACKED shutdown_sysrq_t; /* 4 bytes */
/******************************************************************************
+ * VCPU HOTPLUG CONTROLS
+ */
+
+/*
+ * Subtypes for shutdown messages.
+ */
+#define CMSG_VCPU_HOTPLUG_OFF 0 /* turn vcpu off */
+#define CMSG_VCPU_HOTPLUG_ON 1 /* turn vcpu on */
+
+/*
+ * CMSG_VCPU_HOTPLUG:
+ * Indicate which vcpu's state should change
+ */
+typedef struct {
+ u32 vcpu; /* 0: VCPU's whose state will change */
+ u32 status; /* 4: Return code indicates success or failure. */
+} PACKED vcpu_hotplug_t;
+
+/******************************************************************************
* MEMORY CONTROLS
*/
^ permalink raw reply [flat|nested] 5+ messages in thread
* [PATCH][RESUBMIT] add dom0 vcpu hotplug control
2005-06-07 22:02 [PATCH] add dom0 vcpu hotplug control Ryan Harper
@ 2005-06-14 16:37 ` Ryan Harper
2005-06-21 11:02 ` Christian Limpach
0 siblings, 1 reply; 5+ messages in thread
From: Ryan Harper @ 2005-06-14 16:37 UTC (permalink / raw)
To: xen-devel
* Ryan Harper <ryanh@us.ibm.com> [2005-06-07 17:05]:
> This patch adds new control messages for vcpu hotplug events. Via the
> xm vcpu_hotplug sub-program, VCPUS in domains can be enabled/disabled
> when CONFIG_HOTPLUG_CPU is enabled in the target domain's kernel.
>
Updated this patch to use vcpu_to_cpu up/down info to control whether
state changes are sent when hotplugging vcpus.
Built and tested with vcpu_down patch against 20050614 nightly unstable
snapshot.
--
Ryan Harper
Software Engineer; Linux Technology Center
IBM Corp., Austin, Tx
(512) 838-9253 T/L: 678-9253
ryanh@us.ibm.com
diffstat output:
linux-2.6.11-xen-sparse/arch/xen/i386/kernel/smpboot.c | 81 +++++++++++++++++
tools/python/xen/lowlevel/xu/xu.c | 16 +++
tools/python/xen/xend/XendClient.py | 6 +
tools/python/xen/xend/XendDomain.py | 12 ++
tools/python/xen/xend/XendDomainInfo.py | 12 ++
tools/python/xen/xend/server/SrvDomain.py | 8 +
tools/python/xen/xend/server/messages.py | 18 +++
tools/python/xen/xm/main.py | 30 ++++++
xen/include/public/io/domain_controller.h | 20 ++++
9 files changed, 203 insertions(+)
Signed-off-by: Ryan Harper <ryanh@us.ibm.com>
---
diff -urN vcpu_down/linux-2.6.11-xen-sparse/arch/xen/i386/kernel/smpboot.c vcpu_cntl/linux-2.6.11-xen-sparse/arch/xen/i386/kernel/smpboot.c
--- vcpu_down/linux-2.6.11-xen-sparse/arch/xen/i386/kernel/smpboot.c 2005-06-13 10:41:56.000000000 -0500
+++ vcpu_cntl/linux-2.6.11-xen-sparse/arch/xen/i386/kernel/smpboot.c 2005-06-13 13:41:55.000000000 -0500
@@ -85,6 +85,13 @@
/* Set when the idlers are all forked */
int smp_threads_ready;
+#ifdef CONFIG_HOTPLUG_CPU
+struct vcpu_hotplug_handler_t {
+ void (*fn)();
+ u32 vcpu;
+};
+#endif
+
#if 0
/*
* Trampoline 80x86 program as an array.
@@ -1297,6 +1304,9 @@
}
#ifdef CONFIG_HOTPLUG_CPU
+#include <asm-xen/ctrl_if.h>
+/* hotplug down/up funtion pointer and target vcpu */
+struct vcpu_hotplug_handler_t vcpu_hotplug_handler;
/* must be called with the cpucontrol mutex held */
static int __devinit cpu_enable(unsigned int cpu)
@@ -1357,6 +1367,77 @@
}
printk(KERN_ERR "CPU %u didn't die...\n", cpu);
}
+
+static int vcpu_hotplug_cpu_process(void *unused)
+{
+ struct vcpu_hotplug_handler_t *handler = &vcpu_hotplug_handler;
+
+ if ( handler->fn ) {
+ (*(handler->fn))(handler->vcpu);
+ handler->fn = NULL;
+ }
+ return 0;
+}
+
+static void __vcpu_hotplug_handler(void *unused)
+{
+ int err;
+
+ err = kernel_thread(vcpu_hotplug_cpu_process,
+ NULL, CLONE_FS | CLONE_FILES);
+ if ( err < 0 )
+ printk(KERN_ALERT "Error creating hotplug_cpu process!\n");
+
+}
+
+static void vcpu_hotplug_event_handler(ctrl_msg_t *msg, unsigned long id)
+{
+ static DECLARE_WORK(vcpu_hotplug_work, __vcpu_hotplug_handler, NULL);
+ vcpu_hotplug_t *req = (vcpu_hotplug_t *)&msg->msg[0];
+ struct vcpu_hotplug_handler_t *handler = &vcpu_hotplug_handler;
+ ssize_t ret;
+
+ if ( msg->length != sizeof(vcpu_hotplug_t) )
+ goto parse_error;
+
+ /* grab target vcpu from msg */
+ handler->vcpu = req->vcpu;
+
+ /* determine which function to call based on msg subtype */
+ switch ( msg->subtype ) {
+ case CMSG_VCPU_HOTPLUG_OFF:
+ handler->fn = (void *)&cpu_down;
+ ret = schedule_work(&vcpu_hotplug_work);
+ req->status = (u32) ret;
+ break;
+ case CMSG_VCPU_HOTPLUG_ON:
+ handler->fn = (void *)&cpu_up;
+ ret = schedule_work(&vcpu_hotplug_work);
+ req->status = (u32) ret;
+ break;
+ default:
+ goto parse_error;
+ }
+
+ ctrl_if_send_response(msg);
+ return;
+ parse_error:
+ msg->length = 0;
+ ctrl_if_send_response(msg);
+}
+
+static int __init setup_vcpu_hotplug_event(void)
+{
+ struct vcpu_hotplug_handler_t *handler = &vcpu_hotplug_handler;
+
+ handler->fn = NULL;
+ ctrl_if_register_receiver(CMSG_VCPU_HOTPLUG, vcpu_hotplug_event_handler, 0);
+
+ return 0;
+}
+
+__initcall(setup_vcpu_hotplug_event);
+
#else /* ... !CONFIG_HOTPLUG_CPU */
int __cpu_disable(void)
{
diff -urN vcpu_down/tools/python/xen/lowlevel/xu/xu.c vcpu_cntl/tools/python/xen/lowlevel/xu/xu.c
--- vcpu_down/tools/python/xen/lowlevel/xu/xu.c 2005-06-12 22:13:37.000000000 -0500
+++ vcpu_cntl/tools/python/xen/lowlevel/xu/xu.c 2005-06-13 13:41:55.000000000 -0500
@@ -744,6 +744,14 @@
C2P(mem_request_t, target, Int, Long);
C2P(mem_request_t, status, Int, Long);
return dict;
+ case TYPE(CMSG_VCPU_HOTPLUG, CMSG_VCPU_HOTPLUG_OFF):
+ C2P(vcpu_hotplug_t, vcpu, Int, Long);
+ C2P(vcpu_hotplug_t, status, Int, Long);
+ return dict;
+ case TYPE(CMSG_VCPU_HOTPLUG, CMSG_VCPU_HOTPLUG_ON):
+ C2P(vcpu_hotplug_t, vcpu, Int, Long);
+ C2P(vcpu_hotplug_t, status, Int, Long);
+ return dict;
}
return PyString_FromStringAndSize((char *)xum->msg.msg, xum->msg.length);
@@ -909,6 +917,14 @@
case TYPE(CMSG_MEM_REQUEST, CMSG_MEM_REQUEST_SET):
P2C(mem_request_t, target, u32);
break;
+ case TYPE(CMSG_VCPU_HOTPLUG, CMSG_VCPU_HOTPLUG_OFF):
+ P2C(vcpu_hotplug_t, vcpu, u32);
+ P2C(vcpu_hotplug_t, status, u32);
+ break;
+ case TYPE(CMSG_VCPU_HOTPLUG, CMSG_VCPU_HOTPLUG_ON):
+ P2C(vcpu_hotplug_t, vcpu, u32);
+ P2C(vcpu_hotplug_t, status, u32);
+ break;
case TYPE(CMSG_USBIF_FE, CMSG_USBIF_FE_INTERFACE_STATUS_CHANGED):
P2C(usbif_fe_interface_status_changed_t, status, u32);
P2C(usbif_fe_interface_status_changed_t, evtchn, u16);
diff -urN vcpu_down/tools/python/xen/xend/server/messages.py vcpu_cntl/tools/python/xen/xend/server/messages.py
--- vcpu_down/tools/python/xen/xend/server/messages.py 2005-06-12 22:13:37.000000000 -0500
+++ vcpu_cntl/tools/python/xen/xend/server/messages.py 2005-06-13 13:41:55.000000000 -0500
@@ -309,6 +309,24 @@
msg_formats.update(mem_request_formats)
#============================================================================
+# Domain vcpu hotplug message.
+#============================================================================
+
+CMSG_VCPU_HOTPLUG = 10
+CMSG_VCPU_HOTPLUG_OFF = 0
+CMSG_VCPU_HOTPLUG_ON = 1
+
+vcpu_hotplug_formats = {
+ 'vcpu_hotplug_off_t':
+ (CMSG_VCPU_HOTPLUG, CMSG_VCPU_HOTPLUG_OFF),
+
+ 'vcpu_hotplug_on_t':
+ (CMSG_VCPU_HOTPLUG, CMSG_VCPU_HOTPLUG_ON)
+ }
+
+msg_formats.update(vcpu_hotplug_formats)
+
+#============================================================================
class Msg:
pass
diff -urN vcpu_down/tools/python/xen/xend/server/SrvDomain.py vcpu_cntl/tools/python/xen/xend/server/SrvDomain.py
--- vcpu_down/tools/python/xen/xend/server/SrvDomain.py 2005-06-12 22:13:41.000000000 -0500
+++ vcpu_cntl/tools/python/xen/xend/server/SrvDomain.py 2005-06-13 16:53:32.265760766 -0500
@@ -180,6 +180,14 @@
val = fn(req.args, {'dom': self.dom.id})
return val
+ def op_vcpu_hotplug(self, op, req):
+ fn = FormFn(self.xd.domain_vcpu_hotplug,
+ [['dom', 'int'],
+ ['vcpu', 'int'],
+ ['state', 'int']])
+ val = fn(req.args, {'dom': self.dom.id})
+ return val
+
def render_POST(self, req):
return self.perform(req)
diff -urN vcpu_down/tools/python/xen/xend/XendClient.py vcpu_cntl/tools/python/xen/xend/XendClient.py
--- vcpu_down/tools/python/xen/xend/XendClient.py 2005-06-12 22:13:36.000000000 -0500
+++ vcpu_cntl/tools/python/xen/xend/XendClient.py 2005-06-13 13:41:55.000000000 -0500
@@ -271,6 +271,12 @@
'target' : mem_target })
return val
+ def xend_domain_vcpu_hotplug(self, id, vcpu, state):
+ return self.xendPost(self.domainurl(id),
+ {'op' : 'vcpu_hotplug',
+ 'vcpu' : vcpu,
+ 'state' : state })
+
def xend_domain_vif_limit(self, id, vif, credit, period):
return self.xendPost(self.domainurl(id),
{ 'op' : 'vif_limit_set',
diff -urN vcpu_down/tools/python/xen/xend/XendDomainInfo.py vcpu_cntl/tools/python/xen/xend/XendDomainInfo.py
--- vcpu_down/tools/python/xen/xend/XendDomainInfo.py 2005-06-12 22:13:43.000000000 -0500
+++ vcpu_cntl/tools/python/xen/xend/XendDomainInfo.py 2005-06-13 13:41:55.000000000 -0500
@@ -949,6 +949,18 @@
msg = messages.packMsg('mem_request_t', { 'target' : target * (1 << 8)} )
self.channel.writeRequest(msg)
+ def vcpu_hotplug(self, vcpu, state):
+ """Disable or enable VCPU in domain.
+ """
+ if self.channel:
+ if int(state) == 0:
+ msg = messages.packMsg('vcpu_hotplug_off_t', { 'vcpu' : vcpu} )
+ else:
+ msg = messages.packMsg('vcpu_hotplug_on_t', { 'vcpu' : vcpu} )
+
+ self.channel.writeRequest(msg)
+
+
def shutdown(self, reason, key=0):
msgtype = shutdown_messages.get(reason)
if not msgtype:
diff -urN vcpu_down/tools/python/xen/xend/XendDomain.py vcpu_cntl/tools/python/xen/xend/XendDomain.py
--- vcpu_down/tools/python/xen/xend/XendDomain.py 2005-06-12 22:13:43.000000000 -0500
+++ vcpu_cntl/tools/python/xen/xend/XendDomain.py 2005-06-13 13:45:29.000000000 -0500
@@ -710,6 +710,18 @@
dominfo = self.domain_lookup(id)
return dominfo.mem_target_set(mem)
+ def domain_vcpu_hotplug(self, id, vcpu, state):
+ """Enable or disable VCPU vcpu in DOM id
+
+ @param id: domain
+ @param vcpu: target VCPU in domain
+ @param state: which state VCPU will become
+ @return: 0 on success, -1 on error
+ """
+
+ dominfo = self.domain_lookup(id)
+ return dominfo.vcpu_hotplug(vcpu, state)
+
def domain_dumpcore(self, id):
"""Save a core dump for a crashed domain.
diff -urN vcpu_down/tools/python/xen/xm/main.py vcpu_cntl/tools/python/xen/xm/main.py
--- vcpu_down/tools/python/xen/xm/main.py 2005-06-13 14:46:16.000000000 -0500
+++ vcpu_cntl/tools/python/xen/xm/main.py 2005-06-13 17:28:55.159285868 -0500
@@ -571,6 +571,36 @@
xm.prog(ProgBalloon)
+
+class ProgVcpuhotplug(Prog):
+ group = 'domain'
+ name = 'vcpu_hotplug'
+ info = """Enable or disable a VCPU in a domain."""
+
+ def help(self, args):
+ print args[0], "DOM VCPU [0|1]"
+ print """\nRequest virtual processor VCPU to be disabled or enabled in
+domain DOM"""
+
+ def main(self, args):
+ if len(args) != 4: self.err("%s: Invalid arguments(s)" % args[0])
+ dom = args[1]
+ vcpu = args[2]
+ state = args[3]
+ info = server.xend_domain(dom)
+ vcpu_to_cpu = sxp.child_value(info, 'vcpu_to_cpu', '-1').replace('-1','#')
+ # only send state change if states differ
+ try:
+ # (down going up) or (up going down)
+ if (vcpu_to_cpu[int(vcpu)] == "#" and state == "1") or \
+ (vcpu_to_cpu[int(vcpu)] != "#" and state == "0"):
+ server.xend_domain_vcpu_hotplug(int(dom), int(vcpu), int(state))
+ except IndexError:
+ print "Invalid VCPU(%s)"%(vcpu)
+
+xm.prog(ProgVcpuhotplug)
+
+
class ProgDomid(Prog):
group = 'domain'
name = 'domid'
diff -urN vcpu_down/xen/include/public/io/domain_controller.h vcpu_cntl/xen/include/public/io/domain_controller.h
--- vcpu_down/xen/include/public/io/domain_controller.h 2005-06-12 22:13:41.000000000 -0500
+++ vcpu_cntl/xen/include/public/io/domain_controller.h 2005-06-13 13:41:55.000000000 -0500
@@ -61,6 +61,7 @@
#define CMSG_MEM_REQUEST 7 /* Memory reservation reqs */
#define CMSG_USBIF_BE 8 /* USB controller backend */
#define CMSG_USBIF_FE 9 /* USB controller frontend */
+#define CMSG_VCPU_HOTPLUG 10 /* Hotplug VCPU messages */
/******************************************************************************
* CONSOLE DEFINITIONS
@@ -758,6 +759,25 @@
} PACKED shutdown_sysrq_t; /* 4 bytes */
/******************************************************************************
+ * VCPU HOTPLUG CONTROLS
+ */
+
+/*
+ * Subtypes for shutdown messages.
+ */
+#define CMSG_VCPU_HOTPLUG_OFF 0 /* turn vcpu off */
+#define CMSG_VCPU_HOTPLUG_ON 1 /* turn vcpu on */
+
+/*
+ * CMSG_VCPU_HOTPLUG:
+ * Indicate which vcpu's state should change
+ */
+typedef struct {
+ u32 vcpu; /* 0: VCPU's whose state will change */
+ u32 status; /* 4: Return code indicates success or failure. */
+} PACKED vcpu_hotplug_t;
+
+/******************************************************************************
* MEMORY CONTROLS
*/
^ permalink raw reply [flat|nested] 5+ messages in thread
* Re: [PATCH][RESUBMIT] add dom0 vcpu hotplug control
2005-06-14 16:37 ` [PATCH][RESUBMIT] " Ryan Harper
@ 2005-06-21 11:02 ` Christian Limpach
2005-06-21 14:14 ` Ryan Harper
2005-06-22 21:01 ` Ryan Harper
0 siblings, 2 replies; 5+ messages in thread
From: Christian Limpach @ 2005-06-21 11:02 UTC (permalink / raw)
To: Ryan Harper; +Cc: xen-devel
On 6/14/05, Ryan Harper <ryanh@us.ibm.com> wrote:
> * Ryan Harper <ryanh@us.ibm.com> [2005-06-07 17:05]:
> > This patch adds new control messages for vcpu hotplug events. Via the
> > xm vcpu_hotplug sub-program, VCPUS in domains can be enabled/disabled
> > when CONFIG_HOTPLUG_CPU is enabled in the target domain's kernel.
> >
>
> Updated this patch to use vcpu_to_cpu up/down info to control whether
> state changes are sent when hotplugging vcpus.
I've applied both patches. I think there's a flaw in the way how
vcpu_to_cpu gets passed up to python, the -1 -> # hack is kind of
gross and it won't work for cpus > 9?
Also the whole hotplug stuff is still missing interrupt re-routing
when a vcpu is taken down. To do this, we need an evtchn operation to
change the vcpu affinity of a port by changing notify_vcpu_id.
christian
^ permalink raw reply [flat|nested] 5+ messages in thread
* Re: [PATCH][RESUBMIT] add dom0 vcpu hotplug control
2005-06-21 11:02 ` Christian Limpach
@ 2005-06-21 14:14 ` Ryan Harper
2005-06-22 21:01 ` Ryan Harper
1 sibling, 0 replies; 5+ messages in thread
From: Ryan Harper @ 2005-06-21 14:14 UTC (permalink / raw)
To: Christian.Limpach; +Cc: Ryan Harper, xen-devel
* Christian Limpach <christian.limpach@gmail.com> [2005-06-21 09:11]:
> On 6/14/05, Ryan Harper <ryanh@us.ibm.com> wrote:
> > * Ryan Harper <ryanh@us.ibm.com> [2005-06-07 17:05]:
> > > This patch adds new control messages for vcpu hotplug events. Via the
> > > xm vcpu_hotplug sub-program, VCPUS in domains can be enabled/disabled
> > > when CONFIG_HOTPLUG_CPU is enabled in the target domain's kernel.
> > >
> >
> > Updated this patch to use vcpu_to_cpu up/down info to control whether
> > state changes are sent when hotplugging vcpus.
>
> I've applied both patches. I think there's a flaw in the way how
> vcpu_to_cpu gets passed up to python, the -1 -> # hack is kind of
> gross and it won't work for cpus > 9?
>
> Also the whole hotplug stuff is still missing interrupt re-routing
> when a vcpu is taken down. To do this, we need an evtchn operation to
> change the vcpu affinity of a port by changing notify_vcpu_id.
OK, I'll take a look and address the issues.
Thanks.
--
Ryan Harper
Software Engineer; Linux Technology Center
IBM Corp., Austin, Tx
(512) 838-9253 T/L: 678-9253
ryanh@us.ibm.com
^ permalink raw reply [flat|nested] 5+ messages in thread
* Re: [PATCH][RESUBMIT] add dom0 vcpu hotplug control
2005-06-21 11:02 ` Christian Limpach
2005-06-21 14:14 ` Ryan Harper
@ 2005-06-22 21:01 ` Ryan Harper
1 sibling, 0 replies; 5+ messages in thread
From: Ryan Harper @ 2005-06-22 21:01 UTC (permalink / raw)
To: Christian.Limpach; +Cc: Ryan Harper, xen-devel
* Christian Limpach <christian.limpach@gmail.com> [2005-06-21 09:11]:
> On 6/14/05, Ryan Harper <ryanh@us.ibm.com> wrote:
> > * Ryan Harper <ryanh@us.ibm.com> [2005-06-07 17:05]:
> > > This patch adds new control messages for vcpu hotplug events. Via the
> > > xm vcpu_hotplug sub-program, VCPUS in domains can be enabled/disabled
> > > when CONFIG_HOTPLUG_CPU is enabled in the target domain's kernel.
> > >
> >
> > Updated this patch to use vcpu_to_cpu up/down info to control whether
> > state changes are sent when hotplugging vcpus.
>
> I've applied both patches. I think there's a flaw in the way how
> vcpu_to_cpu gets passed up to python, the -1 -> # hack is kind of
> gross and it won't work for cpus > 9?
I updated the vcpu_to_cpu string creation to include a field separator,
which gets rid of the -1 -> # hack and works for cpus > 9.
I ran into some issues with stale vcpu_to_cpu lists when running the
hotplug subprogram. I would take a vcpu offline, and then issue the
command to bring it back and the vcpu_to_cpu list would not have changed
to indicate the the vcpu actually went down. If I injected a xm list -v
(which always showed the correct mapping) then subsequent hotplug
commands would see the state change and fire off the hotplug request. I
don't know that not sending the event when not changing state saves that
much work so I took the state check out and now just send the hotplug
event directly.
> Also the whole hotplug stuff is still missing interrupt re-routing
> when a vcpu is taken down. To do this, we need an evtchn operation to
> change the vcpu affinity of a port by changing notify_vcpu_id.
I don't fully understand all of the mappings that are happening, so this
part of the patch might be way off. In any case, I've added a new
evtchn op to set the notify_vcpu_id field of a channel. I updated the
HOTPLUG_CPU code to use the new routines when bringing cpus up and down.
When taking down a cpu, I route the IPI irq channels to CPU 0, and when
the cpu comes up, it re-routes the channels back to the awakened CPU.
--
Ryan Harper
Software Engineer; Linux Technology Center
IBM Corp., Austin, Tx
(512) 838-9253 T/L: 678-9253
ryanh@us.ibm.com
diffstat output:
linux-2.6.11-xen-sparse/arch/xen/i386/kernel/smpboot.c | 19 +++++++---
linux-2.6.11-xen-sparse/arch/xen/kernel/evtchn.c | 32 +++++++++++++++++
tools/python/xen/xend/XendDomainInfo.py | 6 ++-
tools/python/xen/xm/main.py | 18 +--------
xen/common/event_channel.c | 29 +++++++++++++++
xen/include/public/event_channel.h | 10 ++++-
6 files changed, 91 insertions(+), 23 deletions(-)
Signed-off-by: Ryan Harper <ryanh@us.ibm.com>
---
diff -urN b/linux-2.6.11-xen-sparse/arch/xen/i386/kernel/smpboot.c c/linux-2.6.11-xen-sparse/arch/xen/i386/kernel/smpboot.c
--- b/linux-2.6.11-xen-sparse/arch/xen/i386/kernel/smpboot.c 2005-06-21 22:10:55.000000000 -0500
+++ c/linux-2.6.11-xen-sparse/arch/xen/i386/kernel/smpboot.c 2005-06-22 14:31:48.765503988 -0500
@@ -103,6 +103,11 @@
DEFINE_PER_CPU(int, cpu_state) = { 0 };
#endif
+static DEFINE_PER_CPU(int, resched_irq);
+static DEFINE_PER_CPU(int, callfunc_irq);
+static char resched_name[NR_CPUS][15];
+static char callfunc_name[NR_CPUS][15];
+
#if 0
/*
* Currently trivial. Write the real->protected mode
@@ -1328,6 +1333,10 @@
while (!cpu_online(cpu))
cpu_relax();
+ /* re-route bound IRQs 0 to cpu */
+ rebind_evtchn_from_irq(0, cpu, per_cpu(resched_irq, cpu));
+ rebind_evtchn_from_irq(0, cpu, per_cpu(callfunc_irq, cpu));
+
fixup_irqs(cpu_online_map);
/* counter the disable in fixup_irqs() */
local_irq_enable();
@@ -1357,6 +1366,11 @@
cpu_clear(cpu, map);
fixup_irqs(map);
+
+ /* re-route IRQs from dead vcpu to another */
+ rebind_evtchn_from_irq(cpu, 0, per_cpu(resched_irq, cpu));
+ rebind_evtchn_from_irq(cpu, 0, per_cpu(callfunc_irq, cpu));
+
/* It's now safe to remove this processor from the online map */
cpu_clear(cpu, cpu_online_map);
@@ -1514,11 +1528,6 @@
extern irqreturn_t smp_reschedule_interrupt(int, void *, struct pt_regs *);
extern irqreturn_t smp_call_function_interrupt(int, void *, struct pt_regs *);
-static DEFINE_PER_CPU(int, resched_irq);
-static DEFINE_PER_CPU(int, callfunc_irq);
-static char resched_name[NR_CPUS][15];
-static char callfunc_name[NR_CPUS][15];
-
void __init smp_intr_init(void)
{
int cpu = smp_processor_id();
diff -urN b/linux-2.6.11-xen-sparse/arch/xen/kernel/evtchn.c c/linux-2.6.11-xen-sparse/arch/xen/kernel/evtchn.c
--- b/linux-2.6.11-xen-sparse/arch/xen/kernel/evtchn.c 2005-06-21 22:10:53.000000000 -0500
+++ c/linux-2.6.11-xen-sparse/arch/xen/kernel/evtchn.c 2005-06-22 14:31:48.767503708 -0500
@@ -271,6 +271,38 @@
return irq;
}
+void rebind_evtchn_from_ipi(int cpu, int newcpu, int ipi)
+{
+ evtchn_op_t op;
+ int evtchn = per_cpu(ipi_to_evtchn, cpu)[ipi];
+
+ spin_lock(&irq_mapping_update_lock);
+
+ op.cmd = EVTCHNOP_rebind;
+ op.u.rebind.port = evtchn;
+ op.u.rebind.vcpu = newcpu;
+ if ( HYPERVISOR_event_channel_op(&op) != 0 )
+ printk(KERN_INFO "Failed to rebind IPI%d to CPU%d\n",ipi,newcpu);
+
+ spin_unlock(&irq_mapping_update_lock);
+}
+
+void rebind_evtchn_from_irq(int cpu, int newcpu, int irq)
+{
+ evtchn_op_t op;
+ int evtchn = irq_to_evtchn[irq];
+
+ spin_lock(&irq_mapping_update_lock);
+
+ op.cmd = EVTCHNOP_rebind;
+ op.u.rebind.port = evtchn;
+ op.u.rebind.vcpu = newcpu;
+ if ( HYPERVISOR_event_channel_op(&op) != 0 )
+ printk(KERN_INFO "Failed to rebind IRQ%d to CPU%d\n",irq,newcpu);
+
+ spin_unlock(&irq_mapping_update_lock);
+}
+
void unbind_ipi_on_cpu_from_irq(int cpu, int ipi)
{
evtchn_op_t op;
diff -urN b/tools/python/xen/xend/XendDomainInfo.py c/tools/python/xen/xend/XendDomainInfo.py
--- b/tools/python/xen/xend/XendDomainInfo.py 2005-06-21 22:10:54.000000000 -0500
+++ c/tools/python/xen/xend/XendDomainInfo.py 2005-06-22 14:31:48.000000000 -0500
@@ -423,8 +423,10 @@
sxpr.append(['cpu_time', self.info['cpu_time']/1e9])
sxpr.append(['vcpus', self.info['vcpus']])
sxpr.append(['cpumap', self.info['cpumap']])
- sxpr.append(['vcpu_to_cpu', ''.join(map(lambda x: str(x),
- self.info['vcpu_to_cpu'][0:self.info['vcpus']]))])
+ # build a string, using '|' to seperate items, show only up
+ # to number of vcpus in domain, and trim the trailing '|'
+ sxpr.append(['vcpu_to_cpu', ''.join(map(lambda x: str(x)+'|',
+ self.info['vcpu_to_cpu'][0:self.info['vcpus']]))[:-1]])
if self.start_time:
up_time = time.time() - self.start_time
diff -urN b/tools/python/xen/xm/main.py c/tools/python/xen/xm/main.py
--- b/tools/python/xen/xm/main.py 2005-06-21 22:10:53.000000000 -0500
+++ c/tools/python/xen/xm/main.py 2005-06-22 14:54:50.810258798 -0500
@@ -410,8 +410,7 @@
print 'Name Id VCPU CPU CPUMAP'
for dom in doms:
info = server.xend_domain(dom)
- # XXX this is quite broken for cpu's > 9
- vcpu_to_cpu = sxp.child_value(info, 'vcpu_to_cpu', '?').replace('-1','#')
+ vcpu_to_cpu = sxp.child_value(info, 'vcpu_to_cpu', '-1').split('|')
cpumap = sxp.child_value(info, 'cpumap', [])
mask = ((int(sxp.child_value(info, 'vcpus', '0')))**2) - 1
count = 0
@@ -420,10 +419,7 @@
d['name'] = sxp.child_value(info, 'name', '??')
d['dom'] = int(sxp.child_value(info, 'id', '-1'))
d['vcpu'] = int(count)
- if cpu == "#":
- d['cpu'] = int("-1")
- else:
- d['cpu'] = int(cpu)
+ d['cpu'] = int(cpu)
d['cpumap'] = int(cpumap[count])&mask
count = count + 1
print ("%(name)-16s %(dom)3d %(vcpu)4d %(cpu)3d 0x%(cpumap)x" % d)
@@ -593,15 +589,7 @@
state = int(args[3])
dom = server.xend_domain(name)
id = sxp.child_value(dom, 'id')
- vcpu_to_cpu = sxp.child_value(dom, 'vcpu_to_cpu', '-1')
- # only send state change if states differ
- try:
- # (down going up) or (up going down)
- if (vcpu_to_cpu[vcpu] == "-1" and state == 1) or \
- (vcpu_to_cpu[vcpu] != "-1" and state == 0):
- server.xend_domain_vcpu_hotplug(id, vcpu, state)
- except IndexError:
- print "Invalid VCPU(%d)"%(vcpu)
+ server.xend_domain_vcpu_hotplug(id, vcpu, state)
xm.prog(ProgVcpuhotplug)
diff -urN b/xen/common/event_channel.c c/xen/common/event_channel.c
--- b/xen/common/event_channel.c 2005-06-21 22:10:55.000000000 -0500
+++ c/xen/common/event_channel.c 2005-06-22 14:55:20.936237917 -0500
@@ -579,6 +579,29 @@
return rc;
}
+static long evtchn_rebind(evtchn_rebind_t *bind)
+{
+ struct domain *d = current->domain;
+ int port = bind->port;
+ int vcpu = bind->vcpu;
+ struct evtchn *chn;
+ long rc = 0;
+
+ spin_lock(&d->evtchn_lock);
+
+ if ( !port_is_valid(d, port) )
+ {
+ rc = -EINVAL;
+ goto out;
+ }
+
+ chn = evtchn_from_port(d, port);
+ chn->notify_vcpu_id = vcpu;
+
+ out:
+ spin_unlock(&d->evtchn_lock);
+ return rc;
+}
long do_event_channel_op(evtchn_op_t *uop)
{
@@ -637,6 +660,12 @@
rc = -EFAULT;
break;
+ case EVTCHNOP_rebind:
+ rc = evtchn_rebind(&op.u.rebind);
+ if ( (rc == 0) && (copy_to_user(uop, &op, sizeof(op)) != 0) )
+ rc = -EFAULT;
+ break;
+
default:
rc = -ENOSYS;
break;
diff -urN b/xen/include/public/event_channel.h c/xen/include/public/event_channel.h
--- b/xen/include/public/event_channel.h 2005-06-21 22:10:55.000000000 -0500
+++ c/xen/include/public/event_channel.h 2005-06-22 14:31:48.801498950 -0500
@@ -162,6 +162,13 @@
} PACKED u;
} PACKED evtchn_status_t; /* 20 bytes */
+#define EVTCHNOP_rebind 8
+typedef struct {
+ /* IN parameters. */
+ u32 port; /* 0 */
+ u32 vcpu; /* 4 */
+} PACKED evtchn_rebind_t; /* 8 bytes */
+
typedef struct {
u32 cmd; /* EVTCHNOP_* */ /* 0 */
u32 __reserved; /* 4 */
@@ -174,7 +181,8 @@
evtchn_close_t close;
evtchn_send_t send;
evtchn_status_t status;
- u8 __dummy[24];
+ evtchn_rebind_t rebind;
+ u8 __dummy[16];
} PACKED u;
} PACKED evtchn_op_t; /* 32 bytes */
^ permalink raw reply [flat|nested] 5+ messages in thread
end of thread, other threads:[~2005-06-22 21:01 UTC | newest]
Thread overview: 5+ messages (download: mbox.gz follow: Atom feed
-- links below jump to the message on this page --
2005-06-07 22:02 [PATCH] add dom0 vcpu hotplug control Ryan Harper
2005-06-14 16:37 ` [PATCH][RESUBMIT] " Ryan Harper
2005-06-21 11:02 ` Christian Limpach
2005-06-21 14:14 ` Ryan Harper
2005-06-22 21:01 ` Ryan Harper
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.