* Re: question on virtio
From: Anthony Liguori @ 2010-05-05 19:40 UTC (permalink / raw)
To: Michael S. Tsirkin; +Cc: qemu-devel, kvm, virtualization
In-Reply-To: <20100505110947.GA27872@redhat.com>
On 05/05/2010 06:09 AM, Michael S. Tsirkin wrote:
> Hi!
> I see this in virtio_ring.c:
>
> /* Put entry in available array (but don't update avail->idx *
> until they do sync). */
>
> Why is it done this way?
> It seems that updating the index straight away would be simpler, while
> this might allow the host to specilatively look up the buffer and handle
> it, without waiting for the kick.
>
It should be okay as long as you don't update idx for partial vectors.
Regards,
Anthony Liguori
^ permalink raw reply
* [PATCH 1/1] staging: hv: Add Time Sync feature to hv_utils module
From: Haiyang Zhang @ 2010-05-05 19:23 UTC (permalink / raw)
To: linux-kernel@vger.kernel.org, devel@driverdev.osuosl.org,
virtualization@lists.osdl.org
Cc: Hank Janssen
[-- Attachment #1: Type: text/plain, Size: 6013 bytes --]
From: Haiyang Zhang <haiyangz@microsoft.com>
Subject: Add Time Sync feature to hv_utils module.
The Time Sync feature synchronizes guest time to host UTC time after reboot,
and restore from saved/paused state.
Cc: Greg Kroah-Hartman <gregkh@suse.de>
Signed-off-by: Hank Janssen <hjanssen@microsoft.com>
Signed-off-by: Haiyang Zhang <haiyangz@microsoft.com>
---
drivers/staging/hv/ChannelMgmt.c | 24 ++++++++++-
drivers/staging/hv/hyperv_utils.c | 84 +++++++++++++++++++++++++++++++++++++
drivers/staging/hv/utils.h | 25 ++++++++++-
3 files changed, 130 insertions(+), 3 deletions(-)
diff --git a/drivers/staging/hv/ChannelMgmt.c b/drivers/staging/hv/ChannelMgmt.c
index 445506d..71fe8dd 100644
--- a/drivers/staging/hv/ChannelMgmt.c
+++ b/drivers/staging/hv/ChannelMgmt.c
@@ -33,8 +33,8 @@ struct vmbus_channel_message_table_entry {
void (*messageHandler)(struct vmbus_channel_message_header *msg);
};
-#define MAX_MSG_TYPES 1
-#define MAX_NUM_DEVICE_CLASSES_SUPPORTED 5
+#define MAX_MSG_TYPES 2
+#define MAX_NUM_DEVICE_CLASSES_SUPPORTED 6
static const struct hv_guid
gSupportedDeviceClasses[MAX_NUM_DEVICE_CLASSES_SUPPORTED] = {
@@ -81,6 +81,14 @@ static const struct hv_guid
0x81, 0x8B, 0x38, 0XD9, 0x0C, 0xED, 0x39, 0xDB
}
},
+ /* {9527E630-D0AE-497b-ADCE-E80AB0175CAF} */
+ /* TimeSync */
+ {
+ .data = {
+ 0x30, 0xe6, 0x27, 0x95, 0xae, 0xd0, 0x7b, 0x49,
+ 0xad, 0xce, 0xe8, 0x0a, 0xb0, 0x17, 0x5c, 0xaf
+ }
+ },
};
@@ -191,6 +199,18 @@ struct hyperv_service_callback hv_cb_utils[MAX_MSG_TYPES] = {
.callback = chn_cb_negotiate,
.log_msg = "Shutdown channel functionality initialized"
},
+
+ /* {9527E630-D0AE-497b-ADCE-E80AB0175CAF} */
+ /* TimeSync */
+ {
+ .msg_type = HV_TIMESYNC_MSG,
+ .data = {
+ 0x30, 0xe6, 0x27, 0x95, 0xae, 0xd0, 0x7b, 0x49,
+ 0xad, 0xce, 0xe8, 0x0a, 0xb0, 0x17, 0x5c, 0xaf
+ },
+ .callback = chn_cb_negotiate,
+ .log_msg = "Timesync channel functionality initialized"
+ },
};
EXPORT_SYMBOL(hv_cb_utils);
diff --git a/drivers/staging/hv/hyperv_utils.c b/drivers/staging/hv/hyperv_utils.c
index cbebad3..9174f79 100644
--- a/drivers/staging/hv/hyperv_utils.c
+++ b/drivers/staging/hv/hyperv_utils.c
@@ -106,6 +106,82 @@ static void shutdown_onchannelcallback(void *context)
orderly_poweroff(false);
}
+
+/*
+ * Synchronize time with host after reboot, restore, etc.
+ */
+static void adj_guesttime(winfiletime_t hosttime, u8 flags)
+{
+ s64 host_tns;
+ struct timespec host_ts;
+ static s32 scnt = 50;
+
+ host_tns = (hosttime - WLTIMEDELTA) * 100;
+ host_ts = ns_to_timespec(host_tns);
+
+ if ((flags & ICTIMESYNCFLAG_SYNC) != 0) {
+ do_settimeofday(&host_ts);
+ return;
+ }
+
+ if ((flags & ICTIMESYNCFLAG_SAMPLE) != 0 &&
+ scnt > 0) {
+ scnt--;
+ do_settimeofday(&host_ts);
+ }
+
+ return;
+}
+
+/*
+ * Time Sync Channel message handler.
+ */
+static void timesync_onchannelcallback(void *context)
+{
+ struct vmbus_channel *channel = context;
+ u8 *buf;
+ u32 buflen, recvlen;
+ u64 requestid;
+ struct icmsg_hdr *icmsghdrp;
+ struct ictimesync_data *timedatap;
+
+ DPRINT_ENTER(VMBUS);
+
+ buflen = PAGE_SIZE;
+ buf = kmalloc(buflen, GFP_ATOMIC);
+
+ VmbusChannelRecvPacket(channel, buf, buflen, &recvlen, &requestid);
+
+ if (recvlen > 0) {
+ DPRINT_DBG(VMBUS, "timesync packet: recvlen=%d, requestid=%lld",
+ recvlen, requestid);
+
+ icmsghdrp = (struct icmsg_hdr *)&buf[
+ sizeof(struct vmbuspipe_hdr)];
+
+ if (icmsghdrp->icmsgtype == ICMSGTYPE_NEGOTIATE) {
+ prep_negotiate_resp(icmsghdrp, NULL, buf);
+ } else {
+ timedatap = (struct ictimesync_data *)&buf[
+ sizeof(struct vmbuspipe_hdr) +
+ sizeof(struct icmsg_hdr)];
+ adj_guesttime(timedatap->parenttime, timedatap->flags);
+ }
+
+ icmsghdrp->icflags = ICMSGHDRFLAG_TRANSACTION
+ | ICMSGHDRFLAG_RESPONSE;
+
+ VmbusChannelSendPacket(channel, buf,
+ recvlen, requestid,
+ VmbusPacketTypeDataInBand, 0);
+ }
+
+ kfree(buf);
+
+ DPRINT_EXIT(VMBUS);
+}
+
+
static int __init init_hyperv_utils(void)
{
printk(KERN_INFO "Registering HyperV Utility Driver\n");
@@ -114,6 +190,10 @@ static int __init init_hyperv_utils(void)
&shutdown_onchannelcallback;
hv_cb_utils[HV_SHUTDOWN_MSG].callback = &shutdown_onchannelcallback;
+ hv_cb_utils[HV_TIMESYNC_MSG].channel->OnChannelCallback =
+ ×ync_onchannelcallback;
+ hv_cb_utils[HV_TIMESYNC_MSG].callback = ×ync_onchannelcallback;
+
return 0;
}
@@ -124,6 +204,10 @@ static void exit_hyperv_utils(void)
hv_cb_utils[HV_SHUTDOWN_MSG].channel->OnChannelCallback =
&chn_cb_negotiate;
hv_cb_utils[HV_SHUTDOWN_MSG].callback = &chn_cb_negotiate;
+
+ hv_cb_utils[HV_TIMESYNC_MSG].channel->OnChannelCallback =
+ &chn_cb_negotiate;
+ hv_cb_utils[HV_TIMESYNC_MSG].callback = &chn_cb_negotiate;
}
module_init(init_hyperv_utils);
diff --git a/drivers/staging/hv/utils.h b/drivers/staging/hv/utils.h
index e404b21..d1d2f28 100644
--- a/drivers/staging/hv/utils.h
+++ b/drivers/staging/hv/utils.h
@@ -75,7 +75,30 @@ struct shutdown_msg_data {
u8 display_message[2048];
} __attribute__((packed));
-#define HV_SHUTDOWN_MSG 0
+
+/* Time Sync IC defs */
+#define ICTIMESYNCFLAG_PROBE 0
+#define ICTIMESYNCFLAG_SYNC 1
+#define ICTIMESYNCFLAG_SAMPLE 2
+
+#ifdef __x86_64__
+#define WLTIMEDELTA 116444736000000000L /* in 100ns unit */
+#else
+#define WLTIMEDELTA 116444736000000000LL
+#endif
+
+typedef u64 winfiletime_t; /* Windows FILETIME type */
+
+struct ictimesync_data{
+ winfiletime_t parenttime;
+ winfiletime_t childtime;
+ winfiletime_t roundtriptime;
+ u8 flags;
+} __attribute__((packed));
+
+/* Index for each IC struct in array hv_cb_utils[] */
+#define HV_SHUTDOWN_MSG 0
+#define HV_TIMESYNC_MSG 1
struct hyperv_service_callback {
u8 msg_type;
--
1.6.3.2
[-- Attachment #2: 0505-timesync-ic.patch --]
[-- Type: application/octet-stream, Size: 5800 bytes --]
From: Haiyang Zhang <haiyangz@microsoft.com>
Subject: Add Time Sync feature to hv_utils module.
The Time Sync feature synchronizes guest time to host UTC time after reboot,
and restore from saved/paused state.
Cc: Greg Kroah-Hartman <gregkh@suse.de>
Signed-off-by: Hank Janssen <hjanssen@microsoft.com>
Signed-off-by: Haiyang Zhang <haiyangz@microsoft.com>
---
drivers/staging/hv/ChannelMgmt.c | 24 ++++++++++-
drivers/staging/hv/hyperv_utils.c | 84 +++++++++++++++++++++++++++++++++++++
drivers/staging/hv/utils.h | 25 ++++++++++-
3 files changed, 130 insertions(+), 3 deletions(-)
diff --git a/drivers/staging/hv/ChannelMgmt.c b/drivers/staging/hv/ChannelMgmt.c
index 445506d..71fe8dd 100644
--- a/drivers/staging/hv/ChannelMgmt.c
+++ b/drivers/staging/hv/ChannelMgmt.c
@@ -33,8 +33,8 @@ struct vmbus_channel_message_table_entry {
void (*messageHandler)(struct vmbus_channel_message_header *msg);
};
-#define MAX_MSG_TYPES 1
-#define MAX_NUM_DEVICE_CLASSES_SUPPORTED 5
+#define MAX_MSG_TYPES 2
+#define MAX_NUM_DEVICE_CLASSES_SUPPORTED 6
static const struct hv_guid
gSupportedDeviceClasses[MAX_NUM_DEVICE_CLASSES_SUPPORTED] = {
@@ -81,6 +81,14 @@ static const struct hv_guid
0x81, 0x8B, 0x38, 0XD9, 0x0C, 0xED, 0x39, 0xDB
}
},
+ /* {9527E630-D0AE-497b-ADCE-E80AB0175CAF} */
+ /* TimeSync */
+ {
+ .data = {
+ 0x30, 0xe6, 0x27, 0x95, 0xae, 0xd0, 0x7b, 0x49,
+ 0xad, 0xce, 0xe8, 0x0a, 0xb0, 0x17, 0x5c, 0xaf
+ }
+ },
};
@@ -191,6 +199,18 @@ struct hyperv_service_callback hv_cb_utils[MAX_MSG_TYPES] = {
.callback = chn_cb_negotiate,
.log_msg = "Shutdown channel functionality initialized"
},
+
+ /* {9527E630-D0AE-497b-ADCE-E80AB0175CAF} */
+ /* TimeSync */
+ {
+ .msg_type = HV_TIMESYNC_MSG,
+ .data = {
+ 0x30, 0xe6, 0x27, 0x95, 0xae, 0xd0, 0x7b, 0x49,
+ 0xad, 0xce, 0xe8, 0x0a, 0xb0, 0x17, 0x5c, 0xaf
+ },
+ .callback = chn_cb_negotiate,
+ .log_msg = "Timesync channel functionality initialized"
+ },
};
EXPORT_SYMBOL(hv_cb_utils);
diff --git a/drivers/staging/hv/hyperv_utils.c b/drivers/staging/hv/hyperv_utils.c
index cbebad3..9174f79 100644
--- a/drivers/staging/hv/hyperv_utils.c
+++ b/drivers/staging/hv/hyperv_utils.c
@@ -106,6 +106,82 @@ static void shutdown_onchannelcallback(void *context)
orderly_poweroff(false);
}
+
+/*
+ * Synchronize time with host after reboot, restore, etc.
+ */
+static void adj_guesttime(winfiletime_t hosttime, u8 flags)
+{
+ s64 host_tns;
+ struct timespec host_ts;
+ static s32 scnt = 50;
+
+ host_tns = (hosttime - WLTIMEDELTA) * 100;
+ host_ts = ns_to_timespec(host_tns);
+
+ if ((flags & ICTIMESYNCFLAG_SYNC) != 0) {
+ do_settimeofday(&host_ts);
+ return;
+ }
+
+ if ((flags & ICTIMESYNCFLAG_SAMPLE) != 0 &&
+ scnt > 0) {
+ scnt--;
+ do_settimeofday(&host_ts);
+ }
+
+ return;
+}
+
+/*
+ * Time Sync Channel message handler.
+ */
+static void timesync_onchannelcallback(void *context)
+{
+ struct vmbus_channel *channel = context;
+ u8 *buf;
+ u32 buflen, recvlen;
+ u64 requestid;
+ struct icmsg_hdr *icmsghdrp;
+ struct ictimesync_data *timedatap;
+
+ DPRINT_ENTER(VMBUS);
+
+ buflen = PAGE_SIZE;
+ buf = kmalloc(buflen, GFP_ATOMIC);
+
+ VmbusChannelRecvPacket(channel, buf, buflen, &recvlen, &requestid);
+
+ if (recvlen > 0) {
+ DPRINT_DBG(VMBUS, "timesync packet: recvlen=%d, requestid=%lld",
+ recvlen, requestid);
+
+ icmsghdrp = (struct icmsg_hdr *)&buf[
+ sizeof(struct vmbuspipe_hdr)];
+
+ if (icmsghdrp->icmsgtype == ICMSGTYPE_NEGOTIATE) {
+ prep_negotiate_resp(icmsghdrp, NULL, buf);
+ } else {
+ timedatap = (struct ictimesync_data *)&buf[
+ sizeof(struct vmbuspipe_hdr) +
+ sizeof(struct icmsg_hdr)];
+ adj_guesttime(timedatap->parenttime, timedatap->flags);
+ }
+
+ icmsghdrp->icflags = ICMSGHDRFLAG_TRANSACTION
+ | ICMSGHDRFLAG_RESPONSE;
+
+ VmbusChannelSendPacket(channel, buf,
+ recvlen, requestid,
+ VmbusPacketTypeDataInBand, 0);
+ }
+
+ kfree(buf);
+
+ DPRINT_EXIT(VMBUS);
+}
+
+
static int __init init_hyperv_utils(void)
{
printk(KERN_INFO "Registering HyperV Utility Driver\n");
@@ -114,6 +190,10 @@ static int __init init_hyperv_utils(void)
&shutdown_onchannelcallback;
hv_cb_utils[HV_SHUTDOWN_MSG].callback = &shutdown_onchannelcallback;
+ hv_cb_utils[HV_TIMESYNC_MSG].channel->OnChannelCallback =
+ ×ync_onchannelcallback;
+ hv_cb_utils[HV_TIMESYNC_MSG].callback = ×ync_onchannelcallback;
+
return 0;
}
@@ -124,6 +204,10 @@ static void exit_hyperv_utils(void)
hv_cb_utils[HV_SHUTDOWN_MSG].channel->OnChannelCallback =
&chn_cb_negotiate;
hv_cb_utils[HV_SHUTDOWN_MSG].callback = &chn_cb_negotiate;
+
+ hv_cb_utils[HV_TIMESYNC_MSG].channel->OnChannelCallback =
+ &chn_cb_negotiate;
+ hv_cb_utils[HV_TIMESYNC_MSG].callback = &chn_cb_negotiate;
}
module_init(init_hyperv_utils);
diff --git a/drivers/staging/hv/utils.h b/drivers/staging/hv/utils.h
index e404b21..d1d2f28 100644
--- a/drivers/staging/hv/utils.h
+++ b/drivers/staging/hv/utils.h
@@ -75,7 +75,30 @@ struct shutdown_msg_data {
u8 display_message[2048];
} __attribute__((packed));
-#define HV_SHUTDOWN_MSG 0
+
+/* Time Sync IC defs */
+#define ICTIMESYNCFLAG_PROBE 0
+#define ICTIMESYNCFLAG_SYNC 1
+#define ICTIMESYNCFLAG_SAMPLE 2
+
+#ifdef __x86_64__
+#define WLTIMEDELTA 116444736000000000L /* in 100ns unit */
+#else
+#define WLTIMEDELTA 116444736000000000LL
+#endif
+
+typedef u64 winfiletime_t; /* Windows FILETIME type */
+
+struct ictimesync_data{
+ winfiletime_t parenttime;
+ winfiletime_t childtime;
+ winfiletime_t roundtriptime;
+ u8 flags;
+} __attribute__((packed));
+
+/* Index for each IC struct in array hv_cb_utils[] */
+#define HV_SHUTDOWN_MSG 0
+#define HV_TIMESYNC_MSG 1
struct hyperv_service_callback {
u8 msg_type;
--
1.6.3.2
^ permalink raw reply related
* Re: RFC: Network Plugin Architecture (NPA) for vmxnet3
From: Pankaj Thakkar @ 2010-05-05 19:00 UTC (permalink / raw)
To: Chris Wright
Cc: kvm@vger.kernel.org, pv-drivers@vmware.com,
netdev@vger.kernel.org, linux-kernel@vger.kernel.org,
virtualization@lists.linux-foundation.org
In-Reply-To: <20100505005852.GA28034@sequoia.sous-sol.org>
On Tue, May 04, 2010 at 05:58:52PM -0700, Chris Wright wrote:
> Date: Tue, 4 May 2010 17:58:52 -0700
> From: Chris Wright <chrisw@sous-sol.org>
> To: Pankaj Thakkar <pthakkar@vmware.com>
> CC: "linux-kernel@vger.kernel.org" <linux-kernel@vger.kernel.org>,
> "netdev@vger.kernel.org" <netdev@vger.kernel.org>,
> "virtualization@lists.linux-foundation.org"
> <virtualization@lists.linux-foundation.org>,
> "pv-drivers@vmware.com" <pv-drivers@vmware.com>,
> Shreyas Bhatewara <sbhatewara@vmware.com>,
> "kvm@vger.kernel.org" <kvm@vger.kernel.org>
> Subject: Re: RFC: Network Plugin Architecture (NPA) for vmxnet3
>
> * Pankaj Thakkar (pthakkar@vmware.com) wrote:
> > We intend to upgrade the upstreamed vmxnet3 driver to implement NPA so that
> > Linux users can exploit the benefits provided by passthrough devices in a
> > seamless manner while retaining the benefits of virtualization. The document
> > below tries to answer most of the questions which we anticipated. Please let us
> > know your comments and queries.
>
> How does the throughput, latency, and host CPU utilization for normal
> data path compare with say NetQueue?
NetQueue is really for scaling across multiple VMs. NPA allows similar scaling
and also helps in improving the CPU efficiency for a single VM since the
hypervisor is bypassed. Througput wise both emulation and passthrough (NPA) can
obtain line rates on 10gig but passthrough saves upto 40% cpu based on the
workload. We did a demo at IDF 2009 where we compared 8 VMs running on NetQueue
v/s 8 VMs running on NPA (using Niantic) and we obtained similar CPU efficiency
gains.
>
> And does this obsolete your UPT implementation?
NPA and UPT share a lot of code in the hypervisor. UPT was adopted only by very
limited IHVs and hence NPA is our way forward to have all IHVs onboard.
> How many cards actually support this NPA interface? What does it look
> like, i.e. where is the NPA specification? (AFAIK, we never got the UPT
> one).
We have it working internally with Intel Niantic (10G) and Kawela (1G) SR-IOV
NIC. We are also working with upcoming Broadcom 10G card and plan to support
other IHVs. This is unlike UPT so we don't dictate the register sets or rings
like we did in UPT. Rather we have guidelines like that the card should have an
embedded switch for inter VF switching or should support programming (rx
filters, VLAN, etc) though the PF driver rather than the VF driver.
> How do you handle hardware which has a more symmetric view of the
> SR-IOV world (SR-IOV is only PCI sepcification, not a network driver
> specification)? Or hardware which has multiple functions per physical
> port (multiqueue, hw filtering, embedded switch, etc.)?
I am not sure what do you mean by symmetric view of SR-IOV world?
NPA allows multi-queue VFs and requires an embedded switch currently. As far as
the PF driver is concerned we require IHVs to support all existing and upcoming
features like NetQueue, FCoE, etc. The PF driver is considered special and is
used to drive the traffic for the emulated/paravirtualized VMs and is also used
to program things on behalf of the VFs through the hypervisor. If the hardware
has multiple physical functions they are treated as separate adapters (with
their own set of VFs) and we require the embedded switch to maintain that
distinction as well.
> > NPA offers several benefits:
> > 1. Performance: Critical performance sensitive paths are not trapped and the
> > guest can directly drive the hardware without incurring virtualization
> > overheads.
>
> Can you demonstrate with data?
The setup is 2.667Ghz Nehalem server running SLES11 VM talking to a 2.33Ghz
Barcelona client box running RHEL 5.1. We had netperf streams with 16k msg size
over 64k socket size running between server VM and client and they are using
Intel Niantic 10G cards. In both cases (NPA and regular) the VM was CPU
saturated (used one full core).
TX: regular vmxnet3 = 3085.5 Mbps/GHz; NPA vmxnet3 = 4397.2 Mbps/GHz
RX: regular vmxnet3 = 1379.6 Mbps/GHz; NPA vmxnet3 = 2349.7 Mbps/GHz
We have similar results for other configuration and in general we have seen NPA
is better in terms of CPU cost and can save upto 40% of CPU cost.
>
> > 2. Hypervisor control: All control operations from the guest such as programming
> > MAC address go through the hypervisor layer and hence can be subjected to
> > hypervisor policies. The PF driver can be further used to put policy decisions
> > like which VLAN the guest should be on.
>
> This can happen without NPA as well. VF simply needs to request
> the change via the PF (in fact, hw does that right now). Also, we
> already have a host side management interface via PF (see, for example,
> RTM_SETLINK IFLA_VF_MAC interface).
>
> What is control plane interface? Just something like a fixed register set?
All operations other than TX/RX go through the vmxnet3 shell to the vmxnet3
device emulation. So the control plane is really the vmxnet3 device emulation
as far as the guest is concerned.
>
> > 3. Guest Management: No hardware specific drivers need to be installed in the
> > guest virtual machine and hence no overheads are incurred for guest management.
> > All software for the driver (including the PF driver and the plugin) is
> > installed in the hypervisor.
>
> So we have a plugin per hardware VF implementation? And the hypervisor
> injects this code into the guest?
One guest-agnostic plugin per VF implementation. Yes, the plugin is injected
into the guest by the hypervisor.
> > The plugin image is provided by the IHVs along with the PF driver and is
> > packaged in the hypervisor. The plugin image is OS agnostic and can be loaded
> > either into a Linux VM or a Windows VM. The plugin is written against the Shell
>
> And it will need to be GPL AFAICT from what you've said thus far. It
> does sound worrisome, although I suppose hw firmware isn't particularly
> different.
Yes it would be GPL and we are thinking of enforcing the license in the
hypervisor as well as in the shell.
> How does the shell switch back to emulated mode for live migration?
The hypervisor sends a notification to the shell to switch out of passthrough
and it quiesces the VF and tears down the mapping between VF and the guest. The
shell free's up the buffers and other resources on behalf of the plugin and
reinitializes the s/w vmxnet3 emulation plugin.
> Please make this shell API interface and the PF/VF requirments available.
We have an internal prototype working but we are not yet ready to post the
patch to LKML. We are still in the process of making changes to our windows
driver and want to ensure that we take into account all changes that could
happen.
Thanks,
-pankaj
^ permalink raw reply
* Re: RFC: Network Plugin Architecture (NPA) for vmxnet3
From: Avi Kivity @ 2010-05-05 17:59 UTC (permalink / raw)
To: Pankaj Thakkar; +Cc: pv-drivers, netdev, linux-kernel, virtualization
In-Reply-To: <20100504230225.GP8323@vmware.com>
On 05/05/2010 02:02 AM, Pankaj Thakkar wrote:
> 2. Hypervisor control: All control operations from the guest such as programming
> MAC address go through the hypervisor layer and hence can be subjected to
> hypervisor policies. The PF driver can be further used to put policy decisions
> like which VLAN the guest should be on.
>
Is this enforced? Since you pass the hardware through, you can't rely
on the guest actually doing this, yes?
> The plugin image is provided by the IHVs along with the PF driver and is
> packaged in the hypervisor. The plugin image is OS agnostic and can be loaded
> either into a Linux VM or a Windows VM. The plugin is written against the Shell
> API interface which the shell is responsible for implementing. The API
> interface allows the plugin to do TX and RX only by programming the hardware
> rings (along with things like buffer allocation and basic initialization). The
> virtual machine comes up in paravirtualized/emulated mode when it is booted.
> The hypervisor allocates the VF and other resources and notifies the shell of
> the availability of the VF. The hypervisor injects the plugin into memory
> location specified by the shell. The shell initializes the plugin by calling
> into a known entry point and the plugin initializes the data path. The control
> path is already initialized by the PF driver when the VF is allocated. At this
> point the shell switches to using the loaded plugin to do all further TX and RX
> operations. The guest networking stack does not participate in these operations
> and continues to function normally. All the control operations continue being
> trapped by the hypervisor and are directed to the PF driver as needed. For
> example, if the MAC address changes the hypervisor updates its internal state
> and changes the state of the embedded switch as well through the PF control
> API.
>
This is essentially a miniature network stack with a its own mini
bonding layer, mini hotplug, and mini API, except s/API/ABI/. Is this a
correct view?
If so, the Linuxy approach would be to use the ordinary drivers and the
Linux networking API, and hide the bond setup using namespaces. The
bond driver, or perhaps a new, similar, driver can be enhanced to
propagate ethtool commands to its (hidden) components, and to have a
control channel with the hypervisor.
This would make the approach hypervisor agnostic, you're just pairing
two devices and presenting them to the rest of the stack as a single device.
> We have reworked our existing Linux vmxnet3 driver to accomodate NPA by
> splitting the driver into two parts: Shell and Plugin. The new split driver is
>
So the Shell would be the reworked or new bond driver, and Plugins would
be ordinary Linux network drivers.
--
Do not meddle in the internals of kernels, for they are subtle and quick to panic.
^ permalink raw reply
* Re: [Pv-drivers] RFC: Network Plugin Architecture (NPA) for vmxnet3
From: Stephen Hemminger @ 2010-05-05 17:52 UTC (permalink / raw)
To: Dmitry Torokhov
Cc: Pankaj Thakkar, pv-drivers@vmware.com, netdev@vger.kernel.org,
linux-kernel@vger.kernel.org,
virtualization@lists.linux-foundation.org, Christoph Hellwig
In-Reply-To: <20100505173951.GA8388@infradead.org>
On Wed, 5 May 2010 13:39:51 -0400
Christoph Hellwig <hch@infradead.org> wrote:
> On Wed, May 05, 2010 at 10:35:28AM -0700, Dmitry Torokhov wrote:
> > Yes, with the exception that the only body of code that will be
> > accepted by the shell should be GPL-licensed and thus open and available
> > for examining. This is not different from having a standard kernel
> > module that is loaded normally and plugs into a certain subsystem.
> > The difference is that the binary resides not on guest filesystem
> > but elsewhere.
>
> Forget about the licensing. Loading binary blobs written to a shim
> layer is a complete pain in the ass and totally unsupportable, and
> also uninteresting because of the overhead.
>
> If you have any interesting in developing this further, do:
>
> (1) move the limited VF drivers directly into the kernel tree,
> talk to them through a normal ops vector
> (2) get rid of the whole shim crap and instead integrate the limited
> VF driver with the full VF driver we already have, instead of
> duplicating the code
> (3) don't make the PV to VF integration VMware-specific but also
> provide an open reference implementation like virtio. We're not
> going to add massive amount of infrastructure that is not actually
> useable in a free software stack.
Let me put it bluntly. Any design that allows external code to run
in the kernel is not going to be accepted. Out of tree kernel modules are enough
of a pain already, why do you expect the developers to add another
interface.
^ permalink raw reply
* RE: [Pv-drivers] RFC: Network Plugin Architecture (NPA) for vmxnet3
From: Pankaj Thakkar @ 2010-05-05 17:47 UTC (permalink / raw)
To: Christoph Hellwig, Dmitry Torokhov
Cc: pv-drivers@vmware.com, netdev@vger.kernel.org,
linux-kernel@vger.kernel.org,
virtualization@lists.linux-foundation.org
In-Reply-To: <20100505173951.GA8388@infradead.org>
> -----Original Message-----
> From: Christoph Hellwig [mailto:hch@infradead.org]
> Sent: Wednesday, May 05, 2010 10:40 AM
> To: Dmitry Torokhov
> Cc: Christoph Hellwig; pv-drivers@vmware.com; Pankaj Thakkar;
> netdev@vger.kernel.org; linux-kernel@vger.kernel.org;
> virtualization@lists.linux-foundation.org
> Subject: Re: [Pv-drivers] RFC: Network Plugin Architecture (NPA) for
> vmxnet3
>
> On Wed, May 05, 2010 at 10:35:28AM -0700, Dmitry Torokhov wrote:
> > Yes, with the exception that the only body of code that will be
> > accepted by the shell should be GPL-licensed and thus open and
> available
> > for examining. This is not different from having a standard kernel
> > module that is loaded normally and plugs into a certain subsystem.
> > The difference is that the binary resides not on guest filesystem
> > but elsewhere.
>
> Forget about the licensing. Loading binary blobs written to a shim
> layer is a complete pain in the ass and totally unsupportable, and
> also uninteresting because of the overhead.
[PT] Why do you think it is unsupportable? How different is it from any module written against a well maintained interface? What overhead are you talking about?
>
> If you have any interesting in developing this further, do:
>
> (1) move the limited VF drivers directly into the kernel tree,
> talk to them through a normal ops vector
[PT] This assumes that all the VF drivers would always be available. Also we have to support windows and our current design supports it nicely in an OS agnostic manner.
> (2) get rid of the whole shim crap and instead integrate the limited
> VF driver with the full VF driver we already have, instead of
> duplicating the code
[PT] Having a full VF driver adds a lot of dependency on the guest VM and this is what NPA tries to avoid.
> (3) don't make the PV to VF integration VMware-specific but also
> provide an open reference implementation like virtio. We're not
> going to add massive amount of infrastructure that is not actually
> useable in a free software stack.
[PT] Today this is tied to vmxnet3 device and is intended to work on ESX hypervisor only (vmxnet3 works on VMware hypervisor only). All the loading support is inside the ESX hypervisor. I am going to post the interface between the shell and the plugin soon and you can see that there is not a whole lot of dependency or infrastructure requirements from the Linux kernel. Please keep in mind that we don't use Linux as a hypervisor but as a guest VM.
^ permalink raw reply
* Re: [Pv-drivers] RFC: Network Plugin Architecture (NPA) for vmxnet3
From: Christoph Hellwig @ 2010-05-05 17:39 UTC (permalink / raw)
To: Dmitry Torokhov
Cc: Pankaj Thakkar, pv-drivers@vmware.com, netdev@vger.kernel.org,
linux-kernel@vger.kernel.org,
virtualization@lists.linux-foundation.org, Christoph Hellwig
In-Reply-To: <201005051035.29831.dtor@vmware.com>
On Wed, May 05, 2010 at 10:35:28AM -0700, Dmitry Torokhov wrote:
> Yes, with the exception that the only body of code that will be
> accepted by the shell should be GPL-licensed and thus open and available
> for examining. This is not different from having a standard kernel
> module that is loaded normally and plugs into a certain subsystem.
> The difference is that the binary resides not on guest filesystem
> but elsewhere.
Forget about the licensing. Loading binary blobs written to a shim
layer is a complete pain in the ass and totally unsupportable, and
also uninteresting because of the overhead.
If you have any interesting in developing this further, do:
(1) move the limited VF drivers directly into the kernel tree,
talk to them through a normal ops vector
(2) get rid of the whole shim crap and instead integrate the limited
VF driver with the full VF driver we already have, instead of
duplicating the code
(3) don't make the PV to VF integration VMware-specific but also
provide an open reference implementation like virtio. We're not
going to add massive amount of infrastructure that is not actually
useable in a free software stack.
^ permalink raw reply
* Re: [Pv-drivers] RFC: Network Plugin Architecture (NPA) for vmxnet3
From: Dmitry Torokhov @ 2010-05-05 17:35 UTC (permalink / raw)
To: Christoph Hellwig
Cc: pv-drivers@vmware.com, Pankaj Thakkar,
virtualization@lists.linux-foundation.org,
linux-kernel@vger.kernel.org, netdev@vger.kernel.org
In-Reply-To: <20100505173120.GA1752@infradead.org>
On Wednesday 05 May 2010 10:31:20 am Christoph Hellwig wrote:
> On Wed, May 05, 2010 at 10:29:40AM -0700, Dmitry Torokhov wrote:
> > > We're not going to add any kind of loader for binry blobs into kernel
> > > space, sorry. Don't even bother wasting your time on this.
> >
> > It would not be a binary blob but software properly released under GPL.
> > The current plan is for the shell to enforce GPL requirement on the
> > plugin code, similar to what module loaded does for regular kernel
> > modules.
>
> The mechanism described in the document is loading a binary blob
> coded to an abstract API.
Yes, with the exception that the only body of code that will be
accepted by the shell should be GPL-licensed and thus open and available
for examining. This is not different from having a standard kernel
module that is loaded normally and plugs into a certain subsystem.
The difference is that the binary resides not on guest filesystem
but elsewhere.
>
> That's something entirely different from having normal modules for
> the Virtual Functions, which we already have for various pieces of
> hardware anyway.
--
Dmitry
^ permalink raw reply
* Re: [Pv-drivers] RFC: Network Plugin Architecture (NPA) for vmxnet3
From: Christoph Hellwig @ 2010-05-05 17:31 UTC (permalink / raw)
To: Dmitry Torokhov
Cc: Pankaj Thakkar, pv-drivers, netdev@vger.kernel.org,
linux-kernel@vger.kernel.org,
virtualization@lists.linux-foundation.org, Christoph Hellwig
In-Reply-To: <201005051029.42052.dtor@vmware.com>
On Wed, May 05, 2010 at 10:29:40AM -0700, Dmitry Torokhov wrote:
> > We're not going to add any kind of loader for binry blobs into kernel
> > space, sorry. Don't even bother wasting your time on this.
> >
>
> It would not be a binary blob but software properly released under GPL.
> The current plan is for the shell to enforce GPL requirement on the
> plugin code, similar to what module loaded does for regular kernel
> modules.
The mechanism described in the document is loading a binary blob
coded to an abstract API.
That's something entirely different from having normal modules for
the Virtual Functions, which we already have for various pieces of
hardware anyway.
^ permalink raw reply
* Re: [Pv-drivers] RFC: Network Plugin Architecture (NPA) for vmxnet3
From: Dmitry Torokhov @ 2010-05-05 17:29 UTC (permalink / raw)
To: pv-drivers
Cc: Christoph Hellwig, Pankaj Thakkar,
virtualization@lists.linux-foundation.org,
linux-kernel@vger.kernel.org, netdev@vger.kernel.org
In-Reply-To: <20100505172316.GA25338@infradead.org>
On Wednesday 05 May 2010 10:23:16 am Christoph Hellwig wrote:
> On Tue, May 04, 2010 at 04:02:25PM -0700, Pankaj Thakkar wrote:
> > The plugin image is provided by the IHVs along with the PF driver and is
> > packaged in the hypervisor. The plugin image is OS agnostic and can be
> > loaded either into a Linux VM or a Windows VM. The plugin is written
> > against the Shell API interface which the shell is responsible for
> > implementing. The API
>
> We're not going to add any kind of loader for binry blobs into kernel
> space, sorry. Don't even bother wasting your time on this.
>
It would not be a binary blob but software properly released under GPL.
The current plan is for the shell to enforce GPL requirement on the
plugin code, similar to what module loaded does for regular kernel
modules.
--
Dmitry
^ permalink raw reply
* Re: RFC: Network Plugin Architecture (NPA) for vmxnet3
From: Christoph Hellwig @ 2010-05-05 17:23 UTC (permalink / raw)
To: Pankaj Thakkar; +Cc: pv-drivers, netdev, linux-kernel, virtualization
In-Reply-To: <20100504230225.GP8323@vmware.com>
On Tue, May 04, 2010 at 04:02:25PM -0700, Pankaj Thakkar wrote:
> The plugin image is provided by the IHVs along with the PF driver and is
> packaged in the hypervisor. The plugin image is OS agnostic and can be loaded
> either into a Linux VM or a Windows VM. The plugin is written against the Shell
> API interface which the shell is responsible for implementing. The API
We're not going to add any kind of loader for binry blobs into kernel
space, sorry. Don't even bother wasting your time on this.
^ permalink raw reply
* question on virtio
From: Michael S. Tsirkin @ 2010-05-05 11:09 UTC (permalink / raw)
To: Rusty Russell; +Cc: kvm, qemu-devel, Anthony Liguori, virtualization
Hi!
I see this in virtio_ring.c:
/* Put entry in available array (but don't update avail->idx *
until they do sync). */
Why is it done this way?
It seems that updating the index straight away would be simpler, while
this might allow the host to specilatively look up the buffer and handle
it, without waiting for the kick.
--
MST
^ permalink raw reply
* Re: [Qemu-devel] Re: [PATCH] virtio-spec: document block CMD and FLUSH
From: Neil Brown @ 2010-05-05 6:03 UTC (permalink / raw)
To: Rusty Russell
Cc: tytso, kvm, Michael S. Tsirkin, Jamie Lokier, qemu-devel,
virtualization, Jens Axboe, hch
In-Reply-To: <201005051428.41735.rusty@rustcorp.com.au>
On Wed, 5 May 2010 14:28:41 +0930
Rusty Russell <rusty@rustcorp.com.au> wrote:
> On Wed, 5 May 2010 05:47:05 am Jamie Lokier wrote:
> > Jens Axboe wrote:
> > > On Tue, May 04 2010, Rusty Russell wrote:
> > > > ISTR someone mentioning a desire for such an API years ago, so CC'ing the
> > > > usual I/O suspects...
> > >
> > > It would be nice to have a more fuller API for this, but the reality is
> > > that only the flush approach is really workable. Even just strict
> > > ordering of requests could only be supported on SCSI, and even there the
> > > kernel still lacks proper guarantees on error handling to prevent
> > > reordering there.
> >
> > There's a few I/O scheduling differences that might be useful:
> >
> > 1. The I/O scheduler could freely move WRITEs before a FLUSH but not
> > before a BARRIER. That might be useful for time-critical WRITEs,
> > and those issued by high I/O priority.
>
> This is only because noone actually wants flushes or barriers, though
> I/O people seem to only offer that. We really want "<these writes> must
> occur before <this write>". That offers maximum choice to the I/O subsystem
> and potentially to smart (virtual?) disks.
>
> > 2. The I/O scheduler could move WRITEs after a FLUSH if the FLUSH is
> > only for data belonging to a particular file (e.g. fdatasync with
> > no file size change, even on btrfs if O_DIRECT was used for the
> > writes being committed). That would entail tagging FLUSHes and
> > WRITEs with a fs-specific identifier (such as inode number), opaque
> > to the scheduler which only checks equality.
>
> This is closer. In userspace I'd be happy with a "all prior writes to this
> struct file before all future writes". Even if the original guarantees were
> stronger (ie. inode basis). We currently implement transactions using 4 fsync
> /msync pairs.
>
> write_recovery_data(fd);
> fsync(fd);
> msync(mmap);
> write_recovery_header(fd);
> fsync(fd);
> msync(mmap);
> overwrite_with_new_data(fd);
> fsync(fd);
> msync(mmap);
> remove_recovery_header(fd);
> fsync(fd);
> msync(mmap);
Seems over-zealous.
If the recovery_header held a strong checksum of the recovery_data you would
not need the first fsync, and as long as you have two places to write recovery
data, you don't need the 3rd and 4th syncs.
Just:
write_internally_checksummed_recovery_data_and_header_to_unused_log_space()
fsync / msync
overwrite_with_new_data()
To recovery you choose the most recent log_space and replay the content.
That may be a redundant operation, but that is no loss.
Also cannot see the point of msync if you have already performed an fsync,
and if there is a point, I would expect you to call msync before
fsync... Maybe there is some subtlety there that I am not aware of.
>
> Yet we really only need ordering, not guarantees about it actually hitting
> disk before returning.
>
> > In other words, FLUSH can be more relaxed than BARRIER inside the
> > kernel. It's ironic that we think of fsync as stronger than
> > fbarrier outside the kernel :-)
>
> It's an implementation detail; barrier has less flexibility because it has
> less information about what is required. I'm saying I want to give you as
> much information as I can, even if you don't use it yet.
Only we know that approach doesn't work.
People will learn that they don't need to give the extra information to still
achieve the same result - just like they did with ext3 and fsync.
Then when we improve the implementation to only provide the guarantees that
you asked for, people will complain that they are getting empty files that
they didn't expect.
The abstraction I would like to see is a simple 'barrier' that contains no
data and has a filesystem-wide effect.
If a filesystem wanted a 'full' barrier such as the current BIO_RW_BARRER,
it would send an empty barrier, then the data, then another empty barrier.
(However I suspect most filesystems don't really need barriers on both sides.)
A low level driver might merge these together if the underlying hardware
supported that combined operation (which I believe some do).
I think this merging would be less complex that the current need to split a
BIO_RW_BARRIER in to the three separate operations when only a flush is
possible (I know it would make md code a lot nicer :-).
I would probably expose this to user-space as extra flags to sync_file_range:
SYNC_FILE_RANGE_BARRIER_BEFORE
SYNC_FILE_RANGE_BARRIER_AFTER
This would make it clear that a barrier does *not* imply a sync, it only
applies to data for which a sync has already been requested. So data that has
already been 'synced' is stored strictly before data which has not yet been
submitted with write() (or by changing a mmapped area).
The barrier would still be filesystem wide in that if you
SYNC_FILE_WRITE_WRITE one file, then SYNC_FILE_RANGE_BARRIER_BEFORE another
file on the same filesystem, the pages scheduled in the first file would be
affect by the barrier request on the second file.
Implementing this would probably require a new address_space_operation so
that the filesystem would have a chance to ensure all necessary writes were
queued before issuing the barrier.
NeilBrown
^ permalink raw reply
* Re: [PATCH] virtio-spec: document block CMD and FLUSH
From: Rusty Russell @ 2010-05-05 5:00 UTC (permalink / raw)
To: Christoph Hellwig
Cc: kvm, virtualization, qemu-devel, Anthony Liguori,
Michael S. Tsirkin
In-Reply-To: <20100504185459.GA24998@lst.de>
On Wed, 5 May 2010 04:24:59 am Christoph Hellwig wrote:
> On Fri, Feb 19, 2010 at 12:22:20AM +0200, Michael S. Tsirkin wrote:
> > I took a stub at documenting CMD and FLUSH request types in virtio
> > block. Christoph, could you look over this please?
> >
> > I note that the interface seems full of warts to me,
> > this might be a first step to cleaning them.
>
> The whole virtio-blk interface is full of warts. It has been
> extended rather ad-hoc, so that is rather expected.
>
> > One issue I struggled with especially is how type
> > field mixes bits and non-bit values. I ended up
> > simply defining all legal values, so that we have
> > CMD = 2, CMD_OUT = 3 and so on.
>
> It's basically a complete mess without much logic behind it.
>
> > +\change_unchanged
> > +the high bit
> > +\change_inserted 0 1266497301
> > + (VIRTIO_BLK_T_BARRIER)
> > +\change_unchanged
> > + indicates that this request acts as a barrier and that all preceeding requests
> > + must be complete before this one, and all following requests must not be
> > + started until this is complete.
> > +
> > +\change_inserted 0 1266504385
> > + Note that a barrier does not flush caches in the underlying backend device
> > + in host, and thus does not serve as data consistency guarantee.
> > + Driver must use FLUSH request to flush the host cache.
> > +\change_unchanged
>
> I'm not sure it's even worth documenting it. I can't see any way to
> actually implement safe behaviour with the VIRTIO_BLK_T_BARRIER-style
> barriers.
>
>
> Btw, did I mention that .lyx is a a really horrible format to review
> diffs for? Plain latex would be a lot better..
Yeah, or export as text post that diff for content review. I do like the
patches to the lyx source though (I check all versions into revision control,
before and after merging changes, which makes it easy to produce annotated
versions).
Cheers,
Rusty.
>
^ permalink raw reply
* Re: [Qemu-devel] Re: [PATCH] virtio-spec: document block CMD and FLUSH
From: Rusty Russell @ 2010-05-05 4:58 UTC (permalink / raw)
To: Jamie Lokier
Cc: tytso, kvm, Michael S. Tsirkin, Neil Brown, qemu-devel,
virtualization, Jens Axboe, hch
In-Reply-To: <20100504201705.GA4360@shareable.org>
On Wed, 5 May 2010 05:47:05 am Jamie Lokier wrote:
> Jens Axboe wrote:
> > On Tue, May 04 2010, Rusty Russell wrote:
> > > ISTR someone mentioning a desire for such an API years ago, so CC'ing the
> > > usual I/O suspects...
> >
> > It would be nice to have a more fuller API for this, but the reality is
> > that only the flush approach is really workable. Even just strict
> > ordering of requests could only be supported on SCSI, and even there the
> > kernel still lacks proper guarantees on error handling to prevent
> > reordering there.
>
> There's a few I/O scheduling differences that might be useful:
>
> 1. The I/O scheduler could freely move WRITEs before a FLUSH but not
> before a BARRIER. That might be useful for time-critical WRITEs,
> and those issued by high I/O priority.
This is only because noone actually wants flushes or barriers, though
I/O people seem to only offer that. We really want "<these writes> must
occur before <this write>". That offers maximum choice to the I/O subsystem
and potentially to smart (virtual?) disks.
> 2. The I/O scheduler could move WRITEs after a FLUSH if the FLUSH is
> only for data belonging to a particular file (e.g. fdatasync with
> no file size change, even on btrfs if O_DIRECT was used for the
> writes being committed). That would entail tagging FLUSHes and
> WRITEs with a fs-specific identifier (such as inode number), opaque
> to the scheduler which only checks equality.
This is closer. In userspace I'd be happy with a "all prior writes to this
struct file before all future writes". Even if the original guarantees were
stronger (ie. inode basis). We currently implement transactions using 4 fsync
/msync pairs.
write_recovery_data(fd);
fsync(fd);
msync(mmap);
write_recovery_header(fd);
fsync(fd);
msync(mmap);
overwrite_with_new_data(fd);
fsync(fd);
msync(mmap);
remove_recovery_header(fd);
fsync(fd);
msync(mmap);
Yet we really only need ordering, not guarantees about it actually hitting
disk before returning.
> In other words, FLUSH can be more relaxed than BARRIER inside the
> kernel. It's ironic that we think of fsync as stronger than
> fbarrier outside the kernel :-)
It's an implementation detail; barrier has less flexibility because it has
less information about what is required. I'm saying I want to give you as
much information as I can, even if you don't use it yet.
Thanks,
Rusty.
^ permalink raw reply
* Re: RFC: Network Plugin Architecture (NPA) for vmxnet3
From: Stephen Hemminger @ 2010-05-05 2:44 UTC (permalink / raw)
To: Pankaj Thakkar
Cc: pv-drivers@vmware.com, netdev@vger.kernel.org,
linux-kernel@vger.kernel.org,
virtualization@lists.linux-foundation.org
In-Reply-To: <20100505001857.GQ8323@vmware.com>
On Tue, 4 May 2010 17:18:57 -0700
Pankaj Thakkar <pthakkar@vmware.com> wrote:
> The purpose of this email is to introduce the architecture and the design principles. The overall project involves more than just changes to vmxnet3 driver and hence we though an overview email would be better. Once people agree to the design in general we intend to provide the code changes to the vmxnet3 driver.
As Dave said, we care more about what the implementation looks like than the high level
goals of the design. I think we all agree that better management of virtualized devices
is necessary, the problem is that their are so many of them (vmware, xen, HV, Xen),
and vendors seem to to lean on their own specific implementation of a offloading,
which makes a general solution more difficult. Please, Please solve this cleanly.
The little things like API's and locking semantics and handling of dynamic versus
static control can make a good design in principle fall apart when someone does a bad
job of implementing them.
Lastly, projects that have had multiple people involved for long periods of time
in the dark often end up building a legacy mentality "but we convinced vendor XXX to include it
in their Enterprise version 666" and require lots of "retraining" before the code
becomes acceptable.
--
^ permalink raw reply
* Re: RFC: Network Plugin Architecture (NPA) for vmxnet3
From: Chris Wright @ 2010-05-05 0:58 UTC (permalink / raw)
To: Pankaj Thakkar; +Cc: kvm, pv-drivers, netdev, linux-kernel, virtualization
In-Reply-To: <20100504230225.GP8323@vmware.com>
* Pankaj Thakkar (pthakkar@vmware.com) wrote:
> We intend to upgrade the upstreamed vmxnet3 driver to implement NPA so that
> Linux users can exploit the benefits provided by passthrough devices in a
> seamless manner while retaining the benefits of virtualization. The document
> below tries to answer most of the questions which we anticipated. Please let us
> know your comments and queries.
How does the throughput, latency, and host CPU utilization for normal
data path compare with say NetQueue?
And does this obsolete your UPT implementation?
> Network Plugin Architecture
> ---------------------------
>
> VMware has been working on various device passthrough technologies for the past
> few years. Passthrough technology is interesting as it can result in better
> performance/cpu utilization for certain demanding applications. In our vSphere
> product we support direct assignment of PCI devices like networking adapters to
> a guest virtual machine. This allows the guest to drive the device using the
> device drivers installed inside the guest. This is similar to the way KVM
> allows for passthrough of PCI devices to the guests. The hypervisor is bypassed
> for all I/O and control operations and hence it can not provide any value add
> features such as live migration, suspend/resume, etc.
>
>
> Network Plugin Architecture (NPA) is an approach which VMware has developed in
> joint partnership with Intel which allows us to retain the best of passthrough
> technology and virtualization. NPA allows for passthrough of the fast data
> (I/O) path and lets the hypervisor deal with the slow control path using
> traditional emulation/paravirtualization techniques. Through this splitting of
> data and control path the hypervisor can still provide the above mentioned
> value add features and exploit the performance benefits of passthrough.
How many cards actually support this NPA interface? What does it look
like, i.e. where is the NPA specification? (AFAIK, we never got the UPT
one).
> NPA requires SR-IOV hardware which allows for sharing of one single NIC adapter
> by multiple guests. SR-IOV hardware has many logically separate functions
> called virtual functions (VF) which can be independently assigned to the guest
> OS. They also have one or more physical functions (PF) (managed by a PF driver)
> which are used by the hypervisor to control certain aspects of the VFs and the
> rest of the hardware.
How do you handle hardware which has a more symmetric view of the
SR-IOV world (SR-IOV is only PCI sepcification, not a network driver
specification)? Or hardware which has multiple functions per physical
port (multiqueue, hw filtering, embedded switch, etc.)?
> NPA splits the guest driver into two components called
> the Shell and the Plugin. The shell is responsible for interacting with the
> guest networking stack and funneling the control operations to the hypervisor.
> The plugin is responsible for driving the data path of the virtual function
> exposed to the guest and is specific to the NIC hardware. NPA also requires an
> embedded switch in the NIC to allow for switching traffic among the virtual
> functions. The PF is also used as an uplink to provide connectivity to other
> VMs which are in emulation mode. The figure below shows the major components in
> a block diagram.
>
> +------------------------------+
> | Guest VM |
> | |
> | +----------------+ |
> | | vmxnet3 driver | |
> | | Shell | |
> | | +============+ | |
> | | | Plugin | | |
> +------+-+------------+-+------+
> | .
> +---------+ .
> | vmxnet3 | .
> |___+-----+ .
> | .
> | .
> +----------------------------+
> | |
> | virtual switch |
> +----------------------------+
> | . \
> | . \
> +=============+ . \
> | PF control | . \
> | | . \
> | L2 driver | . \
> +-------------+ . \
> | . \
> | . \
> +------------------------+ +------------+
> | PF VF1 VF2 ... VFn | | |
> | | | regular |
> | SR-IOV NIC | | nic |
> | +--------------+ | | +--------+
> | | embedded | | +---+
> | | switch | |
> | +--------------+ |
> | +---------------+
> +--------+
>
> NPA offers several benefits:
> 1. Performance: Critical performance sensitive paths are not trapped and the
> guest can directly drive the hardware without incurring virtualization
> overheads.
Can you demonstrate with data?
> 2. Hypervisor control: All control operations from the guest such as programming
> MAC address go through the hypervisor layer and hence can be subjected to
> hypervisor policies. The PF driver can be further used to put policy decisions
> like which VLAN the guest should be on.
This can happen without NPA as well. VF simply needs to request
the change via the PF (in fact, hw does that right now). Also, we
already have a host side management interface via PF (see, for example,
RTM_SETLINK IFLA_VF_MAC interface).
What is control plane interface? Just something like a fixed register set?
> 3. Guest Management: No hardware specific drivers need to be installed in the
> guest virtual machine and hence no overheads are incurred for guest management.
> All software for the driver (including the PF driver and the plugin) is
> installed in the hypervisor.
So we have a plugin per hardware VF implementation? And the hypervisor
injects this code into the guest?
> 4. IHV independence: The architecture provides guidelines for splitting the
> functionality between the VFs and PF but does not dictate how the hardware
> should be implemented. It gives the IHV the freedom to do asynchronous updates
> either to the software or the hardware to work around any defects.
Yes, this is important, esp. instead of the requirement for hw to
implement a specific interface (I suspect you know all about this issue
already).
> The fundamental tenet in NPA is to let the hypervisor control the passthrough
> functionality with minimal guest intervention. This gives a lot of flexibility
> to the hypervisor which can then treat passthrough as an offload feature (just
> like TSO, LRO, etc) which is offered to the guest virtual machine when there
> are no conflicting features present. For example, if the hypervisor wants to
> migrate the virtual machine from one host to another, the hypervisor can switch
> the virtual machine out of passthrough mode into paravirtualized/emulated mode
> and it can use existing technique to migrate the virtual machine. Once the
> virtual machine is migrated to the destination host the hypervisor can switch
> the virtual machine back to passthrough mode if a supporting SR-IOV nic is
> present. This may involve reloading of a different plugin corresponding to the
> new SR-IOV hardware.
>
> Internally we have explored various other options before settling on the NPA
> approach. For example there are approaches which create a bonding driver on top
> of a complete passthrough of a NIC device and an emulated/paravirtualized
> device. Though this approach allows for live migration to work it adds a lot of
> complexity and dependency. First the hypervisor has to rely on a guest with
> hot-add support. Second the hypervisor has to depend on the guest networking
> stack to cooperate to perform migration. Third the guest has to carry the
> driver images for all possible hardware to which the guest may migrate to.
> Fourth the hypervisor does not get full control for all the policy decisions.
> Another approach we have considered is to have a uniform interface for the data
> path between the emulated/paravirtualized device and the hardware device which
> allows the hypervisor to seamlessly switch from the emulated interface to the
> hardware interface. Though this approach is very attractive and can work
> without any guest involvement it is not acceptable to the IHVs as it does not
> give them the freedom to fix bugs/erratas and differentiate from each other. We
> believe NPA approach provides the right level of control and flexibility to the
> hypervisors while letting the guest exploit the benefits of passthrough.
> The plugin image is provided by the IHVs along with the PF driver and is
> packaged in the hypervisor. The plugin image is OS agnostic and can be loaded
> either into a Linux VM or a Windows VM. The plugin is written against the Shell
And it will need to be GPL AFAICT from what you've said thus far. It
does sound worrisome, although I suppose hw firmware isn't particularly
different.
> API interface which the shell is responsible for implementing. The API
> interface allows the plugin to do TX and RX only by programming the hardware
> rings (along with things like buffer allocation and basic initialization). The
> virtual machine comes up in paravirtualized/emulated mode when it is booted.
> The hypervisor allocates the VF and other resources and notifies the shell of
> the availability of the VF. The hypervisor injects the plugin into memory
> location specified by the shell. The shell initializes the plugin by calling
> into a known entry point and the plugin initializes the data path. The control
> path is already initialized by the PF driver when the VF is allocated. At this
> point the shell switches to using the loaded plugin to do all further TX and RX
> operations. The guest networking stack does not participate in these operations
> and continues to function normally. All the control operations continue being
> trapped by the hypervisor and are directed to the PF driver as needed. For
> example, if the MAC address changes the hypervisor updates its internal state
> and changes the state of the embedded switch as well through the PF control
> API.
How does the shell switch back to emulated mode for live migration?
> We have reworked our existing Linux vmxnet3 driver to accomodate NPA by
> splitting the driver into two parts: Shell and Plugin. The new split driver is
> backwards compatible and continues to work on old/existing vmxnet3 device
> emulations. The shell implements the API interface and contains code to do the
> bookkeeping for TX/RX buffers along with interrupt management. The shell code
> also handles the loading of the plugin and verifying the license of the loaded
> plugin. The plugin contains the code specific to vmxnet3 ring and descriptor
> management. The plugin uses the same Shell API interface which would be used by
> other IHVs. This vmxnet3 plugin is compiled statically along with the shell as
> this is needed to provide connectivity when there is no underlying SR-IOV
> device present. The IHV plugins are required to be distributed under GPL
> license and we are currently looking at ways to verify this both within the
> hypervisor and within the shell.
Please make this shell API interface and the PF/VF requirments available.
thanks,
-chris
^ permalink raw reply
* Re: RFC: Network Plugin Architecture (NPA) for vmxnet3
From: Pankaj Thakkar @ 2010-05-05 0:38 UTC (permalink / raw)
To: David Miller
Cc: pv-drivers@vmware.com, netdev@vger.kernel.org,
linux-kernel@vger.kernel.org,
virtualization@lists.linux-foundation.org, shemminger@vyatta.com
In-Reply-To: <20100504.173236.104064409.davem@davemloft.net>
Sure. We have been working on NPA for a while and have the code internally up
and running. Let me sync up internally on how and when we can provide the
vmxnet3 driver code so that people can look at it.
On Tue, May 04, 2010 at 05:32:36PM -0700, David Miller wrote:
> Date: Tue, 4 May 2010 17:32:36 -0700
> From: David Miller <davem@davemloft.net>
> To: Pankaj Thakkar <pthakkar@vmware.com>
> CC: "shemminger@vyatta.com" <shemminger@vyatta.com>,
> "linux-kernel@vger.kernel.org" <linux-kernel@vger.kernel.org>,
> "netdev@vger.kernel.org" <netdev@vger.kernel.org>,
> "virtualization@lists.linux-foundation.org"
> <virtualization@lists.linux-foundation.org>,
> "pv-drivers@vmware.com" <pv-drivers@vmware.com>,
> Shreyas Bhatewara <sbhatewara@vmware.com>
> Subject: Re: RFC: Network Plugin Architecture (NPA) for vmxnet3
>
> From: Pankaj Thakkar <pthakkar@vmware.com>
> Date: Tue, 4 May 2010 17:18:57 -0700
>
> > The purpose of this email is to introduce the architecture and the
> > design principles. The overall project involves more than just
> > changes to vmxnet3 driver and hence we though an overview email
> > would be better. Once people agree to the design in general we
> > intend to provide the code changes to the vmxnet3 driver.
>
> Stephen's point is that code talks and bullshit walks.
>
> Talk about high level designs rarely gets any traction, and often goes
> nowhere. Give us an example implementation so there is something
> concrete for us to sink our teeth into.
^ permalink raw reply
* Re: RFC: Network Plugin Architecture (NPA) for vmxnet3
From: David Miller @ 2010-05-05 0:32 UTC (permalink / raw)
To: pthakkar; +Cc: pv-drivers, netdev, linux-kernel, virtualization, shemminger
In-Reply-To: <20100505001857.GQ8323@vmware.com>
From: Pankaj Thakkar <pthakkar@vmware.com>
Date: Tue, 4 May 2010 17:18:57 -0700
> The purpose of this email is to introduce the architecture and the
> design principles. The overall project involves more than just
> changes to vmxnet3 driver and hence we though an overview email
> would be better. Once people agree to the design in general we
> intend to provide the code changes to the vmxnet3 driver.
Stephen's point is that code talks and bullshit walks.
Talk about high level designs rarely gets any traction, and often goes
nowhere. Give us an example implementation so there is something
concrete for us to sink our teeth into.
^ permalink raw reply
* Re: RFC: Network Plugin Architecture (NPA) for vmxnet3
From: Pankaj Thakkar @ 2010-05-05 0:18 UTC (permalink / raw)
To: Stephen Hemminger
Cc: pv-drivers@vmware.com, netdev@vger.kernel.org,
linux-kernel@vger.kernel.org,
virtualization@lists.linux-foundation.org
In-Reply-To: <20100504170531.1e7122da@nehalam>
The purpose of this email is to introduce the architecture and the design principles. The overall project involves more than just changes to vmxnet3 driver and hence we though an overview email would be better. Once people agree to the design in general we intend to provide the code changes to the vmxnet3 driver.
The architecture supports more than Intel NICs. We started the project with Intel but plan to support all major IHVs including Broadcom, Qlogic, Emulex and others through a certification program. The architecture works on VMware ESX server only as it requires significant support from the hypervisor. Also, the vmxnet3 driver works on VMware platform only. AFAICT Xen has a different model for supporting SR-IOV devices and allowing live migration and the document briefly talks about it (paragraph 6).
Thanks,
-pankaj
On Tue, May 04, 2010 at 05:05:31PM -0700, Stephen Hemminger wrote:
> Date: Tue, 4 May 2010 17:05:31 -0700
> From: Stephen Hemminger <shemminger@vyatta.com>
> To: Pankaj Thakkar <pthakkar@vmware.com>
> CC: "linux-kernel@vger.kernel.org" <linux-kernel@vger.kernel.org>,
> "netdev@vger.kernel.org" <netdev@vger.kernel.org>,
> "virtualization@lists.linux-foundation.org"
> <virtualization@lists.linux-foundation.org>,
> "pv-drivers@vmware.com" <pv-drivers@vmware.com>,
> Shreyas Bhatewara <sbhatewara@vmware.com>
> Subject: Re: RFC: Network Plugin Architecture (NPA) for vmxnet3
>
> On Tue, 4 May 2010 16:02:25 -0700
> Pankaj Thakkar <pthakkar@vmware.com> wrote:
>
> > Device passthrough technology allows a guest to bypass the hypervisor and drive
> > the underlying physical device. VMware has been exploring various ways to
> > deliver this technology to users in a manner which is easy to adopt. In this
> > process we have prepared an architecture along with Intel - NPA (Network Plugin
> > Architecture). NPA allows the guest to use the virtualized NIC vmxnet3 to
> > passthrough to a number of physical NICs which support it. The document below
> > provides an overview of NPA.
> >
> > We intend to upgrade the upstreamed vmxnet3 driver to implement NPA so that
> > Linux users can exploit the benefits provided by passthrough devices in a
> > seamless manner while retaining the benefits of virtualization. The document
> > below tries to answer most of the questions which we anticipated. Please let us
> > know your comments and queries.
> >
> > Thank you.
> >
> > Signed-off-by: Pankaj Thakkar <pthakkar@vmware.com>
>
>
> Code please. Also, it has to work for all architectures not just VMware and
> Intel.
^ permalink raw reply
* Re: RFC: Network Plugin Architecture (NPA) for vmxnet3
From: Stephen Hemminger @ 2010-05-05 0:05 UTC (permalink / raw)
To: Pankaj Thakkar; +Cc: pv-drivers, netdev, linux-kernel, virtualization
In-Reply-To: <20100504230225.GP8323@vmware.com>
On Tue, 4 May 2010 16:02:25 -0700
Pankaj Thakkar <pthakkar@vmware.com> wrote:
> Device passthrough technology allows a guest to bypass the hypervisor and drive
> the underlying physical device. VMware has been exploring various ways to
> deliver this technology to users in a manner which is easy to adopt. In this
> process we have prepared an architecture along with Intel - NPA (Network Plugin
> Architecture). NPA allows the guest to use the virtualized NIC vmxnet3 to
> passthrough to a number of physical NICs which support it. The document below
> provides an overview of NPA.
>
> We intend to upgrade the upstreamed vmxnet3 driver to implement NPA so that
> Linux users can exploit the benefits provided by passthrough devices in a
> seamless manner while retaining the benefits of virtualization. The document
> below tries to answer most of the questions which we anticipated. Please let us
> know your comments and queries.
>
> Thank you.
>
> Signed-off-by: Pankaj Thakkar <pthakkar@vmware.com>
Code please. Also, it has to work for all architectures not just VMware and
Intel.
^ permalink raw reply
* Re: [patch 7/8] Add a bootparameter to reserve high linear address space.
From: Jeremy Fitzhardinge @ 2010-05-04 23:37 UTC (permalink / raw)
To: Pavel Machek
Cc: akpm, Zachary Amsden, xen-devel, Jeremy Fitzhardinge,
linux-kernel, Chris Wright, virtualization
In-Reply-To: <19700101001522.GA3999@ucw.cz>
On 12/31/1969 04:15 PM, Pavel Machek wrote:
> Dynamically loaded hypervisor? Do we really want to support that?
>
(Missed this because your datestamp was 1969...)
The idea was that you could boot a guest under VMWare fully virtualized
then switch it to a VMI guest by loading a module. I don't think it
ever worked out - the transition was too trick to get right (especially
applying all the pvops patching to a live system).
J
^ permalink raw reply
* RFC: Network Plugin Architecture (NPA) for vmxnet3
From: Pankaj Thakkar @ 2010-05-04 23:02 UTC (permalink / raw)
To: linux-kernel, netdev, virtualization; +Cc: pv-drivers
Device passthrough technology allows a guest to bypass the hypervisor and drive
the underlying physical device. VMware has been exploring various ways to
deliver this technology to users in a manner which is easy to adopt. In this
process we have prepared an architecture along with Intel - NPA (Network Plugin
Architecture). NPA allows the guest to use the virtualized NIC vmxnet3 to
passthrough to a number of physical NICs which support it. The document below
provides an overview of NPA.
We intend to upgrade the upstreamed vmxnet3 driver to implement NPA so that
Linux users can exploit the benefits provided by passthrough devices in a
seamless manner while retaining the benefits of virtualization. The document
below tries to answer most of the questions which we anticipated. Please let us
know your comments and queries.
Thank you.
Signed-off-by: Pankaj Thakkar <pthakkar@vmware.com>
Network Plugin Architecture
---------------------------
VMware has been working on various device passthrough technologies for the past
few years. Passthrough technology is interesting as it can result in better
performance/cpu utilization for certain demanding applications. In our vSphere
product we support direct assignment of PCI devices like networking adapters to
a guest virtual machine. This allows the guest to drive the device using the
device drivers installed inside the guest. This is similar to the way KVM
allows for passthrough of PCI devices to the guests. The hypervisor is bypassed
for all I/O and control operations and hence it can not provide any value add
features such as live migration, suspend/resume, etc.
Network Plugin Architecture (NPA) is an approach which VMware has developed in
joint partnership with Intel which allows us to retain the best of passthrough
technology and virtualization. NPA allows for passthrough of the fast data
(I/O) path and lets the hypervisor deal with the slow control path using
traditional emulation/paravirtualization techniques. Through this splitting of
data and control path the hypervisor can still provide the above mentioned
value add features and exploit the performance benefits of passthrough.
NPA requires SR-IOV hardware which allows for sharing of one single NIC adapter
by multiple guests. SR-IOV hardware has many logically separate functions
called virtual functions (VF) which can be independently assigned to the guest
OS. They also have one or more physical functions (PF) (managed by a PF driver)
which are used by the hypervisor to control certain aspects of the VFs and the
rest of the hardware. NPA splits the guest driver into two components called
the Shell and the Plugin. The shell is responsible for interacting with the
guest networking stack and funneling the control operations to the hypervisor.
The plugin is responsible for driving the data path of the virtual function
exposed to the guest and is specific to the NIC hardware. NPA also requires an
embedded switch in the NIC to allow for switching traffic among the virtual
functions. The PF is also used as an uplink to provide connectivity to other
VMs which are in emulation mode. The figure below shows the major components in
a block diagram.
+------------------------------+
| Guest VM |
| |
| +----------------+ |
| | vmxnet3 driver | |
| | Shell | |
| | +============+ | |
| | | Plugin | | |
+------+-+------------+-+------+
| .
+---------+ .
| vmxnet3 | .
|___+-----+ .
| .
| .
+----------------------------+
| |
| virtual switch |
+----------------------------+
| . \
| . \
+=============+ . \
| PF control | . \
| | . \
| L2 driver | . \
+-------------+ . \
| . \
| . \
+------------------------+ +------------+
| PF VF1 VF2 ... VFn | | |
| | | regular |
| SR-IOV NIC | | nic |
| +--------------+ | | +--------+
| | embedded | | +---+
| | switch | |
| +--------------+ |
| +---------------+
+--------+
NPA offers several benefits:
1. Performance: Critical performance sensitive paths are not trapped and the
guest can directly drive the hardware without incurring virtualization
overheads.
2. Hypervisor control: All control operations from the guest such as programming
MAC address go through the hypervisor layer and hence can be subjected to
hypervisor policies. The PF driver can be further used to put policy decisions
like which VLAN the guest should be on.
3. Guest Management: No hardware specific drivers need to be installed in the
guest virtual machine and hence no overheads are incurred for guest management.
All software for the driver (including the PF driver and the plugin) is
installed in the hypervisor.
4. IHV independence: The architecture provides guidelines for splitting the
functionality between the VFs and PF but does not dictate how the hardware
should be implemented. It gives the IHV the freedom to do asynchronous updates
either to the software or the hardware to work around any defects.
The fundamental tenet in NPA is to let the hypervisor control the passthrough
functionality with minimal guest intervention. This gives a lot of flexibility
to the hypervisor which can then treat passthrough as an offload feature (just
like TSO, LRO, etc) which is offered to the guest virtual machine when there
are no conflicting features present. For example, if the hypervisor wants to
migrate the virtual machine from one host to another, the hypervisor can switch
the virtual machine out of passthrough mode into paravirtualized/emulated mode
and it can use existing technique to migrate the virtual machine. Once the
virtual machine is migrated to the destination host the hypervisor can switch
the virtual machine back to passthrough mode if a supporting SR-IOV nic is
present. This may involve reloading of a different plugin corresponding to the
new SR-IOV hardware.
Internally we have explored various other options before settling on the NPA
approach. For example there are approaches which create a bonding driver on top
of a complete passthrough of a NIC device and an emulated/paravirtualized
device. Though this approach allows for live migration to work it adds a lot of
complexity and dependency. First the hypervisor has to rely on a guest with
hot-add support. Second the hypervisor has to depend on the guest networking
stack to cooperate to perform migration. Third the guest has to carry the
driver images for all possible hardware to which the guest may migrate to.
Fourth the hypervisor does not get full control for all the policy decisions.
Another approach we have considered is to have a uniform interface for the data
path between the emulated/paravirtualized device and the hardware device which
allows the hypervisor to seamlessly switch from the emulated interface to the
hardware interface. Though this approach is very attractive and can work
without any guest involvement it is not acceptable to the IHVs as it does not
give them the freedom to fix bugs/erratas and differentiate from each other. We
believe NPA approach provides the right level of control and flexibility to the
hypervisors while letting the guest exploit the benefits of passthrough.
The plugin image is provided by the IHVs along with the PF driver and is
packaged in the hypervisor. The plugin image is OS agnostic and can be loaded
either into a Linux VM or a Windows VM. The plugin is written against the Shell
API interface which the shell is responsible for implementing. The API
interface allows the plugin to do TX and RX only by programming the hardware
rings (along with things like buffer allocation and basic initialization). The
virtual machine comes up in paravirtualized/emulated mode when it is booted.
The hypervisor allocates the VF and other resources and notifies the shell of
the availability of the VF. The hypervisor injects the plugin into memory
location specified by the shell. The shell initializes the plugin by calling
into a known entry point and the plugin initializes the data path. The control
path is already initialized by the PF driver when the VF is allocated. At this
point the shell switches to using the loaded plugin to do all further TX and RX
operations. The guest networking stack does not participate in these operations
and continues to function normally. All the control operations continue being
trapped by the hypervisor and are directed to the PF driver as needed. For
example, if the MAC address changes the hypervisor updates its internal state
and changes the state of the embedded switch as well through the PF control
API.
We have reworked our existing Linux vmxnet3 driver to accomodate NPA by
splitting the driver into two parts: Shell and Plugin. The new split driver is
backwards compatible and continues to work on old/existing vmxnet3 device
emulations. The shell implements the API interface and contains code to do the
bookkeeping for TX/RX buffers along with interrupt management. The shell code
also handles the loading of the plugin and verifying the license of the loaded
plugin. The plugin contains the code specific to vmxnet3 ring and descriptor
management. The plugin uses the same Shell API interface which would be used by
other IHVs. This vmxnet3 plugin is compiled statically along with the shell as
this is needed to provide connectivity when there is no underlying SR-IOV
device present. The IHV plugins are required to be distributed under GPL
license and we are currently looking at ways to verify this both within the
hypervisor and within the shell.
^ permalink raw reply
* [PATCH 1/1] hv: Added new hv_utils driver with shutdown as first functionality - NO OUTLOOK
From: Hank Janssen @ 2010-05-04 22:55 UTC (permalink / raw)
To: hjanssen, linux-kernel, devel, virtualization, gregkh, haiyangz
Resending this patch from my personal Linux server. Exchange server and
outlook at Microsoft seems to badly munge my patch. :(
From: Hank Janssen <hjanssen@sailtheuniverse.com>
Subject: [PATCH 1/1] hv: Added new hv_utils driver with shutdown as first functionality
Addition of new driver for Hyper-V called hv_utils.
This driver is intended to support things like KVP, Timesync, Heartbeat etc.
This first release has support for Gracefull shutdown.
e.g. Select shutdown from the Hyper-V main admin screen and the Linux VM
will do a gracefull shutdown.
Signed-off-by: Hank Janssen <microsoft.com>
Signed-off-by: Hank Janssen <sailtheuniverse.com>
Signed-off-by: Haiyang Zhang <haiyangz@microsoft.com>
---
drivers/staging/hv/Channel.c | 34 ++++++-
drivers/staging/hv/ChannelMgmt.c | 148 +++++++++++++++++++++++++++++++-
drivers/staging/hv/Kconfig | 6 ++
drivers/staging/hv/Makefile | 2 +
drivers/staging/hv/VmbusPacketFormat.h | 1 +
drivers/staging/hv/ext_utils.c | 27 ++++++
drivers/staging/hv/hyperv_utils.c | 134 +++++++++++++++++++++++++++++
drivers/staging/hv/utils.h | 94 ++++++++++++++++++++
8 files changed, 438 insertions(+), 8 deletions(-)
create mode 100644 drivers/staging/hv/ext_utils.c
create mode 100644 drivers/staging/hv/hyperv_utils.c
create mode 100644 drivers/staging/hv/utils.h
diff --git a/drivers/staging/hv/Channel.c b/drivers/staging/hv/Channel.c
index 328d3a0..de2ccb1 100644
--- a/drivers/staging/hv/Channel.c
+++ b/drivers/staging/hv/Channel.c
@@ -21,6 +21,7 @@
#include <linux/kernel.h>
#include <linux/mm.h>
#include <linux/slab.h>
+#include <linux/module.h>
#include "osd.h"
#include "logging.h"
#include "VmbusPrivate.h"
@@ -666,8 +667,19 @@ void VmbusChannelClose(struct vmbus_channel *Channel)
DPRINT_EXIT(VMBUS);
}
-/*
- * VmbusChannelSendPacket - Send the specified buffer on the given channel
+/**
+ * VmbusChannelSendPacket() - Send the specified buffer on the given channel
+ * @Channel: Pointer to vmbus_channel structure.
+ * @Buffer: Pointer to the buffer you want to receive the data into.
+ * @BufferLen: Maximum size of what the the buffer will hold
+ * @RequestId: Identifier of the request
+ * @vmbus_packet_type: Type of packet that is being send e.g. negotiate, time
+ * packet etc.
+ *
+ * Sends data in @Buffer directly to hyper-v via the vmbus
+ * This will send the data unparsed to hyper-v.
+ *
+ * Mainly used by Hyper-V drivers.
*/
int VmbusChannelSendPacket(struct vmbus_channel *Channel, const void *Buffer,
u32 BufferLen, u64 RequestId,
@@ -711,6 +723,7 @@ int VmbusChannelSendPacket(struct vmbus_channel *Channel, const void *Buffer,
return ret;
}
+EXPORT_SYMBOL(VmbusChannelSendPacket);
/*
* VmbusChannelSendPacketPageBuffer - Send a range of single-page buffer
@@ -848,10 +861,20 @@ int VmbusChannelSendPacketMultiPageBuffer(struct vmbus_channel *Channel,
return ret;
}
-/*
- * VmbusChannelRecvPacket - Retrieve the user packet on the specified channel
+
+/**
+ * VmbusChannelRecvPacket() - Retrieve the user packet on the specified channel
+ * @Channel: Pointer to vmbus_channel structure.
+ * @Buffer: Pointer to the buffer you want to receive the data into.
+ * @BufferLen: Maximum size of what the the buffer will hold
+ * @BufferActualLen: The actual size of the data after it was received
+ * @RequestId: Identifier of the request
+ *
+ * Receives directly from the hyper-v vmbus and puts the data it received
+ * into Buffer. This will receive the data unparsed from hyper-v.
+ *
+ * Mainly used by Hyper-V drivers.
*/
-/* TODO: Do we ever receive a gpa direct packet other than the ones we send ? */
int VmbusChannelRecvPacket(struct vmbus_channel *Channel, void *Buffer,
u32 BufferLen, u32 *BufferActualLen, u64 *RequestId)
{
@@ -913,6 +936,7 @@ int VmbusChannelRecvPacket(struct vmbus_channel *Channel, void *Buffer,
return 0;
}
+EXPORT_SYMBOL(VmbusChannelRecvPacket);
/*
* VmbusChannelRecvPacketRaw - Retrieve the raw packet on the specified channel
diff --git a/drivers/staging/hv/ChannelMgmt.c b/drivers/staging/hv/ChannelMgmt.c
index 43f28f2..9c3069f 100644
--- a/drivers/staging/hv/ChannelMgmt.c
+++ b/drivers/staging/hv/ChannelMgmt.c
@@ -22,18 +22,22 @@
#include <linux/mm.h>
#include <linux/slab.h>
#include <linux/list.h>
+#include <linux/module.h>
#include "osd.h"
#include "logging.h"
#include "VmbusPrivate.h"
+#include "utils.h"
struct vmbus_channel_message_table_entry {
enum vmbus_channel_message_type messageType;
void (*messageHandler)(struct vmbus_channel_message_header *msg);
};
-#define MAX_NUM_DEVICE_CLASSES_SUPPORTED 4
+#define MAX_MSG_TYPES 1
+#define MAX_NUM_DEVICE_CLASSES_SUPPORTED 5
+
static const struct hv_guid
- gSupportedDeviceClasses[MAX_NUM_DEVICE_CLASSES_SUPPORTED] = {
+ gSupportedDeviceClasses[MAX_NUM_DEVICE_CLASSES_SUPPORTED] = {
/* {ba6163d9-04a1-4d29-b605-72e2ffb1dc7f} */
/* Storage - SCSI */
{
@@ -69,8 +73,127 @@ static const struct hv_guid
0x9b, 0x5c, 0x50, 0xd1, 0x41, 0x73, 0x54, 0xf5
}
},
+ /* 0E0B6031-5213-4934-818B-38D90CED39DB */
+ /* Shutdown */
+ {
+ .data = {
+ 0x31, 0x60, 0x0B, 0X0E, 0x13, 0x52, 0x34, 0x49,
+ 0x81, 0x8B, 0x38, 0XD9, 0x0C, 0xED, 0x39, 0xDB
+ }
+ },
};
+
+/**
+ * prep_negotiate_resp() - Create default response for Hyper-V Negotiate message
+ * @icmsghdrp: Pointer to msg header structure
+ * @icmsg_negotiate: Pointer to negotiate message structure
+ * @buf: Raw buffer channel data
+ *
+ * @icmsghdrp is of type &struct icmsg_hdr.
+ * @negop is of type &struct icmsg_negotiate.
+ * Set up and fill in default negotiate response message. This response can
+ * come from both the vmbus driver and the hv_utils driver. The current api
+ * will respond properly to both Windows 2008 and Windows 2008-R2 operating
+ * systems.
+ *
+ * Mainly used by Hyper-V drivers.
+ */
+void prep_negotiate_resp(struct icmsg_hdr *icmsghdrp,
+ struct icmsg_negotiate *negop,
+ u8 *buf)
+{
+ if (icmsghdrp->icmsgtype == ICMSGTYPE_NEGOTIATE) {
+ icmsghdrp->icmsgsize = 0x10;
+
+ negop = (struct icmsg_negotiate *)&buf[
+ sizeof(struct vmbuspipe_hdr) +
+ sizeof(struct icmsg_hdr)];
+
+ if (negop->icframe_vercnt == 2 &&
+ negop->icversion_data[1].major == 3) {
+ negop->icversion_data[0].major = 3;
+ negop->icversion_data[0].minor = 0;
+ negop->icversion_data[1].major = 3;
+ negop->icversion_data[1].minor = 0;
+ } else {
+ negop->icversion_data[0].major = 1;
+ negop->icversion_data[0].minor = 0;
+ negop->icversion_data[1].major = 1;
+ negop->icversion_data[1].minor = 0;
+ }
+
+ negop->icframe_vercnt = 1;
+ negop->icmsg_vercnt = 1;
+ }
+}
+EXPORT_SYMBOL(prep_negotiate_resp);
+
+/**
+ * chn_cb_negotiate() - Default handler for non IDE/SCSI/NETWORK
+ * Hyper-V requests
+ * @context: Pointer to argument structure.
+ *
+ * Set up the default handler for non device driver specific requests
+ * from Hyper-V. This stub responds to the default negotiate messages
+ * that come in for every non IDE/SCSI/Network request.
+ * This behavior is normally overwritten in the hv_utils driver. That
+ * driver handles requests like gracefull shutdown, heartbeats etc.
+ *
+ * Mainly used by Hyper-V drivers.
+ */
+void chn_cb_negotiate(void *context)
+{
+ struct vmbus_channel *channel = context;
+ u8 *buf;
+ u32 buflen, recvlen;
+ u64 requestid;
+
+ struct icmsg_hdr *icmsghdrp;
+ struct icmsg_negotiate *negop = NULL;
+
+ buflen = PAGE_SIZE;
+ buf = kmalloc(buflen, GFP_ATOMIC);
+
+ VmbusChannelRecvPacket(channel, buf, buflen, &recvlen, &requestid);
+
+ if (recvlen > 0) {
+ icmsghdrp = (struct icmsg_hdr *)&buf[
+ sizeof(struct vmbuspipe_hdr)];
+
+ prep_negotiate_resp(icmsghdrp, negop, buf);
+
+ icmsghdrp->icflags = ICMSGHDRFLAG_TRANSACTION
+ | ICMSGHDRFLAG_RESPONSE;
+
+ VmbusChannelSendPacket(channel, buf,
+ recvlen, requestid,
+ VmbusPacketTypeDataInBand, 0);
+ }
+
+ kfree(buf);
+}
+EXPORT_SYMBOL(chn_cb_negotiate);
+
+/*
+ * Function table used for message responses for non IDE/SCSI/Network type
+ * messages. (Such as KVP/Shutdown etc)
+ */
+struct hyperv_service_callback hv_cb_utils[MAX_MSG_TYPES] = {
+ /* 0E0B6031-5213-4934-818B-38D90CED39DB */
+ /* Shutdown */
+ {
+ .msg_type = HV_SHUTDOWN_MSG,
+ .data = {
+ 0x31, 0x60, 0x0B, 0X0E, 0x13, 0x52, 0x34, 0x49,
+ 0x81, 0x8B, 0x38, 0XD9, 0x0C, 0xED, 0x39, 0xDB
+ },
+ .callback = chn_cb_negotiate,
+ .log_msg = "Shutdown channel functionality initialized"
+ },
+};
+EXPORT_SYMBOL(hv_cb_utils);
+
/*
* AllocVmbusChannel - Allocate and initialize a vmbus channel object
*/
@@ -132,7 +255,8 @@ void FreeVmbusChannel(struct vmbus_channel *Channel)
}
/*
- * VmbusChannelProcessOffer - Process the offer by creating a channel/device associated with this offer
+ * VmbusChannelProcessOffer - Process the offer by creating a channel/device
+ * associated with this offer
*/
static void VmbusChannelProcessOffer(void *context)
{
@@ -140,6 +264,7 @@ static void VmbusChannelProcessOffer(void *context)
struct vmbus_channel *channel;
bool fNew = true;
int ret;
+ int cnt;
unsigned long flags;
DPRINT_ENTER(VMBUS);
@@ -209,6 +334,23 @@ static void VmbusChannelProcessOffer(void *context)
* can cleanup properly
*/
newChannel->State = CHANNEL_OPEN_STATE;
+ cnt = 0;
+
+ while (cnt != MAX_MSG_TYPES) {
+ if (memcmp(&newChannel->OfferMsg.Offer.InterfaceType,
+ &hv_cb_utils[cnt].data,
+ sizeof(struct hv_guid)) == 0) {
+ DPRINT_INFO(VMBUS, "%s",
+ hv_cb_utils[cnt].log_msg);
+
+ if (VmbusChannelOpen(newChannel, 2 * PAGE_SIZE,
+ 2 * PAGE_SIZE, NULL, 0,
+ hv_cb_utils[cnt].callback,
+ newChannel) == 0)
+ hv_cb_utils[cnt].channel = newChannel;
+ }
+ cnt++;
+ }
}
DPRINT_EXIT(VMBUS);
}
diff --git a/drivers/staging/hv/Kconfig b/drivers/staging/hv/Kconfig
index 4044702..79afb1e 100644
--- a/drivers/staging/hv/Kconfig
+++ b/drivers/staging/hv/Kconfig
@@ -29,4 +29,10 @@ config HYPERV_NET
help
Select this option to enable the Hyper-V virtual network driver.
+config HYPERV_UTILS
+ tristate "Microsoft Hyper-V Utilities driver"
+ default HYPERV
+ help
+ Select this option to enable the Hyper-V Utilities.
+
endif
diff --git a/drivers/staging/hv/Makefile b/drivers/staging/hv/Makefile
index 27ebae8..d2977ab 100644
--- a/drivers/staging/hv/Makefile
+++ b/drivers/staging/hv/Makefile
@@ -2,6 +2,7 @@ obj-$(CONFIG_HYPERV) += hv_vmbus.o
obj-$(CONFIG_HYPERV_STORAGE) += hv_storvsc.o
obj-$(CONFIG_HYPERV_BLOCK) += hv_blkvsc.o
obj-$(CONFIG_HYPERV_NET) += hv_netvsc.o
+obj-$(CONFIG_HYPERV_UTILS) += hv_utils.o
hv_vmbus-objs := vmbus_drv.o osd.o \
Vmbus.o Hv.o Connection.o Channel.o \
@@ -9,3 +10,4 @@ hv_vmbus-objs := vmbus_drv.o osd.o \
hv_storvsc-objs := storvsc_drv.o StorVsc.o
hv_blkvsc-objs := blkvsc_drv.o BlkVsc.o
hv_netvsc-objs := netvsc_drv.o NetVsc.o RndisFilter.o
+hv_utils-objs := hyperv_utils.o ext_utils.o
diff --git a/drivers/staging/hv/VmbusPacketFormat.h b/drivers/staging/hv/VmbusPacketFormat.h
index 79120bc..f9f6b4b 100644
--- a/drivers/staging/hv/VmbusPacketFormat.h
+++ b/drivers/staging/hv/VmbusPacketFormat.h
@@ -22,6 +22,7 @@
*/
#ifndef _VMBUSPACKETFORMAT_H_
+#define _VMBUSPACKETFORMAT_H_
struct vmpacket_descriptor {
u16 Type;
diff --git a/drivers/staging/hv/ext_utils.c b/drivers/staging/hv/ext_utils.c
new file mode 100644
index 0000000..a44cd1b
--- /dev/null
+++ b/drivers/staging/hv/ext_utils.c
@@ -0,0 +1,27 @@
+/*
+ * Copyright (c) 2010, Microsoft Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program; if not, write to the Free Software Foundation, Inc., 59 Temple
+ * Place - Suite 330, Boston, MA 02111-1307 USA.
+ *
+ * Authors:
+ * Haiyang Zhang <haiyangz@microsoft.com>
+ * Hank Janssen <hjanssen@microsoft.com>
+ */
+#include <linux/reboot.h>
+#include "utils.h"
+
+void shutdown_linux_system()
+{
+ orderly_poweroff(false);
+}
diff --git a/drivers/staging/hv/hyperv_utils.c b/drivers/staging/hv/hyperv_utils.c
new file mode 100644
index 0000000..2a48647
--- /dev/null
+++ b/drivers/staging/hv/hyperv_utils.c
@@ -0,0 +1,134 @@
+/*
+ * Copyright (c) 2010, Microsoft Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program; if not, write to the Free Software Foundation, Inc., 59 Temple
+ * Place - Suite 330, Boston, MA 02111-1307 USA.
+ *
+ * Authors:
+ * Haiyang Zhang <haiyangz@microsoft.com>
+ * Hank Janssen <hjanssen@microsoft.com>
+ */
+#include <linux/kernel.h>
+#include <linux/init.h>
+#include <linux/module.h>
+#include <linux/slab.h>
+#include <linux/sysctl.h>
+#include <linux/version.h>
+
+#include "logging.h"
+#include "osd.h"
+#include "vmbus.h"
+#include "VmbusPacketFormat.h"
+#include "VmbusChannelInterface.h"
+#include "VersionInfo.h"
+#include "Channel.h"
+#include "VmbusPrivate.h"
+#include "VmbusApi.h"
+#include "utils.h"
+
+
+void shutdown_onchannelcallback(void *context)
+{
+ struct vmbus_channel *channel = context;
+ u8 *buf;
+ u32 buflen, recvlen;
+ u64 requestid;
+ u8 execute_shutdown = false;
+
+ struct shutdown_msg_data *shutdown_msg;
+
+ struct icmsg_hdr *icmsghdrp;
+ struct icmsg_negotiate *negop = NULL;
+
+ DPRINT_ENTER(VMBUS);
+
+ buflen = PAGE_SIZE;
+ buf = kmalloc(buflen, GFP_ATOMIC);
+
+ VmbusChannelRecvPacket(channel, buf, buflen, &recvlen, &requestid);
+
+ if (recvlen > 0) {
+ DPRINT_DBG(VMBUS, "shutdown packet: len=%d, requestid=%lld",
+ recvlen, requestid);
+
+ icmsghdrp = (struct icmsg_hdr *)&buf[
+ sizeof(struct vmbuspipe_hdr)];
+
+ if (icmsghdrp->icmsgtype == ICMSGTYPE_NEGOTIATE) {
+ prep_negotiate_resp(icmsghdrp, negop, buf);
+ } else {
+ shutdown_msg = (struct shutdown_msg_data *)&buf[
+ sizeof(struct vmbuspipe_hdr) +
+ sizeof(struct icmsg_hdr)];
+
+ switch (shutdown_msg->flags) {
+ case 0:
+ case 1:
+ icmsghdrp->status = HV_S_OK;
+ execute_shutdown = true;
+
+ DPRINT_INFO(VMBUS, "Shutdown request received -"
+ " gracefull shutdown initiated");
+ break;
+ default:
+ icmsghdrp->status = HV_E_FAIL;
+ execute_shutdown = false;
+
+ DPRINT_INFO(VMBUS, "Shutdown request received -"
+ " Invalid request");
+ break;
+ };
+ }
+
+ icmsghdrp->icflags = ICMSGHDRFLAG_TRANSACTION
+ | ICMSGHDRFLAG_RESPONSE;
+
+ VmbusChannelSendPacket(channel, buf,
+ recvlen, requestid,
+ VmbusPacketTypeDataInBand, 0);
+ }
+
+ kfree(buf);
+
+ DPRINT_EXIT(VMBUS);
+
+ if (execute_shutdown == true)
+ shutdown_linux_system();
+}
+
+static int __init init_hyperv_utils(void)
+{
+ printk(KERN_INFO "Registering HyperV Utility Driver\n");
+
+ hv_cb_utils[HV_SHUTDOWN_MSG].channel->OnChannelCallback =
+ &shutdown_onchannelcallback;
+ hv_cb_utils[HV_SHUTDOWN_MSG].callback = &shutdown_onchannelcallback;
+
+ return 0;
+}
+
+static void exit_hyperv_utils(void)
+{
+ printk(KERN_INFO "De-Registered HyperV Utility Driver\n");
+
+ hv_cb_utils[HV_SHUTDOWN_MSG].channel->OnChannelCallback =
+ &chn_cb_negotiate;
+ hv_cb_utils[HV_SHUTDOWN_MSG].callback = &chn_cb_negotiate;
+}
+
+module_init(init_hyperv_utils);
+module_exit(exit_hyperv_utils);
+
+MODULE_DESCRIPTION("Hyper-V Utilities");
+MODULE_VERSION(HV_DRV_VERSION);
+MODULE_LICENSE("GPL");
diff --git a/drivers/staging/hv/utils.h b/drivers/staging/hv/utils.h
new file mode 100644
index 0000000..4e09804
--- /dev/null
+++ b/drivers/staging/hv/utils.h
@@ -0,0 +1,94 @@
+/*
+ * Copyright (c) 2009, Microsoft Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program; if not, write to the Free Software Foundation, Inc., 59 Temple
+ * Place - Suite 330, Boston, MA 02111-1307 USA.
+ *
+ * Authors:
+ * Haiyang Zhang <haiyangz@microsoft.com>
+ * Hank Janssen <hjanssen@microsoft.com>
+ */
+#ifndef _UTILS_H_
+#define _UTILS_H_
+
+/*
+ * Common header for Hyper-V ICs
+ */
+#define ICMSGTYPE_NEGOTIATE 0
+#define ICMSGTYPE_HEARTBEAT 1
+#define ICMSGTYPE_KVPEXCHANGE 2
+#define ICMSGTYPE_SHUTDOWN 3
+#define ICMSGTYPE_TIMESYNC 4
+#define ICMSGTYPE_VSS 5
+
+#define ICMSGHDRFLAG_TRANSACTION 1
+#define ICMSGHDRFLAG_REQUEST 2
+#define ICMSGHDRFLAG_RESPONSE 4
+
+#define HV_S_OK 0x00000000
+#define HV_E_FAIL 0x80004005
+#define HV_ERROR_NOT_SUPPORTED 0x80070032
+#define HV_ERROR_MACHINE_LOCKED 0x800704F7
+
+struct vmbuspipe_hdr {
+ u32 flags;
+ u32 msgsize;
+} __attribute__((packed));
+
+struct ic_version {
+ u16 major;
+ u16 minor;
+} __attribute__((packed));
+
+struct icmsg_hdr {
+ struct ic_version icverframe;
+ u16 icmsgtype;
+ struct ic_version icvermsg;
+ u16 icmsgsize;
+ u32 status;
+ u8 ictransaction_id;
+ u8 icflags;
+ u8 reserved[2];
+} __attribute__((packed));
+
+struct icmsg_negotiate {
+ u16 icframe_vercnt;
+ u16 icmsg_vercnt;
+ u32 reserved;
+ struct ic_version icversion_data[1]; /* any size array */
+} __attribute__((packed));
+
+struct shutdown_msg_data {
+ u32 reason_code;
+ u32 timeout_seconds;
+ u32 flags;
+ u8 display_message[2048];
+} __attribute__((packed));
+
+#define HV_SHUTDOWN_MSG 0
+
+struct hyperv_service_callback {
+ u8 msg_type;
+ char *log_msg;
+ unsigned char data[16];
+ struct vmbus_channel *channel;
+ void (*callback) (void *context);
+};
+
+extern void prep_negotiate_resp(struct icmsg_hdr *,
+ struct icmsg_negotiate *, u8 *);
+extern void shutdown_linux_system(void);
+extern void chn_cb_negotiate(void *);
+extern struct hyperv_service_callback hv_cb_utils[];
+
+#endif /* _UTILS_H_ */
--
1.6.0.2
^ permalink raw reply related
* Re: [Qemu-devel] Re: [PATCH] virtio-spec: document block CMD and FLUSH
From: Jamie Lokier @ 2010-05-04 20:32 UTC (permalink / raw)
To: Rusty Russell
Cc: tytso, kvm, Michael S. Tsirkin, Neil Brown, qemu-devel,
virtualization, Jens Axboe, hch
In-Reply-To: <201005041408.25069.rusty@rustcorp.com.au>
Rusty Russell wrote:
> On Fri, 19 Feb 2010 08:52:20 am Michael S. Tsirkin wrote:
> > I took a stub at documenting CMD and FLUSH request types in virtio
> > block. Christoph, could you look over this please?
> >
> > I note that the interface seems full of warts to me,
> > this might be a first step to cleaning them.
>
> ISTR Christoph had withdrawn some patches in this area, and was waiting
> for him to resubmit?
>
> I've given up on figuring out the block device. What seem to me to be sane
> semantics along the lines of memory barriers are foreign to disk people: they
> want (and depend on) flushing everywhere.
>
> For example, tdb transactions do not require a flush, they only require what
> I would call a barrier: that prior data be written out before any future data.
> Surely that would be more efficient in general than a flush! In fact, TDB
> wants only writes to *that file* (and metadata) written out first; it has no
> ordering issues with other I/O on the same device.
I've just posted elsewhere on this thread, that an I/O level flush can
be more efficient than an I/O level barrier (implemented using a
cache-flush really), because the barrier has stricter ordering
requirements at the I/O scheduling level.
By the time you work up to tdb, another way to think of it is
distinguishing "eager fsync" from "fsync but I'm not in a hurry -
delay as long as is convenient". The latter makes much more sense
with AIO.
> A generic I/O interface would allow you to specify "this request
> depends on these outstanding requests" and leave it at that. It
> might have some sync flush command for dumb applications and OSes.
For filesystems, it would probably be easy to label in-place
overwrites and fdatasync data flushes when there's no file extension
with an opqaue per-file identifier for certain operations. Typically
over-writing in place and fdatasync would match up and wouldn't need
ordering against anything else. Other operations would tend to get
labelled as ordered against everything including these.
-- Jamie
^ permalink raw reply
page: next (older) | prev (newer) | latest
- recent:[subjects (threaded)|topics (new)|topics (active)]
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox