[PATCH V8 3/7] libxl: add pvusb API

[Chunyan Liu <cyliu@suse.com>]

Add pvusb APIs, including:
 - attach/detach (create/destroy) virtual usb controller.
 - attach/detach usb device
 - list usb controller and usb devices
 - some other helper functions

Signed-off-by: Chunyan Liu <cyliu@suse.com>
Signed-off-by: Simon Cao <caobosimon@gmail.com>

---
changes:
  - update COMPARE_USB to compare ctrl and port
  - add check in usb_add/remove to disable non-Dom0 backend so that
    not worring about codes which are effective on Dom0 but not
    compatible on non-Dom0 backend.
  - define READ_SUBPATH macro within functions
  - do not initialize rc but give it value in each return case
  - libxl__strdup gc or NOGC update, internal function using gc,
    external using NOGC.
  - address other comments from George and Ian J.

 tools/libxl/Makefile                 |    2 +-
 tools/libxl/libxl.c                  |   53 ++
 tools/libxl/libxl.h                  |   74 ++
 tools/libxl/libxl_device.c           |    5 +-
 tools/libxl/libxl_internal.h         |   18 +
 tools/libxl/libxl_osdeps.h           |   13 +
 tools/libxl/libxl_pvusb.c            | 1451 ++++++++++++++++++++++++++++++++++
 tools/libxl/libxl_types.idl          |   57 ++
 tools/libxl/libxl_types_internal.idl |    1 +
 tools/libxl/libxl_utils.c            |   16 +
 tools/libxl/libxl_utils.h            |    5 +
 11 files changed, 1693 insertions(+), 2 deletions(-)
 create mode 100644 tools/libxl/libxl_pvusb.c

diff --git a/tools/libxl/Makefile b/tools/libxl/Makefile
index c5ecec1..ef9ccd3 100644
--- a/tools/libxl/Makefile
+++ b/tools/libxl/Makefile
@@ -103,7 +103,7 @@ LIBXL_OBJS = flexarray.o libxl.o libxl_create.o libxl_dm.o libxl_pci.o \
 			libxl_stream_read.o libxl_stream_write.o \
 			libxl_save_callout.o _libxl_save_msgs_callout.o \
 			libxl_qmp.o libxl_event.o libxl_fork.o \
-			libxl_dom_suspend.o $(LIBXL_OBJS-y)
+			libxl_dom_suspend.o libxl_pvusb.o $(LIBXL_OBJS-y)
 LIBXL_OBJS += libxl_genid.o
 LIBXL_OBJS += _libxl_types.o libxl_flask.o _libxl_types_internal.o
 
diff --git a/tools/libxl/libxl.c b/tools/libxl/libxl.c
index dacfaae..a050e8b 100644
--- a/tools/libxl/libxl.c
+++ b/tools/libxl/libxl.c
@@ -4218,11 +4218,54 @@ DEFINE_DEVICE_REMOVE(vtpm, destroy, 1)
 
 /******************************************************************************/
 
+/* Macro for defining device remove/destroy functions for usbctrl */
+/* Following functions are defined:
+ * libxl_device_usbctrl_remove
+ * libxl_device_usbctrl_destroy
+ */
+
+#define DEFINE_DEVICE_REMOVE_EXT(type, removedestroy, f)                \
+    int libxl_device_##type##_##removedestroy(libxl_ctx *ctx,           \
+        uint32_t domid, libxl_device_##type *type,                      \
+        const libxl_asyncop_how *ao_how)                                \
+    {                                                                   \
+        AO_CREATE(ctx, domid, ao_how);                                  \
+        libxl__device *device;                                          \
+        libxl__ao_device *aodev;                                        \
+        int rc;                                                         \
+                                                                        \
+        GCNEW(device);                                                  \
+        rc = libxl__device_from_##type(gc, domid, type, device);        \
+        if (rc != 0) goto out;                                          \
+                                                                        \
+        GCNEW(aodev);                                                   \
+        libxl__prepare_ao_device(ao, aodev);                            \
+        aodev->action = LIBXL__DEVICE_ACTION_REMOVE;                    \
+        aodev->dev = device;                                            \
+        aodev->callback = device_addrm_aocomplete;                      \
+        aodev->force = f;                                               \
+        libxl__initiate_device_##type##_remove(egc, aodev);             \
+                                                                        \
+    out:                                                                \
+        if (rc) return AO_CREATE_FAIL(rc);                              \
+        return AO_INPROGRESS;                                           \
+    }
+
+
+DEFINE_DEVICE_REMOVE_EXT(usbctrl, remove, 0)
+DEFINE_DEVICE_REMOVE_EXT(usbctrl, destroy, 1)
+
+#undef DEFINE_DEVICE_REMOVE_EXT
+
+/******************************************************************************/
+
 /* Macro for defining device addition functions in a compact way */
 /* The following functions are defined:
  * libxl_device_disk_add
  * libxl_device_nic_add
  * libxl_device_vtpm_add
+ * libxl_device_usbctrl_add
+ * libxl_device_usb_add
  */
 
 #define DEFINE_DEVICE_ADD(type)                                         \
@@ -4254,6 +4297,12 @@ DEFINE_DEVICE_ADD(nic)
 /* vtpm */
 DEFINE_DEVICE_ADD(vtpm)
 
+/* usbctrl */
+DEFINE_DEVICE_ADD(usbctrl)
+
+/* usb */
+DEFINE_DEVICE_ADD(usb)
+
 #undef DEFINE_DEVICE_ADD
 
 /******************************************************************************/
@@ -6795,6 +6844,10 @@ int libxl_retrieve_domain_configuration(libxl_ctx *ctx, uint32_t domid,
 
     MERGE(pci, pcidevs, COMPARE_PCI, {});
 
+    MERGE(usbctrl, usbctrls, COMPARE_USBCTRL, {});
+
+    MERGE(usb, usbs, COMPARE_USB, {});
+
     /* Take care of removable device. We maintain invariant in the
      * insert / remove operation so that:
      * 1. if xenstore is "empty" while JSON is not, the result
diff --git a/tools/libxl/libxl.h b/tools/libxl/libxl.h
index 615b1de..594e23d 100644
--- a/tools/libxl/libxl.h
+++ b/tools/libxl/libxl.h
@@ -123,6 +123,12 @@
 #define LIBXL_HAVE_DOMAIN_NODEAFFINITY 1
 
 /*
+ * LIBXL_HAVE_PVUSB indicates functions for plugging in USB devices
+ * through pvusb -- both hotplug and at domain creation time..
+ */
+#define LIBXL_HAVE_PVUSB 1
+
+/*
  * LIBXL_HAVE_BUILDINFO_HVM_VENDOR_DEVICE indicates that the
  * libxl_vendor_device field is present in the hvm sections of
  * libxl_domain_build_info. This field tells libxl which
@@ -1411,6 +1417,74 @@ int libxl_cdrom_insert(libxl_ctx *ctx, uint32_t domid, libxl_device_disk *disk,
                        const libxl_asyncop_how *ao_how)
                        LIBXL_EXTERNAL_CALLERS_ONLY;
 
+/*
+ * USB
+ *
+ * For each device removed or added, one of these protocols is available:
+ * - PV (i.e., PVUSB)
+ * - DEVICEMODEL (i.e, qemu)
+ *
+ * PV is available for either PV or HVM domains.  DEVICEMODEL is only
+ * available for HVM domains.  The caller can additionally specify
+ * "AUTO", in which case the library will try to determine the best
+ * protocol automatically.
+ *
+ * At the moment, the only protocol implemented is PV.
+ *
+ * One can add/remove USB controllers to/from guest, and attach/detach USB
+ * devices to/from USB controllers.
+ *
+ * To add USB controllers and USB devices, one can adding USB controllers
+ * first and then attaching USB devices to some USB controller, or adding
+ * USB devices to guest directly, it will automatically create a USB
+ * controller for USB devices to attach.
+ *
+ * To remove USB controllers or USB devices, one can remove USB devices
+ * under USB controller one by one and then remove USB controller, or
+ * remove USB controller directly, it will remove all USB devices under
+ * it automatically.
+ *
+ */
+/* USB Controllers*/
+int libxl_device_usbctrl_add(libxl_ctx *ctx, uint32_t domid,
+                             libxl_device_usbctrl *usbctrl,
+                             const libxl_asyncop_how *ao_how)
+                             LIBXL_EXTERNAL_CALLERS_ONLY;
+
+int libxl_device_usbctrl_remove(libxl_ctx *ctx, uint32_t domid,
+                                libxl_device_usbctrl *usbctrl,
+                                const libxl_asyncop_how *ao_how)
+                                LIBXL_EXTERNAL_CALLERS_ONLY;
+
+int libxl_device_usbctrl_destroy(libxl_ctx *ctx, uint32_t domid,
+                                 libxl_device_usbctrl *usbctrl,
+                                 const libxl_asyncop_how *ao_how)
+                                 LIBXL_EXTERNAL_CALLERS_ONLY;
+
+libxl_device_usbctrl *libxl_device_usbctrl_list(libxl_ctx *ctx,
+                                                uint32_t domid, int *num);
+
+void libxl_device_usbctrl_list_free(libxl_device_usbctrl *list, int nr);
+
+
+int libxl_device_usbctrl_getinfo(libxl_ctx *ctx, uint32_t domid,
+                                 libxl_device_usbctrl *usbctrl,
+                                 libxl_usbctrlinfo *usbctrlinfo);
+
+/* USB Devices */
+int libxl_device_usb_add(libxl_ctx *ctx, uint32_t domid, libxl_device_usb *usb,
+                         const libxl_asyncop_how *ao_how)
+                         LIBXL_EXTERNAL_CALLERS_ONLY;
+
+int libxl_device_usb_remove(libxl_ctx *ctx, uint32_t domid, libxl_device_usb *usb,
+                            const libxl_asyncop_how *ao_how)
+                            LIBXL_EXTERNAL_CALLERS_ONLY;
+
+libxl_device_usb *
+libxl_device_usb_list(libxl_ctx *ctx, uint32_t domid, int *num);
+
+void libxl_device_usb_list_free(libxl_device_usb *list, int nr);
+
 /* Network Interfaces */
 int libxl_device_nic_add(libxl_ctx *ctx, uint32_t domid, libxl_device_nic *nic,
                          const libxl_asyncop_how *ao_how)
diff --git a/tools/libxl/libxl_device.c b/tools/libxl/libxl_device.c
index 8bb5e93..6715c16 100644
--- a/tools/libxl/libxl_device.c
+++ b/tools/libxl/libxl_device.c
@@ -676,7 +676,10 @@ void libxl__devices_destroy(libxl__egc *egc, libxl__devices_remove_state *drs)
                 aodev->action = LIBXL__DEVICE_ACTION_REMOVE;
                 aodev->dev = dev;
                 aodev->force = drs->force;
-                libxl__initiate_device_remove(egc, aodev);
+                if (dev->backend_kind == LIBXL__DEVICE_KIND_VUSB)
+                    libxl__initiate_device_usbctrl_remove(egc, aodev);
+                else
+                    libxl__initiate_device_remove(egc, aodev);
             }
         }
     }
diff --git a/tools/libxl/libxl_internal.h b/tools/libxl/libxl_internal.h
index a3a5cc1..1aa0aaf 100644
--- a/tools/libxl/libxl_internal.h
+++ b/tools/libxl/libxl_internal.h
@@ -2569,6 +2569,14 @@ _hidden void libxl__device_vtpm_add(libxl__egc *egc, uint32_t domid,
                                    libxl_device_vtpm *vtpm,
                                    libxl__ao_device *aodev);
 
+_hidden void libxl__device_usbctrl_add(libxl__egc *egc, uint32_t domid,
+                                       libxl_device_usbctrl *usbctrl,
+                                       libxl__ao_device *aodev);
+
+_hidden void libxl__device_usb_add(libxl__egc *egc, uint32_t domid,
+                                   libxl_device_usb *usb,
+                                   libxl__ao_device *aodev);
+
 /* Internal function to connect a vkb device */
 _hidden int libxl__device_vkb_add(libxl__gc *gc, uint32_t domid,
                                   libxl_device_vkb *vkb);
@@ -2601,6 +2609,13 @@ _hidden void libxl__wait_device_connection(libxl__egc*,
 _hidden void libxl__initiate_device_remove(libxl__egc *egc,
                                            libxl__ao_device *aodev);
 
+_hidden int libxl__device_from_usbctrl(libxl__gc *gc, uint32_t domid,
+                               libxl_device_usbctrl *usbctrl,
+                               libxl__device *device);
+
+_hidden void libxl__initiate_device_usbctrl_remove(libxl__egc *egc,
+                                                   libxl__ao_device *aodev);
+
 /*
  * libxl__get_hotplug_script_info returns the args and env that should
  * be passed to the hotplug script for the requested device.
@@ -3958,6 +3973,9 @@ static inline void libxl__update_config_vtpm(libxl__gc *gc,
 #define COMPARE_PCI(a, b) ((a)->func == (b)->func &&    \
                            (a)->bus == (b)->bus &&      \
                            (a)->dev == (b)->dev)
+#define COMPARE_USB(a, b) ((a)->ctrl == (b)->ctrl && \
+                           (a)->port == (b)->port)
+#define COMPARE_USBCTRL(a, b) ((a)->devid == (b)->devid)
 
 /* DEVICE_ADD
  *
diff --git a/tools/libxl/libxl_osdeps.h b/tools/libxl/libxl_osdeps.h
index d9661c9..802c762 100644
--- a/tools/libxl/libxl_osdeps.h
+++ b/tools/libxl/libxl_osdeps.h
@@ -24,6 +24,8 @@
 #define _GNU_SOURCE
 
 #if defined(__NetBSD__)
+#define SYSFS_USB_DEV          "/sys/bus/usb/devices"
+#define SYSFS_USBBACK_DRIVER   "/kern/xen/usb"
 #define SYSFS_PCI_DEV          "/sys/bus/pci/devices"
 #define SYSFS_PCIBACK_DRIVER   "/kern/xen/pci"
 #define NETBACK_NIC_NAME       "xvif%ui%d"
@@ -31,6 +33,8 @@
 #elif defined(__OpenBSD__)
 #include <util.h>
 #elif defined(__linux__)
+#define SYSFS_USB_DEV          "/sys/bus/usb/devices"
+#define SYSFS_USBBACK_DRIVER   "/sys/bus/usb/drivers/usbback"
 #define SYSFS_PCI_DEV          "/sys/bus/pci/devices"
 #define SYSFS_PCIBACK_DRIVER   "/sys/bus/pci/drivers/pciback"
 #define NETBACK_NIC_NAME       "vif%u.%d"
@@ -38,6 +42,8 @@
 #elif defined(__sun__)
 #include <stropts.h>
 #elif defined(__FreeBSD__)
+#define SYSFS_USB_DEV          "/dev/null"
+#define SYSFS_USBBACK_DRIVER   "/dev/null"
 #define SYSFS_PCI_DEV          "/dev/null"
 #define SYSFS_PCIBACK_DRIVER   "/dev/null"
 #define NETBACK_NIC_NAME       "xnb%u.%d"
@@ -45,6 +51,13 @@
 #include <sys/endian.h>
 #endif
 
+#ifndef SYSFS_USBBACK_DRIVER
+#error define SYSFS_USBBACK_DRIVER for your platform
+#endif
+#ifndef SYSFS_USB_DEV
+#error define SYSFS_USB_DEV for your platform
+#endif
+
 #ifndef SYSFS_PCIBACK_DRIVER
 #error define SYSFS_PCIBACK_DRIVER for your platform
 #endif
diff --git a/tools/libxl/libxl_pvusb.c b/tools/libxl/libxl_pvusb.c
new file mode 100644
index 0000000..aa1a653
--- /dev/null
+++ b/tools/libxl/libxl_pvusb.c
@@ -0,0 +1,1451 @@
+/*
+ * Copyright (C) 2015 SUSE LINUX Products GmbH, Nuernberg, Germany.
+ * Author Chunyan Liu <cyliu@suse.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published
+ * by the Free Software Foundation; version 2.1 only. with the special
+ * exception on linking described in file LICENSE.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Lesser General Public License for more details.
+ */
+
+#include "libxl_osdeps.h" /* must come before any other headers */
+
+#include "libxl_internal.h"
+#include <inttypes.h>
+
+#define USBBACK_INFO_PATH "/libxl/usbback"
+
+#define USBHUB_CLASS_CODE 9
+
+static int libxl__device_usbctrl_setdefault(libxl__gc *gc, uint32_t domid,
+                                            libxl_device_usbctrl *usbctrl)
+{
+    int rc;
+    libxl_domain_type domtype = libxl__domain_type(gc, domid);
+
+    if (!usbctrl->version)
+        usbctrl->version = 2;
+
+    if (!usbctrl->ports)
+        usbctrl->ports = 8;
+
+    if (usbctrl->type == LIBXL_USBCTRL_TYPE_AUTO) {
+        if (domtype == LIBXL_DOMAIN_TYPE_PV) {
+            usbctrl->type = LIBXL_USBCTRL_TYPE_PV;
+        } else if (domtype == LIBXL_DOMAIN_TYPE_HVM) {
+            /* FIXME: See if we can detect PV frontend */
+            usbctrl->type = LIBXL_USBCTRL_TYPE_DEVICEMODEL;
+        }
+    }
+
+    rc = libxl__resolve_domid(gc, usbctrl->backend_domname,
+                              &usbctrl->backend_domid);
+    return rc;
+}
+
+int libxl__device_from_usbctrl(libxl__gc *gc, uint32_t domid,
+                               libxl_device_usbctrl *usbctrl,
+                               libxl__device *device)
+{
+    device->backend_devid   = usbctrl->devid;
+    device->backend_domid   = usbctrl->backend_domid;
+    device->backend_kind    = LIBXL__DEVICE_KIND_VUSB;
+    device->devid           = usbctrl->devid;
+    device->domid           = domid;
+    device->kind            = LIBXL__DEVICE_KIND_VUSB;
+
+    return 0;
+}
+
+/* Add usbctrl information to xenstore.
+ *
+ * Adding a usb controller will add a new 'vusb' device in xenstore, and
+ * add corresponding frontend, backend information to it. According to
+ * "update_json", decide wether to update json config file.
+ */
+static int libxl__device_usbctrl_add_xenstore(libxl__gc *gc, uint32_t domid,
+                                              libxl_device_usbctrl *usbctrl,
+                                              bool update_json)
+{
+    libxl__device *device;
+    flexarray_t *front;
+    flexarray_t *back;
+    xs_transaction_t t = XBT_NULL;
+    int i, rc;
+    libxl_domain_config d_config;
+    libxl_device_usbctrl usbctrl_saved;
+    libxl__domain_userdata_lock *lock = NULL;
+
+    libxl_domain_config_init(&d_config);
+    libxl_device_usbctrl_init(&usbctrl_saved);
+    libxl_device_usbctrl_copy(CTX, &usbctrl_saved, usbctrl);
+
+    GCNEW(device);
+    rc = libxl__device_from_usbctrl(gc, domid, usbctrl, device);
+    if (rc) goto out;
+
+    front = flexarray_make(gc, 4, 1);
+    back = flexarray_make(gc, 12, 1);
+
+    flexarray_append_pair(back, "frontend-id", GCSPRINTF("%d", domid));
+    flexarray_append_pair(back, "online", "1");
+    flexarray_append_pair(back, "state", "1");
+    flexarray_append_pair(back, "type",
+                    (char *)libxl_usbctrl_type_to_string(usbctrl->type));
+    flexarray_append_pair(back, "usb-ver", GCSPRINTF("%d", usbctrl->version));
+    flexarray_append_pair(back, "num-ports", GCSPRINTF("%d", usbctrl->ports));
+    flexarray_append_pair(back, "port", "");
+    for (i = 0; i < usbctrl->ports; i++)
+        flexarray_append_pair(back, GCSPRINTF("port/%d", i + 1), "");
+
+    flexarray_append_pair(front, "backend-id",
+                          GCSPRINTF("%d", usbctrl->backend_domid));
+    flexarray_append_pair(front, "state", "1");
+
+    if (update_json) {
+        lock = libxl__lock_domain_userdata(gc, domid);
+        if (!lock) {
+            rc = ERROR_LOCK_FAIL;
+            goto out;
+        }
+
+        rc = libxl__get_domain_configuration(gc, domid, &d_config);
+        if (rc) goto out;
+
+        DEVICE_ADD(usbctrl, usbctrls, domid, &usbctrl_saved,
+                   COMPARE_USBCTRL, &d_config);
+    }
+
+    for (;;) {
+        rc = libxl__xs_transaction_start(gc, &t);
+        if (rc) goto out;
+
+        rc = libxl__device_exists(gc, t, device);
+        if (rc < 0) goto out;
+        if (rc == 1) {
+            /* already exists in xenstore */
+            LOG(ERROR, "device already exists in xenstore");
+            rc = ERROR_DEVICE_EXISTS;
+            goto out;
+        }
+
+        if (update_json) {
+            rc = libxl__set_domain_configuration(gc, domid, &d_config);
+            if (rc) goto out;
+        }
+
+        libxl__device_generic_add(gc, t, device,
+                          libxl__xs_kvs_of_flexarray(gc, back, back->count),
+                          libxl__xs_kvs_of_flexarray(gc, front, front->count),
+                          NULL);
+
+        rc = libxl__xs_transaction_commit(gc, &t);
+        if (!rc) break;
+        if (rc < 0) goto out;
+    }
+
+out:
+    libxl__xs_transaction_abort(gc, &t);
+    if (lock) libxl__unlock_domain_userdata(lock);
+    libxl_device_usbctrl_dispose(&usbctrl_saved);
+    libxl_domain_config_dispose(&d_config);
+    return rc;
+}
+
+/* AO operation to add a usb controller.
+ *
+ * Generally, it does:
+ * 1) fill in necessary usb controler information with default value
+ * 2) write usb controller frontend/backend info to xenstore, update json
+ *    config file if necessary.
+ * 3) wait for device connection. PVUSB frontend and backend driver will
+ *    probe xenstore paths and build connection between frontend and backend.
+ *
+ * Before calling this function, aodev should be properly filled:
+ * aodev->ao, aodev->callback, aodev->update_json, ...
+ */
+void libxl__device_usbctrl_add(libxl__egc *egc, uint32_t domid,
+                               libxl_device_usbctrl *usbctrl,
+                               libxl__ao_device *aodev)
+{
+    STATE_AO_GC(aodev->ao);
+    libxl__device *device;
+    int rc;
+
+    rc = libxl__device_usbctrl_setdefault(gc, domid, usbctrl);
+    if (rc < 0) goto out;
+
+    if (usbctrl->devid == -1) {
+        usbctrl->devid = libxl__device_nextid(gc, domid, "vusb");
+        if (usbctrl->devid < 0) {
+            rc = ERROR_FAIL;
+            goto out;
+        }
+    }
+
+    if (usbctrl->type != LIBXL_USBCTRL_TYPE_PV) {
+        LOG(ERROR, "Unsupported USB controller type");
+        rc = ERROR_FAIL;
+        goto out;
+    }
+
+    rc = libxl__device_usbctrl_add_xenstore(gc, domid, usbctrl,
+                                            aodev->update_json);
+    if (rc) goto out;
+
+    GCNEW(device);
+    rc = libxl__device_from_usbctrl(gc, domid, usbctrl, device);
+    if (rc) goto out;
+
+    aodev->dev = device;
+    aodev->action = LIBXL__DEVICE_ACTION_ADD;
+    libxl__wait_device_connection(egc, aodev);
+    return;
+
+out:
+    aodev->rc = rc;
+    if (rc) aodev->callback(egc, aodev);
+    return;
+}
+
+static int
+libxl__device_usb_list_for_usbctrl(libxl__gc *gc, uint32_t domid,
+                                   libxl_devid usbctrl,
+                                   libxl_device_usb **usbs, int *num);
+
+static int
+libxl__device_usb_remove(libxl__gc *gc, uint32_t domid, libxl_device_usb *usb);
+
+/* AO function to remove a usb controller.
+ *
+ * Generally, it does:
+ * 1) check if the usb controller exists or not
+ * 2) remove all usb devices under controller
+ * 3) remove usb controller information from xenstore
+ *
+ * Before calling this function, aodev should be properly filled:
+ * aodev->ao, aodev->dev, aodev->callback, ...
+ */
+void libxl__initiate_device_usbctrl_remove(libxl__egc *egc,
+                                           libxl__ao_device *aodev)
+{
+    STATE_AO_GC(aodev->ao);
+    libxl_ctx *ctx = CTX;
+    libxl_device_usb *usbs = NULL;
+    int numusb = 0;
+    int i, rc;
+    uint32_t domid = ao->domid;
+    int usbctrl_devid = aodev->dev->devid;
+    libxl_device_usbctrl usbctrl;
+    libxl_usbctrlinfo usbctrlinfo;
+
+    libxl_device_usbctrl_init(&usbctrl);
+    libxl_usbctrlinfo_init(&usbctrlinfo);
+    usbctrl.devid = usbctrl_devid;
+
+    rc = libxl_device_usbctrl_getinfo(ctx, domid, &usbctrl, &usbctrlinfo);
+    if (rc) goto out;
+
+    if (usbctrlinfo.type != LIBXL_USBCTRL_TYPE_PV) {
+        LOG(ERROR, "Unsupported USB controller type");
+        rc = ERROR_FAIL;
+        goto out;
+    }
+
+    /* Remove usb devices first */
+    rc  = libxl__device_usb_list_for_usbctrl(gc, domid, usbctrl_devid,
+                                             &usbs, &numusb);
+    if (rc) goto out;
+
+    for (i = 0; i < numusb; i++) {
+        if (libxl__device_usb_remove(gc, domid, &usbs[i])) {
+            LOG(ERROR, "libxl__device_usb_remove failed");
+            rc = ERROR_FAIL;
+            goto out;
+        }
+    }
+
+    libxl_device_usbctrl_dispose(&usbctrl);
+    libxl_usbctrlinfo_dispose(&usbctrlinfo);
+
+    /* Remove usbctrl */
+    return libxl__initiate_device_remove(egc, aodev);
+
+out:
+    libxl_device_usbctrl_dispose(&usbctrl);
+    libxl_usbctrlinfo_dispose(&usbctrlinfo);
+    aodev->rc = rc;
+    if (rc) aodev->callback(egc, aodev);
+    return;
+}
+
+libxl_device_usbctrl *
+libxl_device_usbctrl_list(libxl_ctx *ctx, uint32_t domid, int *num)
+{
+    GC_INIT(ctx);
+    libxl_device_usbctrl *usbctrls = NULL;
+    char *path = NULL;
+    char **entry = NULL;
+    unsigned int nentries = 0;
+
+    *num = 0;
+
+    path = GCSPRINTF("%s/device/vusb",
+                     libxl__xs_get_dompath(gc, domid));
+    entry = libxl__xs_directory(gc, XBT_NULL, path, &nentries);
+
+    if (entry && nentries) {
+        usbctrls = libxl__zalloc(NOGC, sizeof(*usbctrls) * nentries);
+        libxl_device_usbctrl *usbctrl;
+        libxl_device_usbctrl *end = usbctrls + nentries;
+        for (usbctrl = usbctrls; usbctrl < end;
+             usbctrl++, entry++, (*num)++) {
+            const char *tmp, *be_path;
+            int rc;
+            const char *fe_path = GCSPRINTF("%s/%s", path, *entry);
+
+            libxl_device_usbctrl_init(usbctrl);
+            usbctrl->devid = atoi(*entry);
+
+#define READ_SUBPATH(path, subpath) ({                                  \
+        rc = libxl__xs_read_checked(gc, XBT_NULL,                       \
+                                    GCSPRINTF("%s/" subpath, path),     \
+                                    &tmp);                              \
+        if (rc) goto outerr;                                            \
+        (char *)tmp;                                                    \
+    })
+            be_path = READ_SUBPATH(fe_path, "backend");
+            usbctrl->backend_domid = atoi(READ_SUBPATH(fe_path, "backend-id"));
+            usbctrl->version = atoi(READ_SUBPATH(be_path, "usb-ver"));
+            usbctrl->ports = atoi(READ_SUBPATH(be_path, "num-ports"));
+            libxl_usbctrl_type_from_string(READ_SUBPATH(be_path, "type"),
+                                           &usbctrl->type);
+#undef READ_SUBPATH
+       }
+    }
+
+    goto out;
+
+outerr:
+    LOG(ERROR, "Unable to list USB Controllers");
+    libxl_device_usbctrl_list_free(usbctrls, *num);
+    *num = 0;
+    usbctrls = NULL;
+
+out:
+    GC_FREE;
+    return usbctrls;
+}
+
+int libxl_device_usbctrl_getinfo(libxl_ctx *ctx, uint32_t domid,
+                                libxl_device_usbctrl *usbctrl,
+                                libxl_usbctrlinfo *usbctrlinfo)
+{
+    GC_INIT(ctx);
+    const char *dompath, *fe_path, *be_path, *tmp;
+    int rc;
+
+    usbctrlinfo->devid = usbctrl->devid;
+
+#define READ_SUBPATH(path, subpath) ({                                  \
+        rc = libxl__xs_read_checked(gc, XBT_NULL,                       \
+                                    GCSPRINTF("%s/" subpath, path),     \
+                                    &tmp);                              \
+        if (rc) goto out;                                               \
+        (char *)tmp;                                                    \
+    })
+
+    dompath = libxl__xs_get_dompath(gc, domid);
+    fe_path = GCSPRINTF("%s/device/vusb/%d", dompath, usbctrl->devid);
+    be_path = READ_SUBPATH(fe_path, "backend");
+
+    usbctrlinfo->backend = libxl__strdup(NOGC, be_path);
+
+    tmp = READ_SUBPATH(fe_path, "backend-id");
+    usbctrlinfo->backend_id = tmp ? strtoul(tmp, NULL, 10) : -1;
+    tmp = READ_SUBPATH(fe_path, "state");
+    usbctrlinfo->state = tmp ? strtoul(tmp, NULL, 10) : -1;
+    tmp = READ_SUBPATH(fe_path, "event-channel");
+    usbctrlinfo->evtch = tmp ? strtoul(tmp, NULL, 10) : -1;
+    tmp = READ_SUBPATH(fe_path, "urb-ring-ref");
+    usbctrlinfo->ref_urb = tmp ? strtoul(tmp, NULL, 10) : -1;
+    tmp = READ_SUBPATH(fe_path, "conn-ring-ref");
+    usbctrlinfo->ref_conn = tmp ? strtoul(tmp, NULL, 10) : -1;
+    tmp = READ_SUBPATH(be_path, "frontend");
+    usbctrlinfo->frontend = libxl__strdup(NOGC, tmp);
+    tmp = READ_SUBPATH(be_path, "frontend-id");
+    usbctrlinfo->frontend_id = tmp ? strtoul(tmp, NULL, 10) : -1;
+    tmp = READ_SUBPATH(be_path, "num-ports");
+    usbctrlinfo->ports = tmp ? strtoul(tmp, NULL, 10) : -1;
+    tmp = READ_SUBPATH(be_path, "usb-ver");
+    usbctrlinfo->version = tmp ? strtoul(tmp, NULL, 10) : -1;
+    tmp = READ_SUBPATH(be_path, "type");
+    libxl_usbctrl_type_from_string(tmp, &usbctrlinfo->type);
+
+#undef READ_SUBPATH
+
+    rc = 0;
+
+out:
+    GC_FREE;
+    return rc;
+}
+
+int libxl_devid_to_device_usbctrl(libxl_ctx *ctx,
+                                  uint32_t domid,
+                                  int devid,
+                                  libxl_device_usbctrl *usbctrl)
+{
+    libxl_device_usbctrl *usbctrls;
+    int nb = 0;
+    int i, rc;
+
+    usbctrls = libxl_device_usbctrl_list(ctx, domid, &nb);
+    if (!usbctrls)
+        return ERROR_FAIL;
+
+    rc = ERROR_FAIL;
+    for (i = 0; i < nb; i++) {
+        if (devid == usbctrls[i].devid) {
+            libxl_device_usbctrl_copy(ctx, usbctrl, &usbctrls[i]);
+            rc = 0;
+            break;
+        }
+    }
+
+    libxl_device_usbctrl_list_free(usbctrls, nb);
+    return rc;
+}
+
+static char *usb_busaddr_to_busid(libxl__gc *gc, int bus, int addr)
+{
+    struct dirent *de;
+    DIR *dir;
+    char *busid = NULL;
+
+    /* invalid hostbus or hostaddr */
+    if (bus < 1 || addr < 1)
+        return NULL;
+
+    if (!(dir = opendir(SYSFS_USB_DEV)))
+        return NULL;
+
+    while ((de = readdir(dir))) {
+        char *filename;
+        void *buf;
+        int busnum = -1;
+        int devnum = -1;
+
+        if (!de->d_name)
+            continue;
+
+        filename = GCSPRINTF(SYSFS_USB_DEV"/%s/devnum", de->d_name);
+        if (!libxl__read_sysfs_file_contents(gc, filename, &buf, NULL))
+            devnum = atoi((char *)buf);
+
+        filename = GCSPRINTF(SYSFS_USB_DEV"/%s/busnum", de->d_name);
+        if (!libxl__read_sysfs_file_contents(gc, filename, &buf, NULL))
+            busnum = atoi((char *)buf);
+
+        if (bus == busnum && addr == devnum) {
+            busid = libxl__strdup(gc, de->d_name);
+            break;
+        }
+    }
+
+    closedir(dir);
+    return busid;
+}
+
+static int usb_busaddr_from_busid(libxl__gc *gc, const char *busid,
+                                  uint8_t *bus, uint8_t *addr)
+{
+    char *filename;
+    void *buf;
+
+    filename = GCSPRINTF(SYSFS_USB_DEV"/%s/busnum", busid);
+    if (!libxl__read_sysfs_file_contents(gc, filename, &buf, NULL))
+        *bus = atoi((char *)buf);
+    else
+        return ERROR_FAIL;
+
+    filename = GCSPRINTF(SYSFS_USB_DEV"/%s/devnum", busid);
+    if (!libxl__read_sysfs_file_contents(gc, filename, &buf, NULL))
+        *addr = atoi((char *)buf);
+    else
+        return ERROR_FAIL;
+
+    return 0;
+}
+
+static int
+get_assigned_devices(libxl__gc *gc,
+                     libxl_device_usb **list, int *num)
+{
+    char **domlist;
+    unsigned int nd = 0, i, j, k;
+    int rc;
+
+    *list = NULL;
+    *num = 0;
+
+    domlist = libxl__xs_directory(gc, XBT_NULL, "/local/domain", &nd);
+    for (i = 0; i < nd; i++) {
+        char *path, **ctrl_list;
+        unsigned int nc = 0;
+
+        path = GCSPRINTF("/local/domain/%s/device/vusb", domlist[i]);
+        ctrl_list = libxl__xs_directory(gc, XBT_NULL, path, &nc);
+
+        for (j = 0; j < nc; j++) {
+            const char *be_path, *num_ports;
+            int nport;
+
+            rc = libxl__xs_read_checked(gc, XBT_NULL,
+                          GCSPRINTF("%s/%s/backend", path, ctrl_list[j]),
+                          &be_path);
+            if (rc) goto out;
+
+            rc = libxl__xs_read_checked(gc, XBT_NULL,
+                                        GCSPRINTF("%s/num-ports", be_path),
+                                        &num_ports);
+            if (rc) goto out;
+
+            nport = atoi(num_ports);
+            for (k = 0; k < nport; k++) {
+                libxl_device_usb *usb;
+                const char *portpath, *busid;
+
+                portpath = GCSPRINTF("%s/port/%d", be_path, k + 1);
+                busid = libxl__xs_read(gc, XBT_NULL, portpath);
+                /* If there is USB device attached, add it to list */
+                if (busid && strcmp(busid, "")) {
+                    GCREALLOC_ARRAY(*list, *num + 1);
+                    usb = *list + *num;
+                    (*num)++;
+                    libxl_device_usb_init(usb);
+                    usb->ctrl = atoi(ctrl_list[j]);
+                    usb->port = k + 1;
+                    rc = usb_busaddr_from_busid(gc, busid,
+                                                &usb->u.hostdev.hostbus,
+                                                &usb->u.hostdev.hostaddr);
+                    if (rc) goto out;
+                }
+            }
+        }
+    }
+
+    rc = 0;
+
+out:
+    if (rc) {
+        *list = NULL;
+        *num = 0;
+    }
+    return rc;
+}
+
+static bool is_usbdev_in_array(libxl_device_usb *usbs, int num,
+                               libxl_device_usb *usb)
+{
+    int i;
+
+    for (i = 0; i < num; i++) {
+        if (usbs[i].u.hostdev.hostbus == usb->u.hostdev.hostbus &&
+            usbs[i].u.hostdev.hostaddr == usb->u.hostdev.hostaddr)
+            return true;
+    }
+
+    return false;
+}
+
+/* check if USB device is already assigned to a domain */
+/* check if USB device type is assignable */
+static bool is_usb_assignable(libxl__gc *gc, libxl_device_usb *usb)
+{
+    int classcode;
+    char *filename;
+    void *buf = NULL;
+    char *busid = NULL;
+
+    busid = usb_busaddr_to_busid(gc, usb->u.hostdev.hostbus,
+                                 usb->u.hostdev.hostaddr);
+    if (!busid) return false;
+
+    filename = GCSPRINTF(SYSFS_USB_DEV"/%s/bDeviceClass", busid);
+    if (libxl__read_sysfs_file_contents(gc, filename, &buf, NULL))
+        return false;
+
+    classcode = atoi(buf);
+    return classcode != USBHUB_CLASS_CODE;
+}
+
+/* get usb devices under certain usb controller */
+static int
+libxl__device_usb_list_for_usbctrl(libxl__gc *gc, uint32_t domid,
+                                   libxl_devid usbctrl,
+                                   libxl_device_usb **usbs, int *num)
+{
+    const char *fe_path, *be_path, *num_devs;
+    int n, i, rc;
+
+    *usbs = NULL;
+    *num = 0;
+
+    fe_path = GCSPRINTF("%s/device/vusb/%d",
+                        libxl__xs_get_dompath(gc, domid), usbctrl);
+
+    rc = libxl__xs_read_checked(gc, XBT_NULL,
+                                GCSPRINTF("%s/backend", fe_path),
+                                &be_path);
+    if (rc) return rc;
+
+    rc = libxl__xs_read_checked(gc, XBT_NULL,
+                                GCSPRINTF("%s/num-ports", be_path),
+                                &num_devs);
+    if (rc) return rc;
+
+    n = atoi(num_devs);
+
+    for (i = 0; i < n; i++) {
+        char *busid;
+        libxl_device_usb *usb;
+
+        busid = libxl__xs_read(gc, XBT_NULL,
+                               GCSPRINTF("%s/port/%d", be_path, i + 1));
+        if (busid && strcmp(busid, "")) {
+            GCREALLOC_ARRAY(*usbs, *num + 1);
+            usb = *usbs + *num;
+            (*num)++;
+            libxl_device_usb_init(usb);
+            usb->ctrl = usbctrl;
+            usb->port = i + 1;
+            rc = usb_busaddr_from_busid(gc, busid,
+                                        &usb->u.hostdev.hostbus,
+                                        &usb->u.hostdev.hostaddr);
+            if (rc) goto out;
+        }
+    }
+
+    rc = 0;
+
+out:
+    if (rc) {
+        *usbs = NULL;
+        *num = 0;
+    }
+    return rc;
+}
+
+/* get all usb devices of the domain */
+libxl_device_usb *
+libxl_device_usb_list(libxl_ctx *ctx, uint32_t domid, int *num)
+{
+    GC_INIT(ctx);
+    char *path;
+    char **usbctrls;
+    unsigned int nc = 0;
+    int i, j;
+    libxl_device_usb *usbs = NULL;
+
+    *num = 0;
+
+    path = GCSPRINTF("%s/device/vusb",
+                        libxl__xs_get_dompath(gc, domid));
+    usbctrls = libxl__xs_directory(gc, XBT_NULL, path, &nc);
+
+    for (i = 0; i < nc; i++) {
+        int nd = 0;
+        libxl_device_usb *tmp = NULL;
+        libxl__device_usb_list_for_usbctrl(gc, domid,
+                                           atoi(usbctrls[i]), &tmp, &nd);
+        if (!nd) continue;
+
+        usbs = libxl__realloc(NOGC, usbs, sizeof(*usbs) * (*num + nd));
+        for (j = 0; j < nd; j++) {
+            libxl_device_usb_copy(ctx, usbs + *num, tmp + j);
+            (*num)++;
+        }
+    }
+
+    GC_FREE;
+    return usbs;
+}
+
+/* find first unused controller:port and give that to usb device */
+static int
+libxl__device_usb_set_default_usbctrl(libxl__gc *gc, uint32_t domid,
+                                      libxl_device_usb *usb)
+{
+    libxl_ctx *ctx = CTX;
+    libxl_device_usbctrl *usbctrls = NULL;
+    int numctrl = 0;
+    int i, j, rc;
+
+    usbctrls = libxl_device_usbctrl_list(ctx, domid, &numctrl);
+    if (!numctrl) {
+        rc = ERROR_FAIL;
+        goto out;
+    }
+
+    rc = ERROR_FAIL;
+    for (i = 0; i < numctrl; i++) {
+        for (j = 0; j < usbctrls[i].ports; j++) {
+            char *path, *tmp;
+
+            path = GCSPRINTF("%s/backend/vusb/%d/%d/port/%d",
+                             libxl__xs_get_dompath(gc, LIBXL_TOOLSTACK_DOMID),
+                             domid, usbctrls[i].devid, j + 1);
+            tmp = libxl__xs_read(gc, XBT_NULL, path);
+            if (tmp && !strcmp(tmp, "")) {
+                usb->ctrl = usbctrls[i].devid;
+                usb->port = j + 1;
+                rc = 0;
+                goto out;
+            }
+        }
+    }
+
+out:
+    libxl_device_usbctrl_list_free(usbctrls, numctrl);
+    return rc;
+}
+
+/* Fill in usb information with default value.
+ *
+ * Generally, it does:
+ * 1) if "controller" is not specified:
+ *    - if "port" is not specified, try to find an available controller:port,
+ *      if found, use that; otherwise, create a new controller, use this
+ *      controller and its first port
+ *    - if "port" is specified, report error.
+ * 2) if "controller" is specified, but port is not specified:
+ *    try to find an available port under this controller, if found, use
+ *    that, otherwise, report error.
+ * 3) if both "controller" and "port" are specified:
+ *    check the controller:port is available, if not, report error.
+ */
+static int libxl__device_usb_setdefault(libxl__gc *gc, uint32_t domid,
+                                        libxl_device_usb *usb,
+                                        bool update_json)
+{
+    int rc;
+
+    if (!usb->devtype)
+        usb->devtype = LIBXL_USBDEV_TYPE_HOSTDEV;
+
+    if (usb->ctrl == -1) {
+        if (usb->port) {
+            LOG(ERROR, "USB controller must be specified if you specify port");
+            return ERROR_INVAL;
+        }
+
+        rc = libxl__device_usb_set_default_usbctrl(gc, domid, usb);
+        /* If no existing controller to host this usb device, add a new one */
+        if (rc) {
+            libxl_device_usbctrl *usbctrl;
+
+            GCNEW(usbctrl);
+            libxl_device_usbctrl_init(usbctrl);
+            rc = libxl__device_usbctrl_setdefault(gc, domid, usbctrl);
+            if (rc < 0) goto out;
+
+            if (usbctrl->devid == -1) {
+                usbctrl->devid = libxl__device_nextid(gc, domid, "vusb");
+                if (usbctrl->devid < 0) {
+                    rc = ERROR_FAIL;
+                    goto out;
+                }
+            }
+
+            rc = libxl__device_usbctrl_add_xenstore(gc, domid, usbctrl,
+                                                    update_json);
+            if (rc) goto out;
+
+            usb->ctrl = usbctrl->devid;
+            usb->port = 1;
+        }
+    } else if (!usb->port) {
+        /* Valid port starts from 1. Choose port for us. */
+        int i, ports;
+        const char *fe_path, *be_path, *tmp;
+
+        fe_path = GCSPRINTF("%s/device/vusb/%d",
+                         libxl__xs_get_dompath(gc, domid), usb->ctrl);
+
+        rc = libxl__xs_read_checked(gc, XBT_NULL,
+                                    GCSPRINTF("%s/backend", fe_path), &be_path);
+        if (rc) goto out;
+
+        rc = libxl__xs_read_checked(gc, XBT_NULL,
+                                    GCSPRINTF("%s/num-ports", be_path), &tmp);
+        if (rc) goto out;
+
+        ports = atoi(tmp);
+
+        for (i = 0; i < ports; i++) {
+            tmp = libxl__xs_read(gc, XBT_NULL,
+                                 GCSPRINTF("%s/port/%d", be_path, i + 1));
+            if (tmp && !strcmp(tmp, "")) {
+                usb->port = i + 1;
+                break;
+            }
+        }
+
+        if (!usb->port) {
+            LOG(ERROR, "No available port under specified controller");
+            rc = ERROR_FAIL;
+            goto out;
+        }
+    } else {
+        const char *fe_path, *be_path, *tmp;
+
+        fe_path = GCSPRINTF("%s/device/vusb/%d",
+                         libxl__xs_get_dompath(gc, domid), usb->ctrl);
+
+        rc = libxl__xs_read_checked(gc, XBT_NULL,
+                                    GCSPRINTF("%s/backend", fe_path), &be_path);
+        if (rc) goto out;
+
+        rc = libxl__xs_read_checked(gc, XBT_NULL,
+                                    GCSPRINTF("%s/port/%d", be_path, usb->port),
+                                    &tmp);
+        if (rc) goto out;
+
+        if (strcmp(tmp, "")) {
+            LOG(ERROR, "The controller port isn't available");
+            rc = ERROR_FAIL;
+            goto out;
+        }
+    }
+
+    rc = 0;
+
+out:
+    return rc;
+}
+
+/* Add usb information to xenstore
+ *
+ * Adding a usb device won't create new 'vusb' device, but only write
+ * the device busid to the controller:port in xenstore.
+ */
+static int libxl__device_usb_add_xenstore(libxl__gc *gc, uint32_t domid,
+                                          libxl_device_usb *usb,
+                                          bool update_json)
+{
+    char *be_path;
+    char *busid;
+    int rc;
+    xs_transaction_t t = XBT_NULL;
+    libxl_domain_config d_config;
+    libxl_device_usb usb_saved;
+    libxl__domain_userdata_lock *lock = NULL;
+
+    libxl_domain_config_init(&d_config);
+    libxl_device_usb_init(&usb_saved);
+    libxl_device_usb_copy(CTX, &usb_saved, usb);
+
+    busid = usb_busaddr_to_busid(gc, usb->u.hostdev.hostbus,
+                                 usb->u.hostdev.hostaddr);
+    if (!busid) {
+        LOG(DEBUG, "Fail to get busid of usb device");
+        goto out;
+    }
+
+    if (update_json) {
+        lock = libxl__lock_domain_userdata(gc, domid);
+        if (!lock) {
+            rc = ERROR_LOCK_FAIL;
+            goto out;
+        }
+
+        rc = libxl__get_domain_configuration(gc, domid, &d_config);
+        if (rc) goto out;
+
+        DEVICE_ADD(usb, usbs, domid, &usb_saved, COMPARE_USB, &d_config);
+    }
+
+    for (;;) {
+        rc = libxl__xs_transaction_start(gc, &t);
+        if (rc) goto out;
+
+        if (update_json) {
+            rc = libxl__set_domain_configuration(gc, domid, &d_config);
+            if (rc) goto out;
+        }
+
+        be_path = GCSPRINTF("%s/backend/vusb/%d/%d/port/%d",
+                            libxl__xs_get_dompath(gc, LIBXL_TOOLSTACK_DOMID),
+                            domid, usb->ctrl, usb->port);
+
+        LOG(DEBUG, "Adding new usb device to xenstore");
+        if (libxl__xs_write_checked(gc, t, be_path, busid))
+            goto out;
+
+        rc = libxl__xs_transaction_commit(gc, &t);
+        if (!rc) break;
+        if (rc < 0) goto out;
+    }
+
+    rc = 0;
+
+out:
+    if (lock) libxl__unlock_domain_userdata(lock);
+    libxl_device_usb_dispose(&usb_saved);
+    libxl_domain_config_dispose(&d_config);
+    return rc;
+}
+
+static int libxl__device_usb_remove_xenstore(libxl__gc *gc, uint32_t domid,
+                                             libxl_device_usb *usb)
+{
+    char *be_path;
+
+    be_path = GCSPRINTF("%s/backend/vusb/%d/%d/port/%d",
+                        libxl__xs_get_dompath(gc, LIBXL_TOOLSTACK_DOMID),
+                        domid, usb->ctrl, usb->port);
+    LOG(DEBUG, "Removing USB device from xenstore");
+    if (libxl__xs_write_checked(gc, XBT_NULL, be_path, ""))
+        return ERROR_FAIL;
+
+    return 0;
+}
+
+/* bind/unbind usb device interface */
+static int unbind_usb_intf(libxl__gc *gc, char *intf, char **drvpath)
+{
+    char *path, *spath, *dp = NULL;
+    int fd = -1;
+    int rc;
+    struct stat st;
+
+    spath = GCSPRINTF(SYSFS_USB_DEV"/%s/driver", intf);
+    if (!lstat(spath, &st)) {
+        /* Find the canonical path to the driver. */
+        dp = libxl__zalloc(gc, PATH_MAX);
+        dp = realpath(spath, dp);
+
+        path = GCSPRINTF("%s/unbind", spath);
+        fd = open(path, O_WRONLY);
+        if (fd < 0)
+            return ERROR_FAIL;
+        rc = write(fd, intf, strlen(intf));
+        close(fd);
+        if (rc < 0)
+            return ERROR_FAIL;
+    }
+
+    if (drvpath)
+        *drvpath = dp;
+
+    return 0;
+}
+
+static int bind_usb_intf(libxl__gc *gc, char *intf, char *drvpath)
+{
+    char *path;
+    struct stat st;
+    int fd, rc;
+
+    path = GCSPRINTF("%s/%s", drvpath, intf);
+    rc = lstat(path, &st);
+    /* already bind, return */
+    if (!rc)
+        return 0;
+
+    path = GCSPRINTF("%s/bind", drvpath);
+    fd = open(path, O_WRONLY);
+    if (fd < 0)
+        return ERROR_FAIL;
+
+    rc = write(fd, intf, strlen(intf));
+    close(fd);
+    if (rc < 0)
+        return ERROR_FAIL;
+
+    return 0;
+}
+
+/* Is usb interface bound to usbback? */
+static int usb_intf_is_assigned(libxl__gc *gc, char *intf)
+{
+    char *spath;
+    int rc;
+    struct stat st;
+
+    spath = GCSPRINTF(SYSFS_USBBACK_DRIVER"/%s", intf);
+    rc = lstat(spath, &st);
+
+    if (rc == 0)
+        return 1;
+    if (rc < 0 && errno == ENOENT)
+        return 0;
+    LOGE(ERROR, "Accessing %s", spath);
+    return -1;
+}
+
+static int usb_get_all_interfaces(libxl__gc *gc, libxl_device_usb *usb,
+                                  char ***intfs, int *num)
+{
+    DIR *dir;
+    struct dirent *entry;
+    char *buf;
+    char *busid;
+    int rc;
+
+    *intfs = NULL;
+    *num = 0;
+
+    busid = usb_busaddr_to_busid(gc, usb->u.hostdev.hostbus,
+                                 usb->u.hostdev.hostaddr);
+    if (!busid) {
+        rc = ERROR_FAIL;
+        goto out;
+    }
+
+    buf = GCSPRINTF("%s:", busid);
+
+    if (!(dir = opendir(SYSFS_USB_DEV))) {
+        rc = ERROR_FAIL;
+        goto out;
+    }
+
+    while ((entry = readdir(dir)) != NULL) {
+        if (!strncmp(entry->d_name, buf, strlen(buf))) {
+            GCREALLOC_ARRAY(*intfs, *num + 1);
+            if (*intfs == NULL) {
+                rc = ERROR_FAIL;
+                goto out;
+            }
+            (*intfs)[*num] = libxl__strdup(gc, entry->d_name);
+            (*num)++;
+        }
+    }
+
+    closedir(dir);
+    rc = 0;
+
+out:
+    return rc;
+}
+
+/* Encode usb interface so that it could be written to xenstore as a key.
+ *
+ * Since xenstore key cannot include '.' or ':', we'll change '.' to '_',
+ * change ':' to '-'. For example, 3-1:2.1 will be encoded to 3-1-2_1.
+ * This will be used to save original driver of USB device to xenstore.
+ */
+static char *usb_interface_xenstore_encode(char *busid)
+{
+    char *str = strdup(busid);
+    int i, len = strlen(str);
+
+    for (i = 0; i < len; i++) {
+        if (str[i] == '.')
+            str[i] = '_';
+         if (str[i] == ':')
+            str[i] = '-';
+    }
+    return str;
+}
+
+/* Unbind USB device from "usbback" driver.
+ *
+ * If there are many interfaces under USB device, check each interface,
+ * unbind from "usbback" driver and rebind to its original driver.
+ */
+static int usbback_dev_unassign(libxl__gc *gc, libxl_device_usb *usb)
+{
+    char **intfs = NULL;
+    char *path;
+    int num = 0, i;
+    int rc;
+    char *busid;
+    char *usb_encode = NULL;
+
+    if (usb_get_all_interfaces(gc, usb, &intfs, &num) < 0)
+        return ERROR_FAIL;
+
+    busid = usb_busaddr_to_busid(gc, usb->u.hostdev.hostbus,
+                                 usb->u.hostdev.hostaddr);
+    usb_encode = usb_interface_xenstore_encode(busid);
+
+    for (i = 0; i < num; i++) {
+        char *intf = intfs[i];
+        char *drvpath = NULL;
+
+        /* check if the USB interface is already bound to "usbbcak" */
+        if (usb_intf_is_assigned(gc, intf) > 0) {
+            /* unbind interface from usbback driver */
+            if (unbind_usb_intf(gc, intf, NULL) < 0) {
+                rc = ERROR_FAIL;
+                goto out;
+            }
+        }
+
+        /* bind interface to its originial driver */
+        drvpath = libxl__xs_read(gc, XBT_NULL,
+                  GCSPRINTF(USBBACK_INFO_PATH"/%s/%s/driver_path",
+                  usb_encode, usb_interface_xenstore_encode(intf)));
+        if (drvpath && bind_usb_intf(gc, intf, drvpath))
+            LOGE(WARN, "Couldn't bind %s to %s", intf, drvpath);
+    }
+
+    /* finally, remove xs driver path */
+    path = GCSPRINTF(USBBACK_INFO_PATH"/%s", usb_encode);
+    libxl__xs_rm_checked(gc, XBT_NULL, path);
+    rc = 0;
+
+out:
+    free(usb_encode);
+    return rc;
+}
+
+/* Bind USB device to "usbback" driver.
+ *
+ * If there are many interfaces under USB device, check each interface,
+ * unbind from original driver and bind to "usbback" driver.
+ */
+static int usbback_dev_assign(libxl__gc *gc, libxl_device_usb *usb)
+{
+    char **intfs = NULL;
+    int num = 0, i;
+    int rc;
+    char *busid;
+    char *usb_encode = NULL;
+
+    if (usb_get_all_interfaces(gc, usb, &intfs, &num) < 0)
+        return ERROR_FAIL;
+
+    busid = usb_busaddr_to_busid(gc, usb->u.hostdev.hostbus,
+                                 usb->u.hostdev.hostaddr);
+    usb_encode = usb_interface_xenstore_encode(busid);
+
+    for (i = 0; i < num; i++) {
+        char *intf = intfs[i];
+        char *path = NULL;
+        char *drvpath = NULL;
+
+        /* already assigned to usbback */
+        if (usb_intf_is_assigned(gc, intf) > 0)
+            continue;
+
+        /* unbind interface from original driver */
+        if (unbind_usb_intf(gc, intf, &drvpath) < 0) {
+            rc = ERROR_FAIL;
+            goto out_rebind;
+        }
+
+        if (drvpath) {
+            /* write driver path to xenstore for later rebinding */
+            path = GCSPRINTF(USBBACK_INFO_PATH"/%s/%s/driver_path",
+                             usb_encode, usb_interface_xenstore_encode(intf));
+            if (libxl__xs_write_checked(gc, XBT_NULL, path, drvpath) < 0) {
+                LOG(WARN, "Write of %s to node %s failed", drvpath, path);
+            }
+        }
+
+        /* bind interface to usbback */
+        if (bind_usb_intf(gc, intf, SYSFS_USBBACK_DRIVER) < 0) {
+            LOGE(ERROR, "Couldn't bind %s to %s", intf, SYSFS_USBBACK_DRIVER);
+            rc = ERROR_FAIL;
+            goto out_rebind;
+        }
+    }
+
+    rc = 0;
+    goto out;
+
+out_rebind:
+    /* some interfaces might be bound to usbback, unbind it then and
+     * rebind to its original driver
+     */
+    usbback_dev_unassign(gc, usb);
+out:
+    free(usb_encode);
+    return rc;
+}
+
+/*
+ * USB add
+ */
+static int do_usb_add(libxl__gc *gc, uint32_t domid,
+                      libxl_device_usb *usbdev,
+                      libxl__ao_device *aodev)
+{
+    libxl_ctx *ctx = CTX;
+    libxl_usbctrlinfo usbctrlinfo;
+    libxl_device_usbctrl usbctrl;
+    int rc;
+
+    libxl_device_usbctrl_init(&usbctrl);
+    libxl_usbctrlinfo_init(&usbctrlinfo);
+    usbctrl.devid = usbdev->ctrl;
+    rc = libxl_device_usbctrl_getinfo(ctx, domid, &usbctrl, &usbctrlinfo);
+    if (rc)
+        goto out;
+
+    switch (usbctrlinfo.type) {
+    case LIBXL_USBCTRL_TYPE_DEVICEMODEL:
+        LOG(ERROR, "Not supported");
+        break;
+    case LIBXL_USBCTRL_TYPE_PV:
+        rc = libxl__device_usb_add_xenstore(gc, domid, usbdev,
+                                            aodev->update_json);
+        if (rc) goto out;
+
+        rc = usbback_dev_assign(gc, usbdev);
+        if (rc) {
+            libxl__device_usb_remove_xenstore(gc, domid, usbdev);
+            goto out;
+        }
+        break;
+    default:
+        rc = ERROR_FAIL;
+        goto out;
+    }
+
+out:
+    libxl_device_usbctrl_dispose(&usbctrl);
+    libxl_usbctrlinfo_dispose(&usbctrlinfo);
+    return rc;
+}
+
+/* AO operation to add a usb device.
+ *
+ * Generally, it does:
+ * 1) check if the usb device type is assignable
+ * 2) check if the usb device is already assigned to a domain
+ * 3) add 'busid' of the usb device to xenstore contoller/port/.
+ *    (PVUSB driver watches the xenstore changes and will detect that.)
+ * 4) unbind usb device from original driver and bind to usbback.
+ *    If usb device has many interfaces, then:
+ *    - unbind each interface from its original driver and bind to usbback.
+ *    - store the original driver to xenstore for later rebinding when
+ *      detaching the device.
+ *
+ * Before calling this function, aodev should be properly filled:
+ * aodev->ao, aodev->callback, aodev->update_json, ...
+ */
+void libxl__device_usb_add(libxl__egc *egc, uint32_t domid,
+                           libxl_device_usb *usb,
+                           libxl__ao_device *aodev)
+{
+    STATE_AO_GC(aodev->ao);
+    int rc;
+    char *busid = NULL;
+    libxl_device_usb *assigned;
+    int num_assigned;
+    libxl_device_usbctrl usbctrl;
+    libxl_usbctrlinfo usbctrlinfo;
+
+    libxl_device_usbctrl_init(&usbctrl);
+    libxl_usbctrlinfo_init(&usbctrlinfo);
+
+    /* Currently only support adding USB device from Dom0 backend.
+     * So, if USB controller is specified, check backend domain, if
+     * it is not Dom0, report error as not supported.
+     *
+     * -1 is the default value of libxl_device_usbctrl.
+     */
+    if (usb->ctrl != -1) {
+        usbctrl.devid = usb->ctrl;
+        rc = libxl_device_usbctrl_getinfo(CTX, domid, &usbctrl, &usbctrlinfo);
+        if (rc) goto out;
+
+        if (usbctrlinfo.backend_id != 0) {
+            LOG(ERROR, "Don't support adding USB device from non-Dom0 backend");
+            rc = ERROR_INVAL;
+            goto out;
+        }
+    }
+
+    busid = usb_busaddr_to_busid(gc, usb->u.hostdev.hostbus,
+                                 usb->u.hostdev.hostaddr);
+    if (!busid) {
+        LOG(ERROR, "USB device doesn't exist in sysfs");
+        rc = ERROR_FAIL;
+        goto out;
+    }
+
+    if (!is_usb_assignable(gc, usb)) {
+        LOG(ERROR, "USB device is not assignable.");
+        rc = ERROR_FAIL;
+        goto out;
+    }
+
+    /* check usb device is already assigned */
+    rc = get_assigned_devices(gc, &assigned, &num_assigned);
+    if (rc) {
+        LOG(ERROR, "cannot determine if device is assigned,"
+                   " refusing to continue");
+        rc = ERROR_FAIL;
+        goto out;
+    }
+
+    if (is_usbdev_in_array(assigned, num_assigned, usb)) {
+        LOG(ERROR, "USB device already attached to a domain");
+        rc = ERROR_FAIL;
+        goto out;
+    }
+
+    rc = libxl__device_usb_setdefault(gc, domid, usb, aodev->update_json);
+    if (rc) goto out;
+
+    /* Do the add */
+    if (do_usb_add(gc, domid, usb, aodev)) {
+        rc = ERROR_FAIL;
+        goto out;
+    }
+
+    libxl__ao_complete(egc, ao, 0);
+    rc = 0;
+
+out:
+    libxl_device_usbctrl_dispose(&usbctrl);
+    libxl_usbctrlinfo_dispose(&usbctrlinfo);
+    aodev->rc = rc;
+    if (rc) aodev->callback(egc, aodev);
+    return;
+}
+
+static int do_usb_remove(libxl__gc *gc, uint32_t domid,
+                         libxl_device_usb *usbdev)
+{
+    libxl_ctx *ctx = CTX;
+    libxl_usbctrlinfo usbctrlinfo;
+    libxl_device_usbctrl usbctrl;
+    int rc;
+
+    libxl_device_usbctrl_init(&usbctrl);
+    libxl_usbctrlinfo_init(&usbctrlinfo);
+    usbctrl.devid = usbdev->ctrl;
+
+    rc = libxl_device_usbctrl_getinfo(ctx, domid, &usbctrl, &usbctrlinfo);
+    if (rc) goto out;
+
+    if (usbctrlinfo.backend_id != 0) {
+        LOG(ERROR, "Don't support removing USB device from non-Dom0 backend");
+        rc = ERROR_INVAL;
+        goto out;
+    }
+
+    switch (usbctrlinfo.type) {
+    case LIBXL_USBCTRL_TYPE_DEVICEMODEL:
+        LOG(ERROR, "Not supported");
+        break;
+    case LIBXL_USBCTRL_TYPE_PV:
+        rc = libxl__device_usb_remove_xenstore(gc, domid, usbdev);
+        if (rc) goto out;
+
+        usbback_dev_unassign(gc, usbdev);
+        break;
+    default:
+        rc = ERROR_FAIL;
+        goto out;
+    }
+
+    rc = 0;
+
+out:
+    libxl_device_usbctrl_dispose(&usbctrl);
+    libxl_usbctrlinfo_dispose(&usbctrlinfo);
+    return rc;
+}
+
+/* Operation to remove usb device.
+ *
+ * Generally, it does:
+ * 1) check if the usb device is assigned to the domain
+ * 2) remove the usb device from xenstore controller/port.
+ * 3) unbind usb device from usbback and rebind to its original driver.
+ *    If usb device has many interfaces, do it to each interface.
+ */
+static int libxl__device_usb_remove(libxl__gc *gc, uint32_t domid,
+                                    libxl_device_usb *usb)
+{
+    int rc;
+
+    if (usb->ctrl < 0 || usb->port < 1) {
+        LOG(ERROR, "Invalid USB device");
+        rc = ERROR_FAIL;
+        goto out;
+    }
+
+    if (usb->devtype == LIBXL_USBDEV_TYPE_HOSTDEV &&
+        (usb->u.hostdev.hostbus < 1 || usb->u.hostdev.hostaddr < 1)) {
+        LOG(ERROR, "Invalid USB device of hostdev");
+        rc = ERROR_FAIL;
+        goto out;
+    }
+
+    /* Do the remove */
+    rc = do_usb_remove(gc, domid, usb);
+
+out:
+    return rc;
+}
+
+int libxl_device_usb_remove(libxl_ctx *ctx, uint32_t domid,
+                            libxl_device_usb *usb,
+                            const libxl_asyncop_how *ao_how)
+
+{
+    AO_CREATE(ctx, domid, ao_how);
+    int rc;
+
+    rc = libxl__device_usb_remove(gc, domid, usb);
+
+    libxl__ao_complete(egc, ao, rc);
+    return AO_INPROGRESS;
+}
+
+int libxl_ctrlport_to_device_usb(libxl_ctx *ctx,
+                                 uint32_t domid,
+                                 int ctrl,
+                                 int port,
+                                 libxl_device_usb *usb)
+{
+    GC_INIT(ctx);
+    const char *dompath, *be_path, *busid;
+    int rc;
+
+    dompath = libxl__xs_get_dompath(gc, domid);
+
+    rc = libxl__xs_read_checked(gc, XBT_NULL,
+                  GCSPRINTF("%s/device/vusb/%d/backend", dompath, ctrl),
+                  &be_path);
+    if (rc) goto out;
+
+    rc = libxl__xs_read_checked(gc, XBT_NULL,
+                           GCSPRINTF("%s/port/%d", be_path, port),
+                           &busid);
+    if (rc) goto out;
+
+    if (!strcmp(busid, "")) {
+        rc = ERROR_FAIL;
+        goto out;
+    }
+
+    usb->ctrl = ctrl;
+    usb->port = port;
+    rc = usb_busaddr_from_busid(gc, busid, &usb->u.hostdev.hostbus,
+                                &usb->u.hostdev.hostaddr);
+
+out:
+    GC_FREE;
+    return rc;
+}
+/*
+ * Local variables:
+ * mode: C
+ * c-basic-offset: 4
+ * indent-tabs-mode: nil
+ * End:
+ */
diff --git a/tools/libxl/libxl_types.idl b/tools/libxl/libxl_types.idl
index d6ef9a2..dccb317 100644
--- a/tools/libxl/libxl_types.idl
+++ b/tools/libxl/libxl_types.idl
@@ -603,6 +603,35 @@ libxl_device_rdm = Struct("device_rdm", [
     ("policy", libxl_rdm_reserve_policy),
     ])
 
+libxl_usbctrl_type = Enumeration("usbctrl_type", [
+    (0, "AUTO"),
+    (1, "PV"),
+    (2, "DEVICEMODEL"),
+    ])
+
+libxl_usbdev_type = Enumeration("usbdev_type", [
+    (1, "hostdev"),
+    ])
+
+libxl_device_usbctrl = Struct("device_usbctrl", [
+    ("type", libxl_usbctrl_type),
+    ("devid", libxl_devid),
+    ("version", integer),
+    ("ports", integer),
+    ("backend_domid", libxl_domid),
+    ("backend_domname", string),
+   ])
+
+libxl_device_usb = Struct("device_usb", [
+    ("ctrl", libxl_devid),
+    ("port", integer),
+    ("u", KeyedUnion(None, libxl_usbdev_type, "devtype",
+           [("hostdev", Struct(None, [
+                 ("hostbus",   uint8),
+                 ("hostaddr",  uint8)])),
+           ])),
+    ])
+
 libxl_device_dtdev = Struct("device_dtdev", [
     ("path", string),
     ])
@@ -635,6 +664,8 @@ libxl_domain_config = Struct("domain_config", [
     ("pcidevs", Array(libxl_device_pci, "num_pcidevs")),
     ("rdms", Array(libxl_device_rdm, "num_rdms")),
     ("dtdevs", Array(libxl_device_dtdev, "num_dtdevs")),
+    ("usbctrls", Array(libxl_device_usbctrl, "num_usbctrls")),
+    ("usbs", Array(libxl_device_usb, "num_usbs")),
     ("vfbs", Array(libxl_device_vfb, "num_vfbs")),
     ("vkbs", Array(libxl_device_vkb, "num_vkbs")),
     ("vtpms", Array(libxl_device_vtpm, "num_vtpms")),
@@ -684,6 +715,32 @@ libxl_vtpminfo = Struct("vtpminfo", [
     ("uuid", libxl_uuid),
     ], dir=DIR_OUT)
 
+libxl_usbctrlinfo = Struct("usbctrlinfo", [
+    ("type", libxl_usbctrl_type),
+    ("devid", libxl_devid),
+    ("version", integer),
+    ("ports", integer),
+    ("backend", string),
+    ("backend_id", uint32),
+    ("frontend", string),
+    ("frontend_id", uint32),
+    ("state", integer),
+    ("evtch", integer),
+    ("ref_urb", integer),
+    ("ref_conn", integer),
+    ], dir=DIR_OUT)
+
+libxl_usbinfo = Struct("usbinfo", [
+    ("ctrl", libxl_devid),
+    ("port", integer),
+    ("busnum", uint8),
+    ("devnum", uint8),
+    ("idVendor", uint16),
+    ("idProduct", uint16),
+    ("prod", string),
+    ("manuf", string),
+    ], dir=DIR_OUT)
+
 libxl_vcpuinfo = Struct("vcpuinfo", [
     ("vcpuid", uint32),
     ("cpu", uint32),
diff --git a/tools/libxl/libxl_types_internal.idl b/tools/libxl/libxl_types_internal.idl
index 5e55685..696f5f8 100644
--- a/tools/libxl/libxl_types_internal.idl
+++ b/tools/libxl/libxl_types_internal.idl
@@ -22,6 +22,7 @@ libxl__device_kind = Enumeration("device_kind", [
     (6, "VKBD"),
     (7, "CONSOLE"),
     (8, "VTPM"),
+    (9, "VUSB"),
     ])
 
 libxl__console_backend = Enumeration("console_backend", [
diff --git a/tools/libxl/libxl_utils.c b/tools/libxl/libxl_utils.c
index 9c5c4d0..706a0c1 100644
--- a/tools/libxl/libxl_utils.c
+++ b/tools/libxl/libxl_utils.c
@@ -1270,6 +1270,22 @@ int libxl__random_bytes(libxl__gc *gc, uint8_t *buf, size_t len)
     return ret;
 }
 
+void libxl_device_usbctrl_list_free(libxl_device_usbctrl *list, int nr)
+{
+   int i;
+   for (i = 0; i < nr; i++)
+       libxl_device_usbctrl_dispose(&list[i]);
+   free(list);
+}
+
+void libxl_device_usb_list_free(libxl_device_usb *list, int nr)
+{
+   int i;
+   for (i = 0; i < nr; i++)
+       libxl_device_usb_dispose(&list[i]);
+   free(list);
+}
+
 /*
  * Local variables:
  * mode: C
diff --git a/tools/libxl/libxl_utils.h b/tools/libxl/libxl_utils.h
index 339ebdf..60487e3 100644
--- a/tools/libxl/libxl_utils.h
+++ b/tools/libxl/libxl_utils.h
@@ -76,6 +76,11 @@ int libxl_uuid_to_device_vtpm(libxl_ctx *ctx, uint32_t domid,
                                libxl_uuid *uuid, libxl_device_vtpm *vtpm);
 int libxl_devid_to_device_vtpm(libxl_ctx *ctx, uint32_t domid,
                                int devid, libxl_device_vtpm *vtpm);
+int libxl_devid_to_device_usbctrl(libxl_ctx *ctx, uint32_t domid,
+                                  int devid, libxl_device_usbctrl *usbctrl);
+int libxl_ctrlport_to_device_usb(libxl_ctx *ctx, uint32_t domid,
+                                 int ctrl, int port,
+                                 libxl_device_usb *usb);
 
 int libxl_bitmap_alloc(libxl_ctx *ctx, libxl_bitmap *bitmap, int n_bits);
     /* Allocated bimap is from malloc, libxl_bitmap_dispose() to be
-- 
2.1.4