Re: [Qemu-devel] [RFC PATCH v1 1/1] vGPU core driver : to provide common interface for vGPU.

[Kirti Wankhede <kwankhede@nvidia.com>]

Resending this mail again, somehow my previous mail didn't reached every 
to everyone's inbox.

On 2/2/2016 3:16 AM, Kirti Wankhede wrote:
> Design for vGPU Driver:
> Main purpose of vGPU driver is to provide a common interface for vGPU
> management that can be used by differnt GPU drivers.
>
> This module would provide a generic interface to create the device, add
> it to vGPU bus, add device to IOMMU group and then add it to vfio group.
>
> High Level block diagram:
>
>
> +--------------+    vgpu_register_driver()+---------------+
> |     __init() +------------------------->+               |
> |              |                          |               |
> |              +<-------------------------+    vgpu.ko    |
> | vfio_vgpu.ko |   probe()/remove()       |               |
> |              |                +---------+               +---------+
> +--------------+                |         +-------+-------+         |
>                                  |                 ^                 |
>                                  | callback        |                 |
>                                  |         +-------+--------+        |
>                                  |         |vgpu_register_device()   |
>                                  |         |                |        |
>                                  +---^-----+-----+    +-----+------+-+
>                                      | nvidia.ko |    |  i915.ko   |
>                                      |           |    |            |
>                                      +-----------+    +------------+
>
> vGPU driver provides two types of registration interfaces:
> 1. Registration interface for vGPU bus driver:
>
> /**
>   * struct vgpu_driver - vGPU device driver
>   * @name: driver name
>   * @probe: called when new device created
>   * @remove: called when device removed
>   * @driver: device driver structure
>   *
>   **/
> struct vgpu_driver {
>          const char *name;
>          int  (*probe)  (struct device *dev);
>          void (*remove) (struct device *dev);
>          struct device_driver    driver;
> };
>
> int  vgpu_register_driver(struct vgpu_driver *drv, struct module *owner);
> void vgpu_unregister_driver(struct vgpu_driver *drv);
>
> VFIO bus driver for vgpu, should use this interface to register with
> vGPU driver. With this, VFIO bus driver for vGPU devices is responsible
> to add vGPU device to VFIO group.
>
> 2. GPU driver interface
>
> /**
>   * struct gpu_device_ops - Structure to be registered for each physical
> GPU to
>   * register the device to vgpu module.
>   *
>   * @owner:              The module owner.
>   * @vgpu_supported_config: Called to get information about supported
>   *                       vgpu types.
>   *                      @dev : pci device structure of physical GPU.
>   *                      @config: should return string listing supported
>   *                      config
>   *                      Returns integer: success (0) or error (< 0)
>   * @vgpu_create:        Called to allocate basic resouces in graphics
>   *                      driver for a particular vgpu.
>   *                      @dev: physical pci device structure on which vgpu
>   *                            should be created
>   *                      @uuid: uuid for which VM it is intended to
>   *                      @instance: vgpu instance in that VM
>   *                      @vgpu_id: This represents the type of vgpu to be
>   *                                created
>   *                      Returns integer: success (0) or error (< 0)
>   * @vgpu_destroy:       Called to free resources in graphics driver for
>   *                      a vgpu instance of that VM.
>   *                      @dev: physical pci device structure to which
>   *                      this vgpu points to.
>   *                      @uuid: uuid for which the vgpu belongs to.
>   *                      @instance: vgpu instance in that VM
>   *                      Returns integer: success (0) or error (< 0)
>   *                      If VM is running and vgpu_destroy is called that
>   *                      means the vGPU is being hotunpluged. Return error
>   *                      if VM is running and graphics driver doesn't
>   *                      support vgpu hotplug.
>   * @vgpu_start:         Called to do initiate vGPU initialization
>   *                      process in graphics driver when VM boots before
>   *                      qemu starts.
>   *                      @uuid: UUID which is booting.
>   *                      Returns integer: success (0) or error (< 0)
>   * @vgpu_shutdown:      Called to teardown vGPU related resources for
>   *                      the VM
>   *                      @uuid: UUID which is shutting down .
>   *                      Returns integer: success (0) or error (< 0)
>   * @read:               Read emulation callback
>   *                      @vdev: vgpu device structure
>   *                      @buf: read buffer
>   *                      @count: number bytes to read
>   *                      @address_space: specifies for which address space
>   *                      the request is: pci_config_space, IO register
>   *                      space or MMIO space.
>   *                      Retuns number on bytes read on success or error.
>   * @write:              Write emulation callback
>   *                      @vdev: vgpu device structure
>   *                      @buf: write buffer
>   *                      @count: number bytes to be written
>   *                      @address_space: specifies for which address space
>   *                      the request is: pci_config_space, IO register
>   *                      space or MMIO space.
>   *                      Retuns number on bytes written on success or
> error.
>   * @vgpu_set_irqs:      Called to send about interrupts configuration
>   *                      information that qemu set.
>   *                      @vdev: vgpu device structure
>   *                      @flags, index, start, count and *data : same as
>   *                      that of struct vfio_irq_set of
>   *                      VFIO_DEVICE_SET_IRQS API.
>   *
>   * Physical GPU that support vGPU should be register with vgpu module with
>   * gpu_device_ops structure.
>   */
>
> struct gpu_device_ops {
>          struct module   *owner;
>          int     (*vgpu_supported_config)(struct pci_dev *dev,
>                       char  *config);
>          int     (*vgpu_create)(struct pci_dev *dev, uuid_le uuid,
>                                 uint32_t instance, uint32_t vgpu_id);
>          int     (*vgpu_destroy)(struct pci_dev *dev, uuid_le uuid,
>                                  uint32_t instance);
>          int     (*vgpu_start)(uuid_le uuid);
>          int     (*vgpu_shutdown)(uuid_le uuid);
>          ssize_t (*read) (struct vgpu_device *vdev, char *buf,
>               size_t count, uint32_t address_space,
>               loff_t pos);
>          ssize_t (*write)(struct vgpu_device *vdev, char *buf,
>               size_t count, uint32_t address_space,
>               loff_t pos);
>          int     (*vgpu_set_irqs)(struct vgpu_device *vdev,
>                   uint32_t flags, unsigned index,
>                    unsigned start, unsigned count,
>                                   void *data);
> };
>
> int  vgpu_register_device(struct pci_dev *dev, const struct
> gpu_device_ops *ops);
> void vgpu_unregister_device(struct pci_dev *dev);
>
> This registration interface should be used by GPU drivers to register
> each physical device to vGPU driver.
>
> This patch set is to review major APIs and design structure before
> continuing development and don't consider as final patch for review.
> These registration interfaces would evolve as the development proceed.
>
> Regards,
> Kirti
>
> Signed-off-by: Kirti Wankhede <kwankhede@nvidia.com>
> Signed-off-by: Neo Jia <cjia@nvidia.com>
> ---
> ---
>   drivers/vgpu/vgpu-driver.c |  137 ++++++++++++++
>   drivers/vgpu/vgpu_dev.c    |  421
> ++++++++++++++++++++++++++++++++++++++++++++
>   include/linux/vgpu.h       |  189 ++++++++++++++++++++
>   3 files changed, 747 insertions(+), 0 deletions(-)
>   create mode 100644 drivers/vgpu/vgpu-driver.c
>   create mode 100644 drivers/vgpu/vgpu_dev.c
>   create mode 100644 include/linux/vgpu.h
>
> diff --git a/drivers/vgpu/vgpu-driver.c b/drivers/vgpu/vgpu-driver.c
> new file mode 100644
> index 0000000..6b62f19
> --- /dev/null
> +++ b/drivers/vgpu/vgpu-driver.c
> @@ -0,0 +1,137 @@
> +/*
> + * VGPU driver
> + *
> + * Copyright (c) 2016, NVIDIA CORPORATION. All rights reserved.
> + *     Author: Neo Jia <cjia@nvidia.com>
> + *           Kirti Wankhede <kwankhede@nvidia.com>
> + *
> + * This program is free software; you can redistribute it and/or modify
> + * it under the terms of the GNU General Public License version 2 as
> + * published by the Free Software Foundation.
> + */
> +
> +#include <linux/init.h>
> +#include <linux/module.h>
> +#include <linux/device.h>
> +#include <linux/kernel.h>
> +#include <linux/fs.h>
> +#include <linux/vfio.h>
> +#include <linux/iommu.h>
> +#include <linux/sysfs.h>
> +#include <linux/ctype.h>
> +#include <linux/vgpu.h>
> +
> +#include "vgpu_private.h"
> +
> +static int vgpu_device_attach_iommu(struct vgpu_device *vgpu_dev)
> +{
> +        int retval = 0;
> +        struct iommu_group *group = NULL;
> +
> +        group = iommu_group_alloc();
> +        if (IS_ERR(group)) {
> +                printk(KERN_ERR "VGPU: failed to allocate group!\n");
> +                return PTR_ERR(group);
> +        }
> +
> +        retval = iommu_group_add_device(group, &vgpu_dev->dev);
> +        if (retval) {
> +                printk(KERN_ERR "VGPU: failed to add dev to group!\n");
> +                iommu_group_put(group);
> +                return retval;
> +        }
> +
> +        vgpu_dev->group = group;
> +
> +        printk(KERN_INFO "VGPU: group_id = %d \n", iommu_group_id(group));
> +        return retval;
> +}
> +
> +static void vgpu_device_detach_iommu(struct vgpu_device *vgpu_dev)
> +{
> +        iommu_group_put(vgpu_dev->dev.iommu_group);
> +        iommu_group_remove_device(&vgpu_dev->dev);
> +        printk(KERN_INFO "VGPU: detaching iommu \n");
> +}
> +
> +static int vgpu_device_probe(struct device *dev)
> +{
> +    struct vgpu_driver *drv = to_vgpu_driver(dev->driver);
> +    struct vgpu_device *vgpu_dev = to_vgpu_device(dev);
> +    int status = 0;
> +
> +    status = vgpu_device_attach_iommu(vgpu_dev);
> +    if (status) {
> +        printk(KERN_ERR "Failed to attach IOMMU\n");
> +        return status;
> +    }
> +
> +    if (drv && drv->probe) {
> +        status = drv->probe(dev);
> +    }
> +
> +    return status;
> +}
> +
> +static int vgpu_device_remove(struct device *dev)
> +{
> +    struct vgpu_driver *drv = to_vgpu_driver(dev->driver);
> +    struct vgpu_device *vgpu_dev = to_vgpu_device(dev);
> +    int status = 0;
> +
> +    if (drv && drv->remove) {
> +        drv->remove(dev);
> +    }
> +
> +    vgpu_device_detach_iommu(vgpu_dev);
> +
> +    return status;
> +}
> +
> +struct bus_type vgpu_bus_type = {
> +    .name        = "vgpu",
> +    .probe        = vgpu_device_probe,
> +    .remove        = vgpu_device_remove,
> +};
> +EXPORT_SYMBOL_GPL(vgpu_bus_type);
> +
> +/**
> + * vgpu_register_driver - register a new vGPU driver
> + * @drv: the driver to register
> + * @owner: owner module of driver ro register
> + *
> + * Returns a negative value on error, otherwise 0.
> + */
> +int vgpu_register_driver(struct vgpu_driver *drv, struct module *owner)
> +{
> +    /* initialize common driver fields */
> +    drv->driver.name = drv->name;
> +    drv->driver.bus = &vgpu_bus_type;
> +    drv->driver.owner = owner;
> +
> +    /* register with core */
> +    return driver_register(&drv->driver);
> +}
> +EXPORT_SYMBOL(vgpu_register_driver);
> +
> +/**
> + * vgpu_unregister_driver - unregister vGPU driver
> + * @drv: the driver to unregister
> + *
> + */
> +void vgpu_unregister_driver(struct vgpu_driver *drv)
> +{
> +    driver_unregister(&drv->driver);
> +}
> +EXPORT_SYMBOL(vgpu_unregister_driver);
> +
> +int vgpu_bus_register(void)
> +{
> +    return bus_register(&vgpu_bus_type);
> +}
> +
> +void vgpu_bus_unregister(void)
> +{
> +    bus_unregister(&vgpu_bus_type);
> +}
> +
> diff --git a/drivers/vgpu/vgpu_dev.c b/drivers/vgpu/vgpu_dev.c
> new file mode 100644
> index 0000000..e5a92a1
> --- /dev/null
> +++ b/drivers/vgpu/vgpu_dev.c
> @@ -0,0 +1,421 @@
> +/*
> + * VGPU Core Driver
> + *
> + * Copyright (c) 2016, NVIDIA CORPORATION. All rights reserved.
> + *     Author: Neo Jia <cjia@nvidia.com>
> + *           Kirti Wankhede <kwankhede@nvidia.com>
> + *
> + * This program is free software; you can redistribute it and/or modify
> + * it under the terms of the GNU General Public License version 2 as
> + * published by the Free Software Foundation.
> + */
> +
> +#include <linux/init.h>
> +#include <linux/module.h>
> +#include <linux/device.h>
> +#include <linux/kernel.h>
> +#include <linux/fs.h>
> +#include <linux/poll.h>
> +#include <linux/slab.h>
> +#include <linux/cdev.h>
> +#include <linux/sched.h>
> +#include <linux/wait.h>
> +#include <linux/uuid.h>
> +#include <linux/vfio.h>
> +#include <linux/iommu.h>
> +#include <linux/sysfs.h>
> +#include <linux/ctype.h>
> +#include <linux/vgpu.h>
> +
> +#include "vgpu_private.h"
> +
> +#define DRIVER_VERSION    "0.1"
> +#define DRIVER_AUTHOR    "NVIDIA Corporation"
> +#define DRIVER_DESC    "VGPU Core Driver"
> +
> +/*
> + * #defines
> + */
> +
> +#define VGPU_CLASS_NAME        "vgpu"
> +
> +/*
> + * Global Structures
> + */
> +
> +static struct vgpu {
> +    struct list_head    vgpu_devices_list;
> +    struct mutex        vgpu_devices_lock;
> +    struct list_head    gpu_devices_list;
> +    struct mutex        gpu_devices_lock;
> +} vgpu;
> +
> +
> +static struct class vgpu_class;
> +
> +/*
> + * Function prototypes
> + */
> +
> +unsigned int vgpu_poll(struct file *file, poll_table *wait);
> +long vgpu_unlocked_ioctl(struct file *file, unsigned int cmd, unsigned
> long i_arg);
> +int vgpu_mmap(struct file *file, struct vm_area_struct *vma);
> +
> +int vgpu_open(struct inode *inode, struct file *file);
> +int vgpu_close(struct inode *inode, struct file *file);
> +ssize_t vgpu_read(struct file *file, char __user * buf,
> +              size_t len, loff_t * ppos);
> +ssize_t vgpu_write(struct file *file, const char __user *data,
> +               size_t len, loff_t *ppos);
> +
> +/*
> + * Functions
> + */
> +
> +struct vgpu_device *get_vgpu_device_from_group(struct iommu_group *group)
> +{
> +    struct vgpu_device *vdev = NULL;
> +
> +    mutex_lock(&vgpu.vgpu_devices_lock);
> +    list_for_each_entry(vdev, &vgpu.vgpu_devices_list, list) {
> +        if (vdev->group) {
> +            if (iommu_group_id(vdev->group) == iommu_group_id(group)) {
> +                mutex_unlock(&vgpu.vgpu_devices_lock);
> +                return vdev;
> +            }
> +        }
> +    }
> +    mutex_unlock(&vgpu.vgpu_devices_lock);
> +    return NULL;
> +}
> +
> +EXPORT_SYMBOL_GPL(get_vgpu_device_from_group);
> +
> +int vgpu_register_device(struct pci_dev *dev, const struct
> gpu_device_ops *ops)
> +{
> +    int ret = 0;
> +    struct gpu_device *gpu_dev, *tmp;
> +
> +    if (!dev)
> +        return -EINVAL;
> +
> +        gpu_dev = kzalloc(sizeof(*gpu_dev), GFP_KERNEL);
> +        if (!gpu_dev)
> +                return -ENOMEM;
> +
> +    gpu_dev->dev = dev;
> +        gpu_dev->ops = ops;
> +
> +        mutex_lock(&vgpu.gpu_devices_lock);
> +
> +        /* Check for duplicates */
> +        list_for_each_entry(tmp, &vgpu.gpu_devices_list, gpu_next) {
> +                if (tmp->dev == dev) {
> +                        mutex_unlock(&vgpu.gpu_devices_lock);
> +                        kfree(gpu_dev);
> +                        return -EINVAL;
> +                }
> +        }
> +
> +    ret = vgpu_create_pci_device_files(dev);
> +    if (ret) {
> +        mutex_unlock(&vgpu.gpu_devices_lock);
> +        kfree(gpu_dev);
> +        return ret;
> +    }
> +        list_add(&gpu_dev->gpu_next, &vgpu.gpu_devices_list);
> +
> +    printk(KERN_INFO "VGPU: Registered dev 0x%x 0x%x, class 0x%x\n",
> dev->vendor, dev->device, dev->class);
> +        mutex_unlock(&vgpu.gpu_devices_lock);
> +
> +        return 0;
> +}
> +EXPORT_SYMBOL(vgpu_register_device);
> +
> +void vgpu_unregister_device(struct pci_dev *dev)
> +{
> +        struct gpu_device *gpu_dev;
> +
> +        mutex_lock(&vgpu.gpu_devices_lock);
> +        list_for_each_entry(gpu_dev, &vgpu.gpu_devices_list, gpu_next) {
> +                if (gpu_dev->dev == dev) {
> +            printk(KERN_INFO "VGPU: Unregistered dev 0x%x 0x%x, class
> 0x%x\n", dev->vendor, dev->device, dev->class);
> +            vgpu_remove_pci_device_files(dev);
> +                        list_del(&gpu_dev->gpu_next);
> +                        mutex_unlock(&vgpu.gpu_devices_lock);
> +                        kfree(gpu_dev);
> +                        return;
> +                }
> +        }
> +        mutex_unlock(&vgpu.gpu_devices_lock);
> +}
> +EXPORT_SYMBOL(vgpu_unregister_device);
> +
> +/*
> + * Helper Functions
> + */
> +
> +static struct vgpu_device *vgpu_device_alloc(uuid_le uuid, int
> instance, char *name)
> +{
> +    struct vgpu_device *vgpu_dev = NULL;
> +
> +    vgpu_dev = kzalloc(sizeof(*vgpu_dev), GFP_KERNEL);
> +    if (!vgpu_dev)
> +        return ERR_PTR(-ENOMEM);
> +
> +    kref_init(&vgpu_dev->kref);
> +    memcpy(&vgpu_dev->uuid, &uuid, sizeof(uuid_le));
> +    vgpu_dev->vgpu_instance = instance;
> +    strcpy(vgpu_dev->dev_name, name);
> +
> +    mutex_lock(&vgpu.vgpu_devices_lock);
> +    list_add(&vgpu_dev->list, &vgpu.vgpu_devices_list);
> +    mutex_unlock(&vgpu.vgpu_devices_lock);
> +
> +    return vgpu_dev;
> +}
> +
> +static void vgpu_device_free(struct vgpu_device *vgpu_dev)
> +{
> +    if (vgpu_dev) {
> +        mutex_lock(&vgpu.vgpu_devices_lock);
> +        list_del(&vgpu_dev->list);
> +        mutex_unlock(&vgpu.vgpu_devices_lock);
> +        kfree(vgpu_dev);
> +    }
> +    return;
> +}
> +
> +struct vgpu_device *vgpu_drv_get_vgpu_device_by_uuid(uuid_le uuid, int
> instance)
> +{
> +    struct vgpu_device *vdev = NULL;
> +
> +    mutex_lock(&vgpu.vgpu_devices_lock);
> +    list_for_each_entry(vdev, &vgpu.vgpu_devices_list, list) {
> +        if ((uuid_le_cmp(vdev->uuid, uuid) == 0) &&
> +                (vdev->vgpu_instance == instance)) {
> +            mutex_unlock(&vgpu.vgpu_devices_lock);
> +            return vdev;
> +        }
> +    }
> +    mutex_unlock(&vgpu.vgpu_devices_lock);
> +    return NULL;
> +}
> +
> +static void vgpu_device_release(struct device *dev)
> +{
> +    struct vgpu_device *vgpu_dev = to_vgpu_device(dev);
> +    vgpu_device_free(vgpu_dev);
> +}
> +
> +int create_vgpu_device(struct pci_dev *pdev, uuid_le uuid, uint32_t
> instance, uint32_t vgpu_id)
> +{
> +    char name[64];
> +    int numChar = 0;
> +    int retval = 0;
> +    struct vgpu_device *vgpu_dev = NULL;
> +    struct gpu_device *gpu_dev;
> +
> +    printk(KERN_INFO "VGPU: %s: device ", __FUNCTION__);
> +
> +    numChar = sprintf(name, "%pUb-%d", uuid.b, instance);
> +    name[numChar] = '\0';
> +
> +    vgpu_dev = vgpu_device_alloc(uuid, instance, name);
> +    if (IS_ERR(vgpu_dev)) {
> +        return PTR_ERR(vgpu_dev);
> +    }
> +
> +    vgpu_dev->vgpu_id     = vgpu_id;
> +    vgpu_dev->dev.parent  = NULL;
> +    vgpu_dev->dev.bus     = &vgpu_bus_type;
> +    vgpu_dev->dev.release = vgpu_device_release;
> +    dev_set_name(&vgpu_dev->dev, "%s", name);
> +
> +    retval = device_register(&vgpu_dev->dev);
> +    if (retval)
> +        goto create_failed1;
> +
> +    printk(KERN_INFO "UUID %pUb \n", vgpu_dev->uuid.b);
> +
> +    mutex_lock(&vgpu.gpu_devices_lock);
> +    list_for_each_entry(gpu_dev, &vgpu.gpu_devices_list, gpu_next) {
> +        if (gpu_dev->dev == pdev) {
> +            vgpu_dev->gpu_dev = gpu_dev;
> +            if (gpu_dev->ops->vgpu_create) {
> +                retval = gpu_dev->ops->vgpu_create(pdev, vgpu_dev->uuid,
> +                                   instance, vgpu_id);
> +                if (retval) {
> +                    mutex_unlock(&vgpu.gpu_devices_lock);
> +                    goto create_failed2;
> +                }
> +            }
> +            break;
> +        }
> +    }
> +    if (!vgpu_dev->gpu_dev) {
> +        retval = -EINVAL;
> +        goto create_failed2;
> +    }
> +
> +    mutex_unlock(&vgpu.gpu_devices_lock);
> +
> +    return retval;
> +
> +create_failed2:
> +    device_unregister(&vgpu_dev->dev);
> +
> +create_failed1:
> +    vgpu_device_free(vgpu_dev);
> +
> +    return retval;
> +}
> +
> +void destroy_vgpu_device(struct vgpu_device *vgpu_dev)
> +{
> +    printk(KERN_INFO "VGPU: destroying device %s ", vgpu_dev->dev_name);
> +    if (vgpu_dev->gpu_dev->ops->vgpu_destroy) {
> +        int retval = 0;
> +        retval =
> vgpu_dev->gpu_dev->ops->vgpu_destroy(vgpu_dev->gpu_dev->dev,
> +                                  vgpu_dev->uuid,
> +                                  vgpu_dev->vgpu_instance);
> +    /* if vendor driver doesn't return success that means vendor driver
> doesn't
> +     * support hot-unplug */
> +        if (retval)
> +            return;
> +    }
> +
> +    device_unregister(&vgpu_dev->dev);
> +}
> +
> +void destroy_vgpu_device_by_uuid(uuid_le uuid, int instance)
> +{
> +    struct vgpu_device *vdev, *vgpu_dev = NULL;
> +
> +    mutex_lock(&vgpu.vgpu_devices_lock);
> +
> +    // search VGPU device
> +    list_for_each_entry(vdev, &vgpu.vgpu_devices_list, list) {
> +        if ((uuid_le_cmp(vdev->uuid, uuid) == 0) &&
> +                (vdev->vgpu_instance == instance)) {
> +            vgpu_dev = vdev;
> +            break;
> +        }
> +    }
> +
> +    mutex_unlock(&vgpu.vgpu_devices_lock);
> +    if (vgpu_dev)
> +        destroy_vgpu_device(vgpu_dev);
> +}
> +
> +void get_vgpu_supported_types(struct device *dev, char *str)
> +{
> +    struct gpu_device *gpu_dev;
> +
> +    mutex_lock(&vgpu.gpu_devices_lock);
> +    list_for_each_entry(gpu_dev, &vgpu.gpu_devices_list, gpu_next) {
> +        if (&gpu_dev->dev->dev == dev) {
> +            if (gpu_dev->ops->vgpu_supported_config)
> +                gpu_dev->ops->vgpu_supported_config(gpu_dev->dev, str);
> +            break;
> +        }
> +    }
> +    mutex_unlock(&vgpu.gpu_devices_lock);
> +}
> +
> +int vgpu_start_callback(struct vgpu_device *vgpu_dev)
> +{
> +    int ret = 0;
> +
> +    mutex_lock(&vgpu.gpu_devices_lock);
> +    if (vgpu_dev->gpu_dev->ops->vgpu_start)
> +        ret = vgpu_dev->gpu_dev->ops->vgpu_start(vgpu_dev->uuid);
> +    mutex_unlock(&vgpu.gpu_devices_lock);
> +    return ret;
> +}
> +
> +int vgpu_shutdown_callback(struct vgpu_device *vgpu_dev)
> +{
> +    int ret = 0;
> +
> +    mutex_lock(&vgpu.gpu_devices_lock);
> +    if (vgpu_dev->gpu_dev->ops->vgpu_shutdown)
> +        ret = vgpu_dev->gpu_dev->ops->vgpu_shutdown(vgpu_dev->uuid);
> +    mutex_unlock(&vgpu.gpu_devices_lock);
> +    return ret;
> +}
> +
> +int vgpu_set_irqs_callback(struct vgpu_device *vgpu_dev, uint32_t flags,
> +                           unsigned index, unsigned start, unsigned count,
> +                           void *data)
> +{
> +       int ret = 0;
> +
> +       mutex_lock(&vgpu.gpu_devices_lock);
> +       if (vgpu_dev->gpu_dev->ops->vgpu_set_irqs)
> +               ret = vgpu_dev->gpu_dev->ops->vgpu_set_irqs(vgpu_dev,
> flags,
> +                                                          index, start,
> count, data);
> +       mutex_unlock(&vgpu.gpu_devices_lock);
> +       return ret;
> +}
> +
> +char *vgpu_devnode(struct device *dev, umode_t *mode)
> +{
> +    return kasprintf(GFP_KERNEL, "vgpu/%s", dev_name(dev));
> +}
> +
> +static void release_vgpubus_dev(struct device *dev)
> +{
> +    struct vgpu_device *vgpu_dev = to_vgpu_device(dev);
> +    destroy_vgpu_device(vgpu_dev);
> +}
> +
> +static struct class vgpu_class = {
> +    .name        = VGPU_CLASS_NAME,
> +    .owner        = THIS_MODULE,
> +    .class_attrs    = vgpu_class_attrs,
> +    .dev_groups    = vgpu_dev_groups,
> +    .devnode    = vgpu_devnode,
> +    .dev_release    = release_vgpubus_dev,
> +};
> +
> +static int __init vgpu_init(void)
> +{
> +    int rc = 0;
> +
> +    memset(&vgpu, 0 , sizeof(vgpu));
> +
> +    mutex_init(&vgpu.vgpu_devices_lock);
> +    INIT_LIST_HEAD(&vgpu.vgpu_devices_list);
> +    mutex_init(&vgpu.gpu_devices_lock);
> +    INIT_LIST_HEAD(&vgpu.gpu_devices_list);
> +
> +    rc = class_register(&vgpu_class);
> +    if (rc < 0) {
> +        printk(KERN_ERR "Error: failed to register vgpu class\n");
> +        goto failed1;
> +    }
> +
> +    rc = vgpu_bus_register();
> +    if (rc < 0) {
> +        printk(KERN_ERR "Error: failed to register vgpu bus\n");
> +        class_unregister(&vgpu_class);
> +    }
> +
> +failed1:
> +    return rc;
> +}
> +
> +static void __exit vgpu_exit(void)
> +{
> +    vgpu_bus_unregister();
> +    class_unregister(&vgpu_class);
> +}
> +
> +module_init(vgpu_init)
> +module_exit(vgpu_exit)
> +
> +MODULE_VERSION(DRIVER_VERSION);
> +MODULE_LICENSE("GPL");
> +MODULE_AUTHOR(DRIVER_AUTHOR);
> +MODULE_DESCRIPTION(DRIVER_DESC);
> diff --git a/include/linux/vgpu.h b/include/linux/vgpu.h
> new file mode 100644
> index 0000000..8595df5
> --- /dev/null
> +++ b/include/linux/vgpu.h
> @@ -0,0 +1,189 @@
> +/*
> + * VGPU definition
> + *
> + * Copyright (c) 2016, NVIDIA CORPORATION. All rights reserved.
> + *     Author:
> + *
> + * This program is free software; you can redistribute it and/or modify
> + * it under the terms of the GNU General Public License version 2 as
> + * published by the Free Software Foundation.
> + */
> +
> +#ifndef VGPU_H
> +#define VGPU_H
> +
> +// Common Data structures
> +
> +struct pci_bar_info {
> +    uint64_t start;
> +    uint64_t end;
> +    int flags;
> +};
> +
> +enum vgpu_emul_space_e {
> +    vgpu_emul_space_config = 0, /*!< PCI configuration space */
> +    vgpu_emul_space_io = 1,     /*!< I/O register space */
> +    vgpu_emul_space_mmio = 2    /*!< Memory-mapped I/O space */
> +};
> +
> +struct gpu_device;
> +
> +/*
> + * VGPU device
> + */
> +struct vgpu_device {
> +    struct kref        kref;
> +    struct device        dev;
> +    struct gpu_device    *gpu_dev;
> +    struct iommu_group    *group;
> +#define DEVICE_NAME_LEN        (64)
> +    char            dev_name[DEVICE_NAME_LEN];
> +    uuid_le            uuid;
> +    uint32_t        vgpu_instance;
> +    uint32_t        vgpu_id;
> +    char            config_space[0x100];          // 4KB PCI cfg space
> +    struct pci_bar_info    bar[VFIO_PCI_NUM_REGIONS];
> +    struct device_attribute    *dev_attr_vgpu_status;
> +    int            vgpu_device_status;
> +
> +    void            *driver_data;
> +
> +    struct list_head    list;
> +};
> +
> +
> +/**
> + * struct gpu_device_ops - Structure to be registered for each physical
> GPU to
> + * register the device to vgpu module.
> + *
> + * @owner:            The module owner.
> + * @vgpu_supported_config:    Called to get information about supported
> vgpu types.
> + *                @dev : pci device structure of physical GPU.
> + *                @config: should return string listing supported config
> + *                Returns integer: success (0) or error (< 0)
> + * @vgpu_create:        Called to allocate basic resouces in graphics
> + *                driver for a particular vgpu.
> + *                @dev: physical pci device structure on which vgpu
> + *                      should be created
> + *                @uuid: uuid for which VM it is intended to
> + *                @instance: vgpu instance in that VM
> + *                @vgpu_id: This represents the type of vgpu to be
> + *                      created
> + *                Returns integer: success (0) or error (< 0)
> + * @vgpu_destroy:        Called to free resources in graphics driver for
> + *                a vgpu instance of that VM.
> + *                @dev: physical pci device structure to which
> + *                this vgpu points to.
> + *                @uuid: uuid for which the vgpu belongs to.
> + *                @instance: vgpu instance in that VM
> + *                Returns integer: success (0) or error (< 0)
> + *                If VM is running and vgpu_destroy is called that
> + *                means the vGPU is being hotunpluged. Return error
> + *                if VM is running and graphics driver doesn't
> + *                support vgpu hotplug.
> + * @vgpu_start:            Called to do initiate vGPU initialization
> + *                process in graphics driver when VM boots before
> + *                qemu starts.
> + *                @uuid: UUID which is booting.
> + *                Returns integer: success (0) or error (< 0)
> + * @vgpu_shutdown:        Called to teardown vGPU related resources for
> + *                the VM
> + *                @uuid: UUID which is shutting down .
> + *                Returns integer: success (0) or error (< 0)
> + * @read:            Read emulation callback
> + *                @vdev: vgpu device structure
> + *                @buf: read buffer
> + *                @count: number bytes to read
> + *                @address_space: specifies for which address space
> + *                the request is: pci_config_space, IO register
> + *                space or MMIO space.
> + *                Retuns number on bytes read on success or error.
> + * @write:            Write emulation callback
> + *                @vdev: vgpu device structure
> + *                @buf: write buffer
> + *                @count: number bytes to be written
> + *                @address_space: specifies for which address space
> + *                the request is: pci_config_space, IO register
> + *                space or MMIO space.
> + *                Retuns number on bytes written on success or error.
> + * @vgpu_set_irqs:        Called to send about interrupts configuration
> + *                information that qemu set.
> + *                @vdev: vgpu device structure
> + *                @flags, index, start, count and *data : same as
> + *                that of struct vfio_irq_set of
> + *                VFIO_DEVICE_SET_IRQS API.
> + *
> + * Physical GPU that support vGPU should be register with vgpu module with
> + * gpu_device_ops structure.
> + */
> +
> +struct gpu_device_ops {
> +    struct module   *owner;
> +    int    (*vgpu_supported_config)(struct pci_dev *dev, char *config);
> +    int     (*vgpu_create)(struct pci_dev *dev, uuid_le uuid,
> +                   uint32_t instance, uint32_t vgpu_id);
> +    int     (*vgpu_destroy)(struct pci_dev *dev, uuid_le uuid,
> +                    uint32_t instance);
> +    int     (*vgpu_start)(uuid_le uuid);
> +    int     (*vgpu_shutdown)(uuid_le uuid);
> +    ssize_t (*read) (struct vgpu_device *vdev, char *buf, size_t count,
> +             uint32_t address_space, loff_t pos);
> +    ssize_t (*write)(struct vgpu_device *vdev, char *buf, size_t count,
> +             uint32_t address_space,loff_t pos);
> +    int     (*vgpu_set_irqs)(struct vgpu_device *vdev, uint32_t flags,
> +                 unsigned index, unsigned start, unsigned count,
> +                 void *data);
> +
> +};
> +
> +/*
> + * Physical GPU
> + */
> +struct gpu_device {
> +    struct pci_dev                  *dev;
> +    const struct gpu_device_ops     *ops;
> +    struct list_head                gpu_next;
> +};
> +
> +/**
> + * struct vgpu_driver - vGPU device driver
> + * @name: driver name
> + * @probe: called when new device created
> + * @remove: called when device removed
> + * @driver: device driver structure
> + *
> + **/
> +struct vgpu_driver {
> +    const char *name;
> +    int  (*probe)  (struct device *dev);
> +    void (*remove) (struct device *dev);
> +    struct device_driver    driver;
> +};
> +
> +static inline struct vgpu_driver *to_vgpu_driver(struct device_driver
> *drv)
> +{
> +    return drv ? container_of(drv, struct vgpu_driver, driver) : NULL;
> +}
> +
> +static inline struct vgpu_device *to_vgpu_device(struct device *dev)
> +{
> +    return dev ? container_of(dev, struct vgpu_device, dev) : NULL;
> +}
> +
> +extern struct bus_type vgpu_bus_type;
> +
> +#define dev_is_vgpu(d) ((d)->bus == &vgpu_bus_type)
> +
> +extern int  vgpu_register_device(struct pci_dev *dev, const struct
> gpu_device_ops *ops);
> +extern void vgpu_unregister_device(struct pci_dev *dev);
> +
> +extern int  vgpu_register_driver(struct vgpu_driver *drv, struct module
> *owner);
> +extern void vgpu_unregister_driver(struct vgpu_driver *drv);
> +
> +extern int vgpu_map_virtual_bar(uint64_t virt_bar_addr, uint64_t
> phys_bar_addr, uint32_t len, uint32_t flags);
> +extern int vgpu_dma_do_translate(dma_addr_t * gfn_buffer, uint32_t count);
> +
> +struct vgpu_device *get_vgpu_device_from_group(struct iommu_group *group);
> +
> +#endif /* VGPU_H */
> +