[PATCH V1 24/26] vfio/iommufd: preserve descriptors

[Steve Sistare <steven.sistare@oracle.com>]

Save the iommu and vfio device fd in CPR state when it is created.
After CPR, the fd number is found in CPR state and reused.  Remember
the reused status for subsequent patches.  The reused status is cleared
when vmstate load finishes.

Signed-off-by: Steve Sistare <steven.sistare@oracle.com>
---
 backends/iommufd.c       | 24 ++++++++++++++++++++----
 hw/vfio/cpr-iommufd.c    | 15 +++++++++++++++
 hw/vfio/helpers.c        | 10 ++--------
 hw/vfio/iommufd.c        | 10 +++++++++-
 include/system/iommufd.h |  1 +
 5 files changed, 47 insertions(+), 13 deletions(-)

diff --git a/backends/iommufd.c b/backends/iommufd.c
index be5f6a3..e9452e4 100644
--- a/backends/iommufd.c
+++ b/backends/iommufd.c
@@ -16,12 +16,18 @@
 #include "qemu/module.h"
 #include "qom/object_interfaces.h"
 #include "qemu/error-report.h"
+#include "migration/cpr.h"
 #include "monitor/monitor.h"
 #include "trace.h"
 #include "hw/vfio/vfio-common.h"
 #include <sys/ioctl.h>
 #include <linux/iommufd.h>
 
+static const char *iommufd_fd_name(IOMMUFDBackend *be)
+{
+    return object_get_canonical_path_component(OBJECT(be));
+}
+
 static void iommufd_backend_init(Object *obj)
 {
     IOMMUFDBackend *be = IOMMUFD_BACKEND(obj);
@@ -46,10 +52,10 @@ static void iommufd_backend_set_fd(Object *obj, const char *str, Error **errp)
     ERRP_GUARD();
     IOMMUFDBackend *be = IOMMUFD_BACKEND(obj);
     int fd = -1;
+    const char *name = iommufd_fd_name(be);
 
-    fd = monitor_fd_param(monitor_cur(), str, errp);
+    fd = cpr_get_fd_param(name, str, 0, &be->reused, errp);
     if (fd == -1) {
-        error_prepend(errp, "Could not parse remote object fd %s:", str);
         return;
     }
     be->fd = fd;
@@ -98,10 +104,16 @@ bool iommufd_backend_connect(IOMMUFDBackend *be, Error **errp)
     int fd;
 
     if (be->owned && !be->users) {
-        fd = qemu_open("/dev/iommu", O_RDWR, errp);
+        const char *name = iommufd_fd_name(be);
+        fd = cpr_find_fd(name, 0);
+        be->reused = (fd >= 0);
+        if (!be->reused) {
+            fd = qemu_open("/dev/iommu", O_RDWR, errp);
+        }
         if (fd < 0) {
             return false;
         }
+        cpr_resave_fd(name, 0, fd);
         be->fd = fd;
     }
     be->users++;
@@ -112,6 +124,9 @@ bool iommufd_backend_connect(IOMMUFDBackend *be, Error **errp)
 
 void iommufd_backend_disconnect(IOMMUFDBackend *be)
 {
+    int fd = be->fd;
+    const char *name = iommufd_fd_name(be);
+
     if (!be->users) {
         goto out;
     }
@@ -121,7 +136,8 @@ void iommufd_backend_disconnect(IOMMUFDBackend *be)
         be->fd = -1;
     }
 out:
-    trace_iommufd_backend_disconnect(be->fd, be->users);
+    cpr_delete_fd(name, 0);
+    trace_iommufd_backend_disconnect(fd, be->users);
 }
 
 bool iommufd_backend_alloc_ioas(IOMMUFDBackend *be, uint32_t *ioas_id,
diff --git a/hw/vfio/cpr-iommufd.c b/hw/vfio/cpr-iommufd.c
index 4eb358a..053ff8c 100644
--- a/hw/vfio/cpr-iommufd.c
+++ b/hw/vfio/cpr-iommufd.c
@@ -24,10 +24,25 @@ static bool vfio_cpr_supported(VFIOIOMMUFDContainer *container, Error **errp)
     return true;
 }
 
+static int vfio_container_post_load(void *opaque, int version_id)
+{
+    VFIOIOMMUFDContainer *container = opaque;
+    VFIOContainerBase *bcontainer = &container->bcontainer;
+    VFIODevice *vbasedev;
+
+    QLIST_FOREACH(vbasedev, &bcontainer->device_list, container_next) {
+        vbasedev->reused = false;
+    }
+    container->be->reused = false;
+
+    return 0;
+}
+
 static const VMStateDescription vfio_container_vmstate = {
     .name = "vfio-iommufd-container",
     .version_id = 0,
     .minimum_version_id = 0,
+    .post_load = vfio_container_post_load,
     .needed = cpr_needed_for_reuse,
     .fields = (VMStateField[]) {
         VMSTATE_END_OF_LIST()
diff --git a/hw/vfio/helpers.c b/hw/vfio/helpers.c
index bd94b86..7f69707 100644
--- a/hw/vfio/helpers.c
+++ b/hw/vfio/helpers.c
@@ -678,14 +678,8 @@ bool vfio_device_get_name(VFIODevice *vbasedev, Error **errp)
 
 void vfio_device_set_fd(VFIODevice *vbasedev, const char *str, Error **errp)
 {
-    ERRP_GUARD();
-    int fd = monitor_fd_param(monitor_cur(), str, errp);
-
-    if (fd < 0) {
-        error_prepend(errp, "Could not parse remote object fd %s:", str);
-        return;
-    }
-    vbasedev->fd = fd;
+    vbasedev->fd = cpr_get_fd_param(vbasedev->dev->id, str, 0,
+                                    &vbasedev->reused, errp);
 }
 
 void vfio_device_init(VFIODevice *vbasedev, int type, VFIODeviceOps *ops,
diff --git a/hw/vfio/iommufd.c b/hw/vfio/iommufd.c
index ae78e00..abd17b6 100644
--- a/hw/vfio/iommufd.c
+++ b/hw/vfio/iommufd.c
@@ -25,6 +25,7 @@
 #include "qemu/cutils.h"
 #include "qemu/chardev_open.h"
 #include "migration/blocker.h"
+#include "migration/cpr.h"
 #include "pci.h"
 #include "exec/ram_addr.h"
 
@@ -499,13 +500,18 @@ static bool iommufd_cdev_attach(const char *name, VFIODevice *vbasedev,
         VFIO_IOMMU_CLASS(object_class_by_name(TYPE_VFIO_IOMMU_IOMMUFD));
 
     if (vbasedev->fd < 0) {
-        devfd = iommufd_cdev_getfd(vbasedev->sysfsdev, errp);
+        devfd = cpr_find_fd(vbasedev->name, 0);
+        vbasedev->reused = (devfd >= 0);
+        if (!vbasedev->reused) {
+            devfd = iommufd_cdev_getfd(vbasedev->sysfsdev, errp);
+        }
         if (devfd < 0) {
             return false;
         }
         vbasedev->fd = devfd;
     } else {
         devfd = vbasedev->fd;
+        /* reused was set in iommufd_backend_set_fd */
     }
 
     if (!iommufd_cdev_connect_and_bind(vbasedev, errp)) {
@@ -620,6 +626,7 @@ found_container:
     QLIST_INSERT_HEAD(&bcontainer->device_list, vbasedev, container_next);
     QLIST_INSERT_HEAD(&vfio_device_list, vbasedev, global_next);
     vfio_iommufd_cpr_register_device(vbasedev);
+    cpr_resave_fd(vbasedev->name, 0, devfd);
 
     trace_iommufd_cdev_device_info(vbasedev->name, devfd, vbasedev->num_irqs,
                                    vbasedev->num_regions, vbasedev->flags);
@@ -661,6 +668,7 @@ static void iommufd_cdev_detach(VFIODevice *vbasedev)
 
     migrate_del_blocker(&vbasedev->cpr_id_blocker);
     vfio_iommufd_cpr_unregister_device(vbasedev);
+    cpr_delete_fd(vbasedev->name, 0);
     iommufd_cdev_unbind_and_disconnect(vbasedev);
     close(vbasedev->fd);
 }
diff --git a/include/system/iommufd.h b/include/system/iommufd.h
index 4e9c037..6618248 100644
--- a/include/system/iommufd.h
+++ b/include/system/iommufd.h
@@ -32,6 +32,7 @@ struct IOMMUFDBackend {
     /*< protected >*/
     int fd;            /* /dev/iommu file descriptor */
     bool owned;        /* is the /dev/iommu opened internally */
+    bool reused;       /* fd is reused after CPR */
     uint32_t users;
 
     /*< public >*/
-- 
1.8.3.1