[PATCH v11 12/14] tools/libxl: explicitly grant access to needed I/O-memory ranges

[Arianna Avanzini <avanzini.arianna@gmail.com>]

This commit changes the existing libxl code to be sure to grant access
permission to PCI-related I/O memory ranges, while setting up passthrough
of PCI devices specified in the domain's configuration, and to VGA-related
memory ranges, while setting up VGA passthrough (if gfx_passthru = 1 and
gfx_passthru_primary = 1 in the domain's configuration).
As for the latter, the newly-added code does not replace any existing one,
but instead matches the calls to xc_domain_memory_mapping() performed by
QEMU on the path that is executed if gfx passthru is enabled and follows
the registration of a new VGA controller (in register_vga_regions(),
defined in hw/pt-graphics.c). In fact, VGA needs some extra memory
ranges to be mapped with respect to PCI; QEMU expects that access to those
memory ranges is implicitly granted when he calls the hypervisor with the
function xc_domain_memory_mapping(): this commit calls iomem_permission
for it when needed for a primary GPU by checking the passthru PCI devices'
class.

This commit is instrumental to the last one in the series, which will
separate the functions of the iomem_permission and memory_mapping DOMCTLs,
so that requesting an I/O-memory range will not imply that access to such
a range is implicitly granted.

Signed-off-by: Arianna Avanzini <avanzini.arianna@gmail.com>
Cc: Dario Faggioli <dario.faggioli@citrix.com>
Cc: Paolo Valente <paolo.valente@unimore.it>
Cc: Stefano Stabellini <stefano.stabellini@eu.citrix.com>
Cc: Julien Grall <julien.grall@citrix.com>
Cc: Ian Campbell <Ian.Campbell@eu.citrix.com>
Cc: Jan Beulich <JBeulich@suse.com>
Cc: Keir Fraser <keir@xen.org>
Cc: Tim Deegan <tim@xen.org>
Cc: Ian Jackson <Ian.Jackson@eu.citrix.com>
Cc: Andrew Cooper <andrew.cooper3@citrix.com>
Cc: Eric Trudeau <etrudeau@broadcom.com>
Cc: Viktor Kleinik <viktor.kleinik@globallogic.com>
Cc: Andrii Tseglytskyi <andrii.tseglytskyi@globallogic.com>

---

    v11:
        - Move the VGA-related hunk to a libxl helper; evaluate gfx_passthru.val
          only once as it is static in the scope of the helper.
        - Remove leftover debug fprintf().
        - Improve code readability by separating blocks of code.

    v10:
        - Use a class-based check on the PCI device to determine if VGA-related
          I/O-memory ranges must be made accessible to a guest that needs gfx
          passthrough.

    v9:
        - Allow a domain access to the VGA framebuffer only if the user has
          signaled that one of the passthru GPUs is primary via domain config.

    v8:
        - Explain better in the commit description VGA-related code additions.
        - Fix v6 changelog which, being too generic, ended up to uncorrectly
          state that one of the requests had been addressed.
        - Remove tentative phrases from commit description.

    v7:
        - Let libxl not handle I/O ports and I/O memory differently when access
          to a PCI device is allowed to a domain.
        - Change the construct used by libxl during PCI-related initialization
          from a switch to an if to better suit the new execution flow.

---
 tools/libxl/libxl_create.c   |  9 +++++
 tools/libxl/libxl_internal.h |  2 +
 tools/libxl/libxl_pci.c      | 91 ++++++++++++++++++++++++++++++++++++--------
 3 files changed, 86 insertions(+), 16 deletions(-)

diff --git a/tools/libxl/libxl_create.c b/tools/libxl/libxl_create.c
index 3b1ec91..7c02e20 100644
--- a/tools/libxl/libxl_create.c
+++ b/tools/libxl/libxl_create.c
@@ -1197,6 +1197,15 @@ static void domcreate_launch_dm(libxl__egc *egc, libxl__multidev *multidev,
             libxl__spawn_stub_dm(egc, &dcs->dmss);
         else
             libxl__spawn_local_dm(egc, &dcs->dmss.dm);
+
+        /*
+         * Handle the domain's (and the related stubdomain's) access to
+         * the VGA framebuffer.
+         */
+        ret = libxl__grant_vga_iomem_permission(gc, domid, d_config);
+        if ( ret )
+            goto error_out;
+
         return;
     }
     case LIBXL_DOMAIN_TYPE_PV:
diff --git a/tools/libxl/libxl_internal.h b/tools/libxl/libxl_internal.h
index 04c9378..e27861b 100644
--- a/tools/libxl/libxl_internal.h
+++ b/tools/libxl/libxl_internal.h
@@ -951,6 +951,8 @@ _hidden int libxl__domain_cpupool(libxl__gc *gc, uint32_t domid);
 _hidden libxl_scheduler libxl__domain_scheduler(libxl__gc *gc, uint32_t domid);
 _hidden int libxl__sched_set_params(libxl__gc *gc, uint32_t domid,
                                     libxl_domain_sched_params *scparams);
+_hidden int libxl__grant_vga_iomem_permission(libxl__gc *gc, const uint32_t domid,
+                                              libxl_domain_config *const d_config);
 
 typedef struct {
     uint32_t store_port;
diff --git a/tools/libxl/libxl_pci.c b/tools/libxl/libxl_pci.c
index 2782d0e..4fc9218 100644
--- a/tools/libxl/libxl_pci.c
+++ b/tools/libxl/libxl_pci.c
@@ -846,10 +846,13 @@ static int qemu_pci_add_xenstore(libxl__gc *gc, uint32_t domid,
 static int do_pci_add(libxl__gc *gc, uint32_t domid, libxl_device_pci *pcidev, int starting)
 {
     libxl_ctx *ctx = libxl__gc_owner(gc);
+    libxl_domain_type type = libxl__domain_type(gc, domid);
     int rc, hvm = 0;
 
-    switch (libxl__domain_type(gc, domid)) {
-    case LIBXL_DOMAIN_TYPE_HVM:
+    if (type == LIBXL_DOMAIN_TYPE_INVALID)
+        return ERROR_FAIL;
+
+    if (type == LIBXL_DOMAIN_TYPE_HVM) {
         hvm = 1;
         if (libxl__wait_for_device_model_deprecated(gc, domid, "running",
                                          NULL, NULL, NULL) < 0) {
@@ -867,8 +870,8 @@ static int do_pci_add(libxl__gc *gc, uint32_t domid, libxl_device_pci *pcidev, i
         }
         if ( rc )
             return ERROR_FAIL;
-        break;
-    case LIBXL_DOMAIN_TYPE_PV:
+    }
+
     {
         char *sysfs_path = libxl__sprintf(gc, SYSFS_PCI_DEV"/"PCI_BDF"/resource", pcidev->domain,
                                          pcidev->bus, pcidev->dev, pcidev->func);
@@ -937,11 +940,8 @@ static int do_pci_add(libxl__gc *gc, uint32_t domid, libxl_device_pci *pcidev, i
                 return ERROR_FAIL;
             }
         }
-        break;
-    }
-    case LIBXL_DOMAIN_TYPE_INVALID:
-        return ERROR_FAIL;
     }
+
 out:
     if (!libxl_is_stubdom(ctx, domid, NULL)) {
         rc = xc_assign_device(ctx->xch, domid, pcidev_encode_bdf(pcidev));
@@ -1166,6 +1166,7 @@ static int do_pci_remove(libxl__gc *gc, uint32_t domid,
 {
     libxl_ctx *ctx = libxl__gc_owner(gc);
     libxl_device_pci *assigned;
+    libxl_domain_type type = libxl__domain_type(gc, domid);
     int hvm = 0, rc, num;
     int stubdomid = 0;
 
@@ -1181,8 +1182,7 @@ static int do_pci_remove(libxl__gc *gc, uint32_t domid,
     }
 
     rc = ERROR_FAIL;
-    switch (libxl__domain_type(gc, domid)) {
-    case LIBXL_DOMAIN_TYPE_HVM:
+    if (type == LIBXL_DOMAIN_TYPE_HVM) {
         hvm = 1;
         if (libxl__wait_for_device_model_deprecated(gc, domid, "running",
                                          NULL, NULL, NULL) < 0)
@@ -1203,8 +1203,8 @@ static int do_pci_remove(libxl__gc *gc, uint32_t domid,
             rc = ERROR_FAIL;
             goto out_fail;
         }
-        break;
-    case LIBXL_DOMAIN_TYPE_PV:
+    } else if (type != LIBXL_DOMAIN_TYPE_PV)
+        abort();
     {
         char *sysfs_path = libxl__sprintf(gc, SYSFS_PCI_DEV"/"PCI_BDF"/resource", pcidev->domain,
                                          pcidev->bus, pcidev->dev, pcidev->func);
@@ -1254,10 +1254,6 @@ skip1:
             }
         }
         fclose(f);
-        break;
-    }
-    default:
-        abort();
     }
 out:
     /* don't do multiple resets while some functions are still passed through */
@@ -1435,6 +1431,69 @@ int libxl__device_pci_destroy_all(libxl__gc *gc, uint32_t domid)
     return 0;
 }
 
+int libxl__grant_vga_iomem_permission(libxl__gc *gc, const uint32_t domid,
+                                      libxl_domain_config *const d_config)
+{
+    int i, ret;
+
+    if (!d_config->b_info.u.hvm.gfx_passthru.val)
+        return 0;
+
+    for (i = 0 ; i < d_config->num_pcidevs ; i++) {
+        uint64_t vga_iomem_start = 0xa0000 >> XC_PAGE_SHIFT;
+        uint32_t stubdom_domid;
+        libxl_device_pci *pcidev = &d_config->pcidevs[i];
+        char *pci_device_class_path =
+            libxl__sprintf(gc, SYSFS_PCI_DEV"/"PCI_BDF"/class",
+                           pcidev->domain, pcidev->bus, pcidev->dev,
+                           pcidev->func);
+        int read_items;
+        unsigned long pci_device_class;
+
+        FILE *f = fopen(pci_device_class_path, "r");
+        if (!f) {
+            LOGE(ERROR,
+                 "pci device "PCI_BDF" does not have class attribute",
+                 pcidev->domain, pcidev->bus, pcidev->dev, pcidev->func);
+            continue;
+        }
+        read_items = fscanf(f, "0x%lx\n", &pci_device_class);
+        fclose(f);
+        if (read_items != 1) {
+            LOGE(ERROR,
+                 "cannot read class of pci device "PCI_BDF,
+                 pcidev->domain, pcidev->bus, pcidev->dev, pcidev->func);
+            continue;
+        }
+        if (pci_device_class != 0x030000) /* VGA class */
+            continue;
+
+        stubdom_domid = libxl_get_stubdom_id(CTX, domid);
+        ret = xc_domain_iomem_permission(CTX->xch, stubdom_domid,
+                                         vga_iomem_start, 0x20, 1);
+        if (ret < 0) {
+            LOGE(ERROR,
+                 "failed to give stubdom%d access to iomem range "
+                 "%"PRIx64"-%"PRIx64" for VGA passthru",
+                 stubdom_domid,
+                 vga_iomem_start, (vga_iomem_start + 0x20 - 1));
+            return ret;
+        }
+        ret = xc_domain_iomem_permission(CTX->xch, domid,
+                                         vga_iomem_start, 0x20, 1);
+        if (ret < 0) {
+            LOGE(ERROR,
+                 "failed to give dom%d access to iomem range "
+                 "%"PRIx64"-%"PRIx64" for VGA passthru",
+                 domid, vga_iomem_start, (vga_iomem_start + 0x20 - 1));
+            return ret;
+        }
+        break;
+    }
+
+    return 0;
+}
+
 /*
  * Local variables:
  * mode: C
-- 
2.1.0