[PATCH v10 11/12] tools/libxl: explicitly grant access to needed I/O-memory ranges

[Arianna Avanzini <avanzini.arianna@gmail.com>]

This commit changes the existing libxl code to be sure to grant access
permission to PCI-related I/O memory ranges, while setting up passthrough
of PCI devices specified in the domain's configuration, and to VGA-related
memory ranges, while setting up VGA passthrough (if gfx_passthru = 1 and
gfx_passthru_primary = 1 in the domain's configuration).
As for the latter, the newly-added code does not replace any existing one,
but instead matches the calls to xc_domain_memory_mapping() performed by
QEMU on the path that is executed if gfx passthru is enabled and follows
the registration of a new VGA controller (in register_vga_regions(),
defined in hw/pt-graphics.c). In fact, VGA needs some extra memory
ranges to be mapped with respect to PCI; QEMU expects that access to those
memory ranges is implicitly granted when he calls the hypervisor with the
function xc_domain_memory_mapping(): this commit calls iomem_permission
for it when needed for a primary GPU by checking the passthru PCI devices'
class.

This commit is instrumental to the following one, which will separate
the functions of the iomem_permission and memory_mapping DOMCTLs, so
that requesting an I/O-memory range will not imply that access to such a
range is implicitly granted.

Signed-off-by: Arianna Avanzini <avanzini.arianna@gmail.com>
Cc: Dario Faggioli <dario.faggioli@citrix.com>
Cc: Paolo Valente <paolo.valente@unimore.it>
Cc: Stefano Stabellini <stefano.stabellini@eu.citrix.com>
Cc: Julien Grall <julien.grall@citrix.com>
Cc: Ian Campbell <Ian.Campbell@eu.citrix.com>
Cc: Jan Beulich <JBeulich@suse.com>
Cc: Keir Fraser <keir@xen.org>
Cc: Tim Deegan <tim@xen.org>
Cc: Ian Jackson <Ian.Jackson@eu.citrix.com>
Cc: Andrew Cooper <andrew.cooper3@citrix.com>
Cc: Eric Trudeau <etrudeau@broadcom.com>
Cc: Viktor Kleinik <viktor.kleinik@globallogic.com>

---

    v10:
        - Use a class-based check on the PCI device to determine if VGA-related
          I/O-memory ranges must be made accessible to a guest that needs gfx
          passthrough.

    v9:
        - Allow a domain access to the VGA framebuffer only if the user has
          signaled that one of the passthru GPUs is primary via domain config.

    v8:
        - Explain better in the commit description VGA-related code additions.
        - Fix v6 changelog which, being too generic, ended up to uncorrectly
          state that one of the requests had been addressed.
        - Remove tentative phrases from commit description.

    v7:
        - Let libxl not handle I/O ports and I/O memory differently when access
          to a PCI device is allowed to a domain.
        - Change the construct used by libxl during PCI-related initialization
          from a switch to an if to better suit the new execution flow.

---
 tools/libxl/libxl_create.c | 60 ++++++++++++++++++++++++++++++++++++++++++++++
 tools/libxl/libxl_pci.c    | 26 ++++++++------------
 tools/libxl/xl_cmdimpl.c   |  1 +
 3 files changed, 71 insertions(+), 16 deletions(-)

diff --git a/tools/libxl/libxl_create.c b/tools/libxl/libxl_create.c
index 1217310..b5bc07a 100644
--- a/tools/libxl/libxl_create.c
+++ b/tools/libxl/libxl_create.c
@@ -19,6 +19,7 @@
 
 #include "libxl_internal.h"
 #include "libxl_arch.h"
+#include "libxl_pci.h"
 
 #include <xc_dom.h>
 #include <xenguest.h>
@@ -1194,6 +1195,65 @@ static void domcreate_launch_dm(libxl__egc *egc, libxl__multidev *multidev,
             libxl__spawn_stub_dm(egc, &dcs->dmss);
         else
             libxl__spawn_local_dm(egc, &dcs->dmss.dm);
+
+        /*
+         * If VGA passthru is enabled by domain config and one of the
+         * passthru GPUs is of VGA class, be sure that the related
+         * stubdomain and the domain itself can access the VGA
+         * framebuffer.
+         */
+        for (i = 0 ; d_config->b_info.u.hvm.gfx_passthru.val &&
+                     i < d_config->num_pcidevs ; i++) {
+            uint64_t vga_iomem_start = 0xa0000 >> XC_PAGE_SHIFT;
+            int stubdom_domid;
+            libxl_device_pci *pcidev = &d_config->pcidevs[i];
+            char *pci_device_class_path =
+                libxl__sprintf(gc, SYSFS_PCI_DEV"/"PCI_BDF"/class",
+                               pcidev->domain, pcidev->bus, pcidev->dev,
+                               pcidev->func);
+            int read_items;
+            unsigned long pci_device_class;
+            FILE *f = fopen(pci_device_class_path, "r");
+            if (!f) {
+                LOGE(ERROR,
+                     "pci device "PCI_BDF" does not have class attribute",
+                     pcidev->domain, pcidev->bus, pcidev->dev, pcidev->func);
+                continue;
+            }
+            read_items = fscanf(f, "0x%lx\n", &pci_device_class);
+            fclose(f);
+            if (read_items != 1) {
+                LOGE(ERROR,
+                     "cannot read class of pci device "PCI_BDF,
+                     pcidev->domain, pcidev->bus, pcidev->dev, pcidev->func);
+                continue;
+            }
+            if (pci_device_class != 0x030000)
+                continue;
+
+            stubdom_domid = libxl_get_stubdom_id(CTX, domid);
+            ret = xc_domain_iomem_permission(CTX->xch, stubdom_domid,
+                                             vga_iomem_start, 0x20, 1);
+            if (ret < 0) {
+                LOGE(ERROR,
+                     "failed to give stubdom%d access to iomem range "
+                     "%"PRIx64"-%"PRIx64" for VGA passthru",
+                     stubdom_domid,
+                     vga_iomem_start, (vga_iomem_start + 0x20 - 1));
+                goto error_out;
+            }
+            ret = xc_domain_iomem_permission(CTX->xch, domid,
+                                             vga_iomem_start, 0x20, 1);
+            if (ret < 0) {
+                LOGE(ERROR,
+                     "failed to give dom%d access to iomem range "
+                     "%"PRIx64"-%"PRIx64" for VGA passthru",
+                     domid, vga_iomem_start, (vga_iomem_start + 0x20 - 1));
+                goto error_out;
+            }
+            break;
+        }
+
         return;
     }
     case LIBXL_DOMAIN_TYPE_PV:
diff --git a/tools/libxl/libxl_pci.c b/tools/libxl/libxl_pci.c
index 21ecab1..1524170 100644
--- a/tools/libxl/libxl_pci.c
+++ b/tools/libxl/libxl_pci.c
@@ -841,10 +841,13 @@ static int qemu_pci_add_xenstore(libxl__gc *gc, uint32_t domid,
 static int do_pci_add(libxl__gc *gc, uint32_t domid, libxl_device_pci *pcidev, int starting)
 {
     libxl_ctx *ctx = libxl__gc_owner(gc);
+    libxl_domain_type type = libxl__domain_type(gc, domid);
     int rc, hvm = 0;
 
-    switch (libxl__domain_type(gc, domid)) {
-    case LIBXL_DOMAIN_TYPE_HVM:
+    if (type == LIBXL_DOMAIN_TYPE_INVALID)
+        return ERROR_FAIL;
+
+    if (type == LIBXL_DOMAIN_TYPE_HVM) {
         hvm = 1;
         if (libxl__wait_for_device_model_deprecated(gc, domid, "running",
                                          NULL, NULL, NULL) < 0) {
@@ -862,8 +865,7 @@ static int do_pci_add(libxl__gc *gc, uint32_t domid, libxl_device_pci *pcidev, i
         }
         if ( rc )
             return ERROR_FAIL;
-        break;
-    case LIBXL_DOMAIN_TYPE_PV:
+    }
     {
         char *sysfs_path = libxl__sprintf(gc, SYSFS_PCI_DEV"/"PCI_BDF"/resource", pcidev->domain,
                                          pcidev->bus, pcidev->dev, pcidev->func);
@@ -932,10 +934,6 @@ static int do_pci_add(libxl__gc *gc, uint32_t domid, libxl_device_pci *pcidev, i
                 return ERROR_FAIL;
             }
         }
-        break;
-    }
-    case LIBXL_DOMAIN_TYPE_INVALID:
-        return ERROR_FAIL;
     }
 out:
     if (!libxl_is_stubdom(ctx, domid, NULL)) {
@@ -1161,6 +1159,7 @@ static int do_pci_remove(libxl__gc *gc, uint32_t domid,
 {
     libxl_ctx *ctx = libxl__gc_owner(gc);
     libxl_device_pci *assigned;
+    libxl_domain_type type = libxl__domain_type(gc, domid);
     int hvm = 0, rc, num;
     int stubdomid = 0;
 
@@ -1176,8 +1175,7 @@ static int do_pci_remove(libxl__gc *gc, uint32_t domid,
     }
 
     rc = ERROR_FAIL;
-    switch (libxl__domain_type(gc, domid)) {
-    case LIBXL_DOMAIN_TYPE_HVM:
+    if (type == LIBXL_DOMAIN_TYPE_HVM) {
         hvm = 1;
         if (libxl__wait_for_device_model_deprecated(gc, domid, "running",
                                          NULL, NULL, NULL) < 0)
@@ -1198,8 +1196,8 @@ static int do_pci_remove(libxl__gc *gc, uint32_t domid,
             rc = ERROR_FAIL;
             goto out_fail;
         }
-        break;
-    case LIBXL_DOMAIN_TYPE_PV:
+    } else if (type != LIBXL_DOMAIN_TYPE_PV)
+        abort();
     {
         char *sysfs_path = libxl__sprintf(gc, SYSFS_PCI_DEV"/"PCI_BDF"/resource", pcidev->domain,
                                          pcidev->bus, pcidev->dev, pcidev->func);
@@ -1249,10 +1247,6 @@ skip1:
             }
         }
         fclose(f);
-        break;
-    }
-    default:
-        abort();
     }
 out:
     /* don't do multiple resets while some functions are still passed through */
diff --git a/tools/libxl/xl_cmdimpl.c b/tools/libxl/xl_cmdimpl.c
index e035e97..7f1bb6b 100644
--- a/tools/libxl/xl_cmdimpl.c
+++ b/tools/libxl/xl_cmdimpl.c
@@ -1422,6 +1422,7 @@ skip_vfb:
     if (!xlu_cfg_get_list (config, "pci", &pcis, 0, 0)) {
         d_config->num_pcidevs = 0;
         d_config->pcidevs = NULL;
+        fprintf(stderr, "XEN parsing pci list\n");
         for(i = 0; (buf = xlu_cfg_get_listitem (pcis, i)) != NULL; i++) {
             libxl_device_pci *pcidev;
 
-- 
2.0.3