[RFC PATCH 0/1] Add pci_hole_min_size

[RFC PATCH 0/1] Add pci_hole_min_size

[Don Slutz]

This is posted as an RFC because you need the upstream version of qemu with:

http://marc.info/?l=qemu-devel&m=139354036513567&w=2

This allows growing the pci_hole to the size needed.

This may help with using pci passthru and HVM.

Don Slutz (1):
  Add pci_hole_min_size

 tools/firmware/hvmloader/pci.c | 10 ++++++++++
 tools/libxc/xc_hvm_build_x86.c | 22 ++++++++++++++++++++++
 tools/libxc/xenguest.h         | 11 +++++++++++
 tools/libxl/libxl_create.c     |  4 +++-
 tools/libxl/libxl_dm.c         | 13 +++++++++++--
 tools/libxl/libxl_dom.c        |  3 ++-
 tools/libxl/libxl_types.idl    |  1 +
 tools/libxl/xl_cmdimpl.c       |  6 ++++++
 8 files changed, 66 insertions(+), 4 deletions(-)

-- 
1.8.4

[RFC PATCH 0/1] Add pci_hole_min_size

[Don Slutz]

Changes from v1 to v2:
  Boris Ostrovsky:
    Need to init pci_hole_min_size

  Changed the qemu name from pci_hole_min_size to
  pc-memory-layout.max-ram-below-4g.

  Did not change the Xen version for now.

  Added quick note to xl.cfg.pod.5

  Added a check for a too big value. (Most likely in the wrong place.)

If you add enough PCI devices then all mmio may not fit below 4G
which may not be the layout the user wanted. This allows you to
increase the below 4G address space that PCI devices can use and
therefore in more cases not have any mmio that is above 4G.

There are real PCI cards that do not support mmio over 4G, so if you
want to emulate them precisely, you may also need to increase the
space below 4G for them.  There are drivers for these cards that
also do not work if they have their mmio space mapped above 4G.

This is posted as an RFC because you need the upstream version of
qemu with all 4 patches:

http://marc.info/?l=qemu-devel&m=139455360016654&w=2

This allows growing the pci_hole to the size needed.

This may help with using pci passthru and HVM.



Don Slutz (1):
  Add pci_hole_min_size

 docs/man/xl.cfg.pod.5          | 10 ++++++++++
 tools/firmware/hvmloader/pci.c | 12 ++++++++++++
 tools/libxc/xc_hvm_build_x86.c | 22 ++++++++++++++++++++++
 tools/libxc/xenguest.h         | 11 +++++++++++
 tools/libxl/libxl_create.c     |  4 +++-
 tools/libxl/libxl_dm.c         | 15 +++++++++++++++
 tools/libxl/libxl_dom.c        |  3 ++-
 tools/libxl/libxl_types.idl    |  1 +
 tools/libxl/xl_cmdimpl.c       |  6 ++++++
 9 files changed, 82 insertions(+), 2 deletions(-)

-- 
1.8.4

[RFC PATCH 1/1] Add pci_hole_min_size

[Don Slutz]

Add logging of max_ram_below_4g too big.

Signed-off-by: Don Slutz <dslutz@verizon.com>
---
 docs/man/xl.cfg.pod.5          | 10 ++++++++++
 tools/firmware/hvmloader/pci.c | 12 ++++++++++++
 tools/libxc/xc_hvm_build_x86.c | 22 ++++++++++++++++++++++
 tools/libxc/xenguest.h         | 11 +++++++++++
 tools/libxl/libxl_create.c     |  4 +++-
 tools/libxl/libxl_dm.c         | 15 +++++++++++++++
 tools/libxl/libxl_dom.c        |  3 ++-
 tools/libxl/libxl_types.idl    |  1 +
 tools/libxl/xl_cmdimpl.c       |  6 ++++++
 9 files changed, 82 insertions(+), 2 deletions(-)

diff --git a/docs/man/xl.cfg.pod.5 b/docs/man/xl.cfg.pod.5
index c02ad55..7ef7b10 100644
--- a/docs/man/xl.cfg.pod.5
+++ b/docs/man/xl.cfg.pod.5
@@ -1022,6 +1022,16 @@ wallclock (i.e., real) time.
 
 =back
 
+=head3 Memory layout
+
+=over 4
+
+=item B<pci_hole_min_size=BYTES>
+
+Specifies the minimum size the PCI MMIO hole below 4GiB will be.
+
+=back
+
 =head3 Support for Paravirtualisation of HVM Guests
 
 The following options allow Paravirtualised features (such as devices)
diff --git a/tools/firmware/hvmloader/pci.c b/tools/firmware/hvmloader/pci.c
index 627e8cb..4115929 100644
--- a/tools/firmware/hvmloader/pci.c
+++ b/tools/firmware/hvmloader/pci.c
@@ -58,6 +58,7 @@ void pci_setup(void)
         uint64_t bar_sz;
     } *bars = (struct bars *)scratch_start;
     unsigned int i, nr_bars = 0;
+    uint64_t pci_hole_min_size = 0;
 
     const char *s;
     /*
@@ -85,6 +86,10 @@ void pci_setup(void)
     printf("Relocating guest memory for lowmem MMIO space %s\n",
            allow_memory_relocate?"enabled":"disabled");
 
+    s = xenstore_read("platform/pci_hole_min_size", NULL);
+    if ( s )
+        pci_hole_min_size = strtoll(s, NULL, 0);
+
     /* Program PCI-ISA bridge with appropriate link routes. */
     isa_irq = 0;
     for ( link = 0; link < 4; link++ )
@@ -236,6 +241,13 @@ void pci_setup(void)
         pci_writew(devfn, PCI_COMMAND, cmd);
     }
 
+    if (pci_hole_min_size)
+    {
+        pci_mem_start = (1ULL << 32) - pci_hole_min_size;
+        printf("pci_mem_start=0x%lx (was 0x%x) for pci_hole_min_size=%lu\n",
+               pci_mem_start, PCI_MEM_START, (long)pci_hole_min_size);
+    }
+
     /*
      * At the moment qemu-xen can't deal with relocated memory regions.
      * It's too close to the release to make a proper fix; for now,
diff --git a/tools/libxc/xc_hvm_build_x86.c b/tools/libxc/xc_hvm_build_x86.c
index dd3b522..4752d58 100644
--- a/tools/libxc/xc_hvm_build_x86.c
+++ b/tools/libxc/xc_hvm_build_x86.c
@@ -603,16 +603,38 @@ int xc_hvm_build_target_mem(xc_interface *xch,
                            int target,
                            const char *image_name)
 {
+    return xc_hvm_build_target_mem_with_hole(xch, domid, memsize, target, image_name, 0);
+}
+
+int xc_hvm_build_target_mem_with_hole(xc_interface *xch,
+                                      uint32_t domid,
+                                      int memsize,
+                                      int target,
+                                      const char *image_name,
+                                      uint64_t pci_hole_min_size)
+{
     struct xc_hvm_build_args args = {};
 
     memset(&args, 0, sizeof(struct xc_hvm_build_args));
     args.mem_size = (uint64_t)memsize << 20;
     args.mem_target = (uint64_t)target << 20;
     args.image_file_name = image_name;
+    if (pci_hole_min_size > HVM_BELOW_4G_MMIO_LENGTH)
+        args.mmio_size = pci_hole_min_size;
 
     return xc_hvm_build(xch, domid, &args);
 }
 
+int xc_hvm_build_with_hole(xc_interface *xch, uint32_t domid,
+                           struct xc_hvm_build_args *args,
+                           uint64_t pci_hole_min_size)
+{
+    if (pci_hole_min_size > HVM_BELOW_4G_MMIO_LENGTH)
+        args->mmio_size = pci_hole_min_size;
+
+    return xc_hvm_build(xch, domid, args);
+}
+
 /*
  * Local variables:
  * mode: C
diff --git a/tools/libxc/xenguest.h b/tools/libxc/xenguest.h
index a0e30e1..44290bd 100644
--- a/tools/libxc/xenguest.h
+++ b/tools/libxc/xenguest.h
@@ -248,12 +248,23 @@ struct xc_hvm_build_args {
 int xc_hvm_build(xc_interface *xch, uint32_t domid,
                  struct xc_hvm_build_args *hvm_args);
 
+int xc_hvm_build_with_hole(xc_interface *xch, uint32_t domid,
+                           struct xc_hvm_build_args *args,
+                           uint64_t pci_hole_min_size);
+
 int xc_hvm_build_target_mem(xc_interface *xch,
                             uint32_t domid,
                             int memsize,
                             int target,
                             const char *image_name);
 
+int xc_hvm_build_target_mem_with_hole(xc_interface *xch,
+                                      uint32_t domid,
+                                      int memsize,
+                                      int target,
+                                      const char *image_name,
+                                      uint64_t pci_hole_min_size);
+
 int xc_suspend_evtchn_release(xc_interface *xch, xc_evtchn *xce, int domid, int suspend_evtchn);
 
 int xc_suspend_evtchn_init(xc_interface *xch, xc_evtchn *xce, int domid, int port);
diff --git a/tools/libxl/libxl_create.c b/tools/libxl/libxl_create.c
index 53e7cb6..4de8260 100644
--- a/tools/libxl/libxl_create.c
+++ b/tools/libxl/libxl_create.c
@@ -388,13 +388,15 @@ int libxl__domain_build(libxl__gc *gc,
         vments[4] = "start_time";
         vments[5] = libxl__sprintf(gc, "%lu.%02d", start_time.tv_sec,(int)start_time.tv_usec/10000);
 
-        localents = libxl__calloc(gc, 7, sizeof(char *));
+        localents = libxl__calloc(gc, 9, sizeof(char *));
         localents[0] = "platform/acpi";
         localents[1] = libxl_defbool_val(info->u.hvm.acpi) ? "1" : "0";
         localents[2] = "platform/acpi_s3";
         localents[3] = libxl_defbool_val(info->u.hvm.acpi_s3) ? "1" : "0";
         localents[4] = "platform/acpi_s4";
         localents[5] = libxl_defbool_val(info->u.hvm.acpi_s4) ? "1" : "0";
+        localents[6] = "platform/pci_hole_min_size";
+        localents[7] = libxl__sprintf(gc, "%llu", (unsigned long long)info->u.hvm.pci_hole_min_size);
 
         break;
     case LIBXL_DOMAIN_TYPE_PV:
diff --git a/tools/libxl/libxl_dm.c b/tools/libxl/libxl_dm.c
index 5c06dfa..72842aa 100644
--- a/tools/libxl/libxl_dm.c
+++ b/tools/libxl/libxl_dm.c
@@ -656,6 +656,21 @@ static char ** libxl__build_device_model_args_new(libxl__gc *gc,
         } else {
             flexarray_append(dm_args, "xenfv");
         }
+        if (b_info->u.hvm.pci_hole_min_size) {
+            unsigned long long max_ram_below_4g = (1ULL << 32) -
+                b_info->u.hvm.pci_hole_min_size;
+
+            if (max_ram_below_4g > 0xF0000000ULL)
+            {
+                LIBXL__LOG(ctx, LIBXL__LOG_WARNING,
+                           "pci_hole_min_size too big => max_ram_below_4g=%llu > %llu (new adjusted value)\n",
+                           max_ram_below_4g, 0xF0000000ULL);
+                max_ram_below_4g = 0xF0000000ULL;
+            }
+            flexarray_append_pair(dm_args, "-global",
+                           GCSPRINTF("pc-memory-layout.max-ram-below-4g=%llu",
+                                     max_ram_below_4g));
+        }
         for (i = 0; b_info->extra_hvm && b_info->extra_hvm[i] != NULL; i++)
             flexarray_append(dm_args, b_info->extra_hvm[i]);
         break;
diff --git a/tools/libxl/libxl_dom.c b/tools/libxl/libxl_dom.c
index 55f74b2..6ebc606 100644
--- a/tools/libxl/libxl_dom.c
+++ b/tools/libxl/libxl_dom.c
@@ -642,7 +642,8 @@ int libxl__build_hvm(libxl__gc *gc, uint32_t domid,
         goto out;
     }
 
-    ret = xc_hvm_build(ctx->xch, domid, &args);
+    ret = xc_hvm_build_with_hole(ctx->xch, domid, &args,
+                                 info->u.hvm.pci_hole_min_size);
     if (ret) {
         LOGEV(ERROR, ret, "hvm building failed");
         goto out;
diff --git a/tools/libxl/libxl_types.idl b/tools/libxl/libxl_types.idl
index 7d3a62b..bba7076 100644
--- a/tools/libxl/libxl_types.idl
+++ b/tools/libxl/libxl_types.idl
@@ -346,6 +346,7 @@ libxl_domain_build_info = Struct("domain_build_info",[
                                        ("timeoffset",       string),
                                        ("hpet",             libxl_defbool),
                                        ("vpt_align",        libxl_defbool),
+                                       ("pci_hole_min_size",UInt(64, init_val = 0)),
                                        ("timer_mode",       libxl_timer_mode),
                                        ("nested_hvm",       libxl_defbool),
                                        ("smbios_firmware",  string),
diff --git a/tools/libxl/xl_cmdimpl.c b/tools/libxl/xl_cmdimpl.c
index 5f59bbc..8391824 100644
--- a/tools/libxl/xl_cmdimpl.c
+++ b/tools/libxl/xl_cmdimpl.c
@@ -1026,6 +1026,12 @@ static void parse_config_data(const char *config_source,
         xlu_cfg_get_defbool(config, "hpet", &b_info->u.hvm.hpet, 0);
         xlu_cfg_get_defbool(config, "vpt_align", &b_info->u.hvm.vpt_align, 0);
 
+        if (!xlu_cfg_get_long(config, "pci_hole_min_size", &l, 0)) {
+            b_info->u.hvm.pci_hole_min_size = (uint64_t) l;
+            if (dom_info->debug)
+                fprintf(stderr, "pci_hole_min_size: %llu\n", (unsigned long long) b_info->u.hvm.pci_hole_min_size);
+        }
+
         if (!xlu_cfg_get_long(config, "timer_mode", &l, 1)) {
             const char *s = libxl_timer_mode_to_string(l);
             fprintf(stderr, "WARNING: specifying \"timer_mode\" as an integer is deprecated. "
-- 
1.8.4