[Qemu-devel] [PATCH 0/2] pci: 64bit bits

[Qemu-devel] [PATCH 0/2] pci: 64bit bits

[Gerd Hoffmann]

  Hi,

seabios (master branch) just got 64bit pci support.  When running out of
address space in the pci memory window below 4G (0xe0000000+) seabios
will map 64bit pci bars above 4G to make room below 4G.

This patch series carries two little patches for qemu to adapt it to the
seabios changes.  First patch creates a memory window for the 64bit pci
bars.  Second patch adds a 64bit option to the ivshmem driver, which
allows to use huge shared memory chunks.

please review & apply,
  Gerd

Gerd Hoffmann (2):
  pc: add pci64 memory hole
  ivshmem: add 64bit option

 hw/ivshmem.c |   13 ++++++++++---
 hw/pc.c      |   17 ++++++++++++++---
 hw/pc.h      |    6 ++++++
 hw/pc_piix.c |   22 ++++++++++------------
 4 files changed, 40 insertions(+), 18 deletions(-)

[Qemu-devel] [PATCH 1/2] pc: add pci64 memory hole

[Gerd Hoffmann]

This patch adds a address space hole for 64bit PCI ressources.
It starts at 0x8000000000 (512 GB) and ends at 0x10000000000 (1 TB),
thus has 512 GB in size.  This matches what the seabios is doing
(latest master branch).

Signed-off-by: Gerd Hoffmann <kraxel@redhat.com>
---
 hw/pc.c      |   17 ++++++++++++++---
 hw/pc.h      |    6 ++++++
 hw/pc_piix.c |   22 ++++++++++------------
 3 files changed, 30 insertions(+), 15 deletions(-)

diff --git a/hw/pc.c b/hw/pc.c
index 4d34a33..de1b297 100644
--- a/hw/pc.c
+++ b/hw/pc.c
@@ -981,12 +981,13 @@ void pc_memory_init(MemoryRegion *system_memory,
                     const char *initrd_filename,
                     ram_addr_t below_4g_mem_size,
                     ram_addr_t above_4g_mem_size,
+                    ram_addr_t above_1t_mem_size,
                     MemoryRegion *rom_memory,
                     MemoryRegion **ram_memory)
 {
     int linux_boot, i;
     MemoryRegion *ram, *option_rom_mr;
-    MemoryRegion *ram_below_4g, *ram_above_4g;
+    MemoryRegion *ram_below_4g, *ram_above_4g, *ram_above_1t;
     void *fw_cfg;
 
     linux_boot = (kernel_filename != NULL);
@@ -997,7 +998,9 @@ void pc_memory_init(MemoryRegion *system_memory,
      */
     ram = g_malloc(sizeof(*ram));
     memory_region_init_ram(ram, "pc.ram",
-                           below_4g_mem_size + above_4g_mem_size);
+                           below_4g_mem_size +
+                           above_4g_mem_size +
+                           above_1t_mem_size);
     vmstate_register_ram_global(ram);
     *ram_memory = ram;
     ram_below_4g = g_malloc(sizeof(*ram_below_4g));
@@ -1008,9 +1011,17 @@ void pc_memory_init(MemoryRegion *system_memory,
         ram_above_4g = g_malloc(sizeof(*ram_above_4g));
         memory_region_init_alias(ram_above_4g, "ram-above-4g", ram,
                                  below_4g_mem_size, above_4g_mem_size);
-        memory_region_add_subregion(system_memory, 0x100000000ULL,
+        memory_region_add_subregion(system_memory, PCI_HOLE_END,
                                     ram_above_4g);
     }
+    if (above_1t_mem_size > 0) {
+        ram_above_1t = g_malloc(sizeof(*ram_above_1t));
+        memory_region_init_alias(ram_above_1t, "ram-above-1t", ram,
+                                 below_4g_mem_size + above_4g_mem_size,
+                                 above_1t_mem_size);
+        memory_region_add_subregion(system_memory, PCI_HOLE64_END,
+                                    ram_above_1t);
+    }
 
 
     /* Initialize PC system firmware */
diff --git a/hw/pc.h b/hw/pc.h
index 74d3369..0c5e14e 100644
--- a/hw/pc.h
+++ b/hw/pc.h
@@ -12,6 +12,11 @@
 
 /* PC-style peripherals (also used by other machines).  */
 
+#define PCI_HOLE_START   0x0000e0000000ULL
+#define PCI_HOLE_END     0x000100000000ULL
+#define PCI_HOLE64_START 0x008000000000ULL
+#define PCI_HOLE64_END   0x010000000000ULL
+
 /* serial.c */
 
 SerialState *serial_init(int base, qemu_irq irq, int baudbase,
@@ -112,6 +117,7 @@ void pc_memory_init(MemoryRegion *system_memory,
                     const char *initrd_filename,
                     ram_addr_t below_4g_mem_size,
                     ram_addr_t above_4g_mem_size,
+                    ram_addr_t above_1t_mem_size,
                     MemoryRegion *rom_memory,
                     MemoryRegion **ram_memory);
 qemu_irq *pc_allocate_cpu_irq(void);
diff --git a/hw/pc_piix.c b/hw/pc_piix.c
index 6a75718..27f990f 100644
--- a/hw/pc_piix.c
+++ b/hw/pc_piix.c
@@ -133,7 +133,7 @@ static void pc_init1(MemoryRegion *system_memory,
                      int kvmclock_enabled)
 {
     int i;
-    ram_addr_t below_4g_mem_size, above_4g_mem_size;
+    ram_addr_t below_4g_mem_size, above_4g_mem_size, above_1t_mem_size;
     PCIBus *pci_bus;
     ISABus *isa_bus;
     PCII440FXState *i440fx_state;
@@ -157,13 +157,10 @@ static void pc_init1(MemoryRegion *system_memory,
         kvmclock_create();
     }
 
-    if (ram_size >= 0xe0000000 ) {
-        above_4g_mem_size = ram_size - 0xe0000000;
-        below_4g_mem_size = 0xe0000000;
-    } else {
-        above_4g_mem_size = 0;
-        below_4g_mem_size = ram_size;
-    }
+    below_4g_mem_size = MIN(ram_size, PCI_HOLE_START);
+    above_4g_mem_size = MIN(ram_size - below_4g_mem_size,
+                            PCI_HOLE64_START - PCI_HOLE_END);
+    above_1t_mem_size = ram_size - below_4g_mem_size - above_4g_mem_size;
 
     if (pci_enabled) {
         pci_memory = g_new(MemoryRegion, 1);
@@ -178,7 +175,7 @@ static void pc_init1(MemoryRegion *system_memory,
     if (!xen_enabled()) {
         pc_memory_init(system_memory,
                        kernel_filename, kernel_cmdline, initrd_filename,
-                       below_4g_mem_size, above_4g_mem_size,
+                       below_4g_mem_size, above_4g_mem_size, above_1t_mem_size,
                        pci_enabled ? rom_memory : system_memory, &ram_memory);
     }
 
@@ -195,11 +192,12 @@ static void pc_init1(MemoryRegion *system_memory,
         pci_bus = i440fx_init(&i440fx_state, &piix3_devfn, &isa_bus, gsi,
                               system_memory, system_io, ram_size,
                               below_4g_mem_size,
-                              0x100000000ULL - below_4g_mem_size,
-                              0x100000000ULL + above_4g_mem_size,
+                              PCI_HOLE_END - below_4g_mem_size,
+                              PCI_HOLE_END + above_4g_mem_size,
                               (sizeof(target_phys_addr_t) == 4
                                ? 0
-                               : ((uint64_t)1 << 62)),
+                               : (PCI_HOLE64_END - PCI_HOLE_END -
+                                  above_4g_mem_size)),
                               pci_memory, ram_memory);
     } else {
         pci_bus = NULL;
-- 
1.7.1

Re: [Qemu-devel] [PATCH 1/2] pc: add pci64 memory hole

[Avi Kivity]

On 05/02/2012 05:02 PM, Gerd Hoffmann wrote:
> This patch adds a address space hole for 64bit PCI ressources.
> It starts at 0x8000000000 (512 GB) and ends at 0x10000000000 (1 TB),
> thus has 512 GB in size.  This matches what the seabios is doing
> (latest master branch).

We should communicate this to seabios via fwcfg (really seabios should
communicate this to the chipset, but our chipset doesn't support this at
all).

It should also only apply to -M old.

-- 
error compiling committee.c: too many arguments to function

Re: [Qemu-devel] [PATCH 1/2] pc: add pci64 memory hole

[Gerd Hoffmann]

On 05/02/12 17:31, Avi Kivity wrote:
> On 05/02/2012 05:02 PM, Gerd Hoffmann wrote:
>> This patch adds a address space hole for 64bit PCI ressources.
>> It starts at 0x8000000000 (512 GB) and ends at 0x10000000000 (1 TB),
>> thus has 512 GB in size.  This matches what the seabios is doing
>> (latest master branch).
> 
> We should communicate this to seabios via fwcfg

A dsdt entry is needed for the pci64 window (see
http://code.coreboot.org/p/seabios/source/commit/482a020ec25f4cec655ddcb16b67c6f38b0844c0/),
which makes it non-trivial to turn this into a runtime option ...

> It should also only apply to -M old.

Is this the only reason you want this be runtime-switchable?

cheers,
  Gerd

Re: [Qemu-devel] [PATCH 1/2] pc: add pci64 memory hole

[Avi Kivity]

On 05/02/2012 06:46 PM, Gerd Hoffmann wrote:
> On 05/02/12 17:31, Avi Kivity wrote:
> > On 05/02/2012 05:02 PM, Gerd Hoffmann wrote:
> >> This patch adds a address space hole for 64bit PCI ressources.
> >> It starts at 0x8000000000 (512 GB) and ends at 0x10000000000 (1 TB),
> >> thus has 512 GB in size.  This matches what the seabios is doing
> >> (latest master branch).
> > 
> > We should communicate this to seabios via fwcfg
>
> A dsdt entry is needed for the pci64 window (see
> http://code.coreboot.org/p/seabios/source/commit/482a020ec25f4cec655ddcb16b67c6f38b0844c0/),
> which makes it non-trivial to turn this into a runtime option ...

It's a function, isn't it?  So all you need is a ASL driver for fwcfg
and an If().

> > It should also only apply to -M old.
>
> Is this the only reason you want this be runtime-switchable?

No.  The BIOS and qemu shouldn't be making too many assumptions. 
Certainly for Q35 we'd have the bios program qemu for the 64 bit hole.

-- 
error compiling committee.c: too many arguments to function

[Qemu-devel] [PATCH 2/2] ivshmem: add 64bit option

[Gerd Hoffmann]

This patch adds a "use64" property which will make the ivshmem driver
register a 64bit memory bar when set, so you have something to play with
when testing 64bit pci bits.  It also allows to have quite big shared
memory regions, like this:

[root@fedora ~]# lspci -vs1:1
01:01.0 RAM memory: Red Hat, Inc Device 1110
        Subsystem: Red Hat, Inc Device 1100
        Physical Slot: 1-1
        Flags: fast devsel
        Memory at fd400000 (32-bit, non-prefetchable) [disabled] [size=256]
        Memory at 8040000000 (64-bit, prefetchable) [size=1G]

Signed-off-by: Gerd Hoffmann <kraxel@redhat.com>
---
 hw/ivshmem.c |   13 ++++++++++---
 1 files changed, 10 insertions(+), 3 deletions(-)

diff --git a/hw/ivshmem.c b/hw/ivshmem.c
index d48e5f9..9c16c7b 100644
--- a/hw/ivshmem.c
+++ b/hw/ivshmem.c
@@ -72,6 +72,8 @@ typedef struct IVShmemState {
     MemoryRegion ivshmem;
     MemoryRegion msix_bar;
     uint64_t ivshmem_size; /* size of shared memory region */
+    uint32_t ivshmem_attr;
+    uint32_t ivshmem_64bit;
     int shm_fd; /* shared memory file descriptor */
 
     Peer *peers;
@@ -344,7 +346,7 @@ static void create_shared_memory_BAR(IVShmemState *s, int fd) {
     memory_region_add_subregion(&s->bar, 0, &s->ivshmem);
 
     /* region for shared memory */
-    pci_register_bar(&s->dev, 2, PCI_BASE_ADDRESS_SPACE_MEMORY, &s->bar);
+    pci_register_bar(&s->dev, 2, s->ivshmem_attr, &s->bar);
 }
 
 static void close_guest_eventfds(IVShmemState *s, int posn)
@@ -694,6 +696,11 @@ static int pci_ivshmem_init(PCIDevice *dev)
                      &s->ivshmem_mmio);
 
     memory_region_init(&s->bar, "ivshmem-bar2-container", s->ivshmem_size);
+    s->ivshmem_attr = PCI_BASE_ADDRESS_SPACE_MEMORY |
+        PCI_BASE_ADDRESS_MEM_PREFETCH;
+    if (s->ivshmem_64bit) {
+        s->ivshmem_attr |= PCI_BASE_ADDRESS_MEM_TYPE_64;
+    }
 
     if ((s->server_chr != NULL) &&
                         (strncmp(s->server_chr->filename, "unix:", 5) == 0)) {
@@ -719,8 +726,7 @@ static int pci_ivshmem_init(PCIDevice *dev)
         /* allocate/initialize space for interrupt handling */
         s->peers = g_malloc0(s->nb_peers * sizeof(Peer));
 
-        pci_register_bar(&s->dev, 2,
-                         PCI_BASE_ADDRESS_SPACE_MEMORY, &s->bar);
+        pci_register_bar(&s->dev, 2, s->ivshmem_attr, &s->bar);
 
         s->eventfd_chr = g_malloc0(s->vectors * sizeof(CharDriverState *));
 
@@ -792,6 +798,7 @@ static Property ivshmem_properties[] = {
     DEFINE_PROP_BIT("msi", IVShmemState, features, IVSHMEM_MSI, true),
     DEFINE_PROP_STRING("shm", IVShmemState, shmobj),
     DEFINE_PROP_STRING("role", IVShmemState, role),
+    DEFINE_PROP_UINT32("use64", IVShmemState, ivshmem_64bit, 0),
     DEFINE_PROP_END_OF_LIST(),
 };
 
-- 
1.7.1

Re: [Qemu-devel] [PATCH 2/2] ivshmem: add 64bit option

[Avi Kivity]

On 05/02/2012 05:02 PM, Gerd Hoffmann wrote:
> This patch adds a "use64" property which will make the ivshmem driver
> register a 64bit memory bar when set, so you have something to play with
> when testing 64bit pci bits.  It also allows to have quite big shared
> memory regions, like this:
>
> [root@fedora ~]# lspci -vs1:1
> 01:01.0 RAM memory: Red Hat, Inc Device 1110
>         Subsystem: Red Hat, Inc Device 1100
>         Physical Slot: 1-1
>         Flags: fast devsel
>         Memory at fd400000 (32-bit, non-prefetchable) [disabled] [size=256]
>         Memory at 8040000000 (64-bit, prefetchable) [size=1G]
>
>

Suggest making this the default for -M current.

-- 
error compiling committee.c: too many arguments to function

Re: [Qemu-devel] [PATCH 0/2] pci: 64bit bits

[Avi Kivity]

On 05/02/2012 05:02 PM, Gerd Hoffmann wrote:
>   Hi,
>
> seabios (master branch) just got 64bit pci support.  When running out of
> address space in the pci memory window below 4G (0xe0000000+) seabios
> will map 64bit pci bars above 4G to make room below 4G.
>
> This patch series carries two little patches for qemu to adapt it to the
> seabios changes.  First patch creates a memory window for the 64bit pci
> bars.  Second patch adds a 64bit option to the ivshmem driver, which
> allows to use huge shared memory chunks.
>
>

What happens if a pre-64-bit-pci guest boots on such a setup?

-- 
error compiling committee.c: too many arguments to function

Re: [Qemu-devel] [PATCH 0/2] pci: 64bit bits

[Gerd Hoffmann]

On 05/02/12 17:33, Avi Kivity wrote:
> On 05/02/2012 05:02 PM, Gerd Hoffmann wrote:
>>   Hi,
>>
>> seabios (master branch) just got 64bit pci support.  When running out of
>> address space in the pci memory window below 4G (0xe0000000+) seabios
>> will map 64bit pci bars above 4G to make room below 4G.
>>
>> This patch series carries two little patches for qemu to adapt it to the
>> seabios changes.  First patch creates a memory window for the 64bit pci
>> bars.  Second patch adds a 64bit option to the ivshmem driver, which
>> allows to use huge shared memory chunks.
> 
> What happens if a pre-64-bit-pci guest boots on such a setup?

If it worked before it should continue to work just fine.  Note that the
64bit bars are only mapped above 4G in case there is not enough address
space below 4G, so existing setups should never ever see bars mapped high.

cheers,
  Gerd

Re: [Qemu-devel] [PATCH 0/2] pci: 64bit bits

[Avi Kivity]

On 05/02/2012 06:54 PM, Gerd Hoffmann wrote:
> On 05/02/12 17:33, Avi Kivity wrote:
> > On 05/02/2012 05:02 PM, Gerd Hoffmann wrote:
> >>   Hi,
> >>
> >> seabios (master branch) just got 64bit pci support.  When running out of
> >> address space in the pci memory window below 4G (0xe0000000+) seabios
> >> will map 64bit pci bars above 4G to make room below 4G.
> >>
> >> This patch series carries two little patches for qemu to adapt it to the
> >> seabios changes.  First patch creates a memory window for the 64bit pci
> >> bars.  Second patch adds a 64bit option to the ivshmem driver, which
> >> allows to use huge shared memory chunks.
> > 
> > What happens if a pre-64-bit-pci guest boots on such a setup?
>
> If it worked before it should continue to work just fine.  Note that the
> 64bit bars are only mapped above 4G in case there is not enough address
> space below 4G, so existing setups should never ever see bars mapped high.

I see.  I guess it depends on the algorithm for selecting the BAR to be
evicted, but if everything fitted before, it should fit now as well.  If
it didn't, then a different device may break in each of the two cases.

-- 
error compiling committee.c: too many arguments to function