Re: Graphics pass-through

[Gerd Hoffmann <kraxel@redhat.com>]

[-- Attachment #1: Type: text/plain, Size: 546 bytes --]

   Hi,

> Another problem with SeaBIOS which limits the amount of memory space
> is: SeaBIOS allocates the BAR regions as they are encountered. As far
> as I know, the BAR regions should be naturally aligned. Thus the
> simple strategy of the SeaBIOS results in large fragmentation.
> Therefore, even after increasing the PCI memory space to 512MB the BAR
> regions were unallocated.

Ran into this too.  Started fixing that with a second pci pass.  Not 
finished yet.  Patch attached FYI.  Feel free to grab it and run with it.

cheers,
   Gerd

[-- Attachment #2: 0001-wip-pci-move-to-two-pass-pci-initialization.patch --]
[-- Type: text/plain, Size: 9219 bytes --]

>From bf779e443e92872c5e076babb9c1b1a2890402bd Mon Sep 17 00:00:00 2001
From: Gerd Hoffmann <kraxel@redhat.com>
Date: Tue, 3 May 2011 12:38:15 +0200
Subject: [PATCH] [wip] pci: move to two-pass pci initialization

This patch adds a second device scan to the pci initialization, which
counts the memory bars of the various sizes and types.  Then it
calculates the sizes and the packing of the prefetchable and
non-prefetchable pci memory windows and prints the results.

TODO #1: handle pci bridges properly.
TODO #2: actually use the calculated stuff.
---
 src/pciinit.c |  237 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++-
 1 files changed, 234 insertions(+), 3 deletions(-)

diff --git a/src/pciinit.c b/src/pciinit.c
index ee2e72d..993d3cb 100644
--- a/src/pciinit.c
+++ b/src/pciinit.c
@@ -15,12 +15,64 @@
 #define PCI_ROM_SLOT 6
 #define PCI_NUM_REGIONS 7
 
+#define PCI_IO_INDEX_SHIFT 2
+#define PCI_MEM_INDEX_SHIFT 12
+
 static void pci_bios_init_device_in_bus(int bus);
+static void pci_bios_check_device_in_bus(int bus);
 
 static struct pci_region pci_bios_io_region;
 static struct pci_region pci_bios_mem_region;
 static struct pci_region pci_bios_prefmem_region;
 
+static struct pci_bus {
+    /* pci region stats */
+    u32 io_count[16 - PCI_IO_INDEX_SHIFT];
+    u32 mem_count[32 - PCI_MEM_INDEX_SHIFT];
+    u32 prefmem_count[32 - PCI_MEM_INDEX_SHIFT];
+    u32 io_sum, io_max;
+    u32 mem_sum, mem_max;
+    u32 prefmem_sum, prefmem_max;
+    /* pci region assignments */
+    u32 io_bases[16 - PCI_IO_INDEX_SHIFT];
+    u32 mem_bases[32 - PCI_MEM_INDEX_SHIFT];
+    u32 prefmem_bases[32 - PCI_MEM_INDEX_SHIFT];
+    u32 io_base, mem_base, prefmem_base;
+} busses[2];
+
+static int pci_size_to_index(u32 size, int shift)
+{
+    int index = 0;
+
+    while (size > (1 << index)) {
+        index++;
+    }
+    if (index < shift)
+        index = shift;
+    index -= shift;
+    return index;
+}
+
+static int pci_io_size_to_index(u32 size)
+{
+    return pci_size_to_index(size, PCI_IO_INDEX_SHIFT);
+}
+
+static u32 pci_io_index_to_size(int index)
+{
+    return 1 << (index + PCI_IO_INDEX_SHIFT);
+}
+
+static int pci_mem_size_to_index(u32 size)
+{
+    return pci_size_to_index(size, PCI_MEM_INDEX_SHIFT);
+}
+
+static u32 pci_mem_index_to_size(int index)
+{
+    return 1 << (index + PCI_MEM_INDEX_SHIFT);
+}
+
 /* host irqs corresponding to PCI irqs A-D */
 const u8 pci_irqs[4] = {
     10, 10, 11, 11
@@ -393,6 +445,180 @@ pci_bios_init_bus(void)
     pci_bios_init_bus_rec(0 /* host bus */, &pci_bus);
 }
 
+static void pci_bios_check_device(struct pci_bus *bus, u16 bdf)
+{
+    int io_index, mem_index, prefmem_index;
+    u16 class;
+    int i;
+
+    class = pci_config_readw(bdf, PCI_CLASS_DEVICE);
+    if (class == PCI_CLASS_BRIDGE_PCI) {
+        u8 secbus = pci_config_readb(bdf, PCI_SECONDARY_BUS);
+        if (secbus >= ARRAY_SIZE(busses)) {
+            dprintf(1, "PCI: busses array too small, skipping bus %d\n", secbus);
+            return;
+        }
+        pci_bios_check_device_in_bus(secbus);
+        io_index = pci_io_size_to_index(busses[secbus].io_sum);
+        mem_index = pci_mem_size_to_index(busses[secbus].mem_sum);
+        prefmem_index = pci_mem_size_to_index(busses[secbus].prefmem_sum);
+        dprintf(1, "PCI: secondary bus %d sizes: io %x, mem %x, prefmem %x\n",
+                secbus, pci_io_index_to_size(io_index),
+                pci_mem_index_to_size(mem_index),
+                pci_mem_index_to_size(prefmem_index));
+        return;
+    }
+
+    for (i = 0; i < PCI_NUM_REGIONS; i++) {
+        u32 ofs = pci_bar(bdf, i);
+        u32 old = pci_config_readl(bdf, ofs);
+        u32 mask, index;
+        if (i == PCI_ROM_SLOT) {
+            mask = PCI_ROM_ADDRESS_MASK;
+            pci_config_writel(bdf, ofs, mask);
+        } else {
+            if (old & PCI_BASE_ADDRESS_SPACE_IO)
+                mask = PCI_BASE_ADDRESS_IO_MASK;
+            else
+                mask = PCI_BASE_ADDRESS_MEM_MASK;
+            pci_config_writel(bdf, ofs, ~0);
+        }
+        u32 val = pci_config_readl(bdf, ofs);
+        pci_config_writel(bdf, ofs, old);
+        u32 size = (~(val & mask)) + 1;
+        if (val == 0) {
+            continue;
+        }
+
+        if (val & PCI_BASE_ADDRESS_SPACE_IO) {
+            index = pci_io_size_to_index(size);
+            size = pci_io_index_to_size(index);
+            bus->io_count[index]++;
+            bus->io_sum += size;
+            if (bus->io_max < size)
+                bus->io_max = size;
+        } else {
+            index = pci_mem_size_to_index(size);
+            size = pci_mem_index_to_size(index);
+            if (val & PCI_BASE_ADDRESS_MEM_PREFETCH) {
+                bus->prefmem_count[index]++;
+                bus->prefmem_sum += size;
+                if (bus->prefmem_max < size)
+                    bus->prefmem_max = size;
+            } else {
+                bus->mem_count[index]++;
+                bus->mem_sum += size;
+                if (bus->mem_max < size)
+                    bus->mem_max = size;
+            }
+        }
+
+        if (!(val & PCI_BASE_ADDRESS_SPACE_IO) &&
+            (val & PCI_BASE_ADDRESS_MEM_TYPE_MASK) == PCI_BASE_ADDRESS_MEM_TYPE_64) {
+            i++;
+        }
+    }
+}
+
+static void pci_bios_check_device_in_bus(int bus)
+{
+    int bdf, max;
+
+    dprintf(1, "PCI: check devices bus %d\n", bus);
+    foreachpci_in_bus(bdf, max, bus) {
+        pci_bios_check_device(&busses[bus], bdf);
+    }
+}
+
+static void pci_bios_init_bus_bases(struct pci_bus *bus)
+{
+    u32 base, newbase, size;
+    int i;
+
+    /* assign prefetchable memory regions */
+    dprintf(1, "  prefmem max %x sum %x base %x\n",
+            bus->prefmem_max, bus->prefmem_sum, bus->prefmem_base);
+    base = bus->prefmem_base;
+    for (i = ARRAY_SIZE(bus->prefmem_count)-1; i >= 0; i--) {
+        size = pci_mem_index_to_size(i);
+        if (!bus->prefmem_count[i])
+            continue;
+        newbase = base + size * bus->prefmem_count[i];
+        dprintf(1, "    size %8x: %d bar(s), %8x -> %8x\n",
+                size, bus->prefmem_count[i], base, newbase - 1);
+        bus->prefmem_bases[i] = base;
+        base = newbase;
+    }
+
+    /* assign memory regions */
+    dprintf(1, "  mem max %x sum %x base %x\n",
+            bus->mem_max, bus->mem_sum, bus->mem_base);
+    base = bus->mem_base;
+    for (i = ARRAY_SIZE(bus->mem_count)-1; i >= 0; i--) {
+        size = pci_mem_index_to_size(i);
+        if (!bus->mem_count[i])
+            continue;
+        newbase = base + size * bus->mem_count[i];
+        dprintf(1, "    mem size %8x: %d bar(s), %8x -> %8x\n",
+                size, bus->mem_count[i], base, newbase - 1);
+        bus->mem_bases[i] = base;
+        base = newbase;
+    }
+
+    /* assign io regions */
+    dprintf(1, "  io max %x sum %x base %x\n",
+            bus->io_max, bus->io_sum, bus->io_base);
+    base = bus->io_base;
+    for (i = ARRAY_SIZE(bus->io_count)-1; i >= 0; i--) {
+        size = pci_io_index_to_size(i);
+        if (!bus->io_count[i])
+            continue;
+        newbase = base + size * bus->io_count[i];
+        dprintf(1, "    io size %4x: %d bar(s), %4x -> %4x\n",
+                size, bus->io_count[i], base, newbase - 1);
+        bus->io_bases[i] = base;
+        base = newbase;
+    }
+}
+
+static void pci_bios_init_root_regions(void)
+{
+    struct pci_bus *bus = &busses[0];
+
+    /* calculate memory windows */
+    if (bus->prefmem_sum) {
+        u32 reserved = 0xffffffff - BUILD_PCIMEM_END + 1;
+        u32 window = bus->prefmem_max;
+        while (bus->prefmem_sum + reserved > window) {
+            window += bus->prefmem_max;
+        }
+        bus->prefmem_base = 0xffffffff - window + 1;
+    } else {
+        bus->prefmem_base = BUILD_PCIMEM_END;
+    }
+
+    if (bus->mem_sum) {
+        u32 reserved = 0xffffffff - bus->prefmem_base + 1;
+        u32 window = bus->mem_max;
+        while (bus->mem_sum + reserved > window) {
+            window += bus->mem_max;
+        }
+        bus->mem_base = 0xffffffff - window + 1;
+    }
+
+    bus->io_base = 0xc000;
+
+    /* simple sanity check */
+    /* TODO: check e820 table */
+    if (bus->mem_base < RamSize) {
+        dprintf(1, "PCI: out of space for memory bars\n");
+        /* Hmm, what to do now? */
+    }
+
+    dprintf(1, "PCI: init bases bus 0 (primary)\n");
+    pci_bios_init_bus_bases(bus);
+}
+
 void
 pci_setup(void)
 {
@@ -402,15 +628,20 @@ pci_setup(void)
 
     dprintf(3, "pci setup\n");
 
+    pci_bios_init_bus();
+
+    int bdf, max;
+    dprintf(1, "PCI pass 1\n");
+    pci_bios_check_device_in_bus(0 /* host bus */);
+    pci_bios_init_root_regions();
+
     pci_region_init(&pci_bios_io_region, 0xc000, 64 * 1024 - 1);
     pci_region_init(&pci_bios_mem_region,
                     BUILD_PCIMEM_START, BUILD_PCIMEM_END - 1);
     pci_region_init(&pci_bios_prefmem_region,
                     BUILD_PCIPREFMEM_START, BUILD_PCIPREFMEM_END - 1);
 
-    pci_bios_init_bus();
-
-    int bdf, max;
+    dprintf(1, "PCI pass 2\n");
     foreachpci(bdf, max) {
         pci_init_device(pci_isa_bridge_tbl, bdf, NULL);
     }
-- 
1.7.1