Running NUMA kernel on Tiger

["Luck, Tony" <tony.luck@intel.com>]

[-- Attachment #1: Type: text/plain, Size: 1043 bytes --]

In case anyone else would like to experment with NUMA code,
but doesn't have access to a NUMA machine ...

Here is a patch against 2.6.0test4 + ia64-030826 Mosberger patch that
will allow you to configure a kernel with:

	CONFIG_NUMA=y
	CONFIG_ACPI_NUMA=y
	CONFIG_DISCONTIGMEM=y
	CONFIG_VIRTUAL_MEM_MAP=y

and boot on a Tiger.  You may need to change the "phony_srat"
initialization in drivers/acpi/tables.c to match the configuration
of your machine, the version in this patch defines memory below 4G
to be on node 0, and memory in the range 4G-8G to be on node 1.
Cpu 0 & 1 are on node 0, cpu 2 & 3  are on node 1.

This patch includes some extra fixes in addition to adding the SRAT table:

1) execute find_memory() after acpi_numa_init() [arch/ia64/kernel/setup.c]

2) Don't allocate memory using alloc_bootmem() in early acpi initializations
[drivers/acpi/tables.c]

3) Don't use any memory in a granule with a hole [arch/ia64/kernel/efi.c]

-Tony

[Thanks to Martin Hicks and Jesse Barnes for patches 1 & 2]

[-- Attachment #2: phonysrat.patch --]
[-- Type: application/octet-stream, Size: 3797 bytes --]

diff -ru linux-2.6.0-test4/arch/ia64/kernel/efi.c tiger-numa/arch/ia64/kernel/efi.c
--- linux-2.6.0-test4/arch/ia64/kernel/efi.c	Fri Aug 22 16:51:04 2003
+++ tiger-numa/arch/ia64/kernel/efi.c	Wed Aug 27 16:47:27 2003
@@ -324,7 +324,7 @@
 				check_md = q;
 
 				if (check_md->attribute & EFI_MEMORY_WB)
-					trim_bottom(md, granule_addr);
+					trim_bottom(check_md, granule_addr);
 
 				if (check_md->phys_addr < granule_addr)
 					continue;
diff -ru linux-2.6.0-test4/arch/ia64/kernel/setup.c tiger-numa/arch/ia64/kernel/setup.c
--- linux-2.6.0-test4/arch/ia64/kernel/setup.c	Fri Aug 22 16:52:22 2003
+++ tiger-numa/arch/ia64/kernel/setup.c	Tue Aug 26 13:37:41 2003
@@ -372,7 +372,6 @@
 	strlcpy(saved_command_line, *cmdline_p, sizeof(saved_command_line));
 
 	efi_init();
-	find_memory();
 
 #ifdef CONFIG_ACPI_BOOT
 	/* Initialize the ACPI boot-time table parser */
@@ -386,6 +385,8 @@
 # endif
 #endif /* CONFIG_APCI_BOOT */
 
+	find_memory();
+
 	/* process SAL system table: */
 	ia64_sal_init(efi.sal_systab);
 
diff -ru linux-2.6.0-test4/drivers/acpi/tables.c tiger-numa/drivers/acpi/tables.c
--- linux-2.6.0-test4/drivers/acpi/tables.c	Thu Aug 28 09:51:35 2003
+++ tiger-numa/drivers/acpi/tables.c	Thu Aug 28 09:52:22 2003
@@ -69,7 +69,11 @@
 
 static unsigned long		sdt_pa;		/* Physical Address */
 static unsigned long		sdt_count;	/* Table count */
+#if 0
 static struct acpi_table_sdt	*sdt_entry;
+#else
+static struct acpi_table_sdt	sdt_entry[ACPI_MAX_TABLES];
+#endif
 
 void
 acpi_table_print (
@@ -281,6 +285,41 @@
 	return 0;
 }
 	 
+/*
+ * SRAT for multi-node simulation on Tiger:
+ * two nodes, two cpus on each, 0G<mem<4G on node 0, 4G<mem<8G on node 1
+ */
+struct {
+	struct acpi_table_srat s;
+	struct acpi_table_processor_affinity p[4];
+	struct acpi_table_memory_affinity m[2];
+} phony_srat = {
+	{
+		{
+			{ 'S', 'R', 'A', 'T' },
+			0,
+			0,
+			0,
+			{ "OEMID" },
+			{ "OEMTBL" },
+			0,
+			{ "asl" },
+			0
+		},
+		0,
+		0L
+	},
+	{
+		{{ 0, 16 }, 0, 0xC0, { 1, 0 }, 0x18, "" },
+		{{ 0, 16 }, 0, 0xC2, { 1, 0 }, 0x18, "" },
+		{{ 0, 16 }, 1, 0xC4, { 1, 0 }, 0x18, "" },
+		{{ 0, 16 }, 1, 0xC6, { 1, 0 }, 0x18, "" },
+        },
+	{
+		{{ 1, 40 }, 0, "", 0, 0, 0, 1, 1, { 1, 0, 0 }, 0L },
+		{{ 1, 40 }, 1, "", 0, 1, 0, 1, 1, { 1, 0, 0 }, 0L },
+	}
+};
 
 int __init
 acpi_table_parse_madt_family (
@@ -298,6 +337,12 @@
 	if (!handler)
 		return -EINVAL;
 
+	if (id == ACPI_SRAT) {
+		madt = (void *)&phony_srat;
+		madt_end = (unsigned long) madt + sizeof(phony_srat);
+		goto do_parse;
+	}
+
 	/* Locate the MADT (if exists). There should only be one. */
 
 	for (i = 0; i < sdt_count; i++) {
@@ -321,6 +366,7 @@
 
 	madt_end = (unsigned long) madt + sdt_entry[i].size;
 
+do_parse:
 	/* Parse all entries looking for a match. */
 
 	entry = (acpi_table_entry_header *)
@@ -360,6 +406,11 @@
 	if (!handler)
 		return -EINVAL;
 
+	if (id == ACPI_SRAT) {
+		handler(ia64_tpa((unsigned long)&phony_srat), sizeof(phony_srat));
+		return 1;
+	}
+
 	for (i = 0; i < sdt_count; i++) {
 		if (sdt_entry[i].id != id)
 			continue;
@@ -425,11 +476,13 @@
 			sdt_count = ACPI_MAX_TABLES;
 		}
 
+#if 0
 		sdt_entry = alloc_bootmem(sdt_count * sizeof(struct acpi_table_sdt));
 		if (!sdt_entry) {
 			printk(KERN_ERR "ACPI: Could not allocate mem for SDT entries!\n");
 			return -ENOMEM;
 		}
+#endif
 
 		for (i = 0; i < sdt_count; i++)
 			sdt_entry[i].pa = (unsigned long) mapped_xsdt->entry[i];
@@ -477,11 +530,13 @@
 			sdt_count = ACPI_MAX_TABLES;
 		}
 
+#if 0
 		sdt_entry = alloc_bootmem(sdt_count * sizeof(struct acpi_table_sdt));
 		if (!sdt_entry) {
 			printk(KERN_ERR "ACPI: Could not allocate mem for SDT entries!\n");
 			return -ENOMEM;
 		}
+#endif
 
 		for (i = 0; i < sdt_count; i++)
 			sdt_entry[i].pa = (unsigned long) mapped_rsdt->entry[i];