[PATCH 0/4] iommu/amd: IVMD handling cleanup and ivmd= boot override

[PATCH 0/4] iommu/amd: IVMD handling cleanup and ivmd= boot override

[Suravee Suthikulpanit]

This series refreshes how the AMD IOMMU driver models IVRS IVMD entries
after the hardware exclusion range registers were repurposed.

The first patch removes unused per-IOMMU exclusion_start/exclusion_length
state and the MMIO exclusion programming path that never ran, and renames
the associated MMIO offsets to reflect completion-wait store base/limit.

The second patch drops the workaround that forced IVMD exclusion ranges to
r/w unity mappings; the driver no longer programs exclusion range
registers, so that the workaround is obsolete.

The third patch centralizes IVMD flag definitions, renames the unity-map
structures to ivmd_entry, stores the raw IVMD flags byte from ACPI, and
derives IOMMU reserved regions (including exclusion as IOMMU_RESV_RESERVED)
from those flags.

The fourth patch adds an ivmd= kernel parameter so up to four per-device
IVMD-style ranges can be supplied from the command line when the IVRS
table is wrong or incomplete, documented in kernel-parameters.txt.

Thanks,
Suravee

Suravee Suthikulpanit (4):
  iommu/amd: Drop unused global exclusion range fields and init
  iommu/amd: Do not convert IVMD exclusion range to unity range
  iommu/amd: Clean up and simplify IVMD entry handling
  iommu/amd: Introduce boot option ivmd=seg:bus:dev.fun,start,size,flags

 .../admin-guide/kernel-parameters.txt         |  16 ++
 drivers/iommu/amd/amd_iommu_types.h           |  47 +++--
 drivers/iommu/amd/init.c                      | 182 ++++++++++++------
 drivers/iommu/amd/iommu.c                     |  28 ++-
 4 files changed, 186 insertions(+), 87 deletions(-)

-- 
2.34.1

[PATCH 1/4] iommu/amd: Drop unused global exclusion range fields and init

[Suravee Suthikulpanit]

Remove unused struct amd_iommu.exclusion_start/exclusion_length, the
MMIO_EXCL_* flag masks, and iommu_set_exclusion_range(). These struct
amd_iommu fields were never assigned, so early_enable_iommu() never
programmed the hardware exclusion registers through this path.

Please note that the exclusion range registers have been repurposed
to SNP completion-wait store base / limit registers. So, rename
MMIO_EXCL_BASE/LIMIT via to MMIO_COMPL_STORE_BASE/LIMIT instead.

Signed-off-by: Suravee Suthikulpanit <suravee.suthikulpanit@amd.com>
---
 drivers/iommu/amd/amd_iommu_types.h | 13 ++-----------
 drivers/iommu/amd/init.c            | 29 +++--------------------------
 2 files changed, 5 insertions(+), 37 deletions(-)

diff --git a/drivers/iommu/amd/amd_iommu_types.h b/drivers/iommu/amd/amd_iommu_types.h
index c685d3771436..c4ea701b7cab 100644
--- a/drivers/iommu/amd/amd_iommu_types.h
+++ b/drivers/iommu/amd/amd_iommu_types.h
@@ -50,17 +50,13 @@
 #define MMIO_GET_BUS(x) (((x) & MMIO_RANGE_BUS_MASK) >> MMIO_RANGE_BUS_SHIFT)
 #define MMIO_MSI_NUM(x)	((x) & 0x1f)
 
-/* Flag masks for the AMD IOMMU exclusion range */
-#define MMIO_EXCL_ENABLE_MASK 0x01ULL
-#define MMIO_EXCL_ALLOW_MASK  0x02ULL
-
 /* Used offsets into the MMIO space */
 #define MMIO_DEV_TABLE_OFFSET   0x0000
 #define MMIO_CMD_BUF_OFFSET     0x0008
 #define MMIO_EVT_BUF_OFFSET     0x0010
 #define MMIO_CONTROL_OFFSET     0x0018
-#define MMIO_EXCL_BASE_OFFSET   0x0020
-#define MMIO_EXCL_LIMIT_OFFSET  0x0028
+#define MMIO_COMPL_STORE_BASE_OFFSET   0x0020
+#define MMIO_COMPL_STORE_LIMIT_OFFSET  0x0028
 #define MMIO_EXT_FEATURES	0x0030
 #define MMIO_PPR_LOG_OFFSET	0x0038
 #define MMIO_GA_LOG_BASE_OFFSET	0x00e0
@@ -680,11 +676,6 @@ struct amd_iommu {
 	/* pci domain of this IOMMU */
 	struct amd_iommu_pci_seg *pci_seg;
 
-	/* start of exclusion range of that IOMMU */
-	u64 exclusion_start;
-	/* length of exclusion range of that IOMMU */
-	u64 exclusion_length;
-
 	/* command buffer virtual address */
 	u8 *cmd_buf;
 	u32 cmd_buf_head;
diff --git a/drivers/iommu/amd/init.c b/drivers/iommu/amd/init.c
index f3fd7f39efb4..866249d3673e 100644
--- a/drivers/iommu/amd/init.c
+++ b/drivers/iommu/amd/init.c
@@ -352,28 +352,6 @@ static void iommu_write_l2(struct amd_iommu *iommu, u8 address, u32 val)
  *
  ****************************************************************************/
 
-/*
- * This function set the exclusion range in the IOMMU. DMA accesses to the
- * exclusion range are passed through untranslated
- */
-static void iommu_set_exclusion_range(struct amd_iommu *iommu)
-{
-	u64 start = iommu->exclusion_start & PAGE_MASK;
-	u64 limit = (start + iommu->exclusion_length - 1) & PAGE_MASK;
-	u64 entry;
-
-	if (!iommu->exclusion_start)
-		return;
-
-	entry = start | MMIO_EXCL_ENABLE_MASK;
-	memcpy_toio(iommu->mmio_base + MMIO_EXCL_BASE_OFFSET,
-			&entry, sizeof(entry));
-
-	entry = limit;
-	memcpy_toio(iommu->mmio_base + MMIO_EXCL_LIMIT_OFFSET,
-			&entry, sizeof(entry));
-}
-
 static void iommu_set_cwwb_range(struct amd_iommu *iommu)
 {
 	u64 start = iommu_virt_to_phys((void *)iommu->cmd_sem);
@@ -386,14 +364,14 @@ static void iommu_set_cwwb_range(struct amd_iommu *iommu)
 	 * Re-purpose Exclusion base/limit registers for Completion wait
 	 * write-back base/limit.
 	 */
-	memcpy_toio(iommu->mmio_base + MMIO_EXCL_BASE_OFFSET,
+	memcpy_toio(iommu->mmio_base + MMIO_COMPL_STORE_BASE_OFFSET,
 		    &entry, sizeof(entry));
 
 	/* Note:
 	 * Default to 4 Kbytes, which can be specified by setting base
 	 * address equal to the limit address.
 	 */
-	memcpy_toio(iommu->mmio_base + MMIO_EXCL_LIMIT_OFFSET,
+	memcpy_toio(iommu->mmio_base + MMIO_COMPL_STORE_LIMIT_OFFSET,
 		    &entry, sizeof(entry));
 }
 
@@ -1013,7 +991,7 @@ static int __init remap_or_alloc_cwwb_sem(struct amd_iommu *iommu)
 		 * completion wait buffer (CWB) address. Read and re-use it.
 		 */
 		pr_info_once("Re-using CWB buffers from the previous kernel\n");
-		paddr = readq(iommu->mmio_base + MMIO_EXCL_BASE_OFFSET) & PM_ADDR_MASK;
+		paddr = readq(iommu->mmio_base + MMIO_COMPL_STORE_BASE_OFFSET) & PM_ADDR_MASK;
 		iommu->cmd_sem = iommu_memremap(paddr, PAGE_SIZE);
 		if (!iommu->cmd_sem)
 			return -ENOMEM;
@@ -2891,7 +2869,6 @@ static void early_enable_iommu(struct amd_iommu *iommu)
 	iommu_set_device_table(iommu);
 	iommu_enable_command_buffer(iommu);
 	iommu_enable_event_buffer(iommu);
-	iommu_set_exclusion_range(iommu);
 	iommu_enable_gt(iommu);
 	iommu_enable_ga(iommu);
 	iommu_enable_xt(iommu);
-- 
2.34.1

[PATCH 2/4] iommu/amd: Do not convert IVMD exclusion range to unity range

[Suravee Suthikulpanit]

Original problem was there could be multiple IVMD exclusion entries
conflicting over a single exclusion range supported in older Linux
kernel. The fix was first introduced in commit 387caf0b759a ("iommu/amd:
Treat per-device exclusion ranges as r/w unity-mapped regions") and in
commit 0bbe4ced53e3 ("iommu/amd: Fix the overwritten field in IVMD header")
, where IVMD exclusions are converted to separate unity_map_entry objects
in pci_seg->unity_map so amd_iommu_get_resv_regions() treated every such
entry as IOMMU_RESV_DIRECT with R/W.

Since the AMD IOMMU driver no longer support the Exclusion Range base/limit
registers and rely on Linux kernel to exclude the specified IOVA ranges
from being mapped this logic is no longer needed.

Remove the special case that forced e->prot to RW-UNITY mapping
(i.e. IVMD_FLAG_IW | IVMD_FLAG_IR | IVMD_FLAG_UNITY) whenever
IVMD_FLAG_EXCL_RANGE was set.

Fixes: 0bbe4ced53e3 ("iommu/amd: Fix the overwritten field in IVMD header")
Cc: Baoquan He <bhe@redhat.com>
Cc: Adrian Huang <ahuang12@lenovo.com>
Cc: Jerry Snitselaar <jsnitsel@redhat.com>
Signed-off-by: Suravee Suthikulpanit <suravee.suthikulpanit@amd.com>
---
 drivers/iommu/amd/init.c | 10 ----------
 1 file changed, 10 deletions(-)

diff --git a/drivers/iommu/amd/init.c b/drivers/iommu/amd/init.c
index 866249d3673e..2e854f4639a7 100644
--- a/drivers/iommu/amd/init.c
+++ b/drivers/iommu/amd/init.c
@@ -2644,16 +2644,6 @@ static int __init init_unity_map_range(struct ivmd_header *m,
 	e->address_end = e->address_start + PAGE_ALIGN(m->range_length);
 	e->prot = m->flags >> 1;
 
-	/*
-	 * Treat per-device exclusion ranges as r/w unity-mapped regions
-	 * since some buggy BIOSes might lead to the overwritten exclusion
-	 * range (exclusion_start and exclusion_length members). This
-	 * happens when there are multiple exclusion ranges (IVMD entries)
-	 * defined in ACPI table.
-	 */
-	if (m->flags & IVMD_FLAG_EXCL_RANGE)
-		e->prot = (IVMD_FLAG_IW | IVMD_FLAG_IR) >> 1;
-
 	DUMP_printk("%s devid_start: %04x:%02x:%02x.%x devid_end: "
 		    "%04x:%02x:%02x.%x range_start: %016llx range_end: %016llx"
 		    " flags: %x\n", s, m->pci_seg,
-- 
2.34.1

[PATCH 3/4] iommu/amd: Clean up and simplify IVMD entry handling

[Suravee Suthikulpanit]

Clean up IVMD ACPI flags and mask in amd_iommu_types.h, and drop the
IOMMU_PROT_* / IOMMU_UNITY_MAP_FLAG_EXCL_RANGE, which only existed
to carry IVMD semantics.

Rename unity_map_entry and unity_map to ivmd_entry and ivmd_entry_map,
store the raw IVMD flags byte from the ACPI table instead of shifting
protection into a separate int, and derive IOMMU reserved regions from
those flags in amd_iommu_get_resv_regions(). Also, rename struct
list_head unity_map to ivmd_entry_map along with renaming the unity-map
helper functions.

Lastyly, add error message when handling invalid IVMD entry.

Signed-off-by: Suravee Suthikulpanit <suravee.suthikulpanit@amd.com>
---
 drivers/iommu/amd/amd_iommu_types.h | 20 +++++++--------
 drivers/iommu/amd/init.c            | 39 +++++++++++++----------------
 drivers/iommu/amd/iommu.c           | 28 +++++++++++++++------
 3 files changed, 48 insertions(+), 39 deletions(-)

diff --git a/drivers/iommu/amd/amd_iommu_types.h b/drivers/iommu/amd/amd_iommu_types.h
index c4ea701b7cab..e93b4b857dde 100644
--- a/drivers/iommu/amd/amd_iommu_types.h
+++ b/drivers/iommu/amd/amd_iommu_types.h
@@ -376,11 +376,11 @@
 /* DTE[128:179] | DTE[184:191] */
 #define DTE_DATA2_INTR_MASK	~GENMASK_ULL(55, 52)
 
-#define IOMMU_PROT_MASK 0x03
-#define IOMMU_PROT_IR 0x01
-#define IOMMU_PROT_IW 0x02
-
-#define IOMMU_UNITY_MAP_FLAG_EXCL_RANGE	(1 << 2)
+#define IVMD_FLAG_UNITY		BIT(0)
+#define IVMD_FLAG_IR		BIT(1)
+#define IVMD_FLAG_IW		BIT(2)
+#define IVMD_FLAG_EXCL		BIT(3)
+#define IVMD_FLAG_MASK		GENMASK_U32(3, 0)
 
 /* IOMMU capabilities */
 #define IOMMU_CAP_IOTLB   24
@@ -616,11 +616,11 @@ struct amd_iommu_pci_seg {
 	u16 *alias_table;
 
 	/*
-	 * A list of required unity mappings we find in ACPI. It is not locked
+	 * A list of required IVMD entry we find in ACPI. It is not locked
 	 * because as runtime it is only read. It is created at ACPI table
 	 * parsing time.
 	 */
-	struct list_head unity_map;
+	struct list_head ivmd_entry_map;
 };
 
 /*
@@ -876,7 +876,7 @@ struct ivhd_dte_flags {
 /*
  * One entry for unity mappings parsed out of the ACPI table.
  */
-struct unity_map_entry {
+struct ivmd_entry {
 	struct list_head list;
 
 	/* starting device id this entry is used for (including) */
@@ -889,8 +889,8 @@ struct unity_map_entry {
 	/* end address to unity map (including) */
 	u64 address_end;
 
-	/* required protection */
-	int prot;
+	/* IVMD flags for the entry */
+	u8 flags;
 };
 
 /*
diff --git a/drivers/iommu/amd/init.c b/drivers/iommu/amd/init.c
index 2e854f4639a7..4b62bb89a12c 100644
--- a/drivers/iommu/amd/init.c
+++ b/drivers/iommu/amd/init.c
@@ -70,11 +70,6 @@
 #define IVHD_FLAG_RESPASSPW_EN_MASK     0x04
 #define IVHD_FLAG_ISOC_EN_MASK          0x08
 
-#define IVMD_FLAG_EXCL_RANGE            0x08
-#define IVMD_FLAG_IW                    0x04
-#define IVMD_FLAG_IR                    0x02
-#define IVMD_FLAG_UNITY_MAP             0x01
-
 #define ACPI_DEVFLAG_INITPASS           0x01
 #define ACPI_DEVFLAG_EXTINT             0x02
 #define ACPI_DEVFLAG_NMI                0x04
@@ -134,7 +129,7 @@ struct ivhd_entry {
 
 /*
  * An AMD IOMMU memory definition structure. It defines things like exclusion
- * ranges for devices and regions that should be unity mapped.
+ * ranges, unity mapping for devices and regions.
  */
 struct ivmd_header {
 	u8 type;
@@ -1708,7 +1703,7 @@ static struct amd_iommu_pci_seg *__init alloc_pci_segment(u16 id,
 
 	pci_seg->id = id;
 	init_llist_head(&pci_seg->dev_data_list);
-	INIT_LIST_HEAD(&pci_seg->unity_map);
+	INIT_LIST_HEAD(&pci_seg->ivmd_entry_map);
 	list_add_tail(&pci_seg->list, &amd_iommu_pci_seg_list);
 
 	if (alloc_dev_table(pci_seg))
@@ -2301,8 +2296,8 @@ static int __init amd_iommu_init_pci(void)
 	}
 
 	/*
-	 * Order is important here to make sure any unity map requirements are
-	 * fulfilled. The unity mappings are created and written to the device
+	 * Order is important here to make sure any ivmd map requirements are
+	 * fulfilled. The ivmd mappings are created and written to the device
 	 * table during the iommu_init_pci() call.
 	 *
 	 * After that we call init_device_table_dma() to make sure any
@@ -2588,28 +2583,28 @@ static int iommu_init_irq(struct amd_iommu *iommu)
  *
  * The next functions belong to the third pass of parsing the ACPI
  * table. In this last pass the memory mapping requirements are
- * gathered (like exclusion and unity mapping ranges).
+ * gathered (like exclusion and ivmd mapping ranges).
  *
  ****************************************************************************/
 
-static void __init free_unity_maps(void)
+static void __init free_ivmd_maps(void)
 {
-	struct unity_map_entry *entry, *next;
+	struct ivmd_entry *entry, *next;
 	struct amd_iommu_pci_seg *p, *pci_seg;
 
 	for_each_pci_segment_safe(pci_seg, p) {
-		list_for_each_entry_safe(entry, next, &pci_seg->unity_map, list) {
+		list_for_each_entry_safe(entry, next, &pci_seg->ivmd_entry_map, list) {
 			list_del(&entry->list);
 			kfree(entry);
 		}
 	}
 }
 
-/* called for unity map ACPI definition */
-static int __init init_unity_map_range(struct ivmd_header *m,
-				       struct acpi_table_header *ivrs_base)
+/* called for ivmd map ACPI definition */
+static int __init init_ivmd_map_range(struct ivmd_header *m,
+				      struct acpi_table_header *ivrs_base)
 {
-	struct unity_map_entry *e = NULL;
+	struct ivmd_entry *e = NULL;
 	struct amd_iommu_pci_seg *pci_seg;
 	char *s;
 
@@ -2642,7 +2637,7 @@ static int __init init_unity_map_range(struct ivmd_header *m,
 	}
 	e->address_start = PAGE_ALIGN(m->range_start);
 	e->address_end = e->address_start + PAGE_ALIGN(m->range_length);
-	e->prot = m->flags >> 1;
+	e->flags = m->flags;
 
 	DUMP_printk("%s devid_start: %04x:%02x:%02x.%x devid_end: "
 		    "%04x:%02x:%02x.%x range_start: %016llx range_end: %016llx"
@@ -2653,7 +2648,7 @@ static int __init init_unity_map_range(struct ivmd_header *m,
 		    PCI_SLOT(e->devid_end), PCI_FUNC(e->devid_end),
 		    e->address_start, e->address_end, m->flags);
 
-	list_add_tail(&e->list, &pci_seg->unity_map);
+	list_add_tail(&e->list, &pci_seg->ivmd_entry_map);
 
 	return 0;
 }
@@ -2669,8 +2664,8 @@ static int __init init_memory_definitions(struct acpi_table_header *table)
 
 	while (p < end) {
 		m = (struct ivmd_header *)p;
-		if (m->flags & (IVMD_FLAG_UNITY_MAP | IVMD_FLAG_EXCL_RANGE))
-			init_unity_map_range(m, table);
+		if (m->flags & (IVMD_FLAG_UNITY | IVMD_FLAG_EXCL))
+			init_ivmd_map_range(m, table);
 
 		p += m->length;
 	}
@@ -3115,7 +3110,7 @@ static bool __init check_ioapic_information(void)
 static void __init free_dma_resources(void)
 {
 	amd_iommu_pdom_id_destroy();
-	free_unity_maps();
+	free_ivmd_maps();
 }
 
 static void __init ivinfo_init(void *ivrs)
diff --git a/drivers/iommu/amd/iommu.c b/drivers/iommu/amd/iommu.c
index 760d5f4623b5..fc3389ded01a 100644
--- a/drivers/iommu/amd/iommu.c
+++ b/drivers/iommu/amd/iommu.c
@@ -3050,7 +3050,7 @@ static void amd_iommu_get_resv_regions(struct device *dev,
 				       struct list_head *head)
 {
 	struct iommu_resv_region *region;
-	struct unity_map_entry *entry;
+	struct ivmd_entry *entry;
 	struct amd_iommu *iommu;
 	struct amd_iommu_pci_seg *pci_seg;
 	int devid, sbdf;
@@ -3063,28 +3063,42 @@ static void amd_iommu_get_resv_regions(struct device *dev,
 	iommu = get_amd_iommu_from_dev(dev);
 	pci_seg = iommu->pci_seg;
 
-	list_for_each_entry(entry, &pci_seg->unity_map, list) {
+	list_for_each_entry(entry, &pci_seg->ivmd_entry_map, list) {
 		int type, prot = 0;
 		size_t length;
 
 		if (devid < entry->devid_start || devid > entry->devid_end)
 			continue;
 
+		/*
+		 * IVMD_FLAG_UNITY, IVMD_FLAG_IR, IVMD_FLAG_IW are ignored if
+		 * IVMD_FLAG_EXCL is set.
+		 */
+		if ((entry->flags & IVMD_FLAG_EXCL) &&
+		    (entry->flags & (IVMD_FLAG_UNITY | IVMD_FLAG_IR | IVMD_FLAG_IW)))
+			pr_err(FW_BUG "%s: Invalid IVMD flags combination: %#02x for sbdf=%#x. Ignore IW/IR/Unity flags\n",
+			       __func__, entry->flags, sbdf);
+
+		if (entry->flags & ~IVMD_FLAG_MASK)
+			pr_err("%s: Unknown flags: %#02x for sbdf=%#x\n",
+			       __func__, entry->flags & ~IVMD_FLAG_MASK, sbdf);
+
 		type   = IOMMU_RESV_DIRECT;
 		length = entry->address_end - entry->address_start;
-		if (entry->prot & IOMMU_PROT_IR)
+		if (entry->flags & IVMD_FLAG_IR)
 			prot |= IOMMU_READ;
-		if (entry->prot & IOMMU_PROT_IW)
+		if (entry->flags & IVMD_FLAG_IW)
 			prot |= IOMMU_WRITE;
-		if (entry->prot & IOMMU_UNITY_MAP_FLAG_EXCL_RANGE)
-			/* Exclusion range */
+		if (entry->flags & IVMD_FLAG_EXCL) {
 			type = IOMMU_RESV_RESERVED;
+			prot = 0;
+		}
 
 		region = iommu_alloc_resv_region(entry->address_start,
 						 length, prot, type,
 						 GFP_KERNEL);
 		if (!region) {
-			dev_err(dev, "Out of memory allocating dm-regions\n");
+			pr_err("%s: Out of memory allocating reserved regions for sbdf=%#x\n", __func__, sbdf);
 			return;
 		}
 		list_add_tail(&region->list, head);
-- 
2.34.1

[PATCH 4/4] iommu/amd: Introduce boot option ivmd=seg:bus:dev.fun,start,size,flags

[Suravee Suthikulpanit]

IVRS table contains IVMD blocks, which allow firmware to specify memory
usage requirements to communicate to system software based on its needs
or on hardware characteristics. Each IVMD entry may be per-device, range
of devices.

Some BIOS specify incorrect or missing IVMD entry. Introduce a new ivmd
boot option to allow user to specify up-to 4 per-device IVMD entries at
boot time. The entries are stored during driver initialization, and will
be added to the per-segment ivmd_entry_map so that they can be included
during struct iommu_ops.get_resv_regions.

Signed-off-by: Suravee Suthikulpanit <suravee.suthikulpanit@amd.com>
---
 .../admin-guide/kernel-parameters.txt         |  16 +++
 drivers/iommu/amd/amd_iommu_types.h           |  14 +++
 drivers/iommu/amd/init.c                      | 104 +++++++++++++++++-
 3 files changed, 133 insertions(+), 1 deletion(-)

diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt
index 03a550630644..e680a258008f 100644
--- a/Documentation/admin-guide/kernel-parameters.txt
+++ b/Documentation/admin-guide/kernel-parameters.txt
@@ -2897,6 +2897,22 @@ Kernel parameters
 			  PCI device ID 00:14.5, write the parameter as:
 				ivrs_acpihid[0001:00:14.5]=AMD0020:0
 
+	ivmd		[HW,X86-64]
+			Supplement IVMD unity mapping or exclusion ranges from
+			the kernel command line (in addition to the IVRS ACPI
+			table). May be specified multiple times.
+
+			Form:
+				ivmd=segment:bus:dev.fn,start,length,flags
+
+			start and length are byte counts (decimal or 0x hex).
+			flags is the IVMD flags byte (UNITY, IR, IW, EXCL bits
+			per the AMD IOMMU specification). Use segment 0 for
+			PCI segment zero.
+
+			Example:
+				ivmd=0000:00:01.0,0xe0000000,0x100000,0xb
+
 	js=		[HW,JOY] Analog joystick
 			See Documentation/input/joydev/joystick.rst.
 
diff --git a/drivers/iommu/amd/amd_iommu_types.h b/drivers/iommu/amd/amd_iommu_types.h
index e93b4b857dde..25a5bbd8ec03 100644
--- a/drivers/iommu/amd/amd_iommu_types.h
+++ b/drivers/iommu/amd/amd_iommu_types.h
@@ -893,6 +893,20 @@ struct ivmd_entry {
 	u8 flags;
 };
 
+/*
+ * IVMD-style ranges from ivrs_ivmd=... ; applied in
+ * init_memory_definitions() once per-PCI-segment
+ * ivmd_entry_map exists.
+ */
+struct ivmd_cmdline {
+	struct list_head list;
+	u16 pci_seg;
+	u16 devid;
+	u64 range_start;
+	u64 range_length;
+	u8 flags;
+};
+
 /*
  * Data structures for device handling
  */
diff --git a/drivers/iommu/amd/init.c b/drivers/iommu/amd/init.c
index 4b62bb89a12c..7d60143d4711 100644
--- a/drivers/iommu/amd/init.c
+++ b/drivers/iommu/amd/init.c
@@ -208,10 +208,12 @@ enum iommu_init_state {
 static struct devid_map __initdata early_ioapic_map[EARLY_MAP_SIZE];
 static struct devid_map __initdata early_hpet_map[EARLY_MAP_SIZE];
 static struct acpihid_map_entry __initdata early_acpihid_map[EARLY_MAP_SIZE];
+static struct ivmd_cmdline early_ivmd_cmdline_map[EARLY_MAP_SIZE] __initdata;
 
 static int __initdata early_ioapic_map_size;
 static int __initdata early_hpet_map_size;
 static int __initdata early_acpihid_map_size;
+static int early_ivmd_cmdline_map_size __initdata;
 
 static bool __initdata cmdline_maps;
 
@@ -2653,6 +2655,47 @@ static int __init init_ivmd_map_range(struct ivmd_header *m,
 	return 0;
 }
 
+static int __init apply_ivmd_cmdline_entries(struct acpi_table_header *ivrs_base)
+{
+	int i;
+	struct ivmd_entry *e;
+	struct amd_iommu_pci_seg *pci_seg;
+
+	for (i = 0; i < early_ivmd_cmdline_map_size; i++) {
+		struct ivmd_cmdline *cmd = &early_ivmd_cmdline_map[i];
+
+		pci_seg = get_pci_segment(cmd->pci_seg, ivrs_base);
+		if (!pci_seg) {
+			pr_err("ivmd: PCI segment %#x unavailable\n",
+			       cmd->pci_seg);
+			continue;
+		}
+
+		if (cmd->devid > pci_seg->last_bdf) {
+			pr_err("%s: requestor %#x:%#02x:%#02x.%#02x exceeds segment %#x last BDF %#x\n",
+			       __func__, cmd->pci_seg, PCI_BUS_NUM(cmd->devid),
+			       PCI_SLOT(cmd->devid), PCI_FUNC(cmd->devid),
+			       pci_seg->id, pci_seg->last_bdf);
+			continue;
+		}
+
+		e = kzalloc(sizeof(*e), GFP_KERNEL);
+		if (!e) {
+			kfree(cmd);
+			return -ENOMEM;
+		}
+
+		e->devid_start = cmd->devid;
+		e->devid_end = cmd->devid;
+		e->address_start = PAGE_ALIGN(cmd->range_start);
+		e->address_end = e->address_start + PAGE_ALIGN(cmd->range_length);
+		e->flags = cmd->flags;
+		list_add_tail(&e->list, &pci_seg->ivmd_entry_map);
+	}
+
+	return 0;
+}
+
 /* iterates over all memory definitions we find in the ACPI table */
 static int __init init_memory_definitions(struct acpi_table_header *table)
 {
@@ -2670,7 +2713,7 @@ static int __init init_memory_definitions(struct acpi_table_header *table)
 		p += m->length;
 	}
 
-	return 0;
+	return apply_ivmd_cmdline_entries(table);
 }
 
 /*
@@ -3812,12 +3855,71 @@ static int __init parse_ivrs_acpihid(char *str)
 	return 1;
 }
 
+/*
+ * ivmd=seg:bus:dev.fn,start,length,flags
+ *
+ * start and length are in bytes (decimal or 0x hex). flags is the IVMD flags
+ * byte (IVMD_FLAG_UNITY, IVMD_FLAG_IR, IVMD_FLAG_IW, IVMD_FLAG_EXCL). May be
+ * repeated for multiple ranges.
+ */
+static int __init parse_ivmd(char *str)
+{
+	u32 seg, bus, dev, fn;
+	unsigned long long range_start, range_len;
+	unsigned int flags;
+	struct ivmd_cmdline *cmd;
+
+	if (!str)
+		goto invalid;
+
+	if (sscanf(str, "%x:%x:%x.%x,%llu,%llu,%x",
+		   &seg, &bus, &dev, &fn, &range_start,
+		   &range_len, &flags) != 7)
+		goto invalid;
+
+	if (flags & ~IVMD_FLAG_MASK) {
+		pr_err("%s: flags %#x contains unknown flags\n", __func__, flags);
+		return 1;
+	}
+
+	if (!(flags & (IVMD_FLAG_UNITY | IVMD_FLAG_EXCL)))
+		pr_warn("%s: flags %#02x omit UNITY/EXCL; entry may have no effect\n",
+			__func__, (u8)flags);
+
+	if (early_ivmd_cmdline_map_size == EARLY_MAP_SIZE) {
+		pr_err("%s: Early IVMD command line map overflow - ignoring ivmd=%s\n",
+			__func__, str);
+		return 1;
+	}
+
+	cmd = &early_ivmd_cmdline_map[early_ivmd_cmdline_map_size];
+	early_ivmd_cmdline_map_size++;
+
+	cmd->pci_seg = (u16)seg;
+	cmd->devid = PCI_DEVID(bus & 0xff, PCI_DEVFN(dev & 0x1f, fn & 0x7));
+	cmd->range_start = range_start;
+	cmd->range_length = range_len;
+	cmd->flags = (u8)flags;
+
+	pr_info("%s: segment=%04x dev=%02x:%02x.%x start=%#llx len=%#llx flags=%#02x\n",
+		__func__, cmd->pci_seg,
+		PCI_BUS_NUM(cmd->devid), PCI_SLOT(cmd->devid),
+		PCI_FUNC(cmd->devid), range_start, range_len, cmd->flags);
+
+	return 1;
+
+invalid:
+	pr_err("Invalid command line: ivmd=%s\n", str ? str : "");
+	return 1;
+}
+
 __setup("amd_iommu_dump",	parse_amd_iommu_dump);
 __setup("amd_iommu=",		parse_amd_iommu_options);
 __setup("amd_iommu_intr=",	parse_amd_iommu_intr);
 __setup("ivrs_ioapic",		parse_ivrs_ioapic);
 __setup("ivrs_hpet",		parse_ivrs_hpet);
 __setup("ivrs_acpihid",		parse_ivrs_acpihid);
+__setup("ivmd=",		parse_ivmd);
 
 bool amd_iommu_pasid_supported(void)
 {
-- 
2.34.1