[RFC PATCH 0/2] iommu: Expose IOMMU information in sysfs

[RFC PATCH 0/2] iommu: Expose IOMMU information in sysfs

[Alex Williamson]

Users want to know the features of their hardware and we need a better
way to get it than parsing it out of dmesg.  This series adds a simple
registration interface for IOMMUs and an example base implementation
for intel-iommu.

One key hardware feature for device assignment is the IOMMU support
for superpages.  In this example, we can parse it out of the "cap"
attribute exposed, but I expect we'll want to add more human readable
entries for such features.  I'd welcome suggestions on what features
we should pull out into human friendly attributes and how to format
the contents.

I have not attempted to make a common, consistent interface for
attributes between various IOMMU types here.  I'm not entirely sure
such a thing is possible.  Perhaps instead we do like I show in the
intel-iommu example and provide IOMMU driver specific attribute
groups, clearly labeled so that we effectively give each a namespace.
We can promote consistency between drivers, but a common namespace
is probably best left to userspace tools.

Appreciate any thoughts and comments.  Thanks,

Alex

---

Alex Williamson (2):
      iommu: Add sysfs support for IOMMUs
      iommu/intel: Make use of IOMMU sysfs support


 drivers/iommu/Makefile      |    1 
 drivers/iommu/dmar.c        |    8 ++
 drivers/iommu/intel-iommu.c |   75 ++++++++++++++++++++++
 drivers/iommu/iommu-sysfs.c |  147 +++++++++++++++++++++++++++++++++++++++++++
 include/linux/intel-iommu.h |    2 +
 include/linux/iommu.h       |   28 ++++++++
 6 files changed, 260 insertions(+), 1 deletion(-)
 create mode 100644 drivers/iommu/iommu-sysfs.c

[RFC PATCH 1/2] iommu: Add sysfs support for IOMMUs

[Alex Williamson]

IOMMUs currently have no common representation to userspace, most
seem to have no representation at all aside from a few printks
on bootup.  There are, however, features of IOMMUs that are useful
to know about.  For instance, the IOMMU might support superpages,
making use of processor large/huge pages more important in a device
assignment scenario.  It's also useful to create cross links between
devices and IOMMU hardware units, so that users might be able to
load balance their devices to avoid thrashing a single hardware unit.

This patch adds a registration and de-registration interface as well
as device linking, making it very lightweight for an IOMMU driver to
add basic support.  IOMMU drivers can provide additional attributes
automatically by using an attribute_group.

The attributes exposed are expected to be relatively device specific,
the means to retrieve them certainly are.  So there are currently
no common attributes for the new iommu_class created here.

Signed-off-by: Alex Williamson <alex.williamson@redhat.com>
---
 drivers/iommu/Makefile      |    1 
 drivers/iommu/iommu-sysfs.c |  147 +++++++++++++++++++++++++++++++++++++++++++
 include/linux/iommu.h       |   28 ++++++++
 3 files changed, 176 insertions(+)
 create mode 100644 drivers/iommu/iommu-sysfs.c

diff --git a/drivers/iommu/Makefile b/drivers/iommu/Makefile
index 5d58bf1..437c231 100644
--- a/drivers/iommu/Makefile
+++ b/drivers/iommu/Makefile
@@ -1,5 +1,6 @@
 obj-$(CONFIG_IOMMU_API) += iommu.o
 obj-$(CONFIG_IOMMU_API) += iommu-traces.o
+obj-$(CONFIG_IOMMU_API) += iommu-sysfs.o
 obj-$(CONFIG_OF_IOMMU)	+= of_iommu.o
 obj-$(CONFIG_MSM_IOMMU) += msm_iommu.o msm_iommu_dev.o
 obj-$(CONFIG_AMD_IOMMU) += amd_iommu.o amd_iommu_init.o
diff --git a/drivers/iommu/iommu-sysfs.c b/drivers/iommu/iommu-sysfs.c
new file mode 100644
index 0000000..123f20c
--- /dev/null
+++ b/drivers/iommu/iommu-sysfs.c
@@ -0,0 +1,147 @@
+#include <linux/device.h>
+#include <linux/iommu.h>
+#include <linux/module.h>
+
+struct iommu_device {
+	struct device dev;
+	struct kobject *devices;
+};
+
+#define to_iommu_device(dev) container_of(dev, struct iommu_device, dev)
+
+static void iommu_release_device(struct device *dev)
+{
+	struct iommu_device *iommu = to_iommu_device(dev);
+	kobject_put(iommu->devices);
+	kfree(iommu);
+}
+
+static struct class iommu_class = {
+	.name = "iommu",
+	.dev_release = iommu_release_device,
+};
+
+static int __init iommu_dev_init(void)
+{
+	return class_register(&iommu_class);
+}
+postcore_initcall(iommu_dev_init);
+
+static int __match_platform_data(struct device *dev, const void *platform_data)
+{
+	return dev->platform_data == platform_data;
+}
+
+int iommu_register_device(struct device *parent, const char *name,
+			  const struct attribute_group **groups,
+			  void *platform_data)
+{
+	struct device *dev;
+	struct iommu_device *iommu;
+	int ret;
+
+	dev = class_find_device(&iommu_class, NULL,
+				platform_data, __match_platform_data);
+	if (dev) {
+		put_device(dev);
+		return -EEXIST;
+	}
+
+	iommu = kzalloc(sizeof(*iommu), GFP_KERNEL);
+	if (!iommu)
+		return -ENOMEM;
+
+	device_initialize(&iommu->dev);
+
+	iommu->dev.class = &iommu_class;
+	iommu->dev.parent = parent;
+	dev_set_name(&iommu->dev, "%s", name);
+	iommu->dev.groups = groups;
+	iommu->dev.platform_data = platform_data;
+
+	ret = device_add(&iommu->dev);
+	if (ret)
+		goto error;
+
+	iommu->devices = kobject_create_and_add("devices", &iommu->dev.kobj);
+	if (!iommu->devices) {
+		ret = -ENOMEM;
+		goto error;
+	}
+
+	/* devices holds ref */
+	put_device(&iommu->dev);
+
+	return 0;
+
+error:
+	pr_warn("Unable to register iommu device %s\n", name);
+	put_device(&iommu->dev);
+
+	return ret;
+}
+
+void iommu_unregister_device(void *platform_data)
+{
+	struct device *dev;
+	struct iommu_device *iommu;
+
+	dev = class_find_device(&iommu_class, NULL,
+				platform_data, __match_platform_data);
+	if (!dev)
+		return;
+
+	iommu = to_iommu_device(dev);
+	/* class_find_device gives us a new ref to put in device_unregister */
+	device_unregister(&iommu->dev);
+	kobject_put(iommu->devices);
+}
+
+int iommu_device_link(const void *platform_data, struct device *link)
+{
+	struct device *dev;
+	struct iommu_device *iommu;
+	int ret;
+
+	dev = class_find_device(&iommu_class, NULL,
+				platform_data, __match_platform_data);
+	if (!dev)
+		return -ENODEV;
+
+	iommu = to_iommu_device(dev);
+
+	ret = sysfs_create_link(&link->kobj, &dev->kobj, "iommu");
+	if (ret)
+		goto error;
+
+	ret = sysfs_create_link_nowarn(iommu->devices,
+				       &link->kobj, dev_name(link));
+	if (ret) {
+		sysfs_remove_link(&link->kobj, "iommu");
+		goto error;
+	}
+
+	kobject_get(iommu->devices);
+error:
+	put_device(dev);
+	return ret;
+}
+
+void iommu_device_unlink(const void *platform_data, struct device *link)
+{
+	struct device *dev;
+	struct iommu_device *iommu;
+
+	dev = class_find_device(&iommu_class, NULL,
+				platform_data, __match_platform_data);
+	if (!dev)
+		return;
+
+	iommu = to_iommu_device(dev);
+
+	sysfs_remove_link(iommu->devices, dev_name(link));
+	sysfs_remove_link(&link->kobj, "iommu");
+
+	kobject_put(iommu->devices);
+	put_device(dev);
+}
diff --git a/include/linux/iommu.h b/include/linux/iommu.h
index b96a5b2..e9bd3a0 100644
--- a/include/linux/iommu.h
+++ b/include/linux/iommu.h
@@ -186,6 +186,12 @@ extern int iommu_domain_get_attr(struct iommu_domain *domain, enum iommu_attr,
 				 void *data);
 extern int iommu_domain_set_attr(struct iommu_domain *domain, enum iommu_attr,
 				 void *data);
+extern int iommu_register_device(struct device *parent, const char *name,
+				 const struct attribute_group **groups,
+				 void *platform_data);
+extern void iommu_unregister_device(void *platform_data);
+extern int iommu_device_link(const void *platform_data, struct device *link);
+extern void iommu_device_unlink(const void *platform_data, struct device *link);
 
 /* Window handling function prototypes */
 extern int iommu_domain_window_enable(struct iommu_domain *domain, u32 wnd_nr,
@@ -396,6 +402,28 @@ static inline int iommu_domain_set_attr(struct iommu_domain *domain,
 	return -EINVAL;
 }
 
+static inline int iommu_register_device(struct device *parent, const char *name,
+					const struct attribute_group **groups,
+					void *platform_data)
+{
+	return 0;
+}
+
+static inline void iommu_unregister_device(void *platform_data)
+{
+}
+
+static inline int iommu_device_link(const void *platform_data,
+				    struct device *link)
+{
+	return 0;
+}
+
+static inline void iommu_device_unlink(const void *platform_data,
+				       struct device *link)
+{
+}
+
 #endif /* CONFIG_IOMMU_API */
 
 #endif /* __LINUX_IOMMU_H */

[RFC PATCH 2/2] iommu/intel: Make use of IOMMU sysfs support

[Alex Williamson]

Register our DRHD IOMMUs, cross link devices, and provide a base set
of attributes for the IOMMU.  Note that IRQ remapping support parses
the DMAR table very early in boot, well before the iommu_class can
reasonably be setup, so our registration is split between
intel_iommu_init(), which occurs later, and alloc_iommu(), which
typically occurs much earlier, but may happen at any time later
with IOMMU hot-add support.

On a typical desktop system, this provides the following (pruned):

$ find /sys | grep dmar
/sys/devices/virtual/iommu/dmar0
/sys/devices/virtual/iommu/dmar0/devices
/sys/devices/virtual/iommu/dmar0/devices/0000:00:02.0
/sys/devices/virtual/iommu/dmar0/intel-iommu
/sys/devices/virtual/iommu/dmar0/intel-iommu/cap
/sys/devices/virtual/iommu/dmar0/intel-iommu/ecap
/sys/devices/virtual/iommu/dmar0/intel-iommu/address
/sys/devices/virtual/iommu/dmar0/intel-iommu/version
/sys/devices/virtual/iommu/dmar1
/sys/devices/virtual/iommu/dmar1/devices
/sys/devices/virtual/iommu/dmar1/devices/0000:00:00.0
/sys/devices/virtual/iommu/dmar1/devices/0000:00:01.0
/sys/devices/virtual/iommu/dmar1/devices/0000:00:16.0
/sys/devices/virtual/iommu/dmar1/devices/0000:00:1a.0
/sys/devices/virtual/iommu/dmar1/devices/0000:00:1b.0
/sys/devices/virtual/iommu/dmar1/devices/0000:00:1c.0
...
/sys/devices/virtual/iommu/dmar1/intel-iommu
/sys/devices/virtual/iommu/dmar1/intel-iommu/cap
/sys/devices/virtual/iommu/dmar1/intel-iommu/ecap
/sys/devices/virtual/iommu/dmar1/intel-iommu/address
/sys/devices/virtual/iommu/dmar1/intel-iommu/version
/sys/class/iommu/dmar0
/sys/class/iommu/dmar1

(devices also link back to the dmar units)

Putting these under the "virtual" directory may not be correct since
these are physical devices, maybe under /sys/devices/system?  Virtual
is simply the default for NULL parent.

This makes address, version, capabilities, and extended capabilities
available, just like printed on boot.  Nearly everything we might
want can be parsed from cap/ecap, but I expect we'll want to make
some human friendly entries for things that we care about.

Signed-off-by: Alex Williamson <alex.williamson@redhat.com>
---
 drivers/iommu/dmar.c        |    8 +++++
 drivers/iommu/intel-iommu.c |   75 ++++++++++++++++++++++++++++++++++++++++++-
 include/linux/intel-iommu.h |    2 +
 3 files changed, 84 insertions(+), 1 deletion(-)

diff --git a/drivers/iommu/dmar.c b/drivers/iommu/dmar.c
index 39f8b71..68b3d07 100644
--- a/drivers/iommu/dmar.c
+++ b/drivers/iommu/dmar.c
@@ -38,6 +38,7 @@
 #include <linux/tboot.h>
 #include <linux/dmi.h>
 #include <linux/slab.h>
+#include <linux/iommu.h>
 #include <asm/irq_remapping.h>
 #include <asm/iommu_table.h>
 
@@ -980,6 +981,11 @@ static int alloc_iommu(struct dmar_drhd_unit *drhd)
 	raw_spin_lock_init(&iommu->register_lock);
 
 	drhd->iommu = iommu;
+
+	if (intel_iommu_enabled)
+		iommu_register_device(NULL, iommu->name,
+				      intel_iommu_groups, iommu);
+
 	return 0;
 
  err_unmap:
@@ -991,6 +997,8 @@ static int alloc_iommu(struct dmar_drhd_unit *drhd)
 
 static void free_iommu(struct intel_iommu *iommu)
 {
+	iommu_unregister_device(iommu);
+
 	if (iommu->irq) {
 		free_irq(iommu->irq, iommu);
 		irq_set_handler_data(iommu->irq, NULL);
diff --git a/drivers/iommu/intel-iommu.c b/drivers/iommu/intel-iommu.c
index d87b3c9..f8d779c 100644
--- a/drivers/iommu/intel-iommu.c
+++ b/drivers/iommu/intel-iommu.c
@@ -3930,6 +3930,63 @@ static struct notifier_block intel_iommu_memory_nb = {
 	.priority = 0
 };
 
+
+static ssize_t intel_iommu_show_version(struct device *dev,
+					struct device_attribute *attr,
+					char *buf)
+{
+	struct intel_iommu *iommu = dev->platform_data;
+	u32 ver = readl(iommu->reg + DMAR_VER_REG);
+	return sprintf(buf, "%d:%d\n",
+		       DMAR_VER_MAJOR(ver), DMAR_VER_MINOR(ver));
+}
+static DEVICE_ATTR(version, S_IRUGO, intel_iommu_show_version, NULL);
+
+static ssize_t intel_iommu_show_address(struct device *dev,
+					struct device_attribute *attr,
+					char *buf)
+{
+	struct intel_iommu *iommu = dev->platform_data;
+	return sprintf(buf, "%llx\n", iommu->reg_phys);
+}
+static DEVICE_ATTR(address, S_IRUGO, intel_iommu_show_address, NULL);
+
+static ssize_t intel_iommu_show_cap(struct device *dev,
+				    struct device_attribute *attr,
+				    char *buf)
+{
+	struct intel_iommu *iommu = dev->platform_data;
+	return sprintf(buf, "%llx\n", iommu->cap);
+}
+static DEVICE_ATTR(cap, S_IRUGO, intel_iommu_show_cap, NULL);
+
+static ssize_t intel_iommu_show_ecap(struct device *dev,
+				    struct device_attribute *attr,
+				    char *buf)
+{
+	struct intel_iommu *iommu = dev->platform_data;
+	return sprintf(buf, "%llx\n", iommu->ecap);
+}
+static DEVICE_ATTR(ecap, S_IRUGO, intel_iommu_show_ecap, NULL);
+
+static struct attribute *intel_iommu_attrs[] = {
+	&dev_attr_version.attr,
+	&dev_attr_address.attr,
+	&dev_attr_cap.attr,
+	&dev_attr_ecap.attr,
+	NULL,
+};
+
+static struct attribute_group intel_iommu_group = {
+	.name = "intel-iommu",
+	.attrs = intel_iommu_attrs,
+};
+
+const struct attribute_group *intel_iommu_groups[] = {
+	&intel_iommu_group,
+	NULL,
+};
+
 int __init intel_iommu_init(void)
 {
 	int ret = -ENODEV;
@@ -4001,6 +4058,10 @@ int __init intel_iommu_init(void)
 
 	init_iommu_pm_ops();
 
+	for_each_iommu(iommu, drhd)
+		iommu_register_device(NULL, iommu->name,
+				      intel_iommu_groups, iommu);
+
 	bus_set_iommu(&pci_bus_type, &intel_iommu_ops);
 	bus_register_notifier(&pci_bus_type, &device_nb);
 	if (si_domain && !hw_pass_through)
@@ -4338,14 +4399,18 @@ static int intel_iommu_domain_has_cap(struct iommu_domain *domain,
 
 static int intel_iommu_add_device(struct device *dev)
 {
+	struct intel_iommu *iommu;
 	struct pci_dev *pdev;
 	struct iommu_group *group;
 	int ret;
 	u8 bus, devfn;
 
-	if (!device_to_iommu(dev, &bus, &devfn))
+	iommu = device_to_iommu(dev, &bus, &devfn);
+	if (!iommu)
 		return -ENODEV;
 
+	iommu_device_link(iommu, dev);
+
 	group = iommu_group_get(dev);
 	if (group) {
 		iommu_group_put(group);
@@ -4371,7 +4436,15 @@ static int intel_iommu_add_device(struct device *dev)
 
 static void intel_iommu_remove_device(struct device *dev)
 {
+	struct intel_iommu *iommu;
+	u8 bus, devfn;
+
+	iommu = device_to_iommu(dev, &bus, &devfn);
+	if (!iommu)
+		return;
+
 	iommu_group_remove_device(dev);
+	iommu_device_unlink(iommu, dev);
 }
 
 static struct iommu_ops intel_iommu_ops = {
diff --git a/include/linux/intel-iommu.h b/include/linux/intel-iommu.h
index 0a2da51..aa6905e 100644
--- a/include/linux/intel-iommu.h
+++ b/include/linux/intel-iommu.h
@@ -365,4 +365,6 @@ extern int qi_submit_sync(struct qi_desc *desc, struct intel_iommu *iommu);
 
 extern int dmar_ir_support(void);
 
+extern const struct attribute_group *intel_iommu_groups[];
+
 #endif

[RFC PATCH 3/2] iommu/amd: Add sysfs support

[Alex Williamson]

No proof of concept is complete with only a single user, so here's an
example for AMD IOMMU.  This IOMMU is neatly associated with a device,
so we don't need to leave the IOMMU device dangling in virtual space.
This just has a couple fields from struct amd_iommu cherry picked to
expose and looks something like this (pruned):

# find /sys | grep ivhd
/sys/devices/pci0000:00/0000:00:00.2/iommu/ivhd0
/sys/devices/pci0000:00/0000:00:00.2/iommu/ivhd0/devices
/sys/devices/pci0000:00/0000:00:00.2/iommu/ivhd0/devices/0000:00:00.0
/sys/devices/pci0000:00/0000:00:00.2/iommu/ivhd0/devices/0000:00:02.0
/sys/devices/pci0000:00/0000:00:00.2/iommu/ivhd0/devices/0000:00:04.0
/sys/devices/pci0000:00/0000:00:00.2/iommu/ivhd0/devices/0000:00:09.0
/sys/devices/pci0000:00/0000:00:00.2/iommu/ivhd0/devices/0000:00:11.0
/sys/devices/pci0000:00/0000:00:00.2/iommu/ivhd0/devices/0000:00:12.0
/sys/devices/pci0000:00/0000:00:00.2/iommu/ivhd0/devices/0000:00:12.2
/sys/devices/pci0000:00/0000:00:00.2/iommu/ivhd0/devices/0000:00:13.0
...
/sys/devices/pci0000:00/0000:00:00.2/iommu/ivhd0/power
/sys/devices/pci0000:00/0000:00:00.2/iommu/ivhd0/power/control
...
/sys/devices/pci0000:00/0000:00:00.2/iommu/ivhd0/device
/sys/devices/pci0000:00/0000:00:00.2/iommu/ivhd0/subsystem
/sys/devices/pci0000:00/0000:00:00.2/iommu/ivhd0/amd-iommu
/sys/devices/pci0000:00/0000:00:00.2/iommu/ivhd0/amd-iommu/cap
/sys/devices/pci0000:00/0000:00:00.2/iommu/ivhd0/amd-iommu/is_iommu_v2
/sys/devices/pci0000:00/0000:00:00.2/iommu/ivhd0/amd-iommu/features
/sys/devices/pci0000:00/0000:00:00.2/iommu/ivhd0/uevent
/sys/class/iommu/ivhd0

Signed-off-by: Alex Williamson <alex.williamson@redhat.com>
---
 drivers/iommu/amd_iommu.c      |    4 +++
 drivers/iommu/amd_iommu_init.c |   53 ++++++++++++++++++++++++++++++++++++++++
 2 files changed, 57 insertions(+)

diff --git a/drivers/iommu/amd_iommu.c b/drivers/iommu/amd_iommu.c
index 4621692..4849580 100644
--- a/drivers/iommu/amd_iommu.c
+++ b/drivers/iommu/amd_iommu.c
@@ -346,6 +346,8 @@ static int iommu_init_device(struct device *dev)
 
 	dev->archdata.iommu = dev_data;
 
+	iommu_device_link(amd_iommu_rlookup_table[dev_data->devid], dev);
+
 	return 0;
 }
 
@@ -370,6 +372,8 @@ static void iommu_uninit_device(struct device *dev)
 	if (!dev_data)
 		return;
 
+	iommu_device_unlink(amd_iommu_rlookup_table[dev_data->devid], dev);
+
 	iommu_group_remove_device(dev);
 
 	/* Unlink from alias, it may change if another device is re-plugged */
diff --git a/drivers/iommu/amd_iommu_init.c b/drivers/iommu/amd_iommu_init.c
index b76c58d..68f7c08 100644
--- a/drivers/iommu/amd_iommu_init.c
+++ b/drivers/iommu/amd_iommu_init.c
@@ -26,6 +26,8 @@
 #include <linux/msi.h>
 #include <linux/amd-iommu.h>
 #include <linux/export.h>
+#include <linux/iommu.h>
+#include <linux/device.h>
 #include <asm/pci-direct.h>
 #include <asm/iommu.h>
 #include <asm/gart.h>
@@ -1197,11 +1199,55 @@ static void init_iommu_perf_ctr(struct amd_iommu *iommu)
 	iommu->max_counters = (u8) ((val >> 7) & 0xf);
 }
 
+static ssize_t amd_iommu_show_cap(struct device *dev,
+				  struct device_attribute *attr,
+				  char *buf)
+{
+	struct amd_iommu *iommu = dev->platform_data;
+	return sprintf(buf, "%x\n", iommu->cap);
+}
+static DEVICE_ATTR(cap, S_IRUGO, amd_iommu_show_cap, NULL);
+
+static ssize_t amd_iommu_show_features(struct device *dev,
+				       struct device_attribute *attr,
+				       char *buf)
+{
+	struct amd_iommu *iommu = dev->platform_data;
+	return sprintf(buf, "%llx\n", iommu->features);
+}
+static DEVICE_ATTR(features, S_IRUGO, amd_iommu_show_features, NULL);
+
+static ssize_t amd_iommu_show_v2(struct device *dev,
+				 struct device_attribute *attr,
+				 char *buf)
+{
+	struct amd_iommu *iommu = dev->platform_data;
+	return sprintf(buf, "%c\n", iommu->is_iommu_v2 ? 'Y' : 'N');
+}
+static DEVICE_ATTR(is_iommu_v2, S_IRUGO, amd_iommu_show_v2, NULL);
+
+static struct attribute *amd_iommu_attrs[] = {
+	&dev_attr_cap.attr,
+	&dev_attr_features.attr,
+	&dev_attr_is_iommu_v2.attr,
+	NULL,
+};
+
+static struct attribute_group amd_iommu_group = {
+	.name = "amd-iommu",
+	.attrs = amd_iommu_attrs,
+};
+
+const struct attribute_group *amd_iommu_groups[] = {
+	&amd_iommu_group,
+	NULL,
+};
 
 static int iommu_init_pci(struct amd_iommu *iommu)
 {
 	int cap_ptr = iommu->cap_ptr;
 	u32 range, misc, low, high;
+	char *name;
 
 	iommu->dev = pci_get_bus_and_slot(PCI_BUS_NUM(iommu->devid),
 					  iommu->devid & 0xff);
@@ -1297,6 +1343,13 @@ static int iommu_init_pci(struct amd_iommu *iommu)
 
 	amd_iommu_erratum_746_workaround(iommu);
 
+	name = kasprintf(GFP_KERNEL, "ivhd%d", iommu->index);
+	if (name) {
+		iommu_register_device(&iommu->dev->dev, name,
+				      amd_iommu_groups, iommu);
+		kfree(name);
+	}
+
 	return pci_enable_device(iommu->dev);
 }