[PATCH v2 0/7] IOMMU/DMA map_resource support for peer-to-peer

[PATCH v2 0/7] IOMMU/DMA map_resource support for peer-to-peer

[wdavis]

From: Will Davis <wdavis@nvidia.com>

Hi,

This is a second attempt at adding the DMA APIs necessary to map and unmap
a struct resource to and from a PCI device's IOVA domain. This allows a
PCI device to access a peer device's BAR resource when a hardware IOMMU is
enabled.

Thanks,
Will

Changelog:

v2:
- added documentation for the new DMA APIs
- fixed physical-to-bus address conversion in the nommu implementation

v1: http://www.spinics.net/lists/linux-pci/msg40747.html

Will Davis (7):
  dma-debug: add checking for map/unmap_resource
  DMA-API: Introduce dma_(un)map_resource
  dma-mapping: pci: add pci_(un)map_resource
  DMA-API: Add dma_(un)map_resource() documentation
  iommu/amd: Implement (un)map_resource
  iommu/vt-d: implement (un)map_resource
  x86: add pci-nommu implementation of map_resource

 Documentation/DMA-API-HOWTO.txt          | 39 +++++++++++++++-
 Documentation/DMA-API.txt                | 36 ++++++++++++---
 arch/x86/kernel/pci-nommu.c              | 32 ++++++++++++++
 drivers/iommu/amd_iommu.c                | 76 ++++++++++++++++++++++++++------
 drivers/iommu/intel-iommu.c              | 18 ++++++++
 include/asm-generic/dma-mapping-broken.h |  9 ++++
 include/asm-generic/dma-mapping-common.h | 34 ++++++++++++++
 include/asm-generic/pci-dma-compat.h     | 14 ++++++
 include/linux/dma-debug.h                | 20 +++++++++
 include/linux/dma-mapping.h              |  7 +++
 lib/dma-debug.c                          | 47 ++++++++++++++++++++
 11 files changed, 312 insertions(+), 20 deletions(-)

-- 
2.4.0

[PATCH v2 1/7] dma-debug: add checking for map/unmap_resource

[wdavis]

From: Will Davis <wdavis@nvidia.com>

Add debug callbacks for the new dma_map_resource and dma_unmap_resource
functions.

Signed-off-by: Will Davis <wdavis@nvidia.com>
Reviewed-by: Terence Ripperda <tripperda@nvidia.com>
Reviewed-by: John Hubbard <jhubbard@nvidia.com>
---
 include/linux/dma-debug.h | 20 ++++++++++++++++++++
 lib/dma-debug.c           | 47 +++++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 67 insertions(+)

diff --git a/include/linux/dma-debug.h b/include/linux/dma-debug.h
index fe8cb61..19f328c 100644
--- a/include/linux/dma-debug.h
+++ b/include/linux/dma-debug.h
@@ -44,6 +44,13 @@ extern void debug_dma_mapping_error(struct device *dev, dma_addr_t dma_addr);
 extern void debug_dma_unmap_page(struct device *dev, dma_addr_t addr,
 				 size_t size, int direction, bool map_single);
 
+extern void debug_dma_map_resource(struct device *dev, struct resource *res,
+				   size_t offset, size_t size, int direction,
+				   dma_addr_t dma_addr);
+
+extern void debug_dma_unmap_resource(struct device *dev, dma_addr_t addr,
+				     size_t size, int direction);
+
 extern void debug_dma_map_sg(struct device *dev, struct scatterlist *sg,
 			     int nents, int mapped_ents, int direction);
 
@@ -120,6 +127,19 @@ static inline void debug_dma_unmap_page(struct device *dev, dma_addr_t addr,
 {
 }
 
+static inline void debug_dma_map_resource(struct device *dev,
+					  struct resource *res, size_t offset,
+					  size_t size, int direction,
+					  dma_addr_t dma_addr)
+{
+}
+
+static inline void debug_dma_unmap_resource(struct device *dev,
+					    dma_addr_t addr, size_t size,
+					    int direction)
+{
+}
+
 static inline void debug_dma_map_sg(struct device *dev, struct scatterlist *sg,
 				    int nents, int mapped_ents, int direction)
 {
diff --git a/lib/dma-debug.c b/lib/dma-debug.c
index ae4b65e..a6d8fa7 100644
--- a/lib/dma-debug.c
+++ b/lib/dma-debug.c
@@ -43,6 +43,7 @@ enum {
 	dma_debug_page,
 	dma_debug_sg,
 	dma_debug_coherent,
+	dma_debug_resource,
 };
 
 enum map_err_types {
@@ -1348,6 +1349,52 @@ void debug_dma_unmap_page(struct device *dev, dma_addr_t addr,
 }
 EXPORT_SYMBOL(debug_dma_unmap_page);
 
+void debug_dma_map_resource(struct device *dev, struct resource *resource,
+			    size_t offset, size_t size, int direction,
+			    dma_addr_t dma_addr)
+{
+	struct dma_debug_entry *entry;
+
+	if (unlikely(dma_debug_disabled()))
+		return;
+
+	if (dma_mapping_error(dev, dma_addr))
+		return;
+
+	entry = dma_entry_alloc();
+	if (!entry)
+		return;
+
+	entry->dev       = dev;
+	entry->type      = dma_debug_resource;
+	entry->pfn       = resource->start >> PAGE_SHIFT;
+	entry->offset    = offset,
+	entry->dev_addr  = dma_addr;
+	entry->size      = size;
+	entry->direction = direction;
+
+	add_dma_entry(entry);
+}
+EXPORT_SYMBOL(debug_dma_map_resource);
+
+void debug_dma_unmap_resource(struct device *dev, dma_addr_t addr,
+			      size_t size, int direction)
+{
+	struct dma_debug_entry ref = {
+		.type           = dma_debug_resource,
+		.dev            = dev,
+		.dev_addr       = addr,
+		.size           = size,
+		.direction      = direction,
+	};
+
+	if (unlikely(dma_debug_disabled()))
+		return;
+
+	check_unmap(&ref);
+}
+EXPORT_SYMBOL(debug_dma_unmap_resource);
+
 void debug_dma_map_sg(struct device *dev, struct scatterlist *sg,
 		      int nents, int mapped_ents, int direction)
 {
-- 
2.4.0

[PATCH v2 2/7] DMA-API: Introduce dma_(un)map_resource

[wdavis]

From: Will Davis <wdavis@nvidia.com>

Add functions to DMA-map and -unmap a resource for a given device. This
will allow devices to DMA-map a peer device's resource (for example,
another device's BAR region on PCI) to enable peer-to-peer transactions.

Signed-off-by: Will Davis <wdavis@nvidia.com>
Reviewed-by: Terence Ripperda <tripperda@nvidia.com>
Reviewed-by: John Hubbard <jhubbard@nvidia.com>
---
 include/asm-generic/dma-mapping-broken.h |  9 +++++++++
 include/asm-generic/dma-mapping-common.h | 34 ++++++++++++++++++++++++++++++++
 include/linux/dma-mapping.h              |  7 +++++++
 3 files changed, 50 insertions(+)

diff --git a/include/asm-generic/dma-mapping-broken.h b/include/asm-generic/dma-mapping-broken.h
index 6c32af9..d171f01 100644
--- a/include/asm-generic/dma-mapping-broken.h
+++ b/include/asm-generic/dma-mapping-broken.h
@@ -59,6 +59,15 @@ extern void
 dma_unmap_page(struct device *dev, dma_addr_t dma_address, size_t size,
 	       enum dma_data_direction direction);
 
+extern dma_addr_t
+dma_map_resource(struct device *dev, struct resource *res,
+		 unsigned long offset, size_t size,
+		 enum dma_data_direction direction);
+
+extern void
+dma_unmap_resource(struct device *dev, dma_addr_t dma_address, size_t size,
+		   enum dma_data_direction direction);
+
 extern void
 dma_sync_single_for_cpu(struct device *dev, dma_addr_t dma_handle, size_t size,
 			enum dma_data_direction direction);
diff --git a/include/asm-generic/dma-mapping-common.h b/include/asm-generic/dma-mapping-common.h
index 940d5ec..cd9948d 100644
--- a/include/asm-generic/dma-mapping-common.h
+++ b/include/asm-generic/dma-mapping-common.h
@@ -73,6 +73,36 @@ static inline void dma_unmap_sg_attrs(struct device *dev, struct scatterlist *sg
 		ops->unmap_sg(dev, sg, nents, dir, attrs);
 }
 
+static inline dma_addr_t dma_map_resource_attrs(struct device *dev,
+						struct resource *res,
+						size_t offset, size_t size,
+						enum dma_data_direction dir,
+						struct dma_attrs *attrs)
+{
+	const struct dma_map_ops *ops = get_dma_ops(dev);
+	dma_addr_t addr = 0;
+
+	BUG_ON(!valid_dma_direction(dir));
+	if (ops->map_resource)
+		addr = ops->map_resource(dev, res, offset, size, dir, attrs);
+	debug_dma_map_resource(dev, res, offset, size, dir, addr);
+
+	return addr;
+}
+
+static inline void dma_unmap_resource_attrs(struct device *dev, dma_addr_t addr,
+					    size_t size,
+					    enum dma_data_direction dir,
+					    struct dma_attrs *attrs)
+{
+	const struct dma_map_ops *ops = get_dma_ops(dev);
+
+	BUG_ON(!valid_dma_direction(dir));
+	if (ops->unmap_resource)
+		ops->unmap_resource(dev, addr, size, dir, attrs);
+	debug_dma_unmap_resource(dev, addr, size, dir);
+}
+
 static inline dma_addr_t dma_map_page(struct device *dev, struct page *page,
 				      size_t offset, size_t size,
 				      enum dma_data_direction dir)
@@ -180,6 +210,10 @@ dma_sync_sg_for_device(struct device *dev, struct scatterlist *sg,
 #define dma_unmap_single(d, a, s, r) dma_unmap_single_attrs(d, a, s, r, NULL)
 #define dma_map_sg(d, s, n, r) dma_map_sg_attrs(d, s, n, r, NULL)
 #define dma_unmap_sg(d, s, n, r) dma_unmap_sg_attrs(d, s, n, r, NULL)
+#define dma_map_resource(d, e, o, s, r) \
+	dma_map_resource_attrs(d, e, o, s, r, NULL)
+#define dma_unmap_resource(d, a, s, r) \
+	dma_unmap_resource_attrs(d, a, s, r, NULL)
 
 extern int dma_common_mmap(struct device *dev, struct vm_area_struct *vma,
 			   void *cpu_addr, dma_addr_t dma_addr, size_t size);
diff --git a/include/linux/dma-mapping.h b/include/linux/dma-mapping.h
index ac07ff0..05b0b51 100644
--- a/include/linux/dma-mapping.h
+++ b/include/linux/dma-mapping.h
@@ -34,6 +34,13 @@ struct dma_map_ops {
 	void (*unmap_page)(struct device *dev, dma_addr_t dma_handle,
 			   size_t size, enum dma_data_direction dir,
 			   struct dma_attrs *attrs);
+	dma_addr_t (*map_resource)(struct device *dev, struct resource *res,
+				   unsigned long offset, size_t size,
+				   enum dma_data_direction dir,
+				   struct dma_attrs *attrs);
+	void (*unmap_resource)(struct device *dev, dma_addr_t dma_handle,
+			       size_t size, enum dma_data_direction dir,
+			       struct dma_attrs *attrs);
 	/*
 	 * map_sg returns 0 on error and a value > 0 on success.
 	 * It should never return a value < 0.
-- 
2.4.0

Re: [PATCH v2 2/7] DMA-API: Introduce dma_(un)map_resource

[Joerg Roedel]

On Mon, May 18, 2015 at 01:24:59PM -0500, wdavis@nvidia.com wrote:
> +static inline dma_addr_t dma_map_resource_attrs(struct device *dev,
> +						struct resource *res,
> +						size_t offset, size_t size,
> +						enum dma_data_direction dir,
> +						struct dma_attrs *attrs)
> +{
> +	const struct dma_map_ops *ops = get_dma_ops(dev);
> +	dma_addr_t addr = 0;
> +
> +	BUG_ON(!valid_dma_direction(dir));
> +	if (ops->map_resource)
> +		addr = ops->map_resource(dev, res, offset, size, dir,
		> atops->map_resourcetrs);
> +	debug_dma_map_resource(dev, res, offset, size, dir, addr);
> +
> +	return addr;
> +}

Please just do a BUG_ON(ops->map_resource == NULL) instead of checking the
pointer and returning 0 if it is NULL. The 0 could be a valid dma_addr
in some implementations, drivers are supposed to check the returned addr
with dma_mapping_error only (and no '== 0' checks).


	Joerg

[PATCH v2 3/7] dma-mapping: pci: add pci_(un)map_resource

[wdavis]

From: Will Davis <wdavis@nvidia.com>

Simply route these through to the new dma_(un)map_resource APIs.

Signed-off-by: Will Davis <wdavis@nvidia.com>
Reviewed-by: Terence Ripperda <tripperda@nvidia.com>
Reviewed-by: John Hubbard <jhubbard@nvidia.com>
---
 include/asm-generic/pci-dma-compat.h | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/include/asm-generic/pci-dma-compat.h b/include/asm-generic/pci-dma-compat.h
index c110843..ac4a4ad 100644
--- a/include/asm-generic/pci-dma-compat.h
+++ b/include/asm-generic/pci-dma-compat.h
@@ -61,6 +61,20 @@ pci_unmap_page(struct pci_dev *hwdev, dma_addr_t dma_address,
 	dma_unmap_page(hwdev == NULL ? NULL : &hwdev->dev, dma_address, size, (enum dma_data_direction)direction);
 }
 
+static inline dma_addr_t
+pci_map_resource(struct pci_dev *hwdev, struct resource *resource,
+		 unsigned long offset, size_t size, int direction)
+{
+	return dma_map_resource(hwdev == NULL ? NULL : &hwdev->dev, resource, offset, size, (enum dma_data_direction)direction);
+}
+
+static inline void
+pci_unmap_resource(struct pci_dev *hwdev, dma_addr_t dma_address, size_t size,
+		   int direction)
+{
+	dma_unmap_resource(hwdev == NULL ? NULL : &hwdev->dev, dma_address, size, (enum dma_data_direction)direction);
+}
+
 static inline int
 pci_map_sg(struct pci_dev *hwdev, struct scatterlist *sg,
 	   int nents, int direction)
-- 
2.4.0

[PATCH v2 4/7] DMA-API: Add dma_(un)map_resource() documentation

[wdavis]

From: Will Davis <wdavis@nvidia.com>

Add references to both the general API documentation as well as the HOWTO.

Signed-off-by: Will Davis <wdavis@nvidia.com>
---
 Documentation/DMA-API-HOWTO.txt | 39 +++++++++++++++++++++++++++++++++++++--
 Documentation/DMA-API.txt       | 36 +++++++++++++++++++++++++++++++-----
 2 files changed, 68 insertions(+), 7 deletions(-)

diff --git a/Documentation/DMA-API-HOWTO.txt b/Documentation/DMA-API-HOWTO.txt
index 0f7afb2..89bd730 100644
--- a/Documentation/DMA-API-HOWTO.txt
+++ b/Documentation/DMA-API-HOWTO.txt
@@ -138,6 +138,10 @@ What about block I/O and networking buffers?  The block I/O and
 networking subsystems make sure that the buffers they use are valid
 for you to DMA from/to.
 
+In some systems, it may also be possible to DMA to and/or from a peer
+device's MMIO region, as described by a 'struct resource'. This is
+referred to as a peer-to-peer mapping.
+
 			DMA addressing limitations
 
 Does your device have any DMA addressing limitations?  For example, is
@@ -648,6 +652,35 @@ Every dma_map_{single,sg}() call should have its dma_unmap_{single,sg}()
 counterpart, because the bus address space is a shared resource and
 you could render the machine unusable by consuming all bus addresses.
 
+Peer-to-peer DMA mappings can be obtained using dma_map_resource() to map
+another device's MMIO region for the given device:
+
+	struct resource *peer_mmio_res = &other_dev->resource[0];
+	dma_addr_t dma_handle = dma_map_resource(dev, peer_mmio_res,
+						 offset, size, direction);
+	if (dma_handle == 0 || dma_mapping_error(dev, dma_handle))
+	{
+		/*
+		 * If dma_handle == 0, dma_map_resource() is not
+		 * implemented, and peer-to-peer transactions will not
+		 * work.
+		 */
+		goto map_error_handling;
+	}
+
+	...
+
+	dma_unmap_resource(dev, dma_handle, size, direction);
+
+Here, "offset" means byte offset within the given resource.
+
+You should both check for a 0 return value and call dma_mapping_error(),
+as dma_map_resource() can either be not implemented or fail and return
+error as outlined under the dma_map_single() discussion.
+
+You should call dma_unmap_resource() when DMA activity is finished, e.g.,
+from the interrupt which told you that the DMA transfer is done.
+
 If you need to use the same streaming DMA region multiple times and touch
 the data in between the DMA transfers, the buffer needs to be synced
 properly in order for the CPU and device to see the most up-to-date and
@@ -765,8 +798,8 @@ failure can be determined by:
 
 - checking if dma_alloc_coherent() returns NULL or dma_map_sg returns 0
 
-- checking the dma_addr_t returned from dma_map_single() and dma_map_page()
-  by using dma_mapping_error():
+- checking the dma_addr_t returned from dma_map_single(), dma_map_resource(),
+  and dma_map_page() by using dma_mapping_error():
 
 	dma_addr_t dma_handle;
 
@@ -780,6 +813,8 @@ failure can be determined by:
 		goto map_error_handling;
 	}
 
+- checking if dma_map_resource() returns 0
+
 - unmap pages that are already mapped, when mapping error occurs in the middle
   of a multiple page mapping attempt. These example are applicable to
   dma_map_page() as well.
diff --git a/Documentation/DMA-API.txt b/Documentation/DMA-API.txt
index 5208840..c25c549 100644
--- a/Documentation/DMA-API.txt
+++ b/Documentation/DMA-API.txt
@@ -283,14 +283,40 @@ and <size> parameters are provided to do partial page mapping, it is
 recommended that you never use these unless you really know what the
 cache width is.
 
+dma_addr_t
+dma_map_resource(struct device *dev, struct resource *res,
+		 unsigned long offset, size_t size,
+		 enum dma_data_direction_direction)
+
+API for mapping resources. This API allows a driver to map a peer
+device's resource for DMA. All the notes and warnings for the other
+APIs apply here. Also, the success of this API does not validate or
+guarantee that peer-to-peer transactions between the device and its
+peer will be functional. They only grant access so that if such
+transactions are possible, an IOMMU will not prevent them from
+succeeding.
+
+If this API is not provided by the underlying implementation, 0 is
+returned and the driver must take appropriate action. Otherwise, the
+DMA address is returned, and that DMA address should be checked by
+the driver (see dma_mapping_error() below).
+
+void
+dma_unmap_resource(struct device *dev, dma_addr_t dma_address, size_t size,
+		   enum dma_data_direction direction)
+
+Unmaps the resource previously mapped. All the parameters passed in
+must be identical to those passed in to (and returned by) the mapping
+API.
+
 int
 dma_mapping_error(struct device *dev, dma_addr_t dma_addr)
 
-In some circumstances dma_map_single() and dma_map_page() will fail to create
-a mapping. A driver can check for these errors by testing the returned
-DMA address with dma_mapping_error(). A non-zero return value means the mapping
-could not be created and the driver should take appropriate action (e.g.
-reduce current DMA mapping usage or delay and try again later).
+In some circumstances dma_map_single(), dma_map_page() and dma_map_resource()
+will fail to create a mapping. A driver can check for these errors by testing
+the returned DMA address with dma_mapping_error(). A non-zero return value
+means the mapping could not be created and the driver should take appropriate
+action (e.g. reduce current DMA mapping usage or delay and try again later).
 
 	int
 	dma_map_sg(struct device *dev, struct scatterlist *sg,
-- 
2.4.0

Re: [PATCH v2 4/7] DMA-API: Add dma_(un)map_resource() documentation

[Bjorn Helgaas]

[+cc Dave, Jonathan]

On Mon, May 18, 2015 at 01:25:01PM -0500, wdavis@nvidia.com wrote:
> From: Will Davis <wdavis@nvidia.com>
> 
> Add references to both the general API documentation as well as the HOWTO.
> 
> Signed-off-by: Will Davis <wdavis@nvidia.com>
> ---
>  Documentation/DMA-API-HOWTO.txt | 39 +++++++++++++++++++++++++++++++++++++--
>  Documentation/DMA-API.txt       | 36 +++++++++++++++++++++++++++++++-----
>  2 files changed, 68 insertions(+), 7 deletions(-)
> 
> diff --git a/Documentation/DMA-API-HOWTO.txt b/Documentation/DMA-API-HOWTO.txt
> index 0f7afb2..89bd730 100644
> --- a/Documentation/DMA-API-HOWTO.txt
> +++ b/Documentation/DMA-API-HOWTO.txt
> @@ -138,6 +138,10 @@ What about block I/O and networking buffers?  The block I/O and
>  networking subsystems make sure that the buffers they use are valid
>  for you to DMA from/to.
>  
> +In some systems, it may also be possible to DMA to and/or from a peer
> +device's MMIO region, as described by a 'struct resource'. This is
> +referred to as a peer-to-peer mapping.
> +
>  			DMA addressing limitations
>  
>  Does your device have any DMA addressing limitations?  For example, is
> @@ -648,6 +652,35 @@ Every dma_map_{single,sg}() call should have its dma_unmap_{single,sg}()
>  counterpart, because the bus address space is a shared resource and
>  you could render the machine unusable by consuming all bus addresses.
>  
> +Peer-to-peer DMA mappings can be obtained using dma_map_resource() to map
> +another device's MMIO region for the given device:
> +
> +	struct resource *peer_mmio_res = &other_dev->resource[0];
> +	dma_addr_t dma_handle = dma_map_resource(dev, peer_mmio_res,
> +						 offset, size, direction);
> +	if (dma_handle == 0 || dma_mapping_error(dev, dma_handle))
> +	{
> +		/*
> +		 * If dma_handle == 0, dma_map_resource() is not
> +		 * implemented, and peer-to-peer transactions will not
> +		 * work.
> +		 */
> +		goto map_error_handling;
> +	}
> +
> +	...
> +
> +	dma_unmap_resource(dev, dma_handle, size, direction);
> +
> +Here, "offset" means byte offset within the given resource.
> +
> +You should both check for a 0 return value and call dma_mapping_error(),
> +as dma_map_resource() can either be not implemented or fail and return
> +error as outlined under the dma_map_single() discussion.
> +
> +You should call dma_unmap_resource() when DMA activity is finished, e.g.,
> +from the interrupt which told you that the DMA transfer is done.
> +
>  If you need to use the same streaming DMA region multiple times and touch
>  the data in between the DMA transfers, the buffer needs to be synced
>  properly in order for the CPU and device to see the most up-to-date and
> @@ -765,8 +798,8 @@ failure can be determined by:
>  
>  - checking if dma_alloc_coherent() returns NULL or dma_map_sg returns 0
>  
> -- checking the dma_addr_t returned from dma_map_single() and dma_map_page()
> -  by using dma_mapping_error():
> +- checking the dma_addr_t returned from dma_map_single(), dma_map_resource(),
> +  and dma_map_page() by using dma_mapping_error():
>  
>  	dma_addr_t dma_handle;
>  
> @@ -780,6 +813,8 @@ failure can be determined by:
>  		goto map_error_handling;
>  	}
>  
> +- checking if dma_map_resource() returns 0
> +
>  - unmap pages that are already mapped, when mapping error occurs in the middle
>    of a multiple page mapping attempt. These example are applicable to
>    dma_map_page() as well.
> diff --git a/Documentation/DMA-API.txt b/Documentation/DMA-API.txt
> index 5208840..c25c549 100644
> --- a/Documentation/DMA-API.txt
> +++ b/Documentation/DMA-API.txt
> @@ -283,14 +283,40 @@ and <size> parameters are provided to do partial page mapping, it is
>  recommended that you never use these unless you really know what the
>  cache width is.
>  
> +dma_addr_t
> +dma_map_resource(struct device *dev, struct resource *res,
> +		 unsigned long offset, size_t size,
> +		 enum dma_data_direction_direction)
> +
> +API for mapping resources. This API allows a driver to map a peer
> +device's resource for DMA. All the notes and warnings for the other
> +APIs apply here. Also, the success of this API does not validate or
> +guarantee that peer-to-peer transactions between the device and its
> +peer will be functional. They only grant access so that if such
> +transactions are possible, an IOMMU will not prevent them from
> +succeeding.

If the driver can't tell whether peer-to-peer accesses will actually work,
this seems like sort of a dubious API.  I'm trying to imagine how a driver
would handle this.  I guess whether peer-to-peer works depends on the
underlying platform (not the devices themselves)?  If we run the driver on
a platform where peer-to-peer *doesn't* work, what happens?  The driver
can't tell, so we just rely on the user to say "this isn't working as
expected"?

> +If this API is not provided by the underlying implementation, 0 is
> +returned and the driver must take appropriate action. Otherwise, the
> +DMA address is returned, and that DMA address should be checked by
> +the driver (see dma_mapping_error() below).
> +
> +void
> +dma_unmap_resource(struct device *dev, dma_addr_t dma_address, size_t size,
> +		   enum dma_data_direction direction)
> +
> +Unmaps the resource previously mapped. All the parameters passed in
> +must be identical to those passed in to (and returned by) the mapping
> +API.
> +
>  int
>  dma_mapping_error(struct device *dev, dma_addr_t dma_addr)
>  
> -In some circumstances dma_map_single() and dma_map_page() will fail to create
> -a mapping. A driver can check for these errors by testing the returned
> -DMA address with dma_mapping_error(). A non-zero return value means the mapping
> -could not be created and the driver should take appropriate action (e.g.
> -reduce current DMA mapping usage or delay and try again later).
> +In some circumstances dma_map_single(), dma_map_page() and dma_map_resource()
> +will fail to create a mapping. A driver can check for these errors by testing
> +the returned DMA address with dma_mapping_error(). A non-zero return value
> +means the mapping could not be created and the driver should take appropriate
> +action (e.g. reduce current DMA mapping usage or delay and try again later).
>  
>  	int
>  	dma_map_sg(struct device *dev, struct scatterlist *sg,
> -- 
> 2.4.0
> 

Re: [PATCH v2 4/7] DMA-API: Add dma_(un)map_resource() documentation

[Mark Hounschell]

On 05/19/2015 07:43 PM, Bjorn Helgaas wrote:
> [+cc Dave, Jonathan]
>
> On Mon, May 18, 2015 at 01:25:01PM -0500, wdavis-DDmLM1+adcrQT0dZR+AlfA@public.gmane.org wrote:
>> From: Will Davis <wdavis-DDmLM1+adcrQT0dZR+AlfA@public.gmane.org>
>>
>> Add references to both the general API documentation as well as the HOWTO.
>>
>> Signed-off-by: Will Davis <wdavis-DDmLM1+adcrQT0dZR+AlfA@public.gmane.org>
>> ---
>>   Documentation/DMA-API-HOWTO.txt | 39 +++++++++++++++++++++++++++++++++++++--
>>   Documentation/DMA-API.txt       | 36 +++++++++++++++++++++++++++++++-----
>>   2 files changed, 68 insertions(+), 7 deletions(-)
>>
>> diff --git a/Documentation/DMA-API-HOWTO.txt b/Documentation/DMA-API-HOWTO.txt
>> index 0f7afb2..89bd730 100644
>> --- a/Documentation/DMA-API-HOWTO.txt
>> +++ b/Documentation/DMA-API-HOWTO.txt
>> @@ -138,6 +138,10 @@ What about block I/O and networking buffers?  The block I/O and
>>   networking subsystems make sure that the buffers they use are valid
>>   for you to DMA from/to.
>>
>> +In some systems, it may also be possible to DMA to and/or from a peer
>> +device's MMIO region, as described by a 'struct resource'. This is
>> +referred to as a peer-to-peer mapping.
>> +
>>   			DMA addressing limitations
>>
>>   Does your device have any DMA addressing limitations?  For example, is
>> @@ -648,6 +652,35 @@ Every dma_map_{single,sg}() call should have its dma_unmap_{single,sg}()
>>   counterpart, because the bus address space is a shared resource and
>>   you could render the machine unusable by consuming all bus addresses.
>>
>> +Peer-to-peer DMA mappings can be obtained using dma_map_resource() to map
>> +another device's MMIO region for the given device:
>> +
>> +	struct resource *peer_mmio_res = &other_dev->resource[0];
>> +	dma_addr_t dma_handle = dma_map_resource(dev, peer_mmio_res,
>> +						 offset, size, direction);
>> +	if (dma_handle == 0 || dma_mapping_error(dev, dma_handle))
>> +	{
>> +		/*
>> +		 * If dma_handle == 0, dma_map_resource() is not
>> +		 * implemented, and peer-to-peer transactions will not
>> +		 * work.
>> +		 */
>> +		goto map_error_handling;
>> +	}
>> +
>> +	...
>> +
>> +	dma_unmap_resource(dev, dma_handle, size, direction);
>> +
>> +Here, "offset" means byte offset within the given resource.
>> +
>> +You should both check for a 0 return value and call dma_mapping_error(),
>> +as dma_map_resource() can either be not implemented or fail and return
>> +error as outlined under the dma_map_single() discussion.
>> +
>> +You should call dma_unmap_resource() when DMA activity is finished, e.g.,
>> +from the interrupt which told you that the DMA transfer is done.
>> +
>>   If you need to use the same streaming DMA region multiple times and touch
>>   the data in between the DMA transfers, the buffer needs to be synced
>>   properly in order for the CPU and device to see the most up-to-date and
>> @@ -765,8 +798,8 @@ failure can be determined by:
>>
>>   - checking if dma_alloc_coherent() returns NULL or dma_map_sg returns 0
>>
>> -- checking the dma_addr_t returned from dma_map_single() and dma_map_page()
>> -  by using dma_mapping_error():
>> +- checking the dma_addr_t returned from dma_map_single(), dma_map_resource(),
>> +  and dma_map_page() by using dma_mapping_error():
>>
>>   	dma_addr_t dma_handle;
>>
>> @@ -780,6 +813,8 @@ failure can be determined by:
>>   		goto map_error_handling;
>>   	}
>>
>> +- checking if dma_map_resource() returns 0
>> +
>>   - unmap pages that are already mapped, when mapping error occurs in the middle
>>     of a multiple page mapping attempt. These example are applicable to
>>     dma_map_page() as well.
>> diff --git a/Documentation/DMA-API.txt b/Documentation/DMA-API.txt
>> index 5208840..c25c549 100644
>> --- a/Documentation/DMA-API.txt
>> +++ b/Documentation/DMA-API.txt
>> @@ -283,14 +283,40 @@ and <size> parameters are provided to do partial page mapping, it is
>>   recommended that you never use these unless you really know what the
>>   cache width is.
>>
>> +dma_addr_t
>> +dma_map_resource(struct device *dev, struct resource *res,
>> +		 unsigned long offset, size_t size,
>> +		 enum dma_data_direction_direction)
>> +
>> +API for mapping resources. This API allows a driver to map a peer
>> +device's resource for DMA. All the notes and warnings for the other
>> +APIs apply here. Also, the success of this API does not validate or
>> +guarantee that peer-to-peer transactions between the device and its
>> +peer will be functional. They only grant access so that if such
>> +transactions are possible, an IOMMU will not prevent them from
>> +succeeding.
>
> If the driver can't tell whether peer-to-peer accesses will actually work,
> this seems like sort of a dubious API.  I'm trying to imagine how a driver
> would handle this.  I guess whether peer-to-peer works depends on the
> underlying platform (not the devices themselves)?  If we run the driver on
> a platform where peer-to-peer *doesn't* work, what happens?  The driver
> can't tell, so we just rely on the user to say "this isn't working as
> expected"?
>

Most currently available hardware doesn't allow reads but will allow 
writes on PCIe peer-to-peer transfers. All current AMD chipsets are this 
way. I'm pretty sure all Intel chipsets are this way also. What happens 
with reads is they are just dropped with no indication of error other 
than the data will not be as expected. Supposedly the PCIe spec does not 
even require any peer-to-peer support. Regular PCI there is no problem 
and this API could be useful. However I doubt seriously you will find a 
pure PCI motherboard that has an IOMMU.

I don't understand the chipset manufactures reasoning for disabling PCIe 
peer-to-peer reads. We would like to make PCIe versions of our cards but 
their application requires  peer-to-peer reads and writes. So we cannot 
develop PCIe versions of the cards.

Again, Regular PCI there is no problem and this API could be useful. 
IOMMU or not.
If we had a pure PCI with IOMMU env, how will this API handle when the 2 
devices are on the same PCI bus. There will be NO IOMMU between the 
devices on the same bus. Does this API address that configuration?

Mark

>> +If this API is not provided by the underlying implementation, 0 is
>> +returned and the driver must take appropriate action. Otherwise, the
>> +DMA address is returned, and that DMA address should be checked by
>> +the driver (see dma_mapping_error() below).
>> +
>> +void
>> +dma_unmap_resource(struct device *dev, dma_addr_t dma_address, size_t size,
>> +		   enum dma_data_direction direction)
>> +
>> +Unmaps the resource previously mapped. All the parameters passed in
>> +must be identical to those passed in to (and returned by) the mapping
>> +API.
>> +
>>   int
>>   dma_mapping_error(struct device *dev, dma_addr_t dma_addr)
>>
>> -In some circumstances dma_map_single() and dma_map_page() will fail to create
>> -a mapping. A driver can check for these errors by testing the returned
>> -DMA address with dma_mapping_error(). A non-zero return value means the mapping
>> -could not be created and the driver should take appropriate action (e.g.
>> -reduce current DMA mapping usage or delay and try again later).
>> +In some circumstances dma_map_single(), dma_map_page() and dma_map_resource()
>> +will fail to create a mapping. A driver can check for these errors by testing
>> +the returned DMA address with dma_mapping_error(). A non-zero return value
>> +means the mapping could not be created and the driver should take appropriate
>> +action (e.g. reduce current DMA mapping usage or delay and try again later).
>>
>>   	int
>>   	dma_map_sg(struct device *dev, struct scatterlist *sg,
>> --
>> 2.4.0
>>
> --
> To unsubscribe from this list: send the line "unsubscribe linux-pci" in
> the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
> More majordomo info at  http://vger.kernel.org/majordomo-info.html
>

RE: [PATCH v2 4/7] DMA-API: Add dma_(un)map_resource() documentation

[William Davis]

> -----Original Message-----
> From: Mark Hounschell [mailto:markh-n2QNKt385d+sTnJN9+BGXg@public.gmane.org]
> Sent: Wednesday, May 20, 2015 7:11 AM
> To: Bjorn Helgaas; William Davis
> Cc: joro-zLv9SwRftAIdnm+yROfE0A@public.gmane.org; iommu-cunTk1MwBs9QetFLy7KEm3xJsTq8ys+cHZ5vskTnxNA@public.gmane.org; linux-
> pci-u79uwXL29TY76Z2rM5mHXA@public.gmane.org; Terence Ripperda; John Hubbard; jglisse-H+wXaHxf7aLQT0dZR+AlfA@public.gmane.org;
> konrad.wilk-QHcLZuEGTsvQT0dZR+AlfA@public.gmane.org; Jonathan Corbet; David S. Miller
> Subject: Re: [PATCH v2 4/7] DMA-API: Add dma_(un)map_resource()
> documentation
> 
> On 05/19/2015 07:43 PM, Bjorn Helgaas wrote:
> > [+cc Dave, Jonathan]
> >
> > On Mon, May 18, 2015 at 01:25:01PM -0500, wdavis-DDmLM1+adcrQT0dZR+AlfA@public.gmane.org wrote:
> >> From: Will Davis <wdavis-DDmLM1+adcrQT0dZR+AlfA@public.gmane.org>
> >>
> >> Add references to both the general API documentation as well as the
> HOWTO.
> >>
> >> Signed-off-by: Will Davis <wdavis-DDmLM1+adcrQT0dZR+AlfA@public.gmane.org>
> >> ---
> >>   Documentation/DMA-API-HOWTO.txt | 39
> +++++++++++++++++++++++++++++++++++++--
> >>   Documentation/DMA-API.txt       | 36 +++++++++++++++++++++++++++++++--
> ---
> >>   2 files changed, 68 insertions(+), 7 deletions(-)
> >>
> >> diff --git a/Documentation/DMA-API-HOWTO.txt
> >> b/Documentation/DMA-API-HOWTO.txt index 0f7afb2..89bd730 100644
> >> --- a/Documentation/DMA-API-HOWTO.txt
> >> +++ b/Documentation/DMA-API-HOWTO.txt
> >> @@ -138,6 +138,10 @@ What about block I/O and networking buffers?  The
> block I/O and
> >>   networking subsystems make sure that the buffers they use are valid
> >>   for you to DMA from/to.
> >>
> >> +In some systems, it may also be possible to DMA to and/or from a
> >> +peer device's MMIO region, as described by a 'struct resource'. This
> >> +is referred to as a peer-to-peer mapping.
> >> +
> >>   			DMA addressing limitations
> >>
> >>   Does your device have any DMA addressing limitations?  For example,
> >> is @@ -648,6 +652,35 @@ Every dma_map_{single,sg}() call should have its
> dma_unmap_{single,sg}()
> >>   counterpart, because the bus address space is a shared resource and
> >>   you could render the machine unusable by consuming all bus addresses.
> >>
> >> +Peer-to-peer DMA mappings can be obtained using dma_map_resource()
> >> +to map another device's MMIO region for the given device:
> >> +
> >> +	struct resource *peer_mmio_res = &other_dev->resource[0];
> >> +	dma_addr_t dma_handle = dma_map_resource(dev, peer_mmio_res,
> >> +						 offset, size, direction);
> >> +	if (dma_handle == 0 || dma_mapping_error(dev, dma_handle))
> >> +	{
> >> +		/*
> >> +		 * If dma_handle == 0, dma_map_resource() is not
> >> +		 * implemented, and peer-to-peer transactions will not
> >> +		 * work.
> >> +		 */
> >> +		goto map_error_handling;
> >> +	}
> >> +
> >> +	...
> >> +
> >> +	dma_unmap_resource(dev, dma_handle, size, direction);
> >> +
> >> +Here, "offset" means byte offset within the given resource.
> >> +
> >> +You should both check for a 0 return value and call
> >> +dma_mapping_error(), as dma_map_resource() can either be not
> >> +implemented or fail and return error as outlined under the
> dma_map_single() discussion.
> >> +
> >> +You should call dma_unmap_resource() when DMA activity is finished,
> >> +e.g., from the interrupt which told you that the DMA transfer is done.
> >> +
> >>   If you need to use the same streaming DMA region multiple times and
> touch
> >>   the data in between the DMA transfers, the buffer needs to be synced
> >>   properly in order for the CPU and device to see the most up-to-date
> >> and @@ -765,8 +798,8 @@ failure can be determined by:
> >>
> >>   - checking if dma_alloc_coherent() returns NULL or dma_map_sg
> >> returns 0
> >>
> >> -- checking the dma_addr_t returned from dma_map_single() and
> >> dma_map_page()
> >> -  by using dma_mapping_error():
> >> +- checking the dma_addr_t returned from dma_map_single(),
> >> +dma_map_resource(),
> >> +  and dma_map_page() by using dma_mapping_error():
> >>
> >>   	dma_addr_t dma_handle;
> >>
> >> @@ -780,6 +813,8 @@ failure can be determined by:
> >>   		goto map_error_handling;
> >>   	}
> >>
> >> +- checking if dma_map_resource() returns 0
> >> +
> >>   - unmap pages that are already mapped, when mapping error occurs in
> the middle
> >>     of a multiple page mapping attempt. These example are applicable to
> >>     dma_map_page() as well.
> >> diff --git a/Documentation/DMA-API.txt b/Documentation/DMA-API.txt
> >> index 5208840..c25c549 100644
> >> --- a/Documentation/DMA-API.txt
> >> +++ b/Documentation/DMA-API.txt
> >> @@ -283,14 +283,40 @@ and <size> parameters are provided to do partial
> page mapping, it is
> >>   recommended that you never use these unless you really know what the
> >>   cache width is.
> >>
> >> +dma_addr_t
> >> +dma_map_resource(struct device *dev, struct resource *res,
> >> +		 unsigned long offset, size_t size,
> >> +		 enum dma_data_direction_direction)
> >> +
> >> +API for mapping resources. This API allows a driver to map a peer
> >> +device's resource for DMA. All the notes and warnings for the other
> >> +APIs apply here. Also, the success of this API does not validate or
> >> +guarantee that peer-to-peer transactions between the device and its
> >> +peer will be functional. They only grant access so that if such
> >> +transactions are possible, an IOMMU will not prevent them from
> >> +succeeding.
> >
> > If the driver can't tell whether peer-to-peer accesses will actually
> > work, this seems like sort of a dubious API.  I'm trying to imagine
> > how a driver would handle this.  I guess whether peer-to-peer works
> > depends on the underlying platform (not the devices themselves)?  If
> > we run the driver on a platform where peer-to-peer *doesn't* work,
> > what happens?  The driver can't tell, so we just rely on the user to
> > say "this isn't working as expected"?
> >
> 

Yes, it's quite difficult to tell whether peer-to-peer will actually work,
and it usually involves some probing and heuristics on the driver's part.
I wouldn't say that this makes it a dubious API - it's a piece of the
puzzle that's absolutely necessary for a driver to set up peer-to-peer in
an IOMMU environment.

> Most currently available hardware doesn't allow reads but will allow writes
> on PCIe peer-to-peer transfers. All current AMD chipsets are this way. I'm
> pretty sure all Intel chipsets are this way also.

Most != all. As an example, Mellanox offers the ability to do peer-to-peer
transfers:

http://www.mellanox.com/page/products_dyn?product_family=116

which would indicate there is at least some platform out there which allows
peer-to-peer reads. I don't think that that being a minority configuration
should preclude it from support.

> What happens with reads
> is they are just dropped with no indication of error other than the data
> will not be as expected. Supposedly the PCIe spec does not even require any
> peer-to-peer support. Regular PCI there is no problem and this API could be
> useful. However I doubt seriously you will find a pure PCI motherboard that
> has an IOMMU.
> 
> I don't understand the chipset manufactures reasoning for disabling PCIe
> peer-to-peer reads. We would like to make PCIe versions of our cards but
> their application requires  peer-to-peer reads and writes. So we cannot
> develop PCIe versions of the cards.
> 
> Again, Regular PCI there is no problem and this API could be useful.
> IOMMU or not.
> If we had a pure PCI with IOMMU env, how will this API handle when the 2
> devices are on the same PCI bus. There will be NO IOMMU between the devices
> on the same bus. Does this API address that configuration?
> 

What is the expected behavior in this configuration? That the "mapping"
simply be the bus address (as in the nommu case)?

In an IOMMU environment, the DMA ops would be one of the IOMMU
implementations, so these APIs would create a mapping for the peer device
resource, even if it's on the same bus. Would a transaction targeting that
mapping be forwarded upstream until it hits an IOMMU, which would then send
the translated request back downstream? Or is my understanding of this
configuration incorrect?

Thanks,
Will

> Mark
> 
> >> +If this API is not provided by the underlying implementation, 0 is
> >> +returned and the driver must take appropriate action. Otherwise, the
> >> +DMA address is returned, and that DMA address should be checked by
> >> +the driver (see dma_mapping_error() below).
> >> +
> >> +void
> >> +dma_unmap_resource(struct device *dev, dma_addr_t dma_address, size_t
> size,
> >> +		   enum dma_data_direction direction)
> >> +
> >> +Unmaps the resource previously mapped. All the parameters passed in
> >> +must be identical to those passed in to (and returned by) the
> >> +mapping API.
> >> +
> >>   int
> >>   dma_mapping_error(struct device *dev, dma_addr_t dma_addr)
> >>
> >> -In some circumstances dma_map_single() and dma_map_page() will fail
> >> to create -a mapping. A driver can check for these errors by testing
> >> the returned -DMA address with dma_mapping_error(). A non-zero return
> >> value means the mapping -could not be created and the driver should take
> appropriate action (e.g.
> >> -reduce current DMA mapping usage or delay and try again later).
> >> +In some circumstances dma_map_single(), dma_map_page() and
> >> +dma_map_resource() will fail to create a mapping. A driver can check
> >> +for these errors by testing the returned DMA address with
> >> +dma_mapping_error(). A non-zero return value means the mapping could
> >> +not be created and the driver should take appropriate action (e.g.
> reduce current DMA mapping usage or delay and try again later).
> >>
> >>   	int
> >>   	dma_map_sg(struct device *dev, struct scatterlist *sg,
> >> --
> >> 2.4.0
> >>
> > --
> > To unsubscribe from this list: send the line "unsubscribe linux-pci"
> > in the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org More majordomo
> > info at  http://vger.kernel.org/majordomo-info.html
> >
--
nvpublic

Re: [PATCH v2 4/7] DMA-API: Add dma_(un)map_resource() documentation

[Mark Hounschell]

On 05/20/2015 01:30 PM, William Davis wrote:
>
>
>> -----Original Message-----
>> From: Mark Hounschell [mailto:markh@compro.net]
>> Sent: Wednesday, May 20, 2015 7:11 AM
>> To: Bjorn Helgaas; William Davis
>> Cc: joro@8bytes.org; iommu@lists.linux-foundation.org; linux-
>> pci@vger.kernel.org; Terence Ripperda; John Hubbard; jglisse@redhat.com;
>> konrad.wilk@oracle.com; Jonathan Corbet; David S. Miller
>> Subject: Re: [PATCH v2 4/7] DMA-API: Add dma_(un)map_resource()
>> documentation
>>
>> On 05/19/2015 07:43 PM, Bjorn Helgaas wrote:
>>> [+cc Dave, Jonathan]
>>>
>>> On Mon, May 18, 2015 at 01:25:01PM -0500, wdavis@nvidia.com wrote:
>>>> From: Will Davis <wdavis@nvidia.com>
>>>>
>>>> Add references to both the general API documentation as well as the
>> HOWTO.
>>>>
>>>> Signed-off-by: Will Davis <wdavis@nvidia.com>
>>>> ---
>>>>    Documentation/DMA-API-HOWTO.txt | 39
>> +++++++++++++++++++++++++++++++++++++--
>>>>    Documentation/DMA-API.txt       | 36 +++++++++++++++++++++++++++++++--
>> ---
>>>>    2 files changed, 68 insertions(+), 7 deletions(-)
>>>>
>>>> diff --git a/Documentation/DMA-API-HOWTO.txt
>>>> b/Documentation/DMA-API-HOWTO.txt index 0f7afb2..89bd730 100644
>>>> --- a/Documentation/DMA-API-HOWTO.txt
>>>> +++ b/Documentation/DMA-API-HOWTO.txt
>>>> @@ -138,6 +138,10 @@ What about block I/O and networking buffers?  The
>> block I/O and
>>>>    networking subsystems make sure that the buffers they use are valid
>>>>    for you to DMA from/to.
>>>>
>>>> +In some systems, it may also be possible to DMA to and/or from a
>>>> +peer device's MMIO region, as described by a 'struct resource'. This
>>>> +is referred to as a peer-to-peer mapping.
>>>> +
>>>>    			DMA addressing limitations
>>>>
>>>>    Does your device have any DMA addressing limitations?  For example,
>>>> is @@ -648,6 +652,35 @@ Every dma_map_{single,sg}() call should have its
>> dma_unmap_{single,sg}()
>>>>    counterpart, because the bus address space is a shared resource and
>>>>    you could render the machine unusable by consuming all bus addresses.
>>>>
>>>> +Peer-to-peer DMA mappings can be obtained using dma_map_resource()
>>>> +to map another device's MMIO region for the given device:
>>>> +
>>>> +	struct resource *peer_mmio_res = &other_dev->resource[0];
>>>> +	dma_addr_t dma_handle = dma_map_resource(dev, peer_mmio_res,
>>>> +						 offset, size, direction);
>>>> +	if (dma_handle == 0 || dma_mapping_error(dev, dma_handle))
>>>> +	{
>>>> +		/*
>>>> +		 * If dma_handle == 0, dma_map_resource() is not
>>>> +		 * implemented, and peer-to-peer transactions will not
>>>> +		 * work.
>>>> +		 */
>>>> +		goto map_error_handling;
>>>> +	}
>>>> +
>>>> +	...
>>>> +
>>>> +	dma_unmap_resource(dev, dma_handle, size, direction);
>>>> +
>>>> +Here, "offset" means byte offset within the given resource.
>>>> +
>>>> +You should both check for a 0 return value and call
>>>> +dma_mapping_error(), as dma_map_resource() can either be not
>>>> +implemented or fail and return error as outlined under the
>> dma_map_single() discussion.
>>>> +
>>>> +You should call dma_unmap_resource() when DMA activity is finished,
>>>> +e.g., from the interrupt which told you that the DMA transfer is done.
>>>> +
>>>>    If you need to use the same streaming DMA region multiple times and
>> touch
>>>>    the data in between the DMA transfers, the buffer needs to be synced
>>>>    properly in order for the CPU and device to see the most up-to-date
>>>> and @@ -765,8 +798,8 @@ failure can be determined by:
>>>>
>>>>    - checking if dma_alloc_coherent() returns NULL or dma_map_sg
>>>> returns 0
>>>>
>>>> -- checking the dma_addr_t returned from dma_map_single() and
>>>> dma_map_page()
>>>> -  by using dma_mapping_error():
>>>> +- checking the dma_addr_t returned from dma_map_single(),
>>>> +dma_map_resource(),
>>>> +  and dma_map_page() by using dma_mapping_error():
>>>>
>>>>    	dma_addr_t dma_handle;
>>>>
>>>> @@ -780,6 +813,8 @@ failure can be determined by:
>>>>    		goto map_error_handling;
>>>>    	}
>>>>
>>>> +- checking if dma_map_resource() returns 0
>>>> +
>>>>    - unmap pages that are already mapped, when mapping error occurs in
>> the middle
>>>>      of a multiple page mapping attempt. These example are applicable to
>>>>      dma_map_page() as well.
>>>> diff --git a/Documentation/DMA-API.txt b/Documentation/DMA-API.txt
>>>> index 5208840..c25c549 100644
>>>> --- a/Documentation/DMA-API.txt
>>>> +++ b/Documentation/DMA-API.txt
>>>> @@ -283,14 +283,40 @@ and <size> parameters are provided to do partial
>> page mapping, it is
>>>>    recommended that you never use these unless you really know what the
>>>>    cache width is.
>>>>
>>>> +dma_addr_t
>>>> +dma_map_resource(struct device *dev, struct resource *res,
>>>> +		 unsigned long offset, size_t size,
>>>> +		 enum dma_data_direction_direction)
>>>> +
>>>> +API for mapping resources. This API allows a driver to map a peer
>>>> +device's resource for DMA. All the notes and warnings for the other
>>>> +APIs apply here. Also, the success of this API does not validate or
>>>> +guarantee that peer-to-peer transactions between the device and its
>>>> +peer will be functional. They only grant access so that if such
>>>> +transactions are possible, an IOMMU will not prevent them from
>>>> +succeeding.
>>>
>>> If the driver can't tell whether peer-to-peer accesses will actually
>>> work, this seems like sort of a dubious API.  I'm trying to imagine
>>> how a driver would handle this.  I guess whether peer-to-peer works
>>> depends on the underlying platform (not the devices themselves)?  If
>>> we run the driver on a platform where peer-to-peer *doesn't* work,
>>> what happens?  The driver can't tell, so we just rely on the user to
>>> say "this isn't working as expected"?
>>>
>>
>
> Yes, it's quite difficult to tell whether peer-to-peer will actually work,
> and it usually involves some probing and heuristics on the driver's part.
> I wouldn't say that this makes it a dubious API - it's a piece of the
> puzzle that's absolutely necessary for a driver to set up peer-to-peer in
> an IOMMU environment.
>

I currently just do

page = virt_to_page(__va(bus_address));

then just use the normal API. Works for writes anyway.

>> Most currently available hardware doesn't allow reads but will allow writes
>> on PCIe peer-to-peer transfers. All current AMD chipsets are this way. I'm
>> pretty sure all Intel chipsets are this way also.
>
> Most != all. As an example, Mellanox offers the ability to do peer-to-peer
> transfers:
>
> http://www.mellanox.com/page/products_dyn?product_family=116
>
> which would indicate there is at least some platform out there which allows
> peer-to-peer reads. I don't think that that being a minority configuration
> should preclude it from support.
>
>> What happens with reads
>> is they are just dropped with no indication of error other than the data
>> will not be as expected. Supposedly the PCIe spec does not even require any
>> peer-to-peer support. Regular PCI there is no problem and this API could be
>> useful. However I doubt seriously you will find a pure PCI motherboard that
>> has an IOMMU.
>>
>> I don't understand the chipset manufactures reasoning for disabling PCIe
>> peer-to-peer reads. We would like to make PCIe versions of our cards but
>> their application requires  peer-to-peer reads and writes. So we cannot
>> develop PCIe versions of the cards.
>>
>> Again, Regular PCI there is no problem and this API could be useful.
>> IOMMU or not.
>> If we had a pure PCI with IOMMU env, how will this API handle when the 2
>> devices are on the same PCI bus. There will be NO IOMMU between the devices
>> on the same bus. Does this API address that configuration?
>>
>
> What is the expected behavior in this configuration? That the "mapping"
> simply be the bus address (as in the nommu case)?
>

I suspect just using the bus address would sort of defeat one or more 
purposes of the IOMMU. The bus address would certainly be what I would 
want to use though.

> In an IOMMU environment, the DMA ops would be one of the IOMMU
> implementations, so these APIs would create a mapping for the peer device
> resource, even if it's on the same bus. Would a transaction targeting that
> mapping be forwarded upstream until it hits an IOMMU, which would then send
> the translated request back downstream? Or is my understanding of this
> configuration incorrect?
>

It's my understanding of the IOMMU that is lacking here. I have no idea 
if that is actually what would happen. Does it?

Regards
Mark


> Thanks,
> Will
>
>> Mark
>>
>>>> +If this API is not provided by the underlying implementation, 0 is
>>>> +returned and the driver must take appropriate action. Otherwise, the
>>>> +DMA address is returned, and that DMA address should be checked by
>>>> +the driver (see dma_mapping_error() below).
>>>> +
>>>> +void
>>>> +dma_unmap_resource(struct device *dev, dma_addr_t dma_address, size_t
>> size,
>>>> +		   enum dma_data_direction direction)
>>>> +
>>>> +Unmaps the resource previously mapped. All the parameters passed in
>>>> +must be identical to those passed in to (and returned by) the
>>>> +mapping API.
>>>> +
>>>>    int
>>>>    dma_mapping_error(struct device *dev, dma_addr_t dma_addr)
>>>>
>>>> -In some circumstances dma_map_single() and dma_map_page() will fail
>>>> to create -a mapping. A driver can check for these errors by testing
>>>> the returned -DMA address with dma_mapping_error(). A non-zero return
>>>> value means the mapping -could not be created and the driver should take
>> appropriate action (e.g.
>>>> -reduce current DMA mapping usage or delay and try again later).
>>>> +In some circumstances dma_map_single(), dma_map_page() and
>>>> +dma_map_resource() will fail to create a mapping. A driver can check
>>>> +for these errors by testing the returned DMA address with
>>>> +dma_mapping_error(). A non-zero return value means the mapping could
>>>> +not be created and the driver should take appropriate action (e.g.
>> reduce current DMA mapping usage or delay and try again later).
>>>>
>>>>    	int
>>>>    	dma_map_sg(struct device *dev, struct scatterlist *sg,
>>>> --
>>>> 2.4.0
>>>>
>>> --
>>> To unsubscribe from this list: send the line "unsubscribe linux-pci"
>>> in the body of a message to majordomo@vger.kernel.org More majordomo
>>> info at  http://vger.kernel.org/majordomo-info.html
>>>
> --
> nvpublic
> --
> To unsubscribe from this list: send the line "unsubscribe linux-pci" in
> the body of a message to majordomo@vger.kernel.org
> More majordomo info at  http://vger.kernel.org/majordomo-info.html
>

RE: [PATCH v2 4/7] DMA-API: Add dma_(un)map_resource() documentation

[William Davis]

> -----Original Message-----
> From: Mark Hounschell [mailto:markh@compro.net]
> Sent: Wednesday, May 20, 2015 2:16 PM
> To: William Davis; Bjorn Helgaas
> Cc: joro@8bytes.org; iommu@lists.linux-foundation.org; linux-
> pci@vger.kernel.org; Terence Ripperda; John Hubbard; jglisse@redhat.com;
> konrad.wilk@oracle.com; Jonathan Corbet; David S. Miller
> Subject: Re: [PATCH v2 4/7] DMA-API: Add dma_(un)map_resource()
> documentation
> 
> On 05/20/2015 01:30 PM, William Davis wrote:
> >
> >
> >> -----Original Message-----
> >> From: Mark Hounschell [mailto:markh@compro.net]
> >> Sent: Wednesday, May 20, 2015 7:11 AM
> >> To: Bjorn Helgaas; William Davis
> >> Cc: joro@8bytes.org; iommu@lists.linux-foundation.org; linux-
> >> pci@vger.kernel.org; Terence Ripperda; John Hubbard;
> >> jglisse@redhat.com; konrad.wilk@oracle.com; Jonathan Corbet; David S.
> >> Miller
> >> Subject: Re: [PATCH v2 4/7] DMA-API: Add dma_(un)map_resource()
> >> documentation
> >>
> >> On 05/19/2015 07:43 PM, Bjorn Helgaas wrote:
> >>> [+cc Dave, Jonathan]
> >>>
> >>> On Mon, May 18, 2015 at 01:25:01PM -0500, wdavis@nvidia.com wrote:
> >>>> From: Will Davis <wdavis@nvidia.com>
> >>>>
> >>>> Add references to both the general API documentation as well as the
> >> HOWTO.
> >>>>
> >>>> Signed-off-by: Will Davis <wdavis@nvidia.com>
> >>>> ---
> >>>>    Documentation/DMA-API-HOWTO.txt | 39
> >> +++++++++++++++++++++++++++++++++++++--
> >>>>    Documentation/DMA-API.txt       | 36
> +++++++++++++++++++++++++++++++--
> >> ---
> >>>>    2 files changed, 68 insertions(+), 7 deletions(-)
> >>>>
> >>>> diff --git a/Documentation/DMA-API-HOWTO.txt
> >>>> b/Documentation/DMA-API-HOWTO.txt index 0f7afb2..89bd730 100644
> >>>> --- a/Documentation/DMA-API-HOWTO.txt
> >>>> +++ b/Documentation/DMA-API-HOWTO.txt
> >>>> @@ -138,6 +138,10 @@ What about block I/O and networking buffers?
> >>>> The
> >> block I/O and
> >>>>    networking subsystems make sure that the buffers they use are valid
> >>>>    for you to DMA from/to.
> >>>>
> >>>> +In some systems, it may also be possible to DMA to and/or from a
> >>>> +peer device's MMIO region, as described by a 'struct resource'.
> >>>> +This is referred to as a peer-to-peer mapping.
> >>>> +
> >>>>    			DMA addressing limitations
> >>>>
> >>>>    Does your device have any DMA addressing limitations?  For
> >>>> example, is @@ -648,6 +652,35 @@ Every dma_map_{single,sg}() call
> >>>> should have its
> >> dma_unmap_{single,sg}()
> >>>>    counterpart, because the bus address space is a shared resource and
> >>>>    you could render the machine unusable by consuming all bus
> addresses.
> >>>>
> >>>> +Peer-to-peer DMA mappings can be obtained using dma_map_resource()
> >>>> +to map another device's MMIO region for the given device:
> >>>> +
> >>>> +	struct resource *peer_mmio_res = &other_dev->resource[0];
> >>>> +	dma_addr_t dma_handle = dma_map_resource(dev, peer_mmio_res,
> >>>> +						 offset, size, direction);
> >>>> +	if (dma_handle == 0 || dma_mapping_error(dev, dma_handle))
> >>>> +	{
> >>>> +		/*
> >>>> +		 * If dma_handle == 0, dma_map_resource() is not
> >>>> +		 * implemented, and peer-to-peer transactions will not
> >>>> +		 * work.
> >>>> +		 */
> >>>> +		goto map_error_handling;
> >>>> +	}
> >>>> +
> >>>> +	...
> >>>> +
> >>>> +	dma_unmap_resource(dev, dma_handle, size, direction);
> >>>> +
> >>>> +Here, "offset" means byte offset within the given resource.
> >>>> +
> >>>> +You should both check for a 0 return value and call
> >>>> +dma_mapping_error(), as dma_map_resource() can either be not
> >>>> +implemented or fail and return error as outlined under the
> >> dma_map_single() discussion.
> >>>> +
> >>>> +You should call dma_unmap_resource() when DMA activity is
> >>>> +finished, e.g., from the interrupt which told you that the DMA
> transfer is done.
> >>>> +
> >>>>    If you need to use the same streaming DMA region multiple times
> >>>> and
> >> touch
> >>>>    the data in between the DMA transfers, the buffer needs to be
> synced
> >>>>    properly in order for the CPU and device to see the most
> >>>> up-to-date and @@ -765,8 +798,8 @@ failure can be determined by:
> >>>>
> >>>>    - checking if dma_alloc_coherent() returns NULL or dma_map_sg
> >>>> returns 0
> >>>>
> >>>> -- checking the dma_addr_t returned from dma_map_single() and
> >>>> dma_map_page()
> >>>> -  by using dma_mapping_error():
> >>>> +- checking the dma_addr_t returned from dma_map_single(),
> >>>> +dma_map_resource(),
> >>>> +  and dma_map_page() by using dma_mapping_error():
> >>>>
> >>>>    	dma_addr_t dma_handle;
> >>>>
> >>>> @@ -780,6 +813,8 @@ failure can be determined by:
> >>>>    		goto map_error_handling;
> >>>>    	}
> >>>>
> >>>> +- checking if dma_map_resource() returns 0
> >>>> +
> >>>>    - unmap pages that are already mapped, when mapping error occurs
> >>>> in
> >> the middle
> >>>>      of a multiple page mapping attempt. These example are applicable
> to
> >>>>      dma_map_page() as well.
> >>>> diff --git a/Documentation/DMA-API.txt b/Documentation/DMA-API.txt
> >>>> index 5208840..c25c549 100644
> >>>> --- a/Documentation/DMA-API.txt
> >>>> +++ b/Documentation/DMA-API.txt
> >>>> @@ -283,14 +283,40 @@ and <size> parameters are provided to do
> >>>> partial
> >> page mapping, it is
> >>>>    recommended that you never use these unless you really know what
> the
> >>>>    cache width is.
> >>>>
> >>>> +dma_addr_t
> >>>> +dma_map_resource(struct device *dev, struct resource *res,
> >>>> +		 unsigned long offset, size_t size,
> >>>> +		 enum dma_data_direction_direction)
> >>>> +
> >>>> +API for mapping resources. This API allows a driver to map a peer
> >>>> +device's resource for DMA. All the notes and warnings for the
> >>>> +other APIs apply here. Also, the success of this API does not
> >>>> +validate or guarantee that peer-to-peer transactions between the
> >>>> +device and its peer will be functional. They only grant access so
> >>>> +that if such transactions are possible, an IOMMU will not prevent
> >>>> +them from succeeding.
> >>>
> >>> If the driver can't tell whether peer-to-peer accesses will actually
> >>> work, this seems like sort of a dubious API.  I'm trying to imagine
> >>> how a driver would handle this.  I guess whether peer-to-peer works
> >>> depends on the underlying platform (not the devices themselves)?  If
> >>> we run the driver on a platform where peer-to-peer *doesn't* work,
> >>> what happens?  The driver can't tell, so we just rely on the user to
> >>> say "this isn't working as expected"?
> >>>
> >>
> >
> > Yes, it's quite difficult to tell whether peer-to-peer will actually
> > work, and it usually involves some probing and heuristics on the driver's
> part.
> > I wouldn't say that this makes it a dubious API - it's a piece of the
> > puzzle that's absolutely necessary for a driver to set up peer-to-peer
> > in an IOMMU environment.
> >
> 
> I currently just do
> 
> page = virt_to_page(__va(bus_address));
> 
> then just use the normal API. Works for writes anyway.
> 

What platform is this on? I don't understand how that could work for
peer-to-peer. As I understand it, there are no 'struct page's for MMIO
regions, and you could actually end up with a very different page as a
result of that unchecked translation (e.g., a "valid" struct page, but
not in the peer's MMIO range at all).

> >> Most currently available hardware doesn't allow reads but will allow
> >> writes on PCIe peer-to-peer transfers. All current AMD chipsets are
> >> this way. I'm pretty sure all Intel chipsets are this way also.
> >
> > Most != all. As an example, Mellanox offers the ability to do
> > peer-to-peer
> > transfers:
> >
> > http://www.mellanox.com/page/products_dyn?product_family=116
> >
> > which would indicate there is at least some platform out there which
> > allows peer-to-peer reads. I don't think that that being a minority
> > configuration should preclude it from support.
> >
> >> What happens with reads
> >> is they are just dropped with no indication of error other than the
> >> data will not be as expected. Supposedly the PCIe spec does not even
> >> require any peer-to-peer support. Regular PCI there is no problem and
> >> this API could be useful. However I doubt seriously you will find a
> >> pure PCI motherboard that has an IOMMU.
> >>
> >> I don't understand the chipset manufactures reasoning for disabling
> >> PCIe peer-to-peer reads. We would like to make PCIe versions of our
> >> cards but their application requires  peer-to-peer reads and writes.
> >> So we cannot develop PCIe versions of the cards.
> >>
> >> Again, Regular PCI there is no problem and this API could be useful.
> >> IOMMU or not.
> >> If we had a pure PCI with IOMMU env, how will this API handle when
> >> the 2 devices are on the same PCI bus. There will be NO IOMMU between
> >> the devices on the same bus. Does this API address that configuration?
> >>
> >
> > What is the expected behavior in this configuration? That the "mapping"
> > simply be the bus address (as in the nommu case)?
> >
> 
> I suspect just using the bus address would sort of defeat one or more
> purposes of the IOMMU. The bus address would certainly be what I would want
> to use though.
> 
> > In an IOMMU environment, the DMA ops would be one of the IOMMU
> > implementations, so these APIs would create a mapping for the peer
> > device resource, even if it's on the same bus. Would a transaction
> > targeting that mapping be forwarded upstream until it hits an IOMMU,
> > which would then send the translated request back downstream? Or is my
> > understanding of this configuration incorrect?
> >
> 
> It's my understanding of the IOMMU that is lacking here. I have no idea if
> that is actually what would happen. Does it?

It's a mildly educated guess, but I haven't tested this configuration to
know for sure.

Thanks,
Will

--
nvpublic

Re: [PATCH v2 4/7] DMA-API: Add dma_(un)map_resource() documentation

[Mark Hounschell]

On 05/20/2015 03:51 PM, William Davis wrote:
>
>
>> -----Original Message-----
>> From: Mark Hounschell [mailto:markh@compro.net]
>> Sent: Wednesday, May 20, 2015 2:16 PM
>> To: William Davis; Bjorn Helgaas
>> Cc: joro@8bytes.org; iommu@lists.linux-foundation.org; linux-
>> pci@vger.kernel.org; Terence Ripperda; John Hubbard; jglisse@redhat.com;
>> konrad.wilk@oracle.com; Jonathan Corbet; David S. Miller
>> Subject: Re: [PATCH v2 4/7] DMA-API: Add dma_(un)map_resource()
>> documentation
>>
>> On 05/20/2015 01:30 PM, William Davis wrote:
>>>
>>>
>>>> -----Original Message-----
>>>> From: Mark Hounschell [mailto:markh@compro.net]
>>>> Sent: Wednesday, May 20, 2015 7:11 AM
>>>> To: Bjorn Helgaas; William Davis
>>>> Cc: joro@8bytes.org; iommu@lists.linux-foundation.org; linux-
>>>> pci@vger.kernel.org; Terence Ripperda; John Hubbard;
>>>> jglisse@redhat.com; konrad.wilk@oracle.com; Jonathan Corbet; David S.
>>>> Miller
>>>> Subject: Re: [PATCH v2 4/7] DMA-API: Add dma_(un)map_resource()
>>>> documentation
>>>>
>>>> On 05/19/2015 07:43 PM, Bjorn Helgaas wrote:
>>>>> [+cc Dave, Jonathan]
>>>>>
>>>>> On Mon, May 18, 2015 at 01:25:01PM -0500, wdavis@nvidia.com wrote:
>>>>>> From: Will Davis <wdavis@nvidia.com>
>>>>>>
>>>>>> Add references to both the general API documentation as well as the
>>>> HOWTO.
>>>>>>
>>>>>> Signed-off-by: Will Davis <wdavis@nvidia.com>
>>>>>> ---
>>>>>>     Documentation/DMA-API-HOWTO.txt | 39
>>>> +++++++++++++++++++++++++++++++++++++--
>>>>>>     Documentation/DMA-API.txt       | 36
>> +++++++++++++++++++++++++++++++--
>>>> ---
>>>>>>     2 files changed, 68 insertions(+), 7 deletions(-)
>>>>>>
>>>>>> diff --git a/Documentation/DMA-API-HOWTO.txt
>>>>>> b/Documentation/DMA-API-HOWTO.txt index 0f7afb2..89bd730 100644
>>>>>> --- a/Documentation/DMA-API-HOWTO.txt
>>>>>> +++ b/Documentation/DMA-API-HOWTO.txt
>>>>>> @@ -138,6 +138,10 @@ What about block I/O and networking buffers?
>>>>>> The
>>>> block I/O and
>>>>>>     networking subsystems make sure that the buffers they use are valid
>>>>>>     for you to DMA from/to.
>>>>>>
>>>>>> +In some systems, it may also be possible to DMA to and/or from a
>>>>>> +peer device's MMIO region, as described by a 'struct resource'.
>>>>>> +This is referred to as a peer-to-peer mapping.
>>>>>> +
>>>>>>     			DMA addressing limitations
>>>>>>
>>>>>>     Does your device have any DMA addressing limitations?  For
>>>>>> example, is @@ -648,6 +652,35 @@ Every dma_map_{single,sg}() call
>>>>>> should have its
>>>> dma_unmap_{single,sg}()
>>>>>>     counterpart, because the bus address space is a shared resource and
>>>>>>     you could render the machine unusable by consuming all bus
>> addresses.
>>>>>>
>>>>>> +Peer-to-peer DMA mappings can be obtained using dma_map_resource()
>>>>>> +to map another device's MMIO region for the given device:
>>>>>> +
>>>>>> +	struct resource *peer_mmio_res = &other_dev->resource[0];
>>>>>> +	dma_addr_t dma_handle = dma_map_resource(dev, peer_mmio_res,
>>>>>> +						 offset, size, direction);
>>>>>> +	if (dma_handle == 0 || dma_mapping_error(dev, dma_handle))
>>>>>> +	{
>>>>>> +		/*
>>>>>> +		 * If dma_handle == 0, dma_map_resource() is not
>>>>>> +		 * implemented, and peer-to-peer transactions will not
>>>>>> +		 * work.
>>>>>> +		 */
>>>>>> +		goto map_error_handling;
>>>>>> +	}
>>>>>> +
>>>>>> +	...
>>>>>> +
>>>>>> +	dma_unmap_resource(dev, dma_handle, size, direction);
>>>>>> +
>>>>>> +Here, "offset" means byte offset within the given resource.
>>>>>> +
>>>>>> +You should both check for a 0 return value and call
>>>>>> +dma_mapping_error(), as dma_map_resource() can either be not
>>>>>> +implemented or fail and return error as outlined under the
>>>> dma_map_single() discussion.
>>>>>> +
>>>>>> +You should call dma_unmap_resource() when DMA activity is
>>>>>> +finished, e.g., from the interrupt which told you that the DMA
>> transfer is done.
>>>>>> +
>>>>>>     If you need to use the same streaming DMA region multiple times
>>>>>> and
>>>> touch
>>>>>>     the data in between the DMA transfers, the buffer needs to be
>> synced
>>>>>>     properly in order for the CPU and device to see the most
>>>>>> up-to-date and @@ -765,8 +798,8 @@ failure can be determined by:
>>>>>>
>>>>>>     - checking if dma_alloc_coherent() returns NULL or dma_map_sg
>>>>>> returns 0
>>>>>>
>>>>>> -- checking the dma_addr_t returned from dma_map_single() and
>>>>>> dma_map_page()
>>>>>> -  by using dma_mapping_error():
>>>>>> +- checking the dma_addr_t returned from dma_map_single(),
>>>>>> +dma_map_resource(),
>>>>>> +  and dma_map_page() by using dma_mapping_error():
>>>>>>
>>>>>>     	dma_addr_t dma_handle;
>>>>>>
>>>>>> @@ -780,6 +813,8 @@ failure can be determined by:
>>>>>>     		goto map_error_handling;
>>>>>>     	}
>>>>>>
>>>>>> +- checking if dma_map_resource() returns 0
>>>>>> +
>>>>>>     - unmap pages that are already mapped, when mapping error occurs
>>>>>> in
>>>> the middle
>>>>>>       of a multiple page mapping attempt. These example are applicable
>> to
>>>>>>       dma_map_page() as well.
>>>>>> diff --git a/Documentation/DMA-API.txt b/Documentation/DMA-API.txt
>>>>>> index 5208840..c25c549 100644
>>>>>> --- a/Documentation/DMA-API.txt
>>>>>> +++ b/Documentation/DMA-API.txt
>>>>>> @@ -283,14 +283,40 @@ and <size> parameters are provided to do
>>>>>> partial
>>>> page mapping, it is
>>>>>>     recommended that you never use these unless you really know what
>> the
>>>>>>     cache width is.
>>>>>>
>>>>>> +dma_addr_t
>>>>>> +dma_map_resource(struct device *dev, struct resource *res,
>>>>>> +		 unsigned long offset, size_t size,
>>>>>> +		 enum dma_data_direction_direction)
>>>>>> +
>>>>>> +API for mapping resources. This API allows a driver to map a peer
>>>>>> +device's resource for DMA. All the notes and warnings for the
>>>>>> +other APIs apply here. Also, the success of this API does not
>>>>>> +validate or guarantee that peer-to-peer transactions between the
>>>>>> +device and its peer will be functional. They only grant access so
>>>>>> +that if such transactions are possible, an IOMMU will not prevent
>>>>>> +them from succeeding.
>>>>>
>>>>> If the driver can't tell whether peer-to-peer accesses will actually
>>>>> work, this seems like sort of a dubious API.  I'm trying to imagine
>>>>> how a driver would handle this.  I guess whether peer-to-peer works
>>>>> depends on the underlying platform (not the devices themselves)?  If
>>>>> we run the driver on a platform where peer-to-peer *doesn't* work,
>>>>> what happens?  The driver can't tell, so we just rely on the user to
>>>>> say "this isn't working as expected"?
>>>>>
>>>>
>>>
>>> Yes, it's quite difficult to tell whether peer-to-peer will actually
>>> work, and it usually involves some probing and heuristics on the driver's
>> part.
>>> I wouldn't say that this makes it a dubious API - it's a piece of the
>>> puzzle that's absolutely necessary for a driver to set up peer-to-peer
>>> in an IOMMU environment.
>>>
>>
>> I currently just do
>>
>> page = virt_to_page(__va(bus_address));
>>
>> then just use the normal API. Works for writes anyway.
>>
>
> What platform is this on? I don't understand how that could work for
> peer-to-peer. As I understand it, there are no 'struct page's for MMIO
> regions, and you could actually end up with a very different page as a
> result of that unchecked translation (e.g., a "valid" struct page, but
> not in the peer's MMIO range at all).
>

This is an x86-64 AMD 990FX chip set. Works fine. Every time. I do have 
the peers memory that is being written to mmap'd by the task that is 
doing this, but this works.

Mark

RE: [PATCH v2 4/7] DMA-API: Add dma_(un)map_resource() documentation

[William Davis]

> -----Original Message-----
> From: Mark Hounschell [mailto:markh@compro.net]
> Sent: Wednesday, May 20, 2015 3:07 PM
> To: William Davis; Bjorn Helgaas
> Cc: joro@8bytes.org; iommu@lists.linux-foundation.org; linux-pci@vger.kernel.org; Terence Ripperda;
> John Hubbard; jglisse@redhat.com; konrad.wilk@oracle.com; Jonathan Corbet; David S. Miller
> Subject: Re: [PATCH v2 4/7] DMA-API: Add dma_(un)map_resource() documentation
> 
> >>
> >> I currently just do
> >>
> >> page = virt_to_page(__va(bus_address));
> >>
> >> then just use the normal API. Works for writes anyway.
> >>
> >
> > What platform is this on? I don't understand how that could work for
> > peer-to-peer. As I understand it, there are no 'struct page's for MMIO
> > regions, and you could actually end up with a very different page as a
> > result of that unchecked translation (e.g., a "valid" struct page, but
> > not in the peer's MMIO range at all).
> >
> 
> This is an x86-64 AMD 990FX chip set. Works fine. Every time. I do have
> the peers memory that is being written to mmap'd by the task that is
> doing this, but this works.
> 
> Mark
> 

I spent some time looking into this, and I was surprised to find it worked
for me too. At least, it seems like it works, but it is a dangerous
accident that it did, and so I think the patch is still necessary.

In the IOMMU mapping paths, the IOMMU drivers (at least on x86) only
really use the struct page * to get the physical address that it
references, and I think that's usually done exclusively with pointer
arithmetic these days. So you are getting a pointer to a struct page that
actually doesn't exist, because it's either in a hole or off the end of
the memory map.

You can verify this by calling dump_page() on the struct page * returned
by this calculation. On my Intel machine, this BUGs, unable to handle the
paging request when looking at the struct page fields. If I just use the
pointer without dereferencing it, the IOMMU paths can get the correct
physical address without any issues, but this is by accident and not by
design, nor is it portable, so we shouldn't rely on this technique.

Thanks,
Will

--
nvpublic

Re: [PATCH v2 4/7] DMA-API: Add dma_(un)map_resource() documentation

[joro]

On Wed, May 20, 2015 at 03:15:59PM -0400, Mark Hounschell wrote:
> On 05/20/2015 01:30 PM, William Davis wrote:
> >In an IOMMU environment, the DMA ops would be one of the IOMMU
> >implementations, so these APIs would create a mapping for the peer device
> >resource, even if it's on the same bus. Would a transaction targeting that
> >mapping be forwarded upstream until it hits an IOMMU, which would then send
> >the translated request back downstream? Or is my understanding of this
> >configuration incorrect?
> >
> 
> It's my understanding of the IOMMU that is lacking here. I have no
> idea if that is actually what would happen. Does it?

On the x86 side at least the IOMMU drivers enable ACS when available,
forcing all perr-to-peer DMA between PCIe devices to go through the
IOMMU. This only works when the peers are in different iommu groups,
though.


	Joerg

Re: [PATCH v2 4/7] DMA-API: Add dma_(un)map_resource() documentation

[Bjorn Helgaas]

[+cc Alex]

Hi Mark,

On Wed, May 20, 2015 at 08:11:17AM -0400, Mark Hounschell wrote:
> Most currently available hardware doesn't allow reads but will allow
> writes on PCIe peer-to-peer transfers. All current AMD chipsets are
> this way. I'm pretty sure all Intel chipsets are this way also. What
> happens with reads is they are just dropped with no indication of
> error other than the data will not be as expected. Supposedly the
> PCIe spec does not even require any peer-to-peer support. Regular
> PCI there is no problem and this API could be useful. However I
> doubt seriously you will find a pure PCI motherboard that has an
> IOMMU.
> 
> I don't understand the chipset manufactures reasoning for disabling
> PCIe peer-to-peer reads. We would like to make PCIe versions of our
> cards but their application requires  peer-to-peer reads and writes.
> So we cannot develop PCIe versions of the cards.

I'd like to understand this better.  Peer-to-peer between two devices
below the same Root Port should work as long as ACS doesn't prevent
it.  If we find an Intel or AMD IOMMU, I think we configure ACS to
prevent direct peer-to-peer (see "pci_acs_enable"), but maybe it could
still be done with the appropriate IOMMU support.  And if you boot
with "iommu=off", we don't do that ACS configuration, so peer-to-peer
should work.

I suppose the problem is that peer-to-peer doesn't work between
devices under different Root Ports or even devices under different
Root Complexes?

PCIe r3.0, sec 6.12.1.1, says Root Ports that support peer-to-peer
traffic are required to implement ACS P2P Request Redirect, so if a
Root Port doesn't implement RR, we can assume it doesn't support
peer-to-peer.  But unfortunately the converse is not true: if a Root
Port implements RR, that does *not* imply that it supports
peer-to-peer traffic.  

So I don't know how to discover whether peer-to-peer between Root
Ports or Root Complexes is supported.  Maybe there's some clue in the
IOMMU?  The Intel VT-d spec mentions it, but "peer" doesn't even
appear in the AMD spec.

And I'm curious about why writes sometimes work when reads do not.
That sounds like maybe the hardware support is there, but we don't
understand how to configure everything correctly.

Can you give us the specifics of the topology you'd like to use, e.g.,
lspci -vv of the path between the two devices?

Bjorn

Re: [PATCH v2 4/7] DMA-API: Add dma_(un)map_resource() documentation

[Alex Williamson]

On Tue, 2015-07-07 at 10:15 -0500, Bjorn Helgaas wrote:
> [+cc Alex]
> 
> Hi Mark,
> 
> On Wed, May 20, 2015 at 08:11:17AM -0400, Mark Hounschell wrote:
> > Most currently available hardware doesn't allow reads but will allow
> > writes on PCIe peer-to-peer transfers. All current AMD chipsets are
> > this way. I'm pretty sure all Intel chipsets are this way also. What
> > happens with reads is they are just dropped with no indication of
> > error other than the data will not be as expected. Supposedly the
> > PCIe spec does not even require any peer-to-peer support. Regular
> > PCI there is no problem and this API could be useful. However I
> > doubt seriously you will find a pure PCI motherboard that has an
> > IOMMU.
> > 
> > I don't understand the chipset manufactures reasoning for disabling
> > PCIe peer-to-peer reads. We would like to make PCIe versions of our
> > cards but their application requires  peer-to-peer reads and writes.
> > So we cannot develop PCIe versions of the cards.
> 
> I'd like to understand this better.  Peer-to-peer between two devices
> below the same Root Port should work as long as ACS doesn't prevent
> it.  If we find an Intel or AMD IOMMU, I think we configure ACS to
> prevent direct peer-to-peer (see "pci_acs_enable"), but maybe it could
> still be done with the appropriate IOMMU support.  And if you boot
> with "iommu=off", we don't do that ACS configuration, so peer-to-peer
> should work.

The assumption I work from in vfio is that peer-to-peer should be able
to take a path through the IOMMU, that is from one endpoint, up through
to the RC (with ACS enabled), be translated by the IOMMU and directed
back downstream to the specified endpoint.  Whether that actually works
for any/all transactions, I can't say, I don't know how to test it.
It'd be interesting to know if this is a false assumption.

> I suppose the problem is that peer-to-peer doesn't work between
> devices under different Root Ports or even devices under different
> Root Complexes?
> 
> PCIe r3.0, sec 6.12.1.1, says Root Ports that support peer-to-peer
> traffic are required to implement ACS P2P Request Redirect, so if a
> Root Port doesn't implement RR, we can assume it doesn't support
> peer-to-peer.  But unfortunately the converse is not true: if a Root
> Port implements RR, that does *not* imply that it supports
> peer-to-peer traffic.  

ACS is a much overlooked part of the spec, afaict we can't assume
anything about a root port that doesn't implement ACS, and most of them
don't.  Thanks,

Alex

> So I don't know how to discover whether peer-to-peer between Root
> Ports or Root Complexes is supported.  Maybe there's some clue in the
> IOMMU?  The Intel VT-d spec mentions it, but "peer" doesn't even
> appear in the AMD spec.
> 
> And I'm curious about why writes sometimes work when reads do not.
> That sounds like maybe the hardware support is there, but we don't
> understand how to configure everything correctly.
> 
> Can you give us the specifics of the topology you'd like to use, e.g.,
> lspci -vv of the path between the two devices?
> 
> Bjorn
> --
> To unsubscribe from this list: send the line "unsubscribe linux-pci" in
> the body of a message to majordomo@vger.kernel.org
> More majordomo info at  http://vger.kernel.org/majordomo-info.html

Re: [PATCH v2 4/7] DMA-API: Add dma_(un)map_resource() documentation

[Bjorn Helgaas]

On Tue, Jul 7, 2015 at 10:41 AM, Alex Williamson
<alex.williamson@redhat.com> wrote:
> On Tue, 2015-07-07 at 10:15 -0500, Bjorn Helgaas wrote:
>> [+cc Alex]
>>
>> Hi Mark,
>>
>> On Wed, May 20, 2015 at 08:11:17AM -0400, Mark Hounschell wrote:
>> > Most currently available hardware doesn't allow reads but will allow
>> > writes on PCIe peer-to-peer transfers. All current AMD chipsets are
>> > this way. I'm pretty sure all Intel chipsets are this way also. What
>> > happens with reads is they are just dropped with no indication of
>> > error other than the data will not be as expected. Supposedly the
>> > PCIe spec does not even require any peer-to-peer support. Regular
>> > PCI there is no problem and this API could be useful. However I
>> > doubt seriously you will find a pure PCI motherboard that has an
>> > IOMMU.
>> >
>> > I don't understand the chipset manufactures reasoning for disabling
>> > PCIe peer-to-peer reads. We would like to make PCIe versions of our
>> > cards but their application requires  peer-to-peer reads and writes.
>> > So we cannot develop PCIe versions of the cards.
>>
>> I'd like to understand this better.  Peer-to-peer between two devices
>> below the same Root Port should work as long as ACS doesn't prevent
>> it.  If we find an Intel or AMD IOMMU, I think we configure ACS to
>> prevent direct peer-to-peer (see "pci_acs_enable"), but maybe it could
>> still be done with the appropriate IOMMU support.  And if you boot
>> with "iommu=off", we don't do that ACS configuration, so peer-to-peer
>> should work.
>
> The assumption I work from in vfio is that peer-to-peer should be able
> to take a path through the IOMMU, that is from one endpoint, up through
> to the RC (with ACS enabled), be translated by the IOMMU and directed
> back downstream to the specified endpoint.  Whether that actually works
> for any/all transactions, I can't say, I don't know how to test it.
> It'd be interesting to know if this is a false assumption.

That's what I assume as well.

>> I suppose the problem is that peer-to-peer doesn't work between
>> devices under different Root Ports or even devices under different
>> Root Complexes?
>>
>> PCIe r3.0, sec 6.12.1.1, says Root Ports that support peer-to-peer
>> traffic are required to implement ACS P2P Request Redirect, so if a
>> Root Port doesn't implement RR, we can assume it doesn't support
>> peer-to-peer.  But unfortunately the converse is not true: if a Root
>> Port implements RR, that does *not* imply that it supports
>> peer-to-peer traffic.
>
> ACS is a much overlooked part of the spec, afaict we can't assume
> anything about a root port that doesn't implement ACS, and most of them
> don't.

I think we can assume a Root Port that does not implement ACS P2P
Request Redirect does not support peer-to-peer traffic with other Root
Ports, right?  That's how I read sec 6.12.1.1.

If we're trying to set up a peer-to-peer path between devices under
different Root ports, I think it would make sense to check the ACS
support somewhere in the pci_map_resource() path and fail if P2P RR is
not implemented.

Re: [PATCH v2 4/7] DMA-API: Add dma_(un)map_resource() documentation

[Alex Williamson]

On Tue, 2015-07-07 at 11:16 -0500, Bjorn Helgaas wrote:
> On Tue, Jul 7, 2015 at 10:41 AM, Alex Williamson
> <alex.williamson-H+wXaHxf7aLQT0dZR+AlfA@public.gmane.org> wrote:
> > On Tue, 2015-07-07 at 10:15 -0500, Bjorn Helgaas wrote:
> >> [+cc Alex]
> >>
> >> Hi Mark,
> >>
> >> On Wed, May 20, 2015 at 08:11:17AM -0400, Mark Hounschell wrote:
> >> > Most currently available hardware doesn't allow reads but will allow
> >> > writes on PCIe peer-to-peer transfers. All current AMD chipsets are
> >> > this way. I'm pretty sure all Intel chipsets are this way also. What
> >> > happens with reads is they are just dropped with no indication of
> >> > error other than the data will not be as expected. Supposedly the
> >> > PCIe spec does not even require any peer-to-peer support. Regular
> >> > PCI there is no problem and this API could be useful. However I
> >> > doubt seriously you will find a pure PCI motherboard that has an
> >> > IOMMU.
> >> >
> >> > I don't understand the chipset manufactures reasoning for disabling
> >> > PCIe peer-to-peer reads. We would like to make PCIe versions of our
> >> > cards but their application requires  peer-to-peer reads and writes.
> >> > So we cannot develop PCIe versions of the cards.
> >>
> >> I'd like to understand this better.  Peer-to-peer between two devices
> >> below the same Root Port should work as long as ACS doesn't prevent
> >> it.  If we find an Intel or AMD IOMMU, I think we configure ACS to
> >> prevent direct peer-to-peer (see "pci_acs_enable"), but maybe it could
> >> still be done with the appropriate IOMMU support.  And if you boot
> >> with "iommu=off", we don't do that ACS configuration, so peer-to-peer
> >> should work.
> >
> > The assumption I work from in vfio is that peer-to-peer should be able
> > to take a path through the IOMMU, that is from one endpoint, up through
> > to the RC (with ACS enabled), be translated by the IOMMU and directed
> > back downstream to the specified endpoint.  Whether that actually works
> > for any/all transactions, I can't say, I don't know how to test it.
> > It'd be interesting to know if this is a false assumption.
> 
> That's what I assume as well.
> 
> >> I suppose the problem is that peer-to-peer doesn't work between
> >> devices under different Root Ports or even devices under different
> >> Root Complexes?
> >>
> >> PCIe r3.0, sec 6.12.1.1, says Root Ports that support peer-to-peer
> >> traffic are required to implement ACS P2P Request Redirect, so if a
> >> Root Port doesn't implement RR, we can assume it doesn't support
> >> peer-to-peer.  But unfortunately the converse is not true: if a Root
> >> Port implements RR, that does *not* imply that it supports
> >> peer-to-peer traffic.
> >
> > ACS is a much overlooked part of the spec, afaict we can't assume
> > anything about a root port that doesn't implement ACS, and most of them
> > don't.
> 
> I think we can assume a Root Port that does not implement ACS P2P
> Request Redirect does not support peer-to-peer traffic with other Root
> Ports, right?  That's how I read sec 6.12.1.1.

I think that means that *if* the device supports ACS and does not
implement RR then it does not support peer-to-peer.  The assumption we
make in IOMMU grouping is that if ACS is not supported at all, then the
hardware is providing us with no indication for or against peer-to-peer
and we therefore assume peer-to-peer is possible, but that's for
isolation, not actually trying to make use of p2p.  It seems overly
optimistic to assume the designers of a device that does not implement
ACS evaluated the capability and determined it not to be necessary.
Besides, ACS was only introduced in PCIe rev 2.

> If we're trying to set up a peer-to-peer path between devices under
> different Root ports, I think it would make sense to check the ACS
> support somewhere in the pci_map_resource() path and fail if P2P RR is
> not implemented.

There are multiple options here:

1) No ACS
2) ACS, no RR
3) ACS, RR enabled
4) ACS, RR disabled

In case 1), we have no idea.  Cases 2) and 3) are really the same,
neither does peer-to-peer and I think whether we can support any sort of
peer-to-peer mapping depends on whether there's an IOMMU upstream that
can do the redirect.  For case 4), possibly we can assume the redirect
happens at the root port, but you'll never see this case if an IOMMU is
present and enabled.  Thanks,

Alex

Re: [PATCH v2 4/7] DMA-API: Add dma_(un)map_resource() documentation

[Mark Hounschell]

[-- Attachment #1: Type: text/plain, Size: 4759 bytes --]

Hi Bjorn.

On 07/07/2015 11:15 AM, Bjorn Helgaas wrote:
> [+cc Alex]
>
> Hi Mark,
>
> On Wed, May 20, 2015 at 08:11:17AM -0400, Mark Hounschell wrote:
>> Most currently available hardware doesn't allow reads but will allow
>> writes on PCIe peer-to-peer transfers. All current AMD chipsets are
>> this way. I'm pretty sure all Intel chipsets are this way also. What
>> happens with reads is they are just dropped with no indication of
>> error other than the data will not be as expected. Supposedly the
>> PCIe spec does not even require any peer-to-peer support. Regular
>> PCI there is no problem and this API could be useful. However I
>> doubt seriously you will find a pure PCI motherboard that has an
>> IOMMU.
>>
>> I don't understand the chipset manufactures reasoning for disabling
>> PCIe peer-to-peer reads. We would like to make PCIe versions of our
>> cards but their application requires  peer-to-peer reads and writes.
>> So we cannot develop PCIe versions of the cards.
>
> I'd like to understand this better.  Peer-to-peer between two devices
> below the same Root Port should work as long as ACS doesn't prevent
> it.  If we find an Intel or AMD IOMMU, I think we configure ACS to
> prevent direct peer-to-peer (see "pci_acs_enable"), but maybe it could
> still be done with the appropriate IOMMU support.  And if you boot
> with "iommu=off", we don't do that ACS configuration, so peer-to-peer
> should work.
>
> I suppose the problem is that peer-to-peer doesn't work between
> devices under different Root Ports or even devices under different
> Root Complexes?
>
> PCIe r3.0, sec 6.12.1.1, says Root Ports that support peer-to-peer
> traffic are required to implement ACS P2P Request Redirect, so if a
> Root Port doesn't implement RR, we can assume it doesn't support
> peer-to-peer.  But unfortunately the converse is not true: if a Root
> Port implements RR, that does *not* imply that it supports
> peer-to-peer traffic.
>
> So I don't know how to discover whether peer-to-peer between Root
> Ports or Root Complexes is supported.  Maybe there's some clue in the
> IOMMU?  The Intel VT-d spec mentions it, but "peer" doesn't even
> appear in the AMD spec.
>
> And I'm curious about why writes sometimes work when reads do not.
> That sounds like maybe the hardware support is there, but we don't
> understand how to configure everything correctly.
>
> Can you give us the specifics of the topology you'd like to use, e.g.,
> lspci -vv of the path between the two devices?
>

First off, writes always work for me. Not just sometimes. Only reads 
NEVER do.

Reading the AMD-990FX-990X-970-Register-Programming-Requirements-48693.pdf
in section 2.5 "Enabling/Disabling Peer-to-Peer Traffic Access", it 
states specifically that
only P2P memory writes are supported. This has been the case with older 
AMD chipset also. In one of the older chipset documents I read (I think 
the 770 series) , it said this was a security feature. Makes no sense to 
me.

As for the topology I'd like to be able to use. This particular 
configuration (MB) has a single regular pci slot and the rest are pci-e. 
In two of those pci-e slots is a pci-e to pci expansion chassis 
interface card connected to a regular pci expansion rack. I am trying to 
to peer to peer between a regular pci card in one of those chassis to 
another regular pci card in the other chassis. In turn through the pci-e 
subsystem. Attached is the lcpci -vv output from this particular box. 
The cards that initiate the P2P are these:

04:04.0 Intelligent controller [0e80]: PLX Technology, Inc. Device 0480 
(rev 55)
04:05.0 Intelligent controller [0e80]: PLX Technology, Inc. Device 0480 
(rev 55)
04:06.0 Intelligent controller [0e80]: PLX Technology, Inc. Device 0480 
(rev 55)
04:07.0 Intelligent controller [0e80]: PLX Technology, Inc. Device 0480 
(rev 55)

The card they need to P2P to and from is this one.

0a:05.0 Network controller: VMIC GE-IP PCI5565,PMC5565 Reflective Memory 
Node (rev 01)

Like wise, reversing the chassis the initiator lives in from these cards.

0b:00.0 Unassigned class [ff00]: Compro Computer Services, Inc. Device 
4710 (rev 41)
0c:00.0 Unassigned class [ff00]: Compro Computer Services, Inc. Device 
4710 (rev 41)
0d:00.0 Unassigned class [ff00]: Compro Computer Services, Inc. Device 
4710 (rev 41)
0e:00.0 Unassigned class [ff00]: Compro Computer Services, Inc. Device 
0100 (rev 42)

to this card

04:0a.0 Memory controller: Compro Computer Services, Inc. Device 4360 
(rev 4d)

Again, I can go between both pci chassis as long as I am doing writes. 
Only reads do not work.

I can send the AMD-990FX-990X-970-Register-Programming-Requirements if 
you would like. It's available for download on AMD web site. Let me know.

Regards
Mark



[-- Attachment #2: d.lspci --]
[-- Type: text/plain, Size: 85072 bytes --]

00:00.0 Host bridge: Advanced Micro Devices, Inc. [AMD/ATI] RD890 PCI to PCI bridge (external gfx0 port B) (rev 02)
	Subsystem: Advanced Micro Devices, Inc. [AMD/ATI] RD890 PCI to PCI bridge (external gfx0 port B)
	Control: I/O- Mem+ BusMaster- SpecCycle- MemWINV- VGASnoop- ParErr- Stepping- SERR- FastB2B- DisINTx-
	Status: Cap+ 66MHz- UDF- FastB2B- ParErr- DEVSEL=fast >TAbort- <TAbort- <MAbort+ >SERR- <PERR- INTx-
	Capabilities: [f0] HyperTransport: MSI Mapping Enable+ Fixed+
	Capabilities: [c4] HyperTransport: Slave or Primary Interface
		Command: BaseUnitID=0 UnitCnt=20 MastHost- DefDir- DUL-
		Link Control 0: CFlE- CST- CFE- <LkFail- Init+ EOC- TXO- <CRCErr=0 IsocEn- LSEn- ExtCTL- 64b+
		Link Config 0: MLWI=16bit DwFcIn- MLWO=16bit DwFcOut- LWI=16bit DwFcInEn- LWO=16bit DwFcOutEn-
		Link Control 1: CFlE- CST- CFE- <LkFail+ Init- EOC+ TXO+ <CRCErr=0 IsocEn- LSEn- ExtCTL- 64b-
		Link Config 1: MLWI=8bit DwFcIn- MLWO=8bit DwFcOut- LWI=8bit DwFcInEn- LWO=8bit DwFcOutEn-
		Revision ID: 3.00
		Link Frequency 0: [e]
		Link Error 0: <Prot- <Ovfl- <EOC- CTLTm-
		Link Frequency Capability 0: 200MHz+ 300MHz- 400MHz+ 500MHz- 600MHz+ 800MHz+ 1.0GHz+ 1.2GHz+ 1.4GHz- 1.6GHz- Vend-
		Feature Capability: IsocFC+ LDTSTOP+ CRCTM- ECTLT- 64bA+ UIDRD-
		Link Frequency 1: 200MHz
		Link Error 1: <Prot- <Ovfl- <EOC- CTLTm-
		Link Frequency Capability 1: 200MHz- 300MHz- 400MHz- 500MHz- 600MHz- 800MHz- 1.0GHz- 1.2GHz- 1.4GHz- 1.6GHz- Vend-
		Error Handling: PFlE- OFlE- PFE- OFE- EOCFE- RFE- CRCFE- SERRFE- CF- RE- PNFE- ONFE- EOCNFE- RNFE- CRCNFE- SERRNFE-
		Prefetchable memory behind bridge Upper: 00-00
		Bus Number: 00
	Capabilities: [40] HyperTransport: Retry Mode
	Capabilities: [54] HyperTransport: UnitID Clumping
	Capabilities: [9c] HyperTransport: #1a
	Capabilities: [70] MSI: Enable- Count=1/4 Maskable- 64bit-
		Address: 00000000  Data: 0000

00:00.2 IOMMU: Advanced Micro Devices, Inc. [AMD/ATI] RD990 I/O Memory Management Unit (IOMMU)
	Subsystem: Advanced Micro Devices, Inc. [AMD/ATI] RD990 I/O Memory Management Unit (IOMMU)
	Control: I/O- Mem- BusMaster- SpecCycle- MemWINV- VGASnoop- ParErr- Stepping- SERR- FastB2B- DisINTx+
	Status: Cap+ 66MHz- UDF- FastB2B- ParErr- DEVSEL=fast >TAbort- <TAbort- <MAbort- >SERR- <PERR- INTx-
	Interrupt: pin A routed to IRQ 26
	Capabilities: [40] Secure device <?>
	Capabilities: [54] MSI: Enable+ Count=1/1 Maskable- 64bit+
		Address: 00000000fee7f00c  Data: 4142
	Capabilities: [64] HyperTransport: MSI Mapping Enable+ Fixed+

00:02.0 PCI bridge: Advanced Micro Devices, Inc. [AMD/ATI] RD890 PCI to PCI bridge (PCI express gpp port B) (prog-if 00 [Normal decode])
	Control: I/O+ Mem+ BusMaster+ SpecCycle- MemWINV- VGASnoop- ParErr- Stepping- SERR- FastB2B- DisINTx-
	Status: Cap+ 66MHz- UDF- FastB2B- ParErr- DEVSEL=fast >TAbort- <TAbort- <MAbort- >SERR- <PERR- INTx-
	Latency: 0, Cache Line Size: 64 bytes
	Bus: primary=00, secondary=01, subordinate=01, sec-latency=0
	I/O behind bridge: 0000e000-0000efff
	Memory behind bridge: d9000000-db0fffff
	Prefetchable memory behind bridge: 00000000b0000000-00000000bfffffff
	Secondary status: 66MHz- FastB2B- ParErr- DEVSEL=fast >TAbort- <TAbort- <MAbort+ <SERR- <PERR-
	BridgeCtl: Parity- SERR- NoISA- VGA+ MAbort- >Reset- FastB2B-
		PriDiscTmr- SecDiscTmr- DiscTmrStat- DiscTmrSERREn-
	Capabilities: [50] Power Management version 3
		Flags: PMEClk- DSI- D1- D2- AuxCurrent=0mA PME(D0+,D1-,D2-,D3hot+,D3cold+)
		Status: D0 NoSoftRst- PME-Enable- DSel=0 DScale=0 PME-
	Capabilities: [58] Express (v2) Root Port (Slot+), MSI 00
		DevCap:	MaxPayload 128 bytes, PhantFunc 0
			ExtTag+ RBE+
		DevCtl:	Report errors: Correctable- Non-Fatal- Fatal- Unsupported-
			RlxdOrd- ExtTag- PhantFunc- AuxPwr- NoSnoop+
			MaxPayload 128 bytes, MaxReadReq 128 bytes
		DevSta:	CorrErr- UncorrErr- FatalErr- UnsuppReq- AuxPwr- TransPend-
		LnkCap:	Port #0, Speed 5GT/s, Width x16, ASPM L0s L1, Exit Latency L0s <1us, L1 <8us
			ClockPM- Surprise- LLActRep+ BwNot+
		LnkCtl:	ASPM Disabled; RCB 64 bytes Disabled- CommClk+
			ExtSynch- ClockPM- AutWidDis- BWInt- AutBWInt-
		LnkSta:	Speed 2.5GT/s, Width x16, TrErr- Train- SlotClk+ DLActive+ BWMgmt+ ABWMgmt-
		SltCap:	AttnBtn- PwrCtrl- MRL- AttnInd- PwrInd- HotPlug- Surprise-
			Slot #2, PowerLimit 75.000W; Interlock- NoCompl+
		SltCtl:	Enable: AttnBtn- PwrFlt- MRL- PresDet- CmdCplt- HPIrq- LinkChg-
			Control: AttnInd Unknown, PwrInd Unknown, Power- Interlock-
		SltSta:	Status: AttnBtn- PowerFlt- MRL- CmdCplt- PresDet+ Interlock-
			Changed: MRL- PresDet- LinkState-
		RootCtl: ErrCorrectable- ErrNon-Fatal- ErrFatal- PMEIntEna- CRSVisible+
		RootCap: CRSVisible-
		RootSta: PME ReqID 0000, PMEStatus- PMEPending-
		DevCap2: Completion Timeout: Range ABCD, TimeoutDis+, LTR-, OBFF Not Supported ARIFwd+
		DevCtl2: Completion Timeout: 65ms to 210ms, TimeoutDis-, LTR-, OBFF Disabled ARIFwd-
		LnkCtl2: Target Link Speed: 5GT/s, EnterCompliance- SpeedDis-
			 Transmit Margin: Normal Operating Range, EnterModifiedCompliance- ComplianceSOS-
			 Compliance De-emphasis: -6dB
		LnkSta2: Current De-emphasis Level: -3.5dB, EqualizationComplete-, EqualizationPhase1-
			 EqualizationPhase2-, EqualizationPhase3-, LinkEqualizationRequest-
	Capabilities: [a0] MSI: Enable- Count=1/1 Maskable- 64bit-
		Address: 00000000  Data: 0000
	Capabilities: [b0] Subsystem: Advanced Micro Devices, Inc. [AMD/ATI] Device 5a14
	Capabilities: [b8] HyperTransport: MSI Mapping Enable+ Fixed+
	Capabilities: [100 v1] Vendor Specific Information: ID=0001 Rev=1 Len=010 <?>
	Capabilities: [190 v1] Access Control Services
		ACSCap:	SrcValid+ TransBlk+ ReqRedir+ CmpltRedir+ UpstreamFwd+ EgressCtrl- DirectTrans+
		ACSCtl:	SrcValid+ TransBlk- ReqRedir+ CmpltRedir+ UpstreamFwd+ EgressCtrl- DirectTrans-
	Kernel driver in use: pcieport
	Kernel modules: shpchp

00:04.0 PCI bridge: Advanced Micro Devices, Inc. [AMD/ATI] RD890 PCI to PCI bridge (PCI express gpp port D) (prog-if 00 [Normal decode])
	Control: I/O+ Mem+ BusMaster+ SpecCycle- MemWINV- VGASnoop- ParErr- Stepping- SERR- FastB2B- DisINTx-
	Status: Cap+ 66MHz- UDF- FastB2B- ParErr- DEVSEL=fast >TAbort- <TAbort- <MAbort- >SERR- <PERR- INTx-
	Latency: 0, Cache Line Size: 64 bytes
	Bus: primary=00, secondary=02, subordinate=05, sec-latency=0
	I/O behind bridge: 00006000-00007fff
	Memory behind bridge: db800000-dd9fffff
	Prefetchable memory behind bridge: 00000000c0000000-00000000c3ffffff
	Secondary status: 66MHz- FastB2B- ParErr- DEVSEL=fast >TAbort- <TAbort- <MAbort+ <SERR- <PERR-
	BridgeCtl: Parity- SERR- NoISA- VGA- MAbort- >Reset- FastB2B-
		PriDiscTmr- SecDiscTmr- DiscTmrStat- DiscTmrSERREn-
	Capabilities: [50] Power Management version 3
		Flags: PMEClk- DSI- D1- D2- AuxCurrent=0mA PME(D0+,D1-,D2-,D3hot+,D3cold+)
		Status: D0 NoSoftRst- PME-Enable- DSel=0 DScale=0 PME-
	Capabilities: [58] Express (v2) Root Port (Slot+), MSI 00
		DevCap:	MaxPayload 128 bytes, PhantFunc 0
			ExtTag+ RBE+
		DevCtl:	Report errors: Correctable- Non-Fatal- Fatal- Unsupported-
			RlxdOrd- ExtTag- PhantFunc- AuxPwr- NoSnoop+
			MaxPayload 128 bytes, MaxReadReq 128 bytes
		DevSta:	CorrErr- UncorrErr- FatalErr- UnsuppReq- AuxPwr- TransPend-
		LnkCap:	Port #0, Speed 5GT/s, Width x4, ASPM L0s L1, Exit Latency L0s <1us, L1 <8us
			ClockPM- Surprise- LLActRep+ BwNot+
		LnkCtl:	ASPM Disabled; RCB 64 bytes Disabled- CommClk-
			ExtSynch- ClockPM- AutWidDis- BWInt- AutBWInt-
		LnkSta:	Speed 2.5GT/s, Width x1, TrErr- Train- SlotClk+ DLActive+ BWMgmt- ABWMgmt-
		SltCap:	AttnBtn- PwrCtrl- MRL- AttnInd- PwrInd- HotPlug- Surprise-
			Slot #4, PowerLimit 75.000W; Interlock- NoCompl+
		SltCtl:	Enable: AttnBtn- PwrFlt- MRL- PresDet- CmdCplt- HPIrq- LinkChg-
			Control: AttnInd Unknown, PwrInd Unknown, Power- Interlock-
		SltSta:	Status: AttnBtn- PowerFlt- MRL- CmdCplt- PresDet+ Interlock-
			Changed: MRL- PresDet- LinkState-
		RootCtl: ErrCorrectable- ErrNon-Fatal- ErrFatal- PMEIntEna- CRSVisible+
		RootCap: CRSVisible-
		RootSta: PME ReqID 0000, PMEStatus- PMEPending-
		DevCap2: Completion Timeout: Range ABCD, TimeoutDis+, LTR-, OBFF Not Supported ARIFwd+
		DevCtl2: Completion Timeout: 65ms to 210ms, TimeoutDis-, LTR-, OBFF Disabled ARIFwd-
		LnkCtl2: Target Link Speed: 5GT/s, EnterCompliance- SpeedDis-
			 Transmit Margin: Normal Operating Range, EnterModifiedCompliance- ComplianceSOS-
			 Compliance De-emphasis: -6dB
		LnkSta2: Current De-emphasis Level: -3.5dB, EqualizationComplete-, EqualizationPhase1-
			 EqualizationPhase2-, EqualizationPhase3-, LinkEqualizationRequest-
	Capabilities: [a0] MSI: Enable- Count=1/1 Maskable- 64bit-
		Address: 00000000  Data: 0000
	Capabilities: [b0] Subsystem: Advanced Micro Devices, Inc. [AMD/ATI] Device 5a14
	Capabilities: [b8] HyperTransport: MSI Mapping Enable+ Fixed+
	Capabilities: [100 v1] Vendor Specific Information: ID=0001 Rev=1 Len=010 <?>
	Capabilities: [190 v1] Access Control Services
		ACSCap:	SrcValid+ TransBlk+ ReqRedir+ CmpltRedir+ UpstreamFwd+ EgressCtrl- DirectTrans+
		ACSCtl:	SrcValid+ TransBlk- ReqRedir+ CmpltRedir+ UpstreamFwd+ EgressCtrl- DirectTrans-
	Kernel driver in use: pcieport
	Kernel modules: shpchp

00:09.0 PCI bridge: Advanced Micro Devices, Inc. [AMD/ATI] RD890 PCI to PCI bridge (PCI express gpp port H) (prog-if 00 [Normal decode])
	Control: I/O+ Mem+ BusMaster+ SpecCycle- MemWINV- VGASnoop- ParErr- Stepping- SERR- FastB2B- DisINTx-
	Status: Cap+ 66MHz- UDF- FastB2B- ParErr- DEVSEL=fast >TAbort- <TAbort- <MAbort- >SERR- <PERR- INTx-
	Latency: 0, Cache Line Size: 64 bytes
	Bus: primary=00, secondary=06, subordinate=06, sec-latency=0
	Memory behind bridge: de100000-de1fffff
	Secondary status: 66MHz- FastB2B- ParErr- DEVSEL=fast >TAbort- <TAbort- <MAbort+ <SERR- <PERR-
	BridgeCtl: Parity- SERR- NoISA- VGA- MAbort- >Reset- FastB2B-
		PriDiscTmr- SecDiscTmr- DiscTmrStat- DiscTmrSERREn-
	Capabilities: [50] Power Management version 3
		Flags: PMEClk- DSI- D1- D2- AuxCurrent=0mA PME(D0+,D1-,D2-,D3hot+,D3cold+)
		Status: D0 NoSoftRst- PME-Enable- DSel=0 DScale=0 PME-
	Capabilities: [58] Express (v2) Root Port (Slot+), MSI 00
		DevCap:	MaxPayload 128 bytes, PhantFunc 0
			ExtTag+ RBE+
		DevCtl:	Report errors: Correctable- Non-Fatal- Fatal- Unsupported-
			RlxdOrd- ExtTag- PhantFunc- AuxPwr- NoSnoop+
			MaxPayload 128 bytes, MaxReadReq 128 bytes
		DevSta:	CorrErr- UncorrErr- FatalErr- UnsuppReq- AuxPwr- TransPend-
		LnkCap:	Port #1, Speed 5GT/s, Width x2, ASPM L0s L1, Exit Latency L0s <1us, L1 <8us
			ClockPM- Surprise- LLActRep+ BwNot+
		LnkCtl:	ASPM Disabled; RCB 64 bytes Disabled- CommClk+
			ExtSynch- ClockPM- AutWidDis- BWInt- AutBWInt-
		LnkSta:	Speed 5GT/s, Width x1, TrErr- Train- SlotClk+ DLActive+ BWMgmt+ ABWMgmt+
		SltCap:	AttnBtn- PwrCtrl- MRL- AttnInd- PwrInd- HotPlug- Surprise-
			Slot #9, PowerLimit 75.000W; Interlock- NoCompl+
		SltCtl:	Enable: AttnBtn- PwrFlt- MRL- PresDet- CmdCplt- HPIrq- LinkChg-
			Control: AttnInd Unknown, PwrInd Unknown, Power- Interlock-
		SltSta:	Status: AttnBtn- PowerFlt- MRL- CmdCplt- PresDet+ Interlock-
			Changed: MRL- PresDet- LinkState-
		RootCtl: ErrCorrectable- ErrNon-Fatal- ErrFatal- PMEIntEna- CRSVisible+
		RootCap: CRSVisible-
		RootSta: PME ReqID 0000, PMEStatus- PMEPending-
		DevCap2: Completion Timeout: Range ABCD, TimeoutDis+, LTR-, OBFF Not Supported ARIFwd+
		DevCtl2: Completion Timeout: 65ms to 210ms, TimeoutDis-, LTR-, OBFF Disabled ARIFwd-
		LnkCtl2: Target Link Speed: 5GT/s, EnterCompliance- SpeedDis-
			 Transmit Margin: Normal Operating Range, EnterModifiedCompliance- ComplianceSOS-
			 Compliance De-emphasis: -6dB
		LnkSta2: Current De-emphasis Level: -3.5dB, EqualizationComplete-, EqualizationPhase1-
			 EqualizationPhase2-, EqualizationPhase3-, LinkEqualizationRequest-
	Capabilities: [a0] MSI: Enable- Count=1/1 Maskable- 64bit-
		Address: 00000000  Data: 0000
	Capabilities: [b0] Subsystem: Advanced Micro Devices, Inc. [AMD/ATI] Device 5a14
	Capabilities: [b8] HyperTransport: MSI Mapping Enable+ Fixed+
	Capabilities: [100 v1] Vendor Specific Information: ID=0001 Rev=1 Len=010 <?>
	Capabilities: [190 v1] Access Control Services
		ACSCap:	SrcValid+ TransBlk+ ReqRedir+ CmpltRedir+ UpstreamFwd+ EgressCtrl- DirectTrans+
		ACSCtl:	SrcValid+ TransBlk- ReqRedir+ CmpltRedir+ UpstreamFwd+ EgressCtrl- DirectTrans-
	Kernel driver in use: pcieport
	Kernel modules: shpchp

00:0a.0 PCI bridge: Advanced Micro Devices, Inc. [AMD/ATI] RD890 PCI to PCI bridge (external gfx1 port A) (prog-if 00 [Normal decode])
	Control: I/O+ Mem+ BusMaster+ SpecCycle- MemWINV- VGASnoop- ParErr- Stepping- SERR- FastB2B- DisINTx-
	Status: Cap+ 66MHz- UDF- FastB2B- ParErr- DEVSEL=fast >TAbort- <TAbort- <MAbort- >SERR- <PERR- INTx-
	Latency: 0, Cache Line Size: 64 bytes
	Bus: primary=00, secondary=07, subordinate=07, sec-latency=0
	I/O behind bridge: 0000d000-0000dfff
	Memory behind bridge: de000000-de0fffff
	Secondary status: 66MHz- FastB2B- ParErr- DEVSEL=fast >TAbort- <TAbort- <MAbort+ <SERR- <PERR-
	BridgeCtl: Parity- SERR- NoISA- VGA- MAbort- >Reset- FastB2B-
		PriDiscTmr- SecDiscTmr- DiscTmrStat- DiscTmrSERREn-
	Capabilities: [50] Power Management version 3
		Flags: PMEClk- DSI- D1- D2- AuxCurrent=0mA PME(D0+,D1-,D2-,D3hot+,D3cold+)
		Status: D0 NoSoftRst- PME-Enable- DSel=0 DScale=0 PME-
	Capabilities: [58] Express (v2) Root Port (Slot+), MSI 00
		DevCap:	MaxPayload 128 bytes, PhantFunc 0
			ExtTag+ RBE+
		DevCtl:	Report errors: Correctable- Non-Fatal- Fatal- Unsupported-
			RlxdOrd- ExtTag- PhantFunc- AuxPwr- NoSnoop+
			MaxPayload 128 bytes, MaxReadReq 128 bytes
		DevSta:	CorrErr- UncorrErr- FatalErr- UnsuppReq- AuxPwr- TransPend-
		LnkCap:	Port #2, Speed 5GT/s, Width x2, ASPM L0s L1, Exit Latency L0s <1us, L1 <8us
			ClockPM- Surprise- LLActRep+ BwNot+
		LnkCtl:	ASPM Disabled; RCB 64 bytes Disabled- CommClk+
			ExtSynch- ClockPM- AutWidDis- BWInt- AutBWInt-
		LnkSta:	Speed 5GT/s, Width x1, TrErr- Train- SlotClk+ DLActive+ BWMgmt+ ABWMgmt+
		SltCap:	AttnBtn- PwrCtrl- MRL- AttnInd- PwrInd- HotPlug- Surprise-
			Slot #10, PowerLimit 75.000W; Interlock- NoCompl+
		SltCtl:	Enable: AttnBtn- PwrFlt- MRL- PresDet- CmdCplt- HPIrq- LinkChg-
			Control: AttnInd Unknown, PwrInd Unknown, Power- Interlock-
		SltSta:	Status: AttnBtn- PowerFlt- MRL- CmdCplt- PresDet+ Interlock-
			Changed: MRL- PresDet- LinkState-
		RootCtl: ErrCorrectable- ErrNon-Fatal- ErrFatal- PMEIntEna- CRSVisible+
		RootCap: CRSVisible-
		RootSta: PME ReqID 0000, PMEStatus- PMEPending-
		DevCap2: Completion Timeout: Range ABCD, TimeoutDis+, LTR-, OBFF Not Supported ARIFwd+
		DevCtl2: Completion Timeout: 65ms to 210ms, TimeoutDis-, LTR-, OBFF Disabled ARIFwd-
		LnkCtl2: Target Link Speed: 5GT/s, EnterCompliance- SpeedDis-
			 Transmit Margin: Normal Operating Range, EnterModifiedCompliance- ComplianceSOS-
			 Compliance De-emphasis: -6dB
		LnkSta2: Current De-emphasis Level: -3.5dB, EqualizationComplete-, EqualizationPhase1-
			 EqualizationPhase2-, EqualizationPhase3-, LinkEqualizationRequest-
	Capabilities: [a0] MSI: Enable- Count=1/1 Maskable- 64bit-
		Address: 00000000  Data: 0000
	Capabilities: [b0] Subsystem: Advanced Micro Devices, Inc. [AMD/ATI] Device 5a14
	Capabilities: [b8] HyperTransport: MSI Mapping Enable+ Fixed+
	Capabilities: [100 v1] Vendor Specific Information: ID=0001 Rev=1 Len=010 <?>
	Capabilities: [190 v1] Access Control Services
		ACSCap:	SrcValid+ TransBlk+ ReqRedir+ CmpltRedir+ UpstreamFwd+ EgressCtrl- DirectTrans+
		ACSCtl:	SrcValid+ TransBlk- ReqRedir+ CmpltRedir+ UpstreamFwd+ EgressCtrl- DirectTrans-
	Kernel driver in use: pcieport
	Kernel modules: shpchp

00:0b.0 PCI bridge: Advanced Micro Devices, Inc. [AMD/ATI] RD890 PCI to PCI bridge (NB-SB link) (prog-if 00 [Normal decode])
	Control: I/O+ Mem+ BusMaster+ SpecCycle- MemWINV- VGASnoop- ParErr- Stepping- SERR- FastB2B- DisINTx-
	Status: Cap+ 66MHz- UDF- FastB2B- ParErr- DEVSEL=fast >TAbort- <TAbort- <MAbort- >SERR- <PERR- INTx-
	Latency: 0, Cache Line Size: 64 bytes
	Bus: primary=00, secondary=08, subordinate=0e, sec-latency=0
	I/O behind bridge: 0000c000-0000cfff
	Memory behind bridge: d0000000-d84fffff
	Prefetchable memory behind bridge: 00000000c4000000-00000000c6ffffff
	Secondary status: 66MHz- FastB2B- ParErr- DEVSEL=fast >TAbort- <TAbort- <MAbort+ <SERR- <PERR-
	BridgeCtl: Parity- SERR- NoISA- VGA- MAbort- >Reset- FastB2B-
		PriDiscTmr- SecDiscTmr- DiscTmrStat- DiscTmrSERREn-
	Capabilities: [50] Power Management version 3
		Flags: PMEClk- DSI- D1- D2- AuxCurrent=0mA PME(D0+,D1-,D2-,D3hot+,D3cold+)
		Status: D0 NoSoftRst- PME-Enable- DSel=0 DScale=0 PME-
	Capabilities: [58] Express (v2) Root Port (Slot+), MSI 00
		DevCap:	MaxPayload 128 bytes, PhantFunc 0
			ExtTag+ RBE+
		DevCtl:	Report errors: Correctable- Non-Fatal- Fatal- Unsupported-
			RlxdOrd- ExtTag- PhantFunc- AuxPwr- NoSnoop+
			MaxPayload 128 bytes, MaxReadReq 128 bytes
		DevSta:	CorrErr- UncorrErr- FatalErr- UnsuppReq- AuxPwr- TransPend-
		LnkCap:	Port #0, Speed 5GT/s, Width x16, ASPM L0s L1, Exit Latency L0s <1us, L1 <8us
			ClockPM- Surprise- LLActRep+ BwNot+
		LnkCtl:	ASPM Disabled; RCB 64 bytes Disabled- CommClk-
			ExtSynch- ClockPM- AutWidDis- BWInt- AutBWInt-
		LnkSta:	Speed 2.5GT/s, Width x1, TrErr- Train- SlotClk+ DLActive+ BWMgmt- ABWMgmt-
		SltCap:	AttnBtn- PwrCtrl- MRL- AttnInd- PwrInd- HotPlug- Surprise-
			Slot #11, PowerLimit 75.000W; Interlock- NoCompl+
		SltCtl:	Enable: AttnBtn- PwrFlt- MRL- PresDet- CmdCplt- HPIrq- LinkChg-
			Control: AttnInd Unknown, PwrInd Unknown, Power- Interlock-
		SltSta:	Status: AttnBtn- PowerFlt- MRL- CmdCplt- PresDet+ Interlock-
			Changed: MRL- PresDet- LinkState-
		RootCtl: ErrCorrectable- ErrNon-Fatal- ErrFatal- PMEIntEna- CRSVisible+
		RootCap: CRSVisible-
		RootSta: PME ReqID 0000, PMEStatus- PMEPending-
		DevCap2: Completion Timeout: Range ABCD, TimeoutDis+, LTR-, OBFF Not Supported ARIFwd+
		DevCtl2: Completion Timeout: 65ms to 210ms, TimeoutDis-, LTR-, OBFF Disabled ARIFwd-
		LnkCtl2: Target Link Speed: 5GT/s, EnterCompliance- SpeedDis-
			 Transmit Margin: Normal Operating Range, EnterModifiedCompliance- ComplianceSOS-
			 Compliance De-emphasis: -6dB
		LnkSta2: Current De-emphasis Level: -3.5dB, EqualizationComplete-, EqualizationPhase1-
			 EqualizationPhase2-, EqualizationPhase3-, LinkEqualizationRequest-
	Capabilities: [a0] MSI: Enable- Count=1/1 Maskable- 64bit-
		Address: 00000000  Data: 0000
	Capabilities: [b0] Subsystem: Advanced Micro Devices, Inc. [AMD/ATI] Device 5a14
	Capabilities: [b8] HyperTransport: MSI Mapping Enable+ Fixed+
	Capabilities: [100 v1] Vendor Specific Information: ID=0001 Rev=1 Len=010 <?>
	Capabilities: [190 v1] Access Control Services
		ACSCap:	SrcValid+ TransBlk+ ReqRedir+ CmpltRedir+ UpstreamFwd+ EgressCtrl- DirectTrans+
		ACSCtl:	SrcValid+ TransBlk- ReqRedir+ CmpltRedir+ UpstreamFwd+ EgressCtrl- DirectTrans-
	Kernel driver in use: pcieport
	Kernel modules: shpchp

00:0d.0 PCI bridge: Advanced Micro Devices, Inc. [AMD/ATI] RD890 PCI to PCI bridge (external gfx1 port B) (prog-if 00 [Normal decode])
	Control: I/O+ Mem+ BusMaster+ SpecCycle- MemWINV- VGASnoop- ParErr- Stepping- SERR- FastB2B- DisINTx-
	Status: Cap+ 66MHz- UDF- FastB2B- ParErr- DEVSEL=fast >TAbort- <TAbort- <MAbort- >SERR- <PERR- INTx-
	Latency: 0, Cache Line Size: 64 bytes
	Bus: primary=00, secondary=0f, subordinate=10, sec-latency=0
	I/O behind bridge: 0000b000-0000bfff
	Memory behind bridge: ddb00000-ddcfffff
	Secondary status: 66MHz- FastB2B- ParErr- DEVSEL=fast >TAbort- <TAbort- <MAbort+ <SERR- <PERR-
	BridgeCtl: Parity- SERR- NoISA- VGA- MAbort- >Reset- FastB2B-
		PriDiscTmr- SecDiscTmr- DiscTmrStat- DiscTmrSERREn-
	Capabilities: [50] Power Management version 3
		Flags: PMEClk- DSI- D1- D2- AuxCurrent=0mA PME(D0+,D1-,D2-,D3hot+,D3cold+)
		Status: D0 NoSoftRst- PME-Enable- DSel=0 DScale=0 PME-
	Capabilities: [58] Express (v2) Root Port (Slot+), MSI 00
		DevCap:	MaxPayload 128 bytes, PhantFunc 0
			ExtTag+ RBE+
		DevCtl:	Report errors: Correctable- Non-Fatal- Fatal- Unsupported-
			RlxdOrd- ExtTag- PhantFunc- AuxPwr- NoSnoop+
			MaxPayload 128 bytes, MaxReadReq 128 bytes
		DevSta:	CorrErr- UncorrErr- FatalErr- UnsuppReq- AuxPwr- TransPend-
		LnkCap:	Port #0, Speed 5GT/s, Width x4, ASPM L0s L1, Exit Latency L0s <1us, L1 <8us
			ClockPM- Surprise- LLActRep+ BwNot+
		LnkCtl:	ASPM Disabled; RCB 64 bytes Disabled- CommClk-
			ExtSynch- ClockPM- AutWidDis- BWInt- AutBWInt-
		LnkSta:	Speed 2.5GT/s, Width x1, TrErr- Train- SlotClk+ DLActive+ BWMgmt- ABWMgmt-
		SltCap:	AttnBtn- PwrCtrl- MRL- AttnInd- PwrInd- HotPlug- Surprise-
			Slot #13, PowerLimit 75.000W; Interlock- NoCompl+
		SltCtl:	Enable: AttnBtn- PwrFlt- MRL- PresDet- CmdCplt- HPIrq- LinkChg-
			Control: AttnInd Unknown, PwrInd Unknown, Power- Interlock-
		SltSta:	Status: AttnBtn- PowerFlt- MRL- CmdCplt- PresDet+ Interlock-
			Changed: MRL- PresDet- LinkState-
		RootCtl: ErrCorrectable- ErrNon-Fatal- ErrFatal- PMEIntEna- CRSVisible+
		RootCap: CRSVisible-
		RootSta: PME ReqID 0000, PMEStatus- PMEPending-
		DevCap2: Completion Timeout: Range ABCD, TimeoutDis+, LTR-, OBFF Not Supported ARIFwd+
		DevCtl2: Completion Timeout: 65ms to 210ms, TimeoutDis-, LTR-, OBFF Disabled ARIFwd-
		LnkCtl2: Target Link Speed: 5GT/s, EnterCompliance- SpeedDis-
			 Transmit Margin: Normal Operating Range, EnterModifiedCompliance- ComplianceSOS-
			 Compliance De-emphasis: -6dB
		LnkSta2: Current De-emphasis Level: -3.5dB, EqualizationComplete-, EqualizationPhase1-
			 EqualizationPhase2-, EqualizationPhase3-, LinkEqualizationRequest-
	Capabilities: [a0] MSI: Enable- Count=1/1 Maskable- 64bit-
		Address: 00000000  Data: 0000
	Capabilities: [b0] Subsystem: Advanced Micro Devices, Inc. [AMD/ATI] Device 5a14
	Capabilities: [b8] HyperTransport: MSI Mapping Enable+ Fixed+
	Capabilities: [100 v1] Vendor Specific Information: ID=0001 Rev=1 Len=010 <?>
	Capabilities: [190 v1] Access Control Services
		ACSCap:	SrcValid+ TransBlk+ ReqRedir+ CmpltRedir+ UpstreamFwd+ EgressCtrl- DirectTrans+
		ACSCtl:	SrcValid+ TransBlk- ReqRedir+ CmpltRedir+ UpstreamFwd+ EgressCtrl- DirectTrans-
	Kernel driver in use: pcieport
	Kernel modules: shpchp

00:11.0 SATA controller: Advanced Micro Devices, Inc. [AMD/ATI] SB7x0/SB8x0/SB9x0 SATA Controller [AHCI mode] (rev 40) (prog-if 01 [AHCI 1.0])
	Subsystem: Gigabyte Technology Co., Ltd Device b002
	Control: I/O+ Mem+ BusMaster+ SpecCycle- MemWINV- VGASnoop- ParErr- Stepping- SERR- FastB2B- DisINTx-
	Status: Cap+ 66MHz+ UDF- FastB2B- ParErr- DEVSEL=medium >TAbort- <TAbort- <MAbort- >SERR- <PERR- INTx-
	Latency: 32
	Interrupt: pin A routed to IRQ 19
	Region 0: I/O ports at f090 [size=8]
	Region 1: I/O ports at f080 [size=4]
	Region 2: I/O ports at f070 [size=8]
	Region 3: I/O ports at f060 [size=4]
	Region 4: I/O ports at f050 [size=16]
	Region 5: Memory at d850b000 (32-bit, non-prefetchable) [size=1K]
	Capabilities: [70] SATA HBA v1.0 InCfgSpace
	Capabilities: [a4] PCI Advanced Features
		AFCap: TP+ FLR+
		AFCtrl: FLR-
		AFStatus: TP-
	Kernel driver in use: ahci

00:12.0 USB controller: Advanced Micro Devices, Inc. [AMD/ATI] SB7x0/SB8x0/SB9x0 USB OHCI0 Controller (prog-if 10 [OHCI])
	Subsystem: Gigabyte Technology Co., Ltd Device 5004
	Control: I/O+ Mem+ BusMaster+ SpecCycle- MemWINV- VGASnoop- ParErr- Stepping- SERR- FastB2B- DisINTx-
	Status: Cap- 66MHz+ UDF- FastB2B+ ParErr- DEVSEL=medium >TAbort- <TAbort- <MAbort- >SERR- <PERR- INTx-
	Latency: 32, Cache Line Size: 64 bytes
	Interrupt: pin A routed to IRQ 18
	Region 0: Memory at d850a000 (32-bit, non-prefetchable) [size=4K]
	Kernel driver in use: ohci-pci
	Kernel modules: ohci_pci

00:12.2 USB controller: Advanced Micro Devices, Inc. [AMD/ATI] SB7x0/SB8x0/SB9x0 USB EHCI Controller (prog-if 20 [EHCI])
	Subsystem: Gigabyte Technology Co., Ltd Device 5004
	Control: I/O+ Mem+ BusMaster+ SpecCycle- MemWINV+ VGASnoop- ParErr- Stepping- SERR- FastB2B- DisINTx-
	Status: Cap+ 66MHz+ UDF- FastB2B+ ParErr- DEVSEL=medium >TAbort- <TAbort- <MAbort- >SERR- <PERR- INTx-
	Latency: 32, Cache Line Size: 64 bytes
	Interrupt: pin B routed to IRQ 17
	Region 0: Memory at d8509000 (32-bit, non-prefetchable) [size=256]
	Capabilities: [c0] Power Management version 2
		Flags: PMEClk- DSI- D1+ D2+ AuxCurrent=0mA PME(D0+,D1+,D2+,D3hot+,D3cold-)
		Status: D0 NoSoftRst- PME-Enable- DSel=0 DScale=0 PME-
		Bridge: PM- B3+
	Capabilities: [e4] Debug port: BAR=1 offset=00e0
	Kernel driver in use: ehci-pci

00:13.0 USB controller: Advanced Micro Devices, Inc. [AMD/ATI] SB7x0/SB8x0/SB9x0 USB OHCI0 Controller (prog-if 10 [OHCI])
	Subsystem: Gigabyte Technology Co., Ltd Device 5004
	Control: I/O+ Mem+ BusMaster+ SpecCycle- MemWINV- VGASnoop- ParErr- Stepping- SERR- FastB2B- DisINTx-
	Status: Cap- 66MHz+ UDF- FastB2B+ ParErr- DEVSEL=medium >TAbort- <TAbort- <MAbort- >SERR- <PERR- INTx-
	Latency: 32, Cache Line Size: 64 bytes
	Interrupt: pin A routed to IRQ 18
	Region 0: Memory at d8508000 (32-bit, non-prefetchable) [size=4K]
	Kernel driver in use: ohci-pci
	Kernel modules: ohci_pci

00:13.2 USB controller: Advanced Micro Devices, Inc. [AMD/ATI] SB7x0/SB8x0/SB9x0 USB EHCI Controller (prog-if 20 [EHCI])
	Subsystem: Gigabyte Technology Co., Ltd Device 5004
	Control: I/O+ Mem+ BusMaster+ SpecCycle- MemWINV+ VGASnoop- ParErr- Stepping- SERR- FastB2B- DisINTx-
	Status: Cap+ 66MHz+ UDF- FastB2B+ ParErr- DEVSEL=medium >TAbort- <TAbort- <MAbort- >SERR- <PERR- INTx-
	Latency: 32, Cache Line Size: 64 bytes
	Interrupt: pin B routed to IRQ 17
	Region 0: Memory at d8507000 (32-bit, non-prefetchable) [size=256]
	Capabilities: [c0] Power Management version 2
		Flags: PMEClk- DSI- D1+ D2+ AuxCurrent=0mA PME(D0+,D1+,D2+,D3hot+,D3cold-)
		Status: D0 NoSoftRst- PME-Enable- DSel=0 DScale=0 PME-
		Bridge: PM- B3+
	Capabilities: [e4] Debug port: BAR=1 offset=00e0
	Kernel driver in use: ehci-pci

00:14.0 SMBus: Advanced Micro Devices, Inc. [AMD/ATI] SBx00 SMBus Controller (rev 42)
	Subsystem: Advanced Micro Devices, Inc. [AMD/ATI] SBx00 SMBus Controller
	Control: I/O+ Mem+ BusMaster- SpecCycle- MemWINV- VGASnoop- ParErr- Stepping- SERR- FastB2B- DisINTx+
	Status: Cap- 66MHz+ UDF- FastB2B- ParErr- DEVSEL=medium >TAbort- <TAbort- <MAbort- >SERR- <PERR- INTx-
	Kernel driver in use: piix4_smbus
	Kernel modules: i2c_piix4

00:14.1 IDE interface: Advanced Micro Devices, Inc. [AMD/ATI] SB7x0/SB8x0/SB9x0 IDE Controller (rev 40) (prog-if 8a [Master SecP PriP])
	Subsystem: Gigabyte Technology Co., Ltd Device 5002
	Control: I/O+ Mem+ BusMaster+ SpecCycle- MemWINV- VGASnoop- ParErr- Stepping- SERR- FastB2B- DisINTx-
	Status: Cap- 66MHz+ UDF- FastB2B- ParErr- DEVSEL=medium >TAbort- <TAbort- <MAbort- >SERR- <PERR- INTx-
	Latency: 32
	Interrupt: pin B routed to IRQ 17
	Region 0: I/O ports at 01f0 [size=8]
	Region 1: I/O ports at 03f4
	Region 2: I/O ports at 0170 [size=8]
	Region 3: I/O ports at 0374
	Region 4: I/O ports at f000 [size=16]
	Kernel driver in use: pata_atiixp
	Kernel modules: pata_atiixp, pata_acpi, ata_generic

00:14.2 Audio device: Advanced Micro Devices, Inc. [AMD/ATI] SBx00 Azalia (Intel HDA) (rev 40)
	Subsystem: Gigabyte Technology Co., Ltd Device a132
	Control: I/O- Mem+ BusMaster+ SpecCycle- MemWINV- VGASnoop- ParErr- Stepping- SERR- FastB2B- DisINTx-
	Status: Cap+ 66MHz- UDF- FastB2B- ParErr- DEVSEL=slow >TAbort- <TAbort- <MAbort- >SERR- <PERR- INTx-
	Latency: 32, Cache Line Size: 64 bytes
	Interrupt: pin A routed to IRQ 16
	Region 0: Memory at d8500000 (64-bit, non-prefetchable) [size=16K]
	Capabilities: [50] Power Management version 2
		Flags: PMEClk- DSI- D1- D2- AuxCurrent=55mA PME(D0+,D1-,D2-,D3hot+,D3cold+)
		Status: D0 NoSoftRst- PME-Enable- DSel=0 DScale=0 PME-
	Kernel driver in use: snd_hda_intel
	Kernel modules: snd_hda_intel

00:14.3 ISA bridge: Advanced Micro Devices, Inc. [AMD/ATI] SB7x0/SB8x0/SB9x0 LPC host controller (rev 40)
	Subsystem: Advanced Micro Devices, Inc. [AMD/ATI] SB7x0/SB8x0/SB9x0 LPC host controller
	Control: I/O+ Mem+ BusMaster+ SpecCycle+ MemWINV- VGASnoop- ParErr- Stepping- SERR- FastB2B- DisINTx-
	Status: Cap- 66MHz+ UDF- FastB2B- ParErr- DEVSEL=medium >TAbort- <TAbort- <MAbort- >SERR- <PERR- INTx-
	Latency: 0

00:14.4 PCI bridge: Advanced Micro Devices, Inc. [AMD/ATI] SBx00 PCI to PCI Bridge (rev 40) (prog-if 01 [Subtractive decode])
	Control: I/O+ Mem+ BusMaster+ SpecCycle- MemWINV- VGASnoop+ ParErr- Stepping- SERR- FastB2B- DisINTx-
	Status: Cap- 66MHz+ UDF- FastB2B+ ParErr- DEVSEL=medium >TAbort- <TAbort- <MAbort- >SERR- <PERR- INTx-
	Latency: 64
	Bus: primary=00, secondary=11, subordinate=11, sec-latency=64
	I/O behind bridge: 0000a000-0000afff
	Memory behind bridge: ddf00000-ddffffff
	Secondary status: 66MHz- FastB2B+ ParErr- DEVSEL=medium >TAbort- <TAbort- <MAbort+ <SERR- <PERR-
	BridgeCtl: Parity- SERR- NoISA- VGA- MAbort- >Reset- FastB2B-
		PriDiscTmr- SecDiscTmr- DiscTmrStat- DiscTmrSERREn-

00:14.5 USB controller: Advanced Micro Devices, Inc. [AMD/ATI] SB7x0/SB8x0/SB9x0 USB OHCI2 Controller (prog-if 10 [OHCI])
	Subsystem: Gigabyte Technology Co., Ltd Device 5004
	Control: I/O+ Mem+ BusMaster+ SpecCycle- MemWINV- VGASnoop- ParErr- Stepping- SERR- FastB2B- DisINTx-
	Status: Cap- 66MHz+ UDF- FastB2B+ ParErr- DEVSEL=medium >TAbort- <TAbort- <MAbort- >SERR- <PERR- INTx-
	Latency: 32, Cache Line Size: 64 bytes
	Interrupt: pin C routed to IRQ 18
	Region 0: Memory at d8506000 (32-bit, non-prefetchable) [size=4K]
	Kernel driver in use: ohci-pci
	Kernel modules: ohci_pci

00:15.0 PCI bridge: Advanced Micro Devices, Inc. [AMD/ATI] SB700/SB800/SB900 PCI to PCI bridge (PCIE port 0) (prog-if 00 [Normal decode])
	Control: I/O+ Mem+ BusMaster+ SpecCycle- MemWINV- VGASnoop- ParErr- Stepping- SERR- FastB2B- DisINTx-
	Status: Cap+ 66MHz- UDF- FastB2B- ParErr- DEVSEL=fast >TAbort- <TAbort- <MAbort- >SERR- <PERR- INTx-
	Latency: 0, Cache Line Size: 64 bytes
	Bus: primary=00, secondary=12, subordinate=12, sec-latency=0
	I/O behind bridge: 00009000-00009fff
	Prefetchable memory behind bridge: 00000000c7000000-00000000c70fffff
	Secondary status: 66MHz- FastB2B- ParErr- DEVSEL=fast >TAbort- <TAbort- <MAbort+ <SERR- <PERR-
	BridgeCtl: Parity- SERR- NoISA- VGA- MAbort- >Reset- FastB2B-
		PriDiscTmr- SecDiscTmr- DiscTmrStat- DiscTmrSERREn-
	Capabilities: [50] Power Management version 3
		Flags: PMEClk- DSI- D1+ D2+ AuxCurrent=0mA PME(D0-,D1-,D2-,D3hot-,D3cold-)
		Status: D0 NoSoftRst- PME-Enable- DSel=0 DScale=0 PME-
	Capabilities: [58] Express (v2) Root Port (Slot-), MSI 00
		DevCap:	MaxPayload 128 bytes, PhantFunc 0
			ExtTag+ RBE+
		DevCtl:	Report errors: Correctable- Non-Fatal- Fatal- Unsupported-
			RlxdOrd- ExtTag- PhantFunc- AuxPwr- NoSnoop+
			MaxPayload 128 bytes, MaxReadReq 128 bytes
		DevSta:	CorrErr- UncorrErr- FatalErr- UnsuppReq- AuxPwr- TransPend-
		LnkCap:	Port #0, Speed 5GT/s, Width x1, ASPM L0s L1, Exit Latency L0s <64ns, L1 <1us
			ClockPM- Surprise- LLActRep+ BwNot+
		LnkCtl:	ASPM Disabled; RCB 64 bytes Disabled- CommClk+
			ExtSynch- ClockPM- AutWidDis- BWInt- AutBWInt-
		LnkSta:	Speed 2.5GT/s, Width x1, TrErr- Train- SlotClk+ DLActive+ BWMgmt+ ABWMgmt-
		RootCtl: ErrCorrectable- ErrNon-Fatal- ErrFatal- PMEIntEna- CRSVisible+
		RootCap: CRSVisible-
		RootSta: PME ReqID 0000, PMEStatus- PMEPending-
		DevCap2: Completion Timeout: Range ABCD, TimeoutDis+, LTR-, OBFF Not Supported ARIFwd-
		DevCtl2: Completion Timeout: 65ms to 210ms, TimeoutDis-, LTR-, OBFF Disabled ARIFwd-
		LnkCtl2: Target Link Speed: 5GT/s, EnterCompliance- SpeedDis-
			 Transmit Margin: Normal Operating Range, EnterModifiedCompliance- ComplianceSOS-
			 Compliance De-emphasis: -6dB
		LnkSta2: Current De-emphasis Level: -3.5dB, EqualizationComplete-, EqualizationPhase1-
			 EqualizationPhase2-, EqualizationPhase3-, LinkEqualizationRequest-
	Capabilities: [a0] MSI: Enable- Count=1/1 Maskable- 64bit+
		Address: 0000000000000000  Data: 0000
	Capabilities: [b0] Subsystem: Advanced Micro Devices, Inc. [AMD/ATI] Device 0000
	Capabilities: [b8] HyperTransport: MSI Mapping Enable+ Fixed+
	Capabilities: [100 v1] Vendor Specific Information: ID=0001 Rev=1 Len=010 <?>
	Kernel driver in use: pcieport
	Kernel modules: shpchp

00:15.1 PCI bridge: Advanced Micro Devices, Inc. [AMD/ATI] SB700/SB800/SB900 PCI to PCI bridge (PCIE port 1) (prog-if 00 [Normal decode])
	Control: I/O+ Mem+ BusMaster+ SpecCycle- MemWINV- VGASnoop- ParErr- Stepping- SERR- FastB2B- DisINTx-
	Status: Cap+ 66MHz- UDF- FastB2B- ParErr- DEVSEL=fast >TAbort- <TAbort- <MAbort- >SERR- <PERR- INTx-
	Latency: 0, Cache Line Size: 64 bytes
	Bus: primary=00, secondary=13, subordinate=13, sec-latency=0
	Memory behind bridge: dde00000-ddefffff
	Secondary status: 66MHz- FastB2B- ParErr- DEVSEL=fast >TAbort- <TAbort- <MAbort+ <SERR- <PERR-
	BridgeCtl: Parity- SERR- NoISA- VGA- MAbort- >Reset- FastB2B-
		PriDiscTmr- SecDiscTmr- DiscTmrStat- DiscTmrSERREn-
	Capabilities: [50] Power Management version 3
		Flags: PMEClk- DSI- D1+ D2+ AuxCurrent=0mA PME(D0-,D1-,D2-,D3hot-,D3cold-)
		Status: D0 NoSoftRst- PME-Enable- DSel=0 DScale=0 PME-
	Capabilities: [58] Express (v2) Root Port (Slot-), MSI 00
		DevCap:	MaxPayload 128 bytes, PhantFunc 0
			ExtTag+ RBE+
		DevCtl:	Report errors: Correctable- Non-Fatal- Fatal- Unsupported-
			RlxdOrd- ExtTag- PhantFunc- AuxPwr- NoSnoop+
			MaxPayload 128 bytes, MaxReadReq 128 bytes
		DevSta:	CorrErr- UncorrErr- FatalErr- UnsuppReq- AuxPwr- TransPend-
		LnkCap:	Port #1, Speed 5GT/s, Width x1, ASPM L0s L1, Exit Latency L0s <64ns, L1 <1us
			ClockPM- Surprise- LLActRep+ BwNot+
		LnkCtl:	ASPM Disabled; RCB 64 bytes Disabled- CommClk+
			ExtSynch- ClockPM- AutWidDis- BWInt- AutBWInt-
		LnkSta:	Speed 5GT/s, Width x1, TrErr- Train- SlotClk+ DLActive+ BWMgmt+ ABWMgmt+
		RootCtl: ErrCorrectable- ErrNon-Fatal- ErrFatal- PMEIntEna- CRSVisible+
		RootCap: CRSVisible-
		RootSta: PME ReqID 0000, PMEStatus- PMEPending-
		DevCap2: Completion Timeout: Range ABCD, TimeoutDis+, LTR-, OBFF Not Supported ARIFwd-
		DevCtl2: Completion Timeout: 65ms to 210ms, TimeoutDis-, LTR-, OBFF Disabled ARIFwd-
		LnkCtl2: Target Link Speed: 5GT/s, EnterCompliance- SpeedDis-
			 Transmit Margin: Normal Operating Range, EnterModifiedCompliance- ComplianceSOS-
			 Compliance De-emphasis: -6dB
		LnkSta2: Current De-emphasis Level: -3.5dB, EqualizationComplete-, EqualizationPhase1-
			 EqualizationPhase2-, EqualizationPhase3-, LinkEqualizationRequest-
	Capabilities: [a0] MSI: Enable- Count=1/1 Maskable- 64bit+
		Address: 0000000000000000  Data: 0000
	Capabilities: [b0] Subsystem: Advanced Micro Devices, Inc. [AMD/ATI] Device 0000
	Capabilities: [b8] HyperTransport: MSI Mapping Enable+ Fixed+
	Capabilities: [100 v1] Vendor Specific Information: ID=0001 Rev=1 Len=010 <?>
	Kernel driver in use: pcieport
	Kernel modules: shpchp

00:15.2 PCI bridge: Advanced Micro Devices, Inc. [AMD/ATI] SB900 PCI to PCI bridge (PCIE port 2) (prog-if 00 [Normal decode])
	Control: I/O+ Mem+ BusMaster+ SpecCycle- MemWINV- VGASnoop- ParErr- Stepping- SERR- FastB2B- DisINTx-
	Status: Cap+ 66MHz- UDF- FastB2B- ParErr- DEVSEL=fast >TAbort- <TAbort- <MAbort- >SERR- <PERR- INTx-
	Latency: 0, Cache Line Size: 64 bytes
	Bus: primary=00, secondary=14, subordinate=14, sec-latency=0
	I/O behind bridge: 00008000-00008fff
	Memory behind bridge: ddd00000-dddfffff
	Secondary status: 66MHz- FastB2B- ParErr- DEVSEL=fast >TAbort- <TAbort- <MAbort+ <SERR- <PERR-
	BridgeCtl: Parity- SERR- NoISA- VGA- MAbort- >Reset- FastB2B-
		PriDiscTmr- SecDiscTmr- DiscTmrStat- DiscTmrSERREn-
	Capabilities: [50] Power Management version 3
		Flags: PMEClk- DSI- D1+ D2+ AuxCurrent=0mA PME(D0-,D1-,D2-,D3hot-,D3cold-)
		Status: D0 NoSoftRst- PME-Enable- DSel=0 DScale=0 PME-
	Capabilities: [58] Express (v2) Root Port (Slot-), MSI 00
		DevCap:	MaxPayload 128 bytes, PhantFunc 0
			ExtTag+ RBE+
		DevCtl:	Report errors: Correctable- Non-Fatal- Fatal- Unsupported-
			RlxdOrd- ExtTag- PhantFunc- AuxPwr- NoSnoop+
			MaxPayload 128 bytes, MaxReadReq 128 bytes
		DevSta:	CorrErr- UncorrErr- FatalErr- UnsuppReq- AuxPwr- TransPend-
		LnkCap:	Port #2, Speed 5GT/s, Width x1, ASPM L0s L1, Exit Latency L0s <64ns, L1 <1us
			ClockPM- Surprise- LLActRep+ BwNot+
		LnkCtl:	ASPM Disabled; RCB 64 bytes Disabled- CommClk+
			ExtSynch- ClockPM- AutWidDis- BWInt- AutBWInt-
		LnkSta:	Speed 5GT/s, Width x1, TrErr- Train- SlotClk+ DLActive+ BWMgmt+ ABWMgmt-
		RootCtl: ErrCorrectable- ErrNon-Fatal- ErrFatal- PMEIntEna- CRSVisible+
		RootCap: CRSVisible-
		RootSta: PME ReqID 0000, PMEStatus- PMEPending-
		DevCap2: Completion Timeout: Range ABCD, TimeoutDis+, LTR-, OBFF Not Supported ARIFwd-
		DevCtl2: Completion Timeout: 65ms to 210ms, TimeoutDis-, LTR-, OBFF Disabled ARIFwd-
		LnkCtl2: Target Link Speed: 5GT/s, EnterCompliance- SpeedDis-
			 Transmit Margin: Normal Operating Range, EnterModifiedCompliance- ComplianceSOS-
			 Compliance De-emphasis: -6dB
		LnkSta2: Current De-emphasis Level: -3.5dB, EqualizationComplete-, EqualizationPhase1-
			 EqualizationPhase2-, EqualizationPhase3-, LinkEqualizationRequest-
	Capabilities: [a0] MSI: Enable- Count=1/1 Maskable- 64bit+
		Address: 0000000000000000  Data: 0000
	Capabilities: [b0] Subsystem: Advanced Micro Devices, Inc. [AMD/ATI] Device 0000
	Capabilities: [b8] HyperTransport: MSI Mapping Enable+ Fixed+
	Capabilities: [100 v1] Vendor Specific Information: ID=0001 Rev=1 Len=010 <?>
	Kernel driver in use: pcieport
	Kernel modules: shpchp

00:16.0 USB controller: Advanced Micro Devices, Inc. [AMD/ATI] SB7x0/SB8x0/SB9x0 USB OHCI0 Controller (prog-if 10 [OHCI])
	Subsystem: Gigabyte Technology Co., Ltd Device 5004
	Control: I/O+ Mem+ BusMaster+ SpecCycle- MemWINV- VGASnoop- ParErr- Stepping- SERR- FastB2B- DisINTx-
	Status: Cap- 66MHz+ UDF- FastB2B+ ParErr- DEVSEL=medium >TAbort- <TAbort- <MAbort- >SERR- <PERR- INTx-
	Latency: 32, Cache Line Size: 64 bytes
	Interrupt: pin A routed to IRQ 18
	Region 0: Memory at d8505000 (32-bit, non-prefetchable) [size=4K]
	Kernel driver in use: ohci-pci
	Kernel modules: ohci_pci

00:16.2 USB controller: Advanced Micro Devices, Inc. [AMD/ATI] SB7x0/SB8x0/SB9x0 USB EHCI Controller (prog-if 20 [EHCI])
	Subsystem: Gigabyte Technology Co., Ltd Device 5004
	Control: I/O+ Mem+ BusMaster+ SpecCycle- MemWINV+ VGASnoop- ParErr- Stepping- SERR- FastB2B- DisINTx-
	Status: Cap+ 66MHz+ UDF- FastB2B+ ParErr- DEVSEL=medium >TAbort- <TAbort- <MAbort- >SERR- <PERR- INTx-
	Latency: 32, Cache Line Size: 64 bytes
	Interrupt: pin B routed to IRQ 17
	Region 0: Memory at d8504000 (32-bit, non-prefetchable) [size=256]
	Capabilities: [c0] Power Management version 2
		Flags: PMEClk- DSI- D1+ D2+ AuxCurrent=0mA PME(D0+,D1+,D2+,D3hot+,D3cold-)
		Status: D0 NoSoftRst- PME-Enable- DSel=0 DScale=0 PME-
		Bridge: PM- B3+
	Capabilities: [e4] Debug port: BAR=1 offset=00e0
	Kernel driver in use: ehci-pci

00:18.0 Host bridge: Advanced Micro Devices, Inc. [AMD] Family 15h Processor Function 0
	Control: I/O- Mem- BusMaster- SpecCycle- MemWINV- VGASnoop- ParErr- Stepping- SERR- FastB2B- DisINTx-
	Status: Cap+ 66MHz- UDF- FastB2B- ParErr- DEVSEL=fast >TAbort- <TAbort- <MAbort- >SERR- <PERR- INTx-
	Capabilities: [80] HyperTransport: Host or Secondary Interface
		Command: WarmRst+ DblEnd- DevNum=0 ChainSide- HostHide+ Slave- <EOCErr- DUL-
		Link Control: CFlE- CST- CFE- <LkFail- Init+ EOC- TXO- <CRCErr=0 IsocEn- LSEn+ ExtCTL- 64b+
		Link Config: MLWI=16bit DwFcIn- MLWO=16bit DwFcOut- LWI=16bit DwFcInEn- LWO=16bit DwFcOutEn-
		Revision ID: 3.00
		Link Frequency: [e]
		Link Error: <Prot- <Ovfl- <EOC- CTLTm-
		Link Frequency Capability: 200MHz+ 300MHz- 400MHz+ 500MHz- 600MHz+ 800MHz+ 1.0GHz+ 1.2GHz+ 1.4GHz- 1.6GHz- Vend-
		Feature Capability: IsocFC+ LDTSTOP+ CRCTM- ECTLT- 64bA+ UIDRD- ExtRS- UCnfE-

00:18.1 Host bridge: Advanced Micro Devices, Inc. [AMD] Family 15h Processor Function 1
	Control: I/O- Mem- BusMaster- SpecCycle- MemWINV- VGASnoop- ParErr- Stepping- SERR- FastB2B- DisINTx-
	Status: Cap- 66MHz- UDF- FastB2B- ParErr- DEVSEL=fast >TAbort- <TAbort- <MAbort- >SERR- <PERR- INTx-

00:18.2 Host bridge: Advanced Micro Devices, Inc. [AMD] Family 15h Processor Function 2
	Control: I/O- Mem- BusMaster- SpecCycle- MemWINV- VGASnoop- ParErr- Stepping- SERR- FastB2B- DisINTx-
	Status: Cap- 66MHz- UDF- FastB2B- ParErr- DEVSEL=fast >TAbort- <TAbort- <MAbort- >SERR- <PERR- INTx-

00:18.3 Host bridge: Advanced Micro Devices, Inc. [AMD] Family 15h Processor Function 3
	Control: I/O- Mem- BusMaster- SpecCycle- MemWINV- VGASnoop- ParErr- Stepping- SERR- FastB2B- DisINTx-
	Status: Cap+ 66MHz- UDF- FastB2B- ParErr- DEVSEL=fast >TAbort- <TAbort- <MAbort- >SERR- <PERR- INTx-
	Capabilities: [f0] Secure device <?>
	Kernel driver in use: k10temp
	Kernel modules: k10temp

00:18.4 Host bridge: Advanced Micro Devices, Inc. [AMD] Family 15h Processor Function 4
	Control: I/O- Mem- BusMaster- SpecCycle- MemWINV- VGASnoop- ParErr- Stepping- SERR- FastB2B- DisINTx-
	Status: Cap- 66MHz- UDF- FastB2B- ParErr- DEVSEL=fast >TAbort- <TAbort- <MAbort- >SERR- <PERR- INTx-
	Kernel driver in use: fam15h_power
	Kernel modules: fam15h_power

00:18.5 Host bridge: Advanced Micro Devices, Inc. [AMD] Family 15h Processor Function 5
	Control: I/O- Mem- BusMaster- SpecCycle- MemWINV- VGASnoop- ParErr- Stepping- SERR- FastB2B- DisINTx-
	Status: Cap- 66MHz- UDF- FastB2B- ParErr- DEVSEL=fast >TAbort- <TAbort- <MAbort- >SERR- <PERR- INTx-

01:00.0 VGA compatible controller: NVIDIA Corporation G70 [GeForce 7800 GT] (rev a1) (prog-if 00 [VGA controller])
	Subsystem: eVga.com. Corp. Device c517
	Control: I/O+ Mem+ BusMaster+ SpecCycle- MemWINV- VGASnoop- ParErr- Stepping- SERR- FastB2B- DisINTx-
	Status: Cap+ 66MHz- UDF- FastB2B- ParErr- DEVSEL=fast >TAbort- <TAbort- <MAbort- >SERR- <PERR- INTx-
	Latency: 0
	Interrupt: pin A routed to IRQ 43
	Region 0: Memory at da000000 (32-bit, non-prefetchable) [size=16M]
	Region 1: Memory at b0000000 (64-bit, prefetchable) [size=256M]
	Region 3: Memory at d9000000 (64-bit, non-prefetchable) [size=16M]
	Region 5: I/O ports at e000 [size=128]
	[virtual] Expansion ROM at db000000 [disabled] [size=128K]
	Capabilities: [60] Power Management version 2
		Flags: PMEClk- DSI- D1- D2- AuxCurrent=0mA PME(D0-,D1-,D2-,D3hot-,D3cold-)
		Status: D0 NoSoftRst- PME-Enable- DSel=0 DScale=0 PME-
	Capabilities: [68] MSI: Enable- Count=1/1 Maskable- 64bit+
		Address: 0000000000000000  Data: 0000
	Capabilities: [78] Express (v1) Endpoint, MSI 00
		DevCap:	MaxPayload 128 bytes, PhantFunc 0, Latency L0s <512ns, L1 <4us
			ExtTag- AttnBtn- AttnInd- PwrInd- RBE- FLReset-
		DevCtl:	Report errors: Correctable- Non-Fatal- Fatal- Unsupported-
			RlxdOrd- ExtTag- PhantFunc- AuxPwr- NoSnoop+
			MaxPayload 128 bytes, MaxReadReq 512 bytes
		DevSta:	CorrErr- UncorrErr- FatalErr- UnsuppReq- AuxPwr- TransPend+
		LnkCap:	Port #0, Speed 2.5GT/s, Width x16, ASPM L0s L1, Exit Latency L0s <256ns, L1 <4us
			ClockPM- Surprise- LLActRep- BwNot-
		LnkCtl:	ASPM Disabled; RCB 128 bytes Disabled- CommClk+
			ExtSynch- ClockPM- AutWidDis- BWInt- AutBWInt-
		LnkSta:	Speed 2.5GT/s, Width x16, TrErr- Train- SlotClk+ DLActive- BWMgmt- ABWMgmt-
	Capabilities: [100 v1] Virtual Channel
		Caps:	LPEVC=0 RefClk=100ns PATEntryBits=1
		Arb:	Fixed- WRR32- WRR64- WRR128-
		Ctrl:	ArbSelect=Fixed
		Status:	InProgress-
		VC0:	Caps:	PATOffset=00 MaxTimeSlots=1 RejSnoopTrans-
			Arb:	Fixed- WRR32- WRR64- WRR128- TWRR128- WRR256-
			Ctrl:	Enable+ ID=0 ArbSelect=Fixed TC/VC=01
			Status:	NegoPending- InProgress-
	Capabilities: [128 v1] Power Budgeting <?>
	Kernel driver in use: nvidia
	Kernel modules: nvidiafb, nouveau, nvidia

02:00.0 PCI bridge: Texas Instruments XIO2000(A)/XIO2200A PCI Express-to-PCI Bridge (rev 03) (prog-if 00 [Normal decode])
	Control: I/O+ Mem+ BusMaster+ SpecCycle- MemWINV- VGASnoop- ParErr- Stepping- SERR- FastB2B- DisINTx-
	Status: Cap+ 66MHz- UDF- FastB2B- ParErr- DEVSEL=fast >TAbort- <TAbort- <MAbort- >SERR- <PERR- INTx-
	Latency: 0, Cache Line Size: 64 bytes
	Bus: primary=02, secondary=03, subordinate=05, sec-latency=32
	I/O behind bridge: 00006000-00007fff
	Memory behind bridge: db800000-dd9fffff
	Prefetchable memory behind bridge: 00000000c0000000-00000000c3ffffff
	Secondary status: 66MHz- FastB2B+ ParErr- DEVSEL=medium >TAbort- <TAbort- <MAbort+ <SERR- <PERR-
	BridgeCtl: Parity- SERR- NoISA- VGA- MAbort- >Reset- FastB2B-
		PriDiscTmr- SecDiscTmr- DiscTmrStat- DiscTmrSERREn-
	Capabilities: [50] Power Management version 2
		Flags: PMEClk- DSI- D1+ D2+ AuxCurrent=0mA PME(D0-,D1-,D2-,D3hot-,D3cold-)
		Status: D0 NoSoftRst- PME-Enable- DSel=0 DScale=0 PME-
		Bridge: PM- B3+
	Capabilities: [60] MSI: Enable- Count=1/16 Maskable- 64bit+
		Address: 0000000000000000  Data: 0000
	Capabilities: [80] Subsystem: Device 0000:0000
	Capabilities: [90] Express (v1) PCI-Express to PCI/PCI-X Bridge, MSI 00
		DevCap:	MaxPayload 512 bytes, PhantFunc 0
			ExtTag- AttnBtn- AttnInd- PwrInd- RBE-
		DevCtl:	Report errors: Correctable- Non-Fatal- Fatal- Unsupported-
			RlxdOrd- ExtTag- PhantFunc- AuxPwr- NoSnoop+ BrConfRtry-
			MaxPayload 128 bytes, MaxReadReq 512 bytes
		DevSta:	CorrErr- UncorrErr+ FatalErr- UnsuppReq+ AuxPwr+ TransPend-
		LnkCap:	Port #0, Speed 2.5GT/s, Width x1, ASPM L0s L1, Exit Latency L0s <1us, L1 <16us
			ClockPM- Surprise- LLActRep- BwNot-
		LnkCtl:	ASPM Disabled; RCB 64 bytes Disabled- CommClk-
			ExtSynch- ClockPM- AutWidDis- BWInt- AutBWInt-
		LnkSta:	Speed 2.5GT/s, Width x1, TrErr- Train- SlotClk+ DLActive- BWMgmt- ABWMgmt-
	Capabilities: [100 v1] Advanced Error Reporting
		UESta:	DLP- SDES- TLP- FCP- CmpltTO- CmpltAbrt- UnxCmplt- RxOF- MalfTLP- ECRC- UnsupReq+ ACSViol-
		UEMsk:	DLP- SDES- TLP- FCP- CmpltTO- CmpltAbrt- UnxCmplt- RxOF- MalfTLP- ECRC- UnsupReq- ACSViol-
		UESvrt:	DLP+ SDES- TLP- FCP+ CmpltTO- CmpltAbrt- UnxCmplt- RxOF+ MalfTLP+ ECRC- UnsupReq- ACSViol-
		CESta:	RxErr- BadTLP- BadDLLP- Rollover- Timeout- NonFatalErr-
		CEMsk:	RxErr- BadTLP- BadDLLP- Rollover- Timeout- NonFatalErr-
		AERCap:	First Error Pointer: 14, GenCap+ CGenEn- ChkCap+ ChkEn-
	Kernel modules: shpchp

03:04.0 PCI bridge: Pericom Semiconductor PCI to PCI Bridge (rev 02) (prog-if 00 [Normal decode])
	Control: I/O+ Mem+ BusMaster+ SpecCycle- MemWINV- VGASnoop- ParErr- Stepping- SERR- FastB2B- DisINTx-
	Status: Cap+ 66MHz- UDF- FastB2B+ ParErr- DEVSEL=medium >TAbort- <TAbort- <MAbort- >SERR- <PERR- INTx-
	Latency: 32, Cache Line Size: 64 bytes
	Bus: primary=03, secondary=04, subordinate=05, sec-latency=32
	I/O behind bridge: 00006000-00007fff
	Memory behind bridge: db800000-dd9fffff
	Prefetchable memory behind bridge: 00000000c0000000-00000000c3ffffff
	Secondary status: 66MHz- FastB2B+ ParErr- DEVSEL=medium >TAbort- <TAbort- <MAbort+ <SERR- <PERR-
	BridgeCtl: Parity- SERR- NoISA- VGA- MAbort- >Reset- FastB2B-
		PriDiscTmr- SecDiscTmr- DiscTmrStat- DiscTmrSERREn-
	Capabilities: [dc] Power Management version 1
		Flags: PMEClk- DSI- D1- D2- AuxCurrent=0mA PME(D0-,D1-,D2-,D3hot-,D3cold-)
		Status: D0 NoSoftRst- PME-Enable- DSel=0 DScale=0 PME-
	Capabilities: [b0] Slot ID: 0 slots, First-, chassis 00
	Kernel modules: shpchp

04:04.0 Intelligent controller [0e80]: PLX Technology, Inc. Device 0480 (rev 55)
	Subsystem: Compro Computer Services, Inc. Device 4620
	Control: I/O+ Mem+ BusMaster+ SpecCycle- MemWINV- VGASnoop- ParErr- Stepping- SERR- FastB2B- DisINTx-
	Status: Cap+ 66MHz- UDF- FastB2B+ ParErr- DEVSEL=medium >TAbort- <TAbort- <MAbort- >SERR- <PERR- INTx-
	Latency: 32, Cache Line Size: 64 bytes
	Interrupt: pin A routed to IRQ 35
	BIST result: 00
	Region 0: Memory at dd909000 (32-bit, non-prefetchable) [size=1K]
	Region 1: Memory at dd000000 (32-bit, non-prefetchable) [size=8M]
	Region 2: Memory at dd908000 (32-bit, non-prefetchable) [size=16]
	Capabilities: [40] #00 [0000]
	Capabilities: [54] #00 [0080]
	Capabilities: [58] Vital Product Data
		Unknown large resource type 48, will not decode more.
	Kernel driver in use: gpiohsd
	Kernel modules: gpiohsd

04:05.0 Intelligent controller [0e80]: PLX Technology, Inc. Device 0480 (rev 55)
	Subsystem: Compro Computer Services, Inc. Device 4620
	Control: I/O+ Mem+ BusMaster+ SpecCycle- MemWINV- VGASnoop- ParErr- Stepping- SERR- FastB2B- DisINTx-
	Status: Cap+ 66MHz- UDF- FastB2B+ ParErr- DEVSEL=medium >TAbort- <TAbort- <MAbort- >SERR- <PERR- INTx-
	Latency: 32, Cache Line Size: 64 bytes
	Interrupt: pin A routed to IRQ 34
	BIST result: 00
	Region 0: Memory at dd907000 (32-bit, non-prefetchable) [size=1K]
	Region 1: Memory at dc800000 (32-bit, non-prefetchable) [size=8M]
	Region 2: Memory at dd906000 (32-bit, non-prefetchable) [size=16]
	Capabilities: [40] #00 [0000]
	Capabilities: [54] #00 [0080]
	Capabilities: [58] Vital Product Data
		Unknown large resource type 48, will not decode more.
	Kernel driver in use: gpiohsd
	Kernel modules: gpiohsd

04:06.0 Intelligent controller [0e80]: PLX Technology, Inc. Device 0480 (rev 55)
	Subsystem: Compro Computer Services, Inc. Device 4620
	Control: I/O+ Mem+ BusMaster+ SpecCycle- MemWINV- VGASnoop- ParErr- Stepping- SERR- FastB2B- DisINTx-
	Status: Cap+ 66MHz- UDF- FastB2B+ ParErr- DEVSEL=medium >TAbort- <TAbort- <MAbort- >SERR- <PERR- INTx-
	Latency: 32, Cache Line Size: 64 bytes
	Interrupt: pin A routed to IRQ 39
	BIST result: 00
	Region 0: Memory at dd905000 (32-bit, non-prefetchable) [size=1K]
	Region 1: Memory at dc000000 (32-bit, non-prefetchable) [size=8M]
	Region 2: Memory at dd904000 (32-bit, non-prefetchable) [size=16]
	Capabilities: [40] #00 [0000]
	Capabilities: [54] #00 [0080]
	Capabilities: [58] Vital Product Data
		Unknown large resource type 48, will not decode more.
	Kernel driver in use: gpiohsd
	Kernel modules: gpiohsd

04:07.0 Intelligent controller [0e80]: PLX Technology, Inc. Device 0480 (rev 55)
	Subsystem: Compro Computer Services, Inc. Device 4620
	Control: I/O+ Mem+ BusMaster+ SpecCycle- MemWINV- VGASnoop- ParErr- Stepping- SERR- FastB2B- DisINTx-
	Status: Cap+ 66MHz- UDF- FastB2B+ ParErr- DEVSEL=medium >TAbort- <TAbort- <MAbort- >SERR- <PERR- INTx-
	Latency: 32, Cache Line Size: 64 bytes
	Interrupt: pin A routed to IRQ 29
	BIST result: 00
	Region 0: Memory at dd903000 (32-bit, non-prefetchable) [size=1K]
	Region 1: Memory at db800000 (32-bit, non-prefetchable) [size=8M]
	Region 2: Memory at dd902000 (32-bit, non-prefetchable) [size=16]
	Capabilities: [40] #00 [0000]
	Capabilities: [54] #00 [0080]
	Capabilities: [58] Vital Product Data
		Unknown large resource type 48, will not decode more.
	Kernel driver in use: gpiohsd
	Kernel modules: gpiohsd

04:08.0 PCI bridge: Hint Corp HiNT HB4 PCI-PCI Bridge (PCI6150) (rev 04) (prog-if 00 [Normal decode])
	Control: I/O+ Mem+ BusMaster+ SpecCycle- MemWINV- VGASnoop- ParErr- Stepping+ SERR- FastB2B- DisINTx-
	Status: Cap+ 66MHz+ UDF- FastB2B+ ParErr- DEVSEL=medium >TAbort- <TAbort- <MAbort- >SERR- <PERR- INTx-
	Latency: 32, Cache Line Size: 64 bytes
	Bus: primary=04, secondary=05, subordinate=05, sec-latency=32
	I/O behind bridge: 00006000-00006fff
	Memory behind bridge: dd800000-dd8fffff
	Secondary status: 66MHz+ FastB2B+ ParErr- DEVSEL=medium >TAbort- <TAbort- <MAbort- <SERR- <PERR-
	BridgeCtl: Parity- SERR- NoISA- VGA- MAbort- >Reset- FastB2B-
		PriDiscTmr- SecDiscTmr- DiscTmrStat- DiscTmrSERREn-
	Capabilities: [dc] Power Management version 1
		Flags: PMEClk- DSI- D1+ D2+ AuxCurrent=0mA PME(D0+,D1+,D2+,D3hot+,D3cold-)
		Status: D0 NoSoftRst- PME-Enable- DSel=0 DScale=0 PME+
	Capabilities: [e4] CompactPCI hot-swap <?>
	Capabilities: [e8] Vital Product Data
		Unknown small resource type 00, will not decode more.
	Kernel modules: shpchp

04:09.0 Unassigned class [ff00]: Compro Computer Services, Inc. PCI RTOM (rev 03)
	Control: I/O+ Mem+ BusMaster+ SpecCycle- MemWINV- VGASnoop- ParErr- Stepping- SERR- FastB2B- DisINTx-
	Status: Cap- 66MHz- UDF- FastB2B- ParErr- DEVSEL=fast >TAbort- <TAbort- <MAbort- >SERR- <PERR- INTx-
	Latency: 32
	Interrupt: pin A routed to IRQ 34
	Region 0: I/O ports at 7100 [size=64]
	Region 1: Memory at dd901000 (32-bit, non-prefetchable) [size=4K]
	Kernel driver in use: rtom
	Kernel modules: rtom

04:0a.0 Memory controller: Compro Computer Services, Inc. Device 4360 (rev 4d)
	Control: I/O+ Mem+ BusMaster+ SpecCycle- MemWINV- VGASnoop- ParErr- Stepping- SERR- FastB2B- DisINTx-
	Status: Cap- 66MHz- UDF- FastB2B+ ParErr- DEVSEL=medium >TAbort- <TAbort- <MAbort- >SERR- <PERR- INTx-
	Latency: 32, Cache Line Size: 64 bytes
	Interrupt: pin A routed to IRQ 39
	Region 0: Memory at dd900000 (32-bit, non-prefetchable) [size=256]
	Region 1: I/O ports at 7000 [size=256]
	Region 2: Memory at c0000000 (32-bit, non-prefetchable) [size=64M]
	Expansion ROM at dd90a000 [disabled] [size=2K]
	Kernel driver in use: eprm
	Kernel modules: eprm

05:00.0 Unassigned class [ff00]: Compro Computer Services, Inc. Device 0100 (rev 31)
	Subsystem: Compro Computer Services, Inc. Device 4700
	Control: I/O+ Mem+ BusMaster+ SpecCycle- MemWINV- VGASnoop- ParErr- Stepping- SERR- FastB2B- DisINTx-
	Status: Cap- 66MHz- UDF- FastB2B- ParErr- DEVSEL=slow >TAbort- <TAbort- <MAbort- >SERR- <PERR- INTx-
	Latency: 32
	Interrupt: pin A routed to IRQ 35
	Region 0: Memory at dd80a000 (32-bit, non-prefetchable) [size=4K]
	Region 1: Memory at dd808000 (32-bit, non-prefetchable) [size=8K]
	Region 2: Memory at dd800000 (32-bit, non-prefetchable) [size=32K]
	Region 5: I/O ports at 0000
	Kernel driver in use: iom
	Kernel modules: pciiom

06:00.0 USB controller: Etron Technology, Inc. EJ168 USB 3.0 Host Controller (rev 01) (prog-if 30 [XHCI])
	Subsystem: Gigabyte Technology Co., Ltd Device 5007
	Control: I/O- Mem+ BusMaster+ SpecCycle- MemWINV- VGASnoop- ParErr- Stepping- SERR- FastB2B- DisINTx+
	Status: Cap+ 66MHz- UDF- FastB2B- ParErr- DEVSEL=fast >TAbort- <TAbort- <MAbort- >SERR- <PERR- INTx-
	Latency: 0, Cache Line Size: 64 bytes
	Interrupt: pin A routed to IRQ 41
	Region 0: Memory at de100000 (64-bit, non-prefetchable) [size=32K]
	Capabilities: [50] Power Management version 3
		Flags: PMEClk- DSI- D1+ D2+ AuxCurrent=0mA PME(D0+,D1+,D2+,D3hot+,D3cold+)
		Status: D0 NoSoftRst- PME-Enable- DSel=0 DScale=0 PME-
	Capabilities: [70] MSI: Enable+ Count=1/4 Maskable+ 64bit+
		Address: 00000000fee7f00c  Data: 41b3
		Masking: 0000000e  Pending: 00000000
	Capabilities: [a0] Express (v2) Endpoint, MSI 00
		DevCap:	MaxPayload 1024 bytes, PhantFunc 0, Latency L0s <64ns, L1 <1us
			ExtTag+ AttnBtn- AttnInd- PwrInd- RBE+ FLReset+
		DevCtl:	Report errors: Correctable- Non-Fatal- Fatal- Unsupported-
			RlxdOrd- ExtTag- PhantFunc- AuxPwr- NoSnoop+ FLReset-
			MaxPayload 128 bytes, MaxReadReq 512 bytes
		DevSta:	CorrErr+ UncorrErr+ FatalErr- UnsuppReq+ AuxPwr+ TransPend-
		LnkCap:	Port #0, Speed 5GT/s, Width x1, ASPM L0s L1, Exit Latency L0s <1us, L1 <64us
			ClockPM+ Surprise- LLActRep- BwNot-
		LnkCtl:	ASPM Disabled; RCB 64 bytes Disabled- CommClk+
			ExtSynch- ClockPM- AutWidDis- BWInt- AutBWInt-
		LnkSta:	Speed 5GT/s, Width x1, TrErr- Train- SlotClk+ DLActive- BWMgmt- ABWMgmt-
		DevCap2: Completion Timeout: Not Supported, TimeoutDis-, LTR-, OBFF Not Supported
		DevCtl2: Completion Timeout: 50us to 50ms, TimeoutDis-, LTR-, OBFF Disabled
		LnkCtl2: Target Link Speed: 5GT/s, EnterCompliance- SpeedDis-
			 Transmit Margin: Normal Operating Range, EnterModifiedCompliance- ComplianceSOS-
			 Compliance De-emphasis: -6dB
		LnkSta2: Current De-emphasis Level: -6dB, EqualizationComplete-, EqualizationPhase1-
			 EqualizationPhase2-, EqualizationPhase3-, LinkEqualizationRequest-
	Capabilities: [100 v1] Advanced Error Reporting
		UESta:	DLP- SDES- TLP- FCP- CmpltTO- CmpltAbrt- UnxCmplt- RxOF- MalfTLP- ECRC- UnsupReq+ ACSViol-
		UEMsk:	DLP- SDES- TLP- FCP- CmpltTO- CmpltAbrt- UnxCmplt- RxOF- MalfTLP- ECRC- UnsupReq- ACSViol-
		UESvrt:	DLP+ SDES- TLP- FCP+ CmpltTO- CmpltAbrt- UnxCmplt- RxOF+ MalfTLP+ ECRC- UnsupReq- ACSViol-
		CESta:	RxErr- BadTLP- BadDLLP- Rollover- Timeout- NonFatalErr+
		CEMsk:	RxErr- BadTLP- BadDLLP- Rollover- Timeout- NonFatalErr+
		AERCap:	First Error Pointer: 14, GenCap+ CGenEn- ChkCap+ ChkEn-
	Capabilities: [190 v1] Device Serial Number 01-01-01-01-01-01-01-01
	Kernel driver in use: xhci_hcd
	Kernel modules: xhci_pci

07:00.0 SATA controller: Marvell Technology Group Ltd. 88SE9172 SATA 6Gb/s Controller (rev 12) (prog-if 01 [AHCI 1.0])
	Subsystem: Gigabyte Technology Co., Ltd Device b000
	Control: I/O+ Mem+ BusMaster+ SpecCycle- MemWINV- VGASnoop- ParErr- Stepping- SERR- FastB2B- DisINTx+
	Status: Cap+ 66MHz- UDF- FastB2B- ParErr- DEVSEL=fast >TAbort- <TAbort- <MAbort- >SERR- <PERR- INTx-
	Latency: 0, Cache Line Size: 64 bytes
	Interrupt: pin A routed to IRQ 30
	Region 0: I/O ports at d040 [size=8]
	Region 1: I/O ports at d030 [size=4]
	Region 2: I/O ports at d020 [size=8]
	Region 3: I/O ports at d010 [size=4]
	Region 4: I/O ports at d000 [size=16]
	Region 5: Memory at de010000 (32-bit, non-prefetchable) [size=512]
	Expansion ROM at de000000 [disabled] [size=64K]
	Capabilities: [40] Power Management version 3
		Flags: PMEClk- DSI- D1- D2- AuxCurrent=0mA PME(D0-,D1-,D2-,D3hot+,D3cold-)
		Status: D0 NoSoftRst- PME-Enable- DSel=0 DScale=0 PME-
	Capabilities: [50] MSI: Enable+ Count=1/1 Maskable- 64bit-
		Address: fee7f00c  Data: 4192
	Capabilities: [70] Express (v2) Legacy Endpoint, MSI 00
		DevCap:	MaxPayload 512 bytes, PhantFunc 0, Latency L0s <1us, L1 <8us
			ExtTag- AttnBtn- AttnInd- PwrInd- RBE+ FLReset-
		DevCtl:	Report errors: Correctable- Non-Fatal- Fatal- Unsupported-
			RlxdOrd- ExtTag- PhantFunc- AuxPwr- NoSnoop-
			MaxPayload 128 bytes, MaxReadReq 512 bytes
		DevSta:	CorrErr+ UncorrErr- FatalErr- UnsuppReq+ AuxPwr- TransPend-
		LnkCap:	Port #0, Speed 5GT/s, Width x1, ASPM L0s L1, Exit Latency L0s <512ns, L1 <64us
			ClockPM- Surprise- LLActRep- BwNot-
		LnkCtl:	ASPM Disabled; RCB 64 bytes Disabled- CommClk+
			ExtSynch- ClockPM- AutWidDis- BWInt- AutBWInt-
		LnkSta:	Speed 5GT/s, Width x1, TrErr- Train- SlotClk+ DLActive- BWMgmt- ABWMgmt-
		DevCap2: Completion Timeout: Not Supported, TimeoutDis+, LTR-, OBFF Not Supported
		DevCtl2: Completion Timeout: 50us to 50ms, TimeoutDis-, LTR-, OBFF Disabled
		LnkCtl2: Target Link Speed: 5GT/s, EnterCompliance- SpeedDis-
			 Transmit Margin: Normal Operating Range, EnterModifiedCompliance- ComplianceSOS-
			 Compliance De-emphasis: -6dB
		LnkSta2: Current De-emphasis Level: -3.5dB, EqualizationComplete-, EqualizationPhase1-
			 EqualizationPhase2-, EqualizationPhase3-, LinkEqualizationRequest-
	Capabilities: [100 v1] Advanced Error Reporting
		UESta:	DLP- SDES- TLP- FCP- CmpltTO- CmpltAbrt- UnxCmplt- RxOF- MalfTLP- ECRC- UnsupReq- ACSViol-
		UEMsk:	DLP- SDES- TLP- FCP- CmpltTO- CmpltAbrt- UnxCmplt- RxOF- MalfTLP- ECRC- UnsupReq- ACSViol-
		UESvrt:	DLP+ SDES+ TLP- FCP+ CmpltTO- CmpltAbrt- UnxCmplt- RxOF+ MalfTLP+ ECRC- UnsupReq- ACSViol-
		CESta:	RxErr- BadTLP- BadDLLP- Rollover- Timeout- NonFatalErr+
		CEMsk:	RxErr- BadTLP- BadDLLP- Rollover- Timeout- NonFatalErr+
		AERCap:	First Error Pointer: 00, GenCap- CGenEn- ChkCap- ChkEn-
	Kernel driver in use: ahci

08:00.0 PCI bridge: Texas Instruments XIO2000(A)/XIO2200A PCI Express-to-PCI Bridge (rev 03) (prog-if 00 [Normal decode])
	Control: I/O+ Mem+ BusMaster+ SpecCycle- MemWINV- VGASnoop- ParErr- Stepping- SERR- FastB2B- DisINTx-
	Status: Cap+ 66MHz- UDF- FastB2B- ParErr- DEVSEL=fast >TAbort- <TAbort- <MAbort- >SERR- <PERR- INTx-
	Latency: 0, Cache Line Size: 64 bytes
	Bus: primary=08, secondary=09, subordinate=0e, sec-latency=32
	I/O behind bridge: 0000c000-0000cfff
	Memory behind bridge: d0000000-d84fffff
	Prefetchable memory behind bridge: 00000000c4000000-00000000c6ffffff
	Secondary status: 66MHz- FastB2B+ ParErr- DEVSEL=medium >TAbort- <TAbort- <MAbort+ <SERR- <PERR-
	BridgeCtl: Parity- SERR- NoISA- VGA- MAbort- >Reset- FastB2B-
		PriDiscTmr- SecDiscTmr- DiscTmrStat- DiscTmrSERREn-
	Capabilities: [50] Power Management version 2
		Flags: PMEClk- DSI- D1+ D2+ AuxCurrent=0mA PME(D0-,D1-,D2-,D3hot-,D3cold-)
		Status: D0 NoSoftRst- PME-Enable- DSel=0 DScale=0 PME-
		Bridge: PM- B3+
	Capabilities: [60] MSI: Enable- Count=1/16 Maskable- 64bit+
		Address: 0000000000000000  Data: 0000
	Capabilities: [80] Subsystem: Device 0000:0000
	Capabilities: [90] Express (v1) PCI-Express to PCI/PCI-X Bridge, MSI 00
		DevCap:	MaxPayload 512 bytes, PhantFunc 0
			ExtTag- AttnBtn- AttnInd- PwrInd- RBE-
		DevCtl:	Report errors: Correctable- Non-Fatal- Fatal- Unsupported-
			RlxdOrd- ExtTag- PhantFunc- AuxPwr- NoSnoop+ BrConfRtry-
			MaxPayload 128 bytes, MaxReadReq 512 bytes
		DevSta:	CorrErr- UncorrErr+ FatalErr- UnsuppReq+ AuxPwr+ TransPend-
		LnkCap:	Port #0, Speed 2.5GT/s, Width x1, ASPM L0s L1, Exit Latency L0s <1us, L1 <16us
			ClockPM- Surprise- LLActRep- BwNot-
		LnkCtl:	ASPM Disabled; RCB 64 bytes Disabled- CommClk-
			ExtSynch- ClockPM- AutWidDis- BWInt- AutBWInt-
		LnkSta:	Speed 2.5GT/s, Width x1, TrErr- Train- SlotClk+ DLActive- BWMgmt- ABWMgmt-
	Capabilities: [100 v1] Advanced Error Reporting
		UESta:	DLP- SDES- TLP- FCP- CmpltTO- CmpltAbrt- UnxCmplt- RxOF- MalfTLP- ECRC- UnsupReq+ ACSViol-
		UEMsk:	DLP- SDES- TLP- FCP- CmpltTO- CmpltAbrt- UnxCmplt- RxOF- MalfTLP- ECRC- UnsupReq- ACSViol-
		UESvrt:	DLP+ SDES- TLP- FCP+ CmpltTO- CmpltAbrt- UnxCmplt- RxOF+ MalfTLP+ ECRC- UnsupReq- ACSViol-
		CESta:	RxErr- BadTLP- BadDLLP- Rollover- Timeout- NonFatalErr-
		CEMsk:	RxErr- BadTLP- BadDLLP- Rollover- Timeout- NonFatalErr-
		AERCap:	First Error Pointer: 14, GenCap+ CGenEn- ChkCap+ ChkEn-
	Kernel modules: shpchp

09:04.0 PCI bridge: Pericom Semiconductor PCI to PCI Bridge (rev 02) (prog-if 00 [Normal decode])
	Control: I/O+ Mem+ BusMaster+ SpecCycle- MemWINV- VGASnoop- ParErr- Stepping- SERR- FastB2B- DisINTx-
	Status: Cap+ 66MHz- UDF- FastB2B+ ParErr- DEVSEL=medium >TAbort- <TAbort- <MAbort- >SERR- <PERR- INTx-
	Latency: 32, Cache Line Size: 64 bytes
	Bus: primary=09, secondary=0a, subordinate=0e, sec-latency=32
	I/O behind bridge: 0000c000-0000cfff
	Memory behind bridge: d0000000-d84fffff
	Prefetchable memory behind bridge: 00000000c4000000-00000000c6ffffff
	Secondary status: 66MHz- FastB2B+ ParErr- DEVSEL=medium >TAbort- <TAbort- <MAbort+ <SERR- <PERR-
	BridgeCtl: Parity- SERR- NoISA- VGA- MAbort- >Reset- FastB2B-
		PriDiscTmr- SecDiscTmr- DiscTmrStat- DiscTmrSERREn-
	Capabilities: [dc] Power Management version 1
		Flags: PMEClk- DSI- D1- D2- AuxCurrent=0mA PME(D0-,D1-,D2-,D3hot-,D3cold-)
		Status: D0 NoSoftRst- PME-Enable- DSel=0 DScale=0 PME-
	Capabilities: [b0] Slot ID: 0 slots, First-, chassis 00
	Kernel modules: shpchp

0a:04.0 Ethernet controller: 3Com Corporation 3c905C-TX/TX-M [Tornado] (rev 78)
	Subsystem: 3Com Corporation 3C905CX-TX/TX-M Fast Etherlink for PC Management NIC
	Control: I/O+ Mem+ BusMaster+ SpecCycle- MemWINV- VGASnoop- ParErr- Stepping- SERR- FastB2B- DisINTx-
	Status: Cap+ 66MHz- UDF- FastB2B- ParErr- DEVSEL=medium >TAbort- <TAbort- <MAbort- >SERR- <PERR- INTx-
	Latency: 32 (2500ns min, 2500ns max), Cache Line Size: 64 bytes
	Interrupt: pin A routed to IRQ 37
	Region 0: I/O ports at c200 [size=128]
	Region 1: Memory at d8433000 (32-bit, non-prefetchable) [size=128]
	Expansion ROM at d8400000 [disabled] [size=128K]
	Capabilities: [dc] Power Management version 2
		Flags: PMEClk- DSI- D1+ D2+ AuxCurrent=0mA PME(D0+,D1+,D2+,D3hot+,D3cold-)
		Status: D0 NoSoftRst- PME-Enable- DSel=0 DScale=2 PME-
	Kernel driver in use: 3c59x
	Kernel modules: 3c59x

0a:05.0 Network controller: VMIC GE-IP PCI5565,PMC5565 Reflective Memory Node (rev 01)
	Subsystem: PLX Technology, Inc. Device 9656
	Control: I/O+ Mem+ BusMaster+ SpecCycle- MemWINV- VGASnoop- ParErr- Stepping- SERR- FastB2B- DisINTx-
	Status: Cap+ 66MHz+ UDF- FastB2B+ ParErr- DEVSEL=medium >TAbort- <TAbort- <MAbort- >SERR- <PERR- INTx-
	Latency: 32, Cache Line Size: 64 bytes
	Interrupt: pin A routed to IRQ 36
	Region 0: Memory at d8432000 (32-bit, non-prefetchable) [size=512]
	Region 1: I/O ports at c100 [size=256]
	Region 2: Memory at d8431000 (32-bit, non-prefetchable) [size=64]
	Region 3: Memory at d0000000 (32-bit, non-prefetchable) [size=128M]
	Capabilities: [40] Power Management version 0
		Flags: PMEClk- DSI- D1- D2- AuxCurrent=0mA PME(D0-,D1-,D2-,D3hot-,D3cold-)
		Status: D0 NoSoftRst- PME-Enable- DSel=0 DScale=0 PME-
	Capabilities: [48] #00 [0080]
	Kernel driver in use: rfm2g
	Kernel modules: rfm2g

0a:06.0 PCI bridge: Hint Corp HiNT HB4 PCI-PCI Bridge (PCI6150) (rev 04) (prog-if 00 [Normal decode])
	Control: I/O+ Mem+ BusMaster+ SpecCycle- MemWINV- VGASnoop- ParErr- Stepping+ SERR- FastB2B- DisINTx-
	Status: Cap+ 66MHz+ UDF- FastB2B+ ParErr- DEVSEL=medium >TAbort- <TAbort- <MAbort- >SERR- <PERR- INTx-
	Latency: 32, Cache Line Size: 64 bytes
	Bus: primary=0a, secondary=0b, subordinate=0b, sec-latency=32
	Memory behind bridge: d8300000-d83fffff
	Prefetchable memory behind bridge: 00000000c6000000-00000000c6ffffff
	Secondary status: 66MHz+ FastB2B+ ParErr- DEVSEL=medium >TAbort- <TAbort- <MAbort- <SERR- <PERR-
	BridgeCtl: Parity- SERR- NoISA- VGA- MAbort- >Reset- FastB2B-
		PriDiscTmr- SecDiscTmr- DiscTmrStat- DiscTmrSERREn-
	Capabilities: [dc] Power Management version 1
		Flags: PMEClk- DSI- D1+ D2+ AuxCurrent=0mA PME(D0+,D1+,D2+,D3hot+,D3cold-)
		Status: D0 NoSoftRst- PME-Enable- DSel=0 DScale=0 PME+
	Capabilities: [e4] CompactPCI hot-swap <?>
	Capabilities: [e8] Vital Product Data
		No end tag found
	Kernel modules: shpchp

0a:07.0 PCI bridge: Hint Corp HiNT HB4 PCI-PCI Bridge (PCI6150) (rev 04) (prog-if 00 [Normal decode])
	Control: I/O+ Mem+ BusMaster+ SpecCycle- MemWINV- VGASnoop- ParErr- Stepping+ SERR- FastB2B- DisINTx-
	Status: Cap+ 66MHz+ UDF- FastB2B+ ParErr- DEVSEL=medium >TAbort- <TAbort- <MAbort- >SERR- <PERR- INTx-
	Latency: 32, Cache Line Size: 64 bytes
	Bus: primary=0a, secondary=0c, subordinate=0c, sec-latency=32
	Memory behind bridge: d8200000-d82fffff
	Prefetchable memory behind bridge: 00000000c5000000-00000000c5ffffff
	Secondary status: 66MHz+ FastB2B+ ParErr- DEVSEL=medium >TAbort- <TAbort- <MAbort- <SERR- <PERR-
	BridgeCtl: Parity- SERR- NoISA- VGA- MAbort- >Reset- FastB2B-
		PriDiscTmr- SecDiscTmr- DiscTmrStat- DiscTmrSERREn-
	Capabilities: [dc] Power Management version 1
		Flags: PMEClk- DSI- D1+ D2+ AuxCurrent=0mA PME(D0+,D1+,D2+,D3hot+,D3cold-)
		Status: D0 NoSoftRst- PME-Enable- DSel=0 DScale=0 PME+
	Capabilities: [e4] CompactPCI hot-swap <?>
	Capabilities: [e8] Vital Product Data
		No end tag found
	Kernel modules: shpchp

0a:08.0 PCI bridge: Hint Corp HiNT HB4 PCI-PCI Bridge (PCI6150) (rev 04) (prog-if 00 [Normal decode])
	Control: I/O+ Mem+ BusMaster+ SpecCycle- MemWINV- VGASnoop- ParErr- Stepping+ SERR- FastB2B- DisINTx-
	Status: Cap+ 66MHz+ UDF- FastB2B+ ParErr- DEVSEL=medium >TAbort- <TAbort- <MAbort- >SERR- <PERR- INTx-
	Latency: 32, Cache Line Size: 64 bytes
	Bus: primary=0a, secondary=0d, subordinate=0d, sec-latency=32
	Memory behind bridge: d8100000-d81fffff
	Prefetchable memory behind bridge: 00000000c4000000-00000000c4ffffff
	Secondary status: 66MHz+ FastB2B+ ParErr- DEVSEL=medium >TAbort- <TAbort- <MAbort- <SERR- <PERR-
	BridgeCtl: Parity- SERR- NoISA- VGA- MAbort- >Reset- FastB2B-
		PriDiscTmr- SecDiscTmr- DiscTmrStat- DiscTmrSERREn-
	Capabilities: [dc] Power Management version 1
		Flags: PMEClk- DSI- D1+ D2+ AuxCurrent=0mA PME(D0+,D1+,D2+,D3hot+,D3cold-)
		Status: D0 NoSoftRst- PME-Enable- DSel=0 DScale=0 PME+
	Capabilities: [e4] CompactPCI hot-swap <?>
	Capabilities: [e8] Vital Product Data
		No end tag found
	Kernel modules: shpchp

0a:09.0 PCI bridge: Hint Corp HiNT HB4 PCI-PCI Bridge (PCI6150) (rev 04) (prog-if 00 [Normal decode])
	Control: I/O+ Mem+ BusMaster+ SpecCycle- MemWINV- VGASnoop- ParErr- Stepping+ SERR- FastB2B- DisINTx-
	Status: Cap+ 66MHz+ UDF- FastB2B+ ParErr- DEVSEL=medium >TAbort- <TAbort- <MAbort- >SERR- <PERR- INTx-
	Latency: 32, Cache Line Size: 64 bytes
	Bus: primary=0a, secondary=0e, subordinate=0e, sec-latency=32
	Memory behind bridge: d8000000-d80fffff
	Secondary status: 66MHz+ FastB2B+ ParErr- DEVSEL=medium >TAbort- <TAbort- <MAbort- <SERR- <PERR-
	BridgeCtl: Parity- SERR- NoISA- VGA- MAbort- >Reset- FastB2B-
		PriDiscTmr- SecDiscTmr- DiscTmrStat- DiscTmrSERREn-
	Capabilities: [dc] Power Management version 1
		Flags: PMEClk- DSI- D1+ D2+ AuxCurrent=0mA PME(D0+,D1+,D2+,D3hot+,D3cold-)
		Status: D0 NoSoftRst- PME-Enable- DSel=0 DScale=0 PME+
	Capabilities: [e4] CompactPCI hot-swap <?>
	Capabilities: [e8] Vital Product Data
		Unknown small resource type 00, will not decode more.
	Kernel modules: shpchp

0a:0a.0 SCSI storage controller: Adaptec AHA-2930CU (rev 03)
	Subsystem: Adaptec Device 3869
	Control: I/O- Mem+ BusMaster+ SpecCycle- MemWINV- VGASnoop- ParErr- Stepping- SERR- FastB2B- DisINTx-
	Status: Cap+ 66MHz- UDF- FastB2B+ ParErr- DEVSEL=medium >TAbort- <TAbort- <MAbort- >SERR- <PERR- INTx-
	Latency: 32 (1000ns min, 1000ns max), Cache Line Size: 64 bytes
	Interrupt: pin A routed to IRQ 32
	Region 0: I/O ports at c000 [disabled] [size=256]
	Region 1: Memory at d8430000 (32-bit, non-prefetchable) [size=4K]
	Expansion ROM at d8420000 [disabled] [size=64K]
	Capabilities: [dc] Power Management version 1
		Flags: PMEClk- DSI+ D1- D2- AuxCurrent=0mA PME(D0-,D1-,D2-,D3hot-,D3cold-)
		Status: D0 NoSoftRst- PME-Enable- DSel=0 DScale=0 PME-
	Kernel driver in use: aic7xxx
	Kernel modules: aic7xxx

0b:00.0 Unassigned class [ff00]: Compro Computer Services, Inc. Device 4710 (rev 41)
	Subsystem: Compro Computer Services, Inc. Device 4710
	Control: I/O+ Mem+ BusMaster+ SpecCycle- MemWINV- VGASnoop- ParErr- Stepping- SERR- FastB2B- DisINTx-
	Status: Cap+ 66MHz- UDF- FastB2B- ParErr- DEVSEL=slow >TAbort- <TAbort- <MAbort- >SERR- <PERR- INTx-
	Latency: 32
	Interrupt: pin A routed to IRQ 32
	Region 0: Memory at d8300000 (32-bit, non-prefetchable) [size=512]
	Region 1: Memory at c6000000 (32-bit, prefetchable) [size=16M]
	Capabilities: [80] Vital Product Data
		Unknown small resource type 00, will not decode more.
	Kernel driver in use: gpiohsd
	Kernel modules: gpiohsd

0c:00.0 Unassigned class [ff00]: Compro Computer Services, Inc. Device 4710 (rev 41)
	Subsystem: Compro Computer Services, Inc. Device 4710
	Control: I/O+ Mem+ BusMaster+ SpecCycle- MemWINV- VGASnoop- ParErr- Stepping- SERR- FastB2B- DisINTx-
	Status: Cap+ 66MHz- UDF- FastB2B- ParErr- DEVSEL=slow >TAbort- <TAbort- <MAbort- >SERR- <PERR- INTx-
	Latency: 32
	Interrupt: pin A routed to IRQ 44
	Region 0: Memory at d8200000 (32-bit, non-prefetchable) [size=512]
	Region 1: Memory at c5000000 (32-bit, prefetchable) [size=16M]
	Capabilities: [80] Vital Product Data
		Unknown small resource type 00, will not decode more.
	Kernel driver in use: gpiohsd
	Kernel modules: gpiohsd

0d:00.0 Unassigned class [ff00]: Compro Computer Services, Inc. Device 4710 (rev 41)
	Subsystem: Compro Computer Services, Inc. Device 4710
	Control: I/O+ Mem+ BusMaster+ SpecCycle- MemWINV- VGASnoop- ParErr- Stepping- SERR- FastB2B- DisINTx-
	Status: Cap+ 66MHz- UDF- FastB2B- ParErr- DEVSEL=slow >TAbort- <TAbort- <MAbort- >SERR- <PERR- INTx-
	Latency: 32
	Interrupt: pin A routed to IRQ 37
	Region 0: Memory at d8100000 (32-bit, non-prefetchable) [size=512]
	Region 1: Memory at c4000000 (32-bit, prefetchable) [size=16M]
	Capabilities: [80] Vital Product Data
		Unknown small resource type 00, will not decode more.
	Kernel driver in use: gpiohsd
	Kernel modules: gpiohsd

0e:00.0 Unassigned class [ff00]: Compro Computer Services, Inc. Device 0100 (rev 42)
	Subsystem: Compro Computer Services, Inc. Device 4700
	Control: I/O+ Mem+ BusMaster+ SpecCycle- MemWINV- VGASnoop- ParErr- Stepping- SERR- FastB2B- DisINTx-
	Status: Cap- 66MHz- UDF- FastB2B- ParErr- DEVSEL=slow >TAbort- <TAbort- <MAbort- >SERR- <PERR- INTx-
	Latency: 32
	Interrupt: pin A routed to IRQ 36
	Region 0: Memory at d800a000 (32-bit, non-prefetchable) [size=4K]
	Region 1: Memory at d8008000 (32-bit, non-prefetchable) [size=8K]
	Region 2: Memory at d8000000 (32-bit, non-prefetchable) [size=32K]
	Kernel driver in use: iom
	Kernel modules: pciiom

0f:00.0 PCI bridge: PLX Technology, Inc. PEX 8114 PCI Express-to-PCI/PCI-X Bridge (rev bc) (prog-if 00 [Normal decode])
	Control: I/O+ Mem+ BusMaster+ SpecCycle- MemWINV- VGASnoop- ParErr- Stepping- SERR- FastB2B- DisINTx-
	Status: Cap+ 66MHz- UDF- FastB2B- ParErr- DEVSEL=fast >TAbort- <TAbort- <MAbort- >SERR- <PERR- INTx-
	Latency: 0, Cache Line Size: 64 bytes
	Region 0: Memory at ddc00000 (32-bit, non-prefetchable) [size=8K]
	Bus: primary=0f, secondary=10, subordinate=10, sec-latency=32
	I/O behind bridge: 0000b000-0000bfff
	Memory behind bridge: ddb00000-ddbfffff
	Secondary status: 66MHz+ FastB2B+ ParErr- DEVSEL=slow >TAbort- <TAbort- <MAbort+ <SERR- <PERR-
	BridgeCtl: Parity- SERR- NoISA- VGA- MAbort- >Reset- FastB2B-
		PriDiscTmr- SecDiscTmr- DiscTmrStat- DiscTmrSERREn-
	Capabilities: [40] Power Management version 3
		Flags: PMEClk+ DSI- D1- D2- AuxCurrent=0mA PME(D0+,D1-,D2-,D3hot+,D3cold-)
		Status: D0 NoSoftRst+ PME-Enable- DSel=0 DScale=0 PME-
	Capabilities: [48] MSI: Enable- Count=1/1 Maskable- 64bit+
		Address: 0000000000000000  Data: 0000
	Capabilities: [58] PCI-X bridge device
		Secondary Status: 64bit+ 133MHz+ SCD- USC- SCO- SRD- Freq=133MHz
		Status: Dev=0f:00.0 64bit- 133MHz- SCD- USC- SCO+ SRD-
		Upstream: Capacity=16 CommitmentLimit=16
		Downstream: Capacity=16 CommitmentLimit=16
	Capabilities: [68] Express (v1) PCI-Express to PCI/PCI-X Bridge, MSI 00
		DevCap:	MaxPayload 256 bytes, PhantFunc 0
			ExtTag- AttnBtn+ AttnInd+ PwrInd+ RBE-
		DevCtl:	Report errors: Correctable- Non-Fatal- Fatal- Unsupported-
			RlxdOrd- ExtTag- PhantFunc- AuxPwr- NoSnoop- BrConfRtry-
			MaxPayload 128 bytes, MaxReadReq 2048 bytes
		DevSta:	CorrErr- UncorrErr+ FatalErr- UnsuppReq- AuxPwr- TransPend-
		LnkCap:	Port #0, Speed 2.5GT/s, Width x4, ASPM L0s L1, Exit Latency L0s <2us, L1 <32us
			ClockPM- Surprise- LLActRep- BwNot-
		LnkCtl:	ASPM Disabled; RCB 64 bytes Disabled- CommClk-
			ExtSynch- ClockPM- AutWidDis- BWInt- AutBWInt-
		LnkSta:	Speed 2.5GT/s, Width x1, TrErr- Train- SlotClk- DLActive- BWMgmt- ABWMgmt-
	Capabilities: [100 v1] Device Serial Number 00-00-00-01-00-00-0e-df
	Capabilities: [fb4 v1] Advanced Error Reporting
		UESta:	DLP- SDES- TLP- FCP- CmpltTO- CmpltAbrt- UnxCmplt- RxOF- MalfTLP- ECRC- UnsupReq+ ACSViol-
		UEMsk:	DLP- SDES- TLP- FCP- CmpltTO- CmpltAbrt- UnxCmplt- RxOF- MalfTLP- ECRC- UnsupReq- ACSViol-
		UESvrt:	DLP+ SDES- TLP- FCP+ CmpltTO- CmpltAbrt- UnxCmplt- RxOF+ MalfTLP+ ECRC- UnsupReq- ACSViol-
		CESta:	RxErr- BadTLP- BadDLLP- Rollover- Timeout- NonFatalErr-
		CEMsk:	RxErr- BadTLP- BadDLLP- Rollover- Timeout- NonFatalErr-
		AERCap:	First Error Pointer: 14, GenCap+ CGenEn- ChkCap+ ChkEn-
	Capabilities: [138 v1] Power Budgeting <?>
	Capabilities: [148 v1] Virtual Channel
		Caps:	LPEVC=0 RefClk=100ns PATEntryBits=1
		Arb:	Fixed- WRR32- WRR64- WRR128-
		Ctrl:	ArbSelect=Fixed
		Status:	InProgress-
		VC0:	Caps:	PATOffset=00 MaxTimeSlots=1 RejSnoopTrans-
			Arb:	Fixed- WRR32- WRR64- WRR128- TWRR128- WRR256-
			Ctrl:	Enable+ ID=0 ArbSelect=Fixed TC/VC=01
			Status:	NegoPending- InProgress-
	Kernel modules: shpchp

10:04.0 SCSI storage controller: Adaptec ASC-29320ALP U320 (rev 10)
	Subsystem: Adaptec ASC-29320LPE PCIe U320
	Control: I/O- Mem+ BusMaster+ SpecCycle- MemWINV- VGASnoop- ParErr- Stepping- SERR- FastB2B- DisINTx-
	Status: Cap+ 66MHz+ UDF- FastB2B- ParErr- DEVSEL=slow >TAbort- <TAbort- <MAbort- >SERR- <PERR- INTx-
	Latency: 32 (10000ns min, 6250ns max), Cache Line Size: 64 bytes
	Interrupt: pin A routed to IRQ 33
	Region 0: I/O ports at b100 [disabled] [size=256]
	Region 1: Memory at ddb80000 (64-bit, non-prefetchable) [size=8K]
	Region 3: I/O ports at b000 [disabled] [size=256]
	Expansion ROM at ddb00000 [disabled] [size=512K]
	Capabilities: [dc] Power Management version 2
		Flags: PMEClk- DSI- D1- D2- AuxCurrent=0mA PME(D0-,D1-,D2-,D3hot-,D3cold-)
		Status: D0 NoSoftRst- PME-Enable- DSel=0 DScale=0 PME-
	Capabilities: [a0] MSI: Enable- Count=1/2 Maskable- 64bit+
		Address: 0000000000000000  Data: 0000
	Capabilities: [94] PCI-X non-bridge device
		Command: DPERE- ERO+ RBC=2048 OST=8
		Status: Dev=10:04.0 64bit+ 133MHz+ SCD- USC- DC=simple DMMRBC=2048 DMOST=8 DMCRS=64 RSCEM- 266MHz- 533MHz-
	Kernel driver in use: aic79xx
	Kernel modules: aic79xx

11:06.0 Unassigned class [ff00]: Compro Computer Services, Inc. PCI RTOM (rev 02)
	Control: I/O+ Mem+ BusMaster+ SpecCycle- MemWINV- VGASnoop- ParErr- Stepping- SERR- FastB2B- DisINTx-
	Status: Cap- 66MHz- UDF- FastB2B- ParErr- DEVSEL=fast >TAbort- <TAbort- <MAbort- >SERR- <PERR- INTx-
	Latency: 32
	Interrupt: pin A routed to IRQ 20
	Region 0: I/O ports at a000 [size=64]
	Region 1: Memory at ddf00000 (32-bit, non-prefetchable) [size=4K]
	Kernel driver in use: rtom
	Kernel modules: rtom

12:00.0 Ethernet controller: Realtek Semiconductor Co., Ltd. RTL8111/8168/8411 PCI Express Gigabit Ethernet Controller (rev 06)
	Subsystem: Gigabyte Technology Co., Ltd Motherboard
	Control: I/O+ Mem+ BusMaster+ SpecCycle- MemWINV- VGASnoop- ParErr- Stepping- SERR- FastB2B- DisINTx+
	Status: Cap+ 66MHz- UDF- FastB2B- ParErr- DEVSEL=fast >TAbort- <TAbort- <MAbort- >SERR- <PERR- INTx-
	Latency: 0, Cache Line Size: 64 bytes
	Interrupt: pin A routed to IRQ 38
	Region 0: I/O ports at 9000 [size=256]
	Region 2: Memory at c7004000 (64-bit, prefetchable) [size=4K]
	Region 4: Memory at c7000000 (64-bit, prefetchable) [size=16K]
	Capabilities: [40] Power Management version 3
		Flags: PMEClk- DSI- D1+ D2+ AuxCurrent=375mA PME(D0+,D1+,D2+,D3hot+,D3cold+)
		Status: D0 NoSoftRst+ PME-Enable- DSel=0 DScale=0 PME-
	Capabilities: [50] MSI: Enable+ Count=1/1 Maskable- 64bit+
		Address: 00000000feeff00c  Data: 4183
	Capabilities: [70] Express (v2) Endpoint, MSI 01
		DevCap:	MaxPayload 128 bytes, PhantFunc 0, Latency L0s <512ns, L1 <64us
			ExtTag- AttnBtn- AttnInd- PwrInd- RBE+ FLReset-
		DevCtl:	Report errors: Correctable- Non-Fatal- Fatal- Unsupported-
			RlxdOrd- ExtTag- PhantFunc- AuxPwr- NoSnoop-
			MaxPayload 128 bytes, MaxReadReq 4096 bytes
		DevSta:	CorrErr+ UncorrErr- FatalErr- UnsuppReq+ AuxPwr+ TransPend-
		LnkCap:	Port #0, Speed 2.5GT/s, Width x1, ASPM L0s L1, Exit Latency L0s unlimited, L1 <64us
			ClockPM+ Surprise- LLActRep- BwNot-
		LnkCtl:	ASPM Disabled; RCB 64 bytes Disabled- CommClk+
			ExtSynch- ClockPM- AutWidDis- BWInt- AutBWInt-
		LnkSta:	Speed 2.5GT/s, Width x1, TrErr- Train- SlotClk+ DLActive- BWMgmt- ABWMgmt-
		DevCap2: Completion Timeout: Range ABCD, TimeoutDis+, LTR-, OBFF Not Supported
		DevCtl2: Completion Timeout: 50us to 50ms, TimeoutDis-, LTR-, OBFF Disabled
		LnkCtl2: Target Link Speed: 2.5GT/s, EnterCompliance- SpeedDis-
			 Transmit Margin: Normal Operating Range, EnterModifiedCompliance- ComplianceSOS-
			 Compliance De-emphasis: -6dB
		LnkSta2: Current De-emphasis Level: -6dB, EqualizationComplete-, EqualizationPhase1-
			 EqualizationPhase2-, EqualizationPhase3-, LinkEqualizationRequest-
	Capabilities: [b0] MSI-X: Enable- Count=4 Masked-
		Vector table: BAR=4 offset=00000000
		PBA: BAR=4 offset=00000800
	Capabilities: [d0] Vital Product Data
		Unknown small resource type 00, will not decode more.
	Capabilities: [100 v1] Advanced Error Reporting
		UESta:	DLP- SDES- TLP- FCP- CmpltTO+ CmpltAbrt- UnxCmplt- RxOF- MalfTLP- ECRC- UnsupReq+ ACSViol-
		UEMsk:	DLP- SDES- TLP- FCP- CmpltTO- CmpltAbrt- UnxCmplt- RxOF- MalfTLP- ECRC- UnsupReq- ACSViol-
		UESvrt:	DLP+ SDES+ TLP- FCP+ CmpltTO- CmpltAbrt- UnxCmplt- RxOF+ MalfTLP+ ECRC- UnsupReq- ACSViol-
		CESta:	RxErr- BadTLP- BadDLLP- Rollover- Timeout- NonFatalErr+
		CEMsk:	RxErr- BadTLP- BadDLLP- Rollover- Timeout- NonFatalErr+
		AERCap:	First Error Pointer: 0e, GenCap+ CGenEn- ChkCap+ ChkEn-
	Capabilities: [140 v1] Virtual Channel
		Caps:	LPEVC=0 RefClk=100ns PATEntryBits=1
		Arb:	Fixed- WRR32- WRR64- WRR128-
		Ctrl:	ArbSelect=Fixed
		Status:	InProgress-
		VC0:	Caps:	PATOffset=00 MaxTimeSlots=1 RejSnoopTrans-
			Arb:	Fixed- WRR32- WRR64- WRR128- TWRR128- WRR256-
			Ctrl:	Enable+ ID=0 ArbSelect=Fixed TC/VC=01
			Status:	NegoPending- InProgress-
	Capabilities: [160 v1] Device Serial Number 01-00-00-00-68-4c-e0-00
	Kernel driver in use: r8169
	Kernel modules: r8169

13:00.0 USB controller: Etron Technology, Inc. EJ168 USB 3.0 Host Controller (rev 01) (prog-if 30 [XHCI])
	Subsystem: Gigabyte Technology Co., Ltd Device 5007
	Control: I/O- Mem+ BusMaster+ SpecCycle- MemWINV- VGASnoop- ParErr- Stepping- SERR- FastB2B- DisINTx+
	Status: Cap+ 66MHz- UDF- FastB2B- ParErr- DEVSEL=fast >TAbort- <TAbort- <MAbort- >SERR- <PERR- INTx-
	Latency: 0, Cache Line Size: 64 bytes
	Interrupt: pin A routed to IRQ 42
	Region 0: Memory at dde00000 (64-bit, non-prefetchable) [size=32K]
	Capabilities: [50] Power Management version 3
		Flags: PMEClk- DSI- D1+ D2+ AuxCurrent=0mA PME(D0+,D1+,D2+,D3hot+,D3cold+)
		Status: D0 NoSoftRst- PME-Enable- DSel=0 DScale=0 PME-
	Capabilities: [70] MSI: Enable+ Count=1/4 Maskable+ 64bit+
		Address: 00000000fee7f00c  Data: 41c3
		Masking: 0000000e  Pending: 00000000
	Capabilities: [a0] Express (v2) Endpoint, MSI 00
		DevCap:	MaxPayload 1024 bytes, PhantFunc 0, Latency L0s <64ns, L1 <1us
			ExtTag+ AttnBtn- AttnInd- PwrInd- RBE+ FLReset+
		DevCtl:	Report errors: Correctable- Non-Fatal- Fatal- Unsupported-
			RlxdOrd- ExtTag- PhantFunc- AuxPwr- NoSnoop+ FLReset-
			MaxPayload 128 bytes, MaxReadReq 512 bytes
		DevSta:	CorrErr+ UncorrErr+ FatalErr- UnsuppReq+ AuxPwr+ TransPend-
		LnkCap:	Port #0, Speed 5GT/s, Width x1, ASPM L0s L1, Exit Latency L0s <1us, L1 <64us
			ClockPM+ Surprise- LLActRep- BwNot-
		LnkCtl:	ASPM Disabled; RCB 64 bytes Disabled- CommClk+
			ExtSynch- ClockPM- AutWidDis- BWInt- AutBWInt-
		LnkSta:	Speed 5GT/s, Width x1, TrErr- Train- SlotClk+ DLActive- BWMgmt- ABWMgmt-
		DevCap2: Completion Timeout: Not Supported, TimeoutDis-, LTR-, OBFF Not Supported
		DevCtl2: Completion Timeout: 50us to 50ms, TimeoutDis-, LTR-, OBFF Disabled
		LnkCtl2: Target Link Speed: 5GT/s, EnterCompliance- SpeedDis-
			 Transmit Margin: Normal Operating Range, EnterModifiedCompliance- ComplianceSOS-
			 Compliance De-emphasis: -6dB
		LnkSta2: Current De-emphasis Level: -6dB, EqualizationComplete-, EqualizationPhase1-
			 EqualizationPhase2-, EqualizationPhase3-, LinkEqualizationRequest-
	Capabilities: [100 v1] Advanced Error Reporting
		UESta:	DLP- SDES- TLP- FCP- CmpltTO- CmpltAbrt- UnxCmplt- RxOF- MalfTLP- ECRC- UnsupReq+ ACSViol-
		UEMsk:	DLP- SDES- TLP- FCP- CmpltTO- CmpltAbrt- UnxCmplt- RxOF- MalfTLP- ECRC- UnsupReq- ACSViol-
		UESvrt:	DLP+ SDES- TLP- FCP+ CmpltTO- CmpltAbrt- UnxCmplt- RxOF+ MalfTLP+ ECRC- UnsupReq- ACSViol-
		CESta:	RxErr- BadTLP- BadDLLP- Rollover- Timeout- NonFatalErr+
		CEMsk:	RxErr- BadTLP- BadDLLP- Rollover- Timeout- NonFatalErr+
		AERCap:	First Error Pointer: 14, GenCap+ CGenEn- ChkCap+ ChkEn-
	Capabilities: [190 v1] Device Serial Number 01-01-01-01-01-01-01-01
	Kernel driver in use: xhci_hcd
	Kernel modules: xhci_pci

14:00.0 SATA controller: Marvell Technology Group Ltd. 88SE9172 SATA 6Gb/s Controller (rev 12) (prog-if 01 [AHCI 1.0])
	Subsystem: Gigabyte Technology Co., Ltd Device b000
	Control: I/O+ Mem+ BusMaster+ SpecCycle- MemWINV- VGASnoop- ParErr- Stepping- SERR- FastB2B- DisINTx+
	Status: Cap+ 66MHz- UDF- FastB2B- ParErr- DEVSEL=fast >TAbort- <TAbort- <MAbort- >SERR- <PERR- INTx-
	Latency: 0, Cache Line Size: 64 bytes
	Interrupt: pin A routed to IRQ 31
	Region 0: I/O ports at 8040 [size=8]
	Region 1: I/O ports at 8030 [size=4]
	Region 2: I/O ports at 8020 [size=8]
	Region 3: I/O ports at 8010 [size=4]
	Region 4: I/O ports at 8000 [size=16]
	Region 5: Memory at ddd10000 (32-bit, non-prefetchable) [size=512]
	Expansion ROM at ddd00000 [disabled] [size=64K]
	Capabilities: [40] Power Management version 3
		Flags: PMEClk- DSI- D1- D2- AuxCurrent=0mA PME(D0-,D1-,D2-,D3hot+,D3cold-)
		Status: D0 NoSoftRst- PME-Enable- DSel=0 DScale=0 PME-
	Capabilities: [50] MSI: Enable+ Count=1/1 Maskable- 64bit-
		Address: feeff00c  Data: 41b2
	Capabilities: [70] Express (v2) Legacy Endpoint, MSI 00
		DevCap:	MaxPayload 512 bytes, PhantFunc 0, Latency L0s <1us, L1 <8us
			ExtTag- AttnBtn- AttnInd- PwrInd- RBE+ FLReset-
		DevCtl:	Report errors: Correctable- Non-Fatal- Fatal- Unsupported-
			RlxdOrd- ExtTag- PhantFunc- AuxPwr- NoSnoop-
			MaxPayload 128 bytes, MaxReadReq 512 bytes
		DevSta:	CorrErr+ UncorrErr- FatalErr- UnsuppReq+ AuxPwr- TransPend-
		LnkCap:	Port #0, Speed 5GT/s, Width x1, ASPM L0s L1, Exit Latency L0s <512ns, L1 <64us
			ClockPM- Surprise- LLActRep- BwNot-
		LnkCtl:	ASPM Disabled; RCB 64 bytes Disabled- CommClk+
			ExtSynch- ClockPM- AutWidDis- BWInt- AutBWInt-
		LnkSta:	Speed 5GT/s, Width x1, TrErr- Train- SlotClk+ DLActive- BWMgmt- ABWMgmt-
		DevCap2: Completion Timeout: Not Supported, TimeoutDis+, LTR-, OBFF Not Supported
		DevCtl2: Completion Timeout: 50us to 50ms, TimeoutDis-, LTR-, OBFF Disabled
		LnkCtl2: Target Link Speed: 5GT/s, EnterCompliance- SpeedDis-
			 Transmit Margin: Normal Operating Range, EnterModifiedCompliance- ComplianceSOS-
			 Compliance De-emphasis: -6dB
		LnkSta2: Current De-emphasis Level: -3.5dB, EqualizationComplete-, EqualizationPhase1-
			 EqualizationPhase2-, EqualizationPhase3-, LinkEqualizationRequest-
	Capabilities: [100 v1] Advanced Error Reporting
		UESta:	DLP- SDES- TLP- FCP- CmpltTO- CmpltAbrt- UnxCmplt- RxOF- MalfTLP- ECRC- UnsupReq- ACSViol-
		UEMsk:	DLP- SDES- TLP- FCP- CmpltTO- CmpltAbrt- UnxCmplt- RxOF- MalfTLP- ECRC- UnsupReq- ACSViol-
		UESvrt:	DLP+ SDES+ TLP- FCP+ CmpltTO- CmpltAbrt- UnxCmplt- RxOF+ MalfTLP+ ECRC- UnsupReq- ACSViol-
		CESta:	RxErr- BadTLP- BadDLLP- Rollover- Timeout- NonFatalErr+
		CEMsk:	RxErr- BadTLP- BadDLLP- Rollover- Timeout- NonFatalErr+
		AERCap:	First Error Pointer: 00, GenCap- CGenEn- ChkCap- ChkEn-
	Kernel driver in use: ahci


[-- Attachment #3: Type: text/plain, Size: 0 bytes --]

Re: [PATCH v2 4/7] DMA-API: Add dma_(un)map_resource() documentation

[Alex Williamson]

On Tue, 2015-07-07 at 13:14 -0400, Mark Hounschell wrote:
> Hi Bjorn.
> 
> On 07/07/2015 11:15 AM, Bjorn Helgaas wrote:
> > [+cc Alex]
> >
> > Hi Mark,
> >
> > On Wed, May 20, 2015 at 08:11:17AM -0400, Mark Hounschell wrote:
> >> Most currently available hardware doesn't allow reads but will allow
> >> writes on PCIe peer-to-peer transfers. All current AMD chipsets are
> >> this way. I'm pretty sure all Intel chipsets are this way also. What
> >> happens with reads is they are just dropped with no indication of
> >> error other than the data will not be as expected. Supposedly the
> >> PCIe spec does not even require any peer-to-peer support. Regular
> >> PCI there is no problem and this API could be useful. However I
> >> doubt seriously you will find a pure PCI motherboard that has an
> >> IOMMU.
> >>
> >> I don't understand the chipset manufactures reasoning for disabling
> >> PCIe peer-to-peer reads. We would like to make PCIe versions of our
> >> cards but their application requires  peer-to-peer reads and writes.
> >> So we cannot develop PCIe versions of the cards.
> >
> > I'd like to understand this better.  Peer-to-peer between two devices
> > below the same Root Port should work as long as ACS doesn't prevent
> > it.  If we find an Intel or AMD IOMMU, I think we configure ACS to
> > prevent direct peer-to-peer (see "pci_acs_enable"), but maybe it could
> > still be done with the appropriate IOMMU support.  And if you boot
> > with "iommu=off", we don't do that ACS configuration, so peer-to-peer
> > should work.
> >
> > I suppose the problem is that peer-to-peer doesn't work between
> > devices under different Root Ports or even devices under different
> > Root Complexes?
> >
> > PCIe r3.0, sec 6.12.1.1, says Root Ports that support peer-to-peer
> > traffic are required to implement ACS P2P Request Redirect, so if a
> > Root Port doesn't implement RR, we can assume it doesn't support
> > peer-to-peer.  But unfortunately the converse is not true: if a Root
> > Port implements RR, that does *not* imply that it supports
> > peer-to-peer traffic.
> >
> > So I don't know how to discover whether peer-to-peer between Root
> > Ports or Root Complexes is supported.  Maybe there's some clue in the
> > IOMMU?  The Intel VT-d spec mentions it, but "peer" doesn't even
> > appear in the AMD spec.
> >
> > And I'm curious about why writes sometimes work when reads do not.
> > That sounds like maybe the hardware support is there, but we don't
> > understand how to configure everything correctly.
> >
> > Can you give us the specifics of the topology you'd like to use, e.g.,
> > lspci -vv of the path between the two devices?
> >
> 
> First off, writes always work for me. Not just sometimes. Only reads 
> NEVER do.
> 
> Reading the AMD-990FX-990X-970-Register-Programming-Requirements-48693.pdf
> in section 2.5 "Enabling/Disabling Peer-to-Peer Traffic Access", it 
> states specifically that
> only P2P memory writes are supported. This has been the case with older 
> AMD chipset also. In one of the older chipset documents I read (I think 
> the 770 series) , it said this was a security feature. Makes no sense to 
> me.
> 
> As for the topology I'd like to be able to use. This particular 
> configuration (MB) has a single regular pci slot and the rest are pci-e. 
> In two of those pci-e slots is a pci-e to pci expansion chassis 
> interface card connected to a regular pci expansion rack. I am trying to 
> to peer to peer between a regular pci card in one of those chassis to 
> another regular pci card in the other chassis. In turn through the pci-e 
> subsystem. Attached is the lcpci -vv output from this particular box. 
> The cards that initiate the P2P are these:
> 
> 04:04.0 Intelligent controller [0e80]: PLX Technology, Inc. Device 0480 
> (rev 55)
> 04:05.0 Intelligent controller [0e80]: PLX Technology, Inc. Device 0480 
> (rev 55)
> 04:06.0 Intelligent controller [0e80]: PLX Technology, Inc. Device 0480 
> (rev 55)
> 04:07.0 Intelligent controller [0e80]: PLX Technology, Inc. Device 0480 
> (rev 55)
> 
> The card they need to P2P to and from is this one.
> 
> 0a:05.0 Network controller: VMIC GE-IP PCI5565,PMC5565 Reflective Memory 
> Node (rev 01)
> 
> Like wise, reversing the chassis the initiator lives in from these cards.
> 
> 0b:00.0 Unassigned class [ff00]: Compro Computer Services, Inc. Device 
> 4710 (rev 41)
> 0c:00.0 Unassigned class [ff00]: Compro Computer Services, Inc. Device 
> 4710 (rev 41)
> 0d:00.0 Unassigned class [ff00]: Compro Computer Services, Inc. Device 
> 4710 (rev 41)
> 0e:00.0 Unassigned class [ff00]: Compro Computer Services, Inc. Device 
> 0100 (rev 42)
> 
> to this card
> 
> 04:0a.0 Memory controller: Compro Computer Services, Inc. Device 4360 
> (rev 4d)
> 
> Again, I can go between both pci chassis as long as I am doing writes. 
> Only reads do not work.
> 
> I can send the AMD-990FX-990X-970-Register-Programming-Requirements if 
> you would like. It's available for download on AMD web site. Let me know.


It would be interesting to know if this already works if you assign all
the endpoints to a QEMU/KVM VM with vfio-pci.  We make an attempt to map
the device MMIO BARs through the IOMMU, but as I said, I don't know how
to test it.  Does the register programming guide provide any indication
if there are any restrictions on p2p when bounced through the IOMMU?  So
long as the IOMMU does the translation and redirection, I don't see why
the rest of the topology would handle it differently than a DMA to
memory.  Thanks,

Alex

Re: [PATCH v2 4/7] DMA-API: Add dma_(un)map_resource() documentation

[Mark Hounschell]

On 07/07/2015 01:28 PM, Alex Williamson wrote:
> On Tue, 2015-07-07 at 13:14 -0400, Mark Hounschell wrote:
>> Hi Bjorn.
>>
>> On 07/07/2015 11:15 AM, Bjorn Helgaas wrote:
>>> [+cc Alex]
>>>
>>> Hi Mark,
>>>
>>> On Wed, May 20, 2015 at 08:11:17AM -0400, Mark Hounschell wrote:
>>>> Most currently available hardware doesn't allow reads but will allow
>>>> writes on PCIe peer-to-peer transfers. All current AMD chipsets are
>>>> this way. I'm pretty sure all Intel chipsets are this way also. What
>>>> happens with reads is they are just dropped with no indication of
>>>> error other than the data will not be as expected. Supposedly the
>>>> PCIe spec does not even require any peer-to-peer support. Regular
>>>> PCI there is no problem and this API could be useful. However I
>>>> doubt seriously you will find a pure PCI motherboard that has an
>>>> IOMMU.
>>>>
>>>> I don't understand the chipset manufactures reasoning for disabling
>>>> PCIe peer-to-peer reads. We would like to make PCIe versions of our
>>>> cards but their application requires  peer-to-peer reads and writes.
>>>> So we cannot develop PCIe versions of the cards.
>>>
>>> I'd like to understand this better.  Peer-to-peer between two devices
>>> below the same Root Port should work as long as ACS doesn't prevent
>>> it.  If we find an Intel or AMD IOMMU, I think we configure ACS to
>>> prevent direct peer-to-peer (see "pci_acs_enable"), but maybe it could
>>> still be done with the appropriate IOMMU support.  And if you boot
>>> with "iommu=off", we don't do that ACS configuration, so peer-to-peer
>>> should work.
>>>
>>> I suppose the problem is that peer-to-peer doesn't work between
>>> devices under different Root Ports or even devices under different
>>> Root Complexes?
>>>
>>> PCIe r3.0, sec 6.12.1.1, says Root Ports that support peer-to-peer
>>> traffic are required to implement ACS P2P Request Redirect, so if a
>>> Root Port doesn't implement RR, we can assume it doesn't support
>>> peer-to-peer.  But unfortunately the converse is not true: if a Root
>>> Port implements RR, that does *not* imply that it supports
>>> peer-to-peer traffic.
>>>
>>> So I don't know how to discover whether peer-to-peer between Root
>>> Ports or Root Complexes is supported.  Maybe there's some clue in the
>>> IOMMU?  The Intel VT-d spec mentions it, but "peer" doesn't even
>>> appear in the AMD spec.
>>>
>>> And I'm curious about why writes sometimes work when reads do not.
>>> That sounds like maybe the hardware support is there, but we don't
>>> understand how to configure everything correctly.
>>>
>>> Can you give us the specifics of the topology you'd like to use, e.g.,
>>> lspci -vv of the path between the two devices?
>>>
>>
>> First off, writes always work for me. Not just sometimes. Only reads
>> NEVER do.
>>
>> Reading the AMD-990FX-990X-970-Register-Programming-Requirements-48693.pdf
>> in section 2.5 "Enabling/Disabling Peer-to-Peer Traffic Access", it
>> states specifically that
>> only P2P memory writes are supported. This has been the case with older
>> AMD chipset also. In one of the older chipset documents I read (I think
>> the 770 series) , it said this was a security feature. Makes no sense to
>> me.
>>
>> As for the topology I'd like to be able to use. This particular
>> configuration (MB) has a single regular pci slot and the rest are pci-e.
>> In two of those pci-e slots is a pci-e to pci expansion chassis
>> interface card connected to a regular pci expansion rack. I am trying to
>> to peer to peer between a regular pci card in one of those chassis to
>> another regular pci card in the other chassis. In turn through the pci-e
>> subsystem. Attached is the lcpci -vv output from this particular box.
>> The cards that initiate the P2P are these:
>>
>> 04:04.0 Intelligent controller [0e80]: PLX Technology, Inc. Device 0480
>> (rev 55)
>> 04:05.0 Intelligent controller [0e80]: PLX Technology, Inc. Device 0480
>> (rev 55)
>> 04:06.0 Intelligent controller [0e80]: PLX Technology, Inc. Device 0480
>> (rev 55)
>> 04:07.0 Intelligent controller [0e80]: PLX Technology, Inc. Device 0480
>> (rev 55)
>>
>> The card they need to P2P to and from is this one.
>>
>> 0a:05.0 Network controller: VMIC GE-IP PCI5565,PMC5565 Reflective Memory
>> Node (rev 01)
>>
>> Like wise, reversing the chassis the initiator lives in from these cards.
>>
>> 0b:00.0 Unassigned class [ff00]: Compro Computer Services, Inc. Device
>> 4710 (rev 41)
>> 0c:00.0 Unassigned class [ff00]: Compro Computer Services, Inc. Device
>> 4710 (rev 41)
>> 0d:00.0 Unassigned class [ff00]: Compro Computer Services, Inc. Device
>> 4710 (rev 41)
>> 0e:00.0 Unassigned class [ff00]: Compro Computer Services, Inc. Device
>> 0100 (rev 42)
>>
>> to this card
>>
>> 04:0a.0 Memory controller: Compro Computer Services, Inc. Device 4360
>> (rev 4d)
>>
>> Again, I can go between both pci chassis as long as I am doing writes.
>> Only reads do not work.
>>
>> I can send the AMD-990FX-990X-970-Register-Programming-Requirements if
>> you would like. It's available for download on AMD web site. Let me know.
>
>
> It would be interesting to know if this already works if you assign all
> the endpoints to a QEMU/KVM VM with vfio-pci.  We make an attempt to map
> the device MMIO BARs through the IOMMU, but as I said, I don't know how
> to test it.  Does the register programming guide provide any indication
> if there are any restrictions on p2p when bounced through the IOMMU?  So
> long as the IOMMU does the translation and redirection, I don't see why
> the rest of the topology would handle it differently than a DMA to
> memory.  Thanks,
>

Hi Alex,

Somehow I don't think "assigning all the endpoints to a QEMU/KVM VM with 
vfio-pci" would be an easy thing for me to do. I have never used 
QEMU/KVM VM and my particular application is already an emulation. Just 
not an emulation that could use QEMU/KVM. It's an emulation of a totally 
different arch, unknown to any VM. But what I do do, is basically "map 
the device MMIO BARs through the IOMMU". Reads have never worked for me 
even when there was no iommu available. One of the reasons I started 
using the iommu was because I was "hoping" it would fix my long standing 
problems with p2p reads through pcie. The other reason was, I no longer 
had to do (buggie) DAC crap with my 32 bit pci cards.

As far as the manual saying anything about p2p when the iommu is used, 
it actually says nothing about p2p at all in the iommu section, nor the 
iommu in the p2p section.

Regards
Mark

Re: [PATCH v2 4/7] DMA-API: Add dma_(un)map_resource() documentation

[Alex Williamson]

On Tue, 2015-07-07 at 15:17 -0400, Mark Hounschell wrote:
> On 07/07/2015 01:28 PM, Alex Williamson wrote:
> > On Tue, 2015-07-07 at 13:14 -0400, Mark Hounschell wrote:
> >> Hi Bjorn.
> >>
> >> On 07/07/2015 11:15 AM, Bjorn Helgaas wrote:
> >>> [+cc Alex]
> >>>
> >>> Hi Mark,
> >>>
> >>> On Wed, May 20, 2015 at 08:11:17AM -0400, Mark Hounschell wrote:
> >>>> Most currently available hardware doesn't allow reads but will allow
> >>>> writes on PCIe peer-to-peer transfers. All current AMD chipsets are
> >>>> this way. I'm pretty sure all Intel chipsets are this way also. What
> >>>> happens with reads is they are just dropped with no indication of
> >>>> error other than the data will not be as expected. Supposedly the
> >>>> PCIe spec does not even require any peer-to-peer support. Regular
> >>>> PCI there is no problem and this API could be useful. However I
> >>>> doubt seriously you will find a pure PCI motherboard that has an
> >>>> IOMMU.
> >>>>
> >>>> I don't understand the chipset manufactures reasoning for disabling
> >>>> PCIe peer-to-peer reads. We would like to make PCIe versions of our
> >>>> cards but their application requires  peer-to-peer reads and writes.
> >>>> So we cannot develop PCIe versions of the cards.
> >>>
> >>> I'd like to understand this better.  Peer-to-peer between two devices
> >>> below the same Root Port should work as long as ACS doesn't prevent
> >>> it.  If we find an Intel or AMD IOMMU, I think we configure ACS to
> >>> prevent direct peer-to-peer (see "pci_acs_enable"), but maybe it could
> >>> still be done with the appropriate IOMMU support.  And if you boot
> >>> with "iommu=off", we don't do that ACS configuration, so peer-to-peer
> >>> should work.
> >>>
> >>> I suppose the problem is that peer-to-peer doesn't work between
> >>> devices under different Root Ports or even devices under different
> >>> Root Complexes?
> >>>
> >>> PCIe r3.0, sec 6.12.1.1, says Root Ports that support peer-to-peer
> >>> traffic are required to implement ACS P2P Request Redirect, so if a
> >>> Root Port doesn't implement RR, we can assume it doesn't support
> >>> peer-to-peer.  But unfortunately the converse is not true: if a Root
> >>> Port implements RR, that does *not* imply that it supports
> >>> peer-to-peer traffic.
> >>>
> >>> So I don't know how to discover whether peer-to-peer between Root
> >>> Ports or Root Complexes is supported.  Maybe there's some clue in the
> >>> IOMMU?  The Intel VT-d spec mentions it, but "peer" doesn't even
> >>> appear in the AMD spec.
> >>>
> >>> And I'm curious about why writes sometimes work when reads do not.
> >>> That sounds like maybe the hardware support is there, but we don't
> >>> understand how to configure everything correctly.
> >>>
> >>> Can you give us the specifics of the topology you'd like to use, e.g.,
> >>> lspci -vv of the path between the two devices?
> >>>
> >>
> >> First off, writes always work for me. Not just sometimes. Only reads
> >> NEVER do.
> >>
> >> Reading the AMD-990FX-990X-970-Register-Programming-Requirements-48693.pdf
> >> in section 2.5 "Enabling/Disabling Peer-to-Peer Traffic Access", it
> >> states specifically that
> >> only P2P memory writes are supported. This has been the case with older
> >> AMD chipset also. In one of the older chipset documents I read (I think
> >> the 770 series) , it said this was a security feature. Makes no sense to
> >> me.
> >>
> >> As for the topology I'd like to be able to use. This particular
> >> configuration (MB) has a single regular pci slot and the rest are pci-e.
> >> In two of those pci-e slots is a pci-e to pci expansion chassis
> >> interface card connected to a regular pci expansion rack. I am trying to
> >> to peer to peer between a regular pci card in one of those chassis to
> >> another regular pci card in the other chassis. In turn through the pci-e
> >> subsystem. Attached is the lcpci -vv output from this particular box.
> >> The cards that initiate the P2P are these:
> >>
> >> 04:04.0 Intelligent controller [0e80]: PLX Technology, Inc. Device 0480
> >> (rev 55)
> >> 04:05.0 Intelligent controller [0e80]: PLX Technology, Inc. Device 0480
> >> (rev 55)
> >> 04:06.0 Intelligent controller [0e80]: PLX Technology, Inc. Device 0480
> >> (rev 55)
> >> 04:07.0 Intelligent controller [0e80]: PLX Technology, Inc. Device 0480
> >> (rev 55)
> >>
> >> The card they need to P2P to and from is this one.
> >>
> >> 0a:05.0 Network controller: VMIC GE-IP PCI5565,PMC5565 Reflective Memory
> >> Node (rev 01)
> >>
> >> Like wise, reversing the chassis the initiator lives in from these cards.
> >>
> >> 0b:00.0 Unassigned class [ff00]: Compro Computer Services, Inc. Device
> >> 4710 (rev 41)
> >> 0c:00.0 Unassigned class [ff00]: Compro Computer Services, Inc. Device
> >> 4710 (rev 41)
> >> 0d:00.0 Unassigned class [ff00]: Compro Computer Services, Inc. Device
> >> 4710 (rev 41)
> >> 0e:00.0 Unassigned class [ff00]: Compro Computer Services, Inc. Device
> >> 0100 (rev 42)
> >>
> >> to this card
> >>
> >> 04:0a.0 Memory controller: Compro Computer Services, Inc. Device 4360
> >> (rev 4d)
> >>
> >> Again, I can go between both pci chassis as long as I am doing writes.
> >> Only reads do not work.
> >>
> >> I can send the AMD-990FX-990X-970-Register-Programming-Requirements if
> >> you would like. It's available for download on AMD web site. Let me know.
> >
> >
> > It would be interesting to know if this already works if you assign all
> > the endpoints to a QEMU/KVM VM with vfio-pci.  We make an attempt to map
> > the device MMIO BARs through the IOMMU, but as I said, I don't know how
> > to test it.  Does the register programming guide provide any indication
> > if there are any restrictions on p2p when bounced through the IOMMU?  So
> > long as the IOMMU does the translation and redirection, I don't see why
> > the rest of the topology would handle it differently than a DMA to
> > memory.  Thanks,
> >
> 
> Hi Alex,
> 
> Somehow I don't think "assigning all the endpoints to a QEMU/KVM VM with 
> vfio-pci" would be an easy thing for me to do. I have never used 
> QEMU/KVM VM and my particular application is already an emulation. Just 
> not an emulation that could use QEMU/KVM. It's an emulation of a totally 
> different arch, unknown to any VM.

Hi Mark,

Whether the arch is known to the VM is irrelevant, it would simply be an
application running inside an x86 VM with, hopefully, p2p DMA possible
between devices.

>  But what I do do, is basically "map 
> the device MMIO BARs through the IOMMU". Reads have never worked for me 
> even when there was no iommu available. One of the reasons I started 
> using the iommu was because I was "hoping" it would fix my long standing 
> problems with p2p reads through pcie. The other reason was, I no longer 
> had to do (buggie) DAC crap with my 32 bit pci cards.

The thought was that running inside a VM with device assignment would
make all of this transparent because the p2p mappings for the IOMMU are
already configured.  But, we're depending on the IOMMU being able to do
translation (and me to have gotten the mappings right) and redirect the
request back out to I/O rather than memory, which it seems like
something you've already tried.  So maybe it's just another data point
using an alternative approach.  Thanks,

Alex

Re: [PATCH v2 4/7] DMA-API: Add dma_(un)map_resource() documentation

[Bjorn Helgaas]

[+cc Rafael]

On Tue, Jul 07, 2015 at 01:14:27PM -0400, Mark Hounschell wrote:
> On 07/07/2015 11:15 AM, Bjorn Helgaas wrote:
> >On Wed, May 20, 2015 at 08:11:17AM -0400, Mark Hounschell wrote:
> >>Most currently available hardware doesn't allow reads but will allow
> >>writes on PCIe peer-to-peer transfers. All current AMD chipsets are
> >>this way. I'm pretty sure all Intel chipsets are this way also. What
> >>happens with reads is they are just dropped with no indication of
> >>error other than the data will not be as expected. Supposedly the
> >>PCIe spec does not even require any peer-to-peer support. Regular
> >>PCI there is no problem and this API could be useful. However I
> >>doubt seriously you will find a pure PCI motherboard that has an
> >>IOMMU.
> >>
> >>I don't understand the chipset manufactures reasoning for disabling
> >>PCIe peer-to-peer reads. We would like to make PCIe versions of our
> >>cards but their application requires  peer-to-peer reads and writes.
> >>So we cannot develop PCIe versions of the cards.
> >
> >I'd like to understand this better.  Peer-to-peer between two devices
> >below the same Root Port should work as long as ACS doesn't prevent
> >it.  If we find an Intel or AMD IOMMU, I think we configure ACS to
> >prevent direct peer-to-peer (see "pci_acs_enable"), but maybe it could
> >still be done with the appropriate IOMMU support.  And if you boot
> >with "iommu=off", we don't do that ACS configuration, so peer-to-peer
> >should work.
> >
> >I suppose the problem is that peer-to-peer doesn't work between
> >devices under different Root Ports or even devices under different
> >Root Complexes?
> >
> >PCIe r3.0, sec 6.12.1.1, says Root Ports that support peer-to-peer
> >traffic are required to implement ACS P2P Request Redirect, so if a
> >Root Port doesn't implement RR, we can assume it doesn't support
> >peer-to-peer.  But unfortunately the converse is not true: if a Root
> >Port implements RR, that does *not* imply that it supports
> >peer-to-peer traffic.
> >
> >So I don't know how to discover whether peer-to-peer between Root
> >Ports or Root Complexes is supported.  Maybe there's some clue in the
> >IOMMU?  The Intel VT-d spec mentions it, but "peer" doesn't even
> >appear in the AMD spec.
> >
> >And I'm curious about why writes sometimes work when reads do not.
> >That sounds like maybe the hardware support is there, but we don't
> >understand how to configure everything correctly.
> >
> >Can you give us the specifics of the topology you'd like to use, e.g.,
> >lspci -vv of the path between the two devices?
> 
> First off, writes always work for me. Not just sometimes. Only reads
> NEVER do.
> 
> Reading the AMD-990FX-990X-970-Register-Programming-Requirements-48693.pdf
> in section 2.5 "Enabling/Disabling Peer-to-Peer Traffic Access", it
> states specifically that
> only P2P memory writes are supported. This has been the case with
> older AMD chipset also. In one of the older chipset documents I read
> (I think the 770 series) , it said this was a security feature.
> Makes no sense to me.
> 
> As for the topology I'd like to be able to use. This particular
> configuration (MB) has a single regular pci slot and the rest are
> pci-e. In two of those pci-e slots is a pci-e to pci expansion
> chassis interface card connected to a regular pci expansion rack. I
> am trying to to peer to peer between a regular pci card in one of
> those chassis to another regular pci card in the other chassis. In
> turn through the pci-e subsystem. Attached is the lcpci -vv output
> from this particular box. The cards that initiate the P2P are these:
> 
> 04:04.0 Intelligent controller [0e80]: PLX Technology, Inc. Device
> 0480 (rev 55)
> 04:05.0 Intelligent controller [0e80]: PLX Technology, Inc. Device
> 0480 (rev 55)
> 04:06.0 Intelligent controller [0e80]: PLX Technology, Inc. Device
> 0480 (rev 55)
> 04:07.0 Intelligent controller [0e80]: PLX Technology, Inc. Device
> 0480 (rev 55)
> 
> The card they need to P2P to and from is this one.
> 
> 0a:05.0 Network controller: VMIC GE-IP PCI5565,PMC5565 Reflective
> Memory Node (rev 01)

Peer-to-peer traffic initiated by 04:04.0 and targeted at 0a:05.0 has
to be routed up to Root Port 00:04.0, over to Root Port 00:0b.0, and
back down to 0a:05.0:

  00:04.0: Root Port to [bus 02-05] Slot #4 ACS ReqRedir+
  02:00.0: PCIe-to-PCI bridge to [bus 03-05]
  03:04.0: PCI-to-PCI bridge to [bus 04-05]
  04:04.0: PLX intelligent controller

  00:0b.0: Root Port to [bus 08-0e] Slot #11 ACS ReqRedir+
  00:0b.0:   bridge window [mem 0xd0000000-0xd84fffff]
  08:00.0: PCIe-to-PCI bridge to [bus 09-0e]
  08:00.0:   bridge window [mem 0xd0000000-0xd84fffff]
  09:04.0: PCI-to-PCI bridge to [bus 0a-0e]
  09:04.0:   bridge window [mem 0xd0000000-0xd84fffff]
  0a:05.0: VMIC GE-IP reflective memory node
  0a:05.0: BAR 3 [mem 0xd0000000-0xd7ffffff]

Both Root Ports do support ACS, including P2P RR, but that doesn't
tell us anything about whether the Root Complex actually supports
peer-to-peer traffic between the Root Ports.  Per the AMD
990FX/990X/970 spec, your hardware supports it for writes but not
reads.

So your hardware is what it is, and a general-purpose interface should 
probably not allow peer-to-peer at all unless we wanted to complicate
it by adding a read vs. write distinction.

My question is how we can figure that out without having to add a
blacklist or whitelist of specific platforms.  We haven't found
anything in the PCIe specs that tells us whether peer-to-peer is
supported between Root Ports.

The ACPI _DMA method does mention peer-to-peer, and I don't think
Linux looks at _DMA at all.  But you should have a single PNP0A08
bridge that leads to bus 0000:00, with a _CRS that includes the
windows of all the Root Ports, and I don't see how a _DMA method would
help carve that up into separate bus address regions.

Rafael, do you have any idea how we can discover peer-to-peer
capabilities of a platform?

Bjorn

Re: [PATCH v2 4/7] DMA-API: Add dma_(un)map_resource() documentation

[Mark Hounschell]

On 07/08/2015 11:11 AM, Bjorn Helgaas wrote:
> [+cc Rafael]
>
> On Tue, Jul 07, 2015 at 01:14:27PM -0400, Mark Hounschell wrote:
>> On 07/07/2015 11:15 AM, Bjorn Helgaas wrote:
>>> On Wed, May 20, 2015 at 08:11:17AM -0400, Mark Hounschell wrote:
>>>> Most currently available hardware doesn't allow reads but will allow
>>>> writes on PCIe peer-to-peer transfers. All current AMD chipsets are
>>>> this way. I'm pretty sure all Intel chipsets are this way also. What
>>>> happens with reads is they are just dropped with no indication of
>>>> error other than the data will not be as expected. Supposedly the
>>>> PCIe spec does not even require any peer-to-peer support. Regular
>>>> PCI there is no problem and this API could be useful. However I
>>>> doubt seriously you will find a pure PCI motherboard that has an
>>>> IOMMU.
>>>>
>>>> I don't understand the chipset manufactures reasoning for disabling
>>>> PCIe peer-to-peer reads. We would like to make PCIe versions of our
>>>> cards but their application requires  peer-to-peer reads and writes.
>>>> So we cannot develop PCIe versions of the cards.
>>>
>>> I'd like to understand this better.  Peer-to-peer between two devices
>>> below the same Root Port should work as long as ACS doesn't prevent
>>> it.  If we find an Intel or AMD IOMMU, I think we configure ACS to
>>> prevent direct peer-to-peer (see "pci_acs_enable"), but maybe it could
>>> still be done with the appropriate IOMMU support.  And if you boot
>>> with "iommu=off", we don't do that ACS configuration, so peer-to-peer
>>> should work.
>>>
>>> I suppose the problem is that peer-to-peer doesn't work between
>>> devices under different Root Ports or even devices under different
>>> Root Complexes?
>>>
>>> PCIe r3.0, sec 6.12.1.1, says Root Ports that support peer-to-peer
>>> traffic are required to implement ACS P2P Request Redirect, so if a
>>> Root Port doesn't implement RR, we can assume it doesn't support
>>> peer-to-peer.  But unfortunately the converse is not true: if a Root
>>> Port implements RR, that does *not* imply that it supports
>>> peer-to-peer traffic.
>>>
>>> So I don't know how to discover whether peer-to-peer between Root
>>> Ports or Root Complexes is supported.  Maybe there's some clue in the
>>> IOMMU?  The Intel VT-d spec mentions it, but "peer" doesn't even
>>> appear in the AMD spec.
>>>
>>> And I'm curious about why writes sometimes work when reads do not.
>>> That sounds like maybe the hardware support is there, but we don't
>>> understand how to configure everything correctly.
>>>
>>> Can you give us the specifics of the topology you'd like to use, e.g.,
>>> lspci -vv of the path between the two devices?
>>
>> First off, writes always work for me. Not just sometimes. Only reads
>> NEVER do.
>>
>> Reading the AMD-990FX-990X-970-Register-Programming-Requirements-48693.pdf
>> in section 2.5 "Enabling/Disabling Peer-to-Peer Traffic Access", it
>> states specifically that
>> only P2P memory writes are supported. This has been the case with
>> older AMD chipset also. In one of the older chipset documents I read
>> (I think the 770 series) , it said this was a security feature.
>> Makes no sense to me.
>>
>> As for the topology I'd like to be able to use. This particular
>> configuration (MB) has a single regular pci slot and the rest are
>> pci-e. In two of those pci-e slots is a pci-e to pci expansion
>> chassis interface card connected to a regular pci expansion rack. I
>> am trying to to peer to peer between a regular pci card in one of
>> those chassis to another regular pci card in the other chassis. In
>> turn through the pci-e subsystem. Attached is the lcpci -vv output
>> from this particular box. The cards that initiate the P2P are these:
>>
>> 04:04.0 Intelligent controller [0e80]: PLX Technology, Inc. Device
>> 0480 (rev 55)
>> 04:05.0 Intelligent controller [0e80]: PLX Technology, Inc. Device
>> 0480 (rev 55)
>> 04:06.0 Intelligent controller [0e80]: PLX Technology, Inc. Device
>> 0480 (rev 55)
>> 04:07.0 Intelligent controller [0e80]: PLX Technology, Inc. Device
>> 0480 (rev 55)
>>
>> The card they need to P2P to and from is this one.
>>
>> 0a:05.0 Network controller: VMIC GE-IP PCI5565,PMC5565 Reflective
>> Memory Node (rev 01)
>
> Peer-to-peer traffic initiated by 04:04.0 and targeted at 0a:05.0 has
> to be routed up to Root Port 00:04.0, over to Root Port 00:0b.0, and
> back down to 0a:05.0:
>
>    00:04.0: Root Port to [bus 02-05] Slot #4 ACS ReqRedir+
>    02:00.0: PCIe-to-PCI bridge to [bus 03-05]
>    03:04.0: PCI-to-PCI bridge to [bus 04-05]
>    04:04.0: PLX intelligent controller
>
>    00:0b.0: Root Port to [bus 08-0e] Slot #11 ACS ReqRedir+
>    00:0b.0:   bridge window [mem 0xd0000000-0xd84fffff]
>    08:00.0: PCIe-to-PCI bridge to [bus 09-0e]
>    08:00.0:   bridge window [mem 0xd0000000-0xd84fffff]
>    09:04.0: PCI-to-PCI bridge to [bus 0a-0e]
>    09:04.0:   bridge window [mem 0xd0000000-0xd84fffff]
>    0a:05.0: VMIC GE-IP reflective memory node
>    0a:05.0: BAR 3 [mem 0xd0000000-0xd7ffffff]
>
> Both Root Ports do support ACS, including P2P RR, but that doesn't
> tell us anything about whether the Root Complex actually supports
> peer-to-peer traffic between the Root Ports.  Per the AMD
> 990FX/990X/970 spec, your hardware supports it for writes but not
> reads.
>
> So your hardware is what it is, and a general-purpose interface should
> probably not allow peer-to-peer at all unless we wanted to complicate
> it by adding a read vs. write distinction.
>

Perhaps there are some AMD and/or Intel folks here that could influence 
future products to support both reads and writes?

Mark

Re: [PATCH v2 4/7] DMA-API: Add dma_(un)map_resource() documentation

[Rafael J. Wysocki]

On 7/8/2015 5:11 PM, Bjorn Helgaas wrote:
> [+cc Rafael]
>
> On Tue, Jul 07, 2015 at 01:14:27PM -0400, Mark Hounschell wrote:
>> On 07/07/2015 11:15 AM, Bjorn Helgaas wrote:
>>> On Wed, May 20, 2015 at 08:11:17AM -0400, Mark Hounschell wrote:
>>>> Most currently available hardware doesn't allow reads but will allow
>>>> writes on PCIe peer-to-peer transfers. All current AMD chipsets are
>>>> this way. I'm pretty sure all Intel chipsets are this way also. What
>>>> happens with reads is they are just dropped with no indication of
>>>> error other than the data will not be as expected. Supposedly the
>>>> PCIe spec does not even require any peer-to-peer support. Regular
>>>> PCI there is no problem and this API could be useful. However I
>>>> doubt seriously you will find a pure PCI motherboard that has an
>>>> IOMMU.
>>>>
>>>> I don't understand the chipset manufactures reasoning for disabling
>>>> PCIe peer-to-peer reads. We would like to make PCIe versions of our
>>>> cards but their application requires  peer-to-peer reads and writes.
>>>> So we cannot develop PCIe versions of the cards.
>>> I'd like to understand this better.  Peer-to-peer between two devices
>>> below the same Root Port should work as long as ACS doesn't prevent
>>> it.  If we find an Intel or AMD IOMMU, I think we configure ACS to
>>> prevent direct peer-to-peer (see "pci_acs_enable"), but maybe it could
>>> still be done with the appropriate IOMMU support.  And if you boot
>>> with "iommu=off", we don't do that ACS configuration, so peer-to-peer
>>> should work.
>>>
>>> I suppose the problem is that peer-to-peer doesn't work between
>>> devices under different Root Ports or even devices under different
>>> Root Complexes?
>>>
>>> PCIe r3.0, sec 6.12.1.1, says Root Ports that support peer-to-peer
>>> traffic are required to implement ACS P2P Request Redirect, so if a
>>> Root Port doesn't implement RR, we can assume it doesn't support
>>> peer-to-peer.  But unfortunately the converse is not true: if a Root
>>> Port implements RR, that does *not* imply that it supports
>>> peer-to-peer traffic.
>>>
>>> So I don't know how to discover whether peer-to-peer between Root
>>> Ports or Root Complexes is supported.  Maybe there's some clue in the
>>> IOMMU?  The Intel VT-d spec mentions it, but "peer" doesn't even
>>> appear in the AMD spec.
>>>
>>> And I'm curious about why writes sometimes work when reads do not.
>>> That sounds like maybe the hardware support is there, but we don't
>>> understand how to configure everything correctly.
>>>
>>> Can you give us the specifics of the topology you'd like to use, e.g.,
>>> lspci -vv of the path between the two devices?
>> First off, writes always work for me. Not just sometimes. Only reads
>> NEVER do.
>>
>> Reading the AMD-990FX-990X-970-Register-Programming-Requirements-48693.pdf
>> in section 2.5 "Enabling/Disabling Peer-to-Peer Traffic Access", it
>> states specifically that
>> only P2P memory writes are supported. This has been the case with
>> older AMD chipset also. In one of the older chipset documents I read
>> (I think the 770 series) , it said this was a security feature.
>> Makes no sense to me.
>>
>> As for the topology I'd like to be able to use. This particular
>> configuration (MB) has a single regular pci slot and the rest are
>> pci-e. In two of those pci-e slots is a pci-e to pci expansion
>> chassis interface card connected to a regular pci expansion rack. I
>> am trying to to peer to peer between a regular pci card in one of
>> those chassis to another regular pci card in the other chassis. In
>> turn through the pci-e subsystem. Attached is the lcpci -vv output
>> from this particular box. The cards that initiate the P2P are these:
>>
>> 04:04.0 Intelligent controller [0e80]: PLX Technology, Inc. Device
>> 0480 (rev 55)
>> 04:05.0 Intelligent controller [0e80]: PLX Technology, Inc. Device
>> 0480 (rev 55)
>> 04:06.0 Intelligent controller [0e80]: PLX Technology, Inc. Device
>> 0480 (rev 55)
>> 04:07.0 Intelligent controller [0e80]: PLX Technology, Inc. Device
>> 0480 (rev 55)
>>
>> The card they need to P2P to and from is this one.
>>
>> 0a:05.0 Network controller: VMIC GE-IP PCI5565,PMC5565 Reflective
>> Memory Node (rev 01)
> Peer-to-peer traffic initiated by 04:04.0 and targeted at 0a:05.0 has
> to be routed up to Root Port 00:04.0, over to Root Port 00:0b.0, and
> back down to 0a:05.0:
>
>    00:04.0: Root Port to [bus 02-05] Slot #4 ACS ReqRedir+
>    02:00.0: PCIe-to-PCI bridge to [bus 03-05]
>    03:04.0: PCI-to-PCI bridge to [bus 04-05]
>    04:04.0: PLX intelligent controller
>
>    00:0b.0: Root Port to [bus 08-0e] Slot #11 ACS ReqRedir+
>    00:0b.0:   bridge window [mem 0xd0000000-0xd84fffff]
>    08:00.0: PCIe-to-PCI bridge to [bus 09-0e]
>    08:00.0:   bridge window [mem 0xd0000000-0xd84fffff]
>    09:04.0: PCI-to-PCI bridge to [bus 0a-0e]
>    09:04.0:   bridge window [mem 0xd0000000-0xd84fffff]
>    0a:05.0: VMIC GE-IP reflective memory node
>    0a:05.0: BAR 3 [mem 0xd0000000-0xd7ffffff]
>
> Both Root Ports do support ACS, including P2P RR, but that doesn't
> tell us anything about whether the Root Complex actually supports
> peer-to-peer traffic between the Root Ports.  Per the AMD
> 990FX/990X/970 spec, your hardware supports it for writes but not
> reads.
>
> So your hardware is what it is, and a general-purpose interface should
> probably not allow peer-to-peer at all unless we wanted to complicate
> it by adding a read vs. write distinction.
>
> My question is how we can figure that out without having to add a
> blacklist or whitelist of specific platforms.  We haven't found
> anything in the PCIe specs that tells us whether peer-to-peer is
> supported between Root Ports.
>
> The ACPI _DMA method does mention peer-to-peer, and I don't think
> Linux looks at _DMA at all.  But you should have a single PNP0A08
> bridge that leads to bus 0000:00, with a _CRS that includes the
> windows of all the Root Ports, and I don't see how a _DMA method would
> help carve that up into separate bus address regions.
>
> Rafael, do you have any idea how we can discover peer-to-peer
> capabilities of a platform?

No, I don't, sorry.

Rafael

Re: [PATCH v2 4/7] DMA-API: Add dma_(un)map_resource() documentation

[Konrad Rzeszutek Wilk]

On Mon, May 18, 2015 at 01:25:01PM -0500, wdavis-DDmLM1+adcrQT0dZR+AlfA@public.gmane.org wrote:
> From: Will Davis <wdavis-DDmLM1+adcrQT0dZR+AlfA@public.gmane.org>
> 
> Add references to both the general API documentation as well as the HOWTO.
> 
> Signed-off-by: Will Davis <wdavis-DDmLM1+adcrQT0dZR+AlfA@public.gmane.org>
> ---
>  Documentation/DMA-API-HOWTO.txt | 39 +++++++++++++++++++++++++++++++++++++--
>  Documentation/DMA-API.txt       | 36 +++++++++++++++++++++++++++++++-----
>  2 files changed, 68 insertions(+), 7 deletions(-)
> 
> diff --git a/Documentation/DMA-API-HOWTO.txt b/Documentation/DMA-API-HOWTO.txt
> index 0f7afb2..89bd730 100644
> --- a/Documentation/DMA-API-HOWTO.txt
> +++ b/Documentation/DMA-API-HOWTO.txt
> @@ -138,6 +138,10 @@ What about block I/O and networking buffers?  The block I/O and
>  networking subsystems make sure that the buffers they use are valid
>  for you to DMA from/to.
>  
> +In some systems, it may also be possible to DMA to and/or from a peer
> +device's MMIO region, as described by a 'struct resource'. This is
> +referred to as a peer-to-peer mapping.
> +
>  			DMA addressing limitations
>  
>  Does your device have any DMA addressing limitations?  For example, is
> @@ -648,6 +652,35 @@ Every dma_map_{single,sg}() call should have its dma_unmap_{single,sg}()
>  counterpart, because the bus address space is a shared resource and
>  you could render the machine unusable by consuming all bus addresses.
>  
> +Peer-to-peer DMA mappings can be obtained using dma_map_resource() to map
> +another device's MMIO region for the given device:
> +
> +	struct resource *peer_mmio_res = &other_dev->resource[0];
> +	dma_addr_t dma_handle = dma_map_resource(dev, peer_mmio_res,
> +						 offset, size, direction);
> +	if (dma_handle == 0 || dma_mapping_error(dev, dma_handle))

Don't check for 0. That can be a valid value.

> +	{
> +		/*
> +		 * If dma_handle == 0, dma_map_resource() is not
> +		 * implemented, and peer-to-peer transactions will not
> +		 * work.

Perhaps we should just have an dma_can_map_resource function instead
to check for this?

RE: [PATCH v2 4/7] DMA-API: Add dma_(un)map_resource() documentation

[William Davis]

> -----Original Message-----
> From: Konrad Rzeszutek Wilk [mailto:konrad.wilk@oracle.com]
> Sent: Monday, June 1, 2015 4:25 PM
> To: William Davis
> Cc: joro@8bytes.org; bhelgaas@google.com; iommu@lists.linux-foundation.org;
> linux-pci@vger.kernel.org; Terence Ripperda; John Hubbard;
> jglisse@redhat.com
> Subject: Re: [PATCH v2 4/7] DMA-API: Add dma_(un)map_resource()
> documentation
> 
> On Mon, May 18, 2015 at 01:25:01PM -0500, wdavis@nvidia.com wrote:
> > From: Will Davis <wdavis@nvidia.com>
> >
> > Add references to both the general API documentation as well as the
> HOWTO.
> >
> > Signed-off-by: Will Davis <wdavis@nvidia.com>
> > ---
> >  Documentation/DMA-API-HOWTO.txt | 39
> +++++++++++++++++++++++++++++++++++++--
> >  Documentation/DMA-API.txt       | 36 +++++++++++++++++++++++++++++++----
> -
> >  2 files changed, 68 insertions(+), 7 deletions(-)
> >
> > diff --git a/Documentation/DMA-API-HOWTO.txt
> > b/Documentation/DMA-API-HOWTO.txt index 0f7afb2..89bd730 100644
> > --- a/Documentation/DMA-API-HOWTO.txt
> > +++ b/Documentation/DMA-API-HOWTO.txt
> > @@ -138,6 +138,10 @@ What about block I/O and networking buffers?  The
> > block I/O and  networking subsystems make sure that the buffers they
> > use are valid  for you to DMA from/to.
> >
> > +In some systems, it may also be possible to DMA to and/or from a peer
> > +device's MMIO region, as described by a 'struct resource'. This is
> > +referred to as a peer-to-peer mapping.
> > +
> >  			DMA addressing limitations
> >
> >  Does your device have any DMA addressing limitations?  For example,
> > is @@ -648,6 +652,35 @@ Every dma_map_{single,sg}() call should have
> > its dma_unmap_{single,sg}()  counterpart, because the bus address
> > space is a shared resource and  you could render the machine unusable by
> consuming all bus addresses.
> >
> > +Peer-to-peer DMA mappings can be obtained using dma_map_resource() to
> > +map another device's MMIO region for the given device:
> > +
> > +	struct resource *peer_mmio_res = &other_dev->resource[0];
> > +	dma_addr_t dma_handle = dma_map_resource(dev, peer_mmio_res,
> > +						 offset, size, direction);
> > +	if (dma_handle == 0 || dma_mapping_error(dev, dma_handle))
> 
> Don't check for 0. That can be a valid value.
> 
> > +	{
> > +		/*
> > +		 * If dma_handle == 0, dma_map_resource() is not
> > +		 * implemented, and peer-to-peer transactions will not
> > +		 * work.
> 
> Perhaps we should just have an dma_can_map_resource function instead to
> check for this?

In v3 of this patch set, I've added a BUG_ON(ops->map_resource == NULL) to
dma_map_resource() instead of possibly returning 0, at Joerg's suggestion.

In that case, a dma_can_map_resource() would do the same check a driver
would before calling dma_map_resource(), e.g., that
ops->map_resource != NULL. It does seem nicer to wrap that in an API call.

Thanks,
Will

--
nvpublic

[PATCH v2 5/7] iommu/amd: Implement (un)map_resource

[wdavis]

From: Will Davis <wdavis@nvidia.com>

Implement 'map_resource' for the AMD IOMMU driver. Generalize the existing
map_page implementation to operate on a physical address, and make both
map_page and map_resource wrappers around that helper (and similiarly, for
unmap_page and unmap_resource).

This allows a device to map another's resource, to enable peer-to-peer
transactions.

Signed-off-by: Will Davis <wdavis@nvidia.com>
Reviewed-by: Terence Ripperda <tripperda@nvidia.com>
Reviewed-by: John Hubbard <jhubbard@nvidia.com>
---
 drivers/iommu/amd_iommu.c | 76 +++++++++++++++++++++++++++++++++++++++--------
 1 file changed, 63 insertions(+), 13 deletions(-)

diff --git a/drivers/iommu/amd_iommu.c b/drivers/iommu/amd_iommu.c
index e43d489..ca2dac6 100644
--- a/drivers/iommu/amd_iommu.c
+++ b/drivers/iommu/amd_iommu.c
@@ -503,6 +503,8 @@ DECLARE_STATS_COUNTER(cnt_map_single);
 DECLARE_STATS_COUNTER(cnt_unmap_single);
 DECLARE_STATS_COUNTER(cnt_map_sg);
 DECLARE_STATS_COUNTER(cnt_unmap_sg);
+DECLARE_STATS_COUNTER(cnt_map_resource);
+DECLARE_STATS_COUNTER(cnt_unmap_resource);
 DECLARE_STATS_COUNTER(cnt_alloc_coherent);
 DECLARE_STATS_COUNTER(cnt_free_coherent);
 DECLARE_STATS_COUNTER(cross_page);
@@ -541,6 +543,8 @@ static void amd_iommu_stats_init(void)
 	amd_iommu_stats_add(&cnt_unmap_single);
 	amd_iommu_stats_add(&cnt_map_sg);
 	amd_iommu_stats_add(&cnt_unmap_sg);
+	amd_iommu_stats_add(&cnt_map_resource);
+	amd_iommu_stats_add(&cnt_unmap_resource);
 	amd_iommu_stats_add(&cnt_alloc_coherent);
 	amd_iommu_stats_add(&cnt_free_coherent);
 	amd_iommu_stats_add(&cross_page);
@@ -2752,20 +2756,16 @@ static void __unmap_single(struct dma_ops_domain *dma_dom,
 }
 
 /*
- * The exported map_single function for dma_ops.
+ * Wrapper function that contains code common to mapping a physical address
+ * range from a page or a resource.
  */
-static dma_addr_t map_page(struct device *dev, struct page *page,
-			   unsigned long offset, size_t size,
-			   enum dma_data_direction dir,
-			   struct dma_attrs *attrs)
+static dma_addr_t __map_phys(struct device *dev, phys_addr_t paddr,
+			     size_t size, enum dma_data_direction dir)
 {
 	unsigned long flags;
 	struct protection_domain *domain;
 	dma_addr_t addr;
 	u64 dma_mask;
-	phys_addr_t paddr = page_to_phys(page) + offset;
-
-	INC_STATS_COUNTER(cnt_map_single);
 
 	domain = get_domain(dev);
 	if (PTR_ERR(domain) == -EINVAL)
@@ -2791,16 +2791,15 @@ out:
 }
 
 /*
- * The exported unmap_single function for dma_ops.
+ * Wrapper function that contains code common to unmapping a physical address
+ * range from a page or a resource.
  */
-static void unmap_page(struct device *dev, dma_addr_t dma_addr, size_t size,
-		       enum dma_data_direction dir, struct dma_attrs *attrs)
+static void __unmap_phys(struct device *dev, dma_addr_t dma_addr, size_t size,
+			 enum dma_data_direction dir)
 {
 	unsigned long flags;
 	struct protection_domain *domain;
 
-	INC_STATS_COUNTER(cnt_unmap_single);
-
 	domain = get_domain(dev);
 	if (IS_ERR(domain))
 		return;
@@ -2815,6 +2814,55 @@ static void unmap_page(struct device *dev, dma_addr_t dma_addr, size_t size,
 }
 
 /*
+ * The exported map_single function for dma_ops.
+ */
+static dma_addr_t map_page(struct device *dev, struct page *page,
+			   unsigned long offset, size_t size,
+			   enum dma_data_direction dir,
+			   struct dma_attrs *attrs)
+{
+	INC_STATS_COUNTER(cnt_map_single);
+
+	return __map_phys(dev, page_to_phys(page) + offset, size, dir);
+}
+
+/*
+ * The exported unmap_single function for dma_ops.
+ */
+static void unmap_page(struct device *dev, dma_addr_t dma_addr, size_t size,
+		       enum dma_data_direction dir, struct dma_attrs *attrs)
+{
+	INC_STATS_COUNTER(cnt_unmap_single);
+
+	__unmap_phys(dev, dma_addr, size, dir);
+}
+
+/*
+ * The exported map_resource function for dma_ops.
+ */
+static dma_addr_t map_resource(struct device *dev, struct resource *res,
+			       unsigned long offset, size_t size,
+			       enum dma_data_direction dir,
+			       struct dma_attrs *attrs)
+{
+	INC_STATS_COUNTER(cnt_map_resource);
+
+	return __map_phys(dev, res->start + offset, size, dir);
+}
+
+/*
+ * The exported unmap_resource function for dma_ops.
+ */
+static void unmap_resource(struct device *dev, dma_addr_t dma_addr,
+			   size_t size, enum dma_data_direction dir,
+			   struct dma_attrs *attrs)
+{
+	INC_STATS_COUNTER(cnt_unmap_resource);
+
+	__unmap_phys(dev, dma_addr, size, dir);
+}
+
+/*
  * The exported map_sg function for dma_ops (handles scatter-gather
  * lists).
  */
@@ -3066,6 +3114,8 @@ static struct dma_map_ops amd_iommu_dma_ops = {
 	.unmap_page = unmap_page,
 	.map_sg = map_sg,
 	.unmap_sg = unmap_sg,
+	.map_resource = map_resource,
+	.unmap_resource = unmap_resource,
 	.dma_supported = amd_iommu_dma_supported,
 };
 
-- 
2.4.0

[PATCH v2 6/7] iommu/vt-d: implement (un)map_resource

[wdavis]

From: Will Davis <wdavis@nvidia.com>

Implement 'map_resource' for the Intel IOMMU driver. Simply translate the
resource to a physical address and route it to the same handlers used by
the 'map_page' API.

This allows a device to map another's resource, to enable peer-to-peer
transactions.

Signed-off-by: Will Davis <wdavis@nvidia.com>
Reviewed-by: Terence Ripperda <tripperda@nvidia.com>
Reviewed-by: John Hubbard <jhubbard@nvidia.com>
---
 drivers/iommu/intel-iommu.c | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)

diff --git a/drivers/iommu/intel-iommu.c b/drivers/iommu/intel-iommu.c
index 68d43be..0f49eff 100644
--- a/drivers/iommu/intel-iommu.c
+++ b/drivers/iommu/intel-iommu.c
@@ -3095,6 +3095,15 @@ static dma_addr_t intel_map_page(struct device *dev, struct page *page,
 				  dir, *dev->dma_mask);
 }
 
+static dma_addr_t intel_map_resource(struct device *dev, struct resource *res,
+				     unsigned long offset, size_t size,
+				     enum dma_data_direction dir,
+				     struct dma_attrs *attrs)
+{
+	return __intel_map_single(dev, res->start + offset, size,
+				  dir, *dev->dma_mask);
+}
+
 static void flush_unmaps(void)
 {
 	int i, j;
@@ -3226,6 +3235,13 @@ static void intel_unmap_page(struct device *dev, dma_addr_t dev_addr,
 	intel_unmap(dev, dev_addr);
 }
 
+static void intel_unmap_resource(struct device *dev, dma_addr_t dev_addr,
+				 size_t size, enum dma_data_direction dir,
+				 struct dma_attrs *attrs)
+{
+	intel_unmap(dev, dev_addr);
+}
+
 static void *intel_alloc_coherent(struct device *dev, size_t size,
 				  dma_addr_t *dma_handle, gfp_t flags,
 				  struct dma_attrs *attrs)
@@ -3382,6 +3398,8 @@ struct dma_map_ops intel_dma_ops = {
 	.unmap_sg = intel_unmap_sg,
 	.map_page = intel_map_page,
 	.unmap_page = intel_unmap_page,
+	.map_resource = intel_map_resource,
+	.unmap_resource = intel_unmap_resource,
 	.mapping_error = intel_mapping_error,
 };
 
-- 
2.4.0

[PATCH v2 7/7] x86: add pci-nommu implementation of map_resource

[wdavis]

From: Will Davis <wdavis@nvidia.com>

Lookup the bus address of the resource by finding the parent host bridge,
which may be different than the parent host bridge of the target device.

Signed-off-by: Will Davis <wdavis@nvidia.com>
---
 arch/x86/kernel/pci-nommu.c | 32 ++++++++++++++++++++++++++++++++
 1 file changed, 32 insertions(+)

diff --git a/arch/x86/kernel/pci-nommu.c b/arch/x86/kernel/pci-nommu.c
index da15918..6384482 100644
--- a/arch/x86/kernel/pci-nommu.c
+++ b/arch/x86/kernel/pci-nommu.c
@@ -38,6 +38,37 @@ static dma_addr_t nommu_map_page(struct device *dev, struct page *page,
 	return bus;
 }
 
+static dma_addr_t nommu_map_resource(struct device *dev, struct resource *res,
+				     unsigned long offset, size_t size,
+				     enum dma_data_direction dir,
+				     struct dma_attrs *attrs)
+{
+	struct pci_bus *bus;
+	struct pci_host_bridge *bridge;
+	struct resource_entry *window;
+	resource_size_t bus_offset = 0;
+	dma_addr_t dma_address;
+
+	/* Find the parent host bridge of the resource, and determine the
+	 * relative offset.
+	 */
+	list_for_each_entry(bus, &pci_root_buses, node) {
+		bridge = to_pci_host_bridge(bus->bridge);
+		resource_list_for_each_entry(window, &bridge->windows) {
+			if (resource_contains(window->res, res))
+				bus_offset = window->offset;
+		}
+	}
+
+	dma_address = (res->start - bus_offset) + offset;
+	WARN_ON(size == 0);
+	if (!check_addr("map_resource", dev, dma_address, size))
+		return DMA_ERROR_CODE;
+	flush_write_buffers();
+	return dma_address;
+}
+
+
 /* Map a set of buffers described by scatterlist in streaming
  * mode for DMA.  This is the scatter-gather version of the
  * above pci_map_single interface.  Here the scatter gather list
@@ -93,6 +124,7 @@ struct dma_map_ops nommu_dma_ops = {
 	.free			= dma_generic_free_coherent,
 	.map_sg			= nommu_map_sg,
 	.map_page		= nommu_map_page,
+	.map_resource		= nommu_map_resource,
 	.sync_single_for_device = nommu_sync_single_for_device,
 	.sync_sg_for_device	= nommu_sync_sg_for_device,
 	.is_phys		= 1,
-- 
2.4.0