[RFC PATCH 4/5] mm/hmm: add support for peer to peer to HMM device memory

[jglisse@redhat.com]

From: Jérôme Glisse <jglisse@redhat.com>

Signed-off-by: Jérôme Glisse <jglisse@redhat.com>
Cc: Logan Gunthorpe <logang@deltatee.com>
Cc: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
Cc: Rafael J. Wysocki <rafael@kernel.org>
Cc: Bjorn Helgaas <bhelgaas@google.com>
Cc: Christian Koenig <christian.koenig@amd.com>
Cc: Felix Kuehling <Felix.Kuehling@amd.com>
Cc: Jason Gunthorpe <jgg@mellanox.com>
Cc: linux-pci@vger.kernel.org
Cc: dri-devel@lists.freedesktop.org
Cc: Christoph Hellwig <hch@lst.de>
Cc: Marek Szyprowski <m.szyprowski@samsung.com>
Cc: Robin Murphy <robin.murphy@arm.com>
Cc: Joerg Roedel <jroedel@suse.de>
Cc: iommu@lists.linux-foundation.org
---
 include/linux/hmm.h | 47 +++++++++++++++++++++++++++++++++
 mm/hmm.c            | 63 +++++++++++++++++++++++++++++++++++++++++----
 2 files changed, 105 insertions(+), 5 deletions(-)

diff --git a/include/linux/hmm.h b/include/linux/hmm.h
index 4a1454e3efba..7a3ac182cc48 100644
--- a/include/linux/hmm.h
+++ b/include/linux/hmm.h
@@ -710,6 +710,53 @@ struct hmm_devmem_ops {
 		     const struct page *page,
 		     unsigned int flags,
 		     pmd_t *pmdp);
+
+	/*
+	 * p2p_map() - map page for peer to peer between device
+	 * @devmem: device memory structure (see struct hmm_devmem)
+	 * @range: range of virtual address that is being mapped
+	 * @device: device the range is being map to
+	 * @addr: first virtual address in the range to consider
+	 * @pa: device address (where actual mapping is store)
+	 * Returns: number of page successfuly mapped, 0 otherwise
+	 *
+	 * Map page belonging to devmem to another device for peer to peer
+	 * access. Device can decide not to map in which case memory will
+	 * be migrated to main memory.
+	 *
+	 * Also there is no garantee that all the pages in the range does
+	 * belongs to the devmem so it is up to the function to check that
+	 * every single page does belong to devmem.
+	 *
+	 * Note for now we do not care about error exect error, so on failure
+	 * function should just return 0.
+	 */
+	long (*p2p_map)(struct hmm_devmem *devmem,
+			struct hmm_range *range,
+			struct device *device,
+			unsigned long addr,
+			dma_addr_t *pas);
+
+	/*
+	 * p2p_unmap() - unmap page from peer to peer between device
+	 * @devmem: device memory structure (see struct hmm_devmem)
+	 * @range: range of virtual address that is being mapped
+	 * @device: device the range is being map to
+	 * @addr: first virtual address in the range to consider
+	 * @pa: device address (where actual mapping is store)
+	 * Returns: number of page successfuly unmapped, 0 otherwise
+	 *
+	 * Unmap page belonging to devmem previously map with p2p_map().
+	 *
+	 * Note there is no garantee that all the pages in the range does
+	 * belongs to the devmem so it is up to the function to check that
+	 * every single page does belong to devmem.
+	 */
+	unsigned long (*p2p_unmap)(struct hmm_devmem *devmem,
+				   struct hmm_range *range,
+				   struct device *device,
+				   unsigned long addr,
+				   dma_addr_t *pas);
 };
 
 /*
diff --git a/mm/hmm.c b/mm/hmm.c
index 1a444885404e..fd49b1e116d0 100644
--- a/mm/hmm.c
+++ b/mm/hmm.c
@@ -1193,16 +1193,19 @@ long hmm_range_dma_map(struct hmm_range *range,
 		       dma_addr_t *daddrs,
 		       bool block)
 {
-	unsigned long i, npages, mapped, page_size;
+	unsigned long i, npages, mapped, page_size, addr;
 	long ret;
 
+again:
 	ret = hmm_range_fault(range, block);
 	if (ret <= 0)
 		return ret ? ret : -EBUSY;
 
+	mapped = 0;
+	addr = range->start;
 	page_size = hmm_range_page_size(range);
 	npages = (range->end - range->start) >> range->page_shift;
-	for (i = 0, mapped = 0; i < npages; ++i) {
+	for (i = 0; i < npages; ++i, addr += page_size) {
 		enum dma_data_direction dir = DMA_FROM_DEVICE;
 		struct page *page;
 
@@ -1226,6 +1229,29 @@ long hmm_range_dma_map(struct hmm_range *range,
 			goto unmap;
 		}
 
+		if (is_device_private_page(page)) {
+			struct hmm_devmem *devmem = page->pgmap->data;
+
+			if (!devmem->ops->p2p_map || !devmem->ops->p2p_unmap) {
+				/* Fall-back to main memory. */
+				range->default_flags |=
+					range->flags[HMM_PFN_DEVICE_PRIVATE];
+				goto again;
+			}
+
+			ret = devmem->ops->p2p_map(devmem, range, device,
+						   addr, daddrs);
+			if (ret <= 0) {
+				/* Fall-back to main memory. */
+				range->default_flags |=
+					range->flags[HMM_PFN_DEVICE_PRIVATE];
+				goto again;
+			}
+			mapped += ret;
+			i += ret;
+			continue;
+		}
+
 		/* If it is read and write than map bi-directional. */
 		if (range->pfns[i] & range->values[HMM_PFN_WRITE])
 			dir = DMA_BIDIRECTIONAL;
@@ -1242,7 +1268,9 @@ long hmm_range_dma_map(struct hmm_range *range,
 	return mapped;
 
 unmap:
-	for (npages = i, i = 0; (i < npages) && mapped; ++i) {
+	npages = i;
+	addr = range->start;
+	for (i = 0; (i < npages) && mapped; ++i, addr += page_size) {
 		enum dma_data_direction dir = DMA_FROM_DEVICE;
 		struct page *page;
 
@@ -1253,6 +1281,18 @@ long hmm_range_dma_map(struct hmm_range *range,
 		if (dma_mapping_error(device, daddrs[i]))
 			continue;
 
+		if (is_device_private_page(page)) {
+			struct hmm_devmem *devmem = page->pgmap->data;
+			unsigned long inc;
+
+			inc = devmem->ops->p2p_unmap(devmem, range, device,
+						     addr, &daddrs[i]);
+			BUG_ON(inc > npages);
+			mapped += inc;
+			i += inc;
+			continue;
+		}
+
 		/* If it is read and write than map bi-directional. */
 		if (range->pfns[i] & range->values[HMM_PFN_WRITE])
 			dir = DMA_BIDIRECTIONAL;
@@ -1285,7 +1325,7 @@ long hmm_range_dma_unmap(struct hmm_range *range,
 			 dma_addr_t *daddrs,
 			 bool dirty)
 {
-	unsigned long i, npages, page_size;
+	unsigned long i, npages, page_size, addr;
 	long cpages = 0;
 
 	/* Sanity check. */
@@ -1298,7 +1338,7 @@ long hmm_range_dma_unmap(struct hmm_range *range,
 
 	page_size = hmm_range_page_size(range);
 	npages = (range->end - range->start) >> range->page_shift;
-	for (i = 0; i < npages; ++i) {
+	for (i = 0, addr = range->start; i < npages; ++i, addr += page_size) {
 		enum dma_data_direction dir = DMA_FROM_DEVICE;
 		struct page *page;
 
@@ -1318,6 +1358,19 @@ long hmm_range_dma_unmap(struct hmm_range *range,
 				set_page_dirty(page);
 		}
 
+		if (is_device_private_page(page)) {
+			struct hmm_devmem *devmem = page->pgmap->data;
+			unsigned long ret;
+
+			BUG_ON(!devmem->ops->p2p_unmap);
+
+			ret = devmem->ops->p2p_unmap(devmem, range, device,
+						     addr, &daddrs[i]);
+			BUG_ON(ret > npages);
+			i += ret;
+			continue;
+		}
+
 		/* Unmap and clear pfns/dma address */
 		dma_unmap_page(device, daddrs[i], page_size, dir);
 		range->pfns[i] = range->values[HMM_PFN_NONE];
-- 
2.17.2