[PATCH 52/95] drm/amdgpu: Cleanup exported UALink handles

[Alex Deucher <alexander.deucher@amd.com>]

From: Mukul Joshi <mukul.joshi@amd.com>

Add implementation for freeing up exporter GPU's UALink
handles and the corresponding xarray entries. Before freeing,
the exporter GPU would send the NPA-REVOKE message to all the
importers and wait for the NPA-RELEASE messages to come back
before freeing the xarray entry.

Signed-off-by: Mukul Joshi <mukul.joshi@amd.com>
Reviewed-by: Felix Kuehling <felix.kuehling@amd.com>
Signed-off-by: Alex Deucher <alexander.deucher@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/amdgpu_ualink.c | 343 +++++++++++++++++++++
 1 file changed, 343 insertions(+)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ualink.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ualink.c
index 03c061b135afb..44233e6c390ae 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ualink.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ualink.c
@@ -1225,6 +1225,13 @@ static void amdgpu_generate_ualink_handle(struct amdgpu_device *adev,
 
 static void amdgpu_ualink_cleanup_exp_xa_node(struct kref *ref)
 {
+	struct amdgpu_ualink_exp_xa_node *exp_xa_node;
+	struct amdgpu_device *adev;
+
+	exp_xa_node = container_of(ref, struct amdgpu_ualink_exp_xa_node,
+				   refcount);
+	adev = amdgpu_ttm_adev(exp_xa_node->bo->tbo.bdev);
+	queue_work(adev->ualink.npa_wq, &exp_xa_node->cleanup_work);
 }
 
 static void amdgpu_ualink_cleanup_imp_xa_node(struct kref *ref)
@@ -1704,8 +1711,344 @@ static int amdgpu_ualink_setup_connection(struct amdgpu_device *adev,
 	return r;
 }
 
+/* Set PTE.X = 1 for all importer entries to retry RPCs. */
+static void amdgpu_ualink_force_retry_rpcs(struct amdgpu_device *adev,
+				struct amdgpu_ualink_exp_xa_node *exp_xa_node)
+{
+	u32 addr_mode = adev->ualink.info->vpod.addr_mode;
+	struct amdgpu_ualink_importer_entry *imp_entry;
+	struct amdgpu_vm *vm = &adev->ualink.npa_vm;
+	u64 pte_flags, npa_addr, size;
+	struct dma_fence *fence = NULL;
+	struct amdgpu_bo *bo;
+	struct drm_exec exec;
+	u32 remote_acc_id;
+	int r;
+
+	bo = exp_xa_node->bo;
+	size = amdgpu_bo_ngpu_pages(bo);
+
+	amdgpu_ualink_reserve_npa_vm_and_bos(adev, &bo, 1, &exec, false);
+
+	for_each_set_bit(remote_acc_id, exp_xa_node->importers_bitmap,
+			 AMDGPU_UALINK_ACCEL_MAX) {
+		if (addr_mode == AMDGPU_UALINK_ADDR_MODE_SOURCE_ALIAS)
+			imp_entry = &exp_xa_node->importer_entries[0];
+		else
+			imp_entry = &exp_xa_node->importer_entries[remote_acc_id];
+		npa_addr = imp_entry->npa_addr;
+
+		/* If the connection state changed while we are freeing
+		 * the BO, then ignore this importer. We will unmap this
+		 * address eventually in amdgpu_ualink_unmap_all_npa_addr()
+		 * function. Also, we will clear the corresponding bit in
+		 * the importer_bitmap in the same function.
+		 */
+		if (!amdgpu_ualink_check_conn_ready(adev, remote_acc_id,
+					imp_entry->generation_count))
+			continue;
+
+		/* Set PTE.X = 1 */
+		pte_flags = amdgpu_ualink_get_export_pte_flags(adev, bo,
+					AMDGPU_VM_PAGE_EXECUTABLE);
+		dev_dbg(adev->dev,
+			"RETRY-RPC: setting PTE.X=1 for NPA:%llx remote:%u pte:0x%llx\n",
+			npa_addr, remote_acc_id, pte_flags);
+
+		r = amdgpu_vm_update_range(adev, vm, false, false, true,
+					false, NULL, npa_addr, npa_addr + size - 1,
+					pte_flags, 0, adev->vm_manager.vram_base_offset,
+					bo->tbo.resource, NULL, &vm->last_update);
+
+		if (r)
+			dev_warn(adev->dev,
+				"RETRY-RPC: PTE.X update failed for NPA:%llx remote:%u, r: %d\n",
+				npa_addr, remote_acc_id, r);
+
+		if (addr_mode == AMDGPU_UALINK_ADDR_MODE_SOURCE_ALIAS)
+			break;
+	}
+
+	r = amdgpu_vm_update_pdes(adev, vm, false);
+	if (r) {
+		dev_err(adev->dev,
+			"Failed %d to update page directories during force retry rpcs\n",
+			r);
+		amdgpu_ualink_unreserve_npa_vm_and_bos(adev, &exec);
+		return;
+	}
+
+	fence = dma_fence_get(vm->last_update);
+	if (fence) {
+		r = dma_fence_wait(fence, false);
+		dma_fence_put(fence);
+		fence = NULL;
+		if (r)
+			dev_dbg(adev->dev, "RETRY-RPC: dma fence wait failed err:%d\n", r);
+	}
+
+	amdgpu_ualink_unreserve_npa_vm_and_bos(adev, &exec);
+
+	amdgpu_ualink_flush_tlb(adev, TLB_FLUSH_HEAVYWEIGHT);
+}
+
+/* Unmap all NPA addresses associated with a BO (UALink handle). This function is used
+ * only in Source Identification mode.
+ */
+static void amdgpu_ualink_unmap_all_npa_addr(struct amdgpu_device *adev,
+				struct amdgpu_ualink_exp_xa_node *exp_xa_node)
+{
+	u32 addr_mode = adev->ualink.info->vpod.addr_mode;
+	struct amdgpu_ualink_importer_entry *imp_entry;
+	u64 pte_value = adev->gmc.noretry_flags;
+	struct dma_fence *fence = NULL;
+	struct drm_exec exec;
+	u64 npa_addr, size;
+	u32 remote_acc_id;
+	int r;
+
+	size = amdgpu_bo_ngpu_pages(exp_xa_node->bo);
+
+	amdgpu_ualink_reserve_npa_vm_and_bos(adev, &exp_xa_node->bo, 1,
+					     &exec, false);
+
+	for_each_set_bit(remote_acc_id, exp_xa_node->importers_bitmap,
+			 AMDGPU_UALINK_ACCEL_MAX) {
+		if (addr_mode == AMDGPU_UALINK_ADDR_MODE_SOURCE_ALIAS)
+			imp_entry = &exp_xa_node->importer_entries[0];
+		else
+			imp_entry = &exp_xa_node->importer_entries[remote_acc_id];
+		npa_addr = imp_entry->npa_addr;
+
+		if (!amdgpu_ualink_check_conn_ready(adev, remote_acc_id,
+					imp_entry->generation_count)) {
+			clear_bit(remote_acc_id,
+				  exp_xa_node->importers_bitmap);
+			continue;
+		}
+
+		dev_dbg(adev->dev,
+			"UNMAP-NPA: Unmapping NPA:%llx, handle:%llx:%llx remote:%u pte:0x%llx\n",
+			npa_addr, exp_xa_node->handle.handle_hi, exp_xa_node->handle.handle_lo,
+			remote_acc_id, pte_value);
+
+		r = amdgpu_vm_update_range(adev, &adev->ualink.npa_vm, false,
+					   false, true, false, NULL, npa_addr,
+					   npa_addr + size - 1, pte_value, 0,
+					   0, NULL, NULL, &fence);
+
+		if (r)
+			dev_err(adev->dev,
+				"UNMAP-NPA: Unmap failed NPA:%llx, handle:%llx:%llx remote:%u\n",
+				npa_addr, exp_xa_node->handle.handle_hi,
+				exp_xa_node->handle.handle_lo, remote_acc_id);
+
+		if (addr_mode == AMDGPU_UALINK_ADDR_MODE_SOURCE_ALIAS)
+			break;
+	}
+
+	r = amdgpu_vm_update_pdes(adev, &adev->ualink.npa_vm, false);
+	if (r) {
+		dev_err(adev->dev,
+			"Failed %d to update page directories during all NPA addresses unmapping\n",
+			r);
+		amdgpu_ualink_unreserve_npa_vm_and_bos(adev, &exec);
+		return;
+	}
+
+	if (fence) {
+		r = dma_fence_wait(fence, false);
+		dma_fence_put(fence);
+		fence = NULL;
+		if (r)
+			dev_err(adev->dev,
+				"UNMAP-NPA: dma fence wait failed\n");
+	}
+
+	amdgpu_ualink_unreserve_npa_vm_and_bos(adev, &exec);
+
+	amdgpu_ualink_flush_tlb(adev, TLB_FLUSH_HEAVYWEIGHT);
+}
+
+static void amdgpu_ualink_free_all_npa_va(struct amdgpu_device *adev,
+				struct amdgpu_ualink_exp_xa_node *exp_xa_node,
+				unsigned long *importers_bitmap)
+{
+	u32 addr_mode = adev->ualink.info->vpod.addr_mode;
+	struct drm_mm_node *mm_node;
+	u32 remote_acc_id;
+
+	for_each_set_bit(remote_acc_id, importers_bitmap, AMDGPU_UALINK_ACCEL_MAX) {
+		if (addr_mode == AMDGPU_UALINK_ADDR_MODE_SOURCE_ALIAS)
+			mm_node = exp_xa_node->importer_entries[0].mm_node;
+		else
+			mm_node = exp_xa_node->importer_entries[remote_acc_id].mm_node;
+
+		if (!mm_node) {
+			if (addr_mode == AMDGPU_UALINK_ADDR_MODE_SOURCE_ALIAS)
+				break;
+			continue;
+		}
+
+		dev_dbg(adev->dev,
+			"FREE-NPA: freeing NPA address:%llx, handle:%llx:%llx remote:%u\n",
+			mm_node->start, exp_xa_node->handle.handle_hi,
+			exp_xa_node->handle.handle_lo, remote_acc_id);
+
+		amdgpu_ualink_npa_free_va(adev, mm_node);
+		kfree(mm_node);
+
+		if (addr_mode == AMDGPU_UALINK_ADDR_MODE_SOURCE_ALIAS)
+			break;
+	}
+}
+
 static void amdgpu_ualink_exp_cleanup_worker(struct work_struct *work)
 {
+	DECLARE_BITMAP(orig_importers_bitmap, AMDGPU_UALINK_ACCEL_MAX);
+	struct amdgpu_ualink_importer_entry *imp_entry;
+	struct amdgpu_ualink_exp_xa_node *exp_xa_node;
+	struct amdgpu_ualink_handle handle;
+	struct amdgpu_device *adev;
+	struct amdgpu_bo *bo;
+	u32 remote_acc_id;
+	int r;
+
+	exp_xa_node = container_of(work, struct amdgpu_ualink_exp_xa_node,
+				   cleanup_work);
+	bo = exp_xa_node->bo;
+	adev = amdgpu_ttm_adev(bo->tbo.bdev);
+	handle = exp_xa_node->handle;
+
+	/* Revoking access to an exported memory follows the steps:
+	 * 1. Set PTE.X = 1 to retry for RPCs.
+	 * 2. Send Remote TLB Shootdowns to all importers.
+	 * 3. Unmap the NPA address from NPA VM.
+	 * 4. Drop the ref count for the BO.
+	 * 5. Send NPA_REVOKE to all importers.
+	 * 6. Wait for NPA_RELEASE from all importers.
+	 * 7. Free the NPA address once all responses are received.
+	 */
+	/* If there are no importers for this BO/handle */
+	if (bitmap_empty(exp_xa_node->importers_bitmap,
+			 AMDGPU_UALINK_ACCEL_MAX)) {
+		/* Drop the BO reference so it can be freed. */
+		amdgpu_bo_unref(&bo);
+		exp_xa_node->bo = NULL;
+		goto free_node;
+	}
+
+	dev_dbg(adev->dev,
+		"EXP-CLEANUP: handle:%llx:%llx importers bitmap: %*pbl\n",
+		handle.handle_hi, handle.handle_lo,
+		AMDGPU_UALINK_ACCEL_MAX, exp_xa_node->importers_bitmap);
+
+	bitmap_copy(orig_importers_bitmap, exp_xa_node->importers_bitmap,
+		    AMDGPU_UALINK_ACCEL_MAX);
+
+	/* Set PTE.X = 1 for NPA addresses from all importers*/
+	amdgpu_ualink_force_retry_rpcs(adev, exp_xa_node);
+
+	/* Send TLB-shootdown to all importer GPUs */
+	for_each_set_bit(remote_acc_id, exp_xa_node->importers_bitmap,
+			 AMDGPU_UALINK_ACCEL_MAX) {
+		dev_dbg(adev->dev,
+			"EXP-CLEANUP: Sending TLB-shootdown to remote:%u\n",
+			remote_acc_id);
+		r = amdgpu_ualink_send_tlb_shootdown(adev, remote_acc_id);
+		if (r)
+			dev_err(adev->dev,
+				"EXP-CLEANUP: TLB shootdown send failed to remote:%u\n",
+				remote_acc_id);
+	}
+
+	/* Unmap all NPA addresses for this BO from NPA VM */
+	amdgpu_ualink_unmap_all_npa_addr(adev, exp_xa_node);
+
+	dev_dbg(adev->dev, "EXP-CLEANUP: handle:%llx:%llx Unpin BO, pin_count:%u, importers:%u\n",
+		handle.handle_hi, handle.handle_lo, bo->tbo.pin_count,
+		bitmap_weight(orig_importers_bitmap, AMDGPU_UALINK_ACCEL_MAX));
+	WARN_ON(bo->tbo.pin_count < bitmap_weight(orig_importers_bitmap,
+						AMDGPU_UALINK_ACCEL_MAX));
+
+	/* Unpin the BO */
+	if (likely(!amdgpu_bo_reserve(bo, true))) {
+		bo->ualink_handle_lo = 0ULL;
+		for_each_set_bit(remote_acc_id, orig_importers_bitmap,
+				 AMDGPU_UALINK_ACCEL_MAX)
+			amdgpu_bo_unpin(bo);
+		amdgpu_bo_unreserve(bo);
+	} else {
+		dev_warn(adev->dev,
+			"EXP-CLEANUP: BO reserve to unpin failed for handle:%llx:%llx\n",
+			handle.handle_hi, handle.handle_lo);
+	}
+
+	/* Free the DMABuf */
+	dma_buf_put(exp_xa_node->dmabuf);
+	/* Drop the reference to the BO so it can be freed. */
+	amdgpu_bo_unref(&bo);
+	exp_xa_node->bo = NULL;
+
+	/* Send NPA-REVOKE to all importers which have imported this memory */
+	for_each_set_bit(remote_acc_id, exp_xa_node->importers_bitmap,
+				 AMDGPU_UALINK_ACCEL_MAX) {
+		imp_entry = &exp_xa_node->importer_entries[remote_acc_id];
+		if (!amdgpu_ualink_check_conn_ready(adev, remote_acc_id,
+					imp_entry->generation_count)) {
+			clear_bit(remote_acc_id,
+				  exp_xa_node->importers_bitmap);
+			continue;
+		}
+
+		dev_dbg(adev->dev,
+			"EXP-CLEANUP: Sending NPA-REVOKE to remote:%u\n",
+			remote_acc_id);
+		set_bit(remote_acc_id, exp_xa_node->npa_release_bitmap);
+		r = amdgpu_ualink_send_npa_revoke_msg(adev, remote_acc_id, handle);
+		if (r) {
+			dev_err(adev->dev,
+				"EXP-CLEANUP: NPA-REVOKE send failed to remote:%u\n",
+				remote_acc_id);
+			clear_bit(remote_acc_id, exp_xa_node->npa_release_bitmap);
+		}
+	}
+
+	if (!bitmap_empty(exp_xa_node->importers_bitmap,
+			AMDGPU_UALINK_ACCEL_MAX)) {
+		dev_dbg(adev->dev,
+			"EXP-CLEANUP: handle:%llx:%llx NPA-RELEASE bitmap: %*pbl\n",
+			handle.handle_hi, handle.handle_lo,
+			AMDGPU_UALINK_ACCEL_MAX, exp_xa_node->npa_release_bitmap);
+
+		/* Wait for the NPA_RELEASE to come back from all importers */
+		r = wait_for_completion_timeout(&exp_xa_node->npa_done,
+					msecs_to_jiffies(AMDGPU_UALINK_RESP_TIMEOUT));
+
+		if (r == 0)
+			dev_warn(adev->dev,
+				"EXP-CLEANUP: NPA-RELEASE timeout for handle:%llx:%llx\n",
+				handle.handle_hi, handle.handle_lo);
+	}
+
+	/* Free the NPA addresses given to all the importers */
+	amdgpu_ualink_free_all_npa_va(adev, exp_xa_node,
+				      orig_importers_bitmap);
+
+	/* Warn about all importers that didn't respond back with
+	 * NPA-RELEASE message. This will trigger connection timeout
+	 * handling which is added later.
+	 */
+	for_each_set_bit(remote_acc_id, exp_xa_node->npa_release_bitmap,
+			 AMDGPU_UALINK_ACCEL_MAX)
+		dev_warn(adev->dev,
+			"EXP-CLEANUP: handle:%llx:%llx NPA-RELEASE timeout from remote:%u\n",
+			handle.handle_hi, handle.handle_lo, remote_acc_id);
+
+free_node:
+	xa_erase(&adev->ualink.handle_invalid_xa, handle.handle_lo);
+	mutex_destroy(&exp_xa_node->node_lock);
+	kfree(exp_xa_node);
 }
 
 /* This function is a copy of amdgpu_dma_buf_move_notify() function.
-- 
2.55.0