[PATCH 22/31] drm/amdgpu/gmc: rework pasid flushing

[Alex Deucher <alexander.deucher@amd.com>]

Split out all of the various flush methods and use
the new pasid flush method enum to determine which
one to use.

Signed-off-by: Alex Deucher <alexander.deucher@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c | 247 +++++++++++++++++++-----
 drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c  |   2 +
 drivers/gpu/drm/amd/amdgpu/gmc_v11_0.c  |   2 +
 drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c   |   2 +
 4 files changed, 201 insertions(+), 52 deletions(-)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c
index 8a975eddd75c7..2800eebe50649 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c
@@ -788,9 +788,66 @@ void amdgpu_gmc_flush_gpu_tlb_gart(struct amdgpu_device *adev,
 	dev_err(adev->dev, "Error flushing GPU TLB using the SDMA (%d)!\n", r);
 }
 
-int amdgpu_gmc_flush_gpu_tlb_pasid(struct amdgpu_device *adev, uint16_t pasid,
-				   uint32_t flush_type, bool all_hub,
-				   uint32_t inst)
+static int amdgpu_gmc_flush_gpu_tlb_sdma_helper(struct amdgpu_device *adev,
+						uint16_t pasid, uint32_t flush_type,
+						bool all_hub, uint32_t inst)
+{
+	struct amdgpu_ring *ring;
+	struct dma_fence *fence;
+	struct amdgpu_job *job;
+	uint16_t queried;
+	/* Use register 17 for GART */
+	u32 eng = 17;
+	int vmid, r, i, ndw;
+
+	/* flush hdp cache */
+	amdgpu_device_flush_hdp(adev, NULL);
+
+	ndw = ALIGN(adev->mman.buffer_funcs->tlb_inv_num_dw * 16 * AMDGPU_MAX_VMHUBS, 8);
+	ring = to_amdgpu_ring(adev->mman.buffer_funcs_scheds[0]);
+
+	mutex_lock(&adev->mman.default_entity.lock);
+	r = amdgpu_job_alloc_with_ib(ring->adev, &adev->mman.default_entity.base,
+				     AMDGPU_FENCE_OWNER_UNDEFINED,
+				     ndw * 4, AMDGPU_IB_POOL_IMMEDIATE,
+				     AMDGPU_KERNEL_JOB_ID_VM_UPDATE,
+				     &job);
+	if (r)
+		goto exit;
+
+	for (vmid = 1; vmid < 16; vmid++) {
+		bool valid;
+
+		valid = adev->gmc.gmc_funcs->get_vmid_pasid_mapping_info(adev, vmid, inst,
+									 &queried);
+		if (!valid || queried != pasid)
+			continue;
+
+		if (all_hub) {
+			for_each_set_bit(i, adev->vmhubs_mask, AMDGPU_MAX_VMHUBS)
+				amdgpu_emit_tlb_inv(adev, &job->ibs[0], vmid, i, eng,
+						    flush_type, inst);
+		} else {
+			amdgpu_emit_tlb_inv(adev, &job->ibs[0], vmid, AMDGPU_GFXHUB(inst), eng,
+					    flush_type, inst);
+		}
+	}
+	amdgpu_ring_pad_ib(ring, &job->ibs[0]);
+	fence = amdgpu_job_submit(job);
+	mutex_unlock(&adev->mman.default_entity.lock);
+
+	dma_fence_wait(fence, false);
+	dma_fence_put(fence);
+
+exit:
+	mutex_unlock(&adev->mman.default_entity.lock);
+
+	return r;
+}
+
+static int amdgpu_gmc_flush_gpu_tlb_kiq_helper(struct amdgpu_device *adev,
+					       uint16_t pasid, uint32_t flush_type,
+					       bool all_hub, uint32_t inst)
 {
 	struct amdgpu_ring *ring = &adev->gfx.kiq[inst].ring;
 	struct amdgpu_kiq *kiq = &adev->gfx.kiq[inst];
@@ -798,6 +855,110 @@ int amdgpu_gmc_flush_gpu_tlb_pasid(struct amdgpu_device *adev, uint16_t pasid,
 	int r, cnt = 0;
 	uint32_t seq;
 
+	/* 2 dwords flush + 8 dwords fence */
+	ndw = kiq->pmf->invalidate_tlbs_size + 8;
+
+	if (adev->gmc.flush_tlb_needs_extra_type_2)
+		ndw += kiq->pmf->invalidate_tlbs_size;
+
+	if (adev->gmc.flush_tlb_needs_extra_type_0)
+		ndw += kiq->pmf->invalidate_tlbs_size;
+
+	spin_lock(&adev->gfx.kiq[inst].ring_lock);
+	r = amdgpu_ring_alloc(ring, ndw);
+	if (r) {
+		spin_unlock(&adev->gfx.kiq[inst].ring_lock);
+		return r;
+	}
+	if (adev->gmc.flush_tlb_needs_extra_type_2)
+		kiq->pmf->kiq_invalidate_tlbs(ring, pasid, 2, all_hub);
+
+	if (flush_type == 2 && adev->gmc.flush_tlb_needs_extra_type_0)
+		kiq->pmf->kiq_invalidate_tlbs(ring, pasid, 0, all_hub);
+
+	kiq->pmf->kiq_invalidate_tlbs(ring, pasid, flush_type, all_hub);
+	r = amdgpu_fence_emit_polling(ring, &seq, MAX_KIQ_REG_WAIT);
+	if (r) {
+		amdgpu_ring_undo(ring);
+		spin_unlock(&adev->gfx.kiq[inst].ring_lock);
+		return r;
+	}
+
+	amdgpu_ring_commit(ring);
+	spin_unlock(&adev->gfx.kiq[inst].ring_lock);
+
+	r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT);
+
+	might_sleep();
+	while (r < 1 && cnt++ < MAX_KIQ_REG_TRY &&
+	       !amdgpu_reset_pending(adev->reset_domain)) {
+		msleep(MAX_KIQ_REG_BAILOUT_INTERVAL);
+		r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT);
+	}
+
+	if (cnt > MAX_KIQ_REG_TRY) {
+		dev_err(adev->dev, "timeout waiting for kiq fence\n");
+		r = -ETIME;
+	} else
+		r = 0;
+
+	return r;
+}
+
+static int amdgpu_gmc_flush_gpu_tlb_mes_helper(struct amdgpu_device *adev,
+					       uint16_t pasid, uint32_t flush_type,
+					       bool all_hub, uint32_t inst)
+{
+	struct mes_inv_tlbs_pasid_input input = {0};
+	int r;
+
+	input.xcc_id = inst;
+	input.pasid = pasid;
+	input.flush_type = flush_type;
+
+	/* MES will invalidate hubs for the device(including slave xcc)
+	 * from master, ignore request from slave
+	 */
+	if (!amdgpu_gfx_is_master_xcc(adev, inst))
+		return -EINVAL;
+
+	input.hub_id = AMDGPU_GFXHUB(0);
+	amdgpu_mes_lock(&adev->mes);
+	r = adev->mes.funcs->invalidate_tlbs_pasid(&adev->mes, &input);
+	amdgpu_mes_unlock(&adev->mes);
+	if (r)
+		return r;
+
+	if (all_hub) {
+		/* invalidate mm_hub */
+		if (test_bit(AMDGPU_MMHUB0(0), adev->vmhubs_mask)) {
+			input.hub_id = AMDGPU_MMHUB0(0);
+			amdgpu_mes_lock(&adev->mes);
+			r = adev->mes.funcs->invalidate_tlbs_pasid(&adev->mes, &input);
+			amdgpu_mes_unlock(&adev->mes);
+			if (r)
+				return r;
+		}
+		if (test_bit(AMDGPU_MMHUB1(0), adev->vmhubs_mask)) {
+			input.hub_id = AMDGPU_MMHUB1(0);
+			amdgpu_mes_lock(&adev->mes);
+			r = adev->mes.funcs->invalidate_tlbs_pasid(&adev->mes, &input);
+			amdgpu_mes_unlock(&adev->mes);
+			if (r)
+				return r;
+		}
+	}
+	return 0;
+}
+
+int amdgpu_gmc_flush_gpu_tlb_pasid(struct amdgpu_device *adev, uint16_t pasid,
+				   uint32_t flush_type, bool all_hub,
+				   uint32_t inst)
+{
+	struct amdgpu_ring *ring;
+	bool use_mmio = false;
+	int r;
+
 	/*
 	 * A GPU reset should flush all TLBs anyway, so no need to do
 	 * this while one is ongoing.
@@ -805,8 +966,38 @@ int amdgpu_gmc_flush_gpu_tlb_pasid(struct amdgpu_device *adev, uint16_t pasid,
 	if (!down_read_trylock(&adev->reset_domain->sem))
 		return 0;
 
-	if (!adev->gmc.flush_pasid_uses_kiq || !ring->sched.ready) {
+	switch (adev->gmc.pasid_inv_method) {
+	case AMDGPU_TLB_INV_METHOD_MMIO:
+	default:
+		use_mmio = true;
+		break;
+	case AMDGPU_TLB_INV_METHOD_SDMA:
+		ring = to_amdgpu_ring(adev->mman.buffer_funcs_scheds[0]);
+		if (!ring->sched.ready)
+			use_mmio = true;
+		else
+			r = amdgpu_gmc_flush_gpu_tlb_sdma_helper(adev, pasid, flush_type,
+								 all_hub, inst);
+		break;
+	case AMDGPU_TLB_INV_METHOD_KIQ:
+		ring = &adev->gfx.kiq[inst].ring;
+		if (!adev->gmc.flush_pasid_uses_kiq || !ring->sched.ready)
+			use_mmio = true;
+		else
+			r = amdgpu_gmc_flush_gpu_tlb_kiq_helper(adev, pasid, flush_type,
+								all_hub, inst);
+		break;
+	case AMDGPU_TLB_INV_METHOD_MES:
+		ring = &adev->mes.ring[MES_PIPE_INST(inst, 0)];
+		if (!ring->sched.ready)
+			use_mmio = true;
+		else
+			r = amdgpu_gmc_flush_gpu_tlb_mes_helper(adev, pasid, flush_type,
+								all_hub, inst);
+		break;
+	}
 
+	if (use_mmio) {
 		if (!adev->gmc.gmc_funcs->flush_gpu_tlb_pasid) {
 			r = 0;
 			goto error_unlock_reset;
@@ -825,54 +1016,6 @@ int amdgpu_gmc_flush_gpu_tlb_pasid(struct amdgpu_device *adev, uint16_t pasid,
 		adev->gmc.gmc_funcs->flush_gpu_tlb_pasid(adev, pasid,
 							 flush_type, all_hub,
 							 inst);
-		r = 0;
-	} else {
-		/* 2 dwords flush + 8 dwords fence */
-		ndw = kiq->pmf->invalidate_tlbs_size + 8;
-
-		if (adev->gmc.flush_tlb_needs_extra_type_2)
-			ndw += kiq->pmf->invalidate_tlbs_size;
-
-		if (adev->gmc.flush_tlb_needs_extra_type_0)
-			ndw += kiq->pmf->invalidate_tlbs_size;
-
-		spin_lock(&adev->gfx.kiq[inst].ring_lock);
-		r = amdgpu_ring_alloc(ring, ndw);
-		if (r) {
-			spin_unlock(&adev->gfx.kiq[inst].ring_lock);
-			goto error_unlock_reset;
-		}
-		if (adev->gmc.flush_tlb_needs_extra_type_2)
-			kiq->pmf->kiq_invalidate_tlbs(ring, pasid, 2, all_hub);
-
-		if (flush_type == 2 && adev->gmc.flush_tlb_needs_extra_type_0)
-			kiq->pmf->kiq_invalidate_tlbs(ring, pasid, 0, all_hub);
-
-		kiq->pmf->kiq_invalidate_tlbs(ring, pasid, flush_type, all_hub);
-		r = amdgpu_fence_emit_polling(ring, &seq, MAX_KIQ_REG_WAIT);
-		if (r) {
-			amdgpu_ring_undo(ring);
-			spin_unlock(&adev->gfx.kiq[inst].ring_lock);
-			goto error_unlock_reset;
-		}
-
-		amdgpu_ring_commit(ring);
-		spin_unlock(&adev->gfx.kiq[inst].ring_lock);
-
-		r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT);
-
-		might_sleep();
-		while (r < 1 && cnt++ < MAX_KIQ_REG_TRY &&
-		       !amdgpu_reset_pending(adev->reset_domain)) {
-			msleep(MAX_KIQ_REG_BAILOUT_INTERVAL);
-			r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT);
-		}
-
-		if (cnt > MAX_KIQ_REG_TRY) {
-			dev_err(adev->dev, "timeout waiting for kiq fence\n");
-			r = -ETIME;
-		} else
-			r = 0;
 	}
 
 error_unlock_reset:
diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c
index 4bb10a89ce582..e7c529620199b 100644
--- a/drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c
@@ -974,6 +974,8 @@ static int gmc_v10_0_hw_init(struct amdgpu_ip_block *ip_block)
 	int r;
 
 	adev->gmc.flush_pasid_uses_kiq = !amdgpu_emu_mode;
+	if (adev->gmc.flush_pasid_uses_kiq)
+		adev->gmc.pasid_inv_method = AMDGPU_TLB_INV_METHOD_KIQ;
 
 	/* The sequence of these two function calls matters.*/
 	gmc_v10_0_init_golden_registers(adev);
diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v11_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v11_0.c
index acab9f4180da6..a35c84cc7385f 100644
--- a/drivers/gpu/drm/amd/amdgpu/gmc_v11_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gmc_v11_0.c
@@ -950,6 +950,8 @@ static int gmc_v11_0_hw_init(struct amdgpu_ip_block *ip_block)
 	int r;
 
 	adev->gmc.flush_pasid_uses_kiq = !amdgpu_emu_mode;
+	if (adev->gmc.flush_pasid_uses_kiq)
+		adev->gmc.pasid_inv_method = AMDGPU_TLB_INV_METHOD_KIQ;
 
 	/* The sequence of these two function calls matters.*/
 	gmc_v11_0_init_golden_registers(adev);
diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c
index 3c13a6920171a..317b44412b5a6 100644
--- a/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c
@@ -2162,6 +2162,8 @@ static int gmc_v9_0_hw_init(struct amdgpu_ip_block *ip_block)
 	int i, r;
 
 	adev->gmc.flush_pasid_uses_kiq = true;
+	if (adev->gmc.flush_pasid_uses_kiq)
+		adev->gmc.pasid_inv_method = AMDGPU_TLB_INV_METHOD_KIQ;
 
 	/* Vega20+XGMI caches PTEs in TC and TLB. Add a heavy-weight TLB flush
 	 * (type 2), which flushes both. Due to a race condition with
-- 
2.55.0