[PATCH 38/95] drm/amdgpu: Add UALink remote command packets and SDMA dispatch

[Alex Deucher <alexander.deucher@amd.com>]

From: Philip Yang <Philip.Yang@amd.com>

Add 16-dword packet layouts for TLB shootdown, remote interrupt, and
writeback address update. The dispatch path issues three sequential SDMA
copies (ring entry, write pointer, doorbell) and polls the firmware
writeback buffer for the acknowledged sequence number.

Signed-off-by: Philip Yang <Philip.Yang@amd.com>
Reviewed-by: Felix Kuehling <felix.kuehling@amd.com>
Signed-off-by: Alex Deucher <alexander.deucher@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/amdgpu_ualink.c | 347 +++++++++++++++++++++
 1 file changed, 347 insertions(+)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ualink.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ualink.c
index d212416213960..913d53fe91fc1 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ualink.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ualink.c
@@ -2304,3 +2304,350 @@ static int amdgpu_ualink_metadata_init(struct amdgpu_device *adev)
 	return r;
 }
 
+/*
+ * nHT Firmware Error code
+ *
+ * Success				0x0
+ * FIFO overflow			0x1
+ * Invalid RB command			0x2
+ * Timeout				0x3
+ * Invalid metadata entry		0x4
+ * RB overflow				0x5
+ * Invalid wptr address			0x6
+ * Interrupt cookie send failure	0x7
+ * Invalid RB address			0x8
+ * Invalid writeback address		0x9
+ */
+static void amdgpu_ualink_remote_error(struct amdgpu_device *adev, u32 status)
+{
+	const char *fw_err_code_msg[] = {
+		"Unknown FW error status",
+		"FIFO overflow",		/* error code 1 */
+		"Invalid RB command",
+		"Timeout",
+		"Invalid metadata entry",
+		"RB overflow",
+		"Invalid wptr address",
+		"Interrupt cookie send failure",
+		"Invalid RB address",
+		"Invalid writeback address"	/* error code 9 */
+		};
+
+	if (status >= ARRAY_SIZE(fw_err_code_msg))
+		status = 0;
+
+	dev_err(adev->dev, "remote error %s\n", fw_err_code_msg[status]);
+	return;
+}
+
+/**
+ * amdgpu_ualink_remote_wait_timeout - Wait for remote operation completion
+ * @remote_accel_id: remote accelator id to wait for reply
+ * @adev: amdgpu device pointer
+ * @wb_cpu: CPU virtual address of writeback buffer
+ * @seq: Sequence number to wait for
+ *
+ * Polls the writeback buffer waiting for the sequence number to reach or
+ * exceed the expected value. The writeback buffer contains status and data
+ * fields updated by firmware to indicate completion and error conditions.
+ *
+ * Return: 0 on success
+ *         -ETIME on timeout
+ *         -ECOMM on firmware return error status
+ */
+static long amdgpu_ualink_remote_wait_timeout(struct amdgpu_device *adev,
+					      u32 remote_accel_id,
+					      struct amdgpu_ualink_wb *wb_cpu,
+					      u32 seq)
+{
+	/* 2 seconds timeout, long enough for FW to reply */
+	ktime_t timeout = ktime_add_us(ktime_get(), 2 * USEC_PER_SEC);
+	u32 status, data;
+	u64 rptr;
+
+	while (true) {
+		data = READ_ONCE(wb_cpu->data);
+		if (data >= seq) {
+			status = READ_ONCE(wb_cpu->status);
+			rptr = READ_ONCE(wb_cpu->rptr);
+			break;
+		}
+
+		if (ktime_after(ktime_get(), timeout)) {
+			dev_dbg(adev->dev, "remote %d wait timeout\n", remote_accel_id);
+			return -ETIME;
+		}
+
+		usleep_range(10, 50);
+	}
+
+	if (status) {
+		dev_dbg(adev->dev, "remote %u status 0x%x wb data 0x%x seq 0x%x rptr 0x%llx\n",
+			remote_accel_id, status, data, seq, rptr);
+
+		amdgpu_ualink_remote_error(adev, status);
+		return -ECOMM;
+	}
+
+	dev_dbg(adev->dev, "remote %u succeed seq 0x%x wb data 0x%x rptr 0x%llx\n",
+		remote_accel_id, seq, data, rptr);
+	return 0;
+}
+
+#define AMDGPU_UALINK_REMOTE_OP_TLB_INV		0x1
+#define AMDGPU_UALINK_REMOTE_OP_INT		0x2
+#define AMDGPU_UALINK_REMOTE_OP_UPDATE_WB	0x3
+
+static void amdgpu_ualink_emit_shootdown(u32 **cpu_addr_p, u32 wb_data,
+					 u32 dw0,  u32 dw1, u32 dw2, u32 dw3)
+{
+	u64 addr = (((u64)dw1 << 32) | (u64)dw2) << AMDGPU_GPU_PAGE_SHIFT;
+	u32 *cpu_addr = *cpu_addr_p;
+	u32 flush_type = dw0;
+	u32 size_in_pages = dw3;
+	u32 cmd;
+
+	cmd = AMDGPU_UALINK_REMOTE_OP_TLB_INV;
+	cmd |= 1 << 28;	/* headptr update */
+	cmd |= 1 << 29;	/* wb enable */
+
+	*cpu_addr++ = cmd;
+	*cpu_addr++ = wb_data;
+	*cpu_addr++ = 0;
+	*cpu_addr++ = 0;
+	*cpu_addr++ = 0;
+	*cpu_addr++ = 0;
+	*cpu_addr++ = 0; /* metadata */
+	*cpu_addr++ = 0; /* metadata */
+
+	pr_debug("NPA addr 0x%llx npages 0x%x\n", addr, size_in_pages);
+
+	/* Always shootdown everything with full address size s-field coding */
+	addr = GENMASK_ULL(51, 11);
+
+	*cpu_addr++ = lower_32_bits(addr) | flush_type;
+	*cpu_addr++ = upper_32_bits(addr);
+
+	*cpu_addr++ = 0;
+	*cpu_addr++ = 0;
+	*cpu_addr++ = 0;
+	*cpu_addr++ = 0;
+	*cpu_addr++ = 0;
+	*cpu_addr++ = 0;
+	*cpu_addr_p = cpu_addr;
+}
+
+static void amdgpu_ualink_emit_interrupt(u32 **cpu_addr_p, u32 wb_data,
+					 u32 dw0, u32 dw1, u32 dw2, u32 dw3)
+{
+	u32 *cpu_addr = *cpu_addr_p;
+	u32 cmd;
+
+	cmd = AMDGPU_UALINK_REMOTE_OP_INT;
+	cmd |= 1 << 28;	/* headptr_update */
+	cmd |= 1 << 29; /* wb enable */
+
+	*cpu_addr++ = cmd;
+	*cpu_addr++ = wb_data;
+	*cpu_addr++ = 0;
+	*cpu_addr++ = 0;
+	*cpu_addr++ = 0;
+	*cpu_addr++ = 0;
+	*cpu_addr++ = 0; /* metadata */
+	*cpu_addr++ = 0; /* metadata */
+	/* updated by f/w, overwrite PASID with GPU ID */
+	*cpu_addr++ = 0;
+	*cpu_addr++ = 0;
+	*cpu_addr++ = 0;
+	*cpu_addr++ = 0;
+	/* IH ContextID 4 dwords */
+	*cpu_addr++ = dw0;
+	*cpu_addr++ = dw1;
+	*cpu_addr++ = dw2;
+	*cpu_addr++ = dw3;
+	*cpu_addr_p = cpu_addr;
+}
+
+static void amdgpu_ualink_emit_update_wb_addr(u32 **cpu_addr_p, u32 wb_data,
+					      u32 dw0, u32 dw1, u32 dw2, u32 dw3)
+{
+	u32 *cpu_addr = *cpu_addr_p;
+	u32 cmd;
+
+	cmd = AMDGPU_UALINK_REMOTE_OP_UPDATE_WB;
+	cmd |= 1 << 28;	/* headptr update */
+	cmd |= 1 << 29;	/* wb enable */
+
+	*cpu_addr++ = cmd;
+	*cpu_addr++ = wb_data;
+	*cpu_addr++ = dw1 & 0xFFFFFFFC;	/* lower32 wb_npa */
+	*cpu_addr++ = dw0 & 0xFFFFF;	/* upper32 wb_npa */
+	*cpu_addr++ = 0;
+	*cpu_addr++ = 0;
+	*cpu_addr++ = 0; /* metadata */
+	*cpu_addr++ = 0; /* metadata */
+	*cpu_addr++ = 0;
+	*cpu_addr++ = 0;
+	*cpu_addr++ = 0;
+	*cpu_addr++ = 0;
+	*cpu_addr++ = 0;
+	*cpu_addr++ = 0;
+	*cpu_addr++ = 0;
+	*cpu_addr++ = 0;
+	*cpu_addr_p = cpu_addr;
+}
+
+/*
+ * Function proto to generate packet using above different emit_ functions
+ */
+typedef void (*ualink_emit_packet)(u32 **cpu_addr_p, u32 wb, u32 dw0,
+				   u32 dw1, u32 dw2, u32 dw3);
+
+static int amdgpu_ualink_send_command(struct amdgpu_device *adev,
+				      u32 remote_accel_id,
+				      struct amdgpu_ualink_ring *ring,
+				      struct amdgpu_ualink_wb *wb_cpu,
+				      ualink_emit_packet emit_func,
+				      u32 dw0, u32 dw1, u32 dw2, u32 dw3)
+{
+	struct amdgpu_ualink_remote *remote = to_remote(adev);
+	struct amdgpu_ualink_peer *peer;
+	struct amdgpu_ring *sdma_ring;
+	struct dma_fence *fence;
+	struct amdgpu_job *job;
+	struct amdgpu_ib *ib;
+	u32 seq, ndw, ndw_copy_cmd;
+	u64 doorbell_npa_gart;
+	u32 *src_cpu;
+	u64 src;
+	int r;
+
+	peer = &remote->peer[remote_accel_id];
+
+	/*
+	 * 3 sdma copy commands: write data to ring buffer, update wptr, ring doorbell
+	 *
+	 * insert 2 NOP commands to break SDMA back-to-back copy command overlap, to
+	 * ensure the ordering of 3 sdma copy commands execution.
+	 */
+
+	/* buffer after sdma commands, starting at 8 dwords boundary */
+	ndw_copy_cmd = ALIGN(3 * adev->mman.buffer_funcs->copy_num_dw + 2, 8);
+
+	 /* remote command 16 dwords, wptr 2 dwords, doorbell 1 dword */
+	ndw = ndw_copy_cmd + 16 + 2 + 1;
+	r = amdgpu_job_alloc_with_ib(adev, &peer->entity, AMDGPU_FENCE_OWNER_VM,
+				     ndw * 4, AMDGPU_IB_POOL_IMMEDIATE,
+				     AMDGPU_KERNEL_JOB_ID_TTM_COPY_BUFFER, &job);
+	if (r)
+		return r;
+
+	mutex_lock(&peer->lock);
+
+	ring->rptr = READ_ONCE(wb_cpu->rptr);
+
+	if (WARN_ON_ONCE(ring->rptr > ring->wptr)) {
+		dev_err(adev->dev, "accel_id %u ring overflow wptr 0x%llx rptr 0x%llx\n",
+			remote_accel_id, ring->wptr, ring->rptr);
+		r = -EFAULT;
+		goto unlock_free;
+	}
+
+	if ((ring->wptr + 1 - ring->rptr) >= ring->rb_size) {
+		dev_err(adev->dev, "accel_id %u command ring full wptr 0x%llx rptr 0x%llx\n",
+			remote_accel_id, ring->wptr, ring->rptr);
+		r = -ENOSPC;
+		goto unlock_free;
+	}
+
+	ib = &job->ibs[0];
+	src = ib->gpu_addr + ndw_copy_cmd * 4;
+	src_cpu = ib->ptr + ndw_copy_cmd;
+
+	seq = ++ring->seq;
+
+	dev_dbg(adev->dev, "src 0x%llx to npa gart rb 0x%llx wptr 0x%llx doorbell 0x%llx seq 0x%x\n",
+		src, ring->rb_npa_gart + (ring->wptr % ring->rb_size) * 64,
+		ring->wptr_npa_gart, ring->doorbell_npa_gart, seq);
+
+	dev_dbg(adev->dev, "ring wptr 0x%llx rptr 0x%llx\n", ring->wptr, ring->rptr);
+
+	/* remote command packet */
+	emit_func(&src_cpu, seq, dw0, dw1, dw2, dw3);
+
+	/* wptr value */
+	*src_cpu++ = lower_32_bits(ring->wptr + 1);
+	*src_cpu++ = upper_32_bits(ring->wptr + 1);
+
+	/* doorbell value */
+	*src_cpu++ = lower_32_bits(ring->wptr + 1);
+
+	/*
+	 * Add an offset to the doorbell address that cycles through different
+	 * values of bit 11:8 to use different DXS ports.
+	 */
+	peer->dxs_port = (peer->dxs_port + 1) & 0xF;
+	doorbell_npa_gart = ring->doorbell_npa_gart | (peer->dxs_port << 8);
+	dev_dbg(adev->dev, "dxs_port 0x%x, doorbell_npa_gart 0x%llx -> 0x%llx\n",
+		peer->dxs_port, ring->doorbell_npa_gart, doorbell_npa_gart);
+
+	sdma_ring = &adev->sdma.instance[0].ring;
+
+	/*
+	 * SDMA commands copy from ib to NPA, insert NOPs to ensure SDMA copy commands
+	 * execution doesn't overlap, doorbell update is after data and wptr update
+	 * finished.
+	 */
+	amdgpu_emit_copy_buffer(adev, ib, src,
+				ring->rb_npa_gart + (ring->wptr % ring->rb_size) * 64,
+				64, 0);
+
+	ib->ptr[ib->length_dw++] = sdma_ring->funcs->nop;
+
+	amdgpu_emit_copy_buffer(adev, ib, src + 64, ring->wptr_npa_gart, 8, 0);
+
+	ib->ptr[ib->length_dw++] = sdma_ring->funcs->nop;
+
+	amdgpu_emit_copy_buffer(adev, ib, src + 72, doorbell_npa_gart, 4, 0);
+
+	amdgpu_ring_pad_ib(sdma_ring, ib);
+	WARN_ON(ib->length_dw > ndw_copy_cmd);
+
+	fence = amdgpu_job_submit(job);
+	r = dma_fence_wait_timeout(fence, false, AMDGPU_FENCE_JIFFIES_TIMEOUT);
+	dma_fence_put(fence);
+	if (r <= 0) {
+		dev_dbg(adev->dev, "remote %u sdma fence wait return r %d\n",
+			remote_accel_id, r);
+
+		if (r == 0)
+			r = -ETIME;
+
+		mutex_unlock(&peer->lock);
+		return r;
+	}
+
+	/*
+	 * poll remote completion writeback data
+	 */
+	r = amdgpu_ualink_remote_wait_timeout(adev, remote_accel_id, wb_cpu, seq);
+
+	/* increase local copy ring wptr, only if FW not timeout */
+	if (r != -ETIME)
+		ring->wptr++;
+
+	/*
+	 * Release ring lock after the remote FW handle command completes to
+	 * prevent race conditions.
+	 */
+	mutex_unlock(&peer->lock);
+	return r;
+
+
+unlock_free:
+	mutex_unlock(&peer->lock);
+	amdgpu_job_free(job);
+	dev_dbg(adev->dev, "ret r = %d\n", r);
+	return r;
+}
+
-- 
2.55.0