Re: [PATCHv2 2/8] drm/amdgpu: Support VMID 0 GPUVA completion WB address for UALink

[Felix Kuehling <felix.kuehling@amd.com>]

On 2026-10-27 15:54, Mukul Joshi wrote:
> The UPDATE_WB_ADDR remote command hands the MP nHT FW the address it
> writes command completions back to. So far that has always been an NPA.
>
> The FW can also take a VMID 0 GPUVA, selected by setting DW4 of the
> command to 1. Return the VMID 0 GPUVA of the writeback slot from
> amdgpu_ualink_get_wb_addr() alongside the NPA, and use it in
> UALink loopback mode. There the HW only carries importer -> exporter
> traffic, so the FW cannot write the completion back over NPA.
>
> Outside loopback the NPA is still used and DW4 stays 0.
>
> Signed-off-by: Mukul Joshi <mukul.joshi@amd.com>

Reviewed-by: Felix Kuehling <felix.kuehling@amd.com>


> ---
>   drivers/gpu/drm/amd/amdgpu/amdgpu_ualink.c | 48 +++++++++++++++-------
>   1 file changed, 34 insertions(+), 14 deletions(-)
>
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ualink.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ualink.c
> index 2cd6dc71ab9f..a186859dee3c 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ualink.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ualink.c
> @@ -5641,9 +5641,9 @@ static void amdgpu_ualink_emit_update_wb_addr(u32 **cpu_addr_p, u32 wb_data,
>   
>   	*cpu_addr++ = cmd;
>   	*cpu_addr++ = wb_data;
> -	*cpu_addr++ = dw1 & 0xFFFFFFFC;	/* lower32 wb_npa */
> -	*cpu_addr++ = dw0 & 0xFFFFF;	/* upper32 wb_npa */
> -	*cpu_addr++ = 0;
> +	*cpu_addr++ = dw1 & 0xFFFFFFFC;	/* lower32 wb address */
> +	*cpu_addr++ = dw0 & 0xFFFFF;	/* upper32 wb address */
> +	*cpu_addr++ = dw2;		/* 1: wb address is VMID 0 GPUVA, 0: NPA */
>   	*cpu_addr++ = 0;
>   	*cpu_addr++ = 0; /* metadata */
>   	*cpu_addr++ = 0; /* metadata */
> @@ -5857,7 +5857,7 @@ static int amdgpu_ualink_send_command(struct amdgpu_device *adev,
>   static void amdgpu_ualink_get_wb_addr(struct amdgpu_device *adev,
>   				      u32 remote_accel_id,
>   				      struct amdgpu_ualink_wb **wb_cpu,
> -				      u64 *wb_npa, u32 type)
> +				      u64 *wb_npa, u64 *wb_gpu, u32 type)
>   {
>   	struct amdgpu_ualink_remote *remote = to_remote(adev);
>   	/* The GPUID encoded in the write-back NPA address is the routing target
> @@ -5870,15 +5870,17 @@ static void amdgpu_ualink_get_wb_addr(struct amdgpu_device *adev,
>   	u32 accel_id = ualink_exp_accel_id(adev);
>   	u32 rb_size = AMDGPU_GPU_PAGE_ALIGN(2 * AMDGPU_UALINK_RB_SIZE);
>   	uintptr_t wb;
> -	u64 npa;
> +	u64 gpu, npa;
>   	u32 offset = 0;
>   
>   	if (ualink_addr_mode(adev) == AMDGPU_UALINK_ADDR_MODE_SOURCE_IDENT) {
>   		wb = (uintptr_t)remote->rb_cpu_addr + rb_size * remote->num_accel;
> +		gpu = remote->rb_gpu_addr + rb_size * remote->num_accel;
>   		npa = amdgpu_ualink_npa_addr(adev, RB_TYPE_TAILPTR,
>   					     remote_accel_id, accel_id);
>   	} else {
>   		wb = (uintptr_t)remote->rptr_cpu_addr;
> +		gpu = remote->rptr_gpu_addr;
>   		npa = remote->rptr_npa;
>   		npa |= (u64)accel_id << AMDGPU_UALINK_GART_NPA_ADDR_GPUID_SHIFT;
>   	}
> @@ -5891,9 +5893,11 @@ static void amdgpu_ualink_get_wb_addr(struct amdgpu_device *adev,
>   	*wb_cpu = (struct amdgpu_ualink_wb *)(wb + offset);
>   	if (wb_npa)
>   		*wb_npa = npa + offset;
> +	if (wb_gpu)
> +		*wb_gpu = gpu + offset;
>   
> -	dev_dbg(adev->dev, "source %d remote %d wb npa 0x%llx\n", accel_id,
> -		remote_accel_id, npa + offset);
> +	dev_dbg(adev->dev, "source %d remote %d wb npa 0x%llx gpu 0x%llx\n", accel_id,
> +		remote_accel_id, npa + offset, gpu + offset);
>   }
>   
>   struct amdgpu_ualink_wptr_fence_cb {
> @@ -6057,9 +6061,9 @@ static int amdgpu_ualink_ring_resync_ptrs(struct amdgpu_device *adev,
>   		ualink_exp_accel_id(adev), peer->interrupt.wptr, peer->shootdown.wptr,
>   		remote_accel_id, int_wptr, tlb_wptr);
>   
> -	amdgpu_ualink_get_wb_addr(adev, remote_accel_id, &int_wb, NULL,
> +	amdgpu_ualink_get_wb_addr(adev, remote_accel_id, &int_wb, NULL, NULL,
>   				  RB_TYPE_REMOTE_INTERRUPT);
> -	amdgpu_ualink_get_wb_addr(adev, remote_accel_id, &tlb_wb, NULL,
> +	amdgpu_ualink_get_wb_addr(adev, remote_accel_id, &tlb_wb, NULL, NULL,
>   				  RB_TYPE_TLB_INV);
>   
>   	peer->interrupt.wptr = int_wptr;
> @@ -6093,7 +6097,8 @@ static int amdgpu_ualink_update_wb_address(struct amdgpu_device *adev,
>   	struct amdgpu_ualink_peer *peer;
>   	struct amdgpu_ualink_ring *ring;
>   	struct amdgpu_ualink_wb *wb_cpu;
> -	u64 wb_npa;
> +	u64 wb_npa, wb_gpu, wb_addr;
> +	u32 wb_vmid0;
>   	int r;
>   
>   	peer = &remote->peer[remote_accel_id];
> @@ -6105,13 +6110,28 @@ static int amdgpu_ualink_update_wb_address(struct amdgpu_device *adev,
>   		return -EINVAL;
>   
>   	amdgpu_ualink_get_wb_addr(adev, remote_accel_id, &wb_cpu, &wb_npa,
> -				  ring_type);
> +				  &wb_gpu, ring_type);
> +
> +	/* UALink loopback: HW only carries importer -> exporter traffic, so the
> +	 * MPNHT FW cannot write the completion back over NPA. Hand it the VMID 0
> +	 * GPUVA of the writeback instead and flag it in DW4 of the command.
> +	 */
> +	if (adev->ualink.loopback.enabled) {
> +		wb_addr = wb_gpu;
> +		wb_vmid0 = 1;
> +	} else {
> +		wb_addr = wb_npa;
> +		wb_vmid0 = 0;
> +	}
> +
> +	dev_dbg(adev->dev, "remote %u ring type %u wb %s 0x%llx\n",
> +		remote_accel_id, ring_type, wb_vmid0 ? "gpuva" : "npa", wb_addr);
>   
>   	r = amdgpu_ualink_send_command(adev, remote_accel_id, ring, wb_cpu,
>   				       amdgpu_ualink_emit_update_wb_addr,
> -				       upper_32_bits(wb_npa),
> -				       lower_32_bits(wb_npa),
> -				       0, 0);
> +				       upper_32_bits(wb_addr),
> +				       lower_32_bits(wb_addr),
> +				       wb_vmid0, 0);
>   	if (!r) {
>   		ring->wb_cpu = wb_cpu;
>   		ring->ready = true;