* [PATCH v2 1/2] drm/amdgpu/mes: Support 64-bit register reads @ 2026-09-24 6:35 Victor Zhao 2026-09-24 6:35 ` [PATCH v2 2/2] drm/amdgpu: Read RLC GPU clock on GFX12.1 VF Victor Zhao 2026-09-28 4:43 ` [PATCH v2 1/2] drm/amdgpu/mes: Support 64-bit register reads Lazar, Lijo 0 siblings, 2 replies; 7+ messages in thread From: Victor Zhao @ 2026-09-24 6:35 UTC (permalink / raw) To: amd-gfx, Lijo.Lazar, Alexander.Deucher, Shaoyun.Liu Cc: Davis.Ming, Gavin.Wan, Victor Zhao [why] Some register values span two consecutive dwords and must be sampled with one MES operation. [how] Add amdgpu_mes_rreg64(), set READ_REG read64Bits for MES 11 and MES 12, and combine the two returned dwords into a 64-bit value. v2: add MES 11 support with the same read64Bits option Signed-off-by: Victor Zhao <Victor.Zhao@amd.com> --- drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c | 42 +++++++++++++++++++ drivers/gpu/drm/amd/amdgpu/amdgpu_mes.h | 3 ++ drivers/gpu/drm/amd/amdgpu/mes_v11_0.c | 2 + drivers/gpu/drm/amd/amdgpu/mes_v12_0.c | 2 + drivers/gpu/drm/amd/amdgpu/mes_v12_1.c | 2 + drivers/gpu/drm/amd/include/mes_v11_api_def.h | 7 ++++ 6 files changed, 58 insertions(+) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c index 5bebe74de01ad..52b05dc7b74e2 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c @@ -582,6 +582,7 @@ uint32_t amdgpu_mes_rreg(struct amdgpu_device *adev, uint32_t reg, op_input.op = MES_MISC_OP_READ_REG; op_input.read_reg.reg_offset = reg; op_input.read_reg.buffer_addr = read_val_gpu_addr; + op_input.read_reg.read64_bits = false; if (!adev->mes.funcs->misc_op) { dev_err(adev->dev, "mes rreg is not supported!\n"); @@ -602,6 +603,47 @@ uint32_t amdgpu_mes_rreg(struct amdgpu_device *adev, uint32_t reg, return val; } +uint64_t amdgpu_mes_rreg64(struct amdgpu_device *adev, uint32_t reg, + uint32_t xcc_id) +{ + struct mes_misc_op_input op_input = { 0 }; + uint64_t val = 0; + uint32_t addr_offset; + uint32_t *read_val_ptr; + int r; + + if (amdgpu_wb_get(adev, &addr_offset)) { + dev_err(adev->dev, "critical bug! too many mes readers\n"); + return 0; + } + + read_val_ptr = (uint32_t *)&adev->wb.wb[addr_offset]; + op_input.xcc_id = xcc_id; + op_input.op = MES_MISC_OP_READ_REG; + op_input.read_reg.reg_offset = reg; + op_input.read_reg.buffer_addr = adev->wb.gpu_addr + addr_offset * 4; + op_input.read_reg.read64_bits = true; + + if (!adev->mes.funcs->misc_op) { + dev_err(adev->dev, "mes rreg is not supported!\n"); + goto error; + } + + amdgpu_mes_lock(&adev->mes); + r = adev->mes.funcs->misc_op(&adev->mes, &op_input); + amdgpu_mes_unlock(&adev->mes); + if (r) { + dev_err(adev->dev, "failed to read reg (0x%x)\n", reg); + } else { + val = READ_ONCE(read_val_ptr[0]); + val |= (uint64_t)READ_ONCE(read_val_ptr[1]) << 32; + } + +error: + amdgpu_wb_free(adev, addr_offset); + return val; +} + int amdgpu_mes_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t val, uint32_t xcc_id) { diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.h index 33a426f58e723..d67814124f9e1 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.h @@ -373,6 +373,7 @@ struct mes_misc_op_input { struct { uint32_t reg_offset; uint64_t buffer_addr; + bool read64_bits; } read_reg; struct { @@ -545,6 +546,8 @@ int amdgpu_mes_detect_and_reset_hung_queues(struct amdgpu_device *adev, uint32_t amdgpu_mes_rreg(struct amdgpu_device *adev, uint32_t reg, uint32_t xcc_id); +uint64_t amdgpu_mes_rreg64(struct amdgpu_device *adev, uint32_t reg, + uint32_t xcc_id); int amdgpu_mes_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t val, uint32_t xcc_id); int amdgpu_mes_reg_write_reg_wait(struct amdgpu_device *adev, diff --git a/drivers/gpu/drm/amd/amdgpu/mes_v11_0.c b/drivers/gpu/drm/amd/amdgpu/mes_v11_0.c index d8b4d3deff605..304087b851644 100644 --- a/drivers/gpu/drm/amd/amdgpu/mes_v11_0.c +++ b/drivers/gpu/drm/amd/amdgpu/mes_v11_0.c @@ -905,6 +905,8 @@ static int mes_v11_0_misc_op(struct amdgpu_mes *mes, misc_pkt.opcode = MESAPI_MISC__READ_REG; misc_pkt.read_reg.reg_offset = input->read_reg.reg_offset; misc_pkt.read_reg.buffer_addr = input->read_reg.buffer_addr; + misc_pkt.read_reg.option.bits.read64Bits = + input->read_reg.read64_bits; break; case MES_MISC_OP_WRITE_REG: misc_pkt.opcode = MESAPI_MISC__WRITE_REG; diff --git a/drivers/gpu/drm/amd/amdgpu/mes_v12_0.c b/drivers/gpu/drm/amd/amdgpu/mes_v12_0.c index 04465804c2544..0d2a60528802e 100644 --- a/drivers/gpu/drm/amd/amdgpu/mes_v12_0.c +++ b/drivers/gpu/drm/amd/amdgpu/mes_v12_0.c @@ -858,6 +858,8 @@ static int mes_v12_0_misc_op(struct amdgpu_mes *mes, misc_pkt.opcode = MESAPI_MISC__READ_REG; misc_pkt.read_reg.reg_offset = input->read_reg.reg_offset; misc_pkt.read_reg.buffer_addr = input->read_reg.buffer_addr; + misc_pkt.read_reg.option.bits.read64Bits = + input->read_reg.read64_bits; break; case MES_MISC_OP_WRITE_REG: misc_pkt.opcode = MESAPI_MISC__WRITE_REG; diff --git a/drivers/gpu/drm/amd/amdgpu/mes_v12_1.c b/drivers/gpu/drm/amd/amdgpu/mes_v12_1.c index d17ced46d5f73..e0a488e516c09 100644 --- a/drivers/gpu/drm/amd/amdgpu/mes_v12_1.c +++ b/drivers/gpu/drm/amd/amdgpu/mes_v12_1.c @@ -581,6 +581,8 @@ static int mes_v12_1_misc_op(struct amdgpu_mes *mes, case MES_MISC_OP_READ_REG: misc_pkt.opcode = MESAPI_MISC__READ_REG; misc_pkt.read_reg.buffer_addr = input->read_reg.buffer_addr; + misc_pkt.read_reg.option.bits.read64Bits = + input->read_reg.read64_bits; mes_v12_1_get_rrmt(input->read_reg.reg_offset, GET_INST(GC, input->xcc_id), &misc_pkt.read_reg.rrmt_opt, diff --git a/drivers/gpu/drm/amd/include/mes_v11_api_def.h b/drivers/gpu/drm/amd/include/mes_v11_api_def.h index 1fe824b41f2a5..b27ec78da3b08 100644 --- a/drivers/gpu/drm/amd/include/mes_v11_api_def.h +++ b/drivers/gpu/drm/amd/include/mes_v11_api_def.h @@ -635,6 +635,13 @@ struct WRITE_REG { struct READ_REG { uint32_t reg_offset; uint64_t buffer_addr; + union { + struct { + uint32_t read64Bits : 1; + uint32_t reserved : 31; + } bits; + uint32_t all; + } option; }; enum WRM_OPERATION { -- 2.50.1 ^ permalink raw reply related [flat|nested] 7+ messages in thread
* [PATCH v2 2/2] drm/amdgpu: Read RLC GPU clock on GFX12.1 VF 2026-09-24 6:35 [PATCH v2 1/2] drm/amdgpu/mes: Support 64-bit register reads Victor Zhao @ 2026-09-24 6:35 ` Victor Zhao 2026-09-28 3:31 ` Zhao, Victor 2026-09-28 4:47 ` Lazar, Lijo 2026-09-28 4:43 ` [PATCH v2 1/2] drm/amdgpu/mes: Support 64-bit register reads Lazar, Lijo 1 sibling, 2 replies; 7+ messages in thread From: Victor Zhao @ 2026-09-24 6:35 UTC (permalink / raw) To: amd-gfx, Lijo.Lazar, Alexander.Deucher, Shaoyun.Liu Cc: Davis.Ming, Gavin.Wan, Victor Zhao [why] VF timestamp queries need a TSC sample independent of CP/MES. The SMUIO path is not safe on VF. [how] Capture via RLC_CAPTURE_GPU_CLOCK_COUNT and read the consecutive count registers with amdgpu_mes_rreg64(). Signed-off-by: Victor Zhao <Victor.Zhao@amd.com> --- drivers/gpu/drm/amd/amdgpu/gfx_v12_1.c | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v12_1.c b/drivers/gpu/drm/amd/amdgpu/gfx_v12_1.c index d6f70d33201a0..3c8133c94d6fc 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfx_v12_1.c +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v12_1.c @@ -3168,10 +3168,36 @@ static int gfx_v12_1_wait_for_idle(struct amdgpu_ip_block *ip_block) return -ETIMEDOUT; } +static uint64_t gfx_v12_1_get_rlc_gpu_clock_counter(struct amdgpu_device *adev) +{ + uint64_t clock = 0; + uint32_t reg; + + if (!down_read_trylock(&adev->reset_domain->sem)) + return 0; + if (amdgpu_device_skip_hw_access(adev) || + !adev->mes.ring[0].sched.ready) + goto unlock_reset; + + mutex_lock(&adev->gfx.gpu_clock_mutex); + WREG32_SOC15(GC, GET_INST(GC, 0), regRLC_CAPTURE_GPU_CLOCK_COUNT, 1); + reg = SOC15_REG_OFFSET(GC, GET_INST(GC, 0), + regRLC_GPU_CLOCK_COUNT_LSB); + clock = amdgpu_mes_rreg64(adev, reg, 0); + mutex_unlock(&adev->gfx.gpu_clock_mutex); + +unlock_reset: + up_read(&adev->reset_domain->sem); + return clock; +} + static uint64_t gfx_v12_1_get_gpu_clock_counter(struct amdgpu_device *adev) { uint64_t clock = 0; + if (amdgpu_sriov_vf(adev)) + return gfx_v12_1_get_rlc_gpu_clock_counter(adev); + if (adev->smuio.funcs && adev->smuio.funcs->get_gpu_clock_counter) clock = adev->smuio.funcs->get_gpu_clock_counter(adev); -- 2.50.1 ^ permalink raw reply related [flat|nested] 7+ messages in thread
* RE: [PATCH v2 2/2] drm/amdgpu: Read RLC GPU clock on GFX12.1 VF 2026-09-24 6:35 ` [PATCH v2 2/2] drm/amdgpu: Read RLC GPU clock on GFX12.1 VF Victor Zhao @ 2026-09-28 3:31 ` Zhao, Victor 2026-09-28 4:47 ` Lazar, Lijo 1 sibling, 0 replies; 7+ messages in thread From: Zhao, Victor @ 2026-09-28 3:31 UTC (permalink / raw) To: Zhao, Victor, amd-gfx@lists.freedesktop.org, Lazar, Lijo, Deucher, Alexander, Liu, Shaoyun Cc: Ming, Davis, Wan, Gavin AMD General @Lazar, Lijo @Liu, Shaoyun, Can you help review? Thanks, Victor -----Original Message----- From: Victor Zhao <Victor.Zhao@amd.com> Sent: Thursday, September 24, 2026 2:35 PM To: amd-gfx@lists.freedesktop.org; Lazar, Lijo <Lijo.Lazar@amd.com>; Deucher, Alexander <Alexander.Deucher@amd.com>; Liu, Shaoyun <Shaoyun.Liu@amd.com> Cc: Ming, Davis <Davis.Ming@amd.com>; Wan, Gavin <Gavin.Wan@amd.com>; Zhao, Victor <Victor.Zhao@amd.com> Subject: [PATCH v2 2/2] drm/amdgpu: Read RLC GPU clock on GFX12.1 VF [why] VF timestamp queries need a TSC sample independent of CP/MES. The SMUIO path is not safe on VF. [how] Capture via RLC_CAPTURE_GPU_CLOCK_COUNT and read the consecutive count registers with amdgpu_mes_rreg64(). Signed-off-by: Victor Zhao <Victor.Zhao@amd.com> --- drivers/gpu/drm/amd/amdgpu/gfx_v12_1.c | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v12_1.c b/drivers/gpu/drm/amd/amdgpu/gfx_v12_1.c index d6f70d33201a0..3c8133c94d6fc 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfx_v12_1.c +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v12_1.c @@ -3168,10 +3168,36 @@ static int gfx_v12_1_wait_for_idle(struct amdgpu_ip_block *ip_block) return -ETIMEDOUT; } +static uint64_t gfx_v12_1_get_rlc_gpu_clock_counter(struct +amdgpu_device *adev) { + uint64_t clock = 0; + uint32_t reg; + + if (!down_read_trylock(&adev->reset_domain->sem)) + return 0; + if (amdgpu_device_skip_hw_access(adev) || + !adev->mes.ring[0].sched.ready) + goto unlock_reset; + + mutex_lock(&adev->gfx.gpu_clock_mutex); + WREG32_SOC15(GC, GET_INST(GC, 0), regRLC_CAPTURE_GPU_CLOCK_COUNT, 1); + reg = SOC15_REG_OFFSET(GC, GET_INST(GC, 0), + regRLC_GPU_CLOCK_COUNT_LSB); + clock = amdgpu_mes_rreg64(adev, reg, 0); + mutex_unlock(&adev->gfx.gpu_clock_mutex); + +unlock_reset: + up_read(&adev->reset_domain->sem); + return clock; +} + static uint64_t gfx_v12_1_get_gpu_clock_counter(struct amdgpu_device *adev) { uint64_t clock = 0; + if (amdgpu_sriov_vf(adev)) + return gfx_v12_1_get_rlc_gpu_clock_counter(adev); + if (adev->smuio.funcs && adev->smuio.funcs->get_gpu_clock_counter) clock = adev->smuio.funcs->get_gpu_clock_counter(adev); -- 2.50.1 ^ permalink raw reply related [flat|nested] 7+ messages in thread
* Re: [PATCH v2 2/2] drm/amdgpu: Read RLC GPU clock on GFX12.1 VF 2026-09-24 6:35 ` [PATCH v2 2/2] drm/amdgpu: Read RLC GPU clock on GFX12.1 VF Victor Zhao 2026-09-28 3:31 ` Zhao, Victor @ 2026-09-28 4:47 ` Lazar, Lijo 1 sibling, 0 replies; 7+ messages in thread From: Lazar, Lijo @ 2026-09-28 4:47 UTC (permalink / raw) To: Victor Zhao, amd-gfx, Alexander.Deucher, Shaoyun.Liu Cc: Davis.Ming, Gavin.Wan On 24-Sep-26 12:05 PM, Victor Zhao wrote: > [why] > VF timestamp queries need a TSC sample independent of CP/MES. > The SMUIO path is not safe on VF. > > [how] > Capture via RLC_CAPTURE_GPU_CLOCK_COUNT and read the consecutive > count registers with amdgpu_mes_rreg64(). > > Signed-off-by: Victor Zhao <Victor.Zhao@amd.com> > --- > drivers/gpu/drm/amd/amdgpu/gfx_v12_1.c | 26 ++++++++++++++++++++++++++ > 1 file changed, 26 insertions(+) > > diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v12_1.c b/drivers/gpu/drm/amd/amdgpu/gfx_v12_1.c > index d6f70d33201a0..3c8133c94d6fc 100644 > --- a/drivers/gpu/drm/amd/amdgpu/gfx_v12_1.c > +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v12_1.c > @@ -3168,10 +3168,36 @@ static int gfx_v12_1_wait_for_idle(struct amdgpu_ip_block *ip_block) > return -ETIMEDOUT; > } > > +static uint64_t gfx_v12_1_get_rlc_gpu_clock_counter(struct amdgpu_device *adev) > +{ > + uint64_t clock = 0; > + uint32_t reg; > + > + if (!down_read_trylock(&adev->reset_domain->sem)) > + return 0; > + if (amdgpu_device_skip_hw_access(adev) || > + !adev->mes.ring[0].sched.ready) > + goto unlock_reset; > + > + mutex_lock(&adev->gfx.gpu_clock_mutex); > + WREG32_SOC15(GC, GET_INST(GC, 0), regRLC_CAPTURE_GPU_CLOCK_COUNT, 1); > + reg = SOC15_REG_OFFSET(GC, GET_INST(GC, 0), > + regRLC_GPU_CLOCK_COUNT_LSB); > + clock = amdgpu_mes_rreg64(adev, reg, 0); The API error cases should be checked and the clock value (0?) should be returned accordingly to indicate an invalid counter value. Thanks, Lijo > + mutex_unlock(&adev->gfx.gpu_clock_mutex); > + > +unlock_reset: > + up_read(&adev->reset_domain->sem); > + return clock; > +} > + > static uint64_t gfx_v12_1_get_gpu_clock_counter(struct amdgpu_device *adev) > { > uint64_t clock = 0; > > + if (amdgpu_sriov_vf(adev)) > + return gfx_v12_1_get_rlc_gpu_clock_counter(adev); > + > if (adev->smuio.funcs && > adev->smuio.funcs->get_gpu_clock_counter) > clock = adev->smuio.funcs->get_gpu_clock_counter(adev); ^ permalink raw reply [flat|nested] 7+ messages in thread
* Re: [PATCH v2 1/2] drm/amdgpu/mes: Support 64-bit register reads 2026-09-24 6:35 [PATCH v2 1/2] drm/amdgpu/mes: Support 64-bit register reads Victor Zhao 2026-09-24 6:35 ` [PATCH v2 2/2] drm/amdgpu: Read RLC GPU clock on GFX12.1 VF Victor Zhao @ 2026-09-28 4:43 ` Lazar, Lijo 2026-09-28 4:51 ` Lazar, Lijo 2026-09-28 10:29 ` Zhao, Victor 1 sibling, 2 replies; 7+ messages in thread From: Lazar, Lijo @ 2026-09-28 4:43 UTC (permalink / raw) To: Victor Zhao, amd-gfx, Alexander.Deucher, Shaoyun.Liu Cc: Davis.Ming, Gavin.Wan On 24-Sep-26 12:05 PM, Victor Zhao wrote: > [why] > Some register values span two consecutive dwords and must be sampled > with one MES operation. > > [how] > Add amdgpu_mes_rreg64(), set READ_REG read64Bits for MES 11 and > MES 12, and combine the two returned dwords into a 64-bit value. > > v2: add MES 11 support with the same read64Bits option > > Signed-off-by: Victor Zhao <Victor.Zhao@amd.com> > --- > drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c | 42 +++++++++++++++++++ > drivers/gpu/drm/amd/amdgpu/amdgpu_mes.h | 3 ++ > drivers/gpu/drm/amd/amdgpu/mes_v11_0.c | 2 + > drivers/gpu/drm/amd/amdgpu/mes_v12_0.c | 2 + > drivers/gpu/drm/amd/amdgpu/mes_v12_1.c | 2 + > drivers/gpu/drm/amd/include/mes_v11_api_def.h | 7 ++++ > 6 files changed, 58 insertions(+) > > diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c > index 5bebe74de01ad..52b05dc7b74e2 100644 > --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c > +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c > @@ -582,6 +582,7 @@ uint32_t amdgpu_mes_rreg(struct amdgpu_device *adev, uint32_t reg, > op_input.op = MES_MISC_OP_READ_REG; > op_input.read_reg.reg_offset = reg; > op_input.read_reg.buffer_addr = read_val_gpu_addr; > + op_input.read_reg.read64_bits = false; > > if (!adev->mes.funcs->misc_op) { > dev_err(adev->dev, "mes rreg is not supported!\n"); > @@ -602,6 +603,47 @@ uint32_t amdgpu_mes_rreg(struct amdgpu_device *adev, uint32_t reg, > return val; > } > > +uint64_t amdgpu_mes_rreg64(struct amdgpu_device *adev, uint32_t reg, > + uint32_t xcc_id) > +{ > + struct mes_misc_op_input op_input = { 0 }; > + uint64_t val = 0; > + uint32_t addr_offset; > + uint32_t *read_val_ptr; > + int r; > + > + if (amdgpu_wb_get(adev, &addr_offset)) { > + dev_err(adev->dev, "critical bug! too many mes readers\n"); > + return 0; > + } > + > + read_val_ptr = (uint32_t *)&adev->wb.wb[addr_offset]; > + op_input.xcc_id = xcc_id; > + op_input.op = MES_MISC_OP_READ_REG; > + op_input.read_reg.reg_offset = reg; > + op_input.read_reg.buffer_addr = adev->wb.gpu_addr + addr_offset * 4; > + op_input.read_reg.read64_bits = true; > + > + if (!adev->mes.funcs->misc_op) { > + dev_err(adev->dev, "mes rreg is not supported!\n"); > + goto error; > + } > + > + amdgpu_mes_lock(&adev->mes); > + r = adev->mes.funcs->misc_op(&adev->mes, &op_input); > + amdgpu_mes_unlock(&adev->mes); > + if (r) { > + dev_err(adev->dev, "failed to read reg (0x%x)\n", reg); > + } else { > + val = READ_ONCE(read_val_ptr[0]); > + val |= (uint64_t)READ_ONCE(read_val_ptr[1]) << 32; > + } > + > +error: > + amdgpu_wb_free(adev, addr_offset); > + return val; > +} > + > int amdgpu_mes_wreg(struct amdgpu_device *adev, uint32_t reg, > uint32_t val, uint32_t xcc_id) > { > diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.h > index 33a426f58e723..d67814124f9e1 100644 > --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.h > +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.h > @@ -373,6 +373,7 @@ struct mes_misc_op_input { > struct { > uint32_t reg_offset; > uint64_t buffer_addr; > + bool read64_bits; > } read_reg; > > struct { > @@ -545,6 +546,8 @@ int amdgpu_mes_detect_and_reset_hung_queues(struct amdgpu_device *adev, > > uint32_t amdgpu_mes_rreg(struct amdgpu_device *adev, uint32_t reg, > uint32_t xcc_id); > +uint64_t amdgpu_mes_rreg64(struct amdgpu_device *adev, uint32_t reg, > + uint32_t xcc_id); > int amdgpu_mes_wreg(struct amdgpu_device *adev, > uint32_t reg, uint32_t val, uint32_t xcc_id); > int amdgpu_mes_reg_write_reg_wait(struct amdgpu_device *adev, > diff --git a/drivers/gpu/drm/amd/amdgpu/mes_v11_0.c b/drivers/gpu/drm/amd/amdgpu/mes_v11_0.c > index d8b4d3deff605..304087b851644 100644 > --- a/drivers/gpu/drm/amd/amdgpu/mes_v11_0.c > +++ b/drivers/gpu/drm/amd/amdgpu/mes_v11_0.c > @@ -905,6 +905,8 @@ static int mes_v11_0_misc_op(struct amdgpu_mes *mes, > misc_pkt.opcode = MESAPI_MISC__READ_REG; > misc_pkt.read_reg.reg_offset = input->read_reg.reg_offset; > misc_pkt.read_reg.buffer_addr = input->read_reg.buffer_addr; > + misc_pkt.read_reg.option.bits.read64Bits = > + input->read_reg.read64_bits; Is this only a missing interface definition or does this need supported FW version check? Thanks, Lijo > break; > case MES_MISC_OP_WRITE_REG: > misc_pkt.opcode = MESAPI_MISC__WRITE_REG; > diff --git a/drivers/gpu/drm/amd/amdgpu/mes_v12_0.c b/drivers/gpu/drm/amd/amdgpu/mes_v12_0.c > index 04465804c2544..0d2a60528802e 100644 > --- a/drivers/gpu/drm/amd/amdgpu/mes_v12_0.c > +++ b/drivers/gpu/drm/amd/amdgpu/mes_v12_0.c > @@ -858,6 +858,8 @@ static int mes_v12_0_misc_op(struct amdgpu_mes *mes, > misc_pkt.opcode = MESAPI_MISC__READ_REG; > misc_pkt.read_reg.reg_offset = input->read_reg.reg_offset; > misc_pkt.read_reg.buffer_addr = input->read_reg.buffer_addr; > + misc_pkt.read_reg.option.bits.read64Bits = > + input->read_reg.read64_bits; > break; > case MES_MISC_OP_WRITE_REG: > misc_pkt.opcode = MESAPI_MISC__WRITE_REG; > diff --git a/drivers/gpu/drm/amd/amdgpu/mes_v12_1.c b/drivers/gpu/drm/amd/amdgpu/mes_v12_1.c > index d17ced46d5f73..e0a488e516c09 100644 > --- a/drivers/gpu/drm/amd/amdgpu/mes_v12_1.c > +++ b/drivers/gpu/drm/amd/amdgpu/mes_v12_1.c > @@ -581,6 +581,8 @@ static int mes_v12_1_misc_op(struct amdgpu_mes *mes, > case MES_MISC_OP_READ_REG: > misc_pkt.opcode = MESAPI_MISC__READ_REG; > misc_pkt.read_reg.buffer_addr = input->read_reg.buffer_addr; > + misc_pkt.read_reg.option.bits.read64Bits = > + input->read_reg.read64_bits; > mes_v12_1_get_rrmt(input->read_reg.reg_offset, > GET_INST(GC, input->xcc_id), > &misc_pkt.read_reg.rrmt_opt, > diff --git a/drivers/gpu/drm/amd/include/mes_v11_api_def.h b/drivers/gpu/drm/amd/include/mes_v11_api_def.h > index 1fe824b41f2a5..b27ec78da3b08 100644 > --- a/drivers/gpu/drm/amd/include/mes_v11_api_def.h > +++ b/drivers/gpu/drm/amd/include/mes_v11_api_def.h > @@ -635,6 +635,13 @@ struct WRITE_REG { > struct READ_REG { > uint32_t reg_offset; > uint64_t buffer_addr; > + union { > + struct { > + uint32_t read64Bits : 1; > + uint32_t reserved : 31; > + } bits; > + uint32_t all; > + } option; > }; > > enum WRM_OPERATION { ^ permalink raw reply [flat|nested] 7+ messages in thread
* Re: [PATCH v2 1/2] drm/amdgpu/mes: Support 64-bit register reads 2026-09-28 4:43 ` [PATCH v2 1/2] drm/amdgpu/mes: Support 64-bit register reads Lazar, Lijo @ 2026-09-28 4:51 ` Lazar, Lijo 2026-09-28 10:29 ` Zhao, Victor 1 sibling, 0 replies; 7+ messages in thread From: Lazar, Lijo @ 2026-09-28 4:51 UTC (permalink / raw) To: Victor Zhao, amd-gfx, Alexander.Deucher, Shaoyun.Liu Cc: Davis.Ming, Gavin.Wan On 28-Sep-26 10:13 AM, Lazar, Lijo wrote: > > > On 24-Sep-26 12:05 PM, Victor Zhao wrote: >> [why] >> Some register values span two consecutive dwords and must be sampled >> with one MES operation. >> >> [how] >> Add amdgpu_mes_rreg64(), set READ_REG read64Bits for MES 11 and >> MES 12, and combine the two returned dwords into a 64-bit value. >> >> v2: add MES 11 support with the same read64Bits option >> >> Signed-off-by: Victor Zhao <Victor.Zhao@amd.com> >> --- >> drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c | 42 +++++++++++++++++++ >> drivers/gpu/drm/amd/amdgpu/amdgpu_mes.h | 3 ++ >> drivers/gpu/drm/amd/amdgpu/mes_v11_0.c | 2 + >> drivers/gpu/drm/amd/amdgpu/mes_v12_0.c | 2 + >> drivers/gpu/drm/amd/amdgpu/mes_v12_1.c | 2 + >> drivers/gpu/drm/amd/include/mes_v11_api_def.h | 7 ++++ >> 6 files changed, 58 insertions(+) >> >> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c b/drivers/gpu/ >> drm/amd/amdgpu/amdgpu_mes.c >> index 5bebe74de01ad..52b05dc7b74e2 100644 >> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c >> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c >> @@ -582,6 +582,7 @@ uint32_t amdgpu_mes_rreg(struct amdgpu_device >> *adev, uint32_t reg, >> op_input.op = MES_MISC_OP_READ_REG; >> op_input.read_reg.reg_offset = reg; >> op_input.read_reg.buffer_addr = read_val_gpu_addr; >> + op_input.read_reg.read64_bits = false; >> if (!adev->mes.funcs->misc_op) { >> dev_err(adev->dev, "mes rreg is not supported!\n"); >> @@ -602,6 +603,47 @@ uint32_t amdgpu_mes_rreg(struct amdgpu_device >> *adev, uint32_t reg, >> return val; >> } >> +uint64_t amdgpu_mes_rreg64(struct amdgpu_device *adev, uint32_t reg, >> + uint32_t xcc_id) >> +{ >> + struct mes_misc_op_input op_input = { 0 }; >> + uint64_t val = 0; >> + uint32_t addr_offset; >> + uint32_t *read_val_ptr; >> + int r; >> + >> + if (amdgpu_wb_get(adev, &addr_offset)) { >> + dev_err(adev->dev, "critical bug! too many mes readers\n"); >> + return 0; For the comment about patch 2, this API should return an error code. 0 could be misinterpreted as a valid register value. Thanks, Lijo >> + } >> + >> + read_val_ptr = (uint32_t *)&adev->wb.wb[addr_offset]; >> + op_input.xcc_id = xcc_id; >> + op_input.op = MES_MISC_OP_READ_REG; >> + op_input.read_reg.reg_offset = reg; >> + op_input.read_reg.buffer_addr = adev->wb.gpu_addr + addr_offset * 4; >> + op_input.read_reg.read64_bits = true; >> + >> + if (!adev->mes.funcs->misc_op) { >> + dev_err(adev->dev, "mes rreg is not supported!\n"); >> + goto error; >> + } >> + >> + amdgpu_mes_lock(&adev->mes); >> + r = adev->mes.funcs->misc_op(&adev->mes, &op_input); >> + amdgpu_mes_unlock(&adev->mes); >> + if (r) { >> + dev_err(adev->dev, "failed to read reg (0x%x)\n", reg); >> + } else { >> + val = READ_ONCE(read_val_ptr[0]); >> + val |= (uint64_t)READ_ONCE(read_val_ptr[1]) << 32; >> + } >> + >> +error: >> + amdgpu_wb_free(adev, addr_offset); >> + return val; >> +} >> + >> int amdgpu_mes_wreg(struct amdgpu_device *adev, uint32_t reg, >> uint32_t val, uint32_t xcc_id) >> { >> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.h b/drivers/gpu/ >> drm/amd/amdgpu/amdgpu_mes.h >> index 33a426f58e723..d67814124f9e1 100644 >> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.h >> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.h >> @@ -373,6 +373,7 @@ struct mes_misc_op_input { >> struct { >> uint32_t reg_offset; >> uint64_t buffer_addr; >> + bool read64_bits; >> } read_reg; >> struct { >> @@ -545,6 +546,8 @@ int amdgpu_mes_detect_and_reset_hung_queues(struct >> amdgpu_device *adev, >> uint32_t amdgpu_mes_rreg(struct amdgpu_device *adev, uint32_t reg, >> uint32_t xcc_id); >> +uint64_t amdgpu_mes_rreg64(struct amdgpu_device *adev, uint32_t reg, >> + uint32_t xcc_id); >> int amdgpu_mes_wreg(struct amdgpu_device *adev, >> uint32_t reg, uint32_t val, uint32_t xcc_id); >> int amdgpu_mes_reg_write_reg_wait(struct amdgpu_device *adev, >> diff --git a/drivers/gpu/drm/amd/amdgpu/mes_v11_0.c b/drivers/gpu/drm/ >> amd/amdgpu/mes_v11_0.c >> index d8b4d3deff605..304087b851644 100644 >> --- a/drivers/gpu/drm/amd/amdgpu/mes_v11_0.c >> +++ b/drivers/gpu/drm/amd/amdgpu/mes_v11_0.c >> @@ -905,6 +905,8 @@ static int mes_v11_0_misc_op(struct amdgpu_mes *mes, >> misc_pkt.opcode = MESAPI_MISC__READ_REG; >> misc_pkt.read_reg.reg_offset = input->read_reg.reg_offset; >> misc_pkt.read_reg.buffer_addr = input->read_reg.buffer_addr; >> + misc_pkt.read_reg.option.bits.read64Bits = >> + input->read_reg.read64_bits; > > > Is this only a missing interface definition or does this need supported > FW version check? > > Thanks, > Lijo > >> break; >> case MES_MISC_OP_WRITE_REG: >> misc_pkt.opcode = MESAPI_MISC__WRITE_REG; >> diff --git a/drivers/gpu/drm/amd/amdgpu/mes_v12_0.c b/drivers/gpu/drm/ >> amd/amdgpu/mes_v12_0.c >> index 04465804c2544..0d2a60528802e 100644 >> --- a/drivers/gpu/drm/amd/amdgpu/mes_v12_0.c >> +++ b/drivers/gpu/drm/amd/amdgpu/mes_v12_0.c >> @@ -858,6 +858,8 @@ static int mes_v12_0_misc_op(struct amdgpu_mes *mes, >> misc_pkt.opcode = MESAPI_MISC__READ_REG; >> misc_pkt.read_reg.reg_offset = input->read_reg.reg_offset; >> misc_pkt.read_reg.buffer_addr = input->read_reg.buffer_addr; >> + misc_pkt.read_reg.option.bits.read64Bits = >> + input->read_reg.read64_bits; >> break; >> case MES_MISC_OP_WRITE_REG: >> misc_pkt.opcode = MESAPI_MISC__WRITE_REG; >> diff --git a/drivers/gpu/drm/amd/amdgpu/mes_v12_1.c b/drivers/gpu/drm/ >> amd/amdgpu/mes_v12_1.c >> index d17ced46d5f73..e0a488e516c09 100644 >> --- a/drivers/gpu/drm/amd/amdgpu/mes_v12_1.c >> +++ b/drivers/gpu/drm/amd/amdgpu/mes_v12_1.c >> @@ -581,6 +581,8 @@ static int mes_v12_1_misc_op(struct amdgpu_mes *mes, >> case MES_MISC_OP_READ_REG: >> misc_pkt.opcode = MESAPI_MISC__READ_REG; >> misc_pkt.read_reg.buffer_addr = input->read_reg.buffer_addr; >> + misc_pkt.read_reg.option.bits.read64Bits = >> + input->read_reg.read64_bits; >> mes_v12_1_get_rrmt(input->read_reg.reg_offset, >> GET_INST(GC, input->xcc_id), >> &misc_pkt.read_reg.rrmt_opt, >> diff --git a/drivers/gpu/drm/amd/include/mes_v11_api_def.h b/drivers/ >> gpu/drm/amd/include/mes_v11_api_def.h >> index 1fe824b41f2a5..b27ec78da3b08 100644 >> --- a/drivers/gpu/drm/amd/include/mes_v11_api_def.h >> +++ b/drivers/gpu/drm/amd/include/mes_v11_api_def.h >> @@ -635,6 +635,13 @@ struct WRITE_REG { >> struct READ_REG { >> uint32_t reg_offset; >> uint64_t buffer_addr; >> + union { >> + struct { >> + uint32_t read64Bits : 1; >> + uint32_t reserved : 31; >> + } bits; >> + uint32_t all; >> + } option; >> }; >> enum WRM_OPERATION { > ^ permalink raw reply [flat|nested] 7+ messages in thread
* RE: [PATCH v2 1/2] drm/amdgpu/mes: Support 64-bit register reads 2026-09-28 4:43 ` [PATCH v2 1/2] drm/amdgpu/mes: Support 64-bit register reads Lazar, Lijo 2026-09-28 4:51 ` Lazar, Lijo @ 2026-09-28 10:29 ` Zhao, Victor 1 sibling, 0 replies; 7+ messages in thread From: Zhao, Victor @ 2026-09-28 10:29 UTC (permalink / raw) To: Lazar, Lijo, amd-gfx@lists.freedesktop.org, Deucher, Alexander, Liu, Shaoyun Cc: Ming, Davis, Wan, Gavin AMD General Thanks for reviewing. I checked mes change history and the 64bit support was added after 0x42 for mes 11, will add a version check. @Liu, Shaoyun please correct me if its wrong. Thanks, Victor -----Original Message----- From: Lazar, Lijo <Lijo.Lazar@amd.com> Sent: Monday, September 28, 2026 12:43 PM To: Zhao, Victor <Victor.Zhao@amd.com>; amd-gfx@lists.freedesktop.org; Deucher, Alexander <Alexander.Deucher@amd.com>; Liu, Shaoyun <Shaoyun.Liu@amd.com> Cc: Ming, Davis <Davis.Ming@amd.com>; Wan, Gavin <Gavin.Wan@amd.com> Subject: Re: [PATCH v2 1/2] drm/amdgpu/mes: Support 64-bit register reads On 24-Sep-26 12:05 PM, Victor Zhao wrote: > [why] > Some register values span two consecutive dwords and must be sampled > with one MES operation. > > [how] > Add amdgpu_mes_rreg64(), set READ_REG read64Bits for MES 11 and MES > 12, and combine the two returned dwords into a 64-bit value. > > v2: add MES 11 support with the same read64Bits option > > Signed-off-by: Victor Zhao <Victor.Zhao@amd.com> > --- > drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c | 42 +++++++++++++++++++ > drivers/gpu/drm/amd/amdgpu/amdgpu_mes.h | 3 ++ > drivers/gpu/drm/amd/amdgpu/mes_v11_0.c | 2 + > drivers/gpu/drm/amd/amdgpu/mes_v12_0.c | 2 + > drivers/gpu/drm/amd/amdgpu/mes_v12_1.c | 2 + > drivers/gpu/drm/amd/include/mes_v11_api_def.h | 7 ++++ > 6 files changed, 58 insertions(+) > > diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c > b/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c > index 5bebe74de01ad..52b05dc7b74e2 100644 > --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c > +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c > @@ -582,6 +582,7 @@ uint32_t amdgpu_mes_rreg(struct amdgpu_device *adev, uint32_t reg, > op_input.op = MES_MISC_OP_READ_REG; > op_input.read_reg.reg_offset = reg; > op_input.read_reg.buffer_addr = read_val_gpu_addr; > + op_input.read_reg.read64_bits = false; > > if (!adev->mes.funcs->misc_op) { > dev_err(adev->dev, "mes rreg is not supported!\n"); @@ -602,6 > +603,47 @@ uint32_t amdgpu_mes_rreg(struct amdgpu_device *adev, uint32_t reg, > return val; > } > > +uint64_t amdgpu_mes_rreg64(struct amdgpu_device *adev, uint32_t reg, > + uint32_t xcc_id) > +{ > + struct mes_misc_op_input op_input = { 0 }; > + uint64_t val = 0; > + uint32_t addr_offset; > + uint32_t *read_val_ptr; > + int r; > + > + if (amdgpu_wb_get(adev, &addr_offset)) { > + dev_err(adev->dev, "critical bug! too many mes readers\n"); > + return 0; > + } > + > + read_val_ptr = (uint32_t *)&adev->wb.wb[addr_offset]; > + op_input.xcc_id = xcc_id; > + op_input.op = MES_MISC_OP_READ_REG; > + op_input.read_reg.reg_offset = reg; > + op_input.read_reg.buffer_addr = adev->wb.gpu_addr + addr_offset * 4; > + op_input.read_reg.read64_bits = true; > + > + if (!adev->mes.funcs->misc_op) { > + dev_err(adev->dev, "mes rreg is not supported!\n"); > + goto error; > + } > + > + amdgpu_mes_lock(&adev->mes); > + r = adev->mes.funcs->misc_op(&adev->mes, &op_input); > + amdgpu_mes_unlock(&adev->mes); > + if (r) { > + dev_err(adev->dev, "failed to read reg (0x%x)\n", reg); > + } else { > + val = READ_ONCE(read_val_ptr[0]); > + val |= (uint64_t)READ_ONCE(read_val_ptr[1]) << 32; > + } > + > +error: > + amdgpu_wb_free(adev, addr_offset); > + return val; > +} > + > int amdgpu_mes_wreg(struct amdgpu_device *adev, uint32_t reg, > uint32_t val, uint32_t xcc_id) > { > diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.h > b/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.h > index 33a426f58e723..d67814124f9e1 100644 > --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.h > +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.h > @@ -373,6 +373,7 @@ struct mes_misc_op_input { > struct { > uint32_t reg_offset; > uint64_t buffer_addr; > + bool read64_bits; > } read_reg; > > struct { > @@ -545,6 +546,8 @@ int amdgpu_mes_detect_and_reset_hung_queues(struct > amdgpu_device *adev, > > uint32_t amdgpu_mes_rreg(struct amdgpu_device *adev, uint32_t reg, > uint32_t xcc_id); > +uint64_t amdgpu_mes_rreg64(struct amdgpu_device *adev, uint32_t reg, > + uint32_t xcc_id); > int amdgpu_mes_wreg(struct amdgpu_device *adev, > uint32_t reg, uint32_t val, uint32_t xcc_id); > int amdgpu_mes_reg_write_reg_wait(struct amdgpu_device *adev, diff > --git a/drivers/gpu/drm/amd/amdgpu/mes_v11_0.c > b/drivers/gpu/drm/amd/amdgpu/mes_v11_0.c > index d8b4d3deff605..304087b851644 100644 > --- a/drivers/gpu/drm/amd/amdgpu/mes_v11_0.c > +++ b/drivers/gpu/drm/amd/amdgpu/mes_v11_0.c > @@ -905,6 +905,8 @@ static int mes_v11_0_misc_op(struct amdgpu_mes *mes, > misc_pkt.opcode = MESAPI_MISC__READ_REG; > misc_pkt.read_reg.reg_offset = input->read_reg.reg_offset; > misc_pkt.read_reg.buffer_addr = input->read_reg.buffer_addr; > + misc_pkt.read_reg.option.bits.read64Bits = > + input->read_reg.read64_bits; Is this only a missing interface definition or does this need supported FW version check? Thanks, Lijo > break; > case MES_MISC_OP_WRITE_REG: > misc_pkt.opcode = MESAPI_MISC__WRITE_REG; diff --git > a/drivers/gpu/drm/amd/amdgpu/mes_v12_0.c > b/drivers/gpu/drm/amd/amdgpu/mes_v12_0.c > index 04465804c2544..0d2a60528802e 100644 > --- a/drivers/gpu/drm/amd/amdgpu/mes_v12_0.c > +++ b/drivers/gpu/drm/amd/amdgpu/mes_v12_0.c > @@ -858,6 +858,8 @@ static int mes_v12_0_misc_op(struct amdgpu_mes *mes, > misc_pkt.opcode = MESAPI_MISC__READ_REG; > misc_pkt.read_reg.reg_offset = input->read_reg.reg_offset; > misc_pkt.read_reg.buffer_addr = input->read_reg.buffer_addr; > + misc_pkt.read_reg.option.bits.read64Bits = > + input->read_reg.read64_bits; > break; > case MES_MISC_OP_WRITE_REG: > misc_pkt.opcode = MESAPI_MISC__WRITE_REG; diff --git > a/drivers/gpu/drm/amd/amdgpu/mes_v12_1.c > b/drivers/gpu/drm/amd/amdgpu/mes_v12_1.c > index d17ced46d5f73..e0a488e516c09 100644 > --- a/drivers/gpu/drm/amd/amdgpu/mes_v12_1.c > +++ b/drivers/gpu/drm/amd/amdgpu/mes_v12_1.c > @@ -581,6 +581,8 @@ static int mes_v12_1_misc_op(struct amdgpu_mes *mes, > case MES_MISC_OP_READ_REG: > misc_pkt.opcode = MESAPI_MISC__READ_REG; > misc_pkt.read_reg.buffer_addr = input->read_reg.buffer_addr; > + misc_pkt.read_reg.option.bits.read64Bits = > + input->read_reg.read64_bits; > mes_v12_1_get_rrmt(input->read_reg.reg_offset, > GET_INST(GC, input->xcc_id), > &misc_pkt.read_reg.rrmt_opt, > diff --git a/drivers/gpu/drm/amd/include/mes_v11_api_def.h > b/drivers/gpu/drm/amd/include/mes_v11_api_def.h > index 1fe824b41f2a5..b27ec78da3b08 100644 > --- a/drivers/gpu/drm/amd/include/mes_v11_api_def.h > +++ b/drivers/gpu/drm/amd/include/mes_v11_api_def.h > @@ -635,6 +635,13 @@ struct WRITE_REG { > struct READ_REG { > uint32_t reg_offset; > uint64_t buffer_addr; > + union { > + struct { > + uint32_t read64Bits : 1; > + uint32_t reserved : 31; > + } bits; > + uint32_t all; > + } option; > }; > > enum WRM_OPERATION { ^ permalink raw reply [flat|nested] 7+ messages in thread
end of thread, other threads:[~2026-09-28 10:29 UTC | newest] Thread overview: 7+ messages (download: mbox.gz follow: Atom feed -- links below jump to the message on this page -- 2026-09-24 6:35 [PATCH v2 1/2] drm/amdgpu/mes: Support 64-bit register reads Victor Zhao 2026-09-24 6:35 ` [PATCH v2 2/2] drm/amdgpu: Read RLC GPU clock on GFX12.1 VF Victor Zhao 2026-09-28 3:31 ` Zhao, Victor 2026-09-28 4:47 ` Lazar, Lijo 2026-09-28 4:43 ` [PATCH v2 1/2] drm/amdgpu/mes: Support 64-bit register reads Lazar, Lijo 2026-09-28 4:51 ` Lazar, Lijo 2026-09-28 10:29 ` Zhao, Victor
This is a public inbox, see mirroring instructions for how to clone and mirror all data and code used for this inbox