* [RFC PATCH 2/3] media: rockchip: add VEPU510 H.264 encoder driver for RK3576
2026-07-22 7:34 [RFC PATCH 0/3] media: rockchip: VEPU510 H.264 encoder for RK3576 Jiaxing Hu
2026-07-22 7:34 ` [RFC PATCH 1/3] dt-bindings: media: add Rockchip RK3576 VEPU H.264 encoder Jiaxing Hu
@ 2026-07-22 7:34 ` Jiaxing Hu
2026-07-22 7:47 ` sashiko-bot
2026-07-22 10:00 ` Heiko Stübner
2026-07-22 7:34 ` [RFC PATCH 3/3] arm64: dts: rockchip: rk3576: add VEPU H.264 encoder nodes Jiaxing Hu
2 siblings, 2 replies; 7+ messages in thread
From: Jiaxing Hu @ 2026-07-22 7:34 UTC (permalink / raw)
To: mchehab, heiko, robh, krzk+dt, conor+dt, nicolas.dufresne
Cc: ezequiel, linux-media, devicetree, linux-rockchip,
linux-arm-kernel, linux-kernel, Jiaxing Hu
Add a from-scratch stateful V4L2 mem2mem driver for the Rockchip RK3576
VEPU510 H.264 hardware video encoder (raw NV12 in, H.264 Annex-B out),
modelled on the verisilicon/hantro device_run()/codec_ops split rather
than the downstream MPP-service/task-queue model.
The two encoder cores (rkvenc0/rkvenc1) are exposed as two independent
V4L2 M2M device nodes, each driving one physical core standalone; the
downstream vendor CCU cross-core load-balancing is not implemented.
Register field semantics were worked out by trial-and-error against
Rockchip's own open-source userspace codec library (rockchip-linux/mpp)
and cross-checked against a real register-write trace of the downstream
vendor stack running the same encode.
Intra (I-frame) encoding is confirmed working on real hardware (Radxa
ROCK 4D): it produces valid H.264 that the reference decoders accept.
Inter (P-frame) encoding still hits a hardware-watchdog stall in the
reference-read datapath -- this is the main open question for this RFC;
see the cover letter for the full symptom analysis.
Signed-off-by: Jiaxing Hu <gahing@gahingwoo.com>
---
drivers/media/platform/rockchip/Kconfig | 1 +
drivers/media/platform/rockchip/Makefile | 1 +
drivers/media/platform/rockchip/rkvenc/Kconfig | 14 +
drivers/media/platform/rockchip/rkvenc/Makefile | 6 +
.../media/platform/rockchip/rkvenc/rkvenc-h264.c | 1095 ++++++++++++++++++++
.../media/platform/rockchip/rkvenc/rkvenc-regs.h | 929 +++++++++++++++++
drivers/media/platform/rockchip/rkvenc/rkvenc.c | 892 ++++++++++++++++
drivers/media/platform/rockchip/rkvenc/rkvenc.h | 212 ++++
8 files changed, 3150 insertions(+)
diff --git a/drivers/media/platform/rockchip/Kconfig b/drivers/media/platform/rockchip/Kconfig
index ba401d32f..e6afa6092 100644
--- a/drivers/media/platform/rockchip/Kconfig
+++ b/drivers/media/platform/rockchip/Kconfig
@@ -6,3 +6,4 @@ source "drivers/media/platform/rockchip/rga/Kconfig"
source "drivers/media/platform/rockchip/rkcif/Kconfig"
source "drivers/media/platform/rockchip/rkisp1/Kconfig"
source "drivers/media/platform/rockchip/rkvdec/Kconfig"
+source "drivers/media/platform/rockchip/rkvenc/Kconfig"
diff --git a/drivers/media/platform/rockchip/Makefile b/drivers/media/platform/rockchip/Makefile
index 0e0b2cbbd..6a1d847a1 100644
--- a/drivers/media/platform/rockchip/Makefile
+++ b/drivers/media/platform/rockchip/Makefile
@@ -3,3 +3,4 @@ obj-y += rga/
obj-y += rkcif/
obj-y += rkisp1/
obj-y += rkvdec/
+obj-y += rkvenc/
diff --git a/drivers/media/platform/rockchip/rkvenc/Kconfig b/drivers/media/platform/rockchip/rkvenc/Kconfig
new file mode 100644
index 000000000..cd2a9c2eb
--- /dev/null
+++ b/drivers/media/platform/rockchip/rkvenc/Kconfig
@@ -0,0 +1,14 @@
+# SPDX-License-Identifier: GPL-2.0
+config VIDEO_ROCKCHIP_VEPU510
+ tristate "Rockchip VEPU510 Video Encoder driver"
+ depends on ARCH_ROCKCHIP || COMPILE_TEST
+ depends on VIDEO_DEV
+ select VIDEOBUF2_DMA_CONTIG
+ select V4L2_MEM2MEM_DEV
+ help
+ Support for the Rockchip VEPU510 hardware video encoder IP
+ present on the RK3576 SoC, which accelerates H.264 video
+ encoding.
+
+ To compile this driver as a module, choose M here: the module
+ will be called rockchip-vepu510.
diff --git a/drivers/media/platform/rockchip/rkvenc/Makefile b/drivers/media/platform/rockchip/rkvenc/Makefile
new file mode 100644
index 000000000..322bf10af
--- /dev/null
+++ b/drivers/media/platform/rockchip/rkvenc/Makefile
@@ -0,0 +1,6 @@
+# SPDX-License-Identifier: GPL-2.0
+obj-$(CONFIG_VIDEO_ROCKCHIP_VEPU510) += rockchip-vepu510.o
+
+rockchip-vepu510-y += \
+ rkvenc.o \
+ rkvenc-h264.o
diff --git a/drivers/media/platform/rockchip/rkvenc/rkvenc-h264.c b/drivers/media/platform/rockchip/rkvenc/rkvenc-h264.c
new file mode 100644
index 000000000..58b4fb65b
--- /dev/null
+++ b/drivers/media/platform/rockchip/rkvenc/rkvenc-h264.c
@@ -0,0 +1,1095 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Rockchip VEPU510 H.264 codec backend.
+ *
+ * Copyright (C) 2026 Jiaxing Hu <gahing@gahingwoo.com>
+ *
+ * VEPU510 emits complete Annex-B slice NAL units (nal_unit_header +
+ * slice_header + slice_data) directly from hardware — the extensive
+ * synt_sli0/synt_sli1/synt_sli2/synt_nal register set exists precisely
+ * because the slice_header() syntax is generated by the hardware from
+ * those fields (cross-checked against rockchip-linux/mpp's
+ * hal_h264e_vepu510.c, which programs the identical fields from its own
+ * software H264eSlice/H264ePps structs). What the hardware does *not*
+ * generate is the SPS/PPS (parameter sets are a per-stream, not
+ * per-frame, concept, and profile_idc/level_idc/VUI have no register
+ * representation at all) — mpp's own `h264e_vepu_stream_amend_*` stage
+ * confirms parameter sets are assembled in software around the hardware
+ * slice payload, which is exactly what rkvenc_h264_gen_sps_pps() below
+ * does, analogous to the upstream hantro JPEG encoder driver's software
+ * JFIF header prepend.
+ */
+
+#include <linux/delay.h>
+#include <linux/dma-mapping.h>
+#include <linux/log2.h>
+
+#include <media/v4l2-ctrls.h>
+#include <media/videobuf2-dma-contig.h>
+
+#include "rkvenc.h"
+#include "rkvenc-regs.h"
+
+/* SPS log2_max_frame_num_minus4 / log2_max_pic_order_cnt_lsb_minus4: both
+ * fixed at 4 (8-bit fields) so frame_num/poc_lsb wrap well past any sane
+ * GOP size while still fitting the 4-bit synt_sps.max_fnum/mpoc_lm4
+ * hardware fields (valid range 0-12 per spec, 0-15 per register width).
+ */
+#define RKVENC_H264_LOG2_MAX_FRAME_NUM 4
+#define RKVENC_H264_LOG2_MAX_POC_LSB 4
+
+/* PARAM (0x1700-0x19cc) and SQI (0x2000-0x20b8) class content: mpp's own
+ * *default-tuning-profile* constant tables (RDO lambda/cost tables and
+ * subjective-quality tuning respectively), selected by a tuning knob this
+ * driver doesn't expose rather than by resolution/QP/content -- so a
+ * fixed constant table is the correct v1 implementation, not a
+ * placeholder. Captured via wtrace from a real successful
+ * rockchip-linux/mpp `mpi_enc_test` encode and cross-checked against the
+ * declared class boundaries in mpp_rkvenc2.c's hw_info table (SQI's
+ * declared range extends to 0x212c, but a real encode never touches past
+ * 0x20b8, so this driver doesn't either). See the RC_ROI banner comment
+ * in rkvenc-regs.h for why these classes are not actually optional
+ * despite nothing in the userspace HAL ever documenting them as required.
+ */
+static const u32 rkvenc_par_class[] = {
+ 0x00030001, 0x00080006, 0x00030001, 0x00080006, 0x00030001, 0x00080006,
+ 0x00000fff, 0x10101010, 0x10101010, 0x16141a16, 0x13141512, 0x00000000,
+ 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00020001, 0x002d0006,
+ 0x00000000, 0x00000000, 0x00101010, 0x00121112, 0x00101010, 0x00000000,
+ 0x00000103, 0x0500c010, 0x00000000, 0x04040404, 0x00000021, 0x00000000,
+ 0x00401004, 0x00806040, 0x00000000, 0x00000000, 0x00000000, 0x00000000,
+ 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000,
+ 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000,
+ 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000,
+ 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000,
+ 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000,
+ 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000,
+ 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000,
+ 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000,
+ 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000,
+ 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000,
+ 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000,
+ 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000,
+ 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000,
+ 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000,
+ 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000,
+ 0x00000000, 0x00000000, 0x00000009, 0x0000000b, 0x0000000e, 0x00000011,
+ 0x00000016, 0x0000001b, 0x00000022, 0x0000002b, 0x00000036, 0x00000045,
+ 0x00000056, 0x0000006d, 0x00000089, 0x000000ad, 0x000000da, 0x00000112,
+ 0x00000159, 0x000001b3, 0x00000224, 0x000002b3, 0x00000366, 0x00000449,
+ 0x00000566, 0x000006cd, 0x00000891, 0x00000acb, 0x00000d9a, 0x000013c1,
+ 0x000018e4, 0x00001f5c, 0x00002783, 0x000031c8, 0x00003eb8, 0x00004f06,
+ 0x00006390, 0x00008e14, 0x0000b302, 0x0000e18a, 0x00011c29, 0x00016605,
+ 0x0001c313, 0x00027ae1, 0x00031fe6, 0x0003efcf, 0x0004f5c3, 0x0006e785,
+ 0x0008b2ef, 0x000af5c3, 0x000f1e7a, 0x00130c7f, 0x00180000, 0x001e3cf4,
+};
+
+static const u32 rkvenc_sqi_class[] = {
+ 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00002f10,
+ 0x00180000, 0x00400030, 0x00200010, 0x00600030, 0x00180030, 0x00300060,
+ 0x03010301, 0x04010401, 0x03000300, 0x00000301, 0x09060906, 0x08060b06,
+ 0x14061409, 0x00001007, 0x0006000a, 0x00003151, 0x00000000, 0x00000000,
+ 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00070000, 0x0019000f,
+ 0x10101010, 0x00000010, 0x00000000, 0x00000000, 0x00000000, 0x00280014,
+ 0x00000048, 0x10101010, 0x00000000, 0x00000000, 0x00000000, 0x00100008,
+ 0x00000020, 0x10101010, 0x00000000, 0x04010401, 0x08081010,
+};
+
+static void rkvenc_write_table_relaxed(struct rkvenc_dev *dev, u32 base_off,
+ const u32 *words, size_t count)
+{
+ size_t i;
+
+ for (i = 0; i < count; i++)
+ rkvenc_write_relaxed(dev, base_off + 4 * i, words[i]);
+}
+
+/* ---- minimal RBSP bit writer for SPS/PPS ---- */
+
+struct rkvenc_bitwriter {
+ u8 *buf;
+ size_t size;
+ size_t byte_pos;
+ u8 bit_pos;
+};
+
+static void bw_init(struct rkvenc_bitwriter *bw, u8 *buf, size_t size)
+{
+ bw->buf = buf;
+ bw->size = size;
+ bw->byte_pos = 0;
+ bw->bit_pos = 0;
+ memset(buf, 0, size);
+}
+
+static void bw_put_bit(struct rkvenc_bitwriter *bw, u32 bit)
+{
+ if (bw->byte_pos >= bw->size)
+ return;
+
+ if (bit)
+ bw->buf[bw->byte_pos] |= BIT(7 - bw->bit_pos);
+
+ if (++bw->bit_pos == 8) {
+ bw->bit_pos = 0;
+ bw->byte_pos++;
+ }
+}
+
+static void bw_put_bits(struct rkvenc_bitwriter *bw, u32 val, unsigned int n)
+{
+ int i;
+
+ for (i = n - 1; i >= 0; i--)
+ bw_put_bit(bw, (val >> i) & 1);
+}
+
+/* Exp-Golomb ue(v) */
+static void bw_put_ue(struct rkvenc_bitwriter *bw, u32 val)
+{
+ u32 code = val + 1;
+ unsigned int n = code ? fls(code) - 1 : 0;
+
+ bw_put_bits(bw, 0, n);
+ bw_put_bits(bw, code, n + 1);
+}
+
+/* Exp-Golomb se(v) */
+static void bw_put_se(struct rkvenc_bitwriter *bw, int val)
+{
+ u32 code = val <= 0 ? (u32)(-val) * 2 : (u32)val * 2 - 1;
+
+ bw_put_ue(bw, code);
+}
+
+static void bw_rbsp_trailing_bits(struct rkvenc_bitwriter *bw)
+{
+ bw_put_bit(bw, 1);
+ while (bw->bit_pos != 0)
+ bw_put_bit(bw, 0);
+}
+
+/* Emit one Annex-B NAL unit (start code + header + escaped RBSP) into out,
+ * returns bytes written.
+ */
+static size_t rkvenc_write_nal(u8 *out, size_t out_size, u8 nal_ref_idc,
+ u8 nal_unit_type, const u8 *rbsp, size_t rbsp_len)
+{
+ static const u8 start_code[] = { 0x00, 0x00, 0x00, 0x01 };
+ size_t pos = 0;
+ unsigned int zero_run = 0;
+ size_t i;
+
+ if (out_size < sizeof(start_code) + 1)
+ return 0;
+
+ memcpy(out, start_code, sizeof(start_code));
+ pos += sizeof(start_code);
+ out[pos++] = (nal_ref_idc << 5) | nal_unit_type;
+
+ for (i = 0; i < rbsp_len && pos < out_size; i++) {
+ if (zero_run >= 2 && rbsp[i] <= 3) {
+ out[pos++] = 0x03;
+ zero_run = 0;
+ if (pos >= out_size)
+ break;
+ }
+ out[pos++] = rbsp[i];
+ zero_run = rbsp[i] == 0 ? zero_run + 1 : 0;
+ }
+
+ return pos;
+}
+
+static const u8 rkvenc_h264_level_idc[] = {
+ [V4L2_MPEG_VIDEO_H264_LEVEL_1_0] = 10,
+ [V4L2_MPEG_VIDEO_H264_LEVEL_1B] = 11,
+ [V4L2_MPEG_VIDEO_H264_LEVEL_1_1] = 11,
+ [V4L2_MPEG_VIDEO_H264_LEVEL_1_2] = 12,
+ [V4L2_MPEG_VIDEO_H264_LEVEL_1_3] = 13,
+ [V4L2_MPEG_VIDEO_H264_LEVEL_2_0] = 20,
+ [V4L2_MPEG_VIDEO_H264_LEVEL_2_1] = 21,
+ [V4L2_MPEG_VIDEO_H264_LEVEL_2_2] = 22,
+ [V4L2_MPEG_VIDEO_H264_LEVEL_3_0] = 30,
+ [V4L2_MPEG_VIDEO_H264_LEVEL_3_1] = 31,
+ [V4L2_MPEG_VIDEO_H264_LEVEL_3_2] = 32,
+ [V4L2_MPEG_VIDEO_H264_LEVEL_4_0] = 40,
+ [V4L2_MPEG_VIDEO_H264_LEVEL_4_1] = 41,
+ [V4L2_MPEG_VIDEO_H264_LEVEL_4_2] = 42,
+};
+
+static void rkvenc_h264_gen_sps_pps(struct rkvenc_ctx *ctx)
+{
+ struct rkvenc_h264_ctx *h264 = &ctx->h264;
+ struct rkvenc_bitwriter bw;
+ u8 rbsp[32];
+ size_t rbsp_len, pos = 0;
+ unsigned int width = ctx->coded_fmt.width;
+ unsigned int height = ctx->coded_fmt.height;
+ unsigned int mb_w = ALIGN(width, 16) / 16;
+ unsigned int mb_h = ALIGN(height, 16) / 16;
+ unsigned int crop_w = mb_w * 16 - width;
+ unsigned int crop_h = mb_h * 16 - height;
+ bool crop = crop_w || crop_h;
+ bool is_main = h264->profile->val == V4L2_MPEG_VIDEO_H264_PROFILE_MAIN;
+ u8 profile_idc = is_main ? 77 : 66;
+ u8 level_idc = rkvenc_h264_level_idc[h264->level->val];
+ bool cabac = h264->entropy_mode->val == V4L2_MPEG_VIDEO_H264_ENTROPY_MODE_CABAC;
+
+ /* SPS (nal_ref_idc=1, nal_unit_type=7) */
+ bw_init(&bw, rbsp, sizeof(rbsp));
+ bw_put_bits(&bw, profile_idc, 8);
+ bw_put_bits(&bw, 0, 8); /* constraint_set0..5_flag + reserved_zero_2bits */
+ bw_put_bits(&bw, level_idc, 8);
+ bw_put_ue(&bw, 0); /* seq_parameter_set_id */
+ bw_put_ue(&bw, RKVENC_H264_LOG2_MAX_FRAME_NUM);
+ bw_put_ue(&bw, 0); /* pic_order_cnt_type */
+ bw_put_ue(&bw, RKVENC_H264_LOG2_MAX_POC_LSB);
+ bw_put_ue(&bw, 1); /* max_num_ref_frames */
+ bw_put_bit(&bw, 0); /* gaps_in_frame_num_value_allowed_flag */
+ bw_put_ue(&bw, mb_w - 1);
+ bw_put_ue(&bw, mb_h - 1);
+ bw_put_bit(&bw, 1); /* frame_mbs_only_flag */
+ bw_put_bit(&bw, 1); /* direct_8x8_inference_flag */
+ bw_put_bit(&bw, crop);
+ if (crop) {
+ bw_put_ue(&bw, 0);
+ bw_put_ue(&bw, crop_w / 2);
+ bw_put_ue(&bw, 0);
+ bw_put_ue(&bw, crop_h / 2);
+ }
+ bw_put_bit(&bw, 0); /* vui_parameters_present_flag */
+ bw_rbsp_trailing_bits(&bw);
+ rbsp_len = bw.byte_pos;
+ pos += rkvenc_write_nal(h264->sps_pps_nal + pos,
+ sizeof(h264->sps_pps_nal) - pos, 1, 7, rbsp, rbsp_len);
+
+ /* PPS (nal_ref_idc=1, nal_unit_type=8) */
+ bw_init(&bw, rbsp, sizeof(rbsp));
+ bw_put_ue(&bw, 0); /* pic_parameter_set_id */
+ bw_put_ue(&bw, 0); /* seq_parameter_set_id */
+ bw_put_bit(&bw, cabac);
+ bw_put_bit(&bw, 0); /* bottom_field_pic_order_in_frame_present_flag */
+ bw_put_ue(&bw, 0); /* num_slice_groups_minus1 */
+ bw_put_ue(&bw, 0); /* num_ref_idx_l0_default_active_minus1 */
+ bw_put_ue(&bw, 0); /* num_ref_idx_l1_default_active_minus1 */
+ bw_put_bit(&bw, 0); /* weighted_pred_flag */
+ bw_put_bits(&bw, 0, 2); /* weighted_bipred_idc */
+ bw_put_se(&bw, (int)h264->i_qp->val - 26); /* pic_init_qp_minus26 */
+ bw_put_se(&bw, 0); /* pic_init_qs_minus26 */
+ bw_put_se(&bw, 0); /* chroma_qp_index_offset */
+ bw_put_bit(&bw, 1); /* deblocking_filter_control_present_flag */
+ bw_put_bit(&bw, 0); /* constrained_intra_pred_flag */
+ bw_put_bit(&bw, 0); /* redundant_pic_cnt_present_flag */
+ bw_rbsp_trailing_bits(&bw);
+ rbsp_len = bw.byte_pos;
+ pos += rkvenc_write_nal(h264->sps_pps_nal + pos,
+ sizeof(h264->sps_pps_nal) - pos, 1, 8, rbsp, rbsp_len);
+
+ h264->sps_pps_len = pos;
+}
+
+/* ---- controls ---- */
+
+static int rkvenc_h264_s_ctrl(struct v4l2_ctrl *ctrl)
+{
+ return 0;
+}
+
+static const struct v4l2_ctrl_ops rkvenc_h264_ctrl_ops = {
+ .s_ctrl = rkvenc_h264_s_ctrl,
+};
+
+static int rkvenc_h264_init_ctrls(struct rkvenc_ctx *ctx)
+{
+ struct v4l2_ctrl_handler *hdl = &ctx->ctrl_handler;
+ struct rkvenc_h264_ctx *h264 = &ctx->h264;
+
+ /* Advisory only in v1: the hardware rc_en=0 fixed-QP
+ * path (rc_qp) is what's actually wired up, not this bitrate value.
+ */
+ h264->bitrate = v4l2_ctrl_new_std(hdl, &rkvenc_h264_ctrl_ops,
+ V4L2_CID_MPEG_VIDEO_BITRATE,
+ 64000, 100000000, 1, 2000000);
+
+ h264->profile = v4l2_ctrl_new_std_menu(hdl, &rkvenc_h264_ctrl_ops,
+ V4L2_CID_MPEG_VIDEO_H264_PROFILE,
+ V4L2_MPEG_VIDEO_H264_PROFILE_MAIN,
+ BIT(V4L2_MPEG_VIDEO_H264_PROFILE_CONSTRAINED_BASELINE),
+ V4L2_MPEG_VIDEO_H264_PROFILE_BASELINE);
+
+ h264->level = v4l2_ctrl_new_std_menu(hdl, &rkvenc_h264_ctrl_ops,
+ V4L2_CID_MPEG_VIDEO_H264_LEVEL,
+ V4L2_MPEG_VIDEO_H264_LEVEL_4_2,
+ BIT(V4L2_MPEG_VIDEO_H264_LEVEL_1B),
+ V4L2_MPEG_VIDEO_H264_LEVEL_4_0);
+
+ h264->entropy_mode = v4l2_ctrl_new_std_menu(hdl, &rkvenc_h264_ctrl_ops,
+ V4L2_CID_MPEG_VIDEO_H264_ENTROPY_MODE,
+ V4L2_MPEG_VIDEO_H264_ENTROPY_MODE_CABAC, 0,
+ V4L2_MPEG_VIDEO_H264_ENTROPY_MODE_CAVLC);
+
+ h264->gop_size = v4l2_ctrl_new_std(hdl, &rkvenc_h264_ctrl_ops,
+ V4L2_CID_MPEG_VIDEO_GOP_SIZE,
+ 1, 300, 1, 30);
+
+ h264->i_qp = v4l2_ctrl_new_std(hdl, &rkvenc_h264_ctrl_ops,
+ V4L2_CID_MPEG_VIDEO_H264_I_FRAME_QP,
+ 0, 51, 1, 26);
+
+ h264->p_qp = v4l2_ctrl_new_std(hdl, &rkvenc_h264_ctrl_ops,
+ V4L2_CID_MPEG_VIDEO_H264_P_FRAME_QP,
+ 0, 51, 1, 28);
+
+ return hdl->error;
+}
+
+/* ---- start/stop ---- */
+
+static int rkvenc_h264_start(struct rkvenc_ctx *ctx)
+{
+ struct rkvenc_dev *dev = ctx->dev;
+ struct rkvenc_h264_ctx *h264 = &ctx->h264;
+ /* Exact formulas from mpp's recon-buffer sizing function (the one
+ * that feeds hal_bufs_setup()) — aligned_h has the same +16 fudge
+ * mpp applies, not just MPP_ALIGN(height, 16).
+ */
+ unsigned int aligned_w = ALIGN(ctx->coded_fmt.width, 64);
+ unsigned int aligned_h = ALIGN(ctx->coded_fmt.height, 16) + 16;
+ size_t fbc_hdr_size = ALIGN((size_t)aligned_w * aligned_h / 64, SZ_8K);
+ size_t fbc_bdy_size = (size_t)aligned_w * aligned_h * 3 / 2;
+ size_t total_size;
+ int i;
+
+ memset(&h264->frame_num, 0,
+ sizeof(*h264) - offsetof(struct rkvenc_h264_ctx, frame_num));
+
+ h264->pixel_hdr_size = fbc_hdr_size;
+ h264->pixel_buf_size = fbc_hdr_size + fbc_bdy_size;
+ h264->thumb_buf_size = ALIGN((aligned_w / 64) * (aligned_h / 64) * 256, SZ_8K);
+ h264->smear_buf_size = ALIGN(aligned_w / 64, 16) * ALIGN(aligned_h / 16, 16);
+ total_size = h264->pixel_buf_size + h264->thumb_buf_size + h264->smear_buf_size;
+
+ for (i = 0; i < 2; i++) {
+ h264->recn_buf_cpu[i] = dma_alloc_coherent(dev->dev, total_size,
+ &h264->recn_buf_dma[i],
+ GFP_KERNEL);
+ if (!h264->recn_buf_cpu[i]) {
+ if (i)
+ dma_free_coherent(dev->dev, total_size,
+ h264->recn_buf_cpu[0],
+ h264->recn_buf_dma[0]);
+ return -ENOMEM;
+ }
+ }
+
+ h264->meiw_buf_cpu = dma_alloc_coherent(dev->dev, SZ_4K, &h264->meiw_buf_dma,
+ GFP_KERNEL);
+ if (!h264->meiw_buf_cpu) {
+ for (i = 0; i < 2; i++)
+ dma_free_coherent(dev->dev, total_size,
+ h264->recn_buf_cpu[i], h264->recn_buf_dma[i]);
+ return -ENOMEM;
+ }
+
+ /* Catch-all scratch buffer for the auxiliary FRAME-class write
+ * pointers (see the field comment in rkvenc.h). Sized like a full
+ * recon buffer so even a full-frame-sized spurious write fits.
+ */
+ rkvenc_h264_gen_sps_pps(ctx);
+
+ return 0;
+}
+
+static void rkvenc_h264_stop(struct rkvenc_ctx *ctx)
+{
+ struct rkvenc_dev *dev = ctx->dev;
+ struct rkvenc_h264_ctx *h264 = &ctx->h264;
+ size_t total_size = h264->pixel_buf_size + h264->thumb_buf_size +
+ h264->smear_buf_size;
+ int i;
+
+ for (i = 0; i < 2; i++) {
+ if (h264->recn_buf_cpu[i])
+ dma_free_coherent(dev->dev, total_size,
+ h264->recn_buf_cpu[i], h264->recn_buf_dma[i]);
+ h264->recn_buf_cpu[i] = NULL;
+ }
+
+ if (h264->meiw_buf_cpu)
+ dma_free_coherent(dev->dev, SZ_4K, h264->meiw_buf_cpu, h264->meiw_buf_dma);
+ h264->meiw_buf_cpu = NULL;
+}
+
+/* VEPU510-only quirk sequence, written unconditionally on every task by
+ * the downstream vendor kernel driver (drivers/video/rockchip/mpp/
+ * mpp_rkvenc2.c:rkvenc_run(), 0x74/0x308 block). Absent from
+ * rockchip-linux/mpp's userspace HAL entirely (it never touches these
+ * registers), so it cannot be cross-checked against the userspace source
+ * — treated as an opaque required silicon workaround for holding the
+ * encoder off "auto-restart" until this driver's IRQ model reads the
+ * completion status.
+ *
+ * EXPERIMENT, RESULT NEGATIVE (see git log): disabling this quirk was
+ * tried as a hypothesis for a genuine hardware-watchdog stall
+ * (int_sta wdg_sta) seen on first board bring-up. The board hit the
+ * exact same stall with this quirk disabled, so it is not the cause —
+ * re-enabled per the vendor's "unconditionally required" claim, since
+ * removing it demonstrated no benefit. See rkvenc-regs.h RC_ROI-class
+ * comment for the next hypothesis (roi_qthd0-3 never written).
+ */
+static void rkvenc_vepu510_quirk(struct rkvenc_dev *dev)
+{
+ rkvenc_write(dev, RKVENC_REG_DVBM_HOLD, RKVENC_DVBM_HOLD_VAL);
+ rkvenc_write(dev, RKVENC_REG_ENC_ID, RKVENC_ENC_ID_QUIRK_VAL);
+}
+
+static void rkvenc_h264_run(struct rkvenc_ctx *ctx)
+{
+ struct rkvenc_dev *dev = ctx->dev;
+ struct rkvenc_h264_ctx *h264 = &ctx->h264;
+ struct vb2_v4l2_buffer *src_buf = rkvenc_get_src_buf(ctx);
+ struct vb2_v4l2_buffer *dst_buf = rkvenc_get_dst_buf(ctx);
+ dma_addr_t src_dma = vb2_dma_contig_plane_dma_addr(&src_buf->vb2_buf, 0);
+ dma_addr_t dst_dma = vb2_dma_contig_plane_dma_addr(&dst_buf->vb2_buf, 0);
+ unsigned int width = ctx->coded_fmt.width;
+ unsigned int height = ctx->coded_fmt.height;
+ unsigned int aligned_w = ALIGN(width, 16);
+ unsigned int aligned_h = ALIGN(height, 16);
+ unsigned int uv_offset = ctx->src_fmt.bytesperline * ctx->src_fmt.height;
+ unsigned int dst_size = vb2_plane_size(&dst_buf->vb2_buf, 0);
+ bool is_idr = h264->gop_pos == 0;
+ unsigned int header_len = is_idr ? h264->sps_pps_len : 0;
+ unsigned int write_idx = h264->frame_num & 1;
+ /* Frame 0 has no previously-written reference at all -- alias read to
+ * write (real hardware's refr_idx == curr_idx for frame 0, see the
+ * rkvenc_h264_ctx comment in rkvenc.h). From frame 1 on, buf[write_idx
+ * ^ 1] genuinely holds the previous frame's real reconstruction.
+ */
+ unsigned int read_idx = h264->frame_num == 0 ? write_idx : (write_idx ^ 1);
+ bool is_main = h264->profile->val == V4L2_MPEG_VIDEO_H264_PROFILE_MAIN;
+ bool cabac = h264->entropy_mode->val == V4L2_MPEG_VIDEO_H264_ENTROPY_MODE_CABAC;
+ u32 qp = is_idr ? h264->i_qp->val : h264->p_qp->val;
+ union rkvenc_reg_enc_pic enc_pic = { .val = 0 };
+ union rkvenc_reg_enc_rsl enc_rsl = { .val = 0 };
+ union rkvenc_reg_src_fill src_fill = { .val = 0 };
+ union rkvenc_reg_src_fmt src_fmt = { .val = 0 };
+ union rkvenc_reg_src_strd0 src_strd0 = { .val = 0 };
+ union rkvenc_reg_src_strd1 src_strd1 = { .val = 0 };
+ union rkvenc_reg_rc_cfg rc_cfg = { .val = 0 };
+ union rkvenc_reg_rc_qp rc_qp = { .val = 0 };
+ union rkvenc_reg_sli_splt sli_splt = { .val = 0 };
+ union rkvenc_reg_rdo_cfg rdo_cfg = { .val = 0 };
+ union rkvenc_reg_synt_nal synt_nal = { .val = 0 };
+ union rkvenc_reg_synt_sps synt_sps = { .val = 0 };
+ union rkvenc_reg_synt_pps synt_pps = { .val = 0 };
+ union rkvenc_reg_synt_sli0 synt_sli0 = { .val = 0 };
+ union rkvenc_reg_synt_sli1 synt_sli1 = { .val = 0 };
+ union rkvenc_reg_synt_sli2 synt_sli2 = { .val = 0 };
+
+ if (header_len)
+ memcpy(vb2_plane_vaddr(&dst_buf->vb2_buf, 0), h264->sps_pps_nal, header_len);
+
+ rkvenc_write_relaxed(dev, RKVENC_REG_DBG_CLR, RKVENC_DBG_CLR_VAL);
+
+ /* Define every FRAME-class address/pointer register (0x270-0x2e8) by
+ * zeroing it before the specific pointers below overwrite the ones this
+ * driver actually uses. This matches the downstream vendor kernel, which
+ * writes its ENTIRE register file linearly every frame so no register
+ * ever carries stale bootloader/ATF/previous-session garbage. Doing this
+ * moved the long-standing rk_iommu write fault from a boot-varying
+ * garbage IOVA (the tell-tale 0x??b239?? stale-pointer signature) to a
+ * benign, deterministic 0x0.
+ *
+ * That residual write@0 (BUG-2) is a HW-synthesized write through one of
+ * these now-zero pointers (best candidate: the vrsp_rtn_en return-write
+ * to the zero online_addr; the vendor sets the same ENC_ID quirk with
+ * online_addr=0 and does NOT fault, so it is cosmetic). It is confirmed
+ * INDEPENDENT of the P-frame hang and harmless to the I-frame: a
+ * multi-agent evidence audit established the fault is a separate AXI
+ * transaction from the real recon write (rfpw=valid), it never lands
+ * (IOVA 0 pte invalid), and it cannot corrupt the reference the P-frame
+ * reads. An earlier attempt to redirect these pointers to scratch DMA
+ * buffers (colmvw/colmvr/lpfw/ebuft/ebufb) had NO measured effect on
+ * either bug and diverged from the vendor (which keeps them 0), so it was
+ * reverted -- zeroing alone is the vendor-matching behaviour.
+ */
+ {
+ u32 off;
+
+ for (off = RKVENC_REG_FRAME_OFFSET; off <= RKVENC_REG_ADR_ROIR; off += 4)
+ rkvenc_write_relaxed(dev, off, 0);
+ }
+
+ /* Source frame (NV12): adr_src0 = Y, adr_src1 = UV. adr_src2 (a 3rd
+ * plane pointer NV12 doesn't have) must equal adr_src1, not 0 — mpp's
+ * hal_h264e_vepu510.c sets it unconditionally to the same fd as
+ * adr_src1 even for 2-plane formats; confirmed via a real wtrace
+ * capture this is genuinely non-zero on a working encode.
+ */
+ rkvenc_write_relaxed(dev, RKVENC_REG_ADR_SRC0, src_dma);
+ rkvenc_write_relaxed(dev, RKVENC_REG_ADR_SRC1, src_dma + uv_offset);
+ rkvenc_write_relaxed(dev, RKVENC_REG_ADR_SRC2, src_dma + uv_offset);
+
+ /* All three of pixel(FBC)/thumb/smear are written by hardware on
+ * every frame regardless of feature use — see the rkvenc_h264_ctx
+ * comment in rkvenc.h. Leaving any of these at 0 IOMMU-faults in a
+ * tight retry loop instead of failing cleanly. The read side
+ * (rfpr/dspr/smear_rd) aliases the write side on frame 0 via
+ * read_idx == write_idx, see the read_idx comment above.
+ */
+ rkvenc_write_relaxed(dev, RKVENC_REG_RFPW_H_ADDR, h264->recn_buf_dma[write_idx]);
+ rkvenc_write_relaxed(dev, RKVENC_REG_RFPW_B_ADDR,
+ h264->recn_buf_dma[write_idx] + h264->pixel_hdr_size);
+ rkvenc_write_relaxed(dev, RKVENC_REG_DSPW_ADDR,
+ h264->recn_buf_dma[write_idx] + h264->pixel_buf_size);
+ rkvenc_write_relaxed(dev, RKVENC_REG_ADR_SMEAR_WR,
+ h264->recn_buf_dma[write_idx] + h264->pixel_buf_size +
+ h264->thumb_buf_size);
+
+ rkvenc_write_relaxed(dev, RKVENC_REG_RFPR_H_ADDR, h264->recn_buf_dma[read_idx]);
+ rkvenc_write_relaxed(dev, RKVENC_REG_RFPR_B_ADDR,
+ h264->recn_buf_dma[read_idx] + h264->pixel_hdr_size);
+ rkvenc_write_relaxed(dev, RKVENC_REG_DSPR_ADDR,
+ h264->recn_buf_dma[read_idx] + h264->pixel_buf_size);
+ rkvenc_write_relaxed(dev, RKVENC_REG_ADR_SMEAR_RD,
+ h264->recn_buf_dma[read_idx] + h264->pixel_buf_size +
+ h264->thumb_buf_size);
+
+ rkvenc_write_relaxed(dev, RKVENC_REG_MEIW_ADDR, h264->meiw_buf_dma);
+
+ /* Interlaced-field recon addresses: this driver never encodes
+ * interlaced content, so rfpt/rfpb stay disabled -- but real hardware
+ * expects the literal sentinel 0xffffffff in rfpt_h/rfpt_b specifically
+ * (not a valid fd-embedded address), not a plain 0. rfpb_h/adr_rfpb_b
+ * are 0 on a real encode too.
+ */
+ rkvenc_write_relaxed(dev, RKVENC_REG_RFPT_H_ADDR, RKVENC_RFPT_DISABLED);
+ rkvenc_write_relaxed(dev, RKVENC_REG_RFPB_H_ADDR, 0);
+ rkvenc_write_relaxed(dev, RKVENC_REG_RFPT_B_ADDR, RKVENC_RFPT_DISABLED);
+ rkvenc_write_relaxed(dev, RKVENC_REG_ADR_RFPB_B, 0);
+
+ /* Bitstream ring buffer: hardware writes its complete slice NAL
+ * starting right after our software-written SPS/PPS prefix (if any).
+ */
+ rkvenc_write_relaxed(dev, RKVENC_REG_BSBT_ADDR, dst_dma + dst_size - 1);
+ rkvenc_write_relaxed(dev, RKVENC_REG_BSBB_ADDR, dst_dma + header_len);
+ rkvenc_write_relaxed(dev, RKVENC_REG_ADR_BSBS, dst_dma + header_len);
+ rkvenc_write_relaxed(dev, RKVENC_REG_BSBR_ADDR, dst_dma + header_len);
+
+ enc_rsl.pic_wd8_m1 = aligned_w / 8 - 1;
+ enc_rsl.pic_hd8_m1 = aligned_h / 8 - 1;
+ rkvenc_write_relaxed(dev, RKVENC_REG_ENC_RSL, enc_rsl.val);
+
+ src_fill.pic_wfill = aligned_w - width;
+ src_fill.pic_hfill = aligned_h - height;
+ rkvenc_write_relaxed(dev, RKVENC_REG_SRC_FILL, src_fill.val);
+
+ src_fmt.src_cfmt = RKVENC_SRC_FMT_YUV420SP;
+ src_fmt.out_fmt = 1; /* see RKVENC_SRC_FMT_YUV420SP comment in rkvenc-regs.h */
+ rkvenc_write_relaxed(dev, RKVENC_REG_SRC_FMT, src_fmt.val);
+
+ rkvenc_write_relaxed(dev, RKVENC_REG_SRC_PROC, 0);
+ rkvenc_write_relaxed(dev, RKVENC_REG_PIC_OFST, 0);
+
+ src_strd0.src_strd0 = ctx->src_fmt.bytesperline;
+ rkvenc_write_relaxed(dev, RKVENC_REG_SRC_STRD0, src_strd0.val);
+ src_strd1.src_strd1 = ctx->src_fmt.bytesperline;
+ rkvenc_write_relaxed(dev, RKVENC_REG_SRC_STRD1, src_strd1.val);
+
+ /* RC configuration: rc_en=1/aq_en=1 with a real (if approximate --
+ * see below) rc_adj/rc_dthd threshold ladder, but rc_qp_range=0 and
+ * rc_min_qp==rc_max_qp so the AQ engine has no room to actually move
+ * QP away from the requested value. This is genuine fixed-QP
+ * *output* via a real, non-degenerate RC *register configuration* --
+ * NOT the driver's original rc_en=0/all-zero-threshold design, which
+ * board bring-up proved reproducibly stalls the hardware (see the
+ * RC_ROI banner comment in rkvenc-regs.h for the full story and the
+ * exact real-hardware capture that pinned this down).
+ *
+ * bit_target is only a rough per-frame bit budget (bitrate control
+ * divided by an assumed 30fps -- v1 doesn't negotiate real frame
+ * intervals) feeding mpp's real threshold-ladder formula
+ * (setup_vepu510_rc_base()); since rc_qp_range=0 clamps AQ to a
+ * no-op regardless, the exact value only needs to be sane and
+ * non-degenerate, not accurate -- true adaptive bitrate tracking is
+ * a follow-up.
+ */
+ {
+ unsigned int mb_w = aligned_w / 16;
+ unsigned int mb_h = aligned_h / 16;
+ s32 bit_target = (s32)(h264->bitrate->val / 30);
+ s32 mb_target_bits_mul_16 =
+ (s32)(((s64)bit_target << 4) / (mb_w * mb_h));
+ s32 mb_target_bits;
+ s32 negative_bits_thd, positive_bits_thd;
+ union rkvenc_reg_rc_adj0 rc_adj0 = { .val = 0 };
+ union rkvenc_reg_rc_adj1 rc_adj1 = { .val = 0 };
+
+ if (mb_target_bits_mul_16 >= 0x100000)
+ mb_target_bits_mul_16 = 0x50000;
+
+ mb_target_bits = (mb_target_bits_mul_16 * (s32)mb_w) >> 4;
+ negative_bits_thd = 0 - 5 * mb_target_bits / 16;
+ positive_bits_thd = 5 * mb_target_bits / 16;
+
+ rc_cfg.rc_en = 1;
+ rc_cfg.aq_en = 1;
+ rc_cfg.rc_ctu_num = mb_w;
+ rkvenc_write_relaxed(dev, RKVENC_REG_RC_CFG, rc_cfg.val);
+
+ rc_qp.rc_max_qp = qp;
+ rc_qp.rc_min_qp = qp;
+ /* EXPERIMENT, RESULT NEGATIVE (see git log): tried rc_qp_range=1
+ * (matching a real successful mpp capture's adaptive-RC value)
+ * instead of 0, as a hypothesis for the isolated rk_iommu fault
+ * below -- didn't stop it, and combined with other experimental
+ * changes at the time, board testing showed a real regression
+ * (every frame failing with a genuine hardware watchdog under
+ * multi-frame testing). Reverted to 0, which is confirmed safe
+ * for the fixed-QP output this driver actually wants (with
+ * rc_min_qp==rc_max_qp, AQ has no room to move QP regardless of
+ * range, so 0 vs 1 shouldn't matter for correctness -- but 0 is
+ * the value that's actually been confirmed not to regress).
+ */
+ rc_qp.rc_qp_range = 0;
+ rkvenc_write_relaxed(dev, RKVENC_REG_RC_QP, rc_qp.val);
+ rkvenc_write_relaxed(dev, RKVENC_REG_RC_TGT, (u32)mb_target_bits_mul_16);
+
+ /* mpp's own constant qp_adj ladder, same for every frame
+ * regardless of content/resolution/QP.
+ */
+ rc_adj0.qp_adj0 = -2;
+ rc_adj0.qp_adj1 = -1;
+ rc_adj0.qp_adj2 = 0;
+ rc_adj0.qp_adj3 = 1;
+ rc_adj0.qp_adj4 = 2;
+ rkvenc_write_relaxed(dev, RKVENC_REG_RC_ADJ0, rc_adj0.val);
+ rkvenc_write_relaxed(dev, RKVENC_REG_RC_ADJ1, rc_adj1.val);
+
+ rkvenc_write_relaxed(dev, RKVENC_REG_RC_DTHD(0), (u32)(4 * negative_bits_thd));
+ rkvenc_write_relaxed(dev, RKVENC_REG_RC_DTHD(1), (u32)negative_bits_thd);
+ rkvenc_write_relaxed(dev, RKVENC_REG_RC_DTHD(2), (u32)positive_bits_thd);
+ rkvenc_write_relaxed(dev, RKVENC_REG_RC_DTHD(3), (u32)(4 * positive_bits_thd));
+ rkvenc_write_relaxed(dev, RKVENC_REG_RC_DTHD(4), RKVENC_RC_DTHD_SENTINEL);
+ rkvenc_write_relaxed(dev, RKVENC_REG_RC_DTHD(5), RKVENC_RC_DTHD_SENTINEL);
+ rkvenc_write_relaxed(dev, RKVENC_REG_RC_DTHD(6), RKVENC_RC_DTHD_SENTINEL);
+ rkvenc_write_relaxed(dev, RKVENC_REG_RC_DTHD(7), RKVENC_RC_DTHD_SENTINEL);
+ rkvenc_write_relaxed(dev, RKVENC_REG_RC_DTHD(8), RKVENC_RC_DTHD_SENTINEL);
+ }
+
+ /* ROI QP-clamp thresholds: written unconditionally by mpp regardless
+ * of RC mode (see rkvenc-regs.h banner comment) — every area's
+ * [qpmin,qpmax] is set to this frame's QP so the clamp is a no-op
+ * rather than forcing every macroblock to QP 0.
+ */
+ {
+ union rkvenc_reg_roi_qthd0 roi_qthd0 = { .val = 0 };
+ union rkvenc_reg_roi_qthd1 roi_qthd1 = { .val = 0 };
+ union rkvenc_reg_roi_qthd2 roi_qthd2 = { .val = 0 };
+ union rkvenc_reg_roi_qthd3 roi_qthd3 = { .val = 0 };
+
+ roi_qthd0.qpmin_area0 = qp; roi_qthd0.qpmax_area0 = qp;
+ roi_qthd0.qpmin_area1 = qp; roi_qthd0.qpmax_area1 = qp;
+ roi_qthd0.qpmin_area2 = qp;
+ rkvenc_write_relaxed(dev, RKVENC_REG_ROI_QTHD0, roi_qthd0.val);
+
+ roi_qthd1.qpmax_area2 = qp;
+ roi_qthd1.qpmin_area3 = qp; roi_qthd1.qpmax_area3 = qp;
+ roi_qthd1.qpmin_area4 = qp; roi_qthd1.qpmax_area4 = qp;
+ rkvenc_write_relaxed(dev, RKVENC_REG_ROI_QTHD1, roi_qthd1.val);
+
+ roi_qthd2.qpmin_area5 = qp; roi_qthd2.qpmax_area5 = qp;
+ roi_qthd2.qpmin_area6 = qp; roi_qthd2.qpmax_area6 = qp;
+ roi_qthd2.qpmin_area7 = qp;
+ rkvenc_write_relaxed(dev, RKVENC_REG_ROI_QTHD2, roi_qthd2.val);
+
+ roi_qthd3.qpmax_area7 = qp;
+ roi_qthd3.qpmap_mode = 1; /* matches mpp's hardcoded value */
+ rkvenc_write_relaxed(dev, RKVENC_REG_ROI_QTHD3, roi_qthd3.val);
+ }
+
+ /* AQ activity LUT / MAD-statistics thresholds / chroma KLUT offset:
+ * see the banner comment above RKVENC_REG_AQ_TTHD in rkvenc-regs.h --
+ * real fields this driver never wrote at all until now, found by
+ * diffing against mpp's setup_vepu510_aq()/setup_vepu510_me()/
+ * setup_vepu510_rdo_pred(), all written unconditionally every frame
+ * (mpp's own default aq_tthd/aq_step tables are identical for I and P
+ * slices; klut_ofst is 6 for both in the non-IPC-tuning default this
+ * driver matches).
+ */
+ {
+ static const u8 aq_tthd[16] = {
+ 0, 0, 0, 0, 3, 3, 5, 5, 8, 8, 8, 15, 15, 20, 25, 25,
+ };
+ static const s32 aq_step[16] = {
+ -8, -7, -6, -5, -4, -3, -2, -1, 0, 1, 2, 3, 4, 5, 7, 8,
+ };
+ union rkvenc_reg_aq_stp0 aq_stp0 = { .val = 0 };
+ union rkvenc_reg_aq_stp1 aq_stp1 = { .val = 0 };
+ union rkvenc_reg_aq_stp2 aq_stp2 = { .val = 0 };
+ union rkvenc_reg_madi_st_thd madi_st_thd = { .val = 0 };
+ union rkvenc_reg_madp_st_thd0 madp_st_thd0 = { .val = 0 };
+ union rkvenc_reg_madp_st_thd1 madp_st_thd1 = { .val = 0 };
+ union rkvenc_reg_klut_ofst klut_ofst = { .val = 0 };
+ unsigned int i;
+
+ for (i = 0; i < 4; i++) {
+ u32 word = aq_tthd[4 * i] | (aq_tthd[4 * i + 1] << 8) |
+ (aq_tthd[4 * i + 2] << 16) | (aq_tthd[4 * i + 3] << 24);
+
+ rkvenc_write_relaxed(dev, RKVENC_REG_AQ_TTHD + 4 * i, word);
+ }
+
+ aq_stp0.aq_stp_s0 = aq_step[0]; aq_stp0.aq_stp_0t1 = aq_step[1];
+ aq_stp0.aq_stp_1t2 = aq_step[2]; aq_stp0.aq_stp_2t3 = aq_step[3];
+ aq_stp0.aq_stp_3t4 = aq_step[4]; aq_stp0.aq_stp_4t5 = aq_step[5];
+ rkvenc_write_relaxed(dev, RKVENC_REG_AQ_STP0, aq_stp0.val);
+
+ /* aq_stp_7t8 is a hardcoded 0 in mpp's setup_vepu510_aq(), not
+ * aq_step[8] -- a genuine off-by-one quirk in the vendor HAL
+ * (aq_step[8..10] shift into aq_stp_8t9..10t11 instead), real
+ * hardware expects exactly this mapping so it's reproduced
+ * verbatim rather than "corrected".
+ */
+ aq_stp1.aq_stp_5t6 = aq_step[6]; aq_stp1.aq_stp_6t7 = aq_step[7];
+ aq_stp1.aq_stp_7t8 = 0; aq_stp1.aq_stp_8t9 = aq_step[8];
+ aq_stp1.aq_stp_9t10 = aq_step[9]; aq_stp1.aq_stp_10t11 = aq_step[10];
+ rkvenc_write_relaxed(dev, RKVENC_REG_AQ_STP1, aq_stp1.val);
+
+ /* aq_stp2's middle two fields (12t13/13t14) don't match
+ * h264_P/I_aq_step_default's aq_step[12]/[13] verbatim -- cross-
+ * checked against the real vendor wtrace capture from the
+ * original I-frame bring-up (register 0x105c was 0x00839ca3
+ * there), which decodes to 3/5/7/7/8, not 3/4/5/7/8. Using the
+ * confirmed-real values for this one register instead of the
+ * source-derived table, since real silicon is the higher-truth
+ * source when the two disagree.
+ */
+ aq_stp2.aq_stp_11t12 = aq_step[11]; aq_stp2.aq_stp_12t13 = 5;
+ aq_stp2.aq_stp_13t14 = 7; aq_stp2.aq_stp_14t15 = aq_step[14];
+ aq_stp2.aq_stp_b15 = aq_step[15];
+ rkvenc_write_relaxed(dev, RKVENC_REG_AQ_STP2, aq_stp2.val);
+
+ madi_st_thd.madi_th0 = 5;
+ madi_st_thd.madi_th1 = 12;
+ madi_st_thd.madi_th2 = 20;
+ rkvenc_write_relaxed(dev, RKVENC_REG_MADI_ST_THD, madi_st_thd.val);
+
+ madp_st_thd0.madp_th0 = 4 << 4;
+ madp_st_thd0.madp_th1 = 9 << 4;
+ rkvenc_write_relaxed(dev, RKVENC_REG_MADP_ST_THD0, madp_st_thd0.val);
+
+ madp_st_thd1.madp_th2 = 15 << 4;
+ rkvenc_write_relaxed(dev, RKVENC_REG_MADP_ST_THD1, madp_st_thd1.val);
+
+ klut_ofst.chrm_klut_ofst = 6;
+ rkvenc_write_relaxed(dev, RKVENC_REG_KLUT_OFST, klut_ofst.val);
+ }
+
+ /* PARAM/SQI: mpp's default-tuning constant tables -- see the comment
+ * on rkvenc_par_class/rkvenc_sqi_class above. SCL: this driver never
+ * uses a custom H.264 scaling list, so the whole class is zero --
+ * written explicitly every frame rather than relying on POR/leftover
+ * state, since real hardware still expects the class to have been
+ * written (see rkvenc-regs.h banner comment).
+ */
+ rkvenc_write_table_relaxed(dev, RKVENC_REG_PAR_OFFSET, rkvenc_par_class,
+ ARRAY_SIZE(rkvenc_par_class));
+ rkvenc_write_table_relaxed(dev, RKVENC_REG_SQI_OFFSET, rkvenc_sqi_class,
+ ARRAY_SIZE(rkvenc_sqi_class));
+ {
+ u32 off;
+
+ for (off = RKVENC_REG_SCL_OFFSET;
+ off < RKVENC_REG_SCL_OFFSET + RKVENC_REG_SCL_SIZE; off += 4)
+ rkvenc_write_relaxed(dev, off, 0);
+ }
+
+ /* ATR (anti-ringing) weights + smear_opt_cfg.stated_mode: patch the
+ * handful of PARAM/SQI words that are genuinely I/P-slice-dependent on
+ * top of the static blobs above -- see the CORRECTION banner comments
+ * on RKVENC_REG_ATR_THD1 and RKVENC_REG_SMEAR_OPT_CFG in rkvenc-regs.h.
+ * Values below are mpp's real setup_vepu510_anti_ringing()/
+ * setup_vepu510_anti_smear() constants for this driver's non-IPC scene
+ * mode, confirmed exactly against a real 3-frame wtrace capture.
+ */
+ {
+ union rkvenc_reg_atr_thd1 atr_thd1 = { .val = 0 };
+ union rkvenc_reg_atr_wgt atr_wgt16 = { .val = 0 };
+ union rkvenc_reg_atr_wgt atr_wgt8 = { .val = 0 };
+ union rkvenc_reg_atr_wgt atr_wgt4 = { .val = 0 };
+ union rkvenc_reg_smear_opt_cfg smear_opt_cfg = { .val = 0 };
+
+ atr_thd1.thdqp = 45;
+ if (is_idr) {
+ atr_thd1.thd2 = 6;
+ atr_wgt16.wgt0 = 16; atr_wgt16.wgt1 = 16; atr_wgt16.wgt2 = 16;
+ atr_wgt8.wgt0 = 18; atr_wgt8.wgt1 = 17; atr_wgt8.wgt2 = 18;
+ atr_wgt4.wgt0 = 16; atr_wgt4.wgt1 = 16; atr_wgt4.wgt2 = 16;
+ } else {
+ atr_thd1.thd2 = 7;
+ atr_wgt16.wgt0 = 23; atr_wgt16.wgt1 = 22; atr_wgt16.wgt2 = 20;
+ atr_wgt8.wgt0 = 24; atr_wgt8.wgt1 = 24; atr_wgt8.wgt2 = 24;
+ atr_wgt4.wgt0 = 23; atr_wgt4.wgt1 = 22; atr_wgt4.wgt2 = 20;
+ }
+ rkvenc_write_relaxed(dev, RKVENC_REG_ATR_THD1, atr_thd1.val);
+ rkvenc_write_relaxed(dev, RKVENC_REG_ATR_WGT16, atr_wgt16.val);
+ rkvenc_write_relaxed(dev, RKVENC_REG_ATR_WGT8, atr_wgt8.val);
+ rkvenc_write_relaxed(dev, RKVENC_REG_ATR_WGT4, atr_wgt4.val);
+
+ smear_opt_cfg.rdo_smear_lvl16_multi = 16;
+ smear_opt_cfg.rdo_smear_en = 0;
+ smear_opt_cfg.stated_mode = (is_idr || h264->last_frame_was_idr) ? 1 : 2;
+ /* Static approximation of a real per-block-statistics-derived
+ * value -- see the RKVENC_REG_SMEAR_OPT_CFG comment.
+ */
+ smear_opt_cfg.rdo_smear_dlt_qp = is_idr ? -1 : -3;
+ rkvenc_write_relaxed(dev, RKVENC_REG_SMEAR_OPT_CFG, smear_opt_cfg.val);
+ }
+
+ /* Single slice per frame in v1. */
+ rkvenc_write_relaxed(dev, RKVENC_REG_SLI_SPLT, sli_splt.val);
+ rkvenc_write_relaxed(dev, RKVENC_REG_SLI_BYTE, 0);
+ rkvenc_write_relaxed(dev, RKVENC_REG_SLI_CNUM, 0);
+
+ /* Motion-estimation search config — written unconditionally by mpp
+ * on every frame regardless of I/P (see rkvenc-regs.h banner
+ * comment); values are mpp's own hardcoded constants.
+ */
+ {
+ union rkvenc_reg_me_rnge me_rnge = { .val = 0 };
+ union rkvenc_reg_me_cfg me_cfg = { .val = 0 };
+ union rkvenc_reg_me_cach me_cach = { .val = 0 };
+
+ me_rnge.cime_srch_dwnh = 15;
+ me_rnge.cime_srch_uph = 15;
+ me_rnge.cime_srch_rgtw = 12;
+ me_rnge.cime_srch_lftw = 12;
+ rkvenc_write_relaxed(dev, RKVENC_REG_ME_RNGE, me_rnge.val);
+
+ me_cfg.srgn_max_num = 54;
+ me_cfg.cime_dist_thre = 1024;
+ me_cfg.rme_srch_h = 3;
+ me_cfg.rme_srch_v = 3;
+ rkvenc_write_relaxed(dev, RKVENC_REG_ME_CFG, me_cfg.val);
+
+ me_cach.cime_zero_thre = 64;
+ rkvenc_write_relaxed(dev, RKVENC_REG_ME_CACH, me_cach.val);
+ }
+
+ /* Matches setup_vepu510_rdo_pred()'s derivation in the vendor HAL.
+ * chrm_spcl/ccwa_e/atr_mult_sel_e are unconditional 1s in mpp
+ * regardless of profile/level/tune config -- see rkvenc-regs.h.
+ */
+ rdo_cfg.rect_size = !is_main && rkvenc_h264_level_idc[h264->level->val] <= 30;
+ rdo_cfg.vlc_lmt = !is_main && !cabac;
+ rdo_cfg.chrm_spcl = 1;
+ rdo_cfg.ccwa_e = 1;
+ rdo_cfg.atr_mult_sel_e = 1;
+ rkvenc_write_relaxed(dev, RKVENC_REG_RDO_CFG, rdo_cfg.val);
+
+ /* mpp sets synt_nal.nal_ref_idc = slice->nal_reference_idc, which is
+ * NOT a flat 1 for every frame -- a real 3-frame capture showed 3 for
+ * the IDR slice and 2 for P slices (both real references, matching
+ * this driver's unconditional enc_pic.cur_frm_ref=1). This driver
+ * previously hardcoded 1 for both, a real spec-compliance gap in the
+ * hardware-generated slice NAL header (not known to be hang-related --
+ * the hardware just emits whatever value it's given).
+ */
+ synt_nal.nal_ref_idc = is_idr ? 3 : 2;
+ synt_nal.nal_unit_type = is_idr ? 5 : 1;
+ rkvenc_write_relaxed(dev, RKVENC_REG_SYNT_NAL, synt_nal.val);
+
+ synt_sps.max_fnum = RKVENC_H264_LOG2_MAX_FRAME_NUM;
+ synt_sps.drct_8x8 = 1;
+ synt_sps.mpoc_lm4 = RKVENC_H264_LOG2_MAX_POC_LSB;
+ synt_sps.poc_type = 0;
+ rkvenc_write_relaxed(dev, RKVENC_REG_SYNT_SPS, synt_sps.val);
+
+ synt_pps.etpy_mode = cabac;
+ synt_pps.csip_flag = 0;
+ synt_pps.num_ref0_idx = 0;
+ synt_pps.num_ref1_idx = 0;
+ synt_pps.pic_init_qp = h264->i_qp->val;
+ synt_pps.cb_ofst = 0;
+ synt_pps.cr_ofst = 0;
+ synt_pps.dbf_cp_flg = 1;
+ rkvenc_write_relaxed(dev, RKVENC_REG_SYNT_PPS, synt_pps.val);
+
+ synt_sli0.sli_type = is_idr ? RKVENC_SLI_TYPE_I : RKVENC_SLI_TYPE_P;
+ synt_sli0.pps_id = 0;
+ synt_sli0.cbc_init_idc = 0;
+ synt_sli0.frm_num = h264->frame_num;
+ rkvenc_write_relaxed(dev, RKVENC_REG_SYNT_SLI0, synt_sli0.val);
+
+ synt_sli1.idr_pid = h264->idr_pic_id;
+ synt_sli1.poc_lsb = h264->poc_lsb;
+ rkvenc_write_relaxed(dev, RKVENC_REG_SYNT_SLI1, synt_sli1.val);
+
+ synt_sli2.dis_dblk_idc = 0;
+ synt_sli2.sli_alph_ofst = 0;
+ synt_sli2.sli_beta_ofst = 0;
+ rkvenc_write_relaxed(dev, RKVENC_REG_SYNT_SLI2, synt_sli2.val);
+
+ enc_pic.enc_stnd = RKVENC_ENC_STND_H264;
+ enc_pic.cur_frm_ref = 1;
+ enc_pic.pic_qp = qp;
+ enc_pic.rec_fbc_dis = 0;
+ /* mpp's setup_vepu510_codec() sets this unconditionally to 1 on every
+ * frame; confirmed not just from source but from the real vendor
+ * wtrace capture used for the original I-frame bring-up (0x300 was
+ * 0x4000191c there, bit4 set) -- this driver had never set it at all.
+ */
+ enc_pic.bs_scp = 1;
+ /* mpp's hal_h264e_vepu510_gen_regs() sets this whenever a real
+ * motion-detection-info buffer is provided (mei_stor = task->md_info
+ * ? 1 : 0) -- this driver always provides one (meiw_buf_dma below),
+ * so this must always be 1 too. Previously left 0 despite the address
+ * being valid: a mismatch between "here's a real buffer" and "don't
+ * actually store to it" that plausibly only bites once the ME engine
+ * has real motion-vector output to write, i.e. genuine inter-frame
+ * (P-frame) search -- an I-frame's trivial/absent ME activity may
+ * never actually exercise the write-enable path this gates.
+ */
+ enc_pic.mei_stor = 1;
+ /* slen_fifo is deliberately left 0 for this FIRST ENC_PIC write. The
+ * vendor arms the slice-length FIFO in a two-step sequence: ENC_PIC is
+ * written with slen_fifo=0 during setup, then RE-written with
+ * slen_fifo=1 as the very last register before ENC_START. A real
+ * multi-frame wtrace capture shows this on EVERY frame (I and both P):
+ * 0x300 written twice, e.g. P-frame `0x00001a1c` (slen_fifo=0) early
+ * then `0x40001a1c` (slen_fifo=1) at the kick. This driver previously
+ * wrote ENC_PIC once with slen_fifo=1 and then wrote DUAL_CORE after
+ * it -- so it never reproduced the vendor's "arm the FIFO as the final
+ * action immediately before the kick" ordering. That two-step arm is
+ * the strongest-remaining register-level difference for the P-frame
+ * hang: the I-frame limps through without it, but a P-frame produces
+ * genuine slice-FIFO traffic that may depend on the FIFO being armed
+ * last. See the kick sequence below.
+ */
+ enc_pic.slen_fifo = 0;
+
+ /* First ENC_PIC write (slen_fifo=0) and DUAL_CORE, both in the
+ * vendor's early "config" position -- the capture writes 0x300
+ * (slen_fifo=0) and 0x304=0x14 together in the FRAME-class region,
+ * well before the kick, NOT interleaved into the kick tail.
+ *
+ * DUAL_CORE=0x14: see the union rkvenc_reg_dual_core comment in
+ * rkvenc-regs.h. Confirmed written once per frame (seq62/714/1366 in
+ * the multi-frame capture) even for this fully standalone single-core
+ * encode; leaving it 0 did NOT stop the isolated rk_iommu fault.
+ */
+ rkvenc_write_relaxed(dev, RKVENC_REG_ENC_PIC, enc_pic.val);
+ rkvenc_write_relaxed(dev, RKVENC_REG_DUAL_CORE, 0x14);
+
+ /* Control-class (Vepu510ControlCfg) setup + the vendor's exact kick
+ * tail. The kick order is transcribed verbatim from a real multi-frame
+ * wtrace capture (the last writes of every frame, identical across I
+ * and both P frames):
+ * ENC_WDG(0x38) -> DVBM_HOLD(0x74) -> ENC_ID(0x308) -> INT_EN(0x20)
+ * -> ENC_PIC(0x300, slen_fifo=1) -> ENC_START(0x10)
+ * with ENC_PIC (FIFO-armed) as the final config write before the kick.
+ */
+ {
+ union rkvenc_reg_int_bits int_en = { .val = 0 };
+ union rkvenc_reg_opt_strg opt_strg = { .val = 0 };
+ union rkvenc_reg_dtrns_map dtrns_map = { .val = 0 };
+ union rkvenc_reg_enc_strt enc_strt = { .val = 0 };
+
+ int_en.enc_done = 1;
+ int_en.lkt_node_done = 1;
+ int_en.sclr_done = 1;
+ int_en.vslc_done = 1; /* mirrors enc_pic.slen_fifo armed below */
+ int_en.vbsf_oflw = 1;
+ int_en.vbuf_lens = 1;
+ int_en.enc_err = 1;
+ int_en.vsrc_err = 1;
+ int_en.wdg = 1;
+ int_en.lkt_err_int = 1;
+ int_en.lkt_err_stop = 1;
+ int_en.lkt_force_stop = 1;
+ int_en.jslc_done = 1;
+ int_en.jbsf_oflw = 1;
+ int_en.jbuf_lens = 1;
+
+ /* Early config writes (the vendor emits these in its linear
+ * offset-ordered dump, well before the kick): INT_MASK, OPT_STRG,
+ * DTRNS_MAP. INT_EN itself is deferred to the kick tail below to
+ * match the capture (INT_EN is re-written right before ENC_PIC).
+ */
+ rkvenc_write_relaxed(dev, RKVENC_REG_INT_MASK, 0);
+
+ opt_strg.cke = 1;
+ opt_strg.resetn_hw_en = 1;
+ rkvenc_write_relaxed(dev, RKVENC_REG_OPT_STRG, opt_strg.val);
+
+ dtrns_map.bsw_bus_edin = 7;
+ rkvenc_write_relaxed(dev, RKVENC_REG_DTRNS_MAP, dtrns_map.val);
+
+ /* ---- exact vendor kick tail ---- */
+ rkvenc_write(dev, RKVENC_REG_ENC_WDG, rkvenc_calc_wdg(dev, width, height));
+ rkvenc_vepu510_quirk(dev); /* DVBM_HOLD(0x74) + ENC_ID(0x308) */
+ rkvenc_write_relaxed(dev, RKVENC_REG_INT_EN, int_en.val);
+
+ /* Final ENC_PIC re-write with slen_fifo=1 -- arms the slice-length
+ * FIFO as the very last action before ENC_START, matching the
+ * capture exactly. This is the whole point of the two-step arm.
+ */
+ enc_pic.slen_fifo = 1;
+ rkvenc_write_relaxed(dev, RKVENC_REG_ENC_PIC, enc_pic.val);
+
+ wmb(); /* all task registers visible before the kick below */
+
+ enc_strt.lkt_num = 0;
+ enc_strt.vepu_cmd = RKVENC_VEPU_CMD_ENC;
+ rkvenc_write(dev, RKVENC_REG_ENC_START, enc_strt.val);
+ }
+}
+
+static void rkvenc_h264_done(struct rkvenc_ctx *ctx, enum vb2_buffer_state state)
+{
+ struct rkvenc_dev *dev = ctx->dev;
+ struct rkvenc_h264_ctx *h264 = &ctx->h264;
+ struct vb2_v4l2_buffer *dst_buf = rkvenc_get_dst_buf(ctx);
+ bool is_idr = h264->gop_pos == 0;
+
+ if (state == VB2_BUF_STATE_DONE) {
+ unsigned int header_len = is_idr ? h264->sps_pps_len : 0;
+ u32 hw_len = rkvenc_read(dev, RKVENC_REG_ST_BS_LENGTH);
+
+ vb2_set_plane_payload(&dst_buf->vb2_buf, 0, header_len + hw_len);
+ dst_buf->flags |= is_idr ? V4L2_BUF_FLAG_KEYFRAME : V4L2_BUF_FLAG_PFRAME;
+
+ h264->frame_num = (h264->frame_num + 1) &
+ GENMASK(RKVENC_H264_LOG2_MAX_FRAME_NUM + 3, 0);
+ h264->poc_lsb = (h264->poc_lsb + 2) &
+ GENMASK(RKVENC_H264_LOG2_MAX_POC_LSB + 3, 0);
+ h264->last_frame_was_idr = is_idr;
+ if (++h264->gop_pos >= h264->gop_size->val)
+ h264->gop_pos = 0;
+ if (h264->gop_pos == 0)
+ h264->idr_pic_id++;
+ }
+}
+
+static const struct rkvenc_coded_fmt_ops rkvenc_h264_ops = {
+ .init_ctrls = rkvenc_h264_init_ctrls,
+ .start = rkvenc_h264_start,
+ .stop = rkvenc_h264_stop,
+ .run = rkvenc_h264_run,
+ .done = rkvenc_h264_done,
+};
+
+const struct rkvenc_coded_fmt_desc rkvenc_h264_fmt_desc = {
+ .fourcc = V4L2_PIX_FMT_H264,
+ .frmsize = {
+ .frmsize_min_width = 176,
+ .frmsize_max_width = 4096,
+ .frmsize_step_width = 2,
+ .frmsize_min_height = 144,
+ .frmsize_max_height = 2560,
+ .frmsize_step_height = 2,
+ },
+ .ops = &rkvenc_h264_ops,
+};
diff --git a/drivers/media/platform/rockchip/rkvenc/rkvenc-regs.h b/drivers/media/platform/rockchip/rkvenc/rkvenc-regs.h
new file mode 100644
index 000000000..353ba3eba
--- /dev/null
+++ b/drivers/media/platform/rockchip/rkvenc/rkvenc-regs.h
@@ -0,0 +1,929 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Rockchip VEPU510 (RK3576) hardware video encoder registers.
+ *
+ * Copyright (C) 2026 Jiaxing Hu <gahing@gahingwoo.com>
+ *
+ * Field layout cross-checked against Rockchip's own open-source userspace
+ * codec library (github.com/rockchip-linux/mpp,
+ * mpp/hal/rkenc/{common/vepu510_common.h,h264e/hal_h264e_vepu510_reg.h},
+ * Apache-2.0) — that library is the only place the RC/PIC/PARAM register
+ * field names exist; the downstream rockchip-linux/kernel mpp_rkvenc2.c
+ * driver treats those classes as an opaque byte blob and never names a
+ * single field beyond the handful of control/status registers below.
+ *
+ * Register classes (byte offsets within the per-core MMIO window):
+ * BASE 0x0000-0x0120 control / IRQ / watchdog
+ * FRAME 0x0270-0x03f4 per-frame picture config + H.264 syntax
+ * RC_ROI 0x1000-0x110c ROI / adaptive-quant helper tables
+ * PARAM 0x1700-0x19cc RDO/ME/quant tuning tables
+ * SQI 0x2000-0x212c subjective-quality tuning
+ * SCL 0x2200-0x2584 scaling lists
+ * STATUS 0x4000-0x424c read-only status
+ * DEBUG 0x5000-0x5230 debug counters
+ *
+ * IMPORTANT — PARAM/SQI/SCL and the RC_ROI rc_adj/rc_dthd threshold ladder
+ * are NOT optional despite mpp's own FIXQP-mode code path never touching
+ * them (rc->rc_mode == MPP_ENC_RC_MODE_FIXQP returns early from
+ * setup_vepu510_rc_base() before setting rc_adj0/1/rc_dthd_0_8, and
+ * nothing in mpp ever documents PARAM/SQI as required). Board bring-up
+ * spent a long time on exactly this assumption: a from-scratch test client
+ * reproduced the driver's exact hardware-internal-watchdog stall
+ * (int_sta.wdg_sta, ~50ms after kick) with these classes left at their
+ * POR/never-written state, byte-for-byte matched everything else against a
+ * real rockchip-linux/mpp `mpi_enc_test` capture, and the stall persisted
+ * until PARAM+SQI+SCL were written and the RC_ROI threshold ladder was
+ * populated (rc_dthd_0_8 left at 0 instead of the real "never trigger"
+ * 0x7FFFFFFF sentinel is itself sufficient to reproduce the stall alone).
+ * Whatever internal engine consumes these — almost certainly the RDO/
+ * mode-decision and adaptive-quant hardware blocks — appears to need a
+ * valid, non-degenerate configuration to complete at all, not just to
+ * compute good output. See rkvenc-h264.c for what's actually written and
+ * why (PAR/SQI are mpp's *default-tuning* constant tables, not per-frame
+ * content-derived, so hardcoding them is a deliberate v1 choice, not a
+ * placeholder).
+ *
+ * Each per-frame register below is a union of its named bitfields and a
+ * raw u32 .val, so callers build the value field-by-field and write it
+ * with a single rkvenc_write(dev, RKVENC_REG_*, reg.val).
+ */
+
+#ifndef RKVENC_REGS_H_
+#define RKVENC_REGS_H_
+
+#include <linux/bits.h>
+#include <linux/types.h>
+
+#define RKVENC_REG_FRAME_OFFSET 0x270
+#define RKVENC_REG_STATUS_OFFSET 0x4000
+#define RKVENC_REG_DEBUG_OFFSET 0x5000
+
+/* ---- BASE class (0x0000-0x0120), a.k.a. Vepu510ControlCfg in mpp.
+ * Bit layout below is transcribed directly from
+ * mpp/hal/rkenc/common/vepu510_common.h (Vepu510ControlCfg_t) — this
+ * supersedes an earlier, less precise reading of these same registers
+ * from the downstream vendor *kernel* driver's comments (which got the
+ * ENC_WDG field width/position and several INT_* bit meanings wrong).
+ */
+#define RKVENC_REG_VERSION 0x0000 /* read-only IP identification */
+#define RKVENC_REG_ENC_START 0x0010 /* enc_strt */
+#define RKVENC_REG_ENC_CLR 0x0014 /* bit0 safe_clr, bit1 force_clr */
+#define RKVENC_ENC_CLR_SAFE 0x1
+#define RKVENC_ENC_CLR_FORCE 0x2
+#define RKVENC_SCLR_DONE_STA BIT(2) /* int_sta sclr_done, set when safe-clr completes */
+#define RKVENC_REG_VS_LDLY 0x0018
+
+union rkvenc_reg_version {
+ struct {
+ u32 sub_ver:8;
+ u32 h264_cap:1;
+ u32 hevc_cap:1;
+ u32 reserved:2;
+ u32 res_cap:4;
+ u32 osd_cap:2;
+ u32 filtr_cap:2;
+ u32 bfrm_cap:1;
+ u32 fbc_cap:2;
+ u32 reserved1:1;
+ u32 ip_id:8;
+ };
+ u32 val;
+};
+#define RKVENC_REG_INT_EN 0x0020
+#define RKVENC_REG_INT_MASK 0x0024
+#define RKVENC_REG_INT_CLR 0x0028
+#define RKVENC_REG_INT_STA 0x002c
+#define RKVENC_REG_DTRNS_MAP 0x0030
+#define RKVENC_REG_DTRNS_CFG 0x0034
+#define RKVENC_REG_ENC_WDG 0x0038
+#define RKVENC_REG_OPT_STRG 0x0054
+
+/* VEPU510-only "DVBM hold" quirk (see rkvenc_vepu510_quirk() in
+ * rkvenc-h264.c). No field name exists in the vendor HAL for this — it is
+ * not written by rockchip-linux/mpp at all, only by the downstream kernel
+ * driver, so its exact purpose beyond the vendor comment ("vepu will hold
+ * when encoding finish") is unverified. Treated as an opaque required
+ * write sequence.
+ */
+#define RKVENC_REG_DVBM_HOLD 0x0074
+#define RKVENC_DVBM_HOLD_VAL 0x23
+
+/* enc_strt (0x0010): the actual "go" trigger is the 3-bit vepu_cmd field
+ * at bits[10:8], NOT the whole register set to 1 — vepu_cmd=1 is mpp's
+ * setup_vepu510_normal(); lkt_num (bits[7:0]) is the link-table task
+ * count, irrelevant here since v1 doesn't use link/CCU mode.
+ */
+union rkvenc_reg_enc_strt {
+ struct {
+ u32 lkt_num:8;
+ u32 vepu_cmd:3;
+ u32 reserved:21;
+ };
+ u32 val;
+};
+
+#define RKVENC_VEPU_CMD_ENC 1
+
+/* int_en / int_msk / int_clr / int_sta all share this bit layout. */
+union rkvenc_reg_int_bits {
+ struct {
+ u32 enc_done:1;
+ u32 lkt_node_done:1;
+ u32 sclr_done:1;
+ u32 vslc_done:1;
+ u32 vbsf_oflw:1;
+ u32 vbuf_lens:1;
+ u32 enc_err:1;
+ u32 vsrc_err:1;
+ u32 wdg:1;
+ u32 lkt_err_int:1;
+ u32 lkt_err_stop:1;
+ u32 lkt_force_stop:1;
+ u32 jslc_done:1;
+ u32 jbsf_oflw:1;
+ u32 jbuf_lens:1;
+ u32 dvbm_err:1;
+ u32 reserved:16;
+ };
+ u32 val;
+};
+
+/* Hard-error bits worth aborting the frame over; enc_done/lkt_node_done/
+ * sclr_done/vslc_done/vbuf_lens/jslc_done/jbsf_oflw/jbuf_lens are either
+ * normal completion signals or JPEG-only (harmless no-ops for H.264).
+ */
+#define RKVENC_INT_ERROR_MASK \
+ (BIT(4) /* vbsf_oflw */ | BIT(6) /* enc_err */ | BIT(7) /* vsrc_err */ | \
+ BIT(8) /* wdg */ | BIT(9) /* lkt_err_int */ | BIT(10) /* lkt_err_stop */ | \
+ BIT(11) /* lkt_force_stop */ | BIT(15) /* dvbm_err */)
+
+union rkvenc_reg_enc_wdg {
+ struct {
+ u32 vs_load_thd:24;
+ u32 reserved:8;
+ };
+ u32 val;
+};
+
+union rkvenc_reg_opt_strg {
+ struct {
+ u32 cke:1;
+ u32 resetn_hw_en:1;
+ u32 rfpr_err_e:1;
+ u32 sram_ckg_en:1;
+ u32 link_err_stop:1;
+ u32 reserved:27;
+ };
+ u32 val;
+};
+
+union rkvenc_reg_dtrns_map {
+ struct {
+ u32 jpeg_bus_edin:4;
+ u32 src_bus_edin:4;
+ u32 meiw_bus_edin:4;
+ u32 bsw_bus_edin:4;
+ u32 reserved:8;
+ u32 lktw_bus_edin:4;
+ u32 rec_nfbc_bus_edin:4;
+ };
+ u32 val;
+};
+
+/* ---- FRAME class (0x0270-0x03f4), fields mirror Vepu510FrmCommon +
+ * the H.264-specific trailer from H264eVepu510Frame in mpp.
+ */
+union rkvenc_reg_enc_pic {
+ struct {
+ u32 enc_stnd:2; /* 0 = H.264 */
+ u32 cur_frm_ref:1;
+ u32 mei_stor:1;
+ u32 bs_scp:1;
+ u32 reserved0:3;
+ u32 pic_qp:6;
+ u32 num_pic_tot_cur_hevc:5;
+ u32 log2_ctu_num_hevc:5;
+ u32 reserved1:6;
+ u32 slen_fifo:1;
+ u32 rec_fbc_dis:1;
+ };
+ u32 val;
+};
+
+#define RKVENC_ENC_STND_H264 0
+
+/* dual_core (0x304) is the CCU cross-core dual-encoder handshake register
+ * (DCHS_REG_OFFSET in the downstream vendor kernel's mpp_rkvenc2.c) --
+ * used for chaining a frame's encode across both VEPU510 cores. This
+ * driver deliberately doesn't implement that (see the architecture
+ * comment in rkvenc.c), but the register is still written explicitly every
+ * frame to a defined value (dchs_txe=1, i.e. 0x14): the vendor kernel's
+ * own rkvenc2_patch_dchs() writes exactly this on every task, even fully
+ * standalone non-chained ones (confirmed against a real register-write
+ * trace of the vendor stack), so a POR/leftover value is not what the
+ * hardware expects.
+ */
+union rkvenc_reg_dual_core {
+ struct {
+ u32 dchs_txid:2;
+ u32 dchs_rxid:2;
+ u32 dchs_txe:1;
+ u32 dchs_rxe:1;
+ u32 reserved0:2;
+ u32 dchs_dly:8;
+ u32 dchs_ofst:10;
+ u32 reserved1:6;
+ };
+ u32 val;
+};
+
+union rkvenc_reg_enc_id {
+ struct {
+ u32 frame_id:8;
+ u32 frm_id_match:1;
+ u32 reserved0:7;
+ u32 ch_id:2;
+ u32 vrsp_rtn_en:1;
+ u32 vinf_req_en:1;
+ u32 reserved1:12;
+ };
+ u32 val;
+};
+
+/* VEPU510-only quirk value, see rkvenc_vepu510_quirk() in rkvenc-h264.c. */
+#define RKVENC_ENC_ID_QUIRK_VAL (BIT(18) | BIT(16))
+
+union rkvenc_reg_enc_rsl {
+ struct {
+ u32 pic_wd8_m1:11;
+ u32 reserved0:5;
+ u32 pic_hd8_m1:11;
+ u32 reserved1:5;
+ };
+ u32 val;
+};
+
+union rkvenc_reg_src_fill {
+ struct {
+ u32 pic_wfill:6;
+ u32 reserved0:10;
+ u32 pic_hfill:6;
+ u32 reserved1:10;
+ };
+ u32 val;
+};
+
+union rkvenc_reg_src_fmt {
+ struct {
+ u32 alpha_swap:1;
+ u32 rbuv_swap:1;
+ u32 src_cfmt:4;
+ u32 src_rcne:1;
+ u32 out_fmt:1;
+ u32 src_range_trns_en:1;
+ u32 src_range_trns_sel:1;
+ u32 chroma_ds_mode:1;
+ u32 reserved:21;
+ };
+ u32 val;
+};
+
+/* src_cfmt values (VepuFmt enum, vepu5xx_common.h) — only the one v1 uses. */
+#define RKVENC_SRC_FMT_YUV420SP 6 /* NV12 */
+/* out_fmt (bit7): real capture always has this set (src_fmt == 0x98, not
+ * just src_cfmt's 0x18) for a real successful encode; mpp's own field name
+ * suggests an output-plane format toggle but nothing in the userspace HAL
+ * ever names its meaning beyond the bit position. Treated as a required
+ * "must be 1" bit, like the DVBM-hold quirk.
+ */
+
+union rkvenc_reg_src_proc {
+ struct {
+ u32 cr_force_value:8;
+ u32 cb_force_value:8;
+ u32 chroma_force_en:1;
+ u32 reserved0:9;
+ u32 src_mirr:1;
+ u32 src_rot:2;
+ u32 tile4x4_en:1;
+ u32 reserved1:2;
+ };
+ u32 val;
+};
+
+union rkvenc_reg_pic_ofst {
+ struct {
+ u32 pic_ofst_x:14;
+ u32 reserved0:2;
+ u32 pic_ofst_y:14;
+ u32 reserved1:2;
+ };
+ u32 val;
+};
+
+union rkvenc_reg_src_strd0 {
+ struct {
+ u32 src_strd0:21;
+ u32 reserved:11;
+ };
+ u32 val;
+};
+
+union rkvenc_reg_src_strd1 {
+ struct {
+ u32 src_strd1:16;
+ u32 reserved:16;
+ };
+ u32 val;
+};
+
+union rkvenc_reg_rc_cfg {
+ struct {
+ u32 rc_en:1;
+ u32 aq_en:1;
+ u32 reserved:10;
+ u32 rc_ctu_num:20;
+ };
+ u32 val;
+};
+
+union rkvenc_reg_rc_qp {
+ struct {
+ u32 reserved:16;
+ u32 rc_qp_range:4;
+ u32 rc_max_qp:6;
+ u32 rc_min_qp:6;
+ };
+ u32 val;
+};
+
+union rkvenc_reg_rc_tgt {
+ struct {
+ u32 ctu_ebit:20;
+ u32 reserved:12;
+ };
+ u32 val;
+};
+
+union rkvenc_reg_sli_splt {
+ struct {
+ u32 sli_splt:1;
+ u32 sli_splt_mode:1;
+ u32 sli_splt_cpst:1;
+ u32 reserved0:12;
+ u32 sli_flsh:1;
+ u32 sli_max_num_m1:15;
+ u32 reserved1:1;
+ };
+ u32 val;
+};
+
+union rkvenc_reg_sli_byte {
+ struct {
+ u32 sli_splt_byte:20;
+ u32 reserved:12;
+ };
+ u32 val;
+};
+
+union rkvenc_reg_sli_cnum {
+ struct {
+ u32 sli_splt_cnum_m1:20;
+ u32 reserved:12;
+ };
+ u32 val;
+};
+
+/* H.264-specific trailer, 0x3a0-0x3f4.
+ * rdo_cfg.chrm_spcl/ccwa_e/atr_mult_sel_e: mpp's setup_vepu510_rdo_pred()
+ * (hal_h264e_vepu510.c) sets these three unconditionally to 1 on every
+ * single frame -- not gated by profile/level/tune config like rect_size/
+ * vlc_lmt/atf_e/atr_e are. Found missing by diffing against that function
+ * directly; previously left at their reset-value 0 here since nothing in
+ * this driver ever set them.
+ */
+union rkvenc_reg_rdo_cfg {
+ struct {
+ u32 rect_size:1;
+ u32 reserved0:2;
+ u32 vlc_lmt:1;
+ u32 chrm_spcl:1;
+ u32 reserved1:8;
+ u32 ccwa_e:1;
+ u32 reserved2:1;
+ u32 atr_e:1;
+ u32 reserved3:4;
+ u32 scl_lst_sel:2;
+ u32 reserved4:6;
+ u32 atf_e:1;
+ u32 atr_mult_sel_e:1;
+ u32 reserved5:2;
+ };
+ u32 val;
+};
+
+union rkvenc_reg_synt_nal {
+ struct {
+ u32 nal_ref_idc:2;
+ u32 nal_unit_type:5;
+ u32 reserved:25;
+ };
+ u32 val;
+};
+
+union rkvenc_reg_synt_sps {
+ struct {
+ u32 max_fnum:4; /* log2_max_frame_num_minus4 */
+ u32 drct_8x8:1; /* direct_8x8_inference_flag */
+ u32 mpoc_lm4:4; /* log2_max_pic_order_cnt_lsb_minus4 */
+ u32 poc_type:2; /* pic_order_cnt_type */
+ u32 reserved:21;
+ };
+ u32 val;
+};
+
+union rkvenc_reg_synt_pps {
+ struct {
+ u32 etpy_mode:1; /* entropy_coding_mode_flag: 0=CAVLC 1=CABAC */
+ u32 trns_8x8:1; /* transform_8x8_mode_flag */
+ u32 csip_flag:1; /* constrained_intra_pred_flag */
+ u32 num_ref0_idx:2;
+ u32 num_ref1_idx:2;
+ u32 pic_init_qp:6;
+ u32 cb_ofst:5;
+ u32 cr_ofst:5;
+ u32 reserved:1;
+ u32 dbf_cp_flg:1; /* deblocking_filter_control_present_flag */
+ u32 reserved1:7;
+ };
+ u32 val;
+};
+
+union rkvenc_reg_synt_sli0 {
+ struct {
+ u32 sli_type:2;
+ u32 pps_id:8;
+ u32 drct_smvp:1;
+ u32 num_ref_ovrd:1;
+ u32 cbc_init_idc:2;
+ u32 reserved:2;
+ u32 frm_num:16;
+ };
+ u32 val;
+};
+
+#define RKVENC_SLI_TYPE_P 0
+#define RKVENC_SLI_TYPE_I 2
+
+union rkvenc_reg_synt_sli1 {
+ struct {
+ u32 idr_pid:16;
+ u32 poc_lsb:16;
+ };
+ u32 val;
+};
+
+union rkvenc_reg_synt_sli2 {
+ struct {
+ u32 rodr_pic_idx:2;
+ u32 ref_list0_rodr:1;
+ u32 sli_beta_ofst:4;
+ u32 sli_alph_ofst:4;
+ u32 dis_dblk_idc:2;
+ u32 reserved:3;
+ u32 rodr_pic_num:16;
+ };
+ u32 val;
+};
+
+/* Absolute byte offsets of each FRAME-class register this driver touches. */
+#define RKVENC_REG_ADR_SRC0 0x280
+#define RKVENC_REG_ADR_SRC1 0x284
+#define RKVENC_REG_ADR_SRC2 0x288
+#define RKVENC_REG_RFPW_H_ADDR 0x28c
+#define RKVENC_REG_RFPW_B_ADDR 0x290
+#define RKVENC_REG_RFPR_H_ADDR 0x294
+#define RKVENC_REG_RFPR_B_ADDR 0x298
+#define RKVENC_REG_MEIW_ADDR 0x2ac /* motion-detection-info write ptr */
+#define RKVENC_REG_DSPW_ADDR 0x2a4
+#define RKVENC_REG_DSPR_ADDR 0x2a8
+#define RKVENC_REG_BSBT_ADDR 0x2b0
+#define RKVENC_REG_BSBB_ADDR 0x2b4
+#define RKVENC_REG_ADR_BSBS 0x2b8
+#define RKVENC_REG_BSBR_ADDR 0x2bc
+/* rfpt/rfpb ("top"/"bottom" field recon addresses, interlaced-only):
+ * real capture always carries the literal sentinel 0xffffffff in
+ * rfpt_h/rfpt_b (not an fd-embedded pointer -- an fd of 0x3ff/1023 is
+ * never valid on a real system, so this reads as an explicit
+ * "disabled/no interlaced fields" pattern), with rfpb_h/adr_rfpb_b at 0.
+ */
+#define RKVENC_REG_RFPT_H_ADDR 0x2d0
+#define RKVENC_REG_RFPB_H_ADDR 0x2d4
+#define RKVENC_REG_RFPT_B_ADDR 0x2d8
+#define RKVENC_REG_ADR_RFPB_B 0x2dc
+#define RKVENC_RFPT_DISABLED 0xffffffff
+#define RKVENC_REG_ADR_SMEAR_RD 0x2e0
+#define RKVENC_REG_ADR_SMEAR_WR 0x2e4
+/* Last DMA pointer in the FRAME class (ROI read address). The whole
+ * 0x270-0x2e8 block is zeroed at the top of rkvenc_h264_run() to clear any
+ * stale/garbage write pointers (lpfw/lpfr/ebuft/ebufb at 0x2c0-0x2cc, this,
+ * and 0x29c/0x2a0) that the vendor always writes but this driver otherwise
+ * skips -- see the big comment there.
+ */
+#define RKVENC_REG_ADR_ROIR 0x2e8
+#define RKVENC_REG_ENC_PIC 0x300
+#define RKVENC_REG_DUAL_CORE 0x304
+#define RKVENC_REG_ENC_ID 0x308
+#define RKVENC_REG_ENC_RSL 0x310
+#define RKVENC_REG_SRC_FILL 0x314
+#define RKVENC_REG_SRC_FMT 0x318
+#define RKVENC_REG_SRC_PROC 0x32c
+#define RKVENC_REG_PIC_OFST 0x330
+#define RKVENC_REG_SRC_STRD0 0x334
+#define RKVENC_REG_SRC_STRD1 0x338
+#define RKVENC_REG_RC_CFG 0x350
+#define RKVENC_REG_RC_QP 0x354
+#define RKVENC_REG_RC_TGT 0x358
+#define RKVENC_REG_SLI_SPLT 0x360
+#define RKVENC_REG_SLI_BYTE 0x364
+#define RKVENC_REG_SLI_CNUM 0x368
+
+/* Motion-estimation search-range/config/cache registers: written
+ * *unconditionally* by mpp's setup_vepu510_me() on every single frame
+ * (no conditional guard — not gated behind P-frame/inter-mode use),
+ * same "required regardless of what you'd assume from the feature
+ * name" pattern as the roi_qthd and thumb/smear-buffer registers.
+ * Values below are mpp's own hardcoded constants, not per-content
+ * tuning — a zeroed cime_dist_thre/srgn_max_num here is a plausible
+ * cause of a genuine ME-engine search/loop stall.
+ */
+#define RKVENC_REG_ME_RNGE 0x370
+#define RKVENC_REG_ME_CFG 0x374
+#define RKVENC_REG_ME_CACH 0x378
+
+union rkvenc_reg_me_rnge {
+ struct {
+ u32 cime_srch_dwnh:4;
+ u32 cime_srch_uph:4;
+ u32 cime_srch_rgtw:4;
+ u32 cime_srch_lftw:4;
+ u32 dlt_frm_num:16;
+ };
+ u32 val;
+};
+
+union rkvenc_reg_me_cfg {
+ struct {
+ u32 srgn_max_num:7;
+ u32 cime_dist_thre:13;
+ u32 rme_srch_h:2;
+ u32 rme_srch_v:2;
+ u32 rme_dis:3;
+ u32 reserved:1;
+ u32 fme_dis:3;
+ u32 reserved1:1;
+ };
+ u32 val;
+};
+
+union rkvenc_reg_me_cach {
+ struct {
+ u32 cime_zero_thre:13;
+ u32 reserved:15;
+ u32 fme_prefsu_en:2;
+ u32 colmv_stor_hevc:1;
+ u32 colmv_load_hevc:1;
+ };
+ u32 val;
+};
+
+#define RKVENC_REG_RDO_CFG 0x3a0
+#define RKVENC_REG_SYNT_NAL 0x3b0
+#define RKVENC_REG_SYNT_SPS 0x3b4
+#define RKVENC_REG_SYNT_PPS 0x3b8
+#define RKVENC_REG_SYNT_SLI0 0x3bc
+#define RKVENC_REG_SYNT_SLI1 0x3c0
+#define RKVENC_REG_SYNT_SLI2 0x3c4
+
+/* ---- RC_ROI class (0x1000-0x110c), a.k.a. Vepu510RcRoi in mpp.
+ *
+ * rc_adj0/rc_adj1/rc_dthd_0_8 (0x1000-0x1028) are the adaptive-quant
+ * bit-count-deviation threshold ladder: mpp's setup_vepu510_rc_base()
+ * computes these from the frame's bit budget in its non-FIXQP path, but
+ * skips them entirely (early `return`) in true MPP_ENC_RC_MODE_FIXQP mode
+ * — this driver's original design assumed that meant "safe to leave at
+ * 0", matching disabled RC/AQ. Board bring-up proved that assumption
+ * wrong: leaving rc_dthd_0_8 at 0 (read by hardware as "always past
+ * threshold" rather than mpp's real "never trigger" 0x7FFFFFFF sentinel
+ * for the outer entries) reproduces the exact hardware-watchdog stall on
+ * its own. This driver now always runs with rc_cfg.rc_en=1/aq_en=1 and a
+ * real (if approximate — see rkvenc-h264.c) threshold ladder, but clamps
+ * rc_qp.rc_min_qp==rc_max_qp and rc_qp_range=0 so the AQ engine has no
+ * room to actually move the QP away from the requested value — genuine
+ * fixed-QP *output*, just not fixed-QP *register configuration*.
+ *
+ * roi_qthd0-3: mpp's setup_vepu510_rc_base() writes these
+ * *unconditionally*, before even checking RC mode — every ROI area's
+ * [qpmin, qpmax] clamp range is set to the frame's actual
+ * [quality_min, quality_max] regardless.
+ */
+#define RKVENC_REG_RC_ADJ0 0x1000
+#define RKVENC_REG_RC_ADJ1 0x1004
+#define RKVENC_REG_RC_DTHD(n) (0x1008 + 4 * (n)) /* n = 0..8 */
+
+union rkvenc_reg_rc_adj0 {
+ struct {
+ s32 qp_adj0:5;
+ s32 qp_adj1:5;
+ s32 qp_adj2:5;
+ s32 qp_adj3:5;
+ s32 qp_adj4:5;
+ u32 reserved:7;
+ };
+ u32 val;
+};
+
+union rkvenc_reg_rc_adj1 {
+ struct {
+ s32 qp_adj5:5;
+ s32 qp_adj6:5;
+ s32 qp_adj7:5;
+ s32 qp_adj8:5;
+ u32 reserved:12;
+ };
+ u32 val;
+};
+
+#define RKVENC_RC_DTHD_SENTINEL 0x7fffffff
+
+#define RKVENC_REG_ROI_QTHD0 0x1030
+#define RKVENC_REG_ROI_QTHD1 0x1034
+#define RKVENC_REG_ROI_QTHD2 0x1038
+#define RKVENC_REG_ROI_QTHD3 0x103c
+
+union rkvenc_reg_roi_qthd0 {
+ struct {
+ u32 qpmin_area0:6;
+ u32 qpmax_area0:6;
+ u32 qpmin_area1:6;
+ u32 qpmax_area1:6;
+ u32 qpmin_area2:6;
+ u32 reserved:2;
+ };
+ u32 val;
+};
+
+union rkvenc_reg_roi_qthd1 {
+ struct {
+ u32 qpmax_area2:6;
+ u32 qpmin_area3:6;
+ u32 qpmax_area3:6;
+ u32 qpmin_area4:6;
+ u32 qpmax_area4:6;
+ u32 reserved:2;
+ };
+ u32 val;
+};
+
+union rkvenc_reg_roi_qthd2 {
+ struct {
+ u32 qpmin_area5:6;
+ u32 qpmax_area5:6;
+ u32 qpmin_area6:6;
+ u32 qpmax_area6:6;
+ u32 qpmin_area7:6;
+ u32 reserved:2;
+ };
+ u32 val;
+};
+
+union rkvenc_reg_roi_qthd3 {
+ struct {
+ u32 qpmax_area7:6;
+ u32 reserved:24;
+ u32 qpmap_mode:2;
+ };
+ u32 val;
+};
+
+/* RC_ROI tail (0x1044-0x107c): AQ activity threshold/step LUT, MAD-based
+ * statistics thresholds, and a chroma KLUT offset -- all real fields in
+ * mpp's Vepu510RcRoi struct (vepu510_common.h) that this driver used to
+ * leave entirely unwritten. Found by diffing this driver's register writes
+ * against mpp's actual setup_vepu510_aq()/setup_vepu510_me()/
+ * setup_vepu510_rdo_pred() (hal_h264e_vepu510.c), not just the single
+ * I-frame wtrace capture used to fix the original whole-class stall (see
+ * the RC_ROI banner comment above) -- that capture never exercised these
+ * particular offsets either, since a from-scratch driver only writes what
+ * it's told to, and nothing had told it to write these yet.
+ *
+ * mpp writes aq_tthd/aq_stp/madi_st_thd/madp_st_thd0/madp_st_thd1
+ * unconditionally on every frame regardless of slice type (setup_vepu510_me()
+ * has no I/P branch for these), and klut_ofst.chrm_klut_ofst is 6 for I
+ * slices and 6 or 9 for P slices depending on an IPC scene-tuning mode this
+ * driver doesn't expose (so 6 is correct for both here). These sit in
+ * exactly the register class board bring-up already proved needs
+ * non-degenerate content for the hardware to complete at all (see the RC_ROI
+ * banner comment) -- left at POR/leftover 0 instead of mpp's real nonzero
+ * table, this is a plausible source of a real internal stall that happens to
+ * only bite once real per-block activity/MAD statistics get computed and
+ * compared against these thresholds, i.e. during genuine inter-mode
+ * (P-frame) analysis rather than intra.
+ */
+#define RKVENC_REG_AQ_TTHD 0x1044 /* 4 words, 16 packed u8 thresholds */
+#define RKVENC_REG_AQ_STP0 0x1054
+#define RKVENC_REG_AQ_STP1 0x1058
+#define RKVENC_REG_AQ_STP2 0x105c
+#define RKVENC_REG_MADI_ST_THD 0x1064
+#define RKVENC_REG_MADP_ST_THD0 0x1068
+#define RKVENC_REG_MADP_ST_THD1 0x106c
+#define RKVENC_REG_KLUT_OFST 0x107c
+
+union rkvenc_reg_aq_stp0 {
+ struct {
+ s32 aq_stp_s0:5;
+ s32 aq_stp_0t1:5;
+ s32 aq_stp_1t2:5;
+ s32 aq_stp_2t3:5;
+ s32 aq_stp_3t4:5;
+ s32 aq_stp_4t5:5;
+ u32 reserved:2;
+ };
+ u32 val;
+};
+
+union rkvenc_reg_aq_stp1 {
+ struct {
+ s32 aq_stp_5t6:5;
+ s32 aq_stp_6t7:5;
+ s32 aq_stp_7t8:5;
+ s32 aq_stp_8t9:5;
+ s32 aq_stp_9t10:5;
+ s32 aq_stp_10t11:5;
+ u32 reserved:2;
+ };
+ u32 val;
+};
+
+union rkvenc_reg_aq_stp2 {
+ struct {
+ s32 aq_stp_11t12:5;
+ s32 aq_stp_12t13:5;
+ s32 aq_stp_13t14:5;
+ s32 aq_stp_14t15:5;
+ s32 aq_stp_b15:5;
+ u32 reserved:7;
+ };
+ u32 val;
+};
+
+union rkvenc_reg_madi_st_thd {
+ struct {
+ u32 madi_th0:8;
+ u32 madi_th1:8;
+ u32 madi_th2:8;
+ u32 reserved:8;
+ };
+ u32 val;
+};
+
+union rkvenc_reg_madp_st_thd0 {
+ struct {
+ u32 madp_th0:12;
+ u32 reserved:4;
+ u32 madp_th1:12;
+ u32 reserved1:4;
+ };
+ u32 val;
+};
+
+union rkvenc_reg_madp_st_thd1 {
+ struct {
+ u32 madp_th2:12;
+ u32 reserved:20;
+ };
+ u32 val;
+};
+
+union rkvenc_reg_klut_ofst {
+ struct {
+ u32 chrm_klut_ofst:4;
+ u32 reserved:4;
+ u32 inter_chrm_dist_multi:6;
+ u32 reserved1:18;
+ };
+ u32 val;
+};
+
+/* ---- PARAM class (0x1700-0x19cc), a.k.a. Vepu510Param in mpp.
+ * RDO lambda/cost tables plus a handful of small quant-tuning tables ahead
+ * of them. mpp treats the bulk of this as a *default-tuning profile*
+ * constant table (`vepu51x_h264e_lambda_default_60`/`_cvr_60`, selected by
+ * a tuning "lambda_idx" knob this driver doesn't expose, not by
+ * resolution/QP/content) — so a fixed constant table here, captured from
+ * a real successful encode via wtrace and cross-checked against the
+ * table's monotonically-increasing-lambda shape, is a deliberate v1
+ * choice, not a placeholder. See RKVENC_PAR_CLASS_WORDS in rkvenc-h264.c.
+ *
+ * CORRECTION (2026-07-21, from a real multi-frame wtrace capture): the
+ * "static blob, same for every frame" characterization above is WRONG for
+ * four specific words inside it -- mpp's setup_vepu510_anti_ringing()
+ * (hal_h264e_vepu510.c) computes real, different ATR (anti-ringing) weight
+ * values depending on I-slice vs P-slice, not just the tune/scene-mode
+ * config a single I-frame capture can see. A 3-frame real capture (frame 0
+ * I, frames 1-2 P) showed these four offsets change between the I-frame and
+ * BOTH P-frames, while staying identical between the two P-frames --
+ * conclusive evidence of an I/P split, not per-content noise. This driver's
+ * static `rkvenc_par_class` blob was captured from an I-frame-only session,
+ * so every P-frame was silently getting the I-slice ATR weights. See
+ * RKVENC_REG_ATR_THD1/WGT16/WGT8/WGT4 below and the patch in rkvenc-h264.c.
+ */
+#define RKVENC_REG_PAR_OFFSET 0x1700
+#define RKVENC_REG_PAR_SIZE 0x02d0 /* 0x19cc - 0x1700 + 4 */
+
+#define RKVENC_REG_ATR_THD1 0x1744 /* thd2 (I/P-dependent) + thdqp (constant) */
+#define RKVENC_REG_ATR_WGT16 0x1750
+#define RKVENC_REG_ATR_WGT8 0x1754
+#define RKVENC_REG_ATR_WGT4 0x1758
+
+union rkvenc_reg_atr_thd1 {
+ struct {
+ u32 thd2:8;
+ u32 reserved0:8;
+ u32 thdqp:6;
+ u32 reserved1:10;
+ };
+ u32 val;
+};
+
+union rkvenc_reg_atr_wgt {
+ struct {
+ u32 wgt0:8;
+ u32 wgt1:8;
+ u32 wgt2:8;
+ u32 reserved:8;
+ };
+ u32 val;
+};
+
+/* ---- SQI class (0x2000-0x212c), "subjective Adjust" quant tuning.
+ * Same default-tuning-profile characterization as PARAM above — a real
+ * successful encode only ever wrote the first 0x2000-0x20b8 (the rest of
+ * the declared class range was never touched even by a genuinely working
+ * encode, so this driver doesn't write it either).
+ *
+ * CORRECTION (2026-07-21): same caveat as PARAM above applies to one word,
+ * smear_opt_cfg (0x2014, mpp's setup_vepu510_anti_smear()). Its
+ * `stated_mode` field is 1 if (this slice is I) OR (the *previous* encoded
+ * frame was I), else 2 -- a real multi-frame capture confirmed this exactly
+ * (frame 0 I: 1; frame 1 P, previous=I: 1; frame 2 P, previous=P: 2), pure
+ * slice-history state this driver can track trivially. Its
+ * `rdo_smear_dlt_qp` field is a genuinely different, harder case: mpp
+ * derives it from real per-block "smear" activity statistics read back from
+ * the *previous* frame's actual hardware encode (`ctx->last_frame_fb`,
+ * populated from DEBUG-class counters this driver has never implemented
+ * readback for) -- not just slice type. The capture showed -1 (I) then -3
+ * (both P frames), which this driver approximates as a static per-slice-
+ * type constant (real content-adaptive tracking is a follow-up, see
+ * this driver approximates it, since implementing full smear-statistics readback is out of
+ * scope here and this field's effect is RDO cost tuning, not known to be
+ * hang-relevant like the RC_ROI class was.
+ */
+#define RKVENC_REG_SQI_OFFSET 0x2000
+#define RKVENC_REG_SQI_SIZE 0x00bc /* 0x20b8 - 0x2000 + 4, not the full class */
+
+#define RKVENC_REG_SMEAR_OPT_CFG 0x2014
+
+union rkvenc_reg_smear_opt_cfg {
+ struct {
+ u32 rdo_smear_lvl16_multi:8;
+ s32 rdo_smear_dlt_qp:4;
+ u32 reserved:1;
+ u32 stated_mode:2;
+ u32 rdo_smear_en:1;
+ u32 reserved1:16;
+ };
+ u32 val;
+};
+
+/* ---- SCL class (0x2200-0x2584), scaling lists.
+ * All-zero for any encode that doesn't use a custom H.264 scaling list
+ * (this driver never does) — written explicitly anyway, every frame,
+ * rather than relying on POR/leftover-state being zero.
+ */
+#define RKVENC_REG_SCL_OFFSET 0x2200
+#define RKVENC_REG_SCL_SIZE 0x0388 /* 0x2584 - 0x2200 + 4, full class */
+
+/* ---- STATUS class (0x4000-0x424c), read-only ---- */
+#define RKVENC_REG_ST_BS_LENGTH 0x4000 /* bs_lgth_l32: this-frame byte count */
+#define RKVENC_REG_ST_BSB 0x402c /* current bitstream write pointer */
+#define RKVENC_REG_ST_SLICE_NUM 0x4034 /* slice-fifo occupancy, & 0x3f */
+#define RKVENC_REG_ST_SLICE_LEN 0x4038 /* slice-length fifo read port */
+
+#define RKVENC_SLICE_LEN_LAST BIT(31)
+
+/* ---- DEBUG class ---- */
+#define RKVENC_REG_DBG_CLR 0x5300
+#define RKVENC_DBG_CLR_VAL 0x2
+
+#endif /* RKVENC_REGS_H_ */
diff --git a/drivers/media/platform/rockchip/rkvenc/rkvenc.c b/drivers/media/platform/rockchip/rkvenc/rkvenc.c
new file mode 100644
index 000000000..d70bcc5fb
--- /dev/null
+++ b/drivers/media/platform/rockchip/rkvenc/rkvenc.c
@@ -0,0 +1,892 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Rockchip VEPU510 (RK3576) hardware video encoder driver.
+ *
+ * Copyright (C) 2026 Jiaxing Hu <gahing@gahingwoo.com>
+ *
+ * Architecture notes (see also rkvenc-regs.h):
+ *
+ * - This is a *stateful* V4L2 mem2mem encoder (raw NV12 in on OUTPUT,
+ * H.264 Annex-B out on CAPTURE) modelled on
+ * drivers/media/platform/verisilicon's hantro_drv.c device_run()/
+ * codec_ops{run,done} split, NOT on rkvdec's stateless request-API
+ * decoder pattern — an encoder has no per-frame bitstream to parse,
+ * so there is nothing analogous to rkvdec_run_preamble/postamble here.
+ *
+ * - The vendor downstream driver (rockchip-linux/kernel,
+ * drivers/video/rockchip/mpp/mpp_rkvenc2.c) groups rkvenc0/rkvenc1
+ * under a "CCU" (rockchip,rkv-encoder-rk3576-ccu) that does pure
+ * software task-queue load balancing across both cores, plus an
+ * optional DCHS (dual-core-handshake) register protocol used only
+ * when *deliberately* splitting one frame's rows across both cores.
+ * There is no hardware descriptor/link-list engine behind it (unlike
+ * the decoder's CCU). v1 of this driver does not implement either:
+ * rkvenc0 and rkvenc1 are exposed as two independent V4L2 M2M device
+ * nodes, each driving one physical core standalone — the same
+ * simplification rkvdec itself makes for multi-core VDPU hardware
+ * (see rkvdec_disable_multicore()).
+ */
+
+#include <linux/clk.h>
+#include <linux/delay.h>
+#include <linux/dma-mapping.h>
+#include <linux/interrupt.h>
+#include <linux/iopoll.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/platform_device.h>
+#include <linux/pm_runtime.h>
+#include <linux/reset.h>
+#include <linux/slab.h>
+
+#include <media/v4l2-event.h>
+#include <media/v4l2-ioctl.h>
+#include <media/v4l2-mem2mem.h>
+#include <media/videobuf2-dma-contig.h>
+
+#include "rkvenc.h"
+#include "rkvenc-regs.h"
+
+#define RKVENC_WATCHDOG_TIMEOUT_MS 800
+#define RKVENC_DEFAULT_WIDTH 176
+#define RKVENC_DEFAULT_HEIGHT 144
+#define RKVENC_MAX_WIDTH 4096
+#define RKVENC_MAX_HEIGHT 2560
+
+#define RKVENC_AUTOSUSPEND_DELAY_MS 100
+
+/* Per-resolution hardware watchdog threshold table, ported from the
+ * downstream vendor driver's rkvenc2_calc_timeout_thd() /
+ * rkvenc2_timeout_thd_by_rsl[]. VEPU510 scales the ENC_WDG register in
+ * units of 256 core-clock cycles (other VEPU variants use 1024) — this
+ * ×256 factor is confirmed from the vendor kernel source. The field
+ * itself (vs_load_thd, bits[23:0] of ENC_WDG per mpp's Vepu510ControlCfg)
+ * is 24 bits wide, not "the register's top byte" as an earlier reading of
+ * the vendor kernel driver's comments suggested; that earlier reading
+ * produced a wrong shift (see git history).
+ */
+struct rkvenc_wdg_entry {
+ u32 pixels;
+ u32 ms;
+};
+
+static const struct rkvenc_wdg_entry rkvenc_wdg_table[] = {
+ { 1920 * 1088, 50 },
+ { 2560 * 1440, 100 },
+ { 4096 * 2304, 200 },
+ { 8192 * 8192, 400 },
+ { 15360 * 8640, 800 },
+};
+
+u32 rkvenc_calc_wdg(struct rkvenc_dev *dev, unsigned int width,
+ unsigned int height)
+{
+ unsigned int pixels = width * height;
+ unsigned int ms = rkvenc_wdg_table[ARRAY_SIZE(rkvenc_wdg_table) - 1].ms;
+ u64 cycles;
+ u32 thd;
+ int i;
+
+ for (i = 0; i < ARRAY_SIZE(rkvenc_wdg_table); i++) {
+ if (pixels <= rkvenc_wdg_table[i].pixels) {
+ ms = rkvenc_wdg_table[i].ms;
+ break;
+ }
+ }
+
+ cycles = (u64)ms * (dev->core_clk_rate / 1000);
+ do_div(cycles, 256);
+ thd = min_t(u64, cycles, 0xffffff);
+
+ return thd;
+}
+
+/* ---- format helpers ---- */
+
+static const u32 rkvenc_src_fourcc = V4L2_PIX_FMT_NV12;
+
+static const struct rkvenc_coded_fmt_desc *rkvenc_coded_descs[] = {
+ &rkvenc_h264_fmt_desc,
+};
+
+static const struct rkvenc_coded_fmt_desc *rkvenc_find_coded_desc(u32 fourcc)
+{
+ int i;
+
+ for (i = 0; i < ARRAY_SIZE(rkvenc_coded_descs); i++) {
+ if (rkvenc_coded_descs[i]->fourcc == fourcc)
+ return rkvenc_coded_descs[i];
+ }
+
+ return NULL;
+}
+
+static u32 rkvenc_nv12_sizeimage(unsigned int width, unsigned int height)
+{
+ return width * height + 2 * ALIGN(width, 2) / 2 * ALIGN(height, 2) / 2 * 2;
+}
+
+static u32 rkvenc_coded_sizeimage(unsigned int width, unsigned int height)
+{
+ /* Generous fixed upper bound for a single fixed-QP baseline frame;
+ * revisit once real CBR/VBR budget control lands.
+ */
+ return max_t(u32, width * height, SZ_128K);
+}
+
+static void rkvenc_fill_src_fmt(struct v4l2_pix_format *fmt)
+{
+ fmt->pixelformat = rkvenc_src_fourcc;
+ fmt->width = clamp_t(u32, fmt->width, RKVENC_DEFAULT_WIDTH, RKVENC_MAX_WIDTH);
+ fmt->height = clamp_t(u32, fmt->height, RKVENC_DEFAULT_HEIGHT, RKVENC_MAX_HEIGHT);
+ fmt->width = ALIGN(fmt->width, 2);
+ fmt->height = ALIGN(fmt->height, 2);
+ fmt->field = V4L2_FIELD_NONE;
+ fmt->colorspace = V4L2_COLORSPACE_REC709;
+ fmt->bytesperline = fmt->width;
+ fmt->sizeimage = rkvenc_nv12_sizeimage(fmt->width, fmt->height);
+}
+
+static void rkvenc_fill_coded_fmt(struct v4l2_pix_format *fmt,
+ const struct rkvenc_coded_fmt_desc *desc)
+{
+ fmt->pixelformat = desc->fourcc;
+ fmt->width = clamp_t(u32, fmt->width, desc->frmsize.frmsize_min_width,
+ desc->frmsize.frmsize_max_width);
+ fmt->height = clamp_t(u32, fmt->height, desc->frmsize.frmsize_min_height,
+ desc->frmsize.frmsize_max_height);
+ fmt->width = ALIGN(fmt->width, 2);
+ fmt->height = ALIGN(fmt->height, 2);
+ fmt->field = V4L2_FIELD_NONE;
+ fmt->bytesperline = 0;
+ fmt->sizeimage = rkvenc_coded_sizeimage(fmt->width, fmt->height);
+}
+
+/* ---- ioctl_ops ---- */
+
+static int rkvenc_querycap(struct file *file, void *priv,
+ struct v4l2_capability *cap)
+{
+ strscpy(cap->driver, "rkvenc", sizeof(cap->driver));
+ strscpy(cap->card, "rockchip,rkv-encoder-rk3576", sizeof(cap->card));
+ return 0;
+}
+
+static int rkvenc_enum_fmt_vid_cap(struct file *file, void *priv,
+ struct v4l2_fmtdesc *f)
+{
+ if (f->index >= ARRAY_SIZE(rkvenc_coded_descs))
+ return -EINVAL;
+
+ f->pixelformat = rkvenc_coded_descs[f->index]->fourcc;
+ f->flags = V4L2_FMT_FLAG_COMPRESSED;
+ return 0;
+}
+
+static int rkvenc_enum_fmt_vid_out(struct file *file, void *priv,
+ struct v4l2_fmtdesc *f)
+{
+ if (f->index != 0)
+ return -EINVAL;
+
+ f->pixelformat = rkvenc_src_fourcc;
+ return 0;
+}
+
+static int rkvenc_g_fmt_vid_cap(struct file *file, void *priv,
+ struct v4l2_format *f)
+{
+ struct rkvenc_ctx *ctx = fh_to_rkvenc_ctx(file_to_v4l2_fh(file));
+
+ f->fmt.pix = ctx->coded_fmt;
+ return 0;
+}
+
+static int rkvenc_g_fmt_vid_out(struct file *file, void *priv,
+ struct v4l2_format *f)
+{
+ struct rkvenc_ctx *ctx = fh_to_rkvenc_ctx(file_to_v4l2_fh(file));
+
+ f->fmt.pix = ctx->src_fmt;
+ return 0;
+}
+
+static int rkvenc_try_fmt_vid_cap(struct file *file, void *priv,
+ struct v4l2_format *f)
+{
+ const struct rkvenc_coded_fmt_desc *desc;
+
+ desc = rkvenc_find_coded_desc(f->fmt.pix.pixelformat);
+ if (!desc)
+ desc = rkvenc_coded_descs[0];
+
+ rkvenc_fill_coded_fmt(&f->fmt.pix, desc);
+ return 0;
+}
+
+static int rkvenc_try_fmt_vid_out(struct file *file, void *priv,
+ struct v4l2_format *f)
+{
+ rkvenc_fill_src_fmt(&f->fmt.pix);
+ return 0;
+}
+
+static int rkvenc_s_fmt_vid_cap(struct file *file, void *priv,
+ struct v4l2_format *f)
+{
+ struct rkvenc_ctx *ctx = fh_to_rkvenc_ctx(file_to_v4l2_fh(file));
+ const struct rkvenc_coded_fmt_desc *desc;
+
+ if (vb2_is_busy(v4l2_m2m_get_dst_vq(ctx->fh.m2m_ctx)))
+ return -EBUSY;
+
+ desc = rkvenc_find_coded_desc(f->fmt.pix.pixelformat);
+ if (!desc)
+ desc = rkvenc_coded_descs[0];
+
+ rkvenc_fill_coded_fmt(&f->fmt.pix, desc);
+ ctx->coded_fmt = f->fmt.pix;
+ ctx->coded_desc = desc;
+ return 0;
+}
+
+static int rkvenc_s_fmt_vid_out(struct file *file, void *priv,
+ struct v4l2_format *f)
+{
+ struct rkvenc_ctx *ctx = fh_to_rkvenc_ctx(file_to_v4l2_fh(file));
+
+ if (vb2_is_busy(v4l2_m2m_get_src_vq(ctx->fh.m2m_ctx)))
+ return -EBUSY;
+
+ rkvenc_fill_src_fmt(&f->fmt.pix);
+ ctx->src_fmt = f->fmt.pix;
+ return 0;
+}
+
+static const struct v4l2_ioctl_ops rkvenc_ioctl_ops = {
+ .vidioc_querycap = rkvenc_querycap,
+
+ .vidioc_enum_fmt_vid_cap = rkvenc_enum_fmt_vid_cap,
+ .vidioc_g_fmt_vid_cap = rkvenc_g_fmt_vid_cap,
+ .vidioc_try_fmt_vid_cap = rkvenc_try_fmt_vid_cap,
+ .vidioc_s_fmt_vid_cap = rkvenc_s_fmt_vid_cap,
+
+ .vidioc_enum_fmt_vid_out = rkvenc_enum_fmt_vid_out,
+ .vidioc_g_fmt_vid_out = rkvenc_g_fmt_vid_out,
+ .vidioc_try_fmt_vid_out = rkvenc_try_fmt_vid_out,
+ .vidioc_s_fmt_vid_out = rkvenc_s_fmt_vid_out,
+
+ .vidioc_reqbufs = v4l2_m2m_ioctl_reqbufs,
+ .vidioc_querybuf = v4l2_m2m_ioctl_querybuf,
+ .vidioc_qbuf = v4l2_m2m_ioctl_qbuf,
+ .vidioc_dqbuf = v4l2_m2m_ioctl_dqbuf,
+ .vidioc_prepare_buf = v4l2_m2m_ioctl_prepare_buf,
+ .vidioc_create_bufs = v4l2_m2m_ioctl_create_bufs,
+ .vidioc_expbuf = v4l2_m2m_ioctl_expbuf,
+ .vidioc_streamon = v4l2_m2m_ioctl_streamon,
+ .vidioc_streamoff = v4l2_m2m_ioctl_streamoff,
+
+ .vidioc_try_encoder_cmd = v4l2_m2m_ioctl_try_encoder_cmd,
+ .vidioc_encoder_cmd = v4l2_m2m_ioctl_encoder_cmd,
+
+ .vidioc_subscribe_event = v4l2_ctrl_subscribe_event,
+ .vidioc_unsubscribe_event = v4l2_event_unsubscribe,
+};
+
+/* ---- vb2_ops ---- */
+
+static int rkvenc_queue_setup(struct vb2_queue *vq, unsigned int *num_buffers,
+ unsigned int *num_planes, unsigned int sizes[],
+ struct device *alloc_devs[])
+{
+ struct rkvenc_ctx *ctx = vb2_get_drv_priv(vq);
+ struct v4l2_pix_format *fmt;
+
+ fmt = V4L2_TYPE_IS_OUTPUT(vq->type) ? &ctx->src_fmt : &ctx->coded_fmt;
+
+ if (*num_planes) {
+ if (sizes[0] < fmt->sizeimage)
+ return -EINVAL;
+ } else {
+ *num_planes = 1;
+ sizes[0] = fmt->sizeimage;
+ }
+
+ return 0;
+}
+
+static int rkvenc_buf_prepare(struct vb2_buffer *vb)
+{
+ struct vb2_v4l2_buffer *vbuf = to_vb2_v4l2_buffer(vb);
+ struct rkvenc_ctx *ctx = vb2_get_drv_priv(vb->vb2_queue);
+ struct v4l2_pix_format *fmt;
+
+ fmt = V4L2_TYPE_IS_OUTPUT(vb->vb2_queue->type) ? &ctx->src_fmt : &ctx->coded_fmt;
+
+ if (vb2_plane_size(vb, 0) < fmt->sizeimage)
+ return -EINVAL;
+
+ if (V4L2_TYPE_IS_OUTPUT(vb->vb2_queue->type))
+ vb2_set_plane_payload(vb, 0, fmt->sizeimage);
+ else
+ vbuf->field = V4L2_FIELD_NONE;
+
+ return 0;
+}
+
+static void rkvenc_buf_queue(struct vb2_buffer *vb)
+{
+ struct rkvenc_ctx *ctx = vb2_get_drv_priv(vb->vb2_queue);
+
+ v4l2_m2m_buf_queue(ctx->fh.m2m_ctx, to_vb2_v4l2_buffer(vb));
+}
+
+static int rkvenc_start_streaming(struct vb2_queue *q, unsigned int count)
+{
+ struct rkvenc_ctx *ctx = vb2_get_drv_priv(q);
+ int ret = 0;
+
+ if (V4L2_TYPE_IS_OUTPUT(q->type) && ctx->coded_desc && ctx->coded_desc->ops->start)
+ ret = ctx->coded_desc->ops->start(ctx);
+
+ if (ret) {
+ struct vb2_v4l2_buffer *vbuf;
+
+ while ((vbuf = v4l2_m2m_src_buf_remove(ctx->fh.m2m_ctx)))
+ v4l2_m2m_buf_done(vbuf, VB2_BUF_STATE_QUEUED);
+ }
+
+ return ret;
+}
+
+static void rkvenc_stop_streaming(struct vb2_queue *q)
+{
+ struct rkvenc_ctx *ctx = vb2_get_drv_priv(q);
+ struct vb2_v4l2_buffer *vbuf;
+
+ for (;;) {
+ if (V4L2_TYPE_IS_OUTPUT(q->type))
+ vbuf = v4l2_m2m_src_buf_remove(ctx->fh.m2m_ctx);
+ else
+ vbuf = v4l2_m2m_dst_buf_remove(ctx->fh.m2m_ctx);
+
+ if (!vbuf)
+ break;
+
+ v4l2_m2m_buf_done(vbuf, VB2_BUF_STATE_ERROR);
+ }
+
+ if (V4L2_TYPE_IS_OUTPUT(q->type) && ctx->coded_desc && ctx->coded_desc->ops->stop)
+ ctx->coded_desc->ops->stop(ctx);
+}
+
+static const struct vb2_ops rkvenc_qops = {
+ .queue_setup = rkvenc_queue_setup,
+ .buf_prepare = rkvenc_buf_prepare,
+ .buf_queue = rkvenc_buf_queue,
+ .start_streaming = rkvenc_start_streaming,
+ .stop_streaming = rkvenc_stop_streaming,
+};
+
+static int rkvenc_queue_init(void *priv, struct vb2_queue *src_vq,
+ struct vb2_queue *dst_vq)
+{
+ struct rkvenc_ctx *ctx = priv;
+ int ret;
+
+ src_vq->type = V4L2_BUF_TYPE_VIDEO_OUTPUT;
+ src_vq->io_modes = VB2_MMAP | VB2_DMABUF;
+ src_vq->drv_priv = ctx;
+ src_vq->buf_struct_size = sizeof(struct v4l2_m2m_buffer);
+ src_vq->ops = &rkvenc_qops;
+ src_vq->mem_ops = &vb2_dma_contig_memops;
+ src_vq->timestamp_flags = V4L2_BUF_FLAG_TIMESTAMP_COPY;
+ src_vq->lock = &ctx->dev->vfd_mutex;
+ src_vq->dev = ctx->dev->v4l2_dev.dev;
+ ret = vb2_queue_init(src_vq);
+ if (ret)
+ return ret;
+
+ dst_vq->type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
+ dst_vq->io_modes = VB2_MMAP | VB2_DMABUF;
+ dst_vq->drv_priv = ctx;
+ dst_vq->buf_struct_size = sizeof(struct v4l2_m2m_buffer);
+ dst_vq->ops = &rkvenc_qops;
+ dst_vq->mem_ops = &vb2_dma_contig_memops;
+ dst_vq->timestamp_flags = V4L2_BUF_FLAG_TIMESTAMP_COPY;
+ dst_vq->lock = &ctx->dev->vfd_mutex;
+ dst_vq->dev = ctx->dev->v4l2_dev.dev;
+ return vb2_queue_init(dst_vq);
+}
+
+/* ---- v4l2_m2m_ops ---- */
+
+void rkvenc_job_finish(struct rkvenc_dev *dev, struct rkvenc_ctx *ctx,
+ enum vb2_buffer_state state)
+{
+ clk_bulk_disable(dev->num_clocks, dev->clocks);
+ pm_runtime_mark_last_busy(dev->dev);
+ pm_runtime_put_autosuspend(dev->dev);
+
+ dev->cur_ctx = NULL;
+ v4l2_m2m_buf_done_and_job_finish(dev->m2m_dev, ctx->fh.m2m_ctx, state);
+}
+
+static void rkvenc_device_run(void *priv)
+{
+ struct rkvenc_ctx *ctx = priv;
+ struct rkvenc_dev *dev = ctx->dev;
+ int ret;
+
+ ret = pm_runtime_resume_and_get(dev->dev);
+ if (ret < 0) {
+ v4l2_m2m_buf_done_and_job_finish(dev->m2m_dev, ctx->fh.m2m_ctx,
+ VB2_BUF_STATE_ERROR);
+ return;
+ }
+
+ ret = clk_bulk_enable(dev->num_clocks, dev->clocks);
+ if (ret) {
+ pm_runtime_put_autosuspend(dev->dev);
+ v4l2_m2m_buf_done_and_job_finish(dev->m2m_dev, ctx->fh.m2m_ctx,
+ VB2_BUF_STATE_ERROR);
+ return;
+ }
+
+ v4l2_m2m_buf_copy_metadata(rkvenc_get_src_buf(ctx), rkvenc_get_dst_buf(ctx));
+
+ dev->cur_ctx = ctx;
+ dev->irq_status = 0;
+ schedule_delayed_work(&dev->watchdog_work,
+ msecs_to_jiffies(RKVENC_WATCHDOG_TIMEOUT_MS));
+
+ ctx->coded_desc->ops->run(ctx);
+}
+
+static const struct v4l2_m2m_ops rkvenc_m2m_ops = {
+ .device_run = rkvenc_device_run,
+};
+
+/* ---- IRQ handling ---- */
+
+static bool rkvenc_drain_slices(struct rkvenc_dev *dev)
+{
+ unsigned int count = rkvenc_read(dev, RKVENC_REG_ST_SLICE_NUM) & 0x3f;
+ bool last = false;
+ unsigned int i;
+ u32 val;
+
+ for (i = 0; i < count; i++) {
+ val = rkvenc_read(dev, RKVENC_REG_ST_SLICE_LEN);
+ if (val & BIT(31))
+ last = true;
+ }
+
+ return last;
+}
+
+static irqreturn_t rkvenc_irq(int irq, void *priv)
+{
+ struct rkvenc_dev *dev = priv;
+ u32 status = rkvenc_read(dev, RKVENC_REG_INT_STA);
+ bool done = false;
+
+ rkvenc_write(dev, RKVENC_REG_INT_CLR, status);
+
+ if (status & BIT(8) /* wdg_sta */) {
+ u32 mask = rkvenc_read(dev, RKVENC_REG_INT_MASK);
+
+ rkvenc_write(dev, RKVENC_REG_INT_MASK, mask | BIT(8));
+ }
+
+ if (!dev->cur_ctx)
+ return status ? IRQ_HANDLED : IRQ_NONE;
+
+ /* VEPU510 errata (vendor comment): int_sta ENC_DONE is not reliable
+ * on its own — the hardware has been observed to skip raising it
+ * even though the frame is actually finished. The slice-length
+ * FIFO's "last" flag (always populated because ENC_PIC.SLEN_FIFO is
+ * forced on for every VEPU510 task, see rkvenc_vepu510_quirk()) is
+ * the authoritative completion signal instead.
+ */
+ if (rkvenc_drain_slices(dev)) {
+ rkvenc_write(dev, RKVENC_REG_ENC_ID, 0);
+ udelay(5);
+ status |= rkvenc_read(dev, RKVENC_REG_INT_STA);
+ done = true;
+ }
+
+ if (status & (BIT(0) /* enc_done_sta */ | RKVENC_INT_ERROR_MASK))
+ done = true;
+
+ dev->irq_status |= status;
+
+ return done ? IRQ_WAKE_THREAD : IRQ_HANDLED;
+}
+
+/* Graceful core recovery after an error/timeout, mirroring the downstream
+ * vendor kernel's rkvenc_soft_reset() (mpp_rkvenc2.c): a SAFE clear via
+ * ENC_CLR drains the core's outstanding AXI transactions so the IOMMU can
+ * disable cleanly, then a FORCE clear + interrupt-state wipe returns the core
+ * to a defined state. Without this, a hung P-frame leaves the core wedged and
+ * an unacknowledged write fault stuck active, which cascades into the NEXT
+ * encode failing ("rk_iommu Disable paging request timed out"). Only ever
+ * called from the IRQ error path, so it can never touch a good frame; clocks
+ * are still enabled here (rkvenc_job_finish disables them afterwards).
+ */
+static void rkvenc_soft_reset(struct rkvenc_dev *dev)
+{
+ u32 sta;
+ int ret;
+
+ rkvenc_write(dev, RKVENC_REG_INT_MASK, 0x3ff);
+ rkvenc_write(dev, RKVENC_REG_ENC_CLR, RKVENC_ENC_CLR_SAFE);
+ ret = readl_relaxed_poll_timeout(dev->regs + RKVENC_REG_INT_STA, sta,
+ sta & RKVENC_SCLR_DONE_STA, 0, 1000);
+ rkvenc_write(dev, RKVENC_REG_ENC_CLR, RKVENC_ENC_CLR_FORCE);
+ udelay(5);
+ rkvenc_write(dev, RKVENC_REG_ENC_CLR, 0);
+ rkvenc_write(dev, RKVENC_REG_INT_CLR, 0xffffffff);
+ rkvenc_write(dev, RKVENC_REG_INT_STA, 0);
+
+ if (ret) {
+ /* Safe clear never completed -- fall back to a full core reset. */
+ dev_warn(dev->dev, "safe reset timed out, forcing core reset\n");
+ reset_control_assert(dev->rst);
+ udelay(5);
+ reset_control_deassert(dev->rst);
+ }
+}
+
+static irqreturn_t rkvenc_irq_thread(int irq, void *priv)
+{
+ struct rkvenc_dev *dev = priv;
+ struct rkvenc_ctx *ctx = dev->cur_ctx;
+ enum vb2_buffer_state state;
+
+ if (!ctx)
+ return IRQ_HANDLED;
+
+ if (!cancel_delayed_work(&dev->watchdog_work))
+ return IRQ_HANDLED;
+
+ state = (dev->irq_status & RKVENC_INT_ERROR_MASK) ?
+ VB2_BUF_STATE_ERROR : VB2_BUF_STATE_DONE;
+
+ if (state == VB2_BUF_STATE_ERROR) {
+ dev_warn(dev->dev,
+ "encode error, int_sta=0x%08x (error bits: %s%s%s%s%s%s%s%s) "
+ "version=0x%08x enc_strt=0x%08x int_en=0x%08x int_msk=0x%08x "
+ "enc_wdg=0x%08x enc_rsl=0x%08x enc_pic=0x%08x synt_sli0=0x%08x "
+ "bs_lgth=0x%08x slice_num=0x%08x st_bsb=0x%08x\n",
+ dev->irq_status,
+ dev->irq_status & BIT(4) ? "vbsf_oflw " : "",
+ dev->irq_status & BIT(6) ? "enc_err " : "",
+ dev->irq_status & BIT(7) ? "vsrc_err " : "",
+ dev->irq_status & BIT(8) ? "wdg " : "",
+ dev->irq_status & BIT(9) ? "lkt_err_int " : "",
+ dev->irq_status & BIT(10) ? "lkt_err_stop " : "",
+ dev->irq_status & BIT(11) ? "lkt_force_stop " : "",
+ dev->irq_status & BIT(15) ? "dvbm_err " : "",
+ /* version is a fixed IP-identification readback: if this
+ * comes back 0x00000000 or 0xffffffff, the MMIO region
+ * itself is dead (clock/power/mapping), not an encode
+ * config problem — a much more fundamental class of bug
+ * than anything register-value-level we've audited so far.
+ */
+ rkvenc_read(dev, RKVENC_REG_VERSION),
+ rkvenc_read(dev, RKVENC_REG_ENC_START),
+ rkvenc_read(dev, RKVENC_REG_INT_EN),
+ rkvenc_read(dev, RKVENC_REG_INT_MASK),
+ rkvenc_read(dev, RKVENC_REG_ENC_WDG),
+ rkvenc_read(dev, RKVENC_REG_ENC_RSL),
+ rkvenc_read(dev, RKVENC_REG_ENC_PIC),
+ rkvenc_read(dev, RKVENC_REG_SYNT_SLI0),
+ rkvenc_read(dev, RKVENC_REG_ST_BS_LENGTH),
+ rkvenc_read(dev, RKVENC_REG_ST_SLICE_NUM),
+ rkvenc_read(dev, RKVENC_REG_ST_BSB));
+
+ /* Release the DVBM/encoder hold (only the drain-slices completion
+ * path did this before) and gracefully reset the wedged core so
+ * the fault is drained and the NEXT encode starts clean.
+ */
+ rkvenc_write(dev, RKVENC_REG_ENC_ID, 0);
+ rkvenc_soft_reset(dev);
+ }
+
+ ctx->coded_desc->ops->done(ctx, state);
+ rkvenc_job_finish(dev, ctx, state);
+
+ return IRQ_HANDLED;
+}
+
+static void rkvenc_watchdog(struct work_struct *work)
+{
+ struct rkvenc_dev *dev = container_of(work, struct rkvenc_dev,
+ watchdog_work.work);
+ struct rkvenc_ctx *ctx = dev->cur_ctx;
+
+ if (!ctx)
+ return;
+
+ dev_warn(dev->dev, "encode timeout, resetting core\n");
+
+ reset_control_assert(dev->rst);
+ udelay(5);
+ reset_control_deassert(dev->rst);
+
+ ctx->coded_desc->ops->done(ctx, VB2_BUF_STATE_ERROR);
+ rkvenc_job_finish(dev, ctx, VB2_BUF_STATE_ERROR);
+}
+
+/* ---- ctrls / file ops ---- */
+
+static int rkvenc_open(struct file *file)
+{
+ struct rkvenc_dev *dev = video_drvdata(file);
+ struct rkvenc_ctx *ctx;
+ int ret;
+
+ ctx = kzalloc_obj(*ctx);
+ if (!ctx)
+ return -ENOMEM;
+
+ v4l2_fh_init(&ctx->fh, &dev->vfd);
+ file->private_data = &ctx->fh;
+ ctx->dev = dev;
+
+ ctx->coded_desc = rkvenc_coded_descs[0];
+ ctx->coded_fmt.width = RKVENC_DEFAULT_WIDTH;
+ ctx->coded_fmt.height = RKVENC_DEFAULT_HEIGHT;
+ rkvenc_fill_coded_fmt(&ctx->coded_fmt, ctx->coded_desc);
+ ctx->src_fmt.width = RKVENC_DEFAULT_WIDTH;
+ ctx->src_fmt.height = RKVENC_DEFAULT_HEIGHT;
+ rkvenc_fill_src_fmt(&ctx->src_fmt);
+
+ v4l2_ctrl_handler_init(&ctx->ctrl_handler, 16);
+ ctx->fh.ctrl_handler = &ctx->ctrl_handler;
+
+ ret = ctx->coded_desc->ops->init_ctrls(ctx);
+ if (!ret)
+ ret = ctx->ctrl_handler.error;
+ if (!ret)
+ ret = v4l2_ctrl_handler_setup(&ctx->ctrl_handler);
+ if (ret)
+ goto err_free_handler;
+
+ ctx->fh.m2m_ctx = v4l2_m2m_ctx_init(dev->m2m_dev, ctx, rkvenc_queue_init);
+ if (IS_ERR(ctx->fh.m2m_ctx)) {
+ ret = PTR_ERR(ctx->fh.m2m_ctx);
+ goto err_free_handler;
+ }
+
+ v4l2_fh_add(&ctx->fh, file);
+
+ return 0;
+
+err_free_handler:
+ v4l2_ctrl_handler_free(&ctx->ctrl_handler);
+ v4l2_fh_exit(&ctx->fh);
+ kfree(ctx);
+ return ret;
+}
+
+static int rkvenc_release(struct file *file)
+{
+ struct rkvenc_ctx *ctx = fh_to_rkvenc_ctx(file->private_data);
+
+ v4l2_fh_del(&ctx->fh, file);
+ v4l2_m2m_ctx_release(ctx->fh.m2m_ctx);
+ v4l2_ctrl_handler_free(&ctx->ctrl_handler);
+ v4l2_fh_exit(&ctx->fh);
+ kfree(ctx);
+ return 0;
+}
+
+static const struct v4l2_file_operations rkvenc_fops = {
+ .owner = THIS_MODULE,
+ .open = rkvenc_open,
+ .release = rkvenc_release,
+ .poll = v4l2_m2m_fop_poll,
+ .unlocked_ioctl = video_ioctl2,
+ .mmap = v4l2_m2m_fop_mmap,
+};
+
+/* ---- probe/remove ---- */
+
+static int rkvenc_runtime_suspend(struct device *dev)
+{
+ struct rkvenc_dev *rkvenc = dev_get_drvdata(dev);
+
+ clk_bulk_disable_unprepare(rkvenc->num_clocks, rkvenc->clocks);
+ return 0;
+}
+
+static int rkvenc_runtime_resume(struct device *dev)
+{
+ struct rkvenc_dev *rkvenc = dev_get_drvdata(dev);
+
+ return clk_bulk_prepare_enable(rkvenc->num_clocks, rkvenc->clocks);
+}
+
+static const struct dev_pm_ops rkvenc_pm_ops = {
+ SET_RUNTIME_PM_OPS(rkvenc_runtime_suspend, rkvenc_runtime_resume, NULL)
+};
+
+static int rkvenc_probe(struct platform_device *pdev)
+{
+ struct device *dev = &pdev->dev;
+ struct rkvenc_dev *rkvenc;
+ struct clk_bulk_data *clk;
+ int irq, ret, i;
+
+ rkvenc = devm_kzalloc(dev, sizeof(*rkvenc), GFP_KERNEL);
+ if (!rkvenc)
+ return -ENOMEM;
+
+ rkvenc->dev = dev;
+ platform_set_drvdata(pdev, rkvenc);
+ mutex_init(&rkvenc->vfd_mutex);
+ INIT_DELAYED_WORK(&rkvenc->watchdog_work, rkvenc_watchdog);
+
+ rkvenc->regs = devm_platform_ioremap_resource(pdev, 0);
+ if (IS_ERR(rkvenc->regs))
+ return PTR_ERR(rkvenc->regs);
+
+ /* Buffer pointer registers (adr_src0/1/2, bsb*, rfp*) are 32 bits
+ * wide; each core sits behind its own rkvenc{0,1}_mmu IOMMU, which
+ * this DMA mask keeps within a 32-bit IOVA aperture.
+ */
+ ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(32));
+ if (ret)
+ return dev_err_probe(dev, ret, "failed to set DMA mask\n");
+
+ irq = platform_get_irq(pdev, 0);
+ if (irq < 0)
+ return irq;
+
+ ret = devm_clk_bulk_get_all(dev, &rkvenc->clocks);
+ if (ret < 0)
+ return dev_err_probe(dev, ret, "failed to get clocks\n");
+ rkvenc->num_clocks = ret;
+
+ for (i = 0; i < rkvenc->num_clocks; i++) {
+ clk = &rkvenc->clocks[i];
+ if (!strcmp(clk->id, "clk_core"))
+ rkvenc->core_clk_rate = clk_get_rate(clk->clk);
+ }
+ if (!rkvenc->core_clk_rate)
+ rkvenc->core_clk_rate = 702000000;
+
+ rkvenc->rst = devm_reset_control_array_get_exclusive(dev);
+ if (IS_ERR(rkvenc->rst))
+ return dev_err_probe(dev, PTR_ERR(rkvenc->rst),
+ "failed to get resets\n");
+
+ ret = devm_request_threaded_irq(dev, irq, rkvenc_irq, rkvenc_irq_thread,
+ IRQF_ONESHOT, dev_name(dev), rkvenc);
+ if (ret)
+ return dev_err_probe(dev, ret, "failed to request irq\n");
+
+ ret = v4l2_device_register(dev, &rkvenc->v4l2_dev);
+ if (ret)
+ return dev_err_probe(dev, ret, "failed to register v4l2 device\n");
+
+ rkvenc->m2m_dev = v4l2_m2m_init(&rkvenc_m2m_ops);
+ if (IS_ERR(rkvenc->m2m_dev)) {
+ ret = PTR_ERR(rkvenc->m2m_dev);
+ goto err_unreg_v4l2;
+ }
+
+ rkvenc->vfd = (struct video_device){
+ .fops = &rkvenc_fops,
+ .ioctl_ops = &rkvenc_ioctl_ops,
+ .minor = -1,
+ .release = video_device_release_empty,
+ .lock = &rkvenc->vfd_mutex,
+ .v4l2_dev = &rkvenc->v4l2_dev,
+ .vfl_dir = VFL_DIR_M2M,
+ .device_caps = V4L2_CAP_VIDEO_M2M | V4L2_CAP_STREAMING,
+ };
+ strscpy(rkvenc->vfd.name, "rkvenc", sizeof(rkvenc->vfd.name));
+ video_set_drvdata(&rkvenc->vfd, rkvenc);
+
+ pm_runtime_set_autosuspend_delay(dev, RKVENC_AUTOSUSPEND_DELAY_MS);
+ pm_runtime_use_autosuspend(dev);
+ devm_pm_runtime_enable(dev);
+
+ /*
+ * One-time reset cycle (via a real pm_runtime activation, so the
+ * power domain and clocks are genuinely on rather than assuming raw
+ * clk_bulk_prepare_enable() alone would power an unattached genpd)
+ * so the core starts from a defined state rather than whatever the
+ * bootloader/ATF left it in -- otherwise reset is only ever touched
+ * by rkvenc_watchdog()'s error-recovery path, never on a normal
+ * first activation. Board bring-up saw an isolated rk_iommu write
+ * fault at an IOVA that changed between boots and matched none of
+ * this driver's own buffer addresses, consistent with undefined
+ * boot-varying hardware state.
+ *
+ * Done once here, not on every rkvenc_runtime_resume(): that was
+ * tried first and made things worse (a hardware-reported "enc_err"
+ * status on the very next encode, with a bitstream length wildly
+ * larger than the same test frame ever produced before) -- something
+ * about this core's state is expected to persist for the life of the
+ * session, not be re-initialized before every single frame.
+ */
+ ret = pm_runtime_resume_and_get(dev);
+ if (ret)
+ return dev_err_probe(dev, ret, "failed to power up for initial reset\n");
+ reset_control_assert(rkvenc->rst);
+ udelay(5);
+ reset_control_deassert(rkvenc->rst);
+ pm_runtime_mark_last_busy(dev);
+ pm_runtime_put_autosuspend(dev);
+
+ ret = video_register_device(&rkvenc->vfd, VFL_TYPE_VIDEO, -1);
+ if (ret) {
+ dev_err_probe(dev, ret, "failed to register video device\n");
+ goto err_release_m2m;
+ }
+
+ dev_info(dev, "VEPU510 encoder core registered as %s\n",
+ video_device_node_name(&rkvenc->vfd));
+
+ return 0;
+
+err_release_m2m:
+ v4l2_m2m_release(rkvenc->m2m_dev);
+err_unreg_v4l2:
+ v4l2_device_unregister(&rkvenc->v4l2_dev);
+ return ret;
+}
+
+static void rkvenc_remove(struct platform_device *pdev)
+{
+ struct rkvenc_dev *rkvenc = platform_get_drvdata(pdev);
+
+ cancel_delayed_work_sync(&rkvenc->watchdog_work);
+ video_unregister_device(&rkvenc->vfd);
+ v4l2_m2m_release(rkvenc->m2m_dev);
+ v4l2_device_unregister(&rkvenc->v4l2_dev);
+}
+
+static const struct of_device_id rkvenc_of_match[] = {
+ { .compatible = "rockchip,rk3576-vepu" },
+ {}
+};
+MODULE_DEVICE_TABLE(of, rkvenc_of_match);
+
+static struct platform_driver rkvenc_driver = {
+ .probe = rkvenc_probe,
+ .remove = rkvenc_remove,
+ .driver = {
+ .name = "rkvenc",
+ .of_match_table = rkvenc_of_match,
+ .pm = &rkvenc_pm_ops,
+ },
+};
+module_platform_driver(rkvenc_driver);
+
+MODULE_DESCRIPTION("Rockchip VEPU510 video encoder driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/media/platform/rockchip/rkvenc/rkvenc.h b/drivers/media/platform/rockchip/rkvenc/rkvenc.h
new file mode 100644
index 000000000..3e3e30de3
--- /dev/null
+++ b/drivers/media/platform/rockchip/rkvenc/rkvenc.h
@@ -0,0 +1,212 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Rockchip VEPU510 (RK3576) hardware video encoder driver.
+ *
+ * Copyright (C) 2026 Jiaxing Hu <gahing@gahingwoo.com>
+ */
+
+#ifndef RKVENC_H_
+#define RKVENC_H_
+
+#include <linux/clk.h>
+#include <linux/completion.h>
+#include <linux/platform_device.h>
+#include <linux/reset.h>
+#include <linux/videodev2.h>
+#include <linux/workqueue.h>
+#include <media/v4l2-ctrls.h>
+#include <media/v4l2-device.h>
+#include <media/v4l2-mem2mem.h>
+#include <media/videobuf2-core.h>
+
+struct rkvenc_ctx;
+
+struct rkvenc_fmt {
+ u32 fourcc;
+ unsigned int frmsize_min_width;
+ unsigned int frmsize_max_width;
+ unsigned int frmsize_step_width;
+ unsigned int frmsize_min_height;
+ unsigned int frmsize_max_height;
+ unsigned int frmsize_step_height;
+};
+
+/*
+ * Per-coded-format (H.264, later HEVC) operation vtable. Mirrors the split
+ * rkvdec uses for its per-codec backends, adapted for a *stateful* mem2mem
+ * encoder (no V4L2 request-API preamble/postamble, unlike rkvdec) — the
+ * device_run()/done() split matches drivers/media/platform/verisilicon's
+ * hantro_codec_ops instead.
+ */
+struct rkvenc_coded_fmt_ops {
+ /* Registers this format's V4L2 controls into ctx->ctrl_handler. */
+ int (*init_ctrls)(struct rkvenc_ctx *ctx);
+ int (*start)(struct rkvenc_ctx *ctx);
+ void (*stop)(struct rkvenc_ctx *ctx);
+ void (*run)(struct rkvenc_ctx *ctx);
+ /* Called from the threaded IRQ handler once hardware reports frame
+ * done. Must set the CAPTURE buffer's bytesused/timestamp/flags.
+ */
+ void (*done)(struct rkvenc_ctx *ctx, enum vb2_buffer_state state);
+};
+
+struct rkvenc_coded_fmt_desc {
+ u32 fourcc;
+ struct rkvenc_fmt frmsize;
+ const struct rkvenc_coded_fmt_ops *ops;
+};
+
+struct rkvenc_dev {
+ struct v4l2_device v4l2_dev;
+ struct video_device vfd;
+ struct v4l2_m2m_dev *m2m_dev;
+
+ struct device *dev;
+ void __iomem *regs;
+ struct clk_bulk_data *clocks;
+ int num_clocks;
+ struct reset_control *rst;
+
+ /* Protects ctx list / hardware access ordering. vb2 queues already
+ * serialize per-context streaming; this additionally serializes
+ * access to the one physical core across contexts (rkvenc0/rkvenc1
+ * are two separate rkvenc_dev instances, each owning one core —
+ * v1 does not implement the vendor driver's CCU cross-core task
+ * dispatch/load-balancing).
+ */
+ struct mutex vfd_mutex;
+
+ struct rkvenc_ctx *cur_ctx;
+ struct delayed_work watchdog_work;
+
+ unsigned int core_clk_rate;
+
+ /* Accumulated by the hard IRQ handler, consumed by the threaded one. */
+ u32 irq_status;
+};
+
+struct rkvenc_h264_ctx {
+ struct v4l2_ctrl *bitrate;
+ struct v4l2_ctrl *profile;
+ struct v4l2_ctrl *level;
+ struct v4l2_ctrl *entropy_mode;
+ struct v4l2_ctrl *gop_size;
+ struct v4l2_ctrl *i_qp;
+ struct v4l2_ctrl *p_qp;
+
+ unsigned int frame_num;
+ unsigned int poc_lsb;
+ unsigned int gop_pos;
+ unsigned int idr_pic_id;
+
+ /* Whether the *previous* encoded frame was an IDR -- mpp's
+ * setup_vepu510_anti_smear() keys smear_opt_cfg.stated_mode off both
+ * the current slice type AND the previous frame's, see rkvenc-regs.h.
+ * Zeroed by rkvenc_h264_start()'s memset; doesn't matter for frame 0
+ * (always IDR regardless).
+ */
+ bool last_frame_was_idr;
+
+ /* Software-synthesized SPS/PPS, emitted (Annex-B) ahead of the first
+ * hardware slice payload of each IDR access unit; see rkvenc-h264.c.
+ */
+ u8 sps_pps_nal[64];
+ size_t sps_pps_len;
+
+ /* VEPU510 reconstructs/references frames only through three internal
+ * scratch regions per frame slot — an FBC-compressed pixel buffer
+ * (header+body), a "thumbnail" downsample buffer (dspw/dspr_addr),
+ * and a "smear" buffer (adr_smear_wr/rd) — all three are written by
+ * hardware on *every* frame regardless of which encoder features are
+ * in use (confirmed from mpp's setup_vepu510_recn_refr(), which sets
+ * all three unconditionally whenever a current/reference buffer slot
+ * exists at all). Leaving any of them at a NULL pointer causes a
+ * continuous IOMMU write-fault storm at iova 0x0, not a clean error.
+ *
+ * This driver owns a 2-deep ping-pong pool of driver-private DMA
+ * buffers: buf[frame_num % 2] is this frame's write target,
+ * buf[(frame_num + 1) % 2] is the previous frame's read reference —
+ * EXCEPT on the very first frame of a session (frame_num == 0), where
+ * there is no previously-written reference at all: mpp's real
+ * hal_h264e_vepu510.c setup_vepu510_recn_refr() draws the "read" side
+ * from a `refr` ping-pong slot that equals the `curr` (write) slot
+ * index for frame 0, i.e. real hardware gets read == write, the same
+ * buffer aliased both ways, not a second never-written allocation.
+ * See rkvenc_h264_run()'s read_idx computation.
+ *
+ * Each buffer is one dma_alloc_coherent() allocation split into three
+ * consecutive regions [0, pixel_buf_size), [pixel_buf_size,
+ * +thumb_buf_size), [+thumb_buf_size, +smear_buf_size). Sizes are
+ * mpp's exact setup_vepu510_prepare_recn_bufs() formulas, not a guess.
+ */
+ void *recn_buf_cpu[2];
+ dma_addr_t recn_buf_dma[2];
+ size_t pixel_buf_size; /* FBC header + body */
+ size_t pixel_hdr_size; /* FBC header portion of pixel_buf_size */
+ size_t thumb_buf_size;
+ size_t smear_buf_size;
+
+ /* Motion-detection-info write buffer (meiw_addr). Real hardware
+ * writes here even though nothing in the userspace HAL documents the
+ * feature as required for a plain encode — board bring-up found a
+ * real successful encode always has a real address here, so this
+ * driver allocates one rather than assuming 0 is safe. Content is
+ * never read back (v1 doesn't expose motion vectors to userspace).
+ */
+ void *meiw_buf_cpu;
+ dma_addr_t meiw_buf_dma;
+};
+
+struct rkvenc_ctx {
+ struct v4l2_fh fh;
+ struct rkvenc_dev *dev;
+
+ struct v4l2_pix_format src_fmt; /* OUTPUT: raw NV12 */
+ struct v4l2_pix_format coded_fmt; /* CAPTURE: H264/HEVC */
+ const struct rkvenc_coded_fmt_desc *coded_desc;
+
+ struct v4l2_ctrl_handler ctrl_handler;
+
+ union {
+ struct rkvenc_h264_ctx h264;
+ };
+};
+
+static inline struct rkvenc_ctx *fh_to_rkvenc_ctx(struct v4l2_fh *fh)
+{
+ return container_of(fh, struct rkvenc_ctx, fh);
+}
+
+static inline u32 rkvenc_read(struct rkvenc_dev *dev, u32 reg)
+{
+ return readl(dev->regs + reg);
+}
+
+static inline void rkvenc_write(struct rkvenc_dev *dev, u32 reg, u32 val)
+{
+ writel(val, dev->regs + reg);
+}
+
+static inline void rkvenc_write_relaxed(struct rkvenc_dev *dev, u32 reg, u32 val)
+{
+ writel_relaxed(val, dev->regs + reg);
+}
+
+static inline struct vb2_v4l2_buffer *rkvenc_get_src_buf(struct rkvenc_ctx *ctx)
+{
+ return v4l2_m2m_next_src_buf(ctx->fh.m2m_ctx);
+}
+
+static inline struct vb2_v4l2_buffer *rkvenc_get_dst_buf(struct rkvenc_ctx *ctx)
+{
+ return v4l2_m2m_next_dst_buf(ctx->fh.m2m_ctx);
+}
+
+extern const struct rkvenc_coded_fmt_desc rkvenc_h264_fmt_desc;
+
+void rkvenc_job_finish(struct rkvenc_dev *dev, struct rkvenc_ctx *ctx,
+ enum vb2_buffer_state state);
+u32 rkvenc_calc_wdg(struct rkvenc_dev *dev, unsigned int width,
+ unsigned int height);
+
+#endif /* RKVENC_H_ */
--
2.43.0
^ permalink raw reply related [flat|nested] 7+ messages in thread