[RFC PATCH 2/5] media: iris: Update the CAPTURE stride and size on S_FMT(OUTPUT)

[Christian Cortner <ccortner@gmail.com>]

Setting the OUTPUT format also updates the width and height of the
CAPTURE format, but bytesperline and sizeimage keep their old values,
those of the 320x240 default on a new instance. For a 640x360 stream,
VIDIOC_G_FMT(CAPTURE) then reports 640x384 NV12 with a sizeimage of
147456 bytes. VIDIOC_REQBUFS(CAPTURE) allocates buffers of that size,
and iris_vb2_buf_prepare() rejects every one of them with -EINVAL,
because it checks them against the size computed from the new width
and height.

Move the per-format rules S_FMT(CAPTURE) uses into a helper and call it
for both queues, so that S_FMT(OUTPUT) leaves the CAPTURE format exactly
as S_FMT(CAPTURE) would set it for the same size.

Fixes: 8aadfd445373 ("media: iris: Update CAPTURE format info based on OUTPUT format")
Cc: stable@vger.kernel.org
Assisted-by: LLM sparse smatch
---
Reference only, not for merging (no Signed-off-by); see the cover
letter.

 drivers/media/platform/qcom/iris/iris_vdec.c | 61 +++++++++++---------
 1 file changed, 33 insertions(+), 28 deletions(-)

diff --git a/drivers/media/platform/qcom/iris/iris_vdec.c b/drivers/media/platform/qcom/iris/iris_vdec.c
index 0dd33ff447bb..472e8e50830d 100644
--- a/drivers/media/platform/qcom/iris/iris_vdec.c
+++ b/drivers/media/platform/qcom/iris/iris_vdec.c
@@ -212,6 +212,35 @@ int iris_vdec_try_fmt(struct iris_inst *inst, struct v4l2_format *f)
 	return 0;
 }
 
+static void iris_vdec_set_capture_size(struct iris_inst *inst, u32 width,
+				       u32 height)
+{
+	struct v4l2_pix_format_mplane *pix_mp = &inst->fmt_dst->fmt.pix_mp;
+	struct iris_buffers *bufs = &inst->buffers[BUF_OUTPUT];
+
+	switch (pix_mp->pixelformat) {
+	case V4L2_PIX_FMT_P010:
+		pix_mp->width = ALIGN(width, 128);
+		pix_mp->height = ALIGN(height, 32);
+		pix_mp->plane_fmt[0].bytesperline = ALIGN(width * 2, 256);
+		break;
+	case V4L2_PIX_FMT_QC10C:
+		pix_mp->width = roundup(width, 192);
+		pix_mp->height = ALIGN(height, 16);
+		pix_mp->plane_fmt[0].bytesperline = ALIGN(width * 4 / 3, 256);
+		break;
+	case V4L2_PIX_FMT_NV12:
+	case V4L2_PIX_FMT_QC08C:
+		pix_mp->width = ALIGN(width, 128);
+		pix_mp->height = ALIGN(height, 32);
+		pix_mp->plane_fmt[0].bytesperline = ALIGN(width, 128);
+		break;
+	}
+	pix_mp->plane_fmt[0].sizeimage = iris_get_buffer_size(inst, BUF_OUTPUT);
+	bufs->min_count = iris_vpu_buf_count(inst, BUF_OUTPUT);
+	bufs->size = pix_mp->plane_fmt[0].sizeimage;
+}
+
 int iris_vdec_s_fmt(struct iris_inst *inst, struct v4l2_format *f)
 {
 	struct v4l2_format *fmt, *output_fmt;
@@ -263,10 +292,8 @@ int iris_vdec_s_fmt(struct iris_inst *inst, struct v4l2_format *f)
 		output_fmt->fmt.pix_mp.quantization = f->fmt.pix_mp.quantization;
 
 		/* Update capture format based on new ip w/h */
-		output_fmt->fmt.pix_mp.width = ALIGN(f->fmt.pix_mp.width, 128);
-		output_fmt->fmt.pix_mp.height = ALIGN(f->fmt.pix_mp.height, 32);
-		inst->buffers[BUF_OUTPUT].min_count = iris_vpu_buf_count(inst, BUF_OUTPUT);
-		inst->buffers[BUF_OUTPUT].size = iris_get_buffer_size(inst, BUF_OUTPUT);
+		iris_vdec_set_capture_size(inst, f->fmt.pix_mp.width,
+					   f->fmt.pix_mp.height);
 
 		inst->crop.left = 0;
 		inst->crop.top = 0;
@@ -281,30 +308,8 @@ int iris_vdec_s_fmt(struct iris_inst *inst, struct v4l2_format *f)
 		fmt->type = V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE;
 		fmt->fmt.pix_mp.pixelformat = f->fmt.pix_mp.pixelformat;
 		fmt->fmt.pix_mp.num_planes = 1;
-		switch (f->fmt.pix_mp.pixelformat) {
-		case V4L2_PIX_FMT_P010:
-			fmt->fmt.pix_mp.width = ALIGN(f->fmt.pix_mp.width, 128);
-			fmt->fmt.pix_mp.height = ALIGN(f->fmt.pix_mp.height, 32);
-			fmt->fmt.pix_mp.plane_fmt[0].bytesperline =
-				ALIGN(f->fmt.pix_mp.width * 2, 256);
-			break;
-		case V4L2_PIX_FMT_QC10C:
-			fmt->fmt.pix_mp.width = roundup(f->fmt.pix_mp.width, 192);
-			fmt->fmt.pix_mp.height = ALIGN(f->fmt.pix_mp.height, 16);
-			fmt->fmt.pix_mp.plane_fmt[0].bytesperline =
-				ALIGN(f->fmt.pix_mp.width * 4 / 3, 256);
-			break;
-		case V4L2_PIX_FMT_NV12:
-		case V4L2_PIX_FMT_QC08C:
-			fmt->fmt.pix_mp.width = ALIGN(f->fmt.pix_mp.width, 128);
-			fmt->fmt.pix_mp.height = ALIGN(f->fmt.pix_mp.height, 32);
-			fmt->fmt.pix_mp.plane_fmt[0].bytesperline =
-				ALIGN(f->fmt.pix_mp.width, 128);
-			break;
-		}
-		fmt->fmt.pix_mp.plane_fmt[0].sizeimage = iris_get_buffer_size(inst, BUF_OUTPUT);
-		inst->buffers[BUF_OUTPUT].min_count = iris_vpu_buf_count(inst, BUF_OUTPUT);
-		inst->buffers[BUF_OUTPUT].size = fmt->fmt.pix_mp.plane_fmt[0].sizeimage;
+		iris_vdec_set_capture_size(inst, f->fmt.pix_mp.width,
+					   f->fmt.pix_mp.height);
 
 		inst->crop.top = 0;
 		inst->crop.left = 0;
-- 
2.56.0