[PATCH v4 17/19] media: intel/ipu6: support line-based metadata capture support

[Sakari Ailus <sakari.ailus@linux.intel.com>]

From: Bingbu Cao <bingbu.cao@intel.com>

Some camera sensor can output the embedded data in specific data type.
This patch adds the support for metadata capture in IPU6 ISYS driver.

Signed-off-by: Hongju Wang <hongju.wang@intel.com>
Signed-off-by: Bingbu Cao <bingbu.cao@intel.com>
Signed-off-by: Sakari Ailus <sakari.ailus@linux.intel.com>
---
 drivers/media/pci/intel/ipu6/ipu6-isys-csi2.c |   5 +
 .../media/pci/intel/ipu6/ipu6-isys-queue.c    |  33 +--
 .../media/pci/intel/ipu6/ipu6-isys-subdev.c   |   5 +
 .../media/pci/intel/ipu6/ipu6-isys-video.c    | 203 +++++++++++++++---
 .../media/pci/intel/ipu6/ipu6-isys-video.h    |   7 +-
 5 files changed, 212 insertions(+), 41 deletions(-)

diff --git a/drivers/media/pci/intel/ipu6/ipu6-isys-csi2.c b/drivers/media/pci/intel/ipu6/ipu6-isys-csi2.c
index ad0616b96d3e..5f4d4e57d62c 100644
--- a/drivers/media/pci/intel/ipu6/ipu6-isys-csi2.c
+++ b/drivers/media/pci/intel/ipu6/ipu6-isys-csi2.c
@@ -46,6 +46,11 @@ static const u32 csi2_supported_codes[] = {
 	MEDIA_BUS_FMT_SGBRG8_1X8,
 	MEDIA_BUS_FMT_SGRBG8_1X8,
 	MEDIA_BUS_FMT_SRGGB8_1X8,
+	MEDIA_BUS_FMT_META_8,
+	MEDIA_BUS_FMT_META_10,
+	MEDIA_BUS_FMT_META_12,
+	MEDIA_BUS_FMT_META_16,
+	MEDIA_BUS_FMT_META_24,
 	0
 };
 
diff --git a/drivers/media/pci/intel/ipu6/ipu6-isys-queue.c b/drivers/media/pci/intel/ipu6/ipu6-isys-queue.c
index d332b4b3a0f6..6db60d9af3d2 100644
--- a/drivers/media/pci/intel/ipu6/ipu6-isys-queue.c
+++ b/drivers/media/pci/intel/ipu6/ipu6-isys-queue.c
@@ -35,11 +35,14 @@ static int queue_setup(struct vb2_queue *q, unsigned int *num_buffers,
 	/* num_planes == 0: we're being called through VIDIOC_REQBUFS */
 	if (!*num_planes) {
 		use_fmt = true;
-		*num_planes = av->mpix.num_planes;
+		if (av->vfmt.type == V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE)
+			*num_planes = av->vfmt.fmt.pix_mp.num_planes;
+		else if (av->vfmt.type == V4L2_BUF_TYPE_META_CAPTURE)
+			*num_planes = 1;
 	}
 
 	for (i = 0; i < *num_planes; i++) {
-		size = av->mpix.plane_fmt[i].sizeimage;
+		size = ipu6_get_data_size(&av->vfmt, i);
 		if (use_fmt) {
 			sizes[i] = size;
 		} else if (sizes[i] < size) {
@@ -59,16 +62,17 @@ static int ipu6_isys_buf_prepare(struct vb2_buffer *vb)
 	struct ipu6_isys_queue *aq = vb2_queue_to_isys_queue(vb->vb2_queue);
 	struct ipu6_isys_video *av = ipu6_isys_queue_to_video(aq);
 	struct device *dev = &av->isys->adev->auxdev.dev;
+	u32 bytesperline = ipu6_get_bytes_per_line(&av->vfmt);
+	u32 height = ipu6_get_frame_height(&av->vfmt);
+	u32 size = ipu6_get_data_size(&av->vfmt, 0);
 
 	dev_dbg(dev, "buffer: %s: configured size %u, buffer size %lu\n",
-		av->vdev.name, av->mpix.plane_fmt[0].sizeimage,
-		vb2_plane_size(vb, 0));
+		av->vdev.name, size, vb2_plane_size(vb, 0));
 
-	if (av->mpix.plane_fmt[0].sizeimage > vb2_plane_size(vb, 0))
+	if (size > vb2_plane_size(vb, 0))
 		return -EINVAL;
 
-	vb2_set_plane_payload(vb, 0, av->mpix.plane_fmt[0].bytesperline *
-			      av->mpix.height);
+	vb2_set_plane_payload(vb, 0, bytesperline * height);
 	vb->planes[0].data_offset = 0;
 
 	return 0;
@@ -437,11 +441,12 @@ static int ipu6_isys_link_fmt_validate(struct ipu6_isys_queue *aq)
 		return ret;
 	}
 
-	if (format.width != av->mpix.width ||
-	    format.height != av->mpix.height) {
-		dev_dbg(dev, "wrong width or height %ux%u (%ux%u expected)\n",
-			av->mpix.width, av->mpix.height,
-			format.width, format.height);
+	if (format.width != ipu6_get_frame_width(&av->vfmt) ||
+	    format.height != ipu6_get_frame_height(&av->vfmt)) {
+		dev_err(dev, "wrong width or height %ux%u (%ux%u expected)\n",
+			ipu6_get_frame_width(&av->vfmt),
+			ipu6_get_frame_height(&av->vfmt), format.width,
+			format.height);
 		return -EINVAL;
 	}
 
@@ -525,8 +530,8 @@ static int start_streaming(struct vb2_queue *q, unsigned int count)
 	int nr_queues, ret;
 
 	dev_dbg(dev, "stream: %s: width %u, height %u, css pixelformat %u\n",
-		av->vdev.name, av->mpix.width, av->mpix.height,
-		av->pfmt->css_pixelformat);
+		av->vdev.name, ipu6_get_frame_width(&av->vfmt),
+		ipu6_get_frame_height(&av->vfmt), av->pfmt->css_pixelformat);
 
 	ret = ipu6_isys_setup_video(av, &source_entity, &nr_queues);
 	if (ret < 0) {
diff --git a/drivers/media/pci/intel/ipu6/ipu6-isys-subdev.c b/drivers/media/pci/intel/ipu6/ipu6-isys-subdev.c
index 7b85cc9929a5..5ef5a9b44be3 100644
--- a/drivers/media/pci/intel/ipu6/ipu6-isys-subdev.c
+++ b/drivers/media/pci/intel/ipu6/ipu6-isys-subdev.c
@@ -20,10 +20,12 @@ unsigned int ipu6_isys_mbus_code_to_bpp(u32 code)
 {
 	switch (code) {
 	case MEDIA_BUS_FMT_RGB888_1X24:
+	case MEDIA_BUS_FMT_META_24:
 		return 24;
 	case MEDIA_BUS_FMT_RGB565_1X16:
 	case MEDIA_BUS_FMT_UYVY8_1X16:
 	case MEDIA_BUS_FMT_YUYV8_1X16:
+	case MEDIA_BUS_FMT_META_16:
 		return 16;
 	case MEDIA_BUS_FMT_SBGGR16_1X16:
 	case MEDIA_BUS_FMT_SGBRG16_1X16:
@@ -34,16 +36,19 @@ unsigned int ipu6_isys_mbus_code_to_bpp(u32 code)
 	case MEDIA_BUS_FMT_SGBRG12_1X12:
 	case MEDIA_BUS_FMT_SGRBG12_1X12:
 	case MEDIA_BUS_FMT_SRGGB12_1X12:
+	case MEDIA_BUS_FMT_META_12:
 		return 12;
 	case MEDIA_BUS_FMT_SBGGR10_1X10:
 	case MEDIA_BUS_FMT_SGBRG10_1X10:
 	case MEDIA_BUS_FMT_SGRBG10_1X10:
 	case MEDIA_BUS_FMT_SRGGB10_1X10:
+	case MEDIA_BUS_FMT_META_10:
 		return 10;
 	case MEDIA_BUS_FMT_SBGGR8_1X8:
 	case MEDIA_BUS_FMT_SGBRG8_1X8:
 	case MEDIA_BUS_FMT_SGRBG8_1X8:
 	case MEDIA_BUS_FMT_SRGGB8_1X8:
+	case MEDIA_BUS_FMT_META_8:
 		return 8;
 	default:
 		WARN_ON(1);
diff --git a/drivers/media/pci/intel/ipu6/ipu6-isys-video.c b/drivers/media/pci/intel/ipu6/ipu6-isys-video.c
index f40d7e93ebe2..457085177327 100644
--- a/drivers/media/pci/intel/ipu6/ipu6-isys-video.c
+++ b/drivers/media/pci/intel/ipu6/ipu6-isys-video.c
@@ -93,6 +93,14 @@ const struct ipu6_isys_pixelformat ipu6_isys_pfmts[] = {
 	  IPU6_FW_ISYS_FRAME_FORMAT_RGB565 },
 	{ V4L2_PIX_FMT_BGR24, 24, 24, MEDIA_BUS_FMT_RGB888_1X24,
 	  IPU6_FW_ISYS_FRAME_FORMAT_RGBA888 },
+	{ V4L2_META_FMT_GENERIC_8, 8, 8, MEDIA_BUS_FMT_META_8,
+	  IPU6_FW_ISYS_FRAME_FORMAT_RAW8 },
+	{ V4L2_META_FMT_GENERIC_CSI2_10, 10, 10, MEDIA_BUS_FMT_META_10,
+	  IPU6_FW_ISYS_FRAME_FORMAT_RAW10 },
+	{ V4L2_META_FMT_GENERIC_CSI2_12, 12, 12, MEDIA_BUS_FMT_META_12,
+	  IPU6_FW_ISYS_FRAME_FORMAT_RAW12 },
+	{ V4L2_META_FMT_GENERIC_CSI2_16, 16, 16, MEDIA_BUS_FMT_META_16,
+	  IPU6_FW_ISYS_FRAME_FORMAT_RAW16 },
 };
 
 static int video_open(struct file *file)
@@ -184,12 +192,12 @@ static int ipu6_isys_vidioc_enum_framesizes(struct file *file, void *fh,
 	return 0;
 }
 
-static int vidioc_g_fmt_vid_cap_mplane(struct file *file, void *fh,
-				       struct v4l2_format *fmt)
+static int vidioc_get_format(struct file *file, void *fh,
+			     struct v4l2_format *fmt)
 {
 	struct ipu6_isys_video *av = video_drvdata(file);
 
-	fmt->fmt.pix_mp = av->mpix;
+	*fmt = av->vfmt;
 
 	return 0;
 }
@@ -247,30 +255,114 @@ ipu6_isys_video_try_fmt_vid_mplane(struct ipu6_isys_video *av,
 	return pfmt;
 }
 
-static int vidioc_s_fmt_vid_cap_mplane(struct file *file, void *fh,
-				       struct v4l2_format *f)
+static const struct ipu6_isys_pixelformat *
+ipu6_isys_video_try_fmt_meta(struct ipu6_isys_video *av,
+			     struct v4l2_meta_format *meta)
+{
+	const struct ipu6_isys_pixelformat *pfmt =
+		ipu6_isys_get_pixelformat(meta->dataformat);
+
+	memset(&av->vfmt, 0, sizeof(av->vfmt));
+	av->vfmt.type = V4L2_BUF_TYPE_META_CAPTURE;
+	av->pfmt = pfmt;
+
+	meta->dataformat = pfmt->pixelformat;
+	meta->width = clamp(meta->width, IPU6_ISYS_MIN_WIDTH,
+			    IPU6_ISYS_MAX_WIDTH);
+	meta->height = clamp(meta->height, IPU6_ISYS_MIN_HEIGHT,
+			     IPU6_ISYS_MAX_HEIGHT);
+
+	if (pfmt->bpp != pfmt->bpp_packed)
+		meta->bytesperline = meta->width *
+				     DIV_ROUND_UP(pfmt->bpp, BITS_PER_BYTE);
+	else
+		meta->bytesperline =
+			DIV_ROUND_UP(meta->width * pfmt->bpp, BITS_PER_BYTE);
+
+	meta->bytesperline = ALIGN(meta->bytesperline, av->isys->line_align);
+	meta->buffersize =
+		max(max(meta->buffersize, meta->bytesperline * meta->height +
+			max(meta->bytesperline,
+			    av->isys->pdata->ipdata->isys_dma_overshoot)), 1U);
+
+	return pfmt;
+}
+
+static const struct ipu6_isys_pixelformat *
+ipu6_isys_video_try_fmt(struct ipu6_isys_video *av, struct v4l2_format *f)
+{
+	if (f->type == V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE)
+		return ipu6_isys_video_try_fmt_vid_mplane(av, &f->fmt.pix_mp);
+	else if (f->type == V4L2_BUF_TYPE_META_CAPTURE)
+		return ipu6_isys_video_try_fmt_meta(av, &f->fmt.meta);
+	else
+		return &ipu6_isys_pfmts[0];
+}
+
+static int vidioc_set_format(struct file *file, void *fh,
+			     struct v4l2_format *f)
 {
 	struct ipu6_isys_video *av = video_drvdata(file);
 
 	if (av->aq.vbq.streaming)
 		return -EBUSY;
 
-	av->pfmt = ipu6_isys_video_try_fmt_vid_mplane(av, &f->fmt.pix_mp);
-	av->mpix = f->fmt.pix_mp;
+	if (f->type != V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE &&
+	    f->type != V4L2_BUF_TYPE_META_CAPTURE)
+		return -EINVAL;
+
+	av->pfmt = ipu6_isys_video_try_fmt(av, f);
+	av->vfmt = *f;
 
 	return 0;
 }
 
-static int vidioc_try_fmt_vid_cap_mplane(struct file *file, void *fh,
-					 struct v4l2_format *f)
+static int vidioc_try_format(struct file *file, void *fh,
+			     struct v4l2_format *f)
 {
 	struct ipu6_isys_video *av = video_drvdata(file);
 
-	ipu6_isys_video_try_fmt_vid_mplane(av, &f->fmt.pix_mp);
+	if (f->type != V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE &&
+	    f->type != V4L2_BUF_TYPE_META_CAPTURE)
+		return -EINVAL;
+
+	ipu6_isys_video_try_fmt(av, f);
 
 	return 0;
 }
 
+static int vidioc_request_qbufs(struct file *file, void *priv,
+				struct v4l2_requestbuffers *p)
+{
+	struct ipu6_isys_video *av = video_drvdata(file);
+	int ret;
+
+	av->aq.vbq.is_multiplanar = V4L2_TYPE_IS_MULTIPLANAR(p->type);
+	av->aq.vbq.is_output = V4L2_TYPE_IS_OUTPUT(p->type);
+
+	ret = vb2_queue_change_type(&av->aq.vbq, p->type);
+	if (ret)
+		return ret;
+
+	return vb2_ioctl_reqbufs(file, priv, p);
+}
+
+static int vidioc_create_bufs(struct file *file, void *priv,
+			      struct v4l2_create_buffers *p)
+{
+	struct ipu6_isys_video *av = video_drvdata(file);
+	int ret;
+
+	av->aq.vbq.is_multiplanar = V4L2_TYPE_IS_MULTIPLANAR(p->format.type);
+	av->aq.vbq.is_output = V4L2_TYPE_IS_OUTPUT(p->format.type);
+
+	ret = vb2_queue_change_type(&av->aq.vbq, p->format.type);
+	if (ret)
+		return ret;
+
+	return vb2_ioctl_create_bufs(file, priv, p);
+}
+
 static int link_validate(struct media_link *link)
 {
 	struct ipu6_isys_video *av =
@@ -306,11 +398,13 @@ static int link_validate(struct media_link *link)
 		goto unlock;
 	}
 
-	if (s_fmt->width != av->mpix.width ||
-	    s_fmt->height != av->mpix.height || s_fmt->code != av->pfmt->code) {
+	if (s_fmt->width != ipu6_get_frame_width(&av->vfmt) ||
+	    s_fmt->height != ipu6_get_frame_height(&av->vfmt) ||
+	    s_fmt->code != av->pfmt->code) {
 		dev_dbg(dev, "format mismatch %dx%d,%x != %dx%d,%x\n",
 			s_fmt->width, s_fmt->height, s_fmt->code,
-			av->mpix.width, av->mpix.height, av->pfmt->code);
+			ipu6_get_frame_width(&av->vfmt),
+			ipu6_get_frame_height(&av->vfmt), av->pfmt->code);
 		goto unlock;
 	}
 
@@ -394,10 +488,10 @@ static int ipu6_isys_fw_pin_cfg(struct ipu6_isys_video *av,
 
 	output_pin = &cfg->output_pins[output_pins];
 	output_pin->input_pin_id = input_pins;
-	output_pin->output_res.width = av->mpix.width;
-	output_pin->output_res.height = av->mpix.height;
+	output_pin->output_res.width = ipu6_get_frame_width(&av->vfmt);
+	output_pin->output_res.height = ipu6_get_frame_height(&av->vfmt);
 
-	output_pin->stride = av->mpix.plane_fmt[0].bytesperline;
+	output_pin->stride = ipu6_get_bytes_per_line(&av->vfmt);
 	if (av->pfmt->bpp != av->pfmt->bpp_packed)
 		output_pin->pt = IPU6_FW_ISYS_PIN_TYPE_RAW_SOC;
 	else
@@ -664,8 +758,8 @@ void ipu6_isys_configure_stream_watermark(struct ipu6_isys_video *av,
 
 	esd = media_entity_to_v4l2_subdev(av->stream->source_entity);
 
-	av->watermark.width = av->mpix.width;
-	av->watermark.height = av->mpix.height;
+	av->watermark.width = ipu6_get_frame_width(&av->vfmt);
+	av->watermark.height = ipu6_get_frame_height(&av->vfmt);
 	av->watermark.sram_gran_shift = isys->pdata->ipdata->sram_gran_shift;
 	av->watermark.sram_gran_size = isys->pdata->ipdata->sram_gran_size;
 
@@ -990,11 +1084,15 @@ static const struct v4l2_ioctl_ops ioctl_ops_mplane = {
 	.vidioc_querycap = ipu6_isys_vidioc_querycap,
 	.vidioc_enum_fmt_vid_cap = ipu6_isys_vidioc_enum_fmt,
 	.vidioc_enum_framesizes = ipu6_isys_vidioc_enum_framesizes,
-	.vidioc_g_fmt_vid_cap_mplane = vidioc_g_fmt_vid_cap_mplane,
-	.vidioc_s_fmt_vid_cap_mplane = vidioc_s_fmt_vid_cap_mplane,
-	.vidioc_try_fmt_vid_cap_mplane = vidioc_try_fmt_vid_cap_mplane,
-	.vidioc_reqbufs = vb2_ioctl_reqbufs,
-	.vidioc_create_bufs = vb2_ioctl_create_bufs,
+	.vidioc_g_fmt_vid_cap_mplane = vidioc_get_format,
+	.vidioc_s_fmt_vid_cap_mplane = vidioc_set_format,
+	.vidioc_try_fmt_vid_cap_mplane = vidioc_try_format,
+	.vidioc_enum_fmt_meta_cap = ipu6_isys_vidioc_enum_fmt,
+	.vidioc_g_fmt_meta_cap = vidioc_get_format,
+	.vidioc_s_fmt_meta_cap = vidioc_set_format,
+	.vidioc_try_fmt_meta_cap = vidioc_try_format,
+	.vidioc_reqbufs = vidioc_request_qbufs,
+	.vidioc_create_bufs = vidioc_create_bufs,
 	.vidioc_prepare_buf = vb2_ioctl_prepare_buf,
 	.vidioc_querybuf = vb2_ioctl_querybuf,
 	.vidioc_qbuf = vb2_ioctl_qbuf,
@@ -1196,7 +1294,8 @@ int ipu6_isys_video_init(struct ipu6_isys_video *av)
 
 	mutex_init(&av->mutex);
 	av->vdev.device_caps = V4L2_CAP_STREAMING | V4L2_CAP_IO_MC |
-			       V4L2_CAP_VIDEO_CAPTURE_MPLANE;
+			       V4L2_CAP_VIDEO_CAPTURE_MPLANE |
+			       V4L2_CAP_META_CAPTURE;
 	av->vdev.vfl_dir = VFL_DIR_RX;
 
 	ret = ipu6_isys_queue_init(&av->aq);
@@ -1217,8 +1316,8 @@ int ipu6_isys_video_init(struct ipu6_isys_video *av)
 	av->vdev.queue = &av->aq.vbq;
 	av->vdev.lock = &av->mutex;
 
-	ipu6_isys_video_try_fmt_vid_mplane(av, &format.fmt.pix_mp);
-	av->mpix = format.fmt.pix_mp;
+	ipu6_isys_video_try_fmt(av, &format);
+	av->vfmt = format;
 
 	set_bit(V4L2_FL_USES_V4L2_FH, &av->vdev.flags);
 	video_set_drvdata(&av->vdev, av);
@@ -1248,3 +1347,55 @@ void ipu6_isys_video_cleanup(struct ipu6_isys_video *av)
 	media_entity_cleanup(&av->vdev.entity);
 	mutex_destroy(&av->mutex);
 }
+
+u32 ipu6_get_data_size(struct v4l2_format *vfmt, int plane)
+{
+	if (vfmt->type == V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE)
+		return vfmt->fmt.pix_mp.plane_fmt[plane].sizeimage;
+
+	if (vfmt->type == V4L2_BUF_TYPE_META_CAPTURE)
+		return vfmt->fmt.meta.buffersize;
+
+	WARN_ON_ONCE(1);
+
+	return 0;
+}
+
+u32 ipu6_get_bytes_per_line(struct v4l2_format *vfmt)
+{
+	if (vfmt->type == V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE)
+		return vfmt->fmt.pix_mp.plane_fmt[0].bytesperline;
+
+	if (vfmt->type == V4L2_BUF_TYPE_META_CAPTURE)
+		return vfmt->fmt.meta.bytesperline;
+
+	WARN_ON_ONCE(1);
+
+	return 0;
+}
+
+u32 ipu6_get_frame_width(struct v4l2_format *vfmt)
+{
+	if (vfmt->type == V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE)
+		return vfmt->fmt.pix_mp.width;
+
+	if (vfmt->type == V4L2_BUF_TYPE_META_CAPTURE)
+		return vfmt->fmt.meta.width;
+
+	WARN_ON_ONCE(1);
+
+	return 0;
+}
+
+u32 ipu6_get_frame_height(struct v4l2_format *vfmt)
+{
+	if (vfmt->type == V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE)
+		return vfmt->fmt.pix_mp.height;
+
+	if (vfmt->type == V4L2_BUF_TYPE_META_CAPTURE)
+		return vfmt->fmt.meta.height;
+
+	WARN_ON_ONCE(1);
+
+	return 0;
+}
diff --git a/drivers/media/pci/intel/ipu6/ipu6-isys-video.h b/drivers/media/pci/intel/ipu6/ipu6-isys-video.h
index 43bc3f73c070..b9cc7f691c7e 100644
--- a/drivers/media/pci/intel/ipu6/ipu6-isys-video.h
+++ b/drivers/media/pci/intel/ipu6/ipu6-isys-video.h
@@ -90,7 +90,7 @@ struct ipu6_isys_video {
 	struct mutex mutex;
 	struct media_pad pad;
 	struct video_device vdev;
-	struct v4l2_pix_format_mplane mpix;
+	struct v4l2_format vfmt;
 	const struct ipu6_isys_pixelformat *pfmt;
 	struct ipu6_isys *isys;
 	struct ipu6_isys_csi2 *csi2;
@@ -129,4 +129,9 @@ void ipu6_isys_configure_stream_watermark(struct ipu6_isys_video *av,
 					  bool state);
 void ipu6_isys_update_stream_watermark(struct ipu6_isys_video *av, bool state);
 
+u32 ipu6_get_data_size(struct v4l2_format *vfmt, int plane);
+u32 ipu6_get_bytes_per_line(struct v4l2_format *vfmt);
+u32 ipu6_get_frame_width(struct v4l2_format *vfmt);
+u32 ipu6_get_frame_height(struct v4l2_format *vfmt);
+
 #endif /* IPU6_ISYS_VIDEO_H */
-- 
2.39.2