[RFC PATCH v3 01/11] [media] vimc: sen: Integrate the tpg on the sensor

[RFC PATCH v3 01/11] [media] vimc: sen: Integrate the tpg on the sensor

[Helen Koike]

Initialize the test pattern generator on the sensor
Generate a colored bar image instead of a grey one

Signed-off-by: Helen Koike <helen.koike@collabora.com>

---

Changes in v3:
[media] vimc: sen: Integrate the tpg on the sensor
	- Declare frame_size as a local variable
	- Set tpg frame format before starting kthread
	- s_stream(sd, 1): return 0 if stream is already enabled
	- s_stream(sd, 0): return 0 if stream is already disabled
	- s_stream: propagate error from kthread_stop
	- coding style when calling tpg_s_bytesperline
	- s/vimc_thread_sen/vimc_sen_tpg_thread
	- fix multiline comment

Changes in v2:
[media] vimc: sen: Integrate the tpg on the sensor
	- Fix include location
	- Select V4L2_TPG in Kconfig
	- configure tpg on streamon only
	- rm BUG_ON
	- coding style
	- remove V4L2_FIELD_ALTERNATE from tpg_s_field
	- remove V4L2_STD_PAL from tpg_fill_plane_buffer


---
 drivers/media/platform/vimc/Kconfig       |  1 +
 drivers/media/platform/vimc/vimc-sensor.c | 64 ++++++++++++++++++++++++-------
 2 files changed, 52 insertions(+), 13 deletions(-)

diff --git a/drivers/media/platform/vimc/Kconfig b/drivers/media/platform/vimc/Kconfig
index a18f635..71c9fe7 100644
--- a/drivers/media/platform/vimc/Kconfig
+++ b/drivers/media/platform/vimc/Kconfig
@@ -2,6 +2,7 @@ config VIDEO_VIMC
 	tristate "Virtual Media Controller Driver (VIMC)"
 	depends on VIDEO_DEV && VIDEO_V4L2 && VIDEO_V4L2_SUBDEV_API
 	select VIDEOBUF2_VMALLOC
+	select VIDEO_V4L2_TPG
 	default n
 	---help---
 	  Skeleton driver for Virtual Media Controller
diff --git a/drivers/media/platform/vimc/vimc-sensor.c b/drivers/media/platform/vimc/vimc-sensor.c
index 591f6a4..2e83487 100644
--- a/drivers/media/platform/vimc/vimc-sensor.c
+++ b/drivers/media/platform/vimc/vimc-sensor.c
@@ -20,17 +20,20 @@
 #include <linux/v4l2-mediabus.h>
 #include <linux/vmalloc.h>
 #include <media/v4l2-subdev.h>
+#include <media/v4l2-tpg.h>
 
 #include "vimc-sensor.h"
 
+#define VIMC_SEN_FRAME_MAX_WIDTH 4096
+
 struct vimc_sen_device {
 	struct vimc_ent_device ved;
 	struct v4l2_subdev sd;
+	struct tpg_data tpg;
 	struct task_struct *kthread_sen;
 	u8 *frame;
 	/* The active format */
 	struct v4l2_mbus_framefmt mbus_format;
-	int frame_size;
 };
 
 static int vimc_sen_enum_mbus_code(struct v4l2_subdev *sd,
@@ -84,6 +87,24 @@ static int vimc_sen_get_fmt(struct v4l2_subdev *sd,
 	return 0;
 }
 
+static void vimc_sen_tpg_s_format(struct vimc_sen_device *vsen)
+{
+	const struct vimc_pix_map *vpix =
+				vimc_pix_map_by_code(vsen->mbus_format.code);
+
+	tpg_reset_source(&vsen->tpg, vsen->mbus_format.width,
+			 vsen->mbus_format.height, vsen->mbus_format.field);
+	tpg_s_bytesperline(&vsen->tpg, 0, vsen->mbus_format.width * vpix->bpp);
+	tpg_s_buf_height(&vsen->tpg, vsen->mbus_format.height);
+	tpg_s_fourcc(&vsen->tpg, vpix->pixelformat);
+	/* TODO: add support for V4L2_FIELD_ALTERNATE */
+	tpg_s_field(&vsen->tpg, vsen->mbus_format.field, false);
+	tpg_s_colorspace(&vsen->tpg, vsen->mbus_format.colorspace);
+	tpg_s_ycbcr_enc(&vsen->tpg, vsen->mbus_format.ycbcr_enc);
+	tpg_s_quantization(&vsen->tpg, vsen->mbus_format.quantization);
+	tpg_s_xfer_func(&vsen->tpg, vsen->mbus_format.xfer_func);
+}
+
 static const struct v4l2_subdev_pad_ops vimc_sen_pad_ops = {
 	.enum_mbus_code		= vimc_sen_enum_mbus_code,
 	.enum_frame_size	= vimc_sen_enum_frame_size,
@@ -97,7 +118,7 @@ static const struct media_entity_operations vimc_sen_mops = {
 	.link_validate = v4l2_subdev_link_validate,
 };
 
-static int vimc_thread_sen(void *data)
+static int vimc_sen_tpg_thread(void *data)
 {
 	struct vimc_sen_device *vsen = data;
 	unsigned int i;
@@ -110,7 +131,7 @@ static int vimc_thread_sen(void *data)
 		if (kthread_should_stop())
 			break;
 
-		memset(vsen->frame, 100, vsen->frame_size);
+		tpg_fill_plane_buffer(&vsen->tpg, 0, 0, vsen->frame);
 
 		/* Send the frame to all source pads */
 		for (i = 0; i < vsen->sd.entity.num_pads; i++)
@@ -132,26 +153,31 @@ static int vimc_sen_s_stream(struct v4l2_subdev *sd, int enable)
 
 	if (enable) {
 		const struct vimc_pix_map *vpix;
+		unsigned int frame_size;
 
 		if (vsen->kthread_sen)
-			return -EINVAL;
+			/* tpg is already executing */
+			return 0;
 
 		/* Calculate the frame size */
 		vpix = vimc_pix_map_by_code(vsen->mbus_format.code);
-		vsen->frame_size = vsen->mbus_format.width * vpix->bpp *
-				   vsen->mbus_format.height;
+		frame_size = vsen->mbus_format.width * vpix->bpp *
+			     vsen->mbus_format.height;
 
 		/*
 		 * Allocate the frame buffer. Use vmalloc to be able to
 		 * allocate a large amount of memory
 		 */
-		vsen->frame = vmalloc(vsen->frame_size);
+		vsen->frame = vmalloc(frame_size);
 		if (!vsen->frame)
 			return -ENOMEM;
 
+		/* configure the test pattern generator */
+		vimc_sen_tpg_s_format(vsen);
+
 		/* Initialize the image generator thread */
-		vsen->kthread_sen = kthread_run(vimc_thread_sen, vsen, "%s-sen",
-						vsen->sd.v4l2_dev->name);
+		vsen->kthread_sen = kthread_run(vimc_sen_tpg_thread, vsen,
+					"%s-sen", vsen->sd.v4l2_dev->name);
 		if (IS_ERR(vsen->kthread_sen)) {
 			dev_err(vsen->sd.v4l2_dev->dev,
 				"%s: kernel_thread() failed\n",	vsen->sd.name);
@@ -161,15 +187,17 @@ static int vimc_sen_s_stream(struct v4l2_subdev *sd, int enable)
 		}
 	} else {
 		if (!vsen->kthread_sen)
-			return -EINVAL;
+			return 0;
 
 		/* Stop image generator */
 		ret = kthread_stop(vsen->kthread_sen);
-		vsen->kthread_sen = NULL;
+		if (ret)
+			return ret;
 
+		vsen->kthread_sen = NULL;
 		vfree(vsen->frame);
 		vsen->frame = NULL;
-		return ret;
+		return 0;
 	}
 
 	return 0;
@@ -189,6 +217,7 @@ static void vimc_sen_destroy(struct vimc_ent_device *ved)
 	struct vimc_sen_device *vsen =
 				container_of(ved, struct vimc_sen_device, ved);
 
+	tpg_free(&vsen->tpg);
 	v4l2_device_unregister_subdev(&vsen->sd);
 	media_entity_cleanup(ved->ent);
 	kfree(vsen);
@@ -254,17 +283,26 @@ struct vimc_ent_device *vimc_sen_create(struct v4l2_device *v4l2_dev,
 	vsen->mbus_format.quantization = V4L2_QUANTIZATION_FULL_RANGE;
 	vsen->mbus_format.xfer_func = V4L2_XFER_FUNC_SRGB;
 
+	/* Initialize the test pattern generator */
+	tpg_init(&vsen->tpg, vsen->mbus_format.width,
+		 vsen->mbus_format.height);
+	ret = tpg_alloc(&vsen->tpg, VIMC_SEN_FRAME_MAX_WIDTH);
+	if (ret)
+		goto err_clean_m_ent;
+
 	/* Register the subdev with the v4l2 and the media framework */
 	ret = v4l2_device_register_subdev(v4l2_dev, &vsen->sd);
 	if (ret) {
 		dev_err(vsen->sd.v4l2_dev->dev,
 			"%s: subdev register failed (err=%d)\n",
 			vsen->sd.name, ret);
-		goto err_clean_m_ent;
+		goto err_free_tpg;
 	}
 
 	return &vsen->ved;
 
+err_free_tpg:
+	tpg_free(&vsen->tpg);
 err_clean_m_ent:
 	media_entity_cleanup(&vsen->sd.entity);
 err_clean_pads:
-- 
2.7.4

[RFC PATCH v3 02/11] [media] vimc: Move common code from the core

[Helen Koike]

Remove helper functions from vimc-core and add it in vimc-common to
clean up the core.

Signed-off-by: Helen Koike <helen.koike@collabora.com>

---

Changes in v3:
[media] vimc: Move common code from the core
	- This is a new patch in the series

Changes in v2: None


---
 drivers/media/platform/vimc/Makefile               |   2 +-
 drivers/media/platform/vimc/vimc-capture.h         |   2 +-
 drivers/media/platform/vimc/vimc-common.c          | 221 +++++++++++++++++++++
 .../platform/vimc/{vimc-core.h => vimc-common.h}   |   7 +-
 drivers/media/platform/vimc/vimc-core.c            | 205 +------------------
 drivers/media/platform/vimc/vimc-sensor.h          |   2 +-
 6 files changed, 229 insertions(+), 210 deletions(-)
 create mode 100644 drivers/media/platform/vimc/vimc-common.c
 rename drivers/media/platform/vimc/{vimc-core.h => vimc-common.h} (96%)

diff --git a/drivers/media/platform/vimc/Makefile b/drivers/media/platform/vimc/Makefile
index c45195e..6b6ddf4 100644
--- a/drivers/media/platform/vimc/Makefile
+++ b/drivers/media/platform/vimc/Makefile
@@ -1,3 +1,3 @@
-vimc-objs := vimc-core.o vimc-capture.o vimc-sensor.o
+vimc-objs := vimc-core.o vimc-capture.o vimc-common.o vimc-sensor.o
 
 obj-$(CONFIG_VIDEO_VIMC) += vimc.o
diff --git a/drivers/media/platform/vimc/vimc-capture.h b/drivers/media/platform/vimc/vimc-capture.h
index 581a813..7e5c707 100644
--- a/drivers/media/platform/vimc/vimc-capture.h
+++ b/drivers/media/platform/vimc/vimc-capture.h
@@ -18,7 +18,7 @@
 #ifndef _VIMC_CAPTURE_H_
 #define _VIMC_CAPTURE_H_
 
-#include "vimc-core.h"
+#include "vimc-common.h"
 
 struct vimc_ent_device *vimc_cap_create(struct v4l2_device *v4l2_dev,
 					const char *const name,
diff --git a/drivers/media/platform/vimc/vimc-common.c b/drivers/media/platform/vimc/vimc-common.c
new file mode 100644
index 0000000..42f779a
--- /dev/null
+++ b/drivers/media/platform/vimc/vimc-common.c
@@ -0,0 +1,221 @@
+/*
+ * vimc-common.c Virtual Media Controller Driver
+ *
+ * Copyright (C) 2015-2017 Helen Koike <helen.fornazier@gmail.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ */
+
+#include "vimc-common.h"
+
+static const struct vimc_pix_map vimc_pix_map_list[] = {
+	/* TODO: add all missing formats */
+
+	/* RGB formats */
+	{
+		.code = MEDIA_BUS_FMT_BGR888_1X24,
+		.pixelformat = V4L2_PIX_FMT_BGR24,
+		.bpp = 3,
+	},
+	{
+		.code = MEDIA_BUS_FMT_RGB888_1X24,
+		.pixelformat = V4L2_PIX_FMT_RGB24,
+		.bpp = 3,
+	},
+	{
+		.code = MEDIA_BUS_FMT_ARGB8888_1X32,
+		.pixelformat = V4L2_PIX_FMT_ARGB32,
+		.bpp = 4,
+	},
+
+	/* Bayer formats */
+	{
+		.code = MEDIA_BUS_FMT_SBGGR8_1X8,
+		.pixelformat = V4L2_PIX_FMT_SBGGR8,
+		.bpp = 1,
+	},
+	{
+		.code = MEDIA_BUS_FMT_SGBRG8_1X8,
+		.pixelformat = V4L2_PIX_FMT_SGBRG8,
+		.bpp = 1,
+	},
+	{
+		.code = MEDIA_BUS_FMT_SGRBG8_1X8,
+		.pixelformat = V4L2_PIX_FMT_SGRBG8,
+		.bpp = 1,
+	},
+	{
+		.code = MEDIA_BUS_FMT_SRGGB8_1X8,
+		.pixelformat = V4L2_PIX_FMT_SRGGB8,
+		.bpp = 1,
+	},
+	{
+		.code = MEDIA_BUS_FMT_SBGGR10_1X10,
+		.pixelformat = V4L2_PIX_FMT_SBGGR10,
+		.bpp = 2,
+	},
+	{
+		.code = MEDIA_BUS_FMT_SGBRG10_1X10,
+		.pixelformat = V4L2_PIX_FMT_SGBRG10,
+		.bpp = 2,
+	},
+	{
+		.code = MEDIA_BUS_FMT_SGRBG10_1X10,
+		.pixelformat = V4L2_PIX_FMT_SGRBG10,
+		.bpp = 2,
+	},
+	{
+		.code = MEDIA_BUS_FMT_SRGGB10_1X10,
+		.pixelformat = V4L2_PIX_FMT_SRGGB10,
+		.bpp = 2,
+	},
+
+	/* 10bit raw bayer a-law compressed to 8 bits */
+	{
+		.code = MEDIA_BUS_FMT_SBGGR10_ALAW8_1X8,
+		.pixelformat = V4L2_PIX_FMT_SBGGR10ALAW8,
+		.bpp = 1,
+	},
+	{
+		.code = MEDIA_BUS_FMT_SGBRG10_ALAW8_1X8,
+		.pixelformat = V4L2_PIX_FMT_SGBRG10ALAW8,
+		.bpp = 1,
+	},
+	{
+		.code = MEDIA_BUS_FMT_SGRBG10_ALAW8_1X8,
+		.pixelformat = V4L2_PIX_FMT_SGRBG10ALAW8,
+		.bpp = 1,
+	},
+	{
+		.code = MEDIA_BUS_FMT_SRGGB10_ALAW8_1X8,
+		.pixelformat = V4L2_PIX_FMT_SRGGB10ALAW8,
+		.bpp = 1,
+	},
+
+	/* 10bit raw bayer DPCM compressed to 8 bits */
+	{
+		.code = MEDIA_BUS_FMT_SBGGR10_DPCM8_1X8,
+		.pixelformat = V4L2_PIX_FMT_SBGGR10DPCM8,
+		.bpp = 1,
+	},
+	{
+		.code = MEDIA_BUS_FMT_SGBRG10_DPCM8_1X8,
+		.pixelformat = V4L2_PIX_FMT_SGBRG10DPCM8,
+		.bpp = 1,
+	},
+	{
+		.code = MEDIA_BUS_FMT_SGRBG10_DPCM8_1X8,
+		.pixelformat = V4L2_PIX_FMT_SGRBG10DPCM8,
+		.bpp = 1,
+	},
+	{
+		.code = MEDIA_BUS_FMT_SRGGB10_DPCM8_1X8,
+		.pixelformat = V4L2_PIX_FMT_SRGGB10DPCM8,
+		.bpp = 1,
+	},
+	{
+		.code = MEDIA_BUS_FMT_SBGGR12_1X12,
+		.pixelformat = V4L2_PIX_FMT_SBGGR12,
+		.bpp = 2,
+	},
+	{
+		.code = MEDIA_BUS_FMT_SGBRG12_1X12,
+		.pixelformat = V4L2_PIX_FMT_SGBRG12,
+		.bpp = 2,
+	},
+	{
+		.code = MEDIA_BUS_FMT_SGRBG12_1X12,
+		.pixelformat = V4L2_PIX_FMT_SGRBG12,
+		.bpp = 2,
+	},
+	{
+		.code = MEDIA_BUS_FMT_SRGGB12_1X12,
+		.pixelformat = V4L2_PIX_FMT_SRGGB12,
+		.bpp = 2,
+	},
+};
+
+const struct vimc_pix_map *vimc_pix_map_by_code(u32 code)
+{
+	unsigned int i;
+
+	for (i = 0; i < ARRAY_SIZE(vimc_pix_map_list); i++) {
+		if (vimc_pix_map_list[i].code == code)
+			return &vimc_pix_map_list[i];
+	}
+	return NULL;
+}
+
+const struct vimc_pix_map *vimc_pix_map_by_pixelformat(u32 pixelformat)
+{
+	unsigned int i;
+
+	for (i = 0; i < ARRAY_SIZE(vimc_pix_map_list); i++) {
+		if (vimc_pix_map_list[i].pixelformat == pixelformat)
+			return &vimc_pix_map_list[i];
+	}
+	return NULL;
+}
+
+int vimc_propagate_frame(struct media_pad *src, const void *frame)
+{
+	struct media_link *link;
+
+	if (!(src->flags & MEDIA_PAD_FL_SOURCE))
+		return -EINVAL;
+
+	/* Send this frame to all sink pads that are direct linked */
+	list_for_each_entry(link, &src->entity->links, list) {
+		if (link->source == src &&
+		    (link->flags & MEDIA_LNK_FL_ENABLED)) {
+			struct vimc_ent_device *ved = NULL;
+			struct media_entity *entity = link->sink->entity;
+
+			if (is_media_entity_v4l2_subdev(entity)) {
+				struct v4l2_subdev *sd =
+					container_of(entity, struct v4l2_subdev,
+						     entity);
+				ved = v4l2_get_subdevdata(sd);
+			} else if (is_media_entity_v4l2_video_device(entity)) {
+				struct video_device *vdev =
+					container_of(entity,
+						     struct video_device,
+						     entity);
+				ved = video_get_drvdata(vdev);
+			}
+			if (ved && ved->process_frame)
+				ved->process_frame(ved, link->sink, frame);
+		}
+	}
+
+	return 0;
+}
+
+/* Helper function to allocate and initialize pads */
+struct media_pad *vimc_pads_init(u16 num_pads, const unsigned long *pads_flag)
+{
+	struct media_pad *pads;
+	unsigned int i;
+
+	/* Allocate memory for the pads */
+	pads = kcalloc(num_pads, sizeof(*pads), GFP_KERNEL);
+	if (!pads)
+		return ERR_PTR(-ENOMEM);
+
+	/* Initialize the pads */
+	for (i = 0; i < num_pads; i++) {
+		pads[i].index = i;
+		pads[i].flags = pads_flag[i];
+	}
+
+	return pads;
+}
diff --git a/drivers/media/platform/vimc/vimc-core.h b/drivers/media/platform/vimc/vimc-common.h
similarity index 96%
rename from drivers/media/platform/vimc/vimc-core.h
rename to drivers/media/platform/vimc/vimc-common.h
index 4525d23..00d3da4 100644
--- a/drivers/media/platform/vimc/vimc-core.h
+++ b/drivers/media/platform/vimc/vimc-common.h
@@ -1,5 +1,5 @@
 /*
- * vimc-core.h Virtual Media Controller Driver
+ * vimc-ccommon.h Virtual Media Controller Driver
  *
  * Copyright (C) 2015-2017 Helen Koike <helen.fornazier@gmail.com>
  *
@@ -15,10 +15,11 @@
  *
  */
 
-#ifndef _VIMC_CORE_H_
-#define _VIMC_CORE_H_
+#ifndef _VIMC_COMMON_H_
+#define _VIMC_COMMON_H_
 
 #include <linux/slab.h>
+#include <media/media-device.h>
 #include <media/v4l2-device.h>
 
 /**
diff --git a/drivers/media/platform/vimc/vimc-core.c b/drivers/media/platform/vimc/vimc-core.c
index bc107da..afc79e2 100644
--- a/drivers/media/platform/vimc/vimc-core.c
+++ b/drivers/media/platform/vimc/vimc-core.c
@@ -22,7 +22,7 @@
 #include <media/v4l2-device.h>
 
 #include "vimc-capture.h"
-#include "vimc-core.h"
+#include "vimc-common.h"
 #include "vimc-sensor.h"
 
 #define VIMC_PDEV_NAME "vimc"
@@ -197,189 +197,6 @@ static const struct vimc_pipeline_config pipe_cfg = {
 
 /* -------------------------------------------------------------------------- */
 
-static const struct vimc_pix_map vimc_pix_map_list[] = {
-	/* TODO: add all missing formats */
-
-	/* RGB formats */
-	{
-		.code = MEDIA_BUS_FMT_BGR888_1X24,
-		.pixelformat = V4L2_PIX_FMT_BGR24,
-		.bpp = 3,
-	},
-	{
-		.code = MEDIA_BUS_FMT_RGB888_1X24,
-		.pixelformat = V4L2_PIX_FMT_RGB24,
-		.bpp = 3,
-	},
-	{
-		.code = MEDIA_BUS_FMT_ARGB8888_1X32,
-		.pixelformat = V4L2_PIX_FMT_ARGB32,
-		.bpp = 4,
-	},
-
-	/* Bayer formats */
-	{
-		.code = MEDIA_BUS_FMT_SBGGR8_1X8,
-		.pixelformat = V4L2_PIX_FMT_SBGGR8,
-		.bpp = 1,
-	},
-	{
-		.code = MEDIA_BUS_FMT_SGBRG8_1X8,
-		.pixelformat = V4L2_PIX_FMT_SGBRG8,
-		.bpp = 1,
-	},
-	{
-		.code = MEDIA_BUS_FMT_SGRBG8_1X8,
-		.pixelformat = V4L2_PIX_FMT_SGRBG8,
-		.bpp = 1,
-	},
-	{
-		.code = MEDIA_BUS_FMT_SRGGB8_1X8,
-		.pixelformat = V4L2_PIX_FMT_SRGGB8,
-		.bpp = 1,
-	},
-	{
-		.code = MEDIA_BUS_FMT_SBGGR10_1X10,
-		.pixelformat = V4L2_PIX_FMT_SBGGR10,
-		.bpp = 2,
-	},
-	{
-		.code = MEDIA_BUS_FMT_SGBRG10_1X10,
-		.pixelformat = V4L2_PIX_FMT_SGBRG10,
-		.bpp = 2,
-	},
-	{
-		.code = MEDIA_BUS_FMT_SGRBG10_1X10,
-		.pixelformat = V4L2_PIX_FMT_SGRBG10,
-		.bpp = 2,
-	},
-	{
-		.code = MEDIA_BUS_FMT_SRGGB10_1X10,
-		.pixelformat = V4L2_PIX_FMT_SRGGB10,
-		.bpp = 2,
-	},
-
-	/* 10bit raw bayer a-law compressed to 8 bits */
-	{
-		.code = MEDIA_BUS_FMT_SBGGR10_ALAW8_1X8,
-		.pixelformat = V4L2_PIX_FMT_SBGGR10ALAW8,
-		.bpp = 1,
-	},
-	{
-		.code = MEDIA_BUS_FMT_SGBRG10_ALAW8_1X8,
-		.pixelformat = V4L2_PIX_FMT_SGBRG10ALAW8,
-		.bpp = 1,
-	},
-	{
-		.code = MEDIA_BUS_FMT_SGRBG10_ALAW8_1X8,
-		.pixelformat = V4L2_PIX_FMT_SGRBG10ALAW8,
-		.bpp = 1,
-	},
-	{
-		.code = MEDIA_BUS_FMT_SRGGB10_ALAW8_1X8,
-		.pixelformat = V4L2_PIX_FMT_SRGGB10ALAW8,
-		.bpp = 1,
-	},
-
-	/* 10bit raw bayer DPCM compressed to 8 bits */
-	{
-		.code = MEDIA_BUS_FMT_SBGGR10_DPCM8_1X8,
-		.pixelformat = V4L2_PIX_FMT_SBGGR10DPCM8,
-		.bpp = 1,
-	},
-	{
-		.code = MEDIA_BUS_FMT_SGBRG10_DPCM8_1X8,
-		.pixelformat = V4L2_PIX_FMT_SGBRG10DPCM8,
-		.bpp = 1,
-	},
-	{
-		.code = MEDIA_BUS_FMT_SGRBG10_DPCM8_1X8,
-		.pixelformat = V4L2_PIX_FMT_SGRBG10DPCM8,
-		.bpp = 1,
-	},
-	{
-		.code = MEDIA_BUS_FMT_SRGGB10_DPCM8_1X8,
-		.pixelformat = V4L2_PIX_FMT_SRGGB10DPCM8,
-		.bpp = 1,
-	},
-	{
-		.code = MEDIA_BUS_FMT_SBGGR12_1X12,
-		.pixelformat = V4L2_PIX_FMT_SBGGR12,
-		.bpp = 2,
-	},
-	{
-		.code = MEDIA_BUS_FMT_SGBRG12_1X12,
-		.pixelformat = V4L2_PIX_FMT_SGBRG12,
-		.bpp = 2,
-	},
-	{
-		.code = MEDIA_BUS_FMT_SGRBG12_1X12,
-		.pixelformat = V4L2_PIX_FMT_SGRBG12,
-		.bpp = 2,
-	},
-	{
-		.code = MEDIA_BUS_FMT_SRGGB12_1X12,
-		.pixelformat = V4L2_PIX_FMT_SRGGB12,
-		.bpp = 2,
-	},
-};
-
-const struct vimc_pix_map *vimc_pix_map_by_code(u32 code)
-{
-	unsigned int i;
-
-	for (i = 0; i < ARRAY_SIZE(vimc_pix_map_list); i++) {
-		if (vimc_pix_map_list[i].code == code)
-			return &vimc_pix_map_list[i];
-	}
-	return NULL;
-}
-
-const struct vimc_pix_map *vimc_pix_map_by_pixelformat(u32 pixelformat)
-{
-	unsigned int i;
-
-	for (i = 0; i < ARRAY_SIZE(vimc_pix_map_list); i++) {
-		if (vimc_pix_map_list[i].pixelformat == pixelformat)
-			return &vimc_pix_map_list[i];
-	}
-	return NULL;
-}
-
-int vimc_propagate_frame(struct media_pad *src, const void *frame)
-{
-	struct media_link *link;
-
-	if (!(src->flags & MEDIA_PAD_FL_SOURCE))
-		return -EINVAL;
-
-	/* Send this frame to all sink pads that are direct linked */
-	list_for_each_entry(link, &src->entity->links, list) {
-		if (link->source == src &&
-		    (link->flags & MEDIA_LNK_FL_ENABLED)) {
-			struct vimc_ent_device *ved = NULL;
-			struct media_entity *entity = link->sink->entity;
-
-			if (is_media_entity_v4l2_subdev(entity)) {
-				struct v4l2_subdev *sd =
-					container_of(entity, struct v4l2_subdev,
-						     entity);
-				ved = v4l2_get_subdevdata(sd);
-			} else if (is_media_entity_v4l2_video_device(entity)) {
-				struct video_device *vdev =
-					container_of(entity,
-						     struct video_device,
-						     entity);
-				ved = video_get_drvdata(vdev);
-			}
-			if (ved && ved->process_frame)
-				ved->process_frame(ved, link->sink, frame);
-		}
-	}
-
-	return 0;
-}
-
 static void vimc_device_unregister(struct vimc_device *vimc)
 {
 	unsigned int i;
@@ -396,26 +213,6 @@ static void vimc_device_unregister(struct vimc_device *vimc)
 	media_device_cleanup(&vimc->mdev);
 }
 
-/* Helper function to allocate and initialize pads */
-struct media_pad *vimc_pads_init(u16 num_pads, const unsigned long *pads_flag)
-{
-	struct media_pad *pads;
-	unsigned int i;
-
-	/* Allocate memory for the pads */
-	pads = kcalloc(num_pads, sizeof(*pads), GFP_KERNEL);
-	if (!pads)
-		return ERR_PTR(-ENOMEM);
-
-	/* Initialize the pads */
-	for (i = 0; i < num_pads; i++) {
-		pads[i].index = i;
-		pads[i].flags = pads_flag[i];
-	}
-
-	return pads;
-}
-
 /*
  * TODO: remove this function when all the
  * entities specific code are implemented
diff --git a/drivers/media/platform/vimc/vimc-sensor.h b/drivers/media/platform/vimc/vimc-sensor.h
index 505310e..580dcec 100644
--- a/drivers/media/platform/vimc/vimc-sensor.h
+++ b/drivers/media/platform/vimc/vimc-sensor.h
@@ -18,7 +18,7 @@
 #ifndef _VIMC_SENSOR_H_
 #define _VIMC_SENSOR_H_
 
-#include "vimc-core.h"
+#include "vimc-common.h"
 
 struct vimc_ent_device *vimc_sen_create(struct v4l2_device *v4l2_dev,
 					const char *const name,
-- 
2.7.4

[RFC PATCH v3 03/11] [media] vimc: common: Add vimc_ent_sd_* helper

[Helen Koike]

As all the subdevices in the topology will be initialized in the same
way, to avoid code repetition the vimc_ent_sd_{register, unregister}
helper functions were created

Signed-off-by: Helen Koike <helen.koike@collabora.com>

---

Changes in v3:
[media] vimc: Add vimc_ent_sd_* helper functions
	- add it in vimc-common.c instead in vimc-core.c
	- fix vimc_ent_sd_register, use function parameter to assign
	sd->entity.function instead of using a fixed value
	- rename commit to add the "common" tag

Changes in v2:
[media] vimc: Add vimc_ent_sd_* helper functions
	- Comments in vimc_ent_sd_init
	- Update vimc_ent_sd_init with upstream code as media_entity_pads_init
	(instead of media_entity_init), entity->function intead of entity->type
	- Add missing vimc_pads_cleanup in vimc_ent_sd_cleanup
	- remove subdevice v4l2_dev and dev fields
	- change unregister order in vimc_ent_sd_cleanup
	- rename vimc_ent_sd_{init,cleanup} to vimc_ent_sd_{register,unregister}
	- remove struct vimc_ent_subdevice, use ved and sd directly
	- don't impose struct vimc_sen_device to declare ved and sd struct first
	- add kernel docs


---
 drivers/media/platform/vimc/vimc-common.c | 66 +++++++++++++++++++++++++++++++
 drivers/media/platform/vimc/vimc-common.h | 39 ++++++++++++++++++
 drivers/media/platform/vimc/vimc-sensor.c | 58 +++++----------------------
 3 files changed, 114 insertions(+), 49 deletions(-)

diff --git a/drivers/media/platform/vimc/vimc-common.c b/drivers/media/platform/vimc/vimc-common.c
index 42f779a..3afbabd 100644
--- a/drivers/media/platform/vimc/vimc-common.c
+++ b/drivers/media/platform/vimc/vimc-common.c
@@ -219,3 +219,69 @@ struct media_pad *vimc_pads_init(u16 num_pads, const unsigned long *pads_flag)
 
 	return pads;
 }
+
+static const struct media_entity_operations vimc_ent_sd_mops = {
+	.link_validate = v4l2_subdev_link_validate,
+};
+
+int vimc_ent_sd_register(struct vimc_ent_device *ved,
+			 struct v4l2_subdev *sd,
+			 struct v4l2_device *v4l2_dev,
+			 const char *const name,
+			 u32 function,
+			 u16 num_pads,
+			 const unsigned long *pads_flag,
+			 const struct v4l2_subdev_ops *sd_ops,
+			 void (*sd_destroy)(struct vimc_ent_device *))
+{
+	int ret;
+
+	/* Allocate the pads */
+	ved->pads = vimc_pads_init(num_pads, pads_flag);
+	if (IS_ERR(ved->pads))
+		return PTR_ERR(ved->pads);
+
+	/* Fill the vimc_ent_device struct */
+	ved->destroy = sd_destroy;
+	ved->ent = &sd->entity;
+
+	/* Initialize the subdev */
+	v4l2_subdev_init(sd, sd_ops);
+	sd->entity.function = function;
+	sd->entity.ops = &vimc_ent_sd_mops;
+	sd->owner = THIS_MODULE;
+	strlcpy(sd->name, name, sizeof(sd->name));
+	v4l2_set_subdevdata(sd, ved);
+
+	/* Expose this subdev to user space */
+	sd->flags = V4L2_SUBDEV_FL_HAS_DEVNODE;
+
+	/* Initialize the media entity */
+	ret = media_entity_pads_init(&sd->entity, num_pads, ved->pads);
+	if (ret)
+		goto err_clean_pads;
+
+	/* Register the subdev with the v4l2 and the media framework */
+	ret = v4l2_device_register_subdev(v4l2_dev, sd);
+	if (ret) {
+		dev_err(v4l2_dev->dev,
+			"%s: subdev register failed (err=%d)\n",
+			name, ret);
+		goto err_clean_m_ent;
+	}
+
+	return 0;
+
+err_clean_m_ent:
+	media_entity_cleanup(&sd->entity);
+err_clean_pads:
+	vimc_pads_cleanup(ved->pads);
+	return ret;
+}
+
+void vimc_ent_sd_unregister(struct vimc_ent_device *ved, struct v4l2_subdev *sd)
+{
+	v4l2_device_unregister_subdev(sd);
+	media_entity_cleanup(ved->ent);
+	vimc_pads_cleanup(ved->pads);
+}
diff --git a/drivers/media/platform/vimc/vimc-common.h b/drivers/media/platform/vimc/vimc-common.h
index 00d3da4..9ec361c 100644
--- a/drivers/media/platform/vimc/vimc-common.h
+++ b/drivers/media/platform/vimc/vimc-common.h
@@ -110,4 +110,43 @@ const struct vimc_pix_map *vimc_pix_map_by_code(u32 code);
  */
 const struct vimc_pix_map *vimc_pix_map_by_pixelformat(u32 pixelformat);
 
+/**
+ * vimc_ent_sd_register - initialize and register a subdev node
+ *
+ * @ved:	the vimc_ent_device struct to be initialize
+ * @sd:		the v4l2_subdev struct to be initialize and registered
+ * @v4l2_dev:	the v4l2 device to register the v4l2_subdev
+ * @name:	name of the sub-device. Please notice that the name must be
+ *		unique.
+ * @function:	media entity function defined by MEDIA_ENT_F_* macros
+ * @num_pads:	number of pads to initialize
+ * @pads_flag:	flags to use in each pad
+ * @sd_ops:	pointer to &struct v4l2_subdev_ops.
+ * @sd_destroy:	callback to destroy the node
+ *
+ * Helper function initialize and register the struct vimc_ent_device and struct
+ * v4l2_subdev which represents a subdev node in the topology
+ */
+int vimc_ent_sd_register(struct vimc_ent_device *ved,
+			 struct v4l2_subdev *sd,
+			 struct v4l2_device *v4l2_dev,
+			 const char *const name,
+			 u32 function,
+			 u16 num_pads,
+			 const unsigned long *pads_flag,
+			 const struct v4l2_subdev_ops *sd_ops,
+			 void (*sd_destroy)(struct vimc_ent_device *));
+
+/**
+ * vimc_ent_sd_register - initialize and register a subdev node
+ *
+ * @ved:	the vimc_ent_device struct to be initialize
+ * @sd:		the v4l2_subdev struct to be initialize and registered
+ *
+ * Helper function cleanup and unregister the struct vimc_ent_device and struct
+ * v4l2_subdev which represents a subdev node in the topology
+ */
+void vimc_ent_sd_unregister(struct vimc_ent_device *ved,
+			    struct v4l2_subdev *sd);
+
 #endif
diff --git a/drivers/media/platform/vimc/vimc-sensor.c b/drivers/media/platform/vimc/vimc-sensor.c
index 2e83487..6386ac1 100644
--- a/drivers/media/platform/vimc/vimc-sensor.c
+++ b/drivers/media/platform/vimc/vimc-sensor.c
@@ -113,11 +113,6 @@ static const struct v4l2_subdev_pad_ops vimc_sen_pad_ops = {
 	.set_fmt		= vimc_sen_get_fmt,
 };
 
-/* media operations */
-static const struct media_entity_operations vimc_sen_mops = {
-	.link_validate = v4l2_subdev_link_validate,
-};
-
 static int vimc_sen_tpg_thread(void *data)
 {
 	struct vimc_sen_device *vsen = data;
@@ -217,9 +212,8 @@ static void vimc_sen_destroy(struct vimc_ent_device *ved)
 	struct vimc_sen_device *vsen =
 				container_of(ved, struct vimc_sen_device, ved);
 
+	vimc_ent_sd_unregister(ved, &vsen->sd);
 	tpg_free(&vsen->tpg);
-	v4l2_device_unregister_subdev(&vsen->sd);
-	media_entity_cleanup(ved->ent);
 	kfree(vsen);
 }
 
@@ -246,33 +240,12 @@ struct vimc_ent_device *vimc_sen_create(struct v4l2_device *v4l2_dev,
 	if (!vsen)
 		return ERR_PTR(-ENOMEM);
 
-	/* Allocate the pads */
-	vsen->ved.pads = vimc_pads_init(num_pads, pads_flag);
-	if (IS_ERR(vsen->ved.pads)) {
-		ret = PTR_ERR(vsen->ved.pads);
-		goto err_free_vsen;
-	}
-
-	/* Fill the vimc_ent_device struct */
-	vsen->ved.destroy = vimc_sen_destroy;
-	vsen->ved.ent = &vsen->sd.entity;
-
-	/* Initialize the subdev */
-	v4l2_subdev_init(&vsen->sd, &vimc_sen_ops);
-	vsen->sd.entity.function = MEDIA_ENT_F_CAM_SENSOR;
-	vsen->sd.entity.ops = &vimc_sen_mops;
-	vsen->sd.owner = THIS_MODULE;
-	strlcpy(vsen->sd.name, name, sizeof(vsen->sd.name));
-	v4l2_set_subdevdata(&vsen->sd, &vsen->ved);
-
-	/* Expose this subdev to user space */
-	vsen->sd.flags = V4L2_SUBDEV_FL_HAS_DEVNODE;
-
-	/* Initialize the media entity */
-	ret = media_entity_pads_init(&vsen->sd.entity,
-				     num_pads, vsen->ved.pads);
+	/* Initialize ved and sd */
+	ret = vimc_ent_sd_register(&vsen->ved, &vsen->sd, v4l2_dev, name,
+				   MEDIA_ENT_F_CAM_SENSOR, num_pads, pads_flag,
+				   &vimc_sen_ops, vimc_sen_destroy);
 	if (ret)
-		goto err_clean_pads;
+		goto err_free_vsen;
 
 	/* Set the active frame format (this is hardcoded for now) */
 	vsen->mbus_format.width = 640;
@@ -288,25 +261,12 @@ struct vimc_ent_device *vimc_sen_create(struct v4l2_device *v4l2_dev,
 		 vsen->mbus_format.height);
 	ret = tpg_alloc(&vsen->tpg, VIMC_SEN_FRAME_MAX_WIDTH);
 	if (ret)
-		goto err_clean_m_ent;
-
-	/* Register the subdev with the v4l2 and the media framework */
-	ret = v4l2_device_register_subdev(v4l2_dev, &vsen->sd);
-	if (ret) {
-		dev_err(vsen->sd.v4l2_dev->dev,
-			"%s: subdev register failed (err=%d)\n",
-			vsen->sd.name, ret);
-		goto err_free_tpg;
-	}
+		goto err_unregister_ent_sd;
 
 	return &vsen->ved;
 
-err_free_tpg:
-	tpg_free(&vsen->tpg);
-err_clean_m_ent:
-	media_entity_cleanup(&vsen->sd.entity);
-err_clean_pads:
-	vimc_pads_cleanup(vsen->ved.pads);
+err_unregister_ent_sd:
+	vimc_ent_sd_unregister(&vsen->ved,  &vsen->sd);
 err_free_vsen:
 	kfree(vsen);
 
-- 
2.7.4

[RFC PATCH v3 04/11] [media] vimc: common: Add vimc_pipeline_s_stream helper

[Helen Koike]

Move the vimc_cap_pipeline_s_stream from the vimc-cap.c to vimc-common.c
as this core will be reused by other subdevices to activate the stream
in their directly connected nodes

Signed-off-by: Helen Koike <helen.koike@collabora.com>

---

Changes in v3:
[media] vimc: Add vimc_pipeline_s_stream in the core
	- add it in vimc-common instead of vimc-core
	- rename commit with "common" tag

Changes in v2:
[media] vimc: Add vimc_pipeline_s_stream in the core
	- Use is_media_entity_v4l2_subdev instead of comparing with the old
	entity->type
	- Fix comments style
	- add kernel-docs
	- call s_stream across all sink pads


---
 drivers/media/platform/vimc/vimc-capture.c | 29 ++-------------------------
 drivers/media/platform/vimc/vimc-common.c  | 32 ++++++++++++++++++++++++++++++
 drivers/media/platform/vimc/vimc-common.h  | 11 ++++++++++
 3 files changed, 45 insertions(+), 27 deletions(-)

diff --git a/drivers/media/platform/vimc/vimc-capture.c b/drivers/media/platform/vimc/vimc-capture.c
index 9adb06d..93f6a09 100644
--- a/drivers/media/platform/vimc/vimc-capture.c
+++ b/drivers/media/platform/vimc/vimc-capture.c
@@ -132,31 +132,6 @@ static void vimc_cap_return_all_buffers(struct vimc_cap_device *vcap,
 	spin_unlock(&vcap->qlock);
 }
 
-static int vimc_cap_pipeline_s_stream(struct vimc_cap_device *vcap, int enable)
-{
-	struct v4l2_subdev *sd;
-	struct media_pad *pad;
-	int ret;
-
-	/* Start the stream in the subdevice direct connected */
-	pad = media_entity_remote_pad(&vcap->vdev.entity.pads[0]);
-
-	/*
-	 * if it is a raw node from vimc-core, there is nothing to activate
-	 * TODO: remove this when there are no more raw nodes in the
-	 * core and return error instead
-	 */
-	if (pad->entity->obj_type == MEDIA_ENTITY_TYPE_BASE)
-		return 0;
-
-	sd = media_entity_to_v4l2_subdev(pad->entity);
-	ret = v4l2_subdev_call(sd, video, s_stream, enable);
-	if (ret && ret != -ENOIOCTLCMD)
-		return ret;
-
-	return 0;
-}
-
 static int vimc_cap_start_streaming(struct vb2_queue *vq, unsigned int count)
 {
 	struct vimc_cap_device *vcap = vb2_get_drv_priv(vq);
@@ -173,7 +148,7 @@ static int vimc_cap_start_streaming(struct vb2_queue *vq, unsigned int count)
 	}
 
 	/* Enable streaming from the pipe */
-	ret = vimc_cap_pipeline_s_stream(vcap, 1);
+	ret = vimc_pipeline_s_stream(&vcap->vdev.entity, 1);
 	if (ret) {
 		media_pipeline_stop(entity);
 		vimc_cap_return_all_buffers(vcap, VB2_BUF_STATE_QUEUED);
@@ -192,7 +167,7 @@ static void vimc_cap_stop_streaming(struct vb2_queue *vq)
 	struct vimc_cap_device *vcap = vb2_get_drv_priv(vq);
 
 	/* Disable streaming from the pipe */
-	vimc_cap_pipeline_s_stream(vcap, 0);
+	vimc_pipeline_s_stream(&vcap->vdev.entity, 0);
 
 	/* Stop the media pipeline */
 	media_pipeline_stop(&vcap->vdev.entity);
diff --git a/drivers/media/platform/vimc/vimc-common.c b/drivers/media/platform/vimc/vimc-common.c
index 3afbabd..f809a9d 100644
--- a/drivers/media/platform/vimc/vimc-common.c
+++ b/drivers/media/platform/vimc/vimc-common.c
@@ -220,6 +220,38 @@ struct media_pad *vimc_pads_init(u16 num_pads, const unsigned long *pads_flag)
 	return pads;
 }
 
+int vimc_pipeline_s_stream(struct media_entity *ent, int enable)
+{
+	struct v4l2_subdev *sd;
+	struct media_pad *pad;
+	unsigned int i;
+	int ret;
+
+	for (i = 0; i < ent->num_pads; i++) {
+		if (ent->pads[i].flags & MEDIA_PAD_FL_SOURCE)
+			continue;
+
+		/* Start the stream in the subdevice direct connected */
+		pad = media_entity_remote_pad(&ent->pads[i]);
+
+		/*
+		 * if this is a raw node from vimc-core, then there is
+		 * nothing to activate
+		 * TODO: remove this when there are no more raw nodes in the
+		 * core and return error instead
+		 */
+		if (pad->entity->obj_type == MEDIA_ENTITY_TYPE_BASE)
+			continue;
+
+		sd = media_entity_to_v4l2_subdev(pad->entity);
+		ret = v4l2_subdev_call(sd, video, s_stream, enable);
+		if (ret && ret != -ENOIOCTLCMD)
+			return ret;
+	}
+
+	return 0;
+}
+
 static const struct media_entity_operations vimc_ent_sd_mops = {
 	.link_validate = v4l2_subdev_link_validate,
 };
diff --git a/drivers/media/platform/vimc/vimc-common.h b/drivers/media/platform/vimc/vimc-common.h
index 9ec361c..73e7e94 100644
--- a/drivers/media/platform/vimc/vimc-common.h
+++ b/drivers/media/platform/vimc/vimc-common.h
@@ -97,6 +97,17 @@ static inline void vimc_pads_cleanup(struct media_pad *pads)
 }
 
 /**
+ * vimc_pipeline_s_stream - start stream through the pipeline
+ *
+ * @ent:		the pointer to struct media_entity for the node
+ * @enable:		1 to start the stream and 0 to stop
+ *
+ * Helper function to call the s_stream of the subdevices connected
+ * in all the sink pads of the entity
+ */
+int vimc_pipeline_s_stream(struct media_entity *ent, int enable);
+
+/**
  * vimc_pix_map_by_code - get vimc_pix_map struct by media bus code
  *
  * @code:		media bus format code defined by MEDIA_BUS_FMT_* macros
-- 
2.7.4

[RFC PATCH v3 05/11] [media] vimc: common: Add vimc_link_validate

[Helen Koike]

All links will be checked in the same way. Adding a helper function for
that

Signed-off-by: Helen Koike <helen.koike@collabora.com>

---

Changes in v3:
[media] vimc: common: Add vimc_link_validate
	- this is a new patch in the series

Changes in v2: None


---
 drivers/media/platform/vimc/vimc-capture.c |  78 +++---------------
 drivers/media/platform/vimc/vimc-common.c  | 124 ++++++++++++++++++++++++++++-
 drivers/media/platform/vimc/vimc-common.h  |  14 ++++
 3 files changed, 148 insertions(+), 68 deletions(-)

diff --git a/drivers/media/platform/vimc/vimc-capture.c b/drivers/media/platform/vimc/vimc-capture.c
index 93f6a09..5bdecd1 100644
--- a/drivers/media/platform/vimc/vimc-capture.c
+++ b/drivers/media/platform/vimc/vimc-capture.c
@@ -64,6 +64,15 @@ static int vimc_cap_querycap(struct file *file, void *priv,
 	return 0;
 }
 
+static void vimc_cap_get_format(struct vimc_ent_device *ved,
+				struct v4l2_pix_format *fmt)
+{
+	struct vimc_cap_device *vcap = container_of(ved, struct vimc_cap_device,
+						    ved);
+
+	*fmt = vcap->format;
+}
+
 static int vimc_cap_fmt_vid_cap(struct file *file, void *priv,
 				  struct v4l2_format *f)
 {
@@ -231,74 +240,8 @@ static const struct vb2_ops vimc_cap_qops = {
 	.wait_finish		= vb2_ops_wait_finish,
 };
 
-/*
- * NOTE: this function is a copy of v4l2_subdev_link_validate_get_format
- * maybe the v4l2 function should be public
- */
-static int vimc_cap_v4l2_subdev_link_validate_get_format(struct media_pad *pad,
-						struct v4l2_subdev_format *fmt)
-{
-	struct v4l2_subdev *sd = media_entity_to_v4l2_subdev(pad->entity);
-
-	fmt->which = V4L2_SUBDEV_FORMAT_ACTIVE;
-	fmt->pad = pad->index;
-
-	return v4l2_subdev_call(sd, pad, get_fmt, NULL, fmt);
-}
-
-static int vimc_cap_link_validate(struct media_link *link)
-{
-	struct v4l2_subdev_format source_fmt;
-	const struct vimc_pix_map *vpix;
-	struct vimc_cap_device *vcap = container_of(link->sink->entity,
-						    struct vimc_cap_device,
-						    vdev.entity);
-	struct v4l2_pix_format *sink_fmt = &vcap->format;
-	int ret;
-
-	/*
-	 * if it is a raw node from vimc-core, ignore the link for now
-	 * TODO: remove this when there are no more raw nodes in the
-	 * core and return error instead
-	 */
-	if (link->source->entity->obj_type == MEDIA_ENTITY_TYPE_BASE)
-		return 0;
-
-	/* Get the the format of the subdev */
-	ret = vimc_cap_v4l2_subdev_link_validate_get_format(link->source,
-							    &source_fmt);
-	if (ret)
-		return ret;
-
-	dev_dbg(vcap->vdev.v4l2_dev->dev,
-		"%s: link validate formats src:%dx%d %d sink:%dx%d %d\n",
-		vcap->vdev.name,
-		source_fmt.format.width, source_fmt.format.height,
-		source_fmt.format.code,
-		sink_fmt->width, sink_fmt->height,
-		sink_fmt->pixelformat);
-
-	/* The width, height and code must match. */
-	vpix = vimc_pix_map_by_pixelformat(sink_fmt->pixelformat);
-	if (source_fmt.format.width != sink_fmt->width
-	    || source_fmt.format.height != sink_fmt->height
-	    || vpix->code != source_fmt.format.code)
-		return -EPIPE;
-
-	/*
-	 * The field order must match, or the sink field order must be NONE
-	 * to support interlaced hardware connected to bridges that support
-	 * progressive formats only.
-	 */
-	if (source_fmt.format.field != sink_fmt->field &&
-	    sink_fmt->field != V4L2_FIELD_NONE)
-		return -EPIPE;
-
-	return 0;
-}
-
 static const struct media_entity_operations vimc_cap_mops = {
-	.link_validate		= vimc_cap_link_validate,
+	.link_validate		= vimc_link_validate,
 };
 
 static void vimc_cap_destroy(struct vimc_ent_device *ved)
@@ -434,6 +377,7 @@ struct vimc_ent_device *vimc_cap_create(struct v4l2_device *v4l2_dev,
 	vcap->ved.destroy = vimc_cap_destroy;
 	vcap->ved.ent = &vcap->vdev.entity;
 	vcap->ved.process_frame = vimc_cap_process_frame;
+	vcap->ved.vdev_get_format = vimc_cap_get_format;
 
 	/* Initialize the video_device struct */
 	vdev = &vcap->vdev;
diff --git a/drivers/media/platform/vimc/vimc-common.c b/drivers/media/platform/vimc/vimc-common.c
index f809a9d..83d4251 100644
--- a/drivers/media/platform/vimc/vimc-common.c
+++ b/drivers/media/platform/vimc/vimc-common.c
@@ -252,8 +252,130 @@ int vimc_pipeline_s_stream(struct media_entity *ent, int enable)
 	return 0;
 }
 
+static void vimc_fmt_pix_to_mbus(struct v4l2_mbus_framefmt *mfmt,
+				 struct v4l2_pix_format *pfmt)
+{
+	const struct vimc_pix_map *vpix =
+		vimc_pix_map_by_pixelformat(pfmt->pixelformat);
+
+	mfmt->width = pfmt->width;
+	mfmt->height = pfmt->height;
+	mfmt->code = vpix->code;
+	mfmt->field = pfmt->field;
+	mfmt->colorspace = pfmt->colorspace;
+	mfmt->ycbcr_enc = pfmt->ycbcr_enc;
+	mfmt->quantization = pfmt->quantization;
+	mfmt->xfer_func = pfmt->xfer_func;
+}
+
+static int vimc_get_mbus_format(struct media_pad *pad,
+				struct v4l2_subdev_format *fmt)
+{
+	if (is_media_entity_v4l2_subdev(pad->entity)) {
+		struct v4l2_subdev *sd =
+			media_entity_to_v4l2_subdev(pad->entity);
+		int ret;
+
+		fmt->which = V4L2_SUBDEV_FORMAT_ACTIVE;
+		fmt->pad = pad->index;
+
+		ret = v4l2_subdev_call(sd, pad, get_fmt, NULL, fmt);
+		if (ret)
+			return ret;
+
+	} else if (is_media_entity_v4l2_video_device(pad->entity)) {
+		struct video_device *vdev = container_of(pad->entity,
+							 struct video_device,
+							 entity);
+		struct vimc_ent_device *ved = video_get_drvdata(vdev);
+		struct v4l2_pix_format vdev_fmt;
+
+		if (!ved->vdev_get_format)
+			return -ENOIOCTLCMD;
+
+		ved->vdev_get_format(ved, &vdev_fmt);
+		vimc_fmt_pix_to_mbus(&fmt->format, &vdev_fmt);
+	} else {
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+int vimc_link_validate(struct media_link *link)
+{
+	struct v4l2_subdev_format source_fmt, sink_fmt;
+	int ret;
+
+	/*
+	 * if it is a raw node from vimc-core, ignore the link for now
+	 * TODO: remove this when there are no more raw nodes in the
+	 * core and return error instead
+	 */
+	if (link->source->entity->obj_type == MEDIA_ENTITY_TYPE_BASE)
+		return 0;
+
+	ret = vimc_get_mbus_format(link->source, &source_fmt);
+	if (ret)
+		return ret;
+
+	ret = vimc_get_mbus_format(link->sink, &sink_fmt);
+	if (ret)
+		return ret;
+
+	pr_info("vimc link validate: "
+		"%s:src:%dx%d (0x%x, %d, %d, %d, %d) "
+		"%s:snk:%dx%d (0x%x, %d, %d, %d, %d)\n",
+		/* src */
+		link->source->entity->name,
+		source_fmt.format.width, source_fmt.format.height,
+		source_fmt.format.code, source_fmt.format.colorspace,
+		source_fmt.format.quantization, source_fmt.format.xfer_func,
+		source_fmt.format.ycbcr_enc,
+		/* sink */
+		link->sink->entity->name,
+		sink_fmt.format.width, sink_fmt.format.height,
+		sink_fmt.format.code, sink_fmt.format.colorspace,
+		sink_fmt.format.quantization, sink_fmt.format.xfer_func,
+		sink_fmt.format.ycbcr_enc);
+
+	/* The width, height, code and colorspace must match. */
+	if (source_fmt.format.width != sink_fmt.format.width
+	    || source_fmt.format.height != sink_fmt.format.height
+	    || source_fmt.format.code != sink_fmt.format.code
+	    || source_fmt.format.colorspace != sink_fmt.format.colorspace)
+		return -EPIPE;
+
+	/* Colorimetry must match if they are not set to DEFAULT */
+	if (source_fmt.format.ycbcr_enc != V4L2_YCBCR_ENC_DEFAULT
+	    && sink_fmt.format.ycbcr_enc != V4L2_YCBCR_ENC_DEFAULT
+	    && source_fmt.format.ycbcr_enc != sink_fmt.format.ycbcr_enc)
+		return -EPIPE;
+
+	if (source_fmt.format.quantization != V4L2_QUANTIZATION_DEFAULT
+	    && sink_fmt.format.quantization != V4L2_QUANTIZATION_DEFAULT
+	    && source_fmt.format.quantization != sink_fmt.format.quantization)
+		return -EPIPE;
+
+	if (source_fmt.format.xfer_func != V4L2_XFER_FUNC_DEFAULT
+	    && sink_fmt.format.xfer_func != V4L2_XFER_FUNC_DEFAULT
+	    && source_fmt.format.xfer_func != sink_fmt.format.xfer_func)
+		return -EPIPE;
+
+	/* The field order must match, or the sink field order must be NONE
+	 * to support interlaced hardware connected to bridges that support
+	 * progressive formats only.
+	 */
+	if (source_fmt.format.field != sink_fmt.format.field &&
+	    sink_fmt.format.field != V4L2_FIELD_NONE)
+		return -EPIPE;
+
+	return 0;
+}
+EXPORT_SYMBOL(vimc_link_validate);
+
 static const struct media_entity_operations vimc_ent_sd_mops = {
-	.link_validate = v4l2_subdev_link_validate,
+	.link_validate = vimc_link_validate,
 };
 
 int vimc_ent_sd_register(struct vimc_ent_device *ved,
diff --git a/drivers/media/platform/vimc/vimc-common.h b/drivers/media/platform/vimc/vimc-common.h
index 73e7e94..60ebde2 100644
--- a/drivers/media/platform/vimc/vimc-common.h
+++ b/drivers/media/platform/vimc/vimc-common.h
@@ -45,6 +45,9 @@ struct vimc_pix_map {
  * @pads:		the list of pads of the node
  * @destroy:		callback to destroy the node
  * @process_frame:	callback send a frame to that node
+ * @vdev_get_format:	callback that returns the current format a pad, used
+ *			only when is_media_entity_v4l2_video_device(ent) returns
+ *			true
  *
  * Each node of the topology must create a vimc_ent_device struct. Depending on
  * the node it will be of an instance of v4l2_subdev or video_device struct
@@ -60,6 +63,8 @@ struct vimc_ent_device {
 	void (*destroy)(struct vimc_ent_device *);
 	void (*process_frame)(struct vimc_ent_device *ved,
 			      struct media_pad *sink, const void *frame);
+	void (*vdev_get_format)(struct vimc_ent_device *ved,
+			      struct v4l2_pix_format *fmt);
 };
 
 /**
@@ -160,4 +165,13 @@ int vimc_ent_sd_register(struct vimc_ent_device *ved,
 void vimc_ent_sd_unregister(struct vimc_ent_device *ved,
 			    struct v4l2_subdev *sd);
 
+/**
+ * vimc_link_validate - validates a media link
+ *
+ * @link: pointer to &struct media_link
+ *
+ * This function calls validates if a media link is valid for streaming.
+ */
+int vimc_link_validate(struct media_link *link);
+
 #endif
-- 
2.7.4

Re: [RFC PATCH v3 05/11] [media] vimc: common: Add vimc_link_validate

[Hans Verkuil]

On 06/03/2017 04:58 AM, Helen Koike wrote:
> All links will be checked in the same way. Adding a helper function for
> that
> 
> Signed-off-by: Helen Koike <helen.koike@collabora.com>
> 
> ---
> 
> Changes in v3:
> [media] vimc: common: Add vimc_link_validate
> 	- this is a new patch in the series
> 
> Changes in v2: None
> 
> 
> ---
>   drivers/media/platform/vimc/vimc-capture.c |  78 +++---------------
>   drivers/media/platform/vimc/vimc-common.c  | 124 ++++++++++++++++++++++++++++-
>   drivers/media/platform/vimc/vimc-common.h  |  14 ++++
>   3 files changed, 148 insertions(+), 68 deletions(-)
> 
> diff --git a/drivers/media/platform/vimc/vimc-capture.c b/drivers/media/platform/vimc/vimc-capture.c
> index 93f6a09..5bdecd1 100644
> --- a/drivers/media/platform/vimc/vimc-capture.c
> +++ b/drivers/media/platform/vimc/vimc-capture.c
> @@ -64,6 +64,15 @@ static int vimc_cap_querycap(struct file *file, void *priv,
>   	return 0;
>   }
>   
> +static void vimc_cap_get_format(struct vimc_ent_device *ved,
> +				struct v4l2_pix_format *fmt)
> +{
> +	struct vimc_cap_device *vcap = container_of(ved, struct vimc_cap_device,
> +						    ved);
> +
> +	*fmt = vcap->format;
> +}
> +
>   static int vimc_cap_fmt_vid_cap(struct file *file, void *priv,
>   				  struct v4l2_format *f)
>   {
> @@ -231,74 +240,8 @@ static const struct vb2_ops vimc_cap_qops = {
>   	.wait_finish		= vb2_ops_wait_finish,
>   };
>   
> -/*
> - * NOTE: this function is a copy of v4l2_subdev_link_validate_get_format
> - * maybe the v4l2 function should be public
> - */
> -static int vimc_cap_v4l2_subdev_link_validate_get_format(struct media_pad *pad,
> -						struct v4l2_subdev_format *fmt)
> -{
> -	struct v4l2_subdev *sd = media_entity_to_v4l2_subdev(pad->entity);
> -
> -	fmt->which = V4L2_SUBDEV_FORMAT_ACTIVE;
> -	fmt->pad = pad->index;
> -
> -	return v4l2_subdev_call(sd, pad, get_fmt, NULL, fmt);
> -}
> -
> -static int vimc_cap_link_validate(struct media_link *link)
> -{
> -	struct v4l2_subdev_format source_fmt;
> -	const struct vimc_pix_map *vpix;
> -	struct vimc_cap_device *vcap = container_of(link->sink->entity,
> -						    struct vimc_cap_device,
> -						    vdev.entity);
> -	struct v4l2_pix_format *sink_fmt = &vcap->format;
> -	int ret;
> -
> -	/*
> -	 * if it is a raw node from vimc-core, ignore the link for now
> -	 * TODO: remove this when there are no more raw nodes in the
> -	 * core and return error instead
> -	 */
> -	if (link->source->entity->obj_type == MEDIA_ENTITY_TYPE_BASE)
> -		return 0;
> -
> -	/* Get the the format of the subdev */
> -	ret = vimc_cap_v4l2_subdev_link_validate_get_format(link->source,
> -							    &source_fmt);
> -	if (ret)
> -		return ret;
> -
> -	dev_dbg(vcap->vdev.v4l2_dev->dev,
> -		"%s: link validate formats src:%dx%d %d sink:%dx%d %d\n",
> -		vcap->vdev.name,
> -		source_fmt.format.width, source_fmt.format.height,
> -		source_fmt.format.code,
> -		sink_fmt->width, sink_fmt->height,
> -		sink_fmt->pixelformat);
> -
> -	/* The width, height and code must match. */
> -	vpix = vimc_pix_map_by_pixelformat(sink_fmt->pixelformat);
> -	if (source_fmt.format.width != sink_fmt->width
> -	    || source_fmt.format.height != sink_fmt->height
> -	    || vpix->code != source_fmt.format.code)
> -		return -EPIPE;
> -
> -	/*
> -	 * The field order must match, or the sink field order must be NONE
> -	 * to support interlaced hardware connected to bridges that support
> -	 * progressive formats only.
> -	 */
> -	if (source_fmt.format.field != sink_fmt->field &&
> -	    sink_fmt->field != V4L2_FIELD_NONE)
> -		return -EPIPE;
> -
> -	return 0;
> -}
> -
>   static const struct media_entity_operations vimc_cap_mops = {
> -	.link_validate		= vimc_cap_link_validate,
> +	.link_validate		= vimc_link_validate,
>   };
>   
>   static void vimc_cap_destroy(struct vimc_ent_device *ved)
> @@ -434,6 +377,7 @@ struct vimc_ent_device *vimc_cap_create(struct v4l2_device *v4l2_dev,
>   	vcap->ved.destroy = vimc_cap_destroy;
>   	vcap->ved.ent = &vcap->vdev.entity;
>   	vcap->ved.process_frame = vimc_cap_process_frame;
> +	vcap->ved.vdev_get_format = vimc_cap_get_format;
>   
>   	/* Initialize the video_device struct */
>   	vdev = &vcap->vdev;
> diff --git a/drivers/media/platform/vimc/vimc-common.c b/drivers/media/platform/vimc/vimc-common.c
> index f809a9d..83d4251 100644
> --- a/drivers/media/platform/vimc/vimc-common.c
> +++ b/drivers/media/platform/vimc/vimc-common.c
> @@ -252,8 +252,130 @@ int vimc_pipeline_s_stream(struct media_entity *ent, int enable)
>   	return 0;
>   }
>   
> +static void vimc_fmt_pix_to_mbus(struct v4l2_mbus_framefmt *mfmt,
> +				 struct v4l2_pix_format *pfmt)
> +{
> +	const struct vimc_pix_map *vpix =
> +		vimc_pix_map_by_pixelformat(pfmt->pixelformat);
> +
> +	mfmt->width = pfmt->width;
> +	mfmt->height = pfmt->height;
> +	mfmt->code = vpix->code;
> +	mfmt->field = pfmt->field;
> +	mfmt->colorspace = pfmt->colorspace;
> +	mfmt->ycbcr_enc = pfmt->ycbcr_enc;
> +	mfmt->quantization = pfmt->quantization;
> +	mfmt->xfer_func = pfmt->xfer_func;

You can use v4l2_fill_mbus_format() here.

> +}
> +
> +static int vimc_get_mbus_format(struct media_pad *pad,
> +				struct v4l2_subdev_format *fmt)
> +{
> +	if (is_media_entity_v4l2_subdev(pad->entity)) {
> +		struct v4l2_subdev *sd =
> +			media_entity_to_v4l2_subdev(pad->entity);
> +		int ret;
> +
> +		fmt->which = V4L2_SUBDEV_FORMAT_ACTIVE;
> +		fmt->pad = pad->index;
> +
> +		ret = v4l2_subdev_call(sd, pad, get_fmt, NULL, fmt);
> +		if (ret)
> +			return ret;
> +
> +	} else if (is_media_entity_v4l2_video_device(pad->entity)) {
> +		struct video_device *vdev = container_of(pad->entity,
> +							 struct video_device,
> +							 entity);
> +		struct vimc_ent_device *ved = video_get_drvdata(vdev);
> +		struct v4l2_pix_format vdev_fmt;
> +
> +		if (!ved->vdev_get_format)
> +			return -ENOIOCTLCMD;
> +
> +		ved->vdev_get_format(ved, &vdev_fmt);
> +		vimc_fmt_pix_to_mbus(&fmt->format, &vdev_fmt);
> +	} else {
> +		return -EINVAL;
> +	}
> +
> +	return 0;
> +}
> +
> +int vimc_link_validate(struct media_link *link)
> +{
> +	struct v4l2_subdev_format source_fmt, sink_fmt;
> +	int ret;
> +
> +	/*
> +	 * if it is a raw node from vimc-core, ignore the link for now
> +	 * TODO: remove this when there are no more raw nodes in the
> +	 * core and return error instead
> +	 */
> +	if (link->source->entity->obj_type == MEDIA_ENTITY_TYPE_BASE)
> +		return 0;
> +
> +	ret = vimc_get_mbus_format(link->source, &source_fmt);
> +	if (ret)
> +		return ret;
> +
> +	ret = vimc_get_mbus_format(link->sink, &sink_fmt);
> +	if (ret)
> +		return ret;
> +
> +	pr_info("vimc link validate: "
> +		"%s:src:%dx%d (0x%x, %d, %d, %d, %d) "
> +		"%s:snk:%dx%d (0x%x, %d, %d, %d, %d)\n",
> +		/* src */
> +		link->source->entity->name,
> +		source_fmt.format.width, source_fmt.format.height,
> +		source_fmt.format.code, source_fmt.format.colorspace,
> +		source_fmt.format.quantization, source_fmt.format.xfer_func,
> +		source_fmt.format.ycbcr_enc,
> +		/* sink */
> +		link->sink->entity->name,
> +		sink_fmt.format.width, sink_fmt.format.height,
> +		sink_fmt.format.code, sink_fmt.format.colorspace,
> +		sink_fmt.format.quantization, sink_fmt.format.xfer_func,
> +		sink_fmt.format.ycbcr_enc);
> +
> +	/* The width, height, code and colorspace must match. */
> +	if (source_fmt.format.width != sink_fmt.format.width
> +	    || source_fmt.format.height != sink_fmt.format.height
> +	    || source_fmt.format.code != sink_fmt.format.code
> +	    || source_fmt.format.colorspace != sink_fmt.format.colorspace)

Source and/or Sink may be COLORSPACE_DEFAULT. If that's the case, then
you should skip comparing ycbcr_enc, quantization or xfer_func. If colorspace
is DEFAULT, then that implies that the other fields are DEFAULT as well. Nothing
else makes sense in that case.

> +		return -EPIPE;
> +
> +	/* Colorimetry must match if they are not set to DEFAULT */
> +	if (source_fmt.format.ycbcr_enc != V4L2_YCBCR_ENC_DEFAULT
> +	    && sink_fmt.format.ycbcr_enc != V4L2_YCBCR_ENC_DEFAULT
> +	    && source_fmt.format.ycbcr_enc != sink_fmt.format.ycbcr_enc)
> +		return -EPIPE;
> +
> +	if (source_fmt.format.quantization != V4L2_QUANTIZATION_DEFAULT
> +	    && sink_fmt.format.quantization != V4L2_QUANTIZATION_DEFAULT
> +	    && source_fmt.format.quantization != sink_fmt.format.quantization)
> +		return -EPIPE;
> +
> +	if (source_fmt.format.xfer_func != V4L2_XFER_FUNC_DEFAULT
> +	    && sink_fmt.format.xfer_func != V4L2_XFER_FUNC_DEFAULT
> +	    && source_fmt.format.xfer_func != sink_fmt.format.xfer_func)
> +		return -EPIPE;
> +
> +	/* The field order must match, or the sink field order must be NONE
> +	 * to support interlaced hardware connected to bridges that support
> +	 * progressive formats only.
> +	 */
> +	if (source_fmt.format.field != sink_fmt.format.field &&
> +	    sink_fmt.format.field != V4L2_FIELD_NONE)
> +		return -EPIPE;
> +
> +	return 0;
> +}
> +EXPORT_SYMBOL(vimc_link_validate);
> +
>   static const struct media_entity_operations vimc_ent_sd_mops = {
> -	.link_validate = v4l2_subdev_link_validate,
> +	.link_validate = vimc_link_validate,
>   };
>   
>   int vimc_ent_sd_register(struct vimc_ent_device *ved,
> diff --git a/drivers/media/platform/vimc/vimc-common.h b/drivers/media/platform/vimc/vimc-common.h
> index 73e7e94..60ebde2 100644
> --- a/drivers/media/platform/vimc/vimc-common.h
> +++ b/drivers/media/platform/vimc/vimc-common.h
> @@ -45,6 +45,9 @@ struct vimc_pix_map {
>    * @pads:		the list of pads of the node
>    * @destroy:		callback to destroy the node
>    * @process_frame:	callback send a frame to that node
> + * @vdev_get_format:	callback that returns the current format a pad, used
> + *			only when is_media_entity_v4l2_video_device(ent) returns
> + *			true
>    *
>    * Each node of the topology must create a vimc_ent_device struct. Depending on
>    * the node it will be of an instance of v4l2_subdev or video_device struct
> @@ -60,6 +63,8 @@ struct vimc_ent_device {
>   	void (*destroy)(struct vimc_ent_device *);
>   	void (*process_frame)(struct vimc_ent_device *ved,
>   			      struct media_pad *sink, const void *frame);
> +	void (*vdev_get_format)(struct vimc_ent_device *ved,
> +			      struct v4l2_pix_format *fmt);
>   };
>   
>   /**
> @@ -160,4 +165,13 @@ int vimc_ent_sd_register(struct vimc_ent_device *ved,
>   void vimc_ent_sd_unregister(struct vimc_ent_device *ved,
>   			    struct v4l2_subdev *sd);
>   
> +/**
> + * vimc_link_validate - validates a media link
> + *
> + * @link: pointer to &struct media_link
> + *
> + * This function calls validates if a media link is valid for streaming.
> + */
> +int vimc_link_validate(struct media_link *link);
> +
>   #endif
> 

Regards,

	Hans

Re: [RFC PATCH v3 05/11] [media] vimc: common: Add vimc_link_validate

[Helen Koike]

Hi Hans,

Thanks for your review, just a question below

On 2017-06-12 06:50 AM, Hans Verkuil wrote:
> On 06/03/2017 04:58 AM, Helen Koike wrote:
>> All links will be checked in the same way. Adding a helper function for
>> that
>>
>> Signed-off-by: Helen Koike <helen.koike@collabora.com>
>>
>> ---
>>
>> Changes in v3:
>> [media] vimc: common: Add vimc_link_validate
>>     - this is a new patch in the series
>>
>> Changes in v2: None
>>
>>
>> ---
>>   drivers/media/platform/vimc/vimc-capture.c |  78 +++---------------
>>   drivers/media/platform/vimc/vimc-common.c  | 124
>> ++++++++++++++++++++++++++++-
>>   drivers/media/platform/vimc/vimc-common.h  |  14 ++++
>>   3 files changed, 148 insertions(+), 68 deletions(-)
>>
>> diff --git a/drivers/media/platform/vimc/vimc-capture.c
>> b/drivers/media/platform/vimc/vimc-capture.c
>> index 93f6a09..5bdecd1 100644
>> --- a/drivers/media/platform/vimc/vimc-capture.c
>> +++ b/drivers/media/platform/vimc/vimc-capture.c
>> @@ -64,6 +64,15 @@ static int vimc_cap_querycap(struct file *file,
>> void *priv,
>>       return 0;
>>   }
>>   +static void vimc_cap_get_format(struct vimc_ent_device *ved,
>> +                struct v4l2_pix_format *fmt)
>> +{
>> +    struct vimc_cap_device *vcap = container_of(ved, struct
>> vimc_cap_device,
>> +                            ved);
>> +
>> +    *fmt = vcap->format;
>> +}
>> +
>>   static int vimc_cap_fmt_vid_cap(struct file *file, void *priv,
>>                     struct v4l2_format *f)
>>   {
>> @@ -231,74 +240,8 @@ static const struct vb2_ops vimc_cap_qops = {
>>       .wait_finish        = vb2_ops_wait_finish,
>>   };
>>   -/*
>> - * NOTE: this function is a copy of v4l2_subdev_link_validate_get_format
>> - * maybe the v4l2 function should be public
>> - */
>> -static int vimc_cap_v4l2_subdev_link_validate_get_format(struct
>> media_pad *pad,
>> -                        struct v4l2_subdev_format *fmt)
>> -{
>> -    struct v4l2_subdev *sd = media_entity_to_v4l2_subdev(pad->entity);
>> -
>> -    fmt->which = V4L2_SUBDEV_FORMAT_ACTIVE;
>> -    fmt->pad = pad->index;
>> -
>> -    return v4l2_subdev_call(sd, pad, get_fmt, NULL, fmt);
>> -}
>> -
>> -static int vimc_cap_link_validate(struct media_link *link)
>> -{
>> -    struct v4l2_subdev_format source_fmt;
>> -    const struct vimc_pix_map *vpix;
>> -    struct vimc_cap_device *vcap = container_of(link->sink->entity,
>> -                            struct vimc_cap_device,
>> -                            vdev.entity);
>> -    struct v4l2_pix_format *sink_fmt = &vcap->format;
>> -    int ret;
>> -
>> -    /*
>> -     * if it is a raw node from vimc-core, ignore the link for now
>> -     * TODO: remove this when there are no more raw nodes in the
>> -     * core and return error instead
>> -     */
>> -    if (link->source->entity->obj_type == MEDIA_ENTITY_TYPE_BASE)
>> -        return 0;
>> -
>> -    /* Get the the format of the subdev */
>> -    ret = vimc_cap_v4l2_subdev_link_validate_get_format(link->source,
>> -                                &source_fmt);
>> -    if (ret)
>> -        return ret;
>> -
>> -    dev_dbg(vcap->vdev.v4l2_dev->dev,
>> -        "%s: link validate formats src:%dx%d %d sink:%dx%d %d\n",
>> -        vcap->vdev.name,
>> -        source_fmt.format.width, source_fmt.format.height,
>> -        source_fmt.format.code,
>> -        sink_fmt->width, sink_fmt->height,
>> -        sink_fmt->pixelformat);
>> -
>> -    /* The width, height and code must match. */
>> -    vpix = vimc_pix_map_by_pixelformat(sink_fmt->pixelformat);
>> -    if (source_fmt.format.width != sink_fmt->width
>> -        || source_fmt.format.height != sink_fmt->height
>> -        || vpix->code != source_fmt.format.code)
>> -        return -EPIPE;
>> -
>> -    /*
>> -     * The field order must match, or the sink field order must be NONE
>> -     * to support interlaced hardware connected to bridges that support
>> -     * progressive formats only.
>> -     */
>> -    if (source_fmt.format.field != sink_fmt->field &&
>> -        sink_fmt->field != V4L2_FIELD_NONE)
>> -        return -EPIPE;
>> -
>> -    return 0;
>> -}
>> -
>>   static const struct media_entity_operations vimc_cap_mops = {
>> -    .link_validate        = vimc_cap_link_validate,
>> +    .link_validate        = vimc_link_validate,
>>   };
>>     static void vimc_cap_destroy(struct vimc_ent_device *ved)
>> @@ -434,6 +377,7 @@ struct vimc_ent_device *vimc_cap_create(struct
>> v4l2_device *v4l2_dev,
>>       vcap->ved.destroy = vimc_cap_destroy;
>>       vcap->ved.ent = &vcap->vdev.entity;
>>       vcap->ved.process_frame = vimc_cap_process_frame;
>> +    vcap->ved.vdev_get_format = vimc_cap_get_format;
>>         /* Initialize the video_device struct */
>>       vdev = &vcap->vdev;
>> diff --git a/drivers/media/platform/vimc/vimc-common.c
>> b/drivers/media/platform/vimc/vimc-common.c
>> index f809a9d..83d4251 100644
>> --- a/drivers/media/platform/vimc/vimc-common.c
>> +++ b/drivers/media/platform/vimc/vimc-common.c
>> @@ -252,8 +252,130 @@ int vimc_pipeline_s_stream(struct media_entity
>> *ent, int enable)
>>       return 0;
>>   }
>>   +static void vimc_fmt_pix_to_mbus(struct v4l2_mbus_framefmt *mfmt,
>> +                 struct v4l2_pix_format *pfmt)
>> +{
>> +    const struct vimc_pix_map *vpix =
>> +        vimc_pix_map_by_pixelformat(pfmt->pixelformat);
>> +
>> +    mfmt->width = pfmt->width;
>> +    mfmt->height = pfmt->height;
>> +    mfmt->code = vpix->code;
>> +    mfmt->field = pfmt->field;
>> +    mfmt->colorspace = pfmt->colorspace;
>> +    mfmt->ycbcr_enc = pfmt->ycbcr_enc;
>> +    mfmt->quantization = pfmt->quantization;
>> +    mfmt->xfer_func = pfmt->xfer_func;
>
> You can use v4l2_fill_mbus_format() here.
>
>> +}
>> +
>> +static int vimc_get_mbus_format(struct media_pad *pad,
>> +                struct v4l2_subdev_format *fmt)
>> +{
>> +    if (is_media_entity_v4l2_subdev(pad->entity)) {
>> +        struct v4l2_subdev *sd =
>> +            media_entity_to_v4l2_subdev(pad->entity);
>> +        int ret;
>> +
>> +        fmt->which = V4L2_SUBDEV_FORMAT_ACTIVE;
>> +        fmt->pad = pad->index;
>> +
>> +        ret = v4l2_subdev_call(sd, pad, get_fmt, NULL, fmt);
>> +        if (ret)
>> +            return ret;
>> +
>> +    } else if (is_media_entity_v4l2_video_device(pad->entity)) {
>> +        struct video_device *vdev = container_of(pad->entity,
>> +                             struct video_device,
>> +                             entity);
>> +        struct vimc_ent_device *ved = video_get_drvdata(vdev);
>> +        struct v4l2_pix_format vdev_fmt;
>> +
>> +        if (!ved->vdev_get_format)
>> +            return -ENOIOCTLCMD;
>> +
>> +        ved->vdev_get_format(ved, &vdev_fmt);
>> +        vimc_fmt_pix_to_mbus(&fmt->format, &vdev_fmt);
>> +    } else {
>> +        return -EINVAL;
>> +    }
>> +
>> +    return 0;
>> +}
>> +
>> +int vimc_link_validate(struct media_link *link)
>> +{
>> +    struct v4l2_subdev_format source_fmt, sink_fmt;
>> +    int ret;
>> +
>> +    /*
>> +     * if it is a raw node from vimc-core, ignore the link for now
>> +     * TODO: remove this when there are no more raw nodes in the
>> +     * core and return error instead
>> +     */
>> +    if (link->source->entity->obj_type == MEDIA_ENTITY_TYPE_BASE)
>> +        return 0;
>> +
>> +    ret = vimc_get_mbus_format(link->source, &source_fmt);
>> +    if (ret)
>> +        return ret;
>> +
>> +    ret = vimc_get_mbus_format(link->sink, &sink_fmt);
>> +    if (ret)
>> +        return ret;
>> +
>> +    pr_info("vimc link validate: "
>> +        "%s:src:%dx%d (0x%x, %d, %d, %d, %d) "
>> +        "%s:snk:%dx%d (0x%x, %d, %d, %d, %d)\n",
>> +        /* src */
>> +        link->source->entity->name,
>> +        source_fmt.format.width, source_fmt.format.height,
>> +        source_fmt.format.code, source_fmt.format.colorspace,
>> +        source_fmt.format.quantization, source_fmt.format.xfer_func,
>> +        source_fmt.format.ycbcr_enc,
>> +        /* sink */
>> +        link->sink->entity->name,
>> +        sink_fmt.format.width, sink_fmt.format.height,
>> +        sink_fmt.format.code, sink_fmt.format.colorspace,
>> +        sink_fmt.format.quantization, sink_fmt.format.xfer_func,
>> +        sink_fmt.format.ycbcr_enc);
>> +
>> +    /* The width, height, code and colorspace must match. */
>> +    if (source_fmt.format.width != sink_fmt.format.width
>> +        || source_fmt.format.height != sink_fmt.format.height
>> +        || source_fmt.format.code != sink_fmt.format.code
>> +        || source_fmt.format.colorspace != sink_fmt.format.colorspace)
>
> Source and/or Sink may be COLORSPACE_DEFAULT. If that's the case, then
> you should skip comparing ycbcr_enc, quantization or xfer_func. If
> colorspace
> is DEFAULT, then that implies that the other fields are DEFAULT as well.
> Nothing
> else makes sense in that case.

I thought that the colorspace couldn't be COLORSPACE_DEFAULT, in the 
documentation it is written "The default colorspace. This can be used by 
applications to let the driver fill in the colorspace.", so the 
colorspace is always set to something different from default no ?
I thought that the COLORSPACE_DEFAULT was only used by the userspace in 
VIDIOC_{SUBDEV}_S_FMT so say "driver, use wherever colorspace you want", 
but if usespace calls VIDIOC_{SUBDEV}_G_FMT, it would return which exact 
colorspace the driver is using, no?

>
>> +        return -EPIPE;
>> +
>> +    /* Colorimetry must match if they are not set to DEFAULT */
>> +    if (source_fmt.format.ycbcr_enc != V4L2_YCBCR_ENC_DEFAULT
>> +        && sink_fmt.format.ycbcr_enc != V4L2_YCBCR_ENC_DEFAULT
>> +        && source_fmt.format.ycbcr_enc != sink_fmt.format.ycbcr_enc)
>> +        return -EPIPE;
>> +
>> +    if (source_fmt.format.quantization != V4L2_QUANTIZATION_DEFAULT
>> +        && sink_fmt.format.quantization != V4L2_QUANTIZATION_DEFAULT
>> +        && source_fmt.format.quantization !=
>> sink_fmt.format.quantization)
>> +        return -EPIPE;
>> +
>> +    if (source_fmt.format.xfer_func != V4L2_XFER_FUNC_DEFAULT
>> +        && sink_fmt.format.xfer_func != V4L2_XFER_FUNC_DEFAULT
>> +        && source_fmt.format.xfer_func != sink_fmt.format.xfer_func)
>> +        return -EPIPE;
>> +
>> +    /* The field order must match, or the sink field order must be NONE
>> +     * to support interlaced hardware connected to bridges that support
>> +     * progressive formats only.
>> +     */
>> +    if (source_fmt.format.field != sink_fmt.format.field &&
>> +        sink_fmt.format.field != V4L2_FIELD_NONE)
>> +        return -EPIPE;
>> +
>> +    return 0;
>> +}
>> +EXPORT_SYMBOL(vimc_link_validate);
>> +
>>   static const struct media_entity_operations vimc_ent_sd_mops = {
>> -    .link_validate = v4l2_subdev_link_validate,
>> +    .link_validate = vimc_link_validate,
>>   };
>>     int vimc_ent_sd_register(struct vimc_ent_device *ved,
>> diff --git a/drivers/media/platform/vimc/vimc-common.h
>> b/drivers/media/platform/vimc/vimc-common.h
>> index 73e7e94..60ebde2 100644
>> --- a/drivers/media/platform/vimc/vimc-common.h
>> +++ b/drivers/media/platform/vimc/vimc-common.h
>> @@ -45,6 +45,9 @@ struct vimc_pix_map {
>>    * @pads:        the list of pads of the node
>>    * @destroy:        callback to destroy the node
>>    * @process_frame:    callback send a frame to that node
>> + * @vdev_get_format:    callback that returns the current format a
>> pad, used
>> + *            only when is_media_entity_v4l2_video_device(ent) returns
>> + *            true
>>    *
>>    * Each node of the topology must create a vimc_ent_device struct.
>> Depending on
>>    * the node it will be of an instance of v4l2_subdev or video_device
>> struct
>> @@ -60,6 +63,8 @@ struct vimc_ent_device {
>>       void (*destroy)(struct vimc_ent_device *);
>>       void (*process_frame)(struct vimc_ent_device *ved,
>>                     struct media_pad *sink, const void *frame);
>> +    void (*vdev_get_format)(struct vimc_ent_device *ved,
>> +                  struct v4l2_pix_format *fmt);
>>   };
>>     /**
>> @@ -160,4 +165,13 @@ int vimc_ent_sd_register(struct vimc_ent_device
>> *ved,
>>   void vimc_ent_sd_unregister(struct vimc_ent_device *ved,
>>                   struct v4l2_subdev *sd);
>>   +/**
>> + * vimc_link_validate - validates a media link
>> + *
>> + * @link: pointer to &struct media_link
>> + *
>> + * This function calls validates if a media link is valid for streaming.
>> + */
>> +int vimc_link_validate(struct media_link *link);
>> +
>>   #endif
>>
>
> Regards,
>
>     Hans

Thanks,
Helen

Re: [RFC PATCH v3 05/11] [media] vimc: common: Add vimc_link_validate

[Hans Verkuil]

On 06/12/2017 07:20 PM, Helen Koike wrote:
> Hi Hans,
> 
> Thanks for your review, just a question below
> 
> On 2017-06-12 06:50 AM, Hans Verkuil wrote:
>> On 06/03/2017 04:58 AM, Helen Koike wrote:
>>> +    /* The width, height, code and colorspace must match. */
>>> +    if (source_fmt.format.width != sink_fmt.format.width
>>> +        || source_fmt.format.height != sink_fmt.format.height
>>> +        || source_fmt.format.code != sink_fmt.format.code
>>> +        || source_fmt.format.colorspace != sink_fmt.format.colorspace)
>>
>> Source and/or Sink may be COLORSPACE_DEFAULT. If that's the case, then
>> you should skip comparing ycbcr_enc, quantization or xfer_func. If
>> colorspace
>> is DEFAULT, then that implies that the other fields are DEFAULT as well.
>> Nothing
>> else makes sense in that case.
> 
> I thought that the colorspace couldn't be COLORSPACE_DEFAULT, in the
> documentation it is written "The default colorspace. This can be used by
> applications to let the driver fill in the colorspace.", so the
> colorspace is always set to something different from default no ?
> I thought that the COLORSPACE_DEFAULT was only used by the userspace in
> VIDIOC_{SUBDEV}_S_FMT so say "driver, use wherever colorspace you want",
> but if usespace calls VIDIOC_{SUBDEV}_G_FMT, it would return which exact
> colorspace the driver is using, no?

I don't think this rule works for the MC. For regular v4l2 devices the
bridge driver knows what colorspace it receives so it can replace the
colorspace DEFAULT with the real one.

But a e.g. scaler subdev does not actually touch on the colorspace. And
it doesn't know what colorspace it will receive or transmit.

I don't feel it makes sense to require userspace to set and propagate the
colorspace information throughout the pipeline. Allowing it to be set to
DEFAULT (i.e. 'don't care') makes sense to me.

I might change my mind later on this. The simple fact is that the spec isn't
clear what to do with MC devices. That's also where this vimc driver comes
in, so we can try this out without requiring specialized hardware.

Regards,

	Hans

[RFC PATCH v3 06/11] [media] vimc: sen: Support several image formats

[Helen Koike]

Allow user space to change the image format as the frame size, the
media bus pixel format, colorspace, quantization, field YCbCr encoding
and the transfer function

Signed-off-by: Helen Koike <helen.koike@collabora.com>

---

Changes in v3:
[media] vimc: sen: Support several image formats
	- remove support for V4L2_FIELD_ALTERNATE (left as TODO for now)
	- clamp image size to an even dimension for height and width
	- set default values for colorimetry using _DEFAULT macro
	- reset all values of colorimetry to _DEFAULT if user tries to
	set an invalid colorspace

Changes in v2:
[media] vimc: sen: Support several image formats
	- this is a new commit in the serie (the old one was splitted in two)
	- add init_cfg to initialize try_fmt
	- reorder code in vimc_sen_set_fmt
	- allow user space to change all fields from struct v4l2_mbus_framefmt
	  (e.g. colospace, quantization, field, xfer_func, ycbcr_enc)
	- merge with patch for the enum_mbus_code and enum_frame_size
	- change commit message
	- add vimc_pix_map_by_index
	- rename MIN/MAX macros
	- check set_fmt default parameters for quantization, colorspace ...media] vimc: sen: Support several image formats


---
 drivers/media/platform/vimc/vimc-common.c |   8 ++
 drivers/media/platform/vimc/vimc-common.h |  12 +++
 drivers/media/platform/vimc/vimc-sensor.c | 145 ++++++++++++++++++++++++------
 3 files changed, 136 insertions(+), 29 deletions(-)

diff --git a/drivers/media/platform/vimc/vimc-common.c b/drivers/media/platform/vimc/vimc-common.c
index 83d4251..ff59e09 100644
--- a/drivers/media/platform/vimc/vimc-common.c
+++ b/drivers/media/platform/vimc/vimc-common.c
@@ -144,6 +144,14 @@ static const struct vimc_pix_map vimc_pix_map_list[] = {
 	},
 };
 
+const struct vimc_pix_map *vimc_pix_map_by_index(unsigned int i)
+{
+	if (i >= ARRAY_SIZE(vimc_pix_map_list))
+		return NULL;
+
+	return &vimc_pix_map_list[i];
+}
+
 const struct vimc_pix_map *vimc_pix_map_by_code(u32 code)
 {
 	unsigned int i;
diff --git a/drivers/media/platform/vimc/vimc-common.h b/drivers/media/platform/vimc/vimc-common.h
index 60ebde2..2189fd6 100644
--- a/drivers/media/platform/vimc/vimc-common.h
+++ b/drivers/media/platform/vimc/vimc-common.h
@@ -22,6 +22,11 @@
 #include <media/media-device.h>
 #include <media/v4l2-device.h>
 
+#define VIMC_FRAME_MAX_WIDTH 4096
+#define VIMC_FRAME_MAX_HEIGHT 2160
+#define VIMC_FRAME_MIN_WIDTH 16
+#define VIMC_FRAME_MIN_HEIGHT 16
+
 /**
  * struct vimc_pix_map - maps media bus code with v4l2 pixel format
  *
@@ -113,6 +118,13 @@ static inline void vimc_pads_cleanup(struct media_pad *pads)
 int vimc_pipeline_s_stream(struct media_entity *ent, int enable);
 
 /**
+ * vimc_pix_map_by_index - get vimc_pix_map struct by its index
+ *
+ * @i:			index of the vimc_pix_map struct in vimc_pix_map_list
+ */
+const struct vimc_pix_map *vimc_pix_map_by_index(unsigned int i);
+
+/**
  * vimc_pix_map_by_code - get vimc_pix_map struct by media bus code
  *
  * @code:		media bus format code defined by MEDIA_BUS_FMT_* macros
diff --git a/drivers/media/platform/vimc/vimc-sensor.c b/drivers/media/platform/vimc/vimc-sensor.c
index 6386ac1..90c41c6 100644
--- a/drivers/media/platform/vimc/vimc-sensor.c
+++ b/drivers/media/platform/vimc/vimc-sensor.c
@@ -24,8 +24,6 @@
 
 #include "vimc-sensor.h"
 
-#define VIMC_SEN_FRAME_MAX_WIDTH 4096
-
 struct vimc_sen_device {
 	struct vimc_ent_device ved;
 	struct v4l2_subdev sd;
@@ -36,18 +34,39 @@ struct vimc_sen_device {
 	struct v4l2_mbus_framefmt mbus_format;
 };
 
+static const struct v4l2_mbus_framefmt fmt_default = {
+	.width = 640,
+	.height = 480,
+	.code = MEDIA_BUS_FMT_RGB888_1X24,
+	.field = V4L2_FIELD_NONE,
+	.colorspace = V4L2_COLORSPACE_SRGB,
+};
+
+static int vimc_sen_init_cfg(struct v4l2_subdev *sd,
+			     struct v4l2_subdev_pad_config *cfg)
+{
+	unsigned int i;
+
+	for (i = 0; i < sd->entity.num_pads; i++) {
+		struct v4l2_mbus_framefmt *mf;
+
+		mf = v4l2_subdev_get_try_format(sd, cfg, i);
+		*mf = fmt_default;
+	}
+
+	return 0;
+}
+
 static int vimc_sen_enum_mbus_code(struct v4l2_subdev *sd,
 				   struct v4l2_subdev_pad_config *cfg,
 				   struct v4l2_subdev_mbus_code_enum *code)
 {
-	struct vimc_sen_device *vsen =
-				container_of(sd, struct vimc_sen_device, sd);
+	const struct vimc_pix_map *vpix = vimc_pix_map_by_index(code->index);
 
-	/* TODO: Add support for other codes */
-	if (code->index)
+	if (!vpix)
 		return -EINVAL;
 
-	code->code = vsen->mbus_format.code;
+	code->code = vpix->code;
 
 	return 0;
 }
@@ -56,33 +75,34 @@ static int vimc_sen_enum_frame_size(struct v4l2_subdev *sd,
 				    struct v4l2_subdev_pad_config *cfg,
 				    struct v4l2_subdev_frame_size_enum *fse)
 {
-	struct vimc_sen_device *vsen =
-				container_of(sd, struct vimc_sen_device, sd);
+	const struct vimc_pix_map *vpix;
 
-	/* TODO: Add support to other formats */
 	if (fse->index)
 		return -EINVAL;
 
-	/* TODO: Add support for other codes */
-	if (fse->code != vsen->mbus_format.code)
+	/* Only accept code in the pix map table */
+	vpix = vimc_pix_map_by_code(fse->code);
+	if (!vpix)
 		return -EINVAL;
 
-	fse->min_width = vsen->mbus_format.width;
-	fse->max_width = vsen->mbus_format.width;
-	fse->min_height = vsen->mbus_format.height;
-	fse->max_height = vsen->mbus_format.height;
+	fse->min_width = VIMC_FRAME_MIN_WIDTH;
+	fse->max_width = VIMC_FRAME_MAX_WIDTH;
+	fse->min_height = VIMC_FRAME_MIN_HEIGHT;
+	fse->max_height = VIMC_FRAME_MAX_HEIGHT;
 
 	return 0;
 }
 
 static int vimc_sen_get_fmt(struct v4l2_subdev *sd,
 			    struct v4l2_subdev_pad_config *cfg,
-			    struct v4l2_subdev_format *format)
+			    struct v4l2_subdev_format *fmt)
 {
 	struct vimc_sen_device *vsen =
 				container_of(sd, struct vimc_sen_device, sd);
 
-	format->format = vsen->mbus_format;
+	fmt->format = fmt->which == V4L2_SUBDEV_FORMAT_TRY ?
+		      *v4l2_subdev_get_try_format(sd, cfg, fmt->pad) :
+		      vsen->mbus_format;
 
 	return 0;
 }
@@ -105,12 +125,85 @@ static void vimc_sen_tpg_s_format(struct vimc_sen_device *vsen)
 	tpg_s_xfer_func(&vsen->tpg, vsen->mbus_format.xfer_func);
 }
 
+static void vimc_sen_adjust_fmt(struct v4l2_mbus_framefmt *fmt)
+{
+	const struct vimc_pix_map *vpix;
+
+	/* Only accept code in the pix map table */
+	vpix = vimc_pix_map_by_code(fmt->code);
+	if (!vpix)
+		fmt->code = fmt_default.code;
+
+	fmt->width = clamp_t(u32, fmt->width, VIMC_FRAME_MIN_WIDTH,
+			     VIMC_FRAME_MAX_WIDTH) & ~1;
+	fmt->height = clamp_t(u32, fmt->height, VIMC_FRAME_MIN_HEIGHT,
+			      VIMC_FRAME_MAX_HEIGHT) & ~1;
+
+	/* TODO: add support for V4L2_FIELD_ALTERNATE */
+	if (fmt->field == V4L2_FIELD_ANY || fmt->field == V4L2_FIELD_ALTERNATE)
+		fmt->field = fmt_default.field;
+
+	if (fmt->colorspace == V4L2_COLORSPACE_DEFAULT)
+		fmt->colorspace = fmt_default.colorspace;
+
+	/* Check if values are out of range */
+	if (fmt->colorspace > V4L2_COLORSPACE_DCI_P3) {
+		fmt->colorspace = fmt_default.colorspace;
+		fmt->ycbcr_enc = V4L2_YCBCR_ENC_DEFAULT;
+		fmt->quantization = V4L2_QUANTIZATION_DEFAULT;
+		fmt->xfer_func = V4L2_XFER_FUNC_DEFAULT;
+	}
+	if (fmt->ycbcr_enc > V4L2_YCBCR_ENC_SMPTE240M)
+		fmt->ycbcr_enc = V4L2_YCBCR_ENC_DEFAULT;
+	if (fmt->quantization > V4L2_QUANTIZATION_LIM_RANGE)
+		fmt->quantization = V4L2_QUANTIZATION_DEFAULT;
+	if (fmt->xfer_func > V4L2_XFER_FUNC_SMPTE2084)
+		fmt->xfer_func = V4L2_XFER_FUNC_DEFAULT;
+}
+
+static int vimc_sen_set_fmt(struct v4l2_subdev *sd,
+			    struct v4l2_subdev_pad_config *cfg,
+			    struct v4l2_subdev_format *fmt)
+{
+	struct vimc_sen_device *vsen = v4l2_get_subdevdata(sd);
+	struct v4l2_mbus_framefmt *mf;
+
+	if (fmt->which == V4L2_SUBDEV_FORMAT_ACTIVE) {
+		/* Do not change the format while stream is on */
+		if (vsen->frame)
+			return -EBUSY;
+
+		mf = &vsen->mbus_format;
+	} else {
+		mf = v4l2_subdev_get_try_format(sd, cfg, fmt->pad);
+	}
+
+	/* Set the new format */
+	vimc_sen_adjust_fmt(&fmt->format);
+
+	dev_dbg(vsen->sd.v4l2_dev->mdev->dev, "%s: format update: "
+		"old:%dx%d (0x%x, %d, %d, %d, %d) "
+		"new:%dx%d (0x%x, %d, %d, %d, %d)\n", vsen->sd.name,
+		/* old */
+		mf->width, mf->height, mf->code,
+		mf->colorspace,	mf->quantization,
+		mf->xfer_func, mf->ycbcr_enc,
+		/* new */
+		fmt->format.width, fmt->format.height, fmt->format.code,
+		fmt->format.colorspace, fmt->format.quantization,
+		fmt->format.xfer_func, fmt->format.ycbcr_enc);
+
+	*mf = fmt->format;
+
+	return 0;
+}
+
 static const struct v4l2_subdev_pad_ops vimc_sen_pad_ops = {
+	.init_cfg		= vimc_sen_init_cfg,
 	.enum_mbus_code		= vimc_sen_enum_mbus_code,
 	.enum_frame_size	= vimc_sen_enum_frame_size,
 	.get_fmt		= vimc_sen_get_fmt,
-	/* TODO: Add support to other formats */
-	.set_fmt		= vimc_sen_get_fmt,
+	.set_fmt		= vimc_sen_set_fmt,
 };
 
 static int vimc_sen_tpg_thread(void *data)
@@ -247,19 +340,13 @@ struct vimc_ent_device *vimc_sen_create(struct v4l2_device *v4l2_dev,
 	if (ret)
 		goto err_free_vsen;
 
-	/* Set the active frame format (this is hardcoded for now) */
-	vsen->mbus_format.width = 640;
-	vsen->mbus_format.height = 480;
-	vsen->mbus_format.code = MEDIA_BUS_FMT_RGB888_1X24;
-	vsen->mbus_format.field = V4L2_FIELD_NONE;
-	vsen->mbus_format.colorspace = V4L2_COLORSPACE_SRGB;
-	vsen->mbus_format.quantization = V4L2_QUANTIZATION_FULL_RANGE;
-	vsen->mbus_format.xfer_func = V4L2_XFER_FUNC_SRGB;
+	/* Initialize the frame format */
+	vsen->mbus_format = fmt_default;
 
 	/* Initialize the test pattern generator */
 	tpg_init(&vsen->tpg, vsen->mbus_format.width,
 		 vsen->mbus_format.height);
-	ret = tpg_alloc(&vsen->tpg, VIMC_SEN_FRAME_MAX_WIDTH);
+	ret = tpg_alloc(&vsen->tpg, VIMC_FRAME_MAX_WIDTH);
 	if (ret)
 		goto err_unregister_ent_sd;
 
-- 
2.7.4

[RFC PATCH v3 07/11] [media] vimc: cap: Support several image formats

[Helen Koike]

Allow user space to change the image format as the frame size, the
pixel format, colorspace, quantization, field YCbCr encoding
and the transfer function

Signed-off-by: Helen Koike <helen.koike@collabora.com>

---

Changes in v3:
[media] vimc: cap: Support several image formats
	- use *_DEFAULT macros for colorimetry in the default format
	- clamp height and width of the image by an even value
	- is user try to set colorspace to an invalid format, set all
	colorimetry parameters to _DEFAULT
	- remove V4L2_FMT_FLAG_COMPRESSED from vimc_cap_enum_fmt_vid_cap
	- remove V4L2_BUF_TYPE_VIDEO_CAPTURE from vimc_cap_enum_fmt_vid_cap
	- increase step_width and step_height to 2 instead of 1
	- remove link validate function, use the one in vimc-common.c

Changes in v2:
[media] vimc: cap: Support several image formats
	- this is a new commit in the serie (the old one was splitted in two)
	- allow user space to change all fields from struct v4l2_pix_format
	  (e.g. colospace, quantization, field, xfer_func, ycbcr_enc)
	- link_validate and try_fmt: also checks colospace, quantization, field, xfer_func, ycbcr_enc
	- add struct v4l2_pix_format fmt_default
	- add enum_framesizes
	- enum_fmt_vid_cap: enumerate all formats from vimc_pix_map_table
	- add mode dev_dbg


---
 drivers/media/platform/vimc/vimc-capture.c | 131 +++++++++++++++++++++++++----
 1 file changed, 115 insertions(+), 16 deletions(-)

diff --git a/drivers/media/platform/vimc/vimc-capture.c b/drivers/media/platform/vimc/vimc-capture.c
index 5bdecd1..e943267 100644
--- a/drivers/media/platform/vimc/vimc-capture.c
+++ b/drivers/media/platform/vimc/vimc-capture.c
@@ -40,6 +40,14 @@ struct vimc_cap_device {
 	struct media_pipeline pipe;
 };
 
+static const struct v4l2_pix_format fmt_default = {
+	.width = 640,
+	.height = 480,
+	.pixelformat = V4L2_PIX_FMT_RGB24,
+	.field = V4L2_FIELD_NONE,
+	.colorspace = V4L2_COLORSPACE_SRGB,
+};
+
 struct vimc_cap_buffer {
 	/*
 	 * struct vb2_v4l2_buffer must be the first element
@@ -73,7 +81,7 @@ static void vimc_cap_get_format(struct vimc_ent_device *ved,
 	*fmt = vcap->format;
 }
 
-static int vimc_cap_fmt_vid_cap(struct file *file, void *priv,
+static int vimc_cap_g_fmt_vid_cap(struct file *file, void *priv,
 				  struct v4l2_format *f)
 {
 	struct vimc_cap_device *vcap = video_drvdata(file);
@@ -83,16 +91,112 @@ static int vimc_cap_fmt_vid_cap(struct file *file, void *priv,
 	return 0;
 }
 
+static int vimc_cap_try_fmt_vid_cap(struct file *file, void *priv,
+				    struct v4l2_format *f)
+{
+	struct v4l2_pix_format *format = &f->fmt.pix;
+	const struct vimc_pix_map *vpix;
+
+	format->width = clamp_t(u32, format->width, VIMC_FRAME_MIN_WIDTH,
+				VIMC_FRAME_MAX_WIDTH) & ~1;
+	format->height = clamp_t(u32, format->height, VIMC_FRAME_MIN_HEIGHT,
+				 VIMC_FRAME_MAX_HEIGHT) & ~1;
+
+	/* Don't accept a pixelformat that is not on the table */
+	vpix = vimc_pix_map_by_pixelformat(format->pixelformat);
+	if (!vpix) {
+		format->pixelformat = fmt_default.pixelformat;
+		vpix = vimc_pix_map_by_pixelformat(format->pixelformat);
+	}
+	/* TODO: Add support for custom bytesperline values */
+	format->bytesperline = format->width * vpix->bpp;
+	format->sizeimage = format->bytesperline * format->height;
+
+	if (format->field == V4L2_FIELD_ANY)
+		format->field = fmt_default.field;
+
+	if (format->colorspace == V4L2_COLORSPACE_DEFAULT)
+		format->colorspace = fmt_default.colorspace;
+
+	/* Check if values are out of range */
+	if (format->colorspace > V4L2_COLORSPACE_DCI_P3) {
+		format->colorspace = fmt_default.colorspace;
+		format->ycbcr_enc = V4L2_YCBCR_ENC_DEFAULT;
+		format->quantization = V4L2_QUANTIZATION_DEFAULT;
+		format->xfer_func = V4L2_XFER_FUNC_DEFAULT;
+	}
+	if (format->ycbcr_enc > V4L2_YCBCR_ENC_SMPTE240M)
+		format->ycbcr_enc = V4L2_YCBCR_ENC_DEFAULT;
+	if (format->quantization > V4L2_QUANTIZATION_LIM_RANGE)
+		format->quantization = V4L2_QUANTIZATION_DEFAULT;
+	if (format->xfer_func > V4L2_XFER_FUNC_SMPTE2084)
+		format->xfer_func = V4L2_XFER_FUNC_DEFAULT;
+	return 0;
+}
+
+static int vimc_cap_s_fmt_vid_cap(struct file *file, void *priv,
+				  struct v4l2_format *f)
+{
+	struct vimc_cap_device *vcap = video_drvdata(file);
+
+	/* Do not change the format while stream is on */
+	if (vb2_is_busy(&vcap->queue))
+		return -EBUSY;
+
+	vimc_cap_try_fmt_vid_cap(file, priv, f);
+
+	dev_dbg(vcap->vdev.v4l2_dev->dev, "%s: format update: "
+		"old:%dx%d (0x%x, %d, %d, %d, %d) "
+		"new:%dx%d (0x%x, %d, %d, %d, %d)\n", vcap->vdev.name,
+		/* old */
+		vcap->format.width, vcap->format.height,
+		vcap->format.pixelformat, vcap->format.colorspace,
+		vcap->format.quantization, vcap->format.xfer_func,
+		vcap->format.ycbcr_enc,
+		/* new */
+		f->fmt.pix.width, f->fmt.pix.height,
+		f->fmt.pix.pixelformat,	f->fmt.pix.colorspace,
+		f->fmt.pix.quantization, f->fmt.pix.xfer_func,
+		f->fmt.pix.ycbcr_enc);
+
+	vcap->format = f->fmt.pix;
+
+	return 0;
+}
+
 static int vimc_cap_enum_fmt_vid_cap(struct file *file, void *priv,
 				     struct v4l2_fmtdesc *f)
 {
-	struct vimc_cap_device *vcap = video_drvdata(file);
+	const struct vimc_pix_map *vpix = vimc_pix_map_by_index(f->index);
+
+	if (!vpix)
+		return -EINVAL;
+
+	f->pixelformat = vpix->pixelformat;
+
+	return 0;
+}
+
+static int vimc_cap_enum_framesizes(struct file *file, void *fh,
+				    struct v4l2_frmsizeenum *fsize)
+{
+	const struct vimc_pix_map *vpix;
 
-	if (f->index > 0)
+	if (fsize->index)
 		return -EINVAL;
 
-	/* We only support one format for now */
-	f->pixelformat = vcap->format.pixelformat;
+	/* Only accept code in the pix map table */
+	vpix = vimc_pix_map_by_code(fsize->pixel_format);
+	if (!vpix)
+		return -EINVAL;
+
+	fsize->type = V4L2_FRMSIZE_TYPE_CONTINUOUS;
+	fsize->stepwise.min_width = VIMC_FRAME_MIN_WIDTH;
+	fsize->stepwise.max_width = VIMC_FRAME_MAX_WIDTH;
+	fsize->stepwise.min_height = VIMC_FRAME_MIN_HEIGHT;
+	fsize->stepwise.max_height = VIMC_FRAME_MAX_HEIGHT;
+	fsize->stepwise.step_width = 2;
+	fsize->stepwise.step_height = 2;
 
 	return 0;
 }
@@ -110,10 +214,11 @@ static const struct v4l2_file_operations vimc_cap_fops = {
 static const struct v4l2_ioctl_ops vimc_cap_ioctl_ops = {
 	.vidioc_querycap = vimc_cap_querycap,
 
-	.vidioc_g_fmt_vid_cap = vimc_cap_fmt_vid_cap,
-	.vidioc_s_fmt_vid_cap = vimc_cap_fmt_vid_cap,
-	.vidioc_try_fmt_vid_cap = vimc_cap_fmt_vid_cap,
+	.vidioc_g_fmt_vid_cap = vimc_cap_g_fmt_vid_cap,
+	.vidioc_s_fmt_vid_cap = vimc_cap_s_fmt_vid_cap,
+	.vidioc_try_fmt_vid_cap = vimc_cap_try_fmt_vid_cap,
 	.vidioc_enum_fmt_vid_cap = vimc_cap_enum_fmt_vid_cap,
+	.vidioc_enum_framesizes = vimc_cap_enum_framesizes,
 
 	.vidioc_reqbufs = vb2_ioctl_reqbufs,
 	.vidioc_create_bufs = vb2_ioctl_create_bufs,
@@ -360,15 +465,9 @@ struct vimc_ent_device *vimc_cap_create(struct v4l2_device *v4l2_dev,
 	INIT_LIST_HEAD(&vcap->buf_list);
 	spin_lock_init(&vcap->qlock);
 
-	/* Set the frame format (this is hardcoded for now) */
-	vcap->format.width = 640;
-	vcap->format.height = 480;
-	vcap->format.pixelformat = V4L2_PIX_FMT_RGB24;
-	vcap->format.field = V4L2_FIELD_NONE;
-	vcap->format.colorspace = V4L2_COLORSPACE_SRGB;
-
+	/* Set default frame format */
+	vcap->format = fmt_default;
 	vpix = vimc_pix_map_by_pixelformat(vcap->format.pixelformat);
-
 	vcap->format.bytesperline = vcap->format.width * vpix->bpp;
 	vcap->format.sizeimage = vcap->format.bytesperline *
 				 vcap->format.height;
-- 
2.7.4

Re: [RFC PATCH v3 07/11] [media] vimc: cap: Support several image formats

[Hans Verkuil]

On 06/03/2017 04:58 AM, Helen Koike wrote:
> Allow user space to change the image format as the frame size, the
> pixel format, colorspace, quantization, field YCbCr encoding
> and the transfer function
> 
> Signed-off-by: Helen Koike <helen.koike@collabora.com>
> 
> ---
> 
> Changes in v3:
> [media] vimc: cap: Support several image formats
> 	- use *_DEFAULT macros for colorimetry in the default format
> 	- clamp height and width of the image by an even value
> 	- is user try to set colorspace to an invalid format, set all
> 	colorimetry parameters to _DEFAULT
> 	- remove V4L2_FMT_FLAG_COMPRESSED from vimc_cap_enum_fmt_vid_cap
> 	- remove V4L2_BUF_TYPE_VIDEO_CAPTURE from vimc_cap_enum_fmt_vid_cap
> 	- increase step_width and step_height to 2 instead of 1
> 	- remove link validate function, use the one in vimc-common.c
> 
> Changes in v2:
> [media] vimc: cap: Support several image formats
> 	- this is a new commit in the serie (the old one was splitted in two)
> 	- allow user space to change all fields from struct v4l2_pix_format
> 	  (e.g. colospace, quantization, field, xfer_func, ycbcr_enc)
> 	- link_validate and try_fmt: also checks colospace, quantization, field, xfer_func, ycbcr_enc
> 	- add struct v4l2_pix_format fmt_default
> 	- add enum_framesizes
> 	- enum_fmt_vid_cap: enumerate all formats from vimc_pix_map_table
> 	- add mode dev_dbg
> 
> 
> ---
>   drivers/media/platform/vimc/vimc-capture.c | 131 +++++++++++++++++++++++++----
>   1 file changed, 115 insertions(+), 16 deletions(-)
> 
> diff --git a/drivers/media/platform/vimc/vimc-capture.c b/drivers/media/platform/vimc/vimc-capture.c
> index 5bdecd1..e943267 100644
> --- a/drivers/media/platform/vimc/vimc-capture.c
> +++ b/drivers/media/platform/vimc/vimc-capture.c
> @@ -40,6 +40,14 @@ struct vimc_cap_device {
>   	struct media_pipeline pipe;
>   };
>   
> +static const struct v4l2_pix_format fmt_default = {
> +	.width = 640,
> +	.height = 480,
> +	.pixelformat = V4L2_PIX_FMT_RGB24,
> +	.field = V4L2_FIELD_NONE,
> +	.colorspace = V4L2_COLORSPACE_SRGB,
> +};
> +
>   struct vimc_cap_buffer {
>   	/*
>   	 * struct vb2_v4l2_buffer must be the first element
> @@ -73,7 +81,7 @@ static void vimc_cap_get_format(struct vimc_ent_device *ved,
>   	*fmt = vcap->format;
>   }
>   
> -static int vimc_cap_fmt_vid_cap(struct file *file, void *priv,
> +static int vimc_cap_g_fmt_vid_cap(struct file *file, void *priv,
>   				  struct v4l2_format *f)
>   {
>   	struct vimc_cap_device *vcap = video_drvdata(file);
> @@ -83,16 +91,112 @@ static int vimc_cap_fmt_vid_cap(struct file *file, void *priv,
>   	return 0;
>   }
>   
> +static int vimc_cap_try_fmt_vid_cap(struct file *file, void *priv,
> +				    struct v4l2_format *f)
> +{
> +	struct v4l2_pix_format *format = &f->fmt.pix;
> +	const struct vimc_pix_map *vpix;
> +
> +	format->width = clamp_t(u32, format->width, VIMC_FRAME_MIN_WIDTH,
> +				VIMC_FRAME_MAX_WIDTH) & ~1;
> +	format->height = clamp_t(u32, format->height, VIMC_FRAME_MIN_HEIGHT,
> +				 VIMC_FRAME_MAX_HEIGHT) & ~1;
> +
> +	/* Don't accept a pixelformat that is not on the table */
> +	vpix = vimc_pix_map_by_pixelformat(format->pixelformat);
> +	if (!vpix) {
> +		format->pixelformat = fmt_default.pixelformat;
> +		vpix = vimc_pix_map_by_pixelformat(format->pixelformat);
> +	}
> +	/* TODO: Add support for custom bytesperline values */
> +	format->bytesperline = format->width * vpix->bpp;
> +	format->sizeimage = format->bytesperline * format->height;
> +
> +	if (format->field == V4L2_FIELD_ANY)
> +		format->field = fmt_default.field;
> +
> +	if (format->colorspace == V4L2_COLORSPACE_DEFAULT)
> +		format->colorspace = fmt_default.colorspace;
> +
> +	/* Check if values are out of range */
> +	if (format->colorspace > V4L2_COLORSPACE_DCI_P3) {
> +		format->colorspace = fmt_default.colorspace;
> +		format->ycbcr_enc = V4L2_YCBCR_ENC_DEFAULT;
> +		format->quantization = V4L2_QUANTIZATION_DEFAULT;
> +		format->xfer_func = V4L2_XFER_FUNC_DEFAULT;
> +	}
> +	if (format->ycbcr_enc > V4L2_YCBCR_ENC_SMPTE240M)
> +		format->ycbcr_enc = V4L2_YCBCR_ENC_DEFAULT;
> +	if (format->quantization > V4L2_QUANTIZATION_LIM_RANGE)
> +		format->quantization = V4L2_QUANTIZATION_DEFAULT;
> +	if (format->xfer_func > V4L2_XFER_FUNC_SMPTE2084)
> +		format->xfer_func = V4L2_XFER_FUNC_DEFAULT;

You might want to move this to common, since the sensor has the same code.

> +	return 0;
> +}
> +
> +static int vimc_cap_s_fmt_vid_cap(struct file *file, void *priv,
> +				  struct v4l2_format *f)
> +{
> +	struct vimc_cap_device *vcap = video_drvdata(file);
> +
> +	/* Do not change the format while stream is on */
> +	if (vb2_is_busy(&vcap->queue))
> +		return -EBUSY;
> +
> +	vimc_cap_try_fmt_vid_cap(file, priv, f);
> +
> +	dev_dbg(vcap->vdev.v4l2_dev->dev, "%s: format update: "
> +		"old:%dx%d (0x%x, %d, %d, %d, %d) "
> +		"new:%dx%d (0x%x, %d, %d, %d, %d)\n", vcap->vdev.name,
> +		/* old */
> +		vcap->format.width, vcap->format.height,
> +		vcap->format.pixelformat, vcap->format.colorspace,
> +		vcap->format.quantization, vcap->format.xfer_func,
> +		vcap->format.ycbcr_enc,
> +		/* new */
> +		f->fmt.pix.width, f->fmt.pix.height,
> +		f->fmt.pix.pixelformat,	f->fmt.pix.colorspace,
> +		f->fmt.pix.quantization, f->fmt.pix.xfer_func,
> +		f->fmt.pix.ycbcr_enc);
> +
> +	vcap->format = f->fmt.pix;
> +
> +	return 0;
> +}
> +
>   static int vimc_cap_enum_fmt_vid_cap(struct file *file, void *priv,
>   				     struct v4l2_fmtdesc *f)
>   {
> -	struct vimc_cap_device *vcap = video_drvdata(file);
> +	const struct vimc_pix_map *vpix = vimc_pix_map_by_index(f->index);
> +
> +	if (!vpix)
> +		return -EINVAL;
> +
> +	f->pixelformat = vpix->pixelformat;
> +
> +	return 0;
> +}
> +
> +static int vimc_cap_enum_framesizes(struct file *file, void *fh,
> +				    struct v4l2_frmsizeenum *fsize)
> +{
> +	const struct vimc_pix_map *vpix;
>   
> -	if (f->index > 0)
> +	if (fsize->index)
>   		return -EINVAL;
>   
> -	/* We only support one format for now */
> -	f->pixelformat = vcap->format.pixelformat;
> +	/* Only accept code in the pix map table */
> +	vpix = vimc_pix_map_by_code(fsize->pixel_format);
> +	if (!vpix)
> +		return -EINVAL;
> +
> +	fsize->type = V4L2_FRMSIZE_TYPE_CONTINUOUS;
> +	fsize->stepwise.min_width = VIMC_FRAME_MIN_WIDTH;
> +	fsize->stepwise.max_width = VIMC_FRAME_MAX_WIDTH;
> +	fsize->stepwise.min_height = VIMC_FRAME_MIN_HEIGHT;
> +	fsize->stepwise.max_height = VIMC_FRAME_MAX_HEIGHT;
> +	fsize->stepwise.step_width = 2;
> +	fsize->stepwise.step_height = 2;
>   
>   	return 0;
>   }
> @@ -110,10 +214,11 @@ static const struct v4l2_file_operations vimc_cap_fops = {
>   static const struct v4l2_ioctl_ops vimc_cap_ioctl_ops = {
>   	.vidioc_querycap = vimc_cap_querycap,
>   
> -	.vidioc_g_fmt_vid_cap = vimc_cap_fmt_vid_cap,
> -	.vidioc_s_fmt_vid_cap = vimc_cap_fmt_vid_cap,
> -	.vidioc_try_fmt_vid_cap = vimc_cap_fmt_vid_cap,
> +	.vidioc_g_fmt_vid_cap = vimc_cap_g_fmt_vid_cap,
> +	.vidioc_s_fmt_vid_cap = vimc_cap_s_fmt_vid_cap,
> +	.vidioc_try_fmt_vid_cap = vimc_cap_try_fmt_vid_cap,
>   	.vidioc_enum_fmt_vid_cap = vimc_cap_enum_fmt_vid_cap,
> +	.vidioc_enum_framesizes = vimc_cap_enum_framesizes,
>   
>   	.vidioc_reqbufs = vb2_ioctl_reqbufs,
>   	.vidioc_create_bufs = vb2_ioctl_create_bufs,
> @@ -360,15 +465,9 @@ struct vimc_ent_device *vimc_cap_create(struct v4l2_device *v4l2_dev,
>   	INIT_LIST_HEAD(&vcap->buf_list);
>   	spin_lock_init(&vcap->qlock);
>   
> -	/* Set the frame format (this is hardcoded for now) */
> -	vcap->format.width = 640;
> -	vcap->format.height = 480;
> -	vcap->format.pixelformat = V4L2_PIX_FMT_RGB24;
> -	vcap->format.field = V4L2_FIELD_NONE;
> -	vcap->format.colorspace = V4L2_COLORSPACE_SRGB;
> -
> +	/* Set default frame format */
> +	vcap->format = fmt_default;
>   	vpix = vimc_pix_map_by_pixelformat(vcap->format.pixelformat);
> -
>   	vcap->format.bytesperline = vcap->format.width * vpix->bpp;
>   	vcap->format.sizeimage = vcap->format.bytesperline *
>   				 vcap->format.height;
> 

Regards,

	Hans

[RFC PATCH v3 08/11] [media] vimc: Optimize frame generation through the pipe

[Helen Koike]

Add a parameter called vsen_tpg, if true then vimc will work as before:
frames will be generated in the sensor nodes then propagated through the
pipe and processed by each node until a capture node is reached.
If vsen_tpg is false, then the frame is not generated by the sensor, but
directly from the capture node, thus saving intermediate memory buffers
and process time, allowing a higher frame rate.

Signed-off-by: Helen Koike <helen.koike@collabora.com>

---

Changes in v3:
[media] vimc: Optimize frame generation through the pipe
	- This is a new patch in the series

Changes in v2: None


---
 drivers/media/platform/vimc/vimc-capture.c | 178 +++++++++++++++++++++--------
 drivers/media/platform/vimc/vimc-common.h  |   2 +
 drivers/media/platform/vimc/vimc-sensor.c  |  30 ++++-
 3 files changed, 156 insertions(+), 54 deletions(-)

diff --git a/drivers/media/platform/vimc/vimc-capture.c b/drivers/media/platform/vimc/vimc-capture.c
index e943267..b5da0ea 100644
--- a/drivers/media/platform/vimc/vimc-capture.c
+++ b/drivers/media/platform/vimc/vimc-capture.c
@@ -15,7 +15,10 @@
  *
  */
 
+#include <linux/freezer.h>
+#include <linux/kthread.h>
 #include <media/v4l2-ioctl.h>
+#include <media/v4l2-tpg.h>
 #include <media/videobuf2-core.h>
 #include <media/videobuf2-vmalloc.h>
 
@@ -38,6 +41,8 @@ struct vimc_cap_device {
 	struct mutex lock;
 	u32 sequence;
 	struct media_pipeline pipe;
+	struct tpg_data tpg;
+	struct task_struct *kthread_cap;
 };
 
 static const struct v4l2_pix_format fmt_default = {
@@ -246,6 +251,91 @@ static void vimc_cap_return_all_buffers(struct vimc_cap_device *vcap,
 	spin_unlock(&vcap->qlock);
 }
 
+static void vimc_cap_process_frame(struct vimc_ent_device *ved,
+				   struct media_pad *sink, const void *frame)
+{
+	struct vimc_cap_device *vcap = container_of(ved, struct vimc_cap_device,
+						    ved);
+	struct vimc_cap_buffer *vimc_buf;
+	void *vbuf;
+
+	spin_lock(&vcap->qlock);
+
+	/* Get the first entry of the list */
+	vimc_buf = list_first_entry_or_null(&vcap->buf_list,
+					    typeof(*vimc_buf), list);
+	if (!vimc_buf) {
+		spin_unlock(&vcap->qlock);
+		return;
+	}
+
+	/* Remove this entry from the list */
+	list_del(&vimc_buf->list);
+
+	spin_unlock(&vcap->qlock);
+
+	/* Fill the buffer */
+	vimc_buf->vb2.vb2_buf.timestamp = ktime_get_ns();
+	vimc_buf->vb2.sequence = vcap->sequence++;
+	vimc_buf->vb2.field = vcap->format.field;
+
+	vbuf = vb2_plane_vaddr(&vimc_buf->vb2.vb2_buf, 0);
+
+	if (sink)
+		memcpy(vbuf, frame, vcap->format.sizeimage);
+	else
+		tpg_fill_plane_buffer(&vcap->tpg, V4L2_STD_PAL, 0, vbuf);
+
+	/* Set it as ready */
+	vb2_set_plane_payload(&vimc_buf->vb2.vb2_buf, 0,
+			      vcap->format.sizeimage);
+	vb2_buffer_done(&vimc_buf->vb2.vb2_buf, VB2_BUF_STATE_DONE);
+}
+
+
+static int vimc_cap_tpg_thread(void *data)
+{
+	struct vimc_cap_device *vcap = data;
+
+	set_freezable();
+	set_current_state(TASK_UNINTERRUPTIBLE);
+
+	for (;;) {
+		try_to_freeze();
+		if (kthread_should_stop())
+			break;
+
+		vimc_cap_process_frame(&vcap->ved, NULL, NULL);
+
+		/* 60 frames per second */
+		schedule_timeout(HZ/60);
+	}
+
+	return 0;
+}
+
+static void vimc_cap_tpg_s_format(struct vimc_cap_device *vcap)
+{
+	const struct vimc_pix_map *vpix =
+			vimc_pix_map_by_pixelformat(vcap->format.pixelformat);
+
+	tpg_reset_source(&vcap->tpg, vcap->format.width, vcap->format.height,
+			 vcap->format.field);
+	tpg_s_bytesperline(&vcap->tpg, 0, vcap->format.width * vpix->bpp);
+	tpg_s_buf_height(&vcap->tpg, vcap->format.height);
+	tpg_s_fourcc(&vcap->tpg, vpix->pixelformat);
+	/*
+	 * TODO: check why the tpg_s_field need this third argument if
+	 * it is already receiving the field
+	 */
+	tpg_s_field(&vcap->tpg, vcap->format.field,
+		    vcap->format.field == V4L2_FIELD_ALTERNATE);
+	tpg_s_colorspace(&vcap->tpg, vcap->format.colorspace);
+	tpg_s_ycbcr_enc(&vcap->tpg, vcap->format.ycbcr_enc);
+	tpg_s_quantization(&vcap->tpg, vcap->format.quantization);
+	tpg_s_xfer_func(&vcap->tpg, vcap->format.xfer_func);
+}
+
 static int vimc_cap_start_streaming(struct vb2_queue *vq, unsigned int count)
 {
 	struct vimc_cap_device *vcap = vb2_get_drv_priv(vq);
@@ -256,20 +346,43 @@ static int vimc_cap_start_streaming(struct vb2_queue *vq, unsigned int count)
 
 	/* Start the media pipeline */
 	ret = media_pipeline_start(entity, &vcap->pipe);
-	if (ret) {
-		vimc_cap_return_all_buffers(vcap, VB2_BUF_STATE_QUEUED);
-		return ret;
-	}
+	if (ret)
+		goto err_ret_all_buffs;
 
 	/* Enable streaming from the pipe */
 	ret = vimc_pipeline_s_stream(&vcap->vdev.entity, 1);
-	if (ret) {
-		media_pipeline_stop(entity);
-		vimc_cap_return_all_buffers(vcap, VB2_BUF_STATE_QUEUED);
-		return ret;
+	if (ret < 0)
+		goto err_mpipe_stop;
+
+	if (ret == VIMC_PIPE_OPT) {
+		tpg_init(&vcap->tpg, vcap->format.width, vcap->format.height);
+		ret = tpg_alloc(&vcap->tpg, VIMC_FRAME_MAX_WIDTH);
+		if (ret)
+			/* We don't need to call vimc_pipeline_s_stream(e, 0) */
+			goto err_mpipe_stop;
+
+		vimc_cap_tpg_s_format(vcap);
+		vcap->kthread_cap = kthread_run(vimc_cap_tpg_thread, vcap,
+					"%s-cap", vcap->vdev.v4l2_dev->name);
+		if (IS_ERR(vcap->kthread_cap)) {
+			dev_err(vcap->vdev.v4l2_dev->dev,
+				"%s: kernel_thread() failed\n",
+				vcap->vdev.name);
+			ret = PTR_ERR(vcap->kthread_cap);
+			goto err_tpg_free;
+		}
 	}
 
 	return 0;
+
+err_tpg_free:
+	tpg_free(&vcap->tpg);
+err_mpipe_stop:
+	media_pipeline_stop(entity);
+err_ret_all_buffs:
+	vimc_cap_return_all_buffers(vcap, VB2_BUF_STATE_QUEUED);
+
+	return ret;
 }
 
 /*
@@ -280,8 +393,15 @@ static void vimc_cap_stop_streaming(struct vb2_queue *vq)
 {
 	struct vimc_cap_device *vcap = vb2_get_drv_priv(vq);
 
-	/* Disable streaming from the pipe */
-	vimc_pipeline_s_stream(&vcap->vdev.entity, 0);
+	if (vcap->kthread_cap) {
+		/* Stop image generator */
+		kthread_stop(vcap->kthread_cap);
+		vcap->kthread_cap = NULL;
+		tpg_free(&vcap->tpg);
+	} else {
+		/* Disable streaming from the pipe */
+		vimc_pipeline_s_stream(&vcap->vdev.entity, 0);
+	}
 
 	/* Stop the media pipeline */
 	media_pipeline_stop(&vcap->vdev.entity);
@@ -361,44 +481,6 @@ static void vimc_cap_destroy(struct vimc_ent_device *ved)
 	kfree(vcap);
 }
 
-static void vimc_cap_process_frame(struct vimc_ent_device *ved,
-				   struct media_pad *sink, const void *frame)
-{
-	struct vimc_cap_device *vcap = container_of(ved, struct vimc_cap_device,
-						    ved);
-	struct vimc_cap_buffer *vimc_buf;
-	void *vbuf;
-
-	spin_lock(&vcap->qlock);
-
-	/* Get the first entry of the list */
-	vimc_buf = list_first_entry_or_null(&vcap->buf_list,
-					    typeof(*vimc_buf), list);
-	if (!vimc_buf) {
-		spin_unlock(&vcap->qlock);
-		return;
-	}
-
-	/* Remove this entry from the list */
-	list_del(&vimc_buf->list);
-
-	spin_unlock(&vcap->qlock);
-
-	/* Fill the buffer */
-	vimc_buf->vb2.vb2_buf.timestamp = ktime_get_ns();
-	vimc_buf->vb2.sequence = vcap->sequence++;
-	vimc_buf->vb2.field = vcap->format.field;
-
-	vbuf = vb2_plane_vaddr(&vimc_buf->vb2.vb2_buf, 0);
-
-	memcpy(vbuf, frame, vcap->format.sizeimage);
-
-	/* Set it as ready */
-	vb2_set_plane_payload(&vimc_buf->vb2.vb2_buf, 0,
-			      vcap->format.sizeimage);
-	vb2_buffer_done(&vimc_buf->vb2.vb2_buf, VB2_BUF_STATE_DONE);
-}
-
 struct vimc_ent_device *vimc_cap_create(struct v4l2_device *v4l2_dev,
 					const char *const name,
 					u16 num_pads,
diff --git a/drivers/media/platform/vimc/vimc-common.h b/drivers/media/platform/vimc/vimc-common.h
index 2189fd6..5b2691e 100644
--- a/drivers/media/platform/vimc/vimc-common.h
+++ b/drivers/media/platform/vimc/vimc-common.h
@@ -27,6 +27,8 @@
 #define VIMC_FRAME_MIN_WIDTH 16
 #define VIMC_FRAME_MIN_HEIGHT 16
 
+#define VIMC_PIPE_OPT 1
+
 /**
  * struct vimc_pix_map - maps media bus code with v4l2 pixel format
  *
diff --git a/drivers/media/platform/vimc/vimc-sensor.c b/drivers/media/platform/vimc/vimc-sensor.c
index 90c41c6..df89ee7 100644
--- a/drivers/media/platform/vimc/vimc-sensor.c
+++ b/drivers/media/platform/vimc/vimc-sensor.c
@@ -15,6 +15,7 @@
  *
  */
 
+#include <linux/module.h>
 #include <linux/freezer.h>
 #include <linux/kthread.h>
 #include <linux/v4l2-mediabus.h>
@@ -24,6 +25,13 @@
 
 #include "vimc-sensor.h"
 
+static bool vsen_tpg;
+module_param(vsen_tpg, bool, 0000);
+MODULE_PARM_DESC(vsen_tpg, " generate image from sensor node\n"
+	"If set to false, then the pipe will be optimized and image will be "
+	"generated directly in the capture node instead of going through "
+	"the whole pipe");
+
 struct vimc_sen_device {
 	struct vimc_ent_device ved;
 	struct v4l2_subdev sd;
@@ -239,6 +247,13 @@ static int vimc_sen_s_stream(struct v4l2_subdev *sd, int enable)
 				container_of(sd, struct vimc_sen_device, sd);
 	int ret;
 
+	if (!vsen_tpg)
+		/*
+		 * If we are not generating the frames, then inform the caller
+		 * to generate the frame in shallow level of the pipe
+		 */
+		return VIMC_PIPE_OPT;
+
 	if (enable) {
 		const struct vimc_pix_map *vpix;
 		unsigned int frame_size;
@@ -306,7 +321,8 @@ static void vimc_sen_destroy(struct vimc_ent_device *ved)
 				container_of(ved, struct vimc_sen_device, ved);
 
 	vimc_ent_sd_unregister(ved, &vsen->sd);
-	tpg_free(&vsen->tpg);
+	if (vsen_tpg)
+		tpg_free(&vsen->tpg);
 	kfree(vsen);
 }
 
@@ -344,11 +360,13 @@ struct vimc_ent_device *vimc_sen_create(struct v4l2_device *v4l2_dev,
 	vsen->mbus_format = fmt_default;
 
 	/* Initialize the test pattern generator */
-	tpg_init(&vsen->tpg, vsen->mbus_format.width,
-		 vsen->mbus_format.height);
-	ret = tpg_alloc(&vsen->tpg, VIMC_FRAME_MAX_WIDTH);
-	if (ret)
-		goto err_unregister_ent_sd;
+	if (vsen_tpg) {
+		tpg_init(&vsen->tpg, vsen->mbus_format.width,
+			 vsen->mbus_format.height);
+		ret = tpg_alloc(&vsen->tpg, VIMC_FRAME_MAX_WIDTH);
+		if (ret)
+			goto err_unregister_ent_sd;
+	}
 
 	return &vsen->ved;
 
-- 
2.7.4

Re: [RFC PATCH v3 08/11] [media] vimc: Optimize frame generation through the pipe

[Helen Koike]

On 2017-06-02 11:58 PM, Helen Koike wrote:
> Add a parameter called vsen_tpg, if true then vimc will work as before:
> frames will be generated in the sensor nodes then propagated through the
> pipe and processed by each node until a capture node is reached.
> If vsen_tpg is false, then the frame is not generated by the sensor, but
> directly from the capture node, thus saving intermediate memory buffers
> and process time, allowing a higher frame rate.
>
> Signed-off-by: Helen Koike <helen.koike@collabora.com>
>
> ---
>
> Changes in v3:
> [media] vimc: Optimize frame generation through the pipe
> 	- This is a new patch in the series
>
> Changes in v2: None
>
>
> ---
>  drivers/media/platform/vimc/vimc-capture.c | 178 +++++++++++++++++++++--------
>  drivers/media/platform/vimc/vimc-common.h  |   2 +
>  drivers/media/platform/vimc/vimc-sensor.c  |  30 ++++-
>  3 files changed, 156 insertions(+), 54 deletions(-)
>
> diff --git a/drivers/media/platform/vimc/vimc-capture.c b/drivers/media/platform/vimc/vimc-capture.c
> index e943267..b5da0ea 100644
> --- a/drivers/media/platform/vimc/vimc-capture.c
> +++ b/drivers/media/platform/vimc/vimc-capture.c
> @@ -15,7 +15,10 @@
>   *
>   */
>
> +#include <linux/freezer.h>
> +#include <linux/kthread.h>
>  #include <media/v4l2-ioctl.h>
> +#include <media/v4l2-tpg.h>
>  #include <media/videobuf2-core.h>
>  #include <media/videobuf2-vmalloc.h>
>
> @@ -38,6 +41,8 @@ struct vimc_cap_device {
>  	struct mutex lock;
>  	u32 sequence;
>  	struct media_pipeline pipe;
> +	struct tpg_data tpg;
> +	struct task_struct *kthread_cap;
>  };
>
>  static const struct v4l2_pix_format fmt_default = {
> @@ -246,6 +251,91 @@ static void vimc_cap_return_all_buffers(struct vimc_cap_device *vcap,
>  	spin_unlock(&vcap->qlock);
>  }
>
> +static void vimc_cap_process_frame(struct vimc_ent_device *ved,
> +				   struct media_pad *sink, const void *frame)
> +{
> +	struct vimc_cap_device *vcap = container_of(ved, struct vimc_cap_device,
> +						    ved);
> +	struct vimc_cap_buffer *vimc_buf;
> +	void *vbuf;
> +
> +	spin_lock(&vcap->qlock);
> +
> +	/* Get the first entry of the list */
> +	vimc_buf = list_first_entry_or_null(&vcap->buf_list,
> +					    typeof(*vimc_buf), list);
> +	if (!vimc_buf) {
> +		spin_unlock(&vcap->qlock);
> +		return;
> +	}
> +
> +	/* Remove this entry from the list */
> +	list_del(&vimc_buf->list);
> +
> +	spin_unlock(&vcap->qlock);
> +
> +	/* Fill the buffer */
> +	vimc_buf->vb2.vb2_buf.timestamp = ktime_get_ns();
> +	vimc_buf->vb2.sequence = vcap->sequence++;
> +	vimc_buf->vb2.field = vcap->format.field;
> +
> +	vbuf = vb2_plane_vaddr(&vimc_buf->vb2.vb2_buf, 0);
> +
> +	if (sink)
> +		memcpy(vbuf, frame, vcap->format.sizeimage);
> +	else
> +		tpg_fill_plane_buffer(&vcap->tpg, V4L2_STD_PAL, 0, vbuf);
> +
> +	/* Set it as ready */
> +	vb2_set_plane_payload(&vimc_buf->vb2.vb2_buf, 0,
> +			      vcap->format.sizeimage);
> +	vb2_buffer_done(&vimc_buf->vb2.vb2_buf, VB2_BUF_STATE_DONE);
> +}
> +
> +
> +static int vimc_cap_tpg_thread(void *data)
> +{
> +	struct vimc_cap_device *vcap = data;
> +
> +	set_freezable();
> +	set_current_state(TASK_UNINTERRUPTIBLE);
> +
> +	for (;;) {
> +		try_to_freeze();
> +		if (kthread_should_stop())
> +			break;
> +
> +		vimc_cap_process_frame(&vcap->ved, NULL, NULL);
> +
> +		/* 60 frames per second */
> +		schedule_timeout(HZ/60);
> +	}
> +
> +	return 0;
> +}
> +
> +static void vimc_cap_tpg_s_format(struct vimc_cap_device *vcap)
> +{
> +	const struct vimc_pix_map *vpix =
> +			vimc_pix_map_by_pixelformat(vcap->format.pixelformat);
> +
> +	tpg_reset_source(&vcap->tpg, vcap->format.width, vcap->format.height,
> +			 vcap->format.field);
> +	tpg_s_bytesperline(&vcap->tpg, 0, vcap->format.width * vpix->bpp);
> +	tpg_s_buf_height(&vcap->tpg, vcap->format.height);
> +	tpg_s_fourcc(&vcap->tpg, vpix->pixelformat);
> +	/*
> +	 * TODO: check why the tpg_s_field need this third argument if
> +	 * it is already receiving the field
> +	 */
> +	tpg_s_field(&vcap->tpg, vcap->format.field,
> +		    vcap->format.field == V4L2_FIELD_ALTERNATE);
> +	tpg_s_colorspace(&vcap->tpg, vcap->format.colorspace);
> +	tpg_s_ycbcr_enc(&vcap->tpg, vcap->format.ycbcr_enc);
> +	tpg_s_quantization(&vcap->tpg, vcap->format.quantization);
> +	tpg_s_xfer_func(&vcap->tpg, vcap->format.xfer_func);
> +}
> +
>  static int vimc_cap_start_streaming(struct vb2_queue *vq, unsigned int count)
>  {
>  	struct vimc_cap_device *vcap = vb2_get_drv_priv(vq);
> @@ -256,20 +346,43 @@ static int vimc_cap_start_streaming(struct vb2_queue *vq, unsigned int count)
>
>  	/* Start the media pipeline */
>  	ret = media_pipeline_start(entity, &vcap->pipe);
> -	if (ret) {
> -		vimc_cap_return_all_buffers(vcap, VB2_BUF_STATE_QUEUED);
> -		return ret;
> -	}
> +	if (ret)
> +		goto err_ret_all_buffs;
>
>  	/* Enable streaming from the pipe */
>  	ret = vimc_pipeline_s_stream(&vcap->vdev.entity, 1);
> -	if (ret) {
> -		media_pipeline_stop(entity);
> -		vimc_cap_return_all_buffers(vcap, VB2_BUF_STATE_QUEUED);
> -		return ret;
> +	if (ret < 0)
> +		goto err_mpipe_stop;
> +
> +	if (ret == VIMC_PIPE_OPT) {
> +		tpg_init(&vcap->tpg, vcap->format.width, vcap->format.height);
> +		ret = tpg_alloc(&vcap->tpg, VIMC_FRAME_MAX_WIDTH);
> +		if (ret)
> +			/* We don't need to call vimc_pipeline_s_stream(e, 0) */
> +			goto err_mpipe_stop;
> +
> +		vimc_cap_tpg_s_format(vcap);
> +		vcap->kthread_cap = kthread_run(vimc_cap_tpg_thread, vcap,
> +					"%s-cap", vcap->vdev.v4l2_dev->name);
> +		if (IS_ERR(vcap->kthread_cap)) {
> +			dev_err(vcap->vdev.v4l2_dev->dev,
> +				"%s: kernel_thread() failed\n",
> +				vcap->vdev.name);
> +			ret = PTR_ERR(vcap->kthread_cap);
> +			goto err_tpg_free;
> +		}
>  	}
>
>  	return 0;
> +
> +err_tpg_free:
> +	tpg_free(&vcap->tpg);
> +err_mpipe_stop:
> +	media_pipeline_stop(entity);
> +err_ret_all_buffs:
> +	vimc_cap_return_all_buffers(vcap, VB2_BUF_STATE_QUEUED);
> +
> +	return ret;
>  }
>
>  /*
> @@ -280,8 +393,15 @@ static void vimc_cap_stop_streaming(struct vb2_queue *vq)
>  {
>  	struct vimc_cap_device *vcap = vb2_get_drv_priv(vq);
>
> -	/* Disable streaming from the pipe */
> -	vimc_pipeline_s_stream(&vcap->vdev.entity, 0);
> +	if (vcap->kthread_cap) {
> +		/* Stop image generator */
> +		kthread_stop(vcap->kthread_cap);
> +		vcap->kthread_cap = NULL;
> +		tpg_free(&vcap->tpg);
> +	} else {
> +		/* Disable streaming from the pipe */
> +		vimc_pipeline_s_stream(&vcap->vdev.entity, 0);
> +	}
>
>  	/* Stop the media pipeline */
>  	media_pipeline_stop(&vcap->vdev.entity);
> @@ -361,44 +481,6 @@ static void vimc_cap_destroy(struct vimc_ent_device *ved)
>  	kfree(vcap);
>  }
>
> -static void vimc_cap_process_frame(struct vimc_ent_device *ved,
> -				   struct media_pad *sink, const void *frame)
> -{
> -	struct vimc_cap_device *vcap = container_of(ved, struct vimc_cap_device,
> -						    ved);
> -	struct vimc_cap_buffer *vimc_buf;
> -	void *vbuf;
> -
> -	spin_lock(&vcap->qlock);
> -
> -	/* Get the first entry of the list */
> -	vimc_buf = list_first_entry_or_null(&vcap->buf_list,
> -					    typeof(*vimc_buf), list);
> -	if (!vimc_buf) {
> -		spin_unlock(&vcap->qlock);
> -		return;
> -	}
> -
> -	/* Remove this entry from the list */
> -	list_del(&vimc_buf->list);
> -
> -	spin_unlock(&vcap->qlock);
> -
> -	/* Fill the buffer */
> -	vimc_buf->vb2.vb2_buf.timestamp = ktime_get_ns();
> -	vimc_buf->vb2.sequence = vcap->sequence++;
> -	vimc_buf->vb2.field = vcap->format.field;
> -
> -	vbuf = vb2_plane_vaddr(&vimc_buf->vb2.vb2_buf, 0);
> -
> -	memcpy(vbuf, frame, vcap->format.sizeimage);
> -
> -	/* Set it as ready */
> -	vb2_set_plane_payload(&vimc_buf->vb2.vb2_buf, 0,
> -			      vcap->format.sizeimage);
> -	vb2_buffer_done(&vimc_buf->vb2.vb2_buf, VB2_BUF_STATE_DONE);
> -}
> -
>  struct vimc_ent_device *vimc_cap_create(struct v4l2_device *v4l2_dev,
>  					const char *const name,
>  					u16 num_pads,
> diff --git a/drivers/media/platform/vimc/vimc-common.h b/drivers/media/platform/vimc/vimc-common.h
> index 2189fd6..5b2691e 100644
> --- a/drivers/media/platform/vimc/vimc-common.h
> +++ b/drivers/media/platform/vimc/vimc-common.h
> @@ -27,6 +27,8 @@
>  #define VIMC_FRAME_MIN_WIDTH 16
>  #define VIMC_FRAME_MIN_HEIGHT 16
>
> +#define VIMC_PIPE_OPT 1
> +
>  /**
>   * struct vimc_pix_map - maps media bus code with v4l2 pixel format
>   *
> diff --git a/drivers/media/platform/vimc/vimc-sensor.c b/drivers/media/platform/vimc/vimc-sensor.c
> index 90c41c6..df89ee7 100644
> --- a/drivers/media/platform/vimc/vimc-sensor.c
> +++ b/drivers/media/platform/vimc/vimc-sensor.c
> @@ -15,6 +15,7 @@
>   *
>   */
>
> +#include <linux/module.h>
>  #include <linux/freezer.h>
>  #include <linux/kthread.h>
>  #include <linux/v4l2-mediabus.h>
> @@ -24,6 +25,13 @@
>
>  #include "vimc-sensor.h"
>
> +static bool vsen_tpg;
> +module_param(vsen_tpg, bool, 0000);
> +MODULE_PARM_DESC(vsen_tpg, " generate image from sensor node\n"
> +	"If set to false, then the pipe will be optimized and image will be "
> +	"generated directly in the capture node instead of going through "
> +	"the whole pipe");
> +
>  struct vimc_sen_device {
>  	struct vimc_ent_device ved;
>  	struct v4l2_subdev sd;
> @@ -239,6 +247,13 @@ static int vimc_sen_s_stream(struct v4l2_subdev *sd, int enable)
>  				container_of(sd, struct vimc_sen_device, sd);
>  	int ret;
>
> +	if (!vsen_tpg)
> +		/*
> +		 * If we are not generating the frames, then inform the caller
> +		 * to generate the frame in shallow level of the pipe
> +		 */
> +		return VIMC_PIPE_OPT;
> +
>  	if (enable) {
>  		const struct vimc_pix_map *vpix;
>  		unsigned int frame_size;
> @@ -306,7 +321,8 @@ static void vimc_sen_destroy(struct vimc_ent_device *ved)
>  				container_of(ved, struct vimc_sen_device, ved);
>
>  	vimc_ent_sd_unregister(ved, &vsen->sd);
> -	tpg_free(&vsen->tpg);
> +	if (vsen_tpg)
> +		tpg_free(&vsen->tpg);
>  	kfree(vsen);
>  }
>
> @@ -344,11 +360,13 @@ struct vimc_ent_device *vimc_sen_create(struct v4l2_device *v4l2_dev,
>  	vsen->mbus_format = fmt_default;
>
>  	/* Initialize the test pattern generator */
> -	tpg_init(&vsen->tpg, vsen->mbus_format.width,
> -		 vsen->mbus_format.height);
> -	ret = tpg_alloc(&vsen->tpg, VIMC_FRAME_MAX_WIDTH);
> -	if (ret)
> -		goto err_unregister_ent_sd;
> +	if (vsen_tpg) {
> +		tpg_init(&vsen->tpg, vsen->mbus_format.width,
> +			 vsen->mbus_format.height);
> +		ret = tpg_alloc(&vsen->tpg, VIMC_FRAME_MAX_WIDTH);
> +		if (ret)
> +			goto err_unregister_ent_sd;
> +	}
>
>  	return &vsen->ved;
>
>


Hi,

There is a bug in this patch as I didn't consider we could have more 
then one link enabled for a single sink pad. Please, ignore this patch 
in this series, I'll re-work on it and send it in a different patch 
series latter.
I'll will re-send this series without this patch after your comments in 
the other patches.

Thanks
LN Koike

Re: [RFC PATCH v3 08/11] [media] vimc: Optimize frame generation through the pipe

[Hans Verkuil]

On 06/03/2017 04:58 AM, Helen Koike wrote:
> Add a parameter called vsen_tpg, if true then vimc will work as before:
> frames will be generated in the sensor nodes then propagated through the
> pipe and processed by each node until a capture node is reached.
> If vsen_tpg is false, then the frame is not generated by the sensor, but
> directly from the capture node, thus saving intermediate memory buffers
> and process time, allowing a higher frame rate.
> 
> Signed-off-by: Helen Koike <helen.koike@collabora.com>
> 
> ---
> 
> Changes in v3:
> [media] vimc: Optimize frame generation through the pipe
> 	- This is a new patch in the series
> 
> Changes in v2: None
> 
> 
> ---
>   drivers/media/platform/vimc/vimc-capture.c | 178 +++++++++++++++++++++--------
>   drivers/media/platform/vimc/vimc-common.h  |   2 +
>   drivers/media/platform/vimc/vimc-sensor.c  |  30 ++++-
>   3 files changed, 156 insertions(+), 54 deletions(-)
> 
> diff --git a/drivers/media/platform/vimc/vimc-capture.c b/drivers/media/platform/vimc/vimc-capture.c
> index e943267..b5da0ea 100644
> --- a/drivers/media/platform/vimc/vimc-capture.c
> +++ b/drivers/media/platform/vimc/vimc-capture.c
> @@ -15,7 +15,10 @@
>    *
>    */
>   
> +#include <linux/freezer.h>
> +#include <linux/kthread.h>
>   #include <media/v4l2-ioctl.h>
> +#include <media/v4l2-tpg.h>
>   #include <media/videobuf2-core.h>
>   #include <media/videobuf2-vmalloc.h>
>   
> @@ -38,6 +41,8 @@ struct vimc_cap_device {
>   	struct mutex lock;
>   	u32 sequence;
>   	struct media_pipeline pipe;
> +	struct tpg_data tpg;
> +	struct task_struct *kthread_cap;
>   };
>   
>   static const struct v4l2_pix_format fmt_default = {
> @@ -246,6 +251,91 @@ static void vimc_cap_return_all_buffers(struct vimc_cap_device *vcap,
>   	spin_unlock(&vcap->qlock);
>   }
>   
> +static void vimc_cap_process_frame(struct vimc_ent_device *ved,
> +				   struct media_pad *sink, const void *frame)
> +{
> +	struct vimc_cap_device *vcap = container_of(ved, struct vimc_cap_device,
> +						    ved);
> +	struct vimc_cap_buffer *vimc_buf;
> +	void *vbuf;
> +
> +	spin_lock(&vcap->qlock);
> +
> +	/* Get the first entry of the list */
> +	vimc_buf = list_first_entry_or_null(&vcap->buf_list,
> +					    typeof(*vimc_buf), list);
> +	if (!vimc_buf) {
> +		spin_unlock(&vcap->qlock);
> +		return;
> +	}
> +
> +	/* Remove this entry from the list */
> +	list_del(&vimc_buf->list);
> +
> +	spin_unlock(&vcap->qlock);
> +
> +	/* Fill the buffer */
> +	vimc_buf->vb2.vb2_buf.timestamp = ktime_get_ns();
> +	vimc_buf->vb2.sequence = vcap->sequence++;
> +	vimc_buf->vb2.field = vcap->format.field;
> +
> +	vbuf = vb2_plane_vaddr(&vimc_buf->vb2.vb2_buf, 0);
> +
> +	if (sink)
> +		memcpy(vbuf, frame, vcap->format.sizeimage);
> +	else
> +		tpg_fill_plane_buffer(&vcap->tpg, V4L2_STD_PAL, 0, vbuf);
> +
> +	/* Set it as ready */
> +	vb2_set_plane_payload(&vimc_buf->vb2.vb2_buf, 0,
> +			      vcap->format.sizeimage);
> +	vb2_buffer_done(&vimc_buf->vb2.vb2_buf, VB2_BUF_STATE_DONE);
> +}
> +
> +
> +static int vimc_cap_tpg_thread(void *data)
> +{
> +	struct vimc_cap_device *vcap = data;
> +
> +	set_freezable();
> +	set_current_state(TASK_UNINTERRUPTIBLE);
> +
> +	for (;;) {
> +		try_to_freeze();
> +		if (kthread_should_stop())
> +			break;
> +
> +		vimc_cap_process_frame(&vcap->ved, NULL, NULL);
> +
> +		/* 60 frames per second */
> +		schedule_timeout(HZ/60);
> +	}
> +
> +	return 0;
> +}
> +
> +static void vimc_cap_tpg_s_format(struct vimc_cap_device *vcap)
> +{
> +	const struct vimc_pix_map *vpix =
> +			vimc_pix_map_by_pixelformat(vcap->format.pixelformat);
> +
> +	tpg_reset_source(&vcap->tpg, vcap->format.width, vcap->format.height,
> +			 vcap->format.field);
> +	tpg_s_bytesperline(&vcap->tpg, 0, vcap->format.width * vpix->bpp);
> +	tpg_s_buf_height(&vcap->tpg, vcap->format.height);
> +	tpg_s_fourcc(&vcap->tpg, vpix->pixelformat);
> +	/*
> +	 * TODO: check why the tpg_s_field need this third argument if
> +	 * it is already receiving the field
> +	 */
> +	tpg_s_field(&vcap->tpg, vcap->format.field,
> +		    vcap->format.field == V4L2_FIELD_ALTERNATE);

I thought I explained that earlier? When in alternate field mode the format.field
value will be TOP or BOTTOM, but never ALTERNATE. So tpg_s_field needs to know if
it will create TOP or BOTTOM fields only, or if they will be alternating.

Since we don't support ALTERNATE anyway, just pass false as the third argument and
drop the comment.

> +	tpg_s_colorspace(&vcap->tpg, vcap->format.colorspace);
> +	tpg_s_ycbcr_enc(&vcap->tpg, vcap->format.ycbcr_enc);
> +	tpg_s_quantization(&vcap->tpg, vcap->format.quantization);
> +	tpg_s_xfer_func(&vcap->tpg, vcap->format.xfer_func);
> +}
> +
>   static int vimc_cap_start_streaming(struct vb2_queue *vq, unsigned int count)
>   {
>   	struct vimc_cap_device *vcap = vb2_get_drv_priv(vq);
> @@ -256,20 +346,43 @@ static int vimc_cap_start_streaming(struct vb2_queue *vq, unsigned int count)
>   
>   	/* Start the media pipeline */
>   	ret = media_pipeline_start(entity, &vcap->pipe);
> -	if (ret) {
> -		vimc_cap_return_all_buffers(vcap, VB2_BUF_STATE_QUEUED);
> -		return ret;
> -	}
> +	if (ret)
> +		goto err_ret_all_buffs;
>   
>   	/* Enable streaming from the pipe */
>   	ret = vimc_pipeline_s_stream(&vcap->vdev.entity, 1);
> -	if (ret) {
> -		media_pipeline_stop(entity);
> -		vimc_cap_return_all_buffers(vcap, VB2_BUF_STATE_QUEUED);
> -		return ret;
> +	if (ret < 0)
> +		goto err_mpipe_stop;
> +
> +	if (ret == VIMC_PIPE_OPT) {
> +		tpg_init(&vcap->tpg, vcap->format.width, vcap->format.height);
> +		ret = tpg_alloc(&vcap->tpg, VIMC_FRAME_MAX_WIDTH);
> +		if (ret)
> +			/* We don't need to call vimc_pipeline_s_stream(e, 0) */
> +			goto err_mpipe_stop;
> +
> +		vimc_cap_tpg_s_format(vcap);
> +		vcap->kthread_cap = kthread_run(vimc_cap_tpg_thread, vcap,
> +					"%s-cap", vcap->vdev.v4l2_dev->name);
> +		if (IS_ERR(vcap->kthread_cap)) {
> +			dev_err(vcap->vdev.v4l2_dev->dev,
> +				"%s: kernel_thread() failed\n",
> +				vcap->vdev.name);
> +			ret = PTR_ERR(vcap->kthread_cap);
> +			goto err_tpg_free;
> +		}
>   	}
>   
>   	return 0;
> +
> +err_tpg_free:
> +	tpg_free(&vcap->tpg);
> +err_mpipe_stop:
> +	media_pipeline_stop(entity);
> +err_ret_all_buffs:
> +	vimc_cap_return_all_buffers(vcap, VB2_BUF_STATE_QUEUED);
> +
> +	return ret;
>   }
>   
>   /*
> @@ -280,8 +393,15 @@ static void vimc_cap_stop_streaming(struct vb2_queue *vq)
>   {
>   	struct vimc_cap_device *vcap = vb2_get_drv_priv(vq);
>   
> -	/* Disable streaming from the pipe */
> -	vimc_pipeline_s_stream(&vcap->vdev.entity, 0);
> +	if (vcap->kthread_cap) {
> +		/* Stop image generator */
> +		kthread_stop(vcap->kthread_cap);
> +		vcap->kthread_cap = NULL;
> +		tpg_free(&vcap->tpg);
> +	} else {
> +		/* Disable streaming from the pipe */
> +		vimc_pipeline_s_stream(&vcap->vdev.entity, 0);
> +	}
>   
>   	/* Stop the media pipeline */
>   	media_pipeline_stop(&vcap->vdev.entity);
> @@ -361,44 +481,6 @@ static void vimc_cap_destroy(struct vimc_ent_device *ved)
>   	kfree(vcap);
>   }
>   
> -static void vimc_cap_process_frame(struct vimc_ent_device *ved,
> -				   struct media_pad *sink, const void *frame)
> -{
> -	struct vimc_cap_device *vcap = container_of(ved, struct vimc_cap_device,
> -						    ved);
> -	struct vimc_cap_buffer *vimc_buf;
> -	void *vbuf;
> -
> -	spin_lock(&vcap->qlock);
> -
> -	/* Get the first entry of the list */
> -	vimc_buf = list_first_entry_or_null(&vcap->buf_list,
> -					    typeof(*vimc_buf), list);
> -	if (!vimc_buf) {
> -		spin_unlock(&vcap->qlock);
> -		return;
> -	}
> -
> -	/* Remove this entry from the list */
> -	list_del(&vimc_buf->list);
> -
> -	spin_unlock(&vcap->qlock);
> -
> -	/* Fill the buffer */
> -	vimc_buf->vb2.vb2_buf.timestamp = ktime_get_ns();
> -	vimc_buf->vb2.sequence = vcap->sequence++;
> -	vimc_buf->vb2.field = vcap->format.field;
> -
> -	vbuf = vb2_plane_vaddr(&vimc_buf->vb2.vb2_buf, 0);
> -
> -	memcpy(vbuf, frame, vcap->format.sizeimage);
> -
> -	/* Set it as ready */
> -	vb2_set_plane_payload(&vimc_buf->vb2.vb2_buf, 0,
> -			      vcap->format.sizeimage);
> -	vb2_buffer_done(&vimc_buf->vb2.vb2_buf, VB2_BUF_STATE_DONE);
> -}
> -
>   struct vimc_ent_device *vimc_cap_create(struct v4l2_device *v4l2_dev,
>   					const char *const name,
>   					u16 num_pads,
> diff --git a/drivers/media/platform/vimc/vimc-common.h b/drivers/media/platform/vimc/vimc-common.h
> index 2189fd6..5b2691e 100644
> --- a/drivers/media/platform/vimc/vimc-common.h
> +++ b/drivers/media/platform/vimc/vimc-common.h
> @@ -27,6 +27,8 @@
>   #define VIMC_FRAME_MIN_WIDTH 16
>   #define VIMC_FRAME_MIN_HEIGHT 16
>   
> +#define VIMC_PIPE_OPT 1
> +
>   /**
>    * struct vimc_pix_map - maps media bus code with v4l2 pixel format
>    *
> diff --git a/drivers/media/platform/vimc/vimc-sensor.c b/drivers/media/platform/vimc/vimc-sensor.c
> index 90c41c6..df89ee7 100644
> --- a/drivers/media/platform/vimc/vimc-sensor.c
> +++ b/drivers/media/platform/vimc/vimc-sensor.c
> @@ -15,6 +15,7 @@
>    *
>    */
>   
> +#include <linux/module.h>
>   #include <linux/freezer.h>
>   #include <linux/kthread.h>
>   #include <linux/v4l2-mediabus.h>
> @@ -24,6 +25,13 @@
>   
>   #include "vimc-sensor.h"
>   
> +static bool vsen_tpg;
> +module_param(vsen_tpg, bool, 0000);
> +MODULE_PARM_DESC(vsen_tpg, " generate image from sensor node\n"
> +	"If set to false, then the pipe will be optimized and image will be "
> +	"generated directly in the capture node instead of going through "
> +	"the whole pipe");
> +
>   struct vimc_sen_device {
>   	struct vimc_ent_device ved;
>   	struct v4l2_subdev sd;
> @@ -239,6 +247,13 @@ static int vimc_sen_s_stream(struct v4l2_subdev *sd, int enable)
>   				container_of(sd, struct vimc_sen_device, sd);
>   	int ret;
>   
> +	if (!vsen_tpg)
> +		/*
> +		 * If we are not generating the frames, then inform the caller
> +		 * to generate the frame in shallow level of the pipe
> +		 */
> +		return VIMC_PIPE_OPT;
> +
>   	if (enable) {
>   		const struct vimc_pix_map *vpix;
>   		unsigned int frame_size;
> @@ -306,7 +321,8 @@ static void vimc_sen_destroy(struct vimc_ent_device *ved)
>   				container_of(ved, struct vimc_sen_device, ved);
>   
>   	vimc_ent_sd_unregister(ved, &vsen->sd);
> -	tpg_free(&vsen->tpg);
> +	if (vsen_tpg)
> +		tpg_free(&vsen->tpg);
>   	kfree(vsen);
>   }
>   
> @@ -344,11 +360,13 @@ struct vimc_ent_device *vimc_sen_create(struct v4l2_device *v4l2_dev,
>   	vsen->mbus_format = fmt_default;
>   
>   	/* Initialize the test pattern generator */
> -	tpg_init(&vsen->tpg, vsen->mbus_format.width,
> -		 vsen->mbus_format.height);
> -	ret = tpg_alloc(&vsen->tpg, VIMC_FRAME_MAX_WIDTH);
> -	if (ret)
> -		goto err_unregister_ent_sd;
> +	if (vsen_tpg) {
> +		tpg_init(&vsen->tpg, vsen->mbus_format.width,
> +			 vsen->mbus_format.height);
> +		ret = tpg_alloc(&vsen->tpg, VIMC_FRAME_MAX_WIDTH);
> +		if (ret)
> +			goto err_unregister_ent_sd;
> +	}
>   
>   	return &vsen->ved;
>   
> 

Regards,

	Hans

Re: [RFC PATCH v3 08/11] [media] vimc: Optimize frame generation through the pipe

[Helen Koike]

Hi Hans,

Thank you for your review

On 2017-06-12 07:03 AM, Hans Verkuil wrote:
> On 06/03/2017 04:58 AM, Helen Koike wrote:
>> Add a parameter called vsen_tpg, if true then vimc will work as before:
>> frames will be generated in the sensor nodes then propagated through the
>> pipe and processed by each node until a capture node is reached.
>> If vsen_tpg is false, then the frame is not generated by the sensor, but
>> directly from the capture node, thus saving intermediate memory buffers
>> and process time, allowing a higher frame rate.
>>
>> Signed-off-by: Helen Koike <helen.koike@collabora.com>
>>
>> ---
>>
>> Changes in v3:
>> [media] vimc: Optimize frame generation through the pipe
>>     - This is a new patch in the series
>>
>> Changes in v2: None
>>
>>
>> ---
>>   drivers/media/platform/vimc/vimc-capture.c | 178
>> +++++++++++++++++++++--------
>>   drivers/media/platform/vimc/vimc-common.h  |   2 +
>>   drivers/media/platform/vimc/vimc-sensor.c  |  30 ++++-
>>   3 files changed, 156 insertions(+), 54 deletions(-)
>>
>> diff --git a/drivers/media/platform/vimc/vimc-capture.c
>> b/drivers/media/platform/vimc/vimc-capture.c
>> index e943267..b5da0ea 100644
>> --- a/drivers/media/platform/vimc/vimc-capture.c
>> +++ b/drivers/media/platform/vimc/vimc-capture.c
>> @@ -15,7 +15,10 @@
>>    *
>>    */
>>   +#include <linux/freezer.h>
>> +#include <linux/kthread.h>
>>   #include <media/v4l2-ioctl.h>
>> +#include <media/v4l2-tpg.h>
>>   #include <media/videobuf2-core.h>
>>   #include <media/videobuf2-vmalloc.h>
>>   @@ -38,6 +41,8 @@ struct vimc_cap_device {
>>       struct mutex lock;
>>       u32 sequence;
>>       struct media_pipeline pipe;
>> +    struct tpg_data tpg;
>> +    struct task_struct *kthread_cap;
>>   };
>>     static const struct v4l2_pix_format fmt_default = {
>> @@ -246,6 +251,91 @@ static void vimc_cap_return_all_buffers(struct
>> vimc_cap_device *vcap,
>>       spin_unlock(&vcap->qlock);
>>   }
>>   +static void vimc_cap_process_frame(struct vimc_ent_device *ved,
>> +                   struct media_pad *sink, const void *frame)
>> +{
>> +    struct vimc_cap_device *vcap = container_of(ved, struct
>> vimc_cap_device,
>> +                            ved);
>> +    struct vimc_cap_buffer *vimc_buf;
>> +    void *vbuf;
>> +
>> +    spin_lock(&vcap->qlock);
>> +
>> +    /* Get the first entry of the list */
>> +    vimc_buf = list_first_entry_or_null(&vcap->buf_list,
>> +                        typeof(*vimc_buf), list);
>> +    if (!vimc_buf) {
>> +        spin_unlock(&vcap->qlock);
>> +        return;
>> +    }
>> +
>> +    /* Remove this entry from the list */
>> +    list_del(&vimc_buf->list);
>> +
>> +    spin_unlock(&vcap->qlock);
>> +
>> +    /* Fill the buffer */
>> +    vimc_buf->vb2.vb2_buf.timestamp = ktime_get_ns();
>> +    vimc_buf->vb2.sequence = vcap->sequence++;
>> +    vimc_buf->vb2.field = vcap->format.field;
>> +
>> +    vbuf = vb2_plane_vaddr(&vimc_buf->vb2.vb2_buf, 0);
>> +
>> +    if (sink)
>> +        memcpy(vbuf, frame, vcap->format.sizeimage);
>> +    else
>> +        tpg_fill_plane_buffer(&vcap->tpg, V4L2_STD_PAL, 0, vbuf);
>> +
>> +    /* Set it as ready */
>> +    vb2_set_plane_payload(&vimc_buf->vb2.vb2_buf, 0,
>> +                  vcap->format.sizeimage);
>> +    vb2_buffer_done(&vimc_buf->vb2.vb2_buf, VB2_BUF_STATE_DONE);
>> +}
>> +
>> +
>> +static int vimc_cap_tpg_thread(void *data)
>> +{
>> +    struct vimc_cap_device *vcap = data;
>> +
>> +    set_freezable();
>> +    set_current_state(TASK_UNINTERRUPTIBLE);
>> +
>> +    for (;;) {
>> +        try_to_freeze();
>> +        if (kthread_should_stop())
>> +            break;
>> +
>> +        vimc_cap_process_frame(&vcap->ved, NULL, NULL);
>> +
>> +        /* 60 frames per second */
>> +        schedule_timeout(HZ/60);
>> +    }
>> +
>> +    return 0;
>> +}
>> +
>> +static void vimc_cap_tpg_s_format(struct vimc_cap_device *vcap)
>> +{
>> +    const struct vimc_pix_map *vpix =
>> +            vimc_pix_map_by_pixelformat(vcap->format.pixelformat);
>> +
>> +    tpg_reset_source(&vcap->tpg, vcap->format.width,
>> vcap->format.height,
>> +             vcap->format.field);
>> +    tpg_s_bytesperline(&vcap->tpg, 0, vcap->format.width * vpix->bpp);
>> +    tpg_s_buf_height(&vcap->tpg, vcap->format.height);
>> +    tpg_s_fourcc(&vcap->tpg, vpix->pixelformat);
>> +    /*
>> +     * TODO: check why the tpg_s_field need this third argument if
>> +     * it is already receiving the field
>> +     */
>> +    tpg_s_field(&vcap->tpg, vcap->format.field,
>> +            vcap->format.field == V4L2_FIELD_ALTERNATE);
>
> I thought I explained that earlier? When in alternate field mode the
> format.field
> value will be TOP or BOTTOM, but never ALTERNATE. So tpg_s_field needs
> to know if
> it will create TOP or BOTTOM fields only, or if they will be alternating.

Yes, sorry about that. I removed from the vimc_sensor but I forgot it 
here. I am dropping this patch in the series as I found another bug when 
multiple links are enabled going to the same sink pad. I'll re-work in 
this optimization and re-send another version in the future in a 
different patch series.

>
> Since we don't support ALTERNATE anyway, just pass false as the third
> argument and
> drop the comment.
>
>> +    tpg_s_colorspace(&vcap->tpg, vcap->format.colorspace);
>> +    tpg_s_ycbcr_enc(&vcap->tpg, vcap->format.ycbcr_enc);
>> +    tpg_s_quantization(&vcap->tpg, vcap->format.quantization);
>> +    tpg_s_xfer_func(&vcap->tpg, vcap->format.xfer_func);
>> +}
>> +
>>   static int vimc_cap_start_streaming(struct vb2_queue *vq, unsigned
>> int count)
>>   {
>>       struct vimc_cap_device *vcap = vb2_get_drv_priv(vq);
>> @@ -256,20 +346,43 @@ static int vimc_cap_start_streaming(struct
>> vb2_queue *vq, unsigned int count)
>>         /* Start the media pipeline */
>>       ret = media_pipeline_start(entity, &vcap->pipe);
>> -    if (ret) {
>> -        vimc_cap_return_all_buffers(vcap, VB2_BUF_STATE_QUEUED);
>> -        return ret;
>> -    }
>> +    if (ret)
>> +        goto err_ret_all_buffs;
>>         /* Enable streaming from the pipe */
>>       ret = vimc_pipeline_s_stream(&vcap->vdev.entity, 1);
>> -    if (ret) {
>> -        media_pipeline_stop(entity);
>> -        vimc_cap_return_all_buffers(vcap, VB2_BUF_STATE_QUEUED);
>> -        return ret;
>> +    if (ret < 0)
>> +        goto err_mpipe_stop;
>> +
>> +    if (ret == VIMC_PIPE_OPT) {
>> +        tpg_init(&vcap->tpg, vcap->format.width, vcap->format.height);
>> +        ret = tpg_alloc(&vcap->tpg, VIMC_FRAME_MAX_WIDTH);
>> +        if (ret)
>> +            /* We don't need to call vimc_pipeline_s_stream(e, 0) */
>> +            goto err_mpipe_stop;
>> +
>> +        vimc_cap_tpg_s_format(vcap);
>> +        vcap->kthread_cap = kthread_run(vimc_cap_tpg_thread, vcap,
>> +                    "%s-cap", vcap->vdev.v4l2_dev->name);
>> +        if (IS_ERR(vcap->kthread_cap)) {
>> +            dev_err(vcap->vdev.v4l2_dev->dev,
>> +                "%s: kernel_thread() failed\n",
>> +                vcap->vdev.name);
>> +            ret = PTR_ERR(vcap->kthread_cap);
>> +            goto err_tpg_free;
>> +        }
>>       }
>>         return 0;
>> +
>> +err_tpg_free:
>> +    tpg_free(&vcap->tpg);
>> +err_mpipe_stop:
>> +    media_pipeline_stop(entity);
>> +err_ret_all_buffs:
>> +    vimc_cap_return_all_buffers(vcap, VB2_BUF_STATE_QUEUED);
>> +
>> +    return ret;
>>   }
>>     /*
>> @@ -280,8 +393,15 @@ static void vimc_cap_stop_streaming(struct
>> vb2_queue *vq)
>>   {
>>       struct vimc_cap_device *vcap = vb2_get_drv_priv(vq);
>>   -    /* Disable streaming from the pipe */
>> -    vimc_pipeline_s_stream(&vcap->vdev.entity, 0);
>> +    if (vcap->kthread_cap) {
>> +        /* Stop image generator */
>> +        kthread_stop(vcap->kthread_cap);
>> +        vcap->kthread_cap = NULL;
>> +        tpg_free(&vcap->tpg);
>> +    } else {
>> +        /* Disable streaming from the pipe */
>> +        vimc_pipeline_s_stream(&vcap->vdev.entity, 0);
>> +    }
>>         /* Stop the media pipeline */
>>       media_pipeline_stop(&vcap->vdev.entity);
>> @@ -361,44 +481,6 @@ static void vimc_cap_destroy(struct
>> vimc_ent_device *ved)
>>       kfree(vcap);
>>   }
>>   -static void vimc_cap_process_frame(struct vimc_ent_device *ved,
>> -                   struct media_pad *sink, const void *frame)
>> -{
>> -    struct vimc_cap_device *vcap = container_of(ved, struct
>> vimc_cap_device,
>> -                            ved);
>> -    struct vimc_cap_buffer *vimc_buf;
>> -    void *vbuf;
>> -
>> -    spin_lock(&vcap->qlock);
>> -
>> -    /* Get the first entry of the list */
>> -    vimc_buf = list_first_entry_or_null(&vcap->buf_list,
>> -                        typeof(*vimc_buf), list);
>> -    if (!vimc_buf) {
>> -        spin_unlock(&vcap->qlock);
>> -        return;
>> -    }
>> -
>> -    /* Remove this entry from the list */
>> -    list_del(&vimc_buf->list);
>> -
>> -    spin_unlock(&vcap->qlock);
>> -
>> -    /* Fill the buffer */
>> -    vimc_buf->vb2.vb2_buf.timestamp = ktime_get_ns();
>> -    vimc_buf->vb2.sequence = vcap->sequence++;
>> -    vimc_buf->vb2.field = vcap->format.field;
>> -
>> -    vbuf = vb2_plane_vaddr(&vimc_buf->vb2.vb2_buf, 0);
>> -
>> -    memcpy(vbuf, frame, vcap->format.sizeimage);
>> -
>> -    /* Set it as ready */
>> -    vb2_set_plane_payload(&vimc_buf->vb2.vb2_buf, 0,
>> -                  vcap->format.sizeimage);
>> -    vb2_buffer_done(&vimc_buf->vb2.vb2_buf, VB2_BUF_STATE_DONE);
>> -}
>> -
>>   struct vimc_ent_device *vimc_cap_create(struct v4l2_device *v4l2_dev,
>>                       const char *const name,
>>                       u16 num_pads,
>> diff --git a/drivers/media/platform/vimc/vimc-common.h
>> b/drivers/media/platform/vimc/vimc-common.h
>> index 2189fd6..5b2691e 100644
>> --- a/drivers/media/platform/vimc/vimc-common.h
>> +++ b/drivers/media/platform/vimc/vimc-common.h
>> @@ -27,6 +27,8 @@
>>   #define VIMC_FRAME_MIN_WIDTH 16
>>   #define VIMC_FRAME_MIN_HEIGHT 16
>>   +#define VIMC_PIPE_OPT 1
>> +
>>   /**
>>    * struct vimc_pix_map - maps media bus code with v4l2 pixel format
>>    *
>> diff --git a/drivers/media/platform/vimc/vimc-sensor.c
>> b/drivers/media/platform/vimc/vimc-sensor.c
>> index 90c41c6..df89ee7 100644
>> --- a/drivers/media/platform/vimc/vimc-sensor.c
>> +++ b/drivers/media/platform/vimc/vimc-sensor.c
>> @@ -15,6 +15,7 @@
>>    *
>>    */
>>   +#include <linux/module.h>
>>   #include <linux/freezer.h>
>>   #include <linux/kthread.h>
>>   #include <linux/v4l2-mediabus.h>
>> @@ -24,6 +25,13 @@
>>     #include "vimc-sensor.h"
>>   +static bool vsen_tpg;
>> +module_param(vsen_tpg, bool, 0000);
>> +MODULE_PARM_DESC(vsen_tpg, " generate image from sensor node\n"
>> +    "If set to false, then the pipe will be optimized and image will
>> be "
>> +    "generated directly in the capture node instead of going through "
>> +    "the whole pipe");
>> +
>>   struct vimc_sen_device {
>>       struct vimc_ent_device ved;
>>       struct v4l2_subdev sd;
>> @@ -239,6 +247,13 @@ static int vimc_sen_s_stream(struct v4l2_subdev
>> *sd, int enable)
>>                   container_of(sd, struct vimc_sen_device, sd);
>>       int ret;
>>   +    if (!vsen_tpg)
>> +        /*
>> +         * If we are not generating the frames, then inform the caller
>> +         * to generate the frame in shallow level of the pipe
>> +         */
>> +        return VIMC_PIPE_OPT;
>> +
>>       if (enable) {
>>           const struct vimc_pix_map *vpix;
>>           unsigned int frame_size;
>> @@ -306,7 +321,8 @@ static void vimc_sen_destroy(struct
>> vimc_ent_device *ved)
>>                   container_of(ved, struct vimc_sen_device, ved);
>>         vimc_ent_sd_unregister(ved, &vsen->sd);
>> -    tpg_free(&vsen->tpg);
>> +    if (vsen_tpg)
>> +        tpg_free(&vsen->tpg);
>>       kfree(vsen);
>>   }
>>   @@ -344,11 +360,13 @@ struct vimc_ent_device *vimc_sen_create(struct
>> v4l2_device *v4l2_dev,
>>       vsen->mbus_format = fmt_default;
>>         /* Initialize the test pattern generator */
>> -    tpg_init(&vsen->tpg, vsen->mbus_format.width,
>> -         vsen->mbus_format.height);
>> -    ret = tpg_alloc(&vsen->tpg, VIMC_FRAME_MAX_WIDTH);
>> -    if (ret)
>> -        goto err_unregister_ent_sd;
>> +    if (vsen_tpg) {
>> +        tpg_init(&vsen->tpg, vsen->mbus_format.width,
>> +             vsen->mbus_format.height);
>> +        ret = tpg_alloc(&vsen->tpg, VIMC_FRAME_MAX_WIDTH);
>> +        if (ret)
>> +            goto err_unregister_ent_sd;
>> +    }
>>         return &vsen->ved;
>>
>
> Regards,
>
>     Hans

Thanks,

Helen

[RFC PATCH v3 09/11] [media] vimc: Subdevices as modules

[Helen Koike]

Change the core structure for adding subdevices in the topology.
Instead of calling the specific create function for each subdevice,
inject a child platform_device with the driver's name.
Each type of node in the topology (sensor, capture, debayer, scaler)
will register a platform_driver with the corresponding name through the
component subsystem.
Implementing a new subdevice type doesn't require vimc-core to be altered.

This facilitates future implementation of dynamic entities, where
hotpluging an entity in the topology is just a matter of
registering/unregistering a platform_device in the system.
It also facilitates other implementations of different nodes without
touching the core code and remove the need of a header file for each
type of node.

Signed-off-by: Helen Koike <helen.koike@collabora.com>

---

Changes in v3:
[media] vimc: Subdevices as modules
	- This is a new patch in the series

Changes in v2: None


---
 drivers/media/platform/vimc/Makefile       |   7 +-
 drivers/media/platform/vimc/vimc-capture.c |  96 ++++---
 drivers/media/platform/vimc/vimc-capture.h |  28 --
 drivers/media/platform/vimc/vimc-common.c  |  37 ++-
 drivers/media/platform/vimc/vimc-common.h  |  14 +-
 drivers/media/platform/vimc/vimc-core.c    | 406 +++++++++++------------------
 drivers/media/platform/vimc/vimc-sensor.c  |  91 +++++--
 drivers/media/platform/vimc/vimc-sensor.h  |  28 --
 8 files changed, 320 insertions(+), 387 deletions(-)
 delete mode 100644 drivers/media/platform/vimc/vimc-capture.h
 delete mode 100644 drivers/media/platform/vimc/vimc-sensor.h

diff --git a/drivers/media/platform/vimc/Makefile b/drivers/media/platform/vimc/Makefile
index 6b6ddf4..0e5d5ce 100644
--- a/drivers/media/platform/vimc/Makefile
+++ b/drivers/media/platform/vimc/Makefile
@@ -1,3 +1,6 @@
-vimc-objs := vimc-core.o vimc-capture.o vimc-common.o vimc-sensor.o
+vimc-objs := vimc-core.o
+vimc_capture-objs := vimc-capture.o
+vimc_common-objs := vimc-common.o
+vimc_sensor-objs := vimc-sensor.o
 
-obj-$(CONFIG_VIDEO_VIMC) += vimc.o
+obj-$(CONFIG_VIDEO_VIMC) += vimc.o vimc_capture.o vimc_common.o vimc_sensor.o
diff --git a/drivers/media/platform/vimc/vimc-capture.c b/drivers/media/platform/vimc/vimc-capture.c
index b5da0ea..633d99a 100644
--- a/drivers/media/platform/vimc/vimc-capture.c
+++ b/drivers/media/platform/vimc/vimc-capture.c
@@ -15,18 +15,24 @@
  *
  */
 
+#include <linux/component.h>
 #include <linux/freezer.h>
 #include <linux/kthread.h>
+#include <linux/module.h>
+#include <linux/platform_device.h>
 #include <media/v4l2-ioctl.h>
 #include <media/v4l2-tpg.h>
 #include <media/videobuf2-core.h>
 #include <media/videobuf2-vmalloc.h>
 
-#include "vimc-capture.h"
+#include "vimc-common.h"
+
+#define VIMC_CAP_DRV_NAME "vimc-capture"
 
 struct vimc_cap_device {
 	struct vimc_ent_device ved;
 	struct video_device vdev;
+	struct device *dev;
 	struct v4l2_pix_format format;
 	struct vb2_queue queue;
 	struct list_head buf_list;
@@ -150,7 +156,7 @@ static int vimc_cap_s_fmt_vid_cap(struct file *file, void *priv,
 
 	vimc_cap_try_fmt_vid_cap(file, priv, f);
 
-	dev_dbg(vcap->vdev.v4l2_dev->dev, "%s: format update: "
+	dev_dbg(vcap->dev, "%s: format update: "
 		"old:%dx%d (0x%x, %d, %d, %d, %d) "
 		"new:%dx%d (0x%x, %d, %d, %d, %d)\n", vcap->vdev.name,
 		/* old */
@@ -365,8 +371,7 @@ static int vimc_cap_start_streaming(struct vb2_queue *vq, unsigned int count)
 		vcap->kthread_cap = kthread_run(vimc_cap_tpg_thread, vcap,
 					"%s-cap", vcap->vdev.v4l2_dev->name);
 		if (IS_ERR(vcap->kthread_cap)) {
-			dev_err(vcap->vdev.v4l2_dev->dev,
-				"%s: kernel_thread() failed\n",
+			dev_err(vcap->dev, "%s: kernel_thread() failed\n",
 				vcap->vdev.name);
 			ret = PTR_ERR(vcap->kthread_cap);
 			goto err_tpg_free;
@@ -443,8 +448,7 @@ static int vimc_cap_buffer_prepare(struct vb2_buffer *vb)
 	unsigned long size = vcap->format.sizeimage;
 
 	if (vb2_plane_size(vb, 0) < size) {
-		dev_err(vcap->vdev.v4l2_dev->dev,
-			"%s: buffer too small (%lu < %lu)\n",
+		dev_err(vcap->dev, "%s: buffer too small (%lu < %lu)\n",
 			vcap->vdev.name, vb2_plane_size(vb, 0), size);
 		return -EINVAL;
 	}
@@ -469,8 +473,10 @@ static const struct media_entity_operations vimc_cap_mops = {
 	.link_validate		= vimc_link_validate,
 };
 
-static void vimc_cap_destroy(struct vimc_ent_device *ved)
+static void vimc_cap_comp_unbind(struct device *comp, struct device *master,
+				 void *master_data)
 {
+	struct vimc_ent_device *ved = dev_get_drvdata(comp);
 	struct vimc_cap_device *vcap = container_of(ved, struct vimc_cap_device,
 						    ved);
 
@@ -481,32 +487,25 @@ static void vimc_cap_destroy(struct vimc_ent_device *ved)
 	kfree(vcap);
 }
 
-struct vimc_ent_device *vimc_cap_create(struct v4l2_device *v4l2_dev,
-					const char *const name,
-					u16 num_pads,
-					const unsigned long *pads_flag)
+static int vimc_cap_comp_bind(struct device *comp, struct device *master,
+			      void *master_data)
 {
+	struct v4l2_device *v4l2_dev = master_data;
+	char *name = comp->platform_data;
 	const struct vimc_pix_map *vpix;
 	struct vimc_cap_device *vcap;
 	struct video_device *vdev;
 	struct vb2_queue *q;
 	int ret;
 
-	/*
-	 * Check entity configuration params
-	 * NOTE: we only support a single sink pad
-	 */
-	if (!name || num_pads != 1 || !pads_flag ||
-	    !(pads_flag[0] & MEDIA_PAD_FL_SINK))
-		return ERR_PTR(-EINVAL);
-
 	/* Allocate the vimc_cap_device struct */
 	vcap = kzalloc(sizeof(*vcap), GFP_KERNEL);
 	if (!vcap)
-		return ERR_PTR(-ENOMEM);
+		return -ENOMEM;
 
 	/* Allocate the pads */
-	vcap->ved.pads = vimc_pads_init(num_pads, pads_flag);
+	vcap->ved.pads =
+		vimc_pads_init(1, (const unsigned long[1]) {MEDIA_PAD_FL_SINK});
 	if (IS_ERR(vcap->ved.pads)) {
 		ret = PTR_ERR(vcap->ved.pads);
 		goto err_free_vcap;
@@ -516,7 +515,7 @@ struct vimc_ent_device *vimc_cap_create(struct v4l2_device *v4l2_dev,
 	vcap->vdev.entity.name = name;
 	vcap->vdev.entity.function = MEDIA_ENT_F_IO_V4L;
 	ret = media_entity_pads_init(&vcap->vdev.entity,
-				     num_pads, vcap->ved.pads);
+				     1, vcap->ved.pads);
 	if (ret)
 		goto err_clean_pads;
 
@@ -537,8 +536,7 @@ struct vimc_ent_device *vimc_cap_create(struct v4l2_device *v4l2_dev,
 
 	ret = vb2_queue_init(q);
 	if (ret) {
-		dev_err(vcap->vdev.v4l2_dev->dev,
-			"%s: vb2 queue init failed (err=%d)\n",
+		dev_err(comp, "%s: vb2 queue init failed (err=%d)\n",
 			vcap->vdev.name, ret);
 		goto err_clean_m_ent;
 	}
@@ -555,10 +553,11 @@ struct vimc_ent_device *vimc_cap_create(struct v4l2_device *v4l2_dev,
 				 vcap->format.height;
 
 	/* Fill the vimc_ent_device struct */
-	vcap->ved.destroy = vimc_cap_destroy;
 	vcap->ved.ent = &vcap->vdev.entity;
 	vcap->ved.process_frame = vimc_cap_process_frame;
 	vcap->ved.vdev_get_format = vimc_cap_get_format;
+	dev_set_drvdata(comp, &vcap->ved);
+	vcap->dev = comp;
 
 	/* Initialize the video_device struct */
 	vdev = &vcap->vdev;
@@ -577,13 +576,12 @@ struct vimc_ent_device *vimc_cap_create(struct v4l2_device *v4l2_dev,
 	/* Register the video_device with the v4l2 and the media framework */
 	ret = video_register_device(vdev, VFL_TYPE_GRABBER, -1);
 	if (ret) {
-		dev_err(vcap->vdev.v4l2_dev->dev,
-			"%s: video register failed (err=%d)\n",
+		dev_err(comp, "%s: video register failed (err=%d)\n",
 			vcap->vdev.name, ret);
 		goto err_release_queue;
 	}
 
-	return &vcap->ved;
+	return 0;
 
 err_release_queue:
 	vb2_queue_release(q);
@@ -594,5 +592,45 @@ struct vimc_ent_device *vimc_cap_create(struct v4l2_device *v4l2_dev,
 err_free_vcap:
 	kfree(vcap);
 
-	return ERR_PTR(ret);
+	return ret;
+}
+
+static const struct component_ops vimc_cap_comp_ops = {
+	.bind = vimc_cap_comp_bind,
+	.unbind = vimc_cap_comp_unbind,
+};
+
+static int vimc_cap_probe(struct platform_device *pdev)
+{
+	return component_add(&pdev->dev, &vimc_cap_comp_ops);
 }
+
+static int vimc_cap_remove(struct platform_device *pdev)
+{
+	component_del(&pdev->dev, &vimc_cap_comp_ops);
+
+	return 0;
+}
+
+static struct platform_driver vimc_cap_pdrv = {
+	.probe		= vimc_cap_probe,
+	.remove		= vimc_cap_remove,
+	.driver		= {
+		.name	= VIMC_CAP_DRV_NAME,
+	},
+};
+
+static const struct platform_device_id vimc_cap_driver_ids[] = {
+	{
+		.name           = VIMC_CAP_DRV_NAME,
+	},
+	{ }
+};
+
+module_platform_driver(vimc_cap_pdrv);
+
+MODULE_DEVICE_TABLE(platform, vimc_cap_driver_ids);
+
+MODULE_DESCRIPTION("Virtual Media Controller Driver (VIMC) Capture");
+MODULE_AUTHOR("Helen Mae Koike Fornazier <helen.fornazier@gmail.com>");
+MODULE_LICENSE("GPL");
diff --git a/drivers/media/platform/vimc/vimc-capture.h b/drivers/media/platform/vimc/vimc-capture.h
deleted file mode 100644
index 7e5c707..0000000
--- a/drivers/media/platform/vimc/vimc-capture.h
+++ /dev/null
@@ -1,28 +0,0 @@
-/*
- * vimc-capture.h Virtual Media Controller Driver
- *
- * Copyright (C) 2015-2017 Helen Koike <helen.fornazier@gmail.com>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- */
-
-#ifndef _VIMC_CAPTURE_H_
-#define _VIMC_CAPTURE_H_
-
-#include "vimc-common.h"
-
-struct vimc_ent_device *vimc_cap_create(struct v4l2_device *v4l2_dev,
-					const char *const name,
-					u16 num_pads,
-					const unsigned long *pads_flag);
-
-#endif
diff --git a/drivers/media/platform/vimc/vimc-common.c b/drivers/media/platform/vimc/vimc-common.c
index ff59e09..d5ed387 100644
--- a/drivers/media/platform/vimc/vimc-common.c
+++ b/drivers/media/platform/vimc/vimc-common.c
@@ -15,6 +15,9 @@
  *
  */
 
+#include <linux/init.h>
+#include <linux/module.h>
+
 #include "vimc-common.h"
 
 static const struct vimc_pix_map vimc_pix_map_list[] = {
@@ -151,6 +154,7 @@ const struct vimc_pix_map *vimc_pix_map_by_index(unsigned int i)
 
 	return &vimc_pix_map_list[i];
 }
+EXPORT_SYMBOL(vimc_pix_map_by_index);
 
 const struct vimc_pix_map *vimc_pix_map_by_code(u32 code)
 {
@@ -162,6 +166,7 @@ const struct vimc_pix_map *vimc_pix_map_by_code(u32 code)
 	}
 	return NULL;
 }
+EXPORT_SYMBOL(vimc_pix_map_by_code);
 
 const struct vimc_pix_map *vimc_pix_map_by_pixelformat(u32 pixelformat)
 {
@@ -173,6 +178,7 @@ const struct vimc_pix_map *vimc_pix_map_by_pixelformat(u32 pixelformat)
 	}
 	return NULL;
 }
+EXPORT_SYMBOL(vimc_pix_map_by_pixelformat);
 
 int vimc_propagate_frame(struct media_pad *src, const void *frame)
 {
@@ -207,6 +213,7 @@ int vimc_propagate_frame(struct media_pad *src, const void *frame)
 
 	return 0;
 }
+EXPORT_SYMBOL(vimc_propagate_frame);
 
 /* Helper function to allocate and initialize pads */
 struct media_pad *vimc_pads_init(u16 num_pads, const unsigned long *pads_flag)
@@ -227,6 +234,7 @@ struct media_pad *vimc_pads_init(u16 num_pads, const unsigned long *pads_flag)
 
 	return pads;
 }
+EXPORT_SYMBOL(vimc_pads_init);
 
 int vimc_pipeline_s_stream(struct media_entity *ent, int enable)
 {
@@ -242,14 +250,8 @@ int vimc_pipeline_s_stream(struct media_entity *ent, int enable)
 		/* Start the stream in the subdevice direct connected */
 		pad = media_entity_remote_pad(&ent->pads[i]);
 
-		/*
-		 * if this is a raw node from vimc-core, then there is
-		 * nothing to activate
-		 * TODO: remove this when there are no more raw nodes in the
-		 * core and return error instead
-		 */
-		if (pad->entity->obj_type == MEDIA_ENTITY_TYPE_BASE)
-			continue;
+		if (!is_media_entity_v4l2_subdev(pad->entity))
+			return -EINVAL;
 
 		sd = media_entity_to_v4l2_subdev(pad->entity);
 		ret = v4l2_subdev_call(sd, video, s_stream, enable);
@@ -259,6 +261,7 @@ int vimc_pipeline_s_stream(struct media_entity *ent, int enable)
 
 	return 0;
 }
+EXPORT_SYMBOL(vimc_pipeline_s_stream);
 
 static void vimc_fmt_pix_to_mbus(struct v4l2_mbus_framefmt *mfmt,
 				 struct v4l2_pix_format *pfmt)
@@ -315,14 +318,6 @@ int vimc_link_validate(struct media_link *link)
 	struct v4l2_subdev_format source_fmt, sink_fmt;
 	int ret;
 
-	/*
-	 * if it is a raw node from vimc-core, ignore the link for now
-	 * TODO: remove this when there are no more raw nodes in the
-	 * core and return error instead
-	 */
-	if (link->source->entity->obj_type == MEDIA_ENTITY_TYPE_BASE)
-		return 0;
-
 	ret = vimc_get_mbus_format(link->source, &source_fmt);
 	if (ret)
 		return ret;
@@ -393,8 +388,7 @@ int vimc_ent_sd_register(struct vimc_ent_device *ved,
 			 u32 function,
 			 u16 num_pads,
 			 const unsigned long *pads_flag,
-			 const struct v4l2_subdev_ops *sd_ops,
-			 void (*sd_destroy)(struct vimc_ent_device *))
+			 const struct v4l2_subdev_ops *sd_ops)
 {
 	int ret;
 
@@ -404,7 +398,6 @@ int vimc_ent_sd_register(struct vimc_ent_device *ved,
 		return PTR_ERR(ved->pads);
 
 	/* Fill the vimc_ent_device struct */
-	ved->destroy = sd_destroy;
 	ved->ent = &sd->entity;
 
 	/* Initialize the subdev */
@@ -440,6 +433,7 @@ int vimc_ent_sd_register(struct vimc_ent_device *ved,
 	vimc_pads_cleanup(ved->pads);
 	return ret;
 }
+EXPORT_SYMBOL(vimc_ent_sd_register);
 
 void vimc_ent_sd_unregister(struct vimc_ent_device *ved, struct v4l2_subdev *sd)
 {
@@ -447,3 +441,8 @@ void vimc_ent_sd_unregister(struct vimc_ent_device *ved, struct v4l2_subdev *sd)
 	media_entity_cleanup(ved->ent);
 	vimc_pads_cleanup(ved->pads);
 }
+EXPORT_SYMBOL(vimc_ent_sd_unregister);
+
+MODULE_DESCRIPTION("Virtual Media Controller Driver (VIMC) Common");
+MODULE_AUTHOR("Helen Koike <helen.fornazier@gmail.com>");
+MODULE_LICENSE("GPL");
diff --git a/drivers/media/platform/vimc/vimc-common.h b/drivers/media/platform/vimc/vimc-common.h
index 5b2691e..27c9b8c 100644
--- a/drivers/media/platform/vimc/vimc-common.h
+++ b/drivers/media/platform/vimc/vimc-common.h
@@ -1,5 +1,5 @@
 /*
- * vimc-ccommon.h Virtual Media Controller Driver
+ * vimc-common.h Virtual Media Controller Driver
  *
  * Copyright (C) 2015-2017 Helen Koike <helen.fornazier@gmail.com>
  *
@@ -50,7 +50,6 @@ struct vimc_pix_map {
  *
  * @ent:		the pointer to struct media_entity for the node
  * @pads:		the list of pads of the node
- * @destroy:		callback to destroy the node
  * @process_frame:	callback send a frame to that node
  * @vdev_get_format:	callback that returns the current format a pad, used
  *			only when is_media_entity_v4l2_video_device(ent) returns
@@ -67,7 +66,6 @@ struct vimc_pix_map {
 struct vimc_ent_device {
 	struct media_entity *ent;
 	struct media_pad *pads;
-	void (*destroy)(struct vimc_ent_device *);
 	void (*process_frame)(struct vimc_ent_device *ved,
 			      struct media_pad *sink, const void *frame);
 	void (*vdev_get_format)(struct vimc_ent_device *ved,
@@ -152,7 +150,6 @@ const struct vimc_pix_map *vimc_pix_map_by_pixelformat(u32 pixelformat);
  * @num_pads:	number of pads to initialize
  * @pads_flag:	flags to use in each pad
  * @sd_ops:	pointer to &struct v4l2_subdev_ops.
- * @sd_destroy:	callback to destroy the node
  *
  * Helper function initialize and register the struct vimc_ent_device and struct
  * v4l2_subdev which represents a subdev node in the topology
@@ -164,14 +161,13 @@ int vimc_ent_sd_register(struct vimc_ent_device *ved,
 			 u32 function,
 			 u16 num_pads,
 			 const unsigned long *pads_flag,
-			 const struct v4l2_subdev_ops *sd_ops,
-			 void (*sd_destroy)(struct vimc_ent_device *));
+			 const struct v4l2_subdev_ops *sd_ops);
 
 /**
- * vimc_ent_sd_register - initialize and register a subdev node
+ * vimc_ent_sd_unregister - cleanup and unregister a subdev node
  *
- * @ved:	the vimc_ent_device struct to be initialize
- * @sd:		the v4l2_subdev struct to be initialize and registered
+ * @ved:	the vimc_ent_device struct to be cleaned up
+ * @sd:		the v4l2_subdev struct to be unregistered
  *
  * Helper function cleanup and unregister the struct vimc_ent_device and struct
  * v4l2_subdev which represents a subdev node in the topology
diff --git a/drivers/media/platform/vimc/vimc-core.c b/drivers/media/platform/vimc/vimc-core.c
index afc79e2..6148c3e 100644
--- a/drivers/media/platform/vimc/vimc-core.c
+++ b/drivers/media/platform/vimc/vimc-core.c
@@ -15,15 +15,14 @@
  *
  */
 
+#include <linux/component.h>
 #include <linux/init.h>
 #include <linux/module.h>
 #include <linux/platform_device.h>
 #include <media/media-device.h>
 #include <media/v4l2-device.h>
 
-#include "vimc-capture.h"
 #include "vimc-common.h"
-#include "vimc-sensor.h"
 
 #define VIMC_PDEV_NAME "vimc"
 #define VIMC_MDEV_MODEL_NAME "VIMC MDEV"
@@ -37,10 +36,10 @@
 }
 
 struct vimc_device {
-	/*
-	 * The pipeline configuration
-	 * (filled before calling vimc_device_register)
-	 */
+	/* The platform device */
+	struct platform_device pdev;
+
+	/* The pipeline configuration */
 	const struct vimc_pipeline_config *pipe_cfg;
 
 	/* The Associated media_device parent */
@@ -49,43 +48,14 @@ struct vimc_device {
 	/* Internal v4l2 parent device*/
 	struct v4l2_device v4l2_dev;
 
-	/* Internal topology */
-	struct vimc_ent_device **ved;
-};
-
-/**
- * enum vimc_ent_node - Select the functionality of a node in the topology
- * @VIMC_ENT_NODE_SENSOR:	A node of type SENSOR simulates a camera sensor
- *				generating internal images in bayer format and
- *				propagating those images through the pipeline
- * @VIMC_ENT_NODE_CAPTURE:	A node of type CAPTURE is a v4l2 video_device
- *				that exposes the received image from the
- *				pipeline to the user space
- * @VIMC_ENT_NODE_INPUT:	A node of type INPUT is a v4l2 video_device that
- *				receives images from the user space and
- *				propagates them through the pipeline
- * @VIMC_ENT_NODE_DEBAYER:	A node type DEBAYER expects to receive a frame
- *				in bayer format converts it to RGB
- * @VIMC_ENT_NODE_SCALER:	A node of type SCALER scales the received image
- *				by a given multiplier
- *
- * This enum is used in the entity configuration struct to allow the definition
- * of a custom topology specifying the role of each node on it.
- */
-enum vimc_ent_node {
-	VIMC_ENT_NODE_SENSOR,
-	VIMC_ENT_NODE_CAPTURE,
-	VIMC_ENT_NODE_INPUT,
-	VIMC_ENT_NODE_DEBAYER,
-	VIMC_ENT_NODE_SCALER,
+	/* Subdevices */
+	struct platform_device **subdevs;
 };
 
 /* Structure which describes individual configuration for each entity */
 struct vimc_ent_config {
 	const char *name;
-	size_t pads_qty;
-	const unsigned long *pads_flag;
-	enum vimc_ent_node node;
+	const char *drv;
 };
 
 /* Structure which describes links between entities */
@@ -112,60 +82,40 @@ struct vimc_pipeline_config {
 static const struct vimc_ent_config ent_config[] = {
 	{
 		.name = "Sensor A",
-		.pads_qty = 1,
-		.pads_flag = (const unsigned long[]){MEDIA_PAD_FL_SOURCE},
-		.node = VIMC_ENT_NODE_SENSOR,
+		.drv = "vimc-sensor",
 	},
 	{
 		.name = "Sensor B",
-		.pads_qty = 1,
-		.pads_flag = (const unsigned long[]){MEDIA_PAD_FL_SOURCE},
-		.node = VIMC_ENT_NODE_SENSOR,
+		.drv = "vimc-sensor",
 	},
 	{
 		.name = "Debayer A",
-		.pads_qty = 2,
-		.pads_flag = (const unsigned long[]){MEDIA_PAD_FL_SINK,
-						     MEDIA_PAD_FL_SOURCE},
-		.node = VIMC_ENT_NODE_DEBAYER,
+		.drv = "vimc-debayer",
 	},
 	{
 		.name = "Debayer B",
-		.pads_qty = 2,
-		.pads_flag = (const unsigned long[]){MEDIA_PAD_FL_SINK,
-						     MEDIA_PAD_FL_SOURCE},
-		.node = VIMC_ENT_NODE_DEBAYER,
+		.drv = "vimc-debayer",
 	},
 	{
 		.name = "Raw Capture 0",
-		.pads_qty = 1,
-		.pads_flag = (const unsigned long[]){MEDIA_PAD_FL_SINK},
-		.node = VIMC_ENT_NODE_CAPTURE,
+		.drv = "vimc-capture",
 	},
 	{
 		.name = "Raw Capture 1",
-		.pads_qty = 1,
-		.pads_flag = (const unsigned long[]){MEDIA_PAD_FL_SINK},
-		.node = VIMC_ENT_NODE_CAPTURE,
+		.drv = "vimc-capture",
 	},
 	{
 		.name = "RGB/YUV Input",
-		.pads_qty = 1,
-		.pads_flag = (const unsigned long[]){MEDIA_PAD_FL_SOURCE},
-		.node = VIMC_ENT_NODE_INPUT,
+		/* TODO: change to vimc-output when output is implemented */
+		.drv = "vimc-sensor",
 	},
 	{
 		.name = "Scaler",
-		.pads_qty = 2,
-		.pads_flag = (const unsigned long[]){MEDIA_PAD_FL_SINK,
-						     MEDIA_PAD_FL_SOURCE},
-		.node = VIMC_ENT_NODE_SCALER,
+		.drv = "vimc-scaler",
 	},
 	{
 		.name = "RGB/YUV Capture",
-		.pads_qty = 1,
-		.pads_flag = (const unsigned long[]){MEDIA_PAD_FL_SINK},
-		.node = VIMC_ENT_NODE_CAPTURE,
+		.drv = "vimc-capture",
 	},
 };
 
@@ -197,111 +147,40 @@ static const struct vimc_pipeline_config pipe_cfg = {
 
 /* -------------------------------------------------------------------------- */
 
-static void vimc_device_unregister(struct vimc_device *vimc)
+static int vimc_create_links(struct vimc_device *vimc)
 {
 	unsigned int i;
-
-	media_device_unregister(&vimc->mdev);
-	/* Cleanup (only initialized) entities */
-	for (i = 0; i < vimc->pipe_cfg->num_ents; i++) {
-		if (vimc->ved[i] && vimc->ved[i]->destroy)
-			vimc->ved[i]->destroy(vimc->ved[i]);
-
-		vimc->ved[i] = NULL;
-	}
-	v4l2_device_unregister(&vimc->v4l2_dev);
-	media_device_cleanup(&vimc->mdev);
-}
-
-/*
- * TODO: remove this function when all the
- * entities specific code are implemented
- */
-static void vimc_raw_destroy(struct vimc_ent_device *ved)
-{
-	media_device_unregister_entity(ved->ent);
-
-	media_entity_cleanup(ved->ent);
-
-	vimc_pads_cleanup(ved->pads);
-
-	kfree(ved->ent);
-
-	kfree(ved);
-}
-
-/*
- * TODO: remove this function when all the
- * entities specific code are implemented
- */
-static struct vimc_ent_device *vimc_raw_create(struct v4l2_device *v4l2_dev,
-					       const char *const name,
-					       u16 num_pads,
-					       const unsigned long *pads_flag)
-{
-	struct vimc_ent_device *ved;
 	int ret;
 
-	/* Allocate the main ved struct */
-	ved = kzalloc(sizeof(*ved), GFP_KERNEL);
-	if (!ved)
-		return ERR_PTR(-ENOMEM);
-
-	/* Allocate the media entity */
-	ved->ent = kzalloc(sizeof(*ved->ent), GFP_KERNEL);
-	if (!ved->ent) {
-		ret = -ENOMEM;
-		goto err_free_ved;
-	}
-
-	/* Allocate the pads */
-	ved->pads = vimc_pads_init(num_pads, pads_flag);
-	if (IS_ERR(ved->pads)) {
-		ret = PTR_ERR(ved->pads);
-		goto err_free_ent;
+	/* Initialize the links between entities */
+	for (i = 0; i < vimc->pipe_cfg->num_links; i++) {
+		const struct vimc_ent_link *link = &vimc->pipe_cfg->links[i];
+		/*
+		 * TODO: Check another way of retrieving ved struct without
+		 * relying on platform_get_drvdata
+		 */
+		struct vimc_ent_device *ved_src =
+			platform_get_drvdata(vimc->subdevs[link->src_ent]);
+		struct vimc_ent_device *ved_sink =
+			platform_get_drvdata(vimc->subdevs[link->sink_ent]);
+
+		ret = media_create_pad_link(ved_src->ent, link->src_pad,
+					    ved_sink->ent, link->sink_pad,
+					    link->flags);
+		if (ret)
+			return ret;
 	}
 
-	/* Initialize the media entity */
-	ved->ent->name = name;
-	ved->ent->function = MEDIA_ENT_F_V4L2_SUBDEV_UNKNOWN;
-	ret = media_entity_pads_init(ved->ent, num_pads, ved->pads);
-	if (ret)
-		goto err_cleanup_pads;
-
-	/* Register the media entity */
-	ret = media_device_register_entity(v4l2_dev->mdev, ved->ent);
-	if (ret)
-		goto err_cleanup_entity;
-
-	/* Fill out the destroy function and return */
-	ved->destroy = vimc_raw_destroy;
-	return ved;
-
-err_cleanup_entity:
-	media_entity_cleanup(ved->ent);
-err_cleanup_pads:
-	vimc_pads_cleanup(ved->pads);
-err_free_ent:
-	kfree(ved->ent);
-err_free_ved:
-	kfree(ved);
-
-	return ERR_PTR(ret);
+	return 0;
 }
 
-static int vimc_device_register(struct vimc_device *vimc)
+static int vimc_comp_bind(struct device *master)
 {
-	unsigned int i;
+	struct vimc_device *vimc = container_of(to_platform_device(master),
+						struct vimc_device, pdev);
 	int ret;
 
-	/* Allocate memory for the vimc_ent_devices pointers */
-	vimc->ved = devm_kcalloc(vimc->mdev.dev, vimc->pipe_cfg->num_ents,
-				 sizeof(*vimc->ved), GFP_KERNEL);
-	if (!vimc->ved)
-		return -ENOMEM;
-
-	/* Link the media device within the v4l2_device */
-	vimc->v4l2_dev.mdev = &vimc->mdev;
+	dev_dbg(master, "bind");
 
 	/* Register the v4l2 struct */
 	ret = v4l2_device_register(vimc->mdev.dev, &vimc->v4l2_dev);
@@ -311,66 +190,22 @@ static int vimc_device_register(struct vimc_device *vimc)
 		return ret;
 	}
 
-	/* Initialize entities */
-	for (i = 0; i < vimc->pipe_cfg->num_ents; i++) {
-		struct vimc_ent_device *(*create_func)(struct v4l2_device *,
-						       const char *const,
-						       u16,
-						       const unsigned long *);
-
-		/* Register the specific node */
-		switch (vimc->pipe_cfg->ents[i].node) {
-		case VIMC_ENT_NODE_SENSOR:
-			create_func = vimc_sen_create;
-			break;
-
-		case VIMC_ENT_NODE_CAPTURE:
-			create_func = vimc_cap_create;
-			break;
-
-		/* TODO: Instantiate the specific topology node */
-		case VIMC_ENT_NODE_INPUT:
-		case VIMC_ENT_NODE_DEBAYER:
-		case VIMC_ENT_NODE_SCALER:
-		default:
-			/*
-			 * TODO: remove this when all the entities specific
-			 * code are implemented
-			 */
-			create_func = vimc_raw_create;
-			break;
-		}
-
-		vimc->ved[i] = create_func(&vimc->v4l2_dev,
-					   vimc->pipe_cfg->ents[i].name,
-					   vimc->pipe_cfg->ents[i].pads_qty,
-					   vimc->pipe_cfg->ents[i].pads_flag);
-		if (IS_ERR(vimc->ved[i])) {
-			ret = PTR_ERR(vimc->ved[i]);
-			vimc->ved[i] = NULL;
-			goto err;
-		}
-	}
-
-	/* Initialize the links between entities */
-	for (i = 0; i < vimc->pipe_cfg->num_links; i++) {
-		const struct vimc_ent_link *link = &vimc->pipe_cfg->links[i];
+	/* Bind subdevices */
+	ret = component_bind_all(master, &vimc->v4l2_dev);
+	if (ret)
+		goto err_v4l2_unregister;
 
-		ret = media_create_pad_link(vimc->ved[link->src_ent]->ent,
-					    link->src_pad,
-					    vimc->ved[link->sink_ent]->ent,
-					    link->sink_pad,
-					    link->flags);
-		if (ret)
-			goto err;
-	}
+	/* Initialize links */
+	ret = vimc_create_links(vimc);
+	if (ret)
+		goto err_comp_unbind_all;
 
 	/* Register the media device */
 	ret = media_device_register(&vimc->mdev);
 	if (ret) {
 		dev_err(vimc->mdev.dev,
 			"media device register failed (err=%d)\n", ret);
-		return ret;
+		goto err_comp_unbind_all;
 	}
 
 	/* Expose all subdev's nodes*/
@@ -379,32 +214,107 @@ static int vimc_device_register(struct vimc_device *vimc)
 		dev_err(vimc->mdev.dev,
 			"vimc subdev nodes registration failed (err=%d)\n",
 			ret);
-		goto err;
+		goto err_mdev_unregister;
 	}
 
 	return 0;
 
-err:
-	/* Destroy the so far created topology */
-	vimc_device_unregister(vimc);
+err_mdev_unregister:
+	media_device_unregister(&vimc->mdev);
+err_comp_unbind_all:
+	component_unbind_all(master, NULL);
+err_v4l2_unregister:
+	v4l2_device_unregister(&vimc->v4l2_dev);
 
 	return ret;
 }
 
+static void vimc_comp_unbind(struct device *master)
+{
+	struct vimc_device *vimc = container_of(to_platform_device(master),
+						struct vimc_device, pdev);
+
+	dev_dbg(master, "unbind");
+
+	media_device_unregister(&vimc->mdev);
+	component_unbind_all(master, NULL);
+	v4l2_device_unregister(&vimc->v4l2_dev);
+}
+
+static int vimc_comp_compare(struct device *comp, void *data)
+{
+	const struct platform_device *pdev = to_platform_device(comp);
+	const char *name = data;
+
+	return !strcmp(pdev->dev.platform_data, name);
+}
+
+static struct component_match *vimc_add_subdevs(struct vimc_device *vimc)
+{
+	struct component_match *match = NULL;
+	unsigned int i;
+
+	for (i = 0; i < vimc->pipe_cfg->num_ents; i++) {
+		dev_dbg(&vimc->pdev.dev, "new pdev for %s\n",
+			vimc->pipe_cfg->ents[i].drv);
+
+		/*
+		 * TODO: check if using platform_data is indeed the best way to
+		 * pass the name to the driver or if we should add the drv name
+		 * in the platform_device_id table
+		 */
+		vimc->subdevs[i] = platform_device_register_data(&vimc->pdev.dev,
+				vimc->pipe_cfg->ents[i].drv,
+				PLATFORM_DEVID_AUTO,
+				vimc->pipe_cfg->ents[i].name,
+				strlen(vimc->pipe_cfg->ents[i].name) + 1);
+		if (!vimc->subdevs[i]) {
+			while (--i >= 0)
+				platform_device_unregister(vimc->subdevs[i]);
+
+			return ERR_PTR(-ENOMEM);
+		}
+
+		component_match_add(&vimc->pdev.dev, &match, vimc_comp_compare,
+				    (void *)vimc->pipe_cfg->ents[i].name);
+	}
+
+	return match;
+}
+
+static void vimc_rm_subdevs(struct vimc_device *vimc)
+{
+	unsigned int i;
+
+	for (i = 0; i < vimc->pipe_cfg->num_ents; i++)
+		platform_device_unregister(vimc->subdevs[i]);
+}
+
+static const struct component_master_ops vimc_comp_ops = {
+	.bind = vimc_comp_bind,
+	.unbind = vimc_comp_unbind,
+};
+
 static int vimc_probe(struct platform_device *pdev)
 {
-	struct vimc_device *vimc;
+	struct vimc_device *vimc = container_of(pdev, struct vimc_device, pdev);
+	struct component_match *match = NULL;
 	int ret;
 
-	/* Prepare the vimc topology structure */
+	dev_dbg(&pdev->dev, "probe");
 
-	/* Allocate memory for the vimc structure */
-	vimc = kzalloc(sizeof(*vimc), GFP_KERNEL);
-	if (!vimc)
+	/* Create platform_device for each entity in the topology*/
+	vimc->subdevs = devm_kcalloc(&vimc->pdev.dev, vimc->pipe_cfg->num_ents,
+				     sizeof(*vimc->subdevs), GFP_KERNEL);
+	if (!vimc->subdevs)
 		return -ENOMEM;
 
-	/* Set the pipeline configuration struct */
-	vimc->pipe_cfg = &pipe_cfg;
+	match = vimc_add_subdevs(vimc);
+	if (IS_ERR(match))
+		return PTR_ERR(match);
+
+	/* Link the media device within the v4l2_device */
+	vimc->v4l2_dev.mdev = &vimc->mdev;
 
 	/* Initialize media device */
 	strlcpy(vimc->mdev.model, VIMC_MDEV_MODEL_NAME,
@@ -412,28 +322,27 @@ static int vimc_probe(struct platform_device *pdev)
 	vimc->mdev.dev = &pdev->dev;
 	media_device_init(&vimc->mdev);
 
-	/* Create vimc topology */
-	ret = vimc_device_register(vimc);
+	/* Add self to the component system */
+	ret = component_master_add_with_match(&pdev->dev, &vimc_comp_ops,
+					      match);
 	if (ret) {
-		dev_err(vimc->mdev.dev,
-			"vimc device registration failed (err=%d)\n", ret);
+		media_device_cleanup(&vimc->mdev);
+		vimc_rm_subdevs(vimc);
 		kfree(vimc);
 		return ret;
 	}
 
-	/* Link the topology object with the platform device object */
-	platform_set_drvdata(pdev, vimc);
-
 	return 0;
 }
 
 static int vimc_remove(struct platform_device *pdev)
 {
-	struct vimc_device *vimc = platform_get_drvdata(pdev);
+	struct vimc_device *vimc = container_of(pdev, struct vimc_device, pdev);
 
-	/* Destroy all the topology */
-	vimc_device_unregister(vimc);
-	kfree(vimc);
+	dev_dbg(&pdev->dev, "remove");
+
+	component_master_del(&pdev->dev, &vimc_comp_ops);
+	vimc_rm_subdevs(vimc);
 
 	return 0;
 }
@@ -442,9 +351,12 @@ static void vimc_dev_release(struct device *dev)
 {
 }
 
-static struct platform_device vimc_pdev = {
-	.name		= VIMC_PDEV_NAME,
-	.dev.release	= vimc_dev_release,
+static struct vimc_device vimc_dev = {
+	.pipe_cfg = &pipe_cfg,
+	.pdev = {
+		.name = VIMC_PDEV_NAME,
+		.dev.release = vimc_dev_release,
+	}
 };
 
 static struct platform_driver vimc_pdrv = {
@@ -459,29 +371,29 @@ static int __init vimc_init(void)
 {
 	int ret;
 
-	ret = platform_device_register(&vimc_pdev);
+	ret = platform_device_register(&vimc_dev.pdev);
 	if (ret) {
-		dev_err(&vimc_pdev.dev,
+		dev_err(&vimc_dev.pdev.dev,
 			"platform device registration failed (err=%d)\n", ret);
 		return ret;
 	}
 
 	ret = platform_driver_register(&vimc_pdrv);
 	if (ret) {
-		dev_err(&vimc_pdev.dev,
+		dev_err(&vimc_dev.pdev.dev,
 			"platform driver registration failed (err=%d)\n", ret);
-
-		platform_device_unregister(&vimc_pdev);
+		platform_driver_unregister(&vimc_pdrv);
+		return ret;
 	}
 
-	return ret;
+	return 0;
 }
 
 static void __exit vimc_exit(void)
 {
 	platform_driver_unregister(&vimc_pdrv);
 
-	platform_device_unregister(&vimc_pdev);
+	platform_device_unregister(&vimc_dev.pdev);
 }
 
 module_init(vimc_init);
diff --git a/drivers/media/platform/vimc/vimc-sensor.c b/drivers/media/platform/vimc/vimc-sensor.c
index df89ee7..7f16978 100644
--- a/drivers/media/platform/vimc/vimc-sensor.c
+++ b/drivers/media/platform/vimc/vimc-sensor.c
@@ -15,15 +15,19 @@
  *
  */
 
-#include <linux/module.h>
+#include <linux/component.h>
 #include <linux/freezer.h>
 #include <linux/kthread.h>
+#include <linux/module.h>
+#include <linux/platform_device.h>
 #include <linux/v4l2-mediabus.h>
 #include <linux/vmalloc.h>
 #include <media/v4l2-subdev.h>
 #include <media/v4l2-tpg.h>
 
-#include "vimc-sensor.h"
+#include "vimc-common.h"
+
+#define VIMC_SEN_DRV_NAME "vimc-sensor"
 
 static bool vsen_tpg;
 module_param(vsen_tpg, bool, 0000);
@@ -35,6 +39,7 @@ MODULE_PARM_DESC(vsen_tpg, " generate image from sensor node\n"
 struct vimc_sen_device {
 	struct vimc_ent_device ved;
 	struct v4l2_subdev sd;
+	struct device *dev;
 	struct tpg_data tpg;
 	struct task_struct *kthread_sen;
 	u8 *frame;
@@ -189,7 +194,7 @@ static int vimc_sen_set_fmt(struct v4l2_subdev *sd,
 	/* Set the new format */
 	vimc_sen_adjust_fmt(&fmt->format);
 
-	dev_dbg(vsen->sd.v4l2_dev->mdev->dev, "%s: format update: "
+	dev_dbg(vsen->dev, "%s: format update: "
 		"old:%dx%d (0x%x, %d, %d, %d, %d) "
 		"new:%dx%d (0x%x, %d, %d, %d, %d)\n", vsen->sd.name,
 		/* old */
@@ -282,8 +287,8 @@ static int vimc_sen_s_stream(struct v4l2_subdev *sd, int enable)
 		vsen->kthread_sen = kthread_run(vimc_sen_tpg_thread, vsen,
 					"%s-sen", vsen->sd.v4l2_dev->name);
 		if (IS_ERR(vsen->kthread_sen)) {
-			dev_err(vsen->sd.v4l2_dev->dev,
-				"%s: kernel_thread() failed\n",	vsen->sd.name);
+			dev_err(vsen->dev, "%s: kernel_thread() failed\n",
+				vsen->sd.name);
 			vfree(vsen->frame);
 			vsen->frame = NULL;
 			return PTR_ERR(vsen->kthread_sen);
@@ -315,8 +320,10 @@ static const struct v4l2_subdev_ops vimc_sen_ops = {
 	.video = &vimc_sen_video_ops,
 };
 
-static void vimc_sen_destroy(struct vimc_ent_device *ved)
+static void vimc_sen_comp_unbind(struct device *comp, struct device *master,
+				 void *master_data)
 {
+	struct vimc_ent_device *ved = dev_get_drvdata(comp);
 	struct vimc_sen_device *vsen =
 				container_of(ved, struct vimc_sen_device, ved);
 
@@ -326,36 +333,30 @@ static void vimc_sen_destroy(struct vimc_ent_device *ved)
 	kfree(vsen);
 }
 
-struct vimc_ent_device *vimc_sen_create(struct v4l2_device *v4l2_dev,
-					const char *const name,
-					u16 num_pads,
-					const unsigned long *pads_flag)
+static int vimc_sen_comp_bind(struct device *comp, struct device *master,
+			      void *master_data)
 {
+	struct v4l2_device *v4l2_dev = master_data;
+	char *name = comp->platform_data;
 	struct vimc_sen_device *vsen;
-	unsigned int i;
 	int ret;
 
-	/* NOTE: a sensor node may be created with more then one pad */
-	if (!name || !num_pads || !pads_flag)
-		return ERR_PTR(-EINVAL);
-
-	/* check if all pads are sources */
-	for (i = 0; i < num_pads; i++)
-		if (!(pads_flag[i] & MEDIA_PAD_FL_SOURCE))
-			return ERR_PTR(-EINVAL);
-
 	/* Allocate the vsen struct */
 	vsen = kzalloc(sizeof(*vsen), GFP_KERNEL);
 	if (!vsen)
-		return ERR_PTR(-ENOMEM);
+		return -ENOMEM;
 
 	/* Initialize ved and sd */
 	ret = vimc_ent_sd_register(&vsen->ved, &vsen->sd, v4l2_dev, name,
-				   MEDIA_ENT_F_CAM_SENSOR, num_pads, pads_flag,
-				   &vimc_sen_ops, vimc_sen_destroy);
+				   MEDIA_ENT_F_ATV_DECODER, 1,
+				   (const unsigned long[1]) {MEDIA_PAD_FL_SOURCE},
+				   &vimc_sen_ops);
 	if (ret)
 		goto err_free_vsen;
 
+	dev_set_drvdata(comp, &vsen->ved);
+	vsen->dev = comp;
+
 	/* Initialize the frame format */
 	vsen->mbus_format = fmt_default;
 
@@ -368,12 +369,52 @@ struct vimc_ent_device *vimc_sen_create(struct v4l2_device *v4l2_dev,
 			goto err_unregister_ent_sd;
 	}
 
-	return &vsen->ved;
+	return 0;
 
 err_unregister_ent_sd:
 	vimc_ent_sd_unregister(&vsen->ved,  &vsen->sd);
 err_free_vsen:
 	kfree(vsen);
 
-	return ERR_PTR(ret);
+	return ret;
+}
+
+static const struct component_ops vimc_sen_comp_ops = {
+	.bind = vimc_sen_comp_bind,
+	.unbind = vimc_sen_comp_unbind,
+};
+
+static int vimc_sen_probe(struct platform_device *pdev)
+{
+	return component_add(&pdev->dev, &vimc_sen_comp_ops);
 }
+
+static int vimc_sen_remove(struct platform_device *pdev)
+{
+	component_del(&pdev->dev, &vimc_sen_comp_ops);
+
+	return 0;
+}
+
+static struct platform_driver vimc_sen_pdrv = {
+	.probe		= vimc_sen_probe,
+	.remove		= vimc_sen_remove,
+	.driver		= {
+		.name	= VIMC_SEN_DRV_NAME,
+	},
+};
+
+static const struct platform_device_id vimc_sen_driver_ids[] = {
+	{
+		.name           = VIMC_SEN_DRV_NAME,
+	},
+	{ }
+};
+
+module_platform_driver(vimc_sen_pdrv);
+
+MODULE_DEVICE_TABLE(platform, vimc_sen_driver_ids);
+
+MODULE_DESCRIPTION("Virtual Media Controller Driver (VIMC) Sensor");
+MODULE_AUTHOR("Helen Mae Koike Fornazier <helen.fornazier@gmail.com>");
+MODULE_LICENSE("GPL");
diff --git a/drivers/media/platform/vimc/vimc-sensor.h b/drivers/media/platform/vimc/vimc-sensor.h
deleted file mode 100644
index 580dcec..0000000
--- a/drivers/media/platform/vimc/vimc-sensor.h
+++ /dev/null
@@ -1,28 +0,0 @@
-/*
- * vimc-sensor.h Virtual Media Controller Driver
- *
- * Copyright (C) 2015-2017 Helen Koike <helen.fornazier@gmail.com>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- */
-
-#ifndef _VIMC_SENSOR_H_
-#define _VIMC_SENSOR_H_
-
-#include "vimc-common.h"
-
-struct vimc_ent_device *vimc_sen_create(struct v4l2_device *v4l2_dev,
-					const char *const name,
-					u16 num_pads,
-					const unsigned long *pads_flag);
-
-#endif
-- 
2.7.4

Re: [RFC PATCH v3 09/11] [media] vimc: Subdevices as modules

[Hans Verkuil]

On 06/03/2017 04:58 AM, Helen Koike wrote:
> Change the core structure for adding subdevices in the topology.
> Instead of calling the specific create function for each subdevice,
> inject a child platform_device with the driver's name.
> Each type of node in the topology (sensor, capture, debayer, scaler)
> will register a platform_driver with the corresponding name through the
> component subsystem.
> Implementing a new subdevice type doesn't require vimc-core to be altered.
> 
> This facilitates future implementation of dynamic entities, where
> hotpluging an entity in the topology is just a matter of
> registering/unregistering a platform_device in the system.
> It also facilitates other implementations of different nodes without
> touching the core code and remove the need of a header file for each
> type of node.
> 
> Signed-off-by: Helen Koike <helen.koike@collabora.com>
> 
> ---
> 
> Changes in v3:
> [media] vimc: Subdevices as modules
> 	- This is a new patch in the series
> 
> Changes in v2: None
> 
> 
> ---
>   drivers/media/platform/vimc/Makefile       |   7 +-
>   drivers/media/platform/vimc/vimc-capture.c |  96 ++++---
>   drivers/media/platform/vimc/vimc-capture.h |  28 --
>   drivers/media/platform/vimc/vimc-common.c  |  37 ++-
>   drivers/media/platform/vimc/vimc-common.h  |  14 +-
>   drivers/media/platform/vimc/vimc-core.c    | 406 +++++++++++------------------
>   drivers/media/platform/vimc/vimc-sensor.c  |  91 +++++--
>   drivers/media/platform/vimc/vimc-sensor.h  |  28 --
>   8 files changed, 320 insertions(+), 387 deletions(-)
>   delete mode 100644 drivers/media/platform/vimc/vimc-capture.h
>   delete mode 100644 drivers/media/platform/vimc/vimc-sensor.h
> 
> diff --git a/drivers/media/platform/vimc/Makefile b/drivers/media/platform/vimc/Makefile
> index 6b6ddf4..0e5d5ce 100644
> --- a/drivers/media/platform/vimc/Makefile
> +++ b/drivers/media/platform/vimc/Makefile
> @@ -1,3 +1,6 @@
> -vimc-objs := vimc-core.o vimc-capture.o vimc-common.o vimc-sensor.o
> +vimc-objs := vimc-core.o
> +vimc_capture-objs := vimc-capture.o
> +vimc_common-objs := vimc-common.o
> +vimc_sensor-objs := vimc-sensor.o
>   
> -obj-$(CONFIG_VIDEO_VIMC) += vimc.o
> +obj-$(CONFIG_VIDEO_VIMC) += vimc.o vimc_capture.o vimc_common.o vimc_sensor.o
> diff --git a/drivers/media/platform/vimc/vimc-capture.c b/drivers/media/platform/vimc/vimc-capture.c
> index b5da0ea..633d99a 100644
> --- a/drivers/media/platform/vimc/vimc-capture.c
> +++ b/drivers/media/platform/vimc/vimc-capture.c
> @@ -15,18 +15,24 @@
>    *
>    */
>   
> +#include <linux/component.h>
>   #include <linux/freezer.h>
>   #include <linux/kthread.h>
> +#include <linux/module.h>
> +#include <linux/platform_device.h>
>   #include <media/v4l2-ioctl.h>
>   #include <media/v4l2-tpg.h>
>   #include <media/videobuf2-core.h>
>   #include <media/videobuf2-vmalloc.h>
>   
> -#include "vimc-capture.h"
> +#include "vimc-common.h"
> +
> +#define VIMC_CAP_DRV_NAME "vimc-capture"
>   
>   struct vimc_cap_device {
>   	struct vimc_ent_device ved;
>   	struct video_device vdev;
> +	struct device *dev;
>   	struct v4l2_pix_format format;
>   	struct vb2_queue queue;
>   	struct list_head buf_list;
> @@ -150,7 +156,7 @@ static int vimc_cap_s_fmt_vid_cap(struct file *file, void *priv,
>   
>   	vimc_cap_try_fmt_vid_cap(file, priv, f);
>   
> -	dev_dbg(vcap->vdev.v4l2_dev->dev, "%s: format update: "
> +	dev_dbg(vcap->dev, "%s: format update: "
>   		"old:%dx%d (0x%x, %d, %d, %d, %d) "
>   		"new:%dx%d (0x%x, %d, %d, %d, %d)\n", vcap->vdev.name,
>   		/* old */
> @@ -365,8 +371,7 @@ static int vimc_cap_start_streaming(struct vb2_queue *vq, unsigned int count)
>   		vcap->kthread_cap = kthread_run(vimc_cap_tpg_thread, vcap,
>   					"%s-cap", vcap->vdev.v4l2_dev->name);
>   		if (IS_ERR(vcap->kthread_cap)) {
> -			dev_err(vcap->vdev.v4l2_dev->dev,
> -				"%s: kernel_thread() failed\n",
> +			dev_err(vcap->dev, "%s: kernel_thread() failed\n",
>   				vcap->vdev.name);
>   			ret = PTR_ERR(vcap->kthread_cap);
>   			goto err_tpg_free;
> @@ -443,8 +448,7 @@ static int vimc_cap_buffer_prepare(struct vb2_buffer *vb)
>   	unsigned long size = vcap->format.sizeimage;
>   
>   	if (vb2_plane_size(vb, 0) < size) {
> -		dev_err(vcap->vdev.v4l2_dev->dev,
> -			"%s: buffer too small (%lu < %lu)\n",
> +		dev_err(vcap->dev, "%s: buffer too small (%lu < %lu)\n",
>   			vcap->vdev.name, vb2_plane_size(vb, 0), size);
>   		return -EINVAL;
>   	}
> @@ -469,8 +473,10 @@ static const struct media_entity_operations vimc_cap_mops = {
>   	.link_validate		= vimc_link_validate,
>   };
>   
> -static void vimc_cap_destroy(struct vimc_ent_device *ved)
> +static void vimc_cap_comp_unbind(struct device *comp, struct device *master,
> +				 void *master_data)
>   {
> +	struct vimc_ent_device *ved = dev_get_drvdata(comp);
>   	struct vimc_cap_device *vcap = container_of(ved, struct vimc_cap_device,
>   						    ved);
>   
> @@ -481,32 +487,25 @@ static void vimc_cap_destroy(struct vimc_ent_device *ved)
>   	kfree(vcap);
>   }
>   
> -struct vimc_ent_device *vimc_cap_create(struct v4l2_device *v4l2_dev,
> -					const char *const name,
> -					u16 num_pads,
> -					const unsigned long *pads_flag)
> +static int vimc_cap_comp_bind(struct device *comp, struct device *master,
> +			      void *master_data)
>   {
> +	struct v4l2_device *v4l2_dev = master_data;
> +	char *name = comp->platform_data;
>   	const struct vimc_pix_map *vpix;
>   	struct vimc_cap_device *vcap;
>   	struct video_device *vdev;
>   	struct vb2_queue *q;
>   	int ret;
>   
> -	/*
> -	 * Check entity configuration params
> -	 * NOTE: we only support a single sink pad
> -	 */
> -	if (!name || num_pads != 1 || !pads_flag ||
> -	    !(pads_flag[0] & MEDIA_PAD_FL_SINK))
> -		return ERR_PTR(-EINVAL);
> -
>   	/* Allocate the vimc_cap_device struct */
>   	vcap = kzalloc(sizeof(*vcap), GFP_KERNEL);
>   	if (!vcap)
> -		return ERR_PTR(-ENOMEM);
> +		return -ENOMEM;
>   
>   	/* Allocate the pads */
> -	vcap->ved.pads = vimc_pads_init(num_pads, pads_flag);
> +	vcap->ved.pads =
> +		vimc_pads_init(1, (const unsigned long[1]) {MEDIA_PAD_FL_SINK});
>   	if (IS_ERR(vcap->ved.pads)) {
>   		ret = PTR_ERR(vcap->ved.pads);
>   		goto err_free_vcap;
> @@ -516,7 +515,7 @@ struct vimc_ent_device *vimc_cap_create(struct v4l2_device *v4l2_dev,
>   	vcap->vdev.entity.name = name;
>   	vcap->vdev.entity.function = MEDIA_ENT_F_IO_V4L;
>   	ret = media_entity_pads_init(&vcap->vdev.entity,
> -				     num_pads, vcap->ved.pads);
> +				     1, vcap->ved.pads);
>   	if (ret)
>   		goto err_clean_pads;
>   
> @@ -537,8 +536,7 @@ struct vimc_ent_device *vimc_cap_create(struct v4l2_device *v4l2_dev,
>   
>   	ret = vb2_queue_init(q);
>   	if (ret) {
> -		dev_err(vcap->vdev.v4l2_dev->dev,
> -			"%s: vb2 queue init failed (err=%d)\n",
> +		dev_err(comp, "%s: vb2 queue init failed (err=%d)\n",
>   			vcap->vdev.name, ret);
>   		goto err_clean_m_ent;
>   	}
> @@ -555,10 +553,11 @@ struct vimc_ent_device *vimc_cap_create(struct v4l2_device *v4l2_dev,
>   				 vcap->format.height;
>   
>   	/* Fill the vimc_ent_device struct */
> -	vcap->ved.destroy = vimc_cap_destroy;
>   	vcap->ved.ent = &vcap->vdev.entity;
>   	vcap->ved.process_frame = vimc_cap_process_frame;
>   	vcap->ved.vdev_get_format = vimc_cap_get_format;
> +	dev_set_drvdata(comp, &vcap->ved);
> +	vcap->dev = comp;
>   
>   	/* Initialize the video_device struct */
>   	vdev = &vcap->vdev;
> @@ -577,13 +576,12 @@ struct vimc_ent_device *vimc_cap_create(struct v4l2_device *v4l2_dev,
>   	/* Register the video_device with the v4l2 and the media framework */
>   	ret = video_register_device(vdev, VFL_TYPE_GRABBER, -1);
>   	if (ret) {
> -		dev_err(vcap->vdev.v4l2_dev->dev,
> -			"%s: video register failed (err=%d)\n",
> +		dev_err(comp, "%s: video register failed (err=%d)\n",
>   			vcap->vdev.name, ret);
>   		goto err_release_queue;
>   	}
>   
> -	return &vcap->ved;
> +	return 0;
>   
>   err_release_queue:
>   	vb2_queue_release(q);
> @@ -594,5 +592,45 @@ struct vimc_ent_device *vimc_cap_create(struct v4l2_device *v4l2_dev,
>   err_free_vcap:
>   	kfree(vcap);
>   
> -	return ERR_PTR(ret);
> +	return ret;
> +}
> +
> +static const struct component_ops vimc_cap_comp_ops = {
> +	.bind = vimc_cap_comp_bind,
> +	.unbind = vimc_cap_comp_unbind,
> +};
> +
> +static int vimc_cap_probe(struct platform_device *pdev)
> +{
> +	return component_add(&pdev->dev, &vimc_cap_comp_ops);
>   }
> +
> +static int vimc_cap_remove(struct platform_device *pdev)
> +{
> +	component_del(&pdev->dev, &vimc_cap_comp_ops);
> +
> +	return 0;
> +}
> +
> +static struct platform_driver vimc_cap_pdrv = {
> +	.probe		= vimc_cap_probe,
> +	.remove		= vimc_cap_remove,
> +	.driver		= {
> +		.name	= VIMC_CAP_DRV_NAME,
> +	},
> +};
> +
> +static const struct platform_device_id vimc_cap_driver_ids[] = {
> +	{
> +		.name           = VIMC_CAP_DRV_NAME,
> +	},
> +	{ }
> +};
> +
> +module_platform_driver(vimc_cap_pdrv);
> +
> +MODULE_DEVICE_TABLE(platform, vimc_cap_driver_ids);
> +
> +MODULE_DESCRIPTION("Virtual Media Controller Driver (VIMC) Capture");
> +MODULE_AUTHOR("Helen Mae Koike Fornazier <helen.fornazier@gmail.com>");
> +MODULE_LICENSE("GPL");
> diff --git a/drivers/media/platform/vimc/vimc-capture.h b/drivers/media/platform/vimc/vimc-capture.h
> deleted file mode 100644
> index 7e5c707..0000000
> --- a/drivers/media/platform/vimc/vimc-capture.h
> +++ /dev/null
> @@ -1,28 +0,0 @@
> -/*
> - * vimc-capture.h Virtual Media Controller Driver
> - *
> - * Copyright (C) 2015-2017 Helen Koike <helen.fornazier@gmail.com>
> - *
> - * This program is free software; you can redistribute it and/or modify
> - * it under the terms of the GNU General Public License as published by
> - * the Free Software Foundation; either version 2 of the License, or
> - * (at your option) any later version.
> - *
> - * This program is distributed in the hope that it will be useful,
> - * but WITHOUT ANY WARRANTY; without even the implied warranty of
> - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
> - * GNU General Public License for more details.
> - *
> - */
> -
> -#ifndef _VIMC_CAPTURE_H_
> -#define _VIMC_CAPTURE_H_
> -
> -#include "vimc-common.h"
> -
> -struct vimc_ent_device *vimc_cap_create(struct v4l2_device *v4l2_dev,
> -					const char *const name,
> -					u16 num_pads,
> -					const unsigned long *pads_flag);
> -
> -#endif
> diff --git a/drivers/media/platform/vimc/vimc-common.c b/drivers/media/platform/vimc/vimc-common.c
> index ff59e09..d5ed387 100644
> --- a/drivers/media/platform/vimc/vimc-common.c
> +++ b/drivers/media/platform/vimc/vimc-common.c
> @@ -15,6 +15,9 @@
>    *
>    */
>   
> +#include <linux/init.h>
> +#include <linux/module.h>
> +
>   #include "vimc-common.h"
>   
>   static const struct vimc_pix_map vimc_pix_map_list[] = {
> @@ -151,6 +154,7 @@ const struct vimc_pix_map *vimc_pix_map_by_index(unsigned int i)
>   
>   	return &vimc_pix_map_list[i];
>   }
> +EXPORT_SYMBOL(vimc_pix_map_by_index);

I would recommend using EXPORT_SYMBOL_GPL.

>   
>   const struct vimc_pix_map *vimc_pix_map_by_code(u32 code)
>   {
> @@ -162,6 +166,7 @@ const struct vimc_pix_map *vimc_pix_map_by_code(u32 code)
>   	}
>   	return NULL;
>   }
> +EXPORT_SYMBOL(vimc_pix_map_by_code);
>   
>   const struct vimc_pix_map *vimc_pix_map_by_pixelformat(u32 pixelformat)
>   {
> @@ -173,6 +178,7 @@ const struct vimc_pix_map *vimc_pix_map_by_pixelformat(u32 pixelformat)
>   	}
>   	return NULL;
>   }
> +EXPORT_SYMBOL(vimc_pix_map_by_pixelformat);
>   
>   int vimc_propagate_frame(struct media_pad *src, const void *frame)
>   {
> @@ -207,6 +213,7 @@ int vimc_propagate_frame(struct media_pad *src, const void *frame)
>   
>   	return 0;
>   }
> +EXPORT_SYMBOL(vimc_propagate_frame);
>   
>   /* Helper function to allocate and initialize pads */
>   struct media_pad *vimc_pads_init(u16 num_pads, const unsigned long *pads_flag)
> @@ -227,6 +234,7 @@ struct media_pad *vimc_pads_init(u16 num_pads, const unsigned long *pads_flag)
>   
>   	return pads;
>   }
> +EXPORT_SYMBOL(vimc_pads_init);
>   
>   int vimc_pipeline_s_stream(struct media_entity *ent, int enable)
>   {
> @@ -242,14 +250,8 @@ int vimc_pipeline_s_stream(struct media_entity *ent, int enable)
>   		/* Start the stream in the subdevice direct connected */
>   		pad = media_entity_remote_pad(&ent->pads[i]);
>   
> -		/*
> -		 * if this is a raw node from vimc-core, then there is
> -		 * nothing to activate
> -		 * TODO: remove this when there are no more raw nodes in the
> -		 * core and return error instead
> -		 */
> -		if (pad->entity->obj_type == MEDIA_ENTITY_TYPE_BASE)
> -			continue;
> +		if (!is_media_entity_v4l2_subdev(pad->entity))
> +			return -EINVAL;
>   
>   		sd = media_entity_to_v4l2_subdev(pad->entity);
>   		ret = v4l2_subdev_call(sd, video, s_stream, enable);
> @@ -259,6 +261,7 @@ int vimc_pipeline_s_stream(struct media_entity *ent, int enable)
>   
>   	return 0;
>   }
> +EXPORT_SYMBOL(vimc_pipeline_s_stream);
>   
>   static void vimc_fmt_pix_to_mbus(struct v4l2_mbus_framefmt *mfmt,
>   				 struct v4l2_pix_format *pfmt)
> @@ -315,14 +318,6 @@ int vimc_link_validate(struct media_link *link)
>   	struct v4l2_subdev_format source_fmt, sink_fmt;
>   	int ret;
>   
> -	/*
> -	 * if it is a raw node from vimc-core, ignore the link for now
> -	 * TODO: remove this when there are no more raw nodes in the
> -	 * core and return error instead
> -	 */
> -	if (link->source->entity->obj_type == MEDIA_ENTITY_TYPE_BASE)
> -		return 0;
> -
>   	ret = vimc_get_mbus_format(link->source, &source_fmt);
>   	if (ret)
>   		return ret;
> @@ -393,8 +388,7 @@ int vimc_ent_sd_register(struct vimc_ent_device *ved,
>   			 u32 function,
>   			 u16 num_pads,
>   			 const unsigned long *pads_flag,
> -			 const struct v4l2_subdev_ops *sd_ops,
> -			 void (*sd_destroy)(struct vimc_ent_device *))
> +			 const struct v4l2_subdev_ops *sd_ops)
>   {
>   	int ret;
>   
> @@ -404,7 +398,6 @@ int vimc_ent_sd_register(struct vimc_ent_device *ved,
>   		return PTR_ERR(ved->pads);
>   
>   	/* Fill the vimc_ent_device struct */
> -	ved->destroy = sd_destroy;
>   	ved->ent = &sd->entity;
>   
>   	/* Initialize the subdev */
> @@ -440,6 +433,7 @@ int vimc_ent_sd_register(struct vimc_ent_device *ved,
>   	vimc_pads_cleanup(ved->pads);
>   	return ret;
>   }
> +EXPORT_SYMBOL(vimc_ent_sd_register);
>   
>   void vimc_ent_sd_unregister(struct vimc_ent_device *ved, struct v4l2_subdev *sd)
>   {
> @@ -447,3 +441,8 @@ void vimc_ent_sd_unregister(struct vimc_ent_device *ved, struct v4l2_subdev *sd)
>   	media_entity_cleanup(ved->ent);
>   	vimc_pads_cleanup(ved->pads);
>   }
> +EXPORT_SYMBOL(vimc_ent_sd_unregister);
> +
> +MODULE_DESCRIPTION("Virtual Media Controller Driver (VIMC) Common");
> +MODULE_AUTHOR("Helen Koike <helen.fornazier@gmail.com>");
> +MODULE_LICENSE("GPL");
> diff --git a/drivers/media/platform/vimc/vimc-common.h b/drivers/media/platform/vimc/vimc-common.h
> index 5b2691e..27c9b8c 100644
> --- a/drivers/media/platform/vimc/vimc-common.h
> +++ b/drivers/media/platform/vimc/vimc-common.h
> @@ -1,5 +1,5 @@
>   /*
> - * vimc-ccommon.h Virtual Media Controller Driver
> + * vimc-common.h Virtual Media Controller Driver
>    *
>    * Copyright (C) 2015-2017 Helen Koike <helen.fornazier@gmail.com>
>    *
> @@ -50,7 +50,6 @@ struct vimc_pix_map {
>    *
>    * @ent:		the pointer to struct media_entity for the node
>    * @pads:		the list of pads of the node
> - * @destroy:		callback to destroy the node
>    * @process_frame:	callback send a frame to that node
>    * @vdev_get_format:	callback that returns the current format a pad, used
>    *			only when is_media_entity_v4l2_video_device(ent) returns
> @@ -67,7 +66,6 @@ struct vimc_pix_map {
>   struct vimc_ent_device {
>   	struct media_entity *ent;
>   	struct media_pad *pads;
> -	void (*destroy)(struct vimc_ent_device *);
>   	void (*process_frame)(struct vimc_ent_device *ved,
>   			      struct media_pad *sink, const void *frame);
>   	void (*vdev_get_format)(struct vimc_ent_device *ved,
> @@ -152,7 +150,6 @@ const struct vimc_pix_map *vimc_pix_map_by_pixelformat(u32 pixelformat);
>    * @num_pads:	number of pads to initialize
>    * @pads_flag:	flags to use in each pad
>    * @sd_ops:	pointer to &struct v4l2_subdev_ops.
> - * @sd_destroy:	callback to destroy the node
>    *
>    * Helper function initialize and register the struct vimc_ent_device and struct
>    * v4l2_subdev which represents a subdev node in the topology
> @@ -164,14 +161,13 @@ int vimc_ent_sd_register(struct vimc_ent_device *ved,
>   			 u32 function,
>   			 u16 num_pads,
>   			 const unsigned long *pads_flag,
> -			 const struct v4l2_subdev_ops *sd_ops,
> -			 void (*sd_destroy)(struct vimc_ent_device *));
> +			 const struct v4l2_subdev_ops *sd_ops);
>   
>   /**
> - * vimc_ent_sd_register - initialize and register a subdev node
> + * vimc_ent_sd_unregister - cleanup and unregister a subdev node
>    *
> - * @ved:	the vimc_ent_device struct to be initialize
> - * @sd:		the v4l2_subdev struct to be initialize and registered
> + * @ved:	the vimc_ent_device struct to be cleaned up
> + * @sd:		the v4l2_subdev struct to be unregistered
>    *
>    * Helper function cleanup and unregister the struct vimc_ent_device and struct
>    * v4l2_subdev which represents a subdev node in the topology
> diff --git a/drivers/media/platform/vimc/vimc-core.c b/drivers/media/platform/vimc/vimc-core.c
> index afc79e2..6148c3e 100644
> --- a/drivers/media/platform/vimc/vimc-core.c
> +++ b/drivers/media/platform/vimc/vimc-core.c
> @@ -15,15 +15,14 @@
>    *
>    */
>   
> +#include <linux/component.h>
>   #include <linux/init.h>
>   #include <linux/module.h>
>   #include <linux/platform_device.h>
>   #include <media/media-device.h>
>   #include <media/v4l2-device.h>
>   
> -#include "vimc-capture.h"
>   #include "vimc-common.h"
> -#include "vimc-sensor.h"
>   
>   #define VIMC_PDEV_NAME "vimc"
>   #define VIMC_MDEV_MODEL_NAME "VIMC MDEV"
> @@ -37,10 +36,10 @@
>   }
>   
>   struct vimc_device {
> -	/*
> -	 * The pipeline configuration
> -	 * (filled before calling vimc_device_register)
> -	 */
> +	/* The platform device */
> +	struct platform_device pdev;
> +
> +	/* The pipeline configuration */
>   	const struct vimc_pipeline_config *pipe_cfg;
>   
>   	/* The Associated media_device parent */
> @@ -49,43 +48,14 @@ struct vimc_device {
>   	/* Internal v4l2 parent device*/
>   	struct v4l2_device v4l2_dev;
>   
> -	/* Internal topology */
> -	struct vimc_ent_device **ved;
> -};
> -
> -/**
> - * enum vimc_ent_node - Select the functionality of a node in the topology
> - * @VIMC_ENT_NODE_SENSOR:	A node of type SENSOR simulates a camera sensor
> - *				generating internal images in bayer format and
> - *				propagating those images through the pipeline
> - * @VIMC_ENT_NODE_CAPTURE:	A node of type CAPTURE is a v4l2 video_device
> - *				that exposes the received image from the
> - *				pipeline to the user space
> - * @VIMC_ENT_NODE_INPUT:	A node of type INPUT is a v4l2 video_device that
> - *				receives images from the user space and
> - *				propagates them through the pipeline
> - * @VIMC_ENT_NODE_DEBAYER:	A node type DEBAYER expects to receive a frame
> - *				in bayer format converts it to RGB
> - * @VIMC_ENT_NODE_SCALER:	A node of type SCALER scales the received image
> - *				by a given multiplier
> - *
> - * This enum is used in the entity configuration struct to allow the definition
> - * of a custom topology specifying the role of each node on it.
> - */
> -enum vimc_ent_node {
> -	VIMC_ENT_NODE_SENSOR,
> -	VIMC_ENT_NODE_CAPTURE,
> -	VIMC_ENT_NODE_INPUT,
> -	VIMC_ENT_NODE_DEBAYER,
> -	VIMC_ENT_NODE_SCALER,
> +	/* Subdevices */
> +	struct platform_device **subdevs;
>   };
>   
>   /* Structure which describes individual configuration for each entity */
>   struct vimc_ent_config {
>   	const char *name;
> -	size_t pads_qty;
> -	const unsigned long *pads_flag;
> -	enum vimc_ent_node node;
> +	const char *drv;
>   };
>   
>   /* Structure which describes links between entities */
> @@ -112,60 +82,40 @@ struct vimc_pipeline_config {
>   static const struct vimc_ent_config ent_config[] = {
>   	{
>   		.name = "Sensor A",
> -		.pads_qty = 1,
> -		.pads_flag = (const unsigned long[]){MEDIA_PAD_FL_SOURCE},
> -		.node = VIMC_ENT_NODE_SENSOR,
> +		.drv = "vimc-sensor",
>   	},
>   	{
>   		.name = "Sensor B",
> -		.pads_qty = 1,
> -		.pads_flag = (const unsigned long[]){MEDIA_PAD_FL_SOURCE},
> -		.node = VIMC_ENT_NODE_SENSOR,
> +		.drv = "vimc-sensor",
>   	},
>   	{
>   		.name = "Debayer A",
> -		.pads_qty = 2,
> -		.pads_flag = (const unsigned long[]){MEDIA_PAD_FL_SINK,
> -						     MEDIA_PAD_FL_SOURCE},
> -		.node = VIMC_ENT_NODE_DEBAYER,
> +		.drv = "vimc-debayer",
>   	},
>   	{
>   		.name = "Debayer B",
> -		.pads_qty = 2,
> -		.pads_flag = (const unsigned long[]){MEDIA_PAD_FL_SINK,
> -						     MEDIA_PAD_FL_SOURCE},
> -		.node = VIMC_ENT_NODE_DEBAYER,
> +		.drv = "vimc-debayer",
>   	},
>   	{
>   		.name = "Raw Capture 0",
> -		.pads_qty = 1,
> -		.pads_flag = (const unsigned long[]){MEDIA_PAD_FL_SINK},
> -		.node = VIMC_ENT_NODE_CAPTURE,
> +		.drv = "vimc-capture",
>   	},
>   	{
>   		.name = "Raw Capture 1",
> -		.pads_qty = 1,
> -		.pads_flag = (const unsigned long[]){MEDIA_PAD_FL_SINK},
> -		.node = VIMC_ENT_NODE_CAPTURE,
> +		.drv = "vimc-capture",
>   	},
>   	{
>   		.name = "RGB/YUV Input",
> -		.pads_qty = 1,
> -		.pads_flag = (const unsigned long[]){MEDIA_PAD_FL_SOURCE},
> -		.node = VIMC_ENT_NODE_INPUT,
> +		/* TODO: change to vimc-output when output is implemented */
> +		.drv = "vimc-sensor",

This is confusing: the name indicates an input, but the TODO indicates an
output. I assume this is supposed to emulate e.g. an HDMI input? I'm not sure
the TODO text makes sense here.

>   	},
>   	{
>   		.name = "Scaler",
> -		.pads_qty = 2,
> -		.pads_flag = (const unsigned long[]){MEDIA_PAD_FL_SINK,
> -						     MEDIA_PAD_FL_SOURCE},
> -		.node = VIMC_ENT_NODE_SCALER,
> +		.drv = "vimc-scaler",
>   	},
>   	{
>   		.name = "RGB/YUV Capture",
> -		.pads_qty = 1,
> -		.pads_flag = (const unsigned long[]){MEDIA_PAD_FL_SINK},
> -		.node = VIMC_ENT_NODE_CAPTURE,
> +		.drv = "vimc-capture",
>   	},
>   };
>   
> @@ -197,111 +147,40 @@ static const struct vimc_pipeline_config pipe_cfg = {
>   
>   /* -------------------------------------------------------------------------- */
>   
> -static void vimc_device_unregister(struct vimc_device *vimc)
> +static int vimc_create_links(struct vimc_device *vimc)
>   {
>   	unsigned int i;
> -
> -	media_device_unregister(&vimc->mdev);
> -	/* Cleanup (only initialized) entities */
> -	for (i = 0; i < vimc->pipe_cfg->num_ents; i++) {
> -		if (vimc->ved[i] && vimc->ved[i]->destroy)
> -			vimc->ved[i]->destroy(vimc->ved[i]);
> -
> -		vimc->ved[i] = NULL;
> -	}
> -	v4l2_device_unregister(&vimc->v4l2_dev);
> -	media_device_cleanup(&vimc->mdev);
> -}
> -
> -/*
> - * TODO: remove this function when all the
> - * entities specific code are implemented
> - */
> -static void vimc_raw_destroy(struct vimc_ent_device *ved)
> -{
> -	media_device_unregister_entity(ved->ent);
> -
> -	media_entity_cleanup(ved->ent);
> -
> -	vimc_pads_cleanup(ved->pads);
> -
> -	kfree(ved->ent);
> -
> -	kfree(ved);
> -}
> -
> -/*
> - * TODO: remove this function when all the
> - * entities specific code are implemented
> - */
> -static struct vimc_ent_device *vimc_raw_create(struct v4l2_device *v4l2_dev,
> -					       const char *const name,
> -					       u16 num_pads,
> -					       const unsigned long *pads_flag)
> -{
> -	struct vimc_ent_device *ved;
>   	int ret;
>   
> -	/* Allocate the main ved struct */
> -	ved = kzalloc(sizeof(*ved), GFP_KERNEL);
> -	if (!ved)
> -		return ERR_PTR(-ENOMEM);
> -
> -	/* Allocate the media entity */
> -	ved->ent = kzalloc(sizeof(*ved->ent), GFP_KERNEL);
> -	if (!ved->ent) {
> -		ret = -ENOMEM;
> -		goto err_free_ved;
> -	}
> -
> -	/* Allocate the pads */
> -	ved->pads = vimc_pads_init(num_pads, pads_flag);
> -	if (IS_ERR(ved->pads)) {
> -		ret = PTR_ERR(ved->pads);
> -		goto err_free_ent;
> +	/* Initialize the links between entities */
> +	for (i = 0; i < vimc->pipe_cfg->num_links; i++) {
> +		const struct vimc_ent_link *link = &vimc->pipe_cfg->links[i];
> +		/*
> +		 * TODO: Check another way of retrieving ved struct without
> +		 * relying on platform_get_drvdata
> +		 */
> +		struct vimc_ent_device *ved_src =
> +			platform_get_drvdata(vimc->subdevs[link->src_ent]);
> +		struct vimc_ent_device *ved_sink =
> +			platform_get_drvdata(vimc->subdevs[link->sink_ent]);
> +
> +		ret = media_create_pad_link(ved_src->ent, link->src_pad,
> +					    ved_sink->ent, link->sink_pad,
> +					    link->flags);
> +		if (ret)
> +			return ret;
>   	}
>   
> -	/* Initialize the media entity */
> -	ved->ent->name = name;
> -	ved->ent->function = MEDIA_ENT_F_V4L2_SUBDEV_UNKNOWN;
> -	ret = media_entity_pads_init(ved->ent, num_pads, ved->pads);
> -	if (ret)
> -		goto err_cleanup_pads;
> -
> -	/* Register the media entity */
> -	ret = media_device_register_entity(v4l2_dev->mdev, ved->ent);
> -	if (ret)
> -		goto err_cleanup_entity;
> -
> -	/* Fill out the destroy function and return */
> -	ved->destroy = vimc_raw_destroy;
> -	return ved;
> -
> -err_cleanup_entity:
> -	media_entity_cleanup(ved->ent);
> -err_cleanup_pads:
> -	vimc_pads_cleanup(ved->pads);
> -err_free_ent:
> -	kfree(ved->ent);
> -err_free_ved:
> -	kfree(ved);
> -
> -	return ERR_PTR(ret);
> +	return 0;
>   }
>   
> -static int vimc_device_register(struct vimc_device *vimc)
> +static int vimc_comp_bind(struct device *master)
>   {
> -	unsigned int i;
> +	struct vimc_device *vimc = container_of(to_platform_device(master),
> +						struct vimc_device, pdev);
>   	int ret;
>   
> -	/* Allocate memory for the vimc_ent_devices pointers */
> -	vimc->ved = devm_kcalloc(vimc->mdev.dev, vimc->pipe_cfg->num_ents,
> -				 sizeof(*vimc->ved), GFP_KERNEL);
> -	if (!vimc->ved)
> -		return -ENOMEM;
> -
> -	/* Link the media device within the v4l2_device */
> -	vimc->v4l2_dev.mdev = &vimc->mdev;
> +	dev_dbg(master, "bind");
>   
>   	/* Register the v4l2 struct */
>   	ret = v4l2_device_register(vimc->mdev.dev, &vimc->v4l2_dev);
> @@ -311,66 +190,22 @@ static int vimc_device_register(struct vimc_device *vimc)
>   		return ret;
>   	}
>   
> -	/* Initialize entities */
> -	for (i = 0; i < vimc->pipe_cfg->num_ents; i++) {
> -		struct vimc_ent_device *(*create_func)(struct v4l2_device *,
> -						       const char *const,
> -						       u16,
> -						       const unsigned long *);
> -
> -		/* Register the specific node */
> -		switch (vimc->pipe_cfg->ents[i].node) {
> -		case VIMC_ENT_NODE_SENSOR:
> -			create_func = vimc_sen_create;
> -			break;
> -
> -		case VIMC_ENT_NODE_CAPTURE:
> -			create_func = vimc_cap_create;
> -			break;
> -
> -		/* TODO: Instantiate the specific topology node */
> -		case VIMC_ENT_NODE_INPUT:
> -		case VIMC_ENT_NODE_DEBAYER:
> -		case VIMC_ENT_NODE_SCALER:
> -		default:
> -			/*
> -			 * TODO: remove this when all the entities specific
> -			 * code are implemented
> -			 */
> -			create_func = vimc_raw_create;
> -			break;
> -		}
> -
> -		vimc->ved[i] = create_func(&vimc->v4l2_dev,
> -					   vimc->pipe_cfg->ents[i].name,
> -					   vimc->pipe_cfg->ents[i].pads_qty,
> -					   vimc->pipe_cfg->ents[i].pads_flag);
> -		if (IS_ERR(vimc->ved[i])) {
> -			ret = PTR_ERR(vimc->ved[i]);
> -			vimc->ved[i] = NULL;
> -			goto err;
> -		}
> -	}
> -
> -	/* Initialize the links between entities */
> -	for (i = 0; i < vimc->pipe_cfg->num_links; i++) {
> -		const struct vimc_ent_link *link = &vimc->pipe_cfg->links[i];
> +	/* Bind subdevices */
> +	ret = component_bind_all(master, &vimc->v4l2_dev);
> +	if (ret)
> +		goto err_v4l2_unregister;
>   
> -		ret = media_create_pad_link(vimc->ved[link->src_ent]->ent,
> -					    link->src_pad,
> -					    vimc->ved[link->sink_ent]->ent,
> -					    link->sink_pad,
> -					    link->flags);
> -		if (ret)
> -			goto err;
> -	}
> +	/* Initialize links */
> +	ret = vimc_create_links(vimc);
> +	if (ret)
> +		goto err_comp_unbind_all;
>   
>   	/* Register the media device */
>   	ret = media_device_register(&vimc->mdev);
>   	if (ret) {
>   		dev_err(vimc->mdev.dev,
>   			"media device register failed (err=%d)\n", ret);
> -		return ret;
> +		goto err_comp_unbind_all;
>   	}
>   
>   	/* Expose all subdev's nodes*/
> @@ -379,32 +214,107 @@ static int vimc_device_register(struct vimc_device *vimc)
>   		dev_err(vimc->mdev.dev,
>   			"vimc subdev nodes registration failed (err=%d)\n",
>   			ret);
> -		goto err;
> +		goto err_mdev_unregister;
>   	}
>   
>   	return 0;
>   
> -err:
> -	/* Destroy the so far created topology */
> -	vimc_device_unregister(vimc);
> +err_mdev_unregister:
> +	media_device_unregister(&vimc->mdev);
> +err_comp_unbind_all:
> +	component_unbind_all(master, NULL);
> +err_v4l2_unregister:
> +	v4l2_device_unregister(&vimc->v4l2_dev);
>   
>   	return ret;
>   }
>   
> +static void vimc_comp_unbind(struct device *master)
> +{
> +	struct vimc_device *vimc = container_of(to_platform_device(master),
> +						struct vimc_device, pdev);
> +
> +	dev_dbg(master, "unbind");
> +
> +	media_device_unregister(&vimc->mdev);
> +	component_unbind_all(master, NULL);
> +	v4l2_device_unregister(&vimc->v4l2_dev);
> +}
> +
> +static int vimc_comp_compare(struct device *comp, void *data)
> +{
> +	const struct platform_device *pdev = to_platform_device(comp);
> +	const char *name = data;
> +
> +	return !strcmp(pdev->dev.platform_data, name);
> +}
> +
> +static struct component_match *vimc_add_subdevs(struct vimc_device *vimc)
> +{
> +	struct component_match *match = NULL;
> +	unsigned int i;
> +
> +	for (i = 0; i < vimc->pipe_cfg->num_ents; i++) {
> +		dev_dbg(&vimc->pdev.dev, "new pdev for %s\n",
> +			vimc->pipe_cfg->ents[i].drv);
> +
> +		/*
> +		 * TODO: check if using platform_data is indeed the best way to
> +		 * pass the name to the driver or if we should add the drv name
> +		 * in the platform_device_id table
> +		 */

Didn't you set the drv name in the platform_device_id table already?

Using platform_data feels like an abuse to be honest.

Creating these components here makes sense. Wouldn't it also make sense to use
v4l2_async to wait until they have all been bound? It would more closely emulate
standard drivers. Apologies if I misunderstand what is happening here.

> +		vimc->subdevs[i] = platform_device_register_data(&vimc->pdev.dev,
> +				vimc->pipe_cfg->ents[i].drv,
> +				PLATFORM_DEVID_AUTO,
> +				vimc->pipe_cfg->ents[i].name,
> +				strlen(vimc->pipe_cfg->ents[i].name) + 1);
> +		if (!vimc->subdevs[i]) {
> +			while (--i >= 0)
> +				platform_device_unregister(vimc->subdevs[i]);
> +
> +			return ERR_PTR(-ENOMEM);
> +		}
> +
> +		component_match_add(&vimc->pdev.dev, &match, vimc_comp_compare,
> +				    (void *)vimc->pipe_cfg->ents[i].name);
> +	}
> +
> +	return match;
> +}
> +
> +static void vimc_rm_subdevs(struct vimc_device *vimc)
> +{
> +	unsigned int i;
> +
> +	for (i = 0; i < vimc->pipe_cfg->num_ents; i++)
> +		platform_device_unregister(vimc->subdevs[i]);
> +}
> +
> +static const struct component_master_ops vimc_comp_ops = {
> +	.bind = vimc_comp_bind,
> +	.unbind = vimc_comp_unbind,
> +};
> +
>   static int vimc_probe(struct platform_device *pdev)
>   {
> -	struct vimc_device *vimc;
> +	struct vimc_device *vimc = container_of(pdev, struct vimc_device, pdev);
> +	struct component_match *match = NULL;
>   	int ret;
>   
> -	/* Prepare the vimc topology structure */
> +	dev_dbg(&pdev->dev, "probe");
>   
> -	/* Allocate memory for the vimc structure */
> -	vimc = kzalloc(sizeof(*vimc), GFP_KERNEL);
> -	if (!vimc)
> +	/* Create platform_device for each entity in the topology*/
> +	vimc->subdevs = devm_kcalloc(&vimc->pdev.dev, vimc->pipe_cfg->num_ents,
> +				     sizeof(*vimc->subdevs), GFP_KERNEL);
> +	if (!vimc->subdevs)
>   		return -ENOMEM;
>   
> -	/* Set the pipeline configuration struct */
> -	vimc->pipe_cfg = &pipe_cfg;
> +	match = vimc_add_subdevs(vimc);
> +	if (IS_ERR(match))
> +		return PTR_ERR(match);
> +
> +	/* Link the media device within the v4l2_device */
> +	vimc->v4l2_dev.mdev = &vimc->mdev;
>   
>   	/* Initialize media device */
>   	strlcpy(vimc->mdev.model, VIMC_MDEV_MODEL_NAME,
> @@ -412,28 +322,27 @@ static int vimc_probe(struct platform_device *pdev)
>   	vimc->mdev.dev = &pdev->dev;
>   	media_device_init(&vimc->mdev);
>   
> -	/* Create vimc topology */
> -	ret = vimc_device_register(vimc);
> +	/* Add self to the component system */
> +	ret = component_master_add_with_match(&pdev->dev, &vimc_comp_ops,
> +					      match);
>   	if (ret) {
> -		dev_err(vimc->mdev.dev,
> -			"vimc device registration failed (err=%d)\n", ret);
> +		media_device_cleanup(&vimc->mdev);
> +		vimc_rm_subdevs(vimc);
>   		kfree(vimc);
>   		return ret;
>   	}
>   
> -	/* Link the topology object with the platform device object */
> -	platform_set_drvdata(pdev, vimc);
> -
>   	return 0;
>   }
>   
>   static int vimc_remove(struct platform_device *pdev)
>   {
> -	struct vimc_device *vimc = platform_get_drvdata(pdev);
> +	struct vimc_device *vimc = container_of(pdev, struct vimc_device, pdev);
>   
> -	/* Destroy all the topology */
> -	vimc_device_unregister(vimc);
> -	kfree(vimc);
> +	dev_dbg(&pdev->dev, "remove");
> +
> +	component_master_del(&pdev->dev, &vimc_comp_ops);
> +	vimc_rm_subdevs(vimc);
>   
>   	return 0;
>   }
> @@ -442,9 +351,12 @@ static void vimc_dev_release(struct device *dev)
>   {
>   }
>   
> -static struct platform_device vimc_pdev = {
> -	.name		= VIMC_PDEV_NAME,
> -	.dev.release	= vimc_dev_release,
> +static struct vimc_device vimc_dev = {
> +	.pipe_cfg = &pipe_cfg,
> +	.pdev = {
> +		.name = VIMC_PDEV_NAME,
> +		.dev.release = vimc_dev_release,
> +	}
>   };
>   
>   static struct platform_driver vimc_pdrv = {
> @@ -459,29 +371,29 @@ static int __init vimc_init(void)
>   {
>   	int ret;
>   
> -	ret = platform_device_register(&vimc_pdev);
> +	ret = platform_device_register(&vimc_dev.pdev);
>   	if (ret) {
> -		dev_err(&vimc_pdev.dev,
> +		dev_err(&vimc_dev.pdev.dev,
>   			"platform device registration failed (err=%d)\n", ret);
>   		return ret;
>   	}
>   
>   	ret = platform_driver_register(&vimc_pdrv);
>   	if (ret) {
> -		dev_err(&vimc_pdev.dev,
> +		dev_err(&vimc_dev.pdev.dev,
>   			"platform driver registration failed (err=%d)\n", ret);
> -
> -		platform_device_unregister(&vimc_pdev);
> +		platform_driver_unregister(&vimc_pdrv);
> +		return ret;
>   	}
>   
> -	return ret;
> +	return 0;
>   }
>   
>   static void __exit vimc_exit(void)
>   {
>   	platform_driver_unregister(&vimc_pdrv);
>   
> -	platform_device_unregister(&vimc_pdev);
> +	platform_device_unregister(&vimc_dev.pdev);
>   }
>   
>   module_init(vimc_init);
> diff --git a/drivers/media/platform/vimc/vimc-sensor.c b/drivers/media/platform/vimc/vimc-sensor.c
> index df89ee7..7f16978 100644
> --- a/drivers/media/platform/vimc/vimc-sensor.c
> +++ b/drivers/media/platform/vimc/vimc-sensor.c
> @@ -15,15 +15,19 @@
>    *
>    */
>   
> -#include <linux/module.h>
> +#include <linux/component.h>
>   #include <linux/freezer.h>
>   #include <linux/kthread.h>
> +#include <linux/module.h>
> +#include <linux/platform_device.h>
>   #include <linux/v4l2-mediabus.h>
>   #include <linux/vmalloc.h>
>   #include <media/v4l2-subdev.h>
>   #include <media/v4l2-tpg.h>
>   
> -#include "vimc-sensor.h"
> +#include "vimc-common.h"
> +
> +#define VIMC_SEN_DRV_NAME "vimc-sensor"
>   
>   static bool vsen_tpg;
>   module_param(vsen_tpg, bool, 0000);
> @@ -35,6 +39,7 @@ MODULE_PARM_DESC(vsen_tpg, " generate image from sensor node\n"
>   struct vimc_sen_device {
>   	struct vimc_ent_device ved;
>   	struct v4l2_subdev sd;
> +	struct device *dev;
>   	struct tpg_data tpg;
>   	struct task_struct *kthread_sen;
>   	u8 *frame;
> @@ -189,7 +194,7 @@ static int vimc_sen_set_fmt(struct v4l2_subdev *sd,
>   	/* Set the new format */
>   	vimc_sen_adjust_fmt(&fmt->format);
>   
> -	dev_dbg(vsen->sd.v4l2_dev->mdev->dev, "%s: format update: "
> +	dev_dbg(vsen->dev, "%s: format update: "
>   		"old:%dx%d (0x%x, %d, %d, %d, %d) "
>   		"new:%dx%d (0x%x, %d, %d, %d, %d)\n", vsen->sd.name,
>   		/* old */
> @@ -282,8 +287,8 @@ static int vimc_sen_s_stream(struct v4l2_subdev *sd, int enable)
>   		vsen->kthread_sen = kthread_run(vimc_sen_tpg_thread, vsen,
>   					"%s-sen", vsen->sd.v4l2_dev->name);
>   		if (IS_ERR(vsen->kthread_sen)) {
> -			dev_err(vsen->sd.v4l2_dev->dev,
> -				"%s: kernel_thread() failed\n",	vsen->sd.name);
> +			dev_err(vsen->dev, "%s: kernel_thread() failed\n",
> +				vsen->sd.name);
>   			vfree(vsen->frame);
>   			vsen->frame = NULL;
>   			return PTR_ERR(vsen->kthread_sen);
> @@ -315,8 +320,10 @@ static const struct v4l2_subdev_ops vimc_sen_ops = {
>   	.video = &vimc_sen_video_ops,
>   };
>   
> -static void vimc_sen_destroy(struct vimc_ent_device *ved)
> +static void vimc_sen_comp_unbind(struct device *comp, struct device *master,
> +				 void *master_data)
>   {
> +	struct vimc_ent_device *ved = dev_get_drvdata(comp);
>   	struct vimc_sen_device *vsen =
>   				container_of(ved, struct vimc_sen_device, ved);
>   
> @@ -326,36 +333,30 @@ static void vimc_sen_destroy(struct vimc_ent_device *ved)
>   	kfree(vsen);
>   }
>   
> -struct vimc_ent_device *vimc_sen_create(struct v4l2_device *v4l2_dev,
> -					const char *const name,
> -					u16 num_pads,
> -					const unsigned long *pads_flag)
> +static int vimc_sen_comp_bind(struct device *comp, struct device *master,
> +			      void *master_data)
>   {
> +	struct v4l2_device *v4l2_dev = master_data;
> +	char *name = comp->platform_data;
>   	struct vimc_sen_device *vsen;
> -	unsigned int i;
>   	int ret;
>   
> -	/* NOTE: a sensor node may be created with more then one pad */
> -	if (!name || !num_pads || !pads_flag)
> -		return ERR_PTR(-EINVAL);
> -
> -	/* check if all pads are sources */
> -	for (i = 0; i < num_pads; i++)
> -		if (!(pads_flag[i] & MEDIA_PAD_FL_SOURCE))
> -			return ERR_PTR(-EINVAL);
> -
>   	/* Allocate the vsen struct */
>   	vsen = kzalloc(sizeof(*vsen), GFP_KERNEL);
>   	if (!vsen)
> -		return ERR_PTR(-ENOMEM);
> +		return -ENOMEM;
>   
>   	/* Initialize ved and sd */
>   	ret = vimc_ent_sd_register(&vsen->ved, &vsen->sd, v4l2_dev, name,
> -				   MEDIA_ENT_F_CAM_SENSOR, num_pads, pads_flag,
> -				   &vimc_sen_ops, vimc_sen_destroy);
> +				   MEDIA_ENT_F_ATV_DECODER, 1,
> +				   (const unsigned long[1]) {MEDIA_PAD_FL_SOURCE},
> +				   &vimc_sen_ops);
>   	if (ret)
>   		goto err_free_vsen;
>   
> +	dev_set_drvdata(comp, &vsen->ved);
> +	vsen->dev = comp;
> +
>   	/* Initialize the frame format */
>   	vsen->mbus_format = fmt_default;
>   
> @@ -368,12 +369,52 @@ struct vimc_ent_device *vimc_sen_create(struct v4l2_device *v4l2_dev,
>   			goto err_unregister_ent_sd;
>   	}
>   
> -	return &vsen->ved;
> +	return 0;
>   
>   err_unregister_ent_sd:
>   	vimc_ent_sd_unregister(&vsen->ved,  &vsen->sd);
>   err_free_vsen:
>   	kfree(vsen);
>   
> -	return ERR_PTR(ret);
> +	return ret;
> +}
> +
> +static const struct component_ops vimc_sen_comp_ops = {
> +	.bind = vimc_sen_comp_bind,
> +	.unbind = vimc_sen_comp_unbind,
> +};
> +
> +static int vimc_sen_probe(struct platform_device *pdev)
> +{
> +	return component_add(&pdev->dev, &vimc_sen_comp_ops);
>   }
> +
> +static int vimc_sen_remove(struct platform_device *pdev)
> +{
> +	component_del(&pdev->dev, &vimc_sen_comp_ops);
> +
> +	return 0;
> +}
> +
> +static struct platform_driver vimc_sen_pdrv = {
> +	.probe		= vimc_sen_probe,
> +	.remove		= vimc_sen_remove,
> +	.driver		= {
> +		.name	= VIMC_SEN_DRV_NAME,
> +	},
> +};
> +
> +static const struct platform_device_id vimc_sen_driver_ids[] = {
> +	{
> +		.name           = VIMC_SEN_DRV_NAME,
> +	},
> +	{ }
> +};
> +
> +module_platform_driver(vimc_sen_pdrv);
> +
> +MODULE_DEVICE_TABLE(platform, vimc_sen_driver_ids);
> +
> +MODULE_DESCRIPTION("Virtual Media Controller Driver (VIMC) Sensor");
> +MODULE_AUTHOR("Helen Mae Koike Fornazier <helen.fornazier@gmail.com>");
> +MODULE_LICENSE("GPL");
> diff --git a/drivers/media/platform/vimc/vimc-sensor.h b/drivers/media/platform/vimc/vimc-sensor.h
> deleted file mode 100644
> index 580dcec..0000000
> --- a/drivers/media/platform/vimc/vimc-sensor.h
> +++ /dev/null
> @@ -1,28 +0,0 @@
> -/*
> - * vimc-sensor.h Virtual Media Controller Driver
> - *
> - * Copyright (C) 2015-2017 Helen Koike <helen.fornazier@gmail.com>
> - *
> - * This program is free software; you can redistribute it and/or modify
> - * it under the terms of the GNU General Public License as published by
> - * the Free Software Foundation; either version 2 of the License, or
> - * (at your option) any later version.
> - *
> - * This program is distributed in the hope that it will be useful,
> - * but WITHOUT ANY WARRANTY; without even the implied warranty of
> - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
> - * GNU General Public License for more details.
> - *
> - */
> -
> -#ifndef _VIMC_SENSOR_H_
> -#define _VIMC_SENSOR_H_
> -
> -#include "vimc-common.h"
> -
> -struct vimc_ent_device *vimc_sen_create(struct v4l2_device *v4l2_dev,
> -					const char *const name,
> -					u16 num_pads,
> -					const unsigned long *pads_flag);
> -
> -#endif
> 

Regards,

	Hans

Re: [RFC PATCH v3 09/11] [media] vimc: Subdevices as modules

[Helen Koike]

Hi Hans,

Thank you for your review. Please check my comments below

On 2017-06-12 07:37 AM, Hans Verkuil wrote:
> On 06/03/2017 04:58 AM, Helen Koike wrote:
>> Change the core structure for adding subdevices in the topology.
>> Instead of calling the specific create function for each subdevice,
>> inject a child platform_device with the driver's name.
>> Each type of node in the topology (sensor, capture, debayer, scaler)
>> will register a platform_driver with the corresponding name through the
>> component subsystem.
>> Implementing a new subdevice type doesn't require vimc-core to be
>> altered.
>>
>> This facilitates future implementation of dynamic entities, where
>> hotpluging an entity in the topology is just a matter of
>> registering/unregistering a platform_device in the system.
>> It also facilitates other implementations of different nodes without
>> touching the core code and remove the need of a header file for each
>> type of node.
>>
>> Signed-off-by: Helen Koike <helen.koike@collabora.com>
>>
>> ---
>>
>> Changes in v3:
>> [media] vimc: Subdevices as modules
>>     - This is a new patch in the series
>>
>> Changes in v2: None
>>
>>
>> ---
>>   drivers/media/platform/vimc/Makefile       |   7 +-
>>   drivers/media/platform/vimc/vimc-capture.c |  96 ++++---
>>   drivers/media/platform/vimc/vimc-capture.h |  28 --
>>   drivers/media/platform/vimc/vimc-common.c  |  37 ++-
>>   drivers/media/platform/vimc/vimc-common.h  |  14 +-
>>   drivers/media/platform/vimc/vimc-core.c    | 406
>> +++++++++++------------------
>>   drivers/media/platform/vimc/vimc-sensor.c  |  91 +++++--
>>   drivers/media/platform/vimc/vimc-sensor.h  |  28 --
>>   8 files changed, 320 insertions(+), 387 deletions(-)
>>   delete mode 100644 drivers/media/platform/vimc/vimc-capture.h
>>   delete mode 100644 drivers/media/platform/vimc/vimc-sensor.h
>>
>> diff --git a/drivers/media/platform/vimc/Makefile
>> b/drivers/media/platform/vimc/Makefile
>> index 6b6ddf4..0e5d5ce 100644
>> --- a/drivers/media/platform/vimc/Makefile
>> +++ b/drivers/media/platform/vimc/Makefile
>> @@ -1,3 +1,6 @@
>> -vimc-objs := vimc-core.o vimc-capture.o vimc-common.o vimc-sensor.o
>> +vimc-objs := vimc-core.o
>> +vimc_capture-objs := vimc-capture.o
>> +vimc_common-objs := vimc-common.o
>> +vimc_sensor-objs := vimc-sensor.o
>>   -obj-$(CONFIG_VIDEO_VIMC) += vimc.o
>> +obj-$(CONFIG_VIDEO_VIMC) += vimc.o vimc_capture.o vimc_common.o
>> vimc_sensor.o
>> diff --git a/drivers/media/platform/vimc/vimc-capture.c
>> b/drivers/media/platform/vimc/vimc-capture.c
>> index b5da0ea..633d99a 100644
>> --- a/drivers/media/platform/vimc/vimc-capture.c
>> +++ b/drivers/media/platform/vimc/vimc-capture.c
>> @@ -15,18 +15,24 @@
>>    *
>>    */
>>   +#include <linux/component.h>
>>   #include <linux/freezer.h>
>>   #include <linux/kthread.h>
>> +#include <linux/module.h>
>> +#include <linux/platform_device.h>
>>   #include <media/v4l2-ioctl.h>
>>   #include <media/v4l2-tpg.h>
>>   #include <media/videobuf2-core.h>
>>   #include <media/videobuf2-vmalloc.h>
>>   -#include "vimc-capture.h"
>> +#include "vimc-common.h"
>> +
>> +#define VIMC_CAP_DRV_NAME "vimc-capture"
>>     struct vimc_cap_device {
>>       struct vimc_ent_device ved;
>>       struct video_device vdev;
>> +    struct device *dev;
>>       struct v4l2_pix_format format;
>>       struct vb2_queue queue;
>>       struct list_head buf_list;
>> @@ -150,7 +156,7 @@ static int vimc_cap_s_fmt_vid_cap(struct file
>> *file, void *priv,
>>         vimc_cap_try_fmt_vid_cap(file, priv, f);
>>   -    dev_dbg(vcap->vdev.v4l2_dev->dev, "%s: format update: "
>> +    dev_dbg(vcap->dev, "%s: format update: "
>>           "old:%dx%d (0x%x, %d, %d, %d, %d) "
>>           "new:%dx%d (0x%x, %d, %d, %d, %d)\n", vcap->vdev.name,
>>           /* old */
>> @@ -365,8 +371,7 @@ static int vimc_cap_start_streaming(struct
>> vb2_queue *vq, unsigned int count)
>>           vcap->kthread_cap = kthread_run(vimc_cap_tpg_thread, vcap,
>>                       "%s-cap", vcap->vdev.v4l2_dev->name);
>>           if (IS_ERR(vcap->kthread_cap)) {
>> -            dev_err(vcap->vdev.v4l2_dev->dev,
>> -                "%s: kernel_thread() failed\n",
>> +            dev_err(vcap->dev, "%s: kernel_thread() failed\n",
>>                   vcap->vdev.name);
>>               ret = PTR_ERR(vcap->kthread_cap);
>>               goto err_tpg_free;
>> @@ -443,8 +448,7 @@ static int vimc_cap_buffer_prepare(struct
>> vb2_buffer *vb)
>>       unsigned long size = vcap->format.sizeimage;
>>         if (vb2_plane_size(vb, 0) < size) {
>> -        dev_err(vcap->vdev.v4l2_dev->dev,
>> -            "%s: buffer too small (%lu < %lu)\n",
>> +        dev_err(vcap->dev, "%s: buffer too small (%lu < %lu)\n",
>>               vcap->vdev.name, vb2_plane_size(vb, 0), size);
>>           return -EINVAL;
>>       }
>> @@ -469,8 +473,10 @@ static const struct media_entity_operations
>> vimc_cap_mops = {
>>       .link_validate        = vimc_link_validate,
>>   };
>>   -static void vimc_cap_destroy(struct vimc_ent_device *ved)
>> +static void vimc_cap_comp_unbind(struct device *comp, struct device
>> *master,
>> +                 void *master_data)
>>   {
>> +    struct vimc_ent_device *ved = dev_get_drvdata(comp);
>>       struct vimc_cap_device *vcap = container_of(ved, struct
>> vimc_cap_device,
>>                               ved);
>>   @@ -481,32 +487,25 @@ static void vimc_cap_destroy(struct
>> vimc_ent_device *ved)
>>       kfree(vcap);
>>   }
>>   -struct vimc_ent_device *vimc_cap_create(struct v4l2_device *v4l2_dev,
>> -                    const char *const name,
>> -                    u16 num_pads,
>> -                    const unsigned long *pads_flag)
>> +static int vimc_cap_comp_bind(struct device *comp, struct device
>> *master,
>> +                  void *master_data)
>>   {
>> +    struct v4l2_device *v4l2_dev = master_data;
>> +    char *name = comp->platform_data;
>>       const struct vimc_pix_map *vpix;
>>       struct vimc_cap_device *vcap;
>>       struct video_device *vdev;
>>       struct vb2_queue *q;
>>       int ret;
>>   -    /*
>> -     * Check entity configuration params
>> -     * NOTE: we only support a single sink pad
>> -     */
>> -    if (!name || num_pads != 1 || !pads_flag ||
>> -        !(pads_flag[0] & MEDIA_PAD_FL_SINK))
>> -        return ERR_PTR(-EINVAL);
>> -
>>       /* Allocate the vimc_cap_device struct */
>>       vcap = kzalloc(sizeof(*vcap), GFP_KERNEL);
>>       if (!vcap)
>> -        return ERR_PTR(-ENOMEM);
>> +        return -ENOMEM;
>>         /* Allocate the pads */
>> -    vcap->ved.pads = vimc_pads_init(num_pads, pads_flag);
>> +    vcap->ved.pads =
>> +        vimc_pads_init(1, (const unsigned long[1]) {MEDIA_PAD_FL_SINK});
>>       if (IS_ERR(vcap->ved.pads)) {
>>           ret = PTR_ERR(vcap->ved.pads);
>>           goto err_free_vcap;
>> @@ -516,7 +515,7 @@ struct vimc_ent_device *vimc_cap_create(struct
>> v4l2_device *v4l2_dev,
>>       vcap->vdev.entity.name = name;
>>       vcap->vdev.entity.function = MEDIA_ENT_F_IO_V4L;
>>       ret = media_entity_pads_init(&vcap->vdev.entity,
>> -                     num_pads, vcap->ved.pads);
>> +                     1, vcap->ved.pads);
>>       if (ret)
>>           goto err_clean_pads;
>>   @@ -537,8 +536,7 @@ struct vimc_ent_device *vimc_cap_create(struct
>> v4l2_device *v4l2_dev,
>>         ret = vb2_queue_init(q);
>>       if (ret) {
>> -        dev_err(vcap->vdev.v4l2_dev->dev,
>> -            "%s: vb2 queue init failed (err=%d)\n",
>> +        dev_err(comp, "%s: vb2 queue init failed (err=%d)\n",
>>               vcap->vdev.name, ret);
>>           goto err_clean_m_ent;
>>       }
>> @@ -555,10 +553,11 @@ struct vimc_ent_device *vimc_cap_create(struct
>> v4l2_device *v4l2_dev,
>>                    vcap->format.height;
>>         /* Fill the vimc_ent_device struct */
>> -    vcap->ved.destroy = vimc_cap_destroy;
>>       vcap->ved.ent = &vcap->vdev.entity;
>>       vcap->ved.process_frame = vimc_cap_process_frame;
>>       vcap->ved.vdev_get_format = vimc_cap_get_format;
>> +    dev_set_drvdata(comp, &vcap->ved);
>> +    vcap->dev = comp;
>>         /* Initialize the video_device struct */
>>       vdev = &vcap->vdev;
>> @@ -577,13 +576,12 @@ struct vimc_ent_device *vimc_cap_create(struct
>> v4l2_device *v4l2_dev,
>>       /* Register the video_device with the v4l2 and the media
>> framework */
>>       ret = video_register_device(vdev, VFL_TYPE_GRABBER, -1);
>>       if (ret) {
>> -        dev_err(vcap->vdev.v4l2_dev->dev,
>> -            "%s: video register failed (err=%d)\n",
>> +        dev_err(comp, "%s: video register failed (err=%d)\n",
>>               vcap->vdev.name, ret);
>>           goto err_release_queue;
>>       }
>>   -    return &vcap->ved;
>> +    return 0;
>>     err_release_queue:
>>       vb2_queue_release(q);
>> @@ -594,5 +592,45 @@ struct vimc_ent_device *vimc_cap_create(struct
>> v4l2_device *v4l2_dev,
>>   err_free_vcap:
>>       kfree(vcap);
>>   -    return ERR_PTR(ret);
>> +    return ret;
>> +}
>> +
>> +static const struct component_ops vimc_cap_comp_ops = {
>> +    .bind = vimc_cap_comp_bind,
>> +    .unbind = vimc_cap_comp_unbind,
>> +};
>> +
>> +static int vimc_cap_probe(struct platform_device *pdev)
>> +{
>> +    return component_add(&pdev->dev, &vimc_cap_comp_ops);
>>   }
>> +
>> +static int vimc_cap_remove(struct platform_device *pdev)
>> +{
>> +    component_del(&pdev->dev, &vimc_cap_comp_ops);
>> +
>> +    return 0;
>> +}
>> +
>> +static struct platform_driver vimc_cap_pdrv = {
>> +    .probe        = vimc_cap_probe,
>> +    .remove        = vimc_cap_remove,
>> +    .driver        = {
>> +        .name    = VIMC_CAP_DRV_NAME,
>> +    },
>> +};
>> +
>> +static const struct platform_device_id vimc_cap_driver_ids[] = {
>> +    {
>> +        .name           = VIMC_CAP_DRV_NAME,
>> +    },
>> +    { }
>> +};
>> +
>> +module_platform_driver(vimc_cap_pdrv);
>> +
>> +MODULE_DEVICE_TABLE(platform, vimc_cap_driver_ids);
>> +
>> +MODULE_DESCRIPTION("Virtual Media Controller Driver (VIMC) Capture");
>> +MODULE_AUTHOR("Helen Mae Koike Fornazier <helen.fornazier@gmail.com>");
>> +MODULE_LICENSE("GPL");
>> diff --git a/drivers/media/platform/vimc/vimc-capture.h
>> b/drivers/media/platform/vimc/vimc-capture.h
>> deleted file mode 100644
>> index 7e5c707..0000000
>> --- a/drivers/media/platform/vimc/vimc-capture.h
>> +++ /dev/null
>> @@ -1,28 +0,0 @@
>> -/*
>> - * vimc-capture.h Virtual Media Controller Driver
>> - *
>> - * Copyright (C) 2015-2017 Helen Koike <helen.fornazier@gmail.com>
>> - *
>> - * This program is free software; you can redistribute it and/or modify
>> - * it under the terms of the GNU General Public License as published by
>> - * the Free Software Foundation; either version 2 of the License, or
>> - * (at your option) any later version.
>> - *
>> - * This program is distributed in the hope that it will be useful,
>> - * but WITHOUT ANY WARRANTY; without even the implied warranty of
>> - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
>> - * GNU General Public License for more details.
>> - *
>> - */
>> -
>> -#ifndef _VIMC_CAPTURE_H_
>> -#define _VIMC_CAPTURE_H_
>> -
>> -#include "vimc-common.h"
>> -
>> -struct vimc_ent_device *vimc_cap_create(struct v4l2_device *v4l2_dev,
>> -                    const char *const name,
>> -                    u16 num_pads,
>> -                    const unsigned long *pads_flag);
>> -
>> -#endif
>> diff --git a/drivers/media/platform/vimc/vimc-common.c
>> b/drivers/media/platform/vimc/vimc-common.c
>> index ff59e09..d5ed387 100644
>> --- a/drivers/media/platform/vimc/vimc-common.c
>> +++ b/drivers/media/platform/vimc/vimc-common.c
>> @@ -15,6 +15,9 @@
>>    *
>>    */
>>   +#include <linux/init.h>
>> +#include <linux/module.h>
>> +
>>   #include "vimc-common.h"
>>     static const struct vimc_pix_map vimc_pix_map_list[] = {
>> @@ -151,6 +154,7 @@ const struct vimc_pix_map
>> *vimc_pix_map_by_index(unsigned int i)
>>         return &vimc_pix_map_list[i];
>>   }
>> +EXPORT_SYMBOL(vimc_pix_map_by_index);
>
> I would recommend using EXPORT_SYMBOL_GPL.
>
>>     const struct vimc_pix_map *vimc_pix_map_by_code(u32 code)
>>   {
>> @@ -162,6 +166,7 @@ const struct vimc_pix_map
>> *vimc_pix_map_by_code(u32 code)
>>       }
>>       return NULL;
>>   }
>> +EXPORT_SYMBOL(vimc_pix_map_by_code);
>>     const struct vimc_pix_map *vimc_pix_map_by_pixelformat(u32
>> pixelformat)
>>   {
>> @@ -173,6 +178,7 @@ const struct vimc_pix_map
>> *vimc_pix_map_by_pixelformat(u32 pixelformat)
>>       }
>>       return NULL;
>>   }
>> +EXPORT_SYMBOL(vimc_pix_map_by_pixelformat);
>>     int vimc_propagate_frame(struct media_pad *src, const void *frame)
>>   {
>> @@ -207,6 +213,7 @@ int vimc_propagate_frame(struct media_pad *src,
>> const void *frame)
>>         return 0;
>>   }
>> +EXPORT_SYMBOL(vimc_propagate_frame);
>>     /* Helper function to allocate and initialize pads */
>>   struct media_pad *vimc_pads_init(u16 num_pads, const unsigned long
>> *pads_flag)
>> @@ -227,6 +234,7 @@ struct media_pad *vimc_pads_init(u16 num_pads,
>> const unsigned long *pads_flag)
>>         return pads;
>>   }
>> +EXPORT_SYMBOL(vimc_pads_init);
>>     int vimc_pipeline_s_stream(struct media_entity *ent, int enable)
>>   {
>> @@ -242,14 +250,8 @@ int vimc_pipeline_s_stream(struct media_entity
>> *ent, int enable)
>>           /* Start the stream in the subdevice direct connected */
>>           pad = media_entity_remote_pad(&ent->pads[i]);
>>   -        /*
>> -         * if this is a raw node from vimc-core, then there is
>> -         * nothing to activate
>> -         * TODO: remove this when there are no more raw nodes in the
>> -         * core and return error instead
>> -         */
>> -        if (pad->entity->obj_type == MEDIA_ENTITY_TYPE_BASE)
>> -            continue;
>> +        if (!is_media_entity_v4l2_subdev(pad->entity))
>> +            return -EINVAL;
>>             sd = media_entity_to_v4l2_subdev(pad->entity);
>>           ret = v4l2_subdev_call(sd, video, s_stream, enable);
>> @@ -259,6 +261,7 @@ int vimc_pipeline_s_stream(struct media_entity
>> *ent, int enable)
>>         return 0;
>>   }
>> +EXPORT_SYMBOL(vimc_pipeline_s_stream);
>>     static void vimc_fmt_pix_to_mbus(struct v4l2_mbus_framefmt *mfmt,
>>                    struct v4l2_pix_format *pfmt)
>> @@ -315,14 +318,6 @@ int vimc_link_validate(struct media_link *link)
>>       struct v4l2_subdev_format source_fmt, sink_fmt;
>>       int ret;
>>   -    /*
>> -     * if it is a raw node from vimc-core, ignore the link for now
>> -     * TODO: remove this when there are no more raw nodes in the
>> -     * core and return error instead
>> -     */
>> -    if (link->source->entity->obj_type == MEDIA_ENTITY_TYPE_BASE)
>> -        return 0;
>> -
>>       ret = vimc_get_mbus_format(link->source, &source_fmt);
>>       if (ret)
>>           return ret;
>> @@ -393,8 +388,7 @@ int vimc_ent_sd_register(struct vimc_ent_device *ved,
>>                u32 function,
>>                u16 num_pads,
>>                const unsigned long *pads_flag,
>> -             const struct v4l2_subdev_ops *sd_ops,
>> -             void (*sd_destroy)(struct vimc_ent_device *))
>> +             const struct v4l2_subdev_ops *sd_ops)
>>   {
>>       int ret;
>>   @@ -404,7 +398,6 @@ int vimc_ent_sd_register(struct vimc_ent_device
>> *ved,
>>           return PTR_ERR(ved->pads);
>>         /* Fill the vimc_ent_device struct */
>> -    ved->destroy = sd_destroy;
>>       ved->ent = &sd->entity;
>>         /* Initialize the subdev */
>> @@ -440,6 +433,7 @@ int vimc_ent_sd_register(struct vimc_ent_device *ved,
>>       vimc_pads_cleanup(ved->pads);
>>       return ret;
>>   }
>> +EXPORT_SYMBOL(vimc_ent_sd_register);
>>     void vimc_ent_sd_unregister(struct vimc_ent_device *ved, struct
>> v4l2_subdev *sd)
>>   {
>> @@ -447,3 +441,8 @@ void vimc_ent_sd_unregister(struct vimc_ent_device
>> *ved, struct v4l2_subdev *sd)
>>       media_entity_cleanup(ved->ent);
>>       vimc_pads_cleanup(ved->pads);
>>   }
>> +EXPORT_SYMBOL(vimc_ent_sd_unregister);
>> +
>> +MODULE_DESCRIPTION("Virtual Media Controller Driver (VIMC) Common");
>> +MODULE_AUTHOR("Helen Koike <helen.fornazier@gmail.com>");
>> +MODULE_LICENSE("GPL");
>> diff --git a/drivers/media/platform/vimc/vimc-common.h
>> b/drivers/media/platform/vimc/vimc-common.h
>> index 5b2691e..27c9b8c 100644
>> --- a/drivers/media/platform/vimc/vimc-common.h
>> +++ b/drivers/media/platform/vimc/vimc-common.h
>> @@ -1,5 +1,5 @@
>>   /*
>> - * vimc-ccommon.h Virtual Media Controller Driver
>> + * vimc-common.h Virtual Media Controller Driver
>>    *
>>    * Copyright (C) 2015-2017 Helen Koike <helen.fornazier@gmail.com>
>>    *
>> @@ -50,7 +50,6 @@ struct vimc_pix_map {
>>    *
>>    * @ent:        the pointer to struct media_entity for the node
>>    * @pads:        the list of pads of the node
>> - * @destroy:        callback to destroy the node
>>    * @process_frame:    callback send a frame to that node
>>    * @vdev_get_format:    callback that returns the current format a
>> pad, used
>>    *            only when is_media_entity_v4l2_video_device(ent) returns
>> @@ -67,7 +66,6 @@ struct vimc_pix_map {
>>   struct vimc_ent_device {
>>       struct media_entity *ent;
>>       struct media_pad *pads;
>> -    void (*destroy)(struct vimc_ent_device *);
>>       void (*process_frame)(struct vimc_ent_device *ved,
>>                     struct media_pad *sink, const void *frame);
>>       void (*vdev_get_format)(struct vimc_ent_device *ved,
>> @@ -152,7 +150,6 @@ const struct vimc_pix_map
>> *vimc_pix_map_by_pixelformat(u32 pixelformat);
>>    * @num_pads:    number of pads to initialize
>>    * @pads_flag:    flags to use in each pad
>>    * @sd_ops:    pointer to &struct v4l2_subdev_ops.
>> - * @sd_destroy:    callback to destroy the node
>>    *
>>    * Helper function initialize and register the struct
>> vimc_ent_device and struct
>>    * v4l2_subdev which represents a subdev node in the topology
>> @@ -164,14 +161,13 @@ int vimc_ent_sd_register(struct vimc_ent_device
>> *ved,
>>                u32 function,
>>                u16 num_pads,
>>                const unsigned long *pads_flag,
>> -             const struct v4l2_subdev_ops *sd_ops,
>> -             void (*sd_destroy)(struct vimc_ent_device *));
>> +             const struct v4l2_subdev_ops *sd_ops);
>>     /**
>> - * vimc_ent_sd_register - initialize and register a subdev node
>> + * vimc_ent_sd_unregister - cleanup and unregister a subdev node
>>    *
>> - * @ved:    the vimc_ent_device struct to be initialize
>> - * @sd:        the v4l2_subdev struct to be initialize and registered
>> + * @ved:    the vimc_ent_device struct to be cleaned up
>> + * @sd:        the v4l2_subdev struct to be unregistered
>>    *
>>    * Helper function cleanup and unregister the struct vimc_ent_device
>> and struct
>>    * v4l2_subdev which represents a subdev node in the topology
>> diff --git a/drivers/media/platform/vimc/vimc-core.c
>> b/drivers/media/platform/vimc/vimc-core.c
>> index afc79e2..6148c3e 100644
>> --- a/drivers/media/platform/vimc/vimc-core.c
>> +++ b/drivers/media/platform/vimc/vimc-core.c
>> @@ -15,15 +15,14 @@
>>    *
>>    */
>>   +#include <linux/component.h>
>>   #include <linux/init.h>
>>   #include <linux/module.h>
>>   #include <linux/platform_device.h>
>>   #include <media/media-device.h>
>>   #include <media/v4l2-device.h>
>>   -#include "vimc-capture.h"
>>   #include "vimc-common.h"
>> -#include "vimc-sensor.h"
>>     #define VIMC_PDEV_NAME "vimc"
>>   #define VIMC_MDEV_MODEL_NAME "VIMC MDEV"
>> @@ -37,10 +36,10 @@
>>   }
>>     struct vimc_device {
>> -    /*
>> -     * The pipeline configuration
>> -     * (filled before calling vimc_device_register)
>> -     */
>> +    /* The platform device */
>> +    struct platform_device pdev;
>> +
>> +    /* The pipeline configuration */
>>       const struct vimc_pipeline_config *pipe_cfg;
>>         /* The Associated media_device parent */
>> @@ -49,43 +48,14 @@ struct vimc_device {
>>       /* Internal v4l2 parent device*/
>>       struct v4l2_device v4l2_dev;
>>   -    /* Internal topology */
>> -    struct vimc_ent_device **ved;
>> -};
>> -
>> -/**
>> - * enum vimc_ent_node - Select the functionality of a node in the
>> topology
>> - * @VIMC_ENT_NODE_SENSOR:    A node of type SENSOR simulates a camera
>> sensor
>> - *                generating internal images in bayer format and
>> - *                propagating those images through the pipeline
>> - * @VIMC_ENT_NODE_CAPTURE:    A node of type CAPTURE is a v4l2
>> video_device
>> - *                that exposes the received image from the
>> - *                pipeline to the user space
>> - * @VIMC_ENT_NODE_INPUT:    A node of type INPUT is a v4l2
>> video_device that
>> - *                receives images from the user space and
>> - *                propagates them through the pipeline
>> - * @VIMC_ENT_NODE_DEBAYER:    A node type DEBAYER expects to receive
>> a frame
>> - *                in bayer format converts it to RGB
>> - * @VIMC_ENT_NODE_SCALER:    A node of type SCALER scales the
>> received image
>> - *                by a given multiplier
>> - *
>> - * This enum is used in the entity configuration struct to allow the
>> definition
>> - * of a custom topology specifying the role of each node on it.
>> - */
>> -enum vimc_ent_node {
>> -    VIMC_ENT_NODE_SENSOR,
>> -    VIMC_ENT_NODE_CAPTURE,
>> -    VIMC_ENT_NODE_INPUT,
>> -    VIMC_ENT_NODE_DEBAYER,
>> -    VIMC_ENT_NODE_SCALER,
>> +    /* Subdevices */
>> +    struct platform_device **subdevs;
>>   };
>>     /* Structure which describes individual configuration for each
>> entity */
>>   struct vimc_ent_config {
>>       const char *name;
>> -    size_t pads_qty;
>> -    const unsigned long *pads_flag;
>> -    enum vimc_ent_node node;
>> +    const char *drv;
>>   };
>>     /* Structure which describes links between entities */
>> @@ -112,60 +82,40 @@ struct vimc_pipeline_config {
>>   static const struct vimc_ent_config ent_config[] = {
>>       {
>>           .name = "Sensor A",
>> -        .pads_qty = 1,
>> -        .pads_flag = (const unsigned long[]){MEDIA_PAD_FL_SOURCE},
>> -        .node = VIMC_ENT_NODE_SENSOR,
>> +        .drv = "vimc-sensor",
>>       },
>>       {
>>           .name = "Sensor B",
>> -        .pads_qty = 1,
>> -        .pads_flag = (const unsigned long[]){MEDIA_PAD_FL_SOURCE},
>> -        .node = VIMC_ENT_NODE_SENSOR,
>> +        .drv = "vimc-sensor",
>>       },
>>       {
>>           .name = "Debayer A",
>> -        .pads_qty = 2,
>> -        .pads_flag = (const unsigned long[]){MEDIA_PAD_FL_SINK,
>> -                             MEDIA_PAD_FL_SOURCE},
>> -        .node = VIMC_ENT_NODE_DEBAYER,
>> +        .drv = "vimc-debayer",
>>       },
>>       {
>>           .name = "Debayer B",
>> -        .pads_qty = 2,
>> -        .pads_flag = (const unsigned long[]){MEDIA_PAD_FL_SINK,
>> -                             MEDIA_PAD_FL_SOURCE},
>> -        .node = VIMC_ENT_NODE_DEBAYER,
>> +        .drv = "vimc-debayer",
>>       },
>>       {
>>           .name = "Raw Capture 0",
>> -        .pads_qty = 1,
>> -        .pads_flag = (const unsigned long[]){MEDIA_PAD_FL_SINK},
>> -        .node = VIMC_ENT_NODE_CAPTURE,
>> +        .drv = "vimc-capture",
>>       },
>>       {
>>           .name = "Raw Capture 1",
>> -        .pads_qty = 1,
>> -        .pads_flag = (const unsigned long[]){MEDIA_PAD_FL_SINK},
>> -        .node = VIMC_ENT_NODE_CAPTURE,
>> +        .drv = "vimc-capture",
>>       },
>>       {
>>           .name = "RGB/YUV Input",
>> -        .pads_qty = 1,
>> -        .pads_flag = (const unsigned long[]){MEDIA_PAD_FL_SOURCE},
>> -        .node = VIMC_ENT_NODE_INPUT,
>> +        /* TODO: change to vimc-output when output is implemented */
>> +        .drv = "vimc-sensor",
>
> This is confusing: the name indicates an input, but the TODO indicates an
> output. I assume this is supposed to emulate e.g. an HDMI input? I'm not
> sure
> the TODO text makes sense here.

You are right, I was confused about input/output in the pov of kernel or 
userspace and I was with HDMI output in mind (I even implement it, I'll 
send in the next patch series).
I'll change this comment to "change to vimc-input when input is implemented"


>
>>       },
>>       {
>>           .name = "Scaler",
>> -        .pads_qty = 2,
>> -        .pads_flag = (const unsigned long[]){MEDIA_PAD_FL_SINK,
>> -                             MEDIA_PAD_FL_SOURCE},
>> -        .node = VIMC_ENT_NODE_SCALER,
>> +        .drv = "vimc-scaler",
>>       },
>>       {
>>           .name = "RGB/YUV Capture",
>> -        .pads_qty = 1,
>> -        .pads_flag = (const unsigned long[]){MEDIA_PAD_FL_SINK},
>> -        .node = VIMC_ENT_NODE_CAPTURE,
>> +        .drv = "vimc-capture",
>>       },
>>   };
>>   @@ -197,111 +147,40 @@ static const struct vimc_pipeline_config
>> pipe_cfg = {
>>     /*
>> --------------------------------------------------------------------------
>> */
>>   -static void vimc_device_unregister(struct vimc_device *vimc)
>> +static int vimc_create_links(struct vimc_device *vimc)
>>   {
>>       unsigned int i;
>> -
>> -    media_device_unregister(&vimc->mdev);
>> -    /* Cleanup (only initialized) entities */
>> -    for (i = 0; i < vimc->pipe_cfg->num_ents; i++) {
>> -        if (vimc->ved[i] && vimc->ved[i]->destroy)
>> -            vimc->ved[i]->destroy(vimc->ved[i]);
>> -
>> -        vimc->ved[i] = NULL;
>> -    }
>> -    v4l2_device_unregister(&vimc->v4l2_dev);
>> -    media_device_cleanup(&vimc->mdev);
>> -}
>> -
>> -/*
>> - * TODO: remove this function when all the
>> - * entities specific code are implemented
>> - */
>> -static void vimc_raw_destroy(struct vimc_ent_device *ved)
>> -{
>> -    media_device_unregister_entity(ved->ent);
>> -
>> -    media_entity_cleanup(ved->ent);
>> -
>> -    vimc_pads_cleanup(ved->pads);
>> -
>> -    kfree(ved->ent);
>> -
>> -    kfree(ved);
>> -}
>> -
>> -/*
>> - * TODO: remove this function when all the
>> - * entities specific code are implemented
>> - */
>> -static struct vimc_ent_device *vimc_raw_create(struct v4l2_device
>> *v4l2_dev,
>> -                           const char *const name,
>> -                           u16 num_pads,
>> -                           const unsigned long *pads_flag)
>> -{
>> -    struct vimc_ent_device *ved;
>>       int ret;
>>   -    /* Allocate the main ved struct */
>> -    ved = kzalloc(sizeof(*ved), GFP_KERNEL);
>> -    if (!ved)
>> -        return ERR_PTR(-ENOMEM);
>> -
>> -    /* Allocate the media entity */
>> -    ved->ent = kzalloc(sizeof(*ved->ent), GFP_KERNEL);
>> -    if (!ved->ent) {
>> -        ret = -ENOMEM;
>> -        goto err_free_ved;
>> -    }
>> -
>> -    /* Allocate the pads */
>> -    ved->pads = vimc_pads_init(num_pads, pads_flag);
>> -    if (IS_ERR(ved->pads)) {
>> -        ret = PTR_ERR(ved->pads);
>> -        goto err_free_ent;
>> +    /* Initialize the links between entities */
>> +    for (i = 0; i < vimc->pipe_cfg->num_links; i++) {
>> +        const struct vimc_ent_link *link = &vimc->pipe_cfg->links[i];
>> +        /*
>> +         * TODO: Check another way of retrieving ved struct without
>> +         * relying on platform_get_drvdata
>> +         */
>> +        struct vimc_ent_device *ved_src =
>> +            platform_get_drvdata(vimc->subdevs[link->src_ent]);
>> +        struct vimc_ent_device *ved_sink =
>> +            platform_get_drvdata(vimc->subdevs[link->sink_ent]);
>> +
>> +        ret = media_create_pad_link(ved_src->ent, link->src_pad,
>> +                        ved_sink->ent, link->sink_pad,
>> +                        link->flags);
>> +        if (ret)
>> +            return ret;
>>       }
>>   -    /* Initialize the media entity */
>> -    ved->ent->name = name;
>> -    ved->ent->function = MEDIA_ENT_F_V4L2_SUBDEV_UNKNOWN;
>> -    ret = media_entity_pads_init(ved->ent, num_pads, ved->pads);
>> -    if (ret)
>> -        goto err_cleanup_pads;
>> -
>> -    /* Register the media entity */
>> -    ret = media_device_register_entity(v4l2_dev->mdev, ved->ent);
>> -    if (ret)
>> -        goto err_cleanup_entity;
>> -
>> -    /* Fill out the destroy function and return */
>> -    ved->destroy = vimc_raw_destroy;
>> -    return ved;
>> -
>> -err_cleanup_entity:
>> -    media_entity_cleanup(ved->ent);
>> -err_cleanup_pads:
>> -    vimc_pads_cleanup(ved->pads);
>> -err_free_ent:
>> -    kfree(ved->ent);
>> -err_free_ved:
>> -    kfree(ved);
>> -
>> -    return ERR_PTR(ret);
>> +    return 0;
>>   }
>>   -static int vimc_device_register(struct vimc_device *vimc)
>> +static int vimc_comp_bind(struct device *master)
>>   {
>> -    unsigned int i;
>> +    struct vimc_device *vimc = container_of(to_platform_device(master),
>> +                        struct vimc_device, pdev);
>>       int ret;
>>   -    /* Allocate memory for the vimc_ent_devices pointers */
>> -    vimc->ved = devm_kcalloc(vimc->mdev.dev, vimc->pipe_cfg->num_ents,
>> -                 sizeof(*vimc->ved), GFP_KERNEL);
>> -    if (!vimc->ved)
>> -        return -ENOMEM;
>> -
>> -    /* Link the media device within the v4l2_device */
>> -    vimc->v4l2_dev.mdev = &vimc->mdev;
>> +    dev_dbg(master, "bind");
>>         /* Register the v4l2 struct */
>>       ret = v4l2_device_register(vimc->mdev.dev, &vimc->v4l2_dev);
>> @@ -311,66 +190,22 @@ static int vimc_device_register(struct
>> vimc_device *vimc)
>>           return ret;
>>       }
>>   -    /* Initialize entities */
>> -    for (i = 0; i < vimc->pipe_cfg->num_ents; i++) {
>> -        struct vimc_ent_device *(*create_func)(struct v4l2_device *,
>> -                               const char *const,
>> -                               u16,
>> -                               const unsigned long *);
>> -
>> -        /* Register the specific node */
>> -        switch (vimc->pipe_cfg->ents[i].node) {
>> -        case VIMC_ENT_NODE_SENSOR:
>> -            create_func = vimc_sen_create;
>> -            break;
>> -
>> -        case VIMC_ENT_NODE_CAPTURE:
>> -            create_func = vimc_cap_create;
>> -            break;
>> -
>> -        /* TODO: Instantiate the specific topology node */
>> -        case VIMC_ENT_NODE_INPUT:
>> -        case VIMC_ENT_NODE_DEBAYER:
>> -        case VIMC_ENT_NODE_SCALER:
>> -        default:
>> -            /*
>> -             * TODO: remove this when all the entities specific
>> -             * code are implemented
>> -             */
>> -            create_func = vimc_raw_create;
>> -            break;
>> -        }
>> -
>> -        vimc->ved[i] = create_func(&vimc->v4l2_dev,
>> -                       vimc->pipe_cfg->ents[i].name,
>> -                       vimc->pipe_cfg->ents[i].pads_qty,
>> -                       vimc->pipe_cfg->ents[i].pads_flag);
>> -        if (IS_ERR(vimc->ved[i])) {
>> -            ret = PTR_ERR(vimc->ved[i]);
>> -            vimc->ved[i] = NULL;
>> -            goto err;
>> -        }
>> -    }
>> -
>> -    /* Initialize the links between entities */
>> -    for (i = 0; i < vimc->pipe_cfg->num_links; i++) {
>> -        const struct vimc_ent_link *link = &vimc->pipe_cfg->links[i];
>> +    /* Bind subdevices */
>> +    ret = component_bind_all(master, &vimc->v4l2_dev);
>> +    if (ret)
>> +        goto err_v4l2_unregister;
>>   -        ret = media_create_pad_link(vimc->ved[link->src_ent]->ent,
>> -                        link->src_pad,
>> -                        vimc->ved[link->sink_ent]->ent,
>> -                        link->sink_pad,
>> -                        link->flags);
>> -        if (ret)
>> -            goto err;
>> -    }
>> +    /* Initialize links */
>> +    ret = vimc_create_links(vimc);
>> +    if (ret)
>> +        goto err_comp_unbind_all;
>>         /* Register the media device */
>>       ret = media_device_register(&vimc->mdev);
>>       if (ret) {
>>           dev_err(vimc->mdev.dev,
>>               "media device register failed (err=%d)\n", ret);
>> -        return ret;
>> +        goto err_comp_unbind_all;
>>       }
>>         /* Expose all subdev's nodes*/
>> @@ -379,32 +214,107 @@ static int vimc_device_register(struct
>> vimc_device *vimc)
>>           dev_err(vimc->mdev.dev,
>>               "vimc subdev nodes registration failed (err=%d)\n",
>>               ret);
>> -        goto err;
>> +        goto err_mdev_unregister;
>>       }
>>         return 0;
>>   -err:
>> -    /* Destroy the so far created topology */
>> -    vimc_device_unregister(vimc);
>> +err_mdev_unregister:
>> +    media_device_unregister(&vimc->mdev);
>> +err_comp_unbind_all:
>> +    component_unbind_all(master, NULL);
>> +err_v4l2_unregister:
>> +    v4l2_device_unregister(&vimc->v4l2_dev);
>>         return ret;
>>   }
>>   +static void vimc_comp_unbind(struct device *master)
>> +{
>> +    struct vimc_device *vimc = container_of(to_platform_device(master),
>> +                        struct vimc_device, pdev);
>> +
>> +    dev_dbg(master, "unbind");
>> +
>> +    media_device_unregister(&vimc->mdev);
>> +    component_unbind_all(master, NULL);
>> +    v4l2_device_unregister(&vimc->v4l2_dev);
>> +}
>> +
>> +static int vimc_comp_compare(struct device *comp, void *data)
>> +{
>> +    const struct platform_device *pdev = to_platform_device(comp);
>> +    const char *name = data;
>> +
>> +    return !strcmp(pdev->dev.platform_data, name);
>> +}
>> +
>> +static struct component_match *vimc_add_subdevs(struct vimc_device
>> *vimc)
>> +{
>> +    struct component_match *match = NULL;
>> +    unsigned int i;
>> +
>> +    for (i = 0; i < vimc->pipe_cfg->num_ents; i++) {
>> +        dev_dbg(&vimc->pdev.dev, "new pdev for %s\n",
>> +            vimc->pipe_cfg->ents[i].drv);
>> +
>> +        /*
>> +         * TODO: check if using platform_data is indeed the best way to
>> +         * pass the name to the driver or if we should add the drv name
>> +         * in the platform_device_id table
>> +         */
>
> Didn't you set the drv name in the platform_device_id table already?

I refer to the name of the entity, there is the name that identifies the 
driver as "vimc-sensor" that is set in the platform_device_id table but 
there is also the custom name of the entity e.g. "My Sensor A" that I 
need to inform to the vimc-sensor driver.

>
> Using platform_data feels like an abuse to be honest.

Another option would be to make the vimc-sensor driver to populate the 
entity name automatically as "Sensor x", where x could be the entity 
number, but I don't think this is a good option.

>
> Creating these components here makes sense. Wouldn't it also make sense
> to use
> v4l2_async to wait until they have all been bound? It would more closely
> emulate
> standard drivers. Apologies if I misunderstand what is happening here.

I am using linux/component.h for that, when all devices are present and 
all modules are loaded, the component.h system brings up the core by 
calling vimc_comp_bind() function, which calls component_bind_all() to 
call the binding function of each module, then it finishes registering 
the topology.
If any one of the components or module is unload, the component system 
takes down the entire topology calling component_unbind_all which calls 
the unbind functions from each module.
This makes sure that the media device, subdevices and video device are 
only registered in the v4l2 system if all the modules are loaded.

I wans't familiar with v4l2-async.h, but from a quick look it seems that 
it only works with struct v4l2_subdev, but I'll also need for struct 
video_device (for the capture node for example).
And also, if a module is missing we would have vimc partially 
registered, e.g. the debayer could be registered at /dev/subdevX but the 
sensor not yet and the media wouldn't be ready, I am not sure if this is 
a problem though.

Maybe we can use component.h for now, then I can implement 
v4l2_async_{un}register_video_device and migrate to v4l2-sync.h latter. 
What do you think?


>
>> +        vimc->subdevs[i] =
>> platform_device_register_data(&vimc->pdev.dev,
>> +                vimc->pipe_cfg->ents[i].drv,
>> +                PLATFORM_DEVID_AUTO,
>> +                vimc->pipe_cfg->ents[i].name,
>> +                strlen(vimc->pipe_cfg->ents[i].name) + 1);
>> +        if (!vimc->subdevs[i]) {
>> +            while (--i >= 0)
>> +                platform_device_unregister(vimc->subdevs[i]);
>> +
>> +            return ERR_PTR(-ENOMEM);
>> +        }
>> +
>> +        component_match_add(&vimc->pdev.dev, &match, vimc_comp_compare,
>> +                    (void *)vimc->pipe_cfg->ents[i].name);
>> +    }
>> +
>> +    return match;
>> +}
>> +
>> +static void vimc_rm_subdevs(struct vimc_device *vimc)
>> +{
>> +    unsigned int i;
>> +
>> +    for (i = 0; i < vimc->pipe_cfg->num_ents; i++)
>> +        platform_device_unregister(vimc->subdevs[i]);
>> +}
>> +
>> +static const struct component_master_ops vimc_comp_ops = {
>> +    .bind = vimc_comp_bind,
>> +    .unbind = vimc_comp_unbind,
>> +};
>> +
>>   static int vimc_probe(struct platform_device *pdev)
>>   {
>> -    struct vimc_device *vimc;
>> +    struct vimc_device *vimc = container_of(pdev, struct vimc_device,
>> pdev);
>> +    struct component_match *match = NULL;
>>       int ret;
>>   -    /* Prepare the vimc topology structure */
>> +    dev_dbg(&pdev->dev, "probe");
>>   -    /* Allocate memory for the vimc structure */
>> -    vimc = kzalloc(sizeof(*vimc), GFP_KERNEL);
>> -    if (!vimc)
>> +    /* Create platform_device for each entity in the topology*/
>> +    vimc->subdevs = devm_kcalloc(&vimc->pdev.dev,
>> vimc->pipe_cfg->num_ents,
>> +                     sizeof(*vimc->subdevs), GFP_KERNEL);
>> +    if (!vimc->subdevs)
>>           return -ENOMEM;
>>   -    /* Set the pipeline configuration struct */
>> -    vimc->pipe_cfg = &pipe_cfg;
>> +    match = vimc_add_subdevs(vimc);
>> +    if (IS_ERR(match))
>> +        return PTR_ERR(match);
>> +
>> +    /* Link the media device within the v4l2_device */
>> +    vimc->v4l2_dev.mdev = &vimc->mdev;
>>         /* Initialize media device */
>>       strlcpy(vimc->mdev.model, VIMC_MDEV_MODEL_NAME,
>> @@ -412,28 +322,27 @@ static int vimc_probe(struct platform_device *pdev)
>>       vimc->mdev.dev = &pdev->dev;
>>       media_device_init(&vimc->mdev);
>>   -    /* Create vimc topology */
>> -    ret = vimc_device_register(vimc);
>> +    /* Add self to the component system */
>> +    ret = component_master_add_with_match(&pdev->dev, &vimc_comp_ops,
>> +                          match);
>>       if (ret) {
>> -        dev_err(vimc->mdev.dev,
>> -            "vimc device registration failed (err=%d)\n", ret);
>> +        media_device_cleanup(&vimc->mdev);
>> +        vimc_rm_subdevs(vimc);
>>           kfree(vimc);
>>           return ret;
>>       }
>>   -    /* Link the topology object with the platform device object */
>> -    platform_set_drvdata(pdev, vimc);
>> -
>>       return 0;
>>   }
>>     static int vimc_remove(struct platform_device *pdev)
>>   {
>> -    struct vimc_device *vimc = platform_get_drvdata(pdev);
>> +    struct vimc_device *vimc = container_of(pdev, struct vimc_device,
>> pdev);
>>   -    /* Destroy all the topology */
>> -    vimc_device_unregister(vimc);
>> -    kfree(vimc);
>> +    dev_dbg(&pdev->dev, "remove");
>> +
>> +    component_master_del(&pdev->dev, &vimc_comp_ops);
>> +    vimc_rm_subdevs(vimc);
>>         return 0;
>>   }
>> @@ -442,9 +351,12 @@ static void vimc_dev_release(struct device *dev)
>>   {
>>   }
>>   -static struct platform_device vimc_pdev = {
>> -    .name        = VIMC_PDEV_NAME,
>> -    .dev.release    = vimc_dev_release,
>> +static struct vimc_device vimc_dev = {
>> +    .pipe_cfg = &pipe_cfg,
>> +    .pdev = {
>> +        .name = VIMC_PDEV_NAME,
>> +        .dev.release = vimc_dev_release,
>> +    }
>>   };
>>     static struct platform_driver vimc_pdrv = {
>> @@ -459,29 +371,29 @@ static int __init vimc_init(void)
>>   {
>>       int ret;
>>   -    ret = platform_device_register(&vimc_pdev);
>> +    ret = platform_device_register(&vimc_dev.pdev);
>>       if (ret) {
>> -        dev_err(&vimc_pdev.dev,
>> +        dev_err(&vimc_dev.pdev.dev,
>>               "platform device registration failed (err=%d)\n", ret);
>>           return ret;
>>       }
>>         ret = platform_driver_register(&vimc_pdrv);
>>       if (ret) {
>> -        dev_err(&vimc_pdev.dev,
>> +        dev_err(&vimc_dev.pdev.dev,
>>               "platform driver registration failed (err=%d)\n", ret);
>> -
>> -        platform_device_unregister(&vimc_pdev);
>> +        platform_driver_unregister(&vimc_pdrv);
>> +        return ret;
>>       }
>>   -    return ret;
>> +    return 0;
>>   }
>>     static void __exit vimc_exit(void)
>>   {
>>       platform_driver_unregister(&vimc_pdrv);
>>   -    platform_device_unregister(&vimc_pdev);
>> +    platform_device_unregister(&vimc_dev.pdev);
>>   }
>>     module_init(vimc_init);
>> diff --git a/drivers/media/platform/vimc/vimc-sensor.c
>> b/drivers/media/platform/vimc/vimc-sensor.c
>> index df89ee7..7f16978 100644
>> --- a/drivers/media/platform/vimc/vimc-sensor.c
>> +++ b/drivers/media/platform/vimc/vimc-sensor.c
>> @@ -15,15 +15,19 @@
>>    *
>>    */
>>   -#include <linux/module.h>
>> +#include <linux/component.h>
>>   #include <linux/freezer.h>
>>   #include <linux/kthread.h>
>> +#include <linux/module.h>
>> +#include <linux/platform_device.h>
>>   #include <linux/v4l2-mediabus.h>
>>   #include <linux/vmalloc.h>
>>   #include <media/v4l2-subdev.h>
>>   #include <media/v4l2-tpg.h>
>>   -#include "vimc-sensor.h"
>> +#include "vimc-common.h"
>> +
>> +#define VIMC_SEN_DRV_NAME "vimc-sensor"
>>     static bool vsen_tpg;
>>   module_param(vsen_tpg, bool, 0000);
>> @@ -35,6 +39,7 @@ MODULE_PARM_DESC(vsen_tpg, " generate image from
>> sensor node\n"
>>   struct vimc_sen_device {
>>       struct vimc_ent_device ved;
>>       struct v4l2_subdev sd;
>> +    struct device *dev;
>>       struct tpg_data tpg;
>>       struct task_struct *kthread_sen;
>>       u8 *frame;
>> @@ -189,7 +194,7 @@ static int vimc_sen_set_fmt(struct v4l2_subdev *sd,
>>       /* Set the new format */
>>       vimc_sen_adjust_fmt(&fmt->format);
>>   -    dev_dbg(vsen->sd.v4l2_dev->mdev->dev, "%s: format update: "
>> +    dev_dbg(vsen->dev, "%s: format update: "
>>           "old:%dx%d (0x%x, %d, %d, %d, %d) "
>>           "new:%dx%d (0x%x, %d, %d, %d, %d)\n", vsen->sd.name,
>>           /* old */
>> @@ -282,8 +287,8 @@ static int vimc_sen_s_stream(struct v4l2_subdev
>> *sd, int enable)
>>           vsen->kthread_sen = kthread_run(vimc_sen_tpg_thread, vsen,
>>                       "%s-sen", vsen->sd.v4l2_dev->name);
>>           if (IS_ERR(vsen->kthread_sen)) {
>> -            dev_err(vsen->sd.v4l2_dev->dev,
>> -                "%s: kernel_thread() failed\n",    vsen->sd.name);
>> +            dev_err(vsen->dev, "%s: kernel_thread() failed\n",
>> +                vsen->sd.name);
>>               vfree(vsen->frame);
>>               vsen->frame = NULL;
>>               return PTR_ERR(vsen->kthread_sen);
>> @@ -315,8 +320,10 @@ static const struct v4l2_subdev_ops vimc_sen_ops = {
>>       .video = &vimc_sen_video_ops,
>>   };
>>   -static void vimc_sen_destroy(struct vimc_ent_device *ved)
>> +static void vimc_sen_comp_unbind(struct device *comp, struct device
>> *master,
>> +                 void *master_data)
>>   {
>> +    struct vimc_ent_device *ved = dev_get_drvdata(comp);
>>       struct vimc_sen_device *vsen =
>>                   container_of(ved, struct vimc_sen_device, ved);
>>   @@ -326,36 +333,30 @@ static void vimc_sen_destroy(struct
>> vimc_ent_device *ved)
>>       kfree(vsen);
>>   }
>>   -struct vimc_ent_device *vimc_sen_create(struct v4l2_device *v4l2_dev,
>> -                    const char *const name,
>> -                    u16 num_pads,
>> -                    const unsigned long *pads_flag)
>> +static int vimc_sen_comp_bind(struct device *comp, struct device
>> *master,
>> +                  void *master_data)
>>   {
>> +    struct v4l2_device *v4l2_dev = master_data;
>> +    char *name = comp->platform_data;
>>       struct vimc_sen_device *vsen;
>> -    unsigned int i;
>>       int ret;
>>   -    /* NOTE: a sensor node may be created with more then one pad */
>> -    if (!name || !num_pads || !pads_flag)
>> -        return ERR_PTR(-EINVAL);
>> -
>> -    /* check if all pads are sources */
>> -    for (i = 0; i < num_pads; i++)
>> -        if (!(pads_flag[i] & MEDIA_PAD_FL_SOURCE))
>> -            return ERR_PTR(-EINVAL);
>> -
>>       /* Allocate the vsen struct */
>>       vsen = kzalloc(sizeof(*vsen), GFP_KERNEL);
>>       if (!vsen)
>> -        return ERR_PTR(-ENOMEM);
>> +        return -ENOMEM;
>>         /* Initialize ved and sd */
>>       ret = vimc_ent_sd_register(&vsen->ved, &vsen->sd, v4l2_dev, name,
>> -                   MEDIA_ENT_F_CAM_SENSOR, num_pads, pads_flag,
>> -                   &vimc_sen_ops, vimc_sen_destroy);
>> +                   MEDIA_ENT_F_ATV_DECODER, 1,
>> +                   (const unsigned long[1]) {MEDIA_PAD_FL_SOURCE},
>> +                   &vimc_sen_ops);
>>       if (ret)
>>           goto err_free_vsen;
>>   +    dev_set_drvdata(comp, &vsen->ved);
>> +    vsen->dev = comp;
>> +
>>       /* Initialize the frame format */
>>       vsen->mbus_format = fmt_default;
>>   @@ -368,12 +369,52 @@ struct vimc_ent_device *vimc_sen_create(struct
>> v4l2_device *v4l2_dev,
>>               goto err_unregister_ent_sd;
>>       }
>>   -    return &vsen->ved;
>> +    return 0;
>>     err_unregister_ent_sd:
>>       vimc_ent_sd_unregister(&vsen->ved,  &vsen->sd);
>>   err_free_vsen:
>>       kfree(vsen);
>>   -    return ERR_PTR(ret);
>> +    return ret;
>> +}
>> +
>> +static const struct component_ops vimc_sen_comp_ops = {
>> +    .bind = vimc_sen_comp_bind,
>> +    .unbind = vimc_sen_comp_unbind,
>> +};
>> +
>> +static int vimc_sen_probe(struct platform_device *pdev)
>> +{
>> +    return component_add(&pdev->dev, &vimc_sen_comp_ops);
>>   }
>> +
>> +static int vimc_sen_remove(struct platform_device *pdev)
>> +{
>> +    component_del(&pdev->dev, &vimc_sen_comp_ops);
>> +
>> +    return 0;
>> +}
>> +
>> +static struct platform_driver vimc_sen_pdrv = {
>> +    .probe        = vimc_sen_probe,
>> +    .remove        = vimc_sen_remove,
>> +    .driver        = {
>> +        .name    = VIMC_SEN_DRV_NAME,
>> +    },
>> +};
>> +
>> +static const struct platform_device_id vimc_sen_driver_ids[] = {
>> +    {
>> +        .name           = VIMC_SEN_DRV_NAME,
>> +    },
>> +    { }
>> +};
>> +
>> +module_platform_driver(vimc_sen_pdrv);
>> +
>> +MODULE_DEVICE_TABLE(platform, vimc_sen_driver_ids);
>> +
>> +MODULE_DESCRIPTION("Virtual Media Controller Driver (VIMC) Sensor");
>> +MODULE_AUTHOR("Helen Mae Koike Fornazier <helen.fornazier@gmail.com>");
>> +MODULE_LICENSE("GPL");
>> diff --git a/drivers/media/platform/vimc/vimc-sensor.h
>> b/drivers/media/platform/vimc/vimc-sensor.h
>> deleted file mode 100644
>> index 580dcec..0000000
>> --- a/drivers/media/platform/vimc/vimc-sensor.h
>> +++ /dev/null
>> @@ -1,28 +0,0 @@
>> -/*
>> - * vimc-sensor.h Virtual Media Controller Driver
>> - *
>> - * Copyright (C) 2015-2017 Helen Koike <helen.fornazier@gmail.com>
>> - *
>> - * This program is free software; you can redistribute it and/or modify
>> - * it under the terms of the GNU General Public License as published by
>> - * the Free Software Foundation; either version 2 of the License, or
>> - * (at your option) any later version.
>> - *
>> - * This program is distributed in the hope that it will be useful,
>> - * but WITHOUT ANY WARRANTY; without even the implied warranty of
>> - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
>> - * GNU General Public License for more details.
>> - *
>> - */
>> -
>> -#ifndef _VIMC_SENSOR_H_
>> -#define _VIMC_SENSOR_H_
>> -
>> -#include "vimc-common.h"
>> -
>> -struct vimc_ent_device *vimc_sen_create(struct v4l2_device *v4l2_dev,
>> -                    const char *const name,
>> -                    u16 num_pads,
>> -                    const unsigned long *pads_flag);
>> -
>> -#endif
>>
>
> Regards,
>
>     Hans

Thanks
Helen

Re: [RFC PATCH v3 09/11] [media] vimc: Subdevices as modules

[Hans Verkuil]

On 06/12/2017 10:35 PM, Helen Koike wrote:
> Hi Hans,
> 
> Thank you for your review. Please check my comments below
> 
> On 2017-06-12 07:37 AM, Hans Verkuil wrote:
>> On 06/03/2017 04:58 AM, Helen Koike wrote:
>>> +static struct component_match *vimc_add_subdevs(struct vimc_device
>>> *vimc)
>>> +{
>>> +    struct component_match *match = NULL;
>>> +    unsigned int i;
>>> +
>>> +    for (i = 0; i < vimc->pipe_cfg->num_ents; i++) {
>>> +        dev_dbg(&vimc->pdev.dev, "new pdev for %s\n",
>>> +            vimc->pipe_cfg->ents[i].drv);
>>> +
>>> +        /*
>>> +         * TODO: check if using platform_data is indeed the best way to
>>> +         * pass the name to the driver or if we should add the drv name
>>> +         * in the platform_device_id table
>>> +         */
>>
>> Didn't you set the drv name in the platform_device_id table already?
> 
> I refer to the name of the entity, there is the name that identifies the
> driver as "vimc-sensor" that is set in the platform_device_id table but
> there is also the custom name of the entity e.g. "My Sensor A" that I
> need to inform to the vimc-sensor driver.

Ah, so in the TODO you mean:

"the best way to pass the entity name to the driver"

I got confused there.

But in that case I still don't get what you mean with "add the drv name
in the platform_device_id table". Do you mean "entity name" there as
well?

> 
>>
>> Using platform_data feels like an abuse to be honest.
> 
> Another option would be to make the vimc-sensor driver to populate the
> entity name automatically as "Sensor x", where x could be the entity
> number, but I don't think this is a good option.

Why not? Well, probably not the entity number, but a simple instance
counter would do fine (i.e. Sensor 1, 2, 3...).

It can be made fancier later with dynamic reconfiguration where you
might want to use the first unused instance number.

> 
>>
>> Creating these components here makes sense. Wouldn't it also make sense
>> to use
>> v4l2_async to wait until they have all been bound? It would more closely
>> emulate
>> standard drivers. Apologies if I misunderstand what is happening here.
> 
> I am using linux/component.h for that, when all devices are present and
> all modules are loaded, the component.h system brings up the core by
> calling vimc_comp_bind() function, which calls component_bind_all() to
> call the binding function of each module, then it finishes registering
> the topology.
> If any one of the components or module is unload, the component system
> takes down the entire topology calling component_unbind_all which calls
> the unbind functions from each module.
> This makes sure that the media device, subdevices and video device are
> only registered in the v4l2 system if all the modules are loaded.
> 
> I wans't familiar with v4l2-async.h, but from a quick look it seems that
> it only works with struct v4l2_subdev, but I'll also need for struct
> video_device (for the capture node for example).
> And also, if a module is missing we would have vimc partially
> registered, e.g. the debayer could be registered at /dev/subdevX but the
> sensor not yet and the media wouldn't be ready, I am not sure if this is
> a problem though.
> 
> Maybe we can use component.h for now, then I can implement
> v4l2_async_{un}register_video_device and migrate to v4l2-sync.h latter.
> What do you think?

That's OK. The v4l2-async mechanism precedes the component API. We should
probably investigate moving over to the component API. I seem to remember
that it didn't have all the features we needed, but it's a long time ago
since someone looked at that and whatever the objections were, they may
no longer be true.

Regards,

	Hans

Re: [RFC PATCH v3 09/11] [media] vimc: Subdevices as modules

[Helen Koike]

Hi Hans,

On 2017-06-13 03:49 AM, Hans Verkuil wrote:
> On 06/12/2017 10:35 PM, Helen Koike wrote:
>> Hi Hans,
>>
>> Thank you for your review. Please check my comments below
>>
>> On 2017-06-12 07:37 AM, Hans Verkuil wrote:
>>> On 06/03/2017 04:58 AM, Helen Koike wrote:
>>>> +static struct component_match *vimc_add_subdevs(struct vimc_device
>>>> *vimc)
>>>> +{
>>>> +    struct component_match *match = NULL;
>>>> +    unsigned int i;
>>>> +
>>>> +    for (i = 0; i < vimc->pipe_cfg->num_ents; i++) {
>>>> +        dev_dbg(&vimc->pdev.dev, "new pdev for %s\n",
>>>> +            vimc->pipe_cfg->ents[i].drv);
>>>> +
>>>> +        /*
>>>> +         * TODO: check if using platform_data is indeed the best
>>>> way to
>>>> +         * pass the name to the driver or if we should add the drv
>>>> name
>>>> +         * in the platform_device_id table
>>>> +         */
>>>
>>> Didn't you set the drv name in the platform_device_id table already?
>>
>> I refer to the name of the entity, there is the name that identifies the
>> driver as "vimc-sensor" that is set in the platform_device_id table but
>> there is also the custom name of the entity e.g. "My Sensor A" that I
>> need to inform to the vimc-sensor driver.
>
> Ah, so in the TODO you mean:
>
> "the best way to pass the entity name to the driver"

Yes

>
> I got confused there.

Sorry about that, I'll improve this comment (if we don't decide to 
remove it)

>
> But in that case I still don't get what you mean with "add the drv name
> in the platform_device_id table". Do you mean "entity name" there as
> well?

Yes, it is because there is a member of the platform_device_id table 
called driver_data and I was wondering if this would be more 
appropriated then using the platform_data.

Under Documentation/driver-model/platform.txt it is written:
"In many cases, the memory and IRQ resources associated with the 
platform device are not enough to let the device's driver work.  Board 
setup code
will often provide additional information using the device's platform_data
field to hold additional information."

So I thought that using platform_data to pass the entity's name to the 
driver would be acceptable as it seems to be a "Board setup code" for vimc

>
>>
>>>
>>> Using platform_data feels like an abuse to be honest.
>>
>> Another option would be to make the vimc-sensor driver to populate the
>> entity name automatically as "Sensor x", where x could be the entity
>> number, but I don't think this is a good option.
>
> Why not? Well, probably not the entity number, but a simple instance
> counter would do fine (i.e. Sensor 1, 2, 3...).

Because I usually use tests scripts that configure the right pad image 
format to a specific entity's name using media-ctl, I would like an 
assurance that what identifies the entity doesn't change (that doesn't 
depend on the order they are inserted), and I thought that applications 
in userspace might want the same.

>
> It can be made fancier later with dynamic reconfiguration where you
> might want to use the first unused instance number.

We could use configfs for that, but I was wondering what the names of 
the folders that represent an entity would mean, if the user do
$ mkdir vimc-sensor-foo
$ mkdir vimc-sensor-bar
Then foo and bar would unused names as the entities would be first 
created under the names "Sensor 1" and "Sensor 2", I think it would be 
nice if they were created as "Sensor foo" and "Sensor bar".

>
>>
>>>
>>> Creating these components here makes sense. Wouldn't it also make sense
>>> to use
>>> v4l2_async to wait until they have all been bound? It would more closely
>>> emulate
>>> standard drivers. Apologies if I misunderstand what is happening here.
>>
>> I am using linux/component.h for that, when all devices are present and
>> all modules are loaded, the component.h system brings up the core by
>> calling vimc_comp_bind() function, which calls component_bind_all() to
>> call the binding function of each module, then it finishes registering
>> the topology.
>> If any one of the components or module is unload, the component system
>> takes down the entire topology calling component_unbind_all which calls
>> the unbind functions from each module.
>> This makes sure that the media device, subdevices and video device are
>> only registered in the v4l2 system if all the modules are loaded.
>>
>> I wans't familiar with v4l2-async.h, but from a quick look it seems that
>> it only works with struct v4l2_subdev, but I'll also need for struct
>> video_device (for the capture node for example).
>> And also, if a module is missing we would have vimc partially
>> registered, e.g. the debayer could be registered at /dev/subdevX but the
>> sensor not yet and the media wouldn't be ready, I am not sure if this is
>> a problem though.
>>
>> Maybe we can use component.h for now, then I can implement
>> v4l2_async_{un}register_video_device and migrate to v4l2-sync.h latter.
>> What do you think?
>
> That's OK. The v4l2-async mechanism precedes the component API. We should
> probably investigate moving over to the component API. I seem to remember
> that it didn't have all the features we needed, but it's a long time ago
> since someone looked at that and whatever the objections were, they may
> no longer be true.

I see, I can try to take a look on this.

>
> Regards,
>
>     Hans

Thanks
Helen

Re: [RFC PATCH v3 09/11] [media] vimc: Subdevices as modules

[Hans Verkuil]

On 06/13/17 15:23, Helen Koike wrote:
> Hi Hans,
> 
> On 2017-06-13 03:49 AM, Hans Verkuil wrote:
>> On 06/12/2017 10:35 PM, Helen Koike wrote:
>>> Hi Hans,
>>>
>>> Thank you for your review. Please check my comments below
>>>
>>> On 2017-06-12 07:37 AM, Hans Verkuil wrote:
>>>> On 06/03/2017 04:58 AM, Helen Koike wrote:
>>>>> +static struct component_match *vimc_add_subdevs(struct vimc_device
>>>>> *vimc)
>>>>> +{
>>>>> +    struct component_match *match = NULL;
>>>>> +    unsigned int i;
>>>>> +
>>>>> +    for (i = 0; i < vimc->pipe_cfg->num_ents; i++) {
>>>>> +        dev_dbg(&vimc->pdev.dev, "new pdev for %s\n",
>>>>> +            vimc->pipe_cfg->ents[i].drv);
>>>>> +
>>>>> +        /*
>>>>> +         * TODO: check if using platform_data is indeed the best
>>>>> way to
>>>>> +         * pass the name to the driver or if we should add the drv
>>>>> name
>>>>> +         * in the platform_device_id table
>>>>> +         */
>>>>
>>>> Didn't you set the drv name in the platform_device_id table already?
>>>
>>> I refer to the name of the entity, there is the name that identifies the
>>> driver as "vimc-sensor" that is set in the platform_device_id table but
>>> there is also the custom name of the entity e.g. "My Sensor A" that I
>>> need to inform to the vimc-sensor driver.
>>
>> Ah, so in the TODO you mean:
>>
>> "the best way to pass the entity name to the driver"
> 
> Yes
> 
>>
>> I got confused there.
> 
> Sorry about that, I'll improve this comment (if we don't decide to 
> remove it)
> 
>>
>> But in that case I still don't get what you mean with "add the drv name
>> in the platform_device_id table". Do you mean "entity name" there as
>> well?
> 
> Yes, it is because there is a member of the platform_device_id table 
> called driver_data and I was wondering if this would be more 
> appropriated then using the platform_data.
> 
> Under Documentation/driver-model/platform.txt it is written:
> "In many cases, the memory and IRQ resources associated with the 
> platform device are not enough to let the device's driver work.  Board 
> setup code
> will often provide additional information using the device's platform_data
> field to hold additional information."
> 
> So I thought that using platform_data to pass the entity's name to the 
> driver would be acceptable as it seems to be a "Board setup code" for vimc

OK, now I understand. I am OK with this, but I would prefer if you made a
proper struct for this:

struct vimc_platform_data {
	char entity_name[32];
};

Then point platform_data to that. Also comment clearly why you do it.

Part of the confusion comes from the fact that this is unusual for MC
devices: they all use the device tree instead of platform_data, so seeing
platform_data being used in something like this is unexpected.

> 
>>
>>>
>>>>
>>>> Using platform_data feels like an abuse to be honest.
>>>
>>> Another option would be to make the vimc-sensor driver to populate the
>>> entity name automatically as "Sensor x", where x could be the entity
>>> number, but I don't think this is a good option.
>>
>> Why not? Well, probably not the entity number, but a simple instance
>> counter would do fine (i.e. Sensor 1, 2, 3...).
> 
> Because I usually use tests scripts that configure the right pad image 
> format to a specific entity's name using media-ctl, I would like an 
> assurance that what identifies the entity doesn't change (that doesn't 
> depend on the order they are inserted), and I thought that applications 
> in userspace might want the same.

Good point!

> 
>>
>> It can be made fancier later with dynamic reconfiguration where you
>> might want to use the first unused instance number.
> 
> We could use configfs for that, but I was wondering what the names of 
> the folders that represent an entity would mean, if the user do
> $ mkdir vimc-sensor-foo
> $ mkdir vimc-sensor-bar
> Then foo and bar would unused names as the entities would be first 
> created under the names "Sensor 1" and "Sensor 2", I think it would be 
> nice if they were created as "Sensor foo" and "Sensor bar".
> 
>>
>>>
>>>>
>>>> Creating these components here makes sense. Wouldn't it also make sense
>>>> to use
>>>> v4l2_async to wait until they have all been bound? It would more closely
>>>> emulate
>>>> standard drivers. Apologies if I misunderstand what is happening here.
>>>
>>> I am using linux/component.h for that, when all devices are present and
>>> all modules are loaded, the component.h system brings up the core by
>>> calling vimc_comp_bind() function, which calls component_bind_all() to
>>> call the binding function of each module, then it finishes registering
>>> the topology.
>>> If any one of the components or module is unload, the component system
>>> takes down the entire topology calling component_unbind_all which calls
>>> the unbind functions from each module.
>>> This makes sure that the media device, subdevices and video device are
>>> only registered in the v4l2 system if all the modules are loaded.
>>>
>>> I wans't familiar with v4l2-async.h, but from a quick look it seems that
>>> it only works with struct v4l2_subdev, but I'll also need for struct
>>> video_device (for the capture node for example).
>>> And also, if a module is missing we would have vimc partially
>>> registered, e.g. the debayer could be registered at /dev/subdevX but the
>>> sensor not yet and the media wouldn't be ready, I am not sure if this is
>>> a problem though.
>>>
>>> Maybe we can use component.h for now, then I can implement
>>> v4l2_async_{un}register_video_device and migrate to v4l2-sync.h latter.
>>> What do you think?
>>
>> That's OK. The v4l2-async mechanism precedes the component API. We should
>> probably investigate moving over to the component API. I seem to remember
>> that it didn't have all the features we needed, but it's a long time ago
>> since someone looked at that and whatever the objections were, they may
>> no longer be true.
> 
> I see, I can try to take a look on this.

Something for later.

So in other words, this looks good but needs better comments and a proper
platform_data struct.

Regards,

	Hans

[RFC PATCH v3 10/11] [media] vimc: deb: Add debayer filter

[Helen Koike]

Implement the debayer filter and integrate it with the core

Signed-off-by: Helen Koike <helen.koike@collabora.com>

---

Changes in v3:
[media] vimc: deb: Add debayer filter
	- Declare frame_size as a local variable
	- s_stream(sd, 1): return 0 if stream is already enabled
	- s_stream(sd, 0): return 0 if stream is already disabled
	- s_stream: add ret variable to propagate return errors
	- structure code to be a module, use platform_driver and component system
	- fix multiline comment
	- s/thought/through
	- s/RGB8888/RGB888
	- clamp height and width of the image by an even value
	- if user try to set colorspace to an invalid format, set all
        colorimetry parameters to _DEFAULT
	- uset _DEFAULT for colorimetry in the default format

Changes in v2:
[media] vimc: deb: Add debayer filter
	- Using MEDIA_ENT_F_ATV_DECODER in function
	- remove v4l2_dev and dev from vimc_deb_device struct
	- src fmt propagates from the sink
	- coding style
	- remove redundant else if statements
	- check end of enum and remove BUG_ON
	- enum frame size with min and max values
	- set/try fmt
	- remove unecessary include freezer.h
	- check pad types on create
	- return EBUSY when trying to set the format while stream is on
	- remove vsd struct
	- add IS_SRC and IS_SINK macros
	- add deb_mean_win_size as a parameter of the module
	- check set_fmt default parameters for quantization, colorspace ...
	- add more dev_dbg


---
 drivers/media/platform/vimc/Makefile       |   4 +-
 drivers/media/platform/vimc/vimc-common.h  |   2 +
 drivers/media/platform/vimc/vimc-debayer.c | 615 +++++++++++++++++++++++++++++
 3 files changed, 620 insertions(+), 1 deletion(-)
 create mode 100644 drivers/media/platform/vimc/vimc-debayer.c

diff --git a/drivers/media/platform/vimc/Makefile b/drivers/media/platform/vimc/Makefile
index 0e5d5ce..4fba8ef 100644
--- a/drivers/media/platform/vimc/Makefile
+++ b/drivers/media/platform/vimc/Makefile
@@ -1,6 +1,8 @@
 vimc-objs := vimc-core.o
 vimc_capture-objs := vimc-capture.o
 vimc_common-objs := vimc-common.o
+vimc_debayer-objs := vimc-debayer.o
 vimc_sensor-objs := vimc-sensor.o
 
-obj-$(CONFIG_VIDEO_VIMC) += vimc.o vimc_capture.o vimc_common.o vimc_sensor.o
+obj-$(CONFIG_VIDEO_VIMC) += vimc.o vimc_capture.o vimc_common.o vimc-debayer.o \
+				vimc_sensor.o
diff --git a/drivers/media/platform/vimc/vimc-common.h b/drivers/media/platform/vimc/vimc-common.h
index 27c9b8c..c63d51f 100644
--- a/drivers/media/platform/vimc/vimc-common.h
+++ b/drivers/media/platform/vimc/vimc-common.h
@@ -29,6 +29,8 @@
 
 #define VIMC_PIPE_OPT 1
 
+#define VIMC_FRAME_INDEX(lin, col, width, bpp) ((lin * width + col) * bpp)
+
 /**
  * struct vimc_pix_map - maps media bus code with v4l2 pixel format
  *
diff --git a/drivers/media/platform/vimc/vimc-debayer.c b/drivers/media/platform/vimc/vimc-debayer.c
new file mode 100644
index 0000000..9f9604c
--- /dev/null
+++ b/drivers/media/platform/vimc/vimc-debayer.c
@@ -0,0 +1,615 @@
+/*
+ * vimc-debayer.c Virtual Media Controller Driver
+ *
+ * Copyright (C) 2015-2017 Helen Koike <helen.fornazier@gmail.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ */
+
+#include <linux/component.h>
+#include <linux/module.h>
+#include <linux/platform_device.h>
+#include <linux/vmalloc.h>
+#include <linux/v4l2-mediabus.h>
+#include <media/v4l2-subdev.h>
+
+#include "vimc-common.h"
+
+#define VIMC_DEB_DRV_NAME "vimc-debayer"
+
+static unsigned int deb_mean_win_size = 3;
+module_param(deb_mean_win_size, uint, 0000);
+MODULE_PARM_DESC(deb_mean_win_size, " the window size to calculate the mean.\n"
+	"NOTE: the window size need to be an odd number, as the main pixel "
+	"stays in the center of the window, otherwise the next odd number "
+	"is considered");
+
+#define IS_SINK(pad) (!pad)
+#define IS_SRC(pad)  (pad)
+
+enum vimc_deb_rgb_colors {
+	VIMC_DEB_RED = 0,
+	VIMC_DEB_GREEN = 1,
+	VIMC_DEB_BLUE = 2,
+};
+
+struct vimc_deb_pix_map {
+	u32 code;
+	enum vimc_deb_rgb_colors order[2][2];
+};
+
+struct vimc_deb_device {
+	struct vimc_ent_device ved;
+	struct v4l2_subdev sd;
+	struct device *dev;
+	/* The active format */
+	struct v4l2_mbus_framefmt sink_fmt;
+	u32 src_code;
+	void (*set_rgb_src)(struct vimc_deb_device *vdeb, unsigned int lin,
+			    unsigned int col, unsigned int rgb[3]);
+	/* Values calculated when the stream starts */
+	u8 *src_frame;
+	const struct vimc_deb_pix_map *sink_pix_map;
+	unsigned int sink_bpp;
+};
+
+static const struct v4l2_mbus_framefmt sink_fmt_default = {
+	.width = 640,
+	.height = 480,
+	.code = MEDIA_BUS_FMT_RGB888_1X24,
+	.field = V4L2_FIELD_NONE,
+	.colorspace = V4L2_COLORSPACE_SRGB,
+};
+
+static const struct vimc_deb_pix_map vimc_deb_pix_map_list[] = {
+	{
+		.code = MEDIA_BUS_FMT_SBGGR8_1X8,
+		.order = { { VIMC_DEB_BLUE, VIMC_DEB_GREEN },
+			   { VIMC_DEB_GREEN, VIMC_DEB_RED } }
+	},
+	{
+		.code = MEDIA_BUS_FMT_SGBRG8_1X8,
+		.order = { { VIMC_DEB_GREEN, VIMC_DEB_BLUE },
+			   { VIMC_DEB_RED, VIMC_DEB_GREEN } }
+	},
+	{
+		.code = MEDIA_BUS_FMT_SGRBG8_1X8,
+		.order = { { VIMC_DEB_GREEN, VIMC_DEB_RED },
+			   { VIMC_DEB_BLUE, VIMC_DEB_GREEN } }
+	},
+	{
+		.code = MEDIA_BUS_FMT_SRGGB8_1X8,
+		.order = { { VIMC_DEB_RED, VIMC_DEB_GREEN },
+			   { VIMC_DEB_GREEN, VIMC_DEB_BLUE } }
+	},
+	{
+		.code = MEDIA_BUS_FMT_SBGGR10_1X10,
+		.order = { { VIMC_DEB_BLUE, VIMC_DEB_GREEN },
+			   { VIMC_DEB_GREEN, VIMC_DEB_RED } }
+	},
+	{
+		.code = MEDIA_BUS_FMT_SGBRG10_1X10,
+		.order = { { VIMC_DEB_GREEN, VIMC_DEB_BLUE },
+			   { VIMC_DEB_RED, VIMC_DEB_GREEN } }
+	},
+	{
+		.code = MEDIA_BUS_FMT_SGRBG10_1X10,
+		.order = { { VIMC_DEB_GREEN, VIMC_DEB_RED },
+			   { VIMC_DEB_BLUE, VIMC_DEB_GREEN } }
+	},
+	{
+		.code = MEDIA_BUS_FMT_SRGGB10_1X10,
+		.order = { { VIMC_DEB_RED, VIMC_DEB_GREEN },
+			   { VIMC_DEB_GREEN, VIMC_DEB_BLUE } }
+	},
+	{
+		.code = MEDIA_BUS_FMT_SBGGR12_1X12,
+		.order = { { VIMC_DEB_BLUE, VIMC_DEB_GREEN },
+			   { VIMC_DEB_GREEN, VIMC_DEB_RED } }
+	},
+	{
+		.code = MEDIA_BUS_FMT_SGBRG12_1X12,
+		.order = { { VIMC_DEB_GREEN, VIMC_DEB_BLUE },
+			   { VIMC_DEB_RED, VIMC_DEB_GREEN } }
+	},
+	{
+		.code = MEDIA_BUS_FMT_SGRBG12_1X12,
+		.order = { { VIMC_DEB_GREEN, VIMC_DEB_RED },
+			   { VIMC_DEB_BLUE, VIMC_DEB_GREEN } }
+	},
+	{
+		.code = MEDIA_BUS_FMT_SRGGB12_1X12,
+		.order = { { VIMC_DEB_RED, VIMC_DEB_GREEN },
+			   { VIMC_DEB_GREEN, VIMC_DEB_BLUE } }
+	},
+};
+
+static const struct vimc_deb_pix_map *vimc_deb_pix_map_by_code(u32 code)
+{
+	unsigned int i;
+
+	for (i = 0; i < ARRAY_SIZE(vimc_deb_pix_map_list); i++)
+		if (vimc_deb_pix_map_list[i].code == code)
+			return &vimc_deb_pix_map_list[i];
+
+	return NULL;
+}
+
+static int vimc_deb_init_cfg(struct v4l2_subdev *sd,
+			     struct v4l2_subdev_pad_config *cfg)
+{
+	struct vimc_deb_device *vdeb = v4l2_get_subdevdata(sd);
+	struct v4l2_mbus_framefmt *mf;
+	unsigned int i;
+
+	mf = v4l2_subdev_get_try_format(sd, cfg, 0);
+	*mf = sink_fmt_default;
+
+	for (i = 1; i < sd->entity.num_pads; i++) {
+		mf = v4l2_subdev_get_try_format(sd, cfg, i);
+		*mf = sink_fmt_default;
+		mf->code = vdeb->src_code;
+	}
+
+	return 0;
+}
+
+static int vimc_deb_enum_mbus_code(struct v4l2_subdev *sd,
+				   struct v4l2_subdev_pad_config *cfg,
+				   struct v4l2_subdev_mbus_code_enum *code)
+{
+	/* We only support one format for source pads */
+	if (IS_SRC(code->pad)) {
+		struct vimc_deb_device *vdeb = v4l2_get_subdevdata(sd);
+
+		if (code->index)
+			return -EINVAL;
+
+		code->code = vdeb->src_code;
+	} else {
+		if (code->index >= ARRAY_SIZE(vimc_deb_pix_map_list))
+			return -EINVAL;
+
+		code->code = vimc_deb_pix_map_list[code->index].code;
+	}
+
+	return 0;
+}
+
+static int vimc_deb_enum_frame_size(struct v4l2_subdev *sd,
+				    struct v4l2_subdev_pad_config *cfg,
+				    struct v4l2_subdev_frame_size_enum *fse)
+{
+	struct vimc_deb_device *vdeb = v4l2_get_subdevdata(sd);
+
+	if (fse->index)
+		return -EINVAL;
+
+	if (IS_SINK(fse->pad)) {
+		const struct vimc_deb_pix_map *vpix =
+			vimc_deb_pix_map_by_code(fse->code);
+
+		if (!vpix)
+			return -EINVAL;
+	} else if (fse->code != vdeb->src_code) {
+		return -EINVAL;
+	}
+
+	fse->min_width = VIMC_FRAME_MIN_WIDTH;
+	fse->max_width = VIMC_FRAME_MAX_WIDTH;
+	fse->min_height = VIMC_FRAME_MIN_HEIGHT;
+	fse->max_height = VIMC_FRAME_MAX_HEIGHT;
+
+	return 0;
+}
+
+static int vimc_deb_get_fmt(struct v4l2_subdev *sd,
+			    struct v4l2_subdev_pad_config *cfg,
+			    struct v4l2_subdev_format *fmt)
+{
+	struct vimc_deb_device *vdeb = v4l2_get_subdevdata(sd);
+
+	/* Get the current sink format */
+	fmt->format = fmt->which == V4L2_SUBDEV_FORMAT_TRY ?
+		      *v4l2_subdev_get_try_format(sd, cfg, 0) :
+		      vdeb->sink_fmt;
+
+	/* Set the right code for the source pad */
+	if (IS_SRC(fmt->pad))
+		fmt->format.code = vdeb->src_code;
+
+	return 0;
+}
+
+static void vimc_deb_adjust_sink_fmt(struct v4l2_mbus_framefmt *fmt)
+{
+	const struct vimc_deb_pix_map *vpix;
+
+	/* Don't accept a code that is not on the debayer table */
+	vpix = vimc_deb_pix_map_by_code(fmt->code);
+	if (!vpix)
+		fmt->code = sink_fmt_default.code;
+
+	fmt->width = clamp_t(u32, fmt->width, VIMC_FRAME_MIN_WIDTH,
+			     VIMC_FRAME_MAX_WIDTH) & ~1;
+	fmt->height = clamp_t(u32, fmt->height, VIMC_FRAME_MIN_HEIGHT,
+			      VIMC_FRAME_MAX_HEIGHT) & ~1;
+
+	if (fmt->field == V4L2_FIELD_ANY)
+		fmt->field = sink_fmt_default.field;
+
+	if (fmt->colorspace == V4L2_COLORSPACE_DEFAULT)
+		fmt->colorspace = sink_fmt_default.colorspace;
+
+	/* Check if values are out of range */
+	if (fmt->colorspace > V4L2_COLORSPACE_DCI_P3) {
+		fmt->colorspace = sink_fmt_default.colorspace;
+		fmt->ycbcr_enc = V4L2_YCBCR_ENC_DEFAULT;
+		fmt->quantization = V4L2_QUANTIZATION_DEFAULT;
+		fmt->xfer_func = V4L2_XFER_FUNC_DEFAULT;
+	}
+	if (fmt->ycbcr_enc > V4L2_YCBCR_ENC_SMPTE240M)
+		fmt->ycbcr_enc = V4L2_YCBCR_ENC_DEFAULT;
+	if (fmt->quantization > V4L2_QUANTIZATION_LIM_RANGE)
+		fmt->quantization = V4L2_QUANTIZATION_DEFAULT;
+	if (fmt->xfer_func > V4L2_XFER_FUNC_SMPTE2084)
+		fmt->xfer_func = V4L2_XFER_FUNC_DEFAULT;
+}
+
+static int vimc_deb_set_fmt(struct v4l2_subdev *sd,
+			    struct v4l2_subdev_pad_config *cfg,
+			    struct v4l2_subdev_format *fmt)
+{
+	struct vimc_deb_device *vdeb = v4l2_get_subdevdata(sd);
+	struct v4l2_mbus_framefmt *sink_fmt;
+
+	if (fmt->which == V4L2_SUBDEV_FORMAT_ACTIVE) {
+		/* Do not change the format while stream is on */
+		if (vdeb->src_frame)
+			return -EBUSY;
+
+		sink_fmt = &vdeb->sink_fmt;
+	} else {
+		sink_fmt = v4l2_subdev_get_try_format(sd, cfg, 0);
+	}
+
+	/*
+	 * Do not change the format of the source pad,
+	 * it is propagated from the sink
+	 */
+	if (IS_SRC(fmt->pad)) {
+		fmt->format = *sink_fmt;
+		/* TODO: Add support for other formats */
+		fmt->format.code = vdeb->src_code;
+	} else {
+		/* Set the new format in the sink pad */
+		vimc_deb_adjust_sink_fmt(&fmt->format);
+
+		dev_dbg(vdeb->dev, "%s: sink format update: "
+			"old:%dx%d (0x%x, %d, %d, %d, %d) "
+			"new:%dx%d (0x%x, %d, %d, %d, %d)\n", vdeb->sd.name,
+			/* old */
+			sink_fmt->width, sink_fmt->height, sink_fmt->code,
+			sink_fmt->colorspace, sink_fmt->quantization,
+			sink_fmt->xfer_func, sink_fmt->ycbcr_enc,
+			/* new */
+			fmt->format.width, fmt->format.height, fmt->format.code,
+			fmt->format.colorspace,	fmt->format.quantization,
+			fmt->format.xfer_func, fmt->format.ycbcr_enc);
+
+		*sink_fmt = fmt->format;
+	}
+
+	return 0;
+}
+
+static const struct v4l2_subdev_pad_ops vimc_deb_pad_ops = {
+	.init_cfg		= vimc_deb_init_cfg,
+	.enum_mbus_code		= vimc_deb_enum_mbus_code,
+	.enum_frame_size	= vimc_deb_enum_frame_size,
+	.get_fmt		= vimc_deb_get_fmt,
+	.set_fmt		= vimc_deb_set_fmt,
+};
+
+static void vimc_deb_set_rgb_mbus_fmt_rgb888_1x24(struct vimc_deb_device *vdeb,
+						  unsigned int lin,
+						  unsigned int col,
+						  unsigned int rgb[3])
+{
+	unsigned int i, index;
+
+	index = VIMC_FRAME_INDEX(lin, col, vdeb->sink_fmt.width, 3);
+	for (i = 0; i < 3; i++)
+		vdeb->src_frame[index + i] = rgb[i];
+}
+
+static int vimc_deb_s_stream(struct v4l2_subdev *sd, int enable)
+{
+	struct vimc_deb_device *vdeb = v4l2_get_subdevdata(sd);
+	int ret;
+
+	if (enable) {
+		const struct vimc_pix_map *vpix;
+		unsigned int frame_size;
+
+		if (vdeb->src_frame)
+			return 0;
+
+		/* Calculate the frame size of the source pad */
+		vpix = vimc_pix_map_by_code(vdeb->src_code);
+		frame_size = vdeb->sink_fmt.width * vdeb->sink_fmt.height *
+				vpix->bpp;
+
+		/* Save the bytes per pixel of the sink */
+		vpix = vimc_pix_map_by_code(vdeb->sink_fmt.code);
+		vdeb->sink_bpp = vpix->bpp;
+
+		/* Get the corresponding pixel map from the table */
+		vdeb->sink_pix_map =
+			vimc_deb_pix_map_by_code(vdeb->sink_fmt.code);
+
+		/*
+		 * Allocate the frame buffer. Use vmalloc to be able to
+		 * allocate a large amount of memory
+		 */
+		vdeb->src_frame = vmalloc(frame_size);
+		if (!vdeb->src_frame)
+			return -ENOMEM;
+
+		/* Turn the stream on in the subdevices directly connected */
+		ret = vimc_pipeline_s_stream(&vdeb->sd.entity, 1);
+		if (ret) {
+			vfree(vdeb->src_frame);
+			vdeb->src_frame = NULL;
+			return ret;
+		}
+	} else {
+		if (!vdeb->src_frame)
+			return 0;
+
+		/* Disable streaming from the pipe */
+		ret = vimc_pipeline_s_stream(&vdeb->sd.entity, 0);
+		if (ret)
+			return ret;
+
+		vfree(vdeb->src_frame);
+		vdeb->src_frame = NULL;
+	}
+
+	return 0;
+}
+
+struct v4l2_subdev_video_ops vimc_deb_video_ops = {
+	.s_stream = vimc_deb_s_stream,
+};
+
+static const struct v4l2_subdev_ops vimc_deb_ops = {
+	.pad = &vimc_deb_pad_ops,
+	.video = &vimc_deb_video_ops,
+};
+
+static unsigned int vimc_deb_get_val(const u8 *bytes,
+				     const unsigned int n_bytes)
+{
+	unsigned int i;
+	unsigned int acc = 0;
+
+	for (i = 0; i < n_bytes; i++)
+		acc = acc + (bytes[i] << (8 * i));
+
+	return acc;
+}
+
+static void vimc_deb_calc_rgb_sink(struct vimc_deb_device *vdeb,
+				   const u8 *frame,
+				   const unsigned int lin,
+				   const unsigned int col,
+				   unsigned int rgb[3])
+{
+	unsigned int i, seek, wlin, wcol;
+	unsigned int n_rgb[3] = {0, 0, 0};
+
+	for (i = 0; i < 3; i++)
+		rgb[i] = 0;
+
+	/*
+	 * Calculate how many we need to subtract to get to the pixel in
+	 * the top left corner of the mean window (considering the current
+	 * pixel as the center)
+	 */
+	seek = deb_mean_win_size / 2;
+
+	/* Sum the values of the colors in the mean window */
+
+	dev_dbg(vdeb->dev,
+		"deb: %s: --- Calc pixel %dx%d, window mean %d, seek %d ---\n",
+		vdeb->sd.name, lin, col, vdeb->sink_fmt.height, seek);
+
+	/*
+	 * Iterate through all the lines in the mean window, start
+	 * with zero if the pixel is outside the frame and don't pass
+	 * the height when the pixel is in the bottom border of the
+	 * frame
+	 */
+	for (wlin = seek > lin ? 0 : lin - seek;
+	     wlin < lin + seek + 1 && wlin < vdeb->sink_fmt.height;
+	     wlin++) {
+
+		/*
+		 * Iterate through all the columns in the mean window, start
+		 * with zero if the pixel is outside the frame and don't pass
+		 * the width when the pixel is in the right border of the
+		 * frame
+		 */
+		for (wcol = seek > col ? 0 : col - seek;
+		     wcol < col + seek + 1 && wcol < vdeb->sink_fmt.width;
+		     wcol++) {
+			enum vimc_deb_rgb_colors color;
+			unsigned int index;
+
+			/* Check which color this pixel is */
+			color = vdeb->sink_pix_map->order[wlin % 2][wcol % 2];
+
+			index = VIMC_FRAME_INDEX(wlin, wcol,
+						 vdeb->sink_fmt.width,
+						 vdeb->sink_bpp);
+
+			dev_dbg(vdeb->dev,
+				"deb: %s: RGB CALC: frame index %d, win pos %dx%d, color %d\n",
+				vdeb->sd.name, index, wlin, wcol, color);
+
+			/* Get its value */
+			rgb[color] = rgb[color] +
+				vimc_deb_get_val(&frame[index], vdeb->sink_bpp);
+
+			/* Save how many values we already added */
+			n_rgb[color]++;
+
+			dev_dbg(vdeb->dev, "deb: %s: RGB CALC: val %d, n %d\n",
+				vdeb->sd.name, rgb[color], n_rgb[color]);
+		}
+	}
+
+	/* Calculate the mean */
+	for (i = 0; i < 3; i++) {
+		dev_dbg(vdeb->dev,
+			"deb: %s: PRE CALC: %dx%d Color %d, val %d, n %d\n",
+			vdeb->sd.name, lin, col, i, rgb[i], n_rgb[i]);
+
+		if (n_rgb[i])
+			rgb[i] = rgb[i] / n_rgb[i];
+
+		dev_dbg(vdeb->dev,
+			"deb: %s: FINAL CALC: %dx%d Color %d, val %d\n",
+			vdeb->sd.name, lin, col, i, rgb[i]);
+	}
+}
+
+static void vimc_deb_process_frame(struct vimc_ent_device *ved,
+				   struct media_pad *sink,
+				   const void *sink_frame)
+{
+	struct vimc_deb_device *vdeb = container_of(ved, struct vimc_deb_device,
+						    ved);
+	unsigned int rgb[3];
+	unsigned int i, j;
+
+	/* If the stream in this node is not active, just return */
+	if (!vdeb->src_frame)
+		return;
+
+	for (i = 0; i < vdeb->sink_fmt.height; i++)
+		for (j = 0; j < vdeb->sink_fmt.width; j++) {
+			vimc_deb_calc_rgb_sink(vdeb, sink_frame, i, j, rgb);
+			vdeb->set_rgb_src(vdeb, i, j, rgb);
+		}
+
+	/* Propagate the frame through all source pads */
+	for (i = 1; i < vdeb->sd.entity.num_pads; i++) {
+		struct media_pad *pad = &vdeb->sd.entity.pads[i];
+
+		vimc_propagate_frame(pad, vdeb->src_frame);
+	}
+}
+
+static void vimc_deb_comp_unbind(struct device *comp, struct device *master,
+				 void *master_data)
+{
+	struct vimc_ent_device *ved = dev_get_drvdata(comp);
+	struct vimc_deb_device *vdeb = container_of(ved, struct vimc_deb_device,
+						    ved);
+
+	vimc_ent_sd_unregister(ved, &vdeb->sd);
+	kfree(vdeb);
+}
+
+static int vimc_deb_comp_bind(struct device *comp, struct device *master,
+			      void *master_data)
+{
+	struct v4l2_device *v4l2_dev = master_data;
+	char *name = comp->platform_data;
+	struct vimc_deb_device *vdeb;
+	int ret;
+
+	/* Allocate the vdeb struct */
+	vdeb = kzalloc(sizeof(*vdeb), GFP_KERNEL);
+	if (!vdeb)
+		return -ENOMEM;
+
+	/* Initialize ved and sd */
+	ret = vimc_ent_sd_register(&vdeb->ved, &vdeb->sd, v4l2_dev, name,
+				   MEDIA_ENT_F_ATV_DECODER, 2,
+				   (const unsigned long[2]) {MEDIA_PAD_FL_SINK,
+				   MEDIA_PAD_FL_SOURCE},
+				   &vimc_deb_ops);
+	if (ret) {
+		kfree(vdeb);
+		return ret;
+	}
+
+	vdeb->ved.process_frame = vimc_deb_process_frame;
+	dev_set_drvdata(comp, &vdeb->ved);
+	vdeb->dev = comp;
+
+	/* Initialize the frame format */
+	vdeb->sink_fmt = sink_fmt_default;
+	/*
+	 * TODO: Add support for more output formats, we only support
+	 * RGB888 for now
+	 * NOTE: the src format is always the same as the sink, except
+	 * for the code
+	 */
+	vdeb->src_code = MEDIA_BUS_FMT_RGB888_1X24;
+	vdeb->set_rgb_src = vimc_deb_set_rgb_mbus_fmt_rgb888_1x24;
+
+	return 0;
+}
+
+static const struct component_ops vimc_deb_comp_ops = {
+	.bind = vimc_deb_comp_bind,
+	.unbind = vimc_deb_comp_unbind,
+};
+
+static int vimc_deb_probe(struct platform_device *pdev)
+{
+	return component_add(&pdev->dev, &vimc_deb_comp_ops);
+}
+
+static int vimc_deb_remove(struct platform_device *pdev)
+{
+	component_del(&pdev->dev, &vimc_deb_comp_ops);
+
+	return 0;
+}
+
+static struct platform_driver vimc_deb_pdrv = {
+	.probe		= vimc_deb_probe,
+	.remove		= vimc_deb_remove,
+	.driver		= {
+		.name	= VIMC_DEB_DRV_NAME,
+	},
+};
+
+static const struct platform_device_id vimc_deb_driver_ids[] = {
+	{
+		.name           = VIMC_DEB_DRV_NAME,
+	},
+	{ }
+};
+
+module_platform_driver(vimc_deb_pdrv);
+
+MODULE_DEVICE_TABLE(platform, vimc_deb_driver_ids);
+
+MODULE_DESCRIPTION("Virtual Media Controller Driver (VIMC) Debayer");
+MODULE_AUTHOR("Helen Mae Koike Fornazier <helen.fornazier@gmail.com>");
+MODULE_LICENSE("GPL");
-- 
2.7.4

[RFC PATCH v3 11/11] [media] vimc: sca: Add scaler

[Helen Koike]

Implement scaler and integrated with the core

Signed-off-by: Helen Koike <helen.koike@collabora.com>

---

Changes in v3:
[media] vimc: sca: Add scaler
	- Declare frame_size as a local variable
	- s_stream(sd, 1): return 0 if stream is already enabled
	- s_stream(sd, 0): return 0 if stream is already disabled
	- s_stream: add ret variable to propagate return errors
	- structure code to be a module, use platform_driver and component system
	- s/thought/through
	- clamp height and width of the image by an even value
	- if user try to set colorspace to an invalid format, set all
	    colorimetry parameters to _DEFAULT
	- uset _DEFAULT for colorimetry in the default format

Changes in v2:
[media] vimc: sca: Add scaler
	- Add function MEDIA_ENT_F_IO_V4L
	- remove v4l2_dev and dev
	- s/sink_mbus_fmt/sink_fmt
	- remove BUG_ON, remove redundant if else, rewrite TODO, check end of enum
	- rm src_width/height, enum fsize with min and max values
	- set/try fmt
	- remove unecessary include freezer.h
	- core: add bayer boolean in pixel table
	- coding style
	- fix bug in enum frame size
	- check pad types on create
	- return EBUSY when trying to set the format while stream is on
	- remove vsd struct
	- add IS_SRC and IS_SINK macros
	- add sca_mult as a parameter of the module
	- check set_fmt default parameters for quantization, colorspace ...
	- add more dev_dbg


---
 drivers/media/platform/vimc/Makefile      |   3 +-
 drivers/media/platform/vimc/vimc-common.c |  27 ++
 drivers/media/platform/vimc/vimc-common.h |   1 +
 drivers/media/platform/vimc/vimc-scaler.c | 469 ++++++++++++++++++++++++++++++
 4 files changed, 499 insertions(+), 1 deletion(-)
 create mode 100644 drivers/media/platform/vimc/vimc-scaler.c

diff --git a/drivers/media/platform/vimc/Makefile b/drivers/media/platform/vimc/Makefile
index 4fba8ef..68c5d98 100644
--- a/drivers/media/platform/vimc/Makefile
+++ b/drivers/media/platform/vimc/Makefile
@@ -2,7 +2,8 @@ vimc-objs := vimc-core.o
 vimc_capture-objs := vimc-capture.o
 vimc_common-objs := vimc-common.o
 vimc_debayer-objs := vimc-debayer.o
+vimc_scaler-objs := vimc-scaler.o
 vimc_sensor-objs := vimc-sensor.o
 
 obj-$(CONFIG_VIDEO_VIMC) += vimc.o vimc_capture.o vimc_common.o vimc-debayer.o \
-				vimc_sensor.o
+				vimc_scaler.o vimc_sensor.o
diff --git a/drivers/media/platform/vimc/vimc-common.c b/drivers/media/platform/vimc/vimc-common.c
index d5ed387..68887d7 100644
--- a/drivers/media/platform/vimc/vimc-common.c
+++ b/drivers/media/platform/vimc/vimc-common.c
@@ -20,6 +20,10 @@
 
 #include "vimc-common.h"
 
+/*
+ * NOTE: non-bayer formats need to come first (necessary for enum_mbus_code
+ * in the scaler)
+ */
 static const struct vimc_pix_map vimc_pix_map_list[] = {
 	/* TODO: add all missing formats */
 
@@ -28,16 +32,19 @@ static const struct vimc_pix_map vimc_pix_map_list[] = {
 		.code = MEDIA_BUS_FMT_BGR888_1X24,
 		.pixelformat = V4L2_PIX_FMT_BGR24,
 		.bpp = 3,
+		.bayer = false,
 	},
 	{
 		.code = MEDIA_BUS_FMT_RGB888_1X24,
 		.pixelformat = V4L2_PIX_FMT_RGB24,
 		.bpp = 3,
+		.bayer = false,
 	},
 	{
 		.code = MEDIA_BUS_FMT_ARGB8888_1X32,
 		.pixelformat = V4L2_PIX_FMT_ARGB32,
 		.bpp = 4,
+		.bayer = false,
 	},
 
 	/* Bayer formats */
@@ -45,41 +52,49 @@ static const struct vimc_pix_map vimc_pix_map_list[] = {
 		.code = MEDIA_BUS_FMT_SBGGR8_1X8,
 		.pixelformat = V4L2_PIX_FMT_SBGGR8,
 		.bpp = 1,
+		.bayer = true,
 	},
 	{
 		.code = MEDIA_BUS_FMT_SGBRG8_1X8,
 		.pixelformat = V4L2_PIX_FMT_SGBRG8,
 		.bpp = 1,
+		.bayer = true,
 	},
 	{
 		.code = MEDIA_BUS_FMT_SGRBG8_1X8,
 		.pixelformat = V4L2_PIX_FMT_SGRBG8,
 		.bpp = 1,
+		.bayer = true,
 	},
 	{
 		.code = MEDIA_BUS_FMT_SRGGB8_1X8,
 		.pixelformat = V4L2_PIX_FMT_SRGGB8,
 		.bpp = 1,
+		.bayer = true,
 	},
 	{
 		.code = MEDIA_BUS_FMT_SBGGR10_1X10,
 		.pixelformat = V4L2_PIX_FMT_SBGGR10,
 		.bpp = 2,
+		.bayer = true,
 	},
 	{
 		.code = MEDIA_BUS_FMT_SGBRG10_1X10,
 		.pixelformat = V4L2_PIX_FMT_SGBRG10,
 		.bpp = 2,
+		.bayer = true,
 	},
 	{
 		.code = MEDIA_BUS_FMT_SGRBG10_1X10,
 		.pixelformat = V4L2_PIX_FMT_SGRBG10,
 		.bpp = 2,
+		.bayer = true,
 	},
 	{
 		.code = MEDIA_BUS_FMT_SRGGB10_1X10,
 		.pixelformat = V4L2_PIX_FMT_SRGGB10,
 		.bpp = 2,
+		.bayer = true,
 	},
 
 	/* 10bit raw bayer a-law compressed to 8 bits */
@@ -87,21 +102,25 @@ static const struct vimc_pix_map vimc_pix_map_list[] = {
 		.code = MEDIA_BUS_FMT_SBGGR10_ALAW8_1X8,
 		.pixelformat = V4L2_PIX_FMT_SBGGR10ALAW8,
 		.bpp = 1,
+		.bayer = true,
 	},
 	{
 		.code = MEDIA_BUS_FMT_SGBRG10_ALAW8_1X8,
 		.pixelformat = V4L2_PIX_FMT_SGBRG10ALAW8,
 		.bpp = 1,
+		.bayer = true,
 	},
 	{
 		.code = MEDIA_BUS_FMT_SGRBG10_ALAW8_1X8,
 		.pixelformat = V4L2_PIX_FMT_SGRBG10ALAW8,
 		.bpp = 1,
+		.bayer = true,
 	},
 	{
 		.code = MEDIA_BUS_FMT_SRGGB10_ALAW8_1X8,
 		.pixelformat = V4L2_PIX_FMT_SRGGB10ALAW8,
 		.bpp = 1,
+		.bayer = true,
 	},
 
 	/* 10bit raw bayer DPCM compressed to 8 bits */
@@ -109,41 +128,49 @@ static const struct vimc_pix_map vimc_pix_map_list[] = {
 		.code = MEDIA_BUS_FMT_SBGGR10_DPCM8_1X8,
 		.pixelformat = V4L2_PIX_FMT_SBGGR10DPCM8,
 		.bpp = 1,
+		.bayer = true,
 	},
 	{
 		.code = MEDIA_BUS_FMT_SGBRG10_DPCM8_1X8,
 		.pixelformat = V4L2_PIX_FMT_SGBRG10DPCM8,
 		.bpp = 1,
+		.bayer = true,
 	},
 	{
 		.code = MEDIA_BUS_FMT_SGRBG10_DPCM8_1X8,
 		.pixelformat = V4L2_PIX_FMT_SGRBG10DPCM8,
 		.bpp = 1,
+		.bayer = true,
 	},
 	{
 		.code = MEDIA_BUS_FMT_SRGGB10_DPCM8_1X8,
 		.pixelformat = V4L2_PIX_FMT_SRGGB10DPCM8,
 		.bpp = 1,
+		.bayer = true,
 	},
 	{
 		.code = MEDIA_BUS_FMT_SBGGR12_1X12,
 		.pixelformat = V4L2_PIX_FMT_SBGGR12,
 		.bpp = 2,
+		.bayer = true,
 	},
 	{
 		.code = MEDIA_BUS_FMT_SGBRG12_1X12,
 		.pixelformat = V4L2_PIX_FMT_SGBRG12,
 		.bpp = 2,
+		.bayer = true,
 	},
 	{
 		.code = MEDIA_BUS_FMT_SGRBG12_1X12,
 		.pixelformat = V4L2_PIX_FMT_SGRBG12,
 		.bpp = 2,
+		.bayer = true,
 	},
 	{
 		.code = MEDIA_BUS_FMT_SRGGB12_1X12,
 		.pixelformat = V4L2_PIX_FMT_SRGGB12,
 		.bpp = 2,
+		.bayer = true,
 	},
 };
 
diff --git a/drivers/media/platform/vimc/vimc-common.h b/drivers/media/platform/vimc/vimc-common.h
index c63d51f..e1687f0 100644
--- a/drivers/media/platform/vimc/vimc-common.h
+++ b/drivers/media/platform/vimc/vimc-common.h
@@ -45,6 +45,7 @@ struct vimc_pix_map {
 	unsigned int code;
 	unsigned int bpp;
 	u32 pixelformat;
+	bool bayer;
 };
 
 /**
diff --git a/drivers/media/platform/vimc/vimc-scaler.c b/drivers/media/platform/vimc/vimc-scaler.c
new file mode 100644
index 0000000..c98ced2
--- /dev/null
+++ b/drivers/media/platform/vimc/vimc-scaler.c
@@ -0,0 +1,469 @@
+/*
+ * vimc-scaler.c Virtual Media Controller Driver
+ *
+ * Copyright (C) 2015-2017 Helen Koike <helen.fornazier@gmail.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ */
+
+#include <linux/component.h>
+#include <linux/module.h>
+#include <linux/platform_device.h>
+#include <linux/vmalloc.h>
+#include <linux/v4l2-mediabus.h>
+#include <media/v4l2-subdev.h>
+
+#include "vimc-common.h"
+
+#define VIMC_SCA_DRV_NAME "vimc-scaler"
+
+static unsigned int sca_mult = 3;
+module_param(sca_mult, uint, 0000);
+MODULE_PARM_DESC(sca_mult, " the image size multiplier");
+
+#define IS_SINK(pad)	(!pad)
+#define IS_SRC(pad)	(pad)
+#define MAX_ZOOM	8
+
+struct vimc_sca_device {
+	struct vimc_ent_device ved;
+	struct v4l2_subdev sd;
+	struct device *dev;
+	/* NOTE: the source fmt is the same as the sink
+	 * with the width and hight multiplied by mult
+	 */
+	struct v4l2_mbus_framefmt sink_fmt;
+	/* Values calculated when the stream starts */
+	u8 *src_frame;
+	unsigned int src_line_size;
+	unsigned int bpp;
+};
+
+static const struct v4l2_mbus_framefmt sink_fmt_default = {
+	.width = 640,
+	.height = 480,
+	.code = MEDIA_BUS_FMT_RGB888_1X24,
+	.field = V4L2_FIELD_NONE,
+	.colorspace = V4L2_COLORSPACE_SRGB,
+};
+
+static int vimc_sca_init_cfg(struct v4l2_subdev *sd,
+			     struct v4l2_subdev_pad_config *cfg)
+{
+	struct v4l2_mbus_framefmt *mf;
+	unsigned int i;
+
+	mf = v4l2_subdev_get_try_format(sd, cfg, 0);
+	*mf = sink_fmt_default;
+
+	for (i = 1; i < sd->entity.num_pads; i++) {
+		mf = v4l2_subdev_get_try_format(sd, cfg, i);
+		*mf = sink_fmt_default;
+		mf->width = mf->width * sca_mult;
+		mf->height = mf->height * sca_mult;
+	}
+
+	return 0;
+}
+
+static int vimc_sca_enum_mbus_code(struct v4l2_subdev *sd,
+				   struct v4l2_subdev_pad_config *cfg,
+				   struct v4l2_subdev_mbus_code_enum *code)
+{
+	const struct vimc_pix_map *vpix = vimc_pix_map_by_index(code->index);
+
+	/* We don't support bayer format */
+	if (!vpix || vpix->bayer)
+		return -EINVAL;
+
+	code->code = vpix->code;
+
+	return 0;
+}
+
+static int vimc_sca_enum_frame_size(struct v4l2_subdev *sd,
+				    struct v4l2_subdev_pad_config *cfg,
+				    struct v4l2_subdev_frame_size_enum *fse)
+{
+	const struct vimc_pix_map *vpix;
+
+	if (fse->index)
+		return -EINVAL;
+
+	/* Only accept code in the pix map table in non bayer format */
+	vpix = vimc_pix_map_by_code(fse->code);
+	if (!vpix || vpix->bayer)
+		return -EINVAL;
+
+	fse->min_width = VIMC_FRAME_MIN_WIDTH;
+	fse->min_height = VIMC_FRAME_MIN_HEIGHT;
+
+	if (IS_SINK(fse->pad)) {
+		fse->max_width = VIMC_FRAME_MAX_WIDTH;
+		fse->max_height = VIMC_FRAME_MAX_HEIGHT;
+	} else {
+		fse->max_width = VIMC_FRAME_MAX_WIDTH * MAX_ZOOM;
+		fse->max_height = VIMC_FRAME_MAX_HEIGHT * MAX_ZOOM;
+	}
+
+	return 0;
+}
+
+static int vimc_sca_get_fmt(struct v4l2_subdev *sd,
+			    struct v4l2_subdev_pad_config *cfg,
+			    struct v4l2_subdev_format *format)
+{
+	struct vimc_sca_device *vsca = v4l2_get_subdevdata(sd);
+
+	/* Get the current sink format */
+	format->format = (format->which == V4L2_SUBDEV_FORMAT_TRY) ?
+			 *v4l2_subdev_get_try_format(sd, cfg, 0) :
+			 vsca->sink_fmt;
+
+	/* Scale the frame size for the source pad */
+	if (IS_SRC(format->pad)) {
+		format->format.width = vsca->sink_fmt.width * sca_mult;
+		format->format.height = vsca->sink_fmt.height * sca_mult;
+	}
+
+	return 0;
+}
+
+static void vimc_sca_adjust_sink_fmt(struct v4l2_mbus_framefmt *fmt)
+{
+	const struct vimc_pix_map *vpix;
+
+	/* Only accept code in the pix map table in non bayer format */
+	vpix = vimc_pix_map_by_code(fmt->code);
+	if (!vpix || vpix->bayer)
+		fmt->code = sink_fmt_default.code;
+
+	fmt->width = clamp_t(u32, fmt->width, VIMC_FRAME_MIN_WIDTH,
+			     VIMC_FRAME_MAX_WIDTH) & ~1;
+	fmt->height = clamp_t(u32, fmt->height, VIMC_FRAME_MIN_HEIGHT,
+			      VIMC_FRAME_MAX_HEIGHT) & ~1;
+
+	if (fmt->field == V4L2_FIELD_ANY)
+		fmt->field = sink_fmt_default.field;
+
+	if (fmt->colorspace == V4L2_COLORSPACE_DEFAULT)
+		fmt->colorspace = sink_fmt_default.colorspace;
+
+	/* Check if values are out of range */
+	if (fmt->colorspace > V4L2_COLORSPACE_DCI_P3) {
+		fmt->colorspace = sink_fmt_default.colorspace;
+		fmt->ycbcr_enc = V4L2_YCBCR_ENC_DEFAULT;
+		fmt->quantization = V4L2_QUANTIZATION_DEFAULT;
+		fmt->xfer_func = V4L2_XFER_FUNC_DEFAULT;
+	}
+	if (fmt->ycbcr_enc > V4L2_YCBCR_ENC_SMPTE240M)
+		fmt->ycbcr_enc = V4L2_YCBCR_ENC_DEFAULT;
+	if (fmt->quantization > V4L2_QUANTIZATION_LIM_RANGE)
+		fmt->quantization = V4L2_QUANTIZATION_DEFAULT;
+	if (fmt->xfer_func > V4L2_XFER_FUNC_SMPTE2084)
+		fmt->xfer_func = V4L2_XFER_FUNC_DEFAULT;
+}
+
+static int vimc_sca_set_fmt(struct v4l2_subdev *sd,
+			    struct v4l2_subdev_pad_config *cfg,
+			    struct v4l2_subdev_format *fmt)
+{
+	struct vimc_sca_device *vsca = v4l2_get_subdevdata(sd);
+	struct v4l2_mbus_framefmt *sink_fmt;
+
+	if (fmt->which == V4L2_SUBDEV_FORMAT_ACTIVE) {
+		/* Do not change the format while stream is on */
+		if (vsca->src_frame)
+			return -EBUSY;
+
+		sink_fmt = &vsca->sink_fmt;
+	} else {
+		sink_fmt = v4l2_subdev_get_try_format(sd, cfg, 0);
+	}
+
+	/*
+	 * Do not change the format of the source pad,
+	 * it is propagated from the sink
+	 */
+	if (IS_SRC(fmt->pad)) {
+		fmt->format = *sink_fmt;
+		fmt->format.width = sink_fmt->width * sca_mult;
+		fmt->format.height = sink_fmt->height * sca_mult;
+	} else {
+		/* Set the new format in the sink pad */
+		vimc_sca_adjust_sink_fmt(&fmt->format);
+
+		dev_dbg(vsca->dev, "%s: sink format update: "
+			"old:%dx%d (0x%x, %d, %d, %d, %d) "
+			"new:%dx%d (0x%x, %d, %d, %d, %d)\n", vsca->sd.name,
+			/* old */
+			sink_fmt->width, sink_fmt->height, sink_fmt->code,
+			sink_fmt->colorspace, sink_fmt->quantization,
+			sink_fmt->xfer_func, sink_fmt->ycbcr_enc,
+			/* new */
+			fmt->format.width, fmt->format.height, fmt->format.code,
+			fmt->format.colorspace,	fmt->format.quantization,
+			fmt->format.xfer_func, fmt->format.ycbcr_enc);
+
+		*sink_fmt = fmt->format;
+	}
+
+	return 0;
+}
+
+static const struct v4l2_subdev_pad_ops vimc_sca_pad_ops = {
+	.init_cfg		= vimc_sca_init_cfg,
+	.enum_mbus_code		= vimc_sca_enum_mbus_code,
+	.enum_frame_size	= vimc_sca_enum_frame_size,
+	.get_fmt		= vimc_sca_get_fmt,
+	.set_fmt		= vimc_sca_set_fmt,
+};
+
+static int vimc_sca_s_stream(struct v4l2_subdev *sd, int enable)
+{
+	struct vimc_sca_device *vsca = v4l2_get_subdevdata(sd);
+	int ret;
+
+	if (enable) {
+		const struct vimc_pix_map *vpix;
+		unsigned int frame_size;
+
+		if (vsca->src_frame)
+			return 0;
+
+		/* Save the bytes per pixel of the sink */
+		vpix = vimc_pix_map_by_code(vsca->sink_fmt.code);
+		vsca->bpp = vpix->bpp;
+
+		/* Calculate the width in bytes of the src frame */
+		vsca->src_line_size = vsca->sink_fmt.width *
+				      sca_mult * vsca->bpp;
+
+		/* Calculate the frame size of the source pad */
+		frame_size = vsca->src_line_size * vsca->sink_fmt.height *
+			     sca_mult;
+
+		/* Allocate the frame buffer. Use vmalloc to be able to
+		 * allocate a large amount of memory
+		 */
+		vsca->src_frame = vmalloc(frame_size);
+		if (!vsca->src_frame)
+			return -ENOMEM;
+
+		/* Turn the stream on in the subdevices directly connected */
+		ret = vimc_pipeline_s_stream(&vsca->sd.entity, 1);
+		if (ret) {
+			vfree(vsca->src_frame);
+			vsca->src_frame = NULL;
+			return ret;
+		}
+	} else {
+		if (!vsca->src_frame)
+			return 0;
+
+		/* Disable streaming from the pipe */
+		ret = vimc_pipeline_s_stream(&vsca->sd.entity, 0);
+		if (ret)
+			return ret;
+
+		vfree(vsca->src_frame);
+		vsca->src_frame = NULL;
+	}
+
+	return 0;
+}
+
+struct v4l2_subdev_video_ops vimc_sca_video_ops = {
+	.s_stream = vimc_sca_s_stream,
+};
+
+static const struct v4l2_subdev_ops vimc_sca_ops = {
+	.pad = &vimc_sca_pad_ops,
+	.video = &vimc_sca_video_ops,
+};
+
+static void vimc_sca_fill_pix(u8 *const ptr,
+			      const u8 *const pixel,
+			      const unsigned int bpp)
+{
+	unsigned int i;
+
+	/* copy the pixel to the pointer */
+	for (i = 0; i < bpp; i++)
+		ptr[i] = pixel[i];
+}
+
+static void vimc_sca_scale_pix(const struct vimc_sca_device *const vsca,
+			       const unsigned int lin, const unsigned int col,
+			       const u8 *const sink_frame)
+{
+	unsigned int i, j, index;
+	const u8 *pixel;
+
+	/* Point to the pixel value in position (lin, col) in the sink frame */
+	index = VIMC_FRAME_INDEX(lin, col,
+				 vsca->sink_fmt.width,
+				 vsca->bpp);
+	pixel = &sink_frame[index];
+
+	dev_dbg(vsca->dev,
+		"sca: %s: --- scale_pix sink pos %dx%d, index %d ---\n",
+		vsca->sd.name, lin, col, index);
+
+	/* point to the place we are going to put the first pixel
+	 * in the scaled src frame
+	 */
+	index = VIMC_FRAME_INDEX(lin * sca_mult, col * sca_mult,
+				 vsca->sink_fmt.width * sca_mult, vsca->bpp);
+
+	dev_dbg(vsca->dev, "sca: %s: scale_pix src pos %dx%d, index %d\n",
+		vsca->sd.name, lin * sca_mult, col * sca_mult, index);
+
+	/* Repeat this pixel mult times */
+	for (i = 0; i < sca_mult; i++) {
+		/* Iterate through each beginning of a
+		 * pixel repetition in a line
+		 */
+		for (j = 0; j < sca_mult * vsca->bpp; j += vsca->bpp) {
+			dev_dbg(vsca->dev,
+				"sca: %s: sca: scale_pix src pos %d\n",
+				vsca->sd.name, index + j);
+
+			/* copy the pixel to the position index + j */
+			vimc_sca_fill_pix(&vsca->src_frame[index + j],
+					  pixel, vsca->bpp);
+		}
+
+		/* move the index to the next line */
+		index += vsca->src_line_size;
+	}
+}
+
+static void vimc_sca_fill_src_frame(const struct vimc_sca_device *const vsca,
+				    const u8 *const sink_frame)
+{
+	unsigned int i, j;
+
+	/* Scale each pixel from the original sink frame */
+	/* TODO: implement scale down, only scale up is supported for now */
+	for (i = 0; i < vsca->sink_fmt.height; i++)
+		for (j = 0; j < vsca->sink_fmt.width; j++)
+			vimc_sca_scale_pix(vsca, i, j, sink_frame);
+}
+
+static void vimc_sca_process_frame(struct vimc_ent_device *ved,
+				   struct media_pad *sink,
+				   const void *sink_frame)
+{
+	struct vimc_sca_device *vsca = container_of(ved, struct vimc_sca_device,
+						    ved);
+	unsigned int i;
+
+	/* If the stream in this node is not active, just return */
+	if (!vsca->src_frame)
+		return;
+
+	vimc_sca_fill_src_frame(vsca, sink_frame);
+
+	/* Propagate the frame through all source pads */
+	for (i = 1; i < vsca->sd.entity.num_pads; i++) {
+		struct media_pad *pad = &vsca->sd.entity.pads[i];
+
+		vimc_propagate_frame(pad, vsca->src_frame);
+	}
+};
+
+static void vimc_sca_comp_unbind(struct device *comp, struct device *master,
+				 void *master_data)
+{
+	struct vimc_ent_device *ved = dev_get_drvdata(comp);
+	struct vimc_sca_device *vsca = container_of(ved, struct vimc_sca_device,
+						    ved);
+
+	vimc_ent_sd_unregister(ved, &vsca->sd);
+	kfree(vsca);
+}
+
+
+static int vimc_sca_comp_bind(struct device *comp, struct device *master,
+			      void *master_data)
+{
+	struct v4l2_device *v4l2_dev = master_data;
+	char *name = comp->platform_data;
+	struct vimc_sca_device *vsca;
+	int ret;
+
+	/* Allocate the vsca struct */
+	vsca = kzalloc(sizeof(*vsca), GFP_KERNEL);
+	if (!vsca)
+		return -ENOMEM;
+
+	/* Initialize ved and sd */
+	ret = vimc_ent_sd_register(&vsca->ved, &vsca->sd, v4l2_dev, name,
+				   MEDIA_ENT_F_ATV_DECODER, 2,
+				   (const unsigned long[2]) {MEDIA_PAD_FL_SINK,
+				   MEDIA_PAD_FL_SOURCE},
+				   &vimc_sca_ops);
+	if (ret) {
+		kfree(vsca);
+		return ret;
+	}
+
+	vsca->ved.process_frame = vimc_sca_process_frame;
+	dev_set_drvdata(comp, &vsca->ved);
+	vsca->dev = comp;
+
+	/* Initialize the frame format */
+	vsca->sink_fmt = sink_fmt_default;
+
+	return 0;
+}
+
+static const struct component_ops vimc_sca_comp_ops = {
+	.bind = vimc_sca_comp_bind,
+	.unbind = vimc_sca_comp_unbind,
+};
+
+static int vimc_sca_probe(struct platform_device *pdev)
+{
+	return component_add(&pdev->dev, &vimc_sca_comp_ops);
+}
+
+static int vimc_sca_remove(struct platform_device *pdev)
+{
+	component_del(&pdev->dev, &vimc_sca_comp_ops);
+
+	return 0;
+}
+
+static struct platform_driver vimc_sca_pdrv = {
+	.probe		= vimc_sca_probe,
+	.remove		= vimc_sca_remove,
+	.driver		= {
+		.name	= VIMC_SCA_DRV_NAME,
+	},
+};
+
+static const struct platform_device_id vimc_sca_driver_ids[] = {
+	{
+		.name           = VIMC_SCA_DRV_NAME,
+	},
+	{ }
+};
+
+module_platform_driver(vimc_sca_pdrv);
+
+MODULE_DEVICE_TABLE(platform, vimc_sca_driver_ids);
+
+MODULE_DESCRIPTION("Virtual Media Controller Driver (VIMC) Scaler");
+MODULE_AUTHOR("Helen Mae Koike Fornazier <helen.fornazier@gmail.com>");
+MODULE_LICENSE("GPL");
-- 
2.7.4