[PATCH v12 45/86] media: ccs: Add support for embedded data stream

Sakari Ailus
2026-04-09T20:15:42Z

Add support for embedded data stream, in UAPI and frame descriptor.

This patch adds also a new embedded data pad (2) to the source sub-device.

Signed-off-by: Sakari Ailus <sakari.ailus@linux.intel.com>
---
drivers/media/i2c/ccs/ccs-core.c | 342 +++++++++++++++++++++++++++++--
drivers/media/i2c/ccs/ccs.h | 15 +-
2 files changed, 338 insertions(+), 19 deletions(-)

diff --git a/drivers/media/i2c/ccs/ccs-core.c b/drivers/media/i2c/ccs/ccs-core.c
index 0248a51eba8f..c4fe400d1cef 100644
--- a/drivers/media/i2c/ccs/ccs-core.c
+++ b/drivers/media/i2c/ccs/ccs-core.c
@@ -899,7 +899,7 @@ static int ccs_init_controls(struct ccs_sensor *sensor)
struct v4l2_fwnode_device_properties props;
int rval;

- rval = v4l2_ctrl_handler_init(&sensor->pixel_array->ctrl_handler, 21);
+ rval = v4l2_ctrl_handler_init(&sensor->pixel_array->ctrl_handler, 22);
if (rval)
return rval;

@@ -1105,6 +1105,12 @@ static int ccs_init_controls(struct ccs_sensor *sensor)
V4L2_CFA_PATTERN_FLIP_BOTH, 1,
V4L2_CFA_PATTERN_FLIP_BOTH);

+ v4l2_ctrl_new_std(&sensor->pixel_array->ctrl_handler, NULL,
+ V4L2_CID_METADATA_LAYOUT,
+ V4L2_METADATA_LAYOUT_CCS,
+ V4L2_METADATA_LAYOUT_CCS, 1,
+ V4L2_METADATA_LAYOUT_CCS);
+
rval = v4l2_ctrl_handler_init(&sensor->src->ctrl_handler, 0);
if (rval)
return rval;
@@ -1871,6 +1877,57 @@ static int ccs_pm_get_init(struct ccs_sensor *sensor)
return rval;
}

+#define CCS_EMBEDDED_CODE_DEPTH(depth, half_depth) \
+ depth, \
+ CCS_EMB_DATA_CAPABILITY_TWO_BYTES_PER_RAW##depth, \
+ CCS_EMB_DATA_CAPABILITY_NO_ONE_BYTE_PER_RAW##depth, \
+ CCS_EMB_DATA_CTRL_RAW##half_depth##_PACKING_FOR_RAW##depth, \
+ MEDIA_BUS_FMT_META_##half_depth, \
+ MEDIA_BUS_FMT_META_##depth, \
+
+static const struct ccs_embedded_code {
+ u8 depth;
+ u8 cap_two_bytes_per_sample;
+ u8 cap_no_legacy;
+ u8 ctrl;
+ u32 code_two_bytes;
+ u32 code_legacy;
+} ccs_embedded_codes[] = {
+ { CCS_EMBEDDED_CODE_DEPTH(16, 8) },
+ { CCS_EMBEDDED_CODE_DEPTH(20, 10) },
+ { CCS_EMBEDDED_CODE_DEPTH(24, 12) },
+};
+
+static const struct ccs_embedded_code *ccs_embedded_code(unsigned int bpp)
+{
+ unsigned int i;
+
+ for (i = 0; i < ARRAY_SIZE(ccs_embedded_codes); i++)
+ if (ccs_embedded_codes[i].depth == bpp)
+ return ccs_embedded_codes + i;
+
+ WARN_ON(1);
+
+ return ccs_embedded_codes;
+}
+
+static u32
+ccs_default_embedded_code(struct ccs_sensor *sensor,
+ const struct ccs_embedded_code *embedded_code)
+{
+ if (CCS_LIM(sensor, EMB_DATA_CAPABILITY) &
+ BIT(embedded_code->cap_two_bytes_per_sample))
+ return embedded_code->code_two_bytes;
+
+ if (!(CCS_LIM(sensor, EMB_DATA_CAPABILITY) &
+ BIT(embedded_code->cap_no_legacy)))
+ return embedded_code->code_legacy;
+
+ WARN_ON(1);
+
+ return embedded_code->code_legacy;
+}
+
static int ccs_enable_streams(struct v4l2_subdev *subdev,
struct v4l2_subdev_state *state, u32 pad,
u64 streams_mask)
@@ -2003,6 +2060,22 @@ static int ccs_enable_streams(struct v4l2_subdev *subdev,
if (rval < 0)
goto err_pm_put;

+ /* Configure embedded data */
+ if (sensor->csi_format->compressed >= 16) {
+ const struct ccs_embedded_code *embedded_code =
+ ccs_embedded_code(sensor->csi_format->compressed);
+ const struct v4l2_mbus_framefmt *meta_out_fmt =
+ v4l2_subdev_state_get_format(src_state, CCS_PAD_SRC,
+ CCS_STREAM_META);
+
+ rval = ccs_write(sensor, EMB_DATA_CTRL,
+ meta_out_fmt->code ==
+ embedded_code->code_legacy ?
+ 0 : embedded_code->ctrl);
+ if (rval < 0)
+ goto err_pm_put;
+ }
+
if (CCS_LIM(sensor, FLASH_MODE_CAPABILITY) &
(CCS_FLASH_MODE_CAPABILITY_SINGLE_STROBE |
SMIAPP_FLASH_MODE_CAPABILITY_MULTIPLE_STROBE) &&
@@ -2133,6 +2206,61 @@ static int ccs_enum_mbus_code(struct v4l2_subdev *subdev,
dev_err(&client->dev, "subdev %s, pad %u, index %u\n",
subdev->name, code->pad, code->index);

+ if (subdev == &sensor->src->sd) {
+ if (code->pad == CCS_PAD_META ||
+ code->stream == CCS_STREAM_META) {
+ struct v4l2_mbus_framefmt *pix_fmt =
+ v4l2_subdev_state_get_format(sd_state,
+ CCS_PAD_SRC,
+ CCS_STREAM_PIXEL);
+ const struct ccs_csi_data_format *csi_format =
+ ccs_validate_csi_data_format(sensor,
+ pix_fmt->code);
+ unsigned int i = 0;
+ u32 codes[2];
+
+ switch (csi_format->compressed) {
+ case 8:
+ codes[i++] = MEDIA_BUS_FMT_META_8;
+ break;
+ case 10:
+ codes[i++] = MEDIA_BUS_FMT_META_10;
+ break;
+ case 12:
+ codes[i++] = MEDIA_BUS_FMT_META_12;
+ break;
+ case 14:
+ codes[i++] = MEDIA_BUS_FMT_META_14;
+ break;
+ case 16:
+ case 20:
+ case 24: {
+ const struct ccs_embedded_code *embedded_code =
+ ccs_embedded_code(csi_format->compressed);
+
+ if (CCS_LIM(sensor, EMB_DATA_CAPABILITY) &
+ BIT(embedded_code->cap_two_bytes_per_sample))
+ codes[i++] =
+ embedded_code->code_two_bytes;
+
+ if (!(CCS_LIM(sensor, EMB_DATA_CAPABILITY) &
+ BIT(embedded_code->cap_no_legacy)))
+ codes[i++] = embedded_code->code_legacy;
+ break;
+ }
+ default:
+ WARN_ON(1);
+ }
+
+ if (WARN_ON(i > ARRAY_SIZE(codes)) || code->index >= i)
+ return -EINVAL;
+
+ code->code = codes[code->index];
+
+ return 0;
+ }
+ }
+
if (subdev != &sensor->src->sd || code->pad != CCS_PAD_SRC) {
if (code->index)
return -EINVAL;
@@ -2172,8 +2300,11 @@ static int ccs_get_format(struct v4l2_subdev *subdev,
struct v4l2_subdev_state *sd_state,
struct v4l2_subdev_format *fmt)
{
- fmt->format = *v4l2_subdev_state_get_format(sd_state, fmt->pad);
- fmt->format.code = ccs_get_mbus_code(subdev, fmt->pad);
+ fmt->format = *v4l2_subdev_state_get_format(sd_state, fmt->pad,
+ fmt->stream);
+
+ if (fmt->pad != CCS_PAD_META && fmt->stream != CCS_STREAM_META)
+ fmt->format.code = ccs_get_mbus_code(subdev, fmt->pad);

return 0;
}
@@ -2263,6 +2394,86 @@ static int ccs_set_format_source(struct v4l2_subdev *subdev,
return ccs_pll_update(sensor);
}

+static inline unsigned int ccs_embedded_data_lines(struct ccs_sensor *sensor)
+{
+ return sensor->embedded_end - sensor->embedded_start;
+}
+
+static int ccs_set_format_meta(struct v4l2_subdev *subdev,
+ struct v4l2_subdev_state *sd_state,
+ struct v4l2_mbus_framefmt *fmt)
+{
+ struct ccs_sensor *sensor = to_ccs_sensor(subdev);
+ const struct ccs_csi_data_format *csi_format;
+ struct v4l2_mbus_framefmt *pix_fmt;
+ struct v4l2_mbus_framefmt *meta_fmt;
+ struct v4l2_mbus_framefmt *meta_out_fmt;
+ u32 code;
+
+ pix_fmt = v4l2_subdev_state_get_format(sd_state, CCS_PAD_SRC,
+ CCS_STREAM_PIXEL);
+ meta_fmt = v4l2_subdev_state_get_format(sd_state, CCS_PAD_META, 0);
+ meta_out_fmt = v4l2_subdev_state_get_format(sd_state, CCS_PAD_SRC,
+ CCS_STREAM_META);
+
+ code = fmt ? fmt->code : 0;
+
+ meta_fmt->width = pix_fmt->width;
+ meta_fmt->height = ccs_embedded_data_lines(sensor);
+
+ csi_format = ccs_validate_csi_data_format(sensor, pix_fmt->code);
+
+ switch (csi_format->compressed) {
+ case 8:
+ meta_fmt->code = MEDIA_BUS_FMT_META_8;
+ break;
+ case 10:
+ meta_fmt->code = MEDIA_BUS_FMT_META_10;
+ break;
+ case 12:
+ meta_fmt->code = MEDIA_BUS_FMT_META_12;
+ break;
+ case 14:
+ meta_fmt->code = MEDIA_BUS_FMT_META_14;
+ break;
+ case 16:
+ case 20:
+ case 24: {
+ const struct ccs_embedded_code *embedded_code;
+
+ embedded_code = ccs_embedded_code(csi_format->compressed);
+ meta_fmt->code =
+ ccs_default_embedded_code(sensor, embedded_code);
+
+ if (!(CCS_LIM(sensor, EMB_DATA_CAPABILITY) &
+ BIT(embedded_code->cap_no_legacy)) &&
+ code == embedded_code->code_legacy)
+ meta_fmt->code = embedded_code->code_legacy;
+
+ if (CCS_LIM(sensor, EMB_DATA_CAPABILITY) &
+ BIT(embedded_code->cap_two_bytes_per_sample) &&
+ code == embedded_code->code_two_bytes) {
+ meta_fmt->code = embedded_code->code_two_bytes;
+ meta_fmt->width <<= 1;
+ }
+
+ break;
+ }
+ default:
+ WARN_ON(1);
+ return 0;
+ }
+
+ meta_out_fmt->width = meta_fmt->width;
+ meta_out_fmt->height = meta_fmt->height;
+ meta_out_fmt->code = meta_fmt->code;
+
+ if (fmt)
+ *fmt = *meta_out_fmt;
+
+ return 0;
+}
+
static int ccs_set_format(struct v4l2_subdev *subdev,
const struct v4l2_subdev_client_info *ci,
struct v4l2_subdev_state *sd_state,
@@ -2272,10 +2483,21 @@ static int ccs_set_format(struct v4l2_subdev *subdev,
struct ccs_subdev *ssd = to_ccs_subdev(subdev);
struct v4l2_rect *crop;

+ if (ssd == sensor->src && fmt->pad == CCS_PAD_META)
+ return ccs_get_format(subdev, ci, sd_state, fmt);
+
+ if (ssd == sensor->src && fmt->stream == CCS_STREAM_META) {
+ ccs_set_format_meta(subdev, sd_state, &fmt->format);
+
+ return 0;
+ }
+
if (fmt->pad == ssd->source_pad) {
int rval;

rval = ccs_set_format_source(subdev, NULL, sd_state, fmt);
+ if (ccs_embedded_data_lines(sensor) && ssd == sensor->src)
+ ccs_set_format_meta(subdev, sd_state, NULL);

return rval;
}
@@ -2527,6 +2749,12 @@ static int ccs_sel_supported(struct v4l2_subdev *subdev,
struct ccs_sensor *sensor = to_ccs_sensor(subdev);
struct ccs_subdev *ssd = to_ccs_subdev(subdev);

+ if (sel->stream != CCS_STREAM_PIXEL)
+ return -EINVAL;
+
+ if (sel->pad == CCS_PAD_META)
+ return -EINVAL;
+
/* We only implement crop in three places. */
switch (sel->target) {
case V4L2_SEL_TGT_CROP:
@@ -2576,7 +2804,8 @@ static int ccs_set_crop(struct v4l2_subdev *subdev,

if (sel->pad == ssd->sink_pad) {
struct v4l2_mbus_framefmt *mfmt =
- v4l2_subdev_state_get_format(sd_state, sel->pad);
+ v4l2_subdev_state_get_format(sd_state, sel->pad,
+ CCS_STREAM_PIXEL);

src_size.width = mfmt->width;
src_size.height = mfmt->height;
@@ -2638,7 +2867,9 @@ static int ccs_get_selection(struct v4l2_subdev *subdev,
} else if (sel->pad == ssd->sink_pad) {
struct v4l2_mbus_framefmt *sink_fmt =
v4l2_subdev_state_get_format(sd_state,
- ssd->sink_pad);
+ ssd->sink_pad,
+ CCS_STREAM_PIXEL);
+
sel->r.top = sel->r.left = 0;
sel->r.width = sink_fmt->width;
sel->r.height = sink_fmt->height;
@@ -2724,6 +2955,18 @@ static int ccs_get_frame_desc(struct v4l2_subdev *subdev, unsigned int pad,
entry++;
desc->num_entries++;

+ if (ccs_embedded_data_lines(sensor)) {
+ struct v4l2_mbus_framefmt *meta_out_fmt =
+ v4l2_subdev_state_get_format(sd_state, CCS_PAD_SRC,
+ CCS_STREAM_META);
+
+ entry->pixelcode = meta_out_fmt->code;
+ entry->stream = CCS_STREAM_META;
+ entry->bus.csi2.dt = MIPI_CSI2_DT_EMBEDDED_8B;
+ entry++;
+ desc->num_entries++;
+ }
+
v4l2_subdev_unlock_state(sd_state);

return 0;
@@ -3064,7 +3307,8 @@ static int ccs_init_subdev(struct ccs_sensor *sensor,
ssd->sensor = sensor;

ssd->npads = num_pads;
- ssd->source_pad = num_pads - 1;
+ ssd->source_pad =
+ ssd == sensor->pixel_array ? CCS_PA_PAD_SRC : CCS_PAD_SRC;

v4l2_i2c_subdev_set_name(&ssd->sd, client, sensor->minfo.name, name);

@@ -3078,6 +3322,10 @@ static int ccs_init_subdev(struct ccs_sensor *sensor,
ssd->sd.owner = THIS_MODULE;
ssd->sd.dev = &client->dev;
v4l2_set_subdevdata(&ssd->sd, client);
+ } else {
+ ssd->sd.flags |= V4L2_SUBDEV_FL_STREAMS;
+ ssd->pads[CCS_PAD_META].flags =
+ MEDIA_PAD_FL_SINK | MEDIA_PAD_FL_INTERNAL;
}

rval = media_entity_pads_init(&ssd->sd.entity, ssd->npads, ssd->pads);
@@ -3103,9 +3351,9 @@ static int ccs_init_state(struct v4l2_subdev *sd,
unsigned int pad = ssd == sensor->pixel_array ?
CCS_PA_PAD_SRC : CCS_PAD_SINK;
struct v4l2_mbus_framefmt *fmt =
- v4l2_subdev_state_get_format(sd_state, pad);
+ v4l2_subdev_state_get_format(sd_state, pad, CCS_STREAM_PIXEL);
struct v4l2_rect *crop =
- v4l2_subdev_state_get_crop(sd_state, pad);
+ v4l2_subdev_state_get_crop(sd_state, pad, CCS_STREAM_PIXEL);

guard(mutex)(&sensor->mutex);

@@ -3119,7 +3367,8 @@ static int ccs_init_state(struct v4l2_subdev *sd,
if (ssd == sensor->pixel_array)
return 0;

- fmt = v4l2_subdev_state_get_format(sd_state, CCS_PAD_SRC);
+ fmt = v4l2_subdev_state_get_format(sd_state, CCS_PAD_SRC,
+ CCS_STREAM_PIXEL);
fmt->code = ssd == sensor->src ?
sensor->csi_format->code : sensor->internal_csi_format->code;
fmt->field = V4L2_FIELD_NONE;
@@ -3129,6 +3378,47 @@ static int ccs_init_state(struct v4l2_subdev *sd,
return 0;
}

+static int ccs_src_init_state(struct v4l2_subdev *sd,
+ struct v4l2_subdev_state *sd_state)
+{
+ struct v4l2_subdev_route routes[] = {
+ {
+ .sink_pad = CCS_PAD_SINK,
+ .source_pad = CCS_PAD_SRC,
+ .source_stream = CCS_STREAM_PIXEL,
+ .flags = V4L2_SUBDEV_ROUTE_FL_ACTIVE,
+ }, {
+ .sink_pad = CCS_PAD_META,
+ .source_pad = CCS_PAD_SRC,
+ .source_stream = CCS_STREAM_META,
+ .flags = V4L2_SUBDEV_ROUTE_FL_ACTIVE,
+ }
+ };
+ struct v4l2_subdev_krouting routing = {
+ .routes = routes,
+ .num_routes = 1,
+ };
+ struct ccs_subdev *ssd = to_ccs_subdev(sd);
+ struct ccs_sensor *sensor = ssd->sensor;
+ int rval;
+
+ if (ccs_embedded_data_lines(sensor))
+ routing.num_routes++;
+
+ rval = v4l2_subdev_set_routing(sd, sd_state, &routing);
+ if (rval)
+ return rval;
+
+ rval = ccs_init_state(sd, sd_state);
+ if (rval)
+ return rval;
+
+ if (ccs_embedded_data_lines(sensor))
+ ccs_set_format_meta(sd, sd_state, NULL);
+
+ return 0;
+}
+
static const struct v4l2_subdev_video_ops ccs_video_ops = {
.s_stream = v4l2_subdev_s_stream_helper,
.pre_streamon = ccs_pre_streamon,
@@ -3141,6 +3431,14 @@ static const struct v4l2_subdev_pad_ops ccs_pad_ops = {
.set_fmt = ccs_set_format,
.get_selection = ccs_get_selection,
.set_selection = ccs_set_selection,
+};
+
+static const struct v4l2_subdev_pad_ops ccs_src_pad_ops = {
+ .enum_mbus_code = ccs_enum_mbus_code,
+ .get_fmt = ccs_get_format,
+ .set_fmt = ccs_set_format,
+ .get_selection = ccs_get_selection,
+ .set_selection = ccs_set_selection,
.enable_streams = ccs_enable_streams,
.disable_streams = ccs_disable_streams,
.get_frame_desc = ccs_get_frame_desc,
@@ -3157,6 +3455,12 @@ static const struct v4l2_subdev_ops ccs_ops = {
.sensor = &ccs_sensor_ops,
};

+static const struct v4l2_subdev_ops ccs_src_ops = {
+ .video = &ccs_video_ops,
+ .pad = &ccs_src_pad_ops,
+ .sensor = &ccs_sensor_ops,
+};
+
static const struct media_entity_operations ccs_entity_ops = {
.link_validate = v4l2_subdev_link_validate,
};
@@ -3165,8 +3469,8 @@ static const struct v4l2_subdev_internal_ops ccs_internal_ops = {
.init_state = ccs_init_state,
};

-static const struct v4l2_subdev_internal_ops ccs_internal_src_ops = {
- .init_state = ccs_init_state,
+static const struct v4l2_subdev_internal_ops ccs_src_internal_ops = {
+ .init_state = ccs_src_init_state,
.registered = ccs_registered,
.unregistered = ccs_unregistered,
};
@@ -3308,8 +3612,8 @@ static int ccs_probe(struct i2c_client *client)

sensor->src = &sensor->ssds[sensor->ssds_used];

- v4l2_i2c_subdev_init(&sensor->src->sd, client, &ccs_ops);
- sensor->src->sd.internal_ops = &ccs_internal_src_ops;
+ v4l2_i2c_subdev_init(&sensor->src->sd, client, &ccs_src_ops);
+ sensor->src->sd.internal_ops = &ccs_src_internal_ops;

sensor->regulators = devm_kcalloc(&client->dev,
ARRAY_SIZE(ccs_regulators),
@@ -3576,12 +3880,20 @@ static int ccs_probe(struct i2c_client *client)
goto out_cleanup;
}

- rval = ccs_init_subdev(sensor, sensor->scaler, " scaler", 2,
+ rval = ccs_init_subdev(sensor, sensor->scaler, " scaler",
+ sensor->ssds_used != CCS_SUBDEVS ?
+ CCS_PADS_NOMETA :
+ ccs_embedded_data_lines(sensor) ?
+ CCS_PADS : CCS_PADS_NOMETA,
MEDIA_ENT_F_PROC_VIDEO_SCALER,
"ccs scaler mutex", &scaler_lock_key);
if (rval)
goto out_cleanup;
- rval = ccs_init_subdev(sensor, sensor->binner, " binner", 2,
+ rval = ccs_init_subdev(sensor, sensor->binner, " binner",
+ sensor->ssds_used == CCS_SUBDEVS ?
+ CCS_PADS_NOMETA :
+ ccs_embedded_data_lines(sensor) ?
+ CCS_PADS : CCS_PADS_NOMETA,
MEDIA_ENT_F_PROC_VIDEO_SCALER,
"ccs binner mutex", &binner_lock_key);
if (rval)
diff --git a/drivers/media/i2c/ccs/ccs.h b/drivers/media/i2c/ccs/ccs.h
index df044dbde23f..122287ab5399 100644
--- a/drivers/media/i2c/ccs/ccs.h
+++ b/drivers/media/i2c/ccs/ccs.h
@@ -176,11 +176,18 @@ struct ccs_csi_data_format {
#define CCS_SUBDEVS 3

#define CCS_PA_PAD_SRC 0
-#define CCS_PAD_SINK 0
-#define CCS_PAD_SRC 1
-#define CCS_PADS 2
+enum {
+ CCS_PAD_SINK,
+ CCS_PAD_SRC,
+ CCS_PAD_META,
+ CCS_PADS_NOMETA = CCS_PAD_META,
+ CCS_PADS,
+};

-#define CCS_STREAM_PIXEL 0
+enum {
+ CCS_STREAM_PIXEL,
+ CCS_STREAM_META,
+};

struct ccs_binning_subtype {
u8 horizontal:4;
--
2.47.3