[PATCH v2 00/21] IPU6 multi-stream and metadata support preparation

[PATCH v2 00/21] IPU6 multi-stream and metadata support preparation

[Sakari Ailus]

Hi folks,

I split this set off of the metadata series as it seems it can be merged
early. Basically, it prepares the ipu6 driver for streaming multiple
streams from a single source, when the rest of the necessary patches are
merged.

All the patches in the set can be found in my metadata branch. This set
possibly depends on the metadata preparation series
<https://lore.kernel.org/linux-media/20260914114145.574791-1-sakari.ailus@linux.intel.com/T/#t>
I posted earlier today.

since v1:

- Fix intra-set use of v4l2_subdev_get_frame_desc().

- Address Antti's comment on assigning dt for a pin (fw term) from struct
  ipu6_isys_video. Remove vc and dt fields from the same struct, too; they
  were redundant.

Sakari Ailus (21):
  media: ipu6: Fix releasing resources at failing streamon
  media: ipu6: Move streaming control to CSI-2 receiver driver
  media: ipu6: Stream number on CSI-2 receiver source pads is always 0
  media: ipu6: Rename misnamed out_free_watermark label in video init
  media: ipu6: Always request a capture ack
  media: ipu6: Clean up link frequency calculation
  media: ipu6: Get watermark configuration directly from ipdata
  media: ipu6: Collect IPU streams into CSI-2 receiver sub-device
    context
  media: ipu6: Start streaming once all streams have started, stop when
    not
  media: ipu6: Add lockdep checks for CSI-2 streaming enable and disable
  media: ipu6: Remove nr_queues and nr_streaming fields in
    ipu6_isys_stream
  media: ipu6: Collect enabled stream IDs
  media: ipu6: Avoid accessing av->streams before streaming
  media: ipu6: Rework watermark calculation
  media: ipu6: Rework watermark setting
  media: ipu6: Bridge the gap between streams in V4L2 and IPU6 firmware
  media: ipu6: Drop {get,put}_streams_opened()
  media: ipu6: Serialise access to stream pointers by isys stream_lock
  media: ipu6: Move firmware init/cleanup to RPM callbacks
  media: ipu6: Don't track power status, rely on runtime PM
  media: ipu6: Support upstream sub-devices without get_frame_desc()

 drivers/media/pci/intel/ipu6/ipu6-fw-isys.c   | 103 ++-
 drivers/media/pci/intel/ipu6/ipu6-isys-csi2.c | 374 ++++++++--
 drivers/media/pci/intel/ipu6/ipu6-isys-csi2.h |  13 +-
 .../media/pci/intel/ipu6/ipu6-isys-queue.c    | 132 ++--
 .../media/pci/intel/ipu6/ipu6-isys-queue.h    |   2 +
 .../media/pci/intel/ipu6/ipu6-isys-subdev.c   |  22 +-
 .../media/pci/intel/ipu6/ipu6-isys-subdev.h   |   2 +
 .../media/pci/intel/ipu6/ipu6-isys-video.c    | 642 +++++-------------
 .../media/pci/intel/ipu6/ipu6-isys-video.h    |  61 +-
 drivers/media/pci/intel/ipu6/ipu6-isys.c      | 212 +++---
 drivers/media/pci/intel/ipu6/ipu6-isys.h      |  52 +-
 drivers/media/pci/intel/ipu6/ipu7-fw-isys.c   |  78 ++-
 12 files changed, 738 insertions(+), 955 deletions(-)


base-commit: 934ee99f5c020bf55b27edd3975f76181cecf0f4
-- 
2.47.3

[PATCH v2 01/21] media: ipu6: Fix releasing resources at failing streamon

[Sakari Ailus]

If obtaining a firmware message or requesting to queue a buffer list
failed, either the buffer list or the firmware message buffer wasn't
properly released. While this is unlikely to happen, return the remaining
resources now.

This may still leave buffers queued.

Signed-off-by: Sakari Ailus <sakari.ailus@linux.intel.com>
---
 .../media/pci/intel/ipu6/ipu6-isys-queue.c    | 19 ++++++++++---------
 1 file changed, 10 insertions(+), 9 deletions(-)

diff --git a/drivers/media/pci/intel/ipu6/ipu6-isys-queue.c b/drivers/media/pci/intel/ipu6/ipu6-isys-queue.c
index 38db162d64ca..4d862f14c765 100644
--- a/drivers/media/pci/intel/ipu6/ipu6-isys-queue.c
+++ b/drivers/media/pci/intel/ipu6/ipu6-isys-queue.c
@@ -242,6 +242,7 @@ static int ipu6_isys_stream_start(struct ipu6_isys_video *av,
 	struct ipu6_isys_stream *stream = av->stream;
 	struct device *dev = &adev->auxdev.dev;
 	struct ipu6_isys_buffer_list __bl;
+	struct isys_fw_msgs *msg;
 	int ret;
 
 	guard(mutex)(&stream->isys->stream_mutex);
@@ -254,26 +255,26 @@ static int ipu6_isys_stream_start(struct ipu6_isys_video *av,
 	bl = &__bl;
 
 	do {
-		struct isys_fw_msgs *msg;
-
 		ret = buffer_list_get(stream, bl);
 		if (ret < 0)
-			break;
+			return 0;
 
 		msg = ipu6_get_fw_msg_buf(stream);
-		if (!msg)
-			return -ENOMEM;
+		if (WARN_ON(!msg))
+			goto out_requeue;
 
 		fw_ops->prepare_buf_set(msg, stream, bl);
 		fw_ops->dump_frame_buf_set(dev, msg, stream->nr_output_pins);
 		ipu6_isys_buffer_list_queue(bl, IPU6_ISYS_BUFFER_LIST_FL_ACTIVE,
 					    0);
-
 		ret = fw_ops->stream_capture(stream->isys,
 					     stream->stream_handle, msg);
-	} while (!WARN_ON(ret));
+		if (WARN_ON(ret))
+			break;
+	} while (true);
 
-	return 0;
+	/* Error handling begins here. */
+	ipu6_put_fw_msg_buf(stream->isys, msg);
 
 out_requeue:
 	if (bl && bl->nbufs)
@@ -282,7 +283,7 @@ static int ipu6_isys_stream_start(struct ipu6_isys_video *av,
 					    VB2_BUF_STATE_QUEUED);
 	flush_firmware_streamon_fail(stream);
 
-	return ret;
+	return 0;
 }
 
 static void buf_queue(struct vb2_buffer *vb)
-- 
2.47.3

[PATCH v2 02/21] media: ipu6: Move streaming control to CSI-2 receiver driver

[Sakari Ailus]

Move the streaming control out of the video nodes to the CSI-2 receiver
driver which is better positioned to consider stream configuration and so
when to actually start and stop streaming.

Signed-off-by: Sakari Ailus <sakari.ailus@linux.intel.com>
---
 drivers/media/pci/intel/ipu6/ipu6-fw-isys.c   |  2 +-
 drivers/media/pci/intel/ipu6/ipu6-isys-csi2.c | 54 +++++++++++++++----
 .../media/pci/intel/ipu6/ipu6-isys-queue.c    | 42 ++++++---------
 .../media/pci/intel/ipu6/ipu6-isys-queue.h    |  2 +
 .../media/pci/intel/ipu6/ipu6-isys-subdev.c   | 22 +++++---
 .../media/pci/intel/ipu6/ipu6-isys-subdev.h   |  2 +
 .../media/pci/intel/ipu6/ipu6-isys-video.c    | 48 +++++------------
 .../media/pci/intel/ipu6/ipu6-isys-video.h    |  7 ++-
 drivers/media/pci/intel/ipu6/ipu7-fw-isys.c   |  2 +-
 9 files changed, 99 insertions(+), 82 deletions(-)

diff --git a/drivers/media/pci/intel/ipu6/ipu6-fw-isys.c b/drivers/media/pci/intel/ipu6/ipu6-fw-isys.c
index 3cad7d8f9ca9..9fc42c591dc3 100644
--- a/drivers/media/pci/intel/ipu6/ipu6-fw-isys.c
+++ b/drivers/media/pci/intel/ipu6/ipu6-fw-isys.c
@@ -774,7 +774,7 @@ static int ipu6_isys_fw_pin_cfg(struct ipu6_isys_video *av,
 	int output_pins;
 	u32 src_stream;
 
-	src_stream = ipu6_isys_get_src_stream_by_src_pad(sd, src_pad->index);
+	src_stream = __ipu6_isys_get_src_stream_by_src_pad(state, src_pad->index);
 	fmt = *v4l2_subdev_state_get_format(state, src_pad->index, src_stream);
 	v4l2_crop = *v4l2_subdev_state_get_crop(state, src_pad->index, src_stream);
 
diff --git a/drivers/media/pci/intel/ipu6/ipu6-isys-csi2.c b/drivers/media/pci/intel/ipu6/ipu6-isys-csi2.c
index c0e5afd76f6a..0fd4171b50ac 100644
--- a/drivers/media/pci/intel/ipu6/ipu6-isys-csi2.c
+++ b/drivers/media/pci/intel/ipu6/ipu6-isys-csi2.c
@@ -23,6 +23,7 @@
 #include "ipu6-isys.h"
 #include "ipu6-isys-csi2.h"
 #include "ipu6-isys-subdev.h"
+#include "ipu6-isys-video.h"
 #include "ipu6-platform-isys-csi2-reg.h"
 #include "ipu7-isys-csi2-regs.h"
 
@@ -442,10 +443,26 @@ static int ipu6_isys_csi2_enable_streams(struct v4l2_subdev *sd,
 	struct ipu6_isys_csi2 *csi2 = to_ipu6_isys_csi2(asd);
 	struct ipu6_device *isp = asd->isys->adev->isp;
 	struct v4l2_subdev *remote_sd;
-	struct media_pad *remote_pad;
+	struct media_pad *remote_pad,
+		*vdev_pad = media_pad_remote_pad_unique(&sd->entity.pads[pad]);
+	struct ipu6_isys_video *av =
+		container_of_const(vdev_pad, struct ipu6_isys_video, pad);
+	struct ipu6_isys_buffer_list bl;
 	u64 sink_streams;
 	int ret;
 
+	ret = ipu6_isys_buffer_list_get(av->stream, &bl);
+	if (ret < 0) {
+		dev_warn(sd->dev, "no buffer available, DRIVER BUG?\n");
+		return ret;
+	}
+
+	ret = ipu6_isys_start_stream_firmware(av, &bl);
+	if (ret) {
+		dev_err(sd->dev, "start stream of firmware failed\n");
+		goto err_return_buffers;
+	}
+
 	remote_pad = media_pad_remote_pad_first(&sd->entity.pads[CSI2_PAD_SINK]);
 	remote_sd = media_entity_to_v4l2_subdev(remote_pad->entity);
 
@@ -456,30 +473,43 @@ static int ipu6_isys_csi2_enable_streams(struct v4l2_subdev *sd,
 	ret = IS_IPU7(isp) ? ipu7_isys_csi2_stream_enable(csi2) :
 			     ipu6_isys_csi2_stream_enable(csi2);
 	if (ret)
-		return ret;
+		goto err_stop_stream_firmware;
 
 	ret = v4l2_subdev_enable_streams(remote_sd, remote_pad->index,
 					 sink_streams);
-	if (ret) {
-		if IS_IPU7(isp)
-			ipu7_isys_csi2_stream_disable(csi2);
-		else
-			ipu6_isys_csi2_stream_disable(csi2);
-		return ret;
-	}
+	if (ret)
+		goto err_stop_stream_csi2;
 
 	return 0;
+
+err_stop_stream_csi2:
+	if (IS_IPU7(isp))
+		ipu7_isys_csi2_stream_disable(csi2);
+	else
+		ipu6_isys_csi2_stream_disable(csi2);
+
+err_stop_stream_firmware:
+	ipu6_isys_stop_streaming_firmware(av);
+	ipu6_isys_close_streaming_firmware(av);
+
+err_return_buffers:
+	ipu6_isys_buffer_list_queue(&bl, IPU6_ISYS_BUFFER_LIST_FL_INCOMING, 0);
+
+	return ret;
 }
 
 static int ipu6_isys_csi2_disable_streams(struct v4l2_subdev *sd,
 					  struct v4l2_subdev_state *state,
 					  u32 pad, u64 streams_mask)
 {
+	struct media_pad *remote_pad,
+		*vdev_pad = media_pad_remote_pad_unique(&sd->entity.pads[pad]);
+	struct ipu6_isys_video *av =
+		container_of_const(vdev_pad, struct ipu6_isys_video, pad);
 	struct ipu6_isys_subdev *asd = to_ipu6_isys_subdev(sd);
 	struct ipu6_isys_csi2 *csi2 = to_ipu6_isys_csi2(asd);
 	struct ipu6_device *isp = asd->isys->adev->isp;
 	struct v4l2_subdev *remote_sd;
-	struct media_pad *remote_pad;
 	u64 sink_streams;
 
 	sink_streams =
@@ -489,6 +519,8 @@ static int ipu6_isys_csi2_disable_streams(struct v4l2_subdev *sd,
 	remote_pad = media_pad_remote_pad_first(&sd->entity.pads[CSI2_PAD_SINK]);
 	remote_sd = media_entity_to_v4l2_subdev(remote_pad->entity);
 
+	ipu6_isys_stop_streaming_firmware(av);
+
 	if IS_IPU7(isp)
 		ipu7_isys_csi2_stream_disable(csi2);
 	else
@@ -496,6 +528,8 @@ static int ipu6_isys_csi2_disable_streams(struct v4l2_subdev *sd,
 
 	v4l2_subdev_disable_streams(remote_sd, remote_pad->index, sink_streams);
 
+	ipu6_isys_close_streaming_firmware(av);
+
 	return 0;
 }
 
diff --git a/drivers/media/pci/intel/ipu6/ipu6-isys-queue.c b/drivers/media/pci/intel/ipu6/ipu6-isys-queue.c
index 4d862f14c765..91043b531821 100644
--- a/drivers/media/pci/intel/ipu6/ipu6-isys-queue.c
+++ b/drivers/media/pci/intel/ipu6/ipu6-isys-queue.c
@@ -21,6 +21,7 @@
 #include "ipu6-dma.h"
 #include "ipu6-fw-isys.h"
 #include "ipu6-isys.h"
+#include "ipu6-isys-queue.h"
 #include "ipu6-isys-video.h"
 
 static int ipu6_isys_buf_init(struct vb2_buffer *vb)
@@ -191,8 +192,8 @@ static void flush_firmware_streamon_fail(struct ipu6_isys_stream *stream)
  * that contains one entry from each video buffer queue. If a buffer can't be
  * obtained from every queue, the buffers are returned back to the queue.
  */
-static int buffer_list_get(struct ipu6_isys_stream *stream,
-			   struct ipu6_isys_buffer_list *bl)
+int ipu6_isys_buffer_list_get(struct ipu6_isys_stream *stream,
+			      struct ipu6_isys_buffer_list *bl)
 {
 	struct device *dev = &stream->isys->adev->auxdev.dev;
 	struct ipu6_isys_queue *aq;
@@ -234,28 +235,25 @@ static int buffer_list_get(struct ipu6_isys_stream *stream,
 }
 
 /* Start streaming for real. The buffer list must be available. */
-static int ipu6_isys_stream_start(struct ipu6_isys_video *av,
-				  struct ipu6_isys_buffer_list *bl)
+static int ipu6_isys_stream_start(struct ipu6_isys_video *av)
 {
 	struct ipu6_bus_device *adev = av->isys->adev;
 	const struct ipu6_fw_isys_ops *fw_ops = adev->auxdrv_data->fw_ops;
 	struct ipu6_isys_stream *stream = av->stream;
 	struct device *dev = &adev->auxdev.dev;
-	struct ipu6_isys_buffer_list __bl;
+	struct ipu6_isys_buffer_list bl;
 	struct isys_fw_msgs *msg;
 	int ret;
 
 	guard(mutex)(&stream->isys->stream_mutex);
-	ret = ipu6_isys_video_set_streaming(av, 1, bl);
+	ret = ipu6_isys_video_set_streaming(av, 1);
 	if (ret)
-		goto out_requeue;
+		return ret;
 
 	stream->streaming = 1;
 
-	bl = &__bl;
-
 	do {
-		ret = buffer_list_get(stream, bl);
+		ret = ipu6_isys_buffer_list_get(stream, &bl);
 		if (ret < 0)
 			return 0;
 
@@ -263,10 +261,10 @@ static int ipu6_isys_stream_start(struct ipu6_isys_video *av,
 		if (WARN_ON(!msg))
 			goto out_requeue;
 
-		fw_ops->prepare_buf_set(msg, stream, bl);
+		fw_ops->prepare_buf_set(msg, stream, &bl);
 		fw_ops->dump_frame_buf_set(dev, msg, stream->nr_output_pins);
-		ipu6_isys_buffer_list_queue(bl, IPU6_ISYS_BUFFER_LIST_FL_ACTIVE,
-					    0);
+		ipu6_isys_buffer_list_queue(&bl,
+					    IPU6_ISYS_BUFFER_LIST_FL_ACTIVE, 0);
 		ret = fw_ops->stream_capture(stream->isys,
 					     stream->stream_handle, msg);
 		if (WARN_ON(ret))
@@ -277,8 +275,8 @@ static int ipu6_isys_stream_start(struct ipu6_isys_video *av,
 	ipu6_put_fw_msg_buf(stream->isys, msg);
 
 out_requeue:
-	if (bl && bl->nbufs)
-		ipu6_isys_buffer_list_queue(bl,
+	if (bl.nbufs)
+		ipu6_isys_buffer_list_queue(&bl,
 					    IPU6_ISYS_BUFFER_LIST_FL_INCOMING,
 					    VB2_BUF_STATE_QUEUED);
 	flush_firmware_streamon_fail(stream);
@@ -331,7 +329,7 @@ static void buf_queue(struct vb2_buffer *vb)
 	 * (above). Let's see whether all queues in the pipeline would
 	 * have a buffer.
 	 */
-	ret = buffer_list_get(stream, &bl);
+	ret = ipu6_isys_buffer_list_get(stream, &bl);
 	if (ret < 0) {
 		dev_dbg(dev, "No buffers available\n");
 		goto out;
@@ -480,7 +478,6 @@ static int start_streaming(struct vb2_queue *q, unsigned int count)
 	struct device *dev = &av->isys->adev->auxdev.dev;
 	const struct ipu6_isys_pixelformat *pfmt =
 		ipu6_isys_get_isys_format(ipu6_isys_get_format(av), 0);
-	struct ipu6_isys_buffer_list __bl, *bl = NULL;
 	struct ipu6_isys_stream *stream;
 	struct media_pad *source_pad, *remote_pad;
 	int nr_queues, ret;
@@ -541,14 +538,7 @@ static int start_streaming(struct vb2_queue *q, unsigned int count)
 	if (stream->nr_streaming != stream->nr_queues)
 		goto out;
 
-	bl = &__bl;
-	ret = buffer_list_get(stream, bl);
-	if (ret < 0) {
-		dev_warn(dev, "no buffer available, DRIVER BUG?\n");
-		goto out;
-	}
-
-	ret = ipu6_isys_stream_start(av, bl);
+	ret = ipu6_isys_stream_start(av);
 	if (ret)
 		goto out_stream_start;
 
@@ -587,7 +577,7 @@ static void stop_streaming(struct vb2_queue *q)
 
 	mutex_lock(&av->isys->stream_mutex);
 	if (stream->nr_streaming == stream->nr_queues && stream->streaming)
-		ipu6_isys_video_set_streaming(av, 0, NULL);
+		ipu6_isys_video_set_streaming(av, 0);
 	list_del(&aq->node);
 	mutex_unlock(&av->isys->stream_mutex);
 
diff --git a/drivers/media/pci/intel/ipu6/ipu6-isys-queue.h b/drivers/media/pci/intel/ipu6/ipu6-isys-queue.h
index 5df47be0c181..0e0886f59150 100644
--- a/drivers/media/pci/intel/ipu6/ipu6-isys-queue.h
+++ b/drivers/media/pci/intel/ipu6/ipu6-isys-queue.h
@@ -60,6 +60,8 @@ struct ipu6_isys_buffer_list {
 void ipu6_isys_buffer_list_queue(struct ipu6_isys_buffer_list *bl,
 				 unsigned long op_flags,
 				 enum vb2_buffer_state state);
+int ipu6_isys_buffer_list_get(struct ipu6_isys_stream *stream,
+			      struct ipu6_isys_buffer_list *bl);
 void ipu6_isys_queue_buf_ready(struct ipu6_isys_stream *stream,
 			       struct ipu6_fw_isys_resp_info_abi *info);
 int ipu6_isys_queue_init(struct ipu6_isys_queue *aq);
diff --git a/drivers/media/pci/intel/ipu6/ipu6-isys-subdev.c b/drivers/media/pci/intel/ipu6/ipu6-isys-subdev.c
index ab04b5b0ad08..60070842ee68 100644
--- a/drivers/media/pci/intel/ipu6/ipu6-isys-subdev.c
+++ b/drivers/media/pci/intel/ipu6/ipu6-isys-subdev.c
@@ -266,17 +266,13 @@ static int subdev_set_routing(struct v4l2_subdev *sd,
 	return v4l2_subdev_set_routing_with_fmt(sd, state, routing, &format);
 }
 
-u32 ipu6_isys_get_src_stream_by_src_pad(struct v4l2_subdev *sd, u32 pad)
+u32 __ipu6_isys_get_src_stream_by_src_pad(struct v4l2_subdev_state *state,
+					  u32 pad)
 {
-	struct v4l2_subdev_state *state;
 	struct v4l2_subdev_route *routes;
 	unsigned int i;
 	u32 source_stream = 0;
 
-	state = v4l2_subdev_lock_and_get_active_state(sd);
-	if (!state)
-		return 0;
-
 	routes = state->routing.routes;
 	for (i = 0; i < state->routing.num_routes; i++) {
 		if (routes[i].source_pad == pad) {
@@ -285,6 +281,20 @@ u32 ipu6_isys_get_src_stream_by_src_pad(struct v4l2_subdev *sd, u32 pad)
 		}
 	}
 
+	return source_stream;
+}
+
+u32 ipu6_isys_get_src_stream_by_src_pad(struct v4l2_subdev *sd, u32 pad)
+{
+	struct v4l2_subdev_state *state;
+	u32 source_stream = 0;
+
+	state = v4l2_subdev_lock_and_get_active_state(sd);
+	if (!state)
+		return 0;
+
+	source_stream = __ipu6_isys_get_src_stream_by_src_pad(state, pad);
+
 	v4l2_subdev_unlock_state(state);
 
 	return source_stream;
diff --git a/drivers/media/pci/intel/ipu6/ipu6-isys-subdev.h b/drivers/media/pci/intel/ipu6/ipu6-isys-subdev.h
index d4f76d513dc6..b892d96992af 100644
--- a/drivers/media/pci/intel/ipu6/ipu6-isys-subdev.h
+++ b/drivers/media/pci/intel/ipu6/ipu6-isys-subdev.h
@@ -38,6 +38,8 @@ int ipu6_isys_subdev_enum_mbus_code(struct v4l2_subdev *sd,
 				    struct v4l2_subdev_state *state,
 				    struct v4l2_subdev_mbus_code_enum
 				    *code);
+u32 __ipu6_isys_get_src_stream_by_src_pad(struct v4l2_subdev_state *state,
+					  u32 pad);
 u32 ipu6_isys_get_src_stream_by_src_pad(struct v4l2_subdev *sd, u32 pad);
 int ipu6_isys_subdev_set_routing(struct v4l2_subdev *sd,
 				 struct v4l2_subdev_state *state,
diff --git a/drivers/media/pci/intel/ipu6/ipu6-isys-video.c b/drivers/media/pci/intel/ipu6/ipu6-isys-video.c
index 9634b4f232bd..b63fb9673604 100644
--- a/drivers/media/pci/intel/ipu6/ipu6-isys-video.c
+++ b/drivers/media/pci/intel/ipu6/ipu6-isys-video.c
@@ -450,8 +450,8 @@ static void put_stream_opened(struct ipu6_isys_video *av)
 	spin_unlock_irqrestore(&av->isys->streams_lock, flags);
 }
 
-static int start_stream_firmware(struct ipu6_isys_video *av,
-				 struct ipu6_isys_buffer_list *bl)
+int ipu6_isys_start_stream_firmware(struct ipu6_isys_video *av,
+				    struct ipu6_isys_buffer_list *bl)
 {
 	struct ipu6_bus_device *adev = av->isys->adev;
 	const struct ipu6_fw_isys_ops *fw_ops = adev->auxdrv_data->fw_ops;
@@ -499,23 +499,18 @@ static int start_stream_firmware(struct ipu6_isys_video *av,
 	}
 	dev_dbg(dev, "start stream: open complete\n");
 
-	if (bl) {
-		msg = ipu6_get_fw_msg_buf(stream);
-		if (!msg) {
-			ret = -ENOMEM;
-			goto out_put_stream_opened;
-		}
-
-		fw_ops->prepare_buf_set(msg, stream, bl);
-		ipu6_isys_buffer_list_queue(bl,
-					    IPU6_ISYS_BUFFER_LIST_FL_ACTIVE, 0);
+	msg = ipu6_get_fw_msg_buf(stream);
+	if (!msg) {
+		ret = -ENOMEM;
+		goto out_put_stream_opened;
 	}
+	fw_ops->prepare_buf_set(msg, stream, bl);
+	ipu6_isys_buffer_list_queue(bl, IPU6_ISYS_BUFFER_LIST_FL_ACTIVE, 0);
 
 	reinit_completion(&stream->stream_start_completion);
 
 	ret = fw_ops->stream_start(av->isys, stream->stream_handle, msg,
 				   capture);
-
 	if (ret < 0) {
 		dev_err(dev, "can't start streaming (%d)\n", ret);
 		goto out_stream_close;
@@ -561,7 +556,7 @@ static int start_stream_firmware(struct ipu6_isys_video *av,
 	return ret;
 }
 
-static void stop_streaming_firmware(struct ipu6_isys_video *av)
+void ipu6_isys_stop_streaming_firmware(struct ipu6_isys_video *av)
 {
 	struct ipu6_bus_device *adev = av->isys->adev;
 	const struct ipu6_fw_isys_ops *fw_ops = adev->auxdrv_data->fw_ops;
@@ -587,7 +582,7 @@ static void stop_streaming_firmware(struct ipu6_isys_video *av)
 		dev_dbg(dev, "stop stream: complete\n");
 }
 
-static void close_streaming_firmware(struct ipu6_isys_video *av)
+void ipu6_isys_close_streaming_firmware(struct ipu6_isys_video *av)
 {
 	struct ipu6_bus_device *adev = av->isys->adev;
 	const struct ipu6_fw_isys_ops *fw_ops = adev->auxdrv_data->fw_ops;
@@ -889,8 +884,7 @@ static u64 get_stream_mask_by_pipeline(struct ipu6_isys_video *__av)
 	return stream_mask;
 }
 
-int ipu6_isys_video_set_streaming(struct ipu6_isys_video *av, int state,
-				  struct ipu6_isys_buffer_list *bl)
+int ipu6_isys_video_set_streaming(struct ipu6_isys_video *av, int state)
 {
 	struct v4l2_subdev_krouting *routing;
 	struct ipu6_isys_stream *stream = av->stream;
@@ -920,8 +914,6 @@ int ipu6_isys_video_set_streaming(struct ipu6_isys_video *av, int state,
 
 	stream_mask = get_stream_mask_by_pipeline(av);
 	if (!state) {
-		stop_streaming_firmware(av);
-
 		/* stop sub-device which connects with video */
 		dev_dbg(dev, "stream off entity %s pad:%d mask:0x%llx\n",
 			sd->name, r_pad->index, stream_mask);
@@ -930,34 +922,18 @@ int ipu6_isys_video_set_streaming(struct ipu6_isys_video *av, int state,
 		if (ret)
 			dev_err(dev, "stream off %s failed with %d\n", sd->name,
 				ret);
-
-		close_streaming_firmware(av);
 	} else {
-		ret = start_stream_firmware(av, bl);
-		if (ret) {
-			dev_err(dev, "start stream of firmware failed\n");
-			return ret;
-		}
-
 		/* start sub-device which connects with video */
 		dev_dbg(dev, "stream on %s pad %d mask 0x%llx\n", sd->name,
 			r_pad->index, stream_mask);
 		ret = v4l2_subdev_enable_streams(sd, r_pad->index, stream_mask);
-		if (ret) {
+		if (ret)
 			dev_err(dev, "stream on %s failed with %d\n", sd->name,
 				ret);
-			goto out_media_entity_stop_streaming_firmware;
-		}
 	}
 
 	av->streaming = state;
 
-	return 0;
-
-out_media_entity_stop_streaming_firmware:
-	stop_streaming_firmware(av);
-	close_streaming_firmware(av);
-
 	return ret;
 }
 
diff --git a/drivers/media/pci/intel/ipu6/ipu6-isys-video.h b/drivers/media/pci/intel/ipu6/ipu6-isys-video.h
index a7157646d563..5e0ce7979cfd 100644
--- a/drivers/media/pci/intel/ipu6/ipu6-isys-video.h
+++ b/drivers/media/pci/intel/ipu6/ipu6-isys-video.h
@@ -105,11 +105,14 @@ extern const struct ipu6_isys_pixelformat ipu6_isys_pfmts_packed[];
 
 const struct ipu6_isys_pixelformat *
 ipu6_isys_get_isys_format(u32 pixelformat, u32 code);
+int ipu6_isys_start_stream_firmware(struct ipu6_isys_video *av,
+				    struct ipu6_isys_buffer_list *bl);
+void ipu6_isys_stop_streaming_firmware(struct ipu6_isys_video *av);
+void ipu6_isys_close_streaming_firmware(struct ipu6_isys_video *av);
 int ipu6_isys_video_prepare_stream(struct ipu6_isys_video *av,
 				   struct media_entity *source_entity,
 				   int nr_queues);
-int ipu6_isys_video_set_streaming(struct ipu6_isys_video *av, int state,
-				  struct ipu6_isys_buffer_list *bl);
+int ipu6_isys_video_set_streaming(struct ipu6_isys_video *av, int state);
 int ipu6_isys_fw_open(struct ipu6_isys *isys);
 void ipu6_isys_fw_close(struct ipu6_isys *isys);
 int ipu6_isys_setup_video(struct ipu6_isys_video *av,
diff --git a/drivers/media/pci/intel/ipu6/ipu7-fw-isys.c b/drivers/media/pci/intel/ipu6/ipu7-fw-isys.c
index aee9227fd66c..e75ae5f5f939 100644
--- a/drivers/media/pci/intel/ipu6/ipu7-fw-isys.c
+++ b/drivers/media/pci/intel/ipu6/ipu7-fw-isys.c
@@ -173,7 +173,7 @@ static int ipu7_isys_fw_pin_cfg(struct ipu6_isys_video *av,
 	int output_pins;
 	u32 src_stream;
 
-	src_stream = ipu6_isys_get_src_stream_by_src_pad(sd, src_pad->index);
+	src_stream = __ipu6_isys_get_src_stream_by_src_pad(state, src_pad->index);
 	fmt = *v4l2_subdev_state_get_format(state, src_pad->index, src_stream);
 
 	input_pin = &cfg->input_pins[input_pins];
-- 
2.47.3

[PATCH v2 03/21] media: ipu6: Stream number on CSI-2 receiver source pads is always 0

[Sakari Ailus]

The stream number on the video nodes' single pad is always 0, thus the
stream number of the CSI-2 receivers' source pads is also 0. Remove the
complexity that is there to come up with 0.

Signed-off-by: Sakari Ailus <sakari.ailus@linux.intel.com>
---
 .../media/pci/intel/ipu6/ipu6-isys-video.c    | 49 ++-----------------
 1 file changed, 5 insertions(+), 44 deletions(-)

diff --git a/drivers/media/pci/intel/ipu6/ipu6-isys-video.c b/drivers/media/pci/intel/ipu6/ipu6-isys-video.c
index b63fb9673604..b15c24b1b710 100644
--- a/drivers/media/pci/intel/ipu6/ipu6-isys-video.c
+++ b/drivers/media/pci/intel/ipu6/ipu6-isys-video.c
@@ -867,66 +867,27 @@ ipu6_isys_query_stream_by_source(struct ipu6_isys *isys, int source, u8 vc)
 	return stream;
 }
 
-static u64 get_stream_mask_by_pipeline(struct ipu6_isys_video *__av)
-{
-	struct media_pipeline *pipeline =
-		media_entity_pipeline(&__av->vdev.entity);
-	unsigned int i;
-	u64 stream_mask = 0;
-
-	for (i = 0; i < NR_OF_CSI2_SRC_PADS; i++) {
-		struct ipu6_isys_video *av = &__av->csi2->av[i];
-
-		if (pipeline == media_entity_pipeline(&av->vdev.entity))
-			stream_mask |= BIT_ULL(av->source_stream);
-	}
-
-	return stream_mask;
-}
-
 int ipu6_isys_video_set_streaming(struct ipu6_isys_video *av, int state)
 {
-	struct v4l2_subdev_krouting *routing;
-	struct ipu6_isys_stream *stream = av->stream;
-	struct v4l2_subdev_state *subdev_state;
 	struct device *dev = &av->isys->adev->auxdev.dev;
 	struct v4l2_subdev *sd;
 	struct media_pad *r_pad;
-	u32 sink_pad, sink_stream;
-	u64 r_stream;
-	u64 stream_mask = 0;
 	int ret = 0;
 
-	dev_dbg(dev, "set stream: %d\n", state);
-
-	sd = &stream->asd->sd;
+	sd = &av->stream->asd->sd;
 	r_pad = media_pad_remote_pad_first(&av->pad);
-	r_stream = ipu6_isys_get_src_stream_by_src_pad(sd, r_pad->index);
-
-	subdev_state = v4l2_subdev_lock_and_get_active_state(sd);
-	routing = &subdev_state->routing;
-	ret = v4l2_subdev_routing_find_opposite_end(routing, r_pad->index,
-						    r_stream, &sink_pad,
-						    &sink_stream);
-	v4l2_subdev_unlock_state(subdev_state);
-	if (ret)
-		return ret;
 
-	stream_mask = get_stream_mask_by_pipeline(av);
 	if (!state) {
 		/* stop sub-device which connects with video */
-		dev_dbg(dev, "stream off entity %s pad:%d mask:0x%llx\n",
-			sd->name, r_pad->index, stream_mask);
-		ret = v4l2_subdev_disable_streams(sd, r_pad->index,
-						  stream_mask);
+		dev_dbg(dev, "stream off %s pad:%d\n", sd->name, r_pad->index);
+		ret = v4l2_subdev_disable_streams(sd, r_pad->index, 1);
 		if (ret)
 			dev_err(dev, "stream off %s failed with %d\n", sd->name,
 				ret);
 	} else {
 		/* start sub-device which connects with video */
-		dev_dbg(dev, "stream on %s pad %d mask 0x%llx\n", sd->name,
-			r_pad->index, stream_mask);
-		ret = v4l2_subdev_enable_streams(sd, r_pad->index, stream_mask);
+		dev_dbg(dev, "stream on %s pad %d\n", sd->name, r_pad->index);
+		ret = v4l2_subdev_enable_streams(sd, r_pad->index, 1);
 		if (ret)
 			dev_err(dev, "stream on %s failed with %d\n", sd->name,
 				ret);
-- 
2.47.3

[PATCH v2 04/21] media: ipu6: Rename misnamed out_free_watermark label in video init

[Sakari Ailus]

ipu6_isys_video_init() has a label in the error path where it is used to
destroy a mutex. Rename the label as err_mutex_destroy.

Signed-off-by: Sakari Ailus <sakari.ailus@linux.intel.com>
---
 drivers/media/pci/intel/ipu6/ipu6-isys-video.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/drivers/media/pci/intel/ipu6/ipu6-isys-video.c b/drivers/media/pci/intel/ipu6/ipu6-isys-video.c
index b15c24b1b710..781fc0e88417 100644
--- a/drivers/media/pci/intel/ipu6/ipu6-isys-video.c
+++ b/drivers/media/pci/intel/ipu6/ipu6-isys-video.c
@@ -1107,7 +1107,7 @@ int ipu6_isys_video_init(struct ipu6_isys_video *av)
 
 	ret = ipu6_isys_queue_init(&av->aq);
 	if (ret)
-		goto out_free_watermark;
+		goto out_mutex_destroy;
 
 	av->pad.flags = MEDIA_PAD_FL_SINK | MEDIA_PAD_FL_MUST_CONNECT;
 	ret = media_entity_pads_init(&av->vdev.entity, 1, &av->pad);
@@ -1144,7 +1144,7 @@ int ipu6_isys_video_init(struct ipu6_isys_video *av)
 out_vb2_queue_release:
 	vb2_queue_release(&av->aq.vbq);
 
-out_free_watermark:
+out_mutex_destroy:
 	mutex_destroy(&av->mutex);
 
 	return ret;
-- 
2.47.3

[PATCH v2 05/21] media: ipu6: Always request a capture ack

[Sakari Ailus]

Always request a capture ack from a queued buffer. If streaming is
disabled, no buffers would have been queued in the first place. Remove the
streaming field from struct ipu6_isys_stream as well.

Signed-off-by: Sakari Ailus <sakari.ailus@linux.intel.com>
---
 drivers/media/pci/intel/ipu6/ipu6-fw-isys.c    | 5 +----
 drivers/media/pci/intel/ipu6/ipu6-isys-queue.c | 5 +----
 drivers/media/pci/intel/ipu6/ipu6-isys-video.h | 1 -
 3 files changed, 2 insertions(+), 9 deletions(-)

diff --git a/drivers/media/pci/intel/ipu6/ipu6-fw-isys.c b/drivers/media/pci/intel/ipu6/ipu6-fw-isys.c
index 9fc42c591dc3..2c71bf27a91d 100644
--- a/drivers/media/pci/intel/ipu6/ipu6-fw-isys.c
+++ b/drivers/media/pci/intel/ipu6/ipu6-fw-isys.c
@@ -934,10 +934,7 @@ static void ipu6_fw_isys_prepare_buf_set(struct isys_fw_msgs *msg,
 	set->send_irq_eof = 0;
 	set->send_resp_eof = 0;
 
-	if (stream->streaming)
-		set->send_irq_capture_ack = 0;
-	else
-		set->send_irq_capture_ack = 1;
+	set->send_irq_capture_ack = 1;
 	set->send_irq_capture_done = 0;
 
 	set->send_resp_capture_ack = 1;
diff --git a/drivers/media/pci/intel/ipu6/ipu6-isys-queue.c b/drivers/media/pci/intel/ipu6/ipu6-isys-queue.c
index 91043b531821..4754e95e4007 100644
--- a/drivers/media/pci/intel/ipu6/ipu6-isys-queue.c
+++ b/drivers/media/pci/intel/ipu6/ipu6-isys-queue.c
@@ -250,8 +250,6 @@ static int ipu6_isys_stream_start(struct ipu6_isys_video *av)
 	if (ret)
 		return ret;
 
-	stream->streaming = 1;
-
 	do {
 		ret = ipu6_isys_buffer_list_get(stream, &bl);
 		if (ret < 0)
@@ -576,13 +574,12 @@ static void stop_streaming(struct vb2_queue *q)
 	ipu6_isys_update_stream_watermark(av, false);
 
 	mutex_lock(&av->isys->stream_mutex);
-	if (stream->nr_streaming == stream->nr_queues && stream->streaming)
+	if (stream->nr_streaming == stream->nr_queues)
 		ipu6_isys_video_set_streaming(av, 0);
 	list_del(&aq->node);
 	mutex_unlock(&av->isys->stream_mutex);
 
 	stream->nr_streaming--;
-	stream->streaming = 0;
 	mutex_unlock(&stream->mutex);
 
 	ipu6_isys_stream_cleanup(av);
diff --git a/drivers/media/pci/intel/ipu6/ipu6-isys-video.h b/drivers/media/pci/intel/ipu6/ipu6-isys-video.h
index 5e0ce7979cfd..5e522686135f 100644
--- a/drivers/media/pci/intel/ipu6/ipu6-isys-video.h
+++ b/drivers/media/pci/intel/ipu6/ipu6-isys-video.h
@@ -54,7 +54,6 @@ struct ipu6_isys_stream {
 
 	int nr_queues;	/* Number of capture queues */
 	int nr_streaming;
-	int streaming;	/* Has streaming been really started? */
 	struct list_head queues;
 	struct completion stream_open_completion;
 	struct completion stream_close_completion;
-- 
2.47.3

[PATCH v2 06/21] media: ipu6: Clean up link frequency calculation

[Sakari Ailus]

Handle link_freq <= as a special case and continue calculating the pixel
rate otherwise.

Signed-off-by: Sakari Ailus <sakari.ailus@linux.intel.com>
---
 .../media/pci/intel/ipu6/ipu6-isys-video.c    | 37 ++++++++-----------
 1 file changed, 16 insertions(+), 21 deletions(-)

diff --git a/drivers/media/pci/intel/ipu6/ipu6-isys-video.c b/drivers/media/pci/intel/ipu6/ipu6-isys-video.c
index 781fc0e88417..3d455872f29a 100644
--- a/drivers/media/pci/intel/ipu6/ipu6-isys-video.c
+++ b/drivers/media/pci/intel/ipu6/ipu6-isys-video.c
@@ -648,12 +648,9 @@ void ipu6_isys_configure_stream_watermark(struct ipu6_isys_video *av,
 	struct ipu6_isys_csi2 *csi2 = NULL;
 	struct isys_iwake_watermark *iwake_watermark = &isys->iwake_watermark;
 	struct device *dev = &isys->adev->auxdev.dev;
-	struct v4l2_mbus_framefmt format;
 	struct v4l2_subdev *esd;
 	struct v4l2_control hb = { .id = V4L2_CID_HBLANK, .value = 0 };
-	unsigned int bpp, lanes;
-	s64 link_freq = 0;
-	u64 pixel_rate = 0;
+	s64 link_freq;
 	int ret;
 
 	esd = media_entity_to_v4l2_subdev(source);
@@ -671,28 +668,26 @@ void ipu6_isys_configure_stream_watermark(struct ipu6_isys_video *av,
 
 	csi2 = ipu6_isys_subdev_to_csi2(av->stream->asd);
 	link_freq = ipu6_isys_csi2_get_link_freq(csi2);
-	if (link_freq > 0) {
-		struct v4l2_subdev_state *state =
-			v4l2_subdev_lock_and_get_active_state(&csi2->asd.sd);
-
-		lanes = csi2->nlanes;
-		format = *v4l2_subdev_state_get_format(state, 0,
-						       av->source_stream);
-		bpp = ipu6_isys_mbus_code_to_bpp(format.code);
-		pixel_rate = mul_u64_u32_div(link_freq, lanes * 2, bpp);
-
-		v4l2_subdev_unlock_state(state);
-	}
-
-	av->watermark.pixel_rate = pixel_rate;
-
-	if (!pixel_rate) {
+	if (link_freq <= 0) {
 		mutex_lock(&iwake_watermark->mutex);
 		iwake_watermark->force_iwake_disable = true;
 		mutex_unlock(&iwake_watermark->mutex);
-		dev_warn(dev, "unexpected pixel_rate from %s, disable iwake.\n",
+		dev_warn(dev, "unexpected link_freq from %s, disable iwake\n",
 			 source->name);
+		return;
 	}
+
+	struct v4l2_subdev_state *state;
+	struct v4l2_mbus_framefmt *format;
+	unsigned int bpp;
+
+	state = v4l2_subdev_lock_and_get_active_state(&csi2->asd.sd);
+	format = v4l2_subdev_state_get_format(state, 0, av->source_stream);
+	bpp = ipu6_isys_mbus_code_to_bpp(format->code);
+	v4l2_subdev_unlock_state(state);
+
+	av->watermark.pixel_rate = mul_u64_u32_div(link_freq, csi2->nlanes * 2,
+						   bpp);
 }
 
 static void calculate_stream_datarate(struct ipu6_isys_video *av)
-- 
2.47.3

[PATCH v2 07/21] media: ipu6: Get watermark configuration directly from ipdata

[Sakari Ailus]

Instead of passing watermark configuration data around, get it directly
from IPU variant specific static data structure.

Signed-off-by: Sakari Ailus <sakari.ailus@linux.intel.com>
---
 drivers/media/pci/intel/ipu6/ipu6-isys-video.c | 12 ++++--------
 drivers/media/pci/intel/ipu6/ipu6-isys-video.h |  2 --
 2 files changed, 4 insertions(+), 10 deletions(-)

diff --git a/drivers/media/pci/intel/ipu6/ipu6-isys-video.c b/drivers/media/pci/intel/ipu6/ipu6-isys-video.c
index 3d455872f29a..b3cbd4266da1 100644
--- a/drivers/media/pci/intel/ipu6/ipu6-isys-video.c
+++ b/drivers/media/pci/intel/ipu6/ipu6-isys-video.c
@@ -657,8 +657,6 @@ void ipu6_isys_configure_stream_watermark(struct ipu6_isys_video *av,
 
 	av->watermark.width = ipu6_isys_get_frame_width(av);
 	av->watermark.height = ipu6_isys_get_frame_height(av);
-	av->watermark.sram_gran_shift = isys->pdata->ipdata->sram_gran_shift;
-	av->watermark.sram_gran_size = isys->pdata->ipdata->sram_gran_size;
 
 	ret = v4l2_g_ctrl(esd->ctrl_handler, &hb);
 	if (!ret && hb.value >= 0)
@@ -697,17 +695,15 @@ static void calculate_stream_datarate(struct ipu6_isys_video *av)
 		ipu6_isys_get_isys_format(ipu6_isys_get_format(av), 0);
 	u32 pages_per_line, pb_bytes_per_line, pixels_per_line, bytes_per_line;
 	u64 line_time_ns, stream_data_rate;
-	u16 shift, size;
-
-	shift = watermark->sram_gran_shift;
-	size = watermark->sram_gran_size;
 
 	pixels_per_line = watermark->width + watermark->hblank;
 	line_time_ns =  div_u64(pixels_per_line * NSEC_PER_SEC,
 				watermark->pixel_rate);
 	bytes_per_line = watermark->width * pfmt->bpp / 8;
-	pages_per_line = DIV_ROUND_UP(bytes_per_line, size);
-	pb_bytes_per_line = pages_per_line << shift;
+	pages_per_line = DIV_ROUND_UP(bytes_per_line,
+				      av->isys->pdata->ipdata->sram_gran_size);
+	pb_bytes_per_line =
+		pages_per_line << av->isys->pdata->ipdata->sram_gran_shift;
 	stream_data_rate = div64_u64(pb_bytes_per_line * 1000, line_time_ns);
 
 	watermark->stream_data_rate = stream_data_rate;
diff --git a/drivers/media/pci/intel/ipu6/ipu6-isys-video.h b/drivers/media/pci/intel/ipu6/ipu6-isys-video.h
index 5e522686135f..4c890c6af8f5 100644
--- a/drivers/media/pci/intel/ipu6/ipu6-isys-video.h
+++ b/drivers/media/pci/intel/ipu6/ipu6-isys-video.h
@@ -73,8 +73,6 @@ struct video_stream_watermark {
 	u32 frame_rate;
 	u64 pixel_rate;
 	u64 stream_data_rate;
-	u16 sram_gran_shift;
-	u16 sram_gran_size;
 	struct list_head stream_node;
 };
 
-- 
2.47.3

[PATCH v2 08/21] media: ipu6: Collect IPU streams into CSI-2 receiver sub-device context

[Sakari Ailus]

Collect the IPU streams to the CSI-2 receiver sub-device's context.

The background is that an IPU stream is bound to a CSI-2 virtual channel
but in a V4L2 pipeline there's no such assumption. Therefore a pipeline
may contain multiple virtual channels and so multiple IPU streams, too.

Signed-off-by: Sakari Ailus <sakari.ailus@linux.intel.com>
---
 drivers/media/pci/intel/ipu6/ipu6-isys-csi2.c  | 6 ++++++
 drivers/media/pci/intel/ipu6/ipu6-isys-csi2.h  | 1 +
 drivers/media/pci/intel/ipu6/ipu6-isys-video.h | 1 +
 3 files changed, 8 insertions(+)

diff --git a/drivers/media/pci/intel/ipu6/ipu6-isys-csi2.c b/drivers/media/pci/intel/ipu6/ipu6-isys-csi2.c
index 0fd4171b50ac..f92577054d89 100644
--- a/drivers/media/pci/intel/ipu6/ipu6-isys-csi2.c
+++ b/drivers/media/pci/intel/ipu6/ipu6-isys-csi2.c
@@ -457,6 +457,8 @@ static int ipu6_isys_csi2_enable_streams(struct v4l2_subdev *sd,
 		return ret;
 	}
 
+	list_add(&av->csi2_entry, &csi2->av_head);
+
 	ret = ipu6_isys_start_stream_firmware(av, &bl);
 	if (ret) {
 		dev_err(sd->dev, "start stream of firmware failed\n");
@@ -493,6 +495,7 @@ static int ipu6_isys_csi2_enable_streams(struct v4l2_subdev *sd,
 	ipu6_isys_close_streaming_firmware(av);
 
 err_return_buffers:
+	list_del(&av->csi2_entry);
 	ipu6_isys_buffer_list_queue(&bl, IPU6_ISYS_BUFFER_LIST_FL_INCOMING, 0);
 
 	return ret;
@@ -530,6 +533,8 @@ static int ipu6_isys_csi2_disable_streams(struct v4l2_subdev *sd,
 
 	ipu6_isys_close_streaming_firmware(av);
 
+	list_del(&av->csi2_entry);
+
 	return 0;
 }
 
@@ -675,6 +680,7 @@ int ipu6_isys_csi2_init(struct ipu6_isys_csi2 *csi2,
 	if (ret)
 		goto fail;
 
+	INIT_LIST_HEAD(&csi2->av_head);
 	csi2->asd.source = IPU6_FW_ISYS_STREAM_SRC_CSI2_PORT0 + index;
 	csi2->asd.supported_codes = csi2_supported_codes;
 	snprintf(csi2->asd.sd.name, sizeof(csi2->asd.sd.name),
diff --git a/drivers/media/pci/intel/ipu6/ipu6-isys-csi2.h b/drivers/media/pci/intel/ipu6/ipu6-isys-csi2.h
index dbaaf00af623..7c936154691e 100644
--- a/drivers/media/pci/intel/ipu6/ipu6-isys-csi2.h
+++ b/drivers/media/pci/intel/ipu6/ipu6-isys-csi2.h
@@ -41,6 +41,7 @@ struct ipu6_isys_csi2 {
 	struct ipu6_isys_subdev asd;
 	struct ipu6_isys *isys;
 	struct ipu6_isys_video av[NR_OF_CSI2_SRC_PADS];
+	struct list_head av_head;
 
 	void __iomem *base;
 	u32 receiver_errors;
diff --git a/drivers/media/pci/intel/ipu6/ipu6-isys-video.h b/drivers/media/pci/intel/ipu6/ipu6-isys-video.h
index 4c890c6af8f5..cb4b45f0dae2 100644
--- a/drivers/media/pci/intel/ipu6/ipu6-isys-video.h
+++ b/drivers/media/pci/intel/ipu6/ipu6-isys-video.h
@@ -78,6 +78,7 @@ struct video_stream_watermark {
 
 struct ipu6_isys_video {
 	struct ipu6_isys_queue aq;
+	struct list_head csi2_entry;
 	/* Serialise access to other fields in the struct. */
 	struct mutex mutex;
 	struct media_pad pad;
-- 
2.47.3

[PATCH v2 09/21] media: ipu6: Start streaming once all streams have started, stop when not

[Sakari Ailus]

For each virtual channel, start streaming once all buffer queues connected
via enabled routes have been set streaming, similarly stop when the first
queue is stopped.

Signed-off-by: Sakari Ailus <sakari.ailus@linux.intel.com>
---
 drivers/media/pci/intel/ipu6/ipu6-isys-csi2.c | 107 +++++++++++++++++-
 drivers/media/pci/intel/ipu6/ipu6-isys-csi2.h |   1 +
 .../media/pci/intel/ipu6/ipu6-isys-queue.c    |   7 +-
 3 files changed, 105 insertions(+), 10 deletions(-)

diff --git a/drivers/media/pci/intel/ipu6/ipu6-isys-csi2.c b/drivers/media/pci/intel/ipu6/ipu6-isys-csi2.c
index f92577054d89..aa07fabb2afc 100644
--- a/drivers/media/pci/intel/ipu6/ipu6-isys-csi2.c
+++ b/drivers/media/pci/intel/ipu6/ipu6-isys-csi2.c
@@ -12,6 +12,7 @@
 #include <linux/io.h>
 #include <linux/minmax.h>
 #include <linux/sprintf.h>
+#include <linux/string_choices.h>
 
 #include <media/media-entity.h>
 #include <media/v4l2-ctrls.h>
@@ -435,6 +436,89 @@ static int ipu7_isys_csi2_stream_enable(struct ipu6_isys_csi2 *csi2)
 	return 0;
 }
 
+static bool ipu6_isys_csi2_streaming_change(struct ipu6_isys_subdev *asd,
+					    struct v4l2_subdev_state *state,
+					    u32 pad, u8 *vc, bool enable)
+{
+	u32 streams_enabled = 0, nodes_streaming = 0;
+	struct v4l2_subdev_route *route, *this_route = NULL;
+	struct v4l2_mbus_frame_desc desc = { 0 };
+	struct v4l2_mbus_frame_desc_entry *this_entry = NULL;
+	int ret;
+
+	for_each_active_route(&state->routing, this_route)
+		if (pad == this_route->source_pad)
+			break;
+	if (!this_route) {
+		dev_dbg(asd->sd.dev, "no route found for pad %u\n", pad);
+		return false;
+	}
+
+	struct media_pad *remote_pad =
+		media_pad_remote_pad_unique(&asd->sd.entity.pads[this_route->sink_pad]);
+	struct v4l2_subdev *remote_sd =
+		media_entity_to_v4l2_subdev(remote_pad->entity);
+	ret = v4l2_subdev_call(remote_sd, pad, get_frame_desc,
+			       remote_pad->index, &desc);
+	if (ret)
+		return ret;
+
+	for (unsigned int i = 0; i < desc.num_entries; i++) {
+		if (desc.entry[i].stream == this_route->sink_stream) {
+			this_entry = &desc.entry[i];
+			break;
+		}
+	}
+	if (!this_entry) {
+		dev_dbg(asd->sd.dev,
+			"no frame descriptor entry found for stream %u\n",
+			this_route->sink_stream);
+		return false;
+	}
+
+	for_each_active_route(&state->routing, route) {
+		struct v4l2_mbus_frame_desc_entry *entry = NULL;
+
+		for (unsigned int i = 0; i < desc.num_entries; i++) {
+			if (desc.entry[i].stream == this_entry->stream) {
+				entry = &desc.entry[i];
+				break;
+			}
+		}
+
+		if (entry->bus.csi2.vc != this_entry->bus.csi2.vc)
+			continue;
+
+		struct media_pad *video_pad =
+			media_pad_remote_pad_unique(&asd->sd.entity.pads[route->source_pad]);
+		struct ipu6_isys_video *av = !video_pad ? NULL :
+			container_of_const(video_pad,
+					   struct ipu6_isys_video, pad);
+
+		if (!av) {
+			dev_dbg(asd->sd.dev,
+				"can't find source pad for \"%s\":%u\n",
+				asd->sd.entity.name, route->source_pad);
+			return false;
+		}
+
+		streams_enabled++;
+		if (av->streaming || (enable && pad == route->source_pad))
+			nodes_streaming++;
+	}
+
+	*vc = this_entry->bus.csi2.vc;
+
+	if (streams_enabled == nodes_streaming) {
+		dev_dbg(asd->sd.dev,
+			"changing streaming state to %s on \"%s\":%u\n",
+			str_enabled_disabled(enable), asd->sd.entity.name, pad);
+		return true;
+	}
+
+	return false;
+}
+
 static int ipu6_isys_csi2_enable_streams(struct v4l2_subdev *sd,
 					 struct v4l2_subdev_state *state,
 					 u32 pad, u64 streams_mask)
@@ -450,6 +534,7 @@ static int ipu6_isys_csi2_enable_streams(struct v4l2_subdev *sd,
 	struct ipu6_isys_buffer_list bl;
 	u64 sink_streams;
 	int ret;
+	u8 vc;
 
 	ret = ipu6_isys_buffer_list_get(av->stream, &bl);
 	if (ret < 0) {
@@ -459,6 +544,9 @@ static int ipu6_isys_csi2_enable_streams(struct v4l2_subdev *sd,
 
 	list_add(&av->csi2_entry, &csi2->av_head);
 
+	if (!ipu6_isys_csi2_streaming_change(asd, state, pad, &vc, true))
+		return 0;
+
 	ret = ipu6_isys_start_stream_firmware(av, &bl);
 	if (ret) {
 		dev_err(sd->dev, "start stream of firmware failed\n");
@@ -472,16 +560,20 @@ static int ipu6_isys_csi2_enable_streams(struct v4l2_subdev *sd,
 		v4l2_subdev_state_xlate_streams(state, pad, CSI2_PAD_SINK,
 						&streams_mask);
 
-	ret = IS_IPU7(isp) ? ipu7_isys_csi2_stream_enable(csi2) :
-			     ipu6_isys_csi2_stream_enable(csi2);
-	if (ret)
-		goto err_stop_stream_firmware;
+	if (!csi2->streaming_vc) {
+		ret = IS_IPU7(isp) ? ipu7_isys_csi2_stream_enable(csi2) :
+				     ipu6_isys_csi2_stream_enable(csi2);
+		if (ret)
+			goto err_stop_stream_firmware;
+	}
 
 	ret = v4l2_subdev_enable_streams(remote_sd, remote_pad->index,
 					 sink_streams);
 	if (ret)
 		goto err_stop_stream_csi2;
 
+	csi2->streaming_vc |= BIT(vc);
+
 	return 0;
 
 err_stop_stream_csi2:
@@ -514,6 +606,12 @@ static int ipu6_isys_csi2_disable_streams(struct v4l2_subdev *sd,
 	struct ipu6_device *isp = asd->isys->adev->isp;
 	struct v4l2_subdev *remote_sd;
 	u64 sink_streams;
+	u8 vc;
+
+	if (!ipu6_isys_csi2_streaming_change(asd, state, pad, &vc, false))
+		goto out_del_csi2_entry;
+
+	csi2->streaming_vc &= ~BIT(vc);
 
 	sink_streams =
 		v4l2_subdev_state_xlate_streams(state, pad, CSI2_PAD_SINK,
@@ -533,6 +631,7 @@ static int ipu6_isys_csi2_disable_streams(struct v4l2_subdev *sd,
 
 	ipu6_isys_close_streaming_firmware(av);
 
+out_del_csi2_entry:
 	list_del(&av->csi2_entry);
 
 	return 0;
diff --git a/drivers/media/pci/intel/ipu6/ipu6-isys-csi2.h b/drivers/media/pci/intel/ipu6/ipu6-isys-csi2.h
index 7c936154691e..a98a60e443c6 100644
--- a/drivers/media/pci/intel/ipu6/ipu6-isys-csi2.h
+++ b/drivers/media/pci/intel/ipu6/ipu6-isys-csi2.h
@@ -49,6 +49,7 @@ struct ipu6_isys_csi2 {
 	unsigned int port;
 	u32 legacy_irq_mask;
 	unsigned int phy_mode;
+	u32 streaming_vc;
 };
 
 struct ipu6_isys_csi2_timing {
diff --git a/drivers/media/pci/intel/ipu6/ipu6-isys-queue.c b/drivers/media/pci/intel/ipu6/ipu6-isys-queue.c
index 4754e95e4007..5eb6f17bf45b 100644
--- a/drivers/media/pci/intel/ipu6/ipu6-isys-queue.c
+++ b/drivers/media/pci/intel/ipu6/ipu6-isys-queue.c
@@ -533,14 +533,10 @@ static int start_streaming(struct vb2_queue *q, unsigned int count)
 	ipu6_isys_configure_stream_watermark(av, source_pad->entity);
 	ipu6_isys_update_stream_watermark(av, true);
 
-	if (stream->nr_streaming != stream->nr_queues)
-		goto out;
-
 	ret = ipu6_isys_stream_start(av);
 	if (ret)
 		goto out_stream_start;
 
-out:
 	mutex_unlock(&stream->mutex);
 
 	return 0;
@@ -574,8 +570,7 @@ static void stop_streaming(struct vb2_queue *q)
 	ipu6_isys_update_stream_watermark(av, false);
 
 	mutex_lock(&av->isys->stream_mutex);
-	if (stream->nr_streaming == stream->nr_queues)
-		ipu6_isys_video_set_streaming(av, 0);
+	ipu6_isys_video_set_streaming(av, 0);
 	list_del(&aq->node);
 	mutex_unlock(&av->isys->stream_mutex);
 
-- 
2.47.3

Re: [PATCH v2 09/21] media: ipu6: Start streaming once all streams have started, stop when not

[Nicola Fiorillo]

Hi Sakari,

On Thu, Sep 17, 2026 at 02:39:11PM +0300, Sakari Ailus wrote:
> +static bool ipu6_isys_csi2_streaming_change(struct ipu6_isys_subdev *asd,
> +					    struct v4l2_subdev_state *state,
> +					    u32 pad, u8 *vc, bool enable)

Four things in this function, all found by reading it. I cannot build or
test a kernel at the moment, so please take this as review and nothing
more; none of it is reproduced on hardware.

1) The error return does not fit the return type.

> +	ret = v4l2_subdev_call(remote_sd, pad, get_frame_desc,
> +			       remote_pad->index, &desc);
> +	if (ret)
> +		return ret;

The function returns bool, so a negative errno from get_frame_desc()
becomes true, and both callers read true as "go ahead and change the
streaming state". In ipu6_isys_csi2_enable_streams() that starts the
firmware stream and then runs

	csi2->streaming_vc |= BIT(vc);

with vc still uninitialised: *vc is assigned only at the end of the
function, past this early return. ipu6_isys_csi2_disable_streams() does
the mirror of it with &= ~BIT(vc). The two "return false" exits are
fine, they make the caller skip the change; only this one carries a
value.

2) media_pad_remote_pad_unique() never returns NULL, so the guard below
   it is dead code.

> +		struct media_pad *video_pad =
> +			media_pad_remote_pad_unique(&asd->sd.entity.pads[route->source_pad]);
> +		struct ipu6_isys_video *av = !video_pad ? NULL :
> +			container_of_const(video_pad,
> +					   struct ipu6_isys_video, pad);
> +
> +		if (!av) {

It returns ERR_PTR(-ENOLINK) when no enabled link is found and
ERR_PTR(-ENOTUNIQ) when there is more than one. !video_pad is therefore
never true, the dev_dbg() below is unreachable, and the error pointer
goes into container_of_const() instead -- where av->streaming, a few
lines further down, dereferences it.

IS_ERR() is what the check wants. This same file already does it that
way, in ipu6_isys_csi2_get_link_freq():

	src_pad = media_pad_remote_pad_unique(...);
	if (IS_ERR(src_pad)) {
		dev_err(&csi2->isys->adev->auxdev.dev,
			"can't get source pad of %s (%pe)\n",
			csi2->asd.sd.name, src_pad);
		return PTR_ERR(src_pad);
	}

3) The lookup on the sink pad, higher up in the same function, has no
   check at all:

> +	struct media_pad *remote_pad =
> +		media_pad_remote_pad_unique(&asd->sd.entity.pads[this_route->sink_pad]);
> +	struct v4l2_subdev *remote_sd =
> +		media_entity_to_v4l2_subdev(remote_pad->entity);

remote_pad->entity is read immediately, so the same error pointer is
dereferenced here, with no guard to correct.

4) Unless I am misreading it, the inner lookup in the loop does not
   depend on the loop variable:

> +	for_each_active_route(&state->routing, route) {
> +		struct v4l2_mbus_frame_desc_entry *entry = NULL;
> +
> +		for (unsigned int i = 0; i < desc.num_entries; i++) {
> +			if (desc.entry[i].stream == this_entry->stream) {
> +				entry = &desc.entry[i];
> +				break;
> +			}
> +		}
> +
> +		if (entry->bus.csi2.vc != this_entry->bus.csi2.vc)
> +			continue;

The search key is this_entry->stream, which does not change across
iterations, so entry always ends up as this_entry itself and the vc
comparison can never differ. The "continue" is never taken and every
active route is counted, whatever its virtual channel. Going by the
commit message, the key here was meant to be route->sink_stream.

Finally, a note rather than a request. After the whole series,
ipu6_isys_csi2_enable_streams() and ipu6_isys_csi2_disable_streams()
still call media_pad_remote_pad_first() on the sink pad and dereference
the result unchecked -- and that one does return NULL. It is what my
[PATCH v2 1/3] touched. I have dropped that patch and I am not
reopening the question of the scenario behind it; I mention it only
because if the checks in 2) and 3) go in, that pointer is sitting right
next to them.

Thanks,

-- 
Nicola Fiorillo

Re: [PATCH v2 09/21] media: ipu6: Start streaming once all streams have started, stop when not

[Sakari Ailus]

Hi Nicola,

Thank you for the review.

On Fri, Sep 18, 2026 at 09:36:19AM +0200, Nicola Fiorillo wrote:
> Hi Sakari,
> 
> On Thu, Sep 17, 2026 at 02:39:11PM +0300, Sakari Ailus wrote:
> > +static bool ipu6_isys_csi2_streaming_change(struct ipu6_isys_subdev *asd,
> > +					    struct v4l2_subdev_state *state,
> > +					    u32 pad, u8 *vc, bool enable)
> 
> Four things in this function, all found by reading it. I cannot build or
> test a kernel at the moment, so please take this as review and nothing
> more; none of it is reproduced on hardware.
> 
> 1) The error return does not fit the return type.
> 
> > +	ret = v4l2_subdev_call(remote_sd, pad, get_frame_desc,
> > +			       remote_pad->index, &desc);
> > +	if (ret)
> > +		return ret;
> 
> The function returns bool, so a negative errno from get_frame_desc()
> becomes true, and both callers read true as "go ahead and change the
> streaming state". In ipu6_isys_csi2_enable_streams() that starts the

Right; this needs to be reworked a little. I'll switch the return type to
int, I probably chose bool before I realised error handling was actually
required.

> firmware stream and then runs
> 
> 	csi2->streaming_vc |= BIT(vc);
> 
> with vc still uninitialised: *vc is assigned only at the end of the
> function, past this early return. ipu6_isys_csi2_disable_streams() does
> the mirror of it with &= ~BIT(vc). The two "return false" exits are
> fine, they make the caller skip the change; only this one carries a
> value.

vc is actually set if all goes well, so this is related to error handling.

> 
> 2) media_pad_remote_pad_unique() never returns NULL, so the guard below
>    it is dead code.

Indeed. This is where others have tripped, too. I'll fix this for v3.

I think I'll just switch to media_pad_remote_pad_first() as there's just a
single one. The check also can be removed as the MUST_CONNECT pad flag
guarantees there's an enabled link there. Let's see what smatch says...

> 
> > +		struct media_pad *video_pad =
> > +			media_pad_remote_pad_unique(&asd->sd.entity.pads[route->source_pad]);
> > +		struct ipu6_isys_video *av = !video_pad ? NULL :
> > +			container_of_const(video_pad,
> > +					   struct ipu6_isys_video, pad);
> > +
> > +		if (!av) {
> 
> It returns ERR_PTR(-ENOLINK) when no enabled link is found and
> ERR_PTR(-ENOTUNIQ) when there is more than one. !video_pad is therefore
> never true, the dev_dbg() below is unreachable, and the error pointer
> goes into container_of_const() instead -- where av->streaming, a few
> lines further down, dereferences it.
> 
> IS_ERR() is what the check wants. This same file already does it that
> way, in ipu6_isys_csi2_get_link_freq():
> 
> 	src_pad = media_pad_remote_pad_unique(...);
> 	if (IS_ERR(src_pad)) {
> 		dev_err(&csi2->isys->adev->auxdev.dev,
> 			"can't get source pad of %s (%pe)\n",
> 			csi2->asd.sd.name, src_pad);
> 		return PTR_ERR(src_pad);
> 	}
> 
> 3) The lookup on the sink pad, higher up in the same function, has no
>    check at all:
> 
> > +	struct media_pad *remote_pad =
> > +		media_pad_remote_pad_unique(&asd->sd.entity.pads[this_route->sink_pad]);
> > +	struct v4l2_subdev *remote_sd =
> > +		media_entity_to_v4l2_subdev(remote_pad->entity);
> 
> remote_pad->entity is read immediately, so the same error pointer is
> dereferenced here, with no guard to correct.

Also the MUST_CONNECT pad flag is set for the video device's pad. I'll
switch to media_pad_remote_pad_first() also here.

> 
> 4) Unless I am misreading it, the inner lookup in the loop does not
>    depend on the loop variable:
> 
> > +	for_each_active_route(&state->routing, route) {
> > +		struct v4l2_mbus_frame_desc_entry *entry = NULL;
> > +
> > +		for (unsigned int i = 0; i < desc.num_entries; i++) {
> > +			if (desc.entry[i].stream == this_entry->stream) {
> > +				entry = &desc.entry[i];
> > +				break;
> > +			}
> > +		}
> > +
> > +		if (entry->bus.csi2.vc != this_entry->bus.csi2.vc)
> > +			continue;
> 
> The search key is this_entry->stream, which does not change across
> iterations, so entry always ends up as this_entry itself and the vc
> comparison can never differ. The "continue" is never taken and every
> active route is counted, whatever its virtual channel. Going by the
> commit message, the key here was meant to be route->sink_stream.

Yes, the frame descriptor entries' streams was compared with wrong stream,
this needs to come from routing instead. There's also a check missing above
that an entry is actually found.

> 
> Finally, a note rather than a request. After the whole series,
> ipu6_isys_csi2_enable_streams() and ipu6_isys_csi2_disable_streams()
> still call media_pad_remote_pad_first() on the sink pad and dereference
> the result unchecked -- and that one does return NULL. It is what my
> [PATCH v2 1/3] touched. I have dropped that patch and I am not
> reopening the question of the scenario behind it; I mention it only
> because if the checks in 2) and 3) go in, that pointer is sitting right
> next to them.

I guess there are two things to consider here: 1) whether that pointer can
be NULL in the circumstances of this driver (shouldn't be) and 2) whether
static analysers are smart enough to determine the pointer cannot be NULL
(or that there's a bug and we've missed it's actually possible).

I'll reply to the framework patch separately.

-- 
Kind regards,

Sakari Ailus

[PATCH v2 10/21] media: ipu6: Add lockdep checks for CSI-2 streaming enable and disable

[Sakari Ailus]

The callbacks for enabling and disabling streams access data serialised
using the isys stream_mutex; add lockdep checks for this.

Signed-off-by: Sakari Ailus <sakari.ailus@linux.intel.com>
---
 drivers/media/pci/intel/ipu6/ipu6-isys-csi2.c | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/drivers/media/pci/intel/ipu6/ipu6-isys-csi2.c b/drivers/media/pci/intel/ipu6/ipu6-isys-csi2.c
index aa07fabb2afc..431b770cc8e2 100644
--- a/drivers/media/pci/intel/ipu6/ipu6-isys-csi2.c
+++ b/drivers/media/pci/intel/ipu6/ipu6-isys-csi2.c
@@ -536,6 +536,8 @@ static int ipu6_isys_csi2_enable_streams(struct v4l2_subdev *sd,
 	int ret;
 	u8 vc;
 
+	lockdep_assert_held(&csi2->isys->stream_mutex);
+
 	ret = ipu6_isys_buffer_list_get(av->stream, &bl);
 	if (ret < 0) {
 		dev_warn(sd->dev, "no buffer available, DRIVER BUG?\n");
@@ -608,6 +610,8 @@ static int ipu6_isys_csi2_disable_streams(struct v4l2_subdev *sd,
 	u64 sink_streams;
 	u8 vc;
 
+	lockdep_assert_held(&csi2->isys->stream_mutex);
+
 	if (!ipu6_isys_csi2_streaming_change(asd, state, pad, &vc, false))
 		goto out_del_csi2_entry;
 
-- 
2.47.3

[PATCH v2 11/21] media: ipu6: Remove nr_queues and nr_streaming fields in ipu6_isys_stream

[Sakari Ailus]

nr_queues and nr_streaming fields were being used to determine whether
streaming was to be started in the upstream sub-device. As we store the
streaming state in the CSI-2 sub-device, there's no longer need for that.
Drop these fields.

Signed-off-by: Sakari Ailus <sakari.ailus@linux.intel.com>
---
 .../media/pci/intel/ipu6/ipu6-isys-queue.c    | 24 +++++++++----------
 .../media/pci/intel/ipu6/ipu6-isys-video.c    | 20 +++++-----------
 .../media/pci/intel/ipu6/ipu6-isys-video.h    |  7 ++----
 3 files changed, 20 insertions(+), 31 deletions(-)

diff --git a/drivers/media/pci/intel/ipu6/ipu6-isys-queue.c b/drivers/media/pci/intel/ipu6/ipu6-isys-queue.c
index 5eb6f17bf45b..327ce16bdeca 100644
--- a/drivers/media/pci/intel/ipu6/ipu6-isys-queue.c
+++ b/drivers/media/pci/intel/ipu6/ipu6-isys-queue.c
@@ -250,6 +250,9 @@ static int ipu6_isys_stream_start(struct ipu6_isys_video *av)
 	if (ret)
 		return ret;
 
+	if (!(BIT(stream->vc) & av->csi2->streaming_vc))
+		return 0;
+
 	do {
 		ret = ipu6_isys_buffer_list_get(stream, &bl);
 		if (ret < 0)
@@ -294,6 +297,7 @@ static void buf_queue(struct vb2_buffer *vb)
 	struct ipu6_isys_buffer *ib = &ivb->ib;
 	struct device *dev = &adev->auxdev.dev;
 	struct ipu6_isys_stream *stream = av->stream;
+	struct ipu6_isys_csi2 *csi2;
 	struct ipu6_isys_buffer_list bl;
 	struct isys_fw_msgs *msg;
 	unsigned long flags;
@@ -317,7 +321,8 @@ static void buf_queue(struct vb2_buffer *vb)
 
 	mutex_lock(&stream->mutex);
 
-	if (stream->nr_streaming != stream->nr_queues) {
+	csi2 = ipu6_isys_subdev_to_csi2(stream->asd);
+	if (!(BIT(stream->vc) & csi2->streaming_vc)) {
 		dev_dbg(dev, "not streaming yet, adding to incoming\n");
 		goto out;
 	}
@@ -478,7 +483,8 @@ static int start_streaming(struct vb2_queue *q, unsigned int count)
 		ipu6_isys_get_isys_format(ipu6_isys_get_format(av), 0);
 	struct ipu6_isys_stream *stream;
 	struct media_pad *source_pad, *remote_pad;
-	int nr_queues, ret;
+	bool first;
+	int ret;
 
 	dev_dbg(dev, "stream: %s: width %u, height %u, css pixelformat %u\n",
 		av->vdev.name, ipu6_isys_get_frame_width(av),
@@ -498,11 +504,12 @@ static int start_streaming(struct vb2_queue *q, unsigned int count)
 		goto out_return_buffers;
 	}
 
-	ret = ipu6_isys_setup_video(av, remote_pad, source_pad, &nr_queues);
+	ret = ipu6_isys_setup_video(av, remote_pad, source_pad);
 	if (ret < 0) {
 		dev_dbg(dev, "failed to setup video\n");
 		goto out_return_buffers;
 	}
+	first = ret;
 
 	ret = ipu6_isys_link_fmt_validate(aq);
 	if (ret) {
@@ -518,17 +525,12 @@ static int start_streaming(struct vb2_queue *q, unsigned int count)
 
 	stream = av->stream;
 	mutex_lock(&stream->mutex);
-	if (!stream->nr_streaming) {
-		ret = ipu6_isys_video_prepare_stream(av, source_pad->entity,
-						     nr_queues);
+	if (first) {
+		ret = ipu6_isys_video_prepare_stream(av, source_pad->entity);
 		if (ret)
 			goto out_fw_close;
 	}
 
-	stream->nr_streaming++;
-	dev_dbg(dev, "queue %u of %u\n", stream->nr_streaming,
-		stream->nr_queues);
-
 	list_add(&aq->node, &stream->queues);
 	ipu6_isys_configure_stream_watermark(av, source_pad->entity);
 	ipu6_isys_update_stream_watermark(av, true);
@@ -544,7 +546,6 @@ static int start_streaming(struct vb2_queue *q, unsigned int count)
 out_stream_start:
 	ipu6_isys_update_stream_watermark(av, false);
 	list_del(&aq->node);
-	stream->nr_streaming--;
 
 out_fw_close:
 	mutex_unlock(&stream->mutex);
@@ -574,7 +575,6 @@ static void stop_streaming(struct vb2_queue *q)
 	list_del(&aq->node);
 	mutex_unlock(&av->isys->stream_mutex);
 
-	stream->nr_streaming--;
 	mutex_unlock(&stream->mutex);
 
 	ipu6_isys_stream_cleanup(av);
diff --git a/drivers/media/pci/intel/ipu6/ipu6-isys-video.c b/drivers/media/pci/intel/ipu6/ipu6-isys-video.c
index b3cbd4266da1..5c00c98869d9 100644
--- a/drivers/media/pci/intel/ipu6/ipu6-isys-video.c
+++ b/drivers/media/pci/intel/ipu6/ipu6-isys-video.c
@@ -611,16 +611,11 @@ void ipu6_isys_close_streaming_firmware(struct ipu6_isys_video *av)
 }
 
 int ipu6_isys_video_prepare_stream(struct ipu6_isys_video *av,
-				   struct media_entity *source_entity,
-				   int nr_queues)
+				   struct media_entity *source_entity)
 {
 	struct ipu6_isys_stream *stream = av->stream;
 	struct ipu6_isys_csi2 *csi2;
 
-	if (WARN_ON(stream->nr_streaming))
-		return -EINVAL;
-
-	stream->nr_queues = nr_queues;
 	atomic_set(&stream->sequence, 0);
 	atomic_set(&stream->buf_id, 0);
 
@@ -1000,7 +995,7 @@ void ipu6_isys_fw_close(struct ipu6_isys *isys)
 
 int ipu6_isys_setup_video(struct ipu6_isys_video *av,
 			  struct media_pad *remote_pad,
-			  struct media_pad *source_pad, int *nr_queues)
+			  struct media_pad *source_pad)
 {
 	const struct ipu6_isys_pixelformat *pfmt =
 		ipu6_isys_get_isys_format(ipu6_isys_get_format(av), 0);
@@ -1012,18 +1007,14 @@ int ipu6_isys_setup_video(struct ipu6_isys_video *av,
 	struct v4l2_subdev *remote_sd =
 		media_entity_to_v4l2_subdev(remote_pad->entity);
 	struct ipu6_isys_subdev *asd = to_ipu6_isys_subdev(remote_sd);
+	struct media_pipeline *pipeline;
 	int ret = -EINVAL;
 
-	*nr_queues = 0;
-
 	/* Find the root */
 	state = v4l2_subdev_lock_and_get_active_state(remote_sd);
-	for_each_active_route(&state->routing, r) {
-		(*nr_queues)++;
-
+	for_each_active_route(&state->routing, r)
 		if (r->source_pad == remote_pad->index)
 			route = r;
-	}
 
 	if (!route) {
 		v4l2_subdev_unlock_state(state);
@@ -1051,6 +1042,7 @@ int ipu6_isys_setup_video(struct ipu6_isys_video *av,
 		return ret;
 	}
 
+	pipeline = video_device_pipeline(&av->vdev);
 	ret = video_device_pipeline_alloc_start(&av->vdev);
 	if (ret < 0) {
 		dev_dbg(dev, "media pipeline start failed\n");
@@ -1064,7 +1056,7 @@ int ipu6_isys_setup_video(struct ipu6_isys_video *av,
 		return -EINVAL;
 	}
 
-	return 0;
+	return !pipeline;
 }
 
 /*
diff --git a/drivers/media/pci/intel/ipu6/ipu6-isys-video.h b/drivers/media/pci/intel/ipu6/ipu6-isys-video.h
index cb4b45f0dae2..a78aaad63303 100644
--- a/drivers/media/pci/intel/ipu6/ipu6-isys-video.h
+++ b/drivers/media/pci/intel/ipu6/ipu6-isys-video.h
@@ -52,8 +52,6 @@ struct ipu6_isys_stream {
 	unsigned int nr_output_pins;
 	struct ipu6_isys_subdev *asd;
 
-	int nr_queues;	/* Number of capture queues */
-	int nr_streaming;
 	struct list_head queues;
 	struct completion stream_open_completion;
 	struct completion stream_close_completion;
@@ -108,14 +106,13 @@ int ipu6_isys_start_stream_firmware(struct ipu6_isys_video *av,
 void ipu6_isys_stop_streaming_firmware(struct ipu6_isys_video *av);
 void ipu6_isys_close_streaming_firmware(struct ipu6_isys_video *av);
 int ipu6_isys_video_prepare_stream(struct ipu6_isys_video *av,
-				   struct media_entity *source_entity,
-				   int nr_queues);
+				   struct media_entity *source_entity);
 int ipu6_isys_video_set_streaming(struct ipu6_isys_video *av, int state);
 int ipu6_isys_fw_open(struct ipu6_isys *isys);
 void ipu6_isys_fw_close(struct ipu6_isys *isys);
 int ipu6_isys_setup_video(struct ipu6_isys_video *av,
 			  struct media_pad *remote_pad,
-			  struct media_pad *source_pad, int *nr_queues);
+			  struct media_pad *source_pad);
 int ipu6_isys_video_init(struct ipu6_isys_video *av);
 void ipu6_isys_video_cleanup(struct ipu6_isys_video *av);
 void ipu6_isys_put_stream(struct ipu6_isys_stream *stream);
-- 
2.47.3

[PATCH v2 12/21] media: ipu6: Collect enabled stream IDs

[Sakari Ailus]

Collect enable stream IDs as streaming is enabled on video nodes. These
values will be needed once streaming actually starts. This is required for
multiple streams from a single source.

Signed-off-by: Sakari Ailus <sakari.ailus@linux.intel.com>
---
 drivers/media/pci/intel/ipu6/ipu6-isys-csi2.c | 25 ++++++++++++-------
 drivers/media/pci/intel/ipu6/ipu6-isys-csi2.h |  1 +
 2 files changed, 17 insertions(+), 9 deletions(-)

diff --git a/drivers/media/pci/intel/ipu6/ipu6-isys-csi2.c b/drivers/media/pci/intel/ipu6/ipu6-isys-csi2.c
index 431b770cc8e2..dded92cf006b 100644
--- a/drivers/media/pci/intel/ipu6/ipu6-isys-csi2.c
+++ b/drivers/media/pci/intel/ipu6/ipu6-isys-csi2.c
@@ -516,6 +516,10 @@ static bool ipu6_isys_csi2_streaming_change(struct ipu6_isys_subdev *asd,
 		return true;
 	}
 
+	dev_dbg(asd->sd.dev, "not setting streaming %s %s (%u/%u)\n",
+		str_enabled_disabled(enable), enable ? "yet" : "anymore",
+		nodes_streaming, streams_enabled);
+
 	return false;
 }
 
@@ -546,6 +550,11 @@ static int ipu6_isys_csi2_enable_streams(struct v4l2_subdev *sd,
 
 	list_add(&av->csi2_entry, &csi2->av_head);
 
+	sink_streams =
+		v4l2_subdev_state_xlate_streams(state, pad, CSI2_PAD_SINK,
+						&streams_mask);
+	csi2->stream_ids |= sink_streams;
+
 	if (!ipu6_isys_csi2_streaming_change(asd, state, pad, &vc, true))
 		return 0;
 
@@ -555,13 +564,6 @@ static int ipu6_isys_csi2_enable_streams(struct v4l2_subdev *sd,
 		goto err_return_buffers;
 	}
 
-	remote_pad = media_pad_remote_pad_first(&sd->entity.pads[CSI2_PAD_SINK]);
-	remote_sd = media_entity_to_v4l2_subdev(remote_pad->entity);
-
-	sink_streams =
-		v4l2_subdev_state_xlate_streams(state, pad, CSI2_PAD_SINK,
-						&streams_mask);
-
 	if (!csi2->streaming_vc) {
 		ret = IS_IPU7(isp) ? ipu7_isys_csi2_stream_enable(csi2) :
 				     ipu6_isys_csi2_stream_enable(csi2);
@@ -569,8 +571,11 @@ static int ipu6_isys_csi2_enable_streams(struct v4l2_subdev *sd,
 			goto err_stop_stream_firmware;
 	}
 
+	remote_pad = media_pad_remote_pad_first(&sd->entity.pads[CSI2_PAD_SINK]);
+	remote_sd = media_entity_to_v4l2_subdev(remote_pad->entity);
+
 	ret = v4l2_subdev_enable_streams(remote_sd, remote_pad->index,
-					 sink_streams);
+					 csi2->stream_ids);
 	if (ret)
 		goto err_stop_stream_csi2;
 
@@ -589,6 +594,7 @@ static int ipu6_isys_csi2_enable_streams(struct v4l2_subdev *sd,
 	ipu6_isys_close_streaming_firmware(av);
 
 err_return_buffers:
+	csi2->stream_ids &= ~sink_streams;
 	list_del(&av->csi2_entry);
 	ipu6_isys_buffer_list_queue(&bl, IPU6_ISYS_BUFFER_LIST_FL_INCOMING, 0);
 
@@ -631,7 +637,8 @@ static int ipu6_isys_csi2_disable_streams(struct v4l2_subdev *sd,
 	else
 		ipu6_isys_csi2_stream_disable(csi2);
 
-	v4l2_subdev_disable_streams(remote_sd, remote_pad->index, sink_streams);
+	v4l2_subdev_disable_streams(remote_sd, remote_pad->index,
+				    csi2->stream_ids);
 
 	ipu6_isys_close_streaming_firmware(av);
 
diff --git a/drivers/media/pci/intel/ipu6/ipu6-isys-csi2.h b/drivers/media/pci/intel/ipu6/ipu6-isys-csi2.h
index a98a60e443c6..4d84899fde4c 100644
--- a/drivers/media/pci/intel/ipu6/ipu6-isys-csi2.h
+++ b/drivers/media/pci/intel/ipu6/ipu6-isys-csi2.h
@@ -50,6 +50,7 @@ struct ipu6_isys_csi2 {
 	u32 legacy_irq_mask;
 	unsigned int phy_mode;
 	u32 streaming_vc;
+	u64 stream_ids;
 };
 
 struct ipu6_isys_csi2_timing {
-- 
2.47.3

[PATCH v2 13/21] media: ipu6: Avoid accessing av->streams before streaming

[Sakari Ailus]

Streams will soon be allocated just for the duration of streaming
operation. In this case, obtain the CSI-2 sub-device through av->csi2
which is set up during driver probe.

Signed-off-by: Sakari Ailus <sakari.ailus@linux.intel.com>
---
 drivers/media/pci/intel/ipu6/ipu6-isys-video.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/drivers/media/pci/intel/ipu6/ipu6-isys-video.c b/drivers/media/pci/intel/ipu6/ipu6-isys-video.c
index 5c00c98869d9..66f1a28a444e 100644
--- a/drivers/media/pci/intel/ipu6/ipu6-isys-video.c
+++ b/drivers/media/pci/intel/ipu6/ipu6-isys-video.c
@@ -860,7 +860,7 @@ int ipu6_isys_video_set_streaming(struct ipu6_isys_video *av, int state)
 	struct media_pad *r_pad;
 	int ret = 0;
 
-	sd = &av->stream->asd->sd;
+	sd = &av->csi2->asd.sd;
 	r_pad = media_pad_remote_pad_first(&av->pad);
 
 	if (!state) {
-- 
2.47.3

[PATCH v2 14/21] media: ipu6: Rework watermark calculation

[Sakari Ailus]

Move watermark calculation to ipu6-isys-csi2.c and do the whole thing
as streaming is about to start. There's no need to store all the
information so that it is bound to video nodes.

Signed-off-by: Sakari Ailus <sakari.ailus@linux.intel.com>
---
 drivers/media/pci/intel/ipu6/ipu6-isys-csi2.c | 84 ++++++++++++++++-
 drivers/media/pci/intel/ipu6/ipu6-isys-csi2.h |  4 +
 .../media/pci/intel/ipu6/ipu6-isys-queue.c    |  5 -
 .../media/pci/intel/ipu6/ipu6-isys-video.c    | 94 -------------------
 .../media/pci/intel/ipu6/ipu6-isys-video.h    | 15 ---
 drivers/media/pci/intel/ipu6/ipu6-isys.c      | 42 ++++-----
 drivers/media/pci/intel/ipu6/ipu6-isys.h      |  2 +-
 7 files changed, 103 insertions(+), 143 deletions(-)

diff --git a/drivers/media/pci/intel/ipu6/ipu6-isys-csi2.c b/drivers/media/pci/intel/ipu6/ipu6-isys-csi2.c
index dded92cf006b..2bd1392569ff 100644
--- a/drivers/media/pci/intel/ipu6/ipu6-isys-csi2.c
+++ b/drivers/media/pci/intel/ipu6/ipu6-isys-csi2.c
@@ -231,6 +231,79 @@ void ipu6_isys_csi2_error(struct ipu6_isys_csi2 *csi2)
 	}
 }
 
+static void ipu6_isys_csi2_setup_watermark(struct ipu6_isys_csi2 *csi2,
+					   struct v4l2_subdev_state *csi2_state,
+					   struct v4l2_subdev *remote_sd)
+{
+	struct device *dev = &csi2->isys->adev->auxdev.dev;
+	struct v4l2_control hb = { .id = V4L2_CID_HBLANK, .value = 0 };
+	struct v4l2_subdev_route *route;
+	u32 max_stream_data_rate = 0, hblank = 0;
+	s64 link_freq;
+	int ret;
+
+	if (IS_IPU7(csi2->asd.isys->adev->isp))
+		return;
+
+	ret = v4l2_g_ctrl(remote_sd->ctrl_handler, &hb);
+	if (!ret)
+		hblank = max(0, hb.value);
+
+	link_freq = ipu6_isys_csi2_get_link_freq(csi2);
+	if (link_freq <= 0) {
+		csi2->watermark.force_iwake_disable = true;
+		dev_warn(dev, "unexpected link_freq %lld (source %s)\n",
+			 link_freq, remote_sd->entity.name);
+		ipu6_isys_update_watermark_setting(csi2->isys);
+		return;
+	}
+
+	for_each_active_route(&csi2_state->routing, route) {
+		struct v4l2_mbus_framefmt *fmt;
+
+		fmt = v4l2_subdev_state_get_format(csi2_state, CSI2_PAD_SINK,
+						   route->sink_stream);
+		if (WARN_ON(!fmt))
+			continue;
+
+		u32 bpp = ipu6_isys_mbus_code_to_bpp(fmt->code);
+		u64 pixel_rate = mul_u64_u32_div(link_freq, csi2->nlanes * 2,
+						 bpp);
+		u32 pixels_per_line = fmt->width + hblank;
+		u64 line_time_ns = div_u64(pixels_per_line * NSEC_PER_SEC,
+					   pixel_rate);
+		u32 bytes_per_line = fmt->width * bpp / 8;
+		u32 pages_per_line =
+			DIV_ROUND_UP(bytes_per_line,
+				     csi2->isys->pdata->ipdata->sram_gran_size);
+		u32 pb_bytes_per_line =
+			pages_per_line << csi2->isys->pdata->ipdata->sram_gran_shift;
+		u64 stream_data_rate =
+			div64_u64(pb_bytes_per_line * 1000, line_time_ns);
+
+		dev_dbg(dev, "stream %u:%u -> %u:%u data rate %lld\n",
+			route->sink_pad, route->sink_stream, route->source_pad,
+			route->source_stream, stream_data_rate);
+
+		max_stream_data_rate = max(max_stream_data_rate,
+					   stream_data_rate);
+	}
+
+	csi2->watermark.stream_data_rate = max_stream_data_rate;
+
+	ipu6_isys_update_watermark_setting(csi2->isys);
+}
+
+static void ipu6_isys_csi2_clear_watermark(struct ipu6_isys_csi2 *csi2)
+{
+	if (IS_IPU7(csi2->asd.isys->adev->isp))
+		return;
+
+	csi2->watermark.force_iwake_disable = false;
+	csi2->watermark.stream_data_rate = 0;
+	ipu6_isys_update_watermark_setting(csi2->isys);
+}
+
 static void ipu6_isys_csi2_stream_disable(struct ipu6_isys_csi2 *csi2)
 {
 	struct ipu6_isys *isys = csi2->isys;
@@ -558,6 +631,11 @@ static int ipu6_isys_csi2_enable_streams(struct v4l2_subdev *sd,
 	if (!ipu6_isys_csi2_streaming_change(asd, state, pad, &vc, true))
 		return 0;
 
+	remote_pad = media_pad_remote_pad_first(&sd->entity.pads[CSI2_PAD_SINK]);
+	remote_sd = media_entity_to_v4l2_subdev(remote_pad->entity);
+
+	ipu6_isys_csi2_setup_watermark(csi2, state, remote_sd);
+
 	ret = ipu6_isys_start_stream_firmware(av, &bl);
 	if (ret) {
 		dev_err(sd->dev, "start stream of firmware failed\n");
@@ -571,9 +649,6 @@ static int ipu6_isys_csi2_enable_streams(struct v4l2_subdev *sd,
 			goto err_stop_stream_firmware;
 	}
 
-	remote_pad = media_pad_remote_pad_first(&sd->entity.pads[CSI2_PAD_SINK]);
-	remote_sd = media_entity_to_v4l2_subdev(remote_pad->entity);
-
 	ret = v4l2_subdev_enable_streams(remote_sd, remote_pad->index,
 					 csi2->stream_ids);
 	if (ret)
@@ -594,6 +669,7 @@ static int ipu6_isys_csi2_enable_streams(struct v4l2_subdev *sd,
 	ipu6_isys_close_streaming_firmware(av);
 
 err_return_buffers:
+	ipu6_isys_csi2_clear_watermark(csi2);
 	csi2->stream_ids &= ~sink_streams;
 	list_del(&av->csi2_entry);
 	ipu6_isys_buffer_list_queue(&bl, IPU6_ISYS_BUFFER_LIST_FL_INCOMING, 0);
@@ -642,6 +718,8 @@ static int ipu6_isys_csi2_disable_streams(struct v4l2_subdev *sd,
 
 	ipu6_isys_close_streaming_firmware(av);
 
+	ipu6_isys_csi2_clear_watermark(csi2);
+
 out_del_csi2_entry:
 	list_del(&av->csi2_entry);
 
diff --git a/drivers/media/pci/intel/ipu6/ipu6-isys-csi2.h b/drivers/media/pci/intel/ipu6/ipu6-isys-csi2.h
index 4d84899fde4c..49d01e938dc8 100644
--- a/drivers/media/pci/intel/ipu6/ipu6-isys-csi2.h
+++ b/drivers/media/pci/intel/ipu6/ipu6-isys-csi2.h
@@ -49,6 +49,10 @@ struct ipu6_isys_csi2 {
 	unsigned int port;
 	u32 legacy_irq_mask;
 	unsigned int phy_mode;
+	struct {
+		u32 stream_data_rate;
+		bool force_iwake_disable;
+	} watermark;
 	u32 streaming_vc;
 	u64 stream_ids;
 };
diff --git a/drivers/media/pci/intel/ipu6/ipu6-isys-queue.c b/drivers/media/pci/intel/ipu6/ipu6-isys-queue.c
index 327ce16bdeca..725c3603d9e1 100644
--- a/drivers/media/pci/intel/ipu6/ipu6-isys-queue.c
+++ b/drivers/media/pci/intel/ipu6/ipu6-isys-queue.c
@@ -532,8 +532,6 @@ static int start_streaming(struct vb2_queue *q, unsigned int count)
 	}
 
 	list_add(&aq->node, &stream->queues);
-	ipu6_isys_configure_stream_watermark(av, source_pad->entity);
-	ipu6_isys_update_stream_watermark(av, true);
 
 	ret = ipu6_isys_stream_start(av);
 	if (ret)
@@ -544,7 +542,6 @@ static int start_streaming(struct vb2_queue *q, unsigned int count)
 	return 0;
 
 out_stream_start:
-	ipu6_isys_update_stream_watermark(av, false);
 	list_del(&aq->node);
 
 out_fw_close:
@@ -568,8 +565,6 @@ static void stop_streaming(struct vb2_queue *q)
 
 	mutex_lock(&stream->mutex);
 
-	ipu6_isys_update_stream_watermark(av, false);
-
 	mutex_lock(&av->isys->stream_mutex);
 	ipu6_isys_video_set_streaming(av, 0);
 	list_del(&aq->node);
diff --git a/drivers/media/pci/intel/ipu6/ipu6-isys-video.c b/drivers/media/pci/intel/ipu6/ipu6-isys-video.c
index 66f1a28a444e..4037a3c80dc6 100644
--- a/drivers/media/pci/intel/ipu6/ipu6-isys-video.c
+++ b/drivers/media/pci/intel/ipu6/ipu6-isys-video.c
@@ -636,100 +636,6 @@ int ipu6_isys_video_prepare_stream(struct ipu6_isys_video *av,
 	return 0;
 }
 
-void ipu6_isys_configure_stream_watermark(struct ipu6_isys_video *av,
-					  struct media_entity *source)
-{
-	struct ipu6_isys *isys = av->isys;
-	struct ipu6_isys_csi2 *csi2 = NULL;
-	struct isys_iwake_watermark *iwake_watermark = &isys->iwake_watermark;
-	struct device *dev = &isys->adev->auxdev.dev;
-	struct v4l2_subdev *esd;
-	struct v4l2_control hb = { .id = V4L2_CID_HBLANK, .value = 0 };
-	s64 link_freq;
-	int ret;
-
-	esd = media_entity_to_v4l2_subdev(source);
-
-	av->watermark.width = ipu6_isys_get_frame_width(av);
-	av->watermark.height = ipu6_isys_get_frame_height(av);
-
-	ret = v4l2_g_ctrl(esd->ctrl_handler, &hb);
-	if (!ret && hb.value >= 0)
-		av->watermark.hblank = hb.value;
-	else
-		av->watermark.hblank = 0;
-
-	csi2 = ipu6_isys_subdev_to_csi2(av->stream->asd);
-	link_freq = ipu6_isys_csi2_get_link_freq(csi2);
-	if (link_freq <= 0) {
-		mutex_lock(&iwake_watermark->mutex);
-		iwake_watermark->force_iwake_disable = true;
-		mutex_unlock(&iwake_watermark->mutex);
-		dev_warn(dev, "unexpected link_freq from %s, disable iwake\n",
-			 source->name);
-		return;
-	}
-
-	struct v4l2_subdev_state *state;
-	struct v4l2_mbus_framefmt *format;
-	unsigned int bpp;
-
-	state = v4l2_subdev_lock_and_get_active_state(&csi2->asd.sd);
-	format = v4l2_subdev_state_get_format(state, 0, av->source_stream);
-	bpp = ipu6_isys_mbus_code_to_bpp(format->code);
-	v4l2_subdev_unlock_state(state);
-
-	av->watermark.pixel_rate = mul_u64_u32_div(link_freq, csi2->nlanes * 2,
-						   bpp);
-}
-
-static void calculate_stream_datarate(struct ipu6_isys_video *av)
-{
-	struct video_stream_watermark *watermark = &av->watermark;
-	const struct ipu6_isys_pixelformat *pfmt =
-		ipu6_isys_get_isys_format(ipu6_isys_get_format(av), 0);
-	u32 pages_per_line, pb_bytes_per_line, pixels_per_line, bytes_per_line;
-	u64 line_time_ns, stream_data_rate;
-
-	pixels_per_line = watermark->width + watermark->hblank;
-	line_time_ns =  div_u64(pixels_per_line * NSEC_PER_SEC,
-				watermark->pixel_rate);
-	bytes_per_line = watermark->width * pfmt->bpp / 8;
-	pages_per_line = DIV_ROUND_UP(bytes_per_line,
-				      av->isys->pdata->ipdata->sram_gran_size);
-	pb_bytes_per_line =
-		pages_per_line << av->isys->pdata->ipdata->sram_gran_shift;
-	stream_data_rate = div64_u64(pb_bytes_per_line * 1000, line_time_ns);
-
-	watermark->stream_data_rate = stream_data_rate;
-}
-
-void ipu6_isys_update_stream_watermark(struct ipu6_isys_video *av, bool state)
-{
-	struct isys_iwake_watermark *iwake_watermark =
-		&av->isys->iwake_watermark;
-	struct ipu6_device *isp = av->isys->adev->isp;
-
-	if (IS_IPU7(isp) ||
-	    !av->watermark.pixel_rate)
-		return;
-
-	if (state) {
-		calculate_stream_datarate(av);
-		mutex_lock(&iwake_watermark->mutex);
-		list_add(&av->watermark.stream_node,
-			 &iwake_watermark->video_list);
-		mutex_unlock(&iwake_watermark->mutex);
-	} else {
-		av->watermark.stream_data_rate = 0;
-		mutex_lock(&iwake_watermark->mutex);
-		list_del(&av->watermark.stream_node);
-		mutex_unlock(&iwake_watermark->mutex);
-	}
-
-	update_watermark_setting(av->isys);
-}
-
 void ipu6_isys_put_stream(struct ipu6_isys_stream *stream)
 {
 	struct device *dev;
diff --git a/drivers/media/pci/intel/ipu6/ipu6-isys-video.h b/drivers/media/pci/intel/ipu6/ipu6-isys-video.h
index a78aaad63303..f5b773e16d62 100644
--- a/drivers/media/pci/intel/ipu6/ipu6-isys-video.h
+++ b/drivers/media/pci/intel/ipu6/ipu6-isys-video.h
@@ -64,16 +64,6 @@ struct ipu6_isys_stream {
 	u8 vc;
 };
 
-struct video_stream_watermark {
-	u32 width;
-	u32 height;
-	u32 hblank;
-	u32 frame_rate;
-	u64 pixel_rate;
-	u64 stream_data_rate;
-	struct list_head stream_node;
-};
-
 struct ipu6_isys_video {
 	struct ipu6_isys_queue aq;
 	struct list_head csi2_entry;
@@ -87,7 +77,6 @@ struct ipu6_isys_video {
 	struct ipu6_isys_csi2 *csi2;
 	struct ipu6_isys_stream *stream;
 	unsigned int streaming;
-	struct video_stream_watermark watermark;
 	u32 source_stream;
 	u8 vc;
 	u8 dt;
@@ -121,10 +110,6 @@ ipu6_isys_query_stream_by_handle(struct ipu6_isys *isys, u8 stream_handle);
 struct ipu6_isys_stream *
 ipu6_isys_query_stream_by_source(struct ipu6_isys *isys, int source, u8 vc);
 
-void ipu6_isys_configure_stream_watermark(struct ipu6_isys_video *av,
-					  struct media_entity *source);
-void ipu6_isys_update_stream_watermark(struct ipu6_isys_video *av, bool state);
-
 u32 ipu6_isys_get_format(struct ipu6_isys_video *av);
 u32 ipu6_isys_get_data_size(struct ipu6_isys_video *av);
 u32 ipu6_isys_get_bytes_per_line(struct ipu6_isys_video *av);
diff --git a/drivers/media/pci/intel/ipu6/ipu6-isys.c b/drivers/media/pci/intel/ipu6/ipu6-isys.c
index 08f29b678d1b..8e5f6bbf6265 100644
--- a/drivers/media/pci/intel/ipu6/ipu6-isys.c
+++ b/drivers/media/pci/intel/ipu6/ipu6-isys.c
@@ -473,52 +473,47 @@ static void enable_iwake(struct ipu6_isys *isys, bool enable)
 	mutex_unlock(&iwake_watermark->mutex);
 }
 
-void update_watermark_setting(struct ipu6_isys *isys)
+void ipu6_isys_update_watermark_setting(struct ipu6_isys *isys)
 {
-	struct isys_iwake_watermark *iwake_watermark = &isys->iwake_watermark;
 	u32 iwake_threshold, iwake_critical_threshold, page_num;
 	struct device *dev = &isys->adev->auxdev.dev;
 	u32 calc_fill_time_us = 0, ltr = 0, did = 0;
-	struct video_stream_watermark *p_watermark;
 	enum ltr_did_type ltr_did_type;
-	struct list_head *stream_node;
 	u64 isys_pb_datarate_mbs = 0;
 	u32 mem_open_threshold = 0;
 	struct ltr_did ltrdid;
 	u64 threshold_bytes;
 	u32 max_sram_size;
 	u32 shift;
+	bool force_iwake_disable = false;
+
+	lockdep_assert_held(&isys->stream_mutex);
 
 	shift = isys->pdata->ipdata->sram_gran_shift;
 	max_sram_size = isys->pdata->ipdata->max_sram_size;
 
-	mutex_lock(&iwake_watermark->mutex);
-	if (iwake_watermark->force_iwake_disable) {
+	for (unsigned int i = 0; i < isys->pdata->ipdata->csi2.nports; i++) {
+		isys_pb_datarate_mbs +=
+			isys->csi2[i].watermark.stream_data_rate;
+		force_iwake_disable |=
+			isys->csi2[i].watermark.force_iwake_disable;
+	}
+
+	if (force_iwake_disable) {
+		dev_dbg(dev, "watermark: forcing iwake disabled\n");
 		set_iwake_ltrdid(isys, 0, 0, LTR_IWAKE_OFF);
 		set_iwake_register(isys, GDA_IRQ_CRITICAL_THRESHOLD_INDEX,
 				   CRITICAL_THRESHOLD_IWAKE_DISABLE);
-		goto unlock_exit;
-	}
-
-	if (list_empty(&iwake_watermark->video_list)) {
-		isys_pb_datarate_mbs = 0;
-	} else {
-		list_for_each(stream_node, &iwake_watermark->video_list) {
-			p_watermark = list_entry(stream_node,
-						 struct video_stream_watermark,
-						 stream_node);
-			isys_pb_datarate_mbs += p_watermark->stream_data_rate;
-		}
+		return;
 	}
-	mutex_unlock(&iwake_watermark->mutex);
 
 	if (!isys_pb_datarate_mbs) {
+		dev_dbg(dev, "watermark: disabled iwake\n");
 		enable_iwake(isys, false);
 		set_iwake_ltrdid(isys, 0, 0, LTR_IWAKE_OFF);
-		mutex_lock(&iwake_watermark->mutex);
 		set_iwake_register(isys, GDA_IRQ_CRITICAL_THRESHOLD_INDEX,
 				   CRITICAL_THRESHOLD_IWAKE_DISABLE);
-		goto unlock_exit;
+		return;
 	}
 
 	enable_iwake(isys, true);
@@ -553,7 +548,6 @@ void update_watermark_setting(struct ipu6_isys *isys)
 	iwake_threshold = max_t(u32, 1, threshold_bytes >> shift);
 	iwake_threshold = min_t(u32, iwake_threshold, max_sram_size);
 
-	mutex_lock(&iwake_watermark->mutex);
 	if (isys->pdata->ipdata->enhanced_iwake) {
 		set_iwake_register(isys, GDA_IWAKE_THRESHOLD_INDEX,
 				   DEFAULT_IWAKE_THRESHOLD);
@@ -575,7 +569,7 @@ void update_watermark_setting(struct ipu6_isys *isys)
 	iwake_critical_threshold = iwake_threshold +
 		(IS_PIXEL_BUFFER_PAGES - iwake_threshold) / 2;
 
-	dev_dbg(dev, "threshold: %u critical: %u\n", iwake_threshold,
+	dev_dbg(dev, "watermark: threshold: %u critical: %u\n", iwake_threshold,
 		iwake_critical_threshold);
 
 	set_iwake_register(isys, GDA_IRQ_CRITICAL_THRESHOLD_INDEX,
@@ -585,8 +579,6 @@ void update_watermark_setting(struct ipu6_isys *isys)
 	       isys->adev->isp->base + REG_PKGC_PMON_CFG);
 	writel(VAL_PKGC_PMON_CFG_START,
 	       isys->adev->isp->base + REG_PKGC_PMON_CFG);
-unlock_exit:
-	mutex_unlock(&iwake_watermark->mutex);
 }
 
 static void isys_iwake_watermark_init(struct ipu6_isys *isys)
diff --git a/drivers/media/pci/intel/ipu6/ipu6-isys.h b/drivers/media/pci/intel/ipu6/ipu6-isys.h
index 8674c041b994..b85c216b3a46 100644
--- a/drivers/media/pci/intel/ipu6/ipu6-isys.h
+++ b/drivers/media/pci/intel/ipu6/ipu6-isys.h
@@ -222,7 +222,7 @@ void ipu6_cleanup_fw_msg_bufs(struct ipu6_isys *isys);
 
 extern const struct v4l2_ioctl_ops ipu6_isys_ioctl_ops;
 
-void update_watermark_setting(struct ipu6_isys *isys);
+void ipu6_isys_update_watermark_setting(struct ipu6_isys *isys);
 
 int ipu6_isys_mcd_phy_set_power(struct ipu6_isys *isys,
 				struct ipu6_isys_csi2_config *cfg,
-- 
2.47.3

[PATCH v2 15/21] media: ipu6: Rework watermark setting

[Sakari Ailus]

Remove structs to maintain watermark related configuration, by using the
values where they're obtained, apart from keeping watermark enabled status
in struct ipu6_isys.

Signed-off-by: Sakari Ailus <sakari.ailus@linux.intel.com>
---
 drivers/media/pci/intel/ipu6/ipu6-isys.c | 76 ++++++++----------------
 drivers/media/pci/intel/ipu6/ipu6-isys.h | 34 +----------
 2 files changed, 26 insertions(+), 84 deletions(-)

diff --git a/drivers/media/pci/intel/ipu6/ipu6-isys.c b/drivers/media/pci/intel/ipu6/ipu6-isys.c
index 8e5f6bbf6265..0933fb617d3b 100644
--- a/drivers/media/pci/intel/ipu6/ipu6-isys.c
+++ b/drivers/media/pci/intel/ipu6/ipu6-isys.c
@@ -101,6 +101,27 @@ enum ltr_did_type {
 	LTR_TYPE_MAX
 };
 
+struct ltr_did {
+	union {
+		u32 value;
+		struct {
+			u8 val0;
+			u8 val1;
+			u8 val2;
+			u8 val3;
+		} bits;
+	} lut_ltr;
+	union {
+		u32 value;
+		struct {
+			u8 th0;
+			u8 th1;
+			u8 th2;
+			u8 th3;
+		} bits;
+	} lut_fill_time;
+};
+
 #define ISYS_PM_QOS_VALUE	300
 
 static int
@@ -347,20 +368,6 @@ static void ipu6_isys_setup_hw(struct ipu6_isys *isys)
 		writel(thd[i], base + IPU6_REG_ISYS_CDC_THRESHOLD(i));
 }
 
-static void get_lut_ltrdid(struct ipu6_isys *isys, struct ltr_did *pltr_did)
-{
-	struct isys_iwake_watermark *iwake_watermark = &isys->iwake_watermark;
-	struct ltr_did ltrdid_default;
-
-	ltrdid_default.lut_ltr.value = LTR_DEFAULT_VALUE;
-	ltrdid_default.lut_fill_time.value = FILL_TIME_DEFAULT_VALUE;
-
-	if (iwake_watermark->ltrdid.lut_ltr.value)
-		*pltr_did = iwake_watermark->ltrdid;
-	else
-		*pltr_did = ltrdid_default;
-}
-
 static int set_iwake_register(struct ipu6_isys *isys, u32 index, u32 value)
 {
 	struct device *dev = &isys->adev->auxdev.dev;
@@ -456,21 +463,14 @@ static void set_iwake_ltrdid(struct ipu6_isys *isys, u16 ltr, u16 did,
  */
 static void enable_iwake(struct ipu6_isys *isys, bool enable)
 {
-	struct isys_iwake_watermark *iwake_watermark = &isys->iwake_watermark;
 	int ret;
 
-	mutex_lock(&iwake_watermark->mutex);
-
-	if (iwake_watermark->iwake_enabled == enable) {
-		mutex_unlock(&iwake_watermark->mutex);
+	if (isys->iwake_watermark_enabled == enable)
 		return;
-	}
 
 	ret = set_iwake_register(isys, GDA_ENABLE_IWAKE_INDEX, enable);
 	if (!ret)
-		iwake_watermark->iwake_enabled = enable;
-
-	mutex_unlock(&iwake_watermark->mutex);
+		isys->iwake_watermark_enabled = enable;
 }
 
 void ipu6_isys_update_watermark_setting(struct ipu6_isys *isys)
@@ -524,7 +524,8 @@ void ipu6_isys_update_watermark_setting(struct ipu6_isys *isys)
 		did = calc_fill_time_us * DEFAULT_DID_RATIO / 100;
 		ltr_did_type = LTR_ENHANNCE_IWAKE;
 	} else {
-		get_lut_ltrdid(isys, &ltrdid);
+		ltrdid.lut_ltr.value = LTR_DEFAULT_VALUE;
+		ltrdid.lut_fill_time.value = FILL_TIME_DEFAULT_VALUE;
 
 		if (calc_fill_time_us <= ltrdid.lut_fill_time.bits.th0)
 			ltr = 0;
@@ -581,30 +582,6 @@ void ipu6_isys_update_watermark_setting(struct ipu6_isys *isys)
 	       isys->adev->isp->base + REG_PKGC_PMON_CFG);
 }
 
-static void isys_iwake_watermark_init(struct ipu6_isys *isys)
-{
-	struct isys_iwake_watermark *iwake_watermark = &isys->iwake_watermark;
-
-	INIT_LIST_HEAD(&iwake_watermark->video_list);
-	mutex_init(&iwake_watermark->mutex);
-
-	iwake_watermark->ltrdid.lut_ltr.value = 0;
-	iwake_watermark->isys = isys;
-	iwake_watermark->iwake_enabled = false;
-	iwake_watermark->force_iwake_disable = false;
-}
-
-static void isys_iwake_watermark_cleanup(struct ipu6_isys *isys)
-{
-	struct isys_iwake_watermark *iwake_watermark = &isys->iwake_watermark;
-
-	mutex_lock(&iwake_watermark->mutex);
-	list_del(&iwake_watermark->video_list);
-	mutex_unlock(&iwake_watermark->mutex);
-
-	mutex_destroy(&iwake_watermark->mutex);
-}
-
 /* The .bound() notifier callback when a match is found */
 static int isys_notifier_bound(struct v4l2_async_notifier *notifier,
 			       struct v4l2_subdev *sd,
@@ -1037,8 +1014,6 @@ static int isys_probe(struct auxiliary_device *auxdev,
 	if (ret < 0)
 		goto out_remove_pkg_dir_shared_buffer;
 
-	isys_iwake_watermark_init(isys);
-
 	if (IS_IPU7(adev->isp))
 		isys->phy_set_power = ipu7_isys_csi_phy_set_power;
 	else if (IS_IPU6SE(adev->isp))
@@ -1083,7 +1058,6 @@ static void isys_remove(struct auxiliary_device *auxdev)
 	for (i = 0; i < IPU6_ISYS_MAX_STREAMS; i++)
 		mutex_destroy(&isys->streams[i].mutex);
 
-	isys_iwake_watermark_cleanup(isys);
 	mutex_destroy(&isys->stream_mutex);
 	mutex_destroy(&isys->mutex);
 }
diff --git a/drivers/media/pci/intel/ipu6/ipu6-isys.h b/drivers/media/pci/intel/ipu6/ipu6-isys.h
index b85c216b3a46..b9239791dd5e 100644
--- a/drivers/media/pci/intel/ipu6/ipu6-isys.h
+++ b/drivers/media/pci/intel/ipu6/ipu6-isys.h
@@ -62,38 +62,6 @@ struct ipu6_bus_device;
 #define IPU6EP_MTL_LTR_VALUE			1023
 #define IPU6EP_MTL_MIN_MEMOPEN_TH		0xc
 
-struct ltr_did {
-	union {
-		u32 value;
-		struct {
-			u8 val0;
-			u8 val1;
-			u8 val2;
-			u8 val3;
-		} bits;
-	} lut_ltr;
-	union {
-		u32 value;
-		struct {
-			u8 th0;
-			u8 th1;
-			u8 th2;
-			u8 th3;
-		} bits;
-	} lut_fill_time;
-};
-
-struct isys_iwake_watermark {
-	bool iwake_enabled;
-	bool force_iwake_disable;
-	u32 iwake_threshold;
-	u64 isys_pixelbuffer_datarate;
-	struct ltr_did ltrdid;
-	struct mutex mutex; /* protect whole struct */
-	struct ipu6_isys *isys;
-	struct list_head video_list;
-};
-
 struct ipu6_isys_csi2_config {
 	u32 nlanes;
 	u32 port;
@@ -144,6 +112,7 @@ struct ipu6_isys {
 	bool need_reset;
 	bool icache_prefetch;
 	bool csi2_cse_ipc_not_supported;
+	bool iwake_watermark_enabled;
 	unsigned int ref_count;
 	unsigned int stream_opened;
 	unsigned int sensor_type;
@@ -165,7 +134,6 @@ struct ipu6_isys {
 	struct list_head framebuflist;
 	struct list_head framebuflist_fw;
 	struct v4l2_async_notifier notifier;
-	struct isys_iwake_watermark iwake_watermark;
 };
 
 struct isys_fw_msgs {
-- 
2.47.3

[PATCH v2 16/21] media: ipu6: Bridge the gap between streams in V4L2 and IPU6 firmware

[Sakari Ailus]

The stream concept of the IPU6 firmware is aligned with that of the CSI-2,
effectively meaning that a stream equals to a CSI-2 virtual channel. Only
start streaming once all streams on a given virtual channel have been
started, and stop streaming once first one of them has been stopped.

Signed-off-by: Sakari Ailus <sakari.ailus@linux.intel.com>
---
 drivers/media/pci/intel/ipu6/ipu6-fw-isys.c   |  63 ++-
 drivers/media/pci/intel/ipu6/ipu6-isys-csi2.c | 174 ++++----
 drivers/media/pci/intel/ipu6/ipu6-isys-csi2.h |   6 +-
 .../media/pci/intel/ipu6/ipu6-isys-queue.c    |  43 +-
 .../media/pci/intel/ipu6/ipu6-isys-video.c    | 376 ++++++++----------
 .../media/pci/intel/ipu6/ipu6-isys-video.h    |  38 +-
 drivers/media/pci/intel/ipu6/ipu6-isys.c      |  29 +-
 drivers/media/pci/intel/ipu6/ipu6-isys.h      |   8 +-
 drivers/media/pci/intel/ipu6/ipu7-fw-isys.c   |  48 ++-
 9 files changed, 325 insertions(+), 460 deletions(-)

diff --git a/drivers/media/pci/intel/ipu6/ipu6-fw-isys.c b/drivers/media/pci/intel/ipu6/ipu6-fw-isys.c
index 2c71bf27a91d..5e1b57ef650f 100644
--- a/drivers/media/pci/intel/ipu6/ipu6-fw-isys.c
+++ b/drivers/media/pci/intel/ipu6/ipu6-fw-isys.c
@@ -562,7 +562,8 @@ int ipu6_isys_isr_one(struct ipu6_bus_device *adev)
 		goto leave;
 	}
 
-	stream = ipu6_isys_query_stream_by_handle(isys, resp->stream_handle);
+	stream = resp->stream_handle < IPU6_ISYS_MAX_STREAMS ?
+		isys->streams_by_handle[resp->stream_handle] : NULL;
 	if (!stream) {
 		dev_err(&adev->auxdev.dev, "stream of stream_handle %u is unused\n",
 			resp->stream_handle);
@@ -599,7 +600,7 @@ int ipu6_isys_isr_one(struct ipu6_bus_device *adev)
 		if (!resp->buf_id) {
 			dev_warn(&adev->auxdev.dev, "%d: Invalid buf ID\n",
 				 resp->stream_handle);
-			goto leave_put_stream;
+			goto leave;
 		}
 
 		isys_fw_msg = container_of((void *)(uintptr_t)resp->buf_id,
@@ -650,8 +651,6 @@ int ipu6_isys_isr_one(struct ipu6_bus_device *adev)
 		break;
 	}
 
-leave_put_stream:
-	ipu6_isys_put_stream(stream);
 leave:
 	ipu6_fw_isys_put_resp(isys);
 	return 0;
@@ -662,7 +661,6 @@ static void ipu6_isys_csi2_isr(struct ipu6_isys_csi2 *csi2)
 	struct ipu6_isys_stream *stream;
 	unsigned int i;
 	u32 status;
-	int source;
 
 	ipu6_isys_register_errors(csi2);
 
@@ -672,24 +670,17 @@ static void ipu6_isys_csi2_isr(struct ipu6_isys_csi2 *csi2)
 	writel(status, csi2->base + CSI_PORT_REG_BASE_IRQ_CSI_SYNC +
 	       CSI_PORT_REG_BASE_IRQ_CLEAR_OFFSET);
 
-	source = csi2->asd.source;
 	for (i = 0; i < NR_OF_CSI2_VC; i++) {
 		if (status & IPU_CSI_RX_IRQ_FS_VC(i)) {
-			stream = ipu6_isys_query_stream_by_source(csi2->isys,
-								  source, i);
-			if (stream) {
+			stream = csi2->streams_by_vc[i];
+			if (stream)
 				ipu6_isys_csi2_sof_event_by_stream(stream);
-				ipu6_isys_put_stream(stream);
-			}
 		}
 
 		if (status & IPU_CSI_RX_IRQ_FE_VC(i)) {
-			stream = ipu6_isys_query_stream_by_source(csi2->isys,
-								  source, i);
-			if (stream) {
+			stream = csi2->streams_by_vc[i];
+			if (stream)
 				ipu6_isys_csi2_eof_event_by_stream(stream);
-				ipu6_isys_put_stream(stream);
-			}
 		}
 	}
 }
@@ -756,16 +747,18 @@ irqreturn_t ipu6_isys_isr(struct ipu6_bus_device *adev)
 }
 
 static int ipu6_isys_fw_pin_cfg(struct ipu6_isys_video *av,
-				struct ipu6_fw_isys_stream_cfg_data_abi *cfg)
+				struct ipu6_isys_stream *stream,
+				struct media_pad *src_pad,
+				struct v4l2_mbus_frame_desc_entry *entry,
+				void *__cfg)
 {
-	struct media_pad *src_pad = media_pad_remote_pad_first(&av->pad);
 	struct v4l2_subdev *sd = media_entity_to_v4l2_subdev(src_pad->entity);
 	struct v4l2_subdev_state *state = v4l2_subdev_get_locked_active_state(sd);
+	struct ipu6_fw_isys_stream_cfg_data_abi *cfg = __cfg;
 	struct ipu6_fw_isys_input_pin_info_abi *input_pin;
 	struct ipu6_fw_isys_output_pin_info_abi *output_pin;
-	struct ipu6_isys_stream *stream = av->stream;
 	struct ipu6_isys_queue *aq = &av->aq;
-	struct v4l2_mbus_framefmt fmt;
+	struct v4l2_mbus_framefmt *fmt;
 	const struct ipu6_isys_pixelformat *pfmt =
 		ipu6_isys_get_isys_format(ipu6_isys_get_format(av), 0);
 	struct v4l2_rect v4l2_crop;
@@ -775,13 +768,13 @@ static int ipu6_isys_fw_pin_cfg(struct ipu6_isys_video *av,
 	u32 src_stream;
 
 	src_stream = __ipu6_isys_get_src_stream_by_src_pad(state, src_pad->index);
-	fmt = *v4l2_subdev_state_get_format(state, src_pad->index, src_stream);
+	fmt = v4l2_subdev_state_get_format(state, src_pad->index, src_stream);
 	v4l2_crop = *v4l2_subdev_state_get_crop(state, src_pad->index, src_stream);
 
 	input_pin = &cfg->input_pins[input_pins];
-	input_pin->input_res.width = fmt.width;
-	input_pin->input_res.height = fmt.height;
-	input_pin->dt = av->dt;
+	input_pin->input_res.width = fmt->width;
+	input_pin->input_res.height = fmt->height;
+	input_pin->dt = entry->bus.csi2.dt;
 	input_pin->bits_per_pix = pfmt->bpp_packed;
 	input_pin->mapped_dt = 0x40; /* invalid mipi data type */
 	input_pin->mipi_decompression = 0;
@@ -822,28 +815,24 @@ static int ipu6_isys_fw_pin_cfg(struct ipu6_isys_video *av,
 	return 0;
 }
 
-static int ipu6_fw_isys_prepare_stream_cfg(struct ipu6_isys_video *av,
+static int ipu6_fw_isys_prepare_stream_cfg(struct ipu6_isys_stream *stream,
+					   struct v4l2_mbus_frame_desc *desc,
 					   struct isys_fw_msgs *msg)
 {
 	struct ipu6_fw_isys_stream_cfg_data_abi *stream_cfg;
-	struct device *dev = &av->isys->adev->auxdev.dev;
-	struct ipu6_isys_stream *stream = av->stream;
-	struct ipu6_isys_queue *aq;
+	struct device *dev = &stream->isys->adev->auxdev.dev;
+	int ret;
 
 	stream_cfg = &msg->ipu6.stream;
-	stream_cfg->src = stream->stream_source;
+	stream_cfg->src = stream->asd->source;
 	stream_cfg->vc = stream->vc;
 	stream_cfg->isl_use = 0;
 	stream_cfg->sensor_type = IPU6_FW_ISYS_SENSOR_MODE_NORMAL;
 
-	list_for_each_entry(aq, &stream->queues, node) {
-		struct ipu6_isys_video *__av = ipu6_isys_queue_to_video(aq);
-		int ret;
-
-		ret = ipu6_isys_fw_pin_cfg(__av, stream_cfg);
-		if (ret < 0)
-			return ret;
-	}
+	ret = ipu6_isys_fw_pins_prepare(stream, desc, ipu6_isys_fw_pin_cfg,
+					stream_cfg);
+	if (ret)
+		return ret;
 
 	ipu6_fw_isys_dump_stream_cfg(dev, msg);
 
diff --git a/drivers/media/pci/intel/ipu6/ipu6-isys-csi2.c b/drivers/media/pci/intel/ipu6/ipu6-isys-csi2.c
index 2bd1392569ff..5dc2596766f4 100644
--- a/drivers/media/pci/intel/ipu6/ipu6-isys-csi2.c
+++ b/drivers/media/pci/intel/ipu6/ipu6-isys-csi2.c
@@ -511,13 +511,13 @@ static int ipu7_isys_csi2_stream_enable(struct ipu6_isys_csi2 *csi2)
 
 static bool ipu6_isys_csi2_streaming_change(struct ipu6_isys_subdev *asd,
 					    struct v4l2_subdev_state *state,
-					    u32 pad, u8 *vc, bool enable)
+					    u32 pad,
+					    struct v4l2_mbus_frame_desc *desc,
+					    u8 *vc, bool enable)
 {
 	u32 streams_enabled = 0, nodes_streaming = 0;
 	struct v4l2_subdev_route *route, *this_route = NULL;
-	struct v4l2_mbus_frame_desc desc = { 0 };
 	struct v4l2_mbus_frame_desc_entry *this_entry = NULL;
-	int ret;
 
 	for_each_active_route(&state->routing, this_route)
 		if (pad == this_route->source_pad)
@@ -527,18 +527,9 @@ static bool ipu6_isys_csi2_streaming_change(struct ipu6_isys_subdev *asd,
 		return false;
 	}
 
-	struct media_pad *remote_pad =
-		media_pad_remote_pad_unique(&asd->sd.entity.pads[this_route->sink_pad]);
-	struct v4l2_subdev *remote_sd =
-		media_entity_to_v4l2_subdev(remote_pad->entity);
-	ret = v4l2_subdev_call(remote_sd, pad, get_frame_desc,
-			       remote_pad->index, &desc);
-	if (ret)
-		return ret;
-
-	for (unsigned int i = 0; i < desc.num_entries; i++) {
-		if (desc.entry[i].stream == this_route->sink_stream) {
-			this_entry = &desc.entry[i];
+	for (unsigned int i = 0; i < desc->num_entries; i++) {
+		if (desc->entry[i].stream == this_route->sink_stream) {
+			this_entry = &desc->entry[i];
 			break;
 		}
 	}
@@ -552,9 +543,9 @@ static bool ipu6_isys_csi2_streaming_change(struct ipu6_isys_subdev *asd,
 	for_each_active_route(&state->routing, route) {
 		struct v4l2_mbus_frame_desc_entry *entry = NULL;
 
-		for (unsigned int i = 0; i < desc.num_entries; i++) {
-			if (desc.entry[i].stream == this_entry->stream) {
-				entry = &desc.entry[i];
+		for (unsigned int i = 0; i < desc->num_entries; i++) {
+			if (desc->entry[i].stream == this_entry->stream) {
+				entry = &desc->entry[i];
 				break;
 			}
 		}
@@ -580,12 +571,13 @@ static bool ipu6_isys_csi2_streaming_change(struct ipu6_isys_subdev *asd,
 			nodes_streaming++;
 	}
 
-	*vc = this_entry->bus.csi2.vc;
+	if (vc)
+		*vc = this_entry->bus.csi2.vc;
 
 	if (streams_enabled == nodes_streaming) {
 		dev_dbg(asd->sd.dev,
-			"changing streaming state to %s on \"%s\":%u\n",
-			str_enabled_disabled(enable), asd->sd.entity.name, pad);
+			"changing streaming state to %s on \"%s\"\n",
+			str_enabled_disabled(enable), asd->sd.entity.name);
 		return true;
 	}
 
@@ -603,11 +595,15 @@ static int ipu6_isys_csi2_enable_streams(struct v4l2_subdev *sd,
 	struct ipu6_isys_subdev *asd = to_ipu6_isys_subdev(sd);
 	struct ipu6_isys_csi2 *csi2 = to_ipu6_isys_csi2(asd);
 	struct ipu6_device *isp = asd->isys->adev->isp;
-	struct v4l2_subdev *remote_sd;
-	struct media_pad *remote_pad,
+	struct media_pad *remote_pad =
+		media_pad_remote_pad_first(&sd->entity.pads[CSI2_PAD_SINK]),
 		*vdev_pad = media_pad_remote_pad_unique(&sd->entity.pads[pad]);
 	struct ipu6_isys_video *av =
 		container_of_const(vdev_pad, struct ipu6_isys_video, pad);
+	struct v4l2_subdev *remote_sd =
+		media_entity_to_v4l2_subdev(remote_pad->entity);
+	struct v4l2_mbus_frame_desc desc = { 0 };
+	struct ipu6_isys_stream *stream;
 	struct ipu6_isys_buffer_list bl;
 	u64 sink_streams;
 	int ret;
@@ -615,11 +611,10 @@ static int ipu6_isys_csi2_enable_streams(struct v4l2_subdev *sd,
 
 	lockdep_assert_held(&csi2->isys->stream_mutex);
 
-	ret = ipu6_isys_buffer_list_get(av->stream, &bl);
-	if (ret < 0) {
-		dev_warn(sd->dev, "no buffer available, DRIVER BUG?\n");
+	ret = v4l2_subdev_call(remote_sd, pad, get_frame_desc,
+			       remote_pad->index, &desc);
+	if (ret)
 		return ret;
-	}
 
 	list_add(&av->csi2_entry, &csi2->av_head);
 
@@ -628,20 +623,28 @@ static int ipu6_isys_csi2_enable_streams(struct v4l2_subdev *sd,
 						&streams_mask);
 	csi2->stream_ids |= sink_streams;
 
-	if (!ipu6_isys_csi2_streaming_change(asd, state, pad, &vc, true))
+	if (!ipu6_isys_csi2_streaming_change(asd, state, pad, &desc, &vc, true))
 		return 0;
 
-	remote_pad = media_pad_remote_pad_first(&sd->entity.pads[CSI2_PAD_SINK]);
-	remote_sd = media_entity_to_v4l2_subdev(remote_pad->entity);
-
 	ipu6_isys_csi2_setup_watermark(csi2, state, remote_sd);
 
-	ret = ipu6_isys_start_stream_firmware(av, &bl);
-	if (ret) {
-		dev_err(sd->dev, "start stream of firmware failed\n");
-		goto err_return_buffers;
+	stream = ipu6_isys_alloc_stream_firmware(csi2, state, &desc, vc);
+	if (IS_ERR(stream)) {
+		ret = PTR_ERR(stream);
+		dev_err(sd->dev, "allocating firmware stream failed\n");
+		goto err_clear_watermark;
+	}
+
+	ret = ipu6_isys_buffer_list_get(stream, &bl);
+	if (ret < 0) {
+		dev_warn(sd->dev, "no buffer available, DRIVER BUG?\n");
+		goto err_free_stream_firmware;
 	}
 
+	ret = ipu6_isys_start_stream_firmware(stream, &bl, &desc);
+	if (ret)
+		goto err_requeue_buffers;
+
 	if (!csi2->streaming_vc) {
 		ret = IS_IPU7(isp) ? ipu7_isys_csi2_stream_enable(csi2) :
 				     ipu6_isys_csi2_stream_enable(csi2);
@@ -664,15 +667,21 @@ static int ipu6_isys_csi2_enable_streams(struct v4l2_subdev *sd,
 	else
 		ipu6_isys_csi2_stream_disable(csi2);
 
+err_requeue_buffers:
+	ipu6_isys_buffer_list_queue(&bl, IPU6_ISYS_BUFFER_LIST_FL_INCOMING, 0);
+
 err_stop_stream_firmware:
-	ipu6_isys_stop_streaming_firmware(av);
-	ipu6_isys_close_streaming_firmware(av);
+	ipu6_isys_stop_stream_firmware(stream);
+	ipu6_isys_close_stream_firmware(stream);
+
+err_free_stream_firmware:
+	ipu6_isys_free_stream_firmware(stream);
 
-err_return_buffers:
+err_clear_watermark:
 	ipu6_isys_csi2_clear_watermark(csi2);
+	ipu6_isys_csi2_streaming_change(asd, state, pad, &desc, NULL, false);
 	csi2->stream_ids &= ~sink_streams;
 	list_del(&av->csi2_entry);
-	ipu6_isys_buffer_list_queue(&bl, IPU6_ISYS_BUFFER_LIST_FL_INCOMING, 0);
 
 	return ret;
 }
@@ -681,32 +690,42 @@ static int ipu6_isys_csi2_disable_streams(struct v4l2_subdev *sd,
 					  struct v4l2_subdev_state *state,
 					  u32 pad, u64 streams_mask)
 {
-	struct media_pad *remote_pad,
+	struct media_pad *remote_pad =
+		media_pad_remote_pad_first(&sd->entity.pads[CSI2_PAD_SINK]),
 		*vdev_pad = media_pad_remote_pad_unique(&sd->entity.pads[pad]);
 	struct ipu6_isys_video *av =
 		container_of_const(vdev_pad, struct ipu6_isys_video, pad);
+	struct v4l2_subdev *remote_sd =
+		media_entity_to_v4l2_subdev(remote_pad->entity);
 	struct ipu6_isys_subdev *asd = to_ipu6_isys_subdev(sd);
 	struct ipu6_isys_csi2 *csi2 = to_ipu6_isys_csi2(asd);
 	struct ipu6_device *isp = asd->isys->adev->isp;
-	struct v4l2_subdev *remote_sd;
+	struct v4l2_mbus_frame_desc desc = { 0 };
 	u64 sink_streams;
+	int ret;
 	u8 vc;
 
 	lockdep_assert_held(&csi2->isys->stream_mutex);
 
-	if (!ipu6_isys_csi2_streaming_change(asd, state, pad, &vc, false))
-		goto out_del_csi2_entry;
-
-	csi2->streaming_vc &= ~BIT(vc);
+	ret = v4l2_subdev_call(remote_sd, pad, get_frame_desc,
+			       remote_pad->index, &desc);
+	if (ret)
+		return ret;
 
 	sink_streams =
 		v4l2_subdev_state_xlate_streams(state, pad, CSI2_PAD_SINK,
 						&streams_mask);
 
-	remote_pad = media_pad_remote_pad_first(&sd->entity.pads[CSI2_PAD_SINK]);
-	remote_sd = media_entity_to_v4l2_subdev(remote_pad->entity);
+	csi2->stream_ids &= ~sink_streams;
 
-	ipu6_isys_stop_streaming_firmware(av);
+	if (!ipu6_isys_csi2_streaming_change(asd, state, pad, &desc, &vc, false))
+		goto out_del_csi2_entry;
+
+	csi2->streaming_vc &= ~BIT(vc);
+
+	struct ipu6_isys_stream *stream =
+		ipu6_isys_find_stream_firmware(csi2, vc);
+	ipu6_isys_stop_stream_firmware(stream);
 
 	if IS_IPU7(isp)
 		ipu7_isys_csi2_stream_disable(csi2);
@@ -714,9 +733,10 @@ static int ipu6_isys_csi2_disable_streams(struct v4l2_subdev *sd,
 		ipu6_isys_csi2_stream_disable(csi2);
 
 	v4l2_subdev_disable_streams(remote_sd, remote_pad->index,
-				    csi2->stream_ids);
+				    csi2->stream_ids | sink_streams);
 
-	ipu6_isys_close_streaming_firmware(av);
+	ipu6_isys_close_stream_firmware(stream);
+	ipu6_isys_free_stream_firmware(stream);
 
 	ipu6_isys_csi2_clear_watermark(csi2);
 
@@ -869,6 +889,7 @@ int ipu6_isys_csi2_init(struct ipu6_isys_csi2 *csi2,
 		goto fail;
 
 	INIT_LIST_HEAD(&csi2->av_head);
+	INIT_LIST_HEAD(&csi2->streams);
 	csi2->asd.source = IPU6_FW_ISYS_STREAM_SRC_CSI2_PORT0 + index;
 	csi2->asd.supported_codes = csi2_supported_codes;
 	snprintf(csi2->asd.sd.name, sizeof(csi2->asd.sd.name),
@@ -919,56 +940,3 @@ void ipu6_isys_csi2_eof_event_by_stream(struct ipu6_isys_stream *stream)
 	dev_dbg(dev, "eof_event::csi2-%i sequence: %i\n",
 		csi2->port, frame_sequence);
 }
-
-int ipu6_isys_csi2_get_remote_desc(u32 source_stream,
-				   struct ipu6_isys_csi2 *csi2,
-				   struct media_entity *source_entity,
-				   struct v4l2_mbus_frame_desc_entry *entry)
-{
-	struct v4l2_mbus_frame_desc_entry *desc_entry = NULL;
-	struct device *dev = &csi2->isys->adev->auxdev.dev;
-	struct v4l2_mbus_frame_desc desc;
-	struct v4l2_subdev *source;
-	struct media_pad *pad;
-	unsigned int i;
-	int ret;
-
-	source = media_entity_to_v4l2_subdev(source_entity);
-	if (!source)
-		return -EPIPE;
-
-	pad = media_pad_remote_pad_first(&csi2->asd.pad[CSI2_PAD_SINK]);
-	if (!pad)
-		return -EPIPE;
-
-	ret = v4l2_subdev_call(source, pad, get_frame_desc, pad->index, &desc);
-	if (ret)
-		return ret;
-
-	if (desc.type != V4L2_MBUS_FRAME_DESC_TYPE_CSI2) {
-		dev_err(dev, "Unsupported frame descriptor type\n");
-		return -EINVAL;
-	}
-
-	for (i = 0; i < desc.num_entries; i++) {
-		if (source_stream == desc.entry[i].stream) {
-			desc_entry = &desc.entry[i];
-			break;
-		}
-	}
-
-	if (!desc_entry) {
-		dev_err(dev, "Failed to find stream %u from remote subdev\n",
-			source_stream);
-		return -EINVAL;
-	}
-
-	if (desc_entry->bus.csi2.vc >= NR_OF_CSI2_VC) {
-		dev_err(dev, "invalid vc %d\n", desc_entry->bus.csi2.vc);
-		return -EINVAL;
-	}
-
-	*entry = *desc_entry;
-
-	return 0;
-}
diff --git a/drivers/media/pci/intel/ipu6/ipu6-isys-csi2.h b/drivers/media/pci/intel/ipu6/ipu6-isys-csi2.h
index 49d01e938dc8..1620ac16f90d 100644
--- a/drivers/media/pci/intel/ipu6/ipu6-isys-csi2.h
+++ b/drivers/media/pci/intel/ipu6/ipu6-isys-csi2.h
@@ -42,6 +42,8 @@ struct ipu6_isys_csi2 {
 	struct ipu6_isys *isys;
 	struct ipu6_isys_video av[NR_OF_CSI2_SRC_PADS];
 	struct list_head av_head;
+	struct list_head streams;
+	struct ipu6_isys_stream *streams_by_vc[NR_OF_CSI2_VC];
 
 	void __iomem *base;
 	u32 receiver_errors;
@@ -82,9 +84,5 @@ void ipu6_isys_csi2_sof_event_by_stream(struct ipu6_isys_stream *stream);
 void ipu6_isys_csi2_eof_event_by_stream(struct ipu6_isys_stream *stream);
 void ipu6_isys_register_errors(struct ipu6_isys_csi2 *csi2);
 void ipu6_isys_csi2_error(struct ipu6_isys_csi2 *csi2);
-int ipu6_isys_csi2_get_remote_desc(u32 source_stream,
-				   struct ipu6_isys_csi2 *csi2,
-				   struct media_entity *source_entity,
-				   struct v4l2_mbus_frame_desc_entry *entry);
 
 #endif /* IPU6_ISYS_CSI2_H */
diff --git a/drivers/media/pci/intel/ipu6/ipu6-isys-queue.c b/drivers/media/pci/intel/ipu6/ipu6-isys-queue.c
index 725c3603d9e1..fd2c55a5e781 100644
--- a/drivers/media/pci/intel/ipu6/ipu6-isys-queue.c
+++ b/drivers/media/pci/intel/ipu6/ipu6-isys-queue.c
@@ -239,20 +239,23 @@ static int ipu6_isys_stream_start(struct ipu6_isys_video *av)
 {
 	struct ipu6_bus_device *adev = av->isys->adev;
 	const struct ipu6_fw_isys_ops *fw_ops = adev->auxdrv_data->fw_ops;
-	struct ipu6_isys_stream *stream = av->stream;
 	struct device *dev = &adev->auxdev.dev;
 	struct ipu6_isys_buffer_list bl;
 	struct isys_fw_msgs *msg;
 	int ret;
 
-	guard(mutex)(&stream->isys->stream_mutex);
+	guard(mutex)(&av->isys->stream_mutex);
 	ret = ipu6_isys_video_set_streaming(av, 1);
 	if (ret)
 		return ret;
 
-	if (!(BIT(stream->vc) & av->csi2->streaming_vc))
+	if (!av->stream || !(BIT(av->stream->vc) & av->csi2->streaming_vc))
 		return 0;
 
+	struct ipu6_isys_stream *stream = av->stream;
+
+	guard(mutex)(&stream->mutex);
+
 	do {
 		ret = ipu6_isys_buffer_list_get(stream, &bl);
 		if (ret < 0)
@@ -470,7 +473,6 @@ static void return_buffers(struct ipu6_isys_queue *aq,
 static void ipu6_isys_stream_cleanup(struct ipu6_isys_video *av)
 {
 	video_device_pipeline_stop(&av->vdev);
-	ipu6_isys_put_stream(av->stream);
 	av->stream = NULL;
 }
 
@@ -481,9 +483,7 @@ static int start_streaming(struct vb2_queue *q, unsigned int count)
 	struct device *dev = &av->isys->adev->auxdev.dev;
 	const struct ipu6_isys_pixelformat *pfmt =
 		ipu6_isys_get_isys_format(ipu6_isys_get_format(av), 0);
-	struct ipu6_isys_stream *stream;
 	struct media_pad *source_pad, *remote_pad;
-	bool first;
 	int ret;
 
 	dev_dbg(dev, "stream: %s: width %u, height %u, css pixelformat %u\n",
@@ -504,12 +504,9 @@ static int start_streaming(struct vb2_queue *q, unsigned int count)
 		goto out_return_buffers;
 	}
 
-	ret = ipu6_isys_setup_video(av, remote_pad, source_pad);
-	if (ret < 0) {
-		dev_dbg(dev, "failed to setup video\n");
+	ret = video_device_pipeline_alloc_start(&av->vdev);
+	if (ret < 0)
 		goto out_return_buffers;
-	}
-	first = ret;
 
 	ret = ipu6_isys_link_fmt_validate(aq);
 	if (ret) {
@@ -523,29 +520,13 @@ static int start_streaming(struct vb2_queue *q, unsigned int count)
 	if (ret)
 		goto out_pipeline_stop;
 
-	stream = av->stream;
-	mutex_lock(&stream->mutex);
-	if (first) {
-		ret = ipu6_isys_video_prepare_stream(av, source_pad->entity);
-		if (ret)
-			goto out_fw_close;
-	}
-
-	list_add(&aq->node, &stream->queues);
-
 	ret = ipu6_isys_stream_start(av);
 	if (ret)
-		goto out_stream_start;
-
-	mutex_unlock(&stream->mutex);
+		goto out_fw_close;
 
 	return 0;
 
-out_stream_start:
-	list_del(&aq->node);
-
 out_fw_close:
-	mutex_unlock(&stream->mutex);
 	ipu6_isys_fw_close(av->isys);
 
 out_pipeline_stop:
@@ -561,17 +542,11 @@ static void stop_streaming(struct vb2_queue *q)
 {
 	struct ipu6_isys_queue *aq = vb2_queue_to_isys_queue(q);
 	struct ipu6_isys_video *av = ipu6_isys_queue_to_video(aq);
-	struct ipu6_isys_stream *stream = av->stream;
-
-	mutex_lock(&stream->mutex);
 
 	mutex_lock(&av->isys->stream_mutex);
 	ipu6_isys_video_set_streaming(av, 0);
-	list_del(&aq->node);
 	mutex_unlock(&av->isys->stream_mutex);
 
-	mutex_unlock(&stream->mutex);
-
 	ipu6_isys_stream_cleanup(av);
 
 	return_buffers(aq, VB2_BUF_STATE_ERROR);
diff --git a/drivers/media/pci/intel/ipu6/ipu6-isys-video.c b/drivers/media/pci/intel/ipu6/ipu6-isys-video.c
index 4037a3c80dc6..1bb91627c7db 100644
--- a/drivers/media/pci/intel/ipu6/ipu6-isys-video.c
+++ b/drivers/media/pci/intel/ipu6/ipu6-isys-video.c
@@ -432,30 +432,77 @@ static int link_validate(struct media_link *link)
 	return ret;
 }
 
-static void get_stream_opened(struct ipu6_isys_video *av)
+static void get_stream_opened(struct ipu6_isys *isys)
 {
 	unsigned long flags;
 
-	spin_lock_irqsave(&av->isys->streams_lock, flags);
-	av->isys->stream_opened++;
-	spin_unlock_irqrestore(&av->isys->streams_lock, flags);
+	spin_lock_irqsave(&isys->streams_lock, flags);
+	isys->stream_opened++;
+	spin_unlock_irqrestore(&isys->streams_lock, flags);
 }
 
-static void put_stream_opened(struct ipu6_isys_video *av)
+static void put_stream_opened(struct ipu6_isys *isys)
 {
 	unsigned long flags;
 
-	spin_lock_irqsave(&av->isys->streams_lock, flags);
-	av->isys->stream_opened--;
-	spin_unlock_irqrestore(&av->isys->streams_lock, flags);
+	spin_lock_irqsave(&isys->streams_lock, flags);
+	isys->stream_opened--;
+	spin_unlock_irqrestore(&isys->streams_lock, flags);
 }
 
-int ipu6_isys_start_stream_firmware(struct ipu6_isys_video *av,
-				    struct ipu6_isys_buffer_list *bl)
+int ipu6_isys_fw_pins_prepare(struct ipu6_isys_stream *stream,
+			      struct v4l2_mbus_frame_desc *desc,
+			      int (*fw_pin_cfg)(struct ipu6_isys_video *av,
+						struct ipu6_isys_stream *stream,
+						struct media_pad *src_pad,
+						struct v4l2_mbus_frame_desc_entry *entry,
+						void *__cfg), void *stream_cfg)
 {
-	struct ipu6_bus_device *adev = av->isys->adev;
+	struct v4l2_subdev_state *csi2_state =
+		v4l2_subdev_get_locked_active_state(&stream->asd->sd);
+	struct device *dev = &stream->isys->adev->auxdev.dev;
+	struct ipu6_isys_queue *aq;
+
+	list_for_each_entry(aq, &stream->queues, node) {
+		struct ipu6_isys_video *__av = ipu6_isys_queue_to_video(aq);
+		struct media_pad *remote_pad =
+			media_pad_remote_pad_first(&__av->pad);
+		u64 source_streams = 1;
+		unsigned int sink_stream =
+			__ffs(v4l2_subdev_state_xlate_streams(csi2_state,
+							      remote_pad->index,
+							      CSI2_PAD_SINK,
+							      &source_streams));
+		struct v4l2_mbus_frame_desc_entry *entry = NULL;
+		int ret;
+
+		for (unsigned int i = 0; i < desc->num_entries; i++) {
+			if (desc->entry[i].stream == sink_stream) {
+				entry = &desc->entry[i];
+				break;
+			}
+		}
+
+		if (!entry) {
+			dev_err(dev, "cannot find frame desc entry for sink stream %u\n",
+				sink_stream);
+			return -EINVAL;
+		}
+
+		ret = fw_pin_cfg(__av, stream, remote_pad, entry, stream_cfg);
+		if (ret < 0)
+			return ret;
+	}
+
+	return 0;
+}
+
+int ipu6_isys_start_stream_firmware(struct ipu6_isys_stream *stream,
+				    struct ipu6_isys_buffer_list *bl,
+				    struct v4l2_mbus_frame_desc *desc)
+{
+	struct ipu6_bus_device *adev = stream->asd->isys->adev;
 	const struct ipu6_fw_isys_ops *fw_ops = adev->auxdrv_data->fw_ops;
-	struct ipu6_isys_stream *stream = av->stream;
 	struct device *dev = &adev->auxdev.dev;
 	struct isys_fw_msgs *msg = NULL;
 	int ret, retout, tout;
@@ -465,27 +512,27 @@ int ipu6_isys_start_stream_firmware(struct ipu6_isys_video *av,
 	if (!msg)
 		return -ENOMEM;
 
-	ret = fw_ops->prepare_stream_cfg(av, msg);
+	ret = fw_ops->prepare_stream_cfg(stream, desc, msg);
 	if (ret < 0) {
-		ipu6_put_fw_msg_buf(av->isys, msg);
+		ipu6_put_fw_msg_buf(stream->isys, msg);
 		return ret;
 	}
 
 	reinit_completion(&stream->stream_open_completion);
 
-	ret = fw_ops->stream_open(av->isys, stream->stream_handle, msg);
+	ret = fw_ops->stream_open(stream->isys, stream->stream_handle, msg);
 	if (ret < 0) {
 		dev_err(dev, "can't open stream (%d)\n", ret);
-		ipu6_put_fw_msg_buf(av->isys, msg);
+		ipu6_put_fw_msg_buf(stream->isys, msg);
 		return ret;
 	}
 
-	get_stream_opened(av);
+	get_stream_opened(stream->isys);
 
 	tout = wait_for_completion_timeout(&stream->stream_open_completion,
 					   IPU6_FW_CALL_TIMEOUT_JIFFIES);
 
-	ipu6_put_fw_msg_buf(av->isys, msg);
+	ipu6_put_fw_msg_buf(stream->isys, msg);
 
 	if (!tout) {
 		dev_err(dev, "stream open time out\n");
@@ -509,7 +556,7 @@ int ipu6_isys_start_stream_firmware(struct ipu6_isys_video *av,
 
 	reinit_completion(&stream->stream_start_completion);
 
-	ret = fw_ops->stream_start(av->isys, stream->stream_handle, msg,
+	ret = fw_ops->stream_start(stream->isys, stream->stream_handle, msg,
 				   capture);
 	if (ret < 0) {
 		dev_err(dev, "can't start streaming (%d)\n", ret);
@@ -535,7 +582,7 @@ int ipu6_isys_start_stream_firmware(struct ipu6_isys_video *av,
 out_stream_close:
 	reinit_completion(&stream->stream_close_completion);
 
-	retout = fw_ops->stream_close(av->isys, stream->stream_handle);
+	retout = fw_ops->stream_close(stream->isys, stream->stream_handle);
 	if (retout < 0) {
 		dev_dbg(dev, "can't close stream (%d)\n", retout);
 		goto out_put_stream_opened;
@@ -551,22 +598,21 @@ int ipu6_isys_start_stream_firmware(struct ipu6_isys_video *av,
 		dev_dbg(dev, "stream close complete\n");
 
 out_put_stream_opened:
-	put_stream_opened(av);
+	put_stream_opened(stream->isys);
 
 	return ret;
 }
 
-void ipu6_isys_stop_streaming_firmware(struct ipu6_isys_video *av)
+void ipu6_isys_stop_stream_firmware(struct ipu6_isys_stream *stream)
 {
-	struct ipu6_bus_device *adev = av->isys->adev;
+	struct ipu6_bus_device *adev = stream->asd->isys->adev;
 	const struct ipu6_fw_isys_ops *fw_ops = adev->auxdrv_data->fw_ops;
 	struct device *dev = &adev->auxdev.dev;
-	struct ipu6_isys_stream *stream = av->stream;
 	int ret, tout;
 
 	reinit_completion(&stream->stream_stop_completion);
 
-	ret = fw_ops->stream_flush(av->isys, stream->stream_handle);
+	ret = fw_ops->stream_flush(stream->isys, stream->stream_handle);
 	if (ret < 0) {
 		dev_err(dev, "can't stop stream (%d)\n", ret);
 		return;
@@ -582,17 +628,17 @@ void ipu6_isys_stop_streaming_firmware(struct ipu6_isys_video *av)
 		dev_dbg(dev, "stop stream: complete\n");
 }
 
-void ipu6_isys_close_streaming_firmware(struct ipu6_isys_video *av)
+void ipu6_isys_close_stream_firmware(struct ipu6_isys_stream *stream)
 {
-	struct ipu6_bus_device *adev = av->isys->adev;
+	struct ipu6_bus_device *adev = stream->asd->isys->adev;
 	const struct ipu6_fw_isys_ops *fw_ops = adev->auxdrv_data->fw_ops;
-	struct ipu6_isys_stream *stream = av->stream;
 	struct device *dev = &adev->auxdev.dev;
+	struct ipu6_isys_csi2 *csi2 = ipu6_isys_subdev_to_csi2(stream->asd);
 	int ret, tout;
 
 	reinit_completion(&stream->stream_close_completion);
 
-	ret = fw_ops->stream_close(av->isys, stream->stream_handle);
+	ret = fw_ops->stream_close(stream->isys, stream->stream_handle);
 	if (ret < 0) {
 		dev_err(dev, "can't close stream (%d)\n", ret);
 		return;
@@ -607,156 +653,126 @@ void ipu6_isys_close_streaming_firmware(struct ipu6_isys_video *av)
 	else
 		dev_dbg(dev, "close stream: complete\n");
 
-	put_stream_opened(av);
+	put_stream_opened(stream->isys);
+
+	scoped_guard(spinlock_irqsave, &stream->isys->power_lock) {
+		stream->isys->streams_by_handle[stream->stream_handle] = NULL;
+		csi2->streams_by_vc[stream->vc] = NULL;
+	}
 }
 
-int ipu6_isys_video_prepare_stream(struct ipu6_isys_video *av,
-				   struct media_entity *source_entity)
+struct ipu6_isys_stream *
+ipu6_isys_find_stream_firmware(struct ipu6_isys_csi2 *csi2, u8 vc)
 {
-	struct ipu6_isys_stream *stream = av->stream;
-	struct ipu6_isys_csi2 *csi2;
+	struct ipu6_isys_stream *stream;
 
-	atomic_set(&stream->sequence, 0);
-	atomic_set(&stream->buf_id, 0);
+	list_for_each_entry(stream, &csi2->streams, csi2_entry)
+		if (stream->vc == vc)
+			return stream;
 
-	stream->seq_index = 0;
-	memset(stream->seq, 0, sizeof(stream->seq));
+	return NULL;
+}
 
-	if (WARN_ON(!list_empty(&stream->queues)))
-		return -EINVAL;
+void ipu6_isys_free_stream_firmware(struct ipu6_isys_stream *stream)
+{
+	struct ipu6_isys_csi2 *csi2 = ipu6_isys_subdev_to_csi2(stream->asd);
+	struct ipu6_isys_queue *aq, *aq_safe;
 
-	stream->stream_source = stream->asd->source;
-	csi2 = ipu6_isys_subdev_to_csi2(stream->asd);
-	csi2->receiver_errors = 0;
+	list_for_each_entry_safe(aq, aq_safe, &stream->queues, node) {
+		struct ipu6_isys_video *av =
+			container_of_const(aq, struct ipu6_isys_video, aq);
 
-	dev_dbg(&av->isys->adev->auxdev.dev,
-		"prepare stream: external entity %s\n",
-		source_entity->name);
+		list_del(&aq->node);
+		av->stream = NULL;
+	}
 
-	return 0;
+	list_del(&stream->csi2_entry);
+	ida_free(&csi2->isys->streams, stream->stream_handle);
+	kfree(stream);
 }
 
-void ipu6_isys_put_stream(struct ipu6_isys_stream *stream)
+struct ipu6_isys_stream *
+ipu6_isys_alloc_stream_firmware(struct ipu6_isys_csi2 *csi2,
+				struct v4l2_subdev_state *csi2_state,
+				struct v4l2_mbus_frame_desc *desc,
+				u8 vc)
 {
-	struct device *dev;
-	unsigned int i;
-	unsigned long flags;
-
-	if (!stream) {
-		pr_err("ipu6-isys: no available stream\n");
-		return;
-	}
-
-	dev = &stream->isys->adev->auxdev.dev;
+	struct device *dev = &csi2->isys->adev->auxdev.dev;
+	struct ipu6_isys_stream *stream;
+	struct v4l2_subdev_route *route;
+	int ret;
 
-	spin_lock_irqsave(&stream->isys->streams_lock, flags);
-	for (i = 0; i < IPU6_ISYS_MAX_STREAMS; i++) {
-		if (&stream->isys->streams[i] == stream) {
-			if (stream->isys->streams_ref_count[i] > 0)
-				stream->isys->streams_ref_count[i]--;
-			else
-				dev_warn(dev, "invalid stream %d\n", i);
+	stream = kzalloc_obj(*stream);
+	if (!stream)
+		return ERR_PTR(-ENOMEM);
 
-			break;
-		}
+	ret = ida_alloc_max(&csi2->isys->streams, IPU6_ISYS_MAX_STREAMS - 1,
+			    GFP_KERNEL);
+	if (ret < 0)
+		goto err_free_stream;
+
+	stream->stream_handle = ret;
+	mutex_init(&stream->mutex);
+	init_completion(&stream->stream_open_completion);
+	init_completion(&stream->stream_close_completion);
+	init_completion(&stream->stream_start_completion);
+	init_completion(&stream->stream_stop_completion);
+	INIT_LIST_HEAD(&stream->queues);
+	stream->isys = csi2->asd.isys;
+	stream->asd = &csi2->asd;
+	stream->vc = vc;
+
+	scoped_guard(spinlock_irqsave, &stream->isys->power_lock) {
+		stream->isys->streams_by_handle[stream->stream_handle] =
+			stream;
+		csi2->streams_by_vc[stream->vc] = stream;
 	}
-	spin_unlock_irqrestore(&stream->isys->streams_lock, flags);
-}
 
-static struct ipu6_isys_stream *
-ipu6_isys_get_stream(struct ipu6_isys_video *av, struct ipu6_isys_subdev *asd)
-{
-	struct ipu6_isys_stream *stream = NULL;
-	struct ipu6_isys *isys = av->isys;
-	unsigned long flags;
-	unsigned int i;
-	u8 vc = av->vc;
+	list_add(&stream->csi2_entry, &csi2->streams);
 
-	if (!isys)
-		return NULL;
+	for_each_active_route(&csi2_state->routing, route) {
+		struct media_pad *vdev_pad =
+			media_pad_remote_pad_first(&csi2->asd.pad[route->source_pad]);
+		struct v4l2_mbus_frame_desc_entry *entry = NULL;
 
-	spin_lock_irqsave(&isys->streams_lock, flags);
-	for (i = 0; i < IPU6_ISYS_MAX_STREAMS; i++) {
-		if (isys->streams_ref_count[i] && isys->streams[i].vc == vc &&
-		    isys->streams[i].asd == asd) {
-			isys->streams_ref_count[i]++;
-			stream = &isys->streams[i];
+		for (unsigned int i = 0; i < desc->num_entries; i++) {
+			if (desc->entry[i].stream != route->sink_stream)
+				continue;
+
+			entry = &desc->entry[i];
 			break;
 		}
-	}
 
-	if (!stream) {
-		for (i = 0; i < IPU6_ISYS_MAX_STREAMS; i++) {
-			if (!isys->streams_ref_count[i]) {
-				isys->streams_ref_count[i]++;
-				stream = &isys->streams[i];
-				stream->vc = vc;
-				stream->asd = asd;
-				break;
-			}
+		if (!entry) {
+			dev_dbg(dev, "cannot find stream %u in frame desc\n",
+				route->sink_stream);
+			ret = -EINVAL;
+			goto err_ida_free;
 		}
-	}
-	spin_unlock_irqrestore(&isys->streams_lock, flags);
-
-	return stream;
-}
 
-struct ipu6_isys_stream *
-ipu6_isys_query_stream_by_handle(struct ipu6_isys *isys, u8 stream_handle)
-{
-	unsigned long flags;
-	struct ipu6_isys_stream *stream = NULL;
+		if (entry->bus.csi2.vc != vc)
+			continue;
 
-	if (!isys)
-		return NULL;
+		struct ipu6_isys_video *av =
+			container_of_const(vdev_pad, struct ipu6_isys_video,
+					   pad);
 
-	if (stream_handle >= IPU6_ISYS_MAX_STREAMS) {
-		dev_err(&isys->adev->auxdev.dev,
-			"stream_handle %d is invalid\n", stream_handle);
-		return NULL;
-	}
+		list_add(&av->aq.node, &stream->queues);
 
-	spin_lock_irqsave(&isys->streams_lock, flags);
-	if (isys->streams_ref_count[stream_handle] > 0) {
-		isys->streams_ref_count[stream_handle]++;
-		stream = &isys->streams[stream_handle];
+		stream->nr_output_pins++;
+		av->stream = stream;
 	}
-	spin_unlock_irqrestore(&isys->streams_lock, flags);
 
 	return stream;
-}
 
-struct ipu6_isys_stream *
-ipu6_isys_query_stream_by_source(struct ipu6_isys *isys, int source, u8 vc)
-{
-	struct ipu6_isys_stream *stream = NULL;
-	unsigned long flags;
-	unsigned int i;
+err_ida_free:
+	list_del(&stream->csi2_entry);
+	ida_free(&csi2->isys->streams, stream->stream_handle);
 
-	if (!isys)
-		return NULL;
+err_free_stream:
+	kfree(stream);
 
-	if (source < 0) {
-		dev_err(&isys->adev->auxdev.dev,
-			"query stream with invalid port number\n");
-		return NULL;
-	}
-
-	spin_lock_irqsave(&isys->streams_lock, flags);
-	for (i = 0; i < IPU6_ISYS_MAX_STREAMS; i++) {
-		if (!isys->streams_ref_count[i])
-			continue;
-
-		if (isys->streams[i].stream_source == source &&
-		    isys->streams[i].vc == vc) {
-			stream = &isys->streams[i];
-			isys->streams_ref_count[i]++;
-			break;
-		}
-	}
-	spin_unlock_irqrestore(&isys->streams_lock, flags);
-
-	return stream;
+	return ERR_PTR(ret);
 }
 
 int ipu6_isys_video_set_streaming(struct ipu6_isys_video *av, int state)
@@ -899,72 +915,6 @@ void ipu6_isys_fw_close(struct ipu6_isys *isys)
 		pm_runtime_put(&isys->adev->auxdev.dev);
 }
 
-int ipu6_isys_setup_video(struct ipu6_isys_video *av,
-			  struct media_pad *remote_pad,
-			  struct media_pad *source_pad)
-{
-	const struct ipu6_isys_pixelformat *pfmt =
-		ipu6_isys_get_isys_format(ipu6_isys_get_format(av), 0);
-	struct device *dev = &av->isys->adev->auxdev.dev;
-	struct v4l2_mbus_frame_desc_entry entry;
-	struct v4l2_subdev_route *route = NULL;
-	struct v4l2_subdev_route *r;
-	struct v4l2_subdev_state *state;
-	struct v4l2_subdev *remote_sd =
-		media_entity_to_v4l2_subdev(remote_pad->entity);
-	struct ipu6_isys_subdev *asd = to_ipu6_isys_subdev(remote_sd);
-	struct media_pipeline *pipeline;
-	int ret = -EINVAL;
-
-	/* Find the root */
-	state = v4l2_subdev_lock_and_get_active_state(remote_sd);
-	for_each_active_route(&state->routing, r)
-		if (r->source_pad == remote_pad->index)
-			route = r;
-
-	if (!route) {
-		v4l2_subdev_unlock_state(state);
-		dev_dbg(dev, "Failed to find route\n");
-		return -ENODEV;
-	}
-	av->source_stream = route->sink_stream;
-	v4l2_subdev_unlock_state(state);
-
-	ret = ipu6_isys_csi2_get_remote_desc(av->source_stream,
-					     to_ipu6_isys_csi2(asd),
-					     source_pad->entity, &entry);
-	if (ret == -ENOIOCTLCMD) {
-		av->vc = 0;
-		av->dt = ipu6_isys_mbus_code_to_mipi(pfmt->code);
-	} else if (!ret) {
-		dev_dbg(dev, "Framedesc: stream %u, len %u, vc %u, dt %#x\n",
-			entry.stream, entry.length, entry.bus.csi2.vc,
-			entry.bus.csi2.dt);
-
-		av->vc = entry.bus.csi2.vc;
-		av->dt = entry.bus.csi2.dt;
-	} else {
-		dev_err(dev, "failed to get remote frame desc\n");
-		return ret;
-	}
-
-	pipeline = video_device_pipeline(&av->vdev);
-	ret = video_device_pipeline_alloc_start(&av->vdev);
-	if (ret < 0) {
-		dev_dbg(dev, "media pipeline start failed\n");
-		return ret;
-	}
-
-	av->stream = ipu6_isys_get_stream(av, asd);
-	if (!av->stream) {
-		video_device_pipeline_stop(&av->vdev);
-		dev_err(dev, "no available stream for firmware\n");
-		return -EINVAL;
-	}
-
-	return !pipeline;
-}
-
 /*
  * Do everything that's needed to initialise things related to video
  * buffer queue, video node, and the related media entity. The caller
diff --git a/drivers/media/pci/intel/ipu6/ipu6-isys-video.h b/drivers/media/pci/intel/ipu6/ipu6-isys-video.h
index f5b773e16d62..2821b9b1b943 100644
--- a/drivers/media/pci/intel/ipu6/ipu6-isys-video.h
+++ b/drivers/media/pci/intel/ipu6/ipu6-isys-video.h
@@ -22,6 +22,7 @@ struct file;
 struct ipu6_isys;
 struct ipu6_isys_csi2;
 struct ipu6_isys_subdev;
+struct v4l2_mbus_frame_desc;
 
 struct ipu6_isys_pixelformat {
 	u32 pixelformat;
@@ -47,12 +48,12 @@ struct ipu6_isys_stream {
 	atomic_t buf_id;
 	unsigned int seq_index;
 	struct sequence_info seq[IPU6_ISYS_MAX_PARALLEL_SOF];
-	int stream_source;
 	int stream_handle;
 	unsigned int nr_output_pins;
 	struct ipu6_isys_subdev *asd;
-
 	struct list_head queues;
+	struct list_head csi2_entry;
+
 	struct completion stream_open_completion;
 	struct completion stream_close_completion;
 	struct completion stream_start_completion;
@@ -78,8 +79,6 @@ struct ipu6_isys_video {
 	struct ipu6_isys_stream *stream;
 	unsigned int streaming;
 	u32 source_stream;
-	u8 vc;
-	u8 dt;
 };
 
 #define ipu6_isys_queue_to_video(__aq) \
@@ -90,12 +89,26 @@ extern const struct ipu6_isys_pixelformat ipu6_isys_pfmts_packed[];
 
 const struct ipu6_isys_pixelformat *
 ipu6_isys_get_isys_format(u32 pixelformat, u32 code);
-int ipu6_isys_start_stream_firmware(struct ipu6_isys_video *av,
-				    struct ipu6_isys_buffer_list *bl);
-void ipu6_isys_stop_streaming_firmware(struct ipu6_isys_video *av);
-void ipu6_isys_close_streaming_firmware(struct ipu6_isys_video *av);
-int ipu6_isys_video_prepare_stream(struct ipu6_isys_video *av,
-				   struct media_entity *source_entity);
+int ipu6_isys_fw_pins_prepare(struct ipu6_isys_stream *stream,
+			      struct v4l2_mbus_frame_desc *desc,
+			      int (*fw_pin_cfg)(struct ipu6_isys_video *av,
+						struct ipu6_isys_stream *stream,
+						struct media_pad *src_pad,
+						struct v4l2_mbus_frame_desc_entry *entry,
+						void *__cfg), void *stream_cfg);
+int ipu6_isys_start_stream_firmware(struct ipu6_isys_stream *stream,
+				    struct ipu6_isys_buffer_list *bl,
+				    struct v4l2_mbus_frame_desc *desc);
+void ipu6_isys_stop_stream_firmware(struct ipu6_isys_stream *stream);
+void ipu6_isys_close_stream_firmware(struct ipu6_isys_stream *stream);
+struct ipu6_isys_stream *
+ipu6_isys_find_stream_firmware(struct ipu6_isys_csi2 *csi2, u8 vc);
+void ipu6_isys_free_stream_firmware(struct ipu6_isys_stream *stream);
+struct ipu6_isys_stream *
+ipu6_isys_alloc_stream_firmware(struct ipu6_isys_csi2 *csi2,
+				struct v4l2_subdev_state *state,
+				struct v4l2_mbus_frame_desc *desc,
+				u8 vc);
 int ipu6_isys_video_set_streaming(struct ipu6_isys_video *av, int state);
 int ipu6_isys_fw_open(struct ipu6_isys *isys);
 void ipu6_isys_fw_close(struct ipu6_isys *isys);
@@ -104,11 +117,6 @@ int ipu6_isys_setup_video(struct ipu6_isys_video *av,
 			  struct media_pad *source_pad);
 int ipu6_isys_video_init(struct ipu6_isys_video *av);
 void ipu6_isys_video_cleanup(struct ipu6_isys_video *av);
-void ipu6_isys_put_stream(struct ipu6_isys_stream *stream);
-struct ipu6_isys_stream *
-ipu6_isys_query_stream_by_handle(struct ipu6_isys *isys, u8 stream_handle);
-struct ipu6_isys_stream *
-ipu6_isys_query_stream_by_source(struct ipu6_isys *isys, int source, u8 vc);
 
 u32 ipu6_isys_get_format(struct ipu6_isys_video *av);
 u32 ipu6_isys_get_data_size(struct ipu6_isys_video *av);
diff --git a/drivers/media/pci/intel/ipu6/ipu6-isys.c b/drivers/media/pci/intel/ipu6/ipu6-isys.c
index 0933fb617d3b..d7a1f89ab56c 100644
--- a/drivers/media/pci/intel/ipu6/ipu6-isys.c
+++ b/drivers/media/pci/intel/ipu6/ipu6-isys.c
@@ -166,23 +166,6 @@ isys_complete_ext_device_registration(struct ipu6_isys *isys,
 	return ret;
 }
 
-static void isys_stream_init(struct ipu6_isys *isys)
-{
-	u32 i;
-
-	for (i = 0; i < IPU6_ISYS_MAX_STREAMS; i++) {
-		mutex_init(&isys->streams[i].mutex);
-		init_completion(&isys->streams[i].stream_open_completion);
-		init_completion(&isys->streams[i].stream_close_completion);
-		init_completion(&isys->streams[i].stream_start_completion);
-		init_completion(&isys->streams[i].stream_stop_completion);
-		INIT_LIST_HEAD(&isys->streams[i].queues);
-		isys->streams[i].isys = isys;
-		isys->streams[i].stream_handle = i;
-		isys->streams[i].vc = INVALID_VC_ID;
-	}
-}
-
 static void isys_csi2_unregister_subdevices(struct ipu6_isys *isys)
 {
 	const struct ipu6_isys_internal_csi2_pdata *csi2 =
@@ -965,7 +948,6 @@ static int isys_probe(struct auxiliary_device *auxdev,
 	struct ipu6_bus_device *adev = auxdev_to_adev(auxdev);
 	struct ipu6_device *isp = adev->isp;
 	struct ipu6_isys *isys;
-	unsigned int i;
 	int ret;
 
 	if (!isp->bus_ready_to_probe)
@@ -1006,7 +988,7 @@ static int isys_probe(struct auxiliary_device *auxdev,
 
 	dev_set_drvdata(&auxdev->dev, isys);
 
-	isys_stream_init(isys);
+	ida_init(&isys->streams);
 
 	cpu_latency_qos_add_request(&isys->pm_qos, PM_QOS_DEFAULT_VALUE);
 
@@ -1034,9 +1016,6 @@ static int isys_probe(struct auxiliary_device *auxdev,
 out_remove_pkg_dir_shared_buffer:
 	cpu_latency_qos_remove_request(&isys->pm_qos);
 
-	for (i = 0; i < IPU6_ISYS_MAX_STREAMS; i++)
-		mutex_destroy(&isys->streams[i].mutex);
-
 	mutex_destroy(&isys->mutex);
 	mutex_destroy(&isys->stream_mutex);
 
@@ -1046,7 +1025,8 @@ static int isys_probe(struct auxiliary_device *auxdev,
 static void isys_remove(struct auxiliary_device *auxdev)
 {
 	struct ipu6_isys *isys = dev_get_drvdata(&auxdev->dev);
-	unsigned int i;
+
+	ida_destroy(&isys->streams);
 
 	free_fw_msg_bufs(isys);
 
@@ -1055,9 +1035,6 @@ static void isys_remove(struct auxiliary_device *auxdev)
 
 	cpu_latency_qos_remove_request(&isys->pm_qos);
 
-	for (i = 0; i < IPU6_ISYS_MAX_STREAMS; i++)
-		mutex_destroy(&isys->streams[i].mutex);
-
 	mutex_destroy(&isys->stream_mutex);
 	mutex_destroy(&isys->mutex);
 }
diff --git a/drivers/media/pci/intel/ipu6/ipu6-isys.h b/drivers/media/pci/intel/ipu6/ipu6-isys.h
index b9239791dd5e..8a618ded1ef9 100644
--- a/drivers/media/pci/intel/ipu6/ipu6-isys.h
+++ b/drivers/media/pci/intel/ipu6/ipu6-isys.h
@@ -4,6 +4,7 @@
 #ifndef IPU6_ISYS_H
 #define IPU6_ISYS_H
 
+#include <linux/idr.h>
 #include <linux/irqreturn.h>
 #include <linux/list.h>
 #include <linux/mutex.h>
@@ -104,8 +105,7 @@ struct ipu6_isys {
 	u32 isr_csi2_bits;
 	u32 csi2_rx_ctrl_cached;
 	spinlock_t streams_lock;
-	struct ipu6_isys_stream streams[IPU6_ISYS_MAX_STREAMS];
-	int streams_ref_count[IPU6_ISYS_MAX_STREAMS];
+	struct ipu6_isys_stream *streams_by_handle[IPU6_ISYS_MAX_STREAMS];
 	void *fwctx;
 	u32 phy_termcal_val;
 	u32 phy_rext_cal;
@@ -134,6 +134,7 @@ struct ipu6_isys {
 	struct list_head framebuflist;
 	struct list_head framebuflist_fw;
 	struct v4l2_async_notifier notifier;
+	struct ida streams;
 };
 
 struct isys_fw_msgs {
@@ -161,7 +162,8 @@ struct ipu6_fw_isys_ops {
 			dma_addr_t dma_mapped_buf,
 			size_t size, u16 send_type);
 	void (*cleanup)(struct ipu6_isys *isys);
-	int (*prepare_stream_cfg)(struct ipu6_isys_video *av,
+	int (*prepare_stream_cfg)(struct ipu6_isys_stream *stream,
+				  struct v4l2_mbus_frame_desc *desc,
 				  struct isys_fw_msgs *msg);
 	void (*prepare_buf_set)(struct isys_fw_msgs *msg,
 				struct ipu6_isys_stream *stream,
diff --git a/drivers/media/pci/intel/ipu6/ipu7-fw-isys.c b/drivers/media/pci/intel/ipu6/ipu7-fw-isys.c
index e75ae5f5f939..1cfd7b8ee2b3 100644
--- a/drivers/media/pci/intel/ipu6/ipu7-fw-isys.c
+++ b/drivers/media/pci/intel/ipu6/ipu7-fw-isys.c
@@ -157,16 +157,18 @@ static void ipu7_fw_isys_put_resp(struct ipu6_isys *isys)
 }
 
 static int ipu7_isys_fw_pin_cfg(struct ipu6_isys_video *av,
-				struct ipu7_fw_isys_stream_cfg *cfg)
+				struct ipu6_isys_stream *stream,
+				struct media_pad *src_pad,
+				struct v4l2_mbus_frame_desc_entry *entry,
+				void *__cfg)
 {
-	struct media_pad *src_pad = media_pad_remote_pad_first(&av->pad);
 	struct v4l2_subdev *sd = media_entity_to_v4l2_subdev(src_pad->entity);
 	struct v4l2_subdev_state *state = v4l2_subdev_get_locked_active_state(sd);
+	struct ipu7_fw_isys_stream_cfg *cfg = __cfg;
 	struct ipu7_fw_isys_input_pin *input_pin;
 	struct ipu7_fw_isys_output_pin *output_pin;
-	struct ipu6_isys_stream *stream = av->stream;
 	struct ipu6_isys_queue *aq = &av->aq;
-	struct v4l2_mbus_framefmt fmt;
+	struct v4l2_mbus_framefmt *fmt;
 	const struct ipu6_isys_pixelformat *pfmt =
 		ipu6_isys_get_isys_format(ipu6_isys_get_format(av), 0);
 	int input_pins = cfg->nof_input_pins++;
@@ -174,12 +176,12 @@ static int ipu7_isys_fw_pin_cfg(struct ipu6_isys_video *av,
 	u32 src_stream;
 
 	src_stream = __ipu6_isys_get_src_stream_by_src_pad(state, src_pad->index);
-	fmt = *v4l2_subdev_state_get_format(state, src_pad->index, src_stream);
+	fmt = v4l2_subdev_state_get_format(state, src_pad->index, src_stream);
 
 	input_pin = &cfg->input_pins[input_pins];
-	input_pin->input_res.width = fmt.width;
-	input_pin->input_res.height = fmt.height;
-	input_pin->dt = av->dt;
+	input_pin->input_res.width = fmt->width;
+	input_pin->input_res.height = fmt->height;
+	input_pin->dt = entry->bus.csi2.dt;
 	input_pin->disable_mipi_unpacking = 0;
 	if (pfmt->bpp == pfmt->bpp_packed && pfmt->bpp % BITS_PER_BYTE)
 		input_pin->disable_mipi_unpacking = 1;
@@ -193,7 +195,7 @@ static int ipu7_isys_fw_pin_cfg(struct ipu6_isys_video *av,
 
 	output_pins = cfg->nof_output_pins++;
 	aq->fw_output = output_pins;
-	stream->output_pins_queue[output_pins] = aq;
+	av->stream->output_pins_queue[output_pins] = aq;
 
 	output_pin = &cfg->output_pins[output_pins];
 	memset(output_pin, 0, sizeof(*output_pin));
@@ -343,27 +345,24 @@ static void ipu7_fw_isys_dump_frame_buf_set(struct device *dev,
 	dev_dbg(dev, "---------------------------\n");
 }
 
-static int ipu7_fw_isys_prepare_stream_cfg(struct ipu6_isys_video *av,
+static int ipu7_fw_isys_prepare_stream_cfg(struct ipu6_isys_stream *stream,
+					   struct v4l2_mbus_frame_desc *desc,
 					   struct isys_fw_msgs *msg)
 {
 	struct ipu7_fw_isys_stream_cfg *cfg = &msg->ipu7.stream;
-	struct device *dev = &av->isys->adev->auxdev.dev;
-	struct ipu6_isys_stream *stream = av->stream;
-	struct ipu6_isys_queue *aq;
+	struct device *dev = &stream->isys->adev->auxdev.dev;
+	int ret;
 
 	memset(cfg, 0, sizeof(*cfg));
-	cfg->port_id = stream->stream_source;
+	cfg->port_id = stream->asd->source;
 	cfg->vc = stream->vc;
 	cfg->stream_msg_map = IPU7_INSYS_STREAM_ENABLE_MSG_SEND_RESP |
 			      IPU7_INSYS_STREAM_ENABLE_MSG_SEND_IRQ;
 
-	list_for_each_entry(aq, &stream->queues, node) {
-		struct ipu6_isys_video *__av = ipu6_isys_queue_to_video(aq);
-		int ret = ipu7_isys_fw_pin_cfg(__av, cfg);
-
-		if (ret < 0)
-			return ret;
-	}
+	ret = ipu6_isys_fw_pins_prepare(stream, desc, ipu7_isys_fw_pin_cfg,
+					cfg);
+	if (ret)
+		return ret;
 
 	stream->nr_output_pins = cfg->nof_output_pins;
 
@@ -600,7 +599,8 @@ static int ipu7_isys_isr_one(struct ipu6_bus_device *adev)
 		goto leave;
 	}
 
-	stream = ipu6_isys_query_stream_by_handle(isys, resp->stream_id);
+	stream = resp->stream_id < IPU6_ISYS_MAX_STREAMS ?
+		isys->streams_by_handle[resp->stream_id] : NULL;
 	if (!stream) {
 		dev_err(dev, "stream of stream_handle %u is unused\n",
 			resp->stream_id);
@@ -680,7 +680,6 @@ static int ipu7_isys_isr_one(struct ipu6_bus_device *adev)
 		break;
 	}
 
-	ipu6_isys_put_stream(stream);
 leave:
 	ipu7_fw_isys_put_resp(isys);
 
@@ -711,8 +710,7 @@ static void ipu7_isys_csi2_isr(struct ipu6_isys_csi2 *csi2)
 	}
 
 	for (vc = 0; vc < IPU7_NR_OF_CSI2_VC && (sync || fe); vc++) {
-		s = ipu6_isys_query_stream_by_source(csi2->isys,
-						     csi2->asd.source, vc);
+		s = csi2->streams_by_vc[vc];
 		if (!s)
 			continue;
 
-- 
2.47.3

[PATCH v2 17/21] media: ipu6: Drop {get,put}_streams_opened()

[Sakari Ailus]

{get,put}_streams_opened() aren't needed anymore. Drop them. This also
requires acquiring isys->stream_mutex for system suspend.

Signed-off-by: Sakari Ailus <sakari.ailus@linux.intel.com>
---
 .../media/pci/intel/ipu6/ipu6-isys-video.c    | 37 +++----------------
 drivers/media/pci/intel/ipu6/ipu6-isys.c      |  4 +-
 drivers/media/pci/intel/ipu6/ipu6-isys.h      |  1 -
 3 files changed, 8 insertions(+), 34 deletions(-)

diff --git a/drivers/media/pci/intel/ipu6/ipu6-isys-video.c b/drivers/media/pci/intel/ipu6/ipu6-isys-video.c
index 1bb91627c7db..cea37442866e 100644
--- a/drivers/media/pci/intel/ipu6/ipu6-isys-video.c
+++ b/drivers/media/pci/intel/ipu6/ipu6-isys-video.c
@@ -432,24 +432,6 @@ static int link_validate(struct media_link *link)
 	return ret;
 }
 
-static void get_stream_opened(struct ipu6_isys *isys)
-{
-	unsigned long flags;
-
-	spin_lock_irqsave(&isys->streams_lock, flags);
-	isys->stream_opened++;
-	spin_unlock_irqrestore(&isys->streams_lock, flags);
-}
-
-static void put_stream_opened(struct ipu6_isys *isys)
-{
-	unsigned long flags;
-
-	spin_lock_irqsave(&isys->streams_lock, flags);
-	isys->stream_opened--;
-	spin_unlock_irqrestore(&isys->streams_lock, flags);
-}
-
 int ipu6_isys_fw_pins_prepare(struct ipu6_isys_stream *stream,
 			      struct v4l2_mbus_frame_desc *desc,
 			      int (*fw_pin_cfg)(struct ipu6_isys_video *av,
@@ -527,8 +509,6 @@ int ipu6_isys_start_stream_firmware(struct ipu6_isys_stream *stream,
 		return ret;
 	}
 
-	get_stream_opened(stream->isys);
-
 	tout = wait_for_completion_timeout(&stream->stream_open_completion,
 					   IPU6_FW_CALL_TIMEOUT_JIFFIES);
 
@@ -536,21 +516,19 @@ int ipu6_isys_start_stream_firmware(struct ipu6_isys_stream *stream,
 
 	if (!tout) {
 		dev_err(dev, "stream open time out\n");
-		ret = -ETIMEDOUT;
-		goto out_put_stream_opened;
+		return -ETIMEDOUT;
 	}
 	if (stream->error) {
 		dev_err(dev, "stream open error: %d\n", stream->error);
-		ret = -EIO;
-		goto out_put_stream_opened;
+		return -EIO;
 	}
 	dev_dbg(dev, "start stream: open complete\n");
 
 	msg = ipu6_get_fw_msg_buf(stream);
 	if (!msg) {
-		ret = -ENOMEM;
-		goto out_put_stream_opened;
+		return -ENOMEM;
 	}
+
 	fw_ops->prepare_buf_set(msg, stream, bl);
 	ipu6_isys_buffer_list_queue(bl, IPU6_ISYS_BUFFER_LIST_FL_ACTIVE, 0);
 
@@ -585,7 +563,7 @@ int ipu6_isys_start_stream_firmware(struct ipu6_isys_stream *stream,
 	retout = fw_ops->stream_close(stream->isys, stream->stream_handle);
 	if (retout < 0) {
 		dev_dbg(dev, "can't close stream (%d)\n", retout);
-		goto out_put_stream_opened;
+		return retout;
 	}
 
 	tout = wait_for_completion_timeout(&stream->stream_close_completion,
@@ -597,9 +575,6 @@ int ipu6_isys_start_stream_firmware(struct ipu6_isys_stream *stream,
 	else
 		dev_dbg(dev, "stream close complete\n");
 
-out_put_stream_opened:
-	put_stream_opened(stream->isys);
-
 	return ret;
 }
 
@@ -653,8 +628,6 @@ void ipu6_isys_close_stream_firmware(struct ipu6_isys_stream *stream)
 	else
 		dev_dbg(dev, "close stream: complete\n");
 
-	put_stream_opened(stream->isys);
-
 	scoped_guard(spinlock_irqsave, &stream->isys->power_lock) {
 		stream->isys->streams_by_handle[stream->stream_handle] = NULL;
 		csi2->streams_by_vc[stream->vc] = NULL;
diff --git a/drivers/media/pci/intel/ipu6/ipu6-isys.c b/drivers/media/pci/intel/ipu6/ipu6-isys.c
index d7a1f89ab56c..5d220138cf37 100644
--- a/drivers/media/pci/intel/ipu6/ipu6-isys.c
+++ b/drivers/media/pci/intel/ipu6/ipu6-isys.c
@@ -814,8 +814,10 @@ static int isys_suspend(struct device *dev)
 {
 	struct ipu6_isys *isys = dev_get_drvdata(dev);
 
+	guard(mutex)(&isys->stream_mutex);
+
 	/* If stream is open, refuse to suspend */
-	if (isys->stream_opened)
+	if (!ida_is_empty(&isys->streams))
 		return -EBUSY;
 
 	return 0;
diff --git a/drivers/media/pci/intel/ipu6/ipu6-isys.h b/drivers/media/pci/intel/ipu6/ipu6-isys.h
index 8a618ded1ef9..d7d211ad5a54 100644
--- a/drivers/media/pci/intel/ipu6/ipu6-isys.h
+++ b/drivers/media/pci/intel/ipu6/ipu6-isys.h
@@ -114,7 +114,6 @@ struct ipu6_isys {
 	bool csi2_cse_ipc_not_supported;
 	bool iwake_watermark_enabled;
 	unsigned int ref_count;
-	unsigned int stream_opened;
 	unsigned int sensor_type;
 
 	struct mutex mutex;
-- 
2.47.3

[PATCH v2 18/21] media: ipu6: Serialise access to stream pointers by isys stream_lock

[Sakari Ailus]

Re-purpose the unused stream_lock in struct ipu6_isys to serialise access
to streams_by_handle (struct ipu6_isys) and streams_by_vc (struct
ipu6_isys_csi2). Switch to scoped_guard() while at it.

Signed-off-by: Sakari Ailus <sakari.ailus@linux.intel.com>
---
 drivers/media/pci/intel/ipu6/ipu6-fw-isys.c   | 24 ++++++++++++-------
 .../media/pci/intel/ipu6/ipu6-isys-video.c    |  7 +++---
 drivers/media/pci/intel/ipu6/ipu7-fw-isys.c   |  8 ++++++-
 3 files changed, 26 insertions(+), 13 deletions(-)

diff --git a/drivers/media/pci/intel/ipu6/ipu6-fw-isys.c b/drivers/media/pci/intel/ipu6/ipu6-fw-isys.c
index 5e1b57ef650f..0214d7105fbe 100644
--- a/drivers/media/pci/intel/ipu6/ipu6-fw-isys.c
+++ b/drivers/media/pci/intel/ipu6/ipu6-fw-isys.c
@@ -556,6 +556,8 @@ int ipu6_isys_isr_one(struct ipu6_bus_device *adev)
 			"FW error resp error %d, details %d\n",
 			resp->error_info.error, resp->error_info.error_details);
 
+	guard(spinlock_irqsave)(&isys->streams_lock);
+
 	if (resp->stream_handle >= IPU6_ISYS_MAX_STREAMS) {
 		dev_err(&adev->auxdev.dev, "bad stream handle %u\n",
 			resp->stream_handle);
@@ -670,17 +672,23 @@ static void ipu6_isys_csi2_isr(struct ipu6_isys_csi2 *csi2)
 	writel(status, csi2->base + CSI_PORT_REG_BASE_IRQ_CSI_SYNC +
 	       CSI_PORT_REG_BASE_IRQ_CLEAR_OFFSET);
 
-	for (i = 0; i < NR_OF_CSI2_VC; i++) {
-		if (status & IPU_CSI_RX_IRQ_FS_VC(i)) {
-			stream = csi2->streams_by_vc[i];
-			if (stream)
+	scoped_guard(spinlock, &csi2->isys->streams_lock) {
+		for (i = 0; i < NR_OF_CSI2_VC; i++) {
+			if (status & IPU_CSI_RX_IRQ_FS_VC(i)) {
+				stream = csi2->streams_by_vc[i];
+				if (!stream)
+					continue;
+
 				ipu6_isys_csi2_sof_event_by_stream(stream);
-		}
+			}
+
+			if (status & IPU_CSI_RX_IRQ_FE_VC(i)) {
+				stream = csi2->streams_by_vc[i];
+				if (!stream)
+					continue;
 
-		if (status & IPU_CSI_RX_IRQ_FE_VC(i)) {
-			stream = csi2->streams_by_vc[i];
-			if (stream)
 				ipu6_isys_csi2_eof_event_by_stream(stream);
+			}
 		}
 	}
 }
diff --git a/drivers/media/pci/intel/ipu6/ipu6-isys-video.c b/drivers/media/pci/intel/ipu6/ipu6-isys-video.c
index cea37442866e..feb0ad261252 100644
--- a/drivers/media/pci/intel/ipu6/ipu6-isys-video.c
+++ b/drivers/media/pci/intel/ipu6/ipu6-isys-video.c
@@ -628,7 +628,7 @@ void ipu6_isys_close_stream_firmware(struct ipu6_isys_stream *stream)
 	else
 		dev_dbg(dev, "close stream: complete\n");
 
-	scoped_guard(spinlock_irqsave, &stream->isys->power_lock) {
+	scoped_guard(spinlock_irqsave, &stream->isys->streams_lock) {
 		stream->isys->streams_by_handle[stream->stream_handle] = NULL;
 		csi2->streams_by_vc[stream->vc] = NULL;
 	}
@@ -695,9 +695,8 @@ ipu6_isys_alloc_stream_firmware(struct ipu6_isys_csi2 *csi2,
 	stream->asd = &csi2->asd;
 	stream->vc = vc;
 
-	scoped_guard(spinlock_irqsave, &stream->isys->power_lock) {
-		stream->isys->streams_by_handle[stream->stream_handle] =
-			stream;
+	scoped_guard(spinlock_irqsave, &stream->isys->streams_lock) {
+		stream->isys->streams_by_handle[stream->stream_handle] = stream;
 		csi2->streams_by_vc[stream->vc] = stream;
 	}
 
diff --git a/drivers/media/pci/intel/ipu6/ipu7-fw-isys.c b/drivers/media/pci/intel/ipu6/ipu7-fw-isys.c
index 1cfd7b8ee2b3..50219b210ae9 100644
--- a/drivers/media/pci/intel/ipu6/ipu7-fw-isys.c
+++ b/drivers/media/pci/intel/ipu6/ipu7-fw-isys.c
@@ -549,6 +549,7 @@ static int ipu7_isys_isr_one(struct ipu6_bus_device *adev)
 	struct ipu7_fw_isys_msg_err err_info;
 	struct isys_fw_msgs *isys_fw_msg;
 	struct ipu7_insys_resp *resp;
+	unsigned long flags;
 	u64 ts;
 
 	if (!isys->fwctx)
@@ -596,9 +597,11 @@ static int ipu7_isys_isr_one(struct ipu6_bus_device *adev)
 	if (resp->stream_id >= IPU7_ISYS_MAX_STREAMS) {
 		dev_err(dev, "bad stream handle %u\n",
 			resp->stream_id);
-		goto leave;
+		goto leave_nounlock;
 	}
 
+	spin_lock_irqsave(&isys->streams_lock, flags);
+
 	stream = resp->stream_id < IPU6_ISYS_MAX_STREAMS ?
 		isys->streams_by_handle[resp->stream_id] : NULL;
 	if (!stream) {
@@ -681,6 +684,9 @@ static int ipu7_isys_isr_one(struct ipu6_bus_device *adev)
 	}
 
 leave:
+	spin_unlock_irqrestore(&isys->streams_lock, flags);
+
+leave_nounlock:
 	ipu7_fw_isys_put_resp(isys);
 
 	return 0;
-- 
2.47.3

[PATCH v2 19/21] media: ipu6: Move firmware init/cleanup to RPM callbacks

[Sakari Ailus]

Move handling firmware initialisation and cleanup away from the rest of
the driver to the runtime PM callbacks. This simplifies the driver,
including removing the ref_count field in struct ipu6_isys.

Signed-off-by: Sakari Ailus <sakari.ailus@linux.intel.com>
---
 drivers/media/pci/intel/ipu6/ipu6-isys-csi2.c | 10 +++
 .../media/pci/intel/ipu6/ipu6-isys-queue.c    | 11 +--
 .../media/pci/intel/ipu6/ipu6-isys-video.c    | 74 -------------------
 drivers/media/pci/intel/ipu6/ipu6-isys.c      | 55 ++++++++++++--
 drivers/media/pci/intel/ipu6/ipu6-isys.h      |  2 -
 5 files changed, 59 insertions(+), 93 deletions(-)

diff --git a/drivers/media/pci/intel/ipu6/ipu6-isys-csi2.c b/drivers/media/pci/intel/ipu6/ipu6-isys-csi2.c
index 5dc2596766f4..780ac3c4d4e4 100644
--- a/drivers/media/pci/intel/ipu6/ipu6-isys-csi2.c
+++ b/drivers/media/pci/intel/ipu6/ipu6-isys-csi2.c
@@ -11,6 +11,7 @@
 #include <linux/err.h>
 #include <linux/io.h>
 #include <linux/minmax.h>
+#include <linux/pm_runtime.h>
 #include <linux/sprintf.h>
 #include <linux/string_choices.h>
 
@@ -626,6 +627,10 @@ static int ipu6_isys_csi2_enable_streams(struct v4l2_subdev *sd,
 	if (!ipu6_isys_csi2_streaming_change(asd, state, pad, &desc, &vc, true))
 		return 0;
 
+	ret = pm_runtime_resume_and_get(sd->dev);
+	if (ret < 0)
+		goto err_del_av;
+
 	ipu6_isys_csi2_setup_watermark(csi2, state, remote_sd);
 
 	stream = ipu6_isys_alloc_stream_firmware(csi2, state, &desc, vc);
@@ -679,6 +684,9 @@ static int ipu6_isys_csi2_enable_streams(struct v4l2_subdev *sd,
 
 err_clear_watermark:
 	ipu6_isys_csi2_clear_watermark(csi2);
+	pm_runtime_put(sd->dev);
+
+err_del_av:
 	ipu6_isys_csi2_streaming_change(asd, state, pad, &desc, NULL, false);
 	csi2->stream_ids &= ~sink_streams;
 	list_del(&av->csi2_entry);
@@ -740,6 +748,8 @@ static int ipu6_isys_csi2_disable_streams(struct v4l2_subdev *sd,
 
 	ipu6_isys_csi2_clear_watermark(csi2);
 
+	pm_runtime_put(sd->dev);
+
 out_del_csi2_entry:
 	list_del(&av->csi2_entry);
 
diff --git a/drivers/media/pci/intel/ipu6/ipu6-isys-queue.c b/drivers/media/pci/intel/ipu6/ipu6-isys-queue.c
index fd2c55a5e781..0c3576282a42 100644
--- a/drivers/media/pci/intel/ipu6/ipu6-isys-queue.c
+++ b/drivers/media/pci/intel/ipu6/ipu6-isys-queue.c
@@ -516,19 +516,12 @@ static int start_streaming(struct vb2_queue *q, unsigned int count)
 		goto out_pipeline_stop;
 	}
 
-	ret = ipu6_isys_fw_open(av->isys);
-	if (ret)
-		goto out_pipeline_stop;
-
 	ret = ipu6_isys_stream_start(av);
 	if (ret)
-		goto out_fw_close;
+		goto out_pipeline_stop;
 
 	return 0;
 
-out_fw_close:
-	ipu6_isys_fw_close(av->isys);
-
 out_pipeline_stop:
 	ipu6_isys_stream_cleanup(av);
 
@@ -550,8 +543,6 @@ static void stop_streaming(struct vb2_queue *q)
 	ipu6_isys_stream_cleanup(av);
 
 	return_buffers(aq, VB2_BUF_STATE_ERROR);
-
-	ipu6_isys_fw_close(av->isys);
 }
 
 static unsigned int
diff --git a/drivers/media/pci/intel/ipu6/ipu6-isys-video.c b/drivers/media/pci/intel/ipu6/ipu6-isys-video.c
index feb0ad261252..129016e57446 100644
--- a/drivers/media/pci/intel/ipu6/ipu6-isys-video.c
+++ b/drivers/media/pci/intel/ipu6/ipu6-isys-video.c
@@ -813,80 +813,6 @@ static const struct v4l2_file_operations isys_fops = {
 	.release = vb2_fop_release,
 };
 
-int ipu6_isys_fw_open(struct ipu6_isys *isys)
-{
-	struct ipu6_bus_device *adev = isys->adev;
-	const struct ipu6_fw_isys_ops *fw_ops = adev->auxdrv_data->fw_ops;
-	const struct ipu6_isys_internal_pdata *ipdata = isys->pdata->ipdata;
-	int ret;
-
-	ret = pm_runtime_resume_and_get(&adev->auxdev.dev);
-	if (ret < 0)
-		return ret;
-
-	mutex_lock(&isys->mutex);
-
-	if (isys->ref_count++)
-		goto unlock;
-
-	ipu6_configure_spc(adev->isp, &ipdata->hw_variant,
-			   IPU6_CPD_PKG_DIR_ISYS_SERVER_IDX, isys->pdata->base,
-			   adev->pkg_dir, adev->pkg_dir_dma_addr);
-
-	/*
-	 * Buffers could have been left to wrong queue at last closure.
-	 * Move them now back to empty buffer queue.
-	 */
-	ipu6_cleanup_fw_msg_bufs(isys);
-
-	if (isys->fwctx) {
-		/*
-		 * Something went wrong in previous shutdown. As we are now
-		 * restarting isys we can safely delete old context.
-		 */
-		dev_warn(&adev->auxdev.dev, "clearing old context\n");
-		fw_ops->cleanup(isys);
-	}
-
-	ret = fw_ops->init(isys, ipdata->num_parallel_streams);
-	if (ret < 0)
-		goto out;
-
-unlock:
-	mutex_unlock(&isys->mutex);
-
-	return 0;
-
-out:
-	isys->ref_count--;
-	mutex_unlock(&isys->mutex);
-	pm_runtime_put(&adev->auxdev.dev);
-
-	return ret;
-}
-
-void ipu6_isys_fw_close(struct ipu6_isys *isys)
-{
-	mutex_lock(&isys->mutex);
-
-	isys->ref_count--;
-	if (!isys->ref_count) {
-		isys->adev->auxdrv_data->fw_ops->close(isys);
-		if (isys->fwctx) {
-			isys->need_reset = true;
-			dev_warn(&isys->adev->auxdev.dev,
-				 "failed to close fw isys\n");
-		}
-	}
-
-	mutex_unlock(&isys->mutex);
-
-	if (isys->need_reset)
-		pm_runtime_put_sync(&isys->adev->auxdev.dev);
-	else
-		pm_runtime_put(&isys->adev->auxdev.dev);
-}
-
 /*
  * Do everything that's needed to initialise things related to video
  * buffer queue, video node, and the related media entity. The caller
diff --git a/drivers/media/pci/intel/ipu6/ipu6-isys.c b/drivers/media/pci/intel/ipu6/ipu6-isys.c
index 5d220138cf37..fc6e87e9764b 100644
--- a/drivers/media/pci/intel/ipu6/ipu6-isys.c
+++ b/drivers/media/pci/intel/ipu6/ipu6-isys.c
@@ -755,7 +755,9 @@ static void isys_unregister_devices(struct ipu6_isys *isys)
 static int isys_runtime_pm_resume(struct device *dev)
 {
 	struct ipu6_bus_device *adev = to_ipu6_bus_device(dev);
+	const struct ipu6_fw_isys_ops *fw_ops = adev->auxdrv_data->fw_ops;
 	struct ipu6_isys *isys = ipu6_bus_get_drvdata(adev);
+	const struct ipu6_isys_internal_pdata *ipdata = isys->pdata->ipdata;
 	struct ipu6_device *isp = adev->isp;
 	unsigned long flags;
 	int ret;
@@ -768,7 +770,7 @@ static int isys_runtime_pm_resume(struct device *dev)
 
 	ret = ipu6_buttress_start_tsc_sync(isp);
 	if (ret)
-		return ret;
+		goto err_mmu_hw_cleanup;
 
 	spin_lock_irqsave(&isys->power_lock, flags);
 	isys->power = 1;
@@ -781,7 +783,43 @@ static int isys_runtime_pm_resume(struct device *dev)
 		set_iwake_ltrdid(isys, 0, 0, LTR_ISYS_ON);
 	}
 
-	return 0;
+	ipu6_configure_spc(adev->isp, &ipdata->hw_variant,
+			   IPU6_CPD_PKG_DIR_ISYS_SERVER_IDX, isys->pdata->base,
+			   adev->pkg_dir, adev->pkg_dir_dma_addr);
+
+	/*
+	 * Buffers could have been left to wrong queue at last closure.
+	 * Move them now back to empty buffer queue.
+	 */
+	ipu6_cleanup_fw_msg_bufs(isys);
+
+	if (isys->fwctx) {
+		/*
+		 * Something went wrong in previous shutdown. As we are now
+		 * restarting isys we can safely delete old context.
+		 */
+		dev_warn(&adev->auxdev.dev, "clearing old context\n");
+		fw_ops->cleanup(isys);
+	}
+
+	ret = fw_ops->init(isys, ipdata->num_parallel_streams);
+	if (!ret)
+		return 0;
+
+	spin_lock_irqsave(&isys->power_lock, flags);
+	isys->power = 0;
+	spin_unlock_irqrestore(&isys->power_lock, flags);
+
+	isys->phy_termcal_val = 0;
+	cpu_latency_qos_update_request(&isys->pm_qos, PM_QOS_DEFAULT_VALUE);
+
+	if (!IS_IPU7(isp))
+		set_iwake_ltrdid(isys, 0, 0, LTR_ISYS_OFF);
+
+err_mmu_hw_cleanup:
+	ipu6_mmu_hw_cleanup(adev->mmu);
+
+	return ret;
 }
 
 static int isys_runtime_pm_suspend(struct device *dev)
@@ -790,15 +828,18 @@ static int isys_runtime_pm_suspend(struct device *dev)
 	struct ipu6_isys *isys = dev_get_drvdata(dev);
 	struct ipu6_device *isp = adev->isp;
 	unsigned long flags;
+	int ret = 0;
+
+	isys->adev->auxdrv_data->fw_ops->close(isys);
+	if (isys->fwctx) {
+		dev_warn(&isys->adev->auxdev.dev, "failed to close fw isys\n");
+		ret = -EIO;
+	}
 
 	spin_lock_irqsave(&isys->power_lock, flags);
 	isys->power = 0;
 	spin_unlock_irqrestore(&isys->power_lock, flags);
 
-	mutex_lock(&isys->mutex);
-	isys->need_reset = false;
-	mutex_unlock(&isys->mutex);
-
 	isys->phy_termcal_val = 0;
 	cpu_latency_qos_update_request(&isys->pm_qos, PM_QOS_DEFAULT_VALUE);
 
@@ -807,7 +848,7 @@ static int isys_runtime_pm_suspend(struct device *dev)
 
 	ipu6_mmu_hw_cleanup(adev->mmu);
 
-	return 0;
+	return ret;
 }
 
 static int isys_suspend(struct device *dev)
diff --git a/drivers/media/pci/intel/ipu6/ipu6-isys.h b/drivers/media/pci/intel/ipu6/ipu6-isys.h
index d7d211ad5a54..4991078c18b2 100644
--- a/drivers/media/pci/intel/ipu6/ipu6-isys.h
+++ b/drivers/media/pci/intel/ipu6/ipu6-isys.h
@@ -89,7 +89,6 @@ struct sensor_async_sd {
  * @fwctx: fw communication layer context pointer
  * @phy_termcal_val: the termination calibration value, only used for DWC PHY
  * @need_reset: Isys requires d0i0->i3 transition
- * @ref_count: total number of callers fw open
  * @mutex: serialise access isys video open/release related operations
  * @stream_mutex: serialise stream start and stop, queueing requests
  * @pdata: platform data pointer
@@ -113,7 +112,6 @@ struct ipu6_isys {
 	bool icache_prefetch;
 	bool csi2_cse_ipc_not_supported;
 	bool iwake_watermark_enabled;
-	unsigned int ref_count;
 	unsigned int sensor_type;
 
 	struct mutex mutex;
-- 
2.47.3

[PATCH v2 20/21] media: ipu6: Don't track power status, rely on runtime PM

[Sakari Ailus]

Rely on runtime PM suspending and resuming the device and so remove
tracking of power state. Also remove the power_lock spinlock that
serialised firmware initialisation and cleanup with the ISR. This is no
longer necessary as the ISR now runs when the device is in RPM_ACTIVE
state.

Signed-off-by: Sakari Ailus <sakari.ailus@linux.intel.com>
---
 drivers/media/pci/intel/ipu6/ipu6-fw-isys.c | 21 +++++++--------------
 drivers/media/pci/intel/ipu6/ipu6-isys.c    | 16 ----------------
 drivers/media/pci/intel/ipu6/ipu6-isys.h    |  5 -----
 drivers/media/pci/intel/ipu6/ipu7-fw-isys.c | 20 ++++++++++++--------
 4 files changed, 19 insertions(+), 43 deletions(-)

diff --git a/drivers/media/pci/intel/ipu6/ipu6-fw-isys.c b/drivers/media/pci/intel/ipu6/ipu6-fw-isys.c
index 0214d7105fbe..f4f1cf7c86d2 100644
--- a/drivers/media/pci/intel/ipu6/ipu6-fw-isys.c
+++ b/drivers/media/pci/intel/ipu6/ipu6-fw-isys.c
@@ -7,6 +7,7 @@
 #include <linux/delay.h>
 #include <linux/device.h>
 #include <linux/io.h>
+#include <linux/pm_runtime.h>
 #include <linux/spinlock.h>
 #include <linux/types.h>
 
@@ -131,7 +132,6 @@ static int ipu6_fw_isys_close(struct ipu6_isys *isys)
 {
 	struct device *dev = &isys->adev->auxdev.dev;
 	int retry = IPU6_ISYS_CLOSE_RETRY;
-	unsigned long flags;
 	void *fwctx;
 	int ret;
 
@@ -141,11 +141,9 @@ static int ipu6_fw_isys_close(struct ipu6_isys *isys)
 	 * to SP icache.
 	 * spinlock to wait the interrupt handler to be finished
 	 */
-	spin_lock_irqsave(&isys->power_lock, flags);
 	ret = ipu6_fw_com_close(isys->fwctx);
 	fwctx = isys->fwctx;
 	isys->fwctx = NULL;
-	spin_unlock_irqrestore(&isys->power_lock, flags);
 	if (ret)
 		dev_err(dev, "Device close failure: %d\n", ret);
 
@@ -158,9 +156,7 @@ static int ipu6_fw_isys_close(struct ipu6_isys *isys)
 
 	if (ret) {
 		dev_err(dev, "Device release time out %d\n", ret);
-		spin_lock_irqsave(&isys->power_lock, flags);
 		isys->fwctx = fwctx;
-		spin_unlock_irqrestore(&isys->power_lock, flags);
 	}
 
 	return ret;
@@ -531,9 +527,6 @@ int ipu6_isys_isr_one(struct ipu6_bus_device *adev)
 	u32 index;
 	u64 ts;
 
-	if (!isys->fwctx)
-		return 1;
-
 	resp = ipu6_fw_isys_get_resp(isys);
 	if (!resp)
 		return 1;
@@ -699,12 +692,11 @@ irqreturn_t ipu6_isys_isr(struct ipu6_bus_device *adev)
 	void __iomem *base = isys->pdata->base;
 	u32 status_sw, status_csi;
 	u32 ctrl0_status, ctrl0_clear;
+	int pm_status;
 
-	spin_lock(&isys->power_lock);
-	if (!isys->power) {
-		spin_unlock(&isys->power_lock);
-		return IRQ_NONE;
-	}
+	pm_status = pm_runtime_get_if_active(&adev->auxdev.dev);
+	if (!pm_status)
+		return 0;
 
 	ctrl0_status = isys->pdata->ipdata->csi2.ctrl0_irq_status;
 	ctrl0_clear = isys->pdata->ipdata->csi2.ctrl0_irq_clear;
@@ -749,7 +741,8 @@ irqreturn_t ipu6_isys_isr(struct ipu6_bus_device *adev)
 
 	writel(ISYS_UNISPART_IRQS, base + IPU6_REG_ISYS_UNISPART_IRQ_MASK);
 
-	spin_unlock(&isys->power_lock);
+	if (pm_status > 0)
+		pm_runtime_put(&adev->auxdev.dev);
 
 	return IRQ_HANDLED;
 }
diff --git a/drivers/media/pci/intel/ipu6/ipu6-isys.c b/drivers/media/pci/intel/ipu6/ipu6-isys.c
index fc6e87e9764b..60f5f9ea2910 100644
--- a/drivers/media/pci/intel/ipu6/ipu6-isys.c
+++ b/drivers/media/pci/intel/ipu6/ipu6-isys.c
@@ -759,7 +759,6 @@ static int isys_runtime_pm_resume(struct device *dev)
 	struct ipu6_isys *isys = ipu6_bus_get_drvdata(adev);
 	const struct ipu6_isys_internal_pdata *ipdata = isys->pdata->ipdata;
 	struct ipu6_device *isp = adev->isp;
-	unsigned long flags;
 	int ret;
 
 	ret = ipu6_mmu_hw_init(adev->mmu);
@@ -772,10 +771,6 @@ static int isys_runtime_pm_resume(struct device *dev)
 	if (ret)
 		goto err_mmu_hw_cleanup;
 
-	spin_lock_irqsave(&isys->power_lock, flags);
-	isys->power = 1;
-	spin_unlock_irqrestore(&isys->power_lock, flags);
-
 	if (IS_IPU7(isp)) {
 		ipu7_isys_setup_hw(isys);
 	} else {
@@ -806,10 +801,6 @@ static int isys_runtime_pm_resume(struct device *dev)
 	if (!ret)
 		return 0;
 
-	spin_lock_irqsave(&isys->power_lock, flags);
-	isys->power = 0;
-	spin_unlock_irqrestore(&isys->power_lock, flags);
-
 	isys->phy_termcal_val = 0;
 	cpu_latency_qos_update_request(&isys->pm_qos, PM_QOS_DEFAULT_VALUE);
 
@@ -827,7 +818,6 @@ static int isys_runtime_pm_suspend(struct device *dev)
 	struct ipu6_bus_device *adev = to_ipu6_bus_device(dev);
 	struct ipu6_isys *isys = dev_get_drvdata(dev);
 	struct ipu6_device *isp = adev->isp;
-	unsigned long flags;
 	int ret = 0;
 
 	isys->adev->auxdrv_data->fw_ops->close(isys);
@@ -836,10 +826,6 @@ static int isys_runtime_pm_suspend(struct device *dev)
 		ret = -EIO;
 	}
 
-	spin_lock_irqsave(&isys->power_lock, flags);
-	isys->power = 0;
-	spin_unlock_irqrestore(&isys->power_lock, flags);
-
 	isys->phy_termcal_val = 0;
 	cpu_latency_qos_update_request(&isys->pm_qos, PM_QOS_DEFAULT_VALUE);
 
@@ -1016,8 +1002,6 @@ static int isys_probe(struct auxiliary_device *auxdev,
 	isys->sensor_type = isys->pdata->ipdata->sensor_type_start;
 
 	spin_lock_init(&isys->streams_lock);
-	spin_lock_init(&isys->power_lock);
-	isys->power = 0;
 	isys->phy_termcal_val = 0;
 
 	mutex_init(&isys->mutex);
diff --git a/drivers/media/pci/intel/ipu6/ipu6-isys.h b/drivers/media/pci/intel/ipu6/ipu6-isys.h
index 4991078c18b2..2af20f56a965 100644
--- a/drivers/media/pci/intel/ipu6/ipu6-isys.h
+++ b/drivers/media/pci/intel/ipu6/ipu6-isys.h
@@ -80,9 +80,6 @@ struct sensor_async_sd {
  * @media_dev: Media device
  * @v4l2_dev: V4L2 device
  * @adev: ISYS bus device
- * @power: Is ISYS powered on or not?
- * @isr_bits: Which bits does the ISR handle?
- * @power_lock: Serialise access to power (power state in general)
  * @csi2_rx_ctrl_cached: cached shared value between all CSI2 receivers
  * @streams_lock: serialise access to streams
  * @streams: streams per firmware stream ID
@@ -99,8 +96,6 @@ struct ipu6_isys {
 	struct v4l2_device v4l2_dev;
 	struct ipu6_bus_device *adev;
 
-	int power;
-	spinlock_t power_lock;
 	u32 isr_csi2_bits;
 	u32 csi2_rx_ctrl_cached;
 	spinlock_t streams_lock;
diff --git a/drivers/media/pci/intel/ipu6/ipu7-fw-isys.c b/drivers/media/pci/intel/ipu6/ipu7-fw-isys.c
index 50219b210ae9..0876cc54faa7 100644
--- a/drivers/media/pci/intel/ipu6/ipu7-fw-isys.c
+++ b/drivers/media/pci/intel/ipu6/ipu7-fw-isys.c
@@ -5,6 +5,7 @@
 
 #include <linux/cleanup.h>
 #include <linux/cacheflush.h>
+#include <linux/pm_runtime.h>
 
 #include "ipu6-bus.h"
 #include "ipu6-dma.h"
@@ -552,9 +553,6 @@ static int ipu7_isys_isr_one(struct ipu6_bus_device *adev)
 	unsigned long flags;
 	u64 ts;
 
-	if (!isys->fwctx)
-		return 1;
-
 	resp = ipu7_fw_isys_get_resp(isys);
 	if (!resp)
 		return 1;
@@ -752,11 +750,11 @@ irqreturn_t ipu7_isys_isr(struct ipu6_bus_device *adev)
 	void __iomem *base = isys->pdata->base;
 	u32 status_sw, status_csi;
 	u32 csi_offset, sw_offset;
+	int pm_status;
 
-	guard(spinlock)(&isys->power_lock);
-
-	if (!isys->power)
-		return IRQ_NONE;
+	pm_status = pm_runtime_get_if_active(&adev->auxdev.dev);
+	if (!pm_status)
+		return 0;
 
 	csi_offset = IPU7_IS_IO_CSI2_LEGACY_IRQ_CTRL_BASE;
 	sw_offset = IPU7_IS_UC_CTRL_BASE;
@@ -764,8 +762,11 @@ irqreturn_t ipu7_isys_isr(struct ipu6_bus_device *adev)
 	status_csi = readl(base + csi_offset + IPU7_IRQ_CTL_STATUS);
 	status_sw = readl(base + sw_offset + IPU7_TO_SW_IRQ_CNTL_STATUS);
 
-	if (!status_csi && !status_sw)
+	if (!status_csi && !status_sw) {
+		if (pm_status > 0)
+			pm_runtime_put(&adev->auxdev.dev);
 		return IRQ_NONE;
+	}
 
 	do {
 		writel(status_sw, base + sw_offset + IPU7_TO_SW_IRQ_CNTL_CLEAR);
@@ -788,5 +789,8 @@ irqreturn_t ipu7_isys_isr(struct ipu6_bus_device *adev)
 	writel(IPU7_IS_UC_TO_SW_IRQ_MASK,
 	       base + sw_offset + IPU7_TO_SW_IRQ_CNTL_MASK_N);
 
+	if (pm_status > 0)
+		pm_runtime_put(&adev->auxdev.dev);
+
 	return IRQ_HANDLED;
 }
-- 
2.47.3

Re: [PATCH v2 20/21] media: ipu6: Don't track power status, rely on runtime PM

[Manik Bajpai]

Hi Sakari,
          I am hacing issues in applying this patch. Please scroll down 
for seeing the issue

On Thu, 17 Sep 2026, Sakari Ailus wrote:

> Rely on runtime PM suspending and resuming the device and so remove
> tracking of power state. Also remove the power_lock spinlock that
> serialised firmware initialisation and cleanup with the ISR. This is no
> longer necessary as the ISR now runs when the device is in RPM_ACTIVE
> state.
> 
> Signed-off-by: Sakari Ailus <sakari.ailus@linux.intel.com>
> ---
>  drivers/media/pci/intel/ipu6/ipu6-fw-isys.c | 21 +++++++--------------
>  drivers/media/pci/intel/ipu6/ipu6-isys.c    | 16 ----------------
>  drivers/media/pci/intel/ipu6/ipu6-isys.h    |  5 -----
>  drivers/media/pci/intel/ipu6/ipu7-fw-isys.c | 20 ++++++++++++--------
>  4 files changed, 19 insertions(+), 43 deletions(-)
> 
> diff --git a/drivers/media/pci/intel/ipu6/ipu6-fw-isys.c b/drivers/media/pci/intel/ipu6/ipu6-fw-isys.c
> index 0214d7105fbe..f4f1cf7c86d2 100644
> --- a/drivers/media/pci/intel/ipu6/ipu6-fw-isys.c
> +++ b/drivers/media/pci/intel/ipu6/ipu6-fw-isys.c
> @@ -7,6 +7,7 @@
>  #include <linux/delay.h>
>  #include <linux/device.h>
>  #include <linux/io.h>
> +#include <linux/pm_runtime.h>
>  #include <linux/spinlock.h>
>  #include <linux/types.h>
>  
> @@ -131,7 +132,6 @@ static int ipu6_fw_isys_close(struct ipu6_isys *isys)
>  {
>  	struct device *dev = &isys->adev->auxdev.dev;
>  	int retry = IPU6_ISYS_CLOSE_RETRY;
> -	unsigned long flags;
>  	void *fwctx;
>  	int ret;
>  
> @@ -141,11 +141,9 @@ static int ipu6_fw_isys_close(struct ipu6_isys *isys)
>  	 * to SP icache.
>  	 * spinlock to wait the interrupt handler to be finished
>  	 */
> -	spin_lock_irqsave(&isys->power_lock, flags);
>  	ret = ipu6_fw_com_close(isys->fwctx);
>  	fwctx = isys->fwctx;
>  	isys->fwctx = NULL;
> -	spin_unlock_irqrestore(&isys->power_lock, flags);
>  	if (ret)
>  		dev_err(dev, "Device close failure: %d\n", ret);
>  
> @@ -158,9 +156,7 @@ static int ipu6_fw_isys_close(struct ipu6_isys *isys)
>  
>  	if (ret) {
>  		dev_err(dev, "Device release time out %d\n", ret);
> -		spin_lock_irqsave(&isys->power_lock, flags);
>  		isys->fwctx = fwctx;
> -		spin_unlock_irqrestore(&isys->power_lock, flags);
>  	}
>  
>  	return ret;
> @@ -531,9 +527,6 @@ int ipu6_isys_isr_one(struct ipu6_bus_device *adev)
>  	u32 index;
>  	u64 ts;
>  
> -	if (!isys->fwctx)
> -		return 1;
> -
>  	resp = ipu6_fw_isys_get_resp(isys);
>  	if (!resp)
>  		return 1;
> @@ -699,12 +692,11 @@ irqreturn_t ipu6_isys_isr(struct ipu6_bus_device *adev)
>  	void __iomem *base = isys->pdata->base;
>  	u32 status_sw, status_csi;
>  	u32 ctrl0_status, ctrl0_clear;
> +	int pm_status;
>  
> -	spin_lock(&isys->power_lock);
> -	if (!isys->power) {
> -		spin_unlock(&isys->power_lock);
> -		return IRQ_NONE;
> -	}
> +	pm_status = pm_runtime_get_if_active(&adev->auxdev.dev);
> +	if (!pm_status)
> +		return 0;
>  
>  	ctrl0_status = isys->pdata->ipdata->csi2.ctrl0_irq_status;
>  	ctrl0_clear = isys->pdata->ipdata->csi2.ctrl0_irq_clear;
> @@ -749,7 +741,8 @@ irqreturn_t ipu6_isys_isr(struct ipu6_bus_device *adev)
>  
>  	writel(ISYS_UNISPART_IRQS, base + IPU6_REG_ISYS_UNISPART_IRQ_MASK);
>  
> -	spin_unlock(&isys->power_lock);
> +	if (pm_status > 0)
> +		pm_runtime_put(&adev->auxdev.dev);
>  
>  	return IRQ_HANDLED;
>  }
> diff --git a/drivers/media/pci/intel/ipu6/ipu6-isys.c b/drivers/media/pci/intel/ipu6/ipu6-isys.c
> index fc6e87e9764b..60f5f9ea2910 100644
> --- a/drivers/media/pci/intel/ipu6/ipu6-isys.c
> +++ b/drivers/media/pci/intel/ipu6/ipu6-isys.c
> @@ -759,7 +759,6 @@ static int isys_runtime_pm_resume(struct device *dev)
>  	struct ipu6_isys *isys = ipu6_bus_get_drvdata(adev);
>  	const struct ipu6_isys_internal_pdata *ipdata = isys->pdata->ipdata;
>  	struct ipu6_device *isp = adev->isp;
> -	unsigned long flags;
>  	int ret;
>  
>  	ret = ipu6_mmu_hw_init(adev->mmu);
> @@ -772,10 +771,6 @@ static int isys_runtime_pm_resume(struct device *dev)
>  	if (ret)
>  		goto err_mmu_hw_cleanup;
>  
> -	spin_lock_irqsave(&isys->power_lock, flags);
> -	isys->power = 1;
> -	spin_unlock_irqrestore(&isys->power_lock, flags);
> -
>  	if (IS_IPU7(isp)) {
>  		ipu7_isys_setup_hw(isys);
>  	} else {
> @@ -806,10 +801,6 @@ static int isys_runtime_pm_resume(struct device *dev)
>  	if (!ret)
>  		return 0;
>  
> -	spin_lock_irqsave(&isys->power_lock, flags);
> -	isys->power = 0;
> -	spin_unlock_irqrestore(&isys->power_lock, flags);
> -
>  	isys->phy_termcal_val = 0;
>  	cpu_latency_qos_update_request(&isys->pm_qos, PM_QOS_DEFAULT_VALUE);
>  
> @@ -827,7 +818,6 @@ static int isys_runtime_pm_suspend(struct device *dev)
>  	struct ipu6_bus_device *adev = to_ipu6_bus_device(dev);
>  	struct ipu6_isys *isys = dev_get_drvdata(dev);
>  	struct ipu6_device *isp = adev->isp;
> -	unsigned long flags;
>  	int ret = 0;
>  
>  	isys->adev->auxdrv_data->fw_ops->close(isys);
> @@ -836,10 +826,6 @@ static int isys_runtime_pm_suspend(struct device *dev)
>  		ret = -EIO;
>  	}
>  
> -	spin_lock_irqsave(&isys->power_lock, flags);
> -	isys->power = 0;
> -	spin_unlock_irqrestore(&isys->power_lock, flags);
> -
>  	isys->phy_termcal_val = 0;
>  	cpu_latency_qos_update_request(&isys->pm_qos, PM_QOS_DEFAULT_VALUE);
>  
> @@ -1016,8 +1002,6 @@ static int isys_probe(struct auxiliary_device *auxdev,
>  	isys->sensor_type = isys->pdata->ipdata->sensor_type_start;
>  
>  	spin_lock_init(&isys->streams_lock);
> -	spin_lock_init(&isys->power_lock);
> -	isys->power = 0;
>  	isys->phy_termcal_val = 0;
>  
>  	mutex_init(&isys->mutex);
> diff --git a/drivers/media/pci/intel/ipu6/ipu6-isys.h b/drivers/media/pci/intel/ipu6/ipu6-isys.h
> index 4991078c18b2..2af20f56a965 100644
> --- a/drivers/media/pci/intel/ipu6/ipu6-isys.h
> +++ b/drivers/media/pci/intel/ipu6/ipu6-isys.h
> @@ -80,9 +80,6 @@ struct sensor_async_sd {
>   * @media_dev: Media device
>   * @v4l2_dev: V4L2 device
>   * @adev: ISYS bus device
> - * @power: Is ISYS powered on or not?
> - * @isr_bits: Which bits does the ISR handle?
> - * @power_lock: Serialise access to power (power state in general)
>   * @csi2_rx_ctrl_cached: cached shared value between all CSI2 receivers
>   * @streams_lock: serialise access to streams
>   * @streams: streams per firmware stream ID
> @@ -99,8 +96,6 @@ struct ipu6_isys {
>  	struct v4l2_device v4l2_dev;
>  	struct ipu6_bus_device *adev;
>  
> -	int power;
> -	spinlock_t power_lock;
>  	u32 isr_csi2_bits;
>  	u32 csi2_rx_ctrl_cached;
>  	spinlock_t streams_lock;
> diff --git a/drivers/media/pci/intel/ipu6/ipu7-fw-isys.c b/drivers/media/pci/intel/ipu6/ipu7-fw-isys.c
> index 50219b210ae9..0876cc54faa7 100644
> --- a/drivers/media/pci/intel/ipu6/ipu7-fw-isys.c
> +++ b/drivers/media/pci/intel/ipu6/ipu7-fw-isys.c
> @@ -5,6 +5,7 @@
>  
>  #include <linux/cleanup.h>
>  #include <linux/cacheflush.h>
> +#include <linux/pm_runtime.h>
>  
>  #include "ipu6-bus.h"
>  #include "ipu6-dma.h"
> @@ -552,9 +553,6 @@ static int ipu7_isys_isr_one(struct ipu6_bus_device *adev)
>  	unsigned long flags;
>  	u64 ts;
>  
> -	if (!isys->fwctx)
> -		return 1;
> -
>  	resp = ipu7_fw_isys_get_resp(isys);
>  	if (!resp)
>  		return 1;
> @@ -752,11 +750,11 @@ irqreturn_t ipu7_isys_isr(struct ipu6_bus_device *adev)
>  	void __iomem *base = isys->pdata->base;
>  	u32 status_sw, status_csi;
>  	u32 csi_offset, sw_offset;
> +	int pm_status;
>  
> -	guard(spinlock)(&isys->power_lock);
> -
> -	if (!isys->power)
> -		return IRQ_NONE;
> +	pm_status = pm_runtime_get_if_active(&adev->auxdev.dev);
> +	if (!pm_status)
> +		return 0;
>  
>  	csi_offset = IPU7_IS_IO_CSI2_LEGACY_IRQ_CTRL_BASE;
>  	sw_offset = IPU7_IS_UC_CTRL_BASE;
> @@ -764,8 +762,11 @@ irqreturn_t ipu7_isys_isr(struct ipu6_bus_device *adev)
>  	status_csi = readl(base + csi_offset + IPU7_IRQ_CTL_STATUS);
>  	status_sw = readl(base + sw_offset + IPU7_TO_SW_IRQ_CNTL_STATUS);
>  
> -	if (!status_csi && !status_sw)
> +	if (!status_csi && !status_sw) {
> +		if (pm_status > 0)
> +			pm_runtime_put(&adev->auxdev.dev);
>  		return IRQ_NONE;
> +	}
>  
>  	do {
>  		writel(status_sw, base + sw_offset + IPU7_TO_SW_IRQ_CNTL_CLEAR);
> @@ -788,5 +789,8 @@ irqreturn_t ipu7_isys_isr(struct ipu6_bus_device *adev)
>  	writel(IPU7_IS_UC_TO_SW_IRQ_MASK,
>  	       base + sw_offset + IPU7_TO_SW_IRQ_CNTL_MASK_N);
>  
> +	if (pm_status > 0)
> +		pm_runtime_put(&adev->auxdev.dev);
> +
>  	return IRQ_HANDLED;
>  }
> -- 
> 2.47.3
> 
> 
Result of "patch -p1 --dry-run" in pine

checking file drivers/media/pci/intel/ipu6/ipu6-fw-isys.c
checking file drivers/media/pci/intel/ipu6/ipu6-isys.c
checking file drivers/media/pci/intel/ipu6/ipu6-isys.h
checking file drivers/media/pci/intel/ipu6/ipu7-fw-isys.c
Hunk #1 FAILED at 5.
Hunk #2 succeeded at 551 (offset -1 lines).
Hunk #3 succeeded at 748 (offset -1 lines).
Hunk #4 succeeded at 760 (offset -1 lines).
Hunk #5 succeeded at 787 (offset -1 lines).
1 out of 5 hunks FAILED


My current tree looks like this:-

intel@pepper:~/s-ailus/linux-ipu6$ git log --pretty --oneline -25
a67d0bc689db (HEAD -> ipu6) media: ipu6: Move firmware init/cleanup to RPM callbacks
7a11adbbf11c media: ipu6: Serialise access to stream pointers by isys stream_lock
25b80e93ebca media: ipu6: Drop {get,put}_streams_opened()
ae57b074bbf9 media: ipu6: Bridge the gap between streams in V4L2 and IPU6 firmware
6640c40b9185 media: ipu6: Rework watermark setting
27cc9d1434f7 media: ipu6: Rework watermark calculation
bba5324133c7 media: ipu6: Avoid accessing av->streams before streaming
8fe5d941938e media: ipu6: Collect enabled stream IDs
f0144bcba9b1 media: ipu6: Remove nr_queues and nr_streaming fields in ipu6_isys_stream
4e23a8cadad1 media: ipu6: Add lockdep checks for CSI-2 streaming enable and disable
d5d977d60cae media: ipu6: Start streaming once all streams have started, stop when not
6987e506fe88 media: ipu6: Collect IPU streams into CSI-2 receiver sub-device context
ac8bcb8026c7 media: ipu6: Get watermark configuration directly from ipdata
4132a4c2835f media: ipu6: Clean up link frequency calculation
64c42c31525e media: ipu6: Always request a capture ack
3395c33b5160 media: ipu6: Rename misnamed out_free_watermark label in video init
6dbd34bdaf4e media: ipu6: Stream number on CSI-2 receiver source pads is always 0
5ec55ec5646e media: ipu6: Move streaming control to CSI-2 receiver driver
3e3070330170 media: ipu6: Fix releasing resources at failing streamon
8fa3385b6db1 (origin/ipu6) media: ipu6: Enable ipu7 and ipu7.5
09db11b21d85 media: ipu6: Add ipu7.5 mmu initialization data
553f8433c5d9 media: ipu6: Add ipu7.5 buttress support
8d8cd6c357e9 media: ipu6: Set model name for ipu7
0951f102d6c8 media: ipu6: Map ipu7 firmware
58c4286583cb media: ipu6: Move isys fw mapping to pci_probe

[PATCH v2 21/21] media: ipu6: Support upstream sub-devices without get_frame_desc()

[Sakari Ailus]

Provide compatibility support for upstream sub-device drivers that do not
implement get_frame_desc() pad op.

Signed-off-by: Sakari Ailus <sakari.ailus@linux.intel.com>
---
 drivers/media/pci/intel/ipu6/ipu6-isys-csi2.c | 30 ++++++++++++++++---
 1 file changed, 26 insertions(+), 4 deletions(-)

diff --git a/drivers/media/pci/intel/ipu6/ipu6-isys-csi2.c b/drivers/media/pci/intel/ipu6/ipu6-isys-csi2.c
index 780ac3c4d4e4..bf70482029dd 100644
--- a/drivers/media/pci/intel/ipu6/ipu6-isys-csi2.c
+++ b/drivers/media/pci/intel/ipu6/ipu6-isys-csi2.c
@@ -16,6 +16,7 @@
 #include <linux/string_choices.h>
 
 #include <media/media-entity.h>
+#include <media/mipi-csi2.h>
 #include <media/v4l2-ctrls.h>
 #include <media/v4l2-device.h>
 #include <media/v4l2-event.h>
@@ -589,6 +590,29 @@ static bool ipu6_isys_csi2_streaming_change(struct ipu6_isys_subdev *asd,
 	return false;
 }
 
+static int ipu6_isys_get_frame_desc(struct v4l2_subdev *remote_sd,
+				    unsigned int pad,
+				    struct v4l2_mbus_frame_desc *desc)
+{
+	struct v4l2_subdev_format fmt = { .which = V4L2_SUBDEV_FORMAT_ACTIVE };
+	int ret;
+
+	ret = v4l2_subdev_call(remote_sd, pad, get_frame_desc, pad, desc);
+	if (!ret || ret != -ENOIOCTLCMD)
+		return ret;
+
+	ret = v4l2_subdev_call_state_active(remote_sd, pad, get_fmt, &fmt);
+	if (ret)
+		return ret;
+
+	desc->type = V4L2_MBUS_FRAME_DESC_TYPE_CSI2;
+	desc->num_entries = 1;
+	desc->entry[0].pixelcode = fmt.format.code;
+	desc->entry[0].bus.csi2.dt = ipu6_isys_mbus_code_to_mipi(fmt.format.code);
+
+	return 0;
+}
+
 static int ipu6_isys_csi2_enable_streams(struct v4l2_subdev *sd,
 					 struct v4l2_subdev_state *state,
 					 u32 pad, u64 streams_mask)
@@ -612,8 +636,7 @@ static int ipu6_isys_csi2_enable_streams(struct v4l2_subdev *sd,
 
 	lockdep_assert_held(&csi2->isys->stream_mutex);
 
-	ret = v4l2_subdev_call(remote_sd, pad, get_frame_desc,
-			       remote_pad->index, &desc);
+	ret = ipu6_isys_get_frame_desc(remote_sd, remote_pad->index, &desc);
 	if (ret)
 		return ret;
 
@@ -715,8 +738,7 @@ static int ipu6_isys_csi2_disable_streams(struct v4l2_subdev *sd,
 
 	lockdep_assert_held(&csi2->isys->stream_mutex);
 
-	ret = v4l2_subdev_call(remote_sd, pad, get_frame_desc,
-			       remote_pad->index, &desc);
+	ret = ipu6_isys_get_frame_desc(remote_sd, remote_pad->index, &desc);
 	if (ret)
 		return ret;
 
-- 
2.47.3