[PATCH 16/19] coresight: trbe: Support trigger mode

[Leo Yan <leo.yan@arm.com>]

The buffer currently operates in fill mode, where tracing stops when it
reaches the end of the buffer and a maintenance interrupt is raised.
However, due to IRQ latency, trace data may be lost during the window in
which tracing is halted but the program continues to run.

To mitigate the issue, this commit enables the trigger count to support
buffer maintenance without disabling tracing.  This is fulfilled with
two modes:

1) Set a trigger count as a watermark and use fill mode to prevent the
   buffer from being overwritten.  Once the count is decremented to
   zero, an interrupt is raised for buffer maintenance, but the hardware
   continues collecting trace data until limit.

          head            watermark tail
     +----+---------------+---------+-------+
     |$$$$|               |         |$$$$$$$|
     +----+---------------+---------+-------+
     base `---- count ----'         limit   base + nr_pages

     $$$ : Filled trace data

2) Use wrap mode so that tracing continues when reach the top of the
   buffer.  The trigger count is configured as "Stop on trigger" to
   guard the trace data not to be overwritten.

              watermark   tail      head
     +--------+-----------+---------+-------+
     |        |           |$$$$$$$$$|       |
     +--------+-----------+---------+-------+
     base                                   base + nr_pages
                                            limit

                                    `------->
     >-- counter ---------'

     $$$ : Filled trace data

The modes are selected by comparing the limit with the trigger position.

An extra TRBE_FAULT_ACT_TRIG state is introduced for fault action, it is
used to distinguish the trigger event from the WRAP event.

Signed-off-by: Leo Yan <leo.yan@arm.com>
---
 drivers/hwtracing/coresight/coresight-trbe.c | 101 +++++++++++++++++++++------
 drivers/hwtracing/coresight/coresight-trbe.h |  14 ++++
 2 files changed, 94 insertions(+), 21 deletions(-)

diff --git a/drivers/hwtracing/coresight/coresight-trbe.c b/drivers/hwtracing/coresight/coresight-trbe.c
index 8390d0a8fe23d35945610df15f21751279ee37ee..0551ea9b4f8286c156e3c9c7ac94e2ecd3b9dc3f 100644
--- a/drivers/hwtracing/coresight/coresight-trbe.c
+++ b/drivers/hwtracing/coresight/coresight-trbe.c
@@ -48,6 +48,7 @@
 #define TRBE_TRACE_MIN_BUF_SIZE		64
 
 enum trbe_fault_action {
+	TRBE_FAULT_ACT_TRIG,
 	TRBE_FAULT_ACT_WRAP,
 	TRBE_FAULT_ACT_SPURIOUS,
 	TRBE_FAULT_ACT_FATAL,
@@ -67,6 +68,7 @@ struct trbe_buf {
 	unsigned long trbe_hw_base;
 	unsigned long trbe_limit;
 	unsigned long trbe_write;
+	unsigned long trbe_count;
 	int nr_pages;
 	void **pages;
 	bool snapshot;
@@ -478,6 +480,10 @@ static unsigned long __trbe_normal_offset(struct perf_output_handle *handle)
 	if (head < tail)
 		limit = round_down(tail, PAGE_SIZE);
 
+	/* If trigger mode is enabled, no need to use limit for watermark */
+	if (!static_branch_unlikely(&trbe_trigger_mode_bypass))
+		goto out;
+
 	/*
 	 * Wakeup may be arbitrarily far into the future. If it's not in the
 	 * current generation, either we'll wrap before hitting it, or it's
@@ -495,6 +501,7 @@ static unsigned long __trbe_normal_offset(struct perf_output_handle *handle)
 	if (handle->wakeup < (handle->head + handle->size) && head <= wakeup)
 		limit = min(limit, round_up(wakeup, PAGE_SIZE));
 
+out:
 	/*
 	 * There is a situation when this can happen i.e limit is before
 	 * the head and hence TRBE cannot be configured.
@@ -518,6 +525,39 @@ static unsigned long __trbe_normal_offset(struct perf_output_handle *handle)
 	return 0;
 }
 
+static u64 __trbe_normal_trigger_count(struct perf_output_handle *handle)
+{
+	struct trbe_buf *buf = etm_perf_sink_config(handle);
+	struct trbe_cpudata *cpudata = buf->cpudata;
+	u64 limit, head, wakeup;
+	u64 count = 0;
+
+	if (static_branch_unlikely(&trbe_trigger_mode_bypass))
+		return 0;
+
+	limit = buf->trbe_limit - buf->trbe_base;
+	head = PERF_IDX2OFF(handle->head, buf);
+	wakeup = PERF_IDX2OFF(handle->wakeup, buf);
+
+	/* Set the count to guard the end of free buffer after wrap around */
+	if (limit == buf->nr_pages * PAGE_SIZE && (head + handle->size) > limit)
+		count = handle->size;
+
+	/*
+	 * If the watermark is less than the limit, use the trigger count for
+	 * the watermark maintenance.
+	 */
+	if (handle->wakeup < (handle->head + handle->size) && head <= wakeup) {
+		u64 wakeup_count =
+			round_up(wakeup - head, cpudata->trbe_hw_align);
+
+		if (head + wakeup_count < limit)
+			count = wakeup_count;
+	}
+
+	return count;
+}
+
 static int trbe_normal_offset(struct perf_output_handle *handle)
 {
 	struct trbe_buf *buf = etm_perf_sink_config(handle);
@@ -542,6 +582,7 @@ static int trbe_normal_offset(struct perf_output_handle *handle)
 		return -ENOSPC;
 
 	buf->trbe_limit = buf->trbe_base + limit;
+	buf->trbe_count = __trbe_normal_trigger_count(handle);
 	return 0;
 }
 
@@ -594,24 +635,40 @@ static void set_trbe_limit_pointer_enabled(struct trbe_buf *buf)
 	trblimitr &= ~TRBLIMITR_EL1_TM_MASK;
 	trblimitr &= ~TRBLIMITR_EL1_LIMIT_MASK;
 
-	/*
-	 * Fill trace buffer mode is used here while configuring the
-	 * TRBE for trace capture. In this particular mode, the trace
-	 * collection is stopped and a maintenance interrupt is raised
-	 * when the current write pointer wraps. This pause in trace
-	 * collection gives the software an opportunity to capture the
-	 * trace data in the interrupt handler, before reconfiguring
-	 * the TRBE.
-	 */
-	trblimitr |= (TRBLIMITR_EL1_FM_FILL << TRBLIMITR_EL1_FM_SHIFT) &
-		     TRBLIMITR_EL1_FM_MASK;
+	if (!buf->trbe_count ||
+	    buf->trbe_write + buf->trbe_count == buf->trbe_limit) {
+		/*
+		 * Fill trace buffer mode is used here while configuring the
+		 * TRBE for trace capture. In this particular mode, the trace
+		 * collection is stopped and a maintenance interrupt is raised
+		 * when the current write pointer wraps. This pause in trace
+		 * collection gives the software an opportunity to capture the
+		 * trace data in the interrupt handler, before reconfiguring
+		 * the TRBE.
+		 */
+		trblimitr |= FIELD_PREP(TRBLIMITR_EL1_FM_MASK, TRBLIMITR_EL1_FM_FILL) |
+			     FIELD_PREP(TRBLIMITR_EL1_TM_MASK, TRBLIMITR_EL1_TM_IGNR);
+	} else if (buf->trbe_write + buf->trbe_count < buf->trbe_limit) {
+		/*
+		 * Fill mode is used here to stop trace collection and prevent
+		 * the buffer from being overwritten. Trigger mode continues
+		 * trace collection and raises a maintenance interrupt on a
+		 * trigger event, which acts as a watermark for notifying
+		 * userspace.
+		 */
+		trblimitr |= FIELD_PREP(TRBLIMITR_EL1_FM_MASK, TRBLIMITR_EL1_FM_FILL) |
+			     FIELD_PREP(TRBLIMITR_EL1_TM_MASK, TRBLIMITR_EL1_TM_IRQ);
+	} else if (buf->trbe_write + buf->trbe_count > buf->trbe_limit) {
+		/*
+		 * Wrap buffer mode continues trace collection and raises
+		 * maintenance interrupt on buffer wrap. Trigger mode stops
+		 * trace on trigger event to guard the buffer from being
+		 * overwritten.
+		 */
+		trblimitr |= FIELD_PREP(TRBLIMITR_EL1_FM_MASK, TRBLIMITR_EL1_FM_WRAP) |
+			     FIELD_PREP(TRBLIMITR_EL1_TM_MASK, TRBLIMITR_EL1_TM_STOP);
+	}
 
-	/*
-	 * Trigger mode is not used here while configuring the TRBE for
-	 * the trace capture. Hence just keep this in the ignore mode.
-	 */
-	trblimitr |= (TRBLIMITR_EL1_TM_IGNR << TRBLIMITR_EL1_TM_SHIFT) &
-		     TRBLIMITR_EL1_TM_MASK;
 	trblimitr |= (addr & PAGE_MASK);
 	set_trbe_enabled(buf->cpudata, trblimitr);
 }
@@ -623,6 +680,7 @@ static void trbe_enable_hw(struct trbe_buf *buf)
 	WARN_ON(buf->trbe_write >= buf->trbe_limit);
 	set_trbe_base_pointer(buf->trbe_hw_base);
 	set_trbe_write_pointer(buf->trbe_write);
+	set_trbe_trigger_count(buf->trbe_count);
 
 	/*
 	 * Synchronize all the register updates
@@ -639,8 +697,6 @@ static enum trbe_fault_action trbe_get_fault_act(struct perf_output_handle *hand
 	int ec = get_trbe_ec(trbsr);
 	int bsc = get_trbe_bsc(trbsr);
 
-	WARN_ON(is_trbe_running(trbsr));
-
 	if (is_trbe_abort(trbsr)) {
 		err_str = "External abort";
 		goto out_fatal;
@@ -672,8 +728,7 @@ static enum trbe_fault_action trbe_get_fault_act(struct perf_output_handle *hand
 	case TRBE_BSC_FILLED:
 		break;
 	case TRBE_BSC_TRIGGERED:
-		err_str = "Unexpected trigger status";
-		goto out_fatal;
+		break;
 	default:
 		err_str = "Unexpected buffer status code";
 		goto out_fatal;
@@ -692,6 +747,9 @@ static enum trbe_fault_action trbe_get_fault_act(struct perf_output_handle *hand
 	if (is_trbe_wrap(trbsr))
 		return TRBE_FAULT_ACT_WRAP;
 
+	if (is_trbe_trg(trbsr))
+		return TRBE_FAULT_ACT_TRIG;
+
 	return TRBE_FAULT_ACT_SPURIOUS;
 
 out_fatal:
@@ -1180,6 +1238,7 @@ static irqreturn_t arm_trbe_irq_handler(int irq, void *dev)
 		clr_trbe_status();
 
 	switch (act) {
+	case TRBE_FAULT_ACT_TRIG:
 	case TRBE_FAULT_ACT_WRAP:
 		truncated = !!trbe_handle_overflow(handle, act);
 		break;
diff --git a/drivers/hwtracing/coresight/coresight-trbe.h b/drivers/hwtracing/coresight/coresight-trbe.h
index d7f7cd763c0c7139cf322b7336ee563073e3bea0..4c65d164a946ec9860825e7564196745b60d730b 100644
--- a/drivers/hwtracing/coresight/coresight-trbe.h
+++ b/drivers/hwtracing/coresight/coresight-trbe.h
@@ -114,6 +114,20 @@ static inline void set_trbe_write_pointer(unsigned long addr)
 	write_sysreg_s(addr, SYS_TRBPTR_EL1);
 }
 
+static inline void set_trbe_trigger_count(unsigned long count)
+{
+	u64 trbsr;
+
+	write_sysreg_s(count, SYS_TRBTRG_EL1);
+
+	/* TRBSR_EL1.TRG has been cleared in clr_trbe_status() */
+	if (!count)
+		return;
+
+	trbsr = read_sysreg_s(SYS_TRBSR_EL1);
+	write_sysreg_s(trbsr | TRBSR_EL1_TRG, SYS_TRBSR_EL1);
+}
+
 static inline unsigned long get_trbe_limit_pointer(void)
 {
 	u64 trblimitr = read_sysreg_s(SYS_TRBLIMITR_EL1);

-- 
2.34.1