[PATCH 0/6] Preemption support for A6xx targets

[PATCH 0/6] Preemption support for A6xx targets

[Sharat Masetty]

This is a revised preemption support patchset follows Jordan's recent
"drm/msm: Add A6XX device support" patch series. Preemption allows the
GPU to switch to a higher priority ringbuffer when one is ready, thereby
improving user experience. A6xx hardware supports various preemption
levels each with different granularities and different switch-out-switch-in
times.

This series starts off by adding basic preemption support for A6xx targets,
leading up to the tip of the stack which enables L1 level preemption.
This is a more fine grained version and faster than the default level.

Sharat Masetty (6):
  drm/msm: Add submitqueue setup and close
  drm/msm: Add new PM4 type7 opcodes
  drm/msm/A6xx: Implement preemption for A6XX targets
  drm/msm/A6xx: Enable preemption for A6xx targets
  drm/msm/Adreno: Refactor some preemption code
  drm/msm/A6xx: Enable L1 preemption level

 drivers/gpu/drm/msm/Makefile                |   1 +
 drivers/gpu/drm/msm/adreno/a5xx_gpu.h       |  26 --
 drivers/gpu/drm/msm/adreno/a5xx_preempt.c   |  55 ++---
 drivers/gpu/drm/msm/adreno/a6xx_gmu.c       |  44 ++++
 drivers/gpu/drm/msm/adreno/a6xx_gpu.c       | 148 ++++++++++-
 drivers/gpu/drm/msm/adreno/a6xx_gpu.h       | 110 +++++++++
 drivers/gpu/drm/msm/adreno/a6xx_preempt.c   | 366 ++++++++++++++++++++++++++++
 drivers/gpu/drm/msm/adreno/adreno_gpu.h     |  54 ++++
 drivers/gpu/drm/msm/adreno/adreno_pm4.xml.h |   2 +
 drivers/gpu/drm/msm/msm_gpu.h               |   7 +
 drivers/gpu/drm/msm/msm_submitqueue.c       |  15 +-
 11 files changed, 757 insertions(+), 71 deletions(-)
 create mode 100644 drivers/gpu/drm/msm/adreno/a6xx_preempt.c

--
1.9.1

_______________________________________________
dri-devel mailing list
dri-devel@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/dri-devel

[PATCH 1/6] drm/msm: Add submitqueue setup and close

[Sharat Masetty]

This patch adds a bit of infrastructure to give the different Adreno
targets the flexibility to setup the submitqueues per their needs.

Signed-off-by: Sharat Masetty <smasetty@codeaurora.org>
---
 drivers/gpu/drm/msm/msm_gpu.h         |  7 +++++++
 drivers/gpu/drm/msm/msm_submitqueue.c | 15 +++++++++++++--
 2 files changed, 20 insertions(+), 2 deletions(-)

diff --git a/drivers/gpu/drm/msm/msm_gpu.h b/drivers/gpu/drm/msm/msm_gpu.h
index b824117..b9b86ef 100644
--- a/drivers/gpu/drm/msm/msm_gpu.h
+++ b/drivers/gpu/drm/msm/msm_gpu.h
@@ -69,6 +69,10 @@ struct msm_gpu_funcs {
 	int (*debugfs_init)(struct msm_gpu *gpu, struct drm_minor *minor);
 #endif
 	int (*gpu_busy)(struct msm_gpu *gpu, uint64_t *value);
+	int (*submitqueue_setup)(struct msm_gpu *gpu,
+			struct msm_gpu_submitqueue *queue);
+	void (*submitqueue_close)(struct msm_gpu *gpu,
+			struct msm_gpu_submitqueue *queue);
 };
 
 struct msm_gpu {
@@ -173,6 +177,9 @@ struct msm_gpu_submitqueue {
 	int faults;
 	struct list_head node;
 	struct kref ref;
+	struct msm_gpu *gpu;
+	struct drm_gem_object *bo;
+	uint64_t bo_iova;
 };
 
 static inline void gpu_write(struct msm_gpu *gpu, u32 reg, u32 data)
diff --git a/drivers/gpu/drm/msm/msm_submitqueue.c b/drivers/gpu/drm/msm/msm_submitqueue.c
index 5115f75..1e696da 100644
--- a/drivers/gpu/drm/msm/msm_submitqueue.c
+++ b/drivers/gpu/drm/msm/msm_submitqueue.c
@@ -18,6 +18,10 @@ void msm_submitqueue_destroy(struct kref *kref)
 {
 	struct msm_gpu_submitqueue *queue = container_of(kref,
 		struct msm_gpu_submitqueue, ref);
+	struct msm_gpu *gpu = queue->gpu;
+
+	if (gpu && gpu->funcs->submitqueue_close)
+		gpu->funcs->submitqueue_close(gpu, queue);
 
 	kfree(queue);
 }
@@ -65,6 +69,7 @@ int msm_submitqueue_create(struct drm_device *drm, struct msm_file_private *ctx,
 {
 	struct msm_drm_private *priv = drm->dev_private;
 	struct msm_gpu_submitqueue *queue;
+	struct msm_gpu *gpu = priv->gpu;
 
 	if (!ctx)
 		return -ENODEV;
@@ -77,11 +82,14 @@ int msm_submitqueue_create(struct drm_device *drm, struct msm_file_private *ctx,
 	kref_init(&queue->ref);
 	queue->flags = flags;
 
-	if (priv->gpu) {
-		if (prio >= priv->gpu->nr_rings)
+	if (gpu) {
+		if (prio >= gpu->nr_rings) {
+			kfree(queue);
 			return -EINVAL;
+		}
 
 		queue->prio = prio;
+		queue->gpu = gpu;
 	}
 
 	write_lock(&ctx->queuelock);
@@ -95,6 +103,9 @@ int msm_submitqueue_create(struct drm_device *drm, struct msm_file_private *ctx,
 
 	write_unlock(&ctx->queuelock);
 
+	if (gpu && gpu->funcs->submitqueue_setup)
+		gpu->funcs->submitqueue_setup(gpu, queue);
+
 	return 0;
 }
 
-- 
1.9.1

_______________________________________________
dri-devel mailing list
dri-devel@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/dri-devel

[PATCH 2/6] drm/msm: Add new PM4 type7 opcodes

[Sharat Masetty]

This patch adds the following two opcodes:

CP_SET_MARKER opcode is a way to tell CP the current mode of GPU
operation(useful if preemption is in use).

CP_SET_PSEUDO_REG opcode will instruct CP to set a bunch of internal
CP registers, again useful for the preemption save/restore sequence.

Signed-off-by: Sharat Masetty <smasetty@codeaurora.org>
---
 drivers/gpu/drm/msm/adreno/adreno_pm4.xml.h | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/drivers/gpu/drm/msm/adreno/adreno_pm4.xml.h b/drivers/gpu/drm/msm/adreno/adreno_pm4.xml.h
index fb605a3..f0fd80e 100644
--- a/drivers/gpu/drm/msm/adreno/adreno_pm4.xml.h
+++ b/drivers/gpu/drm/msm/adreno/adreno_pm4.xml.h
@@ -202,6 +202,8 @@ enum adreno_pm4_type3_packets {
 	CP_EXEC_CS = 51,
 	CP_PERFCOUNTER_ACTION = 80,
 	CP_SMMU_TABLE_UPDATE = 83,
+	CP_SET_MARKER = 101,
+	CP_SET_PSEUDO_REG = 86,
 	CP_CONTEXT_REG_BUNCH = 92,
 	CP_YIELD_ENABLE = 28,
 	CP_SKIP_IB2_ENABLE_GLOBAL = 29,
-- 
1.9.1

_______________________________________________
Freedreno mailing list
Freedreno@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/freedreno

[PATCH 3/6] drm/msm/A6xx: Implement preemption for A6XX targets

[Sharat Masetty]

This patch implements preemption feature for A6xx targets, this allows
the GPU to switch to a higher priority ringbuffer if one is ready. A6XX
hardware as such supports multiple levels of preemption granularities,
ranging from coarse grained(ringbuffer level) to a more fine grained
such as draw-call level or a bin boundary level preemption. This patch
enables the basic preemption level, with more fine grained preemption
support to follow.

Signed-off-by: Sharat Masetty <smasetty@codeaurora.org>
---
 drivers/gpu/drm/msm/Makefile              |   1 +
 drivers/gpu/drm/msm/adreno/a6xx_gmu.c     |  44 ++++
 drivers/gpu/drm/msm/adreno/a6xx_gpu.c     | 146 +++++++++++-
 drivers/gpu/drm/msm/adreno/a6xx_gpu.h     | 136 +++++++++++
 drivers/gpu/drm/msm/adreno/a6xx_preempt.c | 383 ++++++++++++++++++++++++++++++
 5 files changed, 708 insertions(+), 2 deletions(-)
 create mode 100644 drivers/gpu/drm/msm/adreno/a6xx_preempt.c

diff --git a/drivers/gpu/drm/msm/Makefile b/drivers/gpu/drm/msm/Makefile
index 0b6e150..1978312 100644
--- a/drivers/gpu/drm/msm/Makefile
+++ b/drivers/gpu/drm/msm/Makefile
@@ -13,6 +13,7 @@ msm-y := \
 	adreno/a6xx_gpu.o \
 	adreno/a6xx_gmu.o \
 	adreno/a6xx_hfi.o \
+	adreno/a6xx_preempt.o \
 	hdmi/hdmi.o \
 	hdmi/hdmi_audio.o \
 	hdmi/hdmi_bridge.o \
diff --git a/drivers/gpu/drm/msm/adreno/a6xx_gmu.c b/drivers/gpu/drm/msm/adreno/a6xx_gmu.c
index 8d732e0..5c2a68a 100644
--- a/drivers/gpu/drm/msm/adreno/a6xx_gmu.c
+++ b/drivers/gpu/drm/msm/adreno/a6xx_gmu.c
@@ -1145,6 +1145,50 @@ void a6xx_gmu_remove(struct a6xx_gpu *a6xx_gpu)
 	iommu_domain_free(gmu->domain);
 }
 
+#define A6XX_GMU_FENCED_WRITE_SLEEP_US 10 /* Sleep time between reads in us */
+#define A6XX_GMU_FENCED_WRITE_TIMEOUT  600 /* Timeout in us */
+int a6xx_gmu_fenced_write(struct a6xx_gpu *a6xx_gpu, unsigned int reg,
+		unsigned int value, unsigned int fence_mask)
+{
+	struct a6xx_gmu *gmu = &a6xx_gpu->gmu;
+	struct adreno_gpu *adreno_gpu = &a6xx_gpu->base;
+	struct msm_gpu *gpu = &adreno_gpu->base;
+	unsigned int status;
+	ktime_t timeout = ktime_add_us(ktime_get(),
+			A6XX_GMU_FENCED_WRITE_TIMEOUT);
+
+	/* Write to the GPU register */
+	gpu_write(gpu, reg, value);
+
+	might_sleep_if(A6XX_GMU_FENCED_WRITE_SLEEP_US);
+	for (;;) {
+		status = gmu_read(gmu, REG_A6XX_GMU_AHB_FENCE_STATUS);
+		/*
+		 * If no bits of the fence_mask are set in the status, then the
+		 * write was successful
+		 */
+		if (!(status & fence_mask))
+			return 0;
+
+		if (ktime_compare(ktime_get(), timeout) > 0) {
+			/* Timed out, but check one last time */
+			status = gmu_read(gmu, REG_A6XX_GMU_AHB_FENCE_STATUS);
+			if (!(status & fence_mask))
+				return 0;
+
+			break;
+		}
+
+		usleep_range((A6XX_GMU_FENCED_WRITE_SLEEP_US >> 2) + 1,
+				A6XX_GMU_FENCED_WRITE_SLEEP_US);
+
+		/* Try writing again */
+		gpu_write(gpu, reg, value);
+	}
+
+	return -ETIMEDOUT;
+}
+
 int a6xx_gmu_probe(struct a6xx_gpu *a6xx_gpu, struct device_node *node)
 {
 	struct a6xx_gmu *gmu = &a6xx_gpu->gmu;
diff --git a/drivers/gpu/drm/msm/adreno/a6xx_gpu.c b/drivers/gpu/drm/msm/adreno/a6xx_gpu.c
index c72434b..b1a80ec 100644
--- a/drivers/gpu/drm/msm/adreno/a6xx_gpu.c
+++ b/drivers/gpu/drm/msm/adreno/a6xx_gpu.c
@@ -151,6 +151,8 @@ bool a6xx_idle(struct msm_gpu *gpu, struct msm_ringbuffer *ring)
 
 static void a6xx_flush(struct msm_gpu *gpu, struct msm_ringbuffer *ring)
 {
+	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
+	struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
 	uint32_t wptr;
 	unsigned long flags;
 
@@ -167,16 +169,52 @@ static void a6xx_flush(struct msm_gpu *gpu, struct msm_ringbuffer *ring)
 	/* Make sure everything is posted before making a decision */
 	mb();
 
-	gpu_write(gpu, REG_A6XX_CP_RB_WPTR, wptr);
+	/* Update HW if this is the current ring and we are not in preempt*/
+	if (a6xx_gpu->cur_ring == ring && !a6xx_in_preempt(a6xx_gpu))
+		gpu_write(gpu, REG_A6XX_CP_RB_WPTR, wptr);
 }
 
 static void a6xx_submit(struct msm_gpu *gpu, struct msm_gem_submit *submit,
 	struct msm_file_private *ctx)
 {
+	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
+	struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
 	struct msm_drm_private *priv = gpu->dev->dev_private;
 	struct msm_ringbuffer *ring = submit->ring;
+	uint64_t scratch_dest = SCRATCH_USER_CTX_IOVA(ring->id, a6xx_gpu);
 	unsigned int i;
 
+	/*
+	 * If preemption is enabled, then set the pseudo register for the save
+	 * sequence
+	 */
+	if (gpu->nr_rings > 1) {
+		struct msm_gpu_submitqueue *queue = submit->queue;
+
+		OUT_PKT7(ring, CP_SET_PSEUDO_REG, 6);
+
+		/* privileged and non secure buffer save */
+		OUT_RING(ring, SAVE_REG_PRIV_NON_SECURE_SAVE_ADDR_ID);
+		OUT_RING(ring, lower_32_bits(a6xx_gpu->preempt_iova[ring->id]));
+		OUT_RING(ring, upper_32_bits(a6xx_gpu->preempt_iova[ring->id]));
+
+		/* user context buffer save */
+		OUT_RING(ring, SAVE_REG_NON_PRIV_SAVE_ADDR_ID);
+		OUT_RING(ring, lower_32_bits(queue->bo_iova));
+		OUT_RING(ring, upper_32_bits(queue->bo_iova));
+
+		/*
+		 * Ask CP to save the user context buffer's iova address to a
+		 * scratch memory region, this is needed if the CP preempts
+		 * this ring in between this submit's IB list.
+		 */
+		OUT_PKT7(ring, CP_MEM_WRITE, 4);
+		OUT_RING(ring, lower_32_bits(scratch_dest));
+		OUT_RING(ring, upper_32_bits(scratch_dest));
+		OUT_RING(ring, lower_32_bits(queue->bo_iova));
+		OUT_RING(ring, upper_32_bits(queue->bo_iova));
+	}
+
 	/* Invalidate CCU depth and color */
 	OUT_PKT7(ring, CP_EVENT_WRITE, 1);
 	OUT_RING(ring, PC_CCU_INVALIDATE_DEPTH);
@@ -184,6 +222,14 @@ static void a6xx_submit(struct msm_gpu *gpu, struct msm_gem_submit *submit,
 	OUT_PKT7(ring, CP_EVENT_WRITE, 1);
 	OUT_RING(ring, PC_CCU_INVALIDATE_COLOR);
 
+	if (gpu->nr_rings > 1) {
+		/* Indicate the beginning of IB list, this will implicitly turn
+		 * on preemption.
+		 */
+		OUT_PKT7(ring, CP_SET_MARKER, 1);
+		OUT_RING(ring, 0XD);
+	}
+
 	/* Submit the commands */
 	for (i = 0; i < submit->nr_cmds; i++) {
 		switch (submit->cmd[i].type) {
@@ -201,6 +247,14 @@ static void a6xx_submit(struct msm_gpu *gpu, struct msm_gem_submit *submit,
 		}
 	}
 
+	if (gpu->nr_rings > 1) {
+		/* Indicate the end of IB list, this will implicitly turn
+		 * off preemption.
+		 */
+		OUT_PKT7(ring, CP_SET_MARKER, 1);
+		OUT_RING(ring, 0XE);
+	}
+
 	/* Write the fence to the scratch register */
 	OUT_PKT4(ring, REG_A6XX_CP_SCRATCH_REG(2), 1);
 	OUT_RING(ring, submit->seqno);
@@ -215,7 +269,38 @@ static void a6xx_submit(struct msm_gpu *gpu, struct msm_gem_submit *submit,
 	OUT_RING(ring, upper_32_bits(rbmemptr(ring, fence)));
 	OUT_RING(ring, submit->seqno);
 
+	/* If preemption is enabled */
+	if (gpu->nr_rings > 1) {
+		/*
+		 * Reset the scratch region as we are done with the
+		 * IB list of this submission
+		 */
+		OUT_PKT7(ring, CP_MEM_WRITE, 4);
+		OUT_RING(ring, lower_32_bits(scratch_dest));
+		OUT_RING(ring, upper_32_bits(scratch_dest));
+		OUT_RING(ring, 0x00);
+		OUT_RING(ring, 0x00);
+
+		/* Yield the floor on command completion */
+		OUT_PKT7(ring, CP_CONTEXT_SWITCH_YIELD, 4);
+
+		/*
+		 * If dword[2:1] are non zero, they specify an address for
+		 * the CP to write the value of dword[3] to on preemption
+		 * complete. Write 0 to skip the write
+		 */
+		OUT_RING(ring, 0x00);
+		OUT_RING(ring, 0x00);
+		/* Data value - not used if the address above is 0 */
+		OUT_RING(ring, 0x01);
+		/* generate interrupt on preemption completion */
+		OUT_RING(ring, 0x00);
+	}
+
 	a6xx_flush(gpu, ring);
+
+	/* Check to see if we need to start preemption */
+	a6xx_preempt_trigger(gpu);
 }
 
 static const struct {
@@ -413,11 +498,54 @@ static int a6xx_ucode_init(struct msm_gpu *gpu)
 	return 0;
 }
 
+static int a6xx_preempt_start(struct msm_gpu *gpu)
+{
+	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
+	struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
+	struct msm_ringbuffer *ring = gpu->rb[0];
+
+	if (gpu->nr_rings <= 1)
+		return 0;
+
+	/* Turn CP protection off */
+	OUT_PKT7(ring, CP_SET_PROTECTED_MODE, 1);
+	OUT_RING(ring, 0);
+
+	OUT_PKT7(ring, CP_SET_PSEUDO_REG, 6);
+
+	/* Privileged and non-secure */
+	OUT_RING(ring, SAVE_REG_PRIV_NON_SECURE_SAVE_ADDR_ID);
+	OUT_RING(ring, lower_32_bits(a6xx_gpu->preempt_iova[ring->id]));
+	OUT_RING(ring, upper_32_bits(a6xx_gpu->preempt_iova[ring->id]));
+
+	/* Privileged and secure. We don't do secure yet, so reset to 0 */
+	OUT_RING(ring, SAVE_REG_PRIV_SECURE_SAVE_ADDR_ID);
+	OUT_RING(ring, 0x00);
+	OUT_RING(ring, 0x00);
+
+	/* Turn CP protection back on */
+	OUT_PKT7(ring, CP_SET_PROTECTED_MODE, 1);
+	OUT_RING(ring, 1);
+
+	/* Yield the floor on command completion */
+	OUT_PKT7(ring, CP_CONTEXT_SWITCH_YIELD, 4);
+	OUT_RING(ring, 0x00);
+	OUT_RING(ring, 0x00);
+	OUT_RING(ring, 0x00);
+	/* Generate interrupt on preemption completion */
+	OUT_RING(ring, 0x00);
+
+	gpu->funcs->flush(gpu, ring);
+
+	return a6xx_idle(gpu, ring) ? 0 : -EINVAL;
+}
+
 #define A6XX_INT_MASK (A6XX_RBBM_INT_0_MASK_CP_AHB_ERROR | \
 	  A6XX_RBBM_INT_0_MASK_RBBM_ATB_ASYNCFIFO_OVERFLOW | \
 	  A6XX_RBBM_INT_0_MASK_CP_HW_ERROR | \
 	  A6XX_RBBM_INT_0_MASK_CP_IB2 | \
 	  A6XX_RBBM_INT_0_MASK_CP_IB1 | \
+	  A6XX_RBBM_INT_0_MASK_CP_SW | \
 	  A6XX_RBBM_INT_0_MASK_CP_RB | \
 	  A6XX_RBBM_INT_0_MASK_CP_CACHE_FLUSH_TS | \
 	  A6XX_RBBM_INT_0_MASK_RBBM_ATB_BUS_OVERFLOW | \
@@ -557,6 +685,8 @@ static int a6xx_hw_init(struct msm_gpu *gpu)
 	if (ret)
 		goto out;
 
+	a6xx_preempt_hw_init(gpu);
+
 	ret = a6xx_ucode_init(gpu);
 	if (ret)
 		goto out;
@@ -586,6 +716,8 @@ static int a6xx_hw_init(struct msm_gpu *gpu)
 		gpu_write(gpu, REG_A6XX_RBBM_SECVID_TRUST_CNTL, 0x0);
 	}
 
+	/* Last step - yield the ringbuffer */
+	a6xx_preempt_start(gpu);
 out:
 	/*
 	 * Tell the GMU that we are done touching the GPU and it can start power
@@ -731,8 +863,13 @@ static irqreturn_t a6xx_irq(struct msm_gpu *gpu)
 	if (status & A6XX_RBBM_INT_0_MASK_UCHE_OOB_ACCESS)
 		dev_err_ratelimited(&gpu->pdev->dev, "UCHE | Out of bounds access\n");
 
-	if (status & A6XX_RBBM_INT_0_MASK_CP_CACHE_FLUSH_TS)
+	if (status & A6XX_RBBM_INT_0_MASK_CP_CACHE_FLUSH_TS) {
 		msm_gpu_retire(gpu);
+		a6xx_preempt_trigger(gpu);
+	}
+
+	if (status & A6XX_RBBM_INT_0_MASK_CP_SW)
+		a6xx_preempt_irq(gpu);
 
 	return IRQ_HANDLED;
 }
@@ -874,6 +1011,8 @@ static void a6xx_destroy(struct msm_gpu *gpu)
 		.active_ring = a6xx_active_ring,
 		.irq = a6xx_irq,
 		.destroy = a6xx_destroy,
+		.submitqueue_setup = a6xx_preempt_submitqueue_setup,
+		.submitqueue_close = a6xx_preempt_submitqueue_close,
 #ifdef CONFIG_DEBUG_FS
 		.show = a6xx_show,
 #endif
@@ -923,5 +1062,8 @@ struct msm_gpu *a6xx_gpu_init(struct drm_device *dev)
 		msm_mmu_set_fault_handler(gpu->aspace->mmu, gpu,
 				a6xx_fault_handler);
 
+	/* Set up the preemption specific bits and pieces for each ringbuffer */
+	a6xx_preempt_init(gpu);
+
 	return gpu;
 }
diff --git a/drivers/gpu/drm/msm/adreno/a6xx_gpu.h b/drivers/gpu/drm/msm/adreno/a6xx_gpu.h
index 06e93fd..aca1d7d 100644
--- a/drivers/gpu/drm/msm/adreno/a6xx_gpu.h
+++ b/drivers/gpu/drm/msm/adreno/a6xx_gpu.h
@@ -28,12 +28,126 @@ struct a6xx_gpu {
 	uint64_t sqe_iova;
 
 	struct msm_ringbuffer *cur_ring;
+	struct msm_ringbuffer *next_ring;
+
+	struct drm_gem_object *preempt_bo[MSM_GPU_MAX_RINGS];
+	struct a6xx_preempt_record *preempt[MSM_GPU_MAX_RINGS];
+	uint64_t preempt_iova[MSM_GPU_MAX_RINGS];
+
+	atomic_t preempt_state;
+	struct timer_list preempt_timer;
+
+	unsigned int preempt_level;
+	bool uses_gmem;
+	bool skip_save_restore;
+
+	struct drm_gem_object *scratch_bo;
+	void *scratch_ptr;
+	uint64_t scratch_iova;
 
 	struct a6xx_gmu gmu;
 };
 
 #define to_a6xx_gpu(x) container_of(x, struct a6xx_gpu, base)
 
+#define SCRATCH_USER_CTX_IOVA(ring_id, a6xx_gpu) \
+	(a6xx_gpu->scratch_iova + (ring_id * sizeof(uint64_t)))
+
+/*
+ * In order to do lockless preemption we use a simple state machine to progress
+ * through the process.
+ *
+ * PREEMPT_NONE - no preemption in progress.  Next state START.
+ * PREEMPT_START - The trigger is evaluating if preemption is possible. Next
+ * states: TRIGGERED, NONE
+ * PREEMPT_ABORT - An intermediate state before moving back to NONE. Next
+ * state: NONE.
+ * PREEMPT_TRIGGERED: A preemption has been executed on the hardware. Next
+ * states: FAULTED, PENDING
+ * PREEMPT_FAULTED: A preemption timed out (never completed). This will trigger
+ * recovery.  Next state: N/A
+ * PREEMPT_PENDING: Preemption complete interrupt fired - the callback is
+ * checking the success of the operation. Next state: FAULTED, NONE.
+ */
+
+enum a6xx_preempt_state {
+	PREEMPT_NONE = 0,
+	PREEMPT_START,
+	PREEMPT_ABORT,
+	PREEMPT_TRIGGERED,
+	PREEMPT_FAULTED,
+	PREEMPT_PENDING,
+};
+
+/*
+ * ID values used by SET_PSEUDO_REG PM4 command. These determine which of the
+ * various internal CP registers to write to. Used in the save/restore
+ * preemption sequence.
+ */
+enum a6xx_set_pseudo_register {
+	SAVE_REG_SMMU_INFO_ID = 0,
+	SAVE_REG_PRIV_NON_SECURE_SAVE_ADDR_ID = 1,
+	SAVE_REG_PRIV_SECURE_SAVE_ADDR_ID = 2,
+	SAVE_REG_NON_PRIV_SAVE_ADDR_ID = 3,
+	SAVE_REG_COUNTER_ID = 4,
+};
+
+/*
+ * struct a6xx_preempt_record is a shared buffer between the microcode and the
+ * CPU to store the state for preemption. The record itself is much larger
+ * (2112k) but most of that is used by the CP for storage.
+ *
+ * There is a preemption record assigned per ringbuffer. When the CPU triggers a
+ * preemption, it fills out the record with the useful information (wptr, ring
+ * base, etc) and the microcode uses that information to set up the CP following
+ * the preemption.  When a ring is switched out, the CP will save the ringbuffer
+ * state back to the record. In this way, once the records are properly set up
+ * the CPU can quickly switch back and forth between ringbuffers by only
+ * updating a few registers (often only the wptr).
+ *
+ * These are the CPU aware registers in the record:
+ * @magic: Must always be 0xAE399D6EUL
+ * @info: Type of the record - written 0 by the CPU, updated by the CP
+ * @errno: preemption error record
+ * @data: Data field in YIELD and SET_MARKER packets, Written and used by CP
+ * @cntl: Value of RB_CNTL written by CPU, save/restored by CP
+ * @rptr: Value of RB_RPTR written by CPU, save/restored by CP
+ * @wptr: Value of RB_WPTR written by CPU, save/restored by CP
+ * @_pad: Reserved/padding
+ * @rptr_addr: Value of RB_RPTR_ADDR_LO|HI written by CPU, save/restored by CP
+ * @rbase: Value of RB_BASE written by CPU, save/restored by CP
+ * @counter: GPU address of the storage area for the preemption counters
+ */
+struct a6xx_preempt_record {
+	u32 magic;
+	u32 info;
+	u32 errno;
+	u32 data;
+	u32 cntl;
+	u32 rptr;
+	u32 wptr;
+	u32 _pad;
+	u64 rptr_addr;
+	u64 rbase;
+	u64 counter;
+};
+
+#define A6XX_PREEMPT_RECORD_MAGIC 0xAE399D6EUL
+
+/*
+ * Even though the structure above is only a few bytes, we need a full 2112k to
+ * store the entire preemption record from the CP
+ */
+#define A6XX_PREEMPT_RECORD_SIZE (2112 * 1024)
+
+/*
+ * The preemption counter block is a storage area for the value of the
+ * preemption counters that are saved immediately before context switch. We
+ * append it on to the end of the allocation for the preemption record.
+ */
+#define A6XX_PREEMPT_COUNTER_SIZE (16 * 4)
+
+#define A6XX_PREEMPT_USER_RECORD_SIZE (192 * 1024)
 /*
  * Given a register and a count, return a value to program into
  * REG_CP_PROTECT_REG(n) - this will block both reads and writes for _len
@@ -66,4 +180,26 @@ struct a6xx_gpu {
 int a6xx_gmu_probe(struct a6xx_gpu *a6xx_gpu, struct device_node *node);
 void a6xx_gmu_remove(struct a6xx_gpu *a6xx_gpu);
 
+int a6xx_gmu_fenced_write(struct a6xx_gpu *a6xx_gpu, unsigned int reg,
+		unsigned int value, unsigned int fence_mask);
+
+void a6xx_preempt_init(struct msm_gpu *gpu);
+void a6xx_preempt_hw_init(struct msm_gpu *gpu);
+void a6xx_preempt_trigger(struct msm_gpu *gpu);
+void a6xx_preempt_irq(struct msm_gpu *gpu);
+void a6xx_preempt_fini(struct msm_gpu *gpu);
+int a6xx_preempt_submitqueue_setup(struct msm_gpu *gpu,
+		struct msm_gpu_submitqueue *queue);
+void a6xx_preempt_submitqueue_close(struct msm_gpu *gpu,
+		struct msm_gpu_submitqueue *queue);
+
+/* Return true if we are in a preempt state */
+static inline bool a6xx_in_preempt(struct a6xx_gpu *a6xx_gpu)
+{
+	int preempt_state = atomic_read(&a6xx_gpu->preempt_state);
+
+	return !(preempt_state == PREEMPT_NONE ||
+			preempt_state == PREEMPT_ABORT);
+}
+
 #endif /* __A6XX_GPU_H__ */
diff --git a/drivers/gpu/drm/msm/adreno/a6xx_preempt.c b/drivers/gpu/drm/msm/adreno/a6xx_preempt.c
new file mode 100644
index 0000000..60df6c5
--- /dev/null
+++ b/drivers/gpu/drm/msm/adreno/a6xx_preempt.c
@@ -0,0 +1,383 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/* Copyright (c) 2018, The Linux Foundation. All rights reserved. */
+
+#include "msm_gem.h"
+#include "a6xx_gpu.h"
+#include "a6xx_gmu.xml.h"
+
+/*
+ * Try to transition the preemption state from old to new. Return
+ * true on success or false if the original state wasn't 'old'
+ */
+static inline bool try_preempt_state(struct a6xx_gpu *a6xx_gpu,
+		enum a6xx_preempt_state old, enum a6xx_preempt_state new)
+{
+	enum a6xx_preempt_state cur = atomic_cmpxchg(&a6xx_gpu->preempt_state,
+		old, new);
+
+	return (cur == old);
+}
+
+/*
+ * Force the preemption state to the specified state.  This is used in cases
+ * where the current state is known and won't change
+ */
+static inline void set_preempt_state(struct a6xx_gpu *gpu,
+		enum a6xx_preempt_state new)
+{
+	/*
+	 * preempt_state may be read by other cores trying to trigger a
+	 * preemption or in the interrupt handler so barriers are needed
+	 * before...
+	 */
+	smp_mb__before_atomic();
+	atomic_set(&gpu->preempt_state, new);
+	/* ... and after*/
+	smp_mb__after_atomic();
+}
+
+/* Write the most recent wptr for the given ring into the hardware */
+static inline void update_wptr(struct msm_gpu *gpu, struct msm_ringbuffer *ring)
+{
+	unsigned long flags;
+	uint32_t wptr;
+
+	if (!ring)
+		return;
+
+	spin_lock_irqsave(&ring->lock, flags);
+	wptr = get_wptr(ring);
+	spin_unlock_irqrestore(&ring->lock, flags);
+
+	gpu_write(gpu, REG_A6XX_CP_RB_WPTR, wptr);
+}
+
+/* Return the highest priority ringbuffer with something in it */
+static struct msm_ringbuffer *get_next_ring(struct msm_gpu *gpu)
+{
+	unsigned long flags;
+	int i;
+
+	for (i = 0; i < gpu->nr_rings; i++) {
+		bool empty;
+		struct msm_ringbuffer *ring = gpu->rb[i];
+
+		spin_lock_irqsave(&ring->lock, flags);
+		empty = (get_wptr(ring) == ring->memptrs->rptr);
+		spin_unlock_irqrestore(&ring->lock, flags);
+
+		if (!empty)
+			return ring;
+	}
+
+	return NULL;
+}
+
+static void a6xx_preempt_timer(unsigned long data)
+{
+	struct a6xx_gpu *a6xx_gpu = (struct a6xx_gpu *) data;
+	struct msm_gpu *gpu = &a6xx_gpu->base.base;
+	struct drm_device *dev = gpu->dev;
+	struct msm_drm_private *priv = dev->dev_private;
+
+	if (!try_preempt_state(a6xx_gpu, PREEMPT_TRIGGERED, PREEMPT_FAULTED))
+		return;
+
+	dev_err(dev->dev, "%s: preemption timed out\n", gpu->name);
+	queue_work(priv->wq, &gpu->recover_work);
+}
+
+void a6xx_preempt_irq(struct msm_gpu *gpu)
+{
+	uint32_t status;
+	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
+	struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
+	struct drm_device *dev = gpu->dev;
+	struct msm_drm_private *priv = dev->dev_private;
+
+	if (!try_preempt_state(a6xx_gpu, PREEMPT_TRIGGERED, PREEMPT_PENDING))
+		return;
+
+	/* Delete the preemption watchdog timer */
+	del_timer(&a6xx_gpu->preempt_timer);
+
+	/*
+	 * The hardware should be setting the stop bit of CP_CONTEXT_SWITCH_CNTL
+	 * to zero before firing the interrupt, but there is a non zero chance
+	 * of a hardware condition or a software race that could set it again
+	 * before we have a chance to finish. If that happens, log and go for
+	 * recovery
+	 */
+	status = gpu_read(gpu, REG_A6XX_CP_CONTEXT_SWITCH_CNTL);
+	if (unlikely(status & 0x1)) {
+		set_preempt_state(a6xx_gpu, PREEMPT_FAULTED);
+		dev_err(dev->dev, "%s: Preemption failed to complete\n",
+			gpu->name);
+		queue_work(priv->wq, &gpu->recover_work);
+		return;
+	}
+
+	a6xx_gpu->cur_ring = a6xx_gpu->next_ring;
+	a6xx_gpu->next_ring = NULL;
+
+	update_wptr(gpu, a6xx_gpu->cur_ring);
+
+	set_preempt_state(a6xx_gpu, PREEMPT_NONE);
+}
+
+void a6xx_preempt_hw_init(struct msm_gpu *gpu)
+{
+	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
+	struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
+	int i;
+
+	/* No preemption if we only have one ring */
+	if (gpu->nr_rings == 1)
+		return;
+
+	for (i = 0; i < gpu->nr_rings; i++) {
+		a6xx_gpu->preempt[i]->wptr = 0;
+		a6xx_gpu->preempt[i]->rptr = 0;
+		a6xx_gpu->preempt[i]->info = 0;
+		a6xx_gpu->preempt[i]->data = 0;
+		a6xx_gpu->preempt[i]->rbase = gpu->rb[i]->iova;
+	}
+
+	/* Write a 0 to signal that we aren't switching pagetables */
+	gpu_write64(gpu, REG_A6XX_CP_CONTEXT_SWITCH_SMMU_INFO_LO,
+		REG_A6XX_CP_CONTEXT_SWITCH_SMMU_INFO_HI, 0);
+
+	/* Enable the GMEM save/restore feature for preemption */
+	gpu_write(gpu, REG_A6XX_RB_CONTEXT_SWITCH_GMEM_SAVE_RESTORE, 0x1);
+
+	/* Reset the preemption state */
+	set_preempt_state(a6xx_gpu, PREEMPT_NONE);
+
+	/* Always come up on rb 0 */
+	a6xx_gpu->cur_ring = gpu->rb[0];
+}
+
+#define FENCE_STATUS_WRITEDROPPED0_MASK 0x1
+#define FENCE_STATUS_WRITEDROPPED1_MASK 0x2
+void a6xx_preempt_trigger(struct msm_gpu *gpu)
+{
+	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
+	struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
+	unsigned long flags;
+	struct msm_ringbuffer *ring;
+	uint64_t user_ctx_iova;
+	unsigned int cntl;
+
+	if (gpu->nr_rings == 1)
+		return;
+
+	/*
+	 * Try to start preemption by moving from NONE to START. If
+	 * unsuccessful, a preemption is already in flight
+	 */
+	if (!try_preempt_state(a6xx_gpu, PREEMPT_NONE, PREEMPT_START))
+		return;
+
+	cntl = (((a6xx_gpu->preempt_level << 6) & 0xC0) |
+		((a6xx_gpu->skip_save_restore << 9) & 0x200) |
+		((a6xx_gpu->uses_gmem << 8) & 0x100) | 0x1);
+
+	/* Get the next ring to preempt to */
+	ring = get_next_ring(gpu);
+
+	/*
+	 * If no ring is populated or the highest priority ring is the current
+	 * one do nothing except to update the wptr to the latest and greatest
+	 */
+	if (!ring || (a6xx_gpu->cur_ring == ring)) {
+		set_preempt_state(a6xx_gpu, PREEMPT_ABORT);
+		update_wptr(gpu, a6xx_gpu->cur_ring);
+		set_preempt_state(a6xx_gpu, PREEMPT_NONE);
+		return;
+	}
+
+	spin_lock_irqsave(&ring->lock, flags);
+	a6xx_gpu->preempt[ring->id]->wptr = get_wptr(ring);
+	spin_unlock_irqrestore(&ring->lock, flags);
+
+	/*
+	 * The GPU power collapsing between the following preemption register
+	 * writes can lead to a prolonged preemption trigger sequence, so we
+	 * set a keepalive bit to make sure the GPU is not power collapsed by
+	 * the GMU during this time. The first fenced write will make sure to
+	 * wake up the GPU(if it was power collapsed) and from there on it is
+	 * not going to be power collapsed until we close the keepalive window
+	 * by resetting the keepalive bit.
+	 */
+	gmu_rmw(&a6xx_gpu->gmu, REG_A6XX_GMU_AO_SPARE_CNTL, 0x0, 0x2);
+
+	a6xx_gmu_fenced_write(a6xx_gpu,
+		REG_A6XX_CP_CONTEXT_SWITCH_PRIV_NON_SECURE_RESTORE_ADDR_LO,
+		lower_32_bits(a6xx_gpu->preempt_iova[ring->id]),
+		FENCE_STATUS_WRITEDROPPED1_MASK);
+
+	a6xx_gmu_fenced_write(a6xx_gpu,
+		REG_A6XX_CP_CONTEXT_SWITCH_PRIV_NON_SECURE_RESTORE_ADDR_HI,
+		upper_32_bits(a6xx_gpu->preempt_iova[ring->id]),
+		FENCE_STATUS_WRITEDROPPED1_MASK);
+
+	/*
+	 * Use the user context iova from the scratch memory that the CP may
+	 * have written as part of the ring switch out.
+	 */
+	user_ctx_iova = *((uint64_t *)a6xx_gpu->scratch_ptr + ring->id);
+
+	a6xx_gmu_fenced_write(a6xx_gpu,
+		REG_A6XX_CP_CONTEXT_SWITCH_NON_PRIV_RESTORE_ADDR_LO,
+		lower_32_bits(user_ctx_iova),
+		FENCE_STATUS_WRITEDROPPED1_MASK);
+
+	a6xx_gmu_fenced_write(a6xx_gpu,
+		REG_A6XX_CP_CONTEXT_SWITCH_NON_PRIV_RESTORE_ADDR_HI,
+		upper_32_bits(user_ctx_iova),
+		FENCE_STATUS_WRITEDROPPED1_MASK);
+
+	a6xx_gpu->next_ring = ring;
+
+	/* Start a timer to catch a stuck preemption */
+	mod_timer(&a6xx_gpu->preempt_timer, jiffies + msecs_to_jiffies(10000));
+
+	/* Set the preemption state to triggered */
+	set_preempt_state(a6xx_gpu, PREEMPT_TRIGGERED);
+
+	/* Make sure everything is written before hitting the button */
+	wmb();
+
+	/* Trigger the preemption */
+	a6xx_gmu_fenced_write(a6xx_gpu, REG_A6XX_CP_CONTEXT_SWITCH_CNTL, cntl,
+		FENCE_STATUS_WRITEDROPPED1_MASK);
+
+	/* Close the GPU keelaplive window */
+	gmu_rmw(&a6xx_gpu->gmu, REG_A6XX_GMU_AO_SPARE_CNTL, 0x2, 0x0);
+}
+
+static int preempt_init_ring(struct a6xx_gpu *a6xx_gpu,
+		struct msm_ringbuffer *ring)
+{
+	struct adreno_gpu *adreno_gpu = &a6xx_gpu->base;
+	struct msm_gpu *gpu = &adreno_gpu->base;
+	struct a6xx_preempt_record *ptr;
+	struct drm_gem_object *bo = NULL;
+	u64 iova = 0;
+
+	ptr = msm_gem_kernel_new(gpu->dev,
+		A6XX_PREEMPT_RECORD_SIZE + A6XX_PREEMPT_COUNTER_SIZE,
+		MSM_BO_UNCACHED, gpu->aspace, &bo, &iova);
+
+	if (IS_ERR(ptr))
+		return PTR_ERR(ptr);
+
+	a6xx_gpu->preempt_bo[ring->id] = bo;
+	a6xx_gpu->preempt_iova[ring->id] = iova;
+	a6xx_gpu->preempt[ring->id] = ptr;
+
+	/* Set up the defaults on the preemption record */
+	ptr->magic = A6XX_PREEMPT_RECORD_MAGIC;
+	ptr->info = 0;
+	ptr->data = 0;
+	ptr->rptr = 0;
+	ptr->wptr = 0;
+	ptr->cntl = MSM_GPU_RB_CNTL_DEFAULT;
+	ptr->rptr_addr = rbmemptr(ring, rptr);
+	ptr->rbase = ring->iova;
+	ptr->counter = iova + A6XX_PREEMPT_RECORD_SIZE;
+
+	return 0;
+}
+
+void a6xx_preempt_fini(struct msm_gpu *gpu)
+{
+	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
+	struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
+	int i;
+
+	for (i = 0; i < gpu->nr_rings; i++) {
+		if (!a6xx_gpu->preempt_bo[i])
+			continue;
+
+		msm_gem_put_vaddr(a6xx_gpu->preempt_bo[i]);
+
+		if (a6xx_gpu->preempt_iova[i])
+			msm_gem_put_iova(a6xx_gpu->preempt_bo[i], gpu->aspace);
+
+		drm_gem_object_unreference(a6xx_gpu->preempt_bo[i]);
+		a6xx_gpu->preempt_bo[i] = NULL;
+	}
+}
+
+void a6xx_preempt_init(struct msm_gpu *gpu)
+{
+	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
+	struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
+	int i;
+
+	/* No preemption if we only have one ring */
+	if (gpu->nr_rings <= 1)
+		return;
+
+	for (i = 0; i < gpu->nr_rings; i++) {
+		if (preempt_init_ring(a6xx_gpu, gpu->rb[i]))
+			goto fail;
+	}
+
+	/* TODO: make this configurable? */
+	a6xx_gpu->preempt_level = 0;
+	a6xx_gpu->uses_gmem = 1;
+	a6xx_gpu->skip_save_restore = 0;
+
+	a6xx_gpu->scratch_ptr  = msm_gem_kernel_new(gpu->dev,
+			gpu->nr_rings * sizeof(uint64_t), MSM_BO_WC,
+			gpu->aspace, &a6xx_gpu->scratch_bo,
+			&a6xx_gpu->scratch_iova);
+
+	if (IS_ERR(a6xx_gpu->scratch_ptr))
+		goto fail;
+
+	setup_timer(&a6xx_gpu->preempt_timer, a6xx_preempt_timer,
+			(unsigned long) a6xx_gpu);
+
+	return;
+fail:
+	/*
+	 * On any failure our adventure is over. Clean up and
+	 * set nr_rings to 1 to force preemption off
+	 */
+	a6xx_preempt_fini(gpu);
+	gpu->nr_rings = 1;
+
+	return;
+}
+
+void a6xx_preempt_submitqueue_close(struct msm_gpu *gpu,
+		struct msm_gpu_submitqueue *queue)
+{
+	if (!queue->bo)
+		return;
+
+	msm_gem_put_iova(queue->bo, gpu->aspace);
+	msm_gem_put_vaddr(queue->bo);
+	drm_gem_object_unreference_unlocked(queue->bo);
+}
+
+int a6xx_preempt_submitqueue_setup(struct msm_gpu *gpu,
+		struct msm_gpu_submitqueue *queue)
+{
+	void *ptr;
+
+	/*
+	 * Create a per submitqueue buffer for the CP to save and restore user
+	 * specific information such as the VPC streamout data.
+	 */
+	ptr = msm_gem_kernel_new(gpu->dev, A6XX_PREEMPT_USER_RECORD_SIZE,
+			MSM_BO_WC, gpu->aspace, &queue->bo, &queue->bo_iova);
+
+	if (IS_ERR(ptr))
+		return PTR_ERR(ptr);
+
+	return 0;
+}
-- 
1.9.1

_______________________________________________
dri-devel mailing list
dri-devel@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/dri-devel

[PATCH 4/6] drm/msm/A6xx: Enable preemption for A6xx targets

[Sharat Masetty]

This patch simply increases the number of available ringbuffers,
therefore enabling preemption.

Signed-off-by: Sharat Masetty <smasetty@codeaurora.org>
---
 drivers/gpu/drm/msm/adreno/a6xx_gpu.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/drivers/gpu/drm/msm/adreno/a6xx_gpu.c b/drivers/gpu/drm/msm/adreno/a6xx_gpu.c
index b1a80ec..e83b066 100644
--- a/drivers/gpu/drm/msm/adreno/a6xx_gpu.c
+++ b/drivers/gpu/drm/msm/adreno/a6xx_gpu.c
@@ -1040,7 +1040,7 @@ struct msm_gpu *a6xx_gpu_init(struct drm_device *dev)
 	adreno_gpu->registers = a6xx_registers;
 	adreno_gpu->reg_offsets = a6xx_register_offsets;
 
-	ret = adreno_gpu_init(dev, pdev, adreno_gpu, &funcs, 1);
+	ret = adreno_gpu_init(dev, pdev, adreno_gpu, &funcs, 4);
 	if (ret) {
 		a6xx_destroy(&(a6xx_gpu->base.base));
 		return ERR_PTR(ret);
-- 
1.9.1

_______________________________________________
dri-devel mailing list
dri-devel@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/dri-devel

[PATCH 5/6] drm/msm/Adreno: Refactor some preemption code

[Sharat Masetty]

The preemption state machine related code is same across Adreno targets,
so move the common code to a common header file to avoid code
duplication.

Signed-off-by: Sharat Masetty <smasetty@codeaurora.org>
---
 drivers/gpu/drm/msm/adreno/a5xx_gpu.h     | 26 ---------------
 drivers/gpu/drm/msm/adreno/a5xx_preempt.c | 55 +++++++++----------------------
 drivers/gpu/drm/msm/adreno/a6xx_gpu.h     | 26 ---------------
 drivers/gpu/drm/msm/adreno/a6xx_preempt.c | 55 +++++++++----------------------
 drivers/gpu/drm/msm/adreno/adreno_gpu.h   | 54 ++++++++++++++++++++++++++++++
 5 files changed, 84 insertions(+), 132 deletions(-)

diff --git a/drivers/gpu/drm/msm/adreno/a5xx_gpu.h b/drivers/gpu/drm/msm/adreno/a5xx_gpu.h
index 7d71860..45535f7 100644
--- a/drivers/gpu/drm/msm/adreno/a5xx_gpu.h
+++ b/drivers/gpu/drm/msm/adreno/a5xx_gpu.h
@@ -54,32 +54,6 @@ struct a5xx_gpu {
 #endif
 
 /*
- * In order to do lockless preemption we use a simple state machine to progress
- * through the process.
- *
- * PREEMPT_NONE - no preemption in progress.  Next state START.
- * PREEMPT_START - The trigger is evaulating if preemption is possible. Next
- * states: TRIGGERED, NONE
- * PREEMPT_ABORT - An intermediate state before moving back to NONE. Next
- * state: NONE.
- * PREEMPT_TRIGGERED: A preemption has been executed on the hardware. Next
- * states: FAULTED, PENDING
- * PREEMPT_FAULTED: A preemption timed out (never completed). This will trigger
- * recovery.  Next state: N/A
- * PREEMPT_PENDING: Preemption complete interrupt fired - the callback is
- * checking the success of the operation. Next state: FAULTED, NONE.
- */
-
-enum preempt_state {
-	PREEMPT_NONE = 0,
-	PREEMPT_START,
-	PREEMPT_ABORT,
-	PREEMPT_TRIGGERED,
-	PREEMPT_FAULTED,
-	PREEMPT_PENDING,
-};
-
-/*
  * struct a5xx_preempt_record is a shared buffer between the microcode and the
  * CPU to store the state for preemption. The record itself is much larger
  * (64k) but most of that is used by the CP for storage.
diff --git a/drivers/gpu/drm/msm/adreno/a5xx_preempt.c b/drivers/gpu/drm/msm/adreno/a5xx_preempt.c
index 40f4840..faf844b 100644
--- a/drivers/gpu/drm/msm/adreno/a5xx_preempt.c
+++ b/drivers/gpu/drm/msm/adreno/a5xx_preempt.c
@@ -14,37 +14,6 @@
 #include "msm_gem.h"
 #include "a5xx_gpu.h"
 
-/*
- * Try to transition the preemption state from old to new. Return
- * true on success or false if the original state wasn't 'old'
- */
-static inline bool try_preempt_state(struct a5xx_gpu *a5xx_gpu,
-		enum preempt_state old, enum preempt_state new)
-{
-	enum preempt_state cur = atomic_cmpxchg(&a5xx_gpu->preempt_state,
-		old, new);
-
-	return (cur == old);
-}
-
-/*
- * Force the preemption state to the specified state.  This is used in cases
- * where the current state is known and won't change
- */
-static inline void set_preempt_state(struct a5xx_gpu *gpu,
-		enum preempt_state new)
-{
-	/*
-	 * preempt_state may be read by other cores trying to trigger a
-	 * preemption or in the interrupt handler so barriers are needed
-	 * before...
-	 */
-	smp_mb__before_atomic();
-	atomic_set(&gpu->preempt_state, new);
-	/* ... and after*/
-	smp_mb__after_atomic();
-}
-
 /* Write the most recent wptr for the given ring into the hardware */
 static inline void update_wptr(struct msm_gpu *gpu, struct msm_ringbuffer *ring)
 {
@@ -89,7 +58,8 @@ static void a5xx_preempt_timer(unsigned long data)
 	struct drm_device *dev = gpu->dev;
 	struct msm_drm_private *priv = dev->dev_private;
 
-	if (!try_preempt_state(a5xx_gpu, PREEMPT_TRIGGERED, PREEMPT_FAULTED))
+	if (!adreno_try_preempt_state(&a5xx_gpu->preempt_state,
+				PREEMPT_TRIGGERED, PREEMPT_FAULTED))
 		return;
 
 	dev_err(dev->dev, "%s: preemption timed out\n", gpu->name);
@@ -111,7 +81,8 @@ void a5xx_preempt_trigger(struct msm_gpu *gpu)
 	 * Try to start preemption by moving from NONE to START. If
 	 * unsuccessful, a preemption is already in flight
 	 */
-	if (!try_preempt_state(a5xx_gpu, PREEMPT_NONE, PREEMPT_START))
+	if (!adreno_try_preempt_state(&a5xx_gpu->preempt_state,
+				PREEMPT_NONE, PREEMPT_START))
 		return;
 
 	/* Get the next ring to preempt to */
@@ -134,9 +105,11 @@ void a5xx_preempt_trigger(struct msm_gpu *gpu)
 		 * and can safely update the write pointer.
 		 */
 
-		set_preempt_state(a5xx_gpu, PREEMPT_ABORT);
+		adreno_set_preempt_state(&a5xx_gpu->preempt_state,
+				PREEMPT_ABORT);
 		update_wptr(gpu, a5xx_gpu->cur_ring);
-		set_preempt_state(a5xx_gpu, PREEMPT_NONE);
+		adreno_set_preempt_state(&a5xx_gpu->preempt_state,
+				PREEMPT_NONE);
 		return;
 	}
 
@@ -156,7 +129,7 @@ void a5xx_preempt_trigger(struct msm_gpu *gpu)
 	mod_timer(&a5xx_gpu->preempt_timer, jiffies + msecs_to_jiffies(10000));
 
 	/* Set the preemption state to triggered */
-	set_preempt_state(a5xx_gpu, PREEMPT_TRIGGERED);
+	adreno_set_preempt_state(&a5xx_gpu->preempt_state, PREEMPT_TRIGGERED);
 
 	/* Make sure everything is written before hitting the button */
 	wmb();
@@ -173,7 +146,8 @@ void a5xx_preempt_irq(struct msm_gpu *gpu)
 	struct drm_device *dev = gpu->dev;
 	struct msm_drm_private *priv = dev->dev_private;
 
-	if (!try_preempt_state(a5xx_gpu, PREEMPT_TRIGGERED, PREEMPT_PENDING))
+	if (!adreno_try_preempt_state(&a5xx_gpu->preempt_state,
+				PREEMPT_TRIGGERED, PREEMPT_PENDING))
 		return;
 
 	/* Delete the preemption watchdog timer */
@@ -187,7 +161,8 @@ void a5xx_preempt_irq(struct msm_gpu *gpu)
 	 */
 	status = gpu_read(gpu, REG_A5XX_CP_CONTEXT_SWITCH_CNTL);
 	if (unlikely(status)) {
-		set_preempt_state(a5xx_gpu, PREEMPT_FAULTED);
+		adreno_set_preempt_state(&a5xx_gpu->preempt_state,
+				PREEMPT_FAULTED);
 		dev_err(dev->dev, "%s: Preemption failed to complete\n",
 			gpu->name);
 		queue_work(priv->wq, &gpu->recover_work);
@@ -199,7 +174,7 @@ void a5xx_preempt_irq(struct msm_gpu *gpu)
 
 	update_wptr(gpu, a5xx_gpu->cur_ring);
 
-	set_preempt_state(a5xx_gpu, PREEMPT_NONE);
+	adreno_set_preempt_state(&a5xx_gpu->preempt_state, PREEMPT_NONE);
 }
 
 void a5xx_preempt_hw_init(struct msm_gpu *gpu)
@@ -219,7 +194,7 @@ void a5xx_preempt_hw_init(struct msm_gpu *gpu)
 		REG_A5XX_CP_CONTEXT_SWITCH_SMMU_INFO_HI, 0);
 
 	/* Reset the preemption state */
-	set_preempt_state(a5xx_gpu, PREEMPT_NONE);
+	adreno_set_preempt_state(&a5xx_gpu->preempt_state, PREEMPT_NONE);
 
 	/* Always come up on rb 0 */
 	a5xx_gpu->cur_ring = gpu->rb[0];
diff --git a/drivers/gpu/drm/msm/adreno/a6xx_gpu.h b/drivers/gpu/drm/msm/adreno/a6xx_gpu.h
index aca1d7d..21ab701 100644
--- a/drivers/gpu/drm/msm/adreno/a6xx_gpu.h
+++ b/drivers/gpu/drm/msm/adreno/a6xx_gpu.h
@@ -54,32 +54,6 @@ struct a6xx_gpu {
 	(a6xx_gpu->scratch_iova + (ring_id * sizeof(uint64_t)))
 
 /*
- * In order to do lockless preemption we use a simple state machine to progress
- * through the process.
- *
- * PREEMPT_NONE - no preemption in progress.  Next state START.
- * PREEMPT_START - The trigger is evaluating if preemption is possible. Next
- * states: TRIGGERED, NONE
- * PREEMPT_ABORT - An intermediate state before moving back to NONE. Next
- * state: NONE.
- * PREEMPT_TRIGGERED: A preemption has been executed on the hardware. Next
- * states: FAULTED, PENDING
- * PREEMPT_FAULTED: A preemption timed out (never completed). This will trigger
- * recovery.  Next state: N/A
- * PREEMPT_PENDING: Preemption complete interrupt fired - the callback is
- * checking the success of the operation. Next state: FAULTED, NONE.
- */
-
-enum a6xx_preempt_state {
-	PREEMPT_NONE = 0,
-	PREEMPT_START,
-	PREEMPT_ABORT,
-	PREEMPT_TRIGGERED,
-	PREEMPT_FAULTED,
-	PREEMPT_PENDING,
-};
-
-/*
  * ID values used by SET_PSEUDO_REG PM4 command. These determine which of the
  * various internal CP registers to write to. Used in the save/restore
  * preemption sequence.
diff --git a/drivers/gpu/drm/msm/adreno/a6xx_preempt.c b/drivers/gpu/drm/msm/adreno/a6xx_preempt.c
index 60df6c5..0d2b612 100644
--- a/drivers/gpu/drm/msm/adreno/a6xx_preempt.c
+++ b/drivers/gpu/drm/msm/adreno/a6xx_preempt.c
@@ -5,37 +5,6 @@
 #include "a6xx_gpu.h"
 #include "a6xx_gmu.xml.h"
 
-/*
- * Try to transition the preemption state from old to new. Return
- * true on success or false if the original state wasn't 'old'
- */
-static inline bool try_preempt_state(struct a6xx_gpu *a6xx_gpu,
-		enum a6xx_preempt_state old, enum a6xx_preempt_state new)
-{
-	enum a6xx_preempt_state cur = atomic_cmpxchg(&a6xx_gpu->preempt_state,
-		old, new);
-
-	return (cur == old);
-}
-
-/*
- * Force the preemption state to the specified state.  This is used in cases
- * where the current state is known and won't change
- */
-static inline void set_preempt_state(struct a6xx_gpu *gpu,
-		enum a6xx_preempt_state new)
-{
-	/*
-	 * preempt_state may be read by other cores trying to trigger a
-	 * preemption or in the interrupt handler so barriers are needed
-	 * before...
-	 */
-	smp_mb__before_atomic();
-	atomic_set(&gpu->preempt_state, new);
-	/* ... and after*/
-	smp_mb__after_atomic();
-}
-
 /* Write the most recent wptr for the given ring into the hardware */
 static inline void update_wptr(struct msm_gpu *gpu, struct msm_ringbuffer *ring)
 {
@@ -80,7 +49,8 @@ static void a6xx_preempt_timer(unsigned long data)
 	struct drm_device *dev = gpu->dev;
 	struct msm_drm_private *priv = dev->dev_private;
 
-	if (!try_preempt_state(a6xx_gpu, PREEMPT_TRIGGERED, PREEMPT_FAULTED))
+	if (!adreno_try_preempt_state(&a6xx_gpu->preempt_state,
+				PREEMPT_TRIGGERED, PREEMPT_FAULTED))
 		return;
 
 	dev_err(dev->dev, "%s: preemption timed out\n", gpu->name);
@@ -95,7 +65,8 @@ void a6xx_preempt_irq(struct msm_gpu *gpu)
 	struct drm_device *dev = gpu->dev;
 	struct msm_drm_private *priv = dev->dev_private;
 
-	if (!try_preempt_state(a6xx_gpu, PREEMPT_TRIGGERED, PREEMPT_PENDING))
+	if (!adreno_try_preempt_state(&a6xx_gpu->preempt_state,
+				PREEMPT_TRIGGERED, PREEMPT_PENDING))
 		return;
 
 	/* Delete the preemption watchdog timer */
@@ -110,7 +81,8 @@ void a6xx_preempt_irq(struct msm_gpu *gpu)
 	 */
 	status = gpu_read(gpu, REG_A6XX_CP_CONTEXT_SWITCH_CNTL);
 	if (unlikely(status & 0x1)) {
-		set_preempt_state(a6xx_gpu, PREEMPT_FAULTED);
+		adreno_set_preempt_state(&a6xx_gpu->preempt_state,
+				PREEMPT_FAULTED);
 		dev_err(dev->dev, "%s: Preemption failed to complete\n",
 			gpu->name);
 		queue_work(priv->wq, &gpu->recover_work);
@@ -122,7 +94,7 @@ void a6xx_preempt_irq(struct msm_gpu *gpu)
 
 	update_wptr(gpu, a6xx_gpu->cur_ring);
 
-	set_preempt_state(a6xx_gpu, PREEMPT_NONE);
+	adreno_set_preempt_state(&a6xx_gpu->preempt_state, PREEMPT_NONE);
 }
 
 void a6xx_preempt_hw_init(struct msm_gpu *gpu)
@@ -151,7 +123,7 @@ void a6xx_preempt_hw_init(struct msm_gpu *gpu)
 	gpu_write(gpu, REG_A6XX_RB_CONTEXT_SWITCH_GMEM_SAVE_RESTORE, 0x1);
 
 	/* Reset the preemption state */
-	set_preempt_state(a6xx_gpu, PREEMPT_NONE);
+	adreno_set_preempt_state(&a6xx_gpu->preempt_state, PREEMPT_NONE);
 
 	/* Always come up on rb 0 */
 	a6xx_gpu->cur_ring = gpu->rb[0];
@@ -175,7 +147,8 @@ void a6xx_preempt_trigger(struct msm_gpu *gpu)
 	 * Try to start preemption by moving from NONE to START. If
 	 * unsuccessful, a preemption is already in flight
 	 */
-	if (!try_preempt_state(a6xx_gpu, PREEMPT_NONE, PREEMPT_START))
+	if (!adreno_try_preempt_state(&a6xx_gpu->preempt_state,
+				PREEMPT_NONE, PREEMPT_START))
 		return;
 
 	cntl = (((a6xx_gpu->preempt_level << 6) & 0xC0) |
@@ -190,9 +163,11 @@ void a6xx_preempt_trigger(struct msm_gpu *gpu)
 	 * one do nothing except to update the wptr to the latest and greatest
 	 */
 	if (!ring || (a6xx_gpu->cur_ring == ring)) {
-		set_preempt_state(a6xx_gpu, PREEMPT_ABORT);
+		adreno_set_preempt_state(&a6xx_gpu->preempt_state,
+				PREEMPT_ABORT);
 		update_wptr(gpu, a6xx_gpu->cur_ring);
-		set_preempt_state(a6xx_gpu, PREEMPT_NONE);
+		adreno_set_preempt_state(&a6xx_gpu->preempt_state,
+				PREEMPT_NONE);
 		return;
 	}
 
@@ -243,7 +218,7 @@ void a6xx_preempt_trigger(struct msm_gpu *gpu)
 	mod_timer(&a6xx_gpu->preempt_timer, jiffies + msecs_to_jiffies(10000));
 
 	/* Set the preemption state to triggered */
-	set_preempt_state(a6xx_gpu, PREEMPT_TRIGGERED);
+	adreno_set_preempt_state(&a6xx_gpu->preempt_state, PREEMPT_TRIGGERED);
 
 	/* Make sure everything is written before hitting the button */
 	wmb();
diff --git a/drivers/gpu/drm/msm/adreno/adreno_gpu.h b/drivers/gpu/drm/msm/adreno/adreno_gpu.h
index 94764d0..bb9affd 100644
--- a/drivers/gpu/drm/msm/adreno/adreno_gpu.h
+++ b/drivers/gpu/drm/msm/adreno/adreno_gpu.h
@@ -368,4 +368,58 @@ static inline uint32_t get_wptr(struct msm_ringbuffer *ring)
 	((1 << 29) \
 	((ilog2((_len)) & 0x1F) << 24) | (((_reg) << 2) & 0xFFFFF))
 
+/*
+ * In order to do lockless preemption we use a simple state machine to progress
+ * through the process.
+ *
+ * PREEMPT_NONE - no preemption in progress.  Next state START.
+ * PREEMPT_START - The trigger is evaluating if preemption is possible. Next
+ * states: TRIGGERED, NONE
+ * PREEMPT_ABORT - An intermediate state before moving back to NONE. Next
+ * state: NONE.
+ * PREEMPT_TRIGGERED: A preemption has been executed on the hardware. Next
+ * states: FAULTED, PENDING
+ * PREEMPT_FAULTED: A preemption timed out (never completed). This will trigger
+ * recovery.  Next state: N/A
+ * PREEMPT_PENDING: Preemption complete interrupt fired - the callback is
+ * checking the success of the operation. Next state: FAULTED, NONE.
+ */
+enum adreno_preempt_state {
+	PREEMPT_NONE = 0,
+	PREEMPT_START,
+	PREEMPT_ABORT,
+	PREEMPT_TRIGGERED,
+	PREEMPT_FAULTED,
+	PREEMPT_PENDING,
+};
+
+/*
+ * Try to transition the preemption state from old to new. Return
+ * true on success or false if the original state wasn't 'old'
+ */
+static inline bool adreno_try_preempt_state(atomic_t *preempt_state,
+		enum adreno_preempt_state old, enum adreno_preempt_state new)
+{
+	enum adreno_preempt_state cur = atomic_cmpxchg(preempt_state, old, new);
+
+	return (cur == old);
+}
+
+/*
+ * Force the preemption state to the specified state.  This is used in cases
+ * where the current state is known and won't change
+ */
+static inline void adreno_set_preempt_state(atomic_t *preempt_state,
+		enum adreno_preempt_state new)
+{
+	/*
+	 * adreno_preempt_state may be read by other cores trying to trigger a
+	 * preemption or in the interrupt handler so barriers are needed
+	 * before...
+	 */
+	smp_mb__before_atomic();
+	atomic_set(preempt_state, new);
+	/* ... and after*/
+	smp_mb__after_atomic();
+}
 #endif /* __ADRENO_GPU_H__ */
-- 
1.9.1

_______________________________________________
Freedreno mailing list
Freedreno@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/freedreno

[PATCH 6/6] drm/msm/A6xx: Enable L1 preemption level

[Sharat Masetty]

This patch enables L1 level which is a finer grained preemption
at either a draw call or a bin boundary. The worst case switching
latency is higher in this case but that is a trade off we make for
enabling faster preemption.

Signed-off-by: Sharat Masetty <smasetty@codeaurora.org>
---
 drivers/gpu/drm/msm/adreno/a6xx_preempt.c | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/drivers/gpu/drm/msm/adreno/a6xx_preempt.c b/drivers/gpu/drm/msm/adreno/a6xx_preempt.c
index 0d2b612..4e83a4f 100644
--- a/drivers/gpu/drm/msm/adreno/a6xx_preempt.c
+++ b/drivers/gpu/drm/msm/adreno/a6xx_preempt.c
@@ -301,9 +301,17 @@ void a6xx_preempt_init(struct msm_gpu *gpu)
 	}
 
 	/* TODO: make this configurable? */
-	a6xx_gpu->preempt_level = 0;
+	/*
+	 * Use L1 preemption level which is preemption at either the draw call
+	 * level or the bin boundary level
+	 */
+	a6xx_gpu->preempt_level = 1;
 	a6xx_gpu->uses_gmem = 1;
-	a6xx_gpu->skip_save_restore = 0;
+	/*
+	 * Skip CP save and restore when preempting between bins, use
+	 * this only when the preemption level is 1
+	 */
+	a6xx_gpu->skip_save_restore = 1;
 
 	a6xx_gpu->scratch_ptr  = msm_gem_kernel_new(gpu->dev,
 			gpu->nr_rings * sizeof(uint64_t), MSM_BO_WC,
-- 
1.9.1

_______________________________________________
dri-devel mailing list
dri-devel@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/dri-devel