[PATCH 20/39] drm/i915: Added scheduler flush calls to ring throttle and idle functions

John.C.Harrison
2015-11-23T11:39:56Z

When requesting that all GPU work is completed, it is now necessary to
get the scheduler involved in order to flush out work that queued and
not yet submitted.

v2: Updated to add support for flushing the scheduler queue by time
stamp rather than just doing a blanket flush.

Change-Id: I95dcc2a2ee5c1a844748621c333994ddd6cf6a66
For: VIZ-1587
Signed-off-by: John Harrison <John.C.Harrison@Intel.com>
---
drivers/gpu/drm/i915/i915_gem.c | 24 ++++++++-
drivers/gpu/drm/i915/i915_scheduler.c | 92 +++++++++++++++++++++++++++++++++++
drivers/gpu/drm/i915/i915_scheduler.h | 3 ++
3 files changed, 118 insertions(+), 1 deletion(-)

diff --git a/drivers/gpu/drm/i915/i915_gem.c b/drivers/gpu/drm/i915/i915_gem.c
index 067f213..16bd82c 100644
--- a/drivers/gpu/drm/i915/i915_gem.c
+++ b/drivers/gpu/drm/i915/i915_gem.c
@@ -3768,6 +3768,10 @@ int i915_gpu_idle(struct drm_device *dev)

/* Flush everything onto the inactive list. */
for_each_ring(ring, dev_priv, i) {
+ ret = i915_scheduler_flush(ring, true);
+ if (ret < 0)
+ return ret;
+
if (!i915.enable_execlists) {
struct drm_i915_gem_request *req;

@@ -4481,7 +4485,8 @@ i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file)
unsigned long recent_enough = jiffies - DRM_I915_THROTTLE_JIFFIES;
struct drm_i915_gem_request *request, *target = NULL;
unsigned reset_counter;
- int ret;
+ int i, ret;
+ struct intel_engine_cs *ring;

ret = i915_gem_wait_for_error(&dev_priv->gpu_error);
if (ret)
@@ -4491,6 +4496,23 @@ i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file)
if (ret)
return ret;

+ for_each_ring(ring, dev_priv, i) {
+ /*
+ * Flush out scheduler entries that are getting 'stale'. Note
+ * that the following recent_enough test will only check
+ * against the time at which the request was submitted to the
+ * hardware (i.e. when it left the scheduler) not the time it
+ * was submitted to the driver.
+ *
+ * Also, there is not much point worring about busy return
+ * codes from the scheduler flush call. Even if more work
+ * cannot be submitted right now for whatever reason, we
+ * still want to throttle against stale work that has already
+ * been submitted.
+ */
+ i915_scheduler_flush_stamp(ring, recent_enough, false);
+ }
+
spin_lock(&file_priv->mm.lock);
list_for_each_entry(request, &file_priv->mm.request_list, client_list) {
if (time_after_eq(request->emitted_jiffies, recent_enough))
diff --git a/drivers/gpu/drm/i915/i915_scheduler.c b/drivers/gpu/drm/i915/i915_scheduler.c
index b8b84a6..232f48a 100644
--- a/drivers/gpu/drm/i915/i915_scheduler.c
+++ b/drivers/gpu/drm/i915/i915_scheduler.c
@@ -607,6 +607,98 @@ int i915_scheduler_flush_request(struct drm_i915_gem_request *req,
return flush_count;
}

+int i915_scheduler_flush_stamp(struct intel_engine_cs *ring,
+ unsigned long target,
+ bool is_locked)
+{
+ struct i915_scheduler_queue_entry *node;
+ struct drm_i915_private *dev_priv;
+ struct i915_scheduler *scheduler;
+ unsigned long flags;
+ int flush_count = 0;
+
+ if (!ring)
+ return -EINVAL;
+
+ dev_priv = ring->dev->dev_private;
+ scheduler = dev_priv->scheduler;
+
+ if (!scheduler)
+ return 0;
+
+ if (is_locked && (scheduler->flags[ring->id] & i915_sf_submitting)) {
+ /* Scheduler is busy already submitting another batch,
+ * come back later rather than going recursive... */
+ return -EAGAIN;
+ }
+
+ spin_lock_irqsave(&scheduler->lock, flags);
+ i915_scheduler_priority_bump_clear(scheduler);
+ list_for_each_entry(node, &scheduler->node_queue[ring->id], link) {
+ if (!I915_SQS_IS_QUEUED(node))
+ continue;
+
+ if (node->stamp > target)
+ continue;
+
+ flush_count = i915_scheduler_priority_bump(scheduler,
+ node, scheduler->priority_level_max);
+ }
+ spin_unlock_irqrestore(&scheduler->lock, flags);
+
+ if (flush_count) {
+ DRM_DEBUG_DRIVER("<%s> Bumped %d entries\n", ring->name, flush_count);
+ flush_count = i915_scheduler_submit_max_priority(ring, is_locked);
+ }
+
+ return flush_count;
+}
+
+int i915_scheduler_flush(struct intel_engine_cs *ring, bool is_locked)
+{
+ struct i915_scheduler_queue_entry *node;
+ struct drm_i915_private *dev_priv;
+ struct i915_scheduler *scheduler;
+ unsigned long flags;
+ bool found;
+ int ret;
+ uint32_t count = 0;
+
+ if (!ring)
+ return -EINVAL;
+
+ dev_priv = ring->dev->dev_private;
+ scheduler = dev_priv->scheduler;
+
+ if (!scheduler)
+ return 0;
+
+ BUG_ON(is_locked && (scheduler->flags[ring->id] & i915_sf_submitting));
+
+ do {
+ found = false;
+ spin_lock_irqsave(&scheduler->lock, flags);
+ list_for_each_entry(node, &scheduler->node_queue[ring->id], link) {
+ if (!I915_SQS_IS_QUEUED(node))
+ continue;
+
+ found = true;
+ break;
+ }
+ spin_unlock_irqrestore(&scheduler->lock, flags);
+
+ if (found) {
+ ret = i915_scheduler_submit(ring, is_locked);
+ if (ret < 0)
+ return ret;
+
+ count += ret;
+ }
+ } while (found);
+
+ return count;
+}
+
static void i915_scheduler_priority_bump_clear(struct i915_scheduler *scheduler)
{
struct i915_scheduler_queue_entry *node;
diff --git a/drivers/gpu/drm/i915/i915_scheduler.h b/drivers/gpu/drm/i915/i915_scheduler.h
index 42371b3..29f7248 100644
--- a/drivers/gpu/drm/i915/i915_scheduler.h
+++ b/drivers/gpu/drm/i915/i915_scheduler.h
@@ -92,6 +92,9 @@ int i915_scheduler_queue_execbuffer(struct i915_scheduler_queue_entry *q
bool i915_scheduler_notify_request(struct drm_i915_gem_request *req);
void i915_scheduler_wakeup(struct drm_device *dev);
void i915_gem_scheduler_work_handler(struct work_struct *work);
+int i915_scheduler_flush(struct intel_engine_cs *ring, bool is_locked);
+int i915_scheduler_flush_stamp(struct intel_engine_cs *ring,
+ unsigned long stamp, bool is_locked);
int i915_scheduler_flush_request(struct drm_i915_gem_request *req,
bool is_locked);
bool i915_scheduler_is_request_tracked(struct drm_i915_gem_request *req,