Re: [PATCH 044/190] drm/i915: Move GEM request routines to i915_gem_request.c

[Arun Siluvery <arun.siluvery@linux.intel.com>]

On 11/01/2016 09:16, Chris Wilson wrote:
> Migrate the request operations out of the main body of i915_gem.c and
> into their own C file for easier expansion.
>
> v2: Move __i915_add_request() across as well
>
> Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk>
> ---

don't we lose the history in git blame when moved to a new file? is this 
ok? especially for files like i915_gem.c.

regards
Arun

>   drivers/gpu/drm/i915/Makefile           |   1 +
>   drivers/gpu/drm/i915/i915_drv.h         | 205 +---------
>   drivers/gpu/drm/i915/i915_gem.c         | 652 +------------------------------
>   drivers/gpu/drm/i915/i915_gem_request.c | 659 ++++++++++++++++++++++++++++++++
>   drivers/gpu/drm/i915/i915_gem_request.h | 223 +++++++++++
>   5 files changed, 895 insertions(+), 845 deletions(-)
>   create mode 100644 drivers/gpu/drm/i915/i915_gem_request.c
>   create mode 100644 drivers/gpu/drm/i915/i915_gem_request.h
>
> diff --git a/drivers/gpu/drm/i915/Makefile b/drivers/gpu/drm/i915/Makefile
> index 99ce591c8574..b0a83215db80 100644
> --- a/drivers/gpu/drm/i915/Makefile
> +++ b/drivers/gpu/drm/i915/Makefile
> @@ -31,6 +31,7 @@ i915-y += i915_cmd_parser.o \
>   	  i915_gem_gtt.o \
>   	  i915_gem.o \
>   	  i915_gem_render_state.o \
> +	  i915_gem_request.o \
>   	  i915_gem_shrinker.o \
>   	  i915_gem_stolen.o \
>   	  i915_gem_tiling.o \
> diff --git a/drivers/gpu/drm/i915/i915_drv.h b/drivers/gpu/drm/i915/i915_drv.h
> index 57e450e25ad6..ee146ce02412 100644
> --- a/drivers/gpu/drm/i915/i915_drv.h
> +++ b/drivers/gpu/drm/i915/i915_drv.h
> @@ -41,6 +41,7 @@
>   #include "intel_lrc.h"
>   #include "i915_gem_gtt.h"
>   #include "i915_gem_render_state.h"
> +#include "i915_gem_request.h"
>   #include <linux/io-mapping.h>
>   #include <linux/i2c.h>
>   #include <linux/i2c-algo-bit.h>
> @@ -2162,179 +2163,15 @@ struct drm_i915_gem_object {
>   };
>   #define to_intel_bo(x) container_of(x, struct drm_i915_gem_object, base)
>
> -void i915_gem_track_fb(struct drm_i915_gem_object *old,
> -		       struct drm_i915_gem_object *new,
> -		       unsigned frontbuffer_bits);
> -
> -/**
> - * Request queue structure.
> - *
> - * The request queue allows us to note sequence numbers that have been emitted
> - * and may be associated with active buffers to be retired.
> - *
> - * By keeping this list, we can avoid having to do questionable sequence
> - * number comparisons on buffer last_read|write_seqno. It also allows an
> - * emission time to be associated with the request for tracking how far ahead
> - * of the GPU the submission is.
> - *
> - * The requests are reference counted, so upon creation they should have an
> - * initial reference taken using kref_init
> - */
> -struct drm_i915_gem_request {
> -	struct kref ref;
> -
> -	/** On Which ring this request was generated */
> -	struct drm_i915_private *i915;
> -	struct intel_engine_cs *ring;
> -	unsigned reset_counter;
> -
> -	 /** GEM sequence number associated with the previous request,
> -	  * when the HWS breadcrumb is equal to this the GPU is processing
> -	  * this request.
> -	  */
> -	u32 previous_seqno;
> -
> -	 /** GEM sequence number associated with this request,
> -	  * when the HWS breadcrumb is equal or greater than this the GPU
> -	  * has finished processing this request.
> -	  */
> -	u32 seqno;
> -
> -	/** Position in the ringbuffer of the start of the request */
> -	u32 head;
> -
> -	/**
> -	 * Position in the ringbuffer of the start of the postfix.
> -	 * This is required to calculate the maximum available ringbuffer
> -	 * space without overwriting the postfix.
> -	 */
> -	 u32 postfix;
> -
> -	/** Position in the ringbuffer of the end of the whole request */
> -	u32 tail;
> -
> -	/**
> -	 * Context and ring buffer related to this request
> -	 * Contexts are refcounted, so when this request is associated with a
> -	 * context, we must increment the context's refcount, to guarantee that
> -	 * it persists while any request is linked to it. Requests themselves
> -	 * are also refcounted, so the request will only be freed when the last
> -	 * reference to it is dismissed, and the code in
> -	 * i915_gem_request_free() will then decrement the refcount on the
> -	 * context.
> -	 */
> -	struct intel_context *ctx;
> -	struct intel_ringbuffer *ringbuf;
> -
> -	/** Batch buffer related to this request if any (used for
> -	    error state dump only) */
> -	struct drm_i915_gem_object *batch_obj;
> -
> -	/** Time at which this request was emitted, in jiffies. */
> -	unsigned long emitted_jiffies;
> -
> -	/** global list entry for this request */
> -	struct list_head list;
> -
> -	struct drm_i915_file_private *file_priv;
> -	/** file_priv list entry for this request */
> -	struct list_head client_list;
> -
> -	/** process identifier submitting this request */
> -	struct pid *pid;
> -
> -	/**
> -	 * The ELSP only accepts two elements at a time, so we queue
> -	 * context/tail pairs on a given queue (ring->execlist_queue) until the
> -	 * hardware is available. The queue serves a double purpose: we also use
> -	 * it to keep track of the up to 2 contexts currently in the hardware
> -	 * (usually one in execution and the other queued up by the GPU): We
> -	 * only remove elements from the head of the queue when the hardware
> -	 * informs us that an element has been completed.
> -	 *
> -	 * All accesses to the queue are mediated by a spinlock
> -	 * (ring->execlist_lock).
> -	 */
> -
> -	/** Execlist link in the submission queue.*/
> -	struct list_head execlist_link;
> -
> -	/** Execlists no. of times this request has been sent to the ELSP */
> -	int elsp_submitted;
> -
> -};
> -
>   #ifdef CONFIG_DRM_I915_DEBUG_GEM
>   #define GEM_BUG_ON(expr) BUG_ON(expr)
>   #else
>   #define GEM_BUG_ON(expr)
>   #endif
>
> -int i915_gem_request_alloc(struct intel_engine_cs *ring,
> -			   struct intel_context *ctx,
> -			   struct drm_i915_gem_request **req_out);
> -void i915_gem_request_cancel(struct drm_i915_gem_request *req);
> -void i915_gem_request_free(struct kref *req_ref);
> -int i915_gem_request_add_to_client(struct drm_i915_gem_request *req,
> -				   struct drm_file *file);
> -
> -static inline uint32_t
> -i915_gem_request_get_seqno(struct drm_i915_gem_request *req)
> -{
> -	return req ? req->seqno : 0;
> -}
> -
> -static inline struct intel_engine_cs *
> -i915_gem_request_get_ring(struct drm_i915_gem_request *req)
> -{
> -	return req ? req->ring : NULL;
> -}
> -
> -static inline struct drm_i915_gem_request *
> -i915_gem_request_reference(struct drm_i915_gem_request *req)
> -{
> -	if (req)
> -		kref_get(&req->ref);
> -	return req;
> -}
> -
> -static inline void
> -i915_gem_request_unreference(struct drm_i915_gem_request *req)
> -{
> -	WARN_ON(!mutex_is_locked(&req->ring->dev->struct_mutex));
> -	kref_put(&req->ref, i915_gem_request_free);
> -}
> -
> -static inline void
> -i915_gem_request_unreference__unlocked(struct drm_i915_gem_request *req)
> -{
> -	struct drm_device *dev;
> -
> -	if (!req)
> -		return;
> -
> -	dev = req->ring->dev;
> -	if (kref_put_mutex(&req->ref, i915_gem_request_free, &dev->struct_mutex))
> -		mutex_unlock(&dev->struct_mutex);
> -}
> -
> -static inline void i915_gem_request_assign(struct drm_i915_gem_request **pdst,
> -					   struct drm_i915_gem_request *src)
> -{
> -	if (src)
> -		i915_gem_request_reference(src);
> -
> -	if (*pdst)
> -		i915_gem_request_unreference(*pdst);
> -
> -	*pdst = src;
> -}
> -
> -/*
> - * XXX: i915_gem_request_completed should be here but currently needs the
> - * definition of i915_seqno_passed() which is below. It will be moved in
> - * a later patch when the call to i915_seqno_passed() is obsoleted...
> - */
> +void i915_gem_track_fb(struct drm_i915_gem_object *old,
> +		       struct drm_i915_gem_object *new,
> +		       unsigned frontbuffer_bits);
>
>   /*
>    * A command that requires special handling by the command parser.
> @@ -2956,28 +2793,6 @@ int i915_gem_dumb_create(struct drm_file *file_priv,
>   			 struct drm_mode_create_dumb *args);
>   int i915_gem_mmap_gtt(struct drm_file *file_priv, struct drm_device *dev,
>   		      uint32_t handle, uint64_t *offset);
> -/**
> - * Returns true if seq1 is later than seq2.
> - */
> -static inline bool
> -i915_seqno_passed(uint32_t seq1, uint32_t seq2)
> -{
> -	return (int32_t)(seq1 - seq2) >= 0;
> -}
> -
> -static inline bool i915_gem_request_started(struct drm_i915_gem_request *req)
> -{
> -	return i915_seqno_passed(intel_ring_get_seqno(req->ring),
> -				 req->previous_seqno);
> -}
> -
> -static inline bool i915_gem_request_completed(struct drm_i915_gem_request *req)
> -{
> -	return i915_seqno_passed(intel_ring_get_seqno(req->ring),
> -				 req->seqno);
> -}
> -
> -int __must_check i915_gem_get_seqno(struct drm_device *dev, u32 *seqno);
>   int __must_check i915_gem_set_seqno(struct drm_device *dev, u32 seqno);
>
>   struct drm_i915_gem_request *
> @@ -3036,18 +2851,6 @@ void i915_gem_init_swizzling(struct drm_device *dev);
>   void i915_gem_cleanup_ringbuffer(struct drm_device *dev);
>   int __must_check i915_gpu_idle(struct drm_device *dev);
>   int __must_check i915_gem_suspend(struct drm_device *dev);
> -void __i915_add_request(struct drm_i915_gem_request *req,
> -			struct drm_i915_gem_object *batch_obj,
> -			bool flush_caches);
> -#define i915_add_request(req) \
> -	__i915_add_request(req, NULL, true)
> -#define i915_add_request_no_flush(req) \
> -	__i915_add_request(req, NULL, false)
> -int __i915_wait_request(struct drm_i915_gem_request *req,
> -			bool interruptible,
> -			s64 *timeout,
> -			struct intel_rps_client *rps);
> -int __must_check i915_wait_request(struct drm_i915_gem_request *req);
>   int i915_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf);
>   int __must_check
>   i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj,
> diff --git a/drivers/gpu/drm/i915/i915_gem.c b/drivers/gpu/drm/i915/i915_gem.c
> index ea9344503bf6..68a25617ca7a 100644
> --- a/drivers/gpu/drm/i915/i915_gem.c
> +++ b/drivers/gpu/drm/i915/i915_gem.c
> @@ -1103,365 +1103,6 @@ put_rpm:
>   	return ret;
>   }
>
> -static int
> -i915_gem_check_wedge(unsigned reset_counter, bool interruptible)
> -{
> -	if (__i915_terminally_wedged(reset_counter))
> -		return -EIO;
> -
> -	if (__i915_reset_in_progress(reset_counter)) {
> -		/* Non-interruptible callers can't handle -EAGAIN, hence return
> -		 * -EIO unconditionally for these. */
> -		if (!interruptible)
> -			return -EIO;
> -
> -		return -EAGAIN;
> -	}
> -
> -	return 0;
> -}
> -
> -static unsigned long local_clock_us(unsigned *cpu)
> -{
> -	unsigned long t;
> -
> -	/* Cheaply and approximately convert from nanoseconds to microseconds.
> -	 * The result and subsequent calculations are also defined in the same
> -	 * approximate microseconds units. The principal source of timing
> -	 * error here is from the simple truncation.
> -	 *
> -	 * Note that local_clock() is only defined wrt to the current CPU;
> -	 * the comparisons are no longer valid if we switch CPUs. Instead of
> -	 * blocking preemption for the entire busywait, we can detect the CPU
> -	 * switch and use that as indicator of system load and a reason to
> -	 * stop busywaiting, see busywait_stop().
> -	 */
> -	*cpu = get_cpu();
> -	t = local_clock() >> 10;
> -	put_cpu();
> -
> -	return t;
> -}
> -
> -static bool busywait_stop(unsigned long timeout, unsigned cpu)
> -{
> -	unsigned this_cpu;
> -
> -	if (time_after(local_clock_us(&this_cpu), timeout))
> -		return true;
> -
> -	return this_cpu != cpu;
> -}
> -
> -static bool __i915_spin_request(struct drm_i915_gem_request *req,
> -				struct intel_wait *wait,
> -				int state)
> -{
> -	unsigned long timeout;
> -	unsigned cpu;
> -
> -	/* When waiting for high frequency requests, e.g. during synchronous
> -	 * rendering split between the CPU and GPU, the finite amount of time
> -	 * required to set up the irq and wait upon it limits the response
> -	 * rate. By busywaiting on the request completion for a short while we
> -	 * can service the high frequency waits as quick as possible. However,
> -	 * if it is a slow request, we want to sleep as quickly as possible.
> -	 * The tradeoff between waiting and sleeping is roughly the time it
> -	 * takes to sleep on a request, on the order of a microsecond.
> -	 */
> -
> -	/* Only spin if we know the GPU is processing this request */
> -	if (!i915_gem_request_started(req))
> -		return false;
> -
> -	timeout = local_clock_us(&cpu) + 5;
> -	do {
> -		if (i915_gem_request_completed(req))
> -			return true;
> -
> -		if (signal_pending_state(state, wait->task))
> -			break;
> -
> -		if (busywait_stop(timeout, cpu))
> -			break;
> -
> -		cpu_relax_lowlatency();
> -
> -		/* Break the loop if we have consumed the timeslice (or been
> -		 * preempted) or when either the background thread has
> -		 * enabled the interrupt, or the IRQ itself has fired.
> -		 */
> -	} while (!need_resched() && wait->task->state == state);
> -
> -	return false;
> -}
> -
> -/**
> - * __i915_wait_request - wait until execution of request has finished
> - * @req: duh!
> - * @interruptible: do an interruptible wait (normally yes)
> - * @timeout: in - how long to wait (NULL forever); out - how much time remaining
> - *
> - * Note: It is of utmost importance that the passed in seqno and reset_counter
> - * values have been read by the caller in an smp safe manner. Where read-side
> - * locks are involved, it is sufficient to read the reset_counter before
> - * unlocking the lock that protects the seqno. For lockless tricks, the
> - * reset_counter _must_ be read before, and an appropriate smp_rmb must be
> - * inserted.
> - *
> - * Returns 0 if the request was found within the alloted time. Else returns the
> - * errno with remaining time filled in timeout argument.
> - */
> -int __i915_wait_request(struct drm_i915_gem_request *req,
> -			bool interruptible,
> -			s64 *timeout,
> -			struct intel_rps_client *rps)
> -{
> -	int state = interruptible ? TASK_INTERRUPTIBLE : TASK_UNINTERRUPTIBLE;
> -	struct intel_wait wait;
> -	unsigned long timeout_remain;
> -	int ret = 0;
> -
> -	might_sleep();
> -
> -	if (list_empty(&req->list))
> -		return 0;
> -
> -	if (i915_gem_request_completed(req))
> -		return 0;
> -
> -	timeout_remain = MAX_SCHEDULE_TIMEOUT;
> -	if (timeout) {
> -		if (WARN_ON(*timeout < 0))
> -			return -EINVAL;
> -
> -		if (*timeout == 0)
> -			return -ETIME;
> -
> -		/* Record current time in case interrupted, or wedged */
> -		timeout_remain = nsecs_to_jiffies_timeout(*timeout);
> -		*timeout += ktime_get_raw_ns();
> -	}
> -
> -	trace_i915_gem_request_wait_begin(req);
> -
> -	/* This client is about to stall waiting for the GPU. In many cases
> -	 * this is undesirable and limits the throughput of the system, as
> -	 * many clients cannot continue processing user input/output whilst
> -	 * blocked. RPS autotuning may take tens of milliseconds to respond
> -	 * to the GPU load and thus incurs additional latency for the client.
> -	 * We can circumvent that by promoting the GPU frequency to maximum
> -	 * before we wait. This makes the GPU throttle up much more quickly
> -	 * (good for benchmarks and user experience, e.g. window animations),
> -	 * but at a cost of spending more power processing the workload
> -	 * (bad for battery). Not all clients even want their results
> -	 * immediately and for them we should just let the GPU select its own
> -	 * frequency to maximise efficiency. To prevent a single client from
> -	 * forcing the clocks too high for the whole system, we only allow
> -	 * each client to waitboost once in a busy period.
> -	 */
> -	if (INTEL_INFO(req->i915)->gen >= 6)
> -		gen6_rps_boost(req->i915, rps, req->emitted_jiffies);
> -
> -	intel_wait_init(&wait, req->seqno);
> -	set_task_state(wait.task, state);
> -
> -	/* Optimistic spin for the next ~jiffie before touching IRQs */
> -	if (intel_engine_add_wait(req->ring, &wait)) {
> -		if (__i915_spin_request(req, &wait, state))
> -			goto complete;
> -
> -		/* In order to check that we haven't missed the interrupt
> -		 * as we enabled it, we need to kick ourselves to do a
> -		 * coherent check on the seqno before we sleep.
> -		 */
> -		if (intel_engine_enable_wait_irq(req->ring, &wait))
> -			goto wakeup;
> -	}
> -
> -	for (;;) {
> -		if (signal_pending_state(state, wait.task)) {
> -			ret = -ERESTARTSYS;
> -			break;
> -		}
> -
> -		/* Ensure that even if the GPU hangs, we get woken up. */
> -		i915_queue_hangcheck(req->i915);
> -
> -		timeout_remain = io_schedule_timeout(timeout_remain);
> -		if (timeout_remain == 0) {
> -			ret = -ETIME;
> -			break;
> -		}
> -
> -		if (intel_wait_complete(&wait))
> -			break;
> -
> -wakeup:
> -		set_task_state(wait.task, state);
> -
> -		/* Carefully check if the request is complete, giving time
> -		 * for the seqno to be visible following the interrupt.
> -		 * We also have to check in case we are kicked by the GPU
> -		 * reset in order to drop the struct_mutex.
> -		 */
> -		if (__i915_request_irq_complete(req))
> -			break;
> -	}
> -
> -complete:
> -	intel_engine_remove_wait(req->ring, &wait);
> -	__set_task_state(wait.task, TASK_RUNNING);
> -	trace_i915_gem_request_wait_end(req);
> -
> -	if (timeout) {
> -		*timeout -= ktime_get_raw_ns();
> -		if (*timeout < 0)
> -			*timeout = 0;
> -
> -		/*
> -		 * Apparently ktime isn't accurate enough and occasionally has a
> -		 * bit of mismatch in the jiffies<->nsecs<->ktime loop. So patch
> -		 * things up to make the test happy. We allow up to 1 jiffy.
> -		 *
> -		 * This is a regrssion from the timespec->ktime conversion.
> -		 */
> -		if (ret == -ETIME && *timeout < jiffies_to_usecs(1)*1000)
> -			*timeout = 0;
> -	}
> -
> -	if (ret == 0 && rps && req->seqno == req->ring->last_submitted_seqno) {
> -		/* The GPU is now idle and this client has stalled.
> -		 * Since no other client has submitted a request in the
> -		 * meantime, assume that this client is the only one
> -		 * supplying work to the GPU but is unable to keep that
> -		 * work supplied because it is waiting. Since the GPU is
> -		 * then never kept fully busy, RPS autoclocking will
> -		 * keep the clocks relatively low, causing further delays.
> -		 * Compensate by giving the synchronous client credit for
> -		 * a waitboost next time.
> -		 */
> -		spin_lock(&req->i915->rps.client_lock);
> -		list_del_init(&rps->link);
> -		spin_unlock(&req->i915->rps.client_lock);
> -	}
> -
> -	return ret;
> -}
> -
> -int i915_gem_request_add_to_client(struct drm_i915_gem_request *req,
> -				   struct drm_file *file)
> -{
> -	struct drm_i915_private *dev_private;
> -	struct drm_i915_file_private *file_priv;
> -
> -	WARN_ON(!req || !file || req->file_priv);
> -
> -	if (!req || !file)
> -		return -EINVAL;
> -
> -	if (req->file_priv)
> -		return -EINVAL;
> -
> -	dev_private = req->ring->dev->dev_private;
> -	file_priv = file->driver_priv;
> -
> -	spin_lock(&file_priv->mm.lock);
> -	req->file_priv = file_priv;
> -	list_add_tail(&req->client_list, &file_priv->mm.request_list);
> -	spin_unlock(&file_priv->mm.lock);
> -
> -	req->pid = get_pid(task_pid(current));
> -
> -	return 0;
> -}
> -
> -static inline void
> -i915_gem_request_remove_from_client(struct drm_i915_gem_request *request)
> -{
> -	struct drm_i915_file_private *file_priv = request->file_priv;
> -
> -	if (!file_priv)
> -		return;
> -
> -	spin_lock(&file_priv->mm.lock);
> -	list_del(&request->client_list);
> -	request->file_priv = NULL;
> -	spin_unlock(&file_priv->mm.lock);
> -
> -	put_pid(request->pid);
> -	request->pid = NULL;
> -}
> -
> -static void i915_gem_request_retire(struct drm_i915_gem_request *request)
> -{
> -	trace_i915_gem_request_retire(request);
> -
> -	/* We know the GPU must have read the request to have
> -	 * sent us the seqno + interrupt, so use the position
> -	 * of tail of the request to update the last known position
> -	 * of the GPU head.
> -	 *
> -	 * Note this requires that we are always called in request
> -	 * completion order.
> -	 */
> -	request->ringbuf->last_retired_head = request->postfix;
> -
> -	list_del_init(&request->list);
> -	i915_gem_request_remove_from_client(request);
> -
> -	i915_gem_request_unreference(request);
> -}
> -
> -static void
> -__i915_gem_request_retire__upto(struct drm_i915_gem_request *req)
> -{
> -	struct intel_engine_cs *engine = req->ring;
> -	struct drm_i915_gem_request *tmp;
> -
> -	lockdep_assert_held(&engine->dev->struct_mutex);
> -
> -	if (list_empty(&req->list))
> -		return;
> -
> -	do {
> -		tmp = list_first_entry(&engine->request_list,
> -				       typeof(*tmp), list);
> -
> -		i915_gem_request_retire(tmp);
> -	} while (tmp != req);
> -
> -	WARN_ON(i915_verify_lists(engine->dev));
> -}
> -
> -/**
> - * Waits for a request to be signaled, and cleans up the
> - * request and object lists appropriately for that event.
> - */
> -int
> -i915_wait_request(struct drm_i915_gem_request *req)
> -{
> -	struct drm_device *dev;
> -	struct drm_i915_private *dev_priv;
> -	bool interruptible;
> -	int ret;
> -
> -	BUG_ON(req == NULL);
> -
> -	dev = req->ring->dev;
> -	dev_priv = dev->dev_private;
> -	interruptible = dev_priv->mm.interruptible;
> -
> -	BUG_ON(!mutex_is_locked(&dev->struct_mutex));
> -
> -	ret = __i915_wait_request(req, interruptible, NULL, NULL);
> -	if (ret)
> -		return ret;
> -
> -	__i915_gem_request_retire__upto(req);
> -	return 0;
> -}
> -
>   /**
>    * Ensures that all rendering to the object has completed and the object is
>    * safe to unbind from the GTT or access from the CPU.
> @@ -1515,7 +1156,7 @@ i915_gem_object_retire_request(struct drm_i915_gem_object *obj,
>   	else if (obj->last_write_req == req)
>   		i915_gem_object_retire__write(obj);
>
> -	__i915_gem_request_retire__upto(req);
> +	i915_gem_request_retire_upto(req);
>   }
>
>   /* A nonblocking variant of the above wait. This is a highly dangerous routine
> @@ -2441,94 +2082,6 @@ i915_gem_object_retire__read(struct drm_i915_gem_object *obj, int ring)
>   	drm_gem_object_unreference(&obj->base);
>   }
>
> -static int
> -i915_gem_init_seqno(struct drm_device *dev, u32 seqno)
> -{
> -	struct drm_i915_private *dev_priv = dev->dev_private;
> -	struct intel_engine_cs *ring;
> -	int ret, i, j;
> -
> -	/* Carefully retire all requests without writing to the rings */
> -	for_each_ring(ring, dev_priv, i) {
> -		ret = intel_ring_idle(ring);
> -		if (ret)
> -			return ret;
> -	}
> -	i915_gem_retire_requests(dev);
> -
> -	/* Finally reset hw state */
> -	for_each_ring(ring, dev_priv, i) {
> -		intel_ring_init_seqno(ring, seqno);
> -
> -		for (j = 0; j < ARRAY_SIZE(ring->semaphore.sync_seqno); j++)
> -			ring->semaphore.sync_seqno[j] = 0;
> -	}
> -
> -	return 0;
> -}
> -
> -int i915_gem_set_seqno(struct drm_device *dev, u32 seqno)
> -{
> -	struct drm_i915_private *dev_priv = dev->dev_private;
> -	int ret;
> -
> -	if (seqno == 0)
> -		return -EINVAL;
> -
> -	/* HWS page needs to be set less than what we
> -	 * will inject to ring
> -	 */
> -	ret = i915_gem_init_seqno(dev, seqno - 1);
> -	if (ret)
> -		return ret;
> -
> -	/* Carefully set the last_seqno value so that wrap
> -	 * detection still works
> -	 */
> -	dev_priv->next_seqno = seqno;
> -	dev_priv->last_seqno = seqno - 1;
> -	if (dev_priv->last_seqno == 0)
> -		dev_priv->last_seqno--;
> -
> -	return 0;
> -}
> -
> -int
> -i915_gem_get_seqno(struct drm_device *dev, u32 *seqno)
> -{
> -	struct drm_i915_private *dev_priv = dev->dev_private;
> -
> -	/* reserve 0 for non-seqno */
> -	if (dev_priv->next_seqno == 0) {
> -		int ret = i915_gem_init_seqno(dev, 0);
> -		if (ret)
> -			return ret;
> -
> -		dev_priv->next_seqno = 1;
> -	}
> -
> -	*seqno = dev_priv->last_seqno = dev_priv->next_seqno++;
> -	return 0;
> -}
> -
> -static void i915_gem_mark_busy(struct drm_i915_private *dev_priv)
> -{
> -	if (dev_priv->mm.busy)
> -		return;
> -
> -	intel_runtime_pm_get_noresume(dev_priv);
> -
> -	i915_update_gfx_val(dev_priv);
> -	if (INTEL_INFO(dev_priv)->gen >= 6)
> -		gen6_rps_busy(dev_priv);
> -
> -	queue_delayed_work(dev_priv->wq,
> -			   &dev_priv->mm.retire_work,
> -			   round_jiffies_up_relative(HZ));
> -
> -	dev_priv->mm.busy = true;
> -}
> -
>   static void i915_gem_mark_idle(struct drm_i915_private *dev_priv)
>   {
>   	dev_priv->mm.busy = false;
> @@ -2542,92 +2095,6 @@ static void i915_gem_mark_idle(struct drm_i915_private *dev_priv)
>   	intel_runtime_pm_put(dev_priv);
>   }
>
> -/*
> - * NB: This function is not allowed to fail. Doing so would mean the the
> - * request is not being tracked for completion but the work itself is
> - * going to happen on the hardware. This would be a Bad Thing(tm).
> - */
> -void __i915_add_request(struct drm_i915_gem_request *request,
> -			struct drm_i915_gem_object *obj,
> -			bool flush_caches)
> -{
> -	struct intel_engine_cs *ring;
> -	struct drm_i915_private *dev_priv;
> -	struct intel_ringbuffer *ringbuf;
> -	u32 request_start;
> -	int ret;
> -
> -	if (WARN_ON(request == NULL))
> -		return;
> -
> -	ring = request->ring;
> -	dev_priv = ring->dev->dev_private;
> -	ringbuf = request->ringbuf;
> -
> -	/*
> -	 * To ensure that this call will not fail, space for its emissions
> -	 * should already have been reserved in the ring buffer. Let the ring
> -	 * know that it is time to use that space up.
> -	 */
> -	intel_ring_reserved_space_use(ringbuf);
> -
> -	request_start = intel_ring_get_tail(ringbuf);
> -	/*
> -	 * Emit any outstanding flushes - execbuf can fail to emit the flush
> -	 * after having emitted the batchbuffer command. Hence we need to fix
> -	 * things up similar to emitting the lazy request. The difference here
> -	 * is that the flush _must_ happen before the next request, no matter
> -	 * what.
> -	 */
> -	if (flush_caches) {
> -		if (i915.enable_execlists)
> -			ret = logical_ring_flush_all_caches(request);
> -		else
> -			ret = intel_ring_flush_all_caches(request);
> -		/* Not allowed to fail! */
> -		WARN(ret, "*_ring_flush_all_caches failed: %d!\n", ret);
> -	}
> -
> -	/* Record the position of the start of the request so that
> -	 * should we detect the updated seqno part-way through the
> -	 * GPU processing the request, we never over-estimate the
> -	 * position of the head.
> -	 */
> -	request->postfix = intel_ring_get_tail(ringbuf);
> -
> -	if (i915.enable_execlists)
> -		ret = ring->emit_request(request);
> -	else {
> -		ret = ring->add_request(request);
> -
> -		request->tail = intel_ring_get_tail(ringbuf);
> -	}
> -	/* Not allowed to fail! */
> -	WARN(ret, "emit|add_request failed: %d!\n", ret);
> -
> -	request->head = request_start;
> -
> -	/* Whilst this request exists, batch_obj will be on the
> -	 * active_list, and so will hold the active reference. Only when this
> -	 * request is retired will the the batch_obj be moved onto the
> -	 * inactive_list and lose its active reference. Hence we do not need
> -	 * to explicitly hold another reference here.
> -	 */
> -	request->batch_obj = obj;
> -
> -	request->emitted_jiffies = jiffies;
> -	request->previous_seqno = ring->last_submitted_seqno;
> -	ring->last_submitted_seqno = request->seqno;
> -	list_add_tail(&request->list, &ring->request_list);
> -
> -	trace_i915_gem_request_add(request);
> -
> -	i915_gem_mark_busy(dev_priv);
> -
> -	/* Sanity check that the reserved size was large enough. */
> -	intel_ring_reserved_space_end(ringbuf);
> -}
> -
>   static bool i915_context_is_banned(struct drm_i915_private *dev_priv,
>   				   const struct intel_context *ctx)
>   {
> @@ -2666,109 +2133,6 @@ static void i915_set_reset_status(struct drm_i915_private *dev_priv,
>   	}
>   }
>
> -void i915_gem_request_free(struct kref *req_ref)
> -{
> -	struct drm_i915_gem_request *req = container_of(req_ref,
> -						 typeof(*req), ref);
> -	struct intel_context *ctx = req->ctx;
> -
> -	if (req->file_priv)
> -		i915_gem_request_remove_from_client(req);
> -
> -	if (ctx) {
> -		if (i915.enable_execlists) {
> -			if (ctx != req->ring->default_context)
> -				intel_lr_context_unpin(req);
> -		}
> -
> -		i915_gem_context_unreference(ctx);
> -	}
> -
> -	kmem_cache_free(req->i915->requests, req);
> -}
> -
> -int i915_gem_request_alloc(struct intel_engine_cs *ring,
> -			   struct intel_context *ctx,
> -			   struct drm_i915_gem_request **req_out)
> -{
> -	struct drm_i915_private *dev_priv = to_i915(ring->dev);
> -	unsigned reset_counter = i915_reset_counter(&dev_priv->gpu_error);
> -	struct drm_i915_gem_request *req;
> -	int ret;
> -
> -	if (!req_out)
> -		return -EINVAL;
> -
> -	*req_out = NULL;
> -
> -	/* ABI: Before userspace accesses the GPU (e.g. execbuffer), report
> -	 * EIO if the GPU is already wedged, or EAGAIN to drop the struct_mutex
> -	 * and restart.
> -	 */
> -	ret = i915_gem_check_wedge(reset_counter, dev_priv->mm.interruptible);
> -	if (ret)
> -		return ret;
> -
> -	req = kmem_cache_zalloc(dev_priv->requests, GFP_KERNEL);
> -	if (req == NULL)
> -		return -ENOMEM;
> -
> -	ret = i915_gem_get_seqno(ring->dev, &req->seqno);
> -	if (ret)
> -		goto err;
> -
> -	kref_init(&req->ref);
> -	req->i915 = dev_priv;
> -	req->ring = ring;
> -	req->reset_counter = reset_counter;
> -	req->ctx  = ctx;
> -	i915_gem_context_reference(req->ctx);
> -
> -	if (i915.enable_execlists)
> -		ret = intel_logical_ring_alloc_request_extras(req);
> -	else
> -		ret = intel_ring_alloc_request_extras(req);
> -	if (ret) {
> -		i915_gem_context_unreference(req->ctx);
> -		goto err;
> -	}
> -
> -	/*
> -	 * Reserve space in the ring buffer for all the commands required to
> -	 * eventually emit this request. This is to guarantee that the
> -	 * i915_add_request() call can't fail. Note that the reserve may need
> -	 * to be redone if the request is not actually submitted straight
> -	 * away, e.g. because a GPU scheduler has deferred it.
> -	 */
> -	if (i915.enable_execlists)
> -		ret = intel_logical_ring_reserve_space(req);
> -	else
> -		ret = intel_ring_reserve_space(req);
> -	if (ret) {
> -		/*
> -		 * At this point, the request is fully allocated even if not
> -		 * fully prepared. Thus it can be cleaned up using the proper
> -		 * free code.
> -		 */
> -		i915_gem_request_cancel(req);
> -		return ret;
> -	}
> -
> -	*req_out = req;
> -	return 0;
> -
> -err:
> -	kmem_cache_free(dev_priv->requests, req);
> -	return ret;
> -}
> -
> -void i915_gem_request_cancel(struct drm_i915_gem_request *req)
> -{
> -	intel_ring_reserved_space_cancel(req->ringbuf);
> -
> -	i915_gem_request_unreference(req);
> -}
> -
>   struct drm_i915_gem_request *
>   i915_gem_find_active_request(struct intel_engine_cs *ring)
>   {
> @@ -2850,14 +2214,14 @@ static void i915_gem_reset_ring_cleanup(struct drm_i915_private *dev_priv,
>   	 * implicit references on things like e.g. ppgtt address spaces through
>   	 * the request.
>   	 */
> -	while (!list_empty(&ring->request_list)) {
> +	if (!list_empty(&ring->request_list)) {
>   		struct drm_i915_gem_request *request;
>
> -		request = list_first_entry(&ring->request_list,
> -					   struct drm_i915_gem_request,
> -					   list);
> +		request = list_last_entry(&ring->request_list,
> +					  struct drm_i915_gem_request,
> +					  list);
>
> -		i915_gem_request_retire(request);
> +		i915_gem_request_retire_upto(request);
>   	}
>
>   	/* Having flushed all requests from all queues, we know that all
> @@ -2922,7 +2286,7 @@ i915_gem_retire_requests_ring(struct intel_engine_cs *ring)
>   		if (!i915_gem_request_completed(request))
>   			break;
>
> -		i915_gem_request_retire(request);
> +		i915_gem_request_retire_upto(request);
>   	}
>
>   	/* Move any buffers on the active list that are no longer referenced
> @@ -3053,7 +2417,7 @@ i915_gem_object_flush_active(struct drm_i915_gem_object *obj)
>   			goto retire;
>
>   		if (i915_gem_request_completed(req)) {
> -			__i915_gem_request_retire__upto(req);
> +			i915_gem_request_retire_upto(req);
>   retire:
>   			i915_gem_object_retire__read(obj, i);
>   		}
> diff --git a/drivers/gpu/drm/i915/i915_gem_request.c b/drivers/gpu/drm/i915/i915_gem_request.c
> new file mode 100644
> index 000000000000..b4ede6dd7b20
> --- /dev/null
> +++ b/drivers/gpu/drm/i915/i915_gem_request.c
> @@ -0,0 +1,659 @@
> +/*
> + * Copyright © 2008-2015 Intel Corporation
> + *
> + * Permission is hereby granted, free of charge, to any person obtaining a
> + * copy of this software and associated documentation files (the "Software"),
> + * to deal in the Software without restriction, including without limitation
> + * the rights to use, copy, modify, merge, publish, distribute, sublicense,
> + * and/or sell copies of the Software, and to permit persons to whom the
> + * Software is furnished to do so, subject to the following conditions:
> + *
> + * The above copyright notice and this permission notice (including the next
> + * paragraph) shall be included in all copies or substantial portions of the
> + * Software.
> + *
> + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
> + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
> + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
> + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
> + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
> + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
> + * IN THE SOFTWARE.
> + *
> + */
> +
> +#include "i915_drv.h"
> +
> +static int
> +i915_gem_check_wedge(unsigned reset_counter, bool interruptible)
> +{
> +	if (__i915_terminally_wedged(reset_counter))
> +		return -EIO;
> +
> +	if (__i915_reset_in_progress(reset_counter)) {
> +		/* Non-interruptible callers can't handle -EAGAIN, hence return
> +		 * -EIO unconditionally for these. */
> +		if (!interruptible)
> +			return -EIO;
> +
> +		return -EAGAIN;
> +	}
> +
> +	return 0;
> +}
> +
> +static int
> +i915_gem_init_seqno(struct drm_i915_private *dev_priv, u32 seqno)
> +{
> +	struct intel_engine_cs *ring;
> +	int ret, i, j;
> +
> +	/* Carefully retire all requests without writing to the rings */
> +	for_each_ring(ring, dev_priv, i) {
> +		ret = intel_ring_idle(ring);
> +		if (ret)
> +			return ret;
> +	}
> +	i915_gem_retire_requests(dev_priv->dev);
> +
> +	/* Finally reset hw state */
> +	for_each_ring(ring, dev_priv, i) {
> +		intel_ring_init_seqno(ring, seqno);
> +
> +		for (j = 0; j < ARRAY_SIZE(ring->semaphore.sync_seqno); j++)
> +			ring->semaphore.sync_seqno[j] = 0;
> +	}
> +
> +	return 0;
> +}
> +
> +int i915_gem_set_seqno(struct drm_device *dev, u32 seqno)
> +{
> +	struct drm_i915_private *dev_priv = dev->dev_private;
> +	int ret;
> +
> +	if (seqno == 0)
> +		return -EINVAL;
> +
> +	/* HWS page needs to be set less than what we
> +	 * will inject to ring
> +	 */
> +	ret = i915_gem_init_seqno(dev_priv, seqno - 1);
> +	if (ret)
> +		return ret;
> +
> +	/* Carefully set the last_seqno value so that wrap
> +	 * detection still works
> +	 */
> +	dev_priv->next_seqno = seqno;
> +	dev_priv->last_seqno = seqno - 1;
> +	if (dev_priv->last_seqno == 0)
> +		dev_priv->last_seqno--;
> +
> +	return 0;
> +}
> +
> +static int
> +i915_gem_get_seqno(struct drm_i915_private *dev_priv, u32 *seqno)
> +{
> +	/* reserve 0 for non-seqno */
> +	if (unlikely(dev_priv->next_seqno == 0)) {
> +		int ret = i915_gem_init_seqno(dev_priv, 0);
> +		if (ret)
> +			return ret;
> +
> +		dev_priv->next_seqno = 1;
> +	}
> +
> +	*seqno = dev_priv->last_seqno = dev_priv->next_seqno++;
> +	return 0;
> +}
> +
> +int i915_gem_request_alloc(struct intel_engine_cs *ring,
> +			   struct intel_context *ctx,
> +			   struct drm_i915_gem_request **req_out)
> +{
> +	struct drm_i915_private *dev_priv = to_i915(ring->dev);
> +	unsigned reset_counter = i915_reset_counter(&dev_priv->gpu_error);
> +	struct drm_i915_gem_request *req;
> +	int ret;
> +
> +	if (!req_out)
> +		return -EINVAL;
> +
> +	*req_out = NULL;
> +
> +	/* ABI: Before userspace accesses the GPU (e.g. execbuffer), report
> +	 * EIO if the GPU is already wedged, or EAGAIN to drop the struct_mutex
> +	 * and restart.
> +	 */
> +	ret = i915_gem_check_wedge(reset_counter, dev_priv->mm.interruptible);
> +	if (ret)
> +		return ret;
> +
> +	req = kmem_cache_zalloc(dev_priv->requests, GFP_KERNEL);
> +	if (req == NULL)
> +		return -ENOMEM;
> +
> +	ret = i915_gem_get_seqno(dev_priv, &req->seqno);
> +	if (ret)
> +		goto err;
> +
> +	kref_init(&req->ref);
> +	req->i915 = dev_priv;
> +	req->ring = ring;
> +	req->reset_counter = reset_counter;
> +	req->ctx  = ctx;
> +	i915_gem_context_reference(req->ctx);
> +
> +	if (i915.enable_execlists)
> +		ret = intel_logical_ring_alloc_request_extras(req);
> +	else
> +		ret = intel_ring_alloc_request_extras(req);
> +	if (ret) {
> +		i915_gem_context_unreference(req->ctx);
> +		goto err;
> +	}
> +
> +	/*
> +	 * Reserve space in the ring buffer for all the commands required to
> +	 * eventually emit this request. This is to guarantee that the
> +	 * i915_add_request() call can't fail. Note that the reserve may need
> +	 * to be redone if the request is not actually submitted straight
> +	 * away, e.g. because a GPU scheduler has deferred it.
> +	 */
> +	if (i915.enable_execlists)
> +		ret = intel_logical_ring_reserve_space(req);
> +	else
> +		ret = intel_ring_reserve_space(req);
> +	if (ret) {
> +		/*
> +		 * At this point, the request is fully allocated even if not
> +		 * fully prepared. Thus it can be cleaned up using the proper
> +		 * free code.
> +		 */
> +		i915_gem_request_cancel(req);
> +		return ret;
> +	}
> +
> +	*req_out = req;
> +	return 0;
> +
> +err:
> +	kmem_cache_free(dev_priv->requests, req);
> +	return ret;
> +}
> +
> +void i915_gem_request_cancel(struct drm_i915_gem_request *req)
> +{
> +	intel_ring_reserved_space_cancel(req->ringbuf);
> +
> +	i915_gem_request_unreference(req);
> +}
> +
> +int i915_gem_request_add_to_client(struct drm_i915_gem_request *req,
> +				   struct drm_file *file)
> +{
> +	struct drm_i915_private *dev_private;
> +	struct drm_i915_file_private *file_priv;
> +
> +	WARN_ON(!req || !file || req->file_priv);
> +
> +	if (!req || !file)
> +		return -EINVAL;
> +
> +	if (req->file_priv)
> +		return -EINVAL;
> +
> +	dev_private = req->ring->dev->dev_private;
> +	file_priv = file->driver_priv;
> +
> +	spin_lock(&file_priv->mm.lock);
> +	req->file_priv = file_priv;
> +	list_add_tail(&req->client_list, &file_priv->mm.request_list);
> +	spin_unlock(&file_priv->mm.lock);
> +
> +	req->pid = get_pid(task_pid(current));
> +
> +	return 0;
> +}
> +
> +static inline void
> +i915_gem_request_remove_from_client(struct drm_i915_gem_request *request)
> +{
> +	struct drm_i915_file_private *file_priv = request->file_priv;
> +
> +	if (!file_priv)
> +		return;
> +
> +	spin_lock(&file_priv->mm.lock);
> +	list_del(&request->client_list);
> +	request->file_priv = NULL;
> +	spin_unlock(&file_priv->mm.lock);
> +
> +	put_pid(request->pid);
> +	request->pid = NULL;
> +}
> +
> +static void i915_gem_request_retire(struct drm_i915_gem_request *request)
> +{
> +	trace_i915_gem_request_retire(request);
> +
> +	/* We know the GPU must have read the request to have
> +	 * sent us the seqno + interrupt, so use the position
> +	 * of tail of the request to update the last known position
> +	 * of the GPU head.
> +	 *
> +	 * Note this requires that we are always called in request
> +	 * completion order.
> +	 */
> +	request->ringbuf->last_retired_head = request->postfix;
> +
> +	list_del_init(&request->list);
> +	i915_gem_request_remove_from_client(request);
> +
> +	i915_gem_request_unreference(request);
> +}
> +
> +void
> +i915_gem_request_retire_upto(struct drm_i915_gem_request *req)
> +{
> +	struct intel_engine_cs *engine = req->ring;
> +	struct drm_i915_gem_request *tmp;
> +
> +	lockdep_assert_held(&engine->dev->struct_mutex);
> +
> +	if (list_empty(&req->list))
> +		return;
> +
> +	do {
> +		tmp = list_first_entry(&engine->request_list,
> +				       typeof(*tmp), list);
> +
> +		i915_gem_request_retire(tmp);
> +	} while (tmp != req);
> +
> +	WARN_ON(i915_verify_lists(engine->dev));
> +}
> +
> +static void i915_gem_mark_busy(struct drm_i915_private *dev_priv)
> +{
> +	if (dev_priv->mm.busy)
> +		return;
> +
> +	intel_runtime_pm_get_noresume(dev_priv);
> +
> +	i915_update_gfx_val(dev_priv);
> +	if (INTEL_INFO(dev_priv)->gen >= 6)
> +		gen6_rps_busy(dev_priv);
> +
> +	queue_delayed_work(dev_priv->wq,
> +			   &dev_priv->mm.retire_work,
> +			   round_jiffies_up_relative(HZ));
> +
> +	dev_priv->mm.busy = true;
> +}
> +
> +/*
> + * NB: This function is not allowed to fail. Doing so would mean the the
> + * request is not being tracked for completion but the work itself is
> + * going to happen on the hardware. This would be a Bad Thing(tm).
> + */
> +void __i915_add_request(struct drm_i915_gem_request *request,
> +			struct drm_i915_gem_object *obj,
> +			bool flush_caches)
> +{
> +	struct intel_engine_cs *ring;
> +	struct drm_i915_private *dev_priv;
> +	struct intel_ringbuffer *ringbuf;
> +	u32 request_start;
> +	int ret;
> +
> +	if (WARN_ON(request == NULL))
> +		return;
> +
> +	ring = request->ring;
> +	dev_priv = ring->dev->dev_private;
> +	ringbuf = request->ringbuf;
> +
> +	/*
> +	 * To ensure that this call will not fail, space for its emissions
> +	 * should already have been reserved in the ring buffer. Let the ring
> +	 * know that it is time to use that space up.
> +	 */
> +	intel_ring_reserved_space_use(ringbuf);
> +
> +	request_start = intel_ring_get_tail(ringbuf);
> +	/*
> +	 * Emit any outstanding flushes - execbuf can fail to emit the flush
> +	 * after having emitted the batchbuffer command. Hence we need to fix
> +	 * things up similar to emitting the lazy request. The difference here
> +	 * is that the flush _must_ happen before the next request, no matter
> +	 * what.
> +	 */
> +	if (flush_caches) {
> +		if (i915.enable_execlists)
> +			ret = logical_ring_flush_all_caches(request);
> +		else
> +			ret = intel_ring_flush_all_caches(request);
> +		/* Not allowed to fail! */
> +		WARN(ret, "*_ring_flush_all_caches failed: %d!\n", ret);
> +	}
> +
> +	/* Record the position of the start of the request so that
> +	 * should we detect the updated seqno part-way through the
> +	 * GPU processing the request, we never over-estimate the
> +	 * position of the head.
> +	 */
> +	request->postfix = intel_ring_get_tail(ringbuf);
> +
> +	if (i915.enable_execlists)
> +		ret = ring->emit_request(request);
> +	else {
> +		ret = ring->add_request(request);
> +
> +		request->tail = intel_ring_get_tail(ringbuf);
> +	}
> +	/* Not allowed to fail! */
> +	WARN(ret, "emit|add_request failed: %d!\n", ret);
> +
> +	request->head = request_start;
> +
> +	/* Whilst this request exists, batch_obj will be on the
> +	 * active_list, and so will hold the active reference. Only when this
> +	 * request is retired will the the batch_obj be moved onto the
> +	 * inactive_list and lose its active reference. Hence we do not need
> +	 * to explicitly hold another reference here.
> +	 */
> +	request->batch_obj = obj;
> +
> +	request->emitted_jiffies = jiffies;
> +	request->previous_seqno = ring->last_submitted_seqno;
> +	ring->last_submitted_seqno = request->seqno;
> +	list_add_tail(&request->list, &ring->request_list);
> +
> +	trace_i915_gem_request_add(request);
> +
> +	i915_gem_mark_busy(dev_priv);
> +
> +	/* Sanity check that the reserved size was large enough. */
> +	intel_ring_reserved_space_end(ringbuf);
> +}
> +
> +
> +static unsigned long local_clock_us(unsigned *cpu)
> +{
> +	unsigned long t;
> +
> +	/* Cheaply and approximately convert from nanoseconds to microseconds.
> +	 * The result and subsequent calculations are also defined in the same
> +	 * approximate microseconds units. The principal source of timing
> +	 * error here is from the simple truncation.
> +	 *
> +	 * Note that local_clock() is only defined wrt to the current CPU;
> +	 * the comparisons are no longer valid if we switch CPUs. Instead of
> +	 * blocking preemption for the entire busywait, we can detect the CPU
> +	 * switch and use that as indicator of system load and a reason to
> +	 * stop busywaiting, see busywait_stop().
> +	 */
> +	*cpu = get_cpu();
> +	t = local_clock() >> 10;
> +	put_cpu();
> +
> +	return t;
> +}
> +
> +static bool busywait_stop(unsigned long timeout, unsigned cpu)
> +{
> +	unsigned this_cpu;
> +
> +	if (time_after(local_clock_us(&this_cpu), timeout))
> +		return true;
> +
> +	return this_cpu != cpu;
> +}
> +
> +static bool __i915_spin_request(struct drm_i915_gem_request *req,
> +				struct intel_wait *wait,
> +				int state)
> +{
> +	unsigned long timeout;
> +	unsigned cpu;
> +
> +	/* When waiting for high frequency requests, e.g. during synchronous
> +	 * rendering split between the CPU and GPU, the finite amount of time
> +	 * required to set up the irq and wait upon it limits the response
> +	 * rate. By busywaiting on the request completion for a short while we
> +	 * can service the high frequency waits as quick as possible. However,
> +	 * if it is a slow request, we want to sleep as quickly as possible.
> +	 * The tradeoff between waiting and sleeping is roughly the time it
> +	 * takes to sleep on a request, on the order of a microsecond.
> +	 */
> +
> +	/* Only spin if we know the GPU is processing this request */
> +	if (!i915_gem_request_started(req))
> +		return false;
> +
> +	timeout = local_clock_us(&cpu) + 5;
> +	do {
> +		if (i915_gem_request_completed(req))
> +			return true;
> +
> +		if (signal_pending_state(state, wait->task))
> +			break;
> +
> +		if (busywait_stop(timeout, cpu))
> +			break;
> +
> +		cpu_relax_lowlatency();
> +
> +		/* Break the loop if we have consumed the timeslice (or been
> +		 * preempted) or when either the background thread has
> +		 * enabled the interrupt, or the IRQ itself has fired.
> +		 */
> +	} while (!need_resched() && wait->task->state == state);
> +
> +	return false;
> +}
> +
> +/**
> + * __i915_wait_request - wait until execution of request has finished
> + * @req: duh!
> + * @interruptible: do an interruptible wait (normally yes)
> + * @timeout: in - how long to wait (NULL forever); out - how much time remaining
> + *
> + * Note: It is of utmost importance that the passed in seqno and reset_counter
> + * values have been read by the caller in an smp safe manner. Where read-side
> + * locks are involved, it is sufficient to read the reset_counter before
> + * unlocking the lock that protects the seqno. For lockless tricks, the
> + * reset_counter _must_ be read before, and an appropriate smp_rmb must be
> + * inserted.
> + *
> + * Returns 0 if the request was found within the alloted time. Else returns the
> + * errno with remaining time filled in timeout argument.
> + */
> +int __i915_wait_request(struct drm_i915_gem_request *req,
> +			bool interruptible,
> +			s64 *timeout,
> +			struct intel_rps_client *rps)
> +{
> +	int state = interruptible ? TASK_INTERRUPTIBLE : TASK_UNINTERRUPTIBLE;
> +	struct intel_wait wait;
> +	unsigned long timeout_remain;
> +	int ret = 0;
> +
> +	might_sleep();
> +
> +	if (list_empty(&req->list))
> +		return 0;
> +
> +	if (i915_gem_request_completed(req))
> +		return 0;
> +
> +	timeout_remain = MAX_SCHEDULE_TIMEOUT;
> +	if (timeout) {
> +		if (WARN_ON(*timeout < 0))
> +			return -EINVAL;
> +
> +		if (*timeout == 0)
> +			return -ETIME;
> +
> +		/* Record current time in case interrupted, or wedged */
> +		timeout_remain = nsecs_to_jiffies_timeout(*timeout);
> +		*timeout += ktime_get_raw_ns();
> +	}
> +
> +	trace_i915_gem_request_wait_begin(req);
> +
> +	/* This client is about to stall waiting for the GPU. In many cases
> +	 * this is undesirable and limits the throughput of the system, as
> +	 * many clients cannot continue processing user input/output whilst
> +	 * blocked. RPS autotuning may take tens of milliseconds to respond
> +	 * to the GPU load and thus incurs additional latency for the client.
> +	 * We can circumvent that by promoting the GPU frequency to maximum
> +	 * before we wait. This makes the GPU throttle up much more quickly
> +	 * (good for benchmarks and user experience, e.g. window animations),
> +	 * but at a cost of spending more power processing the workload
> +	 * (bad for battery). Not all clients even want their results
> +	 * immediately and for them we should just let the GPU select its own
> +	 * frequency to maximise efficiency. To prevent a single client from
> +	 * forcing the clocks too high for the whole system, we only allow
> +	 * each client to waitboost once in a busy period.
> +	 */
> +	if (INTEL_INFO(req->i915)->gen >= 6)
> +		gen6_rps_boost(req->i915, rps, req->emitted_jiffies);
> +
> +	intel_wait_init(&wait, req->seqno);
> +	set_task_state(wait.task, state);
> +
> +	/* Optimistic spin for the next ~jiffie before touching IRQs */
> +	if (intel_engine_add_wait(req->ring, &wait)) {
> +		if (__i915_spin_request(req, &wait, state))
> +			goto complete;
> +
> +		/* In order to check that we haven't missed the interrupt
> +		 * as we enabled it, we need to kick ourselves to do a
> +		 * coherent check on the seqno before we sleep.
> +		 */
> +		if (intel_engine_enable_wait_irq(req->ring, &wait))
> +			goto wakeup;
> +	}
> +
> +	for (;;) {
> +		if (signal_pending_state(state, wait.task)) {
> +			ret = -ERESTARTSYS;
> +			break;
> +		}
> +
> +		/* Ensure that even if the GPU hangs, we get woken up. */
> +		i915_queue_hangcheck(req->i915);
> +
> +		timeout_remain = io_schedule_timeout(timeout_remain);
> +		if (timeout_remain == 0) {
> +			ret = -ETIME;
> +			break;
> +		}
> +
> +		if (intel_wait_complete(&wait))
> +			break;
> +
> +wakeup:
> +		set_task_state(wait.task, state);
> +
> +		/* Carefully check if the request is complete, giving time
> +		 * for the seqno to be visible following the interrupt.
> +		 * We also have to check in case we are kicked by the GPU
> +		 * reset in order to drop the struct_mutex.
> +		 */
> +		if (__i915_request_irq_complete(req))
> +			break;
> +	}
> +
> +complete:
> +	intel_engine_remove_wait(req->ring, &wait);
> +	__set_task_state(wait.task, TASK_RUNNING);
> +	trace_i915_gem_request_wait_end(req);
> +
> +	if (timeout) {
> +		*timeout -= ktime_get_raw_ns();
> +		if (*timeout < 0)
> +			*timeout = 0;
> +
> +		/*
> +		 * Apparently ktime isn't accurate enough and occasionally has a
> +		 * bit of mismatch in the jiffies<->nsecs<->ktime loop. So patch
> +		 * things up to make the test happy. We allow up to 1 jiffy.
> +		 *
> +		 * This is a regrssion from the timespec->ktime conversion.
> +		 */
> +		if (ret == -ETIME && *timeout < jiffies_to_usecs(1)*1000)
> +			*timeout = 0;
> +	}
> +
> +	if (ret == 0 && rps && req->seqno == req->ring->last_submitted_seqno) {
> +		/* The GPU is now idle and this client has stalled.
> +		 * Since no other client has submitted a request in the
> +		 * meantime, assume that this client is the only one
> +		 * supplying work to the GPU but is unable to keep that
> +		 * work supplied because it is waiting. Since the GPU is
> +		 * then never kept fully busy, RPS autoclocking will
> +		 * keep the clocks relatively low, causing further delays.
> +		 * Compensate by giving the synchronous client credit for
> +		 * a waitboost next time.
> +		 */
> +		spin_lock(&req->i915->rps.client_lock);
> +		list_del_init(&rps->link);
> +		spin_unlock(&req->i915->rps.client_lock);
> +	}
> +
> +	return ret;
> +}
> +
> +/**
> + * Waits for a request to be signaled, and cleans up the
> + * request and object lists appropriately for that event.
> + */
> +int
> +i915_wait_request(struct drm_i915_gem_request *req)
> +{
> +	struct drm_device *dev;
> +	struct drm_i915_private *dev_priv;
> +	bool interruptible;
> +	int ret;
> +
> +	BUG_ON(req == NULL);
> +
> +	dev = req->ring->dev;
> +	dev_priv = dev->dev_private;
> +	interruptible = dev_priv->mm.interruptible;
> +
> +	BUG_ON(!mutex_is_locked(&dev->struct_mutex));
> +
> +	ret = __i915_wait_request(req, interruptible, NULL, NULL);
> +	if (ret)
> +		return ret;
> +
> +	i915_gem_request_retire_upto(req);
> +	return 0;
> +}
> +
> +void i915_gem_request_free(struct kref *req_ref)
> +{
> +	struct drm_i915_gem_request *req = container_of(req_ref,
> +						 typeof(*req), ref);
> +	struct intel_context *ctx = req->ctx;
> +
> +	if (req->file_priv)
> +		i915_gem_request_remove_from_client(req);
> +
> +	if (ctx) {
> +		if (i915.enable_execlists) {
> +			if (ctx != req->ring->default_context)
> +				intel_lr_context_unpin(req);
> +		}
> +
> +		i915_gem_context_unreference(ctx);
> +	}
> +
> +	kmem_cache_free(req->i915->requests, req);
> +}
> diff --git a/drivers/gpu/drm/i915/i915_gem_request.h b/drivers/gpu/drm/i915/i915_gem_request.h
> new file mode 100644
> index 000000000000..d46f22f30b0a
> --- /dev/null
> +++ b/drivers/gpu/drm/i915/i915_gem_request.h
> @@ -0,0 +1,223 @@
> +/*
> + * Copyright © 2008-2015 Intel Corporation
> + *
> + * Permission is hereby granted, free of charge, to any person obtaining a
> + * copy of this software and associated documentation files (the "Software"),
> + * to deal in the Software without restriction, including without limitation
> + * the rights to use, copy, modify, merge, publish, distribute, sublicense,
> + * and/or sell copies of the Software, and to permit persons to whom the
> + * Software is furnished to do so, subject to the following conditions:
> + *
> + * The above copyright notice and this permission notice (including the next
> + * paragraph) shall be included in all copies or substantial portions of the
> + * Software.
> + *
> + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
> + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
> + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
> + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
> + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
> + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
> + * IN THE SOFTWARE.
> + *
> + */
> +
> +#ifndef I915_GEM_REQUEST_H
> +#define I915_GEM_REQUEST_H
> +
> +/**
> + * Request queue structure.
> + *
> + * The request queue allows us to note sequence numbers that have been emitted
> + * and may be associated with active buffers to be retired.
> + *
> + * By keeping this list, we can avoid having to do questionable sequence
> + * number comparisons on buffer last_read|write_seqno. It also allows an
> + * emission time to be associated with the request for tracking how far ahead
> + * of the GPU the submission is.
> + *
> + * The requests are reference counted, so upon creation they should have an
> + * initial reference taken using kref_init
> + */
> +struct drm_i915_gem_request {
> +	struct kref ref;
> +
> +	/** On Which ring this request was generated */
> +	struct drm_i915_private *i915;
> +	struct intel_engine_cs *ring;
> +	unsigned reset_counter;
> +
> +	 /** GEM sequence number associated with the previous request,
> +	  * when the HWS breadcrumb is equal to this the GPU is processing
> +	  * this request.
> +	  */
> +	u32 previous_seqno;
> +
> +	 /** GEM sequence number associated with this request,
> +	  * when the HWS breadcrumb is equal or greater than this the GPU
> +	  * has finished processing this request.
> +	  */
> +	u32 seqno;
> +
> +	/** Position in the ringbuffer of the start of the request */
> +	u32 head;
> +
> +	/**
> +	 * Position in the ringbuffer of the start of the postfix.
> +	 * This is required to calculate the maximum available ringbuffer
> +	 * space without overwriting the postfix.
> +	 */
> +	 u32 postfix;
> +
> +	/** Position in the ringbuffer of the end of the whole request */
> +	u32 tail;
> +
> +	/**
> +	 * Context and ring buffer related to this request
> +	 * Contexts are refcounted, so when this request is associated with a
> +	 * context, we must increment the context's refcount, to guarantee that
> +	 * it persists while any request is linked to it. Requests themselves
> +	 * are also refcounted, so the request will only be freed when the last
> +	 * reference to it is dismissed, and the code in
> +	 * i915_gem_request_free() will then decrement the refcount on the
> +	 * context.
> +	 */
> +	struct intel_context *ctx;
> +	struct intel_ringbuffer *ringbuf;
> +
> +	/** Batch buffer related to this request if any (used for
> +	    error state dump only) */
> +	struct drm_i915_gem_object *batch_obj;
> +
> +	/** Time at which this request was emitted, in jiffies. */
> +	unsigned long emitted_jiffies;
> +
> +	/** global list entry for this request */
> +	struct list_head list;
> +
> +	struct drm_i915_file_private *file_priv;
> +	/** file_priv list entry for this request */
> +	struct list_head client_list;
> +
> +	/** process identifier submitting this request */
> +	struct pid *pid;
> +
> +	/**
> +	 * The ELSP only accepts two elements at a time, so we queue
> +	 * context/tail pairs on a given queue (ring->execlist_queue) until the
> +	 * hardware is available. The queue serves a double purpose: we also use
> +	 * it to keep track of the up to 2 contexts currently in the hardware
> +	 * (usually one in execution and the other queued up by the GPU): We
> +	 * only remove elements from the head of the queue when the hardware
> +	 * informs us that an element has been completed.
> +	 *
> +	 * All accesses to the queue are mediated by a spinlock
> +	 * (ring->execlist_lock).
> +	 */
> +
> +	/** Execlist link in the submission queue.*/
> +	struct list_head execlist_link;
> +
> +	/** Execlists no. of times this request has been sent to the ELSP */
> +	int elsp_submitted;
> +};
> +
> +int i915_gem_request_alloc(struct intel_engine_cs *ring,
> +			   struct intel_context *ctx,
> +			   struct drm_i915_gem_request **req_out);
> +void i915_gem_request_cancel(struct drm_i915_gem_request *req);
> +void i915_gem_request_free(struct kref *req_ref);
> +int i915_gem_request_add_to_client(struct drm_i915_gem_request *req,
> +				   struct drm_file *file);
> +void i915_gem_request_retire_upto(struct drm_i915_gem_request *req);
> +
> +static inline uint32_t
> +i915_gem_request_get_seqno(struct drm_i915_gem_request *req)
> +{
> +	return req ? req->seqno : 0;
> +}
> +
> +static inline struct intel_engine_cs *
> +i915_gem_request_get_ring(struct drm_i915_gem_request *req)
> +{
> +	return req ? req->ring : NULL;
> +}
> +
> +static inline struct drm_i915_gem_request *
> +i915_gem_request_reference(struct drm_i915_gem_request *req)
> +{
> +	if (req)
> +		kref_get(&req->ref);
> +	return req;
> +}
> +
> +static inline void
> +i915_gem_request_unreference(struct drm_i915_gem_request *req)
> +{
> +	WARN_ON(!mutex_is_locked(&req->ring->dev->struct_mutex));
> +	kref_put(&req->ref, i915_gem_request_free);
> +}
> +
> +static inline void
> +i915_gem_request_unreference__unlocked(struct drm_i915_gem_request *req)
> +{
> +	struct drm_device *dev;
> +
> +	if (!req)
> +		return;
> +
> +	dev = req->ring->dev;
> +	if (kref_put_mutex(&req->ref, i915_gem_request_free, &dev->struct_mutex))
> +		mutex_unlock(&dev->struct_mutex);
> +}
> +
> +static inline void i915_gem_request_assign(struct drm_i915_gem_request **pdst,
> +					   struct drm_i915_gem_request *src)
> +{
> +	if (src)
> +		i915_gem_request_reference(src);
> +
> +	if (*pdst)
> +		i915_gem_request_unreference(*pdst);
> +
> +	*pdst = src;
> +}
> +
> +void __i915_add_request(struct drm_i915_gem_request *req,
> +			struct drm_i915_gem_object *batch_obj,
> +			bool flush_caches);
> +#define i915_add_request(req) \
> +	__i915_add_request(req, NULL, true)
> +#define i915_add_request_no_flush(req) \
> +	__i915_add_request(req, NULL, false)
> +
> +struct intel_rps_client;
> +
> +int __i915_wait_request(struct drm_i915_gem_request *req,
> +			bool interruptible,
> +			s64 *timeout,
> +			struct intel_rps_client *rps);
> +int __must_check i915_wait_request(struct drm_i915_gem_request *req);
> +
> +/**
> + * Returns true if seq1 is later than seq2.
> + */
> +static inline bool
> +i915_seqno_passed(uint32_t seq1, uint32_t seq2)
> +{
> +	return (int32_t)(seq1 - seq2) >= 0;
> +}
> +
> +static inline bool i915_gem_request_started(struct drm_i915_gem_request *req)
> +{
> +	return i915_seqno_passed(intel_ring_get_seqno(req->ring),
> +				 req->previous_seqno);
> +}
> +
> +static inline bool i915_gem_request_completed(struct drm_i915_gem_request *req)
> +{
> +	return i915_seqno_passed(intel_ring_get_seqno(req->ring),
> +				 req->seqno);
> +}
> +
> +#endif /* I915_GEM_REQUEST_H */
>

_______________________________________________
Intel-gfx mailing list
Intel-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/intel-gfx