From: Andi Shyti <andi.shyti@intel.com>
To: Intel GFX <intel-gfx@lists.freedesktop.org>
Subject: Re: [PATCH v2 1/2] drm/i915: Extract GT render power state management
Date: Wed, 23 Oct 2019 12:52:48 +0300 [thread overview]
Message-ID: <20191023095248.GA2593@intel.intel> (raw)
In-Reply-To: <20191023095000.3782-1-andi.shyti@intel.com>
Hi,
Sorry, I messed up with the e-mail (and with the in-reply-to, as
well, and I forgot to add a note to this v2), please take this
as 1/2 because it has the correct e-mail.
Andi
On Wed, Oct 23, 2019 at 12:50:00PM +0300, Andi Shyti wrote:
> i915_irq.c is large. One reason for this is that has a large chunk of
> the GT render power management stashed away in it. Extract that logic
> out of i915_irq.c and intel_pm.c and put it under one roof.
>
> Based on a patch by Chris Wilson.
>
> Signed-off-by: Andi Shyti <andi.shyti@intel.com>
> Cc: Chris Wilson <chris@chris-wilson.co.uk>
> ---
> drivers/gpu/drm/i915/Makefile | 3 +-
> drivers/gpu/drm/i915/display/intel_display.c | 8 +-
> drivers/gpu/drm/i915/gt/intel_gt.c | 6 +-
> drivers/gpu/drm/i915/gt/intel_gt_irq.c | 5 +-
> drivers/gpu/drm/i915/gt/intel_gt_pm.c | 49 +-
> drivers/gpu/drm/i915/gt/intel_gt_pm.h | 1 +
> drivers/gpu/drm/i915/gt/intel_gt_types.h | 2 +
> drivers/gpu/drm/i915/gt/intel_llc.c | 2 +-
> drivers/gpu/drm/i915/gt/intel_rps.c | 1872 +++++++++++++++
> drivers/gpu/drm/i915/gt/intel_rps.h | 37 +
> drivers/gpu/drm/i915/gt/intel_rps_types.h | 93 +
> drivers/gpu/drm/i915/gt/selftest_llc.c | 7 +-
> .../gpu/drm/i915/gt/uc/intel_guc_submission.c | 4 +-
> drivers/gpu/drm/i915/i915_debugfs.c | 73 +-
> drivers/gpu/drm/i915/i915_drv.c | 2 +
> drivers/gpu/drm/i915/i915_drv.h | 96 -
> drivers/gpu/drm/i915/i915_gem.c | 3 +-
> drivers/gpu/drm/i915/i915_irq.c | 359 +--
> drivers/gpu/drm/i915/i915_irq.h | 3 -
> drivers/gpu/drm/i915/i915_pmu.c | 10 +-
> drivers/gpu/drm/i915/i915_request.c | 7 +-
> drivers/gpu/drm/i915/i915_sysfs.c | 74 +-
> drivers/gpu/drm/i915/intel_pm.c | 2105 ++---------------
> drivers/gpu/drm/i915/intel_pm.h | 22 -
> 24 files changed, 2359 insertions(+), 2484 deletions(-)
> create mode 100644 drivers/gpu/drm/i915/gt/intel_rps.c
> create mode 100644 drivers/gpu/drm/i915/gt/intel_rps.h
> create mode 100644 drivers/gpu/drm/i915/gt/intel_rps_types.h
>
> diff --git a/drivers/gpu/drm/i915/Makefile b/drivers/gpu/drm/i915/Makefile
> index 2fd4bed188e5..133ca59e7c48 100644
> --- a/drivers/gpu/drm/i915/Makefile
> +++ b/drivers/gpu/drm/i915/Makefile
> @@ -90,11 +90,12 @@ gt-y += \
> gt/intel_hangcheck.o \
> gt/intel_llc.o \
> gt/intel_lrc.o \
> + gt/intel_mocs.o \
> gt/intel_rc6.o \
> gt/intel_renderstate.o \
> gt/intel_reset.o \
> gt/intel_ringbuffer.o \
> - gt/intel_mocs.o \
> + gt/intel_rps.o \
> gt/intel_sseu.o \
> gt/intel_timeline.o \
> gt/intel_workarounds.o
> diff --git a/drivers/gpu/drm/i915/display/intel_display.c b/drivers/gpu/drm/i915/display/intel_display.c
> index 236fdf122e47..49c2c7f71394 100644
> --- a/drivers/gpu/drm/i915/display/intel_display.c
> +++ b/drivers/gpu/drm/i915/display/intel_display.c
> @@ -55,6 +55,8 @@
> #include "display/intel_tv.h"
> #include "display/intel_vdsc.h"
>
> +#include "gt/intel_rps.h"
> +
> #include "i915_drv.h"
> #include "i915_trace.h"
> #include "intel_acpi.h"
> @@ -14782,7 +14784,7 @@ static int do_rps_boost(struct wait_queue_entry *_wait,
> * vblank without our intervention, so leave RPS alone.
> */
> if (!i915_request_started(rq))
> - gen6_rps_boost(rq);
> + intel_rps_boost(rq);
> i915_request_put(rq);
>
> drm_crtc_vblank_put(wait->crtc);
> @@ -14976,7 +14978,7 @@ intel_prepare_plane_fb(struct drm_plane *plane,
> * maximum clocks following a vblank miss (see do_rps_boost()).
> */
> if (!intel_state->rps_interactive) {
> - intel_rps_mark_interactive(dev_priv, true);
> + intel_rps_mark_interactive(&dev_priv->gt.rps, true);
> intel_state->rps_interactive = true;
> }
>
> @@ -15001,7 +15003,7 @@ intel_cleanup_plane_fb(struct drm_plane *plane,
> struct drm_i915_private *dev_priv = to_i915(plane->dev);
>
> if (intel_state->rps_interactive) {
> - intel_rps_mark_interactive(dev_priv, false);
> + intel_rps_mark_interactive(&dev_priv->gt.rps, false);
> intel_state->rps_interactive = false;
> }
>
> diff --git a/drivers/gpu/drm/i915/gt/intel_gt.c b/drivers/gpu/drm/i915/gt/intel_gt.c
> index 1c4b6c9642ad..7518852cb78a 100644
> --- a/drivers/gpu/drm/i915/gt/intel_gt.c
> +++ b/drivers/gpu/drm/i915/gt/intel_gt.c
> @@ -9,6 +9,7 @@
> #include "intel_gt_requests.h"
> #include "intel_mocs.h"
> #include "intel_rc6.h"
> +#include "intel_rps.h"
> #include "intel_uncore.h"
> #include "intel_pm.h"
>
> @@ -321,8 +322,7 @@ void intel_gt_chipset_flush(struct intel_gt *gt)
>
> void intel_gt_driver_register(struct intel_gt *gt)
> {
> - if (IS_GEN(gt->i915, 5))
> - intel_gpu_ips_init(gt->i915);
> + intel_rps_driver_register(>->rps);
> }
>
> static int intel_gt_init_scratch(struct intel_gt *gt, unsigned int size)
> @@ -385,7 +385,7 @@ void intel_gt_driver_remove(struct intel_gt *gt)
>
> void intel_gt_driver_unregister(struct intel_gt *gt)
> {
> - intel_gpu_ips_teardown();
> + intel_rps_driver_unregister(>->rps);
> }
>
> void intel_gt_driver_release(struct intel_gt *gt)
> diff --git a/drivers/gpu/drm/i915/gt/intel_gt_irq.c b/drivers/gpu/drm/i915/gt/intel_gt_irq.c
> index 34a4fb624bf7..973ee7eded64 100644
> --- a/drivers/gpu/drm/i915/gt/intel_gt_irq.c
> +++ b/drivers/gpu/drm/i915/gt/intel_gt_irq.c
> @@ -11,6 +11,7 @@
> #include "intel_gt.h"
> #include "intel_gt_irq.h"
> #include "intel_uncore.h"
> +#include "intel_rps.h"
>
> static void guc_irq_handler(struct intel_guc *guc, u16 iir)
> {
> @@ -77,7 +78,7 @@ gen11_other_irq_handler(struct intel_gt *gt, const u8 instance,
> return guc_irq_handler(>->uc.guc, iir);
>
> if (instance == OTHER_GTPM_INSTANCE)
> - return gen11_rps_irq_handler(gt, iir);
> + return gen11_rps_irq_handler(>->rps, iir);
>
> WARN_ONCE(1, "unhandled other interrupt instance=0x%x, iir=0x%x\n",
> instance, iir);
> @@ -336,7 +337,7 @@ void gen8_gt_irq_handler(struct intel_gt *gt, u32 master_ctl, u32 gt_iir[4])
> }
>
> if (master_ctl & (GEN8_GT_PM_IRQ | GEN8_GT_GUC_IRQ)) {
> - gen6_rps_irq_handler(gt->i915, gt_iir[2]);
> + gen6_rps_irq_handler(>->rps, gt_iir[2]);
> guc_irq_handler(>->uc.guc, gt_iir[2] >> 16);
> }
> }
> diff --git a/drivers/gpu/drm/i915/gt/intel_gt_pm.c b/drivers/gpu/drm/i915/gt/intel_gt_pm.c
> index 06e73d56cfcf..32ca87265222 100644
> --- a/drivers/gpu/drm/i915/gt/intel_gt_pm.c
> +++ b/drivers/gpu/drm/i915/gt/intel_gt_pm.c
> @@ -12,8 +12,10 @@
> #include "intel_gt.h"
> #include "intel_gt_pm.h"
> #include "intel_gt_requests.h"
> +#include "intel_llc.h"
> #include "intel_pm.h"
> #include "intel_rc6.h"
> +#include "intel_rps.h"
> #include "intel_wakeref.h"
>
> static int __gt_unpark(struct intel_wakeref *wf)
> @@ -39,12 +41,8 @@ static int __gt_unpark(struct intel_wakeref *wf)
> gt->awake = intel_display_power_get(i915, POWER_DOMAIN_GT_IRQ);
> GEM_BUG_ON(!gt->awake);
>
> - intel_enable_gt_powersave(i915);
> -
> - i915_update_gfx_val(i915);
> - if (INTEL_GEN(i915) >= 6)
> - gen6_rps_busy(i915);
> -
> + intel_rps_unpark(>->rps);
> + intel_llc_enable(>->llc);
> i915_pmu_gt_unparked(i915);
>
> intel_gt_queue_hangcheck(gt);
> @@ -65,8 +63,8 @@ static int __gt_park(struct intel_wakeref *wf)
>
> i915_vma_parked(gt);
> i915_pmu_gt_parked(i915);
> - if (INTEL_GEN(i915) >= 6)
> - gen6_rps_idle(i915);
> + intel_llc_disable(>->llc);
> + intel_rps_park(>->rps);
>
> /* Everything switched off, flush any residual interrupt just in case */
> intel_synchronize_irq(i915);
> @@ -98,6 +96,7 @@ void intel_gt_pm_init(struct intel_gt *gt)
> * user.
> */
> intel_rc6_init(>->rc6);
> + intel_rps_init(>->rps);
> }
>
> static bool reset_engines(struct intel_gt *gt)
> @@ -141,10 +140,39 @@ void intel_gt_sanitize(struct intel_gt *gt, bool force)
> engine->reset.finish(engine);
> }
>
> +void intel_gt_pm_enable(struct intel_gt *gt)
> +{
> + struct intel_engine_cs *engine;
> + enum intel_engine_id id;
> +
> + /* Powersaving is controlled by the host when inside a VM */
> + if (intel_vgpu_active(gt->i915))
> + return;
> +
> + if (is_mock_gt(gt))
> + return;
> +
> + intel_gt_pm_get(gt);
> + intel_rps_enable(>->rps);
> + intel_llc_enable(>->llc);
> +
> + for_each_engine(engine, gt->i915, id) {
> + intel_engine_pm_get(engine);
> + engine->serial++; /* force kernel context reload */
> + intel_engine_pm_put(engine);
> + }
> +
> + intel_gt_pm_put(gt);
> +}
> +
> void intel_gt_pm_disable(struct intel_gt *gt)
> {
> - if (!is_mock_gt(gt))
> - intel_sanitize_gt_powersave(gt->i915);
> + if (is_mock_gt(gt))
> + return;
> +
> + intel_rc6_disable(>->rc6);
> + intel_llc_disable(>->llc);
> + intel_rps_disable(>->rps);
> }
>
> void intel_gt_pm_fini(struct intel_gt *gt)
> @@ -165,6 +193,7 @@ int intel_gt_resume(struct intel_gt *gt)
> * allowing us to fixup the user contexts on their first pin.
> */
> intel_gt_pm_get(gt);
> +
> intel_uncore_forcewake_get(gt->uncore, FORCEWAKE_ALL);
> intel_rc6_sanitize(>->rc6);
>
> diff --git a/drivers/gpu/drm/i915/gt/intel_gt_pm.h b/drivers/gpu/drm/i915/gt/intel_gt_pm.h
> index 0ed87da4bb68..df50f853748d 100644
> --- a/drivers/gpu/drm/i915/gt/intel_gt_pm.h
> +++ b/drivers/gpu/drm/i915/gt/intel_gt_pm.h
> @@ -40,6 +40,7 @@ static inline int intel_gt_pm_wait_for_idle(struct intel_gt *gt)
> void intel_gt_pm_init_early(struct intel_gt *gt);
> void intel_gt_pm_init(struct intel_gt *gt);
> void intel_gt_pm_disable(struct intel_gt *gt);
> +void intel_gt_pm_enable(struct intel_gt *gt);
> void intel_gt_pm_fini(struct intel_gt *gt);
>
> void intel_gt_sanitize(struct intel_gt *gt, bool force);
> diff --git a/drivers/gpu/drm/i915/gt/intel_gt_types.h b/drivers/gpu/drm/i915/gt/intel_gt_types.h
> index 980973e66e7f..e4bd23eb8290 100644
> --- a/drivers/gpu/drm/i915/gt/intel_gt_types.h
> +++ b/drivers/gpu/drm/i915/gt/intel_gt_types.h
> @@ -20,6 +20,7 @@
> #include "intel_llc_types.h"
> #include "intel_reset_types.h"
> #include "intel_rc6_types.h"
> +#include "intel_rps_types.h"
> #include "intel_wakeref.h"
>
> struct drm_i915_private;
> @@ -82,6 +83,7 @@ struct intel_gt {
>
> struct intel_llc llc;
> struct intel_rc6 rc6;
> + struct intel_rps rps;
>
> ktime_t last_init_time;
>
> diff --git a/drivers/gpu/drm/i915/gt/intel_llc.c b/drivers/gpu/drm/i915/gt/intel_llc.c
> index 35093eb5f24e..ceb785b75c25 100644
> --- a/drivers/gpu/drm/i915/gt/intel_llc.c
> +++ b/drivers/gpu/drm/i915/gt/intel_llc.c
> @@ -48,7 +48,7 @@ static bool get_ia_constants(struct intel_llc *llc,
> struct ia_constants *consts)
> {
> struct drm_i915_private *i915 = llc_to_gt(llc)->i915;
> - struct intel_rps *rps = &i915->gt_pm.rps;
> + struct intel_rps *rps = &llc_to_gt(llc)->rps;
>
> if (rps->max_freq <= rps->min_freq)
> return false;
> diff --git a/drivers/gpu/drm/i915/gt/intel_rps.c b/drivers/gpu/drm/i915/gt/intel_rps.c
> new file mode 100644
> index 000000000000..30f56c786468
> --- /dev/null
> +++ b/drivers/gpu/drm/i915/gt/intel_rps.c
> @@ -0,0 +1,1872 @@
> +/*
> + * SPDX-License-Identifier: MIT
> + *
> + * Copyright © 2019 Intel Corporation
> + */
> +
> +#include "i915_drv.h"
> +#include "intel_gt.h"
> +#include "intel_gt_irq.h"
> +#include "intel_gt_pm_irq.h"
> +#include "intel_rps.h"
> +#include "intel_sideband.h"
> +#include "../../../platform/x86/intel_ips.h"
> +
> +/*
> + * Lock protecting IPS related data structures
> + */
> +static DEFINE_SPINLOCK(mchdev_lock);
> +
> +static struct intel_gt *rps_to_gt(struct intel_rps *rps)
> +{
> + return container_of(rps, struct intel_gt, rps);
> +}
> +
> +static struct drm_i915_private *rps_to_i915(struct intel_rps *rps)
> +{
> + return rps_to_gt(rps)->i915;
> +}
> +
> +static struct intel_uncore *rps_to_uncore(struct intel_rps *rps)
> +{
> + return rps_to_gt(rps)->uncore;
> +}
> +
> +static u32 rps_pm_sanitize_mask(struct intel_rps *rps, u32 mask)
> +{
> + return mask & ~rps->pm_intrmsk_mbz;
> +}
> +
> +static u32 rps_pm_mask(struct intel_rps *rps, u8 val)
> +{
> + u32 mask = 0;
> +
> + /* We use UP_EI_EXPIRED interrupts for both up/down in manual mode */
> + if (val > rps->min_freq_softlimit)
> + mask |= (GEN6_PM_RP_UP_EI_EXPIRED |
> + GEN6_PM_RP_DOWN_THRESHOLD |
> + GEN6_PM_RP_DOWN_TIMEOUT);
> +
> + if (val < rps->max_freq_softlimit)
> + mask |= GEN6_PM_RP_UP_EI_EXPIRED | GEN6_PM_RP_UP_THRESHOLD;
> +
> + mask &= rps->pm_events;
> +
> + return rps_pm_sanitize_mask(rps, ~mask);
> +}
> +
> +static void rps_reset_ei(struct intel_rps *rps)
> +{
> + memset(&rps->ei, 0, sizeof(rps->ei));
> +}
> +
> +static void rps_enable_interrupts(struct intel_rps *rps)
> +{
> + struct intel_gt *gt = rps_to_gt(rps);
> +
> + rps_reset_ei(rps);
> +
> + if (IS_VALLEYVIEW(gt->i915))
> + /* WaGsvRC0ResidencyMethod:vlv */
> + rps->pm_events = GEN6_PM_RP_UP_EI_EXPIRED;
> + else
> + rps->pm_events = (GEN6_PM_RP_UP_THRESHOLD |
> + GEN6_PM_RP_DOWN_THRESHOLD |
> + GEN6_PM_RP_DOWN_TIMEOUT);
> +
> + spin_lock_irq(>->irq_lock);
> + gen6_gt_pm_enable_irq(gt, rps->pm_events);
> + spin_unlock_irq(>->irq_lock);
> +
> + intel_uncore_write(gt->uncore, GEN6_PMINTRMSK,
> + rps_pm_mask(rps, rps->cur_freq));
> +}
> +
> +static void gen6_rps_reset_interrupts(struct intel_rps *rps)
> +{
> + gen6_gt_pm_reset_iir(rps_to_gt(rps), GEN6_PM_RPS_EVENTS);
> +}
> +
> +static void gen11_rps_reset_interrupts(struct intel_rps *rps)
> +{
> + while (gen11_gt_reset_one_iir(rps_to_gt(rps), 0, GEN11_GTPM))
> + ;
> +}
> +
> +static void rps_reset_interrupts(struct intel_rps *rps)
> +{
> + struct intel_gt *gt = rps_to_gt(rps);
> +
> + spin_lock_irq(>->irq_lock);
> + if (INTEL_GEN(gt->i915) >= 11)
> + gen11_rps_reset_interrupts(rps);
> + else
> + gen6_rps_reset_interrupts(rps);
> +
> + rps->pm_iir = 0;
> + spin_unlock_irq(>->irq_lock);
> +}
> +
> +static void rps_disable_interrupts(struct intel_rps *rps)
> +{
> + struct intel_gt *gt = rps_to_gt(rps);
> +
> + rps->pm_events = 0;
> +
> + intel_uncore_write(gt->uncore, GEN6_PMINTRMSK,
> + rps_pm_sanitize_mask(rps, ~0u));
> +
> + spin_lock_irq(>->irq_lock);
> + gen6_gt_pm_disable_irq(gt, GEN6_PM_RPS_EVENTS);
> + spin_unlock_irq(>->irq_lock);
> +
> + intel_synchronize_irq(gt->i915);
> +
> + /*
> + * Now that we will not be generating any more work, flush any
> + * outstanding tasks. As we are called on the RPS idle path,
> + * we will reset the GPU to minimum frequencies, so the current
> + * state of the worker can be discarded.
> + */
> + cancel_work_sync(&rps->work);
> +
> + rps_reset_interrupts(rps);
> +}
> +
> +static const struct cparams {
> + u16 i;
> + u16 t;
> + u16 m;
> + u16 c;
> +} cparams[] = {
> + { 1, 1333, 301, 28664 },
> + { 1, 1066, 294, 24460 },
> + { 1, 800, 294, 25192 },
> + { 0, 1333, 276, 27605 },
> + { 0, 1066, 276, 27605 },
> + { 0, 800, 231, 23784 },
> +};
> +
> +static void gen5_rps_init(struct intel_rps *rps)
> +{
> + struct drm_i915_private *i915 = rps_to_i915(rps);
> + struct intel_uncore *uncore = rps_to_uncore(rps);
> + u8 fmax, fmin, fstart;
> + u32 rgvmodectl;
> + int c_m, i;
> +
> + if (i915->fsb_freq <= 3200)
> + c_m = 0;
> + else if (i915->fsb_freq <= 4800)
> + c_m = 1;
> + else
> + c_m = 2;
> +
> + for (i = 0; i < ARRAY_SIZE(cparams); i++) {
> + if (cparams[i].i == c_m && cparams[i].t == i915->mem_freq) {
> + rps->ips.m = cparams[i].m;
> + rps->ips.c = cparams[i].c;
> + break;
> + }
> + }
> +
> + rgvmodectl = intel_uncore_read(uncore, MEMMODECTL);
> +
> + /* Set up min, max, and cur for interrupt handling */
> + fmax = (rgvmodectl & MEMMODE_FMAX_MASK) >> MEMMODE_FMAX_SHIFT;
> + fmin = (rgvmodectl & MEMMODE_FMIN_MASK);
> + fstart = (rgvmodectl & MEMMODE_FSTART_MASK) >>
> + MEMMODE_FSTART_SHIFT;
> + DRM_DEBUG_DRIVER("fmax: %d, fmin: %d, fstart: %d\n",
> + fmax, fmin, fstart);
> +
> + rps->min_freq = -fstart;
> + rps->max_freq = -fmin;
> +
> + rps->idle_freq = rps->min_freq;
> + rps->cur_freq = rps->idle_freq;
> +}
> +
> +static unsigned long
> +__ips_chipset_val(struct intel_ips *ips)
> +{
> + struct intel_uncore *uncore =
> + rps_to_uncore(container_of(ips, struct intel_rps, ips));
> + unsigned long now = jiffies_to_msecs(jiffies), dt;
> + unsigned long result;
> + u64 total, delta;
> +
> + lockdep_assert_held(&mchdev_lock);
> +
> + /*
> + * Prevent division-by-zero if we are asking too fast.
> + * Also, we don't get interesting results if we are polling
> + * faster than once in 10ms, so just return the saved value
> + * in such cases.
> + */
> + dt = now - ips->last_time1;
> + if (dt <= 10)
> + return ips->chipset_power;
> +
> + /* FIXME: handle per-counter overflow */
> + total = intel_uncore_read(uncore, DMIEC);
> + total += intel_uncore_read(uncore, DDREC);
> + total += intel_uncore_read(uncore, CSIEC);
> +
> + delta = total - ips->last_count1;
> +
> + result = div_u64(div_u64(ips->m * delta, dt) + ips->c, 10);
> +
> + ips->last_count1 = total;
> + ips->last_time1 = now;
> +
> + ips->chipset_power = result;
> +
> + return result;
> +}
> +
> +static unsigned long ips_mch_val(struct intel_uncore *uncore)
> +{
> + unsigned int m, x, b;
> + u32 tsfs;
> +
> + tsfs = intel_uncore_read(uncore, TSFS);
> + x = intel_uncore_read8(uncore, TR1);
> +
> + b = tsfs & TSFS_INTR_MASK;
> + m = (tsfs & TSFS_SLOPE_MASK) >> TSFS_SLOPE_SHIFT;
> +
> + return m * x / 127 - b;
> +}
> +
> +static int _pxvid_to_vd(u8 pxvid)
> +{
> + if (pxvid == 0)
> + return 0;
> +
> + if (pxvid >= 8 && pxvid < 31)
> + pxvid = 31;
> +
> + return (pxvid + 2) * 125;
> +}
> +
> +static u32 pvid_to_extvid(struct drm_i915_private *i915, u8 pxvid)
> +{
> + const int vd = _pxvid_to_vd(pxvid);
> +
> + if (INTEL_INFO(i915)->is_mobile)
> + return max(vd - 1125, 0);
> +
> + return vd;
> +}
> +
> +static void __gen5_ips_update(struct intel_ips *ips)
> +{
> + struct intel_uncore *uncore =
> + rps_to_uncore(container_of(ips, struct intel_rps, ips));
> + u64 now, delta, dt;
> + u32 count;
> +
> + lockdep_assert_held(&mchdev_lock);
> +
> + now = ktime_get_raw_ns();
> + dt = now - ips->last_time2;
> + do_div(dt, NSEC_PER_MSEC);
> +
> + /* Don't divide by 0 */
> + if (dt <= 10)
> + return;
> +
> + count = intel_uncore_read(uncore, GFXEC);
> + delta = count - ips->last_count2;
> +
> + ips->last_count2 = count;
> + ips->last_time2 = now;
> +
> + /* More magic constants... */
> + ips->gfx_power = div_u64(delta * 1181, dt * 10);
> +}
> +
> +static void gen5_rps_update(struct intel_rps *rps)
> +{
> + spin_lock_irq(&mchdev_lock);
> + __gen5_ips_update(&rps->ips);
> + spin_unlock_irq(&mchdev_lock);
> +}
> +
> +static bool gen5_rps_set(struct intel_rps *rps, u8 val)
> +{
> + struct intel_uncore *uncore = rps_to_uncore(rps);
> + u16 rgvswctl;
> +
> + lockdep_assert_held(&mchdev_lock);
> +
> + rgvswctl = intel_uncore_read16(uncore, MEMSWCTL);
> + if (rgvswctl & MEMCTL_CMD_STS) {
> + DRM_DEBUG("gpu busy, RCS change rejected\n");
> + return false; /* still busy with another command */
> + }
> +
> + val = -val;
> +
> + rgvswctl =
> + (MEMCTL_CMD_CHFREQ << MEMCTL_CMD_SHIFT) |
> + (val << MEMCTL_FREQ_SHIFT) |
> + MEMCTL_SFCAVM;
> + intel_uncore_write16(uncore, MEMSWCTL, rgvswctl);
> + intel_uncore_posting_read16(uncore, MEMSWCTL);
> +
> + rgvswctl |= MEMCTL_CMD_STS;
> + intel_uncore_write16(uncore, MEMSWCTL, rgvswctl);
> +
> + return true;
> +}
> +
> +static unsigned long intel_pxfreq(u32 vidfreq)
> +{
> + int div = (vidfreq & 0x3f0000) >> 16;
> + int post = (vidfreq & 0x3000) >> 12;
> + int pre = (vidfreq & 0x7);
> +
> + if (!pre)
> + return 0;
> +
> + return div * 133333 / (pre << post);
> +}
> +
> +static unsigned int init_emon(struct intel_uncore *uncore)
> +{
> + u8 pxw[16];
> + int i;
> +
> + /* Disable to program */
> + intel_uncore_write(uncore, ECR, 0);
> + intel_uncore_posting_read(uncore, ECR);
> +
> + /* Program energy weights for various events */
> + intel_uncore_write(uncore, SDEW, 0x15040d00);
> + intel_uncore_write(uncore, CSIEW0, 0x007f0000);
> + intel_uncore_write(uncore, CSIEW1, 0x1e220004);
> + intel_uncore_write(uncore, CSIEW2, 0x04000004);
> +
> + for (i = 0; i < 5; i++)
> + intel_uncore_write(uncore, PEW(i), 0);
> + for (i = 0; i < 3; i++)
> + intel_uncore_write(uncore, DEW(i), 0);
> +
> + /* Program P-state weights to account for frequency power adjustment */
> + for (i = 0; i < 16; i++) {
> + u32 pxvidfreq = intel_uncore_read(uncore, PXVFREQ(i));
> + unsigned int freq = intel_pxfreq(pxvidfreq);
> + unsigned int vid =
> + (pxvidfreq & PXVFREQ_PX_MASK) >> PXVFREQ_PX_SHIFT;
> + unsigned int val;
> +
> + val = vid * vid * freq / 1000 * 255;
> + val /= 127 * 127 * 900;
> +
> + pxw[i] = val;
> + }
> + /* Render standby states get 0 weight */
> + pxw[14] = 0;
> + pxw[15] = 0;
> +
> + for (i = 0; i < 4; i++) {
> + intel_uncore_write(uncore, PXW(i),
> + pxw[i * 4 + 0] << 24 |
> + pxw[i * 4 + 1] << 16 |
> + pxw[i * 4 + 2] << 8 |
> + pxw[i * 4 + 3] << 0);
> + }
> +
> + /* Adjust magic regs to magic values (more experimental results) */
> + intel_uncore_write(uncore, OGW0, 0);
> + intel_uncore_write(uncore, OGW1, 0);
> + intel_uncore_write(uncore, EG0, 0x00007f00);
> + intel_uncore_write(uncore, EG1, 0x0000000e);
> + intel_uncore_write(uncore, EG2, 0x000e0000);
> + intel_uncore_write(uncore, EG3, 0x68000300);
> + intel_uncore_write(uncore, EG4, 0x42000000);
> + intel_uncore_write(uncore, EG5, 0x00140031);
> + intel_uncore_write(uncore, EG6, 0);
> + intel_uncore_write(uncore, EG7, 0);
> +
> + for (i = 0; i < 8; i++)
> + intel_uncore_write(uncore, PXWL(i), 0);
> +
> + /* Enable PMON + select events */
> + intel_uncore_write(uncore, ECR, 0x80000019);
> +
> + return intel_uncore_read(uncore, LCFUSE02) & LCFUSE_HIV_MASK;
> +}
> +
> +static bool gen5_rps_enable(struct intel_rps *rps)
> +{
> + struct intel_uncore *uncore = rps_to_uncore(rps);
> + u8 fstart, vstart;
> + u32 rgvmodectl;
> +
> + spin_lock_irq(&mchdev_lock);
> +
> + rgvmodectl = intel_uncore_read(uncore, MEMMODECTL);
> +
> + /* Enable temp reporting */
> + intel_uncore_write16(uncore, PMMISC,
> + intel_uncore_read16(uncore, PMMISC) | MCPPCE_EN);
> + intel_uncore_write16(uncore, TSC1,
> + intel_uncore_read16(uncore, TSC1) | TSE);
> +
> + /* 100ms RC evaluation intervals */
> + intel_uncore_write(uncore, RCUPEI, 100000);
> + intel_uncore_write(uncore, RCDNEI, 100000);
> +
> + /* Set max/min thresholds to 90ms and 80ms respectively */
> + intel_uncore_write(uncore, RCBMAXAVG, 90000);
> + intel_uncore_write(uncore, RCBMINAVG, 80000);
> +
> + intel_uncore_write(uncore, MEMIHYST, 1);
> +
> + /* Set up min, max, and cur for interrupt handling */
> + fstart = (rgvmodectl & MEMMODE_FSTART_MASK) >>
> + MEMMODE_FSTART_SHIFT;
> +
> + vstart = (intel_uncore_read(uncore, PXVFREQ(fstart)) &
> + PXVFREQ_PX_MASK) >> PXVFREQ_PX_SHIFT;
> +
> + intel_uncore_write(uncore,
> + MEMINTREN,
> + MEMINT_CX_SUPR_EN | MEMINT_EVAL_CHG_EN);
> +
> + intel_uncore_write(uncore, VIDSTART, vstart);
> + intel_uncore_posting_read(uncore, VIDSTART);
> +
> + rgvmodectl |= MEMMODE_SWMODE_EN;
> + intel_uncore_write(uncore, MEMMODECTL, rgvmodectl);
> +
> + if (wait_for_atomic((intel_uncore_read(uncore, MEMSWCTL) &
> + MEMCTL_CMD_STS) == 0, 10))
> + DRM_ERROR("stuck trying to change perf mode\n");
> + mdelay(1);
> +
> + gen5_rps_set(rps, rps->cur_freq);
> +
> + rps->ips.last_count1 = intel_uncore_read(uncore, DMIEC);
> + rps->ips.last_count1 += intel_uncore_read(uncore, DDREC);
> + rps->ips.last_count1 += intel_uncore_read(uncore, CSIEC);
> + rps->ips.last_time1 = jiffies_to_msecs(jiffies);
> +
> + rps->ips.last_count2 = intel_uncore_read(uncore, GFXEC);
> + rps->ips.last_time2 = ktime_get_raw_ns();
> +
> + spin_unlock_irq(&mchdev_lock);
> +
> + rps->ips.corr = init_emon(uncore);
> +
> + return true;
> +}
> +
> +static void gen5_rps_disable(struct intel_rps *rps)
> +{
> + struct intel_uncore *uncore = rps_to_uncore(rps);
> + u16 rgvswctl;
> +
> + spin_lock_irq(&mchdev_lock);
> +
> + rgvswctl = intel_uncore_read16(uncore, MEMSWCTL);
> +
> + /* Ack interrupts, disable EFC interrupt */
> + intel_uncore_write(uncore, MEMINTREN,
> + intel_uncore_read(uncore, MEMINTREN) &
> + ~MEMINT_EVAL_CHG_EN);
> + intel_uncore_write(uncore, MEMINTRSTS, MEMINT_EVAL_CHG);
> + intel_uncore_write(uncore, DEIER,
> + intel_uncore_read(uncore, DEIER) & ~DE_PCU_EVENT);
> + intel_uncore_write(uncore, DEIIR, DE_PCU_EVENT);
> + intel_uncore_write(uncore, DEIMR,
> + intel_uncore_read(uncore, DEIMR) | DE_PCU_EVENT);
> +
> + /* Go back to the starting frequency */
> + gen5_rps_set(rps, rps->idle_freq);
> + mdelay(1);
> + rgvswctl |= MEMCTL_CMD_STS;
> + intel_uncore_write(uncore, MEMSWCTL, rgvswctl);
> + mdelay(1);
> +
> + spin_unlock_irq(&mchdev_lock);
> +}
> +
> +static u32 rps_limits(struct intel_rps *rps, u8 val)
> +{
> + u32 limits;
> +
> + /*
> + * Only set the down limit when we've reached the lowest level to avoid
> + * getting more interrupts, otherwise leave this clear. This prevents a
> + * race in the hw when coming out of rc6: There's a tiny window where
> + * the hw runs at the minimal clock before selecting the desired
> + * frequency, if the down threshold expires in that window we will not
> + * receive a down interrupt.
> + */
> + if (INTEL_GEN(rps_to_i915(rps)) >= 9) {
> + limits = rps->max_freq_softlimit << 23;
> + if (val <= rps->min_freq_softlimit)
> + limits |= rps->min_freq_softlimit << 14;
> + } else {
> + limits = rps->max_freq_softlimit << 24;
> + if (val <= rps->min_freq_softlimit)
> + limits |= rps->min_freq_softlimit << 16;
> + }
> +
> + return limits;
> +}
> +
> +static void rps_set_power(struct intel_rps *rps, int new_power)
> +{
> + struct intel_uncore *uncore = rps_to_uncore(rps);
> + struct drm_i915_private *i915 = rps_to_i915(rps);
> + u32 threshold_up = 0, threshold_down = 0; /* in % */
> + u32 ei_up = 0, ei_down = 0;
> +
> + lockdep_assert_held(&rps->power.mutex);
> +
> + if (new_power == rps->power.mode)
> + return;
> +
> + /* Note the units here are not exactly 1us, but 1280ns. */
> + switch (new_power) {
> + case LOW_POWER:
> + /* Upclock if more than 95% busy over 16ms */
> + ei_up = 16000;
> + threshold_up = 95;
> +
> + /* Downclock if less than 85% busy over 32ms */
> + ei_down = 32000;
> + threshold_down = 85;
> + break;
> +
> + case BETWEEN:
> + /* Upclock if more than 90% busy over 13ms */
> + ei_up = 13000;
> + threshold_up = 90;
> +
> + /* Downclock if less than 75% busy over 32ms */
> + ei_down = 32000;
> + threshold_down = 75;
> + break;
> +
> + case HIGH_POWER:
> + /* Upclock if more than 85% busy over 10ms */
> + ei_up = 10000;
> + threshold_up = 85;
> +
> + /* Downclock if less than 60% busy over 32ms */
> + ei_down = 32000;
> + threshold_down = 60;
> + break;
> + }
> +
> + /* When byt can survive without system hang with dynamic
> + * sw freq adjustments, this restriction can be lifted.
> + */
> + if (IS_VALLEYVIEW(i915))
> + goto skip_hw_write;
> +
> + intel_uncore_write(uncore, GEN6_RP_UP_EI,
> + GT_INTERVAL_FROM_US(i915, ei_up));
> + intel_uncore_write(uncore, GEN6_RP_UP_THRESHOLD,
> + GT_INTERVAL_FROM_US(i915,
> + ei_up * threshold_up / 100));
> +
> + intel_uncore_write(uncore, GEN6_RP_DOWN_EI,
> + GT_INTERVAL_FROM_US(i915, ei_down));
> + intel_uncore_write(uncore, GEN6_RP_DOWN_THRESHOLD,
> + GT_INTERVAL_FROM_US(i915,
> + ei_down * threshold_down / 100));
> +
> + intel_uncore_write(uncore, GEN6_RP_CONTROL,
> + (INTEL_GEN(i915) > 9 ? 0 : GEN6_RP_MEDIA_TURBO) |
> + GEN6_RP_MEDIA_HW_NORMAL_MODE |
> + GEN6_RP_MEDIA_IS_GFX |
> + GEN6_RP_ENABLE |
> + GEN6_RP_UP_BUSY_AVG |
> + GEN6_RP_DOWN_IDLE_AVG);
> +
> +skip_hw_write:
> + rps->power.mode = new_power;
> + rps->power.up_threshold = threshold_up;
> + rps->power.down_threshold = threshold_down;
> +}
> +
> +static void gen6_rps_set_thresholds(struct intel_rps *rps, u8 val)
> +{
> + int new_power;
> +
> + new_power = rps->power.mode;
> + switch (rps->power.mode) {
> + case LOW_POWER:
> + if (val > rps->efficient_freq + 1 &&
> + val > rps->cur_freq)
> + new_power = BETWEEN;
> + break;
> +
> + case BETWEEN:
> + if (val <= rps->efficient_freq &&
> + val < rps->cur_freq)
> + new_power = LOW_POWER;
> + else if (val >= rps->rp0_freq &&
> + val > rps->cur_freq)
> + new_power = HIGH_POWER;
> + break;
> +
> + case HIGH_POWER:
> + if (val < (rps->rp1_freq + rps->rp0_freq) >> 1 &&
> + val < rps->cur_freq)
> + new_power = BETWEEN;
> + break;
> + }
> + /* Max/min bins are special */
> + if (val <= rps->min_freq_softlimit)
> + new_power = LOW_POWER;
> + if (val >= rps->max_freq_softlimit)
> + new_power = HIGH_POWER;
> +
> + mutex_lock(&rps->power.mutex);
> + if (rps->power.interactive)
> + new_power = HIGH_POWER;
> + rps_set_power(rps, new_power);
> + mutex_unlock(&rps->power.mutex);
> +}
> +
> +void intel_rps_mark_interactive(struct intel_rps *rps, bool interactive)
> +{
> + if (!rps->enabled)
> + return;
> +
> + mutex_lock(&rps->power.mutex);
> + if (interactive) {
> + if (!rps->power.interactive++ && rps->active)
> + rps_set_power(rps, HIGH_POWER);
> + } else {
> + GEM_BUG_ON(!rps->power.interactive);
> + rps->power.interactive--;
> + }
> + mutex_unlock(&rps->power.mutex);
> +}
> +
> +static int gen6_rps_set(struct intel_rps *rps, u8 val)
> +{
> + struct intel_uncore *uncore = rps_to_uncore(rps);
> + struct drm_i915_private *i915 = rps_to_i915(rps);
> + u32 swreq;
> +
> + if (INTEL_GEN(i915) >= 9)
> + swreq = GEN9_FREQUENCY(val);
> + else if (IS_HASWELL(i915) || IS_BROADWELL(i915))
> + swreq = HSW_FREQUENCY(val);
> + else
> + swreq = (GEN6_FREQUENCY(val) |
> + GEN6_OFFSET(0) |
> + GEN6_AGGRESSIVE_TURBO);
> + intel_uncore_write(uncore, GEN6_RPNSWREQ, swreq);
> +
> + return 0;
> +}
> +
> +static int vlv_rps_set(struct intel_rps *rps, u8 val)
> +{
> + struct drm_i915_private *i915 = rps_to_i915(rps);
> + int err;
> +
> + vlv_punit_get(i915);
> + err = vlv_punit_write(i915, PUNIT_REG_GPU_FREQ_REQ, val);
> + vlv_punit_put(i915);
> +
> + return err;
> +}
> +
> +static int rps_set(struct intel_rps *rps, u8 val)
> +{
> + struct drm_i915_private *i915 = rps_to_i915(rps);
> + int err;
> +
> + if (INTEL_GEN(i915) < 6)
> + return 0;
> +
> + if (val == rps->last_freq)
> + return 0;
> +
> + if (IS_VALLEYVIEW(i915) || IS_CHERRYVIEW(i915))
> + err = vlv_rps_set(rps, val);
> + else
> + err = gen6_rps_set(rps, val);
> + if (err)
> + return err;
> +
> + gen6_rps_set_thresholds(rps, val);
> + rps->last_freq = val;
> +
> + return 0;
> +}
> +
> +void intel_rps_unpark(struct intel_rps *rps)
> +{
> + u8 freq;
> +
> + if (!rps->enabled)
> + return;
> +
> + /*
> + * Use the user's desired frequency as a guide, but for better
> + * performance, jump directly to RPe as our starting frequency.
> + */
> + mutex_lock(&rps->lock);
> + rps->active = true;
> + freq = max(rps->cur_freq, rps->efficient_freq),
> + freq = clamp(freq, rps->min_freq_softlimit, rps->max_freq_softlimit);
> + intel_rps_set(rps, freq);
> + rps->last_adj = 0;
> + mutex_unlock(&rps->lock);
> +
> + if (INTEL_GEN(rps_to_i915(rps)) >= 6)
> + rps_enable_interrupts(rps);
> +
> + if (IS_GEN(rps_to_i915(rps), 5))
> + gen5_rps_update(rps);
> +}
> +
> +void intel_rps_park(struct intel_rps *rps)
> +{
> + struct drm_i915_private *i915 = rps_to_i915(rps);
> +
> + if (!rps->enabled)
> + return;
> +
> + if (INTEL_GEN(i915) >= 6)
> + rps_disable_interrupts(rps);
> +
> + rps->active = false;
> + if (rps->last_freq <= rps->idle_freq)
> + return;
> +
> + /*
> + * The punit delays the write of the frequency and voltage until it
> + * determines the GPU is awake. During normal usage we don't want to
> + * waste power changing the frequency if the GPU is sleeping (rc6).
> + * However, the GPU and driver is now idle and we do not want to delay
> + * switching to minimum voltage (reducing power whilst idle) as we do
> + * not expect to be woken in the near future and so must flush the
> + * change by waking the device.
> + *
> + * We choose to take the media powerwell (either would do to trick the
> + * punit into committing the voltage change) as that takes a lot less
> + * power than the render powerwell.
> + */
> + intel_uncore_forcewake_get(rps_to_uncore(rps), FORCEWAKE_MEDIA);
> + rps_set(rps, rps->idle_freq);
> + intel_uncore_forcewake_put(rps_to_uncore(rps), FORCEWAKE_MEDIA);
> +}
> +
> +void intel_rps_boost(struct i915_request *rq)
> +{
> + struct intel_rps *rps = &rq->engine->gt->rps;
> + unsigned long flags;
> +
> + if (i915_request_signaled(rq) || !rps->active)
> + return;
> +
> + /* Serializes with i915_request_retire() */
> + spin_lock_irqsave(&rq->lock, flags);
> + if (!i915_request_has_waitboost(rq) &&
> + !dma_fence_is_signaled_locked(&rq->fence)) {
> + rq->flags |= I915_REQUEST_WAITBOOST;
> +
> + if (!atomic_fetch_inc(&rps->num_waiters) &&
> + READ_ONCE(rps->cur_freq) < rps->boost_freq)
> + schedule_work(&rps->work);
> +
> + atomic_inc(&rps->boosts);
> + }
> + spin_unlock_irqrestore(&rq->lock, flags);
> +}
> +
> +int intel_rps_set(struct intel_rps *rps, u8 val)
> +{
> + int err = 0;
> +
> + lockdep_assert_held(&rps->lock);
> + GEM_BUG_ON(val > rps->max_freq);
> + GEM_BUG_ON(val < rps->min_freq);
> +
> + if (rps->active) {
> + err = rps_set(rps, val);
> +
> + /*
> + * Make sure we continue to get interrupts
> + * until we hit the minimum or maximum frequencies.
> + */
> + if (INTEL_GEN(rps_to_i915(rps)) >= 6) {
> + struct intel_uncore *uncore = rps_to_uncore(rps);
> +
> + intel_uncore_write(uncore, GEN6_RP_INTERRUPT_LIMITS,
> + rps_limits(rps, val));
> +
> + intel_uncore_write(uncore, GEN6_PMINTRMSK,
> + rps_pm_mask(rps, val));
> + }
> + }
> +
> + if (err == 0)
> + rps->cur_freq = val;
> +
> + return err;
> +}
> +
> +static void gen6_rps_init(struct intel_rps *rps)
> +{
> + struct drm_i915_private *i915 = rps_to_i915(rps);
> + struct intel_uncore *uncore = rps_to_uncore(rps);
> +
> + /* All of these values are in units of 50MHz */
> +
> + /* static values from HW: RP0 > RP1 > RPn (min_freq) */
> + if (IS_GEN9_LP(i915)) {
> + u32 rp_state_cap = intel_uncore_read(uncore, BXT_RP_STATE_CAP);
> +
> + rps->rp0_freq = (rp_state_cap >> 16) & 0xff;
> + rps->rp1_freq = (rp_state_cap >> 8) & 0xff;
> + rps->min_freq = (rp_state_cap >> 0) & 0xff;
> + } else {
> + u32 rp_state_cap = intel_uncore_read(uncore, GEN6_RP_STATE_CAP);
> +
> + rps->rp0_freq = (rp_state_cap >> 0) & 0xff;
> + rps->rp1_freq = (rp_state_cap >> 8) & 0xff;
> + rps->min_freq = (rp_state_cap >> 16) & 0xff;
> + }
> +
> + /* hw_max = RP0 until we check for overclocking */
> + rps->max_freq = rps->rp0_freq;
> +
> + rps->efficient_freq = rps->rp1_freq;
> + if (IS_HASWELL(i915) || IS_BROADWELL(i915) ||
> + IS_GEN9_BC(i915) || INTEL_GEN(i915) >= 10) {
> + u32 ddcc_status = 0;
> +
> + if (sandybridge_pcode_read(i915,
> + HSW_PCODE_DYNAMIC_DUTY_CYCLE_CONTROL,
> + &ddcc_status, NULL) == 0)
> + rps->efficient_freq =
> + clamp_t(u8,
> + (ddcc_status >> 8) & 0xff,
> + rps->min_freq,
> + rps->max_freq);
> + }
> +
> + if (IS_GEN9_BC(i915) || INTEL_GEN(i915) >= 10) {
> + /* Store the frequency values in 16.66 MHZ units, which is
> + * the natural hardware unit for SKL
> + */
> + rps->rp0_freq *= GEN9_FREQ_SCALER;
> + rps->rp1_freq *= GEN9_FREQ_SCALER;
> + rps->min_freq *= GEN9_FREQ_SCALER;
> + rps->max_freq *= GEN9_FREQ_SCALER;
> + rps->efficient_freq *= GEN9_FREQ_SCALER;
> + }
> +}
> +
> +static bool rps_reset(struct intel_rps *rps)
> +{
> + /* force a reset */
> + rps->power.mode = -1;
> + rps->last_freq = -1;
> +
> + if (rps_set(rps, rps->min_freq)) {
> + DRM_ERROR("Failed to reset RPS to initial values\n");
> + return false;
> + }
> +
> + rps->cur_freq = rps->min_freq;
> + return true;
> +}
> +
> +/* See the Gen9_GT_PM_Programming_Guide doc for the below */
> +static bool gen9_rps_enable(struct intel_rps *rps)
> +{
> + struct drm_i915_private *i915 = rps_to_i915(rps);
> + struct intel_uncore *uncore = rps_to_uncore(rps);
> +
> + /* Program defaults and thresholds for RPS */
> + if (IS_GEN(i915, 9))
> + intel_uncore_write_fw(uncore, GEN6_RC_VIDEO_FREQ,
> + GEN9_FREQUENCY(rps->rp1_freq));
> +
> + /* 1 second timeout */
> + intel_uncore_write_fw(uncore, GEN6_RP_DOWN_TIMEOUT,
> + GT_INTERVAL_FROM_US(i915, 1000000));
> +
> + intel_uncore_write_fw(uncore, GEN6_RP_IDLE_HYSTERSIS, 0xa);
> +
> + return rps_reset(rps);
> +}
> +
> +static bool gen8_rps_enable(struct intel_rps *rps)
> +{
> + struct intel_uncore *uncore = rps_to_uncore(rps);
> +
> + intel_uncore_write_fw(uncore, GEN6_RC_VIDEO_FREQ,
> + HSW_FREQUENCY(rps->rp1_freq));
> +
> + /* NB: Docs say 1s, and 1000000 - which aren't equivalent */
> + intel_uncore_write_fw(uncore, GEN6_RP_DOWN_TIMEOUT,
> + 100000000 / 128); /* 1 second timeout */
> +
> + intel_uncore_write_fw(uncore, GEN6_RP_IDLE_HYSTERSIS, 10);
> +
> + return rps_reset(rps);
> +}
> +
> +static bool gen6_rps_enable(struct intel_rps *rps)
> +{
> + struct intel_uncore *uncore = rps_to_uncore(rps);
> +
> + /* Power down if completely idle for over 50ms */
> + intel_uncore_write_fw(uncore, GEN6_RP_DOWN_TIMEOUT, 50000);
> + intel_uncore_write_fw(uncore, GEN6_RP_IDLE_HYSTERSIS, 10);
> +
> + return rps_reset(rps);
> +}
> +
> +static int chv_rps_max_freq(struct intel_rps *rps)
> +{
> + struct drm_i915_private *i915 = rps_to_i915(rps);
> + u32 val;
> +
> + val = vlv_punit_read(i915, FB_GFX_FMAX_AT_VMAX_FUSE);
> +
> + switch (RUNTIME_INFO(i915)->sseu.eu_total) {
> + case 8:
> + /* (2 * 4) config */
> + val >>= FB_GFX_FMAX_AT_VMAX_2SS4EU_FUSE_SHIFT;
> + break;
> + case 12:
> + /* (2 * 6) config */
> + val >>= FB_GFX_FMAX_AT_VMAX_2SS6EU_FUSE_SHIFT;
> + break;
> + case 16:
> + /* (2 * 8) config */
> + default:
> + /* Setting (2 * 8) Min RP0 for any other combination */
> + val >>= FB_GFX_FMAX_AT_VMAX_2SS8EU_FUSE_SHIFT;
> + break;
> + }
> +
> + return val & FB_GFX_FREQ_FUSE_MASK;
> +}
> +
> +static int chv_rps_rpe_freq(struct intel_rps *rps)
> +{
> + struct drm_i915_private *i915 = rps_to_i915(rps);
> + u32 val;
> +
> + val = vlv_punit_read(i915, PUNIT_GPU_DUTYCYCLE_REG);
> + val >>= PUNIT_GPU_DUTYCYCLE_RPE_FREQ_SHIFT;
> +
> + return val & PUNIT_GPU_DUTYCYCLE_RPE_FREQ_MASK;
> +}
> +
> +static int chv_rps_guar_freq(struct intel_rps *rps)
> +{
> + struct drm_i915_private *i915 = rps_to_i915(rps);
> + u32 val;
> +
> + val = vlv_punit_read(i915, FB_GFX_FMAX_AT_VMAX_FUSE);
> +
> + return val & FB_GFX_FREQ_FUSE_MASK;
> +}
> +
> +static u32 chv_rps_min_freq(struct intel_rps *rps)
> +{
> + struct drm_i915_private *i915 = rps_to_i915(rps);
> + u32 val;
> +
> + val = vlv_punit_read(i915, FB_GFX_FMIN_AT_VMIN_FUSE);
> + val >>= FB_GFX_FMIN_AT_VMIN_FUSE_SHIFT;
> +
> + return val & FB_GFX_FREQ_FUSE_MASK;
> +}
> +
> +static bool chv_rps_enable(struct intel_rps *rps)
> +{
> + struct intel_uncore *uncore = rps_to_uncore(rps);
> + struct drm_i915_private *i915 = rps_to_i915(rps);
> + u32 val;
> +
> + /* 1: Program defaults and thresholds for RPS*/
> + intel_uncore_write_fw(uncore, GEN6_RP_DOWN_TIMEOUT, 1000000);
> + intel_uncore_write_fw(uncore, GEN6_RP_UP_THRESHOLD, 59400);
> + intel_uncore_write_fw(uncore, GEN6_RP_DOWN_THRESHOLD, 245000);
> + intel_uncore_write_fw(uncore, GEN6_RP_UP_EI, 66000);
> + intel_uncore_write_fw(uncore, GEN6_RP_DOWN_EI, 350000);
> +
> + intel_uncore_write_fw(uncore, GEN6_RP_IDLE_HYSTERSIS, 10);
> +
> + /* 2: Enable RPS */
> + intel_uncore_write_fw(uncore, GEN6_RP_CONTROL,
> + GEN6_RP_MEDIA_HW_NORMAL_MODE |
> + GEN6_RP_MEDIA_IS_GFX |
> + GEN6_RP_ENABLE |
> + GEN6_RP_UP_BUSY_AVG |
> + GEN6_RP_DOWN_IDLE_AVG);
> +
> + /* Setting Fixed Bias */
> + vlv_punit_get(i915);
> +
> + val = VLV_OVERRIDE_EN | VLV_SOC_TDP_EN | CHV_BIAS_CPU_50_SOC_50;
> + vlv_punit_write(i915, VLV_TURBO_SOC_OVERRIDE, val);
> +
> + val = vlv_punit_read(i915, PUNIT_REG_GPU_FREQ_STS);
> +
> + vlv_punit_put(i915);
> +
> + /* RPS code assumes GPLL is used */
> + WARN_ONCE((val & GPLLENABLE) == 0, "GPLL not enabled\n");
> +
> + DRM_DEBUG_DRIVER("GPLL enabled? %s\n", yesno(val & GPLLENABLE));
> + DRM_DEBUG_DRIVER("GPU status: 0x%08x\n", val);
> +
> + return rps_reset(rps);
> +}
> +
> +static int vlv_rps_guar_freq(struct intel_rps *rps)
> +{
> + struct drm_i915_private *i915 = rps_to_i915(rps);
> + u32 val, rp1;
> +
> + val = vlv_nc_read(i915, IOSF_NC_FB_GFX_FREQ_FUSE);
> +
> + rp1 = val & FB_GFX_FGUARANTEED_FREQ_FUSE_MASK;
> + rp1 >>= FB_GFX_FGUARANTEED_FREQ_FUSE_SHIFT;
> +
> + return rp1;
> +}
> +
> +static int vlv_rps_max_freq(struct intel_rps *rps)
> +{
> + struct drm_i915_private *i915 = rps_to_i915(rps);
> + u32 val, rp0;
> +
> + val = vlv_nc_read(i915, IOSF_NC_FB_GFX_FREQ_FUSE);
> +
> + rp0 = (val & FB_GFX_MAX_FREQ_FUSE_MASK) >> FB_GFX_MAX_FREQ_FUSE_SHIFT;
> + /* Clamp to max */
> + rp0 = min_t(u32, rp0, 0xea);
> +
> + return rp0;
> +}
> +
> +static int vlv_rps_rpe_freq(struct intel_rps *rps)
> +{
> + struct drm_i915_private *i915 = rps_to_i915(rps);
> + u32 val, rpe;
> +
> + val = vlv_nc_read(i915, IOSF_NC_FB_GFX_FMAX_FUSE_LO);
> + rpe = (val & FB_FMAX_VMIN_FREQ_LO_MASK) >> FB_FMAX_VMIN_FREQ_LO_SHIFT;
> + val = vlv_nc_read(i915, IOSF_NC_FB_GFX_FMAX_FUSE_HI);
> + rpe |= (val & FB_FMAX_VMIN_FREQ_HI_MASK) << 5;
> +
> + return rpe;
> +}
> +
> +static int vlv_rps_min_freq(struct intel_rps *rps)
> +{
> + struct drm_i915_private *i915 = rps_to_i915(rps);
> + u32 val;
> +
> + val = vlv_punit_read(i915, PUNIT_REG_GPU_LFM) & 0xff;
> + /*
> + * According to the BYT Punit GPU turbo HAS 1.1.6.3 the minimum value
> + * for the minimum frequency in GPLL mode is 0xc1. Contrary to this on
> + * a BYT-M B0 the above register contains 0xbf. Moreover when setting
> + * a frequency Punit will not allow values below 0xc0. Clamp it 0xc0
> + * to make sure it matches what Punit accepts.
> + */
> + return max_t(u32, val, 0xc0);
> +}
> +
> +static bool vlv_rps_enable(struct intel_rps *rps)
> +{
> + struct intel_uncore *uncore = rps_to_uncore(rps);
> + struct drm_i915_private *i915 = rps_to_i915(rps);
> + u32 val;
> +
> + intel_uncore_write_fw(uncore, GEN6_RP_DOWN_TIMEOUT, 1000000);
> + intel_uncore_write_fw(uncore, GEN6_RP_UP_THRESHOLD, 59400);
> + intel_uncore_write_fw(uncore, GEN6_RP_DOWN_THRESHOLD, 245000);
> + intel_uncore_write_fw(uncore, GEN6_RP_UP_EI, 66000);
> + intel_uncore_write_fw(uncore, GEN6_RP_DOWN_EI, 350000);
> +
> + intel_uncore_write_fw(uncore, GEN6_RP_IDLE_HYSTERSIS, 10);
> +
> + intel_uncore_write_fw(uncore, GEN6_RP_CONTROL,
> + GEN6_RP_MEDIA_TURBO |
> + GEN6_RP_MEDIA_HW_NORMAL_MODE |
> + GEN6_RP_MEDIA_IS_GFX |
> + GEN6_RP_ENABLE |
> + GEN6_RP_UP_BUSY_AVG |
> + GEN6_RP_DOWN_IDLE_CONT);
> +
> + vlv_punit_get(i915);
> +
> + /* Setting Fixed Bias */
> + val = VLV_OVERRIDE_EN | VLV_SOC_TDP_EN | VLV_BIAS_CPU_125_SOC_875;
> + vlv_punit_write(i915, VLV_TURBO_SOC_OVERRIDE, val);
> +
> + val = vlv_punit_read(i915, PUNIT_REG_GPU_FREQ_STS);
> +
> + vlv_punit_put(i915);
> +
> + /* RPS code assumes GPLL is used */
> + WARN_ONCE((val & GPLLENABLE) == 0, "GPLL not enabled\n");
> +
> + DRM_DEBUG_DRIVER("GPLL enabled? %s\n", yesno(val & GPLLENABLE));
> + DRM_DEBUG_DRIVER("GPU status: 0x%08x\n", val);
> +
> + return rps_reset(rps);
> +}
> +
> +static unsigned long __ips_gfx_val(struct intel_ips *ips)
> +{
> + struct intel_rps *rps = container_of(ips, typeof(*rps), ips);
> + struct intel_uncore *uncore = rps_to_uncore(rps);
> + unsigned long t, corr, state1, corr2, state2;
> + u32 pxvid, ext_v;
> +
> + lockdep_assert_held(&mchdev_lock);
> +
> + pxvid = intel_uncore_read(uncore, PXVFREQ(rps->cur_freq));
> + pxvid = (pxvid >> 24) & 0x7f;
> + ext_v = pvid_to_extvid(rps_to_i915(rps), pxvid);
> +
> + state1 = ext_v;
> +
> + /* Revel in the empirically derived constants */
> +
> + /* Correction factor in 1/100000 units */
> + t = ips_mch_val(uncore);
> + if (t > 80)
> + corr = t * 2349 + 135940;
> + else if (t >= 50)
> + corr = t * 964 + 29317;
> + else /* < 50 */
> + corr = t * 301 + 1004;
> +
> + corr = corr * 150142 * state1 / 10000 - 78642;
> + corr /= 100000;
> + corr2 = corr * ips->corr;
> +
> + state2 = corr2 * state1 / 10000;
> + state2 /= 100; /* convert to mW */
> +
> + __gen5_ips_update(ips);
> +
> + return ips->gfx_power + state2;
> +}
> +
> +void intel_rps_enable(struct intel_rps *rps)
> +{
> + struct drm_i915_private *i915 = rps_to_i915(rps);
> + struct intel_uncore *uncore = rps_to_uncore(rps);
> +
> + intel_uncore_forcewake_get(uncore, FORCEWAKE_ALL);
> + if (IS_CHERRYVIEW(i915))
> + rps->enabled = chv_rps_enable(rps);
> + else if (IS_VALLEYVIEW(i915))
> + rps->enabled = vlv_rps_enable(rps);
> + else if (INTEL_GEN(i915) >= 9)
> + rps->enabled = gen9_rps_enable(rps);
> + else if (INTEL_GEN(i915) >= 8)
> + rps->enabled = gen8_rps_enable(rps);
> + else if (INTEL_GEN(i915) >= 6)
> + rps->enabled = gen6_rps_enable(rps);
> + else if (IS_IRONLAKE_M(i915))
> + rps->enabled = gen5_rps_enable(rps);
> + intel_uncore_forcewake_put(uncore, FORCEWAKE_ALL);
> + if (!rps->enabled)
> + return;
> +
> + WARN_ON(rps->max_freq < rps->min_freq);
> + WARN_ON(rps->idle_freq > rps->max_freq);
> +
> + WARN_ON(rps->efficient_freq < rps->min_freq);
> + WARN_ON(rps->efficient_freq > rps->max_freq);
> +}
> +
> +static void gen6_rps_disable(struct intel_rps *rps)
> +{
> + intel_uncore_write(rps_to_uncore(rps), GEN6_RP_CONTROL, 0);
> +}
> +
> +void intel_rps_disable(struct intel_rps *rps)
> +{
> + struct drm_i915_private *i915 = rps_to_i915(rps);
> +
> + rps->enabled = false;
> +
> + if (INTEL_GEN(i915) >= 6)
> + gen6_rps_disable(rps);
> + else if (IS_IRONLAKE_M(i915))
> + gen5_rps_disable(rps);
> +}
> +
> +static int byt_gpu_freq(struct intel_rps *rps, int val)
> +{
> + /*
> + * N = val - 0xb7
> + * Slow = Fast = GPLL ref * N
> + */
> + return DIV_ROUND_CLOSEST(rps->gpll_ref_freq * (val - 0xb7), 1000);
> +}
> +
> +static int byt_freq_opcode(struct intel_rps *rps, int val)
> +{
> + return DIV_ROUND_CLOSEST(1000 * val, rps->gpll_ref_freq) + 0xb7;
> +}
> +
> +static int chv_gpu_freq(struct intel_rps *rps, int val)
> +{
> + /*
> + * N = val / 2
> + * CU (slow) = CU2x (fast) / 2 = GPLL ref * N / 2
> + */
> + return DIV_ROUND_CLOSEST(rps->gpll_ref_freq * val, 2 * 2 * 1000);
> +}
> +
> +static int chv_freq_opcode(struct intel_rps *rps, int val)
> +{
> + /* CHV needs even values */
> + return DIV_ROUND_CLOSEST(2 * 1000 * val, rps->gpll_ref_freq) * 2;
> +}
> +
> +int intel_gpu_freq(struct intel_rps *rps, int val)
> +{
> + struct drm_i915_private *i915 = rps_to_i915(rps);
> +
> + if (INTEL_GEN(i915) >= 9)
> + return DIV_ROUND_CLOSEST(val * GT_FREQUENCY_MULTIPLIER,
> + GEN9_FREQ_SCALER);
> + else if (IS_CHERRYVIEW(i915))
> + return chv_gpu_freq(rps, val);
> + else if (IS_VALLEYVIEW(i915))
> + return byt_gpu_freq(rps, val);
> + else
> + return val * GT_FREQUENCY_MULTIPLIER;
> +}
> +
> +int intel_freq_opcode(struct intel_rps *rps, int val)
> +{
> + struct drm_i915_private *i915 = rps_to_i915(rps);
> +
> + if (INTEL_GEN(i915) >= 9)
> + return DIV_ROUND_CLOSEST(val * GEN9_FREQ_SCALER,
> + GT_FREQUENCY_MULTIPLIER);
> + else if (IS_CHERRYVIEW(i915))
> + return chv_freq_opcode(rps, val);
> + else if (IS_VALLEYVIEW(i915))
> + return byt_freq_opcode(rps, val);
> + else
> + return DIV_ROUND_CLOSEST(val, GT_FREQUENCY_MULTIPLIER);
> +}
> +
> +static void vlv_init_gpll_ref_freq(struct intel_rps *rps)
> +{
> + struct drm_i915_private *i915 = rps_to_i915(rps);
> +
> + rps->gpll_ref_freq =
> + vlv_get_cck_clock(i915, "GPLL ref",
> + CCK_GPLL_CLOCK_CONTROL,
> + i915->czclk_freq);
> +
> + DRM_DEBUG_DRIVER("GPLL reference freq: %d kHz\n", rps->gpll_ref_freq);
> +}
> +
> +static void vlv_rps_init(struct intel_rps *rps)
> +{
> + struct drm_i915_private *i915 = rps_to_i915(rps);
> + u32 val;
> +
> + vlv_iosf_sb_get(i915,
> + BIT(VLV_IOSF_SB_PUNIT) |
> + BIT(VLV_IOSF_SB_NC) |
> + BIT(VLV_IOSF_SB_CCK));
> +
> + vlv_init_gpll_ref_freq(rps);
> +
> + val = vlv_punit_read(i915, PUNIT_REG_GPU_FREQ_STS);
> + switch ((val >> 6) & 3) {
> + case 0:
> + case 1:
> + i915->mem_freq = 800;
> + break;
> + case 2:
> + i915->mem_freq = 1066;
> + break;
> + case 3:
> + i915->mem_freq = 1333;
> + break;
> + }
> + DRM_DEBUG_DRIVER("DDR speed: %d MHz\n", i915->mem_freq);
> +
> + rps->max_freq = vlv_rps_max_freq(rps);
> + rps->rp0_freq = rps->max_freq;
> + DRM_DEBUG_DRIVER("max GPU freq: %d MHz (%u)\n",
> + intel_gpu_freq(rps, rps->max_freq),
> + rps->max_freq);
> +
> + rps->efficient_freq = vlv_rps_rpe_freq(rps);
> + DRM_DEBUG_DRIVER("RPe GPU freq: %d MHz (%u)\n",
> + intel_gpu_freq(rps, rps->efficient_freq),
> + rps->efficient_freq);
> +
> + rps->rp1_freq = vlv_rps_guar_freq(rps);
> + DRM_DEBUG_DRIVER("RP1(Guar Freq) GPU freq: %d MHz (%u)\n",
> + intel_gpu_freq(rps, rps->rp1_freq),
> + rps->rp1_freq);
> +
> + rps->min_freq = vlv_rps_min_freq(rps);
> + DRM_DEBUG_DRIVER("min GPU freq: %d MHz (%u)\n",
> + intel_gpu_freq(rps, rps->min_freq),
> + rps->min_freq);
> +
> + vlv_iosf_sb_put(i915,
> + BIT(VLV_IOSF_SB_PUNIT) |
> + BIT(VLV_IOSF_SB_NC) |
> + BIT(VLV_IOSF_SB_CCK));
> +}
> +
> +static void chv_rps_init(struct intel_rps *rps)
> +{
> + struct drm_i915_private *i915 = rps_to_i915(rps);
> + u32 val;
> +
> + vlv_iosf_sb_get(i915,
> + BIT(VLV_IOSF_SB_PUNIT) |
> + BIT(VLV_IOSF_SB_NC) |
> + BIT(VLV_IOSF_SB_CCK));
> +
> + vlv_init_gpll_ref_freq(rps);
> +
> + val = vlv_cck_read(i915, CCK_FUSE_REG);
> +
> + switch ((val >> 2) & 0x7) {
> + case 3:
> + i915->mem_freq = 2000;
> + break;
> + default:
> + i915->mem_freq = 1600;
> + break;
> + }
> + DRM_DEBUG_DRIVER("DDR speed: %d MHz\n", i915->mem_freq);
> +
> + rps->max_freq = chv_rps_max_freq(rps);
> + rps->rp0_freq = rps->max_freq;
> + DRM_DEBUG_DRIVER("max GPU freq: %d MHz (%u)\n",
> + intel_gpu_freq(rps, rps->max_freq),
> + rps->max_freq);
> +
> + rps->efficient_freq = chv_rps_rpe_freq(rps);
> + DRM_DEBUG_DRIVER("RPe GPU freq: %d MHz (%u)\n",
> + intel_gpu_freq(rps, rps->efficient_freq),
> + rps->efficient_freq);
> +
> + rps->rp1_freq = chv_rps_guar_freq(rps);
> + DRM_DEBUG_DRIVER("RP1(Guar) GPU freq: %d MHz (%u)\n",
> + intel_gpu_freq(rps, rps->rp1_freq),
> + rps->rp1_freq);
> +
> + rps->min_freq = chv_rps_min_freq(rps);
> + DRM_DEBUG_DRIVER("min GPU freq: %d MHz (%u)\n",
> + intel_gpu_freq(rps, rps->min_freq),
> + rps->min_freq);
> +
> + vlv_iosf_sb_put(i915,
> + BIT(VLV_IOSF_SB_PUNIT) |
> + BIT(VLV_IOSF_SB_NC) |
> + BIT(VLV_IOSF_SB_CCK));
> +
> + WARN_ONCE((rps->max_freq | rps->efficient_freq | rps->rp1_freq |
> + rps->min_freq) & 1,
> + "Odd GPU freq values\n");
> +}
> +
> +static void vlv_c0_read(struct intel_uncore *uncore, struct intel_rps_ei *ei)
> +{
> + ei->ktime = ktime_get_raw();
> + ei->render_c0 = intel_uncore_read(uncore, VLV_RENDER_C0_COUNT);
> + ei->media_c0 = intel_uncore_read(uncore, VLV_MEDIA_C0_COUNT);
> +}
> +
> +static u32 vlv_wa_c0_ei(struct intel_rps *rps, u32 pm_iir)
> +{
> + struct intel_uncore *uncore = rps_to_uncore(rps);
> + const struct intel_rps_ei *prev = &rps->ei;
> + struct intel_rps_ei now;
> + u32 events = 0;
> +
> + if ((pm_iir & GEN6_PM_RP_UP_EI_EXPIRED) == 0)
> + return 0;
> +
> + vlv_c0_read(uncore, &now);
> +
> + if (prev->ktime) {
> + u64 time, c0;
> + u32 render, media;
> +
> + time = ktime_us_delta(now.ktime, prev->ktime);
> +
> + time *= rps_to_i915(rps)->czclk_freq;
> +
> + /* Workload can be split between render + media,
> + * e.g. SwapBuffers being blitted in X after being rendered in
> + * mesa. To account for this we need to combine both engines
> + * into our activity counter.
> + */
> + render = now.render_c0 - prev->render_c0;
> + media = now.media_c0 - prev->media_c0;
> + c0 = max(render, media);
> + c0 *= 1000 * 100 << 8; /* to usecs and scale to threshold% */
> +
> + if (c0 > time * rps->power.up_threshold)
> + events = GEN6_PM_RP_UP_THRESHOLD;
> + else if (c0 < time * rps->power.down_threshold)
> + events = GEN6_PM_RP_DOWN_THRESHOLD;
> + }
> +
> + rps->ei = now;
> + return events;
> +}
> +
> +static void rps_work(struct work_struct *work)
> +{
> + struct intel_rps *rps = container_of(work, typeof(*rps), work);
> + struct intel_gt *gt = rps_to_gt(rps);
> + bool client_boost = false;
> + int new_freq, adj, min, max;
> + u32 pm_iir = 0;
> +
> + spin_lock_irq(>->irq_lock);
> + pm_iir = fetch_and_zero(&rps->pm_iir);
> + client_boost = atomic_read(&rps->num_waiters);
> + spin_unlock_irq(>->irq_lock);
> +
> + /* Make sure we didn't queue anything we're not going to process. */
> + if ((pm_iir & rps->pm_events) == 0 && !client_boost)
> + goto out;
> +
> + mutex_lock(&rps->lock);
> +
> + pm_iir |= vlv_wa_c0_ei(rps, pm_iir);
> +
> + adj = rps->last_adj;
> + new_freq = rps->cur_freq;
> + min = rps->min_freq_softlimit;
> + max = rps->max_freq_softlimit;
> + if (client_boost)
> + max = rps->max_freq;
> + if (client_boost && new_freq < rps->boost_freq) {
> + new_freq = rps->boost_freq;
> + adj = 0;
> + } else if (pm_iir & GEN6_PM_RP_UP_THRESHOLD) {
> + if (adj > 0)
> + adj *= 2;
> + else /* CHV needs even encode values */
> + adj = IS_CHERRYVIEW(gt->i915) ? 2 : 1;
> +
> + if (new_freq >= rps->max_freq_softlimit)
> + adj = 0;
> + } else if (client_boost) {
> + adj = 0;
> + } else if (pm_iir & GEN6_PM_RP_DOWN_TIMEOUT) {
> + if (rps->cur_freq > rps->efficient_freq)
> + new_freq = rps->efficient_freq;
> + else if (rps->cur_freq > rps->min_freq_softlimit)
> + new_freq = rps->min_freq_softlimit;
> + adj = 0;
> + } else if (pm_iir & GEN6_PM_RP_DOWN_THRESHOLD) {
> + if (adj < 0)
> + adj *= 2;
> + else /* CHV needs even encode values */
> + adj = IS_CHERRYVIEW(gt->i915) ? -2 : -1;
> +
> + if (new_freq <= rps->min_freq_softlimit)
> + adj = 0;
> + } else { /* unknown event */
> + adj = 0;
> + }
> +
> + rps->last_adj = adj;
> +
> + /*
> + * Limit deboosting and boosting to keep ourselves at the extremes
> + * when in the respective power modes (i.e. slowly decrease frequencies
> + * while in the HIGH_POWER zone and slowly increase frequencies while
> + * in the LOW_POWER zone). On idle, we will hit the timeout and drop
> + * to the next level quickly, and conversely if busy we expect to
> + * hit a waitboost and rapidly switch into max power.
> + */
> + if ((adj < 0 && rps->power.mode == HIGH_POWER) ||
> + (adj > 0 && rps->power.mode == LOW_POWER))
> + rps->last_adj = 0;
> +
> + /* sysfs frequency interfaces may have snuck in while servicing the
> + * interrupt
> + */
> + new_freq += adj;
> + new_freq = clamp_t(int, new_freq, min, max);
> +
> + if (intel_rps_set(rps, new_freq)) {
> + DRM_DEBUG_DRIVER("Failed to set new GPU frequency\n");
> + rps->last_adj = 0;
> + }
> +
> + mutex_unlock(&rps->lock);
> +
> +out:
> + spin_lock_irq(>->irq_lock);
> + gen6_gt_pm_unmask_irq(gt, rps->pm_events);
> + spin_unlock_irq(>->irq_lock);
> +}
> +
> +void gen11_rps_irq_handler(struct intel_rps *rps, u32 pm_iir)
> +{
> + struct intel_gt *gt = rps_to_gt(rps);
> + const u32 events = rps->pm_events & pm_iir;
> +
> + lockdep_assert_held(>->irq_lock);
> +
> + if (unlikely(!events))
> + return;
> +
> + gen6_gt_pm_mask_irq(gt, events);
> +
> + rps->pm_iir |= events;
> + schedule_work(&rps->work);
> +}
> +
> +void gen6_rps_irq_handler(struct intel_rps *rps, u32 pm_iir)
> +{
> + struct drm_i915_private *i915 = rps_to_i915(rps);
> +
> + if (pm_iir & rps->pm_events) {
> + struct intel_gt *gt = rps_to_gt(rps);
> +
> + spin_lock(>->irq_lock);
> + gen6_gt_pm_mask_irq(gt, pm_iir & rps->pm_events);
> + rps->pm_iir |= pm_iir & rps->pm_events;
> + schedule_work(&rps->work);
> + spin_unlock(>->irq_lock);
> + }
> +
> + if (INTEL_GEN(i915) >= 8)
> + return;
> +
> + if (pm_iir & PM_VEBOX_USER_INTERRUPT)
> + intel_engine_breadcrumbs_irq(i915->engine[VECS0]);
> +
> + if (pm_iir & PM_VEBOX_CS_ERROR_INTERRUPT)
> + DRM_DEBUG("Command parser error, pm_iir 0x%08x\n", pm_iir);
> +}
> +
> +void gen5_rps_irq_handler(struct intel_rps *rps)
> +{
> + struct intel_uncore *uncore = rps_to_uncore(rps);
> + u32 busy_up, busy_down, max_avg, min_avg;
> + u8 new_freq;
> +
> + spin_lock(&mchdev_lock);
> +
> + intel_uncore_write16(uncore,
> + MEMINTRSTS,
> + intel_uncore_read(uncore, MEMINTRSTS));
> +
> + intel_uncore_write16(uncore, MEMINTRSTS, MEMINT_EVAL_CHG);
> + busy_up = intel_uncore_read(uncore, RCPREVBSYTUPAVG);
> + busy_down = intel_uncore_read(uncore, RCPREVBSYTDNAVG);
> + max_avg = intel_uncore_read(uncore, RCBMAXAVG);
> + min_avg = intel_uncore_read(uncore, RCBMINAVG);
> +
> + /* Handle RCS change request from hw */
> + new_freq = rps->cur_freq;
> + if (busy_up > max_avg)
> + new_freq++;
> + else if (busy_down < min_avg)
> + new_freq--;
> + new_freq = clamp(new_freq,
> + rps->min_freq_softlimit,
> + rps->max_freq_softlimit);
> +
> + if (new_freq != rps->cur_freq && gen5_rps_set(rps, new_freq))
> + rps->cur_freq = new_freq;
> +
> + spin_unlock(&mchdev_lock);
> +}
> +
> +void intel_rps_init(struct intel_rps *rps)
> +{
> + struct drm_i915_private *i915 = rps_to_i915(rps);
> +
> + mutex_init(&rps->lock);
> + mutex_init(&rps->power.mutex);
> +
> + INIT_WORK(&rps->work, rps_work);
> +
> + atomic_set(&rps->num_waiters, 0);
> +
> + if (IS_CHERRYVIEW(i915))
> + chv_rps_init(rps);
> + else if (IS_VALLEYVIEW(i915))
> + vlv_rps_init(rps);
> + else if (INTEL_GEN(i915) >= 6)
> + gen6_rps_init(rps);
> + else if (IS_IRONLAKE_M(i915))
> + gen5_rps_init(rps);
> +
> + /* Derive initial user preferences/limits from the hardware limits */
> + rps->max_freq_softlimit = rps->max_freq;
> + rps->min_freq_softlimit = rps->min_freq;
> +
> + /* After setting max-softlimit, find the overclock max freq */
> + if (IS_GEN(i915, 6) || IS_IVYBRIDGE(i915) || IS_HASWELL(i915)) {
> + u32 params = 0;
> +
> + sandybridge_pcode_read(i915, GEN6_READ_OC_PARAMS,
> + ¶ms, NULL);
> + if (params & BIT(31)) { /* OC supported */
> + DRM_DEBUG_DRIVER("Overclocking supported, max: %dMHz, overclock: %dMHz\n",
> + (rps->max_freq & 0xff) * 50,
> + (params & 0xff) * 50);
> + rps->max_freq = params & 0xff;
> + }
> + }
> +
> + /* Finally allow us to boost to max by default */
> + rps->boost_freq = rps->max_freq;
> + rps->idle_freq = rps->min_freq;
> + rps->cur_freq = rps->idle_freq;
> +
> + rps->pm_intrmsk_mbz = 0;
> +
> + /*
> + * SNB,IVB,HSW can while VLV,CHV may hard hang on looping batchbuffer
> + * if GEN6_PM_UP_EI_EXPIRED is masked.
> + *
> + * TODO: verify if this can be reproduced on VLV,CHV.
> + */
> + if (INTEL_GEN(i915) <= 7)
> + rps->pm_intrmsk_mbz |= GEN6_PM_RP_UP_EI_EXPIRED;
> +
> + if (INTEL_GEN(i915) >= 8)
> + rps->pm_intrmsk_mbz |= GEN8_PMINTR_DISABLE_REDIRECT_TO_GUC;
> +}
> +
> +u32 intel_get_cagf(struct intel_rps *rps, u32 rpstat)
> +{
> + struct drm_i915_private *i915 = rps_to_i915(rps);
> + u32 cagf;
> +
> + if (INTEL_GEN(i915) >= 9)
> + cagf = (rpstat & GEN9_CAGF_MASK) >> GEN9_CAGF_SHIFT;
> + else if (IS_HASWELL(i915) || IS_BROADWELL(i915))
> + cagf = (rpstat & HSW_CAGF_MASK) >> HSW_CAGF_SHIFT;
> + else
> + cagf = (rpstat & GEN6_CAGF_MASK) >> GEN6_CAGF_SHIFT;
> +
> + return cagf;
> +}
> +
> +/* External interface for intel_ips.ko */
> +
> +static struct drm_i915_private __rcu *ips_mchdev;
> +
> +/**
> + * Tells the intel_ips driver that the i915 driver is now loaded, if
> + * IPS got loaded first.
> + *
> + * This awkward dance is so that neither module has to depend on the
> + * other in order for IPS to do the appropriate communication of
> + * GPU turbo limits to i915.
> + */
> +static void
> +ips_ping_for_i915_load(void)
> +{
> + void (*link)(void);
> +
> + link = symbol_get(ips_link_to_i915_driver);
> + if (link) {
> + link();
> + symbol_put(ips_link_to_i915_driver);
> + }
> +}
> +
> +void intel_rps_driver_register(struct intel_rps *rps)
> +{
> + struct intel_gt *gt = rps_to_gt(rps);
> +
> + /*
> + * We only register the i915 ips part with intel-ips once everything is
> + * set up, to avoid intel-ips sneaking in and reading bogus values.
> + */
> + if (IS_GEN(gt->i915, 5)) {
> + rcu_assign_pointer(ips_mchdev, gt->i915);
> + ips_ping_for_i915_load();
> + }
> +}
> +
> +void intel_rps_driver_unregister(struct intel_rps *rps)
> +{
> + rcu_assign_pointer(ips_mchdev, NULL);
> +}
> +
> +static struct drm_i915_private *mchdev_get(void)
> +{
> + struct drm_i915_private *i915;
> +
> + rcu_read_lock();
> + i915 = rcu_dereference(ips_mchdev);
> + if (!kref_get_unless_zero(&i915->drm.ref))
> + i915 = NULL;
> + rcu_read_unlock();
> +
> + return i915;
> +}
> +
> +/**
> + * i915_read_mch_val - return value for IPS use
> + *
> + * Calculate and return a value for the IPS driver to use when deciding whether
> + * we have thermal and power headroom to increase CPU or GPU power budget.
> + */
> +unsigned long i915_read_mch_val(void)
> +{
> + struct drm_i915_private *i915;
> + unsigned long chipset_val = 0;
> + unsigned long graphics_val = 0;
> + intel_wakeref_t wakeref;
> +
> + i915 = mchdev_get();
> + if (!i915)
> + return 0;
> +
> + with_intel_runtime_pm(&i915->runtime_pm, wakeref) {
> + struct intel_ips *ips = &i915->gt.rps.ips;
> +
> + spin_lock_irq(&mchdev_lock);
> + chipset_val = __ips_chipset_val(ips);
> + graphics_val = __ips_gfx_val(ips);
> + spin_unlock_irq(&mchdev_lock);
> + }
> +
> + drm_dev_put(&i915->drm);
> + return chipset_val + graphics_val;
> +}
> +EXPORT_SYMBOL_GPL(i915_read_mch_val);
> +
> +/**
> + * i915_gpu_raise - raise GPU frequency limit
> + *
> + * Raise the limit; IPS indicates we have thermal headroom.
> + */
> +bool i915_gpu_raise(void)
> +{
> + struct drm_i915_private *i915;
> + struct intel_rps *rps;
> +
> + i915 = mchdev_get();
> + if (!i915)
> + return false;
> +
> + rps = &i915->gt.rps;
> +
> + spin_lock_irq(&mchdev_lock);
> + if (rps->max_freq_softlimit < rps->max_freq)
> + rps->max_freq_softlimit++;
> + spin_unlock_irq(&mchdev_lock);
> +
> + drm_dev_put(&i915->drm);
> + return true;
> +}
> +EXPORT_SYMBOL_GPL(i915_gpu_raise);
> +
> +/**
> + * i915_gpu_lower - lower GPU frequency limit
> + *
> + * IPS indicates we're close to a thermal limit, so throttle back the GPU
> + * frequency maximum.
> + */
> +bool i915_gpu_lower(void)
> +{
> + struct drm_i915_private *i915;
> + struct intel_rps *rps;
> +
> + i915 = mchdev_get();
> + if (!i915)
> + return false;
> +
> + rps = &i915->gt.rps;
> +
> + spin_lock_irq(&mchdev_lock);
> + if (rps->max_freq_softlimit > rps->min_freq)
> + rps->max_freq_softlimit--;
> + spin_unlock_irq(&mchdev_lock);
> +
> + drm_dev_put(&i915->drm);
> + return true;
> +}
> +EXPORT_SYMBOL_GPL(i915_gpu_lower);
> +
> +/**
> + * i915_gpu_busy - indicate GPU business to IPS
> + *
> + * Tell the IPS driver whether or not the GPU is busy.
> + */
> +bool i915_gpu_busy(void)
> +{
> + struct drm_i915_private *i915;
> + bool ret;
> +
> + i915 = mchdev_get();
> + if (!i915)
> + return false;
> +
> + ret = i915->gt.awake;
> +
> + drm_dev_put(&i915->drm);
> + return ret;
> +}
> +EXPORT_SYMBOL_GPL(i915_gpu_busy);
> +
> +/**
> + * i915_gpu_turbo_disable - disable graphics turbo
> + *
> + * Disable graphics turbo by resetting the max frequency and setting the
> + * current frequency to the default.
> + */
> +bool i915_gpu_turbo_disable(void)
> +{
> + struct drm_i915_private *i915;
> + struct intel_rps *rps;
> + bool ret;
> +
> + i915 = mchdev_get();
> + if (!i915)
> + return false;
> +
> + rps = &i915->gt.rps;
> +
> + spin_lock_irq(&mchdev_lock);
> + rps->max_freq_softlimit = rps->min_freq;
> + ret = gen5_rps_set(&i915->gt.rps, rps->min_freq);
> + spin_unlock_irq(&mchdev_lock);
> +
> + drm_dev_put(&i915->drm);
> + return ret;
> +}
> +EXPORT_SYMBOL_GPL(i915_gpu_turbo_disable);
> diff --git a/drivers/gpu/drm/i915/gt/intel_rps.h b/drivers/gpu/drm/i915/gt/intel_rps.h
> new file mode 100644
> index 000000000000..997a4b4e0207
> --- /dev/null
> +++ b/drivers/gpu/drm/i915/gt/intel_rps.h
> @@ -0,0 +1,37 @@
> +/*
> + * SPDX-License-Identifier: MIT
> + *
> + * Copyright © 2019 Intel Corporation
> + */
> +
> +#ifndef INTEL_RPS_H
> +#define INTEL_RPS_H
> +
> +#include "intel_rps_types.h"
> +
> +struct i915_request;
> +
> +void intel_rps_init(struct intel_rps *rps);
> +
> +void intel_rps_driver_register(struct intel_rps *rps);
> +void intel_rps_driver_unregister(struct intel_rps *rps);
> +
> +void intel_rps_enable(struct intel_rps *rps);
> +void intel_rps_disable(struct intel_rps *rps);
> +
> +void intel_rps_park(struct intel_rps *rps);
> +void intel_rps_unpark(struct intel_rps *rps);
> +void intel_rps_boost(struct i915_request *rq);
> +
> +int intel_rps_set(struct intel_rps *rps, u8 val);
> +void intel_rps_mark_interactive(struct intel_rps *rps, bool interactive);
> +
> +int intel_gpu_freq(struct intel_rps *rps, int val);
> +int intel_freq_opcode(struct intel_rps *rps, int val);
> +u32 intel_get_cagf(struct intel_rps *rps, u32 rpstat1);
> +
> +void gen5_rps_irq_handler(struct intel_rps *rps);
> +void gen6_rps_irq_handler(struct intel_rps *rps, u32 pm_iir);
> +void gen11_rps_irq_handler(struct intel_rps *rps, u32 pm_iir);
> +
> +#endif /* INTEL_RPS_H */
> diff --git a/drivers/gpu/drm/i915/gt/intel_rps_types.h b/drivers/gpu/drm/i915/gt/intel_rps_types.h
> new file mode 100644
> index 000000000000..40eb1fb651e7
> --- /dev/null
> +++ b/drivers/gpu/drm/i915/gt/intel_rps_types.h
> @@ -0,0 +1,93 @@
> +/*
> + * SPDX-License-Identifier: MIT
> + *
> + * Copyright © 2019 Intel Corporation
> + */
> +
> +#ifndef INTEL_RPS_TYPES_H
> +#define INTEL_RPS_TYPES_H
> +
> +#include <linux/atomic.h>
> +#include <linux/ktime.h>
> +#include <linux/mutex.h>
> +#include <linux/types.h>
> +#include <linux/workqueue.h>
> +
> +struct intel_ips {
> + u64 last_count1;
> + unsigned long last_time1;
> + unsigned long chipset_power;
> + u64 last_count2;
> + u64 last_time2;
> + unsigned long gfx_power;
> + u8 corr;
> +
> + int c, m;
> +};
> +
> +struct intel_rps_ei {
> + ktime_t ktime;
> + u32 render_c0;
> + u32 media_c0;
> +};
> +
> +struct intel_rps {
> + struct mutex lock; /* protects enabling and the worker */
> +
> + /*
> + * work, interrupts_enabled and pm_iir are protected by
> + * dev_priv->irq_lock
> + */
> + struct work_struct work;
> + bool enabled;
> + bool active;
> + u32 pm_iir;
> +
> + /* PM interrupt bits that should never be masked */
> + u32 pm_intrmsk_mbz;
> + u32 pm_events;
> +
> + /* Frequencies are stored in potentially platform dependent multiples.
> + * In other words, *_freq needs to be multiplied by X to be interesting.
> + * Soft limits are those which are used for the dynamic reclocking done
> + * by the driver (raise frequencies under heavy loads, and lower for
> + * lighter loads). Hard limits are those imposed by the hardware.
> + *
> + * A distinction is made for overclocking, which is never enabled by
> + * default, and is considered to be above the hard limit if it's
> + * possible at all.
> + */
> + u8 cur_freq; /* Current frequency (cached, may not == HW) */
> + u8 last_freq; /* Last swqreq frequency */
> + u8 min_freq_softlimit; /* Minimum frequency permitted by the driver */
> + u8 max_freq_softlimit; /* Max frequency permitted by the driver */
> + u8 max_freq; /* Maximum frequency, RP0 if not overclocking */
> + u8 min_freq; /* AKA RPn. Minimum frequency */
> + u8 boost_freq; /* Frequency to request when wait boosting */
> + u8 idle_freq; /* Frequency to request when we are idle */
> + u8 efficient_freq; /* AKA RPe. Pre-determined balanced frequency */
> + u8 rp1_freq; /* "less than" RP0 power/freqency */
> + u8 rp0_freq; /* Non-overclocked max frequency. */
> + u16 gpll_ref_freq; /* vlv/chv GPLL reference frequency */
> +
> + int last_adj;
> +
> + struct {
> + struct mutex mutex;
> +
> + enum { LOW_POWER, BETWEEN, HIGH_POWER } mode;
> + unsigned int interactive;
> +
> + u8 up_threshold; /* Current %busy required to uplock */
> + u8 down_threshold; /* Current %busy required to downclock */
> + } power;
> +
> + atomic_t num_waiters;
> + atomic_t boosts;
> +
> + /* manual wa residency calculations */
> + struct intel_rps_ei ei;
> + struct intel_ips ips;
> +};
> +
> +#endif /* INTEL_RPS_TYPES_H */
> diff --git a/drivers/gpu/drm/i915/gt/selftest_llc.c b/drivers/gpu/drm/i915/gt/selftest_llc.c
> index a7057785e420..fd3770e48ac7 100644
> --- a/drivers/gpu/drm/i915/gt/selftest_llc.c
> +++ b/drivers/gpu/drm/i915/gt/selftest_llc.c
> @@ -6,6 +6,7 @@
>
> #include "intel_pm.h" /* intel_gpu_freq() */
> #include "selftest_llc.h"
> +#include "intel_rps.h"
>
> static int gen6_verify_ring_freq(struct intel_llc *llc)
> {
> @@ -25,6 +26,8 @@ static int gen6_verify_ring_freq(struct intel_llc *llc)
> for (gpu_freq = consts.min_gpu_freq;
> gpu_freq <= consts.max_gpu_freq;
> gpu_freq++) {
> + struct intel_rps *rps = &llc_to_gt(llc)->rps;
> +
> unsigned int ia_freq, ring_freq, found;
> u32 val;
>
> @@ -44,7 +47,7 @@ static int gen6_verify_ring_freq(struct intel_llc *llc)
> if (found != ia_freq) {
> pr_err("Min freq table(%d/[%d, %d]):%dMHz did not match expected CPU freq, found %d, expected %d\n",
> gpu_freq, consts.min_gpu_freq, consts.max_gpu_freq,
> - intel_gpu_freq(i915, gpu_freq * (INTEL_GEN(i915) >= 9 ? GEN9_FREQ_SCALER : 1)),
> + intel_gpu_freq(rps, gpu_freq * (INTEL_GEN(i915) >= 9 ? GEN9_FREQ_SCALER : 1)),
> found, ia_freq);
> err = -EINVAL;
> break;
> @@ -54,7 +57,7 @@ static int gen6_verify_ring_freq(struct intel_llc *llc)
> if (found != ring_freq) {
> pr_err("Min freq table(%d/[%d, %d]):%dMHz did not match expected ring freq, found %d, expected %d\n",
> gpu_freq, consts.min_gpu_freq, consts.max_gpu_freq,
> - intel_gpu_freq(i915, gpu_freq * (INTEL_GEN(i915) >= 9 ? GEN9_FREQ_SCALER : 1)),
> + intel_gpu_freq(rps, gpu_freq * (INTEL_GEN(i915) >= 9 ? GEN9_FREQ_SCALER : 1)),
> found, ring_freq);
> err = -EINVAL;
> break;
> diff --git a/drivers/gpu/drm/i915/gt/uc/intel_guc_submission.c b/drivers/gpu/drm/i915/gt/uc/intel_guc_submission.c
> index 009e54a3764f..97b858ce7682 100644
> --- a/drivers/gpu/drm/i915/gt/uc/intel_guc_submission.c
> +++ b/drivers/gpu/drm/i915/gt/uc/intel_guc_submission.c
> @@ -1010,7 +1010,7 @@ void intel_guc_submission_fini(struct intel_guc *guc)
>
> static void guc_interrupts_capture(struct intel_gt *gt)
> {
> - struct intel_rps *rps = >->i915->gt_pm.rps;
> + struct intel_rps *rps = >->rps;
> struct intel_uncore *uncore = gt->uncore;
> struct intel_engine_cs *engine;
> enum intel_engine_id id;
> @@ -1056,7 +1056,7 @@ static void guc_interrupts_capture(struct intel_gt *gt)
>
> static void guc_interrupts_release(struct intel_gt *gt)
> {
> - struct intel_rps *rps = >->i915->gt_pm.rps;
> + struct intel_rps *rps = >->rps;
> struct intel_uncore *uncore = gt->uncore;
> struct intel_engine_cs *engine;
> enum intel_engine_id id;
> diff --git a/drivers/gpu/drm/i915/i915_debugfs.c b/drivers/gpu/drm/i915/i915_debugfs.c
> index 16211430eb78..22962373723b 100644
> --- a/drivers/gpu/drm/i915/i915_debugfs.c
> +++ b/drivers/gpu/drm/i915/i915_debugfs.c
> @@ -44,6 +44,7 @@
> #include "gt/intel_gt_requests.h"
> #include "gt/intel_reset.h"
> #include "gt/intel_rc6.h"
> +#include "gt/intel_rps.h"
> #include "gt/uc/intel_guc_submission.h"
>
> #include "i915_debugfs.h"
> @@ -791,7 +792,7 @@ static int i915_frequency_info(struct seq_file *m, void *unused)
> {
> struct drm_i915_private *dev_priv = node_to_i915(m->private);
> struct intel_uncore *uncore = &dev_priv->uncore;
> - struct intel_rps *rps = &dev_priv->gt_pm.rps;
> + struct intel_rps *rps = &dev_priv->gt.rps;
> intel_wakeref_t wakeref;
> int ret = 0;
>
> @@ -827,23 +828,23 @@ static int i915_frequency_info(struct seq_file *m, void *unused)
> seq_printf(m, "DDR freq: %d MHz\n", dev_priv->mem_freq);
>
> seq_printf(m, "actual GPU freq: %d MHz\n",
> - intel_gpu_freq(dev_priv, (freq_sts >> 8) & 0xff));
> + intel_gpu_freq(rps, (freq_sts >> 8) & 0xff));
>
> seq_printf(m, "current GPU freq: %d MHz\n",
> - intel_gpu_freq(dev_priv, rps->cur_freq));
> + intel_gpu_freq(rps, rps->cur_freq));
>
> seq_printf(m, "max GPU freq: %d MHz\n",
> - intel_gpu_freq(dev_priv, rps->max_freq));
> + intel_gpu_freq(rps, rps->max_freq));
>
> seq_printf(m, "min GPU freq: %d MHz\n",
> - intel_gpu_freq(dev_priv, rps->min_freq));
> + intel_gpu_freq(rps, rps->min_freq));
>
> seq_printf(m, "idle GPU freq: %d MHz\n",
> - intel_gpu_freq(dev_priv, rps->idle_freq));
> + intel_gpu_freq(rps, rps->idle_freq));
>
> seq_printf(m,
> "efficient (RPe) frequency: %d MHz\n",
> - intel_gpu_freq(dev_priv, rps->efficient_freq));
> + intel_gpu_freq(rps, rps->efficient_freq));
> } else if (INTEL_GEN(dev_priv) >= 6) {
> u32 rp_state_limits;
> u32 gt_perf_status;
> @@ -877,7 +878,7 @@ static int i915_frequency_info(struct seq_file *m, void *unused)
> else
> reqf >>= 25;
> }
> - reqf = intel_gpu_freq(dev_priv, reqf);
> + reqf = intel_gpu_freq(rps, reqf);
>
> rpmodectl = I915_READ(GEN6_RP_CONTROL);
> rpinclimit = I915_READ(GEN6_RP_UP_THRESHOLD);
> @@ -890,8 +891,7 @@ static int i915_frequency_info(struct seq_file *m, void *unused)
> rpdownei = I915_READ(GEN6_RP_CUR_DOWN_EI) & GEN6_CURIAVG_MASK;
> rpcurdown = I915_READ(GEN6_RP_CUR_DOWN) & GEN6_CURBSYTAVG_MASK;
> rpprevdown = I915_READ(GEN6_RP_PREV_DOWN) & GEN6_CURBSYTAVG_MASK;
> - cagf = intel_gpu_freq(dev_priv,
> - intel_get_cagf(dev_priv, rpstat));
> + cagf = intel_gpu_freq(rps, intel_get_cagf(rps, rpstat));
>
> intel_uncore_forcewake_put(&dev_priv->uncore, FORCEWAKE_ALL);
>
> @@ -968,37 +968,37 @@ static int i915_frequency_info(struct seq_file *m, void *unused)
> max_freq *= (IS_GEN9_BC(dev_priv) ||
> INTEL_GEN(dev_priv) >= 10 ? GEN9_FREQ_SCALER : 1);
> seq_printf(m, "Lowest (RPN) frequency: %dMHz\n",
> - intel_gpu_freq(dev_priv, max_freq));
> + intel_gpu_freq(rps, max_freq));
>
> max_freq = (rp_state_cap & 0xff00) >> 8;
> max_freq *= (IS_GEN9_BC(dev_priv) ||
> INTEL_GEN(dev_priv) >= 10 ? GEN9_FREQ_SCALER : 1);
> seq_printf(m, "Nominal (RP1) frequency: %dMHz\n",
> - intel_gpu_freq(dev_priv, max_freq));
> + intel_gpu_freq(rps, max_freq));
>
> max_freq = (IS_GEN9_LP(dev_priv) ? rp_state_cap >> 16 :
> rp_state_cap >> 0) & 0xff;
> max_freq *= (IS_GEN9_BC(dev_priv) ||
> INTEL_GEN(dev_priv) >= 10 ? GEN9_FREQ_SCALER : 1);
> seq_printf(m, "Max non-overclocked (RP0) frequency: %dMHz\n",
> - intel_gpu_freq(dev_priv, max_freq));
> + intel_gpu_freq(rps, max_freq));
> seq_printf(m, "Max overclocked frequency: %dMHz\n",
> - intel_gpu_freq(dev_priv, rps->max_freq));
> + intel_gpu_freq(rps, rps->max_freq));
>
> seq_printf(m, "Current freq: %d MHz\n",
> - intel_gpu_freq(dev_priv, rps->cur_freq));
> + intel_gpu_freq(rps, rps->cur_freq));
> seq_printf(m, "Actual freq: %d MHz\n", cagf);
> seq_printf(m, "Idle freq: %d MHz\n",
> - intel_gpu_freq(dev_priv, rps->idle_freq));
> + intel_gpu_freq(rps, rps->idle_freq));
> seq_printf(m, "Min freq: %d MHz\n",
> - intel_gpu_freq(dev_priv, rps->min_freq));
> + intel_gpu_freq(rps, rps->min_freq));
> seq_printf(m, "Boost freq: %d MHz\n",
> - intel_gpu_freq(dev_priv, rps->boost_freq));
> + intel_gpu_freq(rps, rps->boost_freq));
> seq_printf(m, "Max freq: %d MHz\n",
> - intel_gpu_freq(dev_priv, rps->max_freq));
> + intel_gpu_freq(rps, rps->max_freq));
> seq_printf(m,
> "efficient (RPe) frequency: %d MHz\n",
> - intel_gpu_freq(dev_priv, rps->efficient_freq));
> + intel_gpu_freq(rps, rps->efficient_freq));
> } else {
> seq_puts(m, "no P-state info available\n");
> }
> @@ -1461,7 +1461,7 @@ static int i915_sr_status(struct seq_file *m, void *unused)
> static int i915_ring_freq_table(struct seq_file *m, void *unused)
> {
> struct drm_i915_private *dev_priv = node_to_i915(m->private);
> - struct intel_rps *rps = &dev_priv->gt_pm.rps;
> + struct intel_rps *rps = &dev_priv->gt.rps;
> unsigned int max_gpu_freq, min_gpu_freq;
> intel_wakeref_t wakeref;
> int gpu_freq, ia_freq;
> @@ -1486,10 +1486,11 @@ static int i915_ring_freq_table(struct seq_file *m, void *unused)
> GEN6_PCODE_READ_MIN_FREQ_TABLE,
> &ia_freq, NULL);
> seq_printf(m, "%d\t\t%d\t\t\t\t%d\n",
> - intel_gpu_freq(dev_priv, (gpu_freq *
> - (IS_GEN9_BC(dev_priv) ||
> - INTEL_GEN(dev_priv) >= 10 ?
> - GEN9_FREQ_SCALER : 1))),
> + intel_gpu_freq(rps,
> + (gpu_freq *
> + (IS_GEN9_BC(dev_priv) ||
> + INTEL_GEN(dev_priv) >= 10 ?
> + GEN9_FREQ_SCALER : 1))),
> ((ia_freq >> 0) & 0xff) * 100,
> ((ia_freq >> 8) & 0xff) * 100);
> }
> @@ -1717,7 +1718,7 @@ static const char *rps_power_to_str(unsigned int power)
> static int i915_rps_boost_info(struct seq_file *m, void *data)
> {
> struct drm_i915_private *dev_priv = node_to_i915(m->private);
> - struct intel_rps *rps = &dev_priv->gt_pm.rps;
> + struct intel_rps *rps = &dev_priv->gt.rps;
> u32 act_freq = rps->cur_freq;
> intel_wakeref_t wakeref;
>
> @@ -1729,7 +1730,7 @@ static int i915_rps_boost_info(struct seq_file *m, void *data)
> vlv_punit_put(dev_priv);
> act_freq = (act_freq >> 8) & 0xff;
> } else {
> - act_freq = intel_get_cagf(dev_priv,
> + act_freq = intel_get_cagf(rps,
> I915_READ(GEN6_RPSTAT1));
> }
> }
> @@ -1740,17 +1741,17 @@ static int i915_rps_boost_info(struct seq_file *m, void *data)
> atomic_read(&rps->num_waiters));
> seq_printf(m, "Interactive? %d\n", READ_ONCE(rps->power.interactive));
> seq_printf(m, "Frequency requested %d, actual %d\n",
> - intel_gpu_freq(dev_priv, rps->cur_freq),
> - intel_gpu_freq(dev_priv, act_freq));
> + intel_gpu_freq(rps, rps->cur_freq),
> + intel_gpu_freq(rps, act_freq));
> seq_printf(m, " min hard:%d, soft:%d; max soft:%d, hard:%d\n",
> - intel_gpu_freq(dev_priv, rps->min_freq),
> - intel_gpu_freq(dev_priv, rps->min_freq_softlimit),
> - intel_gpu_freq(dev_priv, rps->max_freq_softlimit),
> - intel_gpu_freq(dev_priv, rps->max_freq));
> + intel_gpu_freq(rps, rps->min_freq),
> + intel_gpu_freq(rps, rps->min_freq_softlimit),
> + intel_gpu_freq(rps, rps->max_freq_softlimit),
> + intel_gpu_freq(rps, rps->max_freq));
> seq_printf(m, " idle:%d, efficient:%d, boost:%d\n",
> - intel_gpu_freq(dev_priv, rps->idle_freq),
> - intel_gpu_freq(dev_priv, rps->efficient_freq),
> - intel_gpu_freq(dev_priv, rps->boost_freq));
> + intel_gpu_freq(rps, rps->idle_freq),
> + intel_gpu_freq(rps, rps->efficient_freq),
> + intel_gpu_freq(rps, rps->boost_freq));
>
> seq_printf(m, "Wait boosts: %d\n", atomic_read(&rps->boosts));
>
> diff --git a/drivers/gpu/drm/i915/i915_drv.c b/drivers/gpu/drm/i915/i915_drv.c
> index 5138d1eed306..78ac08db48a3 100644
> --- a/drivers/gpu/drm/i915/i915_drv.c
> +++ b/drivers/gpu/drm/i915/i915_drv.c
> @@ -1841,6 +1841,8 @@ static int i915_drm_resume(struct drm_device *dev)
>
> intel_dp_mst_resume(dev_priv);
>
> + intel_gt_pm_enable(&dev_priv->gt);
> +
> intel_display_resume(dev);
>
> drm_kms_helper_poll_enable(dev);
> diff --git a/drivers/gpu/drm/i915/i915_drv.h b/drivers/gpu/drm/i915/i915_drv.h
> index 8882c0908c3b..eb5460290811 100644
> --- a/drivers/gpu/drm/i915/i915_drv.h
> +++ b/drivers/gpu/drm/i915/i915_drv.h
> @@ -543,94 +543,6 @@ struct i915_suspend_saved_registers {
>
> struct vlv_s0ix_state;
>
> -struct intel_rps_ei {
> - ktime_t ktime;
> - u32 render_c0;
> - u32 media_c0;
> -};
> -
> -struct intel_rps {
> - struct mutex lock; /* protects enabling and the worker */
> -
> - /*
> - * work, interrupts_enabled and pm_iir are protected by
> - * dev_priv->irq_lock
> - */
> - struct work_struct work;
> - bool interrupts_enabled;
> - u32 pm_iir;
> -
> - /* PM interrupt bits that should never be masked */
> - u32 pm_intrmsk_mbz;
> -
> - /* Frequencies are stored in potentially platform dependent multiples.
> - * In other words, *_freq needs to be multiplied by X to be interesting.
> - * Soft limits are those which are used for the dynamic reclocking done
> - * by the driver (raise frequencies under heavy loads, and lower for
> - * lighter loads). Hard limits are those imposed by the hardware.
> - *
> - * A distinction is made for overclocking, which is never enabled by
> - * default, and is considered to be above the hard limit if it's
> - * possible at all.
> - */
> - u8 cur_freq; /* Current frequency (cached, may not == HW) */
> - u8 min_freq_softlimit; /* Minimum frequency permitted by the driver */
> - u8 max_freq_softlimit; /* Max frequency permitted by the driver */
> - u8 max_freq; /* Maximum frequency, RP0 if not overclocking */
> - u8 min_freq; /* AKA RPn. Minimum frequency */
> - u8 boost_freq; /* Frequency to request when wait boosting */
> - u8 idle_freq; /* Frequency to request when we are idle */
> - u8 efficient_freq; /* AKA RPe. Pre-determined balanced frequency */
> - u8 rp1_freq; /* "less than" RP0 power/freqency */
> - u8 rp0_freq; /* Non-overclocked max frequency. */
> - u16 gpll_ref_freq; /* vlv/chv GPLL reference frequency */
> -
> - int last_adj;
> -
> - struct {
> - struct mutex mutex;
> -
> - enum { LOW_POWER, BETWEEN, HIGH_POWER } mode;
> - unsigned int interactive;
> -
> - u8 up_threshold; /* Current %busy required to uplock */
> - u8 down_threshold; /* Current %busy required to downclock */
> - } power;
> -
> - bool enabled;
> - atomic_t num_waiters;
> - atomic_t boosts;
> -
> - /* manual wa residency calculations */
> - struct intel_rps_ei ei;
> -};
> -
> -struct intel_gen6_power_mgmt {
> - struct intel_rps rps;
> -};
> -
> -/* defined intel_pm.c */
> -extern spinlock_t mchdev_lock;
> -
> -struct intel_ilk_power_mgmt {
> - u8 cur_delay;
> - u8 min_delay;
> - u8 max_delay;
> - u8 fmax;
> - u8 fstart;
> -
> - u64 last_count1;
> - unsigned long last_time1;
> - unsigned long chipset_power;
> - u64 last_count2;
> - u64 last_time2;
> - unsigned long gfx_power;
> - u8 corr;
> -
> - int c_m;
> - int r_t;
> -};
> -
> #define MAX_L3_SLICES 2
> struct intel_l3_parity {
> u32 *remap_info[MAX_L3_SLICES];
> @@ -1067,7 +979,6 @@ struct drm_i915_private {
> u32 irq_mask;
> u32 de_irq_mask[I915_MAX_PIPES];
> };
> - u32 pm_rps_events;
> u32 pipestat_irq_mask[I915_MAX_PIPES];
>
> struct i915_hotplug hotplug;
> @@ -1202,13 +1113,6 @@ struct drm_i915_private {
> */
> u32 edram_size_mb;
>
> - /* gen6+ GT PM state */
> - struct intel_gen6_power_mgmt gt_pm;
> -
> - /* ilk-only ips/rps state. Everything in here is protected by the global
> - * mchdev_lock in intel_pm.c */
> - struct intel_ilk_power_mgmt ips;
> -
> struct i915_power_domains power_domains;
>
> struct i915_psr psr;
> diff --git a/drivers/gpu/drm/i915/i915_gem.c b/drivers/gpu/drm/i915/i915_gem.c
> index b882988056bd..eab1709d1897 100644
> --- a/drivers/gpu/drm/i915/i915_gem.c
> +++ b/drivers/gpu/drm/i915/i915_gem.c
> @@ -52,6 +52,7 @@
> #include "gt/intel_mocs.h"
> #include "gt/intel_reset.h"
> #include "gt/intel_renderstate.h"
> +#include "gt/intel_rps.h"
> #include "gt/intel_workarounds.h"
>
> #include "i915_drv.h"
> @@ -1272,8 +1273,6 @@ int i915_gem_init(struct drm_i915_private *dev_priv)
> goto err_context;
> }
>
> - intel_init_gt_powersave(dev_priv);
> -
> intel_uc_init(&dev_priv->gt.uc);
>
> ret = intel_gt_init_hw(&dev_priv->gt);
> diff --git a/drivers/gpu/drm/i915/i915_irq.c b/drivers/gpu/drm/i915/i915_irq.c
> index 572a5c37cc61..598924b3c556 100644
> --- a/drivers/gpu/drm/i915/i915_irq.c
> +++ b/drivers/gpu/drm/i915/i915_irq.c
> @@ -45,6 +45,7 @@
> #include "gt/intel_gt.h"
> #include "gt/intel_gt_irq.h"
> #include "gt/intel_gt_pm_irq.h"
> +#include "gt/intel_rps.h"
>
> #include "i915_drv.h"
> #include "i915_irq.h"
> @@ -327,87 +328,6 @@ static i915_reg_t gen6_pm_iir(struct drm_i915_private *dev_priv)
> return INTEL_GEN(dev_priv) >= 8 ? GEN8_GT_IIR(2) : GEN6_PMIIR;
> }
>
> -void gen11_reset_rps_interrupts(struct drm_i915_private *dev_priv)
> -{
> - struct intel_gt *gt = &dev_priv->gt;
> -
> - spin_lock_irq(>->irq_lock);
> -
> - while (gen11_gt_reset_one_iir(gt, 0, GEN11_GTPM))
> - ;
> -
> - dev_priv->gt_pm.rps.pm_iir = 0;
> -
> - spin_unlock_irq(>->irq_lock);
> -}
> -
> -void gen6_reset_rps_interrupts(struct drm_i915_private *dev_priv)
> -{
> - struct intel_gt *gt = &dev_priv->gt;
> -
> - spin_lock_irq(>->irq_lock);
> - gen6_gt_pm_reset_iir(gt, GEN6_PM_RPS_EVENTS);
> - dev_priv->gt_pm.rps.pm_iir = 0;
> - spin_unlock_irq(>->irq_lock);
> -}
> -
> -void gen6_enable_rps_interrupts(struct drm_i915_private *dev_priv)
> -{
> - struct intel_gt *gt = &dev_priv->gt;
> - struct intel_rps *rps = &dev_priv->gt_pm.rps;
> -
> - if (READ_ONCE(rps->interrupts_enabled))
> - return;
> -
> - spin_lock_irq(>->irq_lock);
> - WARN_ON_ONCE(rps->pm_iir);
> -
> - if (INTEL_GEN(dev_priv) >= 11)
> - WARN_ON_ONCE(gen11_gt_reset_one_iir(gt, 0, GEN11_GTPM));
> - else
> - WARN_ON_ONCE(I915_READ(gen6_pm_iir(dev_priv)) & dev_priv->pm_rps_events);
> -
> - rps->interrupts_enabled = true;
> - gen6_gt_pm_enable_irq(gt, dev_priv->pm_rps_events);
> -
> - spin_unlock_irq(>->irq_lock);
> -}
> -
> -u32 gen6_sanitize_rps_pm_mask(const struct drm_i915_private *i915, u32 mask)
> -{
> - return mask & ~i915->gt_pm.rps.pm_intrmsk_mbz;
> -}
> -
> -void gen6_disable_rps_interrupts(struct drm_i915_private *dev_priv)
> -{
> - struct intel_rps *rps = &dev_priv->gt_pm.rps;
> - struct intel_gt *gt = &dev_priv->gt;
> -
> - if (!READ_ONCE(rps->interrupts_enabled))
> - return;
> -
> - spin_lock_irq(>->irq_lock);
> - rps->interrupts_enabled = false;
> -
> - I915_WRITE(GEN6_PMINTRMSK, gen6_sanitize_rps_pm_mask(dev_priv, ~0u));
> -
> - gen6_gt_pm_disable_irq(gt, GEN6_PM_RPS_EVENTS);
> -
> - spin_unlock_irq(>->irq_lock);
> - intel_synchronize_irq(dev_priv);
> -
> - /* Now that we will not be generating any more work, flush any
> - * outstanding tasks. As we are called on the RPS idle path,
> - * we will reset the GPU to minimum frequencies, so the current
> - * state of the worker can be discarded.
> - */
> - cancel_work_sync(&rps->work);
> - if (INTEL_GEN(dev_priv) >= 11)
> - gen11_reset_rps_interrupts(dev_priv);
> - else
> - gen6_reset_rps_interrupts(dev_priv);
> -}
> -
> void gen9_reset_guc_interrupts(struct intel_guc *guc)
> {
> struct intel_gt *gt = guc_to_gt(guc);
> @@ -1065,199 +985,6 @@ int intel_get_crtc_scanline(struct intel_crtc *crtc)
> return position;
> }
>
> -static void ironlake_rps_change_irq_handler(struct drm_i915_private *dev_priv)
> -{
> - struct intel_uncore *uncore = &dev_priv->uncore;
> - u32 busy_up, busy_down, max_avg, min_avg;
> - u8 new_delay;
> -
> - spin_lock(&mchdev_lock);
> -
> - intel_uncore_write16(uncore,
> - MEMINTRSTS,
> - intel_uncore_read(uncore, MEMINTRSTS));
> -
> - new_delay = dev_priv->ips.cur_delay;
> -
> - intel_uncore_write16(uncore, MEMINTRSTS, MEMINT_EVAL_CHG);
> - busy_up = intel_uncore_read(uncore, RCPREVBSYTUPAVG);
> - busy_down = intel_uncore_read(uncore, RCPREVBSYTDNAVG);
> - max_avg = intel_uncore_read(uncore, RCBMAXAVG);
> - min_avg = intel_uncore_read(uncore, RCBMINAVG);
> -
> - /* Handle RCS change request from hw */
> - if (busy_up > max_avg) {
> - if (dev_priv->ips.cur_delay != dev_priv->ips.max_delay)
> - new_delay = dev_priv->ips.cur_delay - 1;
> - if (new_delay < dev_priv->ips.max_delay)
> - new_delay = dev_priv->ips.max_delay;
> - } else if (busy_down < min_avg) {
> - if (dev_priv->ips.cur_delay != dev_priv->ips.min_delay)
> - new_delay = dev_priv->ips.cur_delay + 1;
> - if (new_delay > dev_priv->ips.min_delay)
> - new_delay = dev_priv->ips.min_delay;
> - }
> -
> - if (ironlake_set_drps(dev_priv, new_delay))
> - dev_priv->ips.cur_delay = new_delay;
> -
> - spin_unlock(&mchdev_lock);
> -
> - return;
> -}
> -
> -static void vlv_c0_read(struct drm_i915_private *dev_priv,
> - struct intel_rps_ei *ei)
> -{
> - ei->ktime = ktime_get_raw();
> - ei->render_c0 = I915_READ(VLV_RENDER_C0_COUNT);
> - ei->media_c0 = I915_READ(VLV_MEDIA_C0_COUNT);
> -}
> -
> -void gen6_rps_reset_ei(struct drm_i915_private *dev_priv)
> -{
> - memset(&dev_priv->gt_pm.rps.ei, 0, sizeof(dev_priv->gt_pm.rps.ei));
> -}
> -
> -static u32 vlv_wa_c0_ei(struct drm_i915_private *dev_priv, u32 pm_iir)
> -{
> - struct intel_rps *rps = &dev_priv->gt_pm.rps;
> - const struct intel_rps_ei *prev = &rps->ei;
> - struct intel_rps_ei now;
> - u32 events = 0;
> -
> - if ((pm_iir & GEN6_PM_RP_UP_EI_EXPIRED) == 0)
> - return 0;
> -
> - vlv_c0_read(dev_priv, &now);
> -
> - if (prev->ktime) {
> - u64 time, c0;
> - u32 render, media;
> -
> - time = ktime_us_delta(now.ktime, prev->ktime);
> -
> - time *= dev_priv->czclk_freq;
> -
> - /* Workload can be split between render + media,
> - * e.g. SwapBuffers being blitted in X after being rendered in
> - * mesa. To account for this we need to combine both engines
> - * into our activity counter.
> - */
> - render = now.render_c0 - prev->render_c0;
> - media = now.media_c0 - prev->media_c0;
> - c0 = max(render, media);
> - c0 *= 1000 * 100 << 8; /* to usecs and scale to threshold% */
> -
> - if (c0 > time * rps->power.up_threshold)
> - events = GEN6_PM_RP_UP_THRESHOLD;
> - else if (c0 < time * rps->power.down_threshold)
> - events = GEN6_PM_RP_DOWN_THRESHOLD;
> - }
> -
> - rps->ei = now;
> - return events;
> -}
> -
> -static void gen6_pm_rps_work(struct work_struct *work)
> -{
> - struct drm_i915_private *dev_priv =
> - container_of(work, struct drm_i915_private, gt_pm.rps.work);
> - struct intel_gt *gt = &dev_priv->gt;
> - struct intel_rps *rps = &dev_priv->gt_pm.rps;
> - bool client_boost = false;
> - int new_delay, adj, min, max;
> - u32 pm_iir = 0;
> -
> - spin_lock_irq(>->irq_lock);
> - if (rps->interrupts_enabled) {
> - pm_iir = fetch_and_zero(&rps->pm_iir);
> - client_boost = atomic_read(&rps->num_waiters);
> - }
> - spin_unlock_irq(>->irq_lock);
> -
> - /* Make sure we didn't queue anything we're not going to process. */
> - WARN_ON(pm_iir & ~dev_priv->pm_rps_events);
> - if ((pm_iir & dev_priv->pm_rps_events) == 0 && !client_boost)
> - goto out;
> -
> - mutex_lock(&rps->lock);
> -
> - pm_iir |= vlv_wa_c0_ei(dev_priv, pm_iir);
> -
> - adj = rps->last_adj;
> - new_delay = rps->cur_freq;
> - min = rps->min_freq_softlimit;
> - max = rps->max_freq_softlimit;
> - if (client_boost)
> - max = rps->max_freq;
> - if (client_boost && new_delay < rps->boost_freq) {
> - new_delay = rps->boost_freq;
> - adj = 0;
> - } else if (pm_iir & GEN6_PM_RP_UP_THRESHOLD) {
> - if (adj > 0)
> - adj *= 2;
> - else /* CHV needs even encode values */
> - adj = IS_CHERRYVIEW(dev_priv) ? 2 : 1;
> -
> - if (new_delay >= rps->max_freq_softlimit)
> - adj = 0;
> - } else if (client_boost) {
> - adj = 0;
> - } else if (pm_iir & GEN6_PM_RP_DOWN_TIMEOUT) {
> - if (rps->cur_freq > rps->efficient_freq)
> - new_delay = rps->efficient_freq;
> - else if (rps->cur_freq > rps->min_freq_softlimit)
> - new_delay = rps->min_freq_softlimit;
> - adj = 0;
> - } else if (pm_iir & GEN6_PM_RP_DOWN_THRESHOLD) {
> - if (adj < 0)
> - adj *= 2;
> - else /* CHV needs even encode values */
> - adj = IS_CHERRYVIEW(dev_priv) ? -2 : -1;
> -
> - if (new_delay <= rps->min_freq_softlimit)
> - adj = 0;
> - } else { /* unknown event */
> - adj = 0;
> - }
> -
> - rps->last_adj = adj;
> -
> - /*
> - * Limit deboosting and boosting to keep ourselves at the extremes
> - * when in the respective power modes (i.e. slowly decrease frequencies
> - * while in the HIGH_POWER zone and slowly increase frequencies while
> - * in the LOW_POWER zone). On idle, we will hit the timeout and drop
> - * to the next level quickly, and conversely if busy we expect to
> - * hit a waitboost and rapidly switch into max power.
> - */
> - if ((adj < 0 && rps->power.mode == HIGH_POWER) ||
> - (adj > 0 && rps->power.mode == LOW_POWER))
> - rps->last_adj = 0;
> -
> - /* sysfs frequency interfaces may have snuck in while servicing the
> - * interrupt
> - */
> - new_delay += adj;
> - new_delay = clamp_t(int, new_delay, min, max);
> -
> - if (intel_set_rps(dev_priv, new_delay)) {
> - DRM_DEBUG_DRIVER("Failed to set new GPU frequency\n");
> - rps->last_adj = 0;
> - }
> -
> - mutex_unlock(&rps->lock);
> -
> -out:
> - /* Make sure not to corrupt PMIMR state used by ringbuffer on GEN6 */
> - spin_lock_irq(>->irq_lock);
> - if (rps->interrupts_enabled)
> - gen6_gt_pm_unmask_irq(gt, dev_priv->pm_rps_events);
> - spin_unlock_irq(>->irq_lock);
> -}
> -
> -
> /**
> * ivybridge_parity_work - Workqueue called when a parity error interrupt
> * occurred.
> @@ -1631,54 +1358,6 @@ static void i9xx_pipe_crc_irq_handler(struct drm_i915_private *dev_priv,
> res1, res2);
> }
>
> -/* The RPS events need forcewake, so we add them to a work queue and mask their
> - * IMR bits until the work is done. Other interrupts can be processed without
> - * the work queue. */
> -void gen11_rps_irq_handler(struct intel_gt *gt, u32 pm_iir)
> -{
> - struct drm_i915_private *i915 = gt->i915;
> - struct intel_rps *rps = &i915->gt_pm.rps;
> - const u32 events = i915->pm_rps_events & pm_iir;
> -
> - lockdep_assert_held(>->irq_lock);
> -
> - if (unlikely(!events))
> - return;
> -
> - gen6_gt_pm_mask_irq(gt, events);
> -
> - if (!rps->interrupts_enabled)
> - return;
> -
> - rps->pm_iir |= events;
> - schedule_work(&rps->work);
> -}
> -
> -void gen6_rps_irq_handler(struct drm_i915_private *dev_priv, u32 pm_iir)
> -{
> - struct intel_rps *rps = &dev_priv->gt_pm.rps;
> - struct intel_gt *gt = &dev_priv->gt;
> -
> - if (pm_iir & dev_priv->pm_rps_events) {
> - spin_lock(>->irq_lock);
> - gen6_gt_pm_mask_irq(gt, pm_iir & dev_priv->pm_rps_events);
> - if (rps->interrupts_enabled) {
> - rps->pm_iir |= pm_iir & dev_priv->pm_rps_events;
> - schedule_work(&rps->work);
> - }
> - spin_unlock(>->irq_lock);
> - }
> -
> - if (INTEL_GEN(dev_priv) >= 8)
> - return;
> -
> - if (pm_iir & PM_VEBOX_USER_INTERRUPT)
> - intel_engine_breadcrumbs_irq(dev_priv->engine[VECS0]);
> -
> - if (pm_iir & PM_VEBOX_CS_ERROR_INTERRUPT)
> - DRM_DEBUG("Command parser error, pm_iir 0x%08x\n", pm_iir);
> -}
> -
> static void i9xx_pipestat_irq_reset(struct drm_i915_private *dev_priv)
> {
> enum pipe pipe;
> @@ -1989,7 +1668,7 @@ static irqreturn_t valleyview_irq_handler(int irq, void *arg)
> if (gt_iir)
> gen6_gt_irq_handler(&dev_priv->gt, gt_iir);
> if (pm_iir)
> - gen6_rps_irq_handler(dev_priv, pm_iir);
> + gen6_rps_irq_handler(&dev_priv->gt.rps, pm_iir);
>
> if (hotplug_status)
> i9xx_hpd_irq_handler(dev_priv, hotplug_status);
> @@ -2393,7 +2072,7 @@ static void ilk_display_irq_handler(struct drm_i915_private *dev_priv,
> }
>
> if (IS_GEN(dev_priv, 5) && de_iir & DE_PCU_EVENT)
> - ironlake_rps_change_irq_handler(dev_priv);
> + gen5_rps_irq_handler(&dev_priv->gt.rps);
> }
>
> static void ivb_display_irq_handler(struct drm_i915_private *dev_priv,
> @@ -2498,7 +2177,7 @@ static irqreturn_t ironlake_irq_handler(int irq, void *arg)
> if (pm_iir) {
> I915_WRITE(GEN6_PMIIR, pm_iir);
> ret = IRQ_HANDLED;
> - gen6_rps_irq_handler(dev_priv, pm_iir);
> + gen6_rps_irq_handler(&dev_priv->gt.rps, pm_iir);
> }
> }
>
> @@ -4270,13 +3949,10 @@ static irqreturn_t i965_irq_handler(int irq, void *arg)
> void intel_irq_init(struct drm_i915_private *dev_priv)
> {
> struct drm_device *dev = &dev_priv->drm;
> - struct intel_rps *rps = &dev_priv->gt_pm.rps;
> int i;
>
> intel_hpd_init_work(dev_priv);
>
> - INIT_WORK(&rps->work, gen6_pm_rps_work);
> -
> INIT_WORK(&dev_priv->l3_parity.error_work, ivybridge_parity_work);
> for (i = 0; i < MAX_L3_SLICES; ++i)
> dev_priv->l3_parity.remap_info[i] = NULL;
> @@ -4285,33 +3961,6 @@ void intel_irq_init(struct drm_i915_private *dev_priv)
> if (HAS_GT_UC(dev_priv) && INTEL_GEN(dev_priv) < 11)
> dev_priv->gt.pm_guc_events = GUC_INTR_GUC2HOST << 16;
>
> - /* Let's track the enabled rps events */
> - if (IS_VALLEYVIEW(dev_priv))
> - /* WaGsvRC0ResidencyMethod:vlv */
> - dev_priv->pm_rps_events = GEN6_PM_RP_UP_EI_EXPIRED;
> - else
> - dev_priv->pm_rps_events = (GEN6_PM_RP_UP_THRESHOLD |
> - GEN6_PM_RP_DOWN_THRESHOLD |
> - GEN6_PM_RP_DOWN_TIMEOUT);
> -
> - /* We share the register with other engine */
> - if (INTEL_GEN(dev_priv) > 9)
> - GEM_WARN_ON(dev_priv->pm_rps_events & 0xffff0000);
> -
> - rps->pm_intrmsk_mbz = 0;
> -
> - /*
> - * SNB,IVB,HSW can while VLV,CHV may hard hang on looping batchbuffer
> - * if GEN6_PM_UP_EI_EXPIRED is masked.
> - *
> - * TODO: verify if this can be reproduced on VLV,CHV.
> - */
> - if (INTEL_GEN(dev_priv) <= 7)
> - rps->pm_intrmsk_mbz |= GEN6_PM_RP_UP_EI_EXPIRED;
> -
> - if (INTEL_GEN(dev_priv) >= 8)
> - rps->pm_intrmsk_mbz |= GEN8_PMINTR_DISABLE_REDIRECT_TO_GUC;
> -
> dev->vblank_disable_immediate = true;
>
> /* Most platforms treat the display irq block as an always-on
> diff --git a/drivers/gpu/drm/i915/i915_irq.h b/drivers/gpu/drm/i915/i915_irq.h
> index 19a3bc019535..d0d91c6e00d7 100644
> --- a/drivers/gpu/drm/i915/i915_irq.h
> +++ b/drivers/gpu/drm/i915/i915_irq.h
> @@ -22,9 +22,6 @@ struct intel_gt;
> struct intel_guc;
> struct intel_uncore;
>
> -void gen11_rps_irq_handler(struct intel_gt *gt, u32 pm_iir);
> -void gen6_rps_irq_handler(struct drm_i915_private *dev_priv, u32 pm_iir);
> -
> void intel_irq_init(struct drm_i915_private *dev_priv);
> void intel_irq_fini(struct drm_i915_private *dev_priv);
> int intel_irq_install(struct drm_i915_private *dev_priv);
> diff --git a/drivers/gpu/drm/i915/i915_pmu.c b/drivers/gpu/drm/i915/i915_pmu.c
> index 85912917c062..266d66b1fb7b 100644
> --- a/drivers/gpu/drm/i915/i915_pmu.c
> +++ b/drivers/gpu/drm/i915/i915_pmu.c
> @@ -12,6 +12,7 @@
> #include "gt/intel_engine_user.h"
> #include "gt/intel_gt_pm.h"
> #include "gt/intel_rc6.h"
> +#include "gt/intel_rps.h"
>
> #include "i915_drv.h"
> #include "i915_pmu.h"
> @@ -358,25 +359,26 @@ frequency_sample(struct intel_gt *gt, unsigned int period_ns)
> struct drm_i915_private *i915 = gt->i915;
> struct intel_uncore *uncore = gt->uncore;
> struct i915_pmu *pmu = &i915->pmu;
> + struct intel_rps *rps = >->rps;
>
> if (pmu->enable & config_enabled_mask(I915_PMU_ACTUAL_FREQUENCY)) {
> u32 val;
>
> - val = i915->gt_pm.rps.cur_freq;
> + val = rps->cur_freq;
> if (intel_gt_pm_get_if_awake(gt)) {
> val = intel_uncore_read_notrace(uncore, GEN6_RPSTAT1);
> - val = intel_get_cagf(i915, val);
> + val = intel_get_cagf(rps, val);
> intel_gt_pm_put(gt);
> }
>
> add_sample_mult(&pmu->sample[__I915_SAMPLE_FREQ_ACT],
> - intel_gpu_freq(i915, val),
> + intel_gpu_freq(rps, val),
> period_ns / 1000);
> }
>
> if (pmu->enable & config_enabled_mask(I915_PMU_REQUESTED_FREQUENCY)) {
> add_sample_mult(&pmu->sample[__I915_SAMPLE_FREQ_REQ],
> - intel_gpu_freq(i915, i915->gt_pm.rps.cur_freq),
> + intel_gpu_freq(rps, rps->cur_freq),
> period_ns / 1000);
> }
> }
> diff --git a/drivers/gpu/drm/i915/i915_request.c b/drivers/gpu/drm/i915/i915_request.c
> index 4575f368455d..08ce2eeecf7e 100644
> --- a/drivers/gpu/drm/i915/i915_request.c
> +++ b/drivers/gpu/drm/i915/i915_request.c
> @@ -31,6 +31,7 @@
>
> #include "gem/i915_gem_context.h"
> #include "gt/intel_context.h"
> +#include "gt/intel_rps.h"
>
> #include "i915_active.h"
> #include "i915_drv.h"
> @@ -257,8 +258,8 @@ bool i915_request_retire(struct i915_request *rq)
> if (test_bit(DMA_FENCE_FLAG_ENABLE_SIGNAL_BIT, &rq->fence.flags))
> i915_request_cancel_breadcrumb(rq);
> if (i915_request_has_waitboost(rq)) {
> - GEM_BUG_ON(!atomic_read(&rq->i915->gt_pm.rps.num_waiters));
> - atomic_dec(&rq->i915->gt_pm.rps.num_waiters);
> + GEM_BUG_ON(!atomic_read(&rq->engine->gt->rps.num_waiters));
> + atomic_dec(&rq->engine->gt->rps.num_waiters);
> }
> if (!test_bit(I915_FENCE_FLAG_ACTIVE, &rq->fence.flags)) {
> set_bit(I915_FENCE_FLAG_ACTIVE, &rq->fence.flags);
> @@ -1466,7 +1467,7 @@ long i915_request_wait(struct i915_request *rq,
> */
> if (flags & I915_WAIT_PRIORITY) {
> if (!i915_request_started(rq) && INTEL_GEN(rq->i915) >= 6)
> - gen6_rps_boost(rq);
> + intel_rps_boost(rq);
> i915_schedule_bump_priority(rq, I915_PRIORITY_WAIT);
> }
>
> diff --git a/drivers/gpu/drm/i915/i915_sysfs.c b/drivers/gpu/drm/i915/i915_sysfs.c
> index bf039b8ba593..65476909d1bf 100644
> --- a/drivers/gpu/drm/i915/i915_sysfs.c
> +++ b/drivers/gpu/drm/i915/i915_sysfs.c
> @@ -31,6 +31,7 @@
> #include <linux/sysfs.h>
>
> #include "gt/intel_rc6.h"
> +#include "gt/intel_rps.h"
>
> #include "i915_drv.h"
> #include "i915_sysfs.h"
> @@ -259,6 +260,7 @@ static ssize_t gt_act_freq_mhz_show(struct device *kdev,
> struct device_attribute *attr, char *buf)
> {
> struct drm_i915_private *dev_priv = kdev_minor_to_i915(kdev);
> + struct intel_rps *rps = &dev_priv->gt.rps;
> intel_wakeref_t wakeref;
> u32 freq;
>
> @@ -271,31 +273,31 @@ static ssize_t gt_act_freq_mhz_show(struct device *kdev,
>
> freq = (freq >> 8) & 0xff;
> } else {
> - freq = intel_get_cagf(dev_priv, I915_READ(GEN6_RPSTAT1));
> + freq = intel_get_cagf(rps, I915_READ(GEN6_RPSTAT1));
> }
>
> intel_runtime_pm_put(&dev_priv->runtime_pm, wakeref);
>
> - return snprintf(buf, PAGE_SIZE, "%d\n", intel_gpu_freq(dev_priv, freq));
> + return snprintf(buf, PAGE_SIZE, "%d\n", intel_gpu_freq(rps, freq));
> }
>
> static ssize_t gt_cur_freq_mhz_show(struct device *kdev,
> struct device_attribute *attr, char *buf)
> {
> struct drm_i915_private *dev_priv = kdev_minor_to_i915(kdev);
> + struct intel_rps *rps = &dev_priv->gt.rps;
>
> return snprintf(buf, PAGE_SIZE, "%d\n",
> - intel_gpu_freq(dev_priv,
> - dev_priv->gt_pm.rps.cur_freq));
> + intel_gpu_freq(rps, rps->cur_freq));
> }
>
> static ssize_t gt_boost_freq_mhz_show(struct device *kdev, struct device_attribute *attr, char *buf)
> {
> struct drm_i915_private *dev_priv = kdev_minor_to_i915(kdev);
> + struct intel_rps *rps = &dev_priv->gt.rps;
>
> return snprintf(buf, PAGE_SIZE, "%d\n",
> - intel_gpu_freq(dev_priv,
> - dev_priv->gt_pm.rps.boost_freq));
> + intel_gpu_freq(rps, rps->boost_freq));
> }
>
> static ssize_t gt_boost_freq_mhz_store(struct device *kdev,
> @@ -303,7 +305,7 @@ static ssize_t gt_boost_freq_mhz_store(struct device *kdev,
> const char *buf, size_t count)
> {
> struct drm_i915_private *dev_priv = kdev_minor_to_i915(kdev);
> - struct intel_rps *rps = &dev_priv->gt_pm.rps;
> + struct intel_rps *rps = &dev_priv->gt.rps;
> bool boost = false;
> ssize_t ret;
> u32 val;
> @@ -313,7 +315,7 @@ static ssize_t gt_boost_freq_mhz_store(struct device *kdev,
> return ret;
>
> /* Validate against (static) hardware limits */
> - val = intel_freq_opcode(dev_priv, val);
> + val = intel_freq_opcode(rps, val);
> if (val < rps->min_freq || val > rps->max_freq)
> return -EINVAL;
>
> @@ -333,19 +335,19 @@ static ssize_t vlv_rpe_freq_mhz_show(struct device *kdev,
> struct device_attribute *attr, char *buf)
> {
> struct drm_i915_private *dev_priv = kdev_minor_to_i915(kdev);
> + struct intel_rps *rps = &dev_priv->gt.rps;
>
> return snprintf(buf, PAGE_SIZE, "%d\n",
> - intel_gpu_freq(dev_priv,
> - dev_priv->gt_pm.rps.efficient_freq));
> + intel_gpu_freq(rps, rps->efficient_freq));
> }
>
> static ssize_t gt_max_freq_mhz_show(struct device *kdev, struct device_attribute *attr, char *buf)
> {
> struct drm_i915_private *dev_priv = kdev_minor_to_i915(kdev);
> + struct intel_rps *rps = &dev_priv->gt.rps;
>
> return snprintf(buf, PAGE_SIZE, "%d\n",
> - intel_gpu_freq(dev_priv,
> - dev_priv->gt_pm.rps.max_freq_softlimit));
> + intel_gpu_freq(rps, rps->max_freq_softlimit));
> }
>
> static ssize_t gt_max_freq_mhz_store(struct device *kdev,
> @@ -353,19 +355,17 @@ static ssize_t gt_max_freq_mhz_store(struct device *kdev,
> const char *buf, size_t count)
> {
> struct drm_i915_private *dev_priv = kdev_minor_to_i915(kdev);
> - struct intel_rps *rps = &dev_priv->gt_pm.rps;
> - intel_wakeref_t wakeref;
> - u32 val;
> + struct intel_rps *rps = &dev_priv->gt.rps;
> ssize_t ret;
> + u32 val;
>
> ret = kstrtou32(buf, 0, &val);
> if (ret)
> return ret;
>
> - wakeref = intel_runtime_pm_get(&dev_priv->runtime_pm);
> mutex_lock(&rps->lock);
>
> - val = intel_freq_opcode(dev_priv, val);
> + val = intel_freq_opcode(rps, val);
> if (val < rps->min_freq ||
> val > rps->max_freq ||
> val < rps->min_freq_softlimit) {
> @@ -375,7 +375,7 @@ static ssize_t gt_max_freq_mhz_store(struct device *kdev,
>
> if (val > rps->rp0_freq)
> DRM_DEBUG("User requested overclocking to %d\n",
> - intel_gpu_freq(dev_priv, val));
> + intel_gpu_freq(rps, val));
>
> rps->max_freq_softlimit = val;
>
> @@ -383,14 +383,15 @@ static ssize_t gt_max_freq_mhz_store(struct device *kdev,
> rps->min_freq_softlimit,
> rps->max_freq_softlimit);
>
> - /* We still need *_set_rps to process the new max_delay and
> + /*
> + * We still need *_set_rps to process the new max_delay and
> * update the interrupt limits and PMINTRMSK even though
> - * frequency request may be unchanged. */
> - ret = intel_set_rps(dev_priv, val);
> + * frequency request may be unchanged.
> + */
> + intel_rps_set(rps, val);
>
> unlock:
> mutex_unlock(&rps->lock);
> - intel_runtime_pm_put(&dev_priv->runtime_pm, wakeref);
>
> return ret ?: count;
> }
> @@ -398,10 +399,10 @@ static ssize_t gt_max_freq_mhz_store(struct device *kdev,
> static ssize_t gt_min_freq_mhz_show(struct device *kdev, struct device_attribute *attr, char *buf)
> {
> struct drm_i915_private *dev_priv = kdev_minor_to_i915(kdev);
> + struct intel_rps *rps = &dev_priv->gt.rps;
>
> return snprintf(buf, PAGE_SIZE, "%d\n",
> - intel_gpu_freq(dev_priv,
> - dev_priv->gt_pm.rps.min_freq_softlimit));
> + intel_gpu_freq(rps, rps->min_freq_softlimit));
> }
>
> static ssize_t gt_min_freq_mhz_store(struct device *kdev,
> @@ -409,19 +410,17 @@ static ssize_t gt_min_freq_mhz_store(struct device *kdev,
> const char *buf, size_t count)
> {
> struct drm_i915_private *dev_priv = kdev_minor_to_i915(kdev);
> - struct intel_rps *rps = &dev_priv->gt_pm.rps;
> - intel_wakeref_t wakeref;
> - u32 val;
> + struct intel_rps *rps = &dev_priv->gt.rps;
> ssize_t ret;
> + u32 val;
>
> ret = kstrtou32(buf, 0, &val);
> if (ret)
> return ret;
>
> - wakeref = intel_runtime_pm_get(&dev_priv->runtime_pm);
> mutex_lock(&rps->lock);
>
> - val = intel_freq_opcode(dev_priv, val);
> + val = intel_freq_opcode(rps, val);
> if (val < rps->min_freq ||
> val > rps->max_freq ||
> val > rps->max_freq_softlimit) {
> @@ -435,14 +434,15 @@ static ssize_t gt_min_freq_mhz_store(struct device *kdev,
> rps->min_freq_softlimit,
> rps->max_freq_softlimit);
>
> - /* We still need *_set_rps to process the new min_delay and
> + /*
> + * We still need *_set_rps to process the new min_delay and
> * update the interrupt limits and PMINTRMSK even though
> - * frequency request may be unchanged. */
> - ret = intel_set_rps(dev_priv, val);
> + * frequency request may be unchanged.
> + */
> + intel_rps_set(rps, val);
>
> unlock:
> mutex_unlock(&rps->lock);
> - intel_runtime_pm_put(&dev_priv->runtime_pm, wakeref);
>
> return ret ?: count;
> }
> @@ -464,15 +464,15 @@ static DEVICE_ATTR(gt_RPn_freq_mhz, S_IRUGO, gt_rp_mhz_show, NULL);
> static ssize_t gt_rp_mhz_show(struct device *kdev, struct device_attribute *attr, char *buf)
> {
> struct drm_i915_private *dev_priv = kdev_minor_to_i915(kdev);
> - struct intel_rps *rps = &dev_priv->gt_pm.rps;
> + struct intel_rps *rps = &dev_priv->gt.rps;
> u32 val;
>
> if (attr == &dev_attr_gt_RP0_freq_mhz)
> - val = intel_gpu_freq(dev_priv, rps->rp0_freq);
> + val = intel_gpu_freq(rps, rps->rp0_freq);
> else if (attr == &dev_attr_gt_RP1_freq_mhz)
> - val = intel_gpu_freq(dev_priv, rps->rp1_freq);
> + val = intel_gpu_freq(rps, rps->rp1_freq);
> else if (attr == &dev_attr_gt_RPn_freq_mhz)
> - val = intel_gpu_freq(dev_priv, rps->min_freq);
> + val = intel_gpu_freq(rps, rps->min_freq);
> else
> BUG();
>
> diff --git a/drivers/gpu/drm/i915/intel_pm.c b/drivers/gpu/drm/i915/intel_pm.c
> index 362234449087..6741507c74f3 100644
> --- a/drivers/gpu/drm/i915/intel_pm.c
> +++ b/drivers/gpu/drm/i915/intel_pm.c
> @@ -197,8 +197,6 @@ static void i915_ironlake_get_mem_freq(struct drm_i915_private *dev_priv)
> break;
> }
>
> - dev_priv->ips.r_t = dev_priv->mem_freq;
> -
> switch (csipll & 0x3ff) {
> case 0x00c:
> dev_priv->fsb_freq = 3200;
> @@ -227,14 +225,6 @@ static void i915_ironlake_get_mem_freq(struct drm_i915_private *dev_priv)
> dev_priv->fsb_freq = 0;
> break;
> }
> -
> - if (dev_priv->fsb_freq == 3200) {
> - dev_priv->ips.c_m = 0;
> - } else if (dev_priv->fsb_freq > 3200 && dev_priv->fsb_freq <= 4800) {
> - dev_priv->ips.c_m = 1;
> - } else {
> - dev_priv->ips.c_m = 2;
> - }
> }
>
> static const struct cxsr_latency cxsr_latency_table[] = {
> @@ -6339,1865 +6329,258 @@ void intel_init_ipc(struct drm_i915_private *dev_priv)
> intel_enable_ipc(dev_priv);
> }
>
> -/*
> - * Lock protecting IPS related data structures
> - */
> -DEFINE_SPINLOCK(mchdev_lock);
> +static const struct cparams {
> + u16 i;
> + u16 t;
> + u16 m;
> + u16 c;
> +} cparams[] = {
> + { 1, 1333, 301, 28664 },
> + { 1, 1066, 294, 24460 },
> + { 1, 800, 294, 25192 },
> + { 0, 1333, 276, 27605 },
> + { 0, 1066, 276, 27605 },
> + { 0, 800, 231, 23784 },
> +};
>
> -bool ironlake_set_drps(struct drm_i915_private *i915, u8 val)
> +static void ibx_init_clock_gating(struct drm_i915_private *dev_priv)
> {
> - struct intel_uncore *uncore = &i915->uncore;
> - u16 rgvswctl;
> -
> - lockdep_assert_held(&mchdev_lock);
> -
> - rgvswctl = intel_uncore_read16(uncore, MEMSWCTL);
> - if (rgvswctl & MEMCTL_CMD_STS) {
> - DRM_DEBUG("gpu busy, RCS change rejected\n");
> - return false; /* still busy with another command */
> - }
> -
> - rgvswctl = (MEMCTL_CMD_CHFREQ << MEMCTL_CMD_SHIFT) |
> - (val << MEMCTL_FREQ_SHIFT) | MEMCTL_SFCAVM;
> - intel_uncore_write16(uncore, MEMSWCTL, rgvswctl);
> - intel_uncore_posting_read16(uncore, MEMSWCTL);
> -
> - rgvswctl |= MEMCTL_CMD_STS;
> - intel_uncore_write16(uncore, MEMSWCTL, rgvswctl);
> -
> - return true;
> + /*
> + * On Ibex Peak and Cougar Point, we need to disable clock
> + * gating for the panel power sequencer or it will fail to
> + * start up when no ports are active.
> + */
> + I915_WRITE(SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE);
> }
>
> -static void ironlake_enable_drps(struct drm_i915_private *dev_priv)
> +static void g4x_disable_trickle_feed(struct drm_i915_private *dev_priv)
> {
> - struct intel_uncore *uncore = &dev_priv->uncore;
> - u32 rgvmodectl;
> - u8 fmax, fmin, fstart, vstart;
> -
> - spin_lock_irq(&mchdev_lock);
> -
> - rgvmodectl = intel_uncore_read(uncore, MEMMODECTL);
> -
> - /* Enable temp reporting */
> - intel_uncore_write16(uncore, PMMISC, I915_READ(PMMISC) | MCPPCE_EN);
> - intel_uncore_write16(uncore, TSC1, I915_READ(TSC1) | TSE);
> -
> - /* 100ms RC evaluation intervals */
> - intel_uncore_write(uncore, RCUPEI, 100000);
> - intel_uncore_write(uncore, RCDNEI, 100000);
> -
> - /* Set max/min thresholds to 90ms and 80ms respectively */
> - intel_uncore_write(uncore, RCBMAXAVG, 90000);
> - intel_uncore_write(uncore, RCBMINAVG, 80000);
> -
> - intel_uncore_write(uncore, MEMIHYST, 1);
> + enum pipe pipe;
>
> - /* Set up min, max, and cur for interrupt handling */
> - fmax = (rgvmodectl & MEMMODE_FMAX_MASK) >> MEMMODE_FMAX_SHIFT;
> - fmin = (rgvmodectl & MEMMODE_FMIN_MASK);
> - fstart = (rgvmodectl & MEMMODE_FSTART_MASK) >>
> - MEMMODE_FSTART_SHIFT;
> + for_each_pipe(dev_priv, pipe) {
> + I915_WRITE(DSPCNTR(pipe),
> + I915_READ(DSPCNTR(pipe)) |
> + DISPPLANE_TRICKLE_FEED_DISABLE);
>
> - vstart = (intel_uncore_read(uncore, PXVFREQ(fstart)) &
> - PXVFREQ_PX_MASK) >> PXVFREQ_PX_SHIFT;
> + I915_WRITE(DSPSURF(pipe), I915_READ(DSPSURF(pipe)));
> + POSTING_READ(DSPSURF(pipe));
> + }
> +}
>
> - dev_priv->ips.fmax = fmax; /* IPS callback will increase this */
> - dev_priv->ips.fstart = fstart;
> +static void ilk_init_clock_gating(struct drm_i915_private *dev_priv)
> +{
> + u32 dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE;
>
> - dev_priv->ips.max_delay = fstart;
> - dev_priv->ips.min_delay = fmin;
> - dev_priv->ips.cur_delay = fstart;
> + /*
> + * Required for FBC
> + * WaFbcDisableDpfcClockGating:ilk
> + */
> + dspclk_gate |= ILK_DPFCRUNIT_CLOCK_GATE_DISABLE |
> + ILK_DPFCUNIT_CLOCK_GATE_DISABLE |
> + ILK_DPFDUNIT_CLOCK_GATE_ENABLE;
>
> - DRM_DEBUG_DRIVER("fmax: %d, fmin: %d, fstart: %d\n",
> - fmax, fmin, fstart);
> + I915_WRITE(PCH_3DCGDIS0,
> + MARIUNIT_CLOCK_GATE_DISABLE |
> + SVSMUNIT_CLOCK_GATE_DISABLE);
> + I915_WRITE(PCH_3DCGDIS1,
> + VFMUNIT_CLOCK_GATE_DISABLE);
>
> - intel_uncore_write(uncore,
> - MEMINTREN,
> - MEMINT_CX_SUPR_EN | MEMINT_EVAL_CHG_EN);
> + /*
> + * According to the spec the following bits should be set in
> + * order to enable memory self-refresh
> + * The bit 22/21 of 0x42004
> + * The bit 5 of 0x42020
> + * The bit 15 of 0x45000
> + */
> + I915_WRITE(ILK_DISPLAY_CHICKEN2,
> + (I915_READ(ILK_DISPLAY_CHICKEN2) |
> + ILK_DPARB_GATE | ILK_VSDPFD_FULL));
> + dspclk_gate |= ILK_DPARBUNIT_CLOCK_GATE_ENABLE;
> + I915_WRITE(DISP_ARB_CTL,
> + (I915_READ(DISP_ARB_CTL) |
> + DISP_FBC_WM_DIS));
>
> /*
> - * Interrupts will be enabled in ironlake_irq_postinstall
> + * Based on the document from hardware guys the following bits
> + * should be set unconditionally in order to enable FBC.
> + * The bit 22 of 0x42000
> + * The bit 22 of 0x42004
> + * The bit 7,8,9 of 0x42020.
> */
> + if (IS_IRONLAKE_M(dev_priv)) {
> + /* WaFbcAsynchFlipDisableFbcQueue:ilk */
> + I915_WRITE(ILK_DISPLAY_CHICKEN1,
> + I915_READ(ILK_DISPLAY_CHICKEN1) |
> + ILK_FBCQ_DIS);
> + I915_WRITE(ILK_DISPLAY_CHICKEN2,
> + I915_READ(ILK_DISPLAY_CHICKEN2) |
> + ILK_DPARB_GATE);
> + }
>
> - intel_uncore_write(uncore, VIDSTART, vstart);
> - intel_uncore_posting_read(uncore, VIDSTART);
> + I915_WRITE(ILK_DSPCLK_GATE_D, dspclk_gate);
>
> - rgvmodectl |= MEMMODE_SWMODE_EN;
> - intel_uncore_write(uncore, MEMMODECTL, rgvmodectl);
> + I915_WRITE(ILK_DISPLAY_CHICKEN2,
> + I915_READ(ILK_DISPLAY_CHICKEN2) |
> + ILK_ELPIN_409_SELECT);
> + I915_WRITE(_3D_CHICKEN2,
> + _3D_CHICKEN2_WM_READ_PIPELINED << 16 |
> + _3D_CHICKEN2_WM_READ_PIPELINED);
>
> - if (wait_for_atomic((intel_uncore_read(uncore, MEMSWCTL) &
> - MEMCTL_CMD_STS) == 0, 10))
> - DRM_ERROR("stuck trying to change perf mode\n");
> - mdelay(1);
> + /* WaDisableRenderCachePipelinedFlush:ilk */
> + I915_WRITE(CACHE_MODE_0,
> + _MASKED_BIT_ENABLE(CM0_PIPELINED_RENDER_FLUSH_DISABLE));
>
> - ironlake_set_drps(dev_priv, fstart);
> + /* WaDisable_RenderCache_OperationalFlush:ilk */
> + I915_WRITE(CACHE_MODE_0, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
>
> - dev_priv->ips.last_count1 =
> - intel_uncore_read(uncore, DMIEC) +
> - intel_uncore_read(uncore, DDREC) +
> - intel_uncore_read(uncore, CSIEC);
> - dev_priv->ips.last_time1 = jiffies_to_msecs(jiffies);
> - dev_priv->ips.last_count2 = intel_uncore_read(uncore, GFXEC);
> - dev_priv->ips.last_time2 = ktime_get_raw_ns();
> + g4x_disable_trickle_feed(dev_priv);
>
> - spin_unlock_irq(&mchdev_lock);
> + ibx_init_clock_gating(dev_priv);
> }
>
> -static void ironlake_disable_drps(struct drm_i915_private *i915)
> +static void cpt_init_clock_gating(struct drm_i915_private *dev_priv)
> {
> - struct intel_uncore *uncore = &i915->uncore;
> - u16 rgvswctl;
> -
> - spin_lock_irq(&mchdev_lock);
> -
> - rgvswctl = intel_uncore_read16(uncore, MEMSWCTL);
> -
> - /* Ack interrupts, disable EFC interrupt */
> - intel_uncore_write(uncore,
> - MEMINTREN,
> - intel_uncore_read(uncore, MEMINTREN) &
> - ~MEMINT_EVAL_CHG_EN);
> - intel_uncore_write(uncore, MEMINTRSTS, MEMINT_EVAL_CHG);
> - intel_uncore_write(uncore,
> - DEIER,
> - intel_uncore_read(uncore, DEIER) & ~DE_PCU_EVENT);
> - intel_uncore_write(uncore, DEIIR, DE_PCU_EVENT);
> - intel_uncore_write(uncore,
> - DEIMR,
> - intel_uncore_read(uncore, DEIMR) | DE_PCU_EVENT);
> -
> - /* Go back to the starting frequency */
> - ironlake_set_drps(i915, i915->ips.fstart);
> - mdelay(1);
> - rgvswctl |= MEMCTL_CMD_STS;
> - intel_uncore_write(uncore, MEMSWCTL, rgvswctl);
> - mdelay(1);
> + enum pipe pipe;
> + u32 val;
>
> - spin_unlock_irq(&mchdev_lock);
> + /*
> + * On Ibex Peak and Cougar Point, we need to disable clock
> + * gating for the panel power sequencer or it will fail to
> + * start up when no ports are active.
> + */
> + I915_WRITE(SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE |
> + PCH_DPLUNIT_CLOCK_GATE_DISABLE |
> + PCH_CPUNIT_CLOCK_GATE_DISABLE);
> + I915_WRITE(SOUTH_CHICKEN2, I915_READ(SOUTH_CHICKEN2) |
> + DPLS_EDP_PPS_FIX_DIS);
> + /* The below fixes the weird display corruption, a few pixels shifted
> + * downward, on (only) LVDS of some HP laptops with IVY.
> + */
> + for_each_pipe(dev_priv, pipe) {
> + val = I915_READ(TRANS_CHICKEN2(pipe));
> + val |= TRANS_CHICKEN2_TIMING_OVERRIDE;
> + val &= ~TRANS_CHICKEN2_FDI_POLARITY_REVERSED;
> + if (dev_priv->vbt.fdi_rx_polarity_inverted)
> + val |= TRANS_CHICKEN2_FDI_POLARITY_REVERSED;
> + val &= ~TRANS_CHICKEN2_FRAME_START_DELAY_MASK;
> + val &= ~TRANS_CHICKEN2_DISABLE_DEEP_COLOR_COUNTER;
> + val &= ~TRANS_CHICKEN2_DISABLE_DEEP_COLOR_MODESWITCH;
> + I915_WRITE(TRANS_CHICKEN2(pipe), val);
> + }
> + /* WADP0ClockGatingDisable */
> + for_each_pipe(dev_priv, pipe) {
> + I915_WRITE(TRANS_CHICKEN1(pipe),
> + TRANS_CHICKEN1_DP0UNIT_GC_DISABLE);
> + }
> }
>
> -/* There's a funny hw issue where the hw returns all 0 when reading from
> - * GEN6_RP_INTERRUPT_LIMITS. Hence we always need to compute the desired value
> - * ourselves, instead of doing a rmw cycle (which might result in us clearing
> - * all limits and the gpu stuck at whatever frequency it is at atm).
> - */
> -static u32 intel_rps_limits(struct drm_i915_private *dev_priv, u8 val)
> +static void gen6_check_mch_setup(struct drm_i915_private *dev_priv)
> {
> - struct intel_rps *rps = &dev_priv->gt_pm.rps;
> - u32 limits;
> -
> - /* Only set the down limit when we've reached the lowest level to avoid
> - * getting more interrupts, otherwise leave this clear. This prevents a
> - * race in the hw when coming out of rc6: There's a tiny window where
> - * the hw runs at the minimal clock before selecting the desired
> - * frequency, if the down threshold expires in that window we will not
> - * receive a down interrupt. */
> - if (INTEL_GEN(dev_priv) >= 9) {
> - limits = (rps->max_freq_softlimit) << 23;
> - if (val <= rps->min_freq_softlimit)
> - limits |= (rps->min_freq_softlimit) << 14;
> - } else {
> - limits = rps->max_freq_softlimit << 24;
> - if (val <= rps->min_freq_softlimit)
> - limits |= rps->min_freq_softlimit << 16;
> - }
> + u32 tmp;
>
> - return limits;
> + tmp = I915_READ(MCH_SSKPD);
> + if ((tmp & MCH_SSKPD_WM0_MASK) != MCH_SSKPD_WM0_VAL)
> + DRM_DEBUG_KMS("Wrong MCH_SSKPD value: 0x%08x This can cause underruns.\n",
> + tmp);
> }
>
> -static void rps_set_power(struct drm_i915_private *dev_priv, int new_power)
> +static void gen6_init_clock_gating(struct drm_i915_private *dev_priv)
> {
> - struct intel_rps *rps = &dev_priv->gt_pm.rps;
> - u32 threshold_up = 0, threshold_down = 0; /* in % */
> - u32 ei_up = 0, ei_down = 0;
> -
> - lockdep_assert_held(&rps->power.mutex);
> + u32 dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE;
>
> - if (new_power == rps->power.mode)
> - return;
> + I915_WRITE(ILK_DSPCLK_GATE_D, dspclk_gate);
>
> - /* Note the units here are not exactly 1us, but 1280ns. */
> - switch (new_power) {
> - case LOW_POWER:
> - /* Upclock if more than 95% busy over 16ms */
> - ei_up = 16000;
> - threshold_up = 95;
> + I915_WRITE(ILK_DISPLAY_CHICKEN2,
> + I915_READ(ILK_DISPLAY_CHICKEN2) |
> + ILK_ELPIN_409_SELECT);
>
> - /* Downclock if less than 85% busy over 32ms */
> - ei_down = 32000;
> - threshold_down = 85;
> - break;
> + /* WaDisableHiZPlanesWhenMSAAEnabled:snb */
> + I915_WRITE(_3D_CHICKEN,
> + _MASKED_BIT_ENABLE(_3D_CHICKEN_HIZ_PLANE_DISABLE_MSAA_4X_SNB));
>
> - case BETWEEN:
> - /* Upclock if more than 90% busy over 13ms */
> - ei_up = 13000;
> - threshold_up = 90;
> + /* WaDisable_RenderCache_OperationalFlush:snb */
> + I915_WRITE(CACHE_MODE_0, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
>
> - /* Downclock if less than 75% busy over 32ms */
> - ei_down = 32000;
> - threshold_down = 75;
> - break;
> + /*
> + * BSpec recoomends 8x4 when MSAA is used,
> + * however in practice 16x4 seems fastest.
> + *
> + * Note that PS/WM thread counts depend on the WIZ hashing
> + * disable bit, which we don't touch here, but it's good
> + * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM).
> + */
> + I915_WRITE(GEN6_GT_MODE,
> + _MASKED_FIELD(GEN6_WIZ_HASHING_MASK, GEN6_WIZ_HASHING_16x4));
>
> - case HIGH_POWER:
> - /* Upclock if more than 85% busy over 10ms */
> - ei_up = 10000;
> - threshold_up = 85;
> + I915_WRITE(CACHE_MODE_0,
> + _MASKED_BIT_DISABLE(CM0_STC_EVICT_DISABLE_LRA_SNB));
>
> - /* Downclock if less than 60% busy over 32ms */
> - ei_down = 32000;
> - threshold_down = 60;
> - break;
> - }
> + I915_WRITE(GEN6_UCGCTL1,
> + I915_READ(GEN6_UCGCTL1) |
> + GEN6_BLBUNIT_CLOCK_GATE_DISABLE |
> + GEN6_CSUNIT_CLOCK_GATE_DISABLE);
>
> - /* When byt can survive without system hang with dynamic
> - * sw freq adjustments, this restriction can be lifted.
> + /* According to the BSpec vol1g, bit 12 (RCPBUNIT) clock
> + * gating disable must be set. Failure to set it results in
> + * flickering pixels due to Z write ordering failures after
> + * some amount of runtime in the Mesa "fire" demo, and Unigine
> + * Sanctuary and Tropics, and apparently anything else with
> + * alpha test or pixel discard.
> + *
> + * According to the spec, bit 11 (RCCUNIT) must also be set,
> + * but we didn't debug actual testcases to find it out.
> + *
> + * WaDisableRCCUnitClockGating:snb
> + * WaDisableRCPBUnitClockGating:snb
> */
> - if (IS_VALLEYVIEW(dev_priv))
> - goto skip_hw_write;
> -
> - I915_WRITE(GEN6_RP_UP_EI,
> - GT_INTERVAL_FROM_US(dev_priv, ei_up));
> - I915_WRITE(GEN6_RP_UP_THRESHOLD,
> - GT_INTERVAL_FROM_US(dev_priv,
> - ei_up * threshold_up / 100));
> -
> - I915_WRITE(GEN6_RP_DOWN_EI,
> - GT_INTERVAL_FROM_US(dev_priv, ei_down));
> - I915_WRITE(GEN6_RP_DOWN_THRESHOLD,
> - GT_INTERVAL_FROM_US(dev_priv,
> - ei_down * threshold_down / 100));
> -
> - I915_WRITE(GEN6_RP_CONTROL,
> - (INTEL_GEN(dev_priv) > 9 ? 0 : GEN6_RP_MEDIA_TURBO) |
> - GEN6_RP_MEDIA_HW_NORMAL_MODE |
> - GEN6_RP_MEDIA_IS_GFX |
> - GEN6_RP_ENABLE |
> - GEN6_RP_UP_BUSY_AVG |
> - GEN6_RP_DOWN_IDLE_AVG);
> -
> -skip_hw_write:
> - rps->power.mode = new_power;
> - rps->power.up_threshold = threshold_up;
> - rps->power.down_threshold = threshold_down;
> -}
> -
> -static void gen6_set_rps_thresholds(struct drm_i915_private *dev_priv, u8 val)
> -{
> - struct intel_rps *rps = &dev_priv->gt_pm.rps;
> - int new_power;
> -
> - new_power = rps->power.mode;
> - switch (rps->power.mode) {
> - case LOW_POWER:
> - if (val > rps->efficient_freq + 1 &&
> - val > rps->cur_freq)
> - new_power = BETWEEN;
> - break;
> + I915_WRITE(GEN6_UCGCTL2,
> + GEN6_RCPBUNIT_CLOCK_GATE_DISABLE |
> + GEN6_RCCUNIT_CLOCK_GATE_DISABLE);
>
> - case BETWEEN:
> - if (val <= rps->efficient_freq &&
> - val < rps->cur_freq)
> - new_power = LOW_POWER;
> - else if (val >= rps->rp0_freq &&
> - val > rps->cur_freq)
> - new_power = HIGH_POWER;
> - break;
> + /* WaStripsFansDisableFastClipPerformanceFix:snb */
> + I915_WRITE(_3D_CHICKEN3,
> + _MASKED_BIT_ENABLE(_3D_CHICKEN3_SF_DISABLE_FASTCLIP_CULL));
>
> - case HIGH_POWER:
> - if (val < (rps->rp1_freq + rps->rp0_freq) >> 1 &&
> - val < rps->cur_freq)
> - new_power = BETWEEN;
> - break;
> - }
> - /* Max/min bins are special */
> - if (val <= rps->min_freq_softlimit)
> - new_power = LOW_POWER;
> - if (val >= rps->max_freq_softlimit)
> - new_power = HIGH_POWER;
> + /*
> + * Bspec says:
> + * "This bit must be set if 3DSTATE_CLIP clip mode is set to normal and
> + * 3DSTATE_SF number of SF output attributes is more than 16."
> + */
> + I915_WRITE(_3D_CHICKEN3,
> + _MASKED_BIT_ENABLE(_3D_CHICKEN3_SF_DISABLE_PIPELINED_ATTR_FETCH));
>
> - mutex_lock(&rps->power.mutex);
> - if (rps->power.interactive)
> - new_power = HIGH_POWER;
> - rps_set_power(dev_priv, new_power);
> - mutex_unlock(&rps->power.mutex);
> -}
> + /*
> + * According to the spec the following bits should be
> + * set in order to enable memory self-refresh and fbc:
> + * The bit21 and bit22 of 0x42000
> + * The bit21 and bit22 of 0x42004
> + * The bit5 and bit7 of 0x42020
> + * The bit14 of 0x70180
> + * The bit14 of 0x71180
> + *
> + * WaFbcAsynchFlipDisableFbcQueue:snb
> + */
> + I915_WRITE(ILK_DISPLAY_CHICKEN1,
> + I915_READ(ILK_DISPLAY_CHICKEN1) |
> + ILK_FBCQ_DIS | ILK_PABSTRETCH_DIS);
> + I915_WRITE(ILK_DISPLAY_CHICKEN2,
> + I915_READ(ILK_DISPLAY_CHICKEN2) |
> + ILK_DPARB_GATE | ILK_VSDPFD_FULL);
> + I915_WRITE(ILK_DSPCLK_GATE_D,
> + I915_READ(ILK_DSPCLK_GATE_D) |
> + ILK_DPARBUNIT_CLOCK_GATE_ENABLE |
> + ILK_DPFDUNIT_CLOCK_GATE_ENABLE);
>
> -void intel_rps_mark_interactive(struct drm_i915_private *i915, bool interactive)
> -{
> - struct intel_rps *rps = &i915->gt_pm.rps;
> + g4x_disable_trickle_feed(dev_priv);
>
> - if (INTEL_GEN(i915) < 6)
> - return;
> + cpt_init_clock_gating(dev_priv);
>
> - mutex_lock(&rps->power.mutex);
> - if (interactive) {
> - if (!rps->power.interactive++ && READ_ONCE(i915->gt.awake))
> - rps_set_power(i915, HIGH_POWER);
> - } else {
> - GEM_BUG_ON(!rps->power.interactive);
> - rps->power.interactive--;
> - }
> - mutex_unlock(&rps->power.mutex);
> + gen6_check_mch_setup(dev_priv);
> }
>
> -static u32 gen6_rps_pm_mask(struct drm_i915_private *dev_priv, u8 val)
> +static void gen7_setup_fixed_func_scheduler(struct drm_i915_private *dev_priv)
> {
> - struct intel_rps *rps = &dev_priv->gt_pm.rps;
> - u32 mask = 0;
> -
> - /* We use UP_EI_EXPIRED interupts for both up/down in manual mode */
> - if (val > rps->min_freq_softlimit)
> - mask |= GEN6_PM_RP_UP_EI_EXPIRED | GEN6_PM_RP_DOWN_THRESHOLD | GEN6_PM_RP_DOWN_TIMEOUT;
> - if (val < rps->max_freq_softlimit)
> - mask |= GEN6_PM_RP_UP_EI_EXPIRED | GEN6_PM_RP_UP_THRESHOLD;
> -
> - mask &= dev_priv->pm_rps_events;
> -
> - return gen6_sanitize_rps_pm_mask(dev_priv, ~mask);
> -}
> -
> -/* gen6_set_rps is called to update the frequency request, but should also be
> - * called when the range (min_delay and max_delay) is modified so that we can
> - * update the GEN6_RP_INTERRUPT_LIMITS register accordingly. */
> -static int gen6_set_rps(struct drm_i915_private *dev_priv, u8 val)
> -{
> - struct intel_rps *rps = &dev_priv->gt_pm.rps;
> -
> - /* min/max delay may still have been modified so be sure to
> - * write the limits value.
> - */
> - if (val != rps->cur_freq) {
> - gen6_set_rps_thresholds(dev_priv, val);
> -
> - if (INTEL_GEN(dev_priv) >= 9)
> - I915_WRITE(GEN6_RPNSWREQ,
> - GEN9_FREQUENCY(val));
> - else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
> - I915_WRITE(GEN6_RPNSWREQ,
> - HSW_FREQUENCY(val));
> - else
> - I915_WRITE(GEN6_RPNSWREQ,
> - GEN6_FREQUENCY(val) |
> - GEN6_OFFSET(0) |
> - GEN6_AGGRESSIVE_TURBO);
> - }
> -
> - /* Make sure we continue to get interrupts
> - * until we hit the minimum or maximum frequencies.
> - */
> - I915_WRITE(GEN6_RP_INTERRUPT_LIMITS, intel_rps_limits(dev_priv, val));
> - I915_WRITE(GEN6_PMINTRMSK, gen6_rps_pm_mask(dev_priv, val));
> -
> - rps->cur_freq = val;
> - trace_intel_gpu_freq_change(intel_gpu_freq(dev_priv, val));
> -
> - return 0;
> -}
> -
> -static int valleyview_set_rps(struct drm_i915_private *dev_priv, u8 val)
> -{
> - int err;
> -
> - if (WARN_ONCE(IS_CHERRYVIEW(dev_priv) && (val & 1),
> - "Odd GPU freq value\n"))
> - val &= ~1;
> -
> - I915_WRITE(GEN6_PMINTRMSK, gen6_rps_pm_mask(dev_priv, val));
> -
> - if (val != dev_priv->gt_pm.rps.cur_freq) {
> - vlv_punit_get(dev_priv);
> - err = vlv_punit_write(dev_priv, PUNIT_REG_GPU_FREQ_REQ, val);
> - vlv_punit_put(dev_priv);
> - if (err)
> - return err;
> -
> - gen6_set_rps_thresholds(dev_priv, val);
> - }
> -
> - dev_priv->gt_pm.rps.cur_freq = val;
> - trace_intel_gpu_freq_change(intel_gpu_freq(dev_priv, val));
> -
> - return 0;
> -}
> -
> -/* vlv_set_rps_idle: Set the frequency to idle, if Gfx clocks are down
> - *
> - * * If Gfx is Idle, then
> - * 1. Forcewake Media well.
> - * 2. Request idle freq.
> - * 3. Release Forcewake of Media well.
> -*/
> -static void vlv_set_rps_idle(struct drm_i915_private *dev_priv)
> -{
> - struct intel_rps *rps = &dev_priv->gt_pm.rps;
> - u32 val = rps->idle_freq;
> - int err;
> -
> - if (rps->cur_freq <= val)
> - return;
> -
> - /* The punit delays the write of the frequency and voltage until it
> - * determines the GPU is awake. During normal usage we don't want to
> - * waste power changing the frequency if the GPU is sleeping (rc6).
> - * However, the GPU and driver is now idle and we do not want to delay
> - * switching to minimum voltage (reducing power whilst idle) as we do
> - * not expect to be woken in the near future and so must flush the
> - * change by waking the device.
> - *
> - * We choose to take the media powerwell (either would do to trick the
> - * punit into committing the voltage change) as that takes a lot less
> - * power than the render powerwell.
> - */
> - intel_uncore_forcewake_get(&dev_priv->uncore, FORCEWAKE_MEDIA);
> - err = valleyview_set_rps(dev_priv, val);
> - intel_uncore_forcewake_put(&dev_priv->uncore, FORCEWAKE_MEDIA);
> -
> - if (err)
> - DRM_ERROR("Failed to set RPS for idle\n");
> -}
> -
> -void gen6_rps_busy(struct drm_i915_private *dev_priv)
> -{
> - struct intel_rps *rps = &dev_priv->gt_pm.rps;
> -
> - mutex_lock(&rps->lock);
> - if (rps->enabled) {
> - u8 freq;
> -
> - if (dev_priv->pm_rps_events & GEN6_PM_RP_UP_EI_EXPIRED)
> - gen6_rps_reset_ei(dev_priv);
> - I915_WRITE(GEN6_PMINTRMSK,
> - gen6_rps_pm_mask(dev_priv, rps->cur_freq));
> -
> - gen6_enable_rps_interrupts(dev_priv);
> -
> - /* Use the user's desired frequency as a guide, but for better
> - * performance, jump directly to RPe as our starting frequency.
> - */
> - freq = max(rps->cur_freq,
> - rps->efficient_freq);
> -
> - if (intel_set_rps(dev_priv,
> - clamp(freq,
> - rps->min_freq_softlimit,
> - rps->max_freq_softlimit)))
> - DRM_DEBUG_DRIVER("Failed to set idle frequency\n");
> - }
> - mutex_unlock(&rps->lock);
> -}
> -
> -void gen6_rps_idle(struct drm_i915_private *dev_priv)
> -{
> - struct intel_rps *rps = &dev_priv->gt_pm.rps;
> -
> - /* Flush our bottom-half so that it does not race with us
> - * setting the idle frequency and so that it is bounded by
> - * our rpm wakeref. And then disable the interrupts to stop any
> - * futher RPS reclocking whilst we are asleep.
> - */
> - gen6_disable_rps_interrupts(dev_priv);
> -
> - mutex_lock(&rps->lock);
> - if (rps->enabled) {
> - if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
> - vlv_set_rps_idle(dev_priv);
> - else
> - gen6_set_rps(dev_priv, rps->idle_freq);
> - rps->last_adj = 0;
> - I915_WRITE(GEN6_PMINTRMSK,
> - gen6_sanitize_rps_pm_mask(dev_priv, ~0));
> - }
> - mutex_unlock(&rps->lock);
> -}
> -
> -void gen6_rps_boost(struct i915_request *rq)
> -{
> - struct intel_rps *rps = &rq->i915->gt_pm.rps;
> - unsigned long flags;
> - bool boost;
> -
> - /* This is intentionally racy! We peek at the state here, then
> - * validate inside the RPS worker.
> - */
> - if (!rps->enabled)
> - return;
> -
> - if (i915_request_signaled(rq))
> - return;
> -
> - /* Serializes with i915_request_retire() */
> - boost = false;
> - spin_lock_irqsave(&rq->lock, flags);
> - if (!i915_request_has_waitboost(rq) &&
> - !dma_fence_is_signaled_locked(&rq->fence)) {
> - boost = !atomic_fetch_inc(&rps->num_waiters);
> - rq->flags |= I915_REQUEST_WAITBOOST;
> - }
> - spin_unlock_irqrestore(&rq->lock, flags);
> - if (!boost)
> - return;
> -
> - if (READ_ONCE(rps->cur_freq) < rps->boost_freq)
> - schedule_work(&rps->work);
> -
> - atomic_inc(&rps->boosts);
> -}
> -
> -int intel_set_rps(struct drm_i915_private *dev_priv, u8 val)
> -{
> - struct intel_rps *rps = &dev_priv->gt_pm.rps;
> - int err;
> -
> - lockdep_assert_held(&rps->lock);
> - GEM_BUG_ON(val > rps->max_freq);
> - GEM_BUG_ON(val < rps->min_freq);
> -
> - if (!rps->enabled) {
> - rps->cur_freq = val;
> - return 0;
> - }
> -
> - if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
> - err = valleyview_set_rps(dev_priv, val);
> - else
> - err = gen6_set_rps(dev_priv, val);
> -
> - return err;
> -}
> -
> -static void gen9_disable_rps(struct drm_i915_private *dev_priv)
> -{
> - I915_WRITE(GEN6_RP_CONTROL, 0);
> -}
> -
> -static void gen6_disable_rps(struct drm_i915_private *dev_priv)
> -{
> - I915_WRITE(GEN6_RPNSWREQ, 1 << 31);
> - I915_WRITE(GEN6_RP_CONTROL, 0);
> -}
> -
> -static void cherryview_disable_rps(struct drm_i915_private *dev_priv)
> -{
> - I915_WRITE(GEN6_RP_CONTROL, 0);
> -}
> -
> -static void valleyview_disable_rps(struct drm_i915_private *dev_priv)
> -{
> - I915_WRITE(GEN6_RP_CONTROL, 0);
> -}
> -
> -static void gen6_init_rps_frequencies(struct drm_i915_private *dev_priv)
> -{
> - struct intel_rps *rps = &dev_priv->gt_pm.rps;
> -
> - /* All of these values are in units of 50MHz */
> -
> - /* static values from HW: RP0 > RP1 > RPn (min_freq) */
> - if (IS_GEN9_LP(dev_priv)) {
> - u32 rp_state_cap = I915_READ(BXT_RP_STATE_CAP);
> - rps->rp0_freq = (rp_state_cap >> 16) & 0xff;
> - rps->rp1_freq = (rp_state_cap >> 8) & 0xff;
> - rps->min_freq = (rp_state_cap >> 0) & 0xff;
> - } else {
> - u32 rp_state_cap = I915_READ(GEN6_RP_STATE_CAP);
> - rps->rp0_freq = (rp_state_cap >> 0) & 0xff;
> - rps->rp1_freq = (rp_state_cap >> 8) & 0xff;
> - rps->min_freq = (rp_state_cap >> 16) & 0xff;
> - }
> - /* hw_max = RP0 until we check for overclocking */
> - rps->max_freq = rps->rp0_freq;
> -
> - rps->efficient_freq = rps->rp1_freq;
> - if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv) ||
> - IS_GEN9_BC(dev_priv) || INTEL_GEN(dev_priv) >= 10) {
> - u32 ddcc_status = 0;
> -
> - if (sandybridge_pcode_read(dev_priv,
> - HSW_PCODE_DYNAMIC_DUTY_CYCLE_CONTROL,
> - &ddcc_status, NULL) == 0)
> - rps->efficient_freq =
> - clamp_t(u8,
> - ((ddcc_status >> 8) & 0xff),
> - rps->min_freq,
> - rps->max_freq);
> - }
> -
> - if (IS_GEN9_BC(dev_priv) || INTEL_GEN(dev_priv) >= 10) {
> - /* Store the frequency values in 16.66 MHZ units, which is
> - * the natural hardware unit for SKL
> - */
> - rps->rp0_freq *= GEN9_FREQ_SCALER;
> - rps->rp1_freq *= GEN9_FREQ_SCALER;
> - rps->min_freq *= GEN9_FREQ_SCALER;
> - rps->max_freq *= GEN9_FREQ_SCALER;
> - rps->efficient_freq *= GEN9_FREQ_SCALER;
> - }
> -}
> -
> -static void reset_rps(struct drm_i915_private *dev_priv,
> - int (*set)(struct drm_i915_private *, u8))
> -{
> - struct intel_rps *rps = &dev_priv->gt_pm.rps;
> - u8 freq = rps->cur_freq;
> -
> - /* force a reset */
> - rps->power.mode = -1;
> - rps->cur_freq = -1;
> -
> - if (set(dev_priv, freq))
> - DRM_ERROR("Failed to reset RPS to initial values\n");
> -}
> -
> -/* See the Gen9_GT_PM_Programming_Guide doc for the below */
> -static void gen9_enable_rps(struct drm_i915_private *dev_priv)
> -{
> - intel_uncore_forcewake_get(&dev_priv->uncore, FORCEWAKE_ALL);
> -
> - /* Program defaults and thresholds for RPS */
> - if (IS_GEN(dev_priv, 9))
> - I915_WRITE(GEN6_RC_VIDEO_FREQ,
> - GEN9_FREQUENCY(dev_priv->gt_pm.rps.rp1_freq));
> -
> - /* 1 second timeout*/
> - I915_WRITE(GEN6_RP_DOWN_TIMEOUT,
> - GT_INTERVAL_FROM_US(dev_priv, 1000000));
> -
> - I915_WRITE(GEN6_RP_IDLE_HYSTERSIS, 0xa);
> -
> - /* Leaning on the below call to gen6_set_rps to program/setup the
> - * Up/Down EI & threshold registers, as well as the RP_CONTROL,
> - * RP_INTERRUPT_LIMITS & RPNSWREQ registers */
> - reset_rps(dev_priv, gen6_set_rps);
> -
> - intel_uncore_forcewake_put(&dev_priv->uncore, FORCEWAKE_ALL);
> -}
> -
> -static void gen8_enable_rps(struct drm_i915_private *dev_priv)
> -{
> - struct intel_rps *rps = &dev_priv->gt_pm.rps;
> -
> - intel_uncore_forcewake_get(&dev_priv->uncore, FORCEWAKE_ALL);
> -
> - /* 1 Program defaults and thresholds for RPS*/
> - I915_WRITE(GEN6_RPNSWREQ,
> - HSW_FREQUENCY(rps->rp1_freq));
> - I915_WRITE(GEN6_RC_VIDEO_FREQ,
> - HSW_FREQUENCY(rps->rp1_freq));
> - /* NB: Docs say 1s, and 1000000 - which aren't equivalent */
> - I915_WRITE(GEN6_RP_DOWN_TIMEOUT, 100000000 / 128); /* 1 second timeout */
> -
> - /* Docs recommend 900MHz, and 300 MHz respectively */
> - I915_WRITE(GEN6_RP_INTERRUPT_LIMITS,
> - rps->max_freq_softlimit << 24 |
> - rps->min_freq_softlimit << 16);
> -
> - I915_WRITE(GEN6_RP_UP_THRESHOLD, 7600000 / 128); /* 76ms busyness per EI, 90% */
> - I915_WRITE(GEN6_RP_DOWN_THRESHOLD, 31300000 / 128); /* 313ms busyness per EI, 70%*/
> - I915_WRITE(GEN6_RP_UP_EI, 66000); /* 84.48ms, XXX: random? */
> - I915_WRITE(GEN6_RP_DOWN_EI, 350000); /* 448ms, XXX: random? */
> -
> - I915_WRITE(GEN6_RP_IDLE_HYSTERSIS, 10);
> -
> - /* 2: Enable RPS */
> - I915_WRITE(GEN6_RP_CONTROL,
> - GEN6_RP_MEDIA_TURBO |
> - GEN6_RP_MEDIA_HW_NORMAL_MODE |
> - GEN6_RP_MEDIA_IS_GFX |
> - GEN6_RP_ENABLE |
> - GEN6_RP_UP_BUSY_AVG |
> - GEN6_RP_DOWN_IDLE_AVG);
> -
> - reset_rps(dev_priv, gen6_set_rps);
> -
> - intel_uncore_forcewake_put(&dev_priv->uncore, FORCEWAKE_ALL);
> -}
> -
> -static void gen6_enable_rps(struct drm_i915_private *dev_priv)
> -{
> - /* Here begins a magic sequence of register writes to enable
> - * auto-downclocking.
> - *
> - * Perhaps there might be some value in exposing these to
> - * userspace...
> - */
> - intel_uncore_forcewake_get(&dev_priv->uncore, FORCEWAKE_ALL);
> -
> - /* Power down if completely idle for over 50ms */
> - I915_WRITE(GEN6_RP_DOWN_TIMEOUT, 50000);
> - I915_WRITE(GEN6_RP_IDLE_HYSTERSIS, 10);
> -
> - reset_rps(dev_priv, gen6_set_rps);
> -
> - intel_uncore_forcewake_put(&dev_priv->uncore, FORCEWAKE_ALL);
> -}
> -
> -static int cherryview_rps_max_freq(struct drm_i915_private *dev_priv)
> -{
> - u32 val, rp0;
> -
> - val = vlv_punit_read(dev_priv, FB_GFX_FMAX_AT_VMAX_FUSE);
> -
> - switch (RUNTIME_INFO(dev_priv)->sseu.eu_total) {
> - case 8:
> - /* (2 * 4) config */
> - rp0 = (val >> FB_GFX_FMAX_AT_VMAX_2SS4EU_FUSE_SHIFT);
> - break;
> - case 12:
> - /* (2 * 6) config */
> - rp0 = (val >> FB_GFX_FMAX_AT_VMAX_2SS6EU_FUSE_SHIFT);
> - break;
> - case 16:
> - /* (2 * 8) config */
> - default:
> - /* Setting (2 * 8) Min RP0 for any other combination */
> - rp0 = (val >> FB_GFX_FMAX_AT_VMAX_2SS8EU_FUSE_SHIFT);
> - break;
> - }
> -
> - rp0 = (rp0 & FB_GFX_FREQ_FUSE_MASK);
> -
> - return rp0;
> -}
> -
> -static int cherryview_rps_rpe_freq(struct drm_i915_private *dev_priv)
> -{
> - u32 val, rpe;
> -
> - val = vlv_punit_read(dev_priv, PUNIT_GPU_DUTYCYCLE_REG);
> - rpe = (val >> PUNIT_GPU_DUTYCYCLE_RPE_FREQ_SHIFT) & PUNIT_GPU_DUTYCYCLE_RPE_FREQ_MASK;
> -
> - return rpe;
> -}
> -
> -static int cherryview_rps_guar_freq(struct drm_i915_private *dev_priv)
> -{
> - u32 val, rp1;
> -
> - val = vlv_punit_read(dev_priv, FB_GFX_FMAX_AT_VMAX_FUSE);
> - rp1 = (val & FB_GFX_FREQ_FUSE_MASK);
> -
> - return rp1;
> -}
> -
> -static u32 cherryview_rps_min_freq(struct drm_i915_private *dev_priv)
> -{
> - u32 val, rpn;
> -
> - val = vlv_punit_read(dev_priv, FB_GFX_FMIN_AT_VMIN_FUSE);
> - rpn = ((val >> FB_GFX_FMIN_AT_VMIN_FUSE_SHIFT) &
> - FB_GFX_FREQ_FUSE_MASK);
> -
> - return rpn;
> -}
> -
> -static int valleyview_rps_guar_freq(struct drm_i915_private *dev_priv)
> -{
> - u32 val, rp1;
> -
> - val = vlv_nc_read(dev_priv, IOSF_NC_FB_GFX_FREQ_FUSE);
> -
> - rp1 = (val & FB_GFX_FGUARANTEED_FREQ_FUSE_MASK) >> FB_GFX_FGUARANTEED_FREQ_FUSE_SHIFT;
> -
> - return rp1;
> -}
> -
> -static int valleyview_rps_max_freq(struct drm_i915_private *dev_priv)
> -{
> - u32 val, rp0;
> -
> - val = vlv_nc_read(dev_priv, IOSF_NC_FB_GFX_FREQ_FUSE);
> -
> - rp0 = (val & FB_GFX_MAX_FREQ_FUSE_MASK) >> FB_GFX_MAX_FREQ_FUSE_SHIFT;
> - /* Clamp to max */
> - rp0 = min_t(u32, rp0, 0xea);
> -
> - return rp0;
> -}
> -
> -static int valleyview_rps_rpe_freq(struct drm_i915_private *dev_priv)
> -{
> - u32 val, rpe;
> -
> - val = vlv_nc_read(dev_priv, IOSF_NC_FB_GFX_FMAX_FUSE_LO);
> - rpe = (val & FB_FMAX_VMIN_FREQ_LO_MASK) >> FB_FMAX_VMIN_FREQ_LO_SHIFT;
> - val = vlv_nc_read(dev_priv, IOSF_NC_FB_GFX_FMAX_FUSE_HI);
> - rpe |= (val & FB_FMAX_VMIN_FREQ_HI_MASK) << 5;
> -
> - return rpe;
> -}
> -
> -static int valleyview_rps_min_freq(struct drm_i915_private *dev_priv)
> -{
> - u32 val;
> -
> - val = vlv_punit_read(dev_priv, PUNIT_REG_GPU_LFM) & 0xff;
> - /*
> - * According to the BYT Punit GPU turbo HAS 1.1.6.3 the minimum value
> - * for the minimum frequency in GPLL mode is 0xc1. Contrary to this on
> - * a BYT-M B0 the above register contains 0xbf. Moreover when setting
> - * a frequency Punit will not allow values below 0xc0. Clamp it 0xc0
> - * to make sure it matches what Punit accepts.
> - */
> - return max_t(u32, val, 0xc0);
> -}
> -
> -static void vlv_init_gpll_ref_freq(struct drm_i915_private *dev_priv)
> -{
> - dev_priv->gt_pm.rps.gpll_ref_freq =
> - vlv_get_cck_clock(dev_priv, "GPLL ref",
> - CCK_GPLL_CLOCK_CONTROL,
> - dev_priv->czclk_freq);
> -
> - DRM_DEBUG_DRIVER("GPLL reference freq: %d kHz\n",
> - dev_priv->gt_pm.rps.gpll_ref_freq);
> -}
> -
> -static void valleyview_init_gt_powersave(struct drm_i915_private *dev_priv)
> -{
> - struct intel_rps *rps = &dev_priv->gt_pm.rps;
> - u32 val;
> -
> - vlv_iosf_sb_get(dev_priv,
> - BIT(VLV_IOSF_SB_PUNIT) |
> - BIT(VLV_IOSF_SB_NC) |
> - BIT(VLV_IOSF_SB_CCK));
> -
> - vlv_init_gpll_ref_freq(dev_priv);
> -
> - val = vlv_punit_read(dev_priv, PUNIT_REG_GPU_FREQ_STS);
> - switch ((val >> 6) & 3) {
> - case 0:
> - case 1:
> - dev_priv->mem_freq = 800;
> - break;
> - case 2:
> - dev_priv->mem_freq = 1066;
> - break;
> - case 3:
> - dev_priv->mem_freq = 1333;
> - break;
> - }
> - DRM_DEBUG_DRIVER("DDR speed: %d MHz\n", dev_priv->mem_freq);
> -
> - rps->max_freq = valleyview_rps_max_freq(dev_priv);
> - rps->rp0_freq = rps->max_freq;
> - DRM_DEBUG_DRIVER("max GPU freq: %d MHz (%u)\n",
> - intel_gpu_freq(dev_priv, rps->max_freq),
> - rps->max_freq);
> -
> - rps->efficient_freq = valleyview_rps_rpe_freq(dev_priv);
> - DRM_DEBUG_DRIVER("RPe GPU freq: %d MHz (%u)\n",
> - intel_gpu_freq(dev_priv, rps->efficient_freq),
> - rps->efficient_freq);
> -
> - rps->rp1_freq = valleyview_rps_guar_freq(dev_priv);
> - DRM_DEBUG_DRIVER("RP1(Guar Freq) GPU freq: %d MHz (%u)\n",
> - intel_gpu_freq(dev_priv, rps->rp1_freq),
> - rps->rp1_freq);
> -
> - rps->min_freq = valleyview_rps_min_freq(dev_priv);
> - DRM_DEBUG_DRIVER("min GPU freq: %d MHz (%u)\n",
> - intel_gpu_freq(dev_priv, rps->min_freq),
> - rps->min_freq);
> -
> - vlv_iosf_sb_put(dev_priv,
> - BIT(VLV_IOSF_SB_PUNIT) |
> - BIT(VLV_IOSF_SB_NC) |
> - BIT(VLV_IOSF_SB_CCK));
> -}
> -
> -static void cherryview_init_gt_powersave(struct drm_i915_private *dev_priv)
> -{
> - struct intel_rps *rps = &dev_priv->gt_pm.rps;
> - u32 val;
> -
> - vlv_iosf_sb_get(dev_priv,
> - BIT(VLV_IOSF_SB_PUNIT) |
> - BIT(VLV_IOSF_SB_NC) |
> - BIT(VLV_IOSF_SB_CCK));
> -
> - vlv_init_gpll_ref_freq(dev_priv);
> -
> - val = vlv_cck_read(dev_priv, CCK_FUSE_REG);
> -
> - switch ((val >> 2) & 0x7) {
> - case 3:
> - dev_priv->mem_freq = 2000;
> - break;
> - default:
> - dev_priv->mem_freq = 1600;
> - break;
> - }
> - DRM_DEBUG_DRIVER("DDR speed: %d MHz\n", dev_priv->mem_freq);
> -
> - rps->max_freq = cherryview_rps_max_freq(dev_priv);
> - rps->rp0_freq = rps->max_freq;
> - DRM_DEBUG_DRIVER("max GPU freq: %d MHz (%u)\n",
> - intel_gpu_freq(dev_priv, rps->max_freq),
> - rps->max_freq);
> -
> - rps->efficient_freq = cherryview_rps_rpe_freq(dev_priv);
> - DRM_DEBUG_DRIVER("RPe GPU freq: %d MHz (%u)\n",
> - intel_gpu_freq(dev_priv, rps->efficient_freq),
> - rps->efficient_freq);
> -
> - rps->rp1_freq = cherryview_rps_guar_freq(dev_priv);
> - DRM_DEBUG_DRIVER("RP1(Guar) GPU freq: %d MHz (%u)\n",
> - intel_gpu_freq(dev_priv, rps->rp1_freq),
> - rps->rp1_freq);
> -
> - rps->min_freq = cherryview_rps_min_freq(dev_priv);
> - DRM_DEBUG_DRIVER("min GPU freq: %d MHz (%u)\n",
> - intel_gpu_freq(dev_priv, rps->min_freq),
> - rps->min_freq);
> -
> - vlv_iosf_sb_put(dev_priv,
> - BIT(VLV_IOSF_SB_PUNIT) |
> - BIT(VLV_IOSF_SB_NC) |
> - BIT(VLV_IOSF_SB_CCK));
> -
> - WARN_ONCE((rps->max_freq | rps->efficient_freq | rps->rp1_freq |
> - rps->min_freq) & 1,
> - "Odd GPU freq values\n");
> -}
> -
> -static void cherryview_enable_rps(struct drm_i915_private *dev_priv)
> -{
> - u32 val;
> -
> - intel_uncore_forcewake_get(&dev_priv->uncore, FORCEWAKE_ALL);
> -
> - /* 1: Program defaults and thresholds for RPS*/
> - I915_WRITE(GEN6_RP_DOWN_TIMEOUT, 1000000);
> - I915_WRITE(GEN6_RP_UP_THRESHOLD, 59400);
> - I915_WRITE(GEN6_RP_DOWN_THRESHOLD, 245000);
> - I915_WRITE(GEN6_RP_UP_EI, 66000);
> - I915_WRITE(GEN6_RP_DOWN_EI, 350000);
> -
> - I915_WRITE(GEN6_RP_IDLE_HYSTERSIS, 10);
> -
> - /* 2: Enable RPS */
> - I915_WRITE(GEN6_RP_CONTROL,
> - GEN6_RP_MEDIA_HW_NORMAL_MODE |
> - GEN6_RP_MEDIA_IS_GFX |
> - GEN6_RP_ENABLE |
> - GEN6_RP_UP_BUSY_AVG |
> - GEN6_RP_DOWN_IDLE_AVG);
> -
> - /* Setting Fixed Bias */
> - vlv_punit_get(dev_priv);
> -
> - val = VLV_OVERRIDE_EN | VLV_SOC_TDP_EN | CHV_BIAS_CPU_50_SOC_50;
> - vlv_punit_write(dev_priv, VLV_TURBO_SOC_OVERRIDE, val);
> -
> - val = vlv_punit_read(dev_priv, PUNIT_REG_GPU_FREQ_STS);
> -
> - vlv_punit_put(dev_priv);
> -
> - /* RPS code assumes GPLL is used */
> - WARN_ONCE((val & GPLLENABLE) == 0, "GPLL not enabled\n");
> -
> - DRM_DEBUG_DRIVER("GPLL enabled? %s\n", yesno(val & GPLLENABLE));
> - DRM_DEBUG_DRIVER("GPU status: 0x%08x\n", val);
> -
> - reset_rps(dev_priv, valleyview_set_rps);
> -
> - intel_uncore_forcewake_put(&dev_priv->uncore, FORCEWAKE_ALL);
> -}
> -
> -static void valleyview_enable_rps(struct drm_i915_private *dev_priv)
> -{
> - u32 val;
> -
> - intel_uncore_forcewake_get(&dev_priv->uncore, FORCEWAKE_ALL);
> -
> - I915_WRITE(GEN6_RP_DOWN_TIMEOUT, 1000000);
> - I915_WRITE(GEN6_RP_UP_THRESHOLD, 59400);
> - I915_WRITE(GEN6_RP_DOWN_THRESHOLD, 245000);
> - I915_WRITE(GEN6_RP_UP_EI, 66000);
> - I915_WRITE(GEN6_RP_DOWN_EI, 350000);
> -
> - I915_WRITE(GEN6_RP_IDLE_HYSTERSIS, 10);
> -
> - I915_WRITE(GEN6_RP_CONTROL,
> - GEN6_RP_MEDIA_TURBO |
> - GEN6_RP_MEDIA_HW_NORMAL_MODE |
> - GEN6_RP_MEDIA_IS_GFX |
> - GEN6_RP_ENABLE |
> - GEN6_RP_UP_BUSY_AVG |
> - GEN6_RP_DOWN_IDLE_CONT);
> -
> - vlv_punit_get(dev_priv);
> -
> - /* Setting Fixed Bias */
> - val = VLV_OVERRIDE_EN | VLV_SOC_TDP_EN | VLV_BIAS_CPU_125_SOC_875;
> - vlv_punit_write(dev_priv, VLV_TURBO_SOC_OVERRIDE, val);
> -
> - val = vlv_punit_read(dev_priv, PUNIT_REG_GPU_FREQ_STS);
> -
> - vlv_punit_put(dev_priv);
> -
> - /* RPS code assumes GPLL is used */
> - WARN_ONCE((val & GPLLENABLE) == 0, "GPLL not enabled\n");
> -
> - DRM_DEBUG_DRIVER("GPLL enabled? %s\n", yesno(val & GPLLENABLE));
> - DRM_DEBUG_DRIVER("GPU status: 0x%08x\n", val);
> -
> - reset_rps(dev_priv, valleyview_set_rps);
> -
> - intel_uncore_forcewake_put(&dev_priv->uncore, FORCEWAKE_ALL);
> -}
> -
> -static unsigned long intel_pxfreq(u32 vidfreq)
> -{
> - unsigned long freq;
> - int div = (vidfreq & 0x3f0000) >> 16;
> - int post = (vidfreq & 0x3000) >> 12;
> - int pre = (vidfreq & 0x7);
> -
> - if (!pre)
> - return 0;
> -
> - freq = ((div * 133333) / ((1<<post) * pre));
> -
> - return freq;
> -}
> -
> -static const struct cparams {
> - u16 i;
> - u16 t;
> - u16 m;
> - u16 c;
> -} cparams[] = {
> - { 1, 1333, 301, 28664 },
> - { 1, 1066, 294, 24460 },
> - { 1, 800, 294, 25192 },
> - { 0, 1333, 276, 27605 },
> - { 0, 1066, 276, 27605 },
> - { 0, 800, 231, 23784 },
> -};
> -
> -static unsigned long __i915_chipset_val(struct drm_i915_private *dev_priv)
> -{
> - u64 total_count, diff, ret;
> - u32 count1, count2, count3, m = 0, c = 0;
> - unsigned long now = jiffies_to_msecs(jiffies), diff1;
> - int i;
> -
> - lockdep_assert_held(&mchdev_lock);
> -
> - diff1 = now - dev_priv->ips.last_time1;
> -
> - /* Prevent division-by-zero if we are asking too fast.
> - * Also, we don't get interesting results if we are polling
> - * faster than once in 10ms, so just return the saved value
> - * in such cases.
> - */
> - if (diff1 <= 10)
> - return dev_priv->ips.chipset_power;
> -
> - count1 = I915_READ(DMIEC);
> - count2 = I915_READ(DDREC);
> - count3 = I915_READ(CSIEC);
> -
> - total_count = count1 + count2 + count3;
> -
> - /* FIXME: handle per-counter overflow */
> - if (total_count < dev_priv->ips.last_count1) {
> - diff = ~0UL - dev_priv->ips.last_count1;
> - diff += total_count;
> - } else {
> - diff = total_count - dev_priv->ips.last_count1;
> - }
> -
> - for (i = 0; i < ARRAY_SIZE(cparams); i++) {
> - if (cparams[i].i == dev_priv->ips.c_m &&
> - cparams[i].t == dev_priv->ips.r_t) {
> - m = cparams[i].m;
> - c = cparams[i].c;
> - break;
> - }
> - }
> -
> - diff = div_u64(diff, diff1);
> - ret = ((m * diff) + c);
> - ret = div_u64(ret, 10);
> -
> - dev_priv->ips.last_count1 = total_count;
> - dev_priv->ips.last_time1 = now;
> -
> - dev_priv->ips.chipset_power = ret;
> -
> - return ret;
> -}
> -
> -unsigned long i915_chipset_val(struct drm_i915_private *dev_priv)
> -{
> - intel_wakeref_t wakeref;
> - unsigned long val = 0;
> -
> - if (!IS_GEN(dev_priv, 5))
> - return 0;
> -
> - with_intel_runtime_pm(&dev_priv->runtime_pm, wakeref) {
> - spin_lock_irq(&mchdev_lock);
> - val = __i915_chipset_val(dev_priv);
> - spin_unlock_irq(&mchdev_lock);
> - }
> -
> - return val;
> -}
> -
> -unsigned long i915_mch_val(struct drm_i915_private *i915)
> -{
> - unsigned long m, x, b;
> - u32 tsfs;
> -
> - tsfs = intel_uncore_read(&i915->uncore, TSFS);
> -
> - m = ((tsfs & TSFS_SLOPE_MASK) >> TSFS_SLOPE_SHIFT);
> - x = intel_uncore_read8(&i915->uncore, TR1);
> -
> - b = tsfs & TSFS_INTR_MASK;
> -
> - return ((m * x) / 127) - b;
> -}
> -
> -static int _pxvid_to_vd(u8 pxvid)
> -{
> - if (pxvid == 0)
> - return 0;
> -
> - if (pxvid >= 8 && pxvid < 31)
> - pxvid = 31;
> -
> - return (pxvid + 2) * 125;
> -}
> -
> -static u32 pvid_to_extvid(struct drm_i915_private *dev_priv, u8 pxvid)
> -{
> - const int vd = _pxvid_to_vd(pxvid);
> - const int vm = vd - 1125;
> -
> - if (INTEL_INFO(dev_priv)->is_mobile)
> - return vm > 0 ? vm : 0;
> -
> - return vd;
> -}
> -
> -static void __i915_update_gfx_val(struct drm_i915_private *dev_priv)
> -{
> - u64 now, diff, diffms;
> - u32 count;
> -
> - lockdep_assert_held(&mchdev_lock);
> -
> - now = ktime_get_raw_ns();
> - diffms = now - dev_priv->ips.last_time2;
> - do_div(diffms, NSEC_PER_MSEC);
> -
> - /* Don't divide by 0 */
> - if (!diffms)
> - return;
> -
> - count = I915_READ(GFXEC);
> -
> - if (count < dev_priv->ips.last_count2) {
> - diff = ~0UL - dev_priv->ips.last_count2;
> - diff += count;
> - } else {
> - diff = count - dev_priv->ips.last_count2;
> - }
> -
> - dev_priv->ips.last_count2 = count;
> - dev_priv->ips.last_time2 = now;
> -
> - /* More magic constants... */
> - diff = diff * 1181;
> - diff = div_u64(diff, diffms * 10);
> - dev_priv->ips.gfx_power = diff;
> -}
> -
> -void i915_update_gfx_val(struct drm_i915_private *dev_priv)
> -{
> - intel_wakeref_t wakeref;
> -
> - if (!IS_GEN(dev_priv, 5))
> - return;
> -
> - with_intel_runtime_pm(&dev_priv->runtime_pm, wakeref) {
> - spin_lock_irq(&mchdev_lock);
> - __i915_update_gfx_val(dev_priv);
> - spin_unlock_irq(&mchdev_lock);
> - }
> -}
> -
> -static unsigned long __i915_gfx_val(struct drm_i915_private *dev_priv)
> -{
> - unsigned long t, corr, state1, corr2, state2;
> - u32 pxvid, ext_v;
> -
> - lockdep_assert_held(&mchdev_lock);
> -
> - pxvid = I915_READ(PXVFREQ(dev_priv->gt_pm.rps.cur_freq));
> - pxvid = (pxvid >> 24) & 0x7f;
> - ext_v = pvid_to_extvid(dev_priv, pxvid);
> -
> - state1 = ext_v;
> -
> - t = i915_mch_val(dev_priv);
> -
> - /* Revel in the empirically derived constants */
> -
> - /* Correction factor in 1/100000 units */
> - if (t > 80)
> - corr = ((t * 2349) + 135940);
> - else if (t >= 50)
> - corr = ((t * 964) + 29317);
> - else /* < 50 */
> - corr = ((t * 301) + 1004);
> -
> - corr = corr * ((150142 * state1) / 10000 - 78642);
> - corr /= 100000;
> - corr2 = (corr * dev_priv->ips.corr);
> -
> - state2 = (corr2 * state1) / 10000;
> - state2 /= 100; /* convert to mW */
> -
> - __i915_update_gfx_val(dev_priv);
> -
> - return dev_priv->ips.gfx_power + state2;
> -}
> -
> -unsigned long i915_gfx_val(struct drm_i915_private *dev_priv)
> -{
> - intel_wakeref_t wakeref;
> - unsigned long val = 0;
> -
> - if (!IS_GEN(dev_priv, 5))
> - return 0;
> -
> - with_intel_runtime_pm(&dev_priv->runtime_pm, wakeref) {
> - spin_lock_irq(&mchdev_lock);
> - val = __i915_gfx_val(dev_priv);
> - spin_unlock_irq(&mchdev_lock);
> - }
> -
> - return val;
> -}
> -
> -static struct drm_i915_private __rcu *i915_mch_dev;
> -
> -static struct drm_i915_private *mchdev_get(void)
> -{
> - struct drm_i915_private *i915;
> -
> - rcu_read_lock();
> - i915 = rcu_dereference(i915_mch_dev);
> - if (!kref_get_unless_zero(&i915->drm.ref))
> - i915 = NULL;
> - rcu_read_unlock();
> -
> - return i915;
> -}
> -
> -/**
> - * i915_read_mch_val - return value for IPS use
> - *
> - * Calculate and return a value for the IPS driver to use when deciding whether
> - * we have thermal and power headroom to increase CPU or GPU power budget.
> - */
> -unsigned long i915_read_mch_val(void)
> -{
> - struct drm_i915_private *i915;
> - unsigned long chipset_val = 0;
> - unsigned long graphics_val = 0;
> - intel_wakeref_t wakeref;
> -
> - i915 = mchdev_get();
> - if (!i915)
> - return 0;
> -
> - with_intel_runtime_pm(&i915->runtime_pm, wakeref) {
> - spin_lock_irq(&mchdev_lock);
> - chipset_val = __i915_chipset_val(i915);
> - graphics_val = __i915_gfx_val(i915);
> - spin_unlock_irq(&mchdev_lock);
> - }
> -
> - drm_dev_put(&i915->drm);
> - return chipset_val + graphics_val;
> -}
> -EXPORT_SYMBOL_GPL(i915_read_mch_val);
> -
> -/**
> - * i915_gpu_raise - raise GPU frequency limit
> - *
> - * Raise the limit; IPS indicates we have thermal headroom.
> - */
> -bool i915_gpu_raise(void)
> -{
> - struct drm_i915_private *i915;
> -
> - i915 = mchdev_get();
> - if (!i915)
> - return false;
> -
> - spin_lock_irq(&mchdev_lock);
> - if (i915->ips.max_delay > i915->ips.fmax)
> - i915->ips.max_delay--;
> - spin_unlock_irq(&mchdev_lock);
> -
> - drm_dev_put(&i915->drm);
> - return true;
> -}
> -EXPORT_SYMBOL_GPL(i915_gpu_raise);
> -
> -/**
> - * i915_gpu_lower - lower GPU frequency limit
> - *
> - * IPS indicates we're close to a thermal limit, so throttle back the GPU
> - * frequency maximum.
> - */
> -bool i915_gpu_lower(void)
> -{
> - struct drm_i915_private *i915;
> -
> - i915 = mchdev_get();
> - if (!i915)
> - return false;
> -
> - spin_lock_irq(&mchdev_lock);
> - if (i915->ips.max_delay < i915->ips.min_delay)
> - i915->ips.max_delay++;
> - spin_unlock_irq(&mchdev_lock);
> -
> - drm_dev_put(&i915->drm);
> - return true;
> -}
> -EXPORT_SYMBOL_GPL(i915_gpu_lower);
> -
> -/**
> - * i915_gpu_busy - indicate GPU business to IPS
> - *
> - * Tell the IPS driver whether or not the GPU is busy.
> - */
> -bool i915_gpu_busy(void)
> -{
> - struct drm_i915_private *i915;
> - bool ret;
> -
> - i915 = mchdev_get();
> - if (!i915)
> - return false;
> -
> - ret = i915->gt.awake;
> -
> - drm_dev_put(&i915->drm);
> - return ret;
> -}
> -EXPORT_SYMBOL_GPL(i915_gpu_busy);
> -
> -/**
> - * i915_gpu_turbo_disable - disable graphics turbo
> - *
> - * Disable graphics turbo by resetting the max frequency and setting the
> - * current frequency to the default.
> - */
> -bool i915_gpu_turbo_disable(void)
> -{
> - struct drm_i915_private *i915;
> - bool ret;
> -
> - i915 = mchdev_get();
> - if (!i915)
> - return false;
> -
> - spin_lock_irq(&mchdev_lock);
> - i915->ips.max_delay = i915->ips.fstart;
> - ret = ironlake_set_drps(i915, i915->ips.fstart);
> - spin_unlock_irq(&mchdev_lock);
> -
> - drm_dev_put(&i915->drm);
> - return ret;
> -}
> -EXPORT_SYMBOL_GPL(i915_gpu_turbo_disable);
> -
> -/**
> - * Tells the intel_ips driver that the i915 driver is now loaded, if
> - * IPS got loaded first.
> - *
> - * This awkward dance is so that neither module has to depend on the
> - * other in order for IPS to do the appropriate communication of
> - * GPU turbo limits to i915.
> - */
> -static void
> -ips_ping_for_i915_load(void)
> -{
> - void (*link)(void);
> -
> - link = symbol_get(ips_link_to_i915_driver);
> - if (link) {
> - link();
> - symbol_put(ips_link_to_i915_driver);
> - }
> -}
> -
> -void intel_gpu_ips_init(struct drm_i915_private *dev_priv)
> -{
> - /* We only register the i915 ips part with intel-ips once everything is
> - * set up, to avoid intel-ips sneaking in and reading bogus values. */
> - rcu_assign_pointer(i915_mch_dev, dev_priv);
> -
> - ips_ping_for_i915_load();
> -}
> -
> -void intel_gpu_ips_teardown(void)
> -{
> - rcu_assign_pointer(i915_mch_dev, NULL);
> -}
> -
> -static void intel_init_emon(struct drm_i915_private *dev_priv)
> -{
> - u32 lcfuse;
> - u8 pxw[16];
> - int i;
> -
> - /* Disable to program */
> - I915_WRITE(ECR, 0);
> - POSTING_READ(ECR);
> -
> - /* Program energy weights for various events */
> - I915_WRITE(SDEW, 0x15040d00);
> - I915_WRITE(CSIEW0, 0x007f0000);
> - I915_WRITE(CSIEW1, 0x1e220004);
> - I915_WRITE(CSIEW2, 0x04000004);
> -
> - for (i = 0; i < 5; i++)
> - I915_WRITE(PEW(i), 0);
> - for (i = 0; i < 3; i++)
> - I915_WRITE(DEW(i), 0);
> -
> - /* Program P-state weights to account for frequency power adjustment */
> - for (i = 0; i < 16; i++) {
> - u32 pxvidfreq = I915_READ(PXVFREQ(i));
> - unsigned long freq = intel_pxfreq(pxvidfreq);
> - unsigned long vid = (pxvidfreq & PXVFREQ_PX_MASK) >>
> - PXVFREQ_PX_SHIFT;
> - unsigned long val;
> -
> - val = vid * vid;
> - val *= (freq / 1000);
> - val *= 255;
> - val /= (127*127*900);
> - if (val > 0xff)
> - DRM_ERROR("bad pxval: %ld\n", val);
> - pxw[i] = val;
> - }
> - /* Render standby states get 0 weight */
> - pxw[14] = 0;
> - pxw[15] = 0;
> -
> - for (i = 0; i < 4; i++) {
> - u32 val = (pxw[i*4] << 24) | (pxw[(i*4)+1] << 16) |
> - (pxw[(i*4)+2] << 8) | (pxw[(i*4)+3]);
> - I915_WRITE(PXW(i), val);
> - }
> -
> - /* Adjust magic regs to magic values (more experimental results) */
> - I915_WRITE(OGW0, 0);
> - I915_WRITE(OGW1, 0);
> - I915_WRITE(EG0, 0x00007f00);
> - I915_WRITE(EG1, 0x0000000e);
> - I915_WRITE(EG2, 0x000e0000);
> - I915_WRITE(EG3, 0x68000300);
> - I915_WRITE(EG4, 0x42000000);
> - I915_WRITE(EG5, 0x00140031);
> - I915_WRITE(EG6, 0);
> - I915_WRITE(EG7, 0);
> -
> - for (i = 0; i < 8; i++)
> - I915_WRITE(PXWL(i), 0);
> -
> - /* Enable PMON + select events */
> - I915_WRITE(ECR, 0x80000019);
> -
> - lcfuse = I915_READ(LCFUSE02);
> -
> - dev_priv->ips.corr = (lcfuse & LCFUSE_HIV_MASK);
> -}
> -
> -void intel_init_gt_powersave(struct drm_i915_private *dev_priv)
> -{
> - struct intel_rps *rps = &dev_priv->gt_pm.rps;
> -
> - /* Powersaving is controlled by the host when inside a VM */
> - if (intel_vgpu_active(dev_priv))
> - mkwrite_device_info(dev_priv)->has_rps = false;
> -
> - /* Initialize RPS limits (for userspace) */
> - if (IS_CHERRYVIEW(dev_priv))
> - cherryview_init_gt_powersave(dev_priv);
> - else if (IS_VALLEYVIEW(dev_priv))
> - valleyview_init_gt_powersave(dev_priv);
> - else if (INTEL_GEN(dev_priv) >= 6)
> - gen6_init_rps_frequencies(dev_priv);
> -
> - /* Derive initial user preferences/limits from the hardware limits */
> - rps->max_freq_softlimit = rps->max_freq;
> - rps->min_freq_softlimit = rps->min_freq;
> -
> - /* After setting max-softlimit, find the overclock max freq */
> - if (IS_GEN(dev_priv, 6) ||
> - IS_IVYBRIDGE(dev_priv) || IS_HASWELL(dev_priv)) {
> - u32 params = 0;
> -
> - sandybridge_pcode_read(dev_priv, GEN6_READ_OC_PARAMS,
> - ¶ms, NULL);
> - if (params & BIT(31)) { /* OC supported */
> - DRM_DEBUG_DRIVER("Overclocking supported, max: %dMHz, overclock: %dMHz\n",
> - (rps->max_freq & 0xff) * 50,
> - (params & 0xff) * 50);
> - rps->max_freq = params & 0xff;
> - }
> - }
> -
> - /* Finally allow us to boost to max by default */
> - rps->boost_freq = rps->max_freq;
> - rps->idle_freq = rps->min_freq;
> - rps->cur_freq = rps->idle_freq;
> -}
> -
> -void intel_sanitize_gt_powersave(struct drm_i915_private *dev_priv)
> -{
> - dev_priv->gt_pm.rps.enabled = true; /* force RPS disabling */
> - intel_disable_gt_powersave(dev_priv);
> -
> - if (INTEL_GEN(dev_priv) >= 11)
> - gen11_reset_rps_interrupts(dev_priv);
> - else if (INTEL_GEN(dev_priv) >= 6)
> - gen6_reset_rps_interrupts(dev_priv);
> -}
> -
> -static void intel_disable_rps(struct drm_i915_private *dev_priv)
> -{
> - lockdep_assert_held(&dev_priv->gt_pm.rps.lock);
> -
> - if (!dev_priv->gt_pm.rps.enabled)
> - return;
> -
> - if (INTEL_GEN(dev_priv) >= 9)
> - gen9_disable_rps(dev_priv);
> - else if (IS_CHERRYVIEW(dev_priv))
> - cherryview_disable_rps(dev_priv);
> - else if (IS_VALLEYVIEW(dev_priv))
> - valleyview_disable_rps(dev_priv);
> - else if (INTEL_GEN(dev_priv) >= 6)
> - gen6_disable_rps(dev_priv);
> - else if (IS_IRONLAKE_M(dev_priv))
> - ironlake_disable_drps(dev_priv);
> -
> - dev_priv->gt_pm.rps.enabled = false;
> -}
> -
> -void intel_disable_gt_powersave(struct drm_i915_private *dev_priv)
> -{
> - mutex_lock(&dev_priv->gt_pm.rps.lock);
> -
> - intel_disable_rps(dev_priv);
> - if (HAS_LLC(dev_priv))
> - intel_llc_disable(&dev_priv->gt.llc);
> -
> - mutex_unlock(&dev_priv->gt_pm.rps.lock);
> -}
> -
> -static void intel_enable_rps(struct drm_i915_private *dev_priv)
> -{
> - struct intel_rps *rps = &dev_priv->gt_pm.rps;
> -
> - lockdep_assert_held(&rps->lock);
> -
> - if (rps->enabled)
> - return;
> -
> - if (IS_CHERRYVIEW(dev_priv)) {
> - cherryview_enable_rps(dev_priv);
> - } else if (IS_VALLEYVIEW(dev_priv)) {
> - valleyview_enable_rps(dev_priv);
> - } else if (INTEL_GEN(dev_priv) >= 9) {
> - gen9_enable_rps(dev_priv);
> - } else if (IS_BROADWELL(dev_priv)) {
> - gen8_enable_rps(dev_priv);
> - } else if (INTEL_GEN(dev_priv) >= 6) {
> - gen6_enable_rps(dev_priv);
> - } else if (IS_IRONLAKE_M(dev_priv)) {
> - ironlake_enable_drps(dev_priv);
> - intel_init_emon(dev_priv);
> - }
> -
> - WARN_ON(rps->max_freq < rps->min_freq);
> - WARN_ON(rps->idle_freq > rps->max_freq);
> -
> - WARN_ON(rps->efficient_freq < rps->min_freq);
> - WARN_ON(rps->efficient_freq > rps->max_freq);
> -
> - rps->enabled = true;
> -}
> -
> -void intel_enable_gt_powersave(struct drm_i915_private *dev_priv)
> -{
> - /* Powersaving is controlled by the host when inside a VM */
> - if (intel_vgpu_active(dev_priv))
> - return;
> -
> - mutex_lock(&dev_priv->gt_pm.rps.lock);
> -
> - if (HAS_RPS(dev_priv))
> - intel_enable_rps(dev_priv);
> -
> - intel_llc_enable(&dev_priv->gt.llc);
> -
> - mutex_unlock(&dev_priv->gt_pm.rps.lock);
> -}
> -
> -static void ibx_init_clock_gating(struct drm_i915_private *dev_priv)
> -{
> - /*
> - * On Ibex Peak and Cougar Point, we need to disable clock
> - * gating for the panel power sequencer or it will fail to
> - * start up when no ports are active.
> - */
> - I915_WRITE(SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE);
> -}
> -
> -static void g4x_disable_trickle_feed(struct drm_i915_private *dev_priv)
> -{
> - enum pipe pipe;
> -
> - for_each_pipe(dev_priv, pipe) {
> - I915_WRITE(DSPCNTR(pipe),
> - I915_READ(DSPCNTR(pipe)) |
> - DISPPLANE_TRICKLE_FEED_DISABLE);
> -
> - I915_WRITE(DSPSURF(pipe), I915_READ(DSPSURF(pipe)));
> - POSTING_READ(DSPSURF(pipe));
> - }
> -}
> -
> -static void ilk_init_clock_gating(struct drm_i915_private *dev_priv)
> -{
> - u32 dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE;
> -
> - /*
> - * Required for FBC
> - * WaFbcDisableDpfcClockGating:ilk
> - */
> - dspclk_gate |= ILK_DPFCRUNIT_CLOCK_GATE_DISABLE |
> - ILK_DPFCUNIT_CLOCK_GATE_DISABLE |
> - ILK_DPFDUNIT_CLOCK_GATE_ENABLE;
> -
> - I915_WRITE(PCH_3DCGDIS0,
> - MARIUNIT_CLOCK_GATE_DISABLE |
> - SVSMUNIT_CLOCK_GATE_DISABLE);
> - I915_WRITE(PCH_3DCGDIS1,
> - VFMUNIT_CLOCK_GATE_DISABLE);
> -
> - /*
> - * According to the spec the following bits should be set in
> - * order to enable memory self-refresh
> - * The bit 22/21 of 0x42004
> - * The bit 5 of 0x42020
> - * The bit 15 of 0x45000
> - */
> - I915_WRITE(ILK_DISPLAY_CHICKEN2,
> - (I915_READ(ILK_DISPLAY_CHICKEN2) |
> - ILK_DPARB_GATE | ILK_VSDPFD_FULL));
> - dspclk_gate |= ILK_DPARBUNIT_CLOCK_GATE_ENABLE;
> - I915_WRITE(DISP_ARB_CTL,
> - (I915_READ(DISP_ARB_CTL) |
> - DISP_FBC_WM_DIS));
> -
> - /*
> - * Based on the document from hardware guys the following bits
> - * should be set unconditionally in order to enable FBC.
> - * The bit 22 of 0x42000
> - * The bit 22 of 0x42004
> - * The bit 7,8,9 of 0x42020.
> - */
> - if (IS_IRONLAKE_M(dev_priv)) {
> - /* WaFbcAsynchFlipDisableFbcQueue:ilk */
> - I915_WRITE(ILK_DISPLAY_CHICKEN1,
> - I915_READ(ILK_DISPLAY_CHICKEN1) |
> - ILK_FBCQ_DIS);
> - I915_WRITE(ILK_DISPLAY_CHICKEN2,
> - I915_READ(ILK_DISPLAY_CHICKEN2) |
> - ILK_DPARB_GATE);
> - }
> -
> - I915_WRITE(ILK_DSPCLK_GATE_D, dspclk_gate);
> -
> - I915_WRITE(ILK_DISPLAY_CHICKEN2,
> - I915_READ(ILK_DISPLAY_CHICKEN2) |
> - ILK_ELPIN_409_SELECT);
> - I915_WRITE(_3D_CHICKEN2,
> - _3D_CHICKEN2_WM_READ_PIPELINED << 16 |
> - _3D_CHICKEN2_WM_READ_PIPELINED);
> -
> - /* WaDisableRenderCachePipelinedFlush:ilk */
> - I915_WRITE(CACHE_MODE_0,
> - _MASKED_BIT_ENABLE(CM0_PIPELINED_RENDER_FLUSH_DISABLE));
> -
> - /* WaDisable_RenderCache_OperationalFlush:ilk */
> - I915_WRITE(CACHE_MODE_0, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
> -
> - g4x_disable_trickle_feed(dev_priv);
> -
> - ibx_init_clock_gating(dev_priv);
> -}
> -
> -static void cpt_init_clock_gating(struct drm_i915_private *dev_priv)
> -{
> - enum pipe pipe;
> - u32 val;
> -
> - /*
> - * On Ibex Peak and Cougar Point, we need to disable clock
> - * gating for the panel power sequencer or it will fail to
> - * start up when no ports are active.
> - */
> - I915_WRITE(SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE |
> - PCH_DPLUNIT_CLOCK_GATE_DISABLE |
> - PCH_CPUNIT_CLOCK_GATE_DISABLE);
> - I915_WRITE(SOUTH_CHICKEN2, I915_READ(SOUTH_CHICKEN2) |
> - DPLS_EDP_PPS_FIX_DIS);
> - /* The below fixes the weird display corruption, a few pixels shifted
> - * downward, on (only) LVDS of some HP laptops with IVY.
> - */
> - for_each_pipe(dev_priv, pipe) {
> - val = I915_READ(TRANS_CHICKEN2(pipe));
> - val |= TRANS_CHICKEN2_TIMING_OVERRIDE;
> - val &= ~TRANS_CHICKEN2_FDI_POLARITY_REVERSED;
> - if (dev_priv->vbt.fdi_rx_polarity_inverted)
> - val |= TRANS_CHICKEN2_FDI_POLARITY_REVERSED;
> - val &= ~TRANS_CHICKEN2_FRAME_START_DELAY_MASK;
> - val &= ~TRANS_CHICKEN2_DISABLE_DEEP_COLOR_COUNTER;
> - val &= ~TRANS_CHICKEN2_DISABLE_DEEP_COLOR_MODESWITCH;
> - I915_WRITE(TRANS_CHICKEN2(pipe), val);
> - }
> - /* WADP0ClockGatingDisable */
> - for_each_pipe(dev_priv, pipe) {
> - I915_WRITE(TRANS_CHICKEN1(pipe),
> - TRANS_CHICKEN1_DP0UNIT_GC_DISABLE);
> - }
> -}
> -
> -static void gen6_check_mch_setup(struct drm_i915_private *dev_priv)
> -{
> - u32 tmp;
> -
> - tmp = I915_READ(MCH_SSKPD);
> - if ((tmp & MCH_SSKPD_WM0_MASK) != MCH_SSKPD_WM0_VAL)
> - DRM_DEBUG_KMS("Wrong MCH_SSKPD value: 0x%08x This can cause underruns.\n",
> - tmp);
> -}
> -
> -static void gen6_init_clock_gating(struct drm_i915_private *dev_priv)
> -{
> - u32 dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE;
> -
> - I915_WRITE(ILK_DSPCLK_GATE_D, dspclk_gate);
> -
> - I915_WRITE(ILK_DISPLAY_CHICKEN2,
> - I915_READ(ILK_DISPLAY_CHICKEN2) |
> - ILK_ELPIN_409_SELECT);
> -
> - /* WaDisableHiZPlanesWhenMSAAEnabled:snb */
> - I915_WRITE(_3D_CHICKEN,
> - _MASKED_BIT_ENABLE(_3D_CHICKEN_HIZ_PLANE_DISABLE_MSAA_4X_SNB));
> -
> - /* WaDisable_RenderCache_OperationalFlush:snb */
> - I915_WRITE(CACHE_MODE_0, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
> -
> - /*
> - * BSpec recoomends 8x4 when MSAA is used,
> - * however in practice 16x4 seems fastest.
> - *
> - * Note that PS/WM thread counts depend on the WIZ hashing
> - * disable bit, which we don't touch here, but it's good
> - * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM).
> - */
> - I915_WRITE(GEN6_GT_MODE,
> - _MASKED_FIELD(GEN6_WIZ_HASHING_MASK, GEN6_WIZ_HASHING_16x4));
> -
> - I915_WRITE(CACHE_MODE_0,
> - _MASKED_BIT_DISABLE(CM0_STC_EVICT_DISABLE_LRA_SNB));
> -
> - I915_WRITE(GEN6_UCGCTL1,
> - I915_READ(GEN6_UCGCTL1) |
> - GEN6_BLBUNIT_CLOCK_GATE_DISABLE |
> - GEN6_CSUNIT_CLOCK_GATE_DISABLE);
> -
> - /* According to the BSpec vol1g, bit 12 (RCPBUNIT) clock
> - * gating disable must be set. Failure to set it results in
> - * flickering pixels due to Z write ordering failures after
> - * some amount of runtime in the Mesa "fire" demo, and Unigine
> - * Sanctuary and Tropics, and apparently anything else with
> - * alpha test or pixel discard.
> - *
> - * According to the spec, bit 11 (RCCUNIT) must also be set,
> - * but we didn't debug actual testcases to find it out.
> - *
> - * WaDisableRCCUnitClockGating:snb
> - * WaDisableRCPBUnitClockGating:snb
> - */
> - I915_WRITE(GEN6_UCGCTL2,
> - GEN6_RCPBUNIT_CLOCK_GATE_DISABLE |
> - GEN6_RCCUNIT_CLOCK_GATE_DISABLE);
> -
> - /* WaStripsFansDisableFastClipPerformanceFix:snb */
> - I915_WRITE(_3D_CHICKEN3,
> - _MASKED_BIT_ENABLE(_3D_CHICKEN3_SF_DISABLE_FASTCLIP_CULL));
> -
> - /*
> - * Bspec says:
> - * "This bit must be set if 3DSTATE_CLIP clip mode is set to normal and
> - * 3DSTATE_SF number of SF output attributes is more than 16."
> - */
> - I915_WRITE(_3D_CHICKEN3,
> - _MASKED_BIT_ENABLE(_3D_CHICKEN3_SF_DISABLE_PIPELINED_ATTR_FETCH));
> -
> - /*
> - * According to the spec the following bits should be
> - * set in order to enable memory self-refresh and fbc:
> - * The bit21 and bit22 of 0x42000
> - * The bit21 and bit22 of 0x42004
> - * The bit5 and bit7 of 0x42020
> - * The bit14 of 0x70180
> - * The bit14 of 0x71180
> - *
> - * WaFbcAsynchFlipDisableFbcQueue:snb
> - */
> - I915_WRITE(ILK_DISPLAY_CHICKEN1,
> - I915_READ(ILK_DISPLAY_CHICKEN1) |
> - ILK_FBCQ_DIS | ILK_PABSTRETCH_DIS);
> - I915_WRITE(ILK_DISPLAY_CHICKEN2,
> - I915_READ(ILK_DISPLAY_CHICKEN2) |
> - ILK_DPARB_GATE | ILK_VSDPFD_FULL);
> - I915_WRITE(ILK_DSPCLK_GATE_D,
> - I915_READ(ILK_DSPCLK_GATE_D) |
> - ILK_DPARBUNIT_CLOCK_GATE_ENABLE |
> - ILK_DPFDUNIT_CLOCK_GATE_ENABLE);
> -
> - g4x_disable_trickle_feed(dev_priv);
> -
> - cpt_init_clock_gating(dev_priv);
> -
> - gen6_check_mch_setup(dev_priv);
> -}
> -
> -static void gen7_setup_fixed_func_scheduler(struct drm_i915_private *dev_priv)
> -{
> - u32 reg = I915_READ(GEN7_FF_THREAD_MODE);
> + u32 reg = I915_READ(GEN7_FF_THREAD_MODE);
>
> /*
> * WaVSThreadDispatchOverride:ivb,vlv
> @@ -8942,90 +7325,8 @@ void intel_init_pm(struct drm_i915_private *dev_priv)
> }
> }
>
> -static int byt_gpu_freq(struct drm_i915_private *dev_priv, int val)
> -{
> - struct intel_rps *rps = &dev_priv->gt_pm.rps;
> -
> - /*
> - * N = val - 0xb7
> - * Slow = Fast = GPLL ref * N
> - */
> - return DIV_ROUND_CLOSEST(rps->gpll_ref_freq * (val - 0xb7), 1000);
> -}
> -
> -static int byt_freq_opcode(struct drm_i915_private *dev_priv, int val)
> -{
> - struct intel_rps *rps = &dev_priv->gt_pm.rps;
> -
> - return DIV_ROUND_CLOSEST(1000 * val, rps->gpll_ref_freq) + 0xb7;
> -}
> -
> -static int chv_gpu_freq(struct drm_i915_private *dev_priv, int val)
> -{
> - struct intel_rps *rps = &dev_priv->gt_pm.rps;
> -
> - /*
> - * N = val / 2
> - * CU (slow) = CU2x (fast) / 2 = GPLL ref * N / 2
> - */
> - return DIV_ROUND_CLOSEST(rps->gpll_ref_freq * val, 2 * 2 * 1000);
> -}
> -
> -static int chv_freq_opcode(struct drm_i915_private *dev_priv, int val)
> -{
> - struct intel_rps *rps = &dev_priv->gt_pm.rps;
> -
> - /* CHV needs even values */
> - return DIV_ROUND_CLOSEST(2 * 1000 * val, rps->gpll_ref_freq) * 2;
> -}
> -
> -int intel_gpu_freq(struct drm_i915_private *dev_priv, int val)
> -{
> - if (INTEL_GEN(dev_priv) >= 9)
> - return DIV_ROUND_CLOSEST(val * GT_FREQUENCY_MULTIPLIER,
> - GEN9_FREQ_SCALER);
> - else if (IS_CHERRYVIEW(dev_priv))
> - return chv_gpu_freq(dev_priv, val);
> - else if (IS_VALLEYVIEW(dev_priv))
> - return byt_gpu_freq(dev_priv, val);
> - else
> - return val * GT_FREQUENCY_MULTIPLIER;
> -}
> -
> -int intel_freq_opcode(struct drm_i915_private *dev_priv, int val)
> -{
> - if (INTEL_GEN(dev_priv) >= 9)
> - return DIV_ROUND_CLOSEST(val * GEN9_FREQ_SCALER,
> - GT_FREQUENCY_MULTIPLIER);
> - else if (IS_CHERRYVIEW(dev_priv))
> - return chv_freq_opcode(dev_priv, val);
> - else if (IS_VALLEYVIEW(dev_priv))
> - return byt_freq_opcode(dev_priv, val);
> - else
> - return DIV_ROUND_CLOSEST(val, GT_FREQUENCY_MULTIPLIER);
> -}
> -
> void intel_pm_setup(struct drm_i915_private *dev_priv)
> {
> - mutex_init(&dev_priv->gt_pm.rps.lock);
> - mutex_init(&dev_priv->gt_pm.rps.power.mutex);
> -
> - atomic_set(&dev_priv->gt_pm.rps.num_waiters, 0);
> -
> dev_priv->runtime_pm.suspended = false;
> atomic_set(&dev_priv->runtime_pm.wakeref_count, 0);
> }
> -
> -u32 intel_get_cagf(struct drm_i915_private *dev_priv, u32 rpstat)
> -{
> - u32 cagf;
> -
> - if (INTEL_GEN(dev_priv) >= 9)
> - cagf = (rpstat & GEN9_CAGF_MASK) >> GEN9_CAGF_SHIFT;
> - else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
> - cagf = (rpstat & HSW_CAGF_MASK) >> HSW_CAGF_SHIFT;
> - else
> - cagf = (rpstat & GEN6_CAGF_MASK) >> GEN6_CAGF_SHIFT;
> -
> - return cagf;
> -}
> diff --git a/drivers/gpu/drm/i915/intel_pm.h b/drivers/gpu/drm/i915/intel_pm.h
> index 93d192d0610a..b56e6285d1c3 100644
> --- a/drivers/gpu/drm/i915/intel_pm.h
> +++ b/drivers/gpu/drm/i915/intel_pm.h
> @@ -29,15 +29,6 @@ void intel_update_watermarks(struct intel_crtc *crtc);
> void intel_init_pm(struct drm_i915_private *dev_priv);
> void intel_init_clock_gating_hooks(struct drm_i915_private *dev_priv);
> void intel_pm_setup(struct drm_i915_private *dev_priv);
> -void intel_gpu_ips_init(struct drm_i915_private *dev_priv);
> -void intel_gpu_ips_teardown(void);
> -void intel_init_gt_powersave(struct drm_i915_private *dev_priv);
> -void intel_sanitize_gt_powersave(struct drm_i915_private *dev_priv);
> -void intel_enable_gt_powersave(struct drm_i915_private *dev_priv);
> -void intel_disable_gt_powersave(struct drm_i915_private *dev_priv);
> -void gen6_rps_busy(struct drm_i915_private *dev_priv);
> -void gen6_rps_idle(struct drm_i915_private *dev_priv);
> -void gen6_rps_boost(struct i915_request *rq);
> void g4x_wm_get_hw_state(struct drm_i915_private *dev_priv);
> void vlv_wm_get_hw_state(struct drm_i915_private *dev_priv);
> void ilk_wm_get_hw_state(struct drm_i915_private *dev_priv);
> @@ -69,19 +60,6 @@ int skl_check_pipe_max_pixel_rate(struct intel_crtc *intel_crtc,
> void intel_init_ipc(struct drm_i915_private *dev_priv);
> void intel_enable_ipc(struct drm_i915_private *dev_priv);
>
> -int intel_gpu_freq(struct drm_i915_private *dev_priv, int val);
> -int intel_freq_opcode(struct drm_i915_private *dev_priv, int val);
> -
> -u32 intel_get_cagf(struct drm_i915_private *dev_priv, u32 rpstat1);
> -
> -unsigned long i915_chipset_val(struct drm_i915_private *dev_priv);
> -unsigned long i915_mch_val(struct drm_i915_private *dev_priv);
> -unsigned long i915_gfx_val(struct drm_i915_private *dev_priv);
> -void i915_update_gfx_val(struct drm_i915_private *dev_priv);
> -
> -bool ironlake_set_drps(struct drm_i915_private *dev_priv, u8 val);
> -int intel_set_rps(struct drm_i915_private *dev_priv, u8 val);
> -void intel_rps_mark_interactive(struct drm_i915_private *i915, bool interactive);
> bool intel_set_memory_cxsr(struct drm_i915_private *dev_priv, bool enable);
>
> #endif /* __INTEL_PM_H__ */
> --
> 2.23.0
_______________________________________________
Intel-gfx mailing list
Intel-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/intel-gfx
WARNING: multiple messages have this Message-ID (diff)
From: Andi Shyti <andi.shyti@intel.com>
To: Intel GFX <intel-gfx@lists.freedesktop.org>
Subject: Re: [Intel-gfx] [PATCH v2 1/2] drm/i915: Extract GT render power state management
Date: Wed, 23 Oct 2019 12:52:48 +0300 [thread overview]
Message-ID: <20191023095248.GA2593@intel.intel> (raw)
Message-ID: <20191023095248.dJPU7P0IxUwC-y1g9LCTuZQkhJmnA6EYXD_sD8Xw6zg@z> (raw)
In-Reply-To: <20191023095000.3782-1-andi.shyti@intel.com>
Hi,
Sorry, I messed up with the e-mail (and with the in-reply-to, as
well, and I forgot to add a note to this v2), please take this
as 1/2 because it has the correct e-mail.
Andi
On Wed, Oct 23, 2019 at 12:50:00PM +0300, Andi Shyti wrote:
> i915_irq.c is large. One reason for this is that has a large chunk of
> the GT render power management stashed away in it. Extract that logic
> out of i915_irq.c and intel_pm.c and put it under one roof.
>
> Based on a patch by Chris Wilson.
>
> Signed-off-by: Andi Shyti <andi.shyti@intel.com>
> Cc: Chris Wilson <chris@chris-wilson.co.uk>
> ---
> drivers/gpu/drm/i915/Makefile | 3 +-
> drivers/gpu/drm/i915/display/intel_display.c | 8 +-
> drivers/gpu/drm/i915/gt/intel_gt.c | 6 +-
> drivers/gpu/drm/i915/gt/intel_gt_irq.c | 5 +-
> drivers/gpu/drm/i915/gt/intel_gt_pm.c | 49 +-
> drivers/gpu/drm/i915/gt/intel_gt_pm.h | 1 +
> drivers/gpu/drm/i915/gt/intel_gt_types.h | 2 +
> drivers/gpu/drm/i915/gt/intel_llc.c | 2 +-
> drivers/gpu/drm/i915/gt/intel_rps.c | 1872 +++++++++++++++
> drivers/gpu/drm/i915/gt/intel_rps.h | 37 +
> drivers/gpu/drm/i915/gt/intel_rps_types.h | 93 +
> drivers/gpu/drm/i915/gt/selftest_llc.c | 7 +-
> .../gpu/drm/i915/gt/uc/intel_guc_submission.c | 4 +-
> drivers/gpu/drm/i915/i915_debugfs.c | 73 +-
> drivers/gpu/drm/i915/i915_drv.c | 2 +
> drivers/gpu/drm/i915/i915_drv.h | 96 -
> drivers/gpu/drm/i915/i915_gem.c | 3 +-
> drivers/gpu/drm/i915/i915_irq.c | 359 +--
> drivers/gpu/drm/i915/i915_irq.h | 3 -
> drivers/gpu/drm/i915/i915_pmu.c | 10 +-
> drivers/gpu/drm/i915/i915_request.c | 7 +-
> drivers/gpu/drm/i915/i915_sysfs.c | 74 +-
> drivers/gpu/drm/i915/intel_pm.c | 2105 ++---------------
> drivers/gpu/drm/i915/intel_pm.h | 22 -
> 24 files changed, 2359 insertions(+), 2484 deletions(-)
> create mode 100644 drivers/gpu/drm/i915/gt/intel_rps.c
> create mode 100644 drivers/gpu/drm/i915/gt/intel_rps.h
> create mode 100644 drivers/gpu/drm/i915/gt/intel_rps_types.h
>
> diff --git a/drivers/gpu/drm/i915/Makefile b/drivers/gpu/drm/i915/Makefile
> index 2fd4bed188e5..133ca59e7c48 100644
> --- a/drivers/gpu/drm/i915/Makefile
> +++ b/drivers/gpu/drm/i915/Makefile
> @@ -90,11 +90,12 @@ gt-y += \
> gt/intel_hangcheck.o \
> gt/intel_llc.o \
> gt/intel_lrc.o \
> + gt/intel_mocs.o \
> gt/intel_rc6.o \
> gt/intel_renderstate.o \
> gt/intel_reset.o \
> gt/intel_ringbuffer.o \
> - gt/intel_mocs.o \
> + gt/intel_rps.o \
> gt/intel_sseu.o \
> gt/intel_timeline.o \
> gt/intel_workarounds.o
> diff --git a/drivers/gpu/drm/i915/display/intel_display.c b/drivers/gpu/drm/i915/display/intel_display.c
> index 236fdf122e47..49c2c7f71394 100644
> --- a/drivers/gpu/drm/i915/display/intel_display.c
> +++ b/drivers/gpu/drm/i915/display/intel_display.c
> @@ -55,6 +55,8 @@
> #include "display/intel_tv.h"
> #include "display/intel_vdsc.h"
>
> +#include "gt/intel_rps.h"
> +
> #include "i915_drv.h"
> #include "i915_trace.h"
> #include "intel_acpi.h"
> @@ -14782,7 +14784,7 @@ static int do_rps_boost(struct wait_queue_entry *_wait,
> * vblank without our intervention, so leave RPS alone.
> */
> if (!i915_request_started(rq))
> - gen6_rps_boost(rq);
> + intel_rps_boost(rq);
> i915_request_put(rq);
>
> drm_crtc_vblank_put(wait->crtc);
> @@ -14976,7 +14978,7 @@ intel_prepare_plane_fb(struct drm_plane *plane,
> * maximum clocks following a vblank miss (see do_rps_boost()).
> */
> if (!intel_state->rps_interactive) {
> - intel_rps_mark_interactive(dev_priv, true);
> + intel_rps_mark_interactive(&dev_priv->gt.rps, true);
> intel_state->rps_interactive = true;
> }
>
> @@ -15001,7 +15003,7 @@ intel_cleanup_plane_fb(struct drm_plane *plane,
> struct drm_i915_private *dev_priv = to_i915(plane->dev);
>
> if (intel_state->rps_interactive) {
> - intel_rps_mark_interactive(dev_priv, false);
> + intel_rps_mark_interactive(&dev_priv->gt.rps, false);
> intel_state->rps_interactive = false;
> }
>
> diff --git a/drivers/gpu/drm/i915/gt/intel_gt.c b/drivers/gpu/drm/i915/gt/intel_gt.c
> index 1c4b6c9642ad..7518852cb78a 100644
> --- a/drivers/gpu/drm/i915/gt/intel_gt.c
> +++ b/drivers/gpu/drm/i915/gt/intel_gt.c
> @@ -9,6 +9,7 @@
> #include "intel_gt_requests.h"
> #include "intel_mocs.h"
> #include "intel_rc6.h"
> +#include "intel_rps.h"
> #include "intel_uncore.h"
> #include "intel_pm.h"
>
> @@ -321,8 +322,7 @@ void intel_gt_chipset_flush(struct intel_gt *gt)
>
> void intel_gt_driver_register(struct intel_gt *gt)
> {
> - if (IS_GEN(gt->i915, 5))
> - intel_gpu_ips_init(gt->i915);
> + intel_rps_driver_register(>->rps);
> }
>
> static int intel_gt_init_scratch(struct intel_gt *gt, unsigned int size)
> @@ -385,7 +385,7 @@ void intel_gt_driver_remove(struct intel_gt *gt)
>
> void intel_gt_driver_unregister(struct intel_gt *gt)
> {
> - intel_gpu_ips_teardown();
> + intel_rps_driver_unregister(>->rps);
> }
>
> void intel_gt_driver_release(struct intel_gt *gt)
> diff --git a/drivers/gpu/drm/i915/gt/intel_gt_irq.c b/drivers/gpu/drm/i915/gt/intel_gt_irq.c
> index 34a4fb624bf7..973ee7eded64 100644
> --- a/drivers/gpu/drm/i915/gt/intel_gt_irq.c
> +++ b/drivers/gpu/drm/i915/gt/intel_gt_irq.c
> @@ -11,6 +11,7 @@
> #include "intel_gt.h"
> #include "intel_gt_irq.h"
> #include "intel_uncore.h"
> +#include "intel_rps.h"
>
> static void guc_irq_handler(struct intel_guc *guc, u16 iir)
> {
> @@ -77,7 +78,7 @@ gen11_other_irq_handler(struct intel_gt *gt, const u8 instance,
> return guc_irq_handler(>->uc.guc, iir);
>
> if (instance == OTHER_GTPM_INSTANCE)
> - return gen11_rps_irq_handler(gt, iir);
> + return gen11_rps_irq_handler(>->rps, iir);
>
> WARN_ONCE(1, "unhandled other interrupt instance=0x%x, iir=0x%x\n",
> instance, iir);
> @@ -336,7 +337,7 @@ void gen8_gt_irq_handler(struct intel_gt *gt, u32 master_ctl, u32 gt_iir[4])
> }
>
> if (master_ctl & (GEN8_GT_PM_IRQ | GEN8_GT_GUC_IRQ)) {
> - gen6_rps_irq_handler(gt->i915, gt_iir[2]);
> + gen6_rps_irq_handler(>->rps, gt_iir[2]);
> guc_irq_handler(>->uc.guc, gt_iir[2] >> 16);
> }
> }
> diff --git a/drivers/gpu/drm/i915/gt/intel_gt_pm.c b/drivers/gpu/drm/i915/gt/intel_gt_pm.c
> index 06e73d56cfcf..32ca87265222 100644
> --- a/drivers/gpu/drm/i915/gt/intel_gt_pm.c
> +++ b/drivers/gpu/drm/i915/gt/intel_gt_pm.c
> @@ -12,8 +12,10 @@
> #include "intel_gt.h"
> #include "intel_gt_pm.h"
> #include "intel_gt_requests.h"
> +#include "intel_llc.h"
> #include "intel_pm.h"
> #include "intel_rc6.h"
> +#include "intel_rps.h"
> #include "intel_wakeref.h"
>
> static int __gt_unpark(struct intel_wakeref *wf)
> @@ -39,12 +41,8 @@ static int __gt_unpark(struct intel_wakeref *wf)
> gt->awake = intel_display_power_get(i915, POWER_DOMAIN_GT_IRQ);
> GEM_BUG_ON(!gt->awake);
>
> - intel_enable_gt_powersave(i915);
> -
> - i915_update_gfx_val(i915);
> - if (INTEL_GEN(i915) >= 6)
> - gen6_rps_busy(i915);
> -
> + intel_rps_unpark(>->rps);
> + intel_llc_enable(>->llc);
> i915_pmu_gt_unparked(i915);
>
> intel_gt_queue_hangcheck(gt);
> @@ -65,8 +63,8 @@ static int __gt_park(struct intel_wakeref *wf)
>
> i915_vma_parked(gt);
> i915_pmu_gt_parked(i915);
> - if (INTEL_GEN(i915) >= 6)
> - gen6_rps_idle(i915);
> + intel_llc_disable(>->llc);
> + intel_rps_park(>->rps);
>
> /* Everything switched off, flush any residual interrupt just in case */
> intel_synchronize_irq(i915);
> @@ -98,6 +96,7 @@ void intel_gt_pm_init(struct intel_gt *gt)
> * user.
> */
> intel_rc6_init(>->rc6);
> + intel_rps_init(>->rps);
> }
>
> static bool reset_engines(struct intel_gt *gt)
> @@ -141,10 +140,39 @@ void intel_gt_sanitize(struct intel_gt *gt, bool force)
> engine->reset.finish(engine);
> }
>
> +void intel_gt_pm_enable(struct intel_gt *gt)
> +{
> + struct intel_engine_cs *engine;
> + enum intel_engine_id id;
> +
> + /* Powersaving is controlled by the host when inside a VM */
> + if (intel_vgpu_active(gt->i915))
> + return;
> +
> + if (is_mock_gt(gt))
> + return;
> +
> + intel_gt_pm_get(gt);
> + intel_rps_enable(>->rps);
> + intel_llc_enable(>->llc);
> +
> + for_each_engine(engine, gt->i915, id) {
> + intel_engine_pm_get(engine);
> + engine->serial++; /* force kernel context reload */
> + intel_engine_pm_put(engine);
> + }
> +
> + intel_gt_pm_put(gt);
> +}
> +
> void intel_gt_pm_disable(struct intel_gt *gt)
> {
> - if (!is_mock_gt(gt))
> - intel_sanitize_gt_powersave(gt->i915);
> + if (is_mock_gt(gt))
> + return;
> +
> + intel_rc6_disable(>->rc6);
> + intel_llc_disable(>->llc);
> + intel_rps_disable(>->rps);
> }
>
> void intel_gt_pm_fini(struct intel_gt *gt)
> @@ -165,6 +193,7 @@ int intel_gt_resume(struct intel_gt *gt)
> * allowing us to fixup the user contexts on their first pin.
> */
> intel_gt_pm_get(gt);
> +
> intel_uncore_forcewake_get(gt->uncore, FORCEWAKE_ALL);
> intel_rc6_sanitize(>->rc6);
>
> diff --git a/drivers/gpu/drm/i915/gt/intel_gt_pm.h b/drivers/gpu/drm/i915/gt/intel_gt_pm.h
> index 0ed87da4bb68..df50f853748d 100644
> --- a/drivers/gpu/drm/i915/gt/intel_gt_pm.h
> +++ b/drivers/gpu/drm/i915/gt/intel_gt_pm.h
> @@ -40,6 +40,7 @@ static inline int intel_gt_pm_wait_for_idle(struct intel_gt *gt)
> void intel_gt_pm_init_early(struct intel_gt *gt);
> void intel_gt_pm_init(struct intel_gt *gt);
> void intel_gt_pm_disable(struct intel_gt *gt);
> +void intel_gt_pm_enable(struct intel_gt *gt);
> void intel_gt_pm_fini(struct intel_gt *gt);
>
> void intel_gt_sanitize(struct intel_gt *gt, bool force);
> diff --git a/drivers/gpu/drm/i915/gt/intel_gt_types.h b/drivers/gpu/drm/i915/gt/intel_gt_types.h
> index 980973e66e7f..e4bd23eb8290 100644
> --- a/drivers/gpu/drm/i915/gt/intel_gt_types.h
> +++ b/drivers/gpu/drm/i915/gt/intel_gt_types.h
> @@ -20,6 +20,7 @@
> #include "intel_llc_types.h"
> #include "intel_reset_types.h"
> #include "intel_rc6_types.h"
> +#include "intel_rps_types.h"
> #include "intel_wakeref.h"
>
> struct drm_i915_private;
> @@ -82,6 +83,7 @@ struct intel_gt {
>
> struct intel_llc llc;
> struct intel_rc6 rc6;
> + struct intel_rps rps;
>
> ktime_t last_init_time;
>
> diff --git a/drivers/gpu/drm/i915/gt/intel_llc.c b/drivers/gpu/drm/i915/gt/intel_llc.c
> index 35093eb5f24e..ceb785b75c25 100644
> --- a/drivers/gpu/drm/i915/gt/intel_llc.c
> +++ b/drivers/gpu/drm/i915/gt/intel_llc.c
> @@ -48,7 +48,7 @@ static bool get_ia_constants(struct intel_llc *llc,
> struct ia_constants *consts)
> {
> struct drm_i915_private *i915 = llc_to_gt(llc)->i915;
> - struct intel_rps *rps = &i915->gt_pm.rps;
> + struct intel_rps *rps = &llc_to_gt(llc)->rps;
>
> if (rps->max_freq <= rps->min_freq)
> return false;
> diff --git a/drivers/gpu/drm/i915/gt/intel_rps.c b/drivers/gpu/drm/i915/gt/intel_rps.c
> new file mode 100644
> index 000000000000..30f56c786468
> --- /dev/null
> +++ b/drivers/gpu/drm/i915/gt/intel_rps.c
> @@ -0,0 +1,1872 @@
> +/*
> + * SPDX-License-Identifier: MIT
> + *
> + * Copyright © 2019 Intel Corporation
> + */
> +
> +#include "i915_drv.h"
> +#include "intel_gt.h"
> +#include "intel_gt_irq.h"
> +#include "intel_gt_pm_irq.h"
> +#include "intel_rps.h"
> +#include "intel_sideband.h"
> +#include "../../../platform/x86/intel_ips.h"
> +
> +/*
> + * Lock protecting IPS related data structures
> + */
> +static DEFINE_SPINLOCK(mchdev_lock);
> +
> +static struct intel_gt *rps_to_gt(struct intel_rps *rps)
> +{
> + return container_of(rps, struct intel_gt, rps);
> +}
> +
> +static struct drm_i915_private *rps_to_i915(struct intel_rps *rps)
> +{
> + return rps_to_gt(rps)->i915;
> +}
> +
> +static struct intel_uncore *rps_to_uncore(struct intel_rps *rps)
> +{
> + return rps_to_gt(rps)->uncore;
> +}
> +
> +static u32 rps_pm_sanitize_mask(struct intel_rps *rps, u32 mask)
> +{
> + return mask & ~rps->pm_intrmsk_mbz;
> +}
> +
> +static u32 rps_pm_mask(struct intel_rps *rps, u8 val)
> +{
> + u32 mask = 0;
> +
> + /* We use UP_EI_EXPIRED interrupts for both up/down in manual mode */
> + if (val > rps->min_freq_softlimit)
> + mask |= (GEN6_PM_RP_UP_EI_EXPIRED |
> + GEN6_PM_RP_DOWN_THRESHOLD |
> + GEN6_PM_RP_DOWN_TIMEOUT);
> +
> + if (val < rps->max_freq_softlimit)
> + mask |= GEN6_PM_RP_UP_EI_EXPIRED | GEN6_PM_RP_UP_THRESHOLD;
> +
> + mask &= rps->pm_events;
> +
> + return rps_pm_sanitize_mask(rps, ~mask);
> +}
> +
> +static void rps_reset_ei(struct intel_rps *rps)
> +{
> + memset(&rps->ei, 0, sizeof(rps->ei));
> +}
> +
> +static void rps_enable_interrupts(struct intel_rps *rps)
> +{
> + struct intel_gt *gt = rps_to_gt(rps);
> +
> + rps_reset_ei(rps);
> +
> + if (IS_VALLEYVIEW(gt->i915))
> + /* WaGsvRC0ResidencyMethod:vlv */
> + rps->pm_events = GEN6_PM_RP_UP_EI_EXPIRED;
> + else
> + rps->pm_events = (GEN6_PM_RP_UP_THRESHOLD |
> + GEN6_PM_RP_DOWN_THRESHOLD |
> + GEN6_PM_RP_DOWN_TIMEOUT);
> +
> + spin_lock_irq(>->irq_lock);
> + gen6_gt_pm_enable_irq(gt, rps->pm_events);
> + spin_unlock_irq(>->irq_lock);
> +
> + intel_uncore_write(gt->uncore, GEN6_PMINTRMSK,
> + rps_pm_mask(rps, rps->cur_freq));
> +}
> +
> +static void gen6_rps_reset_interrupts(struct intel_rps *rps)
> +{
> + gen6_gt_pm_reset_iir(rps_to_gt(rps), GEN6_PM_RPS_EVENTS);
> +}
> +
> +static void gen11_rps_reset_interrupts(struct intel_rps *rps)
> +{
> + while (gen11_gt_reset_one_iir(rps_to_gt(rps), 0, GEN11_GTPM))
> + ;
> +}
> +
> +static void rps_reset_interrupts(struct intel_rps *rps)
> +{
> + struct intel_gt *gt = rps_to_gt(rps);
> +
> + spin_lock_irq(>->irq_lock);
> + if (INTEL_GEN(gt->i915) >= 11)
> + gen11_rps_reset_interrupts(rps);
> + else
> + gen6_rps_reset_interrupts(rps);
> +
> + rps->pm_iir = 0;
> + spin_unlock_irq(>->irq_lock);
> +}
> +
> +static void rps_disable_interrupts(struct intel_rps *rps)
> +{
> + struct intel_gt *gt = rps_to_gt(rps);
> +
> + rps->pm_events = 0;
> +
> + intel_uncore_write(gt->uncore, GEN6_PMINTRMSK,
> + rps_pm_sanitize_mask(rps, ~0u));
> +
> + spin_lock_irq(>->irq_lock);
> + gen6_gt_pm_disable_irq(gt, GEN6_PM_RPS_EVENTS);
> + spin_unlock_irq(>->irq_lock);
> +
> + intel_synchronize_irq(gt->i915);
> +
> + /*
> + * Now that we will not be generating any more work, flush any
> + * outstanding tasks. As we are called on the RPS idle path,
> + * we will reset the GPU to minimum frequencies, so the current
> + * state of the worker can be discarded.
> + */
> + cancel_work_sync(&rps->work);
> +
> + rps_reset_interrupts(rps);
> +}
> +
> +static const struct cparams {
> + u16 i;
> + u16 t;
> + u16 m;
> + u16 c;
> +} cparams[] = {
> + { 1, 1333, 301, 28664 },
> + { 1, 1066, 294, 24460 },
> + { 1, 800, 294, 25192 },
> + { 0, 1333, 276, 27605 },
> + { 0, 1066, 276, 27605 },
> + { 0, 800, 231, 23784 },
> +};
> +
> +static void gen5_rps_init(struct intel_rps *rps)
> +{
> + struct drm_i915_private *i915 = rps_to_i915(rps);
> + struct intel_uncore *uncore = rps_to_uncore(rps);
> + u8 fmax, fmin, fstart;
> + u32 rgvmodectl;
> + int c_m, i;
> +
> + if (i915->fsb_freq <= 3200)
> + c_m = 0;
> + else if (i915->fsb_freq <= 4800)
> + c_m = 1;
> + else
> + c_m = 2;
> +
> + for (i = 0; i < ARRAY_SIZE(cparams); i++) {
> + if (cparams[i].i == c_m && cparams[i].t == i915->mem_freq) {
> + rps->ips.m = cparams[i].m;
> + rps->ips.c = cparams[i].c;
> + break;
> + }
> + }
> +
> + rgvmodectl = intel_uncore_read(uncore, MEMMODECTL);
> +
> + /* Set up min, max, and cur for interrupt handling */
> + fmax = (rgvmodectl & MEMMODE_FMAX_MASK) >> MEMMODE_FMAX_SHIFT;
> + fmin = (rgvmodectl & MEMMODE_FMIN_MASK);
> + fstart = (rgvmodectl & MEMMODE_FSTART_MASK) >>
> + MEMMODE_FSTART_SHIFT;
> + DRM_DEBUG_DRIVER("fmax: %d, fmin: %d, fstart: %d\n",
> + fmax, fmin, fstart);
> +
> + rps->min_freq = -fstart;
> + rps->max_freq = -fmin;
> +
> + rps->idle_freq = rps->min_freq;
> + rps->cur_freq = rps->idle_freq;
> +}
> +
> +static unsigned long
> +__ips_chipset_val(struct intel_ips *ips)
> +{
> + struct intel_uncore *uncore =
> + rps_to_uncore(container_of(ips, struct intel_rps, ips));
> + unsigned long now = jiffies_to_msecs(jiffies), dt;
> + unsigned long result;
> + u64 total, delta;
> +
> + lockdep_assert_held(&mchdev_lock);
> +
> + /*
> + * Prevent division-by-zero if we are asking too fast.
> + * Also, we don't get interesting results if we are polling
> + * faster than once in 10ms, so just return the saved value
> + * in such cases.
> + */
> + dt = now - ips->last_time1;
> + if (dt <= 10)
> + return ips->chipset_power;
> +
> + /* FIXME: handle per-counter overflow */
> + total = intel_uncore_read(uncore, DMIEC);
> + total += intel_uncore_read(uncore, DDREC);
> + total += intel_uncore_read(uncore, CSIEC);
> +
> + delta = total - ips->last_count1;
> +
> + result = div_u64(div_u64(ips->m * delta, dt) + ips->c, 10);
> +
> + ips->last_count1 = total;
> + ips->last_time1 = now;
> +
> + ips->chipset_power = result;
> +
> + return result;
> +}
> +
> +static unsigned long ips_mch_val(struct intel_uncore *uncore)
> +{
> + unsigned int m, x, b;
> + u32 tsfs;
> +
> + tsfs = intel_uncore_read(uncore, TSFS);
> + x = intel_uncore_read8(uncore, TR1);
> +
> + b = tsfs & TSFS_INTR_MASK;
> + m = (tsfs & TSFS_SLOPE_MASK) >> TSFS_SLOPE_SHIFT;
> +
> + return m * x / 127 - b;
> +}
> +
> +static int _pxvid_to_vd(u8 pxvid)
> +{
> + if (pxvid == 0)
> + return 0;
> +
> + if (pxvid >= 8 && pxvid < 31)
> + pxvid = 31;
> +
> + return (pxvid + 2) * 125;
> +}
> +
> +static u32 pvid_to_extvid(struct drm_i915_private *i915, u8 pxvid)
> +{
> + const int vd = _pxvid_to_vd(pxvid);
> +
> + if (INTEL_INFO(i915)->is_mobile)
> + return max(vd - 1125, 0);
> +
> + return vd;
> +}
> +
> +static void __gen5_ips_update(struct intel_ips *ips)
> +{
> + struct intel_uncore *uncore =
> + rps_to_uncore(container_of(ips, struct intel_rps, ips));
> + u64 now, delta, dt;
> + u32 count;
> +
> + lockdep_assert_held(&mchdev_lock);
> +
> + now = ktime_get_raw_ns();
> + dt = now - ips->last_time2;
> + do_div(dt, NSEC_PER_MSEC);
> +
> + /* Don't divide by 0 */
> + if (dt <= 10)
> + return;
> +
> + count = intel_uncore_read(uncore, GFXEC);
> + delta = count - ips->last_count2;
> +
> + ips->last_count2 = count;
> + ips->last_time2 = now;
> +
> + /* More magic constants... */
> + ips->gfx_power = div_u64(delta * 1181, dt * 10);
> +}
> +
> +static void gen5_rps_update(struct intel_rps *rps)
> +{
> + spin_lock_irq(&mchdev_lock);
> + __gen5_ips_update(&rps->ips);
> + spin_unlock_irq(&mchdev_lock);
> +}
> +
> +static bool gen5_rps_set(struct intel_rps *rps, u8 val)
> +{
> + struct intel_uncore *uncore = rps_to_uncore(rps);
> + u16 rgvswctl;
> +
> + lockdep_assert_held(&mchdev_lock);
> +
> + rgvswctl = intel_uncore_read16(uncore, MEMSWCTL);
> + if (rgvswctl & MEMCTL_CMD_STS) {
> + DRM_DEBUG("gpu busy, RCS change rejected\n");
> + return false; /* still busy with another command */
> + }
> +
> + val = -val;
> +
> + rgvswctl =
> + (MEMCTL_CMD_CHFREQ << MEMCTL_CMD_SHIFT) |
> + (val << MEMCTL_FREQ_SHIFT) |
> + MEMCTL_SFCAVM;
> + intel_uncore_write16(uncore, MEMSWCTL, rgvswctl);
> + intel_uncore_posting_read16(uncore, MEMSWCTL);
> +
> + rgvswctl |= MEMCTL_CMD_STS;
> + intel_uncore_write16(uncore, MEMSWCTL, rgvswctl);
> +
> + return true;
> +}
> +
> +static unsigned long intel_pxfreq(u32 vidfreq)
> +{
> + int div = (vidfreq & 0x3f0000) >> 16;
> + int post = (vidfreq & 0x3000) >> 12;
> + int pre = (vidfreq & 0x7);
> +
> + if (!pre)
> + return 0;
> +
> + return div * 133333 / (pre << post);
> +}
> +
> +static unsigned int init_emon(struct intel_uncore *uncore)
> +{
> + u8 pxw[16];
> + int i;
> +
> + /* Disable to program */
> + intel_uncore_write(uncore, ECR, 0);
> + intel_uncore_posting_read(uncore, ECR);
> +
> + /* Program energy weights for various events */
> + intel_uncore_write(uncore, SDEW, 0x15040d00);
> + intel_uncore_write(uncore, CSIEW0, 0x007f0000);
> + intel_uncore_write(uncore, CSIEW1, 0x1e220004);
> + intel_uncore_write(uncore, CSIEW2, 0x04000004);
> +
> + for (i = 0; i < 5; i++)
> + intel_uncore_write(uncore, PEW(i), 0);
> + for (i = 0; i < 3; i++)
> + intel_uncore_write(uncore, DEW(i), 0);
> +
> + /* Program P-state weights to account for frequency power adjustment */
> + for (i = 0; i < 16; i++) {
> + u32 pxvidfreq = intel_uncore_read(uncore, PXVFREQ(i));
> + unsigned int freq = intel_pxfreq(pxvidfreq);
> + unsigned int vid =
> + (pxvidfreq & PXVFREQ_PX_MASK) >> PXVFREQ_PX_SHIFT;
> + unsigned int val;
> +
> + val = vid * vid * freq / 1000 * 255;
> + val /= 127 * 127 * 900;
> +
> + pxw[i] = val;
> + }
> + /* Render standby states get 0 weight */
> + pxw[14] = 0;
> + pxw[15] = 0;
> +
> + for (i = 0; i < 4; i++) {
> + intel_uncore_write(uncore, PXW(i),
> + pxw[i * 4 + 0] << 24 |
> + pxw[i * 4 + 1] << 16 |
> + pxw[i * 4 + 2] << 8 |
> + pxw[i * 4 + 3] << 0);
> + }
> +
> + /* Adjust magic regs to magic values (more experimental results) */
> + intel_uncore_write(uncore, OGW0, 0);
> + intel_uncore_write(uncore, OGW1, 0);
> + intel_uncore_write(uncore, EG0, 0x00007f00);
> + intel_uncore_write(uncore, EG1, 0x0000000e);
> + intel_uncore_write(uncore, EG2, 0x000e0000);
> + intel_uncore_write(uncore, EG3, 0x68000300);
> + intel_uncore_write(uncore, EG4, 0x42000000);
> + intel_uncore_write(uncore, EG5, 0x00140031);
> + intel_uncore_write(uncore, EG6, 0);
> + intel_uncore_write(uncore, EG7, 0);
> +
> + for (i = 0; i < 8; i++)
> + intel_uncore_write(uncore, PXWL(i), 0);
> +
> + /* Enable PMON + select events */
> + intel_uncore_write(uncore, ECR, 0x80000019);
> +
> + return intel_uncore_read(uncore, LCFUSE02) & LCFUSE_HIV_MASK;
> +}
> +
> +static bool gen5_rps_enable(struct intel_rps *rps)
> +{
> + struct intel_uncore *uncore = rps_to_uncore(rps);
> + u8 fstart, vstart;
> + u32 rgvmodectl;
> +
> + spin_lock_irq(&mchdev_lock);
> +
> + rgvmodectl = intel_uncore_read(uncore, MEMMODECTL);
> +
> + /* Enable temp reporting */
> + intel_uncore_write16(uncore, PMMISC,
> + intel_uncore_read16(uncore, PMMISC) | MCPPCE_EN);
> + intel_uncore_write16(uncore, TSC1,
> + intel_uncore_read16(uncore, TSC1) | TSE);
> +
> + /* 100ms RC evaluation intervals */
> + intel_uncore_write(uncore, RCUPEI, 100000);
> + intel_uncore_write(uncore, RCDNEI, 100000);
> +
> + /* Set max/min thresholds to 90ms and 80ms respectively */
> + intel_uncore_write(uncore, RCBMAXAVG, 90000);
> + intel_uncore_write(uncore, RCBMINAVG, 80000);
> +
> + intel_uncore_write(uncore, MEMIHYST, 1);
> +
> + /* Set up min, max, and cur for interrupt handling */
> + fstart = (rgvmodectl & MEMMODE_FSTART_MASK) >>
> + MEMMODE_FSTART_SHIFT;
> +
> + vstart = (intel_uncore_read(uncore, PXVFREQ(fstart)) &
> + PXVFREQ_PX_MASK) >> PXVFREQ_PX_SHIFT;
> +
> + intel_uncore_write(uncore,
> + MEMINTREN,
> + MEMINT_CX_SUPR_EN | MEMINT_EVAL_CHG_EN);
> +
> + intel_uncore_write(uncore, VIDSTART, vstart);
> + intel_uncore_posting_read(uncore, VIDSTART);
> +
> + rgvmodectl |= MEMMODE_SWMODE_EN;
> + intel_uncore_write(uncore, MEMMODECTL, rgvmodectl);
> +
> + if (wait_for_atomic((intel_uncore_read(uncore, MEMSWCTL) &
> + MEMCTL_CMD_STS) == 0, 10))
> + DRM_ERROR("stuck trying to change perf mode\n");
> + mdelay(1);
> +
> + gen5_rps_set(rps, rps->cur_freq);
> +
> + rps->ips.last_count1 = intel_uncore_read(uncore, DMIEC);
> + rps->ips.last_count1 += intel_uncore_read(uncore, DDREC);
> + rps->ips.last_count1 += intel_uncore_read(uncore, CSIEC);
> + rps->ips.last_time1 = jiffies_to_msecs(jiffies);
> +
> + rps->ips.last_count2 = intel_uncore_read(uncore, GFXEC);
> + rps->ips.last_time2 = ktime_get_raw_ns();
> +
> + spin_unlock_irq(&mchdev_lock);
> +
> + rps->ips.corr = init_emon(uncore);
> +
> + return true;
> +}
> +
> +static void gen5_rps_disable(struct intel_rps *rps)
> +{
> + struct intel_uncore *uncore = rps_to_uncore(rps);
> + u16 rgvswctl;
> +
> + spin_lock_irq(&mchdev_lock);
> +
> + rgvswctl = intel_uncore_read16(uncore, MEMSWCTL);
> +
> + /* Ack interrupts, disable EFC interrupt */
> + intel_uncore_write(uncore, MEMINTREN,
> + intel_uncore_read(uncore, MEMINTREN) &
> + ~MEMINT_EVAL_CHG_EN);
> + intel_uncore_write(uncore, MEMINTRSTS, MEMINT_EVAL_CHG);
> + intel_uncore_write(uncore, DEIER,
> + intel_uncore_read(uncore, DEIER) & ~DE_PCU_EVENT);
> + intel_uncore_write(uncore, DEIIR, DE_PCU_EVENT);
> + intel_uncore_write(uncore, DEIMR,
> + intel_uncore_read(uncore, DEIMR) | DE_PCU_EVENT);
> +
> + /* Go back to the starting frequency */
> + gen5_rps_set(rps, rps->idle_freq);
> + mdelay(1);
> + rgvswctl |= MEMCTL_CMD_STS;
> + intel_uncore_write(uncore, MEMSWCTL, rgvswctl);
> + mdelay(1);
> +
> + spin_unlock_irq(&mchdev_lock);
> +}
> +
> +static u32 rps_limits(struct intel_rps *rps, u8 val)
> +{
> + u32 limits;
> +
> + /*
> + * Only set the down limit when we've reached the lowest level to avoid
> + * getting more interrupts, otherwise leave this clear. This prevents a
> + * race in the hw when coming out of rc6: There's a tiny window where
> + * the hw runs at the minimal clock before selecting the desired
> + * frequency, if the down threshold expires in that window we will not
> + * receive a down interrupt.
> + */
> + if (INTEL_GEN(rps_to_i915(rps)) >= 9) {
> + limits = rps->max_freq_softlimit << 23;
> + if (val <= rps->min_freq_softlimit)
> + limits |= rps->min_freq_softlimit << 14;
> + } else {
> + limits = rps->max_freq_softlimit << 24;
> + if (val <= rps->min_freq_softlimit)
> + limits |= rps->min_freq_softlimit << 16;
> + }
> +
> + return limits;
> +}
> +
> +static void rps_set_power(struct intel_rps *rps, int new_power)
> +{
> + struct intel_uncore *uncore = rps_to_uncore(rps);
> + struct drm_i915_private *i915 = rps_to_i915(rps);
> + u32 threshold_up = 0, threshold_down = 0; /* in % */
> + u32 ei_up = 0, ei_down = 0;
> +
> + lockdep_assert_held(&rps->power.mutex);
> +
> + if (new_power == rps->power.mode)
> + return;
> +
> + /* Note the units here are not exactly 1us, but 1280ns. */
> + switch (new_power) {
> + case LOW_POWER:
> + /* Upclock if more than 95% busy over 16ms */
> + ei_up = 16000;
> + threshold_up = 95;
> +
> + /* Downclock if less than 85% busy over 32ms */
> + ei_down = 32000;
> + threshold_down = 85;
> + break;
> +
> + case BETWEEN:
> + /* Upclock if more than 90% busy over 13ms */
> + ei_up = 13000;
> + threshold_up = 90;
> +
> + /* Downclock if less than 75% busy over 32ms */
> + ei_down = 32000;
> + threshold_down = 75;
> + break;
> +
> + case HIGH_POWER:
> + /* Upclock if more than 85% busy over 10ms */
> + ei_up = 10000;
> + threshold_up = 85;
> +
> + /* Downclock if less than 60% busy over 32ms */
> + ei_down = 32000;
> + threshold_down = 60;
> + break;
> + }
> +
> + /* When byt can survive without system hang with dynamic
> + * sw freq adjustments, this restriction can be lifted.
> + */
> + if (IS_VALLEYVIEW(i915))
> + goto skip_hw_write;
> +
> + intel_uncore_write(uncore, GEN6_RP_UP_EI,
> + GT_INTERVAL_FROM_US(i915, ei_up));
> + intel_uncore_write(uncore, GEN6_RP_UP_THRESHOLD,
> + GT_INTERVAL_FROM_US(i915,
> + ei_up * threshold_up / 100));
> +
> + intel_uncore_write(uncore, GEN6_RP_DOWN_EI,
> + GT_INTERVAL_FROM_US(i915, ei_down));
> + intel_uncore_write(uncore, GEN6_RP_DOWN_THRESHOLD,
> + GT_INTERVAL_FROM_US(i915,
> + ei_down * threshold_down / 100));
> +
> + intel_uncore_write(uncore, GEN6_RP_CONTROL,
> + (INTEL_GEN(i915) > 9 ? 0 : GEN6_RP_MEDIA_TURBO) |
> + GEN6_RP_MEDIA_HW_NORMAL_MODE |
> + GEN6_RP_MEDIA_IS_GFX |
> + GEN6_RP_ENABLE |
> + GEN6_RP_UP_BUSY_AVG |
> + GEN6_RP_DOWN_IDLE_AVG);
> +
> +skip_hw_write:
> + rps->power.mode = new_power;
> + rps->power.up_threshold = threshold_up;
> + rps->power.down_threshold = threshold_down;
> +}
> +
> +static void gen6_rps_set_thresholds(struct intel_rps *rps, u8 val)
> +{
> + int new_power;
> +
> + new_power = rps->power.mode;
> + switch (rps->power.mode) {
> + case LOW_POWER:
> + if (val > rps->efficient_freq + 1 &&
> + val > rps->cur_freq)
> + new_power = BETWEEN;
> + break;
> +
> + case BETWEEN:
> + if (val <= rps->efficient_freq &&
> + val < rps->cur_freq)
> + new_power = LOW_POWER;
> + else if (val >= rps->rp0_freq &&
> + val > rps->cur_freq)
> + new_power = HIGH_POWER;
> + break;
> +
> + case HIGH_POWER:
> + if (val < (rps->rp1_freq + rps->rp0_freq) >> 1 &&
> + val < rps->cur_freq)
> + new_power = BETWEEN;
> + break;
> + }
> + /* Max/min bins are special */
> + if (val <= rps->min_freq_softlimit)
> + new_power = LOW_POWER;
> + if (val >= rps->max_freq_softlimit)
> + new_power = HIGH_POWER;
> +
> + mutex_lock(&rps->power.mutex);
> + if (rps->power.interactive)
> + new_power = HIGH_POWER;
> + rps_set_power(rps, new_power);
> + mutex_unlock(&rps->power.mutex);
> +}
> +
> +void intel_rps_mark_interactive(struct intel_rps *rps, bool interactive)
> +{
> + if (!rps->enabled)
> + return;
> +
> + mutex_lock(&rps->power.mutex);
> + if (interactive) {
> + if (!rps->power.interactive++ && rps->active)
> + rps_set_power(rps, HIGH_POWER);
> + } else {
> + GEM_BUG_ON(!rps->power.interactive);
> + rps->power.interactive--;
> + }
> + mutex_unlock(&rps->power.mutex);
> +}
> +
> +static int gen6_rps_set(struct intel_rps *rps, u8 val)
> +{
> + struct intel_uncore *uncore = rps_to_uncore(rps);
> + struct drm_i915_private *i915 = rps_to_i915(rps);
> + u32 swreq;
> +
> + if (INTEL_GEN(i915) >= 9)
> + swreq = GEN9_FREQUENCY(val);
> + else if (IS_HASWELL(i915) || IS_BROADWELL(i915))
> + swreq = HSW_FREQUENCY(val);
> + else
> + swreq = (GEN6_FREQUENCY(val) |
> + GEN6_OFFSET(0) |
> + GEN6_AGGRESSIVE_TURBO);
> + intel_uncore_write(uncore, GEN6_RPNSWREQ, swreq);
> +
> + return 0;
> +}
> +
> +static int vlv_rps_set(struct intel_rps *rps, u8 val)
> +{
> + struct drm_i915_private *i915 = rps_to_i915(rps);
> + int err;
> +
> + vlv_punit_get(i915);
> + err = vlv_punit_write(i915, PUNIT_REG_GPU_FREQ_REQ, val);
> + vlv_punit_put(i915);
> +
> + return err;
> +}
> +
> +static int rps_set(struct intel_rps *rps, u8 val)
> +{
> + struct drm_i915_private *i915 = rps_to_i915(rps);
> + int err;
> +
> + if (INTEL_GEN(i915) < 6)
> + return 0;
> +
> + if (val == rps->last_freq)
> + return 0;
> +
> + if (IS_VALLEYVIEW(i915) || IS_CHERRYVIEW(i915))
> + err = vlv_rps_set(rps, val);
> + else
> + err = gen6_rps_set(rps, val);
> + if (err)
> + return err;
> +
> + gen6_rps_set_thresholds(rps, val);
> + rps->last_freq = val;
> +
> + return 0;
> +}
> +
> +void intel_rps_unpark(struct intel_rps *rps)
> +{
> + u8 freq;
> +
> + if (!rps->enabled)
> + return;
> +
> + /*
> + * Use the user's desired frequency as a guide, but for better
> + * performance, jump directly to RPe as our starting frequency.
> + */
> + mutex_lock(&rps->lock);
> + rps->active = true;
> + freq = max(rps->cur_freq, rps->efficient_freq),
> + freq = clamp(freq, rps->min_freq_softlimit, rps->max_freq_softlimit);
> + intel_rps_set(rps, freq);
> + rps->last_adj = 0;
> + mutex_unlock(&rps->lock);
> +
> + if (INTEL_GEN(rps_to_i915(rps)) >= 6)
> + rps_enable_interrupts(rps);
> +
> + if (IS_GEN(rps_to_i915(rps), 5))
> + gen5_rps_update(rps);
> +}
> +
> +void intel_rps_park(struct intel_rps *rps)
> +{
> + struct drm_i915_private *i915 = rps_to_i915(rps);
> +
> + if (!rps->enabled)
> + return;
> +
> + if (INTEL_GEN(i915) >= 6)
> + rps_disable_interrupts(rps);
> +
> + rps->active = false;
> + if (rps->last_freq <= rps->idle_freq)
> + return;
> +
> + /*
> + * The punit delays the write of the frequency and voltage until it
> + * determines the GPU is awake. During normal usage we don't want to
> + * waste power changing the frequency if the GPU is sleeping (rc6).
> + * However, the GPU and driver is now idle and we do not want to delay
> + * switching to minimum voltage (reducing power whilst idle) as we do
> + * not expect to be woken in the near future and so must flush the
> + * change by waking the device.
> + *
> + * We choose to take the media powerwell (either would do to trick the
> + * punit into committing the voltage change) as that takes a lot less
> + * power than the render powerwell.
> + */
> + intel_uncore_forcewake_get(rps_to_uncore(rps), FORCEWAKE_MEDIA);
> + rps_set(rps, rps->idle_freq);
> + intel_uncore_forcewake_put(rps_to_uncore(rps), FORCEWAKE_MEDIA);
> +}
> +
> +void intel_rps_boost(struct i915_request *rq)
> +{
> + struct intel_rps *rps = &rq->engine->gt->rps;
> + unsigned long flags;
> +
> + if (i915_request_signaled(rq) || !rps->active)
> + return;
> +
> + /* Serializes with i915_request_retire() */
> + spin_lock_irqsave(&rq->lock, flags);
> + if (!i915_request_has_waitboost(rq) &&
> + !dma_fence_is_signaled_locked(&rq->fence)) {
> + rq->flags |= I915_REQUEST_WAITBOOST;
> +
> + if (!atomic_fetch_inc(&rps->num_waiters) &&
> + READ_ONCE(rps->cur_freq) < rps->boost_freq)
> + schedule_work(&rps->work);
> +
> + atomic_inc(&rps->boosts);
> + }
> + spin_unlock_irqrestore(&rq->lock, flags);
> +}
> +
> +int intel_rps_set(struct intel_rps *rps, u8 val)
> +{
> + int err = 0;
> +
> + lockdep_assert_held(&rps->lock);
> + GEM_BUG_ON(val > rps->max_freq);
> + GEM_BUG_ON(val < rps->min_freq);
> +
> + if (rps->active) {
> + err = rps_set(rps, val);
> +
> + /*
> + * Make sure we continue to get interrupts
> + * until we hit the minimum or maximum frequencies.
> + */
> + if (INTEL_GEN(rps_to_i915(rps)) >= 6) {
> + struct intel_uncore *uncore = rps_to_uncore(rps);
> +
> + intel_uncore_write(uncore, GEN6_RP_INTERRUPT_LIMITS,
> + rps_limits(rps, val));
> +
> + intel_uncore_write(uncore, GEN6_PMINTRMSK,
> + rps_pm_mask(rps, val));
> + }
> + }
> +
> + if (err == 0)
> + rps->cur_freq = val;
> +
> + return err;
> +}
> +
> +static void gen6_rps_init(struct intel_rps *rps)
> +{
> + struct drm_i915_private *i915 = rps_to_i915(rps);
> + struct intel_uncore *uncore = rps_to_uncore(rps);
> +
> + /* All of these values are in units of 50MHz */
> +
> + /* static values from HW: RP0 > RP1 > RPn (min_freq) */
> + if (IS_GEN9_LP(i915)) {
> + u32 rp_state_cap = intel_uncore_read(uncore, BXT_RP_STATE_CAP);
> +
> + rps->rp0_freq = (rp_state_cap >> 16) & 0xff;
> + rps->rp1_freq = (rp_state_cap >> 8) & 0xff;
> + rps->min_freq = (rp_state_cap >> 0) & 0xff;
> + } else {
> + u32 rp_state_cap = intel_uncore_read(uncore, GEN6_RP_STATE_CAP);
> +
> + rps->rp0_freq = (rp_state_cap >> 0) & 0xff;
> + rps->rp1_freq = (rp_state_cap >> 8) & 0xff;
> + rps->min_freq = (rp_state_cap >> 16) & 0xff;
> + }
> +
> + /* hw_max = RP0 until we check for overclocking */
> + rps->max_freq = rps->rp0_freq;
> +
> + rps->efficient_freq = rps->rp1_freq;
> + if (IS_HASWELL(i915) || IS_BROADWELL(i915) ||
> + IS_GEN9_BC(i915) || INTEL_GEN(i915) >= 10) {
> + u32 ddcc_status = 0;
> +
> + if (sandybridge_pcode_read(i915,
> + HSW_PCODE_DYNAMIC_DUTY_CYCLE_CONTROL,
> + &ddcc_status, NULL) == 0)
> + rps->efficient_freq =
> + clamp_t(u8,
> + (ddcc_status >> 8) & 0xff,
> + rps->min_freq,
> + rps->max_freq);
> + }
> +
> + if (IS_GEN9_BC(i915) || INTEL_GEN(i915) >= 10) {
> + /* Store the frequency values in 16.66 MHZ units, which is
> + * the natural hardware unit for SKL
> + */
> + rps->rp0_freq *= GEN9_FREQ_SCALER;
> + rps->rp1_freq *= GEN9_FREQ_SCALER;
> + rps->min_freq *= GEN9_FREQ_SCALER;
> + rps->max_freq *= GEN9_FREQ_SCALER;
> + rps->efficient_freq *= GEN9_FREQ_SCALER;
> + }
> +}
> +
> +static bool rps_reset(struct intel_rps *rps)
> +{
> + /* force a reset */
> + rps->power.mode = -1;
> + rps->last_freq = -1;
> +
> + if (rps_set(rps, rps->min_freq)) {
> + DRM_ERROR("Failed to reset RPS to initial values\n");
> + return false;
> + }
> +
> + rps->cur_freq = rps->min_freq;
> + return true;
> +}
> +
> +/* See the Gen9_GT_PM_Programming_Guide doc for the below */
> +static bool gen9_rps_enable(struct intel_rps *rps)
> +{
> + struct drm_i915_private *i915 = rps_to_i915(rps);
> + struct intel_uncore *uncore = rps_to_uncore(rps);
> +
> + /* Program defaults and thresholds for RPS */
> + if (IS_GEN(i915, 9))
> + intel_uncore_write_fw(uncore, GEN6_RC_VIDEO_FREQ,
> + GEN9_FREQUENCY(rps->rp1_freq));
> +
> + /* 1 second timeout */
> + intel_uncore_write_fw(uncore, GEN6_RP_DOWN_TIMEOUT,
> + GT_INTERVAL_FROM_US(i915, 1000000));
> +
> + intel_uncore_write_fw(uncore, GEN6_RP_IDLE_HYSTERSIS, 0xa);
> +
> + return rps_reset(rps);
> +}
> +
> +static bool gen8_rps_enable(struct intel_rps *rps)
> +{
> + struct intel_uncore *uncore = rps_to_uncore(rps);
> +
> + intel_uncore_write_fw(uncore, GEN6_RC_VIDEO_FREQ,
> + HSW_FREQUENCY(rps->rp1_freq));
> +
> + /* NB: Docs say 1s, and 1000000 - which aren't equivalent */
> + intel_uncore_write_fw(uncore, GEN6_RP_DOWN_TIMEOUT,
> + 100000000 / 128); /* 1 second timeout */
> +
> + intel_uncore_write_fw(uncore, GEN6_RP_IDLE_HYSTERSIS, 10);
> +
> + return rps_reset(rps);
> +}
> +
> +static bool gen6_rps_enable(struct intel_rps *rps)
> +{
> + struct intel_uncore *uncore = rps_to_uncore(rps);
> +
> + /* Power down if completely idle for over 50ms */
> + intel_uncore_write_fw(uncore, GEN6_RP_DOWN_TIMEOUT, 50000);
> + intel_uncore_write_fw(uncore, GEN6_RP_IDLE_HYSTERSIS, 10);
> +
> + return rps_reset(rps);
> +}
> +
> +static int chv_rps_max_freq(struct intel_rps *rps)
> +{
> + struct drm_i915_private *i915 = rps_to_i915(rps);
> + u32 val;
> +
> + val = vlv_punit_read(i915, FB_GFX_FMAX_AT_VMAX_FUSE);
> +
> + switch (RUNTIME_INFO(i915)->sseu.eu_total) {
> + case 8:
> + /* (2 * 4) config */
> + val >>= FB_GFX_FMAX_AT_VMAX_2SS4EU_FUSE_SHIFT;
> + break;
> + case 12:
> + /* (2 * 6) config */
> + val >>= FB_GFX_FMAX_AT_VMAX_2SS6EU_FUSE_SHIFT;
> + break;
> + case 16:
> + /* (2 * 8) config */
> + default:
> + /* Setting (2 * 8) Min RP0 for any other combination */
> + val >>= FB_GFX_FMAX_AT_VMAX_2SS8EU_FUSE_SHIFT;
> + break;
> + }
> +
> + return val & FB_GFX_FREQ_FUSE_MASK;
> +}
> +
> +static int chv_rps_rpe_freq(struct intel_rps *rps)
> +{
> + struct drm_i915_private *i915 = rps_to_i915(rps);
> + u32 val;
> +
> + val = vlv_punit_read(i915, PUNIT_GPU_DUTYCYCLE_REG);
> + val >>= PUNIT_GPU_DUTYCYCLE_RPE_FREQ_SHIFT;
> +
> + return val & PUNIT_GPU_DUTYCYCLE_RPE_FREQ_MASK;
> +}
> +
> +static int chv_rps_guar_freq(struct intel_rps *rps)
> +{
> + struct drm_i915_private *i915 = rps_to_i915(rps);
> + u32 val;
> +
> + val = vlv_punit_read(i915, FB_GFX_FMAX_AT_VMAX_FUSE);
> +
> + return val & FB_GFX_FREQ_FUSE_MASK;
> +}
> +
> +static u32 chv_rps_min_freq(struct intel_rps *rps)
> +{
> + struct drm_i915_private *i915 = rps_to_i915(rps);
> + u32 val;
> +
> + val = vlv_punit_read(i915, FB_GFX_FMIN_AT_VMIN_FUSE);
> + val >>= FB_GFX_FMIN_AT_VMIN_FUSE_SHIFT;
> +
> + return val & FB_GFX_FREQ_FUSE_MASK;
> +}
> +
> +static bool chv_rps_enable(struct intel_rps *rps)
> +{
> + struct intel_uncore *uncore = rps_to_uncore(rps);
> + struct drm_i915_private *i915 = rps_to_i915(rps);
> + u32 val;
> +
> + /* 1: Program defaults and thresholds for RPS*/
> + intel_uncore_write_fw(uncore, GEN6_RP_DOWN_TIMEOUT, 1000000);
> + intel_uncore_write_fw(uncore, GEN6_RP_UP_THRESHOLD, 59400);
> + intel_uncore_write_fw(uncore, GEN6_RP_DOWN_THRESHOLD, 245000);
> + intel_uncore_write_fw(uncore, GEN6_RP_UP_EI, 66000);
> + intel_uncore_write_fw(uncore, GEN6_RP_DOWN_EI, 350000);
> +
> + intel_uncore_write_fw(uncore, GEN6_RP_IDLE_HYSTERSIS, 10);
> +
> + /* 2: Enable RPS */
> + intel_uncore_write_fw(uncore, GEN6_RP_CONTROL,
> + GEN6_RP_MEDIA_HW_NORMAL_MODE |
> + GEN6_RP_MEDIA_IS_GFX |
> + GEN6_RP_ENABLE |
> + GEN6_RP_UP_BUSY_AVG |
> + GEN6_RP_DOWN_IDLE_AVG);
> +
> + /* Setting Fixed Bias */
> + vlv_punit_get(i915);
> +
> + val = VLV_OVERRIDE_EN | VLV_SOC_TDP_EN | CHV_BIAS_CPU_50_SOC_50;
> + vlv_punit_write(i915, VLV_TURBO_SOC_OVERRIDE, val);
> +
> + val = vlv_punit_read(i915, PUNIT_REG_GPU_FREQ_STS);
> +
> + vlv_punit_put(i915);
> +
> + /* RPS code assumes GPLL is used */
> + WARN_ONCE((val & GPLLENABLE) == 0, "GPLL not enabled\n");
> +
> + DRM_DEBUG_DRIVER("GPLL enabled? %s\n", yesno(val & GPLLENABLE));
> + DRM_DEBUG_DRIVER("GPU status: 0x%08x\n", val);
> +
> + return rps_reset(rps);
> +}
> +
> +static int vlv_rps_guar_freq(struct intel_rps *rps)
> +{
> + struct drm_i915_private *i915 = rps_to_i915(rps);
> + u32 val, rp1;
> +
> + val = vlv_nc_read(i915, IOSF_NC_FB_GFX_FREQ_FUSE);
> +
> + rp1 = val & FB_GFX_FGUARANTEED_FREQ_FUSE_MASK;
> + rp1 >>= FB_GFX_FGUARANTEED_FREQ_FUSE_SHIFT;
> +
> + return rp1;
> +}
> +
> +static int vlv_rps_max_freq(struct intel_rps *rps)
> +{
> + struct drm_i915_private *i915 = rps_to_i915(rps);
> + u32 val, rp0;
> +
> + val = vlv_nc_read(i915, IOSF_NC_FB_GFX_FREQ_FUSE);
> +
> + rp0 = (val & FB_GFX_MAX_FREQ_FUSE_MASK) >> FB_GFX_MAX_FREQ_FUSE_SHIFT;
> + /* Clamp to max */
> + rp0 = min_t(u32, rp0, 0xea);
> +
> + return rp0;
> +}
> +
> +static int vlv_rps_rpe_freq(struct intel_rps *rps)
> +{
> + struct drm_i915_private *i915 = rps_to_i915(rps);
> + u32 val, rpe;
> +
> + val = vlv_nc_read(i915, IOSF_NC_FB_GFX_FMAX_FUSE_LO);
> + rpe = (val & FB_FMAX_VMIN_FREQ_LO_MASK) >> FB_FMAX_VMIN_FREQ_LO_SHIFT;
> + val = vlv_nc_read(i915, IOSF_NC_FB_GFX_FMAX_FUSE_HI);
> + rpe |= (val & FB_FMAX_VMIN_FREQ_HI_MASK) << 5;
> +
> + return rpe;
> +}
> +
> +static int vlv_rps_min_freq(struct intel_rps *rps)
> +{
> + struct drm_i915_private *i915 = rps_to_i915(rps);
> + u32 val;
> +
> + val = vlv_punit_read(i915, PUNIT_REG_GPU_LFM) & 0xff;
> + /*
> + * According to the BYT Punit GPU turbo HAS 1.1.6.3 the minimum value
> + * for the minimum frequency in GPLL mode is 0xc1. Contrary to this on
> + * a BYT-M B0 the above register contains 0xbf. Moreover when setting
> + * a frequency Punit will not allow values below 0xc0. Clamp it 0xc0
> + * to make sure it matches what Punit accepts.
> + */
> + return max_t(u32, val, 0xc0);
> +}
> +
> +static bool vlv_rps_enable(struct intel_rps *rps)
> +{
> + struct intel_uncore *uncore = rps_to_uncore(rps);
> + struct drm_i915_private *i915 = rps_to_i915(rps);
> + u32 val;
> +
> + intel_uncore_write_fw(uncore, GEN6_RP_DOWN_TIMEOUT, 1000000);
> + intel_uncore_write_fw(uncore, GEN6_RP_UP_THRESHOLD, 59400);
> + intel_uncore_write_fw(uncore, GEN6_RP_DOWN_THRESHOLD, 245000);
> + intel_uncore_write_fw(uncore, GEN6_RP_UP_EI, 66000);
> + intel_uncore_write_fw(uncore, GEN6_RP_DOWN_EI, 350000);
> +
> + intel_uncore_write_fw(uncore, GEN6_RP_IDLE_HYSTERSIS, 10);
> +
> + intel_uncore_write_fw(uncore, GEN6_RP_CONTROL,
> + GEN6_RP_MEDIA_TURBO |
> + GEN6_RP_MEDIA_HW_NORMAL_MODE |
> + GEN6_RP_MEDIA_IS_GFX |
> + GEN6_RP_ENABLE |
> + GEN6_RP_UP_BUSY_AVG |
> + GEN6_RP_DOWN_IDLE_CONT);
> +
> + vlv_punit_get(i915);
> +
> + /* Setting Fixed Bias */
> + val = VLV_OVERRIDE_EN | VLV_SOC_TDP_EN | VLV_BIAS_CPU_125_SOC_875;
> + vlv_punit_write(i915, VLV_TURBO_SOC_OVERRIDE, val);
> +
> + val = vlv_punit_read(i915, PUNIT_REG_GPU_FREQ_STS);
> +
> + vlv_punit_put(i915);
> +
> + /* RPS code assumes GPLL is used */
> + WARN_ONCE((val & GPLLENABLE) == 0, "GPLL not enabled\n");
> +
> + DRM_DEBUG_DRIVER("GPLL enabled? %s\n", yesno(val & GPLLENABLE));
> + DRM_DEBUG_DRIVER("GPU status: 0x%08x\n", val);
> +
> + return rps_reset(rps);
> +}
> +
> +static unsigned long __ips_gfx_val(struct intel_ips *ips)
> +{
> + struct intel_rps *rps = container_of(ips, typeof(*rps), ips);
> + struct intel_uncore *uncore = rps_to_uncore(rps);
> + unsigned long t, corr, state1, corr2, state2;
> + u32 pxvid, ext_v;
> +
> + lockdep_assert_held(&mchdev_lock);
> +
> + pxvid = intel_uncore_read(uncore, PXVFREQ(rps->cur_freq));
> + pxvid = (pxvid >> 24) & 0x7f;
> + ext_v = pvid_to_extvid(rps_to_i915(rps), pxvid);
> +
> + state1 = ext_v;
> +
> + /* Revel in the empirically derived constants */
> +
> + /* Correction factor in 1/100000 units */
> + t = ips_mch_val(uncore);
> + if (t > 80)
> + corr = t * 2349 + 135940;
> + else if (t >= 50)
> + corr = t * 964 + 29317;
> + else /* < 50 */
> + corr = t * 301 + 1004;
> +
> + corr = corr * 150142 * state1 / 10000 - 78642;
> + corr /= 100000;
> + corr2 = corr * ips->corr;
> +
> + state2 = corr2 * state1 / 10000;
> + state2 /= 100; /* convert to mW */
> +
> + __gen5_ips_update(ips);
> +
> + return ips->gfx_power + state2;
> +}
> +
> +void intel_rps_enable(struct intel_rps *rps)
> +{
> + struct drm_i915_private *i915 = rps_to_i915(rps);
> + struct intel_uncore *uncore = rps_to_uncore(rps);
> +
> + intel_uncore_forcewake_get(uncore, FORCEWAKE_ALL);
> + if (IS_CHERRYVIEW(i915))
> + rps->enabled = chv_rps_enable(rps);
> + else if (IS_VALLEYVIEW(i915))
> + rps->enabled = vlv_rps_enable(rps);
> + else if (INTEL_GEN(i915) >= 9)
> + rps->enabled = gen9_rps_enable(rps);
> + else if (INTEL_GEN(i915) >= 8)
> + rps->enabled = gen8_rps_enable(rps);
> + else if (INTEL_GEN(i915) >= 6)
> + rps->enabled = gen6_rps_enable(rps);
> + else if (IS_IRONLAKE_M(i915))
> + rps->enabled = gen5_rps_enable(rps);
> + intel_uncore_forcewake_put(uncore, FORCEWAKE_ALL);
> + if (!rps->enabled)
> + return;
> +
> + WARN_ON(rps->max_freq < rps->min_freq);
> + WARN_ON(rps->idle_freq > rps->max_freq);
> +
> + WARN_ON(rps->efficient_freq < rps->min_freq);
> + WARN_ON(rps->efficient_freq > rps->max_freq);
> +}
> +
> +static void gen6_rps_disable(struct intel_rps *rps)
> +{
> + intel_uncore_write(rps_to_uncore(rps), GEN6_RP_CONTROL, 0);
> +}
> +
> +void intel_rps_disable(struct intel_rps *rps)
> +{
> + struct drm_i915_private *i915 = rps_to_i915(rps);
> +
> + rps->enabled = false;
> +
> + if (INTEL_GEN(i915) >= 6)
> + gen6_rps_disable(rps);
> + else if (IS_IRONLAKE_M(i915))
> + gen5_rps_disable(rps);
> +}
> +
> +static int byt_gpu_freq(struct intel_rps *rps, int val)
> +{
> + /*
> + * N = val - 0xb7
> + * Slow = Fast = GPLL ref * N
> + */
> + return DIV_ROUND_CLOSEST(rps->gpll_ref_freq * (val - 0xb7), 1000);
> +}
> +
> +static int byt_freq_opcode(struct intel_rps *rps, int val)
> +{
> + return DIV_ROUND_CLOSEST(1000 * val, rps->gpll_ref_freq) + 0xb7;
> +}
> +
> +static int chv_gpu_freq(struct intel_rps *rps, int val)
> +{
> + /*
> + * N = val / 2
> + * CU (slow) = CU2x (fast) / 2 = GPLL ref * N / 2
> + */
> + return DIV_ROUND_CLOSEST(rps->gpll_ref_freq * val, 2 * 2 * 1000);
> +}
> +
> +static int chv_freq_opcode(struct intel_rps *rps, int val)
> +{
> + /* CHV needs even values */
> + return DIV_ROUND_CLOSEST(2 * 1000 * val, rps->gpll_ref_freq) * 2;
> +}
> +
> +int intel_gpu_freq(struct intel_rps *rps, int val)
> +{
> + struct drm_i915_private *i915 = rps_to_i915(rps);
> +
> + if (INTEL_GEN(i915) >= 9)
> + return DIV_ROUND_CLOSEST(val * GT_FREQUENCY_MULTIPLIER,
> + GEN9_FREQ_SCALER);
> + else if (IS_CHERRYVIEW(i915))
> + return chv_gpu_freq(rps, val);
> + else if (IS_VALLEYVIEW(i915))
> + return byt_gpu_freq(rps, val);
> + else
> + return val * GT_FREQUENCY_MULTIPLIER;
> +}
> +
> +int intel_freq_opcode(struct intel_rps *rps, int val)
> +{
> + struct drm_i915_private *i915 = rps_to_i915(rps);
> +
> + if (INTEL_GEN(i915) >= 9)
> + return DIV_ROUND_CLOSEST(val * GEN9_FREQ_SCALER,
> + GT_FREQUENCY_MULTIPLIER);
> + else if (IS_CHERRYVIEW(i915))
> + return chv_freq_opcode(rps, val);
> + else if (IS_VALLEYVIEW(i915))
> + return byt_freq_opcode(rps, val);
> + else
> + return DIV_ROUND_CLOSEST(val, GT_FREQUENCY_MULTIPLIER);
> +}
> +
> +static void vlv_init_gpll_ref_freq(struct intel_rps *rps)
> +{
> + struct drm_i915_private *i915 = rps_to_i915(rps);
> +
> + rps->gpll_ref_freq =
> + vlv_get_cck_clock(i915, "GPLL ref",
> + CCK_GPLL_CLOCK_CONTROL,
> + i915->czclk_freq);
> +
> + DRM_DEBUG_DRIVER("GPLL reference freq: %d kHz\n", rps->gpll_ref_freq);
> +}
> +
> +static void vlv_rps_init(struct intel_rps *rps)
> +{
> + struct drm_i915_private *i915 = rps_to_i915(rps);
> + u32 val;
> +
> + vlv_iosf_sb_get(i915,
> + BIT(VLV_IOSF_SB_PUNIT) |
> + BIT(VLV_IOSF_SB_NC) |
> + BIT(VLV_IOSF_SB_CCK));
> +
> + vlv_init_gpll_ref_freq(rps);
> +
> + val = vlv_punit_read(i915, PUNIT_REG_GPU_FREQ_STS);
> + switch ((val >> 6) & 3) {
> + case 0:
> + case 1:
> + i915->mem_freq = 800;
> + break;
> + case 2:
> + i915->mem_freq = 1066;
> + break;
> + case 3:
> + i915->mem_freq = 1333;
> + break;
> + }
> + DRM_DEBUG_DRIVER("DDR speed: %d MHz\n", i915->mem_freq);
> +
> + rps->max_freq = vlv_rps_max_freq(rps);
> + rps->rp0_freq = rps->max_freq;
> + DRM_DEBUG_DRIVER("max GPU freq: %d MHz (%u)\n",
> + intel_gpu_freq(rps, rps->max_freq),
> + rps->max_freq);
> +
> + rps->efficient_freq = vlv_rps_rpe_freq(rps);
> + DRM_DEBUG_DRIVER("RPe GPU freq: %d MHz (%u)\n",
> + intel_gpu_freq(rps, rps->efficient_freq),
> + rps->efficient_freq);
> +
> + rps->rp1_freq = vlv_rps_guar_freq(rps);
> + DRM_DEBUG_DRIVER("RP1(Guar Freq) GPU freq: %d MHz (%u)\n",
> + intel_gpu_freq(rps, rps->rp1_freq),
> + rps->rp1_freq);
> +
> + rps->min_freq = vlv_rps_min_freq(rps);
> + DRM_DEBUG_DRIVER("min GPU freq: %d MHz (%u)\n",
> + intel_gpu_freq(rps, rps->min_freq),
> + rps->min_freq);
> +
> + vlv_iosf_sb_put(i915,
> + BIT(VLV_IOSF_SB_PUNIT) |
> + BIT(VLV_IOSF_SB_NC) |
> + BIT(VLV_IOSF_SB_CCK));
> +}
> +
> +static void chv_rps_init(struct intel_rps *rps)
> +{
> + struct drm_i915_private *i915 = rps_to_i915(rps);
> + u32 val;
> +
> + vlv_iosf_sb_get(i915,
> + BIT(VLV_IOSF_SB_PUNIT) |
> + BIT(VLV_IOSF_SB_NC) |
> + BIT(VLV_IOSF_SB_CCK));
> +
> + vlv_init_gpll_ref_freq(rps);
> +
> + val = vlv_cck_read(i915, CCK_FUSE_REG);
> +
> + switch ((val >> 2) & 0x7) {
> + case 3:
> + i915->mem_freq = 2000;
> + break;
> + default:
> + i915->mem_freq = 1600;
> + break;
> + }
> + DRM_DEBUG_DRIVER("DDR speed: %d MHz\n", i915->mem_freq);
> +
> + rps->max_freq = chv_rps_max_freq(rps);
> + rps->rp0_freq = rps->max_freq;
> + DRM_DEBUG_DRIVER("max GPU freq: %d MHz (%u)\n",
> + intel_gpu_freq(rps, rps->max_freq),
> + rps->max_freq);
> +
> + rps->efficient_freq = chv_rps_rpe_freq(rps);
> + DRM_DEBUG_DRIVER("RPe GPU freq: %d MHz (%u)\n",
> + intel_gpu_freq(rps, rps->efficient_freq),
> + rps->efficient_freq);
> +
> + rps->rp1_freq = chv_rps_guar_freq(rps);
> + DRM_DEBUG_DRIVER("RP1(Guar) GPU freq: %d MHz (%u)\n",
> + intel_gpu_freq(rps, rps->rp1_freq),
> + rps->rp1_freq);
> +
> + rps->min_freq = chv_rps_min_freq(rps);
> + DRM_DEBUG_DRIVER("min GPU freq: %d MHz (%u)\n",
> + intel_gpu_freq(rps, rps->min_freq),
> + rps->min_freq);
> +
> + vlv_iosf_sb_put(i915,
> + BIT(VLV_IOSF_SB_PUNIT) |
> + BIT(VLV_IOSF_SB_NC) |
> + BIT(VLV_IOSF_SB_CCK));
> +
> + WARN_ONCE((rps->max_freq | rps->efficient_freq | rps->rp1_freq |
> + rps->min_freq) & 1,
> + "Odd GPU freq values\n");
> +}
> +
> +static void vlv_c0_read(struct intel_uncore *uncore, struct intel_rps_ei *ei)
> +{
> + ei->ktime = ktime_get_raw();
> + ei->render_c0 = intel_uncore_read(uncore, VLV_RENDER_C0_COUNT);
> + ei->media_c0 = intel_uncore_read(uncore, VLV_MEDIA_C0_COUNT);
> +}
> +
> +static u32 vlv_wa_c0_ei(struct intel_rps *rps, u32 pm_iir)
> +{
> + struct intel_uncore *uncore = rps_to_uncore(rps);
> + const struct intel_rps_ei *prev = &rps->ei;
> + struct intel_rps_ei now;
> + u32 events = 0;
> +
> + if ((pm_iir & GEN6_PM_RP_UP_EI_EXPIRED) == 0)
> + return 0;
> +
> + vlv_c0_read(uncore, &now);
> +
> + if (prev->ktime) {
> + u64 time, c0;
> + u32 render, media;
> +
> + time = ktime_us_delta(now.ktime, prev->ktime);
> +
> + time *= rps_to_i915(rps)->czclk_freq;
> +
> + /* Workload can be split between render + media,
> + * e.g. SwapBuffers being blitted in X after being rendered in
> + * mesa. To account for this we need to combine both engines
> + * into our activity counter.
> + */
> + render = now.render_c0 - prev->render_c0;
> + media = now.media_c0 - prev->media_c0;
> + c0 = max(render, media);
> + c0 *= 1000 * 100 << 8; /* to usecs and scale to threshold% */
> +
> + if (c0 > time * rps->power.up_threshold)
> + events = GEN6_PM_RP_UP_THRESHOLD;
> + else if (c0 < time * rps->power.down_threshold)
> + events = GEN6_PM_RP_DOWN_THRESHOLD;
> + }
> +
> + rps->ei = now;
> + return events;
> +}
> +
> +static void rps_work(struct work_struct *work)
> +{
> + struct intel_rps *rps = container_of(work, typeof(*rps), work);
> + struct intel_gt *gt = rps_to_gt(rps);
> + bool client_boost = false;
> + int new_freq, adj, min, max;
> + u32 pm_iir = 0;
> +
> + spin_lock_irq(>->irq_lock);
> + pm_iir = fetch_and_zero(&rps->pm_iir);
> + client_boost = atomic_read(&rps->num_waiters);
> + spin_unlock_irq(>->irq_lock);
> +
> + /* Make sure we didn't queue anything we're not going to process. */
> + if ((pm_iir & rps->pm_events) == 0 && !client_boost)
> + goto out;
> +
> + mutex_lock(&rps->lock);
> +
> + pm_iir |= vlv_wa_c0_ei(rps, pm_iir);
> +
> + adj = rps->last_adj;
> + new_freq = rps->cur_freq;
> + min = rps->min_freq_softlimit;
> + max = rps->max_freq_softlimit;
> + if (client_boost)
> + max = rps->max_freq;
> + if (client_boost && new_freq < rps->boost_freq) {
> + new_freq = rps->boost_freq;
> + adj = 0;
> + } else if (pm_iir & GEN6_PM_RP_UP_THRESHOLD) {
> + if (adj > 0)
> + adj *= 2;
> + else /* CHV needs even encode values */
> + adj = IS_CHERRYVIEW(gt->i915) ? 2 : 1;
> +
> + if (new_freq >= rps->max_freq_softlimit)
> + adj = 0;
> + } else if (client_boost) {
> + adj = 0;
> + } else if (pm_iir & GEN6_PM_RP_DOWN_TIMEOUT) {
> + if (rps->cur_freq > rps->efficient_freq)
> + new_freq = rps->efficient_freq;
> + else if (rps->cur_freq > rps->min_freq_softlimit)
> + new_freq = rps->min_freq_softlimit;
> + adj = 0;
> + } else if (pm_iir & GEN6_PM_RP_DOWN_THRESHOLD) {
> + if (adj < 0)
> + adj *= 2;
> + else /* CHV needs even encode values */
> + adj = IS_CHERRYVIEW(gt->i915) ? -2 : -1;
> +
> + if (new_freq <= rps->min_freq_softlimit)
> + adj = 0;
> + } else { /* unknown event */
> + adj = 0;
> + }
> +
> + rps->last_adj = adj;
> +
> + /*
> + * Limit deboosting and boosting to keep ourselves at the extremes
> + * when in the respective power modes (i.e. slowly decrease frequencies
> + * while in the HIGH_POWER zone and slowly increase frequencies while
> + * in the LOW_POWER zone). On idle, we will hit the timeout and drop
> + * to the next level quickly, and conversely if busy we expect to
> + * hit a waitboost and rapidly switch into max power.
> + */
> + if ((adj < 0 && rps->power.mode == HIGH_POWER) ||
> + (adj > 0 && rps->power.mode == LOW_POWER))
> + rps->last_adj = 0;
> +
> + /* sysfs frequency interfaces may have snuck in while servicing the
> + * interrupt
> + */
> + new_freq += adj;
> + new_freq = clamp_t(int, new_freq, min, max);
> +
> + if (intel_rps_set(rps, new_freq)) {
> + DRM_DEBUG_DRIVER("Failed to set new GPU frequency\n");
> + rps->last_adj = 0;
> + }
> +
> + mutex_unlock(&rps->lock);
> +
> +out:
> + spin_lock_irq(>->irq_lock);
> + gen6_gt_pm_unmask_irq(gt, rps->pm_events);
> + spin_unlock_irq(>->irq_lock);
> +}
> +
> +void gen11_rps_irq_handler(struct intel_rps *rps, u32 pm_iir)
> +{
> + struct intel_gt *gt = rps_to_gt(rps);
> + const u32 events = rps->pm_events & pm_iir;
> +
> + lockdep_assert_held(>->irq_lock);
> +
> + if (unlikely(!events))
> + return;
> +
> + gen6_gt_pm_mask_irq(gt, events);
> +
> + rps->pm_iir |= events;
> + schedule_work(&rps->work);
> +}
> +
> +void gen6_rps_irq_handler(struct intel_rps *rps, u32 pm_iir)
> +{
> + struct drm_i915_private *i915 = rps_to_i915(rps);
> +
> + if (pm_iir & rps->pm_events) {
> + struct intel_gt *gt = rps_to_gt(rps);
> +
> + spin_lock(>->irq_lock);
> + gen6_gt_pm_mask_irq(gt, pm_iir & rps->pm_events);
> + rps->pm_iir |= pm_iir & rps->pm_events;
> + schedule_work(&rps->work);
> + spin_unlock(>->irq_lock);
> + }
> +
> + if (INTEL_GEN(i915) >= 8)
> + return;
> +
> + if (pm_iir & PM_VEBOX_USER_INTERRUPT)
> + intel_engine_breadcrumbs_irq(i915->engine[VECS0]);
> +
> + if (pm_iir & PM_VEBOX_CS_ERROR_INTERRUPT)
> + DRM_DEBUG("Command parser error, pm_iir 0x%08x\n", pm_iir);
> +}
> +
> +void gen5_rps_irq_handler(struct intel_rps *rps)
> +{
> + struct intel_uncore *uncore = rps_to_uncore(rps);
> + u32 busy_up, busy_down, max_avg, min_avg;
> + u8 new_freq;
> +
> + spin_lock(&mchdev_lock);
> +
> + intel_uncore_write16(uncore,
> + MEMINTRSTS,
> + intel_uncore_read(uncore, MEMINTRSTS));
> +
> + intel_uncore_write16(uncore, MEMINTRSTS, MEMINT_EVAL_CHG);
> + busy_up = intel_uncore_read(uncore, RCPREVBSYTUPAVG);
> + busy_down = intel_uncore_read(uncore, RCPREVBSYTDNAVG);
> + max_avg = intel_uncore_read(uncore, RCBMAXAVG);
> + min_avg = intel_uncore_read(uncore, RCBMINAVG);
> +
> + /* Handle RCS change request from hw */
> + new_freq = rps->cur_freq;
> + if (busy_up > max_avg)
> + new_freq++;
> + else if (busy_down < min_avg)
> + new_freq--;
> + new_freq = clamp(new_freq,
> + rps->min_freq_softlimit,
> + rps->max_freq_softlimit);
> +
> + if (new_freq != rps->cur_freq && gen5_rps_set(rps, new_freq))
> + rps->cur_freq = new_freq;
> +
> + spin_unlock(&mchdev_lock);
> +}
> +
> +void intel_rps_init(struct intel_rps *rps)
> +{
> + struct drm_i915_private *i915 = rps_to_i915(rps);
> +
> + mutex_init(&rps->lock);
> + mutex_init(&rps->power.mutex);
> +
> + INIT_WORK(&rps->work, rps_work);
> +
> + atomic_set(&rps->num_waiters, 0);
> +
> + if (IS_CHERRYVIEW(i915))
> + chv_rps_init(rps);
> + else if (IS_VALLEYVIEW(i915))
> + vlv_rps_init(rps);
> + else if (INTEL_GEN(i915) >= 6)
> + gen6_rps_init(rps);
> + else if (IS_IRONLAKE_M(i915))
> + gen5_rps_init(rps);
> +
> + /* Derive initial user preferences/limits from the hardware limits */
> + rps->max_freq_softlimit = rps->max_freq;
> + rps->min_freq_softlimit = rps->min_freq;
> +
> + /* After setting max-softlimit, find the overclock max freq */
> + if (IS_GEN(i915, 6) || IS_IVYBRIDGE(i915) || IS_HASWELL(i915)) {
> + u32 params = 0;
> +
> + sandybridge_pcode_read(i915, GEN6_READ_OC_PARAMS,
> + ¶ms, NULL);
> + if (params & BIT(31)) { /* OC supported */
> + DRM_DEBUG_DRIVER("Overclocking supported, max: %dMHz, overclock: %dMHz\n",
> + (rps->max_freq & 0xff) * 50,
> + (params & 0xff) * 50);
> + rps->max_freq = params & 0xff;
> + }
> + }
> +
> + /* Finally allow us to boost to max by default */
> + rps->boost_freq = rps->max_freq;
> + rps->idle_freq = rps->min_freq;
> + rps->cur_freq = rps->idle_freq;
> +
> + rps->pm_intrmsk_mbz = 0;
> +
> + /*
> + * SNB,IVB,HSW can while VLV,CHV may hard hang on looping batchbuffer
> + * if GEN6_PM_UP_EI_EXPIRED is masked.
> + *
> + * TODO: verify if this can be reproduced on VLV,CHV.
> + */
> + if (INTEL_GEN(i915) <= 7)
> + rps->pm_intrmsk_mbz |= GEN6_PM_RP_UP_EI_EXPIRED;
> +
> + if (INTEL_GEN(i915) >= 8)
> + rps->pm_intrmsk_mbz |= GEN8_PMINTR_DISABLE_REDIRECT_TO_GUC;
> +}
> +
> +u32 intel_get_cagf(struct intel_rps *rps, u32 rpstat)
> +{
> + struct drm_i915_private *i915 = rps_to_i915(rps);
> + u32 cagf;
> +
> + if (INTEL_GEN(i915) >= 9)
> + cagf = (rpstat & GEN9_CAGF_MASK) >> GEN9_CAGF_SHIFT;
> + else if (IS_HASWELL(i915) || IS_BROADWELL(i915))
> + cagf = (rpstat & HSW_CAGF_MASK) >> HSW_CAGF_SHIFT;
> + else
> + cagf = (rpstat & GEN6_CAGF_MASK) >> GEN6_CAGF_SHIFT;
> +
> + return cagf;
> +}
> +
> +/* External interface for intel_ips.ko */
> +
> +static struct drm_i915_private __rcu *ips_mchdev;
> +
> +/**
> + * Tells the intel_ips driver that the i915 driver is now loaded, if
> + * IPS got loaded first.
> + *
> + * This awkward dance is so that neither module has to depend on the
> + * other in order for IPS to do the appropriate communication of
> + * GPU turbo limits to i915.
> + */
> +static void
> +ips_ping_for_i915_load(void)
> +{
> + void (*link)(void);
> +
> + link = symbol_get(ips_link_to_i915_driver);
> + if (link) {
> + link();
> + symbol_put(ips_link_to_i915_driver);
> + }
> +}
> +
> +void intel_rps_driver_register(struct intel_rps *rps)
> +{
> + struct intel_gt *gt = rps_to_gt(rps);
> +
> + /*
> + * We only register the i915 ips part with intel-ips once everything is
> + * set up, to avoid intel-ips sneaking in and reading bogus values.
> + */
> + if (IS_GEN(gt->i915, 5)) {
> + rcu_assign_pointer(ips_mchdev, gt->i915);
> + ips_ping_for_i915_load();
> + }
> +}
> +
> +void intel_rps_driver_unregister(struct intel_rps *rps)
> +{
> + rcu_assign_pointer(ips_mchdev, NULL);
> +}
> +
> +static struct drm_i915_private *mchdev_get(void)
> +{
> + struct drm_i915_private *i915;
> +
> + rcu_read_lock();
> + i915 = rcu_dereference(ips_mchdev);
> + if (!kref_get_unless_zero(&i915->drm.ref))
> + i915 = NULL;
> + rcu_read_unlock();
> +
> + return i915;
> +}
> +
> +/**
> + * i915_read_mch_val - return value for IPS use
> + *
> + * Calculate and return a value for the IPS driver to use when deciding whether
> + * we have thermal and power headroom to increase CPU or GPU power budget.
> + */
> +unsigned long i915_read_mch_val(void)
> +{
> + struct drm_i915_private *i915;
> + unsigned long chipset_val = 0;
> + unsigned long graphics_val = 0;
> + intel_wakeref_t wakeref;
> +
> + i915 = mchdev_get();
> + if (!i915)
> + return 0;
> +
> + with_intel_runtime_pm(&i915->runtime_pm, wakeref) {
> + struct intel_ips *ips = &i915->gt.rps.ips;
> +
> + spin_lock_irq(&mchdev_lock);
> + chipset_val = __ips_chipset_val(ips);
> + graphics_val = __ips_gfx_val(ips);
> + spin_unlock_irq(&mchdev_lock);
> + }
> +
> + drm_dev_put(&i915->drm);
> + return chipset_val + graphics_val;
> +}
> +EXPORT_SYMBOL_GPL(i915_read_mch_val);
> +
> +/**
> + * i915_gpu_raise - raise GPU frequency limit
> + *
> + * Raise the limit; IPS indicates we have thermal headroom.
> + */
> +bool i915_gpu_raise(void)
> +{
> + struct drm_i915_private *i915;
> + struct intel_rps *rps;
> +
> + i915 = mchdev_get();
> + if (!i915)
> + return false;
> +
> + rps = &i915->gt.rps;
> +
> + spin_lock_irq(&mchdev_lock);
> + if (rps->max_freq_softlimit < rps->max_freq)
> + rps->max_freq_softlimit++;
> + spin_unlock_irq(&mchdev_lock);
> +
> + drm_dev_put(&i915->drm);
> + return true;
> +}
> +EXPORT_SYMBOL_GPL(i915_gpu_raise);
> +
> +/**
> + * i915_gpu_lower - lower GPU frequency limit
> + *
> + * IPS indicates we're close to a thermal limit, so throttle back the GPU
> + * frequency maximum.
> + */
> +bool i915_gpu_lower(void)
> +{
> + struct drm_i915_private *i915;
> + struct intel_rps *rps;
> +
> + i915 = mchdev_get();
> + if (!i915)
> + return false;
> +
> + rps = &i915->gt.rps;
> +
> + spin_lock_irq(&mchdev_lock);
> + if (rps->max_freq_softlimit > rps->min_freq)
> + rps->max_freq_softlimit--;
> + spin_unlock_irq(&mchdev_lock);
> +
> + drm_dev_put(&i915->drm);
> + return true;
> +}
> +EXPORT_SYMBOL_GPL(i915_gpu_lower);
> +
> +/**
> + * i915_gpu_busy - indicate GPU business to IPS
> + *
> + * Tell the IPS driver whether or not the GPU is busy.
> + */
> +bool i915_gpu_busy(void)
> +{
> + struct drm_i915_private *i915;
> + bool ret;
> +
> + i915 = mchdev_get();
> + if (!i915)
> + return false;
> +
> + ret = i915->gt.awake;
> +
> + drm_dev_put(&i915->drm);
> + return ret;
> +}
> +EXPORT_SYMBOL_GPL(i915_gpu_busy);
> +
> +/**
> + * i915_gpu_turbo_disable - disable graphics turbo
> + *
> + * Disable graphics turbo by resetting the max frequency and setting the
> + * current frequency to the default.
> + */
> +bool i915_gpu_turbo_disable(void)
> +{
> + struct drm_i915_private *i915;
> + struct intel_rps *rps;
> + bool ret;
> +
> + i915 = mchdev_get();
> + if (!i915)
> + return false;
> +
> + rps = &i915->gt.rps;
> +
> + spin_lock_irq(&mchdev_lock);
> + rps->max_freq_softlimit = rps->min_freq;
> + ret = gen5_rps_set(&i915->gt.rps, rps->min_freq);
> + spin_unlock_irq(&mchdev_lock);
> +
> + drm_dev_put(&i915->drm);
> + return ret;
> +}
> +EXPORT_SYMBOL_GPL(i915_gpu_turbo_disable);
> diff --git a/drivers/gpu/drm/i915/gt/intel_rps.h b/drivers/gpu/drm/i915/gt/intel_rps.h
> new file mode 100644
> index 000000000000..997a4b4e0207
> --- /dev/null
> +++ b/drivers/gpu/drm/i915/gt/intel_rps.h
> @@ -0,0 +1,37 @@
> +/*
> + * SPDX-License-Identifier: MIT
> + *
> + * Copyright © 2019 Intel Corporation
> + */
> +
> +#ifndef INTEL_RPS_H
> +#define INTEL_RPS_H
> +
> +#include "intel_rps_types.h"
> +
> +struct i915_request;
> +
> +void intel_rps_init(struct intel_rps *rps);
> +
> +void intel_rps_driver_register(struct intel_rps *rps);
> +void intel_rps_driver_unregister(struct intel_rps *rps);
> +
> +void intel_rps_enable(struct intel_rps *rps);
> +void intel_rps_disable(struct intel_rps *rps);
> +
> +void intel_rps_park(struct intel_rps *rps);
> +void intel_rps_unpark(struct intel_rps *rps);
> +void intel_rps_boost(struct i915_request *rq);
> +
> +int intel_rps_set(struct intel_rps *rps, u8 val);
> +void intel_rps_mark_interactive(struct intel_rps *rps, bool interactive);
> +
> +int intel_gpu_freq(struct intel_rps *rps, int val);
> +int intel_freq_opcode(struct intel_rps *rps, int val);
> +u32 intel_get_cagf(struct intel_rps *rps, u32 rpstat1);
> +
> +void gen5_rps_irq_handler(struct intel_rps *rps);
> +void gen6_rps_irq_handler(struct intel_rps *rps, u32 pm_iir);
> +void gen11_rps_irq_handler(struct intel_rps *rps, u32 pm_iir);
> +
> +#endif /* INTEL_RPS_H */
> diff --git a/drivers/gpu/drm/i915/gt/intel_rps_types.h b/drivers/gpu/drm/i915/gt/intel_rps_types.h
> new file mode 100644
> index 000000000000..40eb1fb651e7
> --- /dev/null
> +++ b/drivers/gpu/drm/i915/gt/intel_rps_types.h
> @@ -0,0 +1,93 @@
> +/*
> + * SPDX-License-Identifier: MIT
> + *
> + * Copyright © 2019 Intel Corporation
> + */
> +
> +#ifndef INTEL_RPS_TYPES_H
> +#define INTEL_RPS_TYPES_H
> +
> +#include <linux/atomic.h>
> +#include <linux/ktime.h>
> +#include <linux/mutex.h>
> +#include <linux/types.h>
> +#include <linux/workqueue.h>
> +
> +struct intel_ips {
> + u64 last_count1;
> + unsigned long last_time1;
> + unsigned long chipset_power;
> + u64 last_count2;
> + u64 last_time2;
> + unsigned long gfx_power;
> + u8 corr;
> +
> + int c, m;
> +};
> +
> +struct intel_rps_ei {
> + ktime_t ktime;
> + u32 render_c0;
> + u32 media_c0;
> +};
> +
> +struct intel_rps {
> + struct mutex lock; /* protects enabling and the worker */
> +
> + /*
> + * work, interrupts_enabled and pm_iir are protected by
> + * dev_priv->irq_lock
> + */
> + struct work_struct work;
> + bool enabled;
> + bool active;
> + u32 pm_iir;
> +
> + /* PM interrupt bits that should never be masked */
> + u32 pm_intrmsk_mbz;
> + u32 pm_events;
> +
> + /* Frequencies are stored in potentially platform dependent multiples.
> + * In other words, *_freq needs to be multiplied by X to be interesting.
> + * Soft limits are those which are used for the dynamic reclocking done
> + * by the driver (raise frequencies under heavy loads, and lower for
> + * lighter loads). Hard limits are those imposed by the hardware.
> + *
> + * A distinction is made for overclocking, which is never enabled by
> + * default, and is considered to be above the hard limit if it's
> + * possible at all.
> + */
> + u8 cur_freq; /* Current frequency (cached, may not == HW) */
> + u8 last_freq; /* Last swqreq frequency */
> + u8 min_freq_softlimit; /* Minimum frequency permitted by the driver */
> + u8 max_freq_softlimit; /* Max frequency permitted by the driver */
> + u8 max_freq; /* Maximum frequency, RP0 if not overclocking */
> + u8 min_freq; /* AKA RPn. Minimum frequency */
> + u8 boost_freq; /* Frequency to request when wait boosting */
> + u8 idle_freq; /* Frequency to request when we are idle */
> + u8 efficient_freq; /* AKA RPe. Pre-determined balanced frequency */
> + u8 rp1_freq; /* "less than" RP0 power/freqency */
> + u8 rp0_freq; /* Non-overclocked max frequency. */
> + u16 gpll_ref_freq; /* vlv/chv GPLL reference frequency */
> +
> + int last_adj;
> +
> + struct {
> + struct mutex mutex;
> +
> + enum { LOW_POWER, BETWEEN, HIGH_POWER } mode;
> + unsigned int interactive;
> +
> + u8 up_threshold; /* Current %busy required to uplock */
> + u8 down_threshold; /* Current %busy required to downclock */
> + } power;
> +
> + atomic_t num_waiters;
> + atomic_t boosts;
> +
> + /* manual wa residency calculations */
> + struct intel_rps_ei ei;
> + struct intel_ips ips;
> +};
> +
> +#endif /* INTEL_RPS_TYPES_H */
> diff --git a/drivers/gpu/drm/i915/gt/selftest_llc.c b/drivers/gpu/drm/i915/gt/selftest_llc.c
> index a7057785e420..fd3770e48ac7 100644
> --- a/drivers/gpu/drm/i915/gt/selftest_llc.c
> +++ b/drivers/gpu/drm/i915/gt/selftest_llc.c
> @@ -6,6 +6,7 @@
>
> #include "intel_pm.h" /* intel_gpu_freq() */
> #include "selftest_llc.h"
> +#include "intel_rps.h"
>
> static int gen6_verify_ring_freq(struct intel_llc *llc)
> {
> @@ -25,6 +26,8 @@ static int gen6_verify_ring_freq(struct intel_llc *llc)
> for (gpu_freq = consts.min_gpu_freq;
> gpu_freq <= consts.max_gpu_freq;
> gpu_freq++) {
> + struct intel_rps *rps = &llc_to_gt(llc)->rps;
> +
> unsigned int ia_freq, ring_freq, found;
> u32 val;
>
> @@ -44,7 +47,7 @@ static int gen6_verify_ring_freq(struct intel_llc *llc)
> if (found != ia_freq) {
> pr_err("Min freq table(%d/[%d, %d]):%dMHz did not match expected CPU freq, found %d, expected %d\n",
> gpu_freq, consts.min_gpu_freq, consts.max_gpu_freq,
> - intel_gpu_freq(i915, gpu_freq * (INTEL_GEN(i915) >= 9 ? GEN9_FREQ_SCALER : 1)),
> + intel_gpu_freq(rps, gpu_freq * (INTEL_GEN(i915) >= 9 ? GEN9_FREQ_SCALER : 1)),
> found, ia_freq);
> err = -EINVAL;
> break;
> @@ -54,7 +57,7 @@ static int gen6_verify_ring_freq(struct intel_llc *llc)
> if (found != ring_freq) {
> pr_err("Min freq table(%d/[%d, %d]):%dMHz did not match expected ring freq, found %d, expected %d\n",
> gpu_freq, consts.min_gpu_freq, consts.max_gpu_freq,
> - intel_gpu_freq(i915, gpu_freq * (INTEL_GEN(i915) >= 9 ? GEN9_FREQ_SCALER : 1)),
> + intel_gpu_freq(rps, gpu_freq * (INTEL_GEN(i915) >= 9 ? GEN9_FREQ_SCALER : 1)),
> found, ring_freq);
> err = -EINVAL;
> break;
> diff --git a/drivers/gpu/drm/i915/gt/uc/intel_guc_submission.c b/drivers/gpu/drm/i915/gt/uc/intel_guc_submission.c
> index 009e54a3764f..97b858ce7682 100644
> --- a/drivers/gpu/drm/i915/gt/uc/intel_guc_submission.c
> +++ b/drivers/gpu/drm/i915/gt/uc/intel_guc_submission.c
> @@ -1010,7 +1010,7 @@ void intel_guc_submission_fini(struct intel_guc *guc)
>
> static void guc_interrupts_capture(struct intel_gt *gt)
> {
> - struct intel_rps *rps = >->i915->gt_pm.rps;
> + struct intel_rps *rps = >->rps;
> struct intel_uncore *uncore = gt->uncore;
> struct intel_engine_cs *engine;
> enum intel_engine_id id;
> @@ -1056,7 +1056,7 @@ static void guc_interrupts_capture(struct intel_gt *gt)
>
> static void guc_interrupts_release(struct intel_gt *gt)
> {
> - struct intel_rps *rps = >->i915->gt_pm.rps;
> + struct intel_rps *rps = >->rps;
> struct intel_uncore *uncore = gt->uncore;
> struct intel_engine_cs *engine;
> enum intel_engine_id id;
> diff --git a/drivers/gpu/drm/i915/i915_debugfs.c b/drivers/gpu/drm/i915/i915_debugfs.c
> index 16211430eb78..22962373723b 100644
> --- a/drivers/gpu/drm/i915/i915_debugfs.c
> +++ b/drivers/gpu/drm/i915/i915_debugfs.c
> @@ -44,6 +44,7 @@
> #include "gt/intel_gt_requests.h"
> #include "gt/intel_reset.h"
> #include "gt/intel_rc6.h"
> +#include "gt/intel_rps.h"
> #include "gt/uc/intel_guc_submission.h"
>
> #include "i915_debugfs.h"
> @@ -791,7 +792,7 @@ static int i915_frequency_info(struct seq_file *m, void *unused)
> {
> struct drm_i915_private *dev_priv = node_to_i915(m->private);
> struct intel_uncore *uncore = &dev_priv->uncore;
> - struct intel_rps *rps = &dev_priv->gt_pm.rps;
> + struct intel_rps *rps = &dev_priv->gt.rps;
> intel_wakeref_t wakeref;
> int ret = 0;
>
> @@ -827,23 +828,23 @@ static int i915_frequency_info(struct seq_file *m, void *unused)
> seq_printf(m, "DDR freq: %d MHz\n", dev_priv->mem_freq);
>
> seq_printf(m, "actual GPU freq: %d MHz\n",
> - intel_gpu_freq(dev_priv, (freq_sts >> 8) & 0xff));
> + intel_gpu_freq(rps, (freq_sts >> 8) & 0xff));
>
> seq_printf(m, "current GPU freq: %d MHz\n",
> - intel_gpu_freq(dev_priv, rps->cur_freq));
> + intel_gpu_freq(rps, rps->cur_freq));
>
> seq_printf(m, "max GPU freq: %d MHz\n",
> - intel_gpu_freq(dev_priv, rps->max_freq));
> + intel_gpu_freq(rps, rps->max_freq));
>
> seq_printf(m, "min GPU freq: %d MHz\n",
> - intel_gpu_freq(dev_priv, rps->min_freq));
> + intel_gpu_freq(rps, rps->min_freq));
>
> seq_printf(m, "idle GPU freq: %d MHz\n",
> - intel_gpu_freq(dev_priv, rps->idle_freq));
> + intel_gpu_freq(rps, rps->idle_freq));
>
> seq_printf(m,
> "efficient (RPe) frequency: %d MHz\n",
> - intel_gpu_freq(dev_priv, rps->efficient_freq));
> + intel_gpu_freq(rps, rps->efficient_freq));
> } else if (INTEL_GEN(dev_priv) >= 6) {
> u32 rp_state_limits;
> u32 gt_perf_status;
> @@ -877,7 +878,7 @@ static int i915_frequency_info(struct seq_file *m, void *unused)
> else
> reqf >>= 25;
> }
> - reqf = intel_gpu_freq(dev_priv, reqf);
> + reqf = intel_gpu_freq(rps, reqf);
>
> rpmodectl = I915_READ(GEN6_RP_CONTROL);
> rpinclimit = I915_READ(GEN6_RP_UP_THRESHOLD);
> @@ -890,8 +891,7 @@ static int i915_frequency_info(struct seq_file *m, void *unused)
> rpdownei = I915_READ(GEN6_RP_CUR_DOWN_EI) & GEN6_CURIAVG_MASK;
> rpcurdown = I915_READ(GEN6_RP_CUR_DOWN) & GEN6_CURBSYTAVG_MASK;
> rpprevdown = I915_READ(GEN6_RP_PREV_DOWN) & GEN6_CURBSYTAVG_MASK;
> - cagf = intel_gpu_freq(dev_priv,
> - intel_get_cagf(dev_priv, rpstat));
> + cagf = intel_gpu_freq(rps, intel_get_cagf(rps, rpstat));
>
> intel_uncore_forcewake_put(&dev_priv->uncore, FORCEWAKE_ALL);
>
> @@ -968,37 +968,37 @@ static int i915_frequency_info(struct seq_file *m, void *unused)
> max_freq *= (IS_GEN9_BC(dev_priv) ||
> INTEL_GEN(dev_priv) >= 10 ? GEN9_FREQ_SCALER : 1);
> seq_printf(m, "Lowest (RPN) frequency: %dMHz\n",
> - intel_gpu_freq(dev_priv, max_freq));
> + intel_gpu_freq(rps, max_freq));
>
> max_freq = (rp_state_cap & 0xff00) >> 8;
> max_freq *= (IS_GEN9_BC(dev_priv) ||
> INTEL_GEN(dev_priv) >= 10 ? GEN9_FREQ_SCALER : 1);
> seq_printf(m, "Nominal (RP1) frequency: %dMHz\n",
> - intel_gpu_freq(dev_priv, max_freq));
> + intel_gpu_freq(rps, max_freq));
>
> max_freq = (IS_GEN9_LP(dev_priv) ? rp_state_cap >> 16 :
> rp_state_cap >> 0) & 0xff;
> max_freq *= (IS_GEN9_BC(dev_priv) ||
> INTEL_GEN(dev_priv) >= 10 ? GEN9_FREQ_SCALER : 1);
> seq_printf(m, "Max non-overclocked (RP0) frequency: %dMHz\n",
> - intel_gpu_freq(dev_priv, max_freq));
> + intel_gpu_freq(rps, max_freq));
> seq_printf(m, "Max overclocked frequency: %dMHz\n",
> - intel_gpu_freq(dev_priv, rps->max_freq));
> + intel_gpu_freq(rps, rps->max_freq));
>
> seq_printf(m, "Current freq: %d MHz\n",
> - intel_gpu_freq(dev_priv, rps->cur_freq));
> + intel_gpu_freq(rps, rps->cur_freq));
> seq_printf(m, "Actual freq: %d MHz\n", cagf);
> seq_printf(m, "Idle freq: %d MHz\n",
> - intel_gpu_freq(dev_priv, rps->idle_freq));
> + intel_gpu_freq(rps, rps->idle_freq));
> seq_printf(m, "Min freq: %d MHz\n",
> - intel_gpu_freq(dev_priv, rps->min_freq));
> + intel_gpu_freq(rps, rps->min_freq));
> seq_printf(m, "Boost freq: %d MHz\n",
> - intel_gpu_freq(dev_priv, rps->boost_freq));
> + intel_gpu_freq(rps, rps->boost_freq));
> seq_printf(m, "Max freq: %d MHz\n",
> - intel_gpu_freq(dev_priv, rps->max_freq));
> + intel_gpu_freq(rps, rps->max_freq));
> seq_printf(m,
> "efficient (RPe) frequency: %d MHz\n",
> - intel_gpu_freq(dev_priv, rps->efficient_freq));
> + intel_gpu_freq(rps, rps->efficient_freq));
> } else {
> seq_puts(m, "no P-state info available\n");
> }
> @@ -1461,7 +1461,7 @@ static int i915_sr_status(struct seq_file *m, void *unused)
> static int i915_ring_freq_table(struct seq_file *m, void *unused)
> {
> struct drm_i915_private *dev_priv = node_to_i915(m->private);
> - struct intel_rps *rps = &dev_priv->gt_pm.rps;
> + struct intel_rps *rps = &dev_priv->gt.rps;
> unsigned int max_gpu_freq, min_gpu_freq;
> intel_wakeref_t wakeref;
> int gpu_freq, ia_freq;
> @@ -1486,10 +1486,11 @@ static int i915_ring_freq_table(struct seq_file *m, void *unused)
> GEN6_PCODE_READ_MIN_FREQ_TABLE,
> &ia_freq, NULL);
> seq_printf(m, "%d\t\t%d\t\t\t\t%d\n",
> - intel_gpu_freq(dev_priv, (gpu_freq *
> - (IS_GEN9_BC(dev_priv) ||
> - INTEL_GEN(dev_priv) >= 10 ?
> - GEN9_FREQ_SCALER : 1))),
> + intel_gpu_freq(rps,
> + (gpu_freq *
> + (IS_GEN9_BC(dev_priv) ||
> + INTEL_GEN(dev_priv) >= 10 ?
> + GEN9_FREQ_SCALER : 1))),
> ((ia_freq >> 0) & 0xff) * 100,
> ((ia_freq >> 8) & 0xff) * 100);
> }
> @@ -1717,7 +1718,7 @@ static const char *rps_power_to_str(unsigned int power)
> static int i915_rps_boost_info(struct seq_file *m, void *data)
> {
> struct drm_i915_private *dev_priv = node_to_i915(m->private);
> - struct intel_rps *rps = &dev_priv->gt_pm.rps;
> + struct intel_rps *rps = &dev_priv->gt.rps;
> u32 act_freq = rps->cur_freq;
> intel_wakeref_t wakeref;
>
> @@ -1729,7 +1730,7 @@ static int i915_rps_boost_info(struct seq_file *m, void *data)
> vlv_punit_put(dev_priv);
> act_freq = (act_freq >> 8) & 0xff;
> } else {
> - act_freq = intel_get_cagf(dev_priv,
> + act_freq = intel_get_cagf(rps,
> I915_READ(GEN6_RPSTAT1));
> }
> }
> @@ -1740,17 +1741,17 @@ static int i915_rps_boost_info(struct seq_file *m, void *data)
> atomic_read(&rps->num_waiters));
> seq_printf(m, "Interactive? %d\n", READ_ONCE(rps->power.interactive));
> seq_printf(m, "Frequency requested %d, actual %d\n",
> - intel_gpu_freq(dev_priv, rps->cur_freq),
> - intel_gpu_freq(dev_priv, act_freq));
> + intel_gpu_freq(rps, rps->cur_freq),
> + intel_gpu_freq(rps, act_freq));
> seq_printf(m, " min hard:%d, soft:%d; max soft:%d, hard:%d\n",
> - intel_gpu_freq(dev_priv, rps->min_freq),
> - intel_gpu_freq(dev_priv, rps->min_freq_softlimit),
> - intel_gpu_freq(dev_priv, rps->max_freq_softlimit),
> - intel_gpu_freq(dev_priv, rps->max_freq));
> + intel_gpu_freq(rps, rps->min_freq),
> + intel_gpu_freq(rps, rps->min_freq_softlimit),
> + intel_gpu_freq(rps, rps->max_freq_softlimit),
> + intel_gpu_freq(rps, rps->max_freq));
> seq_printf(m, " idle:%d, efficient:%d, boost:%d\n",
> - intel_gpu_freq(dev_priv, rps->idle_freq),
> - intel_gpu_freq(dev_priv, rps->efficient_freq),
> - intel_gpu_freq(dev_priv, rps->boost_freq));
> + intel_gpu_freq(rps, rps->idle_freq),
> + intel_gpu_freq(rps, rps->efficient_freq),
> + intel_gpu_freq(rps, rps->boost_freq));
>
> seq_printf(m, "Wait boosts: %d\n", atomic_read(&rps->boosts));
>
> diff --git a/drivers/gpu/drm/i915/i915_drv.c b/drivers/gpu/drm/i915/i915_drv.c
> index 5138d1eed306..78ac08db48a3 100644
> --- a/drivers/gpu/drm/i915/i915_drv.c
> +++ b/drivers/gpu/drm/i915/i915_drv.c
> @@ -1841,6 +1841,8 @@ static int i915_drm_resume(struct drm_device *dev)
>
> intel_dp_mst_resume(dev_priv);
>
> + intel_gt_pm_enable(&dev_priv->gt);
> +
> intel_display_resume(dev);
>
> drm_kms_helper_poll_enable(dev);
> diff --git a/drivers/gpu/drm/i915/i915_drv.h b/drivers/gpu/drm/i915/i915_drv.h
> index 8882c0908c3b..eb5460290811 100644
> --- a/drivers/gpu/drm/i915/i915_drv.h
> +++ b/drivers/gpu/drm/i915/i915_drv.h
> @@ -543,94 +543,6 @@ struct i915_suspend_saved_registers {
>
> struct vlv_s0ix_state;
>
> -struct intel_rps_ei {
> - ktime_t ktime;
> - u32 render_c0;
> - u32 media_c0;
> -};
> -
> -struct intel_rps {
> - struct mutex lock; /* protects enabling and the worker */
> -
> - /*
> - * work, interrupts_enabled and pm_iir are protected by
> - * dev_priv->irq_lock
> - */
> - struct work_struct work;
> - bool interrupts_enabled;
> - u32 pm_iir;
> -
> - /* PM interrupt bits that should never be masked */
> - u32 pm_intrmsk_mbz;
> -
> - /* Frequencies are stored in potentially platform dependent multiples.
> - * In other words, *_freq needs to be multiplied by X to be interesting.
> - * Soft limits are those which are used for the dynamic reclocking done
> - * by the driver (raise frequencies under heavy loads, and lower for
> - * lighter loads). Hard limits are those imposed by the hardware.
> - *
> - * A distinction is made for overclocking, which is never enabled by
> - * default, and is considered to be above the hard limit if it's
> - * possible at all.
> - */
> - u8 cur_freq; /* Current frequency (cached, may not == HW) */
> - u8 min_freq_softlimit; /* Minimum frequency permitted by the driver */
> - u8 max_freq_softlimit; /* Max frequency permitted by the driver */
> - u8 max_freq; /* Maximum frequency, RP0 if not overclocking */
> - u8 min_freq; /* AKA RPn. Minimum frequency */
> - u8 boost_freq; /* Frequency to request when wait boosting */
> - u8 idle_freq; /* Frequency to request when we are idle */
> - u8 efficient_freq; /* AKA RPe. Pre-determined balanced frequency */
> - u8 rp1_freq; /* "less than" RP0 power/freqency */
> - u8 rp0_freq; /* Non-overclocked max frequency. */
> - u16 gpll_ref_freq; /* vlv/chv GPLL reference frequency */
> -
> - int last_adj;
> -
> - struct {
> - struct mutex mutex;
> -
> - enum { LOW_POWER, BETWEEN, HIGH_POWER } mode;
> - unsigned int interactive;
> -
> - u8 up_threshold; /* Current %busy required to uplock */
> - u8 down_threshold; /* Current %busy required to downclock */
> - } power;
> -
> - bool enabled;
> - atomic_t num_waiters;
> - atomic_t boosts;
> -
> - /* manual wa residency calculations */
> - struct intel_rps_ei ei;
> -};
> -
> -struct intel_gen6_power_mgmt {
> - struct intel_rps rps;
> -};
> -
> -/* defined intel_pm.c */
> -extern spinlock_t mchdev_lock;
> -
> -struct intel_ilk_power_mgmt {
> - u8 cur_delay;
> - u8 min_delay;
> - u8 max_delay;
> - u8 fmax;
> - u8 fstart;
> -
> - u64 last_count1;
> - unsigned long last_time1;
> - unsigned long chipset_power;
> - u64 last_count2;
> - u64 last_time2;
> - unsigned long gfx_power;
> - u8 corr;
> -
> - int c_m;
> - int r_t;
> -};
> -
> #define MAX_L3_SLICES 2
> struct intel_l3_parity {
> u32 *remap_info[MAX_L3_SLICES];
> @@ -1067,7 +979,6 @@ struct drm_i915_private {
> u32 irq_mask;
> u32 de_irq_mask[I915_MAX_PIPES];
> };
> - u32 pm_rps_events;
> u32 pipestat_irq_mask[I915_MAX_PIPES];
>
> struct i915_hotplug hotplug;
> @@ -1202,13 +1113,6 @@ struct drm_i915_private {
> */
> u32 edram_size_mb;
>
> - /* gen6+ GT PM state */
> - struct intel_gen6_power_mgmt gt_pm;
> -
> - /* ilk-only ips/rps state. Everything in here is protected by the global
> - * mchdev_lock in intel_pm.c */
> - struct intel_ilk_power_mgmt ips;
> -
> struct i915_power_domains power_domains;
>
> struct i915_psr psr;
> diff --git a/drivers/gpu/drm/i915/i915_gem.c b/drivers/gpu/drm/i915/i915_gem.c
> index b882988056bd..eab1709d1897 100644
> --- a/drivers/gpu/drm/i915/i915_gem.c
> +++ b/drivers/gpu/drm/i915/i915_gem.c
> @@ -52,6 +52,7 @@
> #include "gt/intel_mocs.h"
> #include "gt/intel_reset.h"
> #include "gt/intel_renderstate.h"
> +#include "gt/intel_rps.h"
> #include "gt/intel_workarounds.h"
>
> #include "i915_drv.h"
> @@ -1272,8 +1273,6 @@ int i915_gem_init(struct drm_i915_private *dev_priv)
> goto err_context;
> }
>
> - intel_init_gt_powersave(dev_priv);
> -
> intel_uc_init(&dev_priv->gt.uc);
>
> ret = intel_gt_init_hw(&dev_priv->gt);
> diff --git a/drivers/gpu/drm/i915/i915_irq.c b/drivers/gpu/drm/i915/i915_irq.c
> index 572a5c37cc61..598924b3c556 100644
> --- a/drivers/gpu/drm/i915/i915_irq.c
> +++ b/drivers/gpu/drm/i915/i915_irq.c
> @@ -45,6 +45,7 @@
> #include "gt/intel_gt.h"
> #include "gt/intel_gt_irq.h"
> #include "gt/intel_gt_pm_irq.h"
> +#include "gt/intel_rps.h"
>
> #include "i915_drv.h"
> #include "i915_irq.h"
> @@ -327,87 +328,6 @@ static i915_reg_t gen6_pm_iir(struct drm_i915_private *dev_priv)
> return INTEL_GEN(dev_priv) >= 8 ? GEN8_GT_IIR(2) : GEN6_PMIIR;
> }
>
> -void gen11_reset_rps_interrupts(struct drm_i915_private *dev_priv)
> -{
> - struct intel_gt *gt = &dev_priv->gt;
> -
> - spin_lock_irq(>->irq_lock);
> -
> - while (gen11_gt_reset_one_iir(gt, 0, GEN11_GTPM))
> - ;
> -
> - dev_priv->gt_pm.rps.pm_iir = 0;
> -
> - spin_unlock_irq(>->irq_lock);
> -}
> -
> -void gen6_reset_rps_interrupts(struct drm_i915_private *dev_priv)
> -{
> - struct intel_gt *gt = &dev_priv->gt;
> -
> - spin_lock_irq(>->irq_lock);
> - gen6_gt_pm_reset_iir(gt, GEN6_PM_RPS_EVENTS);
> - dev_priv->gt_pm.rps.pm_iir = 0;
> - spin_unlock_irq(>->irq_lock);
> -}
> -
> -void gen6_enable_rps_interrupts(struct drm_i915_private *dev_priv)
> -{
> - struct intel_gt *gt = &dev_priv->gt;
> - struct intel_rps *rps = &dev_priv->gt_pm.rps;
> -
> - if (READ_ONCE(rps->interrupts_enabled))
> - return;
> -
> - spin_lock_irq(>->irq_lock);
> - WARN_ON_ONCE(rps->pm_iir);
> -
> - if (INTEL_GEN(dev_priv) >= 11)
> - WARN_ON_ONCE(gen11_gt_reset_one_iir(gt, 0, GEN11_GTPM));
> - else
> - WARN_ON_ONCE(I915_READ(gen6_pm_iir(dev_priv)) & dev_priv->pm_rps_events);
> -
> - rps->interrupts_enabled = true;
> - gen6_gt_pm_enable_irq(gt, dev_priv->pm_rps_events);
> -
> - spin_unlock_irq(>->irq_lock);
> -}
> -
> -u32 gen6_sanitize_rps_pm_mask(const struct drm_i915_private *i915, u32 mask)
> -{
> - return mask & ~i915->gt_pm.rps.pm_intrmsk_mbz;
> -}
> -
> -void gen6_disable_rps_interrupts(struct drm_i915_private *dev_priv)
> -{
> - struct intel_rps *rps = &dev_priv->gt_pm.rps;
> - struct intel_gt *gt = &dev_priv->gt;
> -
> - if (!READ_ONCE(rps->interrupts_enabled))
> - return;
> -
> - spin_lock_irq(>->irq_lock);
> - rps->interrupts_enabled = false;
> -
> - I915_WRITE(GEN6_PMINTRMSK, gen6_sanitize_rps_pm_mask(dev_priv, ~0u));
> -
> - gen6_gt_pm_disable_irq(gt, GEN6_PM_RPS_EVENTS);
> -
> - spin_unlock_irq(>->irq_lock);
> - intel_synchronize_irq(dev_priv);
> -
> - /* Now that we will not be generating any more work, flush any
> - * outstanding tasks. As we are called on the RPS idle path,
> - * we will reset the GPU to minimum frequencies, so the current
> - * state of the worker can be discarded.
> - */
> - cancel_work_sync(&rps->work);
> - if (INTEL_GEN(dev_priv) >= 11)
> - gen11_reset_rps_interrupts(dev_priv);
> - else
> - gen6_reset_rps_interrupts(dev_priv);
> -}
> -
> void gen9_reset_guc_interrupts(struct intel_guc *guc)
> {
> struct intel_gt *gt = guc_to_gt(guc);
> @@ -1065,199 +985,6 @@ int intel_get_crtc_scanline(struct intel_crtc *crtc)
> return position;
> }
>
> -static void ironlake_rps_change_irq_handler(struct drm_i915_private *dev_priv)
> -{
> - struct intel_uncore *uncore = &dev_priv->uncore;
> - u32 busy_up, busy_down, max_avg, min_avg;
> - u8 new_delay;
> -
> - spin_lock(&mchdev_lock);
> -
> - intel_uncore_write16(uncore,
> - MEMINTRSTS,
> - intel_uncore_read(uncore, MEMINTRSTS));
> -
> - new_delay = dev_priv->ips.cur_delay;
> -
> - intel_uncore_write16(uncore, MEMINTRSTS, MEMINT_EVAL_CHG);
> - busy_up = intel_uncore_read(uncore, RCPREVBSYTUPAVG);
> - busy_down = intel_uncore_read(uncore, RCPREVBSYTDNAVG);
> - max_avg = intel_uncore_read(uncore, RCBMAXAVG);
> - min_avg = intel_uncore_read(uncore, RCBMINAVG);
> -
> - /* Handle RCS change request from hw */
> - if (busy_up > max_avg) {
> - if (dev_priv->ips.cur_delay != dev_priv->ips.max_delay)
> - new_delay = dev_priv->ips.cur_delay - 1;
> - if (new_delay < dev_priv->ips.max_delay)
> - new_delay = dev_priv->ips.max_delay;
> - } else if (busy_down < min_avg) {
> - if (dev_priv->ips.cur_delay != dev_priv->ips.min_delay)
> - new_delay = dev_priv->ips.cur_delay + 1;
> - if (new_delay > dev_priv->ips.min_delay)
> - new_delay = dev_priv->ips.min_delay;
> - }
> -
> - if (ironlake_set_drps(dev_priv, new_delay))
> - dev_priv->ips.cur_delay = new_delay;
> -
> - spin_unlock(&mchdev_lock);
> -
> - return;
> -}
> -
> -static void vlv_c0_read(struct drm_i915_private *dev_priv,
> - struct intel_rps_ei *ei)
> -{
> - ei->ktime = ktime_get_raw();
> - ei->render_c0 = I915_READ(VLV_RENDER_C0_COUNT);
> - ei->media_c0 = I915_READ(VLV_MEDIA_C0_COUNT);
> -}
> -
> -void gen6_rps_reset_ei(struct drm_i915_private *dev_priv)
> -{
> - memset(&dev_priv->gt_pm.rps.ei, 0, sizeof(dev_priv->gt_pm.rps.ei));
> -}
> -
> -static u32 vlv_wa_c0_ei(struct drm_i915_private *dev_priv, u32 pm_iir)
> -{
> - struct intel_rps *rps = &dev_priv->gt_pm.rps;
> - const struct intel_rps_ei *prev = &rps->ei;
> - struct intel_rps_ei now;
> - u32 events = 0;
> -
> - if ((pm_iir & GEN6_PM_RP_UP_EI_EXPIRED) == 0)
> - return 0;
> -
> - vlv_c0_read(dev_priv, &now);
> -
> - if (prev->ktime) {
> - u64 time, c0;
> - u32 render, media;
> -
> - time = ktime_us_delta(now.ktime, prev->ktime);
> -
> - time *= dev_priv->czclk_freq;
> -
> - /* Workload can be split between render + media,
> - * e.g. SwapBuffers being blitted in X after being rendered in
> - * mesa. To account for this we need to combine both engines
> - * into our activity counter.
> - */
> - render = now.render_c0 - prev->render_c0;
> - media = now.media_c0 - prev->media_c0;
> - c0 = max(render, media);
> - c0 *= 1000 * 100 << 8; /* to usecs and scale to threshold% */
> -
> - if (c0 > time * rps->power.up_threshold)
> - events = GEN6_PM_RP_UP_THRESHOLD;
> - else if (c0 < time * rps->power.down_threshold)
> - events = GEN6_PM_RP_DOWN_THRESHOLD;
> - }
> -
> - rps->ei = now;
> - return events;
> -}
> -
> -static void gen6_pm_rps_work(struct work_struct *work)
> -{
> - struct drm_i915_private *dev_priv =
> - container_of(work, struct drm_i915_private, gt_pm.rps.work);
> - struct intel_gt *gt = &dev_priv->gt;
> - struct intel_rps *rps = &dev_priv->gt_pm.rps;
> - bool client_boost = false;
> - int new_delay, adj, min, max;
> - u32 pm_iir = 0;
> -
> - spin_lock_irq(>->irq_lock);
> - if (rps->interrupts_enabled) {
> - pm_iir = fetch_and_zero(&rps->pm_iir);
> - client_boost = atomic_read(&rps->num_waiters);
> - }
> - spin_unlock_irq(>->irq_lock);
> -
> - /* Make sure we didn't queue anything we're not going to process. */
> - WARN_ON(pm_iir & ~dev_priv->pm_rps_events);
> - if ((pm_iir & dev_priv->pm_rps_events) == 0 && !client_boost)
> - goto out;
> -
> - mutex_lock(&rps->lock);
> -
> - pm_iir |= vlv_wa_c0_ei(dev_priv, pm_iir);
> -
> - adj = rps->last_adj;
> - new_delay = rps->cur_freq;
> - min = rps->min_freq_softlimit;
> - max = rps->max_freq_softlimit;
> - if (client_boost)
> - max = rps->max_freq;
> - if (client_boost && new_delay < rps->boost_freq) {
> - new_delay = rps->boost_freq;
> - adj = 0;
> - } else if (pm_iir & GEN6_PM_RP_UP_THRESHOLD) {
> - if (adj > 0)
> - adj *= 2;
> - else /* CHV needs even encode values */
> - adj = IS_CHERRYVIEW(dev_priv) ? 2 : 1;
> -
> - if (new_delay >= rps->max_freq_softlimit)
> - adj = 0;
> - } else if (client_boost) {
> - adj = 0;
> - } else if (pm_iir & GEN6_PM_RP_DOWN_TIMEOUT) {
> - if (rps->cur_freq > rps->efficient_freq)
> - new_delay = rps->efficient_freq;
> - else if (rps->cur_freq > rps->min_freq_softlimit)
> - new_delay = rps->min_freq_softlimit;
> - adj = 0;
> - } else if (pm_iir & GEN6_PM_RP_DOWN_THRESHOLD) {
> - if (adj < 0)
> - adj *= 2;
> - else /* CHV needs even encode values */
> - adj = IS_CHERRYVIEW(dev_priv) ? -2 : -1;
> -
> - if (new_delay <= rps->min_freq_softlimit)
> - adj = 0;
> - } else { /* unknown event */
> - adj = 0;
> - }
> -
> - rps->last_adj = adj;
> -
> - /*
> - * Limit deboosting and boosting to keep ourselves at the extremes
> - * when in the respective power modes (i.e. slowly decrease frequencies
> - * while in the HIGH_POWER zone and slowly increase frequencies while
> - * in the LOW_POWER zone). On idle, we will hit the timeout and drop
> - * to the next level quickly, and conversely if busy we expect to
> - * hit a waitboost and rapidly switch into max power.
> - */
> - if ((adj < 0 && rps->power.mode == HIGH_POWER) ||
> - (adj > 0 && rps->power.mode == LOW_POWER))
> - rps->last_adj = 0;
> -
> - /* sysfs frequency interfaces may have snuck in while servicing the
> - * interrupt
> - */
> - new_delay += adj;
> - new_delay = clamp_t(int, new_delay, min, max);
> -
> - if (intel_set_rps(dev_priv, new_delay)) {
> - DRM_DEBUG_DRIVER("Failed to set new GPU frequency\n");
> - rps->last_adj = 0;
> - }
> -
> - mutex_unlock(&rps->lock);
> -
> -out:
> - /* Make sure not to corrupt PMIMR state used by ringbuffer on GEN6 */
> - spin_lock_irq(>->irq_lock);
> - if (rps->interrupts_enabled)
> - gen6_gt_pm_unmask_irq(gt, dev_priv->pm_rps_events);
> - spin_unlock_irq(>->irq_lock);
> -}
> -
> -
> /**
> * ivybridge_parity_work - Workqueue called when a parity error interrupt
> * occurred.
> @@ -1631,54 +1358,6 @@ static void i9xx_pipe_crc_irq_handler(struct drm_i915_private *dev_priv,
> res1, res2);
> }
>
> -/* The RPS events need forcewake, so we add them to a work queue and mask their
> - * IMR bits until the work is done. Other interrupts can be processed without
> - * the work queue. */
> -void gen11_rps_irq_handler(struct intel_gt *gt, u32 pm_iir)
> -{
> - struct drm_i915_private *i915 = gt->i915;
> - struct intel_rps *rps = &i915->gt_pm.rps;
> - const u32 events = i915->pm_rps_events & pm_iir;
> -
> - lockdep_assert_held(>->irq_lock);
> -
> - if (unlikely(!events))
> - return;
> -
> - gen6_gt_pm_mask_irq(gt, events);
> -
> - if (!rps->interrupts_enabled)
> - return;
> -
> - rps->pm_iir |= events;
> - schedule_work(&rps->work);
> -}
> -
> -void gen6_rps_irq_handler(struct drm_i915_private *dev_priv, u32 pm_iir)
> -{
> - struct intel_rps *rps = &dev_priv->gt_pm.rps;
> - struct intel_gt *gt = &dev_priv->gt;
> -
> - if (pm_iir & dev_priv->pm_rps_events) {
> - spin_lock(>->irq_lock);
> - gen6_gt_pm_mask_irq(gt, pm_iir & dev_priv->pm_rps_events);
> - if (rps->interrupts_enabled) {
> - rps->pm_iir |= pm_iir & dev_priv->pm_rps_events;
> - schedule_work(&rps->work);
> - }
> - spin_unlock(>->irq_lock);
> - }
> -
> - if (INTEL_GEN(dev_priv) >= 8)
> - return;
> -
> - if (pm_iir & PM_VEBOX_USER_INTERRUPT)
> - intel_engine_breadcrumbs_irq(dev_priv->engine[VECS0]);
> -
> - if (pm_iir & PM_VEBOX_CS_ERROR_INTERRUPT)
> - DRM_DEBUG("Command parser error, pm_iir 0x%08x\n", pm_iir);
> -}
> -
> static void i9xx_pipestat_irq_reset(struct drm_i915_private *dev_priv)
> {
> enum pipe pipe;
> @@ -1989,7 +1668,7 @@ static irqreturn_t valleyview_irq_handler(int irq, void *arg)
> if (gt_iir)
> gen6_gt_irq_handler(&dev_priv->gt, gt_iir);
> if (pm_iir)
> - gen6_rps_irq_handler(dev_priv, pm_iir);
> + gen6_rps_irq_handler(&dev_priv->gt.rps, pm_iir);
>
> if (hotplug_status)
> i9xx_hpd_irq_handler(dev_priv, hotplug_status);
> @@ -2393,7 +2072,7 @@ static void ilk_display_irq_handler(struct drm_i915_private *dev_priv,
> }
>
> if (IS_GEN(dev_priv, 5) && de_iir & DE_PCU_EVENT)
> - ironlake_rps_change_irq_handler(dev_priv);
> + gen5_rps_irq_handler(&dev_priv->gt.rps);
> }
>
> static void ivb_display_irq_handler(struct drm_i915_private *dev_priv,
> @@ -2498,7 +2177,7 @@ static irqreturn_t ironlake_irq_handler(int irq, void *arg)
> if (pm_iir) {
> I915_WRITE(GEN6_PMIIR, pm_iir);
> ret = IRQ_HANDLED;
> - gen6_rps_irq_handler(dev_priv, pm_iir);
> + gen6_rps_irq_handler(&dev_priv->gt.rps, pm_iir);
> }
> }
>
> @@ -4270,13 +3949,10 @@ static irqreturn_t i965_irq_handler(int irq, void *arg)
> void intel_irq_init(struct drm_i915_private *dev_priv)
> {
> struct drm_device *dev = &dev_priv->drm;
> - struct intel_rps *rps = &dev_priv->gt_pm.rps;
> int i;
>
> intel_hpd_init_work(dev_priv);
>
> - INIT_WORK(&rps->work, gen6_pm_rps_work);
> -
> INIT_WORK(&dev_priv->l3_parity.error_work, ivybridge_parity_work);
> for (i = 0; i < MAX_L3_SLICES; ++i)
> dev_priv->l3_parity.remap_info[i] = NULL;
> @@ -4285,33 +3961,6 @@ void intel_irq_init(struct drm_i915_private *dev_priv)
> if (HAS_GT_UC(dev_priv) && INTEL_GEN(dev_priv) < 11)
> dev_priv->gt.pm_guc_events = GUC_INTR_GUC2HOST << 16;
>
> - /* Let's track the enabled rps events */
> - if (IS_VALLEYVIEW(dev_priv))
> - /* WaGsvRC0ResidencyMethod:vlv */
> - dev_priv->pm_rps_events = GEN6_PM_RP_UP_EI_EXPIRED;
> - else
> - dev_priv->pm_rps_events = (GEN6_PM_RP_UP_THRESHOLD |
> - GEN6_PM_RP_DOWN_THRESHOLD |
> - GEN6_PM_RP_DOWN_TIMEOUT);
> -
> - /* We share the register with other engine */
> - if (INTEL_GEN(dev_priv) > 9)
> - GEM_WARN_ON(dev_priv->pm_rps_events & 0xffff0000);
> -
> - rps->pm_intrmsk_mbz = 0;
> -
> - /*
> - * SNB,IVB,HSW can while VLV,CHV may hard hang on looping batchbuffer
> - * if GEN6_PM_UP_EI_EXPIRED is masked.
> - *
> - * TODO: verify if this can be reproduced on VLV,CHV.
> - */
> - if (INTEL_GEN(dev_priv) <= 7)
> - rps->pm_intrmsk_mbz |= GEN6_PM_RP_UP_EI_EXPIRED;
> -
> - if (INTEL_GEN(dev_priv) >= 8)
> - rps->pm_intrmsk_mbz |= GEN8_PMINTR_DISABLE_REDIRECT_TO_GUC;
> -
> dev->vblank_disable_immediate = true;
>
> /* Most platforms treat the display irq block as an always-on
> diff --git a/drivers/gpu/drm/i915/i915_irq.h b/drivers/gpu/drm/i915/i915_irq.h
> index 19a3bc019535..d0d91c6e00d7 100644
> --- a/drivers/gpu/drm/i915/i915_irq.h
> +++ b/drivers/gpu/drm/i915/i915_irq.h
> @@ -22,9 +22,6 @@ struct intel_gt;
> struct intel_guc;
> struct intel_uncore;
>
> -void gen11_rps_irq_handler(struct intel_gt *gt, u32 pm_iir);
> -void gen6_rps_irq_handler(struct drm_i915_private *dev_priv, u32 pm_iir);
> -
> void intel_irq_init(struct drm_i915_private *dev_priv);
> void intel_irq_fini(struct drm_i915_private *dev_priv);
> int intel_irq_install(struct drm_i915_private *dev_priv);
> diff --git a/drivers/gpu/drm/i915/i915_pmu.c b/drivers/gpu/drm/i915/i915_pmu.c
> index 85912917c062..266d66b1fb7b 100644
> --- a/drivers/gpu/drm/i915/i915_pmu.c
> +++ b/drivers/gpu/drm/i915/i915_pmu.c
> @@ -12,6 +12,7 @@
> #include "gt/intel_engine_user.h"
> #include "gt/intel_gt_pm.h"
> #include "gt/intel_rc6.h"
> +#include "gt/intel_rps.h"
>
> #include "i915_drv.h"
> #include "i915_pmu.h"
> @@ -358,25 +359,26 @@ frequency_sample(struct intel_gt *gt, unsigned int period_ns)
> struct drm_i915_private *i915 = gt->i915;
> struct intel_uncore *uncore = gt->uncore;
> struct i915_pmu *pmu = &i915->pmu;
> + struct intel_rps *rps = >->rps;
>
> if (pmu->enable & config_enabled_mask(I915_PMU_ACTUAL_FREQUENCY)) {
> u32 val;
>
> - val = i915->gt_pm.rps.cur_freq;
> + val = rps->cur_freq;
> if (intel_gt_pm_get_if_awake(gt)) {
> val = intel_uncore_read_notrace(uncore, GEN6_RPSTAT1);
> - val = intel_get_cagf(i915, val);
> + val = intel_get_cagf(rps, val);
> intel_gt_pm_put(gt);
> }
>
> add_sample_mult(&pmu->sample[__I915_SAMPLE_FREQ_ACT],
> - intel_gpu_freq(i915, val),
> + intel_gpu_freq(rps, val),
> period_ns / 1000);
> }
>
> if (pmu->enable & config_enabled_mask(I915_PMU_REQUESTED_FREQUENCY)) {
> add_sample_mult(&pmu->sample[__I915_SAMPLE_FREQ_REQ],
> - intel_gpu_freq(i915, i915->gt_pm.rps.cur_freq),
> + intel_gpu_freq(rps, rps->cur_freq),
> period_ns / 1000);
> }
> }
> diff --git a/drivers/gpu/drm/i915/i915_request.c b/drivers/gpu/drm/i915/i915_request.c
> index 4575f368455d..08ce2eeecf7e 100644
> --- a/drivers/gpu/drm/i915/i915_request.c
> +++ b/drivers/gpu/drm/i915/i915_request.c
> @@ -31,6 +31,7 @@
>
> #include "gem/i915_gem_context.h"
> #include "gt/intel_context.h"
> +#include "gt/intel_rps.h"
>
> #include "i915_active.h"
> #include "i915_drv.h"
> @@ -257,8 +258,8 @@ bool i915_request_retire(struct i915_request *rq)
> if (test_bit(DMA_FENCE_FLAG_ENABLE_SIGNAL_BIT, &rq->fence.flags))
> i915_request_cancel_breadcrumb(rq);
> if (i915_request_has_waitboost(rq)) {
> - GEM_BUG_ON(!atomic_read(&rq->i915->gt_pm.rps.num_waiters));
> - atomic_dec(&rq->i915->gt_pm.rps.num_waiters);
> + GEM_BUG_ON(!atomic_read(&rq->engine->gt->rps.num_waiters));
> + atomic_dec(&rq->engine->gt->rps.num_waiters);
> }
> if (!test_bit(I915_FENCE_FLAG_ACTIVE, &rq->fence.flags)) {
> set_bit(I915_FENCE_FLAG_ACTIVE, &rq->fence.flags);
> @@ -1466,7 +1467,7 @@ long i915_request_wait(struct i915_request *rq,
> */
> if (flags & I915_WAIT_PRIORITY) {
> if (!i915_request_started(rq) && INTEL_GEN(rq->i915) >= 6)
> - gen6_rps_boost(rq);
> + intel_rps_boost(rq);
> i915_schedule_bump_priority(rq, I915_PRIORITY_WAIT);
> }
>
> diff --git a/drivers/gpu/drm/i915/i915_sysfs.c b/drivers/gpu/drm/i915/i915_sysfs.c
> index bf039b8ba593..65476909d1bf 100644
> --- a/drivers/gpu/drm/i915/i915_sysfs.c
> +++ b/drivers/gpu/drm/i915/i915_sysfs.c
> @@ -31,6 +31,7 @@
> #include <linux/sysfs.h>
>
> #include "gt/intel_rc6.h"
> +#include "gt/intel_rps.h"
>
> #include "i915_drv.h"
> #include "i915_sysfs.h"
> @@ -259,6 +260,7 @@ static ssize_t gt_act_freq_mhz_show(struct device *kdev,
> struct device_attribute *attr, char *buf)
> {
> struct drm_i915_private *dev_priv = kdev_minor_to_i915(kdev);
> + struct intel_rps *rps = &dev_priv->gt.rps;
> intel_wakeref_t wakeref;
> u32 freq;
>
> @@ -271,31 +273,31 @@ static ssize_t gt_act_freq_mhz_show(struct device *kdev,
>
> freq = (freq >> 8) & 0xff;
> } else {
> - freq = intel_get_cagf(dev_priv, I915_READ(GEN6_RPSTAT1));
> + freq = intel_get_cagf(rps, I915_READ(GEN6_RPSTAT1));
> }
>
> intel_runtime_pm_put(&dev_priv->runtime_pm, wakeref);
>
> - return snprintf(buf, PAGE_SIZE, "%d\n", intel_gpu_freq(dev_priv, freq));
> + return snprintf(buf, PAGE_SIZE, "%d\n", intel_gpu_freq(rps, freq));
> }
>
> static ssize_t gt_cur_freq_mhz_show(struct device *kdev,
> struct device_attribute *attr, char *buf)
> {
> struct drm_i915_private *dev_priv = kdev_minor_to_i915(kdev);
> + struct intel_rps *rps = &dev_priv->gt.rps;
>
> return snprintf(buf, PAGE_SIZE, "%d\n",
> - intel_gpu_freq(dev_priv,
> - dev_priv->gt_pm.rps.cur_freq));
> + intel_gpu_freq(rps, rps->cur_freq));
> }
>
> static ssize_t gt_boost_freq_mhz_show(struct device *kdev, struct device_attribute *attr, char *buf)
> {
> struct drm_i915_private *dev_priv = kdev_minor_to_i915(kdev);
> + struct intel_rps *rps = &dev_priv->gt.rps;
>
> return snprintf(buf, PAGE_SIZE, "%d\n",
> - intel_gpu_freq(dev_priv,
> - dev_priv->gt_pm.rps.boost_freq));
> + intel_gpu_freq(rps, rps->boost_freq));
> }
>
> static ssize_t gt_boost_freq_mhz_store(struct device *kdev,
> @@ -303,7 +305,7 @@ static ssize_t gt_boost_freq_mhz_store(struct device *kdev,
> const char *buf, size_t count)
> {
> struct drm_i915_private *dev_priv = kdev_minor_to_i915(kdev);
> - struct intel_rps *rps = &dev_priv->gt_pm.rps;
> + struct intel_rps *rps = &dev_priv->gt.rps;
> bool boost = false;
> ssize_t ret;
> u32 val;
> @@ -313,7 +315,7 @@ static ssize_t gt_boost_freq_mhz_store(struct device *kdev,
> return ret;
>
> /* Validate against (static) hardware limits */
> - val = intel_freq_opcode(dev_priv, val);
> + val = intel_freq_opcode(rps, val);
> if (val < rps->min_freq || val > rps->max_freq)
> return -EINVAL;
>
> @@ -333,19 +335,19 @@ static ssize_t vlv_rpe_freq_mhz_show(struct device *kdev,
> struct device_attribute *attr, char *buf)
> {
> struct drm_i915_private *dev_priv = kdev_minor_to_i915(kdev);
> + struct intel_rps *rps = &dev_priv->gt.rps;
>
> return snprintf(buf, PAGE_SIZE, "%d\n",
> - intel_gpu_freq(dev_priv,
> - dev_priv->gt_pm.rps.efficient_freq));
> + intel_gpu_freq(rps, rps->efficient_freq));
> }
>
> static ssize_t gt_max_freq_mhz_show(struct device *kdev, struct device_attribute *attr, char *buf)
> {
> struct drm_i915_private *dev_priv = kdev_minor_to_i915(kdev);
> + struct intel_rps *rps = &dev_priv->gt.rps;
>
> return snprintf(buf, PAGE_SIZE, "%d\n",
> - intel_gpu_freq(dev_priv,
> - dev_priv->gt_pm.rps.max_freq_softlimit));
> + intel_gpu_freq(rps, rps->max_freq_softlimit));
> }
>
> static ssize_t gt_max_freq_mhz_store(struct device *kdev,
> @@ -353,19 +355,17 @@ static ssize_t gt_max_freq_mhz_store(struct device *kdev,
> const char *buf, size_t count)
> {
> struct drm_i915_private *dev_priv = kdev_minor_to_i915(kdev);
> - struct intel_rps *rps = &dev_priv->gt_pm.rps;
> - intel_wakeref_t wakeref;
> - u32 val;
> + struct intel_rps *rps = &dev_priv->gt.rps;
> ssize_t ret;
> + u32 val;
>
> ret = kstrtou32(buf, 0, &val);
> if (ret)
> return ret;
>
> - wakeref = intel_runtime_pm_get(&dev_priv->runtime_pm);
> mutex_lock(&rps->lock);
>
> - val = intel_freq_opcode(dev_priv, val);
> + val = intel_freq_opcode(rps, val);
> if (val < rps->min_freq ||
> val > rps->max_freq ||
> val < rps->min_freq_softlimit) {
> @@ -375,7 +375,7 @@ static ssize_t gt_max_freq_mhz_store(struct device *kdev,
>
> if (val > rps->rp0_freq)
> DRM_DEBUG("User requested overclocking to %d\n",
> - intel_gpu_freq(dev_priv, val));
> + intel_gpu_freq(rps, val));
>
> rps->max_freq_softlimit = val;
>
> @@ -383,14 +383,15 @@ static ssize_t gt_max_freq_mhz_store(struct device *kdev,
> rps->min_freq_softlimit,
> rps->max_freq_softlimit);
>
> - /* We still need *_set_rps to process the new max_delay and
> + /*
> + * We still need *_set_rps to process the new max_delay and
> * update the interrupt limits and PMINTRMSK even though
> - * frequency request may be unchanged. */
> - ret = intel_set_rps(dev_priv, val);
> + * frequency request may be unchanged.
> + */
> + intel_rps_set(rps, val);
>
> unlock:
> mutex_unlock(&rps->lock);
> - intel_runtime_pm_put(&dev_priv->runtime_pm, wakeref);
>
> return ret ?: count;
> }
> @@ -398,10 +399,10 @@ static ssize_t gt_max_freq_mhz_store(struct device *kdev,
> static ssize_t gt_min_freq_mhz_show(struct device *kdev, struct device_attribute *attr, char *buf)
> {
> struct drm_i915_private *dev_priv = kdev_minor_to_i915(kdev);
> + struct intel_rps *rps = &dev_priv->gt.rps;
>
> return snprintf(buf, PAGE_SIZE, "%d\n",
> - intel_gpu_freq(dev_priv,
> - dev_priv->gt_pm.rps.min_freq_softlimit));
> + intel_gpu_freq(rps, rps->min_freq_softlimit));
> }
>
> static ssize_t gt_min_freq_mhz_store(struct device *kdev,
> @@ -409,19 +410,17 @@ static ssize_t gt_min_freq_mhz_store(struct device *kdev,
> const char *buf, size_t count)
> {
> struct drm_i915_private *dev_priv = kdev_minor_to_i915(kdev);
> - struct intel_rps *rps = &dev_priv->gt_pm.rps;
> - intel_wakeref_t wakeref;
> - u32 val;
> + struct intel_rps *rps = &dev_priv->gt.rps;
> ssize_t ret;
> + u32 val;
>
> ret = kstrtou32(buf, 0, &val);
> if (ret)
> return ret;
>
> - wakeref = intel_runtime_pm_get(&dev_priv->runtime_pm);
> mutex_lock(&rps->lock);
>
> - val = intel_freq_opcode(dev_priv, val);
> + val = intel_freq_opcode(rps, val);
> if (val < rps->min_freq ||
> val > rps->max_freq ||
> val > rps->max_freq_softlimit) {
> @@ -435,14 +434,15 @@ static ssize_t gt_min_freq_mhz_store(struct device *kdev,
> rps->min_freq_softlimit,
> rps->max_freq_softlimit);
>
> - /* We still need *_set_rps to process the new min_delay and
> + /*
> + * We still need *_set_rps to process the new min_delay and
> * update the interrupt limits and PMINTRMSK even though
> - * frequency request may be unchanged. */
> - ret = intel_set_rps(dev_priv, val);
> + * frequency request may be unchanged.
> + */
> + intel_rps_set(rps, val);
>
> unlock:
> mutex_unlock(&rps->lock);
> - intel_runtime_pm_put(&dev_priv->runtime_pm, wakeref);
>
> return ret ?: count;
> }
> @@ -464,15 +464,15 @@ static DEVICE_ATTR(gt_RPn_freq_mhz, S_IRUGO, gt_rp_mhz_show, NULL);
> static ssize_t gt_rp_mhz_show(struct device *kdev, struct device_attribute *attr, char *buf)
> {
> struct drm_i915_private *dev_priv = kdev_minor_to_i915(kdev);
> - struct intel_rps *rps = &dev_priv->gt_pm.rps;
> + struct intel_rps *rps = &dev_priv->gt.rps;
> u32 val;
>
> if (attr == &dev_attr_gt_RP0_freq_mhz)
> - val = intel_gpu_freq(dev_priv, rps->rp0_freq);
> + val = intel_gpu_freq(rps, rps->rp0_freq);
> else if (attr == &dev_attr_gt_RP1_freq_mhz)
> - val = intel_gpu_freq(dev_priv, rps->rp1_freq);
> + val = intel_gpu_freq(rps, rps->rp1_freq);
> else if (attr == &dev_attr_gt_RPn_freq_mhz)
> - val = intel_gpu_freq(dev_priv, rps->min_freq);
> + val = intel_gpu_freq(rps, rps->min_freq);
> else
> BUG();
>
> diff --git a/drivers/gpu/drm/i915/intel_pm.c b/drivers/gpu/drm/i915/intel_pm.c
> index 362234449087..6741507c74f3 100644
> --- a/drivers/gpu/drm/i915/intel_pm.c
> +++ b/drivers/gpu/drm/i915/intel_pm.c
> @@ -197,8 +197,6 @@ static void i915_ironlake_get_mem_freq(struct drm_i915_private *dev_priv)
> break;
> }
>
> - dev_priv->ips.r_t = dev_priv->mem_freq;
> -
> switch (csipll & 0x3ff) {
> case 0x00c:
> dev_priv->fsb_freq = 3200;
> @@ -227,14 +225,6 @@ static void i915_ironlake_get_mem_freq(struct drm_i915_private *dev_priv)
> dev_priv->fsb_freq = 0;
> break;
> }
> -
> - if (dev_priv->fsb_freq == 3200) {
> - dev_priv->ips.c_m = 0;
> - } else if (dev_priv->fsb_freq > 3200 && dev_priv->fsb_freq <= 4800) {
> - dev_priv->ips.c_m = 1;
> - } else {
> - dev_priv->ips.c_m = 2;
> - }
> }
>
> static const struct cxsr_latency cxsr_latency_table[] = {
> @@ -6339,1865 +6329,258 @@ void intel_init_ipc(struct drm_i915_private *dev_priv)
> intel_enable_ipc(dev_priv);
> }
>
> -/*
> - * Lock protecting IPS related data structures
> - */
> -DEFINE_SPINLOCK(mchdev_lock);
> +static const struct cparams {
> + u16 i;
> + u16 t;
> + u16 m;
> + u16 c;
> +} cparams[] = {
> + { 1, 1333, 301, 28664 },
> + { 1, 1066, 294, 24460 },
> + { 1, 800, 294, 25192 },
> + { 0, 1333, 276, 27605 },
> + { 0, 1066, 276, 27605 },
> + { 0, 800, 231, 23784 },
> +};
>
> -bool ironlake_set_drps(struct drm_i915_private *i915, u8 val)
> +static void ibx_init_clock_gating(struct drm_i915_private *dev_priv)
> {
> - struct intel_uncore *uncore = &i915->uncore;
> - u16 rgvswctl;
> -
> - lockdep_assert_held(&mchdev_lock);
> -
> - rgvswctl = intel_uncore_read16(uncore, MEMSWCTL);
> - if (rgvswctl & MEMCTL_CMD_STS) {
> - DRM_DEBUG("gpu busy, RCS change rejected\n");
> - return false; /* still busy with another command */
> - }
> -
> - rgvswctl = (MEMCTL_CMD_CHFREQ << MEMCTL_CMD_SHIFT) |
> - (val << MEMCTL_FREQ_SHIFT) | MEMCTL_SFCAVM;
> - intel_uncore_write16(uncore, MEMSWCTL, rgvswctl);
> - intel_uncore_posting_read16(uncore, MEMSWCTL);
> -
> - rgvswctl |= MEMCTL_CMD_STS;
> - intel_uncore_write16(uncore, MEMSWCTL, rgvswctl);
> -
> - return true;
> + /*
> + * On Ibex Peak and Cougar Point, we need to disable clock
> + * gating for the panel power sequencer or it will fail to
> + * start up when no ports are active.
> + */
> + I915_WRITE(SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE);
> }
>
> -static void ironlake_enable_drps(struct drm_i915_private *dev_priv)
> +static void g4x_disable_trickle_feed(struct drm_i915_private *dev_priv)
> {
> - struct intel_uncore *uncore = &dev_priv->uncore;
> - u32 rgvmodectl;
> - u8 fmax, fmin, fstart, vstart;
> -
> - spin_lock_irq(&mchdev_lock);
> -
> - rgvmodectl = intel_uncore_read(uncore, MEMMODECTL);
> -
> - /* Enable temp reporting */
> - intel_uncore_write16(uncore, PMMISC, I915_READ(PMMISC) | MCPPCE_EN);
> - intel_uncore_write16(uncore, TSC1, I915_READ(TSC1) | TSE);
> -
> - /* 100ms RC evaluation intervals */
> - intel_uncore_write(uncore, RCUPEI, 100000);
> - intel_uncore_write(uncore, RCDNEI, 100000);
> -
> - /* Set max/min thresholds to 90ms and 80ms respectively */
> - intel_uncore_write(uncore, RCBMAXAVG, 90000);
> - intel_uncore_write(uncore, RCBMINAVG, 80000);
> -
> - intel_uncore_write(uncore, MEMIHYST, 1);
> + enum pipe pipe;
>
> - /* Set up min, max, and cur for interrupt handling */
> - fmax = (rgvmodectl & MEMMODE_FMAX_MASK) >> MEMMODE_FMAX_SHIFT;
> - fmin = (rgvmodectl & MEMMODE_FMIN_MASK);
> - fstart = (rgvmodectl & MEMMODE_FSTART_MASK) >>
> - MEMMODE_FSTART_SHIFT;
> + for_each_pipe(dev_priv, pipe) {
> + I915_WRITE(DSPCNTR(pipe),
> + I915_READ(DSPCNTR(pipe)) |
> + DISPPLANE_TRICKLE_FEED_DISABLE);
>
> - vstart = (intel_uncore_read(uncore, PXVFREQ(fstart)) &
> - PXVFREQ_PX_MASK) >> PXVFREQ_PX_SHIFT;
> + I915_WRITE(DSPSURF(pipe), I915_READ(DSPSURF(pipe)));
> + POSTING_READ(DSPSURF(pipe));
> + }
> +}
>
> - dev_priv->ips.fmax = fmax; /* IPS callback will increase this */
> - dev_priv->ips.fstart = fstart;
> +static void ilk_init_clock_gating(struct drm_i915_private *dev_priv)
> +{
> + u32 dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE;
>
> - dev_priv->ips.max_delay = fstart;
> - dev_priv->ips.min_delay = fmin;
> - dev_priv->ips.cur_delay = fstart;
> + /*
> + * Required for FBC
> + * WaFbcDisableDpfcClockGating:ilk
> + */
> + dspclk_gate |= ILK_DPFCRUNIT_CLOCK_GATE_DISABLE |
> + ILK_DPFCUNIT_CLOCK_GATE_DISABLE |
> + ILK_DPFDUNIT_CLOCK_GATE_ENABLE;
>
> - DRM_DEBUG_DRIVER("fmax: %d, fmin: %d, fstart: %d\n",
> - fmax, fmin, fstart);
> + I915_WRITE(PCH_3DCGDIS0,
> + MARIUNIT_CLOCK_GATE_DISABLE |
> + SVSMUNIT_CLOCK_GATE_DISABLE);
> + I915_WRITE(PCH_3DCGDIS1,
> + VFMUNIT_CLOCK_GATE_DISABLE);
>
> - intel_uncore_write(uncore,
> - MEMINTREN,
> - MEMINT_CX_SUPR_EN | MEMINT_EVAL_CHG_EN);
> + /*
> + * According to the spec the following bits should be set in
> + * order to enable memory self-refresh
> + * The bit 22/21 of 0x42004
> + * The bit 5 of 0x42020
> + * The bit 15 of 0x45000
> + */
> + I915_WRITE(ILK_DISPLAY_CHICKEN2,
> + (I915_READ(ILK_DISPLAY_CHICKEN2) |
> + ILK_DPARB_GATE | ILK_VSDPFD_FULL));
> + dspclk_gate |= ILK_DPARBUNIT_CLOCK_GATE_ENABLE;
> + I915_WRITE(DISP_ARB_CTL,
> + (I915_READ(DISP_ARB_CTL) |
> + DISP_FBC_WM_DIS));
>
> /*
> - * Interrupts will be enabled in ironlake_irq_postinstall
> + * Based on the document from hardware guys the following bits
> + * should be set unconditionally in order to enable FBC.
> + * The bit 22 of 0x42000
> + * The bit 22 of 0x42004
> + * The bit 7,8,9 of 0x42020.
> */
> + if (IS_IRONLAKE_M(dev_priv)) {
> + /* WaFbcAsynchFlipDisableFbcQueue:ilk */
> + I915_WRITE(ILK_DISPLAY_CHICKEN1,
> + I915_READ(ILK_DISPLAY_CHICKEN1) |
> + ILK_FBCQ_DIS);
> + I915_WRITE(ILK_DISPLAY_CHICKEN2,
> + I915_READ(ILK_DISPLAY_CHICKEN2) |
> + ILK_DPARB_GATE);
> + }
>
> - intel_uncore_write(uncore, VIDSTART, vstart);
> - intel_uncore_posting_read(uncore, VIDSTART);
> + I915_WRITE(ILK_DSPCLK_GATE_D, dspclk_gate);
>
> - rgvmodectl |= MEMMODE_SWMODE_EN;
> - intel_uncore_write(uncore, MEMMODECTL, rgvmodectl);
> + I915_WRITE(ILK_DISPLAY_CHICKEN2,
> + I915_READ(ILK_DISPLAY_CHICKEN2) |
> + ILK_ELPIN_409_SELECT);
> + I915_WRITE(_3D_CHICKEN2,
> + _3D_CHICKEN2_WM_READ_PIPELINED << 16 |
> + _3D_CHICKEN2_WM_READ_PIPELINED);
>
> - if (wait_for_atomic((intel_uncore_read(uncore, MEMSWCTL) &
> - MEMCTL_CMD_STS) == 0, 10))
> - DRM_ERROR("stuck trying to change perf mode\n");
> - mdelay(1);
> + /* WaDisableRenderCachePipelinedFlush:ilk */
> + I915_WRITE(CACHE_MODE_0,
> + _MASKED_BIT_ENABLE(CM0_PIPELINED_RENDER_FLUSH_DISABLE));
>
> - ironlake_set_drps(dev_priv, fstart);
> + /* WaDisable_RenderCache_OperationalFlush:ilk */
> + I915_WRITE(CACHE_MODE_0, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
>
> - dev_priv->ips.last_count1 =
> - intel_uncore_read(uncore, DMIEC) +
> - intel_uncore_read(uncore, DDREC) +
> - intel_uncore_read(uncore, CSIEC);
> - dev_priv->ips.last_time1 = jiffies_to_msecs(jiffies);
> - dev_priv->ips.last_count2 = intel_uncore_read(uncore, GFXEC);
> - dev_priv->ips.last_time2 = ktime_get_raw_ns();
> + g4x_disable_trickle_feed(dev_priv);
>
> - spin_unlock_irq(&mchdev_lock);
> + ibx_init_clock_gating(dev_priv);
> }
>
> -static void ironlake_disable_drps(struct drm_i915_private *i915)
> +static void cpt_init_clock_gating(struct drm_i915_private *dev_priv)
> {
> - struct intel_uncore *uncore = &i915->uncore;
> - u16 rgvswctl;
> -
> - spin_lock_irq(&mchdev_lock);
> -
> - rgvswctl = intel_uncore_read16(uncore, MEMSWCTL);
> -
> - /* Ack interrupts, disable EFC interrupt */
> - intel_uncore_write(uncore,
> - MEMINTREN,
> - intel_uncore_read(uncore, MEMINTREN) &
> - ~MEMINT_EVAL_CHG_EN);
> - intel_uncore_write(uncore, MEMINTRSTS, MEMINT_EVAL_CHG);
> - intel_uncore_write(uncore,
> - DEIER,
> - intel_uncore_read(uncore, DEIER) & ~DE_PCU_EVENT);
> - intel_uncore_write(uncore, DEIIR, DE_PCU_EVENT);
> - intel_uncore_write(uncore,
> - DEIMR,
> - intel_uncore_read(uncore, DEIMR) | DE_PCU_EVENT);
> -
> - /* Go back to the starting frequency */
> - ironlake_set_drps(i915, i915->ips.fstart);
> - mdelay(1);
> - rgvswctl |= MEMCTL_CMD_STS;
> - intel_uncore_write(uncore, MEMSWCTL, rgvswctl);
> - mdelay(1);
> + enum pipe pipe;
> + u32 val;
>
> - spin_unlock_irq(&mchdev_lock);
> + /*
> + * On Ibex Peak and Cougar Point, we need to disable clock
> + * gating for the panel power sequencer or it will fail to
> + * start up when no ports are active.
> + */
> + I915_WRITE(SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE |
> + PCH_DPLUNIT_CLOCK_GATE_DISABLE |
> + PCH_CPUNIT_CLOCK_GATE_DISABLE);
> + I915_WRITE(SOUTH_CHICKEN2, I915_READ(SOUTH_CHICKEN2) |
> + DPLS_EDP_PPS_FIX_DIS);
> + /* The below fixes the weird display corruption, a few pixels shifted
> + * downward, on (only) LVDS of some HP laptops with IVY.
> + */
> + for_each_pipe(dev_priv, pipe) {
> + val = I915_READ(TRANS_CHICKEN2(pipe));
> + val |= TRANS_CHICKEN2_TIMING_OVERRIDE;
> + val &= ~TRANS_CHICKEN2_FDI_POLARITY_REVERSED;
> + if (dev_priv->vbt.fdi_rx_polarity_inverted)
> + val |= TRANS_CHICKEN2_FDI_POLARITY_REVERSED;
> + val &= ~TRANS_CHICKEN2_FRAME_START_DELAY_MASK;
> + val &= ~TRANS_CHICKEN2_DISABLE_DEEP_COLOR_COUNTER;
> + val &= ~TRANS_CHICKEN2_DISABLE_DEEP_COLOR_MODESWITCH;
> + I915_WRITE(TRANS_CHICKEN2(pipe), val);
> + }
> + /* WADP0ClockGatingDisable */
> + for_each_pipe(dev_priv, pipe) {
> + I915_WRITE(TRANS_CHICKEN1(pipe),
> + TRANS_CHICKEN1_DP0UNIT_GC_DISABLE);
> + }
> }
>
> -/* There's a funny hw issue where the hw returns all 0 when reading from
> - * GEN6_RP_INTERRUPT_LIMITS. Hence we always need to compute the desired value
> - * ourselves, instead of doing a rmw cycle (which might result in us clearing
> - * all limits and the gpu stuck at whatever frequency it is at atm).
> - */
> -static u32 intel_rps_limits(struct drm_i915_private *dev_priv, u8 val)
> +static void gen6_check_mch_setup(struct drm_i915_private *dev_priv)
> {
> - struct intel_rps *rps = &dev_priv->gt_pm.rps;
> - u32 limits;
> -
> - /* Only set the down limit when we've reached the lowest level to avoid
> - * getting more interrupts, otherwise leave this clear. This prevents a
> - * race in the hw when coming out of rc6: There's a tiny window where
> - * the hw runs at the minimal clock before selecting the desired
> - * frequency, if the down threshold expires in that window we will not
> - * receive a down interrupt. */
> - if (INTEL_GEN(dev_priv) >= 9) {
> - limits = (rps->max_freq_softlimit) << 23;
> - if (val <= rps->min_freq_softlimit)
> - limits |= (rps->min_freq_softlimit) << 14;
> - } else {
> - limits = rps->max_freq_softlimit << 24;
> - if (val <= rps->min_freq_softlimit)
> - limits |= rps->min_freq_softlimit << 16;
> - }
> + u32 tmp;
>
> - return limits;
> + tmp = I915_READ(MCH_SSKPD);
> + if ((tmp & MCH_SSKPD_WM0_MASK) != MCH_SSKPD_WM0_VAL)
> + DRM_DEBUG_KMS("Wrong MCH_SSKPD value: 0x%08x This can cause underruns.\n",
> + tmp);
> }
>
> -static void rps_set_power(struct drm_i915_private *dev_priv, int new_power)
> +static void gen6_init_clock_gating(struct drm_i915_private *dev_priv)
> {
> - struct intel_rps *rps = &dev_priv->gt_pm.rps;
> - u32 threshold_up = 0, threshold_down = 0; /* in % */
> - u32 ei_up = 0, ei_down = 0;
> -
> - lockdep_assert_held(&rps->power.mutex);
> + u32 dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE;
>
> - if (new_power == rps->power.mode)
> - return;
> + I915_WRITE(ILK_DSPCLK_GATE_D, dspclk_gate);
>
> - /* Note the units here are not exactly 1us, but 1280ns. */
> - switch (new_power) {
> - case LOW_POWER:
> - /* Upclock if more than 95% busy over 16ms */
> - ei_up = 16000;
> - threshold_up = 95;
> + I915_WRITE(ILK_DISPLAY_CHICKEN2,
> + I915_READ(ILK_DISPLAY_CHICKEN2) |
> + ILK_ELPIN_409_SELECT);
>
> - /* Downclock if less than 85% busy over 32ms */
> - ei_down = 32000;
> - threshold_down = 85;
> - break;
> + /* WaDisableHiZPlanesWhenMSAAEnabled:snb */
> + I915_WRITE(_3D_CHICKEN,
> + _MASKED_BIT_ENABLE(_3D_CHICKEN_HIZ_PLANE_DISABLE_MSAA_4X_SNB));
>
> - case BETWEEN:
> - /* Upclock if more than 90% busy over 13ms */
> - ei_up = 13000;
> - threshold_up = 90;
> + /* WaDisable_RenderCache_OperationalFlush:snb */
> + I915_WRITE(CACHE_MODE_0, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
>
> - /* Downclock if less than 75% busy over 32ms */
> - ei_down = 32000;
> - threshold_down = 75;
> - break;
> + /*
> + * BSpec recoomends 8x4 when MSAA is used,
> + * however in practice 16x4 seems fastest.
> + *
> + * Note that PS/WM thread counts depend on the WIZ hashing
> + * disable bit, which we don't touch here, but it's good
> + * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM).
> + */
> + I915_WRITE(GEN6_GT_MODE,
> + _MASKED_FIELD(GEN6_WIZ_HASHING_MASK, GEN6_WIZ_HASHING_16x4));
>
> - case HIGH_POWER:
> - /* Upclock if more than 85% busy over 10ms */
> - ei_up = 10000;
> - threshold_up = 85;
> + I915_WRITE(CACHE_MODE_0,
> + _MASKED_BIT_DISABLE(CM0_STC_EVICT_DISABLE_LRA_SNB));
>
> - /* Downclock if less than 60% busy over 32ms */
> - ei_down = 32000;
> - threshold_down = 60;
> - break;
> - }
> + I915_WRITE(GEN6_UCGCTL1,
> + I915_READ(GEN6_UCGCTL1) |
> + GEN6_BLBUNIT_CLOCK_GATE_DISABLE |
> + GEN6_CSUNIT_CLOCK_GATE_DISABLE);
>
> - /* When byt can survive without system hang with dynamic
> - * sw freq adjustments, this restriction can be lifted.
> + /* According to the BSpec vol1g, bit 12 (RCPBUNIT) clock
> + * gating disable must be set. Failure to set it results in
> + * flickering pixels due to Z write ordering failures after
> + * some amount of runtime in the Mesa "fire" demo, and Unigine
> + * Sanctuary and Tropics, and apparently anything else with
> + * alpha test or pixel discard.
> + *
> + * According to the spec, bit 11 (RCCUNIT) must also be set,
> + * but we didn't debug actual testcases to find it out.
> + *
> + * WaDisableRCCUnitClockGating:snb
> + * WaDisableRCPBUnitClockGating:snb
> */
> - if (IS_VALLEYVIEW(dev_priv))
> - goto skip_hw_write;
> -
> - I915_WRITE(GEN6_RP_UP_EI,
> - GT_INTERVAL_FROM_US(dev_priv, ei_up));
> - I915_WRITE(GEN6_RP_UP_THRESHOLD,
> - GT_INTERVAL_FROM_US(dev_priv,
> - ei_up * threshold_up / 100));
> -
> - I915_WRITE(GEN6_RP_DOWN_EI,
> - GT_INTERVAL_FROM_US(dev_priv, ei_down));
> - I915_WRITE(GEN6_RP_DOWN_THRESHOLD,
> - GT_INTERVAL_FROM_US(dev_priv,
> - ei_down * threshold_down / 100));
> -
> - I915_WRITE(GEN6_RP_CONTROL,
> - (INTEL_GEN(dev_priv) > 9 ? 0 : GEN6_RP_MEDIA_TURBO) |
> - GEN6_RP_MEDIA_HW_NORMAL_MODE |
> - GEN6_RP_MEDIA_IS_GFX |
> - GEN6_RP_ENABLE |
> - GEN6_RP_UP_BUSY_AVG |
> - GEN6_RP_DOWN_IDLE_AVG);
> -
> -skip_hw_write:
> - rps->power.mode = new_power;
> - rps->power.up_threshold = threshold_up;
> - rps->power.down_threshold = threshold_down;
> -}
> -
> -static void gen6_set_rps_thresholds(struct drm_i915_private *dev_priv, u8 val)
> -{
> - struct intel_rps *rps = &dev_priv->gt_pm.rps;
> - int new_power;
> -
> - new_power = rps->power.mode;
> - switch (rps->power.mode) {
> - case LOW_POWER:
> - if (val > rps->efficient_freq + 1 &&
> - val > rps->cur_freq)
> - new_power = BETWEEN;
> - break;
> + I915_WRITE(GEN6_UCGCTL2,
> + GEN6_RCPBUNIT_CLOCK_GATE_DISABLE |
> + GEN6_RCCUNIT_CLOCK_GATE_DISABLE);
>
> - case BETWEEN:
> - if (val <= rps->efficient_freq &&
> - val < rps->cur_freq)
> - new_power = LOW_POWER;
> - else if (val >= rps->rp0_freq &&
> - val > rps->cur_freq)
> - new_power = HIGH_POWER;
> - break;
> + /* WaStripsFansDisableFastClipPerformanceFix:snb */
> + I915_WRITE(_3D_CHICKEN3,
> + _MASKED_BIT_ENABLE(_3D_CHICKEN3_SF_DISABLE_FASTCLIP_CULL));
>
> - case HIGH_POWER:
> - if (val < (rps->rp1_freq + rps->rp0_freq) >> 1 &&
> - val < rps->cur_freq)
> - new_power = BETWEEN;
> - break;
> - }
> - /* Max/min bins are special */
> - if (val <= rps->min_freq_softlimit)
> - new_power = LOW_POWER;
> - if (val >= rps->max_freq_softlimit)
> - new_power = HIGH_POWER;
> + /*
> + * Bspec says:
> + * "This bit must be set if 3DSTATE_CLIP clip mode is set to normal and
> + * 3DSTATE_SF number of SF output attributes is more than 16."
> + */
> + I915_WRITE(_3D_CHICKEN3,
> + _MASKED_BIT_ENABLE(_3D_CHICKEN3_SF_DISABLE_PIPELINED_ATTR_FETCH));
>
> - mutex_lock(&rps->power.mutex);
> - if (rps->power.interactive)
> - new_power = HIGH_POWER;
> - rps_set_power(dev_priv, new_power);
> - mutex_unlock(&rps->power.mutex);
> -}
> + /*
> + * According to the spec the following bits should be
> + * set in order to enable memory self-refresh and fbc:
> + * The bit21 and bit22 of 0x42000
> + * The bit21 and bit22 of 0x42004
> + * The bit5 and bit7 of 0x42020
> + * The bit14 of 0x70180
> + * The bit14 of 0x71180
> + *
> + * WaFbcAsynchFlipDisableFbcQueue:snb
> + */
> + I915_WRITE(ILK_DISPLAY_CHICKEN1,
> + I915_READ(ILK_DISPLAY_CHICKEN1) |
> + ILK_FBCQ_DIS | ILK_PABSTRETCH_DIS);
> + I915_WRITE(ILK_DISPLAY_CHICKEN2,
> + I915_READ(ILK_DISPLAY_CHICKEN2) |
> + ILK_DPARB_GATE | ILK_VSDPFD_FULL);
> + I915_WRITE(ILK_DSPCLK_GATE_D,
> + I915_READ(ILK_DSPCLK_GATE_D) |
> + ILK_DPARBUNIT_CLOCK_GATE_ENABLE |
> + ILK_DPFDUNIT_CLOCK_GATE_ENABLE);
>
> -void intel_rps_mark_interactive(struct drm_i915_private *i915, bool interactive)
> -{
> - struct intel_rps *rps = &i915->gt_pm.rps;
> + g4x_disable_trickle_feed(dev_priv);
>
> - if (INTEL_GEN(i915) < 6)
> - return;
> + cpt_init_clock_gating(dev_priv);
>
> - mutex_lock(&rps->power.mutex);
> - if (interactive) {
> - if (!rps->power.interactive++ && READ_ONCE(i915->gt.awake))
> - rps_set_power(i915, HIGH_POWER);
> - } else {
> - GEM_BUG_ON(!rps->power.interactive);
> - rps->power.interactive--;
> - }
> - mutex_unlock(&rps->power.mutex);
> + gen6_check_mch_setup(dev_priv);
> }
>
> -static u32 gen6_rps_pm_mask(struct drm_i915_private *dev_priv, u8 val)
> +static void gen7_setup_fixed_func_scheduler(struct drm_i915_private *dev_priv)
> {
> - struct intel_rps *rps = &dev_priv->gt_pm.rps;
> - u32 mask = 0;
> -
> - /* We use UP_EI_EXPIRED interupts for both up/down in manual mode */
> - if (val > rps->min_freq_softlimit)
> - mask |= GEN6_PM_RP_UP_EI_EXPIRED | GEN6_PM_RP_DOWN_THRESHOLD | GEN6_PM_RP_DOWN_TIMEOUT;
> - if (val < rps->max_freq_softlimit)
> - mask |= GEN6_PM_RP_UP_EI_EXPIRED | GEN6_PM_RP_UP_THRESHOLD;
> -
> - mask &= dev_priv->pm_rps_events;
> -
> - return gen6_sanitize_rps_pm_mask(dev_priv, ~mask);
> -}
> -
> -/* gen6_set_rps is called to update the frequency request, but should also be
> - * called when the range (min_delay and max_delay) is modified so that we can
> - * update the GEN6_RP_INTERRUPT_LIMITS register accordingly. */
> -static int gen6_set_rps(struct drm_i915_private *dev_priv, u8 val)
> -{
> - struct intel_rps *rps = &dev_priv->gt_pm.rps;
> -
> - /* min/max delay may still have been modified so be sure to
> - * write the limits value.
> - */
> - if (val != rps->cur_freq) {
> - gen6_set_rps_thresholds(dev_priv, val);
> -
> - if (INTEL_GEN(dev_priv) >= 9)
> - I915_WRITE(GEN6_RPNSWREQ,
> - GEN9_FREQUENCY(val));
> - else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
> - I915_WRITE(GEN6_RPNSWREQ,
> - HSW_FREQUENCY(val));
> - else
> - I915_WRITE(GEN6_RPNSWREQ,
> - GEN6_FREQUENCY(val) |
> - GEN6_OFFSET(0) |
> - GEN6_AGGRESSIVE_TURBO);
> - }
> -
> - /* Make sure we continue to get interrupts
> - * until we hit the minimum or maximum frequencies.
> - */
> - I915_WRITE(GEN6_RP_INTERRUPT_LIMITS, intel_rps_limits(dev_priv, val));
> - I915_WRITE(GEN6_PMINTRMSK, gen6_rps_pm_mask(dev_priv, val));
> -
> - rps->cur_freq = val;
> - trace_intel_gpu_freq_change(intel_gpu_freq(dev_priv, val));
> -
> - return 0;
> -}
> -
> -static int valleyview_set_rps(struct drm_i915_private *dev_priv, u8 val)
> -{
> - int err;
> -
> - if (WARN_ONCE(IS_CHERRYVIEW(dev_priv) && (val & 1),
> - "Odd GPU freq value\n"))
> - val &= ~1;
> -
> - I915_WRITE(GEN6_PMINTRMSK, gen6_rps_pm_mask(dev_priv, val));
> -
> - if (val != dev_priv->gt_pm.rps.cur_freq) {
> - vlv_punit_get(dev_priv);
> - err = vlv_punit_write(dev_priv, PUNIT_REG_GPU_FREQ_REQ, val);
> - vlv_punit_put(dev_priv);
> - if (err)
> - return err;
> -
> - gen6_set_rps_thresholds(dev_priv, val);
> - }
> -
> - dev_priv->gt_pm.rps.cur_freq = val;
> - trace_intel_gpu_freq_change(intel_gpu_freq(dev_priv, val));
> -
> - return 0;
> -}
> -
> -/* vlv_set_rps_idle: Set the frequency to idle, if Gfx clocks are down
> - *
> - * * If Gfx is Idle, then
> - * 1. Forcewake Media well.
> - * 2. Request idle freq.
> - * 3. Release Forcewake of Media well.
> -*/
> -static void vlv_set_rps_idle(struct drm_i915_private *dev_priv)
> -{
> - struct intel_rps *rps = &dev_priv->gt_pm.rps;
> - u32 val = rps->idle_freq;
> - int err;
> -
> - if (rps->cur_freq <= val)
> - return;
> -
> - /* The punit delays the write of the frequency and voltage until it
> - * determines the GPU is awake. During normal usage we don't want to
> - * waste power changing the frequency if the GPU is sleeping (rc6).
> - * However, the GPU and driver is now idle and we do not want to delay
> - * switching to minimum voltage (reducing power whilst idle) as we do
> - * not expect to be woken in the near future and so must flush the
> - * change by waking the device.
> - *
> - * We choose to take the media powerwell (either would do to trick the
> - * punit into committing the voltage change) as that takes a lot less
> - * power than the render powerwell.
> - */
> - intel_uncore_forcewake_get(&dev_priv->uncore, FORCEWAKE_MEDIA);
> - err = valleyview_set_rps(dev_priv, val);
> - intel_uncore_forcewake_put(&dev_priv->uncore, FORCEWAKE_MEDIA);
> -
> - if (err)
> - DRM_ERROR("Failed to set RPS for idle\n");
> -}
> -
> -void gen6_rps_busy(struct drm_i915_private *dev_priv)
> -{
> - struct intel_rps *rps = &dev_priv->gt_pm.rps;
> -
> - mutex_lock(&rps->lock);
> - if (rps->enabled) {
> - u8 freq;
> -
> - if (dev_priv->pm_rps_events & GEN6_PM_RP_UP_EI_EXPIRED)
> - gen6_rps_reset_ei(dev_priv);
> - I915_WRITE(GEN6_PMINTRMSK,
> - gen6_rps_pm_mask(dev_priv, rps->cur_freq));
> -
> - gen6_enable_rps_interrupts(dev_priv);
> -
> - /* Use the user's desired frequency as a guide, but for better
> - * performance, jump directly to RPe as our starting frequency.
> - */
> - freq = max(rps->cur_freq,
> - rps->efficient_freq);
> -
> - if (intel_set_rps(dev_priv,
> - clamp(freq,
> - rps->min_freq_softlimit,
> - rps->max_freq_softlimit)))
> - DRM_DEBUG_DRIVER("Failed to set idle frequency\n");
> - }
> - mutex_unlock(&rps->lock);
> -}
> -
> -void gen6_rps_idle(struct drm_i915_private *dev_priv)
> -{
> - struct intel_rps *rps = &dev_priv->gt_pm.rps;
> -
> - /* Flush our bottom-half so that it does not race with us
> - * setting the idle frequency and so that it is bounded by
> - * our rpm wakeref. And then disable the interrupts to stop any
> - * futher RPS reclocking whilst we are asleep.
> - */
> - gen6_disable_rps_interrupts(dev_priv);
> -
> - mutex_lock(&rps->lock);
> - if (rps->enabled) {
> - if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
> - vlv_set_rps_idle(dev_priv);
> - else
> - gen6_set_rps(dev_priv, rps->idle_freq);
> - rps->last_adj = 0;
> - I915_WRITE(GEN6_PMINTRMSK,
> - gen6_sanitize_rps_pm_mask(dev_priv, ~0));
> - }
> - mutex_unlock(&rps->lock);
> -}
> -
> -void gen6_rps_boost(struct i915_request *rq)
> -{
> - struct intel_rps *rps = &rq->i915->gt_pm.rps;
> - unsigned long flags;
> - bool boost;
> -
> - /* This is intentionally racy! We peek at the state here, then
> - * validate inside the RPS worker.
> - */
> - if (!rps->enabled)
> - return;
> -
> - if (i915_request_signaled(rq))
> - return;
> -
> - /* Serializes with i915_request_retire() */
> - boost = false;
> - spin_lock_irqsave(&rq->lock, flags);
> - if (!i915_request_has_waitboost(rq) &&
> - !dma_fence_is_signaled_locked(&rq->fence)) {
> - boost = !atomic_fetch_inc(&rps->num_waiters);
> - rq->flags |= I915_REQUEST_WAITBOOST;
> - }
> - spin_unlock_irqrestore(&rq->lock, flags);
> - if (!boost)
> - return;
> -
> - if (READ_ONCE(rps->cur_freq) < rps->boost_freq)
> - schedule_work(&rps->work);
> -
> - atomic_inc(&rps->boosts);
> -}
> -
> -int intel_set_rps(struct drm_i915_private *dev_priv, u8 val)
> -{
> - struct intel_rps *rps = &dev_priv->gt_pm.rps;
> - int err;
> -
> - lockdep_assert_held(&rps->lock);
> - GEM_BUG_ON(val > rps->max_freq);
> - GEM_BUG_ON(val < rps->min_freq);
> -
> - if (!rps->enabled) {
> - rps->cur_freq = val;
> - return 0;
> - }
> -
> - if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
> - err = valleyview_set_rps(dev_priv, val);
> - else
> - err = gen6_set_rps(dev_priv, val);
> -
> - return err;
> -}
> -
> -static void gen9_disable_rps(struct drm_i915_private *dev_priv)
> -{
> - I915_WRITE(GEN6_RP_CONTROL, 0);
> -}
> -
> -static void gen6_disable_rps(struct drm_i915_private *dev_priv)
> -{
> - I915_WRITE(GEN6_RPNSWREQ, 1 << 31);
> - I915_WRITE(GEN6_RP_CONTROL, 0);
> -}
> -
> -static void cherryview_disable_rps(struct drm_i915_private *dev_priv)
> -{
> - I915_WRITE(GEN6_RP_CONTROL, 0);
> -}
> -
> -static void valleyview_disable_rps(struct drm_i915_private *dev_priv)
> -{
> - I915_WRITE(GEN6_RP_CONTROL, 0);
> -}
> -
> -static void gen6_init_rps_frequencies(struct drm_i915_private *dev_priv)
> -{
> - struct intel_rps *rps = &dev_priv->gt_pm.rps;
> -
> - /* All of these values are in units of 50MHz */
> -
> - /* static values from HW: RP0 > RP1 > RPn (min_freq) */
> - if (IS_GEN9_LP(dev_priv)) {
> - u32 rp_state_cap = I915_READ(BXT_RP_STATE_CAP);
> - rps->rp0_freq = (rp_state_cap >> 16) & 0xff;
> - rps->rp1_freq = (rp_state_cap >> 8) & 0xff;
> - rps->min_freq = (rp_state_cap >> 0) & 0xff;
> - } else {
> - u32 rp_state_cap = I915_READ(GEN6_RP_STATE_CAP);
> - rps->rp0_freq = (rp_state_cap >> 0) & 0xff;
> - rps->rp1_freq = (rp_state_cap >> 8) & 0xff;
> - rps->min_freq = (rp_state_cap >> 16) & 0xff;
> - }
> - /* hw_max = RP0 until we check for overclocking */
> - rps->max_freq = rps->rp0_freq;
> -
> - rps->efficient_freq = rps->rp1_freq;
> - if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv) ||
> - IS_GEN9_BC(dev_priv) || INTEL_GEN(dev_priv) >= 10) {
> - u32 ddcc_status = 0;
> -
> - if (sandybridge_pcode_read(dev_priv,
> - HSW_PCODE_DYNAMIC_DUTY_CYCLE_CONTROL,
> - &ddcc_status, NULL) == 0)
> - rps->efficient_freq =
> - clamp_t(u8,
> - ((ddcc_status >> 8) & 0xff),
> - rps->min_freq,
> - rps->max_freq);
> - }
> -
> - if (IS_GEN9_BC(dev_priv) || INTEL_GEN(dev_priv) >= 10) {
> - /* Store the frequency values in 16.66 MHZ units, which is
> - * the natural hardware unit for SKL
> - */
> - rps->rp0_freq *= GEN9_FREQ_SCALER;
> - rps->rp1_freq *= GEN9_FREQ_SCALER;
> - rps->min_freq *= GEN9_FREQ_SCALER;
> - rps->max_freq *= GEN9_FREQ_SCALER;
> - rps->efficient_freq *= GEN9_FREQ_SCALER;
> - }
> -}
> -
> -static void reset_rps(struct drm_i915_private *dev_priv,
> - int (*set)(struct drm_i915_private *, u8))
> -{
> - struct intel_rps *rps = &dev_priv->gt_pm.rps;
> - u8 freq = rps->cur_freq;
> -
> - /* force a reset */
> - rps->power.mode = -1;
> - rps->cur_freq = -1;
> -
> - if (set(dev_priv, freq))
> - DRM_ERROR("Failed to reset RPS to initial values\n");
> -}
> -
> -/* See the Gen9_GT_PM_Programming_Guide doc for the below */
> -static void gen9_enable_rps(struct drm_i915_private *dev_priv)
> -{
> - intel_uncore_forcewake_get(&dev_priv->uncore, FORCEWAKE_ALL);
> -
> - /* Program defaults and thresholds for RPS */
> - if (IS_GEN(dev_priv, 9))
> - I915_WRITE(GEN6_RC_VIDEO_FREQ,
> - GEN9_FREQUENCY(dev_priv->gt_pm.rps.rp1_freq));
> -
> - /* 1 second timeout*/
> - I915_WRITE(GEN6_RP_DOWN_TIMEOUT,
> - GT_INTERVAL_FROM_US(dev_priv, 1000000));
> -
> - I915_WRITE(GEN6_RP_IDLE_HYSTERSIS, 0xa);
> -
> - /* Leaning on the below call to gen6_set_rps to program/setup the
> - * Up/Down EI & threshold registers, as well as the RP_CONTROL,
> - * RP_INTERRUPT_LIMITS & RPNSWREQ registers */
> - reset_rps(dev_priv, gen6_set_rps);
> -
> - intel_uncore_forcewake_put(&dev_priv->uncore, FORCEWAKE_ALL);
> -}
> -
> -static void gen8_enable_rps(struct drm_i915_private *dev_priv)
> -{
> - struct intel_rps *rps = &dev_priv->gt_pm.rps;
> -
> - intel_uncore_forcewake_get(&dev_priv->uncore, FORCEWAKE_ALL);
> -
> - /* 1 Program defaults and thresholds for RPS*/
> - I915_WRITE(GEN6_RPNSWREQ,
> - HSW_FREQUENCY(rps->rp1_freq));
> - I915_WRITE(GEN6_RC_VIDEO_FREQ,
> - HSW_FREQUENCY(rps->rp1_freq));
> - /* NB: Docs say 1s, and 1000000 - which aren't equivalent */
> - I915_WRITE(GEN6_RP_DOWN_TIMEOUT, 100000000 / 128); /* 1 second timeout */
> -
> - /* Docs recommend 900MHz, and 300 MHz respectively */
> - I915_WRITE(GEN6_RP_INTERRUPT_LIMITS,
> - rps->max_freq_softlimit << 24 |
> - rps->min_freq_softlimit << 16);
> -
> - I915_WRITE(GEN6_RP_UP_THRESHOLD, 7600000 / 128); /* 76ms busyness per EI, 90% */
> - I915_WRITE(GEN6_RP_DOWN_THRESHOLD, 31300000 / 128); /* 313ms busyness per EI, 70%*/
> - I915_WRITE(GEN6_RP_UP_EI, 66000); /* 84.48ms, XXX: random? */
> - I915_WRITE(GEN6_RP_DOWN_EI, 350000); /* 448ms, XXX: random? */
> -
> - I915_WRITE(GEN6_RP_IDLE_HYSTERSIS, 10);
> -
> - /* 2: Enable RPS */
> - I915_WRITE(GEN6_RP_CONTROL,
> - GEN6_RP_MEDIA_TURBO |
> - GEN6_RP_MEDIA_HW_NORMAL_MODE |
> - GEN6_RP_MEDIA_IS_GFX |
> - GEN6_RP_ENABLE |
> - GEN6_RP_UP_BUSY_AVG |
> - GEN6_RP_DOWN_IDLE_AVG);
> -
> - reset_rps(dev_priv, gen6_set_rps);
> -
> - intel_uncore_forcewake_put(&dev_priv->uncore, FORCEWAKE_ALL);
> -}
> -
> -static void gen6_enable_rps(struct drm_i915_private *dev_priv)
> -{
> - /* Here begins a magic sequence of register writes to enable
> - * auto-downclocking.
> - *
> - * Perhaps there might be some value in exposing these to
> - * userspace...
> - */
> - intel_uncore_forcewake_get(&dev_priv->uncore, FORCEWAKE_ALL);
> -
> - /* Power down if completely idle for over 50ms */
> - I915_WRITE(GEN6_RP_DOWN_TIMEOUT, 50000);
> - I915_WRITE(GEN6_RP_IDLE_HYSTERSIS, 10);
> -
> - reset_rps(dev_priv, gen6_set_rps);
> -
> - intel_uncore_forcewake_put(&dev_priv->uncore, FORCEWAKE_ALL);
> -}
> -
> -static int cherryview_rps_max_freq(struct drm_i915_private *dev_priv)
> -{
> - u32 val, rp0;
> -
> - val = vlv_punit_read(dev_priv, FB_GFX_FMAX_AT_VMAX_FUSE);
> -
> - switch (RUNTIME_INFO(dev_priv)->sseu.eu_total) {
> - case 8:
> - /* (2 * 4) config */
> - rp0 = (val >> FB_GFX_FMAX_AT_VMAX_2SS4EU_FUSE_SHIFT);
> - break;
> - case 12:
> - /* (2 * 6) config */
> - rp0 = (val >> FB_GFX_FMAX_AT_VMAX_2SS6EU_FUSE_SHIFT);
> - break;
> - case 16:
> - /* (2 * 8) config */
> - default:
> - /* Setting (2 * 8) Min RP0 for any other combination */
> - rp0 = (val >> FB_GFX_FMAX_AT_VMAX_2SS8EU_FUSE_SHIFT);
> - break;
> - }
> -
> - rp0 = (rp0 & FB_GFX_FREQ_FUSE_MASK);
> -
> - return rp0;
> -}
> -
> -static int cherryview_rps_rpe_freq(struct drm_i915_private *dev_priv)
> -{
> - u32 val, rpe;
> -
> - val = vlv_punit_read(dev_priv, PUNIT_GPU_DUTYCYCLE_REG);
> - rpe = (val >> PUNIT_GPU_DUTYCYCLE_RPE_FREQ_SHIFT) & PUNIT_GPU_DUTYCYCLE_RPE_FREQ_MASK;
> -
> - return rpe;
> -}
> -
> -static int cherryview_rps_guar_freq(struct drm_i915_private *dev_priv)
> -{
> - u32 val, rp1;
> -
> - val = vlv_punit_read(dev_priv, FB_GFX_FMAX_AT_VMAX_FUSE);
> - rp1 = (val & FB_GFX_FREQ_FUSE_MASK);
> -
> - return rp1;
> -}
> -
> -static u32 cherryview_rps_min_freq(struct drm_i915_private *dev_priv)
> -{
> - u32 val, rpn;
> -
> - val = vlv_punit_read(dev_priv, FB_GFX_FMIN_AT_VMIN_FUSE);
> - rpn = ((val >> FB_GFX_FMIN_AT_VMIN_FUSE_SHIFT) &
> - FB_GFX_FREQ_FUSE_MASK);
> -
> - return rpn;
> -}
> -
> -static int valleyview_rps_guar_freq(struct drm_i915_private *dev_priv)
> -{
> - u32 val, rp1;
> -
> - val = vlv_nc_read(dev_priv, IOSF_NC_FB_GFX_FREQ_FUSE);
> -
> - rp1 = (val & FB_GFX_FGUARANTEED_FREQ_FUSE_MASK) >> FB_GFX_FGUARANTEED_FREQ_FUSE_SHIFT;
> -
> - return rp1;
> -}
> -
> -static int valleyview_rps_max_freq(struct drm_i915_private *dev_priv)
> -{
> - u32 val, rp0;
> -
> - val = vlv_nc_read(dev_priv, IOSF_NC_FB_GFX_FREQ_FUSE);
> -
> - rp0 = (val & FB_GFX_MAX_FREQ_FUSE_MASK) >> FB_GFX_MAX_FREQ_FUSE_SHIFT;
> - /* Clamp to max */
> - rp0 = min_t(u32, rp0, 0xea);
> -
> - return rp0;
> -}
> -
> -static int valleyview_rps_rpe_freq(struct drm_i915_private *dev_priv)
> -{
> - u32 val, rpe;
> -
> - val = vlv_nc_read(dev_priv, IOSF_NC_FB_GFX_FMAX_FUSE_LO);
> - rpe = (val & FB_FMAX_VMIN_FREQ_LO_MASK) >> FB_FMAX_VMIN_FREQ_LO_SHIFT;
> - val = vlv_nc_read(dev_priv, IOSF_NC_FB_GFX_FMAX_FUSE_HI);
> - rpe |= (val & FB_FMAX_VMIN_FREQ_HI_MASK) << 5;
> -
> - return rpe;
> -}
> -
> -static int valleyview_rps_min_freq(struct drm_i915_private *dev_priv)
> -{
> - u32 val;
> -
> - val = vlv_punit_read(dev_priv, PUNIT_REG_GPU_LFM) & 0xff;
> - /*
> - * According to the BYT Punit GPU turbo HAS 1.1.6.3 the minimum value
> - * for the minimum frequency in GPLL mode is 0xc1. Contrary to this on
> - * a BYT-M B0 the above register contains 0xbf. Moreover when setting
> - * a frequency Punit will not allow values below 0xc0. Clamp it 0xc0
> - * to make sure it matches what Punit accepts.
> - */
> - return max_t(u32, val, 0xc0);
> -}
> -
> -static void vlv_init_gpll_ref_freq(struct drm_i915_private *dev_priv)
> -{
> - dev_priv->gt_pm.rps.gpll_ref_freq =
> - vlv_get_cck_clock(dev_priv, "GPLL ref",
> - CCK_GPLL_CLOCK_CONTROL,
> - dev_priv->czclk_freq);
> -
> - DRM_DEBUG_DRIVER("GPLL reference freq: %d kHz\n",
> - dev_priv->gt_pm.rps.gpll_ref_freq);
> -}
> -
> -static void valleyview_init_gt_powersave(struct drm_i915_private *dev_priv)
> -{
> - struct intel_rps *rps = &dev_priv->gt_pm.rps;
> - u32 val;
> -
> - vlv_iosf_sb_get(dev_priv,
> - BIT(VLV_IOSF_SB_PUNIT) |
> - BIT(VLV_IOSF_SB_NC) |
> - BIT(VLV_IOSF_SB_CCK));
> -
> - vlv_init_gpll_ref_freq(dev_priv);
> -
> - val = vlv_punit_read(dev_priv, PUNIT_REG_GPU_FREQ_STS);
> - switch ((val >> 6) & 3) {
> - case 0:
> - case 1:
> - dev_priv->mem_freq = 800;
> - break;
> - case 2:
> - dev_priv->mem_freq = 1066;
> - break;
> - case 3:
> - dev_priv->mem_freq = 1333;
> - break;
> - }
> - DRM_DEBUG_DRIVER("DDR speed: %d MHz\n", dev_priv->mem_freq);
> -
> - rps->max_freq = valleyview_rps_max_freq(dev_priv);
> - rps->rp0_freq = rps->max_freq;
> - DRM_DEBUG_DRIVER("max GPU freq: %d MHz (%u)\n",
> - intel_gpu_freq(dev_priv, rps->max_freq),
> - rps->max_freq);
> -
> - rps->efficient_freq = valleyview_rps_rpe_freq(dev_priv);
> - DRM_DEBUG_DRIVER("RPe GPU freq: %d MHz (%u)\n",
> - intel_gpu_freq(dev_priv, rps->efficient_freq),
> - rps->efficient_freq);
> -
> - rps->rp1_freq = valleyview_rps_guar_freq(dev_priv);
> - DRM_DEBUG_DRIVER("RP1(Guar Freq) GPU freq: %d MHz (%u)\n",
> - intel_gpu_freq(dev_priv, rps->rp1_freq),
> - rps->rp1_freq);
> -
> - rps->min_freq = valleyview_rps_min_freq(dev_priv);
> - DRM_DEBUG_DRIVER("min GPU freq: %d MHz (%u)\n",
> - intel_gpu_freq(dev_priv, rps->min_freq),
> - rps->min_freq);
> -
> - vlv_iosf_sb_put(dev_priv,
> - BIT(VLV_IOSF_SB_PUNIT) |
> - BIT(VLV_IOSF_SB_NC) |
> - BIT(VLV_IOSF_SB_CCK));
> -}
> -
> -static void cherryview_init_gt_powersave(struct drm_i915_private *dev_priv)
> -{
> - struct intel_rps *rps = &dev_priv->gt_pm.rps;
> - u32 val;
> -
> - vlv_iosf_sb_get(dev_priv,
> - BIT(VLV_IOSF_SB_PUNIT) |
> - BIT(VLV_IOSF_SB_NC) |
> - BIT(VLV_IOSF_SB_CCK));
> -
> - vlv_init_gpll_ref_freq(dev_priv);
> -
> - val = vlv_cck_read(dev_priv, CCK_FUSE_REG);
> -
> - switch ((val >> 2) & 0x7) {
> - case 3:
> - dev_priv->mem_freq = 2000;
> - break;
> - default:
> - dev_priv->mem_freq = 1600;
> - break;
> - }
> - DRM_DEBUG_DRIVER("DDR speed: %d MHz\n", dev_priv->mem_freq);
> -
> - rps->max_freq = cherryview_rps_max_freq(dev_priv);
> - rps->rp0_freq = rps->max_freq;
> - DRM_DEBUG_DRIVER("max GPU freq: %d MHz (%u)\n",
> - intel_gpu_freq(dev_priv, rps->max_freq),
> - rps->max_freq);
> -
> - rps->efficient_freq = cherryview_rps_rpe_freq(dev_priv);
> - DRM_DEBUG_DRIVER("RPe GPU freq: %d MHz (%u)\n",
> - intel_gpu_freq(dev_priv, rps->efficient_freq),
> - rps->efficient_freq);
> -
> - rps->rp1_freq = cherryview_rps_guar_freq(dev_priv);
> - DRM_DEBUG_DRIVER("RP1(Guar) GPU freq: %d MHz (%u)\n",
> - intel_gpu_freq(dev_priv, rps->rp1_freq),
> - rps->rp1_freq);
> -
> - rps->min_freq = cherryview_rps_min_freq(dev_priv);
> - DRM_DEBUG_DRIVER("min GPU freq: %d MHz (%u)\n",
> - intel_gpu_freq(dev_priv, rps->min_freq),
> - rps->min_freq);
> -
> - vlv_iosf_sb_put(dev_priv,
> - BIT(VLV_IOSF_SB_PUNIT) |
> - BIT(VLV_IOSF_SB_NC) |
> - BIT(VLV_IOSF_SB_CCK));
> -
> - WARN_ONCE((rps->max_freq | rps->efficient_freq | rps->rp1_freq |
> - rps->min_freq) & 1,
> - "Odd GPU freq values\n");
> -}
> -
> -static void cherryview_enable_rps(struct drm_i915_private *dev_priv)
> -{
> - u32 val;
> -
> - intel_uncore_forcewake_get(&dev_priv->uncore, FORCEWAKE_ALL);
> -
> - /* 1: Program defaults and thresholds for RPS*/
> - I915_WRITE(GEN6_RP_DOWN_TIMEOUT, 1000000);
> - I915_WRITE(GEN6_RP_UP_THRESHOLD, 59400);
> - I915_WRITE(GEN6_RP_DOWN_THRESHOLD, 245000);
> - I915_WRITE(GEN6_RP_UP_EI, 66000);
> - I915_WRITE(GEN6_RP_DOWN_EI, 350000);
> -
> - I915_WRITE(GEN6_RP_IDLE_HYSTERSIS, 10);
> -
> - /* 2: Enable RPS */
> - I915_WRITE(GEN6_RP_CONTROL,
> - GEN6_RP_MEDIA_HW_NORMAL_MODE |
> - GEN6_RP_MEDIA_IS_GFX |
> - GEN6_RP_ENABLE |
> - GEN6_RP_UP_BUSY_AVG |
> - GEN6_RP_DOWN_IDLE_AVG);
> -
> - /* Setting Fixed Bias */
> - vlv_punit_get(dev_priv);
> -
> - val = VLV_OVERRIDE_EN | VLV_SOC_TDP_EN | CHV_BIAS_CPU_50_SOC_50;
> - vlv_punit_write(dev_priv, VLV_TURBO_SOC_OVERRIDE, val);
> -
> - val = vlv_punit_read(dev_priv, PUNIT_REG_GPU_FREQ_STS);
> -
> - vlv_punit_put(dev_priv);
> -
> - /* RPS code assumes GPLL is used */
> - WARN_ONCE((val & GPLLENABLE) == 0, "GPLL not enabled\n");
> -
> - DRM_DEBUG_DRIVER("GPLL enabled? %s\n", yesno(val & GPLLENABLE));
> - DRM_DEBUG_DRIVER("GPU status: 0x%08x\n", val);
> -
> - reset_rps(dev_priv, valleyview_set_rps);
> -
> - intel_uncore_forcewake_put(&dev_priv->uncore, FORCEWAKE_ALL);
> -}
> -
> -static void valleyview_enable_rps(struct drm_i915_private *dev_priv)
> -{
> - u32 val;
> -
> - intel_uncore_forcewake_get(&dev_priv->uncore, FORCEWAKE_ALL);
> -
> - I915_WRITE(GEN6_RP_DOWN_TIMEOUT, 1000000);
> - I915_WRITE(GEN6_RP_UP_THRESHOLD, 59400);
> - I915_WRITE(GEN6_RP_DOWN_THRESHOLD, 245000);
> - I915_WRITE(GEN6_RP_UP_EI, 66000);
> - I915_WRITE(GEN6_RP_DOWN_EI, 350000);
> -
> - I915_WRITE(GEN6_RP_IDLE_HYSTERSIS, 10);
> -
> - I915_WRITE(GEN6_RP_CONTROL,
> - GEN6_RP_MEDIA_TURBO |
> - GEN6_RP_MEDIA_HW_NORMAL_MODE |
> - GEN6_RP_MEDIA_IS_GFX |
> - GEN6_RP_ENABLE |
> - GEN6_RP_UP_BUSY_AVG |
> - GEN6_RP_DOWN_IDLE_CONT);
> -
> - vlv_punit_get(dev_priv);
> -
> - /* Setting Fixed Bias */
> - val = VLV_OVERRIDE_EN | VLV_SOC_TDP_EN | VLV_BIAS_CPU_125_SOC_875;
> - vlv_punit_write(dev_priv, VLV_TURBO_SOC_OVERRIDE, val);
> -
> - val = vlv_punit_read(dev_priv, PUNIT_REG_GPU_FREQ_STS);
> -
> - vlv_punit_put(dev_priv);
> -
> - /* RPS code assumes GPLL is used */
> - WARN_ONCE((val & GPLLENABLE) == 0, "GPLL not enabled\n");
> -
> - DRM_DEBUG_DRIVER("GPLL enabled? %s\n", yesno(val & GPLLENABLE));
> - DRM_DEBUG_DRIVER("GPU status: 0x%08x\n", val);
> -
> - reset_rps(dev_priv, valleyview_set_rps);
> -
> - intel_uncore_forcewake_put(&dev_priv->uncore, FORCEWAKE_ALL);
> -}
> -
> -static unsigned long intel_pxfreq(u32 vidfreq)
> -{
> - unsigned long freq;
> - int div = (vidfreq & 0x3f0000) >> 16;
> - int post = (vidfreq & 0x3000) >> 12;
> - int pre = (vidfreq & 0x7);
> -
> - if (!pre)
> - return 0;
> -
> - freq = ((div * 133333) / ((1<<post) * pre));
> -
> - return freq;
> -}
> -
> -static const struct cparams {
> - u16 i;
> - u16 t;
> - u16 m;
> - u16 c;
> -} cparams[] = {
> - { 1, 1333, 301, 28664 },
> - { 1, 1066, 294, 24460 },
> - { 1, 800, 294, 25192 },
> - { 0, 1333, 276, 27605 },
> - { 0, 1066, 276, 27605 },
> - { 0, 800, 231, 23784 },
> -};
> -
> -static unsigned long __i915_chipset_val(struct drm_i915_private *dev_priv)
> -{
> - u64 total_count, diff, ret;
> - u32 count1, count2, count3, m = 0, c = 0;
> - unsigned long now = jiffies_to_msecs(jiffies), diff1;
> - int i;
> -
> - lockdep_assert_held(&mchdev_lock);
> -
> - diff1 = now - dev_priv->ips.last_time1;
> -
> - /* Prevent division-by-zero if we are asking too fast.
> - * Also, we don't get interesting results if we are polling
> - * faster than once in 10ms, so just return the saved value
> - * in such cases.
> - */
> - if (diff1 <= 10)
> - return dev_priv->ips.chipset_power;
> -
> - count1 = I915_READ(DMIEC);
> - count2 = I915_READ(DDREC);
> - count3 = I915_READ(CSIEC);
> -
> - total_count = count1 + count2 + count3;
> -
> - /* FIXME: handle per-counter overflow */
> - if (total_count < dev_priv->ips.last_count1) {
> - diff = ~0UL - dev_priv->ips.last_count1;
> - diff += total_count;
> - } else {
> - diff = total_count - dev_priv->ips.last_count1;
> - }
> -
> - for (i = 0; i < ARRAY_SIZE(cparams); i++) {
> - if (cparams[i].i == dev_priv->ips.c_m &&
> - cparams[i].t == dev_priv->ips.r_t) {
> - m = cparams[i].m;
> - c = cparams[i].c;
> - break;
> - }
> - }
> -
> - diff = div_u64(diff, diff1);
> - ret = ((m * diff) + c);
> - ret = div_u64(ret, 10);
> -
> - dev_priv->ips.last_count1 = total_count;
> - dev_priv->ips.last_time1 = now;
> -
> - dev_priv->ips.chipset_power = ret;
> -
> - return ret;
> -}
> -
> -unsigned long i915_chipset_val(struct drm_i915_private *dev_priv)
> -{
> - intel_wakeref_t wakeref;
> - unsigned long val = 0;
> -
> - if (!IS_GEN(dev_priv, 5))
> - return 0;
> -
> - with_intel_runtime_pm(&dev_priv->runtime_pm, wakeref) {
> - spin_lock_irq(&mchdev_lock);
> - val = __i915_chipset_val(dev_priv);
> - spin_unlock_irq(&mchdev_lock);
> - }
> -
> - return val;
> -}
> -
> -unsigned long i915_mch_val(struct drm_i915_private *i915)
> -{
> - unsigned long m, x, b;
> - u32 tsfs;
> -
> - tsfs = intel_uncore_read(&i915->uncore, TSFS);
> -
> - m = ((tsfs & TSFS_SLOPE_MASK) >> TSFS_SLOPE_SHIFT);
> - x = intel_uncore_read8(&i915->uncore, TR1);
> -
> - b = tsfs & TSFS_INTR_MASK;
> -
> - return ((m * x) / 127) - b;
> -}
> -
> -static int _pxvid_to_vd(u8 pxvid)
> -{
> - if (pxvid == 0)
> - return 0;
> -
> - if (pxvid >= 8 && pxvid < 31)
> - pxvid = 31;
> -
> - return (pxvid + 2) * 125;
> -}
> -
> -static u32 pvid_to_extvid(struct drm_i915_private *dev_priv, u8 pxvid)
> -{
> - const int vd = _pxvid_to_vd(pxvid);
> - const int vm = vd - 1125;
> -
> - if (INTEL_INFO(dev_priv)->is_mobile)
> - return vm > 0 ? vm : 0;
> -
> - return vd;
> -}
> -
> -static void __i915_update_gfx_val(struct drm_i915_private *dev_priv)
> -{
> - u64 now, diff, diffms;
> - u32 count;
> -
> - lockdep_assert_held(&mchdev_lock);
> -
> - now = ktime_get_raw_ns();
> - diffms = now - dev_priv->ips.last_time2;
> - do_div(diffms, NSEC_PER_MSEC);
> -
> - /* Don't divide by 0 */
> - if (!diffms)
> - return;
> -
> - count = I915_READ(GFXEC);
> -
> - if (count < dev_priv->ips.last_count2) {
> - diff = ~0UL - dev_priv->ips.last_count2;
> - diff += count;
> - } else {
> - diff = count - dev_priv->ips.last_count2;
> - }
> -
> - dev_priv->ips.last_count2 = count;
> - dev_priv->ips.last_time2 = now;
> -
> - /* More magic constants... */
> - diff = diff * 1181;
> - diff = div_u64(diff, diffms * 10);
> - dev_priv->ips.gfx_power = diff;
> -}
> -
> -void i915_update_gfx_val(struct drm_i915_private *dev_priv)
> -{
> - intel_wakeref_t wakeref;
> -
> - if (!IS_GEN(dev_priv, 5))
> - return;
> -
> - with_intel_runtime_pm(&dev_priv->runtime_pm, wakeref) {
> - spin_lock_irq(&mchdev_lock);
> - __i915_update_gfx_val(dev_priv);
> - spin_unlock_irq(&mchdev_lock);
> - }
> -}
> -
> -static unsigned long __i915_gfx_val(struct drm_i915_private *dev_priv)
> -{
> - unsigned long t, corr, state1, corr2, state2;
> - u32 pxvid, ext_v;
> -
> - lockdep_assert_held(&mchdev_lock);
> -
> - pxvid = I915_READ(PXVFREQ(dev_priv->gt_pm.rps.cur_freq));
> - pxvid = (pxvid >> 24) & 0x7f;
> - ext_v = pvid_to_extvid(dev_priv, pxvid);
> -
> - state1 = ext_v;
> -
> - t = i915_mch_val(dev_priv);
> -
> - /* Revel in the empirically derived constants */
> -
> - /* Correction factor in 1/100000 units */
> - if (t > 80)
> - corr = ((t * 2349) + 135940);
> - else if (t >= 50)
> - corr = ((t * 964) + 29317);
> - else /* < 50 */
> - corr = ((t * 301) + 1004);
> -
> - corr = corr * ((150142 * state1) / 10000 - 78642);
> - corr /= 100000;
> - corr2 = (corr * dev_priv->ips.corr);
> -
> - state2 = (corr2 * state1) / 10000;
> - state2 /= 100; /* convert to mW */
> -
> - __i915_update_gfx_val(dev_priv);
> -
> - return dev_priv->ips.gfx_power + state2;
> -}
> -
> -unsigned long i915_gfx_val(struct drm_i915_private *dev_priv)
> -{
> - intel_wakeref_t wakeref;
> - unsigned long val = 0;
> -
> - if (!IS_GEN(dev_priv, 5))
> - return 0;
> -
> - with_intel_runtime_pm(&dev_priv->runtime_pm, wakeref) {
> - spin_lock_irq(&mchdev_lock);
> - val = __i915_gfx_val(dev_priv);
> - spin_unlock_irq(&mchdev_lock);
> - }
> -
> - return val;
> -}
> -
> -static struct drm_i915_private __rcu *i915_mch_dev;
> -
> -static struct drm_i915_private *mchdev_get(void)
> -{
> - struct drm_i915_private *i915;
> -
> - rcu_read_lock();
> - i915 = rcu_dereference(i915_mch_dev);
> - if (!kref_get_unless_zero(&i915->drm.ref))
> - i915 = NULL;
> - rcu_read_unlock();
> -
> - return i915;
> -}
> -
> -/**
> - * i915_read_mch_val - return value for IPS use
> - *
> - * Calculate and return a value for the IPS driver to use when deciding whether
> - * we have thermal and power headroom to increase CPU or GPU power budget.
> - */
> -unsigned long i915_read_mch_val(void)
> -{
> - struct drm_i915_private *i915;
> - unsigned long chipset_val = 0;
> - unsigned long graphics_val = 0;
> - intel_wakeref_t wakeref;
> -
> - i915 = mchdev_get();
> - if (!i915)
> - return 0;
> -
> - with_intel_runtime_pm(&i915->runtime_pm, wakeref) {
> - spin_lock_irq(&mchdev_lock);
> - chipset_val = __i915_chipset_val(i915);
> - graphics_val = __i915_gfx_val(i915);
> - spin_unlock_irq(&mchdev_lock);
> - }
> -
> - drm_dev_put(&i915->drm);
> - return chipset_val + graphics_val;
> -}
> -EXPORT_SYMBOL_GPL(i915_read_mch_val);
> -
> -/**
> - * i915_gpu_raise - raise GPU frequency limit
> - *
> - * Raise the limit; IPS indicates we have thermal headroom.
> - */
> -bool i915_gpu_raise(void)
> -{
> - struct drm_i915_private *i915;
> -
> - i915 = mchdev_get();
> - if (!i915)
> - return false;
> -
> - spin_lock_irq(&mchdev_lock);
> - if (i915->ips.max_delay > i915->ips.fmax)
> - i915->ips.max_delay--;
> - spin_unlock_irq(&mchdev_lock);
> -
> - drm_dev_put(&i915->drm);
> - return true;
> -}
> -EXPORT_SYMBOL_GPL(i915_gpu_raise);
> -
> -/**
> - * i915_gpu_lower - lower GPU frequency limit
> - *
> - * IPS indicates we're close to a thermal limit, so throttle back the GPU
> - * frequency maximum.
> - */
> -bool i915_gpu_lower(void)
> -{
> - struct drm_i915_private *i915;
> -
> - i915 = mchdev_get();
> - if (!i915)
> - return false;
> -
> - spin_lock_irq(&mchdev_lock);
> - if (i915->ips.max_delay < i915->ips.min_delay)
> - i915->ips.max_delay++;
> - spin_unlock_irq(&mchdev_lock);
> -
> - drm_dev_put(&i915->drm);
> - return true;
> -}
> -EXPORT_SYMBOL_GPL(i915_gpu_lower);
> -
> -/**
> - * i915_gpu_busy - indicate GPU business to IPS
> - *
> - * Tell the IPS driver whether or not the GPU is busy.
> - */
> -bool i915_gpu_busy(void)
> -{
> - struct drm_i915_private *i915;
> - bool ret;
> -
> - i915 = mchdev_get();
> - if (!i915)
> - return false;
> -
> - ret = i915->gt.awake;
> -
> - drm_dev_put(&i915->drm);
> - return ret;
> -}
> -EXPORT_SYMBOL_GPL(i915_gpu_busy);
> -
> -/**
> - * i915_gpu_turbo_disable - disable graphics turbo
> - *
> - * Disable graphics turbo by resetting the max frequency and setting the
> - * current frequency to the default.
> - */
> -bool i915_gpu_turbo_disable(void)
> -{
> - struct drm_i915_private *i915;
> - bool ret;
> -
> - i915 = mchdev_get();
> - if (!i915)
> - return false;
> -
> - spin_lock_irq(&mchdev_lock);
> - i915->ips.max_delay = i915->ips.fstart;
> - ret = ironlake_set_drps(i915, i915->ips.fstart);
> - spin_unlock_irq(&mchdev_lock);
> -
> - drm_dev_put(&i915->drm);
> - return ret;
> -}
> -EXPORT_SYMBOL_GPL(i915_gpu_turbo_disable);
> -
> -/**
> - * Tells the intel_ips driver that the i915 driver is now loaded, if
> - * IPS got loaded first.
> - *
> - * This awkward dance is so that neither module has to depend on the
> - * other in order for IPS to do the appropriate communication of
> - * GPU turbo limits to i915.
> - */
> -static void
> -ips_ping_for_i915_load(void)
> -{
> - void (*link)(void);
> -
> - link = symbol_get(ips_link_to_i915_driver);
> - if (link) {
> - link();
> - symbol_put(ips_link_to_i915_driver);
> - }
> -}
> -
> -void intel_gpu_ips_init(struct drm_i915_private *dev_priv)
> -{
> - /* We only register the i915 ips part with intel-ips once everything is
> - * set up, to avoid intel-ips sneaking in and reading bogus values. */
> - rcu_assign_pointer(i915_mch_dev, dev_priv);
> -
> - ips_ping_for_i915_load();
> -}
> -
> -void intel_gpu_ips_teardown(void)
> -{
> - rcu_assign_pointer(i915_mch_dev, NULL);
> -}
> -
> -static void intel_init_emon(struct drm_i915_private *dev_priv)
> -{
> - u32 lcfuse;
> - u8 pxw[16];
> - int i;
> -
> - /* Disable to program */
> - I915_WRITE(ECR, 0);
> - POSTING_READ(ECR);
> -
> - /* Program energy weights for various events */
> - I915_WRITE(SDEW, 0x15040d00);
> - I915_WRITE(CSIEW0, 0x007f0000);
> - I915_WRITE(CSIEW1, 0x1e220004);
> - I915_WRITE(CSIEW2, 0x04000004);
> -
> - for (i = 0; i < 5; i++)
> - I915_WRITE(PEW(i), 0);
> - for (i = 0; i < 3; i++)
> - I915_WRITE(DEW(i), 0);
> -
> - /* Program P-state weights to account for frequency power adjustment */
> - for (i = 0; i < 16; i++) {
> - u32 pxvidfreq = I915_READ(PXVFREQ(i));
> - unsigned long freq = intel_pxfreq(pxvidfreq);
> - unsigned long vid = (pxvidfreq & PXVFREQ_PX_MASK) >>
> - PXVFREQ_PX_SHIFT;
> - unsigned long val;
> -
> - val = vid * vid;
> - val *= (freq / 1000);
> - val *= 255;
> - val /= (127*127*900);
> - if (val > 0xff)
> - DRM_ERROR("bad pxval: %ld\n", val);
> - pxw[i] = val;
> - }
> - /* Render standby states get 0 weight */
> - pxw[14] = 0;
> - pxw[15] = 0;
> -
> - for (i = 0; i < 4; i++) {
> - u32 val = (pxw[i*4] << 24) | (pxw[(i*4)+1] << 16) |
> - (pxw[(i*4)+2] << 8) | (pxw[(i*4)+3]);
> - I915_WRITE(PXW(i), val);
> - }
> -
> - /* Adjust magic regs to magic values (more experimental results) */
> - I915_WRITE(OGW0, 0);
> - I915_WRITE(OGW1, 0);
> - I915_WRITE(EG0, 0x00007f00);
> - I915_WRITE(EG1, 0x0000000e);
> - I915_WRITE(EG2, 0x000e0000);
> - I915_WRITE(EG3, 0x68000300);
> - I915_WRITE(EG4, 0x42000000);
> - I915_WRITE(EG5, 0x00140031);
> - I915_WRITE(EG6, 0);
> - I915_WRITE(EG7, 0);
> -
> - for (i = 0; i < 8; i++)
> - I915_WRITE(PXWL(i), 0);
> -
> - /* Enable PMON + select events */
> - I915_WRITE(ECR, 0x80000019);
> -
> - lcfuse = I915_READ(LCFUSE02);
> -
> - dev_priv->ips.corr = (lcfuse & LCFUSE_HIV_MASK);
> -}
> -
> -void intel_init_gt_powersave(struct drm_i915_private *dev_priv)
> -{
> - struct intel_rps *rps = &dev_priv->gt_pm.rps;
> -
> - /* Powersaving is controlled by the host when inside a VM */
> - if (intel_vgpu_active(dev_priv))
> - mkwrite_device_info(dev_priv)->has_rps = false;
> -
> - /* Initialize RPS limits (for userspace) */
> - if (IS_CHERRYVIEW(dev_priv))
> - cherryview_init_gt_powersave(dev_priv);
> - else if (IS_VALLEYVIEW(dev_priv))
> - valleyview_init_gt_powersave(dev_priv);
> - else if (INTEL_GEN(dev_priv) >= 6)
> - gen6_init_rps_frequencies(dev_priv);
> -
> - /* Derive initial user preferences/limits from the hardware limits */
> - rps->max_freq_softlimit = rps->max_freq;
> - rps->min_freq_softlimit = rps->min_freq;
> -
> - /* After setting max-softlimit, find the overclock max freq */
> - if (IS_GEN(dev_priv, 6) ||
> - IS_IVYBRIDGE(dev_priv) || IS_HASWELL(dev_priv)) {
> - u32 params = 0;
> -
> - sandybridge_pcode_read(dev_priv, GEN6_READ_OC_PARAMS,
> - ¶ms, NULL);
> - if (params & BIT(31)) { /* OC supported */
> - DRM_DEBUG_DRIVER("Overclocking supported, max: %dMHz, overclock: %dMHz\n",
> - (rps->max_freq & 0xff) * 50,
> - (params & 0xff) * 50);
> - rps->max_freq = params & 0xff;
> - }
> - }
> -
> - /* Finally allow us to boost to max by default */
> - rps->boost_freq = rps->max_freq;
> - rps->idle_freq = rps->min_freq;
> - rps->cur_freq = rps->idle_freq;
> -}
> -
> -void intel_sanitize_gt_powersave(struct drm_i915_private *dev_priv)
> -{
> - dev_priv->gt_pm.rps.enabled = true; /* force RPS disabling */
> - intel_disable_gt_powersave(dev_priv);
> -
> - if (INTEL_GEN(dev_priv) >= 11)
> - gen11_reset_rps_interrupts(dev_priv);
> - else if (INTEL_GEN(dev_priv) >= 6)
> - gen6_reset_rps_interrupts(dev_priv);
> -}
> -
> -static void intel_disable_rps(struct drm_i915_private *dev_priv)
> -{
> - lockdep_assert_held(&dev_priv->gt_pm.rps.lock);
> -
> - if (!dev_priv->gt_pm.rps.enabled)
> - return;
> -
> - if (INTEL_GEN(dev_priv) >= 9)
> - gen9_disable_rps(dev_priv);
> - else if (IS_CHERRYVIEW(dev_priv))
> - cherryview_disable_rps(dev_priv);
> - else if (IS_VALLEYVIEW(dev_priv))
> - valleyview_disable_rps(dev_priv);
> - else if (INTEL_GEN(dev_priv) >= 6)
> - gen6_disable_rps(dev_priv);
> - else if (IS_IRONLAKE_M(dev_priv))
> - ironlake_disable_drps(dev_priv);
> -
> - dev_priv->gt_pm.rps.enabled = false;
> -}
> -
> -void intel_disable_gt_powersave(struct drm_i915_private *dev_priv)
> -{
> - mutex_lock(&dev_priv->gt_pm.rps.lock);
> -
> - intel_disable_rps(dev_priv);
> - if (HAS_LLC(dev_priv))
> - intel_llc_disable(&dev_priv->gt.llc);
> -
> - mutex_unlock(&dev_priv->gt_pm.rps.lock);
> -}
> -
> -static void intel_enable_rps(struct drm_i915_private *dev_priv)
> -{
> - struct intel_rps *rps = &dev_priv->gt_pm.rps;
> -
> - lockdep_assert_held(&rps->lock);
> -
> - if (rps->enabled)
> - return;
> -
> - if (IS_CHERRYVIEW(dev_priv)) {
> - cherryview_enable_rps(dev_priv);
> - } else if (IS_VALLEYVIEW(dev_priv)) {
> - valleyview_enable_rps(dev_priv);
> - } else if (INTEL_GEN(dev_priv) >= 9) {
> - gen9_enable_rps(dev_priv);
> - } else if (IS_BROADWELL(dev_priv)) {
> - gen8_enable_rps(dev_priv);
> - } else if (INTEL_GEN(dev_priv) >= 6) {
> - gen6_enable_rps(dev_priv);
> - } else if (IS_IRONLAKE_M(dev_priv)) {
> - ironlake_enable_drps(dev_priv);
> - intel_init_emon(dev_priv);
> - }
> -
> - WARN_ON(rps->max_freq < rps->min_freq);
> - WARN_ON(rps->idle_freq > rps->max_freq);
> -
> - WARN_ON(rps->efficient_freq < rps->min_freq);
> - WARN_ON(rps->efficient_freq > rps->max_freq);
> -
> - rps->enabled = true;
> -}
> -
> -void intel_enable_gt_powersave(struct drm_i915_private *dev_priv)
> -{
> - /* Powersaving is controlled by the host when inside a VM */
> - if (intel_vgpu_active(dev_priv))
> - return;
> -
> - mutex_lock(&dev_priv->gt_pm.rps.lock);
> -
> - if (HAS_RPS(dev_priv))
> - intel_enable_rps(dev_priv);
> -
> - intel_llc_enable(&dev_priv->gt.llc);
> -
> - mutex_unlock(&dev_priv->gt_pm.rps.lock);
> -}
> -
> -static void ibx_init_clock_gating(struct drm_i915_private *dev_priv)
> -{
> - /*
> - * On Ibex Peak and Cougar Point, we need to disable clock
> - * gating for the panel power sequencer or it will fail to
> - * start up when no ports are active.
> - */
> - I915_WRITE(SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE);
> -}
> -
> -static void g4x_disable_trickle_feed(struct drm_i915_private *dev_priv)
> -{
> - enum pipe pipe;
> -
> - for_each_pipe(dev_priv, pipe) {
> - I915_WRITE(DSPCNTR(pipe),
> - I915_READ(DSPCNTR(pipe)) |
> - DISPPLANE_TRICKLE_FEED_DISABLE);
> -
> - I915_WRITE(DSPSURF(pipe), I915_READ(DSPSURF(pipe)));
> - POSTING_READ(DSPSURF(pipe));
> - }
> -}
> -
> -static void ilk_init_clock_gating(struct drm_i915_private *dev_priv)
> -{
> - u32 dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE;
> -
> - /*
> - * Required for FBC
> - * WaFbcDisableDpfcClockGating:ilk
> - */
> - dspclk_gate |= ILK_DPFCRUNIT_CLOCK_GATE_DISABLE |
> - ILK_DPFCUNIT_CLOCK_GATE_DISABLE |
> - ILK_DPFDUNIT_CLOCK_GATE_ENABLE;
> -
> - I915_WRITE(PCH_3DCGDIS0,
> - MARIUNIT_CLOCK_GATE_DISABLE |
> - SVSMUNIT_CLOCK_GATE_DISABLE);
> - I915_WRITE(PCH_3DCGDIS1,
> - VFMUNIT_CLOCK_GATE_DISABLE);
> -
> - /*
> - * According to the spec the following bits should be set in
> - * order to enable memory self-refresh
> - * The bit 22/21 of 0x42004
> - * The bit 5 of 0x42020
> - * The bit 15 of 0x45000
> - */
> - I915_WRITE(ILK_DISPLAY_CHICKEN2,
> - (I915_READ(ILK_DISPLAY_CHICKEN2) |
> - ILK_DPARB_GATE | ILK_VSDPFD_FULL));
> - dspclk_gate |= ILK_DPARBUNIT_CLOCK_GATE_ENABLE;
> - I915_WRITE(DISP_ARB_CTL,
> - (I915_READ(DISP_ARB_CTL) |
> - DISP_FBC_WM_DIS));
> -
> - /*
> - * Based on the document from hardware guys the following bits
> - * should be set unconditionally in order to enable FBC.
> - * The bit 22 of 0x42000
> - * The bit 22 of 0x42004
> - * The bit 7,8,9 of 0x42020.
> - */
> - if (IS_IRONLAKE_M(dev_priv)) {
> - /* WaFbcAsynchFlipDisableFbcQueue:ilk */
> - I915_WRITE(ILK_DISPLAY_CHICKEN1,
> - I915_READ(ILK_DISPLAY_CHICKEN1) |
> - ILK_FBCQ_DIS);
> - I915_WRITE(ILK_DISPLAY_CHICKEN2,
> - I915_READ(ILK_DISPLAY_CHICKEN2) |
> - ILK_DPARB_GATE);
> - }
> -
> - I915_WRITE(ILK_DSPCLK_GATE_D, dspclk_gate);
> -
> - I915_WRITE(ILK_DISPLAY_CHICKEN2,
> - I915_READ(ILK_DISPLAY_CHICKEN2) |
> - ILK_ELPIN_409_SELECT);
> - I915_WRITE(_3D_CHICKEN2,
> - _3D_CHICKEN2_WM_READ_PIPELINED << 16 |
> - _3D_CHICKEN2_WM_READ_PIPELINED);
> -
> - /* WaDisableRenderCachePipelinedFlush:ilk */
> - I915_WRITE(CACHE_MODE_0,
> - _MASKED_BIT_ENABLE(CM0_PIPELINED_RENDER_FLUSH_DISABLE));
> -
> - /* WaDisable_RenderCache_OperationalFlush:ilk */
> - I915_WRITE(CACHE_MODE_0, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
> -
> - g4x_disable_trickle_feed(dev_priv);
> -
> - ibx_init_clock_gating(dev_priv);
> -}
> -
> -static void cpt_init_clock_gating(struct drm_i915_private *dev_priv)
> -{
> - enum pipe pipe;
> - u32 val;
> -
> - /*
> - * On Ibex Peak and Cougar Point, we need to disable clock
> - * gating for the panel power sequencer or it will fail to
> - * start up when no ports are active.
> - */
> - I915_WRITE(SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE |
> - PCH_DPLUNIT_CLOCK_GATE_DISABLE |
> - PCH_CPUNIT_CLOCK_GATE_DISABLE);
> - I915_WRITE(SOUTH_CHICKEN2, I915_READ(SOUTH_CHICKEN2) |
> - DPLS_EDP_PPS_FIX_DIS);
> - /* The below fixes the weird display corruption, a few pixels shifted
> - * downward, on (only) LVDS of some HP laptops with IVY.
> - */
> - for_each_pipe(dev_priv, pipe) {
> - val = I915_READ(TRANS_CHICKEN2(pipe));
> - val |= TRANS_CHICKEN2_TIMING_OVERRIDE;
> - val &= ~TRANS_CHICKEN2_FDI_POLARITY_REVERSED;
> - if (dev_priv->vbt.fdi_rx_polarity_inverted)
> - val |= TRANS_CHICKEN2_FDI_POLARITY_REVERSED;
> - val &= ~TRANS_CHICKEN2_FRAME_START_DELAY_MASK;
> - val &= ~TRANS_CHICKEN2_DISABLE_DEEP_COLOR_COUNTER;
> - val &= ~TRANS_CHICKEN2_DISABLE_DEEP_COLOR_MODESWITCH;
> - I915_WRITE(TRANS_CHICKEN2(pipe), val);
> - }
> - /* WADP0ClockGatingDisable */
> - for_each_pipe(dev_priv, pipe) {
> - I915_WRITE(TRANS_CHICKEN1(pipe),
> - TRANS_CHICKEN1_DP0UNIT_GC_DISABLE);
> - }
> -}
> -
> -static void gen6_check_mch_setup(struct drm_i915_private *dev_priv)
> -{
> - u32 tmp;
> -
> - tmp = I915_READ(MCH_SSKPD);
> - if ((tmp & MCH_SSKPD_WM0_MASK) != MCH_SSKPD_WM0_VAL)
> - DRM_DEBUG_KMS("Wrong MCH_SSKPD value: 0x%08x This can cause underruns.\n",
> - tmp);
> -}
> -
> -static void gen6_init_clock_gating(struct drm_i915_private *dev_priv)
> -{
> - u32 dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE;
> -
> - I915_WRITE(ILK_DSPCLK_GATE_D, dspclk_gate);
> -
> - I915_WRITE(ILK_DISPLAY_CHICKEN2,
> - I915_READ(ILK_DISPLAY_CHICKEN2) |
> - ILK_ELPIN_409_SELECT);
> -
> - /* WaDisableHiZPlanesWhenMSAAEnabled:snb */
> - I915_WRITE(_3D_CHICKEN,
> - _MASKED_BIT_ENABLE(_3D_CHICKEN_HIZ_PLANE_DISABLE_MSAA_4X_SNB));
> -
> - /* WaDisable_RenderCache_OperationalFlush:snb */
> - I915_WRITE(CACHE_MODE_0, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
> -
> - /*
> - * BSpec recoomends 8x4 when MSAA is used,
> - * however in practice 16x4 seems fastest.
> - *
> - * Note that PS/WM thread counts depend on the WIZ hashing
> - * disable bit, which we don't touch here, but it's good
> - * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM).
> - */
> - I915_WRITE(GEN6_GT_MODE,
> - _MASKED_FIELD(GEN6_WIZ_HASHING_MASK, GEN6_WIZ_HASHING_16x4));
> -
> - I915_WRITE(CACHE_MODE_0,
> - _MASKED_BIT_DISABLE(CM0_STC_EVICT_DISABLE_LRA_SNB));
> -
> - I915_WRITE(GEN6_UCGCTL1,
> - I915_READ(GEN6_UCGCTL1) |
> - GEN6_BLBUNIT_CLOCK_GATE_DISABLE |
> - GEN6_CSUNIT_CLOCK_GATE_DISABLE);
> -
> - /* According to the BSpec vol1g, bit 12 (RCPBUNIT) clock
> - * gating disable must be set. Failure to set it results in
> - * flickering pixels due to Z write ordering failures after
> - * some amount of runtime in the Mesa "fire" demo, and Unigine
> - * Sanctuary and Tropics, and apparently anything else with
> - * alpha test or pixel discard.
> - *
> - * According to the spec, bit 11 (RCCUNIT) must also be set,
> - * but we didn't debug actual testcases to find it out.
> - *
> - * WaDisableRCCUnitClockGating:snb
> - * WaDisableRCPBUnitClockGating:snb
> - */
> - I915_WRITE(GEN6_UCGCTL2,
> - GEN6_RCPBUNIT_CLOCK_GATE_DISABLE |
> - GEN6_RCCUNIT_CLOCK_GATE_DISABLE);
> -
> - /* WaStripsFansDisableFastClipPerformanceFix:snb */
> - I915_WRITE(_3D_CHICKEN3,
> - _MASKED_BIT_ENABLE(_3D_CHICKEN3_SF_DISABLE_FASTCLIP_CULL));
> -
> - /*
> - * Bspec says:
> - * "This bit must be set if 3DSTATE_CLIP clip mode is set to normal and
> - * 3DSTATE_SF number of SF output attributes is more than 16."
> - */
> - I915_WRITE(_3D_CHICKEN3,
> - _MASKED_BIT_ENABLE(_3D_CHICKEN3_SF_DISABLE_PIPELINED_ATTR_FETCH));
> -
> - /*
> - * According to the spec the following bits should be
> - * set in order to enable memory self-refresh and fbc:
> - * The bit21 and bit22 of 0x42000
> - * The bit21 and bit22 of 0x42004
> - * The bit5 and bit7 of 0x42020
> - * The bit14 of 0x70180
> - * The bit14 of 0x71180
> - *
> - * WaFbcAsynchFlipDisableFbcQueue:snb
> - */
> - I915_WRITE(ILK_DISPLAY_CHICKEN1,
> - I915_READ(ILK_DISPLAY_CHICKEN1) |
> - ILK_FBCQ_DIS | ILK_PABSTRETCH_DIS);
> - I915_WRITE(ILK_DISPLAY_CHICKEN2,
> - I915_READ(ILK_DISPLAY_CHICKEN2) |
> - ILK_DPARB_GATE | ILK_VSDPFD_FULL);
> - I915_WRITE(ILK_DSPCLK_GATE_D,
> - I915_READ(ILK_DSPCLK_GATE_D) |
> - ILK_DPARBUNIT_CLOCK_GATE_ENABLE |
> - ILK_DPFDUNIT_CLOCK_GATE_ENABLE);
> -
> - g4x_disable_trickle_feed(dev_priv);
> -
> - cpt_init_clock_gating(dev_priv);
> -
> - gen6_check_mch_setup(dev_priv);
> -}
> -
> -static void gen7_setup_fixed_func_scheduler(struct drm_i915_private *dev_priv)
> -{
> - u32 reg = I915_READ(GEN7_FF_THREAD_MODE);
> + u32 reg = I915_READ(GEN7_FF_THREAD_MODE);
>
> /*
> * WaVSThreadDispatchOverride:ivb,vlv
> @@ -8942,90 +7325,8 @@ void intel_init_pm(struct drm_i915_private *dev_priv)
> }
> }
>
> -static int byt_gpu_freq(struct drm_i915_private *dev_priv, int val)
> -{
> - struct intel_rps *rps = &dev_priv->gt_pm.rps;
> -
> - /*
> - * N = val - 0xb7
> - * Slow = Fast = GPLL ref * N
> - */
> - return DIV_ROUND_CLOSEST(rps->gpll_ref_freq * (val - 0xb7), 1000);
> -}
> -
> -static int byt_freq_opcode(struct drm_i915_private *dev_priv, int val)
> -{
> - struct intel_rps *rps = &dev_priv->gt_pm.rps;
> -
> - return DIV_ROUND_CLOSEST(1000 * val, rps->gpll_ref_freq) + 0xb7;
> -}
> -
> -static int chv_gpu_freq(struct drm_i915_private *dev_priv, int val)
> -{
> - struct intel_rps *rps = &dev_priv->gt_pm.rps;
> -
> - /*
> - * N = val / 2
> - * CU (slow) = CU2x (fast) / 2 = GPLL ref * N / 2
> - */
> - return DIV_ROUND_CLOSEST(rps->gpll_ref_freq * val, 2 * 2 * 1000);
> -}
> -
> -static int chv_freq_opcode(struct drm_i915_private *dev_priv, int val)
> -{
> - struct intel_rps *rps = &dev_priv->gt_pm.rps;
> -
> - /* CHV needs even values */
> - return DIV_ROUND_CLOSEST(2 * 1000 * val, rps->gpll_ref_freq) * 2;
> -}
> -
> -int intel_gpu_freq(struct drm_i915_private *dev_priv, int val)
> -{
> - if (INTEL_GEN(dev_priv) >= 9)
> - return DIV_ROUND_CLOSEST(val * GT_FREQUENCY_MULTIPLIER,
> - GEN9_FREQ_SCALER);
> - else if (IS_CHERRYVIEW(dev_priv))
> - return chv_gpu_freq(dev_priv, val);
> - else if (IS_VALLEYVIEW(dev_priv))
> - return byt_gpu_freq(dev_priv, val);
> - else
> - return val * GT_FREQUENCY_MULTIPLIER;
> -}
> -
> -int intel_freq_opcode(struct drm_i915_private *dev_priv, int val)
> -{
> - if (INTEL_GEN(dev_priv) >= 9)
> - return DIV_ROUND_CLOSEST(val * GEN9_FREQ_SCALER,
> - GT_FREQUENCY_MULTIPLIER);
> - else if (IS_CHERRYVIEW(dev_priv))
> - return chv_freq_opcode(dev_priv, val);
> - else if (IS_VALLEYVIEW(dev_priv))
> - return byt_freq_opcode(dev_priv, val);
> - else
> - return DIV_ROUND_CLOSEST(val, GT_FREQUENCY_MULTIPLIER);
> -}
> -
> void intel_pm_setup(struct drm_i915_private *dev_priv)
> {
> - mutex_init(&dev_priv->gt_pm.rps.lock);
> - mutex_init(&dev_priv->gt_pm.rps.power.mutex);
> -
> - atomic_set(&dev_priv->gt_pm.rps.num_waiters, 0);
> -
> dev_priv->runtime_pm.suspended = false;
> atomic_set(&dev_priv->runtime_pm.wakeref_count, 0);
> }
> -
> -u32 intel_get_cagf(struct drm_i915_private *dev_priv, u32 rpstat)
> -{
> - u32 cagf;
> -
> - if (INTEL_GEN(dev_priv) >= 9)
> - cagf = (rpstat & GEN9_CAGF_MASK) >> GEN9_CAGF_SHIFT;
> - else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
> - cagf = (rpstat & HSW_CAGF_MASK) >> HSW_CAGF_SHIFT;
> - else
> - cagf = (rpstat & GEN6_CAGF_MASK) >> GEN6_CAGF_SHIFT;
> -
> - return cagf;
> -}
> diff --git a/drivers/gpu/drm/i915/intel_pm.h b/drivers/gpu/drm/i915/intel_pm.h
> index 93d192d0610a..b56e6285d1c3 100644
> --- a/drivers/gpu/drm/i915/intel_pm.h
> +++ b/drivers/gpu/drm/i915/intel_pm.h
> @@ -29,15 +29,6 @@ void intel_update_watermarks(struct intel_crtc *crtc);
> void intel_init_pm(struct drm_i915_private *dev_priv);
> void intel_init_clock_gating_hooks(struct drm_i915_private *dev_priv);
> void intel_pm_setup(struct drm_i915_private *dev_priv);
> -void intel_gpu_ips_init(struct drm_i915_private *dev_priv);
> -void intel_gpu_ips_teardown(void);
> -void intel_init_gt_powersave(struct drm_i915_private *dev_priv);
> -void intel_sanitize_gt_powersave(struct drm_i915_private *dev_priv);
> -void intel_enable_gt_powersave(struct drm_i915_private *dev_priv);
> -void intel_disable_gt_powersave(struct drm_i915_private *dev_priv);
> -void gen6_rps_busy(struct drm_i915_private *dev_priv);
> -void gen6_rps_idle(struct drm_i915_private *dev_priv);
> -void gen6_rps_boost(struct i915_request *rq);
> void g4x_wm_get_hw_state(struct drm_i915_private *dev_priv);
> void vlv_wm_get_hw_state(struct drm_i915_private *dev_priv);
> void ilk_wm_get_hw_state(struct drm_i915_private *dev_priv);
> @@ -69,19 +60,6 @@ int skl_check_pipe_max_pixel_rate(struct intel_crtc *intel_crtc,
> void intel_init_ipc(struct drm_i915_private *dev_priv);
> void intel_enable_ipc(struct drm_i915_private *dev_priv);
>
> -int intel_gpu_freq(struct drm_i915_private *dev_priv, int val);
> -int intel_freq_opcode(struct drm_i915_private *dev_priv, int val);
> -
> -u32 intel_get_cagf(struct drm_i915_private *dev_priv, u32 rpstat1);
> -
> -unsigned long i915_chipset_val(struct drm_i915_private *dev_priv);
> -unsigned long i915_mch_val(struct drm_i915_private *dev_priv);
> -unsigned long i915_gfx_val(struct drm_i915_private *dev_priv);
> -void i915_update_gfx_val(struct drm_i915_private *dev_priv);
> -
> -bool ironlake_set_drps(struct drm_i915_private *dev_priv, u8 val);
> -int intel_set_rps(struct drm_i915_private *dev_priv, u8 val);
> -void intel_rps_mark_interactive(struct drm_i915_private *i915, bool interactive);
> bool intel_set_memory_cxsr(struct drm_i915_private *dev_priv, bool enable);
>
> #endif /* __INTEL_PM_H__ */
> --
> 2.23.0
_______________________________________________
Intel-gfx mailing list
Intel-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/intel-gfx
next prev parent reply other threads:[~2019-10-23 9:52 UTC|newest]
Thread overview: 24+ messages / expand[flat|nested] mbox.gz Atom feed top
2019-10-23 9:38 [PATCH 0/2] Extract rps and guc Andi Shyti
2019-10-23 9:38 ` [Intel-gfx] " Andi Shyti
2019-10-23 9:38 ` [PATCH 1/2] drm/i915: Extract GT render power state management Andi Shyti
2019-10-23 9:38 ` [Intel-gfx] " Andi Shyti
2019-10-23 9:38 ` [PATCH 2/2] drm/i915: Extract the GuC interrupt handlers Andi Shyti
2019-10-23 9:38 ` [Intel-gfx] " Andi Shyti
2019-10-23 9:50 ` [PATCH v2 1/2] drm/i915: Extract GT render power state management Andi Shyti
2019-10-23 9:50 ` [Intel-gfx] " Andi Shyti
2019-10-23 9:52 ` Andi Shyti [this message]
2019-10-23 9:52 ` Andi Shyti
2019-10-23 12:11 ` Chris Wilson
2019-10-23 12:11 ` [Intel-gfx] " Chris Wilson
2019-10-23 12:32 ` Andi Shyti
2019-10-23 12:32 ` [Intel-gfx] " Andi Shyti
2019-10-23 12:36 ` Chris Wilson
2019-10-23 12:36 ` [Intel-gfx] " Chris Wilson
2019-10-23 12:49 ` Andi Shyti
2019-10-23 12:49 ` [Intel-gfx] " Andi Shyti
2019-10-23 17:48 ` ✗ Fi.CI.CHECKPATCH: warning for Extract rps and guc Patchwork
2019-10-23 17:48 ` [Intel-gfx] " Patchwork
2019-10-23 18:17 ` ✓ Fi.CI.BAT: success " Patchwork
2019-10-23 18:17 ` [Intel-gfx] " Patchwork
2019-10-24 10:24 ` ✗ Fi.CI.IGT: failure " Patchwork
2019-10-24 10:24 ` [Intel-gfx] " Patchwork
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20191023095248.GA2593@intel.intel \
--to=andi.shyti@intel.com \
--cc=intel-gfx@lists.freedesktop.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.