Re: [PATCH 01/10] drm/i915: Disable preemption and sleeping while using the punit sideband

["Ville Syrjälä" <ville.syrjala@linux.intel.com>]

On Fri, Apr 19, 2019 at 06:13:53PM +0100, Chris Wilson wrote:
> While we talk to the punit over its sideband, we need to prevent the cpu
> from sleeping in order to prevent a potential machine hang.
> 
> Note that by itself, it appears that pm_qos_update_request (via
> intel_idle) doesn't provide a sufficient barrier to ensure that all core
> are indeed awake (out of Cstate) and that the package is awake. To do so,
> we need to supplement the pm_qos with a manual ping on_each_cpu.
> 
> v2: Restrict the heavy-weight wakeup to just the ISOF_PORT_PUNIT, there
> is insufficient evidence to implicate a wider problem atm. Similarly,
> restrict the w/a to Valleyview, as Cherryview doesn't have an angry cadre
> of users.
> 
> The working theory, courtesy of Ville and Hans, is the issue lies within
> the power delivery and so is likely to be unit and board specific and
> occurs when both the unit/fw require extra power at the same time as the
> cpu package is changing its own power state.
> 
> References: https://bugzilla.kernel.org/show_bug.cgi?id=109051
> References: https://bugs.freedesktop.org/show_bug.cgi?id=102657
> References: https://bugzilla.kernel.org/show_bug.cgi?id=195255
> Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk>
> Cc: Mika Kuoppala <mika.kuoppala@linux.intel.com>
> Cc: Hans de Goede <hdegoede@redhat.com>
> Cc: Ville Syrjälä <ville.syrjala@linux.intel.com>
> ---
>  drivers/gpu/drm/i915/i915_drv.c       |   6 +
>  drivers/gpu/drm/i915/i915_drv.h       |   1 +
>  drivers/gpu/drm/i915/intel_sideband.c | 203 +++++++++++++++++---------
>  3 files changed, 139 insertions(+), 71 deletions(-)
> 
> diff --git a/drivers/gpu/drm/i915/i915_drv.c b/drivers/gpu/drm/i915/i915_drv.c
> index 5e2ae2300454..9e657a0410c2 100644
> --- a/drivers/gpu/drm/i915/i915_drv.c
> +++ b/drivers/gpu/drm/i915/i915_drv.c
> @@ -884,6 +884,9 @@ static int i915_driver_init_early(struct drm_i915_private *dev_priv)
>  	mutex_init(&dev_priv->backlight_lock);
>  
>  	mutex_init(&dev_priv->sb_lock);
> +	pm_qos_add_request(&dev_priv->sb_qos,
> +			   PM_QOS_CPU_DMA_LATENCY, PM_QOS_DEFAULT_VALUE);
> +
>  	mutex_init(&dev_priv->av_mutex);
>  	mutex_init(&dev_priv->wm.wm_mutex);
>  	mutex_init(&dev_priv->pps_mutex);
> @@ -943,6 +946,9 @@ static void i915_driver_cleanup_early(struct drm_i915_private *dev_priv)
>  	i915_gem_cleanup_early(dev_priv);
>  	i915_workqueues_cleanup(dev_priv);
>  	i915_engines_cleanup(dev_priv);
> +
> +	pm_qos_remove_request(&dev_priv->sb_qos);
> +	mutex_destroy(&dev_priv->sb_lock);
>  }
>  
>  /**
> diff --git a/drivers/gpu/drm/i915/i915_drv.h b/drivers/gpu/drm/i915/i915_drv.h
> index 71612e7fc8bc..afb979ff416f 100644
> --- a/drivers/gpu/drm/i915/i915_drv.h
> +++ b/drivers/gpu/drm/i915/i915_drv.h
> @@ -1561,6 +1561,7 @@ struct drm_i915_private {
>  
>  	/* Sideband mailbox protection */
>  	struct mutex sb_lock;
> +	struct pm_qos_request sb_qos;

Should we call it punit_qos maybe?

I'm a bit sad to all the complexity for this, but no one has come up
with a way to really fix it so I guess we just have to swallow this.

Reviewed-by: Ville Syrjälä <ville.syrjala@linux.intel.com>

>  
>  	/** Cached value of IMR to avoid reads in updating the bitfield */
>  	union {
> diff --git a/drivers/gpu/drm/i915/intel_sideband.c b/drivers/gpu/drm/i915/intel_sideband.c
> index 57de41b1f989..fc8913461622 100644
> --- a/drivers/gpu/drm/i915/intel_sideband.c
> +++ b/drivers/gpu/drm/i915/intel_sideband.c
> @@ -22,6 +22,8 @@
>   *
>   */
>  
> +#include <asm/iosf_mbi.h>
> +
>  #include "i915_drv.h"
>  #include "intel_drv.h"
>  
> @@ -39,19 +41,50 @@
>  /* Private register write, double-word addressing, non-posted */
>  #define SB_CRWRDA_NP	0x07
>  
> -static int vlv_sideband_rw(struct drm_i915_private *dev_priv, u32 devfn,
> -			   u32 port, u32 opcode, u32 addr, u32 *val)
> +static void ping(void *info)
>  {
> -	u32 cmd, be = 0xf, bar = 0;
> -	bool is_read = (opcode == SB_MRD_NP || opcode == SB_CRRDDA_NP);
> +}
>  
> -	cmd = (devfn << IOSF_DEVFN_SHIFT) | (opcode << IOSF_OPCODE_SHIFT) |
> -		(port << IOSF_PORT_SHIFT) | (be << IOSF_BYTE_ENABLES_SHIFT) |
> -		(bar << IOSF_BAR_SHIFT);
> +static void __vlv_punit_get(struct drm_i915_private *i915)
> +{
> +	iosf_mbi_punit_acquire();
>  
> -	WARN_ON(!mutex_is_locked(&dev_priv->sb_lock));
> +	/*
> +	 * Prevent the cpu from sleeping while we use this sideband, otherwise
> +	 * the punit may cause a machine hang. The issue appears to be isolated
> +	 * with changing the power state of the CPU package while changing
> +	 * the power state via the punit, and we have only observed it
> +	 * reliably on 4-core Baytail systems suggesting the issue is in the
> +	 * power delivery mechanism and likely to be be board/function
> +	 * specific. Hence we presume the workaround needs only be applied
> +	 * to the Valleyview P-unit and not all sideband communications.
> +	 */
> +	if (IS_VALLEYVIEW(i915)) {
> +		pm_qos_update_request(&i915->sb_qos, 0);
> +		on_each_cpu(ping, NULL, 1);
> +	}
> +}
>  
> -	if (intel_wait_for_register(&dev_priv->uncore,
> +static void __vlv_punit_put(struct drm_i915_private *i915)
> +{
> +	if (IS_VALLEYVIEW(i915))
> +		pm_qos_update_request(&i915->sb_qos, PM_QOS_DEFAULT_VALUE);
> +
> +	iosf_mbi_punit_release();
> +}
> +
> +static int vlv_sideband_rw(struct drm_i915_private *i915,
> +			   u32 devfn, u32 port, u32 opcode,
> +			   u32 addr, u32 *val)
> +{
> +	struct intel_uncore *uncore = &i915->uncore;
> +	const bool is_read = (opcode == SB_MRD_NP || opcode == SB_CRRDDA_NP);
> +	int err;
> +
> +	lockdep_assert_held(&i915->sb_lock);
> +
> +	/* Flush the previous comms, just in case it failed last time. */
> +	if (intel_wait_for_register(uncore,
>  				    VLV_IOSF_DOORBELL_REQ, IOSF_SB_BUSY, 0,
>  				    5)) {
>  		DRM_DEBUG_DRIVER("IOSF sideband idle wait (%s) timed out\n",
> @@ -59,131 +92,156 @@ static int vlv_sideband_rw(struct drm_i915_private *dev_priv, u32 devfn,
>  		return -EAGAIN;
>  	}
>  
> -	I915_WRITE(VLV_IOSF_ADDR, addr);
> -	I915_WRITE(VLV_IOSF_DATA, is_read ? 0 : *val);
> -	I915_WRITE(VLV_IOSF_DOORBELL_REQ, cmd);
> -
> -	if (intel_wait_for_register(&dev_priv->uncore,
> -				    VLV_IOSF_DOORBELL_REQ, IOSF_SB_BUSY, 0,
> -				    5)) {
> +	preempt_disable();
> +
> +	intel_uncore_write_fw(uncore, VLV_IOSF_ADDR, addr);
> +	intel_uncore_write_fw(uncore, VLV_IOSF_DATA, is_read ? 0 : *val);
> +	intel_uncore_write_fw(uncore, VLV_IOSF_DOORBELL_REQ,
> +			      (devfn << IOSF_DEVFN_SHIFT) |
> +			      (opcode << IOSF_OPCODE_SHIFT) |
> +			      (port << IOSF_PORT_SHIFT) |
> +			      (0xf << IOSF_BYTE_ENABLES_SHIFT) |
> +			      (0 << IOSF_BAR_SHIFT) |
> +			      IOSF_SB_BUSY);
> +
> +	if (__intel_wait_for_register_fw(uncore,
> +					 VLV_IOSF_DOORBELL_REQ, IOSF_SB_BUSY, 0,
> +					 10000, 0, NULL) == 0) {
> +		if (is_read)
> +			*val = intel_uncore_read_fw(uncore, VLV_IOSF_DATA);
> +		err = 0;
> +	} else {
>  		DRM_DEBUG_DRIVER("IOSF sideband finish wait (%s) timed out\n",
>  				 is_read ? "read" : "write");
> -		return -ETIMEDOUT;
> +		err = -ETIMEDOUT;
>  	}
>  
> -	if (is_read)
> -		*val = I915_READ(VLV_IOSF_DATA);
> +	preempt_enable();
>  
> -	return 0;
> +	return err;
>  }
>  
> -u32 vlv_punit_read(struct drm_i915_private *dev_priv, u32 addr)
> +u32 vlv_punit_read(struct drm_i915_private *i915, u32 addr)
>  {
>  	u32 val = 0;
>  
> -	WARN_ON(!mutex_is_locked(&dev_priv->pcu_lock));
> +	WARN_ON(!mutex_is_locked(&i915->pcu_lock));
> +
> +	mutex_lock(&i915->sb_lock);
> +	__vlv_punit_get(i915);
>  
> -	mutex_lock(&dev_priv->sb_lock);
> -	vlv_sideband_rw(dev_priv, PCI_DEVFN(0, 0), IOSF_PORT_PUNIT,
> +	vlv_sideband_rw(i915, PCI_DEVFN(0, 0), IOSF_PORT_PUNIT,
>  			SB_CRRDDA_NP, addr, &val);
> -	mutex_unlock(&dev_priv->sb_lock);
> +
> +	__vlv_punit_put(i915);
> +	mutex_unlock(&i915->sb_lock);
>  
>  	return val;
>  }
>  
> -int vlv_punit_write(struct drm_i915_private *dev_priv, u32 addr, u32 val)
> +int vlv_punit_write(struct drm_i915_private *i915, u32 addr, u32 val)
>  {
>  	int err;
>  
> -	WARN_ON(!mutex_is_locked(&dev_priv->pcu_lock));
> +	WARN_ON(!mutex_is_locked(&i915->pcu_lock));
>  
> -	mutex_lock(&dev_priv->sb_lock);
> -	err = vlv_sideband_rw(dev_priv, PCI_DEVFN(0, 0), IOSF_PORT_PUNIT,
> +	mutex_lock(&i915->sb_lock);
> +	__vlv_punit_get(i915);
> +
> +	err = vlv_sideband_rw(i915, PCI_DEVFN(0, 0), IOSF_PORT_PUNIT,
>  			      SB_CRWRDA_NP, addr, &val);
> -	mutex_unlock(&dev_priv->sb_lock);
> +
> +	__vlv_punit_put(i915);
> +	mutex_unlock(&i915->sb_lock);
>  
>  	return err;
>  }
>  
> -u32 vlv_bunit_read(struct drm_i915_private *dev_priv, u32 reg)
> +u32 vlv_bunit_read(struct drm_i915_private *i915, u32 reg)
>  {
>  	u32 val = 0;
>  
> -	vlv_sideband_rw(dev_priv, PCI_DEVFN(0, 0), IOSF_PORT_BUNIT,
> +	vlv_sideband_rw(i915, PCI_DEVFN(0, 0), IOSF_PORT_BUNIT,
>  			SB_CRRDDA_NP, reg, &val);
>  
>  	return val;
>  }
>  
> -void vlv_bunit_write(struct drm_i915_private *dev_priv, u32 reg, u32 val)
> +void vlv_bunit_write(struct drm_i915_private *i915, u32 reg, u32 val)
>  {
> -	vlv_sideband_rw(dev_priv, PCI_DEVFN(0, 0), IOSF_PORT_BUNIT,
> +	vlv_sideband_rw(i915, PCI_DEVFN(0, 0), IOSF_PORT_BUNIT,
>  			SB_CRWRDA_NP, reg, &val);
>  }
>  
> -u32 vlv_nc_read(struct drm_i915_private *dev_priv, u8 addr)
> +u32 vlv_nc_read(struct drm_i915_private *i915, u8 addr)
>  {
>  	u32 val = 0;
>  
> -	WARN_ON(!mutex_is_locked(&dev_priv->pcu_lock));
> +	WARN_ON(!mutex_is_locked(&i915->pcu_lock));
>  
> -	mutex_lock(&dev_priv->sb_lock);
> -	vlv_sideband_rw(dev_priv, PCI_DEVFN(0, 0), IOSF_PORT_NC,
> +	mutex_lock(&i915->sb_lock);
> +	vlv_sideband_rw(i915, PCI_DEVFN(0, 0), IOSF_PORT_NC,
>  			SB_CRRDDA_NP, addr, &val);
> -	mutex_unlock(&dev_priv->sb_lock);
> +	mutex_unlock(&i915->sb_lock);
>  
>  	return val;
>  }
>  
> -u32 vlv_iosf_sb_read(struct drm_i915_private *dev_priv, u8 port, u32 reg)
> +u32 vlv_iosf_sb_read(struct drm_i915_private *i915, u8 port, u32 reg)
>  {
>  	u32 val = 0;
> -	vlv_sideband_rw(dev_priv, PCI_DEVFN(0, 0), port,
> +
> +	vlv_sideband_rw(i915, PCI_DEVFN(0, 0), port,
>  			SB_CRRDDA_NP, reg, &val);
> +
>  	return val;
>  }
>  
> -void vlv_iosf_sb_write(struct drm_i915_private *dev_priv,
> +void vlv_iosf_sb_write(struct drm_i915_private *i915,
>  		       u8 port, u32 reg, u32 val)
>  {
> -	vlv_sideband_rw(dev_priv, PCI_DEVFN(0, 0), port,
> +	vlv_sideband_rw(i915, PCI_DEVFN(0, 0), port,
>  			SB_CRWRDA_NP, reg, &val);
>  }
>  
> -u32 vlv_cck_read(struct drm_i915_private *dev_priv, u32 reg)
> +u32 vlv_cck_read(struct drm_i915_private *i915, u32 reg)
>  {
>  	u32 val = 0;
> -	vlv_sideband_rw(dev_priv, PCI_DEVFN(0, 0), IOSF_PORT_CCK,
> +
> +	vlv_sideband_rw(i915, PCI_DEVFN(0, 0), IOSF_PORT_CCK,
>  			SB_CRRDDA_NP, reg, &val);
> +
>  	return val;
>  }
>  
> -void vlv_cck_write(struct drm_i915_private *dev_priv, u32 reg, u32 val)
> +void vlv_cck_write(struct drm_i915_private *i915, u32 reg, u32 val)
>  {
> -	vlv_sideband_rw(dev_priv, PCI_DEVFN(0, 0), IOSF_PORT_CCK,
> +	vlv_sideband_rw(i915, PCI_DEVFN(0, 0), IOSF_PORT_CCK,
>  			SB_CRWRDA_NP, reg, &val);
>  }
>  
> -u32 vlv_ccu_read(struct drm_i915_private *dev_priv, u32 reg)
> +u32 vlv_ccu_read(struct drm_i915_private *i915, u32 reg)
>  {
>  	u32 val = 0;
> -	vlv_sideband_rw(dev_priv, PCI_DEVFN(0, 0), IOSF_PORT_CCU,
> +
> +	vlv_sideband_rw(i915, PCI_DEVFN(0, 0), IOSF_PORT_CCU,
>  			SB_CRRDDA_NP, reg, &val);
> +
>  	return val;
>  }
>  
> -void vlv_ccu_write(struct drm_i915_private *dev_priv, u32 reg, u32 val)
> +void vlv_ccu_write(struct drm_i915_private *i915, u32 reg, u32 val)
>  {
> -	vlv_sideband_rw(dev_priv, PCI_DEVFN(0, 0), IOSF_PORT_CCU,
> +	vlv_sideband_rw(i915, PCI_DEVFN(0, 0), IOSF_PORT_CCU,
>  			SB_CRWRDA_NP, reg, &val);
>  }
>  
> -u32 vlv_dpio_read(struct drm_i915_private *dev_priv, enum pipe pipe, int reg)
> +u32 vlv_dpio_read(struct drm_i915_private *i915, enum pipe pipe, int reg)
>  {
> +	int port = i915->dpio_phy_iosf_port[DPIO_PHY(pipe)];
>  	u32 val = 0;
>  
> -	vlv_sideband_rw(dev_priv, DPIO_DEVFN, DPIO_PHY_IOSF_PORT(DPIO_PHY(pipe)),
> -			SB_MRD_NP, reg, &val);
> +	vlv_sideband_rw(i915, DPIO_DEVFN, port, SB_MRD_NP, reg, &val);
>  
>  	/*
>  	 * FIXME: There might be some registers where all 1's is a valid value,
> @@ -195,10 +253,27 @@ u32 vlv_dpio_read(struct drm_i915_private *dev_priv, enum pipe pipe, int reg)
>  	return val;
>  }
>  
> -void vlv_dpio_write(struct drm_i915_private *dev_priv, enum pipe pipe, int reg, u32 val)
> +void vlv_dpio_write(struct drm_i915_private *i915,
> +		    enum pipe pipe, int reg, u32 val)
> +{
> +	int port = i915->dpio_phy_iosf_port[DPIO_PHY(pipe)];
> +
> +	vlv_sideband_rw(i915, DPIO_DEVFN, port, SB_MWR_NP, reg, &val);
> +}
> +
> +u32 vlv_flisdsi_read(struct drm_i915_private *i915, u32 reg)
>  {
> -	vlv_sideband_rw(dev_priv, DPIO_DEVFN, DPIO_PHY_IOSF_PORT(DPIO_PHY(pipe)),
> -			SB_MWR_NP, reg, &val);
> +	u32 val = 0;
> +
> +	vlv_sideband_rw(i915, DPIO_DEVFN, IOSF_PORT_FLISDSI, SB_CRRDDA_NP,
> +			reg, &val);
> +	return val;
> +}
> +
> +void vlv_flisdsi_write(struct drm_i915_private *i915, u32 reg, u32 val)
> +{
> +	vlv_sideband_rw(i915, DPIO_DEVFN, IOSF_PORT_FLISDSI, SB_CRWRDA_NP,
> +			reg, &val);
>  }
>  
>  /* SBI access */
> @@ -279,17 +354,3 @@ void intel_sbi_write(struct drm_i915_private *dev_priv, u16 reg, u32 value,
>  		return;
>  	}
>  }
> -
> -u32 vlv_flisdsi_read(struct drm_i915_private *dev_priv, u32 reg)
> -{
> -	u32 val = 0;
> -	vlv_sideband_rw(dev_priv, DPIO_DEVFN, IOSF_PORT_FLISDSI, SB_CRRDDA_NP,
> -			reg, &val);
> -	return val;
> -}
> -
> -void vlv_flisdsi_write(struct drm_i915_private *dev_priv, u32 reg, u32 val)
> -{
> -	vlv_sideband_rw(dev_priv, DPIO_DEVFN, IOSF_PORT_FLISDSI, SB_CRWRDA_NP,
> -			reg, &val);
> -}
> -- 
> 2.20.1

-- 
Ville Syrjälä
Intel
_______________________________________________
Intel-gfx mailing list
Intel-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/intel-gfx