Patchwork drm/i915: add multi-threaded forcewake support

login
register
mail settings
Submitter Manoj Iyer
Date Nov. 21, 2011, 7:39 p.m.
Message ID <1321904356-7499-1-git-send-email-manoj.iyer@canonical.com>
Download mbox | patch
Permalink /patch/126911/
State New
Headers show

Comments

Manoj Iyer - Nov. 21, 2011, 7:39 p.m.
From: Keith Packard <keithp at keithp.com>

On IVB C0+ with newer BIOSes, the forcewake handshake has changed.  There's
now a bitfield for different driver components to keep the GT powered
on.  On Linux, we centralize forcewake handling in one place, so we
still just need a single bit, but we need to use the new registers if MT
forcewake is enabled.

This needs testing on affected machines.  Please reply with your
tested-by if you had problems after a BIOS upgrade and this patch fixes
them.

v2: force MT mode. shift by 16
v3: set MT force wake bits then check ECOBUS

This patch was submitted to intel-gfx by upstream and has not yet made it
to the linus tree, the patch was backported to Oneiric, and tested on
ivybridge system.

BugLink: http://bugs.launchpad.net/bugs/891270

Tested-by: Keith Packard <keithp at keithp.com>
Signed-off-by: Jesse Barnes <jbarnes at virtuousgeek.org>
Signed-off-by: Keith Packard <keithp at keithp.com>

Backported to 3.0
Signed-off-by: Robert Hooker <robert.hooker@canonical.com>
Signed-off-by: Manoj Iyer <manoj.iyer@canonical.com>
---
 drivers/gpu/drm/i915/i915_drv.c      |   30 ++++++++++++++++++++++++++----
 drivers/gpu/drm/i915/i915_drv.h      |    8 ++++++++
 drivers/gpu/drm/i915/i915_reg.h      |    4 ++++
 drivers/gpu/drm/i915/intel_display.c |   22 ++++++++++++++++++++++
 4 files changed, 60 insertions(+), 4 deletions(-)
Tim Gardner - Nov. 21, 2011, 8:53 p.m.
On 11/21/2011 12:39 PM, Manoj Iyer wrote:
> From: Keith Packard<keithp at keithp.com>
>
> On IVB C0+ with newer BIOSes, the forcewake handshake has changed.  There's
> now a bitfield for different driver components to keep the GT powered
> on.  On Linux, we centralize forcewake handling in one place, so we
> still just need a single bit, but we need to use the new registers if MT
> forcewake is enabled.
>
> This needs testing on affected machines.  Please reply with your
> tested-by if you had problems after a BIOS upgrade and this patch fixes
> them.
>
> v2: force MT mode. shift by 16
> v3: set MT force wake bits then check ECOBUS
>
> This patch was submitted to intel-gfx by upstream and has not yet made it
> to the linus tree, the patch was backported to Oneiric, and tested on
> ivybridge system.
>
> BugLink: http://bugs.launchpad.net/bugs/891270
>
> Tested-by: Keith Packard<keithp at keithp.com>
> Signed-off-by: Jesse Barnes<jbarnes at virtuousgeek.org>
> Signed-off-by: Keith Packard<keithp at keithp.com>
>
> Backported to 3.0
> Signed-off-by: Robert Hooker<robert.hooker@canonical.com>
> Signed-off-by: Manoj Iyer<manoj.iyer@canonical.com>
> ---
>   drivers/gpu/drm/i915/i915_drv.c      |   30 ++++++++++++++++++++++++++----
>   drivers/gpu/drm/i915/i915_drv.h      |    8 ++++++++
>   drivers/gpu/drm/i915/i915_reg.h      |    4 ++++
>   drivers/gpu/drm/i915/intel_display.c |   22 ++++++++++++++++++++++
>   4 files changed, 60 insertions(+), 4 deletions(-)
>
> diff --git a/drivers/gpu/drm/i915/i915_drv.c b/drivers/gpu/drm/i915/i915_drv.c
> index eb91e2d..0b1a752 100644
> --- a/drivers/gpu/drm/i915/i915_drv.c
> +++ b/drivers/gpu/drm/i915/i915_drv.c
> @@ -295,7 +295,7 @@ void intel_detect_pch (struct drm_device *dev)
>   	}
>   }
>
> -static void __gen6_gt_force_wake_get(struct drm_i915_private *dev_priv)
> +void __gen6_gt_force_wake_get(struct drm_i915_private *dev_priv)
>   {
>   	int count;
>
> @@ -311,6 +311,22 @@ static void __gen6_gt_force_wake_get(struct drm_i915_private *dev_priv)
>   		udelay(10);
>   }
>
> +void __gen6_gt_force_wake_mt_get(struct drm_i915_private *dev_priv)
> +{
> +	int count;
> +
> +	count = 0;
> +	while (count++<  50&&  (I915_READ_NOTRACE(FORCEWAKE_MT_ACK)&  1))
> +		udelay(10);
> +
> +	I915_WRITE_NOTRACE(FORCEWAKE_MT, (1<<16) | 1);
> +	POSTING_READ(FORCEWAKE_MT);
> +
> +	count = 0;
> +	while (count++<  50&&  (I915_READ_NOTRACE(FORCEWAKE_MT_ACK)&  1) == 0)
> +		udelay(10);
> +}
> +
>   /*
>    * Generally this is called implicitly by the register read function. However,
>    * if some sequence requires the GT to not power down then this function should
> @@ -323,15 +339,21 @@ void gen6_gt_force_wake_get(struct drm_i915_private *dev_priv)
>
>   	/* Forcewake is atomic in case we get in here without the lock */
>   	if (atomic_add_return(1,&dev_priv->forcewake_count) == 1)
> -		__gen6_gt_force_wake_get(dev_priv);
> +		dev_priv->display.force_wake_get(dev_priv);
>   }
>
> -static void __gen6_gt_force_wake_put(struct drm_i915_private *dev_priv)
> +void __gen6_gt_force_wake_put(struct drm_i915_private *dev_priv)
>   {
>   	I915_WRITE_NOTRACE(FORCEWAKE, 0);
>   	POSTING_READ(FORCEWAKE);
>   }
>
> +void __gen6_gt_force_wake_mt_put(struct drm_i915_private *dev_priv)
> +{
> +	I915_WRITE_NOTRACE(FORCEWAKE_MT, (1<<16) | 0);
> +	POSTING_READ(FORCEWAKE_MT);
> +}
> +
>   /*
>    * see gen6_gt_force_wake_get()
>    */
> @@ -340,7 +362,7 @@ void gen6_gt_force_wake_put(struct drm_i915_private *dev_priv)
>   	WARN_ON(!mutex_is_locked(&dev_priv->dev->struct_mutex));
>
>   	if (atomic_dec_and_test(&dev_priv->forcewake_count))
> -		__gen6_gt_force_wake_put(dev_priv);
> +		dev_priv->display.force_wake_put(dev_priv);
>   }
>
>   void __gen6_gt_wait_for_fifo(struct drm_i915_private *dev_priv)
> diff --git a/drivers/gpu/drm/i915/i915_drv.h b/drivers/gpu/drm/i915/i915_drv.h
> index 1c44613..0d1ddca 100644
> --- a/drivers/gpu/drm/i915/i915_drv.h
> +++ b/drivers/gpu/drm/i915/i915_drv.h
> @@ -107,6 +107,7 @@ struct opregion_header;
>   struct opregion_acpi;
>   struct opregion_swsci;
>   struct opregion_asle;
> +struct drm_i915_private;
>
>   struct intel_opregion {
>   	struct opregion_header *header;
> @@ -215,6 +216,8 @@ struct drm_i915_display_funcs {
>   	int (*queue_flip)(struct drm_device *dev, struct drm_crtc *crtc,
>   			  struct drm_framebuffer *fb,
>   			  struct drm_i915_gem_object *obj);
> +	void (*force_wake_get)(struct drm_i915_private *dev_priv);
> +	void (*force_wake_put)(struct drm_i915_private *dev_priv);
>   	/* clock updates for mode set */
>   	/* cursor updates */
>   	/* render clock increase/decrease */
> @@ -1299,6 +1302,11 @@ extern void gen6_set_rps(struct drm_device *dev, u8 val);
>   extern void intel_detect_pch (struct drm_device *dev);
>   extern int intel_trans_dp_port_sel (struct drm_crtc *crtc);
>
> +extern void __gen6_gt_force_wake_get(struct drm_i915_private *dev_priv);
> +extern void __gen6_gt_force_wake_mt_get(struct drm_i915_private *dev_priv);
> +extern void __gen6_gt_force_wake_put(struct drm_i915_private *dev_priv);
> +extern void __gen6_gt_force_wake_mt_put(struct drm_i915_private *dev_priv);
> +
>   /* overlay */
>   #ifdef CONFIG_DEBUG_FS
>   extern struct intel_overlay_error_state *intel_overlay_capture_error_state(struct drm_device *dev);
> diff --git a/drivers/gpu/drm/i915/i915_reg.h b/drivers/gpu/drm/i915/i915_reg.h
> index 2c34d07..4c333d5 100644
> --- a/drivers/gpu/drm/i915/i915_reg.h
> +++ b/drivers/gpu/drm/i915/i915_reg.h
> @@ -3362,6 +3362,10 @@
>
>   #define  FORCEWAKE				0xA18C
>   #define  FORCEWAKE_ACK				0x130090
> +#define  FORCEWAKE_MT				0xa188 /* multi-threaded */
> +#define  FORCEWAKE_MT_ACK			0x130040
> +#define  ECOBUS					0xa180
> +#define    FORCEWAKE_MT_ENABLE			(1<<5)
>
>   #define  GT_FIFO_FREE_ENTRIES			0x120008
>
> diff --git a/drivers/gpu/drm/i915/intel_display.c b/drivers/gpu/drm/i915/intel_display.c
> index e917c7b..f0cddd6 100644
> --- a/drivers/gpu/drm/i915/intel_display.c
> +++ b/drivers/gpu/drm/i915/intel_display.c
> @@ -7689,6 +7689,28 @@ static void intel_init_display(struct drm_device *dev)
>
>   	/* For FIFO watermark updates */
>   	if (HAS_PCH_SPLIT(dev)) {
> +		dev_priv->display.force_wake_get = __gen6_gt_force_wake_get;
> +		dev_priv->display.force_wake_put = __gen6_gt_force_wake_put;
> +
> +		/* IVB configs may use multi-threaded forcewake */
> +		if (IS_IVYBRIDGE(dev)) {
> +			u32	ecobus;
> +
> +			mutex_lock(&dev->struct_mutex);
> +			__gen6_gt_force_wake_mt_get(dev_priv);
> +			ecobus = I915_READ(ECOBUS);
> +			__gen6_gt_force_wake_mt_put(dev_priv);
> +			mutex_unlock(&dev->struct_mutex);
> +
> +			if (ecobus&  FORCEWAKE_MT_ENABLE) {
> +				DRM_DEBUG_KMS("Using MT version of forcewake\n");
> +				dev_priv->display.force_wake_get =
> +					__gen6_gt_force_wake_mt_get;
> +				dev_priv->display.force_wake_put =
> +					__gen6_gt_force_wake_mt_put;
> +			}
> +		}
> +
>   		if (HAS_PCH_IBX(dev))
>   			dev_priv->display.init_pch_clock_gating = ibx_init_clock_gating;
>   		else if (HAS_PCH_CPT(dev))
Herton Ronaldo Krzesinski - Nov. 22, 2011, 12:02 p.m.
On Mon, Nov 21, 2011 at 01:39:16PM -0600, Manoj Iyer wrote:
> From: Keith Packard <keithp at keithp.com>
> 
> On IVB C0+ with newer BIOSes, the forcewake handshake has changed.  There's
> now a bitfield for different driver components to keep the GT powered
> on.  On Linux, we centralize forcewake handling in one place, so we
> still just need a single bit, but we need to use the new registers if MT
> forcewake is enabled.
> 
> This needs testing on affected machines.  Please reply with your
> tested-by if you had problems after a BIOS upgrade and this patch fixes
> them.
> 
> v2: force MT mode. shift by 16
> v3: set MT force wake bits then check ECOBUS
> 
> This patch was submitted to intel-gfx by upstream and has not yet made it
> to the linus tree, the patch was backported to Oneiric, and tested on
> ivybridge system.
> 
> BugLink: http://bugs.launchpad.net/bugs/891270
> 
> Tested-by: Keith Packard <keithp at keithp.com>
> Signed-off-by: Jesse Barnes <jbarnes at virtuousgeek.org>
> Signed-off-by: Keith Packard <keithp at keithp.com>
> 
> Backported to 3.0
> Signed-off-by: Robert Hooker <robert.hooker@canonical.com>
> Signed-off-by: Manoj Iyer <manoj.iyer@canonical.com>

It seems the following is missing from this backport, looking at the
posted patch on
http://permalink.gmane.org/gmane.comp.freedesktop.xorg.drivers.intel/7192

#define NEEDS_FORCE_WAKE(dev_priv, reg) \
 	(((dev_priv)->info->gen >= 6) && \
-	((reg) < 0x40000) && \
-	((reg) != FORCEWAKE))
+	 ((reg) < 0x40000) &&		 \
+	 ((reg) != FORCEWAKE) &&	 \
+	 ((reg) != ECOBUS))

Any reason to not update the NEEDS_FORCE_WAKE macro?

> ---
>  drivers/gpu/drm/i915/i915_drv.c      |   30 ++++++++++++++++++++++++++----
>  drivers/gpu/drm/i915/i915_drv.h      |    8 ++++++++
>  drivers/gpu/drm/i915/i915_reg.h      |    4 ++++
>  drivers/gpu/drm/i915/intel_display.c |   22 ++++++++++++++++++++++
>  4 files changed, 60 insertions(+), 4 deletions(-)
> 
> diff --git a/drivers/gpu/drm/i915/i915_drv.c b/drivers/gpu/drm/i915/i915_drv.c
> index eb91e2d..0b1a752 100644
> --- a/drivers/gpu/drm/i915/i915_drv.c
> +++ b/drivers/gpu/drm/i915/i915_drv.c
> @@ -295,7 +295,7 @@ void intel_detect_pch (struct drm_device *dev)
>  	}
>  }
>  
> -static void __gen6_gt_force_wake_get(struct drm_i915_private *dev_priv)
> +void __gen6_gt_force_wake_get(struct drm_i915_private *dev_priv)
>  {
>  	int count;
>  
> @@ -311,6 +311,22 @@ static void __gen6_gt_force_wake_get(struct drm_i915_private *dev_priv)
>  		udelay(10);
>  }
>  
> +void __gen6_gt_force_wake_mt_get(struct drm_i915_private *dev_priv)
> +{
> +	int count;
> +
> +	count = 0;
> +	while (count++ < 50 && (I915_READ_NOTRACE(FORCEWAKE_MT_ACK) & 1))
> +		udelay(10);
> +
> +	I915_WRITE_NOTRACE(FORCEWAKE_MT, (1<<16) | 1);
> +	POSTING_READ(FORCEWAKE_MT);
> +
> +	count = 0;
> +	while (count++ < 50 && (I915_READ_NOTRACE(FORCEWAKE_MT_ACK) & 1) == 0)
> +		udelay(10);
> +}
> +
>  /*
>   * Generally this is called implicitly by the register read function. However,
>   * if some sequence requires the GT to not power down then this function should
> @@ -323,15 +339,21 @@ void gen6_gt_force_wake_get(struct drm_i915_private *dev_priv)
>  
>  	/* Forcewake is atomic in case we get in here without the lock */
>  	if (atomic_add_return(1, &dev_priv->forcewake_count) == 1)
> -		__gen6_gt_force_wake_get(dev_priv);
> +		dev_priv->display.force_wake_get(dev_priv);
>  }
>  
> -static void __gen6_gt_force_wake_put(struct drm_i915_private *dev_priv)
> +void __gen6_gt_force_wake_put(struct drm_i915_private *dev_priv)
>  {
>  	I915_WRITE_NOTRACE(FORCEWAKE, 0);
>  	POSTING_READ(FORCEWAKE);
>  }
>  
> +void __gen6_gt_force_wake_mt_put(struct drm_i915_private *dev_priv)
> +{
> +	I915_WRITE_NOTRACE(FORCEWAKE_MT, (1<<16) | 0);
> +	POSTING_READ(FORCEWAKE_MT);
> +}
> +
>  /*
>   * see gen6_gt_force_wake_get()
>   */
> @@ -340,7 +362,7 @@ void gen6_gt_force_wake_put(struct drm_i915_private *dev_priv)
>  	WARN_ON(!mutex_is_locked(&dev_priv->dev->struct_mutex));
>  
>  	if (atomic_dec_and_test(&dev_priv->forcewake_count))
> -		__gen6_gt_force_wake_put(dev_priv);
> +		dev_priv->display.force_wake_put(dev_priv);
>  }
>  
>  void __gen6_gt_wait_for_fifo(struct drm_i915_private *dev_priv)
> diff --git a/drivers/gpu/drm/i915/i915_drv.h b/drivers/gpu/drm/i915/i915_drv.h
> index 1c44613..0d1ddca 100644
> --- a/drivers/gpu/drm/i915/i915_drv.h
> +++ b/drivers/gpu/drm/i915/i915_drv.h
> @@ -107,6 +107,7 @@ struct opregion_header;
>  struct opregion_acpi;
>  struct opregion_swsci;
>  struct opregion_asle;
> +struct drm_i915_private;
>  
>  struct intel_opregion {
>  	struct opregion_header *header;
> @@ -215,6 +216,8 @@ struct drm_i915_display_funcs {
>  	int (*queue_flip)(struct drm_device *dev, struct drm_crtc *crtc,
>  			  struct drm_framebuffer *fb,
>  			  struct drm_i915_gem_object *obj);
> +	void (*force_wake_get)(struct drm_i915_private *dev_priv);
> +	void (*force_wake_put)(struct drm_i915_private *dev_priv);
>  	/* clock updates for mode set */
>  	/* cursor updates */
>  	/* render clock increase/decrease */
> @@ -1299,6 +1302,11 @@ extern void gen6_set_rps(struct drm_device *dev, u8 val);
>  extern void intel_detect_pch (struct drm_device *dev);
>  extern int intel_trans_dp_port_sel (struct drm_crtc *crtc);
>  
> +extern void __gen6_gt_force_wake_get(struct drm_i915_private *dev_priv);
> +extern void __gen6_gt_force_wake_mt_get(struct drm_i915_private *dev_priv);
> +extern void __gen6_gt_force_wake_put(struct drm_i915_private *dev_priv);
> +extern void __gen6_gt_force_wake_mt_put(struct drm_i915_private *dev_priv);
> +
>  /* overlay */
>  #ifdef CONFIG_DEBUG_FS
>  extern struct intel_overlay_error_state *intel_overlay_capture_error_state(struct drm_device *dev);
> diff --git a/drivers/gpu/drm/i915/i915_reg.h b/drivers/gpu/drm/i915/i915_reg.h
> index 2c34d07..4c333d5 100644
> --- a/drivers/gpu/drm/i915/i915_reg.h
> +++ b/drivers/gpu/drm/i915/i915_reg.h
> @@ -3362,6 +3362,10 @@
>  
>  #define  FORCEWAKE				0xA18C
>  #define  FORCEWAKE_ACK				0x130090
> +#define  FORCEWAKE_MT				0xa188 /* multi-threaded */
> +#define  FORCEWAKE_MT_ACK			0x130040
> +#define  ECOBUS					0xa180
> +#define    FORCEWAKE_MT_ENABLE			(1<<5)
>  
>  #define  GT_FIFO_FREE_ENTRIES			0x120008
>  
> diff --git a/drivers/gpu/drm/i915/intel_display.c b/drivers/gpu/drm/i915/intel_display.c
> index e917c7b..f0cddd6 100644
> --- a/drivers/gpu/drm/i915/intel_display.c
> +++ b/drivers/gpu/drm/i915/intel_display.c
> @@ -7689,6 +7689,28 @@ static void intel_init_display(struct drm_device *dev)
>  
>  	/* For FIFO watermark updates */
>  	if (HAS_PCH_SPLIT(dev)) {
> +		dev_priv->display.force_wake_get = __gen6_gt_force_wake_get;
> +		dev_priv->display.force_wake_put = __gen6_gt_force_wake_put;
> +
> +		/* IVB configs may use multi-threaded forcewake */
> +		if (IS_IVYBRIDGE(dev)) {
> +			u32	ecobus;
> +
> +			mutex_lock(&dev->struct_mutex);
> +			__gen6_gt_force_wake_mt_get(dev_priv);
> +			ecobus = I915_READ(ECOBUS);
> +			__gen6_gt_force_wake_mt_put(dev_priv);
> +			mutex_unlock(&dev->struct_mutex);
> +
> +			if (ecobus & FORCEWAKE_MT_ENABLE) {
> +				DRM_DEBUG_KMS("Using MT version of forcewake\n");
> +				dev_priv->display.force_wake_get =
> +					__gen6_gt_force_wake_mt_get;
> +				dev_priv->display.force_wake_put =
> +					__gen6_gt_force_wake_mt_put;
> +			}
> +		}
> +
>  		if (HAS_PCH_IBX(dev))
>  			dev_priv->display.init_pch_clock_gating = ibx_init_clock_gating;
>  		else if (HAS_PCH_CPT(dev))
> -- 
> 1.7.5.4
> 
>
Robert Hooker - Nov. 22, 2011, 4:27 p.m.
On Tue, Nov 22, 2011 at 7:02 AM, Herton Ronaldo Krzesinski
<herton.krzesinski@canonical.com> wrote:
> It seems the following is missing from this backport, looking at the
> posted patch on
> http://permalink.gmane.org/gmane.comp.freedesktop.xorg.drivers.intel/7192
>
> #define NEEDS_FORCE_WAKE(dev_priv, reg) \
>        (((dev_priv)->info->gen >= 6) && \
> -       ((reg) < 0x40000) && \
> -       ((reg) != FORCEWAKE))
> +        ((reg) < 0x40000) &&            \
> +        ((reg) != FORCEWAKE) &&         \
> +        ((reg) != ECOBUS))
>
> Any reason to not update the NEEDS_FORCE_WAKE macro?
>
>
> --
> []'s
> Herton
>
> --
> kernel-team mailing list
> kernel-team@lists.ubuntu.com
> https://lists.ubuntu.com/mailman/listinfo/kernel-team
>

My mistake, thanks for catching that! I only meant to drop the
identical hunk from i915_drv.c which isn't there in 3.0 due to
f700088333c5c7e5a7f4ab71b642362290259e26 but ended up dropping it from
i915_drv.h too.

Patch

diff --git a/drivers/gpu/drm/i915/i915_drv.c b/drivers/gpu/drm/i915/i915_drv.c
index eb91e2d..0b1a752 100644
--- a/drivers/gpu/drm/i915/i915_drv.c
+++ b/drivers/gpu/drm/i915/i915_drv.c
@@ -295,7 +295,7 @@  void intel_detect_pch (struct drm_device *dev)
 	}
 }
 
-static void __gen6_gt_force_wake_get(struct drm_i915_private *dev_priv)
+void __gen6_gt_force_wake_get(struct drm_i915_private *dev_priv)
 {
 	int count;
 
@@ -311,6 +311,22 @@  static void __gen6_gt_force_wake_get(struct drm_i915_private *dev_priv)
 		udelay(10);
 }
 
+void __gen6_gt_force_wake_mt_get(struct drm_i915_private *dev_priv)
+{
+	int count;
+
+	count = 0;
+	while (count++ < 50 && (I915_READ_NOTRACE(FORCEWAKE_MT_ACK) & 1))
+		udelay(10);
+
+	I915_WRITE_NOTRACE(FORCEWAKE_MT, (1<<16) | 1);
+	POSTING_READ(FORCEWAKE_MT);
+
+	count = 0;
+	while (count++ < 50 && (I915_READ_NOTRACE(FORCEWAKE_MT_ACK) & 1) == 0)
+		udelay(10);
+}
+
 /*
  * Generally this is called implicitly by the register read function. However,
  * if some sequence requires the GT to not power down then this function should
@@ -323,15 +339,21 @@  void gen6_gt_force_wake_get(struct drm_i915_private *dev_priv)
 
 	/* Forcewake is atomic in case we get in here without the lock */
 	if (atomic_add_return(1, &dev_priv->forcewake_count) == 1)
-		__gen6_gt_force_wake_get(dev_priv);
+		dev_priv->display.force_wake_get(dev_priv);
 }
 
-static void __gen6_gt_force_wake_put(struct drm_i915_private *dev_priv)
+void __gen6_gt_force_wake_put(struct drm_i915_private *dev_priv)
 {
 	I915_WRITE_NOTRACE(FORCEWAKE, 0);
 	POSTING_READ(FORCEWAKE);
 }
 
+void __gen6_gt_force_wake_mt_put(struct drm_i915_private *dev_priv)
+{
+	I915_WRITE_NOTRACE(FORCEWAKE_MT, (1<<16) | 0);
+	POSTING_READ(FORCEWAKE_MT);
+}
+
 /*
  * see gen6_gt_force_wake_get()
  */
@@ -340,7 +362,7 @@  void gen6_gt_force_wake_put(struct drm_i915_private *dev_priv)
 	WARN_ON(!mutex_is_locked(&dev_priv->dev->struct_mutex));
 
 	if (atomic_dec_and_test(&dev_priv->forcewake_count))
-		__gen6_gt_force_wake_put(dev_priv);
+		dev_priv->display.force_wake_put(dev_priv);
 }
 
 void __gen6_gt_wait_for_fifo(struct drm_i915_private *dev_priv)
diff --git a/drivers/gpu/drm/i915/i915_drv.h b/drivers/gpu/drm/i915/i915_drv.h
index 1c44613..0d1ddca 100644
--- a/drivers/gpu/drm/i915/i915_drv.h
+++ b/drivers/gpu/drm/i915/i915_drv.h
@@ -107,6 +107,7 @@  struct opregion_header;
 struct opregion_acpi;
 struct opregion_swsci;
 struct opregion_asle;
+struct drm_i915_private;
 
 struct intel_opregion {
 	struct opregion_header *header;
@@ -215,6 +216,8 @@  struct drm_i915_display_funcs {
 	int (*queue_flip)(struct drm_device *dev, struct drm_crtc *crtc,
 			  struct drm_framebuffer *fb,
 			  struct drm_i915_gem_object *obj);
+	void (*force_wake_get)(struct drm_i915_private *dev_priv);
+	void (*force_wake_put)(struct drm_i915_private *dev_priv);
 	/* clock updates for mode set */
 	/* cursor updates */
 	/* render clock increase/decrease */
@@ -1299,6 +1302,11 @@  extern void gen6_set_rps(struct drm_device *dev, u8 val);
 extern void intel_detect_pch (struct drm_device *dev);
 extern int intel_trans_dp_port_sel (struct drm_crtc *crtc);
 
+extern void __gen6_gt_force_wake_get(struct drm_i915_private *dev_priv);
+extern void __gen6_gt_force_wake_mt_get(struct drm_i915_private *dev_priv);
+extern void __gen6_gt_force_wake_put(struct drm_i915_private *dev_priv);
+extern void __gen6_gt_force_wake_mt_put(struct drm_i915_private *dev_priv);
+
 /* overlay */
 #ifdef CONFIG_DEBUG_FS
 extern struct intel_overlay_error_state *intel_overlay_capture_error_state(struct drm_device *dev);
diff --git a/drivers/gpu/drm/i915/i915_reg.h b/drivers/gpu/drm/i915/i915_reg.h
index 2c34d07..4c333d5 100644
--- a/drivers/gpu/drm/i915/i915_reg.h
+++ b/drivers/gpu/drm/i915/i915_reg.h
@@ -3362,6 +3362,10 @@ 
 
 #define  FORCEWAKE				0xA18C
 #define  FORCEWAKE_ACK				0x130090
+#define  FORCEWAKE_MT				0xa188 /* multi-threaded */
+#define  FORCEWAKE_MT_ACK			0x130040
+#define  ECOBUS					0xa180
+#define    FORCEWAKE_MT_ENABLE			(1<<5)
 
 #define  GT_FIFO_FREE_ENTRIES			0x120008
 
diff --git a/drivers/gpu/drm/i915/intel_display.c b/drivers/gpu/drm/i915/intel_display.c
index e917c7b..f0cddd6 100644
--- a/drivers/gpu/drm/i915/intel_display.c
+++ b/drivers/gpu/drm/i915/intel_display.c
@@ -7689,6 +7689,28 @@  static void intel_init_display(struct drm_device *dev)
 
 	/* For FIFO watermark updates */
 	if (HAS_PCH_SPLIT(dev)) {
+		dev_priv->display.force_wake_get = __gen6_gt_force_wake_get;
+		dev_priv->display.force_wake_put = __gen6_gt_force_wake_put;
+
+		/* IVB configs may use multi-threaded forcewake */
+		if (IS_IVYBRIDGE(dev)) {
+			u32	ecobus;
+
+			mutex_lock(&dev->struct_mutex);
+			__gen6_gt_force_wake_mt_get(dev_priv);
+			ecobus = I915_READ(ECOBUS);
+			__gen6_gt_force_wake_mt_put(dev_priv);
+			mutex_unlock(&dev->struct_mutex);
+
+			if (ecobus & FORCEWAKE_MT_ENABLE) {
+				DRM_DEBUG_KMS("Using MT version of forcewake\n");
+				dev_priv->display.force_wake_get =
+					__gen6_gt_force_wake_mt_get;
+				dev_priv->display.force_wake_put =
+					__gen6_gt_force_wake_mt_put;
+			}
+		}
+
 		if (HAS_PCH_IBX(dev))
 			dev_priv->display.init_pch_clock_gating = ibx_init_clock_gating;
 		else if (HAS_PCH_CPT(dev))