[RFC] powerpc/64s: Move ISAv3.0 / POWER9 idle code to powernv C code

Message ID 20180708142436.15714-1-npiggin@gmail.com
State RFC
Headers show
Series
  • [RFC] powerpc/64s: Move ISAv3.0 / POWER9 idle code to powernv C code
Related show

Commit Message

Nicholas Piggin July 8, 2018, 2:24 p.m.
Reimplement POWER9 idle code in C, in the powernv platform code.
Assembly stubs are used to save and restore the stack frame and
non-volatile GPRs before going to idle, but these are small and
mostly agnostic to microarchitecture implementation details.

POWER7/8 code is not converted (yet), but that's not a moving
target, and it doesn't make you want to claw your eyes out so
much with the POWER9 code untangled from it.

The optimisation where EC=ESL=0 idle modes did not have to save
GPRs or mtmsrd L=0 is restored, because it's simple to do.

Idle wakeup no longer uses the ->cpu_restore call to reinit SPRs,
but saves and restores them all explicitly.

Moving the HMI, SPR, OPAL, locking, etc. to C is the only real
way this stuff will cope with non-trivial new CPU implementation
details, firmware changes, etc., without becoming unmaintainable.
---
 arch/powerpc/include/asm/book3s/64/mmu-hash.h |   1 +
 arch/powerpc/include/asm/cpuidle.h            |  14 +-
 arch/powerpc/include/asm/paca.h               |  38 +-
 arch/powerpc/include/asm/processor.h          |   3 +-
 arch/powerpc/include/asm/reg.h                |   7 +-
 arch/powerpc/kernel/Makefile                  |   2 +-
 arch/powerpc/kernel/asm-offsets.c             |  11 +-
 arch/powerpc/kernel/exceptions-64s.S          |  10 +-
 arch/powerpc/kernel/idle_book3s.S             | 348 ++-------------
 arch/powerpc/kernel/idle_isa3.S               |  73 ++++
 arch/powerpc/kernel/setup-common.c            |   4 +-
 arch/powerpc/mm/slb.c                         |   7 +-
 arch/powerpc/platforms/powernv/idle.c         | 402 +++++++++++++++---
 arch/powerpc/xmon/xmon.c                      |  25 +-
 14 files changed, 496 insertions(+), 449 deletions(-)
 create mode 100644 arch/powerpc/kernel/idle_isa3.S

Comments

Gautham R. Shenoy July 10, 2018, 11:06 a.m. | #1
Hello Nicholas,


On Mon, Jul 09, 2018 at 12:24:36AM +1000, Nicholas Piggin wrote:
> Reimplement POWER9 idle code in C, in the powernv platform code.
> Assembly stubs are used to save and restore the stack frame and
> non-volatile GPRs before going to idle, but these are small and
> mostly agnostic to microarchitecture implementation details.
>

Thanks for this patch.  It is indeed hard to maneuver through the
current assembly code and change things without introducing new bugs.


> POWER7/8 code is not converted (yet), but that's not a moving
> target, and it doesn't make you want to claw your eyes out so
> much with the POWER9 code untangled from it.
> 
> The optimisation where EC=ESL=0 idle modes did not have to save
> GPRs or mtmsrd L=0 is restored, because it's simple to do.
> 
> Idle wakeup no longer uses the ->cpu_restore call to reinit SPRs,
> but saves and restores them all explicitly.

Right! The ->cpu_restore call in the current code is rendered
ineffective by "restore_additional_sprs" which is called after the
cpu_restore. 

> 
> Moving the HMI, SPR, OPAL, locking, etc. to C is the only real
> way this stuff will cope with non-trivial new CPU implementation
> details, firmware changes, etc., without becoming unmaintainable.

Some comments inline. 

> ---
>  arch/powerpc/include/asm/book3s/64/mmu-hash.h |   1 +
>  arch/powerpc/include/asm/cpuidle.h            |  14 +-
>  arch/powerpc/include/asm/paca.h               |  38 +-
>  arch/powerpc/include/asm/processor.h          |   3 +-
>  arch/powerpc/include/asm/reg.h                |   7 +-
>  arch/powerpc/kernel/Makefile                  |   2 +-
>  arch/powerpc/kernel/asm-offsets.c             |  11 +-
>  arch/powerpc/kernel/exceptions-64s.S          |  10 +-
>  arch/powerpc/kernel/idle_book3s.S             | 348 ++-------------
>  arch/powerpc/kernel/idle_isa3.S               |  73 ++++
>  arch/powerpc/kernel/setup-common.c            |   4 +-
>  arch/powerpc/mm/slb.c                         |   7 +-
>  arch/powerpc/platforms/powernv/idle.c         | 402 +++++++++++++++---
>  arch/powerpc/xmon/xmon.c                      |  25 +-
>  14 files changed, 496 insertions(+), 449 deletions(-)
>  create mode 100644 arch/powerpc/kernel/idle_isa3.S
> 
> diff --git a/arch/powerpc/include/asm/book3s/64/mmu-hash.h b/arch/powerpc/include/asm/book3s/64/mmu-hash.h
> index 50ed64fba4ae..c626319a962d 100644
> --- a/arch/powerpc/include/asm/book3s/64/mmu-hash.h
> +++ b/arch/powerpc/include/asm/book3s/64/mmu-hash.h
> @@ -486,6 +486,7 @@ static inline void hpte_init_pseries(void) { }
>  extern void hpte_init_native(void);
> 
>  extern void slb_initialize(void);
> +extern void __slb_flush_and_rebolt(void);
>  extern void slb_flush_and_rebolt(void);
> 
>  extern void slb_vmalloc_update(void);
> diff --git a/arch/powerpc/include/asm/cpuidle.h b/arch/powerpc/include/asm/cpuidle.h
> index e210a83eb196..b668f030d531 100644
> --- a/arch/powerpc/include/asm/cpuidle.h
> +++ b/arch/powerpc/include/asm/cpuidle.h
> @@ -28,6 +28,7 @@
>   * yet woken from the winkle state.
>   */
>  #define PNV_CORE_IDLE_LOCK_BIT			0x10000000
> +#define NR_PNV_CORE_IDLE_LOCK_BIT		28

We can define PNV_CORE_IDLE_LOCK_BIT mask based on
NR_PNV_CORE_IDLE_LOCK_BIT ?

> 
>  #define PNV_CORE_IDLE_WINKLE_COUNT		0x00010000
>  #define PNV_CORE_IDLE_WINKLE_COUNT_ALL_BIT	0x00080000
> @@ -68,22 +69,9 @@
>  #define ERR_DEEP_STATE_ESL_MISMATCH	-2
> 
>  #ifndef __ASSEMBLY__
> -/* Additional SPRs that need to be saved/restored during stop */
> -struct stop_sprs {
> -	u64 pid;
> -	u64 ldbar;
> -	u64 fscr;
> -	u64 hfscr;
> -	u64 mmcr1;
> -	u64 mmcr2;
> -	u64 mmcra;
> -};
> -
>  extern u32 pnv_fastsleep_workaround_at_entry[];
>  extern u32 pnv_fastsleep_workaround_at_exit[];
> 
> -extern u64 pnv_first_deep_stop_state;
> -
>  unsigned long pnv_cpu_offline(unsigned int cpu);
>  int validate_psscr_val_mask(u64 *psscr_val, u64 *psscr_mask, u32 flags);
>  static inline void report_invalid_psscr_val(u64 psscr_val, int err)
> diff --git a/arch/powerpc/include/asm/paca.h b/arch/powerpc/include/asm/paca.h
> index 4e9cede5a7e7..a7a4892d39c0 100644
> --- a/arch/powerpc/include/asm/paca.h
> +++ b/arch/powerpc/include/asm/paca.h
> @@ -178,23 +178,29 @@ struct paca_struct {
>  #endif
> 
>  #ifdef CONFIG_PPC_POWERNV
> -	/* Per-core mask tracking idle threads and a lock bit-[L][TTTTTTTT] */
> -	u32 *core_idle_state_ptr;
> -	u8 thread_idle_state;		/* PNV_THREAD_RUNNING/NAP/SLEEP	*/
> -	/* Mask to indicate thread id in core */
> -	u8 thread_mask;
> -	/* Mask to denote subcore sibling threads */
> -	u8 subcore_sibling_mask;
> -	/* Flag to request this thread not to stop */
> -	atomic_t dont_stop;
> -	/* The PSSCR value that the kernel requested before going to stop */
> -	u64 requested_psscr;
> +	union {
> +		/* P7/P8 specific fields */
> +		struct {
> +			/* Per-core mask tracking idle threads and a lock bit-[L][TTTTTTTT] */
> +			unsigned long *core_idle_state_ptr;
> +			u8 thread_idle_state;	/* PNV_THREAD_RUNNING/NAP/SLEEP	*/
> +			/* Mask to indicate thread id in core */
> +			u8 thread_mask;
> +			/* Mask to denote subcore sibling threads */
> +			u8 subcore_sibling_mask;
> +		};
> 
> -	/*
> -	 * Save area for additional SPRs that need to be
> -	 * saved/restored during cpuidle stop.
> -	 */
> -	struct stop_sprs stop_sprs;
> +		/* P9 specific fields */
> +		struct {
> +			/* The PSSCR value that the kernel requested before going to stop */
> +			u64 requested_psscr;
> +			unsigned long idle_state;
> +#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE
> +			/* Flag to request this thread not to stop */
> +			atomic_t dont_stop;
> +#endif
> +		};
> +	};
>  #endif
> 
>  #ifdef CONFIG_PPC_BOOK3S_64
> diff --git a/arch/powerpc/include/asm/processor.h b/arch/powerpc/include/asm/processor.h
> index 5debe337ea9d..4774ec7603ee 100644
> --- a/arch/powerpc/include/asm/processor.h
> +++ b/arch/powerpc/include/asm/processor.h
> @@ -513,7 +513,8 @@ enum idle_boot_override {IDLE_NO_OVERRIDE = 0, IDLE_POWERSAVE_OFF};
>  extern int powersave_nap;	/* set if nap mode can be used in idle loop */
>  extern unsigned long power7_idle_insn(unsigned long type); /* PNV_THREAD_NAP/etc*/
>  extern void power7_idle_type(unsigned long type);
> -extern unsigned long power9_idle_stop(unsigned long psscr_val);
> +extern unsigned long isa3_idle_stop_noloss(unsigned long psscr_val);
> +extern unsigned long isa3_idle_stop_mayloss(unsigned long psscr_val);
>  extern unsigned long power9_offline_stop(unsigned long psscr_val);
>  extern void power9_idle_type(unsigned long stop_psscr_val,
>  			      unsigned long stop_psscr_mask);
> diff --git a/arch/powerpc/include/asm/reg.h b/arch/powerpc/include/asm/reg.h
> index 562568414cf4..3c7d97b2abb0 100644
> --- a/arch/powerpc/include/asm/reg.h
> +++ b/arch/powerpc/include/asm/reg.h
> @@ -750,10 +750,9 @@
>  #define	  SRR1_WAKERESET	0x00100000 /* System reset */
>  #define   SRR1_WAKEHDBELL	0x000c0000 /* Hypervisor doorbell on P8 */
>  #define	  SRR1_WAKESTATE	0x00030000 /* Powersave exit mask [46:47] */
> -#define	  SRR1_WS_DEEPEST	0x00030000 /* Some resources not maintained,
> -					  * may not be recoverable */
> -#define	  SRR1_WS_DEEPER	0x00020000 /* Some resources not maintained */
> -#define	  SRR1_WS_DEEP		0x00010000 /* All resources maintained */
> +#define	  SRR1_WS_HVLOSS	0x00030000 /* HV resources not maintained */
> +#define	  SRR1_WS_GPRLOSS	0x00020000 /* GPRs not maintained */
> +#define	  SRR1_WS_NOLOSS	0x00010000 /* All resources maintained */

This is nice. More consistent with the ISA. And we weren't using the
SRR1_WS_DEEP* so far due to relying on the rotate, shift bit
manipulations in the asm code.




>  #define   SRR1_PROGTM		0x00200000 /* TM Bad Thing */
>  #define   SRR1_PROGFPE		0x00100000 /* Floating Point Enabled */
>  #define   SRR1_PROGILL		0x00080000 /* Illegal instruction */
> diff --git a/arch/powerpc/kernel/Makefile b/arch/powerpc/kernel/Makefile
> index 2b4c40b255e4..6914fab16e2c 100644
> --- a/arch/powerpc/kernel/Makefile
> +++ b/arch/powerpc/kernel/Makefile
> @@ -48,7 +48,7 @@ obj-$(CONFIG_PPC_BOOK3E_64)	+= exceptions-64e.o idle_book3e.o
>  obj-$(CONFIG_PPC64)		+= vdso64/
>  obj-$(CONFIG_ALTIVEC)		+= vecemu.o
>  obj-$(CONFIG_PPC_970_NAP)	+= idle_power4.o
> -obj-$(CONFIG_PPC_P7_NAP)	+= idle_book3s.o
> +obj-$(CONFIG_PPC_P7_NAP)	+= idle_book3s.o idle_isa3.o
>  procfs-y			:= proc_powerpc.o
>  obj-$(CONFIG_PROC_FS)		+= $(procfs-y)
>  rtaspci-$(CONFIG_PPC64)-$(CONFIG_PCI)	:= rtas_pci.o
> diff --git a/arch/powerpc/kernel/asm-offsets.c b/arch/powerpc/kernel/asm-offsets.c
> index 89cf15566c4e..b8162bb80ddb 100644
> --- a/arch/powerpc/kernel/asm-offsets.c
> +++ b/arch/powerpc/kernel/asm-offsets.c
> @@ -762,20 +762,11 @@ int main(void)
>  #endif
> 
>  #ifdef CONFIG_PPC_POWERNV
> +	/* POWER7/8 specific idle fields (kernel/idle_book3s.S) */
>  	OFFSET(PACA_CORE_IDLE_STATE_PTR, paca_struct, core_idle_state_ptr);
>  	OFFSET(PACA_THREAD_IDLE_STATE, paca_struct, thread_idle_state);
>  	OFFSET(PACA_THREAD_MASK, paca_struct, thread_mask);
>  	OFFSET(PACA_SUBCORE_SIBLING_MASK, paca_struct, subcore_sibling_mask);
> -	OFFSET(PACA_REQ_PSSCR, paca_struct, requested_psscr);
> -	OFFSET(PACA_DONT_STOP, paca_struct, dont_stop);
> -#define STOP_SPR(x, f)	OFFSET(x, paca_struct, stop_sprs.f)
> -	STOP_SPR(STOP_PID, pid);
> -	STOP_SPR(STOP_LDBAR, ldbar);
> -	STOP_SPR(STOP_FSCR, fscr);
> -	STOP_SPR(STOP_HFSCR, hfscr);
> -	STOP_SPR(STOP_MMCR1, mmcr1);
> -	STOP_SPR(STOP_MMCR2, mmcr2);
> -	STOP_SPR(STOP_MMCRA, mmcra);
>  #endif
> 
>  	DEFINE(PPC_DBELL_SERVER, PPC_DBELL_SERVER);
> diff --git a/arch/powerpc/kernel/exceptions-64s.S b/arch/powerpc/kernel/exceptions-64s.S
> index 76a14702cb9c..36b5f0e18c0c 100644
> --- a/arch/powerpc/kernel/exceptions-64s.S
> +++ b/arch/powerpc/kernel/exceptions-64s.S
> @@ -135,8 +135,14 @@ TRAMP_KVM(PACA_EXNMI, 0x100)
> 
>  #ifdef CONFIG_PPC_P7_NAP
>  EXC_COMMON_BEGIN(system_reset_idle_common)
> +BEGIN_FTR_SECTION
> +	mfspr	r3,SPRN_SRR1
> +	bltlr	cr3	/* no state loss, return to idle caller */

This is a nice optimization. But perhaps needs a carefully checked.
We wakeup at SYSTEM_RESET only when we have executed stop via the
mayloss() call which creates an additional stack frame. When we return
back to the caller, we will go back with that stack frame. Perhaps we
need to jump to a isa3_idle_wake_gpr_noloss ?

> +	b	isa3_idle_wake_gpr_loss
> +FTR_SECTION_ELSE
>  	mfspr	r12,SPRN_SRR1
>  	b	pnv_powersave_wakeup
> +ALT_FTR_SECTION_END_IFSET(CPU_FTR_ARCH_300)
>  #endif
> 
>  /*
> @@ -425,7 +431,9 @@ EXC_COMMON_BEGIN(machine_check_idle_common)
>  	li	r11,0
>  	mtmsrd	r11,1
> 
> -	b	pnv_powersave_wakeup_mce
> +	/* XXX fixup
> +	b	pnv_powersave_wakeup_mce */
> +	b	.
>  #endif
>  	/*
>  	 * Handle machine check early in real mode. We come here with
> diff --git a/arch/powerpc/kernel/idle_book3s.S b/arch/powerpc/kernel/idle_book3s.S
> index d85d5515a091..506b88768767 100644
> --- a/arch/powerpc/kernel/idle_book3s.S
> +++ b/arch/powerpc/kernel/idle_book3s.S
> @@ -1,6 +1,6 @@
>  /*
> - *  This file contains idle entry/exit functions for POWER7,
> - *  POWER8 and POWER9 CPUs.
> + *  This file contains idle entry/exit functions for POWER7 and
> + *  POWER8 CPUs.
>   *
>   *  This program is free software; you can redistribute it and/or
>   *  modify it under the terms of the GNU General Public License
> @@ -56,19 +56,8 @@ save_sprs_to_stack:
>  	 * Note all register i.e per-core, per-subcore or per-thread is saved
>  	 * here since any thread in the core might wake up first
>  	 */
> -BEGIN_FTR_SECTION
> -	/*
> -	 * Note - SDR1 is dropped in Power ISA v3. Hence not restoring
> -	 * SDR1 here
> -	 */
> -	mfspr	r3,SPRN_PTCR
> -	std	r3,_PTCR(r1)
> -	mfspr	r3,SPRN_LPCR
> -	std	r3,_LPCR(r1)
> -FTR_SECTION_ELSE
>  	mfspr	r3,SPRN_SDR1
>  	std	r3,_SDR1(r1)
> -ALT_FTR_SECTION_END_IFSET(CPU_FTR_ARCH_300)
>  	mfspr	r3,SPRN_RPR
>  	std	r3,_RPR(r1)
>  	mfspr	r3,SPRN_SPURR
> @@ -85,66 +74,6 @@ ALT_FTR_SECTION_END_IFSET(CPU_FTR_ARCH_300)
>  	std	r3,_WORT(r1)
>  	mfspr	r3,SPRN_WORC
>  	std	r3,_WORC(r1)
> -/*
> - * On POWER9, there are idle states such as stop4, invoked via cpuidle,
> - * that lose hypervisor resources. In such cases, we need to save
> - * additional SPRs before entering those idle states so that they can
> - * be restored to their older values on wakeup from the idle state.
> - *
> - * On POWER8, the only such deep idle state is winkle which is used
> - * only in the context of CPU-Hotplug, where these additional SPRs are
> - * reinitiazed to a sane value. Hence there is no need to save/restore
> - * these SPRs.
> - */
> -BEGIN_FTR_SECTION
> -	blr
> -END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_300)
> -
> -power9_save_additional_sprs:
> -	mfspr	r3, SPRN_PID
> -	mfspr	r4, SPRN_LDBAR
> -	std	r3, STOP_PID(r13)
> -	std	r4, STOP_LDBAR(r13)
> -
> -	mfspr	r3, SPRN_FSCR
> -	mfspr	r4, SPRN_HFSCR
> -	std	r3, STOP_FSCR(r13)
> -	std	r4, STOP_HFSCR(r13)
> -
> -	mfspr	r3, SPRN_MMCRA
> -	mfspr	r4, SPRN_MMCR0
> -	std	r3, STOP_MMCRA(r13)
> -	std	r4, _MMCR0(r1)
> -
> -	mfspr	r3, SPRN_MMCR1
> -	mfspr	r4, SPRN_MMCR2
> -	std	r3, STOP_MMCR1(r13)
> -	std	r4, STOP_MMCR2(r13)
> -	blr
> -
> -power9_restore_additional_sprs:
> -	ld	r3,_LPCR(r1)
> -	ld	r4, STOP_PID(r13)
> -	mtspr	SPRN_LPCR,r3
> -	mtspr	SPRN_PID, r4
> -
> -	ld	r3, STOP_LDBAR(r13)
> -	ld	r4, STOP_FSCR(r13)
> -	mtspr	SPRN_LDBAR, r3
> -	mtspr	SPRN_FSCR, r4
> -
> -	ld	r3, STOP_HFSCR(r13)
> -	ld	r4, STOP_MMCRA(r13)
> -	mtspr	SPRN_HFSCR, r3
> -	mtspr	SPRN_MMCRA, r4
> -
> -	ld	r3, _MMCR0(r1)
> -	ld	r4, STOP_MMCR1(r13)
> -	mtspr	SPRN_MMCR0, r3
> -	mtspr	SPRN_MMCR1, r4
> -
> -	ld	r3, STOP_MMCR2(r13)
> -	mtspr	SPRN_MMCR2, r3
>  	blr
> 
>  /*
> @@ -167,13 +96,23 @@ core_idle_lock_held:
>  	blr
> 
>  /*
> - * Pass requested state in r3:
> - *	r3 - PNV_THREAD_NAP/SLEEP/WINKLE in POWER8
> - *	   - Requested PSSCR value in POWER9
> - *
> - * Address of idle handler to branch to in realmode in r4
> + * This is the sequence required to execute idle instructions, as
> + * specified in ISA v2.07 (and earlier). MSR[IR] and MSR[DR] must be 0.
> + */
> +#define IDLE_STATE_ENTER_SEQ_NORET(IDLE_INST)			\
> +	/* Magic NAP/SLEEP/WINKLE mode enter sequence */	\
> +	std	r0,0(r1);					\
> +	ptesync;						\
> +	ld	r0,0(r1);					\
> +236:	cmpd	cr0,r0,r0;					\
> +	bne	236b;						\
> +	IDLE_INST;
> +
> +/*
> + * Entered with MSR[EE]=0 and no soft-masked interrupts pending.
> + * r3 contains desired idle state (PNV_THREAD_NAP/SLEEP/WINKLE).
>   */
> -pnv_powersave_common:
> +_GLOBAL(power7_idle_insn)
>  	/* Use r3 to pass state nap/sleep/winkle */
>  	/* NAP is a state loss, we create a regs frame on the
>  	 * stack, fill it up with the state we care about and
> @@ -181,8 +120,6 @@ pnv_powersave_common:
>  	 * need to save PC, some CR bits and the NV GPRs,
>  	 * but for now an interrupt frame will do.
>  	 */
> -	mtctr	r4
> -
>  	mflr	r0
>  	std	r0,16(r1)
>  	stdu	r1,-INT_FRAME_SIZE(r1)
> @@ -200,16 +137,7 @@ pnv_powersave_common:
>  	std	r5,_CCR(r1)
>  	std	r1,PACAR1(r13)
> 
> -BEGIN_FTR_SECTION
> -	/*
> -	 * POWER9 does not require real mode to stop, and presently does not
> -	 * set hwthread_state for KVM (threads don't share MMU context), so
> -	 * we can remain in virtual mode for this.
> -	 */
> -	bctr
> -END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300)
>  	/*
> -	 * POWER8
>  	 * Go to real mode to do the nap, as required by the architecture.
>  	 * Also, we need to be in real mode before setting hwthread_state,
>  	 * because as soon as we do that, another thread can switch
> @@ -217,24 +145,7 @@ END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300)
>  	 */
>  	LOAD_REG_IMMEDIATE(r7, MSR_IDLE)
>  	mtmsrd	r7,0
> -	bctr
> 
> -/*
> - * This is the sequence required to execute idle instructions, as
> - * specified in ISA v2.07 (and earlier). MSR[IR] and MSR[DR] must be 0.
> - */
> -#define IDLE_STATE_ENTER_SEQ_NORET(IDLE_INST)			\
> -	/* Magic NAP/SLEEP/WINKLE mode enter sequence */	\
> -	std	r0,0(r1);					\
> -	ptesync;						\
> -	ld	r0,0(r1);					\
> -236:	cmpd	cr0,r0,r0;					\
> -	bne	236b;						\
> -	IDLE_INST;
> -
> -
> -	.globl pnv_enter_arch207_idle_mode
> -pnv_enter_arch207_idle_mode:
>  #ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE
>  	/* Tell KVM we're entering idle */
>  	li	r4,KVM_HWTHREAD_IN_IDLE
> @@ -321,86 +232,6 @@ enter_winkle:
> 
>  	IDLE_STATE_ENTER_SEQ_NORET(PPC_WINKLE)
> 
> -/*
> - * r3 - PSSCR value corresponding to the requested stop state.
> - */
> -power_enter_stop:
> -/*
> - * Check if we are executing the lite variant with ESL=EC=0
> - */
> -	andis.   r4,r3,PSSCR_EC_ESL_MASK_SHIFTED
> -	clrldi   r3,r3,60 /* r3 = Bits[60:63] = Requested Level (RL) */
> -	bne	 .Lhandle_esl_ec_set
> -	PPC_STOP
> -	li	r3,0  /* Since we didn't lose state, return 0 */
> -	std	r3, PACA_REQ_PSSCR(r13)
> -
> -	/*
> -	 * pnv_wakeup_noloss() expects r12 to contain the SRR1 value so
> -	 * it can determine if the wakeup reason is an HMI in
> -	 * CHECK_HMI_INTERRUPT.
> -	 *
> -	 * However, when we wakeup with ESL=0, SRR1 will not contain the wakeup
> -	 * reason, so there is no point setting r12 to SRR1.
> -	 *
> -	 * Further, we clear r12 here, so that we don't accidentally enter the
> -	 * HMI in pnv_wakeup_noloss() if the value of r12[42:45] == WAKE_HMI.
> -	 */
> -	li	r12, 0
> -	b 	pnv_wakeup_noloss
> -
> -.Lhandle_esl_ec_set:
> -BEGIN_FTR_SECTION
> -	/*
> -	 * POWER9 DD2.0 or earlier can incorrectly set PMAO when waking up after
> -	 * a state-loss idle. Saving and restoring MMCR0 over idle is a
> -	 * workaround.
> -	 */
> -	mfspr	r4,SPRN_MMCR0
> -	std	r4,_MMCR0(r1)
> -END_FTR_SECTION_IFCLR(CPU_FTR_POWER9_DD2_1)
> -
> -/*
> - * Check if the requested state is a deep idle state.
> - */
> -	LOAD_REG_ADDRBASE(r5,pnv_first_deep_stop_state)
> -	ld	r4,ADDROFF(pnv_first_deep_stop_state)(r5)
> -	cmpd	r3,r4
> -	bge	.Lhandle_deep_stop
> -	PPC_STOP	/* Does not return (system reset interrupt) */
> -
> -.Lhandle_deep_stop:
> -/*
> - * Entering deep idle state.
> - * Clear thread bit in PACA_CORE_IDLE_STATE, save SPRs to
> - * stack and enter stop
> - */
> -	lbz     r7,PACA_THREAD_MASK(r13)
> -	ld      r14,PACA_CORE_IDLE_STATE_PTR(r13)
> -
> -lwarx_loop_stop:
> -	lwarx   r15,0,r14
> -	andis.	r9,r15,PNV_CORE_IDLE_LOCK_BIT@h
> -	bnel-	core_idle_lock_held
> -	andc    r15,r15,r7                      /* Clear thread bit */
> -
> -	stwcx.  r15,0,r14
> -	bne-    lwarx_loop_stop
> -	isync
> -
> -	bl	save_sprs_to_stack
> -
> -	PPC_STOP	/* Does not return (system reset interrupt) */
> -
> -/*
> - * Entered with MSR[EE]=0 and no soft-masked interrupts pending.
> - * r3 contains desired idle state (PNV_THREAD_NAP/SLEEP/WINKLE).
> - */
> -_GLOBAL(power7_idle_insn)
> -	/* Now check if user or arch enabled NAP mode */
> -	LOAD_REG_ADDR(r4, pnv_enter_arch207_idle_mode)
> -	b	pnv_powersave_common
> -
>  #define CHECK_HMI_INTERRUPT						\
>  BEGIN_FTR_SECTION_NESTED(66);						\
>  	rlwinm	r0,r12,45-31,0xf;  /* extract wake reason field (P8) */	\
> @@ -419,53 +250,6 @@ ALT_FTR_SECTION_END_NESTED_IFSET(CPU_FTR_ARCH_207S, 66);		\
>  	ld	r3,ORIG_GPR3(r1);	/* Restore original r3 */	\
>  20:	nop;
> 
> -/*
> - * Entered with MSR[EE]=0 and no soft-masked interrupts pending.
> - * r3 contains desired PSSCR register value.
> - *
> - * Offline (CPU unplug) case also must notify KVM that the CPU is
> - * idle.
> - */
> -_GLOBAL(power9_offline_stop)
> -#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE
> -	/*
> -	 * Tell KVM we're entering idle.
> -	 * This does not have to be done in real mode because the P9 MMU
> -	 * is independent per-thread. Some steppings share radix/hash mode
> -	 * between threads, but in that case KVM has a barrier sync in real
> -	 * mode before and after switching between radix and hash.
> -	 */
> -	li	r4,KVM_HWTHREAD_IN_IDLE
> -	stb	r4,HSTATE_HWTHREAD_STATE(r13)
> -#endif
> -	/* fall through */
> -
> -_GLOBAL(power9_idle_stop)
> -	std	r3, PACA_REQ_PSSCR(r13)
> -#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE
> -BEGIN_FTR_SECTION
> -	sync
> -	lwz	r5, PACA_DONT_STOP(r13)
> -	cmpwi	r5, 0
> -	bne	1f
> -END_FTR_SECTION_IFSET(CPU_FTR_P9_TM_XER_SO_BUG)
> -#endif
> -	mtspr 	SPRN_PSSCR,r3
> -	LOAD_REG_ADDR(r4,power_enter_stop)
> -	b	pnv_powersave_common
> -	/* No return */
> -#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE
> -1:
> -	/*
> -	 * We get here when TM / thread reconfiguration bug workaround
> -	 * code wants to get the CPU into SMT4 mode, and therefore
> -	 * we are being asked not to stop.
> -	 */
> -	li	r3, 0
> -	std	r3, PACA_REQ_PSSCR(r13)
> -	blr		/* return 0 for wakeup cause / SRR1 value */
> -#endif
> -
>  /*
>   * Called from machine check handler for powersave wakeups.
>   * Low level machine check processing has already been done. Now just
> @@ -499,11 +283,7 @@ pnv_powersave_wakeup_mce:
>  pnv_powersave_wakeup:
>  	ld	r2, PACATOC(r13)
> 
> -BEGIN_FTR_SECTION
> -	bl	pnv_restore_hyp_resource_arch300
> -FTR_SECTION_ELSE
> -	bl	pnv_restore_hyp_resource_arch207
> -ALT_FTR_SECTION_END_IFSET(CPU_FTR_ARCH_300)
> +	bl	pnv_restore_hyp_resource
> 
>  	li	r0,PNV_THREAD_RUNNING
>  	stb	r0,PACA_THREAD_IDLE_STATE(r13)	/* Clear thread state */
> @@ -535,50 +315,8 @@ ALT_FTR_SECTION_END_IFSET(CPU_FTR_ARCH_300)
>   *
>   * cr3 - set to gt if waking up with partial/complete hypervisor state loss
>   */
> -pnv_restore_hyp_resource_arch300:
> -	/*
> -	 * Workaround for POWER9, if we lost resources, the ERAT
> -	 * might have been mixed up and needs flushing. We also need
> -	 * to reload MMCR0 (see comment above). We also need to set
> -	 * then clear bit 60 in MMCRA to ensure the PMU starts running.
> -	 */
> -	blt	cr3,1f
> -BEGIN_FTR_SECTION
> -	PPC_INVALIDATE_ERAT
> -	ld	r1,PACAR1(r13)
> -	ld	r4,_MMCR0(r1)
> -	mtspr	SPRN_MMCR0,r4
> -END_FTR_SECTION_IFCLR(CPU_FTR_POWER9_DD2_1)
> -	mfspr	r4,SPRN_MMCRA
> -	ori	r4,r4,(1 << (63-60))
> -	mtspr	SPRN_MMCRA,r4
> -	xori	r4,r4,(1 << (63-60))
> -	mtspr	SPRN_MMCRA,r4
> -1:
> -	/*
> -	 * POWER ISA 3. Use PSSCR to determine if we
> -	 * are waking up from deep idle state
> -	 */
> -	LOAD_REG_ADDRBASE(r5,pnv_first_deep_stop_state)
> -	ld	r4,ADDROFF(pnv_first_deep_stop_state)(r5)
> -
> +pnv_restore_hyp_resource:
>  	/*
> -	 * 0-3 bits correspond to Power-Saving Level Status
> -	 * which indicates the idle state we are waking up from
> -	 */
> -	mfspr	r5, SPRN_PSSCR
> -	rldicl  r5,r5,4,60
> -	li	r0, 0		/* clear requested_psscr to say we're awake */
> -	std	r0, PACA_REQ_PSSCR(r13)
> -	cmpd	cr4,r5,r4
> -	bge	cr4,pnv_wakeup_tb_loss /* returns to caller */
> -
> -	blr	/* Waking up without hypervisor state loss. */
> -
> -/* Same calling convention as arch300 */
> -pnv_restore_hyp_resource_arch207:
> -	/*
> -	 * POWER ISA 2.07 or less.
>  	 * Check if we slept with sleep or winkle.
>  	 */
>  	lbz	r4,PACA_THREAD_IDLE_STATE(r13)
> @@ -598,15 +336,9 @@ pnv_restore_hyp_resource_arch207:
>   * Called if waking up from idle state which can cause either partial or
>   * complete hyp state loss.
>   * In POWER8, called if waking up from fastsleep or winkle
> - * In POWER9, called if waking up from stop state >= pnv_first_deep_stop_state
>   *
>   * r13 - PACA
>   * cr3 - gt if waking up with partial/complete hypervisor state loss
> - *
> - * If ISA300:
> - * cr4 - gt or eq if waking up from complete hypervisor state loss.
> - *
> - * If ISA207:
>   * r4 - PACA_THREAD_IDLE_STATE
>   */
>  pnv_wakeup_tb_loss:
> @@ -621,9 +353,7 @@ pnv_wakeup_tb_loss:
>  	 * and SRR1 test for restoring NVGPRs.
>  	 *
>  	 * We are about to clobber NVGPRs now, so set NAPSTATELOST to
> -	 * guarantee they will always be restored. This might be tightened
> -	 * with careful reading of specs (particularly for ISA300) but this
> -	 * is already a slow wakeup path and it's simpler to be safe.
> +	 * guarantee they will always be restored.
>  	 */
>  	li	r0,1
>  	stb	r0,PACA_NAPSTATELOST(r13)
> @@ -672,19 +402,16 @@ END_FTR_SECTION_IFSET(CPU_FTR_HVMODE)
>  	 * At this stage
>  	 * cr2 - eq if first thread to wakeup in core
>  	 * cr3-  gt if waking up with partial/complete hypervisor state loss
> -	 * ISA300:
> -	 * cr4 - gt or eq if waking up from complete hypervisor state loss.
>  	 */
> 
> -BEGIN_FTR_SECTION
>  	/*
>  	 * Were we in winkle?
>  	 * If yes, check if all threads were in winkle, decrement our
>  	 * winkle count, set all thread winkle bits if all were in winkle.
> -	 * Check if our thread has a winkle bit set, and set cr4 accordingly
> -	 * (to match ISA300, above). Pseudo-code for core idle state
> -	 * transitions for ISA207 is as follows (everything happens atomically
> -	 * due to store conditional and/or lock bit):
> +	 * Check if our thread has a winkle bit set, and set cr4 accordingly.
> +	 * Pseudo-code for core idle state transitions for ISA207 is as follows
> +	 * (everything happens atomically due to store conditional and/or lock
> +	 * bit):
>  	 *
>  	 * nap_idle() { }
>  	 * nap_wake() { }
> @@ -749,7 +476,6 @@ BEGIN_FTR_SECTION
> 
>  	or	r15,r15,r7		/* Set thread bit */
>  	beq	first_thread_in_subcore
> -END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_300)
> 
>  	or	r15,r15,r7		/* Set thread bit */
>  	beq	cr2,first_thread_in_core
> @@ -815,15 +541,6 @@ timebase_resync:
>  	 * complete hypervisor state loss. Restore per core hypervisor
>  	 * state.
>  	 */
> -BEGIN_FTR_SECTION
> -	ld	r4,_PTCR(r1)
> -	mtspr	SPRN_PTCR,r4
> -	ld	r4,_RPR(r1)
> -	mtspr	SPRN_RPR,r4
> -	ld	r4,_AMOR(r1)
> -	mtspr	SPRN_AMOR,r4
> -END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300)
> -
>  	ld	r4,_TSCR(r1)
>  	mtspr	SPRN_TSCR,r4
>  	ld	r4,_WORC(r1)
> @@ -845,9 +562,6 @@ common_exit:
> 
>  	/* Waking up from winkle */
> 
> -BEGIN_MMU_FTR_SECTION
> -	b	no_segments
> -END_MMU_FTR_SECTION_IFSET(MMU_FTR_TYPE_RADIX)
>  	/* Restore SLB  from PACA */
>  	ld	r8,PACA_SLBSHADOWPTR(r13)
> 
> @@ -861,7 +575,6 @@ END_MMU_FTR_SECTION_IFSET(MMU_FTR_TYPE_RADIX)
>  	slbmte	r6,r5
>  1:	addi	r8,r8,16
>  	.endr
> -no_segments:
> 
>  	/* Restore per thread state */
> 
> @@ -884,17 +597,6 @@ no_segments:
>  	mtctr	r12
>  	bctrl
> 
> -/*
> - * On POWER9, we can come here on wakeup from a cpuidle stop state.
> - * Hence restore the additional SPRs to the saved value.
> - *
> - * On POWER8, we come here only on winkle. Since winkle is used
> - * only in the case of CPU-Hotplug, we don't need to restore
> - * the additional SPRs.
> - */
> -BEGIN_FTR_SECTION
> -	bl 	power9_restore_additional_sprs
> -END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300)
>  hypervisor_state_restored:
> 
>  	mr	r12,r19
> diff --git a/arch/powerpc/kernel/idle_isa3.S b/arch/powerpc/kernel/idle_isa3.S
> new file mode 100644
> index 000000000000..c869512b716a
> --- /dev/null
> +++ b/arch/powerpc/kernel/idle_isa3.S
> @@ -0,0 +1,73 @@
> +/*
> + *  This file contains general idle entry/exit functions. The platform / CPU
> + *  must call the correct save/restore functions and ensure SPRs are saved
> + *  and restored correctly, handle KVM, interrupts, etc.
> + *
> + *  This program is free software; you can redistribute it and/or
> + *  modify it under the terms of the GNU General Public License
> + *  as published by the Free Software Foundation; either version
> + *  2 of the License, or (at your option) any later version.
> + */
> +
> +#include <asm/ppc_asm.h>
> +#include <asm/asm-offsets.h>
> +#include <asm/ppc-opcode.h>
> +
> +/*
> + * Desired PSSCR in r3
> + *
> + * No state will be lost regardless of wakeup mechanism (interrupt or NIA).
> + * Interrupt driven wakeup may clobber volatiles, and should blr (with LR
> + * unchanged) to return to caller with r3 set according to caller's expected
> + * return code (for Book3S/64 that is SRR1).
> + *
> + * Caller is responsible for restoring SPRs, MSR, etc.
> + */
> +_GLOBAL(isa3_idle_stop_noloss)
> +	mtspr 	SPRN_PSSCR,r3
> +	PPC_STOP
> +	li	r3,0
> +	blr
> +
> +/*
> + * Desired PSSCR in r3
> + *
> + * GPRs may be lost, so they are saved here. Wakeup is by interrupt only.
> + * Wakeup can return to caller by calling pnv_powersave_wakeup_gpr_loss
> + * with r3 set to return value.
> + *
> + * A wakeup without GPR loss may alterateively be handled as in
> + * isa3_idle_stop_noloss as an optimisation.
> + *
> + * Caller is responsible for restoring SPRs, MSR, etc.
> + */
> +_GLOBAL(isa3_idle_stop_mayloss)
> +	std	r1,PACAR1(r13)
> +	mflr	r4
> +	mfcr	r5
> +	/* use stack red zone rather than a new frame */
> +	addi	r6,r1,-INT_FRAME_SIZE
> +	SAVE_GPR(2, r6)
> +	SAVE_NVGPRS(r6)
> +	std	r4,_LINK(r6)
> +	std	r5,_CCR(r6)
> +	mtspr 	SPRN_PSSCR,r3
> +	PPC_STOP


Suppose we enter into a isa3_idle_stop_may_loss (ESL=EC=1), but as
soon as we execute PPC_STOP, if we get interrupted, we will go back to
SYSTEM_RESET vector.

The SRR1[46:47] should be 0x1, due to which we will do a bl there,
thereby going back to the caller of isa3_idle_stop_mayloss. Which
implies that we need to perform the stack unwinding there.


> +	b	.
> +
> +/*
> + * Desired return value in r3
> + *
> + * Idle wakeup can call this after calling isa3_idle_stop_loss to
> + * return to caller with r3 as return code.
> + */
> +_GLOBAL(isa3_idle_wake_gpr_loss)
> +	ld	r1,PACAR1(r13)
> +	addi	r6,r1,-INT_FRAME_SIZE
> +	ld	r4,_LINK(r6)
> +	ld	r5,_CCR(r6)
> +	REST_NVGPRS(r6)
> +	REST_GPR(2, r6)
> +	mtlr	r4
> +	mtcr	r5
> +	blr
> diff --git a/arch/powerpc/kernel/setup-common.c b/arch/powerpc/kernel/setup-common.c
> index 40b44bb53a4e..e089da156ef3 100644
> --- a/arch/powerpc/kernel/setup-common.c
> +++ b/arch/powerpc/kernel/setup-common.c
> @@ -401,8 +401,8 @@ void __init check_for_initrd(void)
> 
>  #ifdef CONFIG_SMP
> 
> -int threads_per_core, threads_per_subcore, threads_shift;
> -cpumask_t threads_core_mask;
> +int threads_per_core, threads_per_subcore, threads_shift __read_mostly;
> +cpumask_t threads_core_mask __read_mostly;
>  EXPORT_SYMBOL_GPL(threads_per_core);
>  EXPORT_SYMBOL_GPL(threads_per_subcore);
>  EXPORT_SYMBOL_GPL(threads_shift);
> diff --git a/arch/powerpc/mm/slb.c b/arch/powerpc/mm/slb.c
> index cb796724a6fc..2d5db5e0132e 100644
> --- a/arch/powerpc/mm/slb.c
> +++ b/arch/powerpc/mm/slb.c
> @@ -90,7 +90,7 @@ static inline void create_shadowed_slbe(unsigned long ea, int ssize,
>  		     : "memory" );
>  }
> 
> -static void __slb_flush_and_rebolt(void)
> +void __slb_flush_and_rebolt(void)
>  {
>  	/* If you change this make sure you change SLB_NUM_BOLTED
>  	 * and PR KVM appropriately too. */
> @@ -128,6 +128,8 @@ static void __slb_flush_and_rebolt(void)
>  		        "r"(ksp_vsid_data),
>  		        "r"(ksp_esid_data)
>  		     : "memory");
> +
> +	get_paca()->slb_cache_ptr = 0;
>  }
> 
>  void slb_flush_and_rebolt(void)
> @@ -142,7 +144,6 @@ void slb_flush_and_rebolt(void)
>  	hard_irq_disable();
> 
>  	__slb_flush_and_rebolt();
> -	get_paca()->slb_cache_ptr = 0;
>  }
> 
>  void slb_vmalloc_update(void)
> @@ -213,6 +214,7 @@ void switch_slb(struct task_struct *tsk, struct mm_struct *mm)
>  			asm volatile("slbie %0" : : "r" (slbie_data));
>  		}
>  		asm volatile("isync" : : : "memory");
> +		get_paca()->slb_cache_ptr = 0;
>  	} else {
>  		__slb_flush_and_rebolt();
>  	}
> @@ -221,7 +223,6 @@ void switch_slb(struct task_struct *tsk, struct mm_struct *mm)
>  	if (offset == 1 || offset > SLB_CACHE_ENTRIES)
>  		asm volatile("slbie %0" : : "r" (slbie_data));
> 
> -	get_paca()->slb_cache_ptr = 0;
>  	copy_mm_to_paca(mm);
> 
>  	/*
> diff --git a/arch/powerpc/platforms/powernv/idle.c b/arch/powerpc/platforms/powernv/idle.c
> index 12f13acee1f6..2e129b882727 100644
> --- a/arch/powerpc/platforms/powernv/idle.c
> +++ b/arch/powerpc/platforms/powernv/idle.c
> @@ -16,6 +16,7 @@
>  #include <linux/device.h>
>  #include <linux/cpu.h>
> 
> +#include <asm/asm-prototypes.h>
>  #include <asm/firmware.h>
>  #include <asm/machdep.h>
>  #include <asm/opal.h>
> @@ -46,10 +47,10 @@ static u64 pnv_default_stop_mask;
>  static bool default_stop_found;
> 
>  /*
> - * First deep stop state. Used to figure out when to save/restore
> - * hypervisor context.
> + * First stop state levels when HV and TB loss can occur.
>   */
> -u64 pnv_first_deep_stop_state = MAX_STOP_STATE;
> +static u64 pnv_first_tb_loss_level = MAX_STOP_STATE + 1;
> +static u64 pnv_first_hv_loss_level = MAX_STOP_STATE + 1;


> 
>  /*
>   * psscr value and mask of the deepest stop idle state.
> @@ -135,11 +136,11 @@ static int pnv_save_sprs_for_deep_states(void)
>  	return 0;
>  }
> 
> -static void pnv_alloc_idle_core_states(void)
> +static void pnv_alloc_idle_core_states_p8(void)
>  {
>  	int i, j;
>  	int nr_cores = cpu_nr_cores();
> -	u32 *core_idle_state;
> +	unsigned long *core_idle_state;
> 
>  	/*
>  	 * core_idle_state - The lower 8 bits track the idle state of
> @@ -166,7 +167,7 @@ static void pnv_alloc_idle_core_states(void)
>  		int node = cpu_to_node(first_cpu);
>  		size_t paca_ptr_array_size;
> 
> -		core_idle_state = kmalloc_node(sizeof(u32), GFP_KERNEL, node);
> +		core_idle_state = kmalloc_node(sizeof(unsigned long), GFP_KERNEL, node);
>  		*core_idle_state = (1 << threads_per_core) - 1;
>  		paca_ptr_array_size = (threads_per_core *
>  				       sizeof(struct paca_struct *));
> @@ -181,41 +182,6 @@ static void pnv_alloc_idle_core_states(void)
>  	}
> 
>  	update_subcore_sibling_mask();
> -
> -	if (supported_cpuidle_states & OPAL_PM_LOSE_FULL_CONTEXT) {
> -		int rc = pnv_save_sprs_for_deep_states();
> -
> -		if (likely(!rc))
> -			return;
> -
> -		/*
> -		 * The stop-api is unable to restore hypervisor
> -		 * resources on wakeup from platform idle states which
> -		 * lose full context. So disable such states.
> -		 */
> -		supported_cpuidle_states &= ~OPAL_PM_LOSE_FULL_CONTEXT;
> -		pr_warn("cpuidle-powernv: Disabling idle states that lose full context\n");
> -		pr_warn("cpuidle-powernv: Idle power-savings, CPU-Hotplug affected\n");
> -
> -		if (cpu_has_feature(CPU_FTR_ARCH_300) &&
> -		    (pnv_deepest_stop_flag & OPAL_PM_LOSE_FULL_CONTEXT)) {
> -			/*
> -			 * Use the default stop state for CPU-Hotplug
> -			 * if available.
> -			 */
> -			if (default_stop_found) {
> -				pnv_deepest_stop_psscr_val =
> -					pnv_default_stop_val;
> -				pnv_deepest_stop_psscr_mask =
> -					pnv_default_stop_mask;
> -				pr_warn("cpuidle-powernv: Offlined CPUs will stop with psscr = 0x%016llx\n",
> -					pnv_deepest_stop_psscr_val);
> -			} else { /* Fallback to snooze loop for CPU-Hotplug */
> -				deepest_stop_found = false;
> -				pr_warn("cpuidle-powernv: Offlined CPUs will busy wait\n");
> -			}
> -		}
> -	}
>  }
> 
>  u32 pnv_get_supported_cpuidle_states(void)
> @@ -345,6 +311,263 @@ void power7_idle(void)
>  	power7_idle_type(PNV_THREAD_NAP);
>  }
> 
> +struct p9_sprs {
> +	/* per core */
> +	u64 ptcr;
> +	u64 rpr;
> +	u64 tscr;
> +	u64 ldbar;
> +
> +	/* per thread */
> +	u64 lpcr;
> +	u64 hfscr;
> +	u64 fscr;
> +	u64 pid;
> +	u64 purr;
> +	u64 spurr;
> +	u64 dscr;
> +
> +	u64 mmcra;
> +	u32 mmcr0;
> +	u32 mmcr1;
> +	u64 mmcr2;
> +};
> +
> +static inline void atomic_start_thread_idle(void)
> +{
> +	int cpu = raw_smp_processor_id();
> +	int first = cpu_first_thread_sibling(cpu);
> +	int thread = cpu_thread_in_core(cpu);
> +	unsigned long *state = &paca_ptrs[first]->idle_state;
> +	u64 s = READ_ONCE(*state);
> +	u64 new, tmp;
> +
> +again:
> +	if (unlikely(s & PNV_CORE_IDLE_LOCK_BIT)) {
> +		spin_begin();
> +		do {
> +			spin_cpu_relax();
> +			s = READ_ONCE(*state);
> +		} while (s & PNV_CORE_IDLE_LOCK_BIT);
> +		spin_end();
> +	}
> +
> +	BUG_ON(!(s & thread));
> +
> +	new = s & ~thread;
> +	tmp = cmpxchg(state, s, new);
> +	if (unlikely(tmp != s)) {
> +		s = tmp;
> +		goto again;
> +	}
> +}
> +
> +static inline void atomic_lock_thread_idle(void)
> +{
> +	int cpu = raw_smp_processor_id();
> +	int first = cpu_first_thread_sibling(cpu);
> +	unsigned long *state = &paca_ptrs[first]->idle_state;
> +
> +	while (unlikely(test_and_set_bit_lock(NR_PNV_CORE_IDLE_LOCK_BIT, state)))
> +		barrier();
> +}
> +
> +static inline void atomic_unlock_thread_idle(void)
> +{
> +	int cpu = raw_smp_processor_id();
> +	int first = cpu_first_thread_sibling(cpu);
> +	unsigned long *state = &paca_ptrs[first]->idle_state;
> +
> +	clear_bit_unlock(NR_PNV_CORE_IDLE_LOCK_BIT, state);
> +}
> +
> +static unsigned long power9_idle_stop(unsigned long psscr, bool  mmu_on)

We are not using the mmu_on anywhere in the code. Is this for
restoring the IR|DR bit in the MSR ?

> +{
> +	int cpu = raw_smp_processor_id();
> +	int first = cpu_first_thread_sibling(cpu);
> +	int thread = cpu_thread_in_core(cpu);
> +	unsigned long *state = &paca_ptrs[first]->idle_state;
> +	unsigned long srr1;
> +	unsigned long mmcr0 = 0;
> +	struct p9_sprs sprs;

Ok, so we save the sprs on the stack.

> +
> +	/* XXX: this gets rid of the uninitialized warning. Should use attributes because this is expensive */
> +	memset(&sprs, 0, sizeof(sprs));
> +
> +	if (!(psscr & (PSSCR_EC|PSSCR_ESL))) {
> +		/*
> +		 * Wake synchronously. SRESET via xscom may still cause
> +		 * a 0x100 powersave wakeup with SRR1 reason!
> +		 */
> +		srr1 = isa3_idle_stop_noloss(psscr);
> +		if (likely(!srr1))
> +			return 0;
> +
> +	} else {
> +		if (!cpu_has_feature(CPU_FTR_POWER9_DD2_1)) {
> +			 /*
> +			  * POWER9 DD2 can incorrectly set PMAO when waking up
> +			  * after a state-loss idle. Saving and restoring MMCR0
> +			  * over idle is a workaround.
> +			  */
> +			mmcr0 = mfspr(SPRN_MMCR0);
> +		}
> +		if ((psscr & PSSCR_RL_MASK) >= pnv_first_hv_loss_level) {
> +			atomic_start_thread_idle();
> +
> +			sprs.ptcr = mfspr(SPRN_PTCR);
> +			sprs.rpr = mfspr(SPRN_RPR);
> +			sprs.tscr = mfspr(SPRN_TSCR);
> +			sprs.ldbar = mfspr(SPRN_LDBAR);
> +
> +			sprs.lpcr = mfspr(SPRN_LPCR);
> +			sprs.hfscr = mfspr(SPRN_HFSCR);
> +			sprs.fscr = mfspr(SPRN_FSCR);
> +			sprs.pid = mfspr(SPRN_PID);
> +			sprs.purr = mfspr(SPRN_PURR);
> +			sprs.spurr = mfspr(SPRN_SPURR);
> +			sprs.dscr = mfspr(SPRN_DSCR);
> +
> +			sprs.mmcra = mfspr(SPRN_MMCRA);
> +			sprs.mmcr0 = mfspr(SPRN_MMCR0);
> +			sprs.mmcr1 = mfspr(SPRN_MMCR1);
> +			sprs.mmcr2 = mfspr(SPRN_MMCR2);
> +		}
> +#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE
> +		if (cpu_has_feature(CPU_FTR_P9_TM_XER_SO_BUG)) {
> +			local_paca->requested_psscr = psscr;
> +			/* order setting requested_psscr vs testing dont_stop */
> +			smp_mb();
> +			if (atomic_read(&local_paca->dont_stop)) {
> +				local_paca->requested_psscr = 0;
> +				return 0;
> +			}
> +
> +			srr1 = isa3_idle_stop_mayloss(psscr);
> +			local_paca->requested_psscr = 0;
> +		} else
> +#endif
> +			srr1 = isa3_idle_stop_mayloss(psscr);


As pointed out earlier, In both the mayloss cases if we return with
SRR1[46:47] = SRR_WS_NOLOSS, the topmost stack frame will still be the
one corresponding to the redzone, and not this function's stack frame.


> +	}
> +
> +	WARN_ON_ONCE(!srr1);
> +	WARN_ON_ONCE(mfmsr() & (MSR_IR|MSR_DR));
> +
> +	if ((srr1 & SRR1_WAKESTATE) != SRR1_WS_NOLOSS) {
> +		unsigned long mmcra;
> +
> +		/*
> +		 * Workaround for POWER9 DD2, if we lost resources, the ERAT
> +		 * might have been mixed up and needs flushing. We also need
> +		 * to reload MMCR0 (see mmcr0 comment above).
> +		 */
> +		if (!cpu_has_feature(CPU_FTR_POWER9_DD2_1)) {
> +			asm volatile(PPC_INVALIDATE_ERAT);
> +			mtspr(SPRN_MMCR0, mmcr0);
> +		}
> +
> +		/*
> +		 * DD2.2 and earlier need to set then clear bit 60 in MMCRA
> +		 * to ensure the PMU starts running.
> +		 */
> +		mmcra = mfspr(SPRN_MMCRA);
> +		mmcra |= PPC_BIT(60);
> +		mtspr(SPRN_MMCRA, mmcra);
> +		mmcra &= ~PPC_BIT(60);
> +		mtspr(SPRN_MMCRA, mmcra);
> +	}
> +
> +	if (unlikely((srr1 & SRR1_WAKEMASK_P8) == SRR1_WAKEHMI))
> +		hmi_exception_realmode(NULL);
> +
> +	if (likely((srr1 & SRR1_WAKESTATE) != SRR1_WS_HVLOSS)) {
> +		mtmsr(MSR_KERNEL);
> +		return srr1;
> +	}
> +
> +	/* HV state loss */
> +	WARN_ON((psscr & PSSCR_RL_MASK) < pnv_first_hv_loss_level);
> +
> +	atomic_lock_thread_idle();
> +
> +	WARN_ON(*state & thread);
> +
> +	if ((*state & ((1 << threads_per_core) - 1)) != 0)
> +		goto core_woken;
> +
> +	/* Per-core SPRs */
> +	mtspr(SPRN_PTCR, sprs.ptcr);
> +	mtspr(SPRN_RPR, sprs.rpr);
> +	mtspr(SPRN_TSCR, sprs.tscr);
> +	mtspr(SPRN_LDBAR, sprs.ldbar);
> +
> +	if ((psscr & PSSCR_RL_MASK) >= pnv_first_tb_loss_level) {
> +		unsigned long level = mfspr(SPRN_PSSCR) & PSSCR_RL_MASK;
> +		if (level >= pnv_first_tb_loss_level) {
> +			/* TB loss */
> +			if (opal_resync_timebase() != OPAL_SUCCESS)
> +				BUG();
> +		}
> +	}
> +
> +core_woken:
> +	*state |= thread;
> +	atomic_unlock_thread_idle();
> +
> +	/* Per-thread SPRs */
> +	mtspr(SPRN_LPCR, sprs.lpcr);
> +	mtspr(SPRN_HFSCR, sprs.hfscr);
> +	mtspr(SPRN_FSCR, sprs.fscr);
> +	mtspr(SPRN_PID, sprs.pid);
> +	mtspr(SPRN_PURR, sprs.purr);
> +	mtspr(SPRN_SPURR, sprs.spurr);
> +	mtspr(SPRN_DSCR, sprs.dscr);
> +
> +	mtspr(SPRN_MMCRA, sprs.mmcra);
> +	mtspr(SPRN_MMCR0, sprs.mmcr0);
> +	mtspr(SPRN_MMCR1, sprs.mmcr1);
> +	mtspr(SPRN_MMCR2, sprs.mmcr2);
> +
> +	if (!radix_enabled())
> +		__slb_flush_and_rebolt();
> +
> +	mtmsr(MSR_KERNEL);
> +
> +	return srr1;
> +}
> +
> +unsigned long power9_offline_stop(unsigned long psscr)
> +{
> +#ifndef CONFIG_KVM_BOOK3S_HV_POSSIBLE
> +	return power9_idle_stop(psscr, true);
> +#else
> +	unsigned long srr1;
> +
> +	/*
> +	 * Tell KVM we're entering idle.
> +	 * This does not have to be done in real mode because the P9 MMU
> +	 * is independent per-thread. Some steppings share radix/hash mode
> +	 * between threads, but in that case KVM has a barrier sync in real
> +	 * mode before and after switching between radix and hash.
> +	 */
> +	local_paca->kvm_hstate.hwthread_state = KVM_HWTHREAD_IN_IDLE;
> +	srr1 = power9_idle_stop(psscr, false);
> +
> +	if (local_paca->kvm_hstate.hwthread_state == KVM_HWTHREAD_IN_KERNEL) {
> +		local_paca->kvm_hstate.hwthread_state = KVM_HWTHREAD_IN_KERNEL;
> +		/* Order setting hwthread_state vs. testing hwthread_req */
> +		smp_mb();
> +	}
> +	if (local_paca->kvm_hstate.hwthread_req) {
> +		/* XXX: fix this so it's not garbage */
> +		asm volatile("b	kvm_start_guest" ::: "memory");
> +	}
> +	mtmsr(MSR_KERNEL);
> +
> +	return srr1;
> +#endif
> +}
> +
>  static unsigned long __power9_idle_type(unsigned long stop_psscr_val,
>  				      unsigned long stop_psscr_mask)
>  {
> @@ -358,7 +581,7 @@ static unsigned long __power9_idle_type(unsigned long stop_psscr_val,
>  	psscr = (psscr & ~stop_psscr_mask) | stop_psscr_val;
> 
>  	__ppc64_runlatch_off();
> -	srr1 = power9_idle_stop(psscr);
> +	srr1 = power9_idle_stop(psscr, true);
>  	__ppc64_runlatch_on();
> 
>  	fini_irq_for_idle_irqsoff();
> @@ -407,7 +630,7 @@ void pnv_power9_force_smt4_catch(void)
>  			atomic_inc(&paca_ptrs[cpu0+thr]->dont_stop);
>  	}
>  	/* order setting dont_stop vs testing requested_psscr */
> -	mb();
> +	smp_mb();
>  	for (thr = 0; thr < threads_per_core; ++thr) {
>  		if (!paca_ptrs[cpu0+thr]->requested_psscr)
>  			++awake_threads;
> @@ -623,7 +846,8 @@ static int __init pnv_power9_idle_init(struct device_node *np, u32 *flags,
>  	u64 *psscr_val = NULL;
>  	u64 *psscr_mask = NULL;
>  	u32 *residency_ns = NULL;
> -	u64 max_residency_ns = 0;
> +	u64 max_deep_residency_ns = 0;
> +	u64 max_default_residency_ns = 0;
>  	int rc = 0, i;
> 
>  	psscr_val = kcalloc(dt_idle_states, sizeof(*psscr_val), GFP_KERNEL);
> @@ -661,26 +885,25 @@ static int __init pnv_power9_idle_init(struct device_node *np, u32 *flags,
>  	}
> 
>  	/*
> -	 * Set pnv_first_deep_stop_state, pnv_deepest_stop_psscr_{val,mask},
> -	 * and the pnv_default_stop_{val,mask}.
> -	 *
> -	 * pnv_first_deep_stop_state should be set to the first stop
> -	 * level to cause hypervisor state loss.
> -	 *
>  	 * pnv_deepest_stop_{val,mask} should be set to values corresponding to
>  	 * the deepest stop state.
>  	 *
>  	 * pnv_default_stop_{val,mask} should be set to values corresponding to
> -	 * the shallowest (OPAL_PM_STOP_INST_FAST) loss-less stop state.
> +	 * the deepest loss-less (OPAL_PM_STOP_INST_FAST) stop state.
>  	 */
> -	pnv_first_deep_stop_state = MAX_STOP_STATE;
> +	pnv_first_tb_loss_level = MAX_STOP_STATE + 1;
> +	pnv_first_hv_loss_level = MAX_STOP_STATE + 1;
>  	for (i = 0; i < dt_idle_states; i++) {
>  		int err;
>  		u64 psscr_rl = psscr_val[i] & PSSCR_RL_MASK;
> 
> +		if ((flags[i] & OPAL_PM_TIMEBASE_STOP) &&
> +		     (pnv_first_tb_loss_level > psscr_rl))
> +			pnv_first_tb_loss_level = psscr_rl;
> +
>  		if ((flags[i] & OPAL_PM_LOSE_FULL_CONTEXT) &&
> -		     (pnv_first_deep_stop_state > psscr_rl))
> -			pnv_first_deep_stop_state = psscr_rl;
> +		     (pnv_first_hv_loss_level > psscr_rl))
> +			pnv_first_hv_loss_level = psscr_rl;
> 
>  		err = validate_psscr_val_mask(&psscr_val[i], &psscr_mask[i],
>  					      flags[i]);
> @@ -689,19 +912,21 @@ static int __init pnv_power9_idle_init(struct device_node *np, u32 *flags,
>  			continue;
>  		}
> 
> -		if (max_residency_ns < residency_ns[i]) {
> -			max_residency_ns = residency_ns[i];
> +		if (max_deep_residency_ns < residency_ns[i]) {
> +			max_deep_residency_ns = residency_ns[i];
>  			pnv_deepest_stop_psscr_val = psscr_val[i];
>  			pnv_deepest_stop_psscr_mask = psscr_mask[i];
>  			pnv_deepest_stop_flag = flags[i];
>  			deepest_stop_found = true;
>  		}
> 
> -		if (!default_stop_found &&
> +		if (max_default_residency_ns < residency_ns[i] &&
>  		    (flags[i] & OPAL_PM_STOP_INST_FAST)) {
> +			max_default_residency_ns = residency_ns[i];
>  			pnv_default_stop_val = psscr_val[i];
>  			pnv_default_stop_mask = psscr_mask[i];
>  			default_stop_found = true;
> +			WARN_ON(flags[i] & OPAL_PM_LOSE_FULL_CONTEXT);
>  		}
>  	}
> 
> @@ -721,15 +946,48 @@ static int __init pnv_power9_idle_init(struct device_node *np, u32 *flags,
>  			pnv_deepest_stop_psscr_mask);
>  	}
> 
> -	pr_info("cpuidle-powernv: Requested Level (RL) value of first deep stop = 0x%llx\n",
> -		pnv_first_deep_stop_state);
> +	pr_info("cpuidle-powernv: First stop level that may lose SPRs = 0x%lld\n",
> +		pnv_first_hv_loss_level);
> +
> +	pr_info("cpuidle-powernv: First stop level that may lose timebase = 0x%lld\n",
> +		pnv_first_tb_loss_level);
>  out:
>  	kfree(psscr_val);
>  	kfree(psscr_mask);
>  	kfree(residency_ns);
> +
>  	return rc;
>  }
> 
> +static void __init pnv_disable_deep_states(void)
> +{
> +	/*
> +	 * The stop-api is unable to restore hypervisor
> +	 * resources on wakeup from platform idle states which
> +	 * lose full context. So disable such states.
> +	 */
> +	supported_cpuidle_states &= ~OPAL_PM_LOSE_FULL_CONTEXT;
> +	pr_warn("cpuidle-powernv: Disabling idle states that lose full context\n");
> +	pr_warn("cpuidle-powernv: Idle power-savings, CPU-Hotplug affected\n");
> +
> +	if (cpu_has_feature(CPU_FTR_ARCH_300) &&
> +	    (pnv_deepest_stop_flag & OPAL_PM_LOSE_FULL_CONTEXT)) {
> +		/*
> +		 * Use the default stop state for CPU-Hotplug
> +		 * if available.
> +		 */
> +		if (default_stop_found) {
> +			pnv_deepest_stop_psscr_val = pnv_default_stop_val;
> +			pnv_deepest_stop_psscr_mask = pnv_default_stop_mask;
> +			pr_warn("cpuidle-powernv: Offlined CPUs will stop with psscr = 0x%016llx\n",
> +				pnv_deepest_stop_psscr_val);
> +		} else { /* Fallback to snooze loop for CPU-Hotplug */
> +			deepest_stop_found = false;
> +			pr_warn("cpuidle-powernv: Offlined CPUs will busy wait\n");
> +		}
> +	}
> +}
> +
>  /*
>   * Probe device tree for supported idle states
>   */
> @@ -771,6 +1029,7 @@ static void __init pnv_probe_idle_states(void)
>  out:
>  	kfree(flags);
>  }
> +
>  static int __init pnv_init_idle_states(void)
>  {
> 
> @@ -798,10 +1057,29 @@ static int __init pnv_init_idle_states(void)
>  				&dev_attr_fastsleep_workaround_applyonce);
>  	}
> 
> -	pnv_alloc_idle_core_states();
> +	if (!cpu_has_feature(CPU_FTR_ARCH_300)) {
> +		pnv_alloc_idle_core_states_p8();
> +		if (supported_cpuidle_states & OPAL_PM_NAP_ENABLED)
> +			ppc_md.power_save = power7_idle;
> +	} else {
> +		int cpu;
> +
> +		for_each_present_cpu(cpu) {
> +			paca_ptrs[cpu]->requested_psscr = 0;
> +			paca_ptrs[cpu]->idle_state = 0;
> +			if (cpu == cpu_first_thread_sibling(cpu))
> +				paca_ptrs[cpu]->idle_state =
> +					(1 << threads_per_core) - 1;
> +#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE
> +			atomic_set(&paca_ptrs[cpu]->dont_stop, 0);
> +#endif
> +		}
> +	}
> 
> -	if (supported_cpuidle_states & OPAL_PM_NAP_ENABLED)
> -		ppc_md.power_save = power7_idle;
> +	if (supported_cpuidle_states & OPAL_PM_LOSE_FULL_CONTEXT) {
> +		if (pnv_save_sprs_for_deep_states())
> +			pnv_disable_deep_states();
> +	}
> 
>  out:
>  	return 0;
> diff --git a/arch/powerpc/xmon/xmon.c b/arch/powerpc/xmon/xmon.c
> index 21119cfe8474..09120d4ec12b 100644
> --- a/arch/powerpc/xmon/xmon.c
> +++ b/arch/powerpc/xmon/xmon.c
> @@ -2425,19 +2425,18 @@ static void dump_one_paca(int cpu)
>  #endif
> 
>  #ifdef CONFIG_PPC_POWERNV
> -	DUMP(p, core_idle_state_ptr, "%-*px");
> -	DUMP(p, thread_idle_state, "%#-*x");
> -	DUMP(p, thread_mask, "%#-*x");
> -	DUMP(p, subcore_sibling_mask, "%#-*x");
> -	DUMP(p, requested_psscr, "%#-*llx");
> -	DUMP(p, stop_sprs.pid, "%#-*llx");
> -	DUMP(p, stop_sprs.ldbar, "%#-*llx");
> -	DUMP(p, stop_sprs.fscr, "%#-*llx");
> -	DUMP(p, stop_sprs.hfscr, "%#-*llx");
> -	DUMP(p, stop_sprs.mmcr1, "%#-*llx");
> -	DUMP(p, stop_sprs.mmcr2, "%#-*llx");
> -	DUMP(p, stop_sprs.mmcra, "%#-*llx");
> -	DUMP(p, dont_stop.counter, "%#-*x");
> +	if (!cpu_has_feature(CPU_FTR_ARCH_300)) {
> +		DUMP(p, core_idle_state_ptr, "%-*px");
> +		DUMP(p, thread_idle_state, "%#-*x");
> +		DUMP(p, thread_mask, "%#-*x");
> +		DUMP(p, subcore_sibling_mask, "%#-*x");
> +	} else {
> +		DUMP(p, idle_state, "%#-*lx");
> +		DUMP(p, requested_psscr, "%#-*llx");
> +#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE
> +		DUMP(p, dont_stop.counter, "%#-*x");
> +#endif
> +	}
>  #endif
> 
>  	DUMP(p, accounting.utime, "%#-*lx");
> -- 
> 2.17.0
>
Nicholas Piggin July 11, 2018, 8:30 a.m. | #2
On Tue, 10 Jul 2018 16:36:34 +0530
Gautham R Shenoy <ego@linux.vnet.ibm.com> wrote:

> Hello Nicholas,
> 
> 
> On Mon, Jul 09, 2018 at 12:24:36AM +1000, Nicholas Piggin wrote:
> > Reimplement POWER9 idle code in C, in the powernv platform code.
> > Assembly stubs are used to save and restore the stack frame and
> > non-volatile GPRs before going to idle, but these are small and
> > mostly agnostic to microarchitecture implementation details.
> >  
> 
> Thanks for this patch.  It is indeed hard to maneuver through the
> current assembly code and change things without introducing new bugs.

Hey thanks for the very good review. I don't mean to disparage the
existing asm code too much :) We were doing our best there to get
something working, now I think it's a good time to step back and see
what we can improve.

> > POWER7/8 code is not converted (yet), but that's not a moving
> > target, and it doesn't make you want to claw your eyes out so
> > much with the POWER9 code untangled from it.
> > 
> > The optimisation where EC=ESL=0 idle modes did not have to save
> > GPRs or mtmsrd L=0 is restored, because it's simple to do.
> > 
> > Idle wakeup no longer uses the ->cpu_restore call to reinit SPRs,
> > but saves and restores them all explicitly.  
> 
> Right! The ->cpu_restore call in the current code is rendered
> ineffective by "restore_additional_sprs" which is called after the
> cpu_restore. 

Yes, I would actually like to do an initial incremental patch for this,
and remove it from P7/8 CPU types as well. I think that's quite easy to
do, and a useful cleanup to remove it entirely.

> > diff --git a/arch/powerpc/include/asm/cpuidle.h b/arch/powerpc/include/asm/cpuidle.h
> > index e210a83eb196..b668f030d531 100644
> > --- a/arch/powerpc/include/asm/cpuidle.h
> > +++ b/arch/powerpc/include/asm/cpuidle.h
> > @@ -28,6 +28,7 @@
> >   * yet woken from the winkle state.
> >   */
> >  #define PNV_CORE_IDLE_LOCK_BIT			0x10000000
> > +#define NR_PNV_CORE_IDLE_LOCK_BIT		28  
> 
> We can define PNV_CORE_IDLE_LOCK_BIT mask based on
> NR_PNV_CORE_IDLE_LOCK_BIT ?

Yep good point.

> > diff --git a/arch/powerpc/kernel/exceptions-64s.S b/arch/powerpc/kernel/exceptions-64s.S
> > index 76a14702cb9c..36b5f0e18c0c 100644
> > --- a/arch/powerpc/kernel/exceptions-64s.S
> > +++ b/arch/powerpc/kernel/exceptions-64s.S
> > @@ -135,8 +135,14 @@ TRAMP_KVM(PACA_EXNMI, 0x100)
> > 
> >  #ifdef CONFIG_PPC_P7_NAP
> >  EXC_COMMON_BEGIN(system_reset_idle_common)
> > +BEGIN_FTR_SECTION
> > +	mfspr	r3,SPRN_SRR1
> > +	bltlr	cr3	/* no state loss, return to idle caller */  
> 
> This is a nice optimization. But perhaps needs a carefully checked.
> We wakeup at SYSTEM_RESET only when we have executed stop via the
> mayloss() call which creates an additional stack frame. When we return
> back to the caller, we will go back with that stack frame. Perhaps we
> need to jump to a isa3_idle_wake_gpr_noloss ?

I wasn't 100% sure of all this so yes it's something I do need to
verify carefully. I did test a few different cases in mambo and stepped
through things, but could easily have bugs.

I could have got something wrong, but I think mayloss does not create
an additional stack frame, so this works okay. It's just using red zone.
... I think...

> > +_GLOBAL(isa3_idle_stop_mayloss)
> > +	std	r1,PACAR1(r13)
> > +	mflr	r4
> > +	mfcr	r5
> > +	/* use stack red zone rather than a new frame */
> > +	addi	r6,r1,-INT_FRAME_SIZE
> > +	SAVE_GPR(2, r6)
> > +	SAVE_NVGPRS(r6)
> > +	std	r4,_LINK(r6)
> > +	std	r5,_CCR(r6)
> > +	mtspr 	SPRN_PSSCR,r3
> > +	PPC_STOP  
> 
> 
> Suppose we enter into a isa3_idle_stop_may_loss (ESL=EC=1), but as
> soon as we execute PPC_STOP, if we get interrupted, we will go back to
> SYSTEM_RESET vector.
> 
> The SRR1[46:47] should be 0x1, due to which we will do a bl there,
> thereby going back to the caller of isa3_idle_stop_mayloss. Which
> implies that we need to perform the stack unwinding there.

Well we haven't modified r1 here, so I thought it should be okay. It
seems to do the right thing in mambo.

> > +static inline void atomic_unlock_thread_idle(void)
> > +{
> > +	int cpu = raw_smp_processor_id();
> > +	int first = cpu_first_thread_sibling(cpu);
> > +	unsigned long *state = &paca_ptrs[first]->idle_state;
> > +
> > +	clear_bit_unlock(NR_PNV_CORE_IDLE_LOCK_BIT, state);
> > +}
> > +
> > +static unsigned long power9_idle_stop(unsigned long psscr, bool  mmu_on)  
> 
> We are not using the mmu_on anywhere in the code. Is this for
> restoring the IR|DR bit in the MSR ?

Hmm, it's supposed to be for KVM secondaries that have to have the MMU
switched off around idle/wake. I haven't got all the KVM stuff right
yet though.


Thanks,
Nick
Gautham R. Shenoy July 11, 2018, 10:24 a.m. | #3
On Wed, Jul 11, 2018 at 06:30:36PM +1000, Nicholas Piggin wrote:
> On Tue, 10 Jul 2018 16:36:34 +0530
> Gautham R Shenoy <ego@linux.vnet.ibm.com> wrote:
> 
> > Hello Nicholas,
> > 
> > 
> > On Mon, Jul 09, 2018 at 12:24:36AM +1000, Nicholas Piggin wrote:
> > > Reimplement POWER9 idle code in C, in the powernv platform code.
> > > Assembly stubs are used to save and restore the stack frame and
> > > non-volatile GPRs before going to idle, but these are small and
> > > mostly agnostic to microarchitecture implementation details.
> > >  
> > 
> > Thanks for this patch.  It is indeed hard to maneuver through the
> > current assembly code and change things without introducing new bugs.
> 
> Hey thanks for the very good review. I don't mean to disparage the
> existing asm code too much :) We were doing our best there to get
> something working, now I think it's a good time to step back and see
> what we can improve.
> 
> > > POWER7/8 code is not converted (yet), but that's not a moving
> > > target, and it doesn't make you want to claw your eyes out so
> > > much with the POWER9 code untangled from it.
> > > 
> > > The optimisation where EC=ESL=0 idle modes did not have to save
> > > GPRs or mtmsrd L=0 is restored, because it's simple to do.
> > > 
> > > Idle wakeup no longer uses the ->cpu_restore call to reinit SPRs,
> > > but saves and restores them all explicitly.  
> > 
> > Right! The ->cpu_restore call in the current code is rendered
> > ineffective by "restore_additional_sprs" which is called after the
> > cpu_restore. 
> 
> Yes, I would actually like to do an initial incremental patch for this,
> and remove it from P7/8 CPU types as well. I think that's quite easy to
> do, and a useful cleanup to remove it entirely.
> 
> > > diff --git a/arch/powerpc/include/asm/cpuidle.h b/arch/powerpc/include/asm/cpuidle.h
> > > index e210a83eb196..b668f030d531 100644
> > > --- a/arch/powerpc/include/asm/cpuidle.h
> > > +++ b/arch/powerpc/include/asm/cpuidle.h
> > > @@ -28,6 +28,7 @@
> > >   * yet woken from the winkle state.
> > >   */
> > >  #define PNV_CORE_IDLE_LOCK_BIT			0x10000000
> > > +#define NR_PNV_CORE_IDLE_LOCK_BIT		28  
> > 
> > We can define PNV_CORE_IDLE_LOCK_BIT mask based on
> > NR_PNV_CORE_IDLE_LOCK_BIT ?
> 
> Yep good point.
> 
> > > diff --git a/arch/powerpc/kernel/exceptions-64s.S b/arch/powerpc/kernel/exceptions-64s.S
> > > index 76a14702cb9c..36b5f0e18c0c 100644
> > > --- a/arch/powerpc/kernel/exceptions-64s.S
> > > +++ b/arch/powerpc/kernel/exceptions-64s.S
> > > @@ -135,8 +135,14 @@ TRAMP_KVM(PACA_EXNMI, 0x100)
> > > 
> > >  #ifdef CONFIG_PPC_P7_NAP
> > >  EXC_COMMON_BEGIN(system_reset_idle_common)
> > > +BEGIN_FTR_SECTION
> > > +	mfspr	r3,SPRN_SRR1
> > > +	bltlr	cr3	/* no state loss, return to idle caller */  
> > 
> > This is a nice optimization. But perhaps needs a carefully checked.
> > We wakeup at SYSTEM_RESET only when we have executed stop via the
> > mayloss() call which creates an additional stack frame. When we return
> > back to the caller, we will go back with that stack frame. Perhaps we
> > need to jump to a isa3_idle_wake_gpr_noloss ?
> 
> I wasn't 100% sure of all this so yes it's something I do need to
> verify carefully. I did test a few different cases in mambo and stepped
> through things, but could easily have bugs.
> 
> I could have got something wrong, but I think mayloss does not create
> an additional stack frame, so this works okay. It's just using red zone.
> ... I think...
> 
> > > +_GLOBAL(isa3_idle_stop_mayloss)
> > > +	std	r1,PACAR1(r13)
> > > +	mflr	r4
> > > +	mfcr	r5
> > > +	/* use stack red zone rather than a new frame */
> > > +	addi	r6,r1,-INT_FRAME_SIZE
> > > +	SAVE_GPR(2, r6)
> > > +	SAVE_NVGPRS(r6)
> > > +	std	r4,_LINK(r6)
> > > +	std	r5,_CCR(r6)
> > > +	mtspr 	SPRN_PSSCR,r3
> > > +	PPC_STOP  
> > 
> > 
> > Suppose we enter into a isa3_idle_stop_may_loss (ESL=EC=1), but as
> > soon as we execute PPC_STOP, if we get interrupted, we will go back to
> > SYSTEM_RESET vector.
> > 
> > The SRR1[46:47] should be 0x1, due to which we will do a bl there,
> > thereby going back to the caller of isa3_idle_stop_mayloss. Which
> > implies that we need to perform the stack unwinding there.
> 
> Well we haven't modified r1 here, so I thought it should be okay. It
> seems to do the right thing in mambo.

Yeah, my bad. We aren't updating the r1. The code is fine in that
case.

> 
> > > +static inline void atomic_unlock_thread_idle(void)
> > > +{
> > > +	int cpu = raw_smp_processor_id();
> > > +	int first = cpu_first_thread_sibling(cpu);
> > > +	unsigned long *state = &paca_ptrs[first]->idle_state;
> > > +
> > > +	clear_bit_unlock(NR_PNV_CORE_IDLE_LOCK_BIT, state);
> > > +}
> > > +
> > > +static unsigned long power9_idle_stop(unsigned long psscr, bool  mmu_on)  
> > 
> > We are not using the mmu_on anywhere in the code. Is this for
> > restoring the IR|DR bit in the MSR ?
> 
> Hmm, it's supposed to be for KVM secondaries that have to have the MMU
> switched off around idle/wake. I haven't got all the KVM stuff right
> yet though.

Ah, right. But I was under the impression that it was a quirk only on 
POWER8 so that we set the KVM_HWTHREAD_IN_IDLE on the secondaries with
MMU off, so that in the event the primary thread in the core switches
to guest, the secondaries won't execute the guest code.

On P9, the comment in power9_offline_stop says that we don't need to
set the KVM_HWTHREAD_IN_IDLE in the real-mode.

	/*
	 * Tell KVM we're entering idle.
	 * This does not have to be done in real mode because the P9 MMU
	 * is independent per-thread. Some steppings share radix/hash mode
	 * between threads, but in that case KVM has a barrier sync in real
	 * mode before and after switching between radix and hash.
	 */

Am I missing something?

> 
> 
> Thanks,
> Nick
> 
--
Thanks and Regards
gautham.

Patch

diff --git a/arch/powerpc/include/asm/book3s/64/mmu-hash.h b/arch/powerpc/include/asm/book3s/64/mmu-hash.h
index 50ed64fba4ae..c626319a962d 100644
--- a/arch/powerpc/include/asm/book3s/64/mmu-hash.h
+++ b/arch/powerpc/include/asm/book3s/64/mmu-hash.h
@@ -486,6 +486,7 @@  static inline void hpte_init_pseries(void) { }
 extern void hpte_init_native(void);
 
 extern void slb_initialize(void);
+extern void __slb_flush_and_rebolt(void);
 extern void slb_flush_and_rebolt(void);
 
 extern void slb_vmalloc_update(void);
diff --git a/arch/powerpc/include/asm/cpuidle.h b/arch/powerpc/include/asm/cpuidle.h
index e210a83eb196..b668f030d531 100644
--- a/arch/powerpc/include/asm/cpuidle.h
+++ b/arch/powerpc/include/asm/cpuidle.h
@@ -28,6 +28,7 @@ 
  * yet woken from the winkle state.
  */
 #define PNV_CORE_IDLE_LOCK_BIT			0x10000000
+#define NR_PNV_CORE_IDLE_LOCK_BIT		28
 
 #define PNV_CORE_IDLE_WINKLE_COUNT		0x00010000
 #define PNV_CORE_IDLE_WINKLE_COUNT_ALL_BIT	0x00080000
@@ -68,22 +69,9 @@ 
 #define ERR_DEEP_STATE_ESL_MISMATCH	-2
 
 #ifndef __ASSEMBLY__
-/* Additional SPRs that need to be saved/restored during stop */
-struct stop_sprs {
-	u64 pid;
-	u64 ldbar;
-	u64 fscr;
-	u64 hfscr;
-	u64 mmcr1;
-	u64 mmcr2;
-	u64 mmcra;
-};
-
 extern u32 pnv_fastsleep_workaround_at_entry[];
 extern u32 pnv_fastsleep_workaround_at_exit[];
 
-extern u64 pnv_first_deep_stop_state;
-
 unsigned long pnv_cpu_offline(unsigned int cpu);
 int validate_psscr_val_mask(u64 *psscr_val, u64 *psscr_mask, u32 flags);
 static inline void report_invalid_psscr_val(u64 psscr_val, int err)
diff --git a/arch/powerpc/include/asm/paca.h b/arch/powerpc/include/asm/paca.h
index 4e9cede5a7e7..a7a4892d39c0 100644
--- a/arch/powerpc/include/asm/paca.h
+++ b/arch/powerpc/include/asm/paca.h
@@ -178,23 +178,29 @@  struct paca_struct {
 #endif
 
 #ifdef CONFIG_PPC_POWERNV
-	/* Per-core mask tracking idle threads and a lock bit-[L][TTTTTTTT] */
-	u32 *core_idle_state_ptr;
-	u8 thread_idle_state;		/* PNV_THREAD_RUNNING/NAP/SLEEP	*/
-	/* Mask to indicate thread id in core */
-	u8 thread_mask;
-	/* Mask to denote subcore sibling threads */
-	u8 subcore_sibling_mask;
-	/* Flag to request this thread not to stop */
-	atomic_t dont_stop;
-	/* The PSSCR value that the kernel requested before going to stop */
-	u64 requested_psscr;
+	union {
+		/* P7/P8 specific fields */
+		struct {
+			/* Per-core mask tracking idle threads and a lock bit-[L][TTTTTTTT] */
+			unsigned long *core_idle_state_ptr;
+			u8 thread_idle_state;	/* PNV_THREAD_RUNNING/NAP/SLEEP	*/
+			/* Mask to indicate thread id in core */
+			u8 thread_mask;
+			/* Mask to denote subcore sibling threads */
+			u8 subcore_sibling_mask;
+		};
 
-	/*
-	 * Save area for additional SPRs that need to be
-	 * saved/restored during cpuidle stop.
-	 */
-	struct stop_sprs stop_sprs;
+		/* P9 specific fields */
+		struct {
+			/* The PSSCR value that the kernel requested before going to stop */
+			u64 requested_psscr;
+			unsigned long idle_state;
+#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE
+			/* Flag to request this thread not to stop */
+			atomic_t dont_stop;
+#endif
+		};
+	};
 #endif
 
 #ifdef CONFIG_PPC_BOOK3S_64
diff --git a/arch/powerpc/include/asm/processor.h b/arch/powerpc/include/asm/processor.h
index 5debe337ea9d..4774ec7603ee 100644
--- a/arch/powerpc/include/asm/processor.h
+++ b/arch/powerpc/include/asm/processor.h
@@ -513,7 +513,8 @@  enum idle_boot_override {IDLE_NO_OVERRIDE = 0, IDLE_POWERSAVE_OFF};
 extern int powersave_nap;	/* set if nap mode can be used in idle loop */
 extern unsigned long power7_idle_insn(unsigned long type); /* PNV_THREAD_NAP/etc*/
 extern void power7_idle_type(unsigned long type);
-extern unsigned long power9_idle_stop(unsigned long psscr_val);
+extern unsigned long isa3_idle_stop_noloss(unsigned long psscr_val);
+extern unsigned long isa3_idle_stop_mayloss(unsigned long psscr_val);
 extern unsigned long power9_offline_stop(unsigned long psscr_val);
 extern void power9_idle_type(unsigned long stop_psscr_val,
 			      unsigned long stop_psscr_mask);
diff --git a/arch/powerpc/include/asm/reg.h b/arch/powerpc/include/asm/reg.h
index 562568414cf4..3c7d97b2abb0 100644
--- a/arch/powerpc/include/asm/reg.h
+++ b/arch/powerpc/include/asm/reg.h
@@ -750,10 +750,9 @@ 
 #define	  SRR1_WAKERESET	0x00100000 /* System reset */
 #define   SRR1_WAKEHDBELL	0x000c0000 /* Hypervisor doorbell on P8 */
 #define	  SRR1_WAKESTATE	0x00030000 /* Powersave exit mask [46:47] */
-#define	  SRR1_WS_DEEPEST	0x00030000 /* Some resources not maintained,
-					  * may not be recoverable */
-#define	  SRR1_WS_DEEPER	0x00020000 /* Some resources not maintained */
-#define	  SRR1_WS_DEEP		0x00010000 /* All resources maintained */
+#define	  SRR1_WS_HVLOSS	0x00030000 /* HV resources not maintained */
+#define	  SRR1_WS_GPRLOSS	0x00020000 /* GPRs not maintained */
+#define	  SRR1_WS_NOLOSS	0x00010000 /* All resources maintained */
 #define   SRR1_PROGTM		0x00200000 /* TM Bad Thing */
 #define   SRR1_PROGFPE		0x00100000 /* Floating Point Enabled */
 #define   SRR1_PROGILL		0x00080000 /* Illegal instruction */
diff --git a/arch/powerpc/kernel/Makefile b/arch/powerpc/kernel/Makefile
index 2b4c40b255e4..6914fab16e2c 100644
--- a/arch/powerpc/kernel/Makefile
+++ b/arch/powerpc/kernel/Makefile
@@ -48,7 +48,7 @@  obj-$(CONFIG_PPC_BOOK3E_64)	+= exceptions-64e.o idle_book3e.o
 obj-$(CONFIG_PPC64)		+= vdso64/
 obj-$(CONFIG_ALTIVEC)		+= vecemu.o
 obj-$(CONFIG_PPC_970_NAP)	+= idle_power4.o
-obj-$(CONFIG_PPC_P7_NAP)	+= idle_book3s.o
+obj-$(CONFIG_PPC_P7_NAP)	+= idle_book3s.o idle_isa3.o
 procfs-y			:= proc_powerpc.o
 obj-$(CONFIG_PROC_FS)		+= $(procfs-y)
 rtaspci-$(CONFIG_PPC64)-$(CONFIG_PCI)	:= rtas_pci.o
diff --git a/arch/powerpc/kernel/asm-offsets.c b/arch/powerpc/kernel/asm-offsets.c
index 89cf15566c4e..b8162bb80ddb 100644
--- a/arch/powerpc/kernel/asm-offsets.c
+++ b/arch/powerpc/kernel/asm-offsets.c
@@ -762,20 +762,11 @@  int main(void)
 #endif
 
 #ifdef CONFIG_PPC_POWERNV
+	/* POWER7/8 specific idle fields (kernel/idle_book3s.S) */
 	OFFSET(PACA_CORE_IDLE_STATE_PTR, paca_struct, core_idle_state_ptr);
 	OFFSET(PACA_THREAD_IDLE_STATE, paca_struct, thread_idle_state);
 	OFFSET(PACA_THREAD_MASK, paca_struct, thread_mask);
 	OFFSET(PACA_SUBCORE_SIBLING_MASK, paca_struct, subcore_sibling_mask);
-	OFFSET(PACA_REQ_PSSCR, paca_struct, requested_psscr);
-	OFFSET(PACA_DONT_STOP, paca_struct, dont_stop);
-#define STOP_SPR(x, f)	OFFSET(x, paca_struct, stop_sprs.f)
-	STOP_SPR(STOP_PID, pid);
-	STOP_SPR(STOP_LDBAR, ldbar);
-	STOP_SPR(STOP_FSCR, fscr);
-	STOP_SPR(STOP_HFSCR, hfscr);
-	STOP_SPR(STOP_MMCR1, mmcr1);
-	STOP_SPR(STOP_MMCR2, mmcr2);
-	STOP_SPR(STOP_MMCRA, mmcra);
 #endif
 
 	DEFINE(PPC_DBELL_SERVER, PPC_DBELL_SERVER);
diff --git a/arch/powerpc/kernel/exceptions-64s.S b/arch/powerpc/kernel/exceptions-64s.S
index 76a14702cb9c..36b5f0e18c0c 100644
--- a/arch/powerpc/kernel/exceptions-64s.S
+++ b/arch/powerpc/kernel/exceptions-64s.S
@@ -135,8 +135,14 @@  TRAMP_KVM(PACA_EXNMI, 0x100)
 
 #ifdef CONFIG_PPC_P7_NAP
 EXC_COMMON_BEGIN(system_reset_idle_common)
+BEGIN_FTR_SECTION
+	mfspr	r3,SPRN_SRR1
+	bltlr	cr3	/* no state loss, return to idle caller */
+	b	isa3_idle_wake_gpr_loss
+FTR_SECTION_ELSE
 	mfspr	r12,SPRN_SRR1
 	b	pnv_powersave_wakeup
+ALT_FTR_SECTION_END_IFSET(CPU_FTR_ARCH_300)
 #endif
 
 /*
@@ -425,7 +431,9 @@  EXC_COMMON_BEGIN(machine_check_idle_common)
 	li	r11,0
 	mtmsrd	r11,1
 
-	b	pnv_powersave_wakeup_mce
+	/* XXX fixup
+	b	pnv_powersave_wakeup_mce */
+	b	.
 #endif
 	/*
 	 * Handle machine check early in real mode. We come here with
diff --git a/arch/powerpc/kernel/idle_book3s.S b/arch/powerpc/kernel/idle_book3s.S
index d85d5515a091..506b88768767 100644
--- a/arch/powerpc/kernel/idle_book3s.S
+++ b/arch/powerpc/kernel/idle_book3s.S
@@ -1,6 +1,6 @@ 
 /*
- *  This file contains idle entry/exit functions for POWER7,
- *  POWER8 and POWER9 CPUs.
+ *  This file contains idle entry/exit functions for POWER7 and
+ *  POWER8 CPUs.
  *
  *  This program is free software; you can redistribute it and/or
  *  modify it under the terms of the GNU General Public License
@@ -56,19 +56,8 @@  save_sprs_to_stack:
 	 * Note all register i.e per-core, per-subcore or per-thread is saved
 	 * here since any thread in the core might wake up first
 	 */
-BEGIN_FTR_SECTION
-	/*
-	 * Note - SDR1 is dropped in Power ISA v3. Hence not restoring
-	 * SDR1 here
-	 */
-	mfspr	r3,SPRN_PTCR
-	std	r3,_PTCR(r1)
-	mfspr	r3,SPRN_LPCR
-	std	r3,_LPCR(r1)
-FTR_SECTION_ELSE
 	mfspr	r3,SPRN_SDR1
 	std	r3,_SDR1(r1)
-ALT_FTR_SECTION_END_IFSET(CPU_FTR_ARCH_300)
 	mfspr	r3,SPRN_RPR
 	std	r3,_RPR(r1)
 	mfspr	r3,SPRN_SPURR
@@ -85,66 +74,6 @@  ALT_FTR_SECTION_END_IFSET(CPU_FTR_ARCH_300)
 	std	r3,_WORT(r1)
 	mfspr	r3,SPRN_WORC
 	std	r3,_WORC(r1)
-/*
- * On POWER9, there are idle states such as stop4, invoked via cpuidle,
- * that lose hypervisor resources. In such cases, we need to save
- * additional SPRs before entering those idle states so that they can
- * be restored to their older values on wakeup from the idle state.
- *
- * On POWER8, the only such deep idle state is winkle which is used
- * only in the context of CPU-Hotplug, where these additional SPRs are
- * reinitiazed to a sane value. Hence there is no need to save/restore
- * these SPRs.
- */
-BEGIN_FTR_SECTION
-	blr
-END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_300)
-
-power9_save_additional_sprs:
-	mfspr	r3, SPRN_PID
-	mfspr	r4, SPRN_LDBAR
-	std	r3, STOP_PID(r13)
-	std	r4, STOP_LDBAR(r13)
-
-	mfspr	r3, SPRN_FSCR
-	mfspr	r4, SPRN_HFSCR
-	std	r3, STOP_FSCR(r13)
-	std	r4, STOP_HFSCR(r13)
-
-	mfspr	r3, SPRN_MMCRA
-	mfspr	r4, SPRN_MMCR0
-	std	r3, STOP_MMCRA(r13)
-	std	r4, _MMCR0(r1)
-
-	mfspr	r3, SPRN_MMCR1
-	mfspr	r4, SPRN_MMCR2
-	std	r3, STOP_MMCR1(r13)
-	std	r4, STOP_MMCR2(r13)
-	blr
-
-power9_restore_additional_sprs:
-	ld	r3,_LPCR(r1)
-	ld	r4, STOP_PID(r13)
-	mtspr	SPRN_LPCR,r3
-	mtspr	SPRN_PID, r4
-
-	ld	r3, STOP_LDBAR(r13)
-	ld	r4, STOP_FSCR(r13)
-	mtspr	SPRN_LDBAR, r3
-	mtspr	SPRN_FSCR, r4
-
-	ld	r3, STOP_HFSCR(r13)
-	ld	r4, STOP_MMCRA(r13)
-	mtspr	SPRN_HFSCR, r3
-	mtspr	SPRN_MMCRA, r4
-
-	ld	r3, _MMCR0(r1)
-	ld	r4, STOP_MMCR1(r13)
-	mtspr	SPRN_MMCR0, r3
-	mtspr	SPRN_MMCR1, r4
-
-	ld	r3, STOP_MMCR2(r13)
-	mtspr	SPRN_MMCR2, r3
 	blr
 
 /*
@@ -167,13 +96,23 @@  core_idle_lock_held:
 	blr
 
 /*
- * Pass requested state in r3:
- *	r3 - PNV_THREAD_NAP/SLEEP/WINKLE in POWER8
- *	   - Requested PSSCR value in POWER9
- *
- * Address of idle handler to branch to in realmode in r4
+ * This is the sequence required to execute idle instructions, as
+ * specified in ISA v2.07 (and earlier). MSR[IR] and MSR[DR] must be 0.
+ */
+#define IDLE_STATE_ENTER_SEQ_NORET(IDLE_INST)			\
+	/* Magic NAP/SLEEP/WINKLE mode enter sequence */	\
+	std	r0,0(r1);					\
+	ptesync;						\
+	ld	r0,0(r1);					\
+236:	cmpd	cr0,r0,r0;					\
+	bne	236b;						\
+	IDLE_INST;
+
+/*
+ * Entered with MSR[EE]=0 and no soft-masked interrupts pending.
+ * r3 contains desired idle state (PNV_THREAD_NAP/SLEEP/WINKLE).
  */
-pnv_powersave_common:
+_GLOBAL(power7_idle_insn)
 	/* Use r3 to pass state nap/sleep/winkle */
 	/* NAP is a state loss, we create a regs frame on the
 	 * stack, fill it up with the state we care about and
@@ -181,8 +120,6 @@  pnv_powersave_common:
 	 * need to save PC, some CR bits and the NV GPRs,
 	 * but for now an interrupt frame will do.
 	 */
-	mtctr	r4
-
 	mflr	r0
 	std	r0,16(r1)
 	stdu	r1,-INT_FRAME_SIZE(r1)
@@ -200,16 +137,7 @@  pnv_powersave_common:
 	std	r5,_CCR(r1)
 	std	r1,PACAR1(r13)
 
-BEGIN_FTR_SECTION
-	/*
-	 * POWER9 does not require real mode to stop, and presently does not
-	 * set hwthread_state for KVM (threads don't share MMU context), so
-	 * we can remain in virtual mode for this.
-	 */
-	bctr
-END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300)
 	/*
-	 * POWER8
 	 * Go to real mode to do the nap, as required by the architecture.
 	 * Also, we need to be in real mode before setting hwthread_state,
 	 * because as soon as we do that, another thread can switch
@@ -217,24 +145,7 @@  END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300)
 	 */
 	LOAD_REG_IMMEDIATE(r7, MSR_IDLE)
 	mtmsrd	r7,0
-	bctr
 
-/*
- * This is the sequence required to execute idle instructions, as
- * specified in ISA v2.07 (and earlier). MSR[IR] and MSR[DR] must be 0.
- */
-#define IDLE_STATE_ENTER_SEQ_NORET(IDLE_INST)			\
-	/* Magic NAP/SLEEP/WINKLE mode enter sequence */	\
-	std	r0,0(r1);					\
-	ptesync;						\
-	ld	r0,0(r1);					\
-236:	cmpd	cr0,r0,r0;					\
-	bne	236b;						\
-	IDLE_INST;
-
-
-	.globl pnv_enter_arch207_idle_mode
-pnv_enter_arch207_idle_mode:
 #ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE
 	/* Tell KVM we're entering idle */
 	li	r4,KVM_HWTHREAD_IN_IDLE
@@ -321,86 +232,6 @@  enter_winkle:
 
 	IDLE_STATE_ENTER_SEQ_NORET(PPC_WINKLE)
 
-/*
- * r3 - PSSCR value corresponding to the requested stop state.
- */
-power_enter_stop:
-/*
- * Check if we are executing the lite variant with ESL=EC=0
- */
-	andis.   r4,r3,PSSCR_EC_ESL_MASK_SHIFTED
-	clrldi   r3,r3,60 /* r3 = Bits[60:63] = Requested Level (RL) */
-	bne	 .Lhandle_esl_ec_set
-	PPC_STOP
-	li	r3,0  /* Since we didn't lose state, return 0 */
-	std	r3, PACA_REQ_PSSCR(r13)
-
-	/*
-	 * pnv_wakeup_noloss() expects r12 to contain the SRR1 value so
-	 * it can determine if the wakeup reason is an HMI in
-	 * CHECK_HMI_INTERRUPT.
-	 *
-	 * However, when we wakeup with ESL=0, SRR1 will not contain the wakeup
-	 * reason, so there is no point setting r12 to SRR1.
-	 *
-	 * Further, we clear r12 here, so that we don't accidentally enter the
-	 * HMI in pnv_wakeup_noloss() if the value of r12[42:45] == WAKE_HMI.
-	 */
-	li	r12, 0
-	b 	pnv_wakeup_noloss
-
-.Lhandle_esl_ec_set:
-BEGIN_FTR_SECTION
-	/*
-	 * POWER9 DD2.0 or earlier can incorrectly set PMAO when waking up after
-	 * a state-loss idle. Saving and restoring MMCR0 over idle is a
-	 * workaround.
-	 */
-	mfspr	r4,SPRN_MMCR0
-	std	r4,_MMCR0(r1)
-END_FTR_SECTION_IFCLR(CPU_FTR_POWER9_DD2_1)
-
-/*
- * Check if the requested state is a deep idle state.
- */
-	LOAD_REG_ADDRBASE(r5,pnv_first_deep_stop_state)
-	ld	r4,ADDROFF(pnv_first_deep_stop_state)(r5)
-	cmpd	r3,r4
-	bge	.Lhandle_deep_stop
-	PPC_STOP	/* Does not return (system reset interrupt) */
-
-.Lhandle_deep_stop:
-/*
- * Entering deep idle state.
- * Clear thread bit in PACA_CORE_IDLE_STATE, save SPRs to
- * stack and enter stop
- */
-	lbz     r7,PACA_THREAD_MASK(r13)
-	ld      r14,PACA_CORE_IDLE_STATE_PTR(r13)
-
-lwarx_loop_stop:
-	lwarx   r15,0,r14
-	andis.	r9,r15,PNV_CORE_IDLE_LOCK_BIT@h
-	bnel-	core_idle_lock_held
-	andc    r15,r15,r7                      /* Clear thread bit */
-
-	stwcx.  r15,0,r14
-	bne-    lwarx_loop_stop
-	isync
-
-	bl	save_sprs_to_stack
-
-	PPC_STOP	/* Does not return (system reset interrupt) */
-
-/*
- * Entered with MSR[EE]=0 and no soft-masked interrupts pending.
- * r3 contains desired idle state (PNV_THREAD_NAP/SLEEP/WINKLE).
- */
-_GLOBAL(power7_idle_insn)
-	/* Now check if user or arch enabled NAP mode */
-	LOAD_REG_ADDR(r4, pnv_enter_arch207_idle_mode)
-	b	pnv_powersave_common
-
 #define CHECK_HMI_INTERRUPT						\
 BEGIN_FTR_SECTION_NESTED(66);						\
 	rlwinm	r0,r12,45-31,0xf;  /* extract wake reason field (P8) */	\
@@ -419,53 +250,6 @@  ALT_FTR_SECTION_END_NESTED_IFSET(CPU_FTR_ARCH_207S, 66);		\
 	ld	r3,ORIG_GPR3(r1);	/* Restore original r3 */	\
 20:	nop;
 
-/*
- * Entered with MSR[EE]=0 and no soft-masked interrupts pending.
- * r3 contains desired PSSCR register value.
- *
- * Offline (CPU unplug) case also must notify KVM that the CPU is
- * idle.
- */
-_GLOBAL(power9_offline_stop)
-#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE
-	/*
-	 * Tell KVM we're entering idle.
-	 * This does not have to be done in real mode because the P9 MMU
-	 * is independent per-thread. Some steppings share radix/hash mode
-	 * between threads, but in that case KVM has a barrier sync in real
-	 * mode before and after switching between radix and hash.
-	 */
-	li	r4,KVM_HWTHREAD_IN_IDLE
-	stb	r4,HSTATE_HWTHREAD_STATE(r13)
-#endif
-	/* fall through */
-
-_GLOBAL(power9_idle_stop)
-	std	r3, PACA_REQ_PSSCR(r13)
-#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE
-BEGIN_FTR_SECTION
-	sync
-	lwz	r5, PACA_DONT_STOP(r13)
-	cmpwi	r5, 0
-	bne	1f
-END_FTR_SECTION_IFSET(CPU_FTR_P9_TM_XER_SO_BUG)
-#endif
-	mtspr 	SPRN_PSSCR,r3
-	LOAD_REG_ADDR(r4,power_enter_stop)
-	b	pnv_powersave_common
-	/* No return */
-#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE
-1:
-	/*
-	 * We get here when TM / thread reconfiguration bug workaround
-	 * code wants to get the CPU into SMT4 mode, and therefore
-	 * we are being asked not to stop.
-	 */
-	li	r3, 0
-	std	r3, PACA_REQ_PSSCR(r13)
-	blr		/* return 0 for wakeup cause / SRR1 value */
-#endif
-
 /*
  * Called from machine check handler for powersave wakeups.
  * Low level machine check processing has already been done. Now just
@@ -499,11 +283,7 @@  pnv_powersave_wakeup_mce:
 pnv_powersave_wakeup:
 	ld	r2, PACATOC(r13)
 
-BEGIN_FTR_SECTION
-	bl	pnv_restore_hyp_resource_arch300
-FTR_SECTION_ELSE
-	bl	pnv_restore_hyp_resource_arch207
-ALT_FTR_SECTION_END_IFSET(CPU_FTR_ARCH_300)
+	bl	pnv_restore_hyp_resource
 
 	li	r0,PNV_THREAD_RUNNING
 	stb	r0,PACA_THREAD_IDLE_STATE(r13)	/* Clear thread state */
@@ -535,50 +315,8 @@  ALT_FTR_SECTION_END_IFSET(CPU_FTR_ARCH_300)
  *
  * cr3 - set to gt if waking up with partial/complete hypervisor state loss
  */
-pnv_restore_hyp_resource_arch300:
-	/*
-	 * Workaround for POWER9, if we lost resources, the ERAT
-	 * might have been mixed up and needs flushing. We also need
-	 * to reload MMCR0 (see comment above). We also need to set
-	 * then clear bit 60 in MMCRA to ensure the PMU starts running.
-	 */
-	blt	cr3,1f
-BEGIN_FTR_SECTION
-	PPC_INVALIDATE_ERAT
-	ld	r1,PACAR1(r13)
-	ld	r4,_MMCR0(r1)
-	mtspr	SPRN_MMCR0,r4
-END_FTR_SECTION_IFCLR(CPU_FTR_POWER9_DD2_1)
-	mfspr	r4,SPRN_MMCRA
-	ori	r4,r4,(1 << (63-60))
-	mtspr	SPRN_MMCRA,r4
-	xori	r4,r4,(1 << (63-60))
-	mtspr	SPRN_MMCRA,r4
-1:
-	/*
-	 * POWER ISA 3. Use PSSCR to determine if we
-	 * are waking up from deep idle state
-	 */
-	LOAD_REG_ADDRBASE(r5,pnv_first_deep_stop_state)
-	ld	r4,ADDROFF(pnv_first_deep_stop_state)(r5)
-
+pnv_restore_hyp_resource:
 	/*
-	 * 0-3 bits correspond to Power-Saving Level Status
-	 * which indicates the idle state we are waking up from
-	 */
-	mfspr	r5, SPRN_PSSCR
-	rldicl  r5,r5,4,60
-	li	r0, 0		/* clear requested_psscr to say we're awake */
-	std	r0, PACA_REQ_PSSCR(r13)
-	cmpd	cr4,r5,r4
-	bge	cr4,pnv_wakeup_tb_loss /* returns to caller */
-
-	blr	/* Waking up without hypervisor state loss. */
-
-/* Same calling convention as arch300 */
-pnv_restore_hyp_resource_arch207:
-	/*
-	 * POWER ISA 2.07 or less.
 	 * Check if we slept with sleep or winkle.
 	 */
 	lbz	r4,PACA_THREAD_IDLE_STATE(r13)
@@ -598,15 +336,9 @@  pnv_restore_hyp_resource_arch207:
  * Called if waking up from idle state which can cause either partial or
  * complete hyp state loss.
  * In POWER8, called if waking up from fastsleep or winkle
- * In POWER9, called if waking up from stop state >= pnv_first_deep_stop_state
  *
  * r13 - PACA
  * cr3 - gt if waking up with partial/complete hypervisor state loss
- *
- * If ISA300:
- * cr4 - gt or eq if waking up from complete hypervisor state loss.
- *
- * If ISA207:
  * r4 - PACA_THREAD_IDLE_STATE
  */
 pnv_wakeup_tb_loss:
@@ -621,9 +353,7 @@  pnv_wakeup_tb_loss:
 	 * and SRR1 test for restoring NVGPRs.
 	 *
 	 * We are about to clobber NVGPRs now, so set NAPSTATELOST to
-	 * guarantee they will always be restored. This might be tightened
-	 * with careful reading of specs (particularly for ISA300) but this
-	 * is already a slow wakeup path and it's simpler to be safe.
+	 * guarantee they will always be restored.
 	 */
 	li	r0,1
 	stb	r0,PACA_NAPSTATELOST(r13)
@@ -672,19 +402,16 @@  END_FTR_SECTION_IFSET(CPU_FTR_HVMODE)
 	 * At this stage
 	 * cr2 - eq if first thread to wakeup in core
 	 * cr3-  gt if waking up with partial/complete hypervisor state loss
-	 * ISA300:
-	 * cr4 - gt or eq if waking up from complete hypervisor state loss.
 	 */
 
-BEGIN_FTR_SECTION
 	/*
 	 * Were we in winkle?
 	 * If yes, check if all threads were in winkle, decrement our
 	 * winkle count, set all thread winkle bits if all were in winkle.
-	 * Check if our thread has a winkle bit set, and set cr4 accordingly
-	 * (to match ISA300, above). Pseudo-code for core idle state
-	 * transitions for ISA207 is as follows (everything happens atomically
-	 * due to store conditional and/or lock bit):
+	 * Check if our thread has a winkle bit set, and set cr4 accordingly.
+	 * Pseudo-code for core idle state transitions for ISA207 is as follows
+	 * (everything happens atomically due to store conditional and/or lock
+	 * bit):
 	 *
 	 * nap_idle() { }
 	 * nap_wake() { }
@@ -749,7 +476,6 @@  BEGIN_FTR_SECTION
 
 	or	r15,r15,r7		/* Set thread bit */
 	beq	first_thread_in_subcore
-END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_300)
 
 	or	r15,r15,r7		/* Set thread bit */
 	beq	cr2,first_thread_in_core
@@ -815,15 +541,6 @@  timebase_resync:
 	 * complete hypervisor state loss. Restore per core hypervisor
 	 * state.
 	 */
-BEGIN_FTR_SECTION
-	ld	r4,_PTCR(r1)
-	mtspr	SPRN_PTCR,r4
-	ld	r4,_RPR(r1)
-	mtspr	SPRN_RPR,r4
-	ld	r4,_AMOR(r1)
-	mtspr	SPRN_AMOR,r4
-END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300)
-
 	ld	r4,_TSCR(r1)
 	mtspr	SPRN_TSCR,r4
 	ld	r4,_WORC(r1)
@@ -845,9 +562,6 @@  common_exit:
 
 	/* Waking up from winkle */
 
-BEGIN_MMU_FTR_SECTION
-	b	no_segments
-END_MMU_FTR_SECTION_IFSET(MMU_FTR_TYPE_RADIX)
 	/* Restore SLB  from PACA */
 	ld	r8,PACA_SLBSHADOWPTR(r13)
 
@@ -861,7 +575,6 @@  END_MMU_FTR_SECTION_IFSET(MMU_FTR_TYPE_RADIX)
 	slbmte	r6,r5
 1:	addi	r8,r8,16
 	.endr
-no_segments:
 
 	/* Restore per thread state */
 
@@ -884,17 +597,6 @@  no_segments:
 	mtctr	r12
 	bctrl
 
-/*
- * On POWER9, we can come here on wakeup from a cpuidle stop state.
- * Hence restore the additional SPRs to the saved value.
- *
- * On POWER8, we come here only on winkle. Since winkle is used
- * only in the case of CPU-Hotplug, we don't need to restore
- * the additional SPRs.
- */
-BEGIN_FTR_SECTION
-	bl 	power9_restore_additional_sprs
-END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300)
 hypervisor_state_restored:
 
 	mr	r12,r19
diff --git a/arch/powerpc/kernel/idle_isa3.S b/arch/powerpc/kernel/idle_isa3.S
new file mode 100644
index 000000000000..c869512b716a
--- /dev/null
+++ b/arch/powerpc/kernel/idle_isa3.S
@@ -0,0 +1,73 @@ 
+/*
+ *  This file contains general idle entry/exit functions. The platform / CPU
+ *  must call the correct save/restore functions and ensure SPRs are saved
+ *  and restored correctly, handle KVM, interrupts, etc.
+ *
+ *  This program is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU General Public License
+ *  as published by the Free Software Foundation; either version
+ *  2 of the License, or (at your option) any later version.
+ */
+
+#include <asm/ppc_asm.h>
+#include <asm/asm-offsets.h>
+#include <asm/ppc-opcode.h>
+
+/*
+ * Desired PSSCR in r3
+ *
+ * No state will be lost regardless of wakeup mechanism (interrupt or NIA).
+ * Interrupt driven wakeup may clobber volatiles, and should blr (with LR
+ * unchanged) to return to caller with r3 set according to caller's expected
+ * return code (for Book3S/64 that is SRR1).
+ *
+ * Caller is responsible for restoring SPRs, MSR, etc.
+ */
+_GLOBAL(isa3_idle_stop_noloss)
+	mtspr 	SPRN_PSSCR,r3
+	PPC_STOP
+	li	r3,0
+	blr
+
+/*
+ * Desired PSSCR in r3
+ *
+ * GPRs may be lost, so they are saved here. Wakeup is by interrupt only.
+ * Wakeup can return to caller by calling pnv_powersave_wakeup_gpr_loss
+ * with r3 set to return value.
+ *
+ * A wakeup without GPR loss may alterateively be handled as in
+ * isa3_idle_stop_noloss as an optimisation.
+ *
+ * Caller is responsible for restoring SPRs, MSR, etc.
+ */
+_GLOBAL(isa3_idle_stop_mayloss)
+	std	r1,PACAR1(r13)
+	mflr	r4
+	mfcr	r5
+	/* use stack red zone rather than a new frame */
+	addi	r6,r1,-INT_FRAME_SIZE
+	SAVE_GPR(2, r6)
+	SAVE_NVGPRS(r6)
+	std	r4,_LINK(r6)
+	std	r5,_CCR(r6)
+	mtspr 	SPRN_PSSCR,r3
+	PPC_STOP
+	b	.
+
+/*
+ * Desired return value in r3
+ *
+ * Idle wakeup can call this after calling isa3_idle_stop_loss to
+ * return to caller with r3 as return code.
+ */
+_GLOBAL(isa3_idle_wake_gpr_loss)
+	ld	r1,PACAR1(r13)
+	addi	r6,r1,-INT_FRAME_SIZE
+	ld	r4,_LINK(r6)
+	ld	r5,_CCR(r6)
+	REST_NVGPRS(r6)
+	REST_GPR(2, r6)
+	mtlr	r4
+	mtcr	r5
+	blr
diff --git a/arch/powerpc/kernel/setup-common.c b/arch/powerpc/kernel/setup-common.c
index 40b44bb53a4e..e089da156ef3 100644
--- a/arch/powerpc/kernel/setup-common.c
+++ b/arch/powerpc/kernel/setup-common.c
@@ -401,8 +401,8 @@  void __init check_for_initrd(void)
 
 #ifdef CONFIG_SMP
 
-int threads_per_core, threads_per_subcore, threads_shift;
-cpumask_t threads_core_mask;
+int threads_per_core, threads_per_subcore, threads_shift __read_mostly;
+cpumask_t threads_core_mask __read_mostly;
 EXPORT_SYMBOL_GPL(threads_per_core);
 EXPORT_SYMBOL_GPL(threads_per_subcore);
 EXPORT_SYMBOL_GPL(threads_shift);
diff --git a/arch/powerpc/mm/slb.c b/arch/powerpc/mm/slb.c
index cb796724a6fc..2d5db5e0132e 100644
--- a/arch/powerpc/mm/slb.c
+++ b/arch/powerpc/mm/slb.c
@@ -90,7 +90,7 @@  static inline void create_shadowed_slbe(unsigned long ea, int ssize,
 		     : "memory" );
 }
 
-static void __slb_flush_and_rebolt(void)
+void __slb_flush_and_rebolt(void)
 {
 	/* If you change this make sure you change SLB_NUM_BOLTED
 	 * and PR KVM appropriately too. */
@@ -128,6 +128,8 @@  static void __slb_flush_and_rebolt(void)
 		        "r"(ksp_vsid_data),
 		        "r"(ksp_esid_data)
 		     : "memory");
+
+	get_paca()->slb_cache_ptr = 0;
 }
 
 void slb_flush_and_rebolt(void)
@@ -142,7 +144,6 @@  void slb_flush_and_rebolt(void)
 	hard_irq_disable();
 
 	__slb_flush_and_rebolt();
-	get_paca()->slb_cache_ptr = 0;
 }
 
 void slb_vmalloc_update(void)
@@ -213,6 +214,7 @@  void switch_slb(struct task_struct *tsk, struct mm_struct *mm)
 			asm volatile("slbie %0" : : "r" (slbie_data));
 		}
 		asm volatile("isync" : : : "memory");
+		get_paca()->slb_cache_ptr = 0;
 	} else {
 		__slb_flush_and_rebolt();
 	}
@@ -221,7 +223,6 @@  void switch_slb(struct task_struct *tsk, struct mm_struct *mm)
 	if (offset == 1 || offset > SLB_CACHE_ENTRIES)
 		asm volatile("slbie %0" : : "r" (slbie_data));
 
-	get_paca()->slb_cache_ptr = 0;
 	copy_mm_to_paca(mm);
 
 	/*
diff --git a/arch/powerpc/platforms/powernv/idle.c b/arch/powerpc/platforms/powernv/idle.c
index 12f13acee1f6..2e129b882727 100644
--- a/arch/powerpc/platforms/powernv/idle.c
+++ b/arch/powerpc/platforms/powernv/idle.c
@@ -16,6 +16,7 @@ 
 #include <linux/device.h>
 #include <linux/cpu.h>
 
+#include <asm/asm-prototypes.h>
 #include <asm/firmware.h>
 #include <asm/machdep.h>
 #include <asm/opal.h>
@@ -46,10 +47,10 @@  static u64 pnv_default_stop_mask;
 static bool default_stop_found;
 
 /*
- * First deep stop state. Used to figure out when to save/restore
- * hypervisor context.
+ * First stop state levels when HV and TB loss can occur.
  */
-u64 pnv_first_deep_stop_state = MAX_STOP_STATE;
+static u64 pnv_first_tb_loss_level = MAX_STOP_STATE + 1;
+static u64 pnv_first_hv_loss_level = MAX_STOP_STATE + 1;
 
 /*
  * psscr value and mask of the deepest stop idle state.
@@ -135,11 +136,11 @@  static int pnv_save_sprs_for_deep_states(void)
 	return 0;
 }
 
-static void pnv_alloc_idle_core_states(void)
+static void pnv_alloc_idle_core_states_p8(void)
 {
 	int i, j;
 	int nr_cores = cpu_nr_cores();
-	u32 *core_idle_state;
+	unsigned long *core_idle_state;
 
 	/*
 	 * core_idle_state - The lower 8 bits track the idle state of
@@ -166,7 +167,7 @@  static void pnv_alloc_idle_core_states(void)
 		int node = cpu_to_node(first_cpu);
 		size_t paca_ptr_array_size;
 
-		core_idle_state = kmalloc_node(sizeof(u32), GFP_KERNEL, node);
+		core_idle_state = kmalloc_node(sizeof(unsigned long), GFP_KERNEL, node);
 		*core_idle_state = (1 << threads_per_core) - 1;
 		paca_ptr_array_size = (threads_per_core *
 				       sizeof(struct paca_struct *));
@@ -181,41 +182,6 @@  static void pnv_alloc_idle_core_states(void)
 	}
 
 	update_subcore_sibling_mask();
-
-	if (supported_cpuidle_states & OPAL_PM_LOSE_FULL_CONTEXT) {
-		int rc = pnv_save_sprs_for_deep_states();
-
-		if (likely(!rc))
-			return;
-
-		/*
-		 * The stop-api is unable to restore hypervisor
-		 * resources on wakeup from platform idle states which
-		 * lose full context. So disable such states.
-		 */
-		supported_cpuidle_states &= ~OPAL_PM_LOSE_FULL_CONTEXT;
-		pr_warn("cpuidle-powernv: Disabling idle states that lose full context\n");
-		pr_warn("cpuidle-powernv: Idle power-savings, CPU-Hotplug affected\n");
-
-		if (cpu_has_feature(CPU_FTR_ARCH_300) &&
-		    (pnv_deepest_stop_flag & OPAL_PM_LOSE_FULL_CONTEXT)) {
-			/*
-			 * Use the default stop state for CPU-Hotplug
-			 * if available.
-			 */
-			if (default_stop_found) {
-				pnv_deepest_stop_psscr_val =
-					pnv_default_stop_val;
-				pnv_deepest_stop_psscr_mask =
-					pnv_default_stop_mask;
-				pr_warn("cpuidle-powernv: Offlined CPUs will stop with psscr = 0x%016llx\n",
-					pnv_deepest_stop_psscr_val);
-			} else { /* Fallback to snooze loop for CPU-Hotplug */
-				deepest_stop_found = false;
-				pr_warn("cpuidle-powernv: Offlined CPUs will busy wait\n");
-			}
-		}
-	}
 }
 
 u32 pnv_get_supported_cpuidle_states(void)
@@ -345,6 +311,263 @@  void power7_idle(void)
 	power7_idle_type(PNV_THREAD_NAP);
 }
 
+struct p9_sprs {
+	/* per core */
+	u64 ptcr;
+	u64 rpr;
+	u64 tscr;
+	u64 ldbar;
+
+	/* per thread */
+	u64 lpcr;
+	u64 hfscr;
+	u64 fscr;
+	u64 pid;
+	u64 purr;
+	u64 spurr;
+	u64 dscr;
+
+	u64 mmcra;
+	u32 mmcr0;
+	u32 mmcr1;
+	u64 mmcr2;
+};
+
+static inline void atomic_start_thread_idle(void)
+{
+	int cpu = raw_smp_processor_id();
+	int first = cpu_first_thread_sibling(cpu);
+	int thread = cpu_thread_in_core(cpu);
+	unsigned long *state = &paca_ptrs[first]->idle_state;
+	u64 s = READ_ONCE(*state);
+	u64 new, tmp;
+
+again:
+	if (unlikely(s & PNV_CORE_IDLE_LOCK_BIT)) {
+		spin_begin();
+		do {
+			spin_cpu_relax();
+			s = READ_ONCE(*state);
+		} while (s & PNV_CORE_IDLE_LOCK_BIT);
+		spin_end();
+	}
+
+	BUG_ON(!(s & thread));
+
+	new = s & ~thread;
+	tmp = cmpxchg(state, s, new);
+	if (unlikely(tmp != s)) {
+		s = tmp;
+		goto again;
+	}
+}
+
+static inline void atomic_lock_thread_idle(void)
+{
+	int cpu = raw_smp_processor_id();
+	int first = cpu_first_thread_sibling(cpu);
+	unsigned long *state = &paca_ptrs[first]->idle_state;
+
+	while (unlikely(test_and_set_bit_lock(NR_PNV_CORE_IDLE_LOCK_BIT, state)))
+		barrier();
+}
+
+static inline void atomic_unlock_thread_idle(void)
+{
+	int cpu = raw_smp_processor_id();
+	int first = cpu_first_thread_sibling(cpu);
+	unsigned long *state = &paca_ptrs[first]->idle_state;
+
+	clear_bit_unlock(NR_PNV_CORE_IDLE_LOCK_BIT, state);
+}
+
+static unsigned long power9_idle_stop(unsigned long psscr, bool mmu_on)
+{
+	int cpu = raw_smp_processor_id();
+	int first = cpu_first_thread_sibling(cpu);
+	int thread = cpu_thread_in_core(cpu);
+	unsigned long *state = &paca_ptrs[first]->idle_state;
+	unsigned long srr1;
+	unsigned long mmcr0 = 0;
+	struct p9_sprs sprs;
+
+	/* XXX: this gets rid of the uninitialized warning. Should use attributes because this is expensive */
+	memset(&sprs, 0, sizeof(sprs));
+
+	if (!(psscr & (PSSCR_EC|PSSCR_ESL))) {
+		/*
+		 * Wake synchronously. SRESET via xscom may still cause
+		 * a 0x100 powersave wakeup with SRR1 reason!
+		 */
+		srr1 = isa3_idle_stop_noloss(psscr);
+		if (likely(!srr1))
+			return 0;
+
+	} else {
+		if (!cpu_has_feature(CPU_FTR_POWER9_DD2_1)) {
+			 /*
+			  * POWER9 DD2 can incorrectly set PMAO when waking up
+			  * after a state-loss idle. Saving and restoring MMCR0
+			  * over idle is a workaround.
+			  */
+			mmcr0 = mfspr(SPRN_MMCR0);
+		}
+		if ((psscr & PSSCR_RL_MASK) >= pnv_first_hv_loss_level) {
+			atomic_start_thread_idle();
+
+			sprs.ptcr = mfspr(SPRN_PTCR);
+			sprs.rpr = mfspr(SPRN_RPR);
+			sprs.tscr = mfspr(SPRN_TSCR);
+			sprs.ldbar = mfspr(SPRN_LDBAR);
+
+			sprs.lpcr = mfspr(SPRN_LPCR);
+			sprs.hfscr = mfspr(SPRN_HFSCR);
+			sprs.fscr = mfspr(SPRN_FSCR);
+			sprs.pid = mfspr(SPRN_PID);
+			sprs.purr = mfspr(SPRN_PURR);
+			sprs.spurr = mfspr(SPRN_SPURR);
+			sprs.dscr = mfspr(SPRN_DSCR);
+
+			sprs.mmcra = mfspr(SPRN_MMCRA);
+			sprs.mmcr0 = mfspr(SPRN_MMCR0);
+			sprs.mmcr1 = mfspr(SPRN_MMCR1);
+			sprs.mmcr2 = mfspr(SPRN_MMCR2);
+		}
+#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE
+		if (cpu_has_feature(CPU_FTR_P9_TM_XER_SO_BUG)) {
+			local_paca->requested_psscr = psscr;
+			/* order setting requested_psscr vs testing dont_stop */
+			smp_mb();
+			if (atomic_read(&local_paca->dont_stop)) {
+				local_paca->requested_psscr = 0;
+				return 0;
+			}
+
+			srr1 = isa3_idle_stop_mayloss(psscr);
+			local_paca->requested_psscr = 0;
+		} else
+#endif
+			srr1 = isa3_idle_stop_mayloss(psscr);
+	}
+
+	WARN_ON_ONCE(!srr1);
+	WARN_ON_ONCE(mfmsr() & (MSR_IR|MSR_DR));
+
+	if ((srr1 & SRR1_WAKESTATE) != SRR1_WS_NOLOSS) {
+		unsigned long mmcra;
+
+		/*
+		 * Workaround for POWER9 DD2, if we lost resources, the ERAT
+		 * might have been mixed up and needs flushing. We also need
+		 * to reload MMCR0 (see mmcr0 comment above).
+		 */
+		if (!cpu_has_feature(CPU_FTR_POWER9_DD2_1)) {
+			asm volatile(PPC_INVALIDATE_ERAT);
+			mtspr(SPRN_MMCR0, mmcr0);
+		}
+
+		/*
+		 * DD2.2 and earlier need to set then clear bit 60 in MMCRA
+		 * to ensure the PMU starts running.
+		 */
+		mmcra = mfspr(SPRN_MMCRA);
+		mmcra |= PPC_BIT(60);
+		mtspr(SPRN_MMCRA, mmcra);
+		mmcra &= ~PPC_BIT(60);
+		mtspr(SPRN_MMCRA, mmcra);
+	}
+
+	if (unlikely((srr1 & SRR1_WAKEMASK_P8) == SRR1_WAKEHMI))
+		hmi_exception_realmode(NULL);
+
+	if (likely((srr1 & SRR1_WAKESTATE) != SRR1_WS_HVLOSS)) {
+		mtmsr(MSR_KERNEL);
+		return srr1;
+	}
+
+	/* HV state loss */
+	WARN_ON((psscr & PSSCR_RL_MASK) < pnv_first_hv_loss_level);
+
+	atomic_lock_thread_idle();
+
+	WARN_ON(*state & thread);
+
+	if ((*state & ((1 << threads_per_core) - 1)) != 0)
+		goto core_woken;
+
+	/* Per-core SPRs */
+	mtspr(SPRN_PTCR, sprs.ptcr);
+	mtspr(SPRN_RPR, sprs.rpr);
+	mtspr(SPRN_TSCR, sprs.tscr);
+	mtspr(SPRN_LDBAR, sprs.ldbar);
+
+	if ((psscr & PSSCR_RL_MASK) >= pnv_first_tb_loss_level) {
+		unsigned long level = mfspr(SPRN_PSSCR) & PSSCR_RL_MASK;
+		if (level >= pnv_first_tb_loss_level) {
+			/* TB loss */
+			if (opal_resync_timebase() != OPAL_SUCCESS)
+				BUG();
+		}
+	}
+
+core_woken:
+	*state |= thread;
+	atomic_unlock_thread_idle();
+
+	/* Per-thread SPRs */
+	mtspr(SPRN_LPCR, sprs.lpcr);
+	mtspr(SPRN_HFSCR, sprs.hfscr);
+	mtspr(SPRN_FSCR, sprs.fscr);
+	mtspr(SPRN_PID, sprs.pid);
+	mtspr(SPRN_PURR, sprs.purr);
+	mtspr(SPRN_SPURR, sprs.spurr);
+	mtspr(SPRN_DSCR, sprs.dscr);
+
+	mtspr(SPRN_MMCRA, sprs.mmcra);
+	mtspr(SPRN_MMCR0, sprs.mmcr0);
+	mtspr(SPRN_MMCR1, sprs.mmcr1);
+	mtspr(SPRN_MMCR2, sprs.mmcr2);
+
+	if (!radix_enabled())
+		__slb_flush_and_rebolt();
+
+	mtmsr(MSR_KERNEL);
+
+	return srr1;
+}
+
+unsigned long power9_offline_stop(unsigned long psscr)
+{
+#ifndef CONFIG_KVM_BOOK3S_HV_POSSIBLE
+	return power9_idle_stop(psscr, true);
+#else
+	unsigned long srr1;
+
+	/*
+	 * Tell KVM we're entering idle.
+	 * This does not have to be done in real mode because the P9 MMU
+	 * is independent per-thread. Some steppings share radix/hash mode
+	 * between threads, but in that case KVM has a barrier sync in real
+	 * mode before and after switching between radix and hash.
+	 */
+	local_paca->kvm_hstate.hwthread_state = KVM_HWTHREAD_IN_IDLE;
+	srr1 = power9_idle_stop(psscr, false);
+
+	if (local_paca->kvm_hstate.hwthread_state == KVM_HWTHREAD_IN_KERNEL) {
+		local_paca->kvm_hstate.hwthread_state = KVM_HWTHREAD_IN_KERNEL;
+		/* Order setting hwthread_state vs. testing hwthread_req */
+		smp_mb();
+	}
+	if (local_paca->kvm_hstate.hwthread_req) {
+		/* XXX: fix this so it's not garbage */
+		asm volatile("b	kvm_start_guest" ::: "memory");
+	}
+	mtmsr(MSR_KERNEL);
+
+	return srr1;
+#endif
+}
+
 static unsigned long __power9_idle_type(unsigned long stop_psscr_val,
 				      unsigned long stop_psscr_mask)
 {
@@ -358,7 +581,7 @@  static unsigned long __power9_idle_type(unsigned long stop_psscr_val,
 	psscr = (psscr & ~stop_psscr_mask) | stop_psscr_val;
 
 	__ppc64_runlatch_off();
-	srr1 = power9_idle_stop(psscr);
+	srr1 = power9_idle_stop(psscr, true);
 	__ppc64_runlatch_on();
 
 	fini_irq_for_idle_irqsoff();
@@ -407,7 +630,7 @@  void pnv_power9_force_smt4_catch(void)
 			atomic_inc(&paca_ptrs[cpu0+thr]->dont_stop);
 	}
 	/* order setting dont_stop vs testing requested_psscr */
-	mb();
+	smp_mb();
 	for (thr = 0; thr < threads_per_core; ++thr) {
 		if (!paca_ptrs[cpu0+thr]->requested_psscr)
 			++awake_threads;
@@ -623,7 +846,8 @@  static int __init pnv_power9_idle_init(struct device_node *np, u32 *flags,
 	u64 *psscr_val = NULL;
 	u64 *psscr_mask = NULL;
 	u32 *residency_ns = NULL;
-	u64 max_residency_ns = 0;
+	u64 max_deep_residency_ns = 0;
+	u64 max_default_residency_ns = 0;
 	int rc = 0, i;
 
 	psscr_val = kcalloc(dt_idle_states, sizeof(*psscr_val), GFP_KERNEL);
@@ -661,26 +885,25 @@  static int __init pnv_power9_idle_init(struct device_node *np, u32 *flags,
 	}
 
 	/*
-	 * Set pnv_first_deep_stop_state, pnv_deepest_stop_psscr_{val,mask},
-	 * and the pnv_default_stop_{val,mask}.
-	 *
-	 * pnv_first_deep_stop_state should be set to the first stop
-	 * level to cause hypervisor state loss.
-	 *
 	 * pnv_deepest_stop_{val,mask} should be set to values corresponding to
 	 * the deepest stop state.
 	 *
 	 * pnv_default_stop_{val,mask} should be set to values corresponding to
-	 * the shallowest (OPAL_PM_STOP_INST_FAST) loss-less stop state.
+	 * the deepest loss-less (OPAL_PM_STOP_INST_FAST) stop state.
 	 */
-	pnv_first_deep_stop_state = MAX_STOP_STATE;
+	pnv_first_tb_loss_level = MAX_STOP_STATE + 1;
+	pnv_first_hv_loss_level = MAX_STOP_STATE + 1;
 	for (i = 0; i < dt_idle_states; i++) {
 		int err;
 		u64 psscr_rl = psscr_val[i] & PSSCR_RL_MASK;
 
+		if ((flags[i] & OPAL_PM_TIMEBASE_STOP) &&
+		     (pnv_first_tb_loss_level > psscr_rl))
+			pnv_first_tb_loss_level = psscr_rl;
+
 		if ((flags[i] & OPAL_PM_LOSE_FULL_CONTEXT) &&
-		     (pnv_first_deep_stop_state > psscr_rl))
-			pnv_first_deep_stop_state = psscr_rl;
+		     (pnv_first_hv_loss_level > psscr_rl))
+			pnv_first_hv_loss_level = psscr_rl;
 
 		err = validate_psscr_val_mask(&psscr_val[i], &psscr_mask[i],
 					      flags[i]);
@@ -689,19 +912,21 @@  static int __init pnv_power9_idle_init(struct device_node *np, u32 *flags,
 			continue;
 		}
 
-		if (max_residency_ns < residency_ns[i]) {
-			max_residency_ns = residency_ns[i];
+		if (max_deep_residency_ns < residency_ns[i]) {
+			max_deep_residency_ns = residency_ns[i];
 			pnv_deepest_stop_psscr_val = psscr_val[i];
 			pnv_deepest_stop_psscr_mask = psscr_mask[i];
 			pnv_deepest_stop_flag = flags[i];
 			deepest_stop_found = true;
 		}
 
-		if (!default_stop_found &&
+		if (max_default_residency_ns < residency_ns[i] &&
 		    (flags[i] & OPAL_PM_STOP_INST_FAST)) {
+			max_default_residency_ns = residency_ns[i];
 			pnv_default_stop_val = psscr_val[i];
 			pnv_default_stop_mask = psscr_mask[i];
 			default_stop_found = true;
+			WARN_ON(flags[i] & OPAL_PM_LOSE_FULL_CONTEXT);
 		}
 	}
 
@@ -721,15 +946,48 @@  static int __init pnv_power9_idle_init(struct device_node *np, u32 *flags,
 			pnv_deepest_stop_psscr_mask);
 	}
 
-	pr_info("cpuidle-powernv: Requested Level (RL) value of first deep stop = 0x%llx\n",
-		pnv_first_deep_stop_state);
+	pr_info("cpuidle-powernv: First stop level that may lose SPRs = 0x%lld\n",
+		pnv_first_hv_loss_level);
+
+	pr_info("cpuidle-powernv: First stop level that may lose timebase = 0x%lld\n",
+		pnv_first_tb_loss_level);
 out:
 	kfree(psscr_val);
 	kfree(psscr_mask);
 	kfree(residency_ns);
+
 	return rc;
 }
 
+static void __init pnv_disable_deep_states(void)
+{
+	/*
+	 * The stop-api is unable to restore hypervisor
+	 * resources on wakeup from platform idle states which
+	 * lose full context. So disable such states.
+	 */
+	supported_cpuidle_states &= ~OPAL_PM_LOSE_FULL_CONTEXT;
+	pr_warn("cpuidle-powernv: Disabling idle states that lose full context\n");
+	pr_warn("cpuidle-powernv: Idle power-savings, CPU-Hotplug affected\n");
+
+	if (cpu_has_feature(CPU_FTR_ARCH_300) &&
+	    (pnv_deepest_stop_flag & OPAL_PM_LOSE_FULL_CONTEXT)) {
+		/*
+		 * Use the default stop state for CPU-Hotplug
+		 * if available.
+		 */
+		if (default_stop_found) {
+			pnv_deepest_stop_psscr_val = pnv_default_stop_val;
+			pnv_deepest_stop_psscr_mask = pnv_default_stop_mask;
+			pr_warn("cpuidle-powernv: Offlined CPUs will stop with psscr = 0x%016llx\n",
+				pnv_deepest_stop_psscr_val);
+		} else { /* Fallback to snooze loop for CPU-Hotplug */
+			deepest_stop_found = false;
+			pr_warn("cpuidle-powernv: Offlined CPUs will busy wait\n");
+		}
+	}
+}
+
 /*
  * Probe device tree for supported idle states
  */
@@ -771,6 +1029,7 @@  static void __init pnv_probe_idle_states(void)
 out:
 	kfree(flags);
 }
+
 static int __init pnv_init_idle_states(void)
 {
 
@@ -798,10 +1057,29 @@  static int __init pnv_init_idle_states(void)
 				&dev_attr_fastsleep_workaround_applyonce);
 	}
 
-	pnv_alloc_idle_core_states();
+	if (!cpu_has_feature(CPU_FTR_ARCH_300)) {
+		pnv_alloc_idle_core_states_p8();
+		if (supported_cpuidle_states & OPAL_PM_NAP_ENABLED)
+			ppc_md.power_save = power7_idle;
+	} else {
+		int cpu;
+
+		for_each_present_cpu(cpu) {
+			paca_ptrs[cpu]->requested_psscr = 0;
+			paca_ptrs[cpu]->idle_state = 0;
+			if (cpu == cpu_first_thread_sibling(cpu))
+				paca_ptrs[cpu]->idle_state =
+					(1 << threads_per_core) - 1;
+#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE
+			atomic_set(&paca_ptrs[cpu]->dont_stop, 0);
+#endif
+		}
+	}
 
-	if (supported_cpuidle_states & OPAL_PM_NAP_ENABLED)
-		ppc_md.power_save = power7_idle;
+	if (supported_cpuidle_states & OPAL_PM_LOSE_FULL_CONTEXT) {
+		if (pnv_save_sprs_for_deep_states())
+			pnv_disable_deep_states();
+	}
 
 out:
 	return 0;
diff --git a/arch/powerpc/xmon/xmon.c b/arch/powerpc/xmon/xmon.c
index 21119cfe8474..09120d4ec12b 100644
--- a/arch/powerpc/xmon/xmon.c
+++ b/arch/powerpc/xmon/xmon.c
@@ -2425,19 +2425,18 @@  static void dump_one_paca(int cpu)
 #endif
 
 #ifdef CONFIG_PPC_POWERNV
-	DUMP(p, core_idle_state_ptr, "%-*px");
-	DUMP(p, thread_idle_state, "%#-*x");
-	DUMP(p, thread_mask, "%#-*x");
-	DUMP(p, subcore_sibling_mask, "%#-*x");
-	DUMP(p, requested_psscr, "%#-*llx");
-	DUMP(p, stop_sprs.pid, "%#-*llx");
-	DUMP(p, stop_sprs.ldbar, "%#-*llx");
-	DUMP(p, stop_sprs.fscr, "%#-*llx");
-	DUMP(p, stop_sprs.hfscr, "%#-*llx");
-	DUMP(p, stop_sprs.mmcr1, "%#-*llx");
-	DUMP(p, stop_sprs.mmcr2, "%#-*llx");
-	DUMP(p, stop_sprs.mmcra, "%#-*llx");
-	DUMP(p, dont_stop.counter, "%#-*x");
+	if (!cpu_has_feature(CPU_FTR_ARCH_300)) {
+		DUMP(p, core_idle_state_ptr, "%-*px");
+		DUMP(p, thread_idle_state, "%#-*x");
+		DUMP(p, thread_mask, "%#-*x");
+		DUMP(p, subcore_sibling_mask, "%#-*x");
+	} else {
+		DUMP(p, idle_state, "%#-*lx");
+		DUMP(p, requested_psscr, "%#-*llx");
+#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE
+		DUMP(p, dont_stop.counter, "%#-*x");
+#endif
+	}
 #endif
 
 	DUMP(p, accounting.utime, "%#-*lx");