[RFC] opal : Support for pre-entry and post-exit of stop state in opal

This patch provides opal support for save restore of sprs in idle stop
loop for LE opal. Opal support for stop states is needed to selectively
enable stop states or to introduce a quirk quickly in case a buggy
stop state is present.
We make a opal call from kernel if firmware-stop-support for stop
states is enabled. All the quirks for pre-entry of stop state is
handled inside opal. A call from opal is made into kernel where we
execute stop afer saving of NVGPRs.
After waking up from 0x100 vector in kernel, we enter back into opal.
All the quirks in post exit path, if any, are then handled in opal,
from where we return successfully back to kernel.
For deep stop states in which additional SPRs are lost, saving and
restoration will be done in OPAL.

This idea was first proposed by Nick here:
https://patchwork.ozlabs.org/patch/1208159/

Will combine this patch with the idle-stop versioning patch for BE
opal proposed here : https://patchwork.ozlabs.org/patch/1249114/

Signed-off-by: Abhishek Goel <huntbag@linux.vnet.ibm.com>
Signed-off-by: Nicholas Piggin <npiggin@gmail.com>
---
 core/opal.c             | 57 +++++++++++++++++++++++++++++++++++++++++
 hw/slw.c                |  3 +++
 include/opal-api.h      |  8 +++++-
 include/opal-internal.h | 10 ++++++++
 4 files changed, 77 insertions(+), 1 deletion(-)

Hi Abhishek,

On Fri, Apr 03, 2020 at 03:56:38AM -0500, Abhishek Goel wrote:
> This patch provides opal support for save restore of sprs in idle stop
> loop for LE opal. Opal support for stop states is needed to selectively
> enable stop states or to introduce a quirk quickly in case a buggy
> stop state is present.
> We make a opal call from kernel if firmware-stop-support for stop
> states is enabled. All the quirks for pre-entry of stop state is
> handled inside opal. A call from opal is made into kernel where we
> execute stop afer saving of NVGPRs.
> After waking up from 0x100 vector in kernel, we enter back into opal.
> All the quirks in post exit path, if any, are then handled in opal,
> from where we return successfully back to kernel.
> For deep stop states in which additional SPRs are lost, saving and
> restoration will be done in OPAL.
> 
> This idea was first proposed by Nick here:
> https://patchwork.ozlabs.org/patch/1208159/
> 
> Will combine this patch with the idle-stop versioning patch for BE
> opal proposed here : https://patchwork.ozlabs.org/patch/1249114/


> Signed-off-by: Abhishek Goel <huntbag@linux.vnet.ibm.com>
> Signed-off-by: Nicholas Piggin <npiggin@gmail.com>

Since the patch originally came from Nicholas, his Signed-off-by
should be above yours. And since he is authored major portions of the
patch, Author credit should go to him.

> ---
>  core/opal.c             | 57 +++++++++++++++++++++++++++++++++++++++++
>  hw/slw.c                |  3 +++
>  include/opal-api.h      |  8 +++++-
>  include/opal-internal.h | 10 ++++++++
>  4 files changed, 77 insertions(+), 1 deletion(-)
> 
> diff --git a/core/opal.c b/core/opal.c
> index 64fdfe62..e7fa087c 100644
> --- a/core/opal.c
> +++ b/core/opal.c
> @@ -44,6 +44,7 @@ static uint64_t opal_dynamic_events;
>  extern uint32_t attn_trigger;
>  extern uint32_t hir_trigger;
> 
> +struct os_ops os_ops;
> 
>  void opal_table_init(void)
>  {
> @@ -422,6 +423,62 @@ void add_opal_node(void)
>  	memcons_add_properties();
>  }
> 
> +/*
> + * Function to register all the os operations in opal.
> + * Currently registering a os_ops that will handle idle stop
> + * saving and restoring of sprs in kernel.
> + */
> +static int64_t opal_register_os_ops(struct opal_os_ops *__os_ops)
> +{
> +	struct cpu_thread *cpu;
> +
> +	for_each_cpu(cpu) {
> +		if (cpu == this_cpu())
> +			continue;
> +		if (cpu->state == cpu_state_os)
> +			return OPAL_BUSY;
> +	}

Why is this synchronization required?

> +
> +	os_ops.os_idle_stop = (void *)be64_to_cpu(__os_ops->os_idle_stop);
> +
> +	return OPAL_SUCCESS;
> +}
> +opal_call(OPAL_REGISTER_OS_OPS, opal_register_os_ops, 1);
> +
> +/*
> + * Opal function to handle idle stop in kernel.
> + */
> +static uint64_t opal_cpu_idle(__be64 srr1_addr, uint64_t psscr)
> +{
> +	u64 *le_srr1 = (u64 *)be64_to_cpu(srr1_addr);
> +
> +        if (!os_ops.os_idle_stop)
> +                return OPAL_UNSUPPORTED;
> +
> +        if (proc_gen != proc_gen_p9)
> +                return OPAL_UNSUPPORTED;
> +
> +	/*
> +	 * This will contain all the kernel code or quirks which
> +	 * manages saving of sprs before entering into stop.
> +	 * Saving of Additional SPRs required for deep stop states will
> +	 * be done here.
> +	 */
> +	if (!(psscr & (OPAL_PM_PSSCR_EC|OPAL_PM_PSSCR_ESL)))
> +	        *le_srr1 = os_ops.os_idle_stop(psscr, false);
> +	else
> +		*le_srr1 = os_ops.os_idle_stop(psscr, true);


Just curious.. Does this work with stop4/5 as well or only stop0-2 ?

> +	/*
> +	 * This will contain all the kernel code or quirks which
> +	 * manages restoring of sprs after exiting from stop.
> +	 * Restoration of additional SPRs that are lost for deep stop
> +	 * states will be done here.
> +	 */
> +
> +        return OPAL_SUCCESS;
> +}
> +opal_call(OPAL_CPU_IDLE, opal_cpu_idle, 2);
> +
>  static struct lock evt_lock = LOCK_UNLOCKED;
> 
>  void opal_update_pending_evt(uint64_t evt_mask, uint64_t evt_values)
> diff --git a/hw/slw.c b/hw/slw.c
> index beb129a8..96e7152f 100644
> --- a/hw/slw.c
> +++ b/hw/slw.c
> @@ -958,6 +958,9 @@ void add_cpu_idle_state_properties(void)
>  		dt_add_property(power_mgt, "ibm,cpu-idle-state-psscr-mask",
>  				pm_ctrl_reg_mask_buf,
>  				num_supported_idle_states * sizeof(u64));
> +		if (__BYTE_ORDER == __LITTLE_ENDIAN)
> +			dt_add_property_string(power_mgt, "compatible",
> +					       "firmware-stop-supported");

This can be a dt_cpu_feature instead of another property under
power_mgt, no ?


>  	} else {
>  		dt_add_property(power_mgt, "ibm,cpu-idle-state-pmicr",
>  				pm_ctrl_reg_val_buf,
> diff --git a/include/opal-api.h b/include/opal-api.h
> index e90cab1e..a1e7d122 100644
> --- a/include/opal-api.h
> +++ b/include/opal-api.h
> @@ -227,7 +227,9 @@
>  #define OPAL_SECVAR_ENQUEUE_UPDATE		178
>  #define OPAL_PHB_SET_OPTION			179
>  #define OPAL_PHB_GET_OPTION			180
> -#define OPAL_LAST				180
> +#define OPAL_REGISTER_OS_OPS			181
> +#define OPAL_CPU_IDLE				182
> +#define OPAL_LAST				182
> 
>  #define QUIESCE_HOLD			1 /* Spin all calls at entry */
>  #define QUIESCE_REJECT			2 /* Fail all calls with OPAL_BUSY */
> @@ -1255,6 +1257,10 @@ struct opal_mpipl_fadump {
>  	struct	opal_mpipl_region region[];
>  };
> 
> +struct opal_os_ops {
> +	__be64 os_idle_stop;
> +};
> +
>  #endif /* __ASSEMBLY__ */
> 
>  #endif /* __OPAL_API_H */
> diff --git a/include/opal-internal.h b/include/opal-internal.h
> index f6ca7ac3..9368fb79 100644
> --- a/include/opal-internal.h
> +++ b/include/opal-internal.h
> @@ -18,6 +18,14 @@ struct opal_table_entry {
>  	u32	nargs;
>  };
> 
> +struct os_ops {
> +	/*
> +	 * save_gprs help us distinguish between lite states and
> +	 * non-lite states.
> +	 */
> +	int64_t (*os_idle_stop)(uint64_t psscr, bool save_gprs);
> +};
> +
>  #ifdef __CHECKER__
>  #define __opal_func_test_arg(__func, __nargs) 0
>  #else
> @@ -75,6 +83,8 @@ extern void opal_run_pollers(void);
>  extern void opal_add_host_sync_notifier(bool (*notify)(void *data), void *data);
>  extern void opal_del_host_sync_notifier(bool (*notify)(void *data), void *data);
> 
> +extern int64_t os_idle_stop(uint64_t psscr, bool save_gprs);
> +
>  /*
>   * Opal internal function prototype
>   */
> -- 
> 2.17.1
>