[V5,11/18] clk: tegra210: Add support for Tegra210 clocks

This patch adds system suspend and resume support for Tegra210
clocks.

All the CAR controller settings are lost on suspend when core power
goes off.

This patch has implementation for saving and restoring all the PLLs
and clocks context during system suspend and resume to have the
clocks back to same state for normal operation.

Acked-by: Thierry Reding <treding@nvidia.com>
Signed-off-by: Sowjanya Komatineni <skomatineni@nvidia.com>
---
 drivers/clk/tegra/clk-tegra210.c | 115 ++++++++++++++++++++++++++++++++++++++-
 drivers/clk/tegra/clk.c          |  14 +++++
 drivers/clk/tegra/clk.h          |   1 +
 3 files changed, 127 insertions(+), 3 deletions(-)

28.06.2019 5:12, Sowjanya Komatineni пишет:
> This patch adds system suspend and resume support for Tegra210
> clocks.
> 
> All the CAR controller settings are lost on suspend when core power
> goes off.
> 
> This patch has implementation for saving and restoring all the PLLs
> and clocks context during system suspend and resume to have the
> clocks back to same state for normal operation.
> 
> Acked-by: Thierry Reding <treding@nvidia.com>
> Signed-off-by: Sowjanya Komatineni <skomatineni@nvidia.com>
> ---
>  drivers/clk/tegra/clk-tegra210.c | 115 ++++++++++++++++++++++++++++++++++++++-
>  drivers/clk/tegra/clk.c          |  14 +++++
>  drivers/clk/tegra/clk.h          |   1 +
>  3 files changed, 127 insertions(+), 3 deletions(-)
> 
> diff --git a/drivers/clk/tegra/clk-tegra210.c b/drivers/clk/tegra/clk-tegra210.c
> index 1c08c53482a5..1b839544e086 100644
> --- a/drivers/clk/tegra/clk-tegra210.c
> +++ b/drivers/clk/tegra/clk-tegra210.c
> @@ -9,10 +9,12 @@
>  #include <linux/clkdev.h>
>  #include <linux/of.h>
>  #include <linux/of_address.h>
> +#include <linux/of_platform.h>
>  #include <linux/delay.h>
>  #include <linux/export.h>
>  #include <linux/mutex.h>
>  #include <linux/clk/tegra.h>
> +#include <linux/syscore_ops.h>
>  #include <dt-bindings/clock/tegra210-car.h>
>  #include <dt-bindings/reset/tegra210-car.h>
>  #include <linux/iopoll.h>
> @@ -20,6 +22,7 @@
>  #include <soc/tegra/pmc.h>
>  
>  #include "clk.h"
> +#include "clk-dfll.h"
>  #include "clk-id.h"
>  
>  /*
> @@ -225,6 +228,7 @@
>  
>  #define CLK_RST_CONTROLLER_RST_DEV_Y_SET 0x2a8
>  #define CLK_RST_CONTROLLER_RST_DEV_Y_CLR 0x2ac
> +#define CPU_SOFTRST_CTRL 0x380
>  
>  #define LVL2_CLK_GATE_OVRA 0xf8
>  #define LVL2_CLK_GATE_OVRC 0x3a0
> @@ -2820,6 +2824,7 @@ static int tegra210_enable_pllu(void)
>  	struct tegra_clk_pll_freq_table *fentry;
>  	struct tegra_clk_pll pllu;
>  	u32 reg;
> +	int ret;
>  
>  	for (fentry = pll_u_freq_table; fentry->input_rate; fentry++) {
>  		if (fentry->input_rate == pll_ref_freq)
> @@ -2847,10 +2852,10 @@ static int tegra210_enable_pllu(void)
>  	fence_udelay(1, clk_base);
>  	reg |= PLL_ENABLE;
>  	writel(reg, clk_base + PLLU_BASE);
> +	fence_udelay(1, clk_base);
>  
> -	readl_relaxed_poll_timeout_atomic(clk_base + PLLU_BASE, reg,
> -					  reg & PLL_BASE_LOCK, 2, 1000);
> -	if (!(reg & PLL_BASE_LOCK)) {
> +	ret = tegra210_wait_for_mask(&pllu, PLLU_BASE, PLL_BASE_LOCK);
> +	if (ret) {
>  		pr_err("Timed out waiting for PLL_U to lock\n");
>  		return -ETIMEDOUT;
>  	}
> @@ -3283,6 +3288,103 @@ static void tegra210_disable_cpu_clock(u32 cpu)
>  }
>  
>  #ifdef CONFIG_PM_SLEEP
> +static u32 cpu_softrst_ctx[3];
> +static struct platform_device *dfll_pdev;
> +#define car_readl(_base, _off) readl_relaxed(clk_base + (_base) + ((_off) * 4))
> +#define car_writel(_val, _base, _off) \
> +		writel_relaxed(_val, clk_base + (_base) + ((_off) * 4))
> +
> +static int tegra210_clk_suspend(void)
> +{
> +	unsigned int i;
> +	struct device_node *node;
> +
> +	tegra_cclkg_burst_policy_save_context();
> +
> +	if (!dfll_pdev) {
> +		node = of_find_compatible_node(NULL, NULL,
> +					       "nvidia,tegra210-dfll");
> +		if (node)
> +			dfll_pdev = of_find_device_by_node(node);
> +
> +		of_node_put(node);
> +		if (!dfll_pdev)
> +			pr_err("dfll node not found. no suspend for dfll\n");
> +	}
> +
> +	if (dfll_pdev)
> +		tegra_dfll_suspend(dfll_pdev);
> +
> +	/* Enable PLLP_OUT_CPU after dfll suspend */
> +	tegra_clk_set_pllp_out_cpu(true);
> +
> +	tegra_sclk_cclklp_burst_policy_save_context();
> +
> +	clk_save_context();
> +
> +	for (i = 0; i < ARRAY_SIZE(cpu_softrst_ctx); i++)
> +		cpu_softrst_ctx[i] = car_readl(CPU_SOFTRST_CTRL, i);
> +
> +	return 0;
> +}
> +
> +static void tegra210_clk_resume(void)
> +{
> +	unsigned int i;
> +	struct clk_hw *parent;
> +	struct clk *clk;
> +
> +	/*
> +	 * clk_restore_context restores clocks as per the clock tree.
> +	 *
> +	 * dfllCPU_out is first in the clock tree to get restored and it
> +	 * involves programming DFLL controller along with restoring CPUG
> +	 * clock burst policy.
> +	 *
> +	 * DFLL programming needs dfll_ref and dfll_soc peripheral clocks
> +	 * to be restores which are part ofthe peripheral clocks.
> +	 * So, peripheral clocks restore should happen prior to dfll clock
> +	 * restore.
> +	 */
> +
> +	tegra_clk_osc_resume(clk_base);
> +	for (i = 0; i < ARRAY_SIZE(cpu_softrst_ctx); i++)
> +		car_writel(cpu_softrst_ctx[i], CPU_SOFTRST_CTRL, i);
> +
> +	/* restore all plls and peripheral clocks */
> +	tegra210_init_pllu();
> +	clk_restore_context();
> +
> +	fence_udelay(5, clk_base);
> +
> +	/* resume SCLK and CPULP clocks */
> +	tegra_sclk_cpulp_burst_policy_restore_context();
> +
> +	/*
> +	 * restore CPUG clocks:
> +	 * - enable DFLL in open loop mode
> +	 * - switch CPUG to DFLL clock source
> +	 * - close DFLL loop
> +	 * - sync PLLX state
> +	 */
> +	if (dfll_pdev)
> +		tegra_dfll_resume(dfll_pdev, false);
> +
> +	tegra_cclkg_burst_policy_restore_context();
> +	fence_udelay(2, clk_base);
> +
> +	if (dfll_pdev)
> +		tegra_dfll_resume(dfll_pdev, true);
> +
> +	parent = clk_hw_get_parent(__clk_get_hw(clks[TEGRA210_CLK_CCLK_G]));
> +	clk = clks[TEGRA210_CLK_PLL_X];
> +	if (parent != __clk_get_hw(clk))
> +		tegra_clk_sync_state_pll(__clk_get_hw(clk));
> +
> +	/* Disable PLL_OUT_CPU after DFLL resume */
> +	tegra_clk_set_pllp_out_cpu(false);
> +}
> +
>  static void tegra210_cpu_clock_suspend(void)
>  {
>  	/* switch coresite to clk_m, save off original source */
> @@ -3298,6 +3400,11 @@ static void tegra210_cpu_clock_resume(void)
>  }
>  #endif
>  
> +static struct syscore_ops tegra_clk_syscore_ops = {
> +	.suspend = tegra210_clk_suspend,
> +	.resume = tegra210_clk_resume,
> +};
> +
>  static struct tegra_cpu_car_ops tegra210_cpu_car_ops = {
>  	.wait_for_reset	= tegra210_wait_cpu_in_reset,
>  	.disable_clock	= tegra210_disable_cpu_clock,
> @@ -3583,5 +3690,7 @@ static void __init tegra210_clock_init(struct device_node *np)
>  	tegra210_mbist_clk_init();
>  
>  	tegra_cpu_car_ops = &tegra210_cpu_car_ops;
> +
> +	register_syscore_ops(&tegra_clk_syscore_ops);
>  }
>  CLK_OF_DECLARE(tegra210, "nvidia,tegra210-car", tegra210_clock_init);
> diff --git a/drivers/clk/tegra/clk.c b/drivers/clk/tegra/clk.c
> index 9e863362d2bf..96cc9937ea37 100644
> --- a/drivers/clk/tegra/clk.c
> +++ b/drivers/clk/tegra/clk.c
> @@ -23,6 +23,7 @@
>  #define CLK_OUT_ENB_W			0x364
>  #define CLK_OUT_ENB_X			0x280
>  #define CLK_OUT_ENB_Y			0x298
> +#define CLK_ENB_PLLP_OUT_CPU		BIT(31)
>  #define CLK_OUT_ENB_SET_L		0x320
>  #define CLK_OUT_ENB_CLR_L		0x324
>  #define CLK_OUT_ENB_SET_H		0x328
> @@ -205,6 +206,19 @@ const struct tegra_clk_periph_regs *get_reg_bank(int clkid)
>  	}
>  }
>  
> +void tegra_clk_set_pllp_out_cpu(bool enable)
> +{
> +	u32 val;
> +
> +	val = readl_relaxed(clk_base + CLK_OUT_ENB_Y);
> +	if (enable)
> +		val |= CLK_ENB_PLLP_OUT_CPU;
> +	else
> +		val &= ~CLK_ENB_PLLP_OUT_CPU;
> +
> +	writel_relaxed(val, clk_base + CLK_OUT_ENB_Y);
> +}

Do we needed fence_udelay() here? I'm a bit confused about when the fence should be
used for the CLK hardware.. Maybe Peter could clarify?

28.06.2019 5:12, Sowjanya Komatineni пишет:
> This patch adds system suspend and resume support for Tegra210
> clocks.
> 
> All the CAR controller settings are lost on suspend when core power
> goes off.
> 
> This patch has implementation for saving and restoring all the PLLs
> and clocks context during system suspend and resume to have the
> clocks back to same state for normal operation.
> 
> Acked-by: Thierry Reding <treding@nvidia.com>
> Signed-off-by: Sowjanya Komatineni <skomatineni@nvidia.com>
> ---
>  drivers/clk/tegra/clk-tegra210.c | 115 ++++++++++++++++++++++++++++++++++++++-
>  drivers/clk/tegra/clk.c          |  14 +++++
>  drivers/clk/tegra/clk.h          |   1 +
>  3 files changed, 127 insertions(+), 3 deletions(-)
> 
> diff --git a/drivers/clk/tegra/clk-tegra210.c b/drivers/clk/tegra/clk-tegra210.c
> index 1c08c53482a5..1b839544e086 100644
> --- a/drivers/clk/tegra/clk-tegra210.c
> +++ b/drivers/clk/tegra/clk-tegra210.c
> @@ -9,10 +9,12 @@
>  #include <linux/clkdev.h>
>  #include <linux/of.h>
>  #include <linux/of_address.h>
> +#include <linux/of_platform.h>
>  #include <linux/delay.h>
>  #include <linux/export.h>
>  #include <linux/mutex.h>
>  #include <linux/clk/tegra.h>
> +#include <linux/syscore_ops.h>
>  #include <dt-bindings/clock/tegra210-car.h>
>  #include <dt-bindings/reset/tegra210-car.h>
>  #include <linux/iopoll.h>
> @@ -20,6 +22,7 @@
>  #include <soc/tegra/pmc.h>
>  
>  #include "clk.h"
> +#include "clk-dfll.h"
>  #include "clk-id.h"
>  
>  /*
> @@ -225,6 +228,7 @@
>  
>  #define CLK_RST_CONTROLLER_RST_DEV_Y_SET 0x2a8
>  #define CLK_RST_CONTROLLER_RST_DEV_Y_CLR 0x2ac
> +#define CPU_SOFTRST_CTRL 0x380
>  
>  #define LVL2_CLK_GATE_OVRA 0xf8
>  #define LVL2_CLK_GATE_OVRC 0x3a0
> @@ -2820,6 +2824,7 @@ static int tegra210_enable_pllu(void)
>  	struct tegra_clk_pll_freq_table *fentry;
>  	struct tegra_clk_pll pllu;
>  	u32 reg;
> +	int ret;
>  
>  	for (fentry = pll_u_freq_table; fentry->input_rate; fentry++) {
>  		if (fentry->input_rate == pll_ref_freq)
> @@ -2847,10 +2852,10 @@ static int tegra210_enable_pllu(void)
>  	fence_udelay(1, clk_base);
>  	reg |= PLL_ENABLE;
>  	writel(reg, clk_base + PLLU_BASE);
> +	fence_udelay(1, clk_base);
>  
> -	readl_relaxed_poll_timeout_atomic(clk_base + PLLU_BASE, reg,
> -					  reg & PLL_BASE_LOCK, 2, 1000);
> -	if (!(reg & PLL_BASE_LOCK)) {
> +	ret = tegra210_wait_for_mask(&pllu, PLLU_BASE, PLL_BASE_LOCK);
> +	if (ret) {
>  		pr_err("Timed out waiting for PLL_U to lock\n");
>  		return -ETIMEDOUT;
>  	}
> @@ -3283,6 +3288,103 @@ static void tegra210_disable_cpu_clock(u32 cpu)
>  }
>  
>  #ifdef CONFIG_PM_SLEEP
> +static u32 cpu_softrst_ctx[3];
> +static struct platform_device *dfll_pdev;
> +#define car_readl(_base, _off) readl_relaxed(clk_base + (_base) + ((_off) * 4))
> +#define car_writel(_val, _base, _off) \
> +		writel_relaxed(_val, clk_base + (_base) + ((_off) * 4))
> +
> +static int tegra210_clk_suspend(void)
> +{
> +	unsigned int i;
> +	struct device_node *node;
> +
> +	tegra_cclkg_burst_policy_save_context();
> +
> +	if (!dfll_pdev) {
> +		node = of_find_compatible_node(NULL, NULL,
> +					       "nvidia,tegra210-dfll");
> +		if (node)
> +			dfll_pdev = of_find_device_by_node(node);
> +
> +		of_node_put(node);
> +		if (!dfll_pdev)
> +			pr_err("dfll node not found. no suspend for dfll\n");
> +	}
> +
> +	if (dfll_pdev)
> +		tegra_dfll_suspend(dfll_pdev);
> +
> +	/* Enable PLLP_OUT_CPU after dfll suspend */
> +	tegra_clk_set_pllp_out_cpu(true);
> +
> +	tegra_sclk_cclklp_burst_policy_save_context();
> +
> +	clk_save_context();
> +
> +	for (i = 0; i < ARRAY_SIZE(cpu_softrst_ctx); i++)
> +		cpu_softrst_ctx[i] = car_readl(CPU_SOFTRST_CTRL, i);
> +
> +	return 0;
> +}
> +
> +static void tegra210_clk_resume(void)
> +{
> +	unsigned int i;
> +	struct clk_hw *parent;
> +	struct clk *clk;
> +
> +	/*
> +	 * clk_restore_context restores clocks as per the clock tree.
> +	 *
> +	 * dfllCPU_out is first in the clock tree to get restored and it
> +	 * involves programming DFLL controller along with restoring CPUG
> +	 * clock burst policy.
> +	 *
> +	 * DFLL programming needs dfll_ref and dfll_soc peripheral clocks
> +	 * to be restores which are part ofthe peripheral clocks.
> +	 * So, peripheral clocks restore should happen prior to dfll clock
> +	 * restore.
> +	 */
> +
> +	tegra_clk_osc_resume(clk_base);
> +	for (i = 0; i < ARRAY_SIZE(cpu_softrst_ctx); i++)
> +		car_writel(cpu_softrst_ctx[i], CPU_SOFTRST_CTRL, i);
> +
> +	/* restore all plls and peripheral clocks */
> +	tegra210_init_pllu();
> +	clk_restore_context();
> +
> +	fence_udelay(5, clk_base);
> +
> +	/* resume SCLK and CPULP clocks */
> +	tegra_sclk_cpulp_burst_policy_restore_context();
> +
> +	/*
> +	 * restore CPUG clocks:
> +	 * - enable DFLL in open loop mode
> +	 * - switch CPUG to DFLL clock source
> +	 * - close DFLL loop
> +	 * - sync PLLX state
> +	 */
> +	if (dfll_pdev)
> +		tegra_dfll_resume(dfll_pdev, false);
> +
> +	tegra_cclkg_burst_policy_restore_context();
> +	fence_udelay(2, clk_base);
> +
> +	if (dfll_pdev)
> +		tegra_dfll_resume(dfll_pdev, true);
> +
> +	parent = clk_hw_get_parent(__clk_get_hw(clks[TEGRA210_CLK_CCLK_G]));
> +	clk = clks[TEGRA210_CLK_PLL_X];
> +	if (parent != __clk_get_hw(clk))
> +		tegra_clk_sync_state_pll(__clk_get_hw(clk));
> +
> +	/* Disable PLL_OUT_CPU after DFLL resume */
> +	tegra_clk_set_pllp_out_cpu(false);
> +}
> +
>  static void tegra210_cpu_clock_suspend(void)
>  {
>  	/* switch coresite to clk_m, save off original source */
> @@ -3298,6 +3400,11 @@ static void tegra210_cpu_clock_resume(void)
>  }
>  #endif
>  
> +static struct syscore_ops tegra_clk_syscore_ops = {
> +	.suspend = tegra210_clk_suspend,
> +	.resume = tegra210_clk_resume,
> +};
> +
>  static struct tegra_cpu_car_ops tegra210_cpu_car_ops = {
>  	.wait_for_reset	= tegra210_wait_cpu_in_reset,
>  	.disable_clock	= tegra210_disable_cpu_clock,
> @@ -3583,5 +3690,7 @@ static void __init tegra210_clock_init(struct device_node *np)
>  	tegra210_mbist_clk_init();
>  
>  	tegra_cpu_car_ops = &tegra210_cpu_car_ops;
> +
> +	register_syscore_ops(&tegra_clk_syscore_ops);
>  }

Is it really worthwhile to use syscore_ops for suspend/resume given that drivers for
won't resume before the CLK driver anyway? Are there any other options for CLK
suspend/resume?

I'm also not sure whether PM runtime API could be used at all in the context of
syscore_ops ..

Secondly, what about to use generic clk_save_context() / clk_restore_context()
helpers for the suspend-resume? It looks to me that some other essential (and proper)
platform driver (soc/tegra/? PMC?) should suspend-resume the clocks using the generic
CLK Framework API.

28.06.2019 5:12, Sowjanya Komatineni пишет:
> This patch adds system suspend and resume support for Tegra210
> clocks.
> 
> All the CAR controller settings are lost on suspend when core power
> goes off.
> 
> This patch has implementation for saving and restoring all the PLLs
> and clocks context during system suspend and resume to have the
> clocks back to same state for normal operation.
> 
> Acked-by: Thierry Reding <treding@nvidia.com>
> Signed-off-by: Sowjanya Komatineni <skomatineni@nvidia.com>
> ---

I'd also change the commit's title to something more brief and explicit, like "clk:
tegra210: Support suspend-resume".

On 6/29/19 8:10 AM, Dmitry Osipenko wrote:
> 28.06.2019 5:12, Sowjanya Komatineni пишет:
>> This patch adds system suspend and resume support for Tegra210
>> clocks.
>>
>> All the CAR controller settings are lost on suspend when core power
>> goes off.
>>
>> This patch has implementation for saving and restoring all the PLLs
>> and clocks context during system suspend and resume to have the
>> clocks back to same state for normal operation.
>>
>> Acked-by: Thierry Reding <treding@nvidia.com>
>> Signed-off-by: Sowjanya Komatineni <skomatineni@nvidia.com>
>> ---
>>   drivers/clk/tegra/clk-tegra210.c | 115 ++++++++++++++++++++++++++++++++++++++-
>>   drivers/clk/tegra/clk.c          |  14 +++++
>>   drivers/clk/tegra/clk.h          |   1 +
>>   3 files changed, 127 insertions(+), 3 deletions(-)
>>
>> diff --git a/drivers/clk/tegra/clk-tegra210.c b/drivers/clk/tegra/clk-tegra210.c
>> index 1c08c53482a5..1b839544e086 100644
>> --- a/drivers/clk/tegra/clk-tegra210.c
>> +++ b/drivers/clk/tegra/clk-tegra210.c
>> @@ -9,10 +9,12 @@
>>   #include <linux/clkdev.h>
>>   #include <linux/of.h>
>>   #include <linux/of_address.h>
>> +#include <linux/of_platform.h>
>>   #include <linux/delay.h>
>>   #include <linux/export.h>
>>   #include <linux/mutex.h>
>>   #include <linux/clk/tegra.h>
>> +#include <linux/syscore_ops.h>
>>   #include <dt-bindings/clock/tegra210-car.h>
>>   #include <dt-bindings/reset/tegra210-car.h>
>>   #include <linux/iopoll.h>
>> @@ -20,6 +22,7 @@
>>   #include <soc/tegra/pmc.h>
>>   
>>   #include "clk.h"
>> +#include "clk-dfll.h"
>>   #include "clk-id.h"
>>   
>>   /*
>> @@ -225,6 +228,7 @@
>>   
>>   #define CLK_RST_CONTROLLER_RST_DEV_Y_SET 0x2a8
>>   #define CLK_RST_CONTROLLER_RST_DEV_Y_CLR 0x2ac
>> +#define CPU_SOFTRST_CTRL 0x380
>>   
>>   #define LVL2_CLK_GATE_OVRA 0xf8
>>   #define LVL2_CLK_GATE_OVRC 0x3a0
>> @@ -2820,6 +2824,7 @@ static int tegra210_enable_pllu(void)
>>   	struct tegra_clk_pll_freq_table *fentry;
>>   	struct tegra_clk_pll pllu;
>>   	u32 reg;
>> +	int ret;
>>   
>>   	for (fentry = pll_u_freq_table; fentry->input_rate; fentry++) {
>>   		if (fentry->input_rate == pll_ref_freq)
>> @@ -2847,10 +2852,10 @@ static int tegra210_enable_pllu(void)
>>   	fence_udelay(1, clk_base);
>>   	reg |= PLL_ENABLE;
>>   	writel(reg, clk_base + PLLU_BASE);
>> +	fence_udelay(1, clk_base);
>>   
>> -	readl_relaxed_poll_timeout_atomic(clk_base + PLLU_BASE, reg,
>> -					  reg & PLL_BASE_LOCK, 2, 1000);
>> -	if (!(reg & PLL_BASE_LOCK)) {
>> +	ret = tegra210_wait_for_mask(&pllu, PLLU_BASE, PLL_BASE_LOCK);
>> +	if (ret) {
>>   		pr_err("Timed out waiting for PLL_U to lock\n");
>>   		return -ETIMEDOUT;
>>   	}
>> @@ -3283,6 +3288,103 @@ static void tegra210_disable_cpu_clock(u32 cpu)
>>   }
>>   
>>   #ifdef CONFIG_PM_SLEEP
>> +static u32 cpu_softrst_ctx[3];
>> +static struct platform_device *dfll_pdev;
>> +#define car_readl(_base, _off) readl_relaxed(clk_base + (_base) + ((_off) * 4))
>> +#define car_writel(_val, _base, _off) \
>> +		writel_relaxed(_val, clk_base + (_base) + ((_off) * 4))
>> +
>> +static int tegra210_clk_suspend(void)
>> +{
>> +	unsigned int i;
>> +	struct device_node *node;
>> +
>> +	tegra_cclkg_burst_policy_save_context();
>> +
>> +	if (!dfll_pdev) {
>> +		node = of_find_compatible_node(NULL, NULL,
>> +					       "nvidia,tegra210-dfll");
>> +		if (node)
>> +			dfll_pdev = of_find_device_by_node(node);
>> +
>> +		of_node_put(node);
>> +		if (!dfll_pdev)
>> +			pr_err("dfll node not found. no suspend for dfll\n");
>> +	}
>> +
>> +	if (dfll_pdev)
>> +		tegra_dfll_suspend(dfll_pdev);
>> +
>> +	/* Enable PLLP_OUT_CPU after dfll suspend */
>> +	tegra_clk_set_pllp_out_cpu(true);
>> +
>> +	tegra_sclk_cclklp_burst_policy_save_context();
>> +
>> +	clk_save_context();
>> +
>> +	for (i = 0; i < ARRAY_SIZE(cpu_softrst_ctx); i++)
>> +		cpu_softrst_ctx[i] = car_readl(CPU_SOFTRST_CTRL, i);
>> +
>> +	return 0;
>> +}
>> +
>> +static void tegra210_clk_resume(void)
>> +{
>> +	unsigned int i;
>> +	struct clk_hw *parent;
>> +	struct clk *clk;
>> +
>> +	/*
>> +	 * clk_restore_context restores clocks as per the clock tree.
>> +	 *
>> +	 * dfllCPU_out is first in the clock tree to get restored and it
>> +	 * involves programming DFLL controller along with restoring CPUG
>> +	 * clock burst policy.
>> +	 *
>> +	 * DFLL programming needs dfll_ref and dfll_soc peripheral clocks
>> +	 * to be restores which are part ofthe peripheral clocks.
>> +	 * So, peripheral clocks restore should happen prior to dfll clock
>> +	 * restore.
>> +	 */
>> +
>> +	tegra_clk_osc_resume(clk_base);
>> +	for (i = 0; i < ARRAY_SIZE(cpu_softrst_ctx); i++)
>> +		car_writel(cpu_softrst_ctx[i], CPU_SOFTRST_CTRL, i);
>> +
>> +	/* restore all plls and peripheral clocks */
>> +	tegra210_init_pllu();
>> +	clk_restore_context();
>> +
>> +	fence_udelay(5, clk_base);
>> +
>> +	/* resume SCLK and CPULP clocks */
>> +	tegra_sclk_cpulp_burst_policy_restore_context();
>> +
>> +	/*
>> +	 * restore CPUG clocks:
>> +	 * - enable DFLL in open loop mode
>> +	 * - switch CPUG to DFLL clock source
>> +	 * - close DFLL loop
>> +	 * - sync PLLX state
>> +	 */
>> +	if (dfll_pdev)
>> +		tegra_dfll_resume(dfll_pdev, false);
>> +
>> +	tegra_cclkg_burst_policy_restore_context();
>> +	fence_udelay(2, clk_base);
>> +
>> +	if (dfll_pdev)
>> +		tegra_dfll_resume(dfll_pdev, true);
>> +
>> +	parent = clk_hw_get_parent(__clk_get_hw(clks[TEGRA210_CLK_CCLK_G]));
>> +	clk = clks[TEGRA210_CLK_PLL_X];
>> +	if (parent != __clk_get_hw(clk))
>> +		tegra_clk_sync_state_pll(__clk_get_hw(clk));
>> +
>> +	/* Disable PLL_OUT_CPU after DFLL resume */
>> +	tegra_clk_set_pllp_out_cpu(false);
>> +}
>> +
>>   static void tegra210_cpu_clock_suspend(void)
>>   {
>>   	/* switch coresite to clk_m, save off original source */
>> @@ -3298,6 +3400,11 @@ static void tegra210_cpu_clock_resume(void)
>>   }
>>   #endif
>>   
>> +static struct syscore_ops tegra_clk_syscore_ops = {
>> +	.suspend = tegra210_clk_suspend,
>> +	.resume = tegra210_clk_resume,
>> +};
>> +
>>   static struct tegra_cpu_car_ops tegra210_cpu_car_ops = {
>>   	.wait_for_reset	= tegra210_wait_cpu_in_reset,
>>   	.disable_clock	= tegra210_disable_cpu_clock,
>> @@ -3583,5 +3690,7 @@ static void __init tegra210_clock_init(struct device_node *np)
>>   	tegra210_mbist_clk_init();
>>   
>>   	tegra_cpu_car_ops = &tegra210_cpu_car_ops;
>> +
>> +	register_syscore_ops(&tegra_clk_syscore_ops);
>>   }
> Is it really worthwhile to use syscore_ops for suspend/resume given that drivers for
> won't resume before the CLK driver anyway? Are there any other options for CLK
> suspend/resume?
>
> I'm also not sure whether PM runtime API could be used at all in the context of
> syscore_ops ..
>
> Secondly, what about to use generic clk_save_context() / clk_restore_context()
> helpers for the suspend-resume? It looks to me that some other essential (and proper)
> platform driver (soc/tegra/? PMC?) should suspend-resume the clocks using the generic
> CLK Framework API.

Clock resume should happen very early to restore peripheral and cpu 
clocks very early than peripheral drivers resume happens.

this patch series uses clk_save_context and clk_restore_context for 
corresponding divider, pll, pllout.. save and restore context.

But as there is dependency on dfll resume and cpu and pllx clocks 
restore, couldnt use clk_save_context and clk_restore_context for dfll.

So implemented recommended dfll resume sequence in main Tegra210 clock 
driver along with invoking clk_save_context/clk_restore_context where 
all other clocks save/restore happens as per clock tree traversal.

13.07.2019 8:54, Sowjanya Komatineni пишет:
> 
> On 6/29/19 8:10 AM, Dmitry Osipenko wrote:
>> 28.06.2019 5:12, Sowjanya Komatineni пишет:
>>> This patch adds system suspend and resume support for Tegra210
>>> clocks.
>>>
>>> All the CAR controller settings are lost on suspend when core power
>>> goes off.
>>>
>>> This patch has implementation for saving and restoring all the PLLs
>>> and clocks context during system suspend and resume to have the
>>> clocks back to same state for normal operation.
>>>
>>> Acked-by: Thierry Reding <treding@nvidia.com>
>>> Signed-off-by: Sowjanya Komatineni <skomatineni@nvidia.com>
>>> ---
>>>   drivers/clk/tegra/clk-tegra210.c | 115
>>> ++++++++++++++++++++++++++++++++++++++-
>>>   drivers/clk/tegra/clk.c          |  14 +++++
>>>   drivers/clk/tegra/clk.h          |   1 +
>>>   3 files changed, 127 insertions(+), 3 deletions(-)
>>>
>>> diff --git a/drivers/clk/tegra/clk-tegra210.c
>>> b/drivers/clk/tegra/clk-tegra210.c
>>> index 1c08c53482a5..1b839544e086 100644
>>> --- a/drivers/clk/tegra/clk-tegra210.c
>>> +++ b/drivers/clk/tegra/clk-tegra210.c
>>> @@ -9,10 +9,12 @@
>>>   #include <linux/clkdev.h>
>>>   #include <linux/of.h>
>>>   #include <linux/of_address.h>
>>> +#include <linux/of_platform.h>
>>>   #include <linux/delay.h>
>>>   #include <linux/export.h>
>>>   #include <linux/mutex.h>
>>>   #include <linux/clk/tegra.h>
>>> +#include <linux/syscore_ops.h>
>>>   #include <dt-bindings/clock/tegra210-car.h>
>>>   #include <dt-bindings/reset/tegra210-car.h>
>>>   #include <linux/iopoll.h>
>>> @@ -20,6 +22,7 @@
>>>   #include <soc/tegra/pmc.h>
>>>     #include "clk.h"
>>> +#include "clk-dfll.h"
>>>   #include "clk-id.h"
>>>     /*
>>> @@ -225,6 +228,7 @@
>>>     #define CLK_RST_CONTROLLER_RST_DEV_Y_SET 0x2a8
>>>   #define CLK_RST_CONTROLLER_RST_DEV_Y_CLR 0x2ac
>>> +#define CPU_SOFTRST_CTRL 0x380
>>>     #define LVL2_CLK_GATE_OVRA 0xf8
>>>   #define LVL2_CLK_GATE_OVRC 0x3a0
>>> @@ -2820,6 +2824,7 @@ static int tegra210_enable_pllu(void)
>>>       struct tegra_clk_pll_freq_table *fentry;
>>>       struct tegra_clk_pll pllu;
>>>       u32 reg;
>>> +    int ret;
>>>         for (fentry = pll_u_freq_table; fentry->input_rate; fentry++) {
>>>           if (fentry->input_rate == pll_ref_freq)
>>> @@ -2847,10 +2852,10 @@ static int tegra210_enable_pllu(void)
>>>       fence_udelay(1, clk_base);
>>>       reg |= PLL_ENABLE;
>>>       writel(reg, clk_base + PLLU_BASE);
>>> +    fence_udelay(1, clk_base);
>>>   -    readl_relaxed_poll_timeout_atomic(clk_base + PLLU_BASE, reg,
>>> -                      reg & PLL_BASE_LOCK, 2, 1000);
>>> -    if (!(reg & PLL_BASE_LOCK)) {
>>> +    ret = tegra210_wait_for_mask(&pllu, PLLU_BASE, PLL_BASE_LOCK);
>>> +    if (ret) {
>>>           pr_err("Timed out waiting for PLL_U to lock\n");
>>>           return -ETIMEDOUT;
>>>       }
>>> @@ -3283,6 +3288,103 @@ static void tegra210_disable_cpu_clock(u32 cpu)
>>>   }
>>>     #ifdef CONFIG_PM_SLEEP
>>> +static u32 cpu_softrst_ctx[3];
>>> +static struct platform_device *dfll_pdev;
>>> +#define car_readl(_base, _off) readl_relaxed(clk_base + (_base) +
>>> ((_off) * 4))
>>> +#define car_writel(_val, _base, _off) \
>>> +        writel_relaxed(_val, clk_base + (_base) + ((_off) * 4))
>>> +
>>> +static int tegra210_clk_suspend(void)
>>> +{
>>> +    unsigned int i;
>>> +    struct device_node *node;
>>> +
>>> +    tegra_cclkg_burst_policy_save_context();
>>> +
>>> +    if (!dfll_pdev) {
>>> +        node = of_find_compatible_node(NULL, NULL,
>>> +                           "nvidia,tegra210-dfll");
>>> +        if (node)
>>> +            dfll_pdev = of_find_device_by_node(node);
>>> +
>>> +        of_node_put(node);
>>> +        if (!dfll_pdev)
>>> +            pr_err("dfll node not found. no suspend for dfll\n");
>>> +    }
>>> +
>>> +    if (dfll_pdev)
>>> +        tegra_dfll_suspend(dfll_pdev);
>>> +
>>> +    /* Enable PLLP_OUT_CPU after dfll suspend */
>>> +    tegra_clk_set_pllp_out_cpu(true);
>>> +
>>> +    tegra_sclk_cclklp_burst_policy_save_context();
>>> +
>>> +    clk_save_context();
>>> +
>>> +    for (i = 0; i < ARRAY_SIZE(cpu_softrst_ctx); i++)
>>> +        cpu_softrst_ctx[i] = car_readl(CPU_SOFTRST_CTRL, i);
>>> +
>>> +    return 0;
>>> +}
>>> +
>>> +static void tegra210_clk_resume(void)
>>> +{
>>> +    unsigned int i;
>>> +    struct clk_hw *parent;
>>> +    struct clk *clk;
>>> +
>>> +    /*
>>> +     * clk_restore_context restores clocks as per the clock tree.
>>> +     *
>>> +     * dfllCPU_out is first in the clock tree to get restored and it
>>> +     * involves programming DFLL controller along with restoring CPUG
>>> +     * clock burst policy.
>>> +     *
>>> +     * DFLL programming needs dfll_ref and dfll_soc peripheral clocks
>>> +     * to be restores which are part ofthe peripheral clocks.

                                            ^ white-space

Please use spellchecker to avoid typos.

>>> +     * So, peripheral clocks restore should happen prior to dfll clock
>>> +     * restore.
>>> +     */
>>> +
>>> +    tegra_clk_osc_resume(clk_base);
>>> +    for (i = 0; i < ARRAY_SIZE(cpu_softrst_ctx); i++)
>>> +        car_writel(cpu_softrst_ctx[i], CPU_SOFTRST_CTRL, i);
>>> +
>>> +    /* restore all plls and peripheral clocks */
>>> +    tegra210_init_pllu();
>>> +    clk_restore_context();
>>> +
>>> +    fence_udelay(5, clk_base);
>>> +
>>> +    /* resume SCLK and CPULP clocks */
>>> +    tegra_sclk_cpulp_burst_policy_restore_context();
>>> +
>>> +    /*
>>> +     * restore CPUG clocks:
>>> +     * - enable DFLL in open loop mode
>>> +     * - switch CPUG to DFLL clock source
>>> +     * - close DFLL loop
>>> +     * - sync PLLX state
>>> +     */
>>> +    if (dfll_pdev)
>>> +        tegra_dfll_resume(dfll_pdev, false);
>>> +
>>> +    tegra_cclkg_burst_policy_restore_context();
>>> +    fence_udelay(2, clk_base);
>>> +
>>> +    if (dfll_pdev)
>>> +        tegra_dfll_resume(dfll_pdev, true);
>>> +
>>> +    parent =
>>> clk_hw_get_parent(__clk_get_hw(clks[TEGRA210_CLK_CCLK_G]));
>>> +    clk = clks[TEGRA210_CLK_PLL_X];
>>> +    if (parent != __clk_get_hw(clk))
>>> +        tegra_clk_sync_state_pll(__clk_get_hw(clk));
>>> +
>>> +    /* Disable PLL_OUT_CPU after DFLL resume */
>>> +    tegra_clk_set_pllp_out_cpu(false);
>>> +}
>>> +
>>>   static void tegra210_cpu_clock_suspend(void)
>>>   {
>>>       /* switch coresite to clk_m, save off original source */
>>> @@ -3298,6 +3400,11 @@ static void tegra210_cpu_clock_resume(void)
>>>   }
>>>   #endif
>>>   +static struct syscore_ops tegra_clk_syscore_ops = {
>>> +    .suspend = tegra210_clk_suspend,
>>> +    .resume = tegra210_clk_resume,
>>> +};
>>> +
>>>   static struct tegra_cpu_car_ops tegra210_cpu_car_ops = {
>>>       .wait_for_reset    = tegra210_wait_cpu_in_reset,
>>>       .disable_clock    = tegra210_disable_cpu_clock,
>>> @@ -3583,5 +3690,7 @@ static void __init tegra210_clock_init(struct
>>> device_node *np)
>>>       tegra210_mbist_clk_init();
>>>         tegra_cpu_car_ops = &tegra210_cpu_car_ops;
>>> +
>>> +    register_syscore_ops(&tegra_clk_syscore_ops);
>>>   }
>> Is it really worthwhile to use syscore_ops for suspend/resume given
>> that drivers for
>> won't resume before the CLK driver anyway? Are there any other options
>> for CLK
>> suspend/resume?
>>
>> I'm also not sure whether PM runtime API could be used at all in the
>> context of
>> syscore_ops ..
>>
>> Secondly, what about to use generic clk_save_context() /
>> clk_restore_context()
>> helpers for the suspend-resume? It looks to me that some other
>> essential (and proper)
>> platform driver (soc/tegra/? PMC?) should suspend-resume the clocks
>> using the generic
>> CLK Framework API.
> 
> Clock resume should happen very early to restore peripheral and cpu
> clocks very early than peripheral drivers resume happens.

If all peripheral drivers properly requested all of the necessary clocks
and CLK driver was a platform driver, then I guess the probe should have
been naturally ordered. But that's not very achievable with the
currently available infrastructure in the kernel, so I'm not arguing
that the clocks should be explicitly resumed before the users.

> this patch series uses clk_save_context and clk_restore_context for
> corresponding divider, pll, pllout.. save and restore context.

Now I see that indeed this API is utilized in this patch, thank you for
the clarification.

> But as there is dependency on dfll resume and cpu and pllx clocks
> restore, couldnt use clk_save_context and clk_restore_context for dfll.
> 
> So implemented recommended dfll resume sequence in main Tegra210 clock
> driver along with invoking clk_save_context/clk_restore_context where
> all other clocks save/restore happens as per clock tree traversal.

Could you please clarify what part of peripherals clocks is required for
DFLL's restore? Couldn't DFLL driver be changed to avoid that quirkness
and thus to make DFLL driver suspend/resume the clock?

On 7/14/19 2:41 PM, Dmitry Osipenko wrote:
> 13.07.2019 8:54, Sowjanya Komatineni пишет:
>> On 6/29/19 8:10 AM, Dmitry Osipenko wrote:
>>> 28.06.2019 5:12, Sowjanya Komatineni пишет:
>>>> This patch adds system suspend and resume support for Tegra210
>>>> clocks.
>>>>
>>>> All the CAR controller settings are lost on suspend when core power
>>>> goes off.
>>>>
>>>> This patch has implementation for saving and restoring all the PLLs
>>>> and clocks context during system suspend and resume to have the
>>>> clocks back to same state for normal operation.
>>>>
>>>> Acked-by: Thierry Reding <treding@nvidia.com>
>>>> Signed-off-by: Sowjanya Komatineni <skomatineni@nvidia.com>
>>>> ---
>>>>    drivers/clk/tegra/clk-tegra210.c | 115
>>>> ++++++++++++++++++++++++++++++++++++++-
>>>>    drivers/clk/tegra/clk.c          |  14 +++++
>>>>    drivers/clk/tegra/clk.h          |   1 +
>>>>    3 files changed, 127 insertions(+), 3 deletions(-)
>>>>
>>>> diff --git a/drivers/clk/tegra/clk-tegra210.c
>>>> b/drivers/clk/tegra/clk-tegra210.c
>>>> index 1c08c53482a5..1b839544e086 100644
>>>> --- a/drivers/clk/tegra/clk-tegra210.c
>>>> +++ b/drivers/clk/tegra/clk-tegra210.c
>>>> @@ -9,10 +9,12 @@
>>>>    #include <linux/clkdev.h>
>>>>    #include <linux/of.h>
>>>>    #include <linux/of_address.h>
>>>> +#include <linux/of_platform.h>
>>>>    #include <linux/delay.h>
>>>>    #include <linux/export.h>
>>>>    #include <linux/mutex.h>
>>>>    #include <linux/clk/tegra.h>
>>>> +#include <linux/syscore_ops.h>
>>>>    #include <dt-bindings/clock/tegra210-car.h>
>>>>    #include <dt-bindings/reset/tegra210-car.h>
>>>>    #include <linux/iopoll.h>
>>>> @@ -20,6 +22,7 @@
>>>>    #include <soc/tegra/pmc.h>
>>>>      #include "clk.h"
>>>> +#include "clk-dfll.h"
>>>>    #include "clk-id.h"
>>>>      /*
>>>> @@ -225,6 +228,7 @@
>>>>      #define CLK_RST_CONTROLLER_RST_DEV_Y_SET 0x2a8
>>>>    #define CLK_RST_CONTROLLER_RST_DEV_Y_CLR 0x2ac
>>>> +#define CPU_SOFTRST_CTRL 0x380
>>>>      #define LVL2_CLK_GATE_OVRA 0xf8
>>>>    #define LVL2_CLK_GATE_OVRC 0x3a0
>>>> @@ -2820,6 +2824,7 @@ static int tegra210_enable_pllu(void)
>>>>        struct tegra_clk_pll_freq_table *fentry;
>>>>        struct tegra_clk_pll pllu;
>>>>        u32 reg;
>>>> +    int ret;
>>>>          for (fentry = pll_u_freq_table; fentry->input_rate; fentry++) {
>>>>            if (fentry->input_rate == pll_ref_freq)
>>>> @@ -2847,10 +2852,10 @@ static int tegra210_enable_pllu(void)
>>>>        fence_udelay(1, clk_base);
>>>>        reg |= PLL_ENABLE;
>>>>        writel(reg, clk_base + PLLU_BASE);
>>>> +    fence_udelay(1, clk_base);
>>>>    -    readl_relaxed_poll_timeout_atomic(clk_base + PLLU_BASE, reg,
>>>> -                      reg & PLL_BASE_LOCK, 2, 1000);
>>>> -    if (!(reg & PLL_BASE_LOCK)) {
>>>> +    ret = tegra210_wait_for_mask(&pllu, PLLU_BASE, PLL_BASE_LOCK);
>>>> +    if (ret) {
>>>>            pr_err("Timed out waiting for PLL_U to lock\n");
>>>>            return -ETIMEDOUT;
>>>>        }
>>>> @@ -3283,6 +3288,103 @@ static void tegra210_disable_cpu_clock(u32 cpu)
>>>>    }
>>>>      #ifdef CONFIG_PM_SLEEP
>>>> +static u32 cpu_softrst_ctx[3];
>>>> +static struct platform_device *dfll_pdev;
>>>> +#define car_readl(_base, _off) readl_relaxed(clk_base + (_base) +
>>>> ((_off) * 4))
>>>> +#define car_writel(_val, _base, _off) \
>>>> +        writel_relaxed(_val, clk_base + (_base) + ((_off) * 4))
>>>> +
>>>> +static int tegra210_clk_suspend(void)
>>>> +{
>>>> +    unsigned int i;
>>>> +    struct device_node *node;
>>>> +
>>>> +    tegra_cclkg_burst_policy_save_context();
>>>> +
>>>> +    if (!dfll_pdev) {
>>>> +        node = of_find_compatible_node(NULL, NULL,
>>>> +                           "nvidia,tegra210-dfll");
>>>> +        if (node)
>>>> +            dfll_pdev = of_find_device_by_node(node);
>>>> +
>>>> +        of_node_put(node);
>>>> +        if (!dfll_pdev)
>>>> +            pr_err("dfll node not found. no suspend for dfll\n");
>>>> +    }
>>>> +
>>>> +    if (dfll_pdev)
>>>> +        tegra_dfll_suspend(dfll_pdev);
>>>> +
>>>> +    /* Enable PLLP_OUT_CPU after dfll suspend */
>>>> +    tegra_clk_set_pllp_out_cpu(true);
>>>> +
>>>> +    tegra_sclk_cclklp_burst_policy_save_context();
>>>> +
>>>> +    clk_save_context();
>>>> +
>>>> +    for (i = 0; i < ARRAY_SIZE(cpu_softrst_ctx); i++)
>>>> +        cpu_softrst_ctx[i] = car_readl(CPU_SOFTRST_CTRL, i);
>>>> +
>>>> +    return 0;
>>>> +}
>>>> +
>>>> +static void tegra210_clk_resume(void)
>>>> +{
>>>> +    unsigned int i;
>>>> +    struct clk_hw *parent;
>>>> +    struct clk *clk;
>>>> +
>>>> +    /*
>>>> +     * clk_restore_context restores clocks as per the clock tree.
>>>> +     *
>>>> +     * dfllCPU_out is first in the clock tree to get restored and it
>>>> +     * involves programming DFLL controller along with restoring CPUG
>>>> +     * clock burst policy.
>>>> +     *
>>>> +     * DFLL programming needs dfll_ref and dfll_soc peripheral clocks
>>>> +     * to be restores which are part ofthe peripheral clocks.
>                                              ^ white-space
>
> Please use spellchecker to avoid typos.
>
>>>> +     * So, peripheral clocks restore should happen prior to dfll clock
>>>> +     * restore.
>>>> +     */
>>>> +
>>>> +    tegra_clk_osc_resume(clk_base);
>>>> +    for (i = 0; i < ARRAY_SIZE(cpu_softrst_ctx); i++)
>>>> +        car_writel(cpu_softrst_ctx[i], CPU_SOFTRST_CTRL, i);
>>>> +
>>>> +    /* restore all plls and peripheral clocks */
>>>> +    tegra210_init_pllu();
>>>> +    clk_restore_context();
>>>> +
>>>> +    fence_udelay(5, clk_base);
>>>> +
>>>> +    /* resume SCLK and CPULP clocks */
>>>> +    tegra_sclk_cpulp_burst_policy_restore_context();
>>>> +
>>>> +    /*
>>>> +     * restore CPUG clocks:
>>>> +     * - enable DFLL in open loop mode
>>>> +     * - switch CPUG to DFLL clock source
>>>> +     * - close DFLL loop
>>>> +     * - sync PLLX state
>>>> +     */
>>>> +    if (dfll_pdev)
>>>> +        tegra_dfll_resume(dfll_pdev, false);
>>>> +
>>>> +    tegra_cclkg_burst_policy_restore_context();
>>>> +    fence_udelay(2, clk_base);
>>>> +
>>>> +    if (dfll_pdev)
>>>> +        tegra_dfll_resume(dfll_pdev, true);
>>>> +
>>>> +    parent =
>>>> clk_hw_get_parent(__clk_get_hw(clks[TEGRA210_CLK_CCLK_G]));
>>>> +    clk = clks[TEGRA210_CLK_PLL_X];
>>>> +    if (parent != __clk_get_hw(clk))
>>>> +        tegra_clk_sync_state_pll(__clk_get_hw(clk));
>>>> +
>>>> +    /* Disable PLL_OUT_CPU after DFLL resume */
>>>> +    tegra_clk_set_pllp_out_cpu(false);
>>>> +}
>>>> +
>>>>    static void tegra210_cpu_clock_suspend(void)
>>>>    {
>>>>        /* switch coresite to clk_m, save off original source */
>>>> @@ -3298,6 +3400,11 @@ static void tegra210_cpu_clock_resume(void)
>>>>    }
>>>>    #endif
>>>>    +static struct syscore_ops tegra_clk_syscore_ops = {
>>>> +    .suspend = tegra210_clk_suspend,
>>>> +    .resume = tegra210_clk_resume,
>>>> +};
>>>> +
>>>>    static struct tegra_cpu_car_ops tegra210_cpu_car_ops = {
>>>>        .wait_for_reset    = tegra210_wait_cpu_in_reset,
>>>>        .disable_clock    = tegra210_disable_cpu_clock,
>>>> @@ -3583,5 +3690,7 @@ static void __init tegra210_clock_init(struct
>>>> device_node *np)
>>>>        tegra210_mbist_clk_init();
>>>>          tegra_cpu_car_ops = &tegra210_cpu_car_ops;
>>>> +
>>>> +    register_syscore_ops(&tegra_clk_syscore_ops);
>>>>    }
>>> Is it really worthwhile to use syscore_ops for suspend/resume given
>>> that drivers for
>>> won't resume before the CLK driver anyway? Are there any other options
>>> for CLK
>>> suspend/resume?
>>>
>>> I'm also not sure whether PM runtime API could be used at all in the
>>> context of
>>> syscore_ops ..
>>>
>>> Secondly, what about to use generic clk_save_context() /
>>> clk_restore_context()
>>> helpers for the suspend-resume? It looks to me that some other
>>> essential (and proper)
>>> platform driver (soc/tegra/? PMC?) should suspend-resume the clocks
>>> using the generic
>>> CLK Framework API.
>> Clock resume should happen very early to restore peripheral and cpu
>> clocks very early than peripheral drivers resume happens.
> If all peripheral drivers properly requested all of the necessary clocks
> and CLK driver was a platform driver, then I guess the probe should have
> been naturally ordered. But that's not very achievable with the
> currently available infrastructure in the kernel, so I'm not arguing
> that the clocks should be explicitly resumed before the users.
>
>> this patch series uses clk_save_context and clk_restore_context for
>> corresponding divider, pll, pllout.. save and restore context.
> Now I see that indeed this API is utilized in this patch, thank you for
> the clarification.
>
>> But as there is dependency on dfll resume and cpu and pllx clocks
>> restore, couldnt use clk_save_context and clk_restore_context for dfll.
>>
>> So implemented recommended dfll resume sequence in main Tegra210 clock
>> driver along with invoking clk_save_context/clk_restore_context where
>> all other clocks save/restore happens as per clock tree traversal.
> Could you please clarify what part of peripherals clocks is required for
> DFLL's restore? Couldn't DFLL driver be changed to avoid that quirkness
> and thus to make DFLL driver suspend/resume the clock?

DFLL source ref_clk and soc_clk need to be restored prior to dfll.

I see dfllCPU_out parent to CCLK_G first in the clock tree and dfll_ref 
and dfll_soc peripheral clocks are not resumed by the time dfll resume 
happens first.

ref_clk and soc_clk source is from pll_p and clock tree has these 
registered under pll_p which happens later.

tegra210_clock_init registers in order plls, peripheral clocks, 
super_clk init for cclk_g during clock driver probe and dfll probe and 
register happens later.

On 7/15/19 5:35 PM, Sowjanya Komatineni wrote:
>
> On 7/14/19 2:41 PM, Dmitry Osipenko wrote:
>> 13.07.2019 8:54, Sowjanya Komatineni пишет:
>>> On 6/29/19 8:10 AM, Dmitry Osipenko wrote:
>>>> 28.06.2019 5:12, Sowjanya Komatineni пишет:
>>>>> This patch adds system suspend and resume support for Tegra210
>>>>> clocks.
>>>>>
>>>>> All the CAR controller settings are lost on suspend when core power
>>>>> goes off.
>>>>>
>>>>> This patch has implementation for saving and restoring all the PLLs
>>>>> and clocks context during system suspend and resume to have the
>>>>> clocks back to same state for normal operation.
>>>>>
>>>>> Acked-by: Thierry Reding <treding@nvidia.com>
>>>>> Signed-off-by: Sowjanya Komatineni <skomatineni@nvidia.com>
>>>>> ---
>>>>>    drivers/clk/tegra/clk-tegra210.c | 115
>>>>> ++++++++++++++++++++++++++++++++++++++-
>>>>>    drivers/clk/tegra/clk.c          |  14 +++++
>>>>>    drivers/clk/tegra/clk.h          |   1 +
>>>>>    3 files changed, 127 insertions(+), 3 deletions(-)
>>>>>
>>>>> diff --git a/drivers/clk/tegra/clk-tegra210.c
>>>>> b/drivers/clk/tegra/clk-tegra210.c
>>>>> index 1c08c53482a5..1b839544e086 100644
>>>>> --- a/drivers/clk/tegra/clk-tegra210.c
>>>>> +++ b/drivers/clk/tegra/clk-tegra210.c
>>>>> @@ -9,10 +9,12 @@
>>>>>    #include <linux/clkdev.h>
>>>>>    #include <linux/of.h>
>>>>>    #include <linux/of_address.h>
>>>>> +#include <linux/of_platform.h>
>>>>>    #include <linux/delay.h>
>>>>>    #include <linux/export.h>
>>>>>    #include <linux/mutex.h>
>>>>>    #include <linux/clk/tegra.h>
>>>>> +#include <linux/syscore_ops.h>
>>>>>    #include <dt-bindings/clock/tegra210-car.h>
>>>>>    #include <dt-bindings/reset/tegra210-car.h>
>>>>>    #include <linux/iopoll.h>
>>>>> @@ -20,6 +22,7 @@
>>>>>    #include <soc/tegra/pmc.h>
>>>>>      #include "clk.h"
>>>>> +#include "clk-dfll.h"
>>>>>    #include "clk-id.h"
>>>>>      /*
>>>>> @@ -225,6 +228,7 @@
>>>>>      #define CLK_RST_CONTROLLER_RST_DEV_Y_SET 0x2a8
>>>>>    #define CLK_RST_CONTROLLER_RST_DEV_Y_CLR 0x2ac
>>>>> +#define CPU_SOFTRST_CTRL 0x380
>>>>>      #define LVL2_CLK_GATE_OVRA 0xf8
>>>>>    #define LVL2_CLK_GATE_OVRC 0x3a0
>>>>> @@ -2820,6 +2824,7 @@ static int tegra210_enable_pllu(void)
>>>>>        struct tegra_clk_pll_freq_table *fentry;
>>>>>        struct tegra_clk_pll pllu;
>>>>>        u32 reg;
>>>>> +    int ret;
>>>>>          for (fentry = pll_u_freq_table; fentry->input_rate; 
>>>>> fentry++) {
>>>>>            if (fentry->input_rate == pll_ref_freq)
>>>>> @@ -2847,10 +2852,10 @@ static int tegra210_enable_pllu(void)
>>>>>        fence_udelay(1, clk_base);
>>>>>        reg |= PLL_ENABLE;
>>>>>        writel(reg, clk_base + PLLU_BASE);
>>>>> +    fence_udelay(1, clk_base);
>>>>>    -    readl_relaxed_poll_timeout_atomic(clk_base + PLLU_BASE, reg,
>>>>> -                      reg & PLL_BASE_LOCK, 2, 1000);
>>>>> -    if (!(reg & PLL_BASE_LOCK)) {
>>>>> +    ret = tegra210_wait_for_mask(&pllu, PLLU_BASE, PLL_BASE_LOCK);
>>>>> +    if (ret) {
>>>>>            pr_err("Timed out waiting for PLL_U to lock\n");
>>>>>            return -ETIMEDOUT;
>>>>>        }
>>>>> @@ -3283,6 +3288,103 @@ static void tegra210_disable_cpu_clock(u32 
>>>>> cpu)
>>>>>    }
>>>>>      #ifdef CONFIG_PM_SLEEP
>>>>> +static u32 cpu_softrst_ctx[3];
>>>>> +static struct platform_device *dfll_pdev;
>>>>> +#define car_readl(_base, _off) readl_relaxed(clk_base + (_base) +
>>>>> ((_off) * 4))
>>>>> +#define car_writel(_val, _base, _off) \
>>>>> +        writel_relaxed(_val, clk_base + (_base) + ((_off) * 4))
>>>>> +
>>>>> +static int tegra210_clk_suspend(void)
>>>>> +{
>>>>> +    unsigned int i;
>>>>> +    struct device_node *node;
>>>>> +
>>>>> +    tegra_cclkg_burst_policy_save_context();
>>>>> +
>>>>> +    if (!dfll_pdev) {
>>>>> +        node = of_find_compatible_node(NULL, NULL,
>>>>> +                           "nvidia,tegra210-dfll");
>>>>> +        if (node)
>>>>> +            dfll_pdev = of_find_device_by_node(node);
>>>>> +
>>>>> +        of_node_put(node);
>>>>> +        if (!dfll_pdev)
>>>>> +            pr_err("dfll node not found. no suspend for dfll\n");
>>>>> +    }
>>>>> +
>>>>> +    if (dfll_pdev)
>>>>> +        tegra_dfll_suspend(dfll_pdev);
>>>>> +
>>>>> +    /* Enable PLLP_OUT_CPU after dfll suspend */
>>>>> +    tegra_clk_set_pllp_out_cpu(true);
>>>>> +
>>>>> +    tegra_sclk_cclklp_burst_policy_save_context();
>>>>> +
>>>>> +    clk_save_context();
>>>>> +
>>>>> +    for (i = 0; i < ARRAY_SIZE(cpu_softrst_ctx); i++)
>>>>> +        cpu_softrst_ctx[i] = car_readl(CPU_SOFTRST_CTRL, i);
>>>>> +
>>>>> +    return 0;
>>>>> +}
>>>>> +
>>>>> +static void tegra210_clk_resume(void)
>>>>> +{
>>>>> +    unsigned int i;
>>>>> +    struct clk_hw *parent;
>>>>> +    struct clk *clk;
>>>>> +
>>>>> +    /*
>>>>> +     * clk_restore_context restores clocks as per the clock tree.
>>>>> +     *
>>>>> +     * dfllCPU_out is first in the clock tree to get restored and it
>>>>> +     * involves programming DFLL controller along with restoring 
>>>>> CPUG
>>>>> +     * clock burst policy.
>>>>> +     *
>>>>> +     * DFLL programming needs dfll_ref and dfll_soc peripheral 
>>>>> clocks
>>>>> +     * to be restores which are part ofthe peripheral clocks.
>>                                              ^ white-space
>>
>> Please use spellchecker to avoid typos.
>>
>>>>> +     * So, peripheral clocks restore should happen prior to dfll 
>>>>> clock
>>>>> +     * restore.
>>>>> +     */
>>>>> +
>>>>> +    tegra_clk_osc_resume(clk_base);
>>>>> +    for (i = 0; i < ARRAY_SIZE(cpu_softrst_ctx); i++)
>>>>> +        car_writel(cpu_softrst_ctx[i], CPU_SOFTRST_CTRL, i);
>>>>> +
>>>>> +    /* restore all plls and peripheral clocks */
>>>>> +    tegra210_init_pllu();
>>>>> +    clk_restore_context();
>>>>> +
>>>>> +    fence_udelay(5, clk_base);
>>>>> +
>>>>> +    /* resume SCLK and CPULP clocks */
>>>>> +    tegra_sclk_cpulp_burst_policy_restore_context();
>>>>> +
>>>>> +    /*
>>>>> +     * restore CPUG clocks:
>>>>> +     * - enable DFLL in open loop mode
>>>>> +     * - switch CPUG to DFLL clock source
>>>>> +     * - close DFLL loop
>>>>> +     * - sync PLLX state
>>>>> +     */
>>>>> +    if (dfll_pdev)
>>>>> +        tegra_dfll_resume(dfll_pdev, false);
>>>>> +
>>>>> +    tegra_cclkg_burst_policy_restore_context();
>>>>> +    fence_udelay(2, clk_base);
>>>>> +
>>>>> +    if (dfll_pdev)
>>>>> +        tegra_dfll_resume(dfll_pdev, true);
>>>>> +
>>>>> +    parent =
>>>>> clk_hw_get_parent(__clk_get_hw(clks[TEGRA210_CLK_CCLK_G]));
>>>>> +    clk = clks[TEGRA210_CLK_PLL_X];
>>>>> +    if (parent != __clk_get_hw(clk))
>>>>> +        tegra_clk_sync_state_pll(__clk_get_hw(clk));
>>>>> +
>>>>> +    /* Disable PLL_OUT_CPU after DFLL resume */
>>>>> +    tegra_clk_set_pllp_out_cpu(false);
>>>>> +}
>>>>> +
>>>>>    static void tegra210_cpu_clock_suspend(void)
>>>>>    {
>>>>>        /* switch coresite to clk_m, save off original source */
>>>>> @@ -3298,6 +3400,11 @@ static void tegra210_cpu_clock_resume(void)
>>>>>    }
>>>>>    #endif
>>>>>    +static struct syscore_ops tegra_clk_syscore_ops = {
>>>>> +    .suspend = tegra210_clk_suspend,
>>>>> +    .resume = tegra210_clk_resume,
>>>>> +};
>>>>> +
>>>>>    static struct tegra_cpu_car_ops tegra210_cpu_car_ops = {
>>>>>        .wait_for_reset    = tegra210_wait_cpu_in_reset,
>>>>>        .disable_clock    = tegra210_disable_cpu_clock,
>>>>> @@ -3583,5 +3690,7 @@ static void __init tegra210_clock_init(struct
>>>>> device_node *np)
>>>>>        tegra210_mbist_clk_init();
>>>>>          tegra_cpu_car_ops = &tegra210_cpu_car_ops;
>>>>> +
>>>>> +    register_syscore_ops(&tegra_clk_syscore_ops);
>>>>>    }
>>>> Is it really worthwhile to use syscore_ops for suspend/resume given
>>>> that drivers for
>>>> won't resume before the CLK driver anyway? Are there any other options
>>>> for CLK
>>>> suspend/resume?
>>>>
>>>> I'm also not sure whether PM runtime API could be used at all in the
>>>> context of
>>>> syscore_ops ..
>>>>
>>>> Secondly, what about to use generic clk_save_context() /
>>>> clk_restore_context()
>>>> helpers for the suspend-resume? It looks to me that some other
>>>> essential (and proper)
>>>> platform driver (soc/tegra/? PMC?) should suspend-resume the clocks
>>>> using the generic
>>>> CLK Framework API.
>>> Clock resume should happen very early to restore peripheral and cpu
>>> clocks very early than peripheral drivers resume happens.
>> If all peripheral drivers properly requested all of the necessary clocks
>> and CLK driver was a platform driver, then I guess the probe should have
>> been naturally ordered. But that's not very achievable with the
>> currently available infrastructure in the kernel, so I'm not arguing
>> that the clocks should be explicitly resumed before the users.
>>
>>> this patch series uses clk_save_context and clk_restore_context for
>>> corresponding divider, pll, pllout.. save and restore context.
>> Now I see that indeed this API is utilized in this patch, thank you for
>> the clarification.
>>
>>> But as there is dependency on dfll resume and cpu and pllx clocks
>>> restore, couldnt use clk_save_context and clk_restore_context for dfll.
>>>
>>> So implemented recommended dfll resume sequence in main Tegra210 clock
>>> driver along with invoking clk_save_context/clk_restore_context where
>>> all other clocks save/restore happens as per clock tree traversal.
>> Could you please clarify what part of peripherals clocks is required for
>> DFLL's restore? Couldn't DFLL driver be changed to avoid that quirkness
>> and thus to make DFLL driver suspend/resume the clock?
>
> DFLL source ref_clk and soc_clk need to be restored prior to dfll.
>
> I see dfllCPU_out parent to CCLK_G first in the clock tree and 
> dfll_ref and dfll_soc peripheral clocks are not resumed by the time 
> dfll resume happens first.
>
> ref_clk and soc_clk source is from pll_p and clock tree has these 
> registered under pll_p which happens later.
>
> tegra210_clock_init registers in order plls, peripheral clocks, 
> super_clk init for cclk_g during clock driver probe and dfll probe and 
> register happens later.
>
One more thing, CLDVFS peripheral clock enable is also needed to be 
enabled to program DFLL Controller and all peripheral clock context is 
restored only after their PLL sources are restored.

DFLL restore involves dfll source clock resume along with CLDVFS 
periheral clock enable and reset

On 7/15/19 8:00 PM, Sowjanya Komatineni wrote:
>
> On 7/15/19 5:35 PM, Sowjanya Komatineni wrote:
>>
>> On 7/14/19 2:41 PM, Dmitry Osipenko wrote:
>>> 13.07.2019 8:54, Sowjanya Komatineni пишет:
>>>> On 6/29/19 8:10 AM, Dmitry Osipenko wrote:
>>>>> 28.06.2019 5:12, Sowjanya Komatineni пишет:
>>>>>> This patch adds system suspend and resume support for Tegra210
>>>>>> clocks.
>>>>>>
>>>>>> All the CAR controller settings are lost on suspend when core power
>>>>>> goes off.
>>>>>>
>>>>>> This patch has implementation for saving and restoring all the PLLs
>>>>>> and clocks context during system suspend and resume to have the
>>>>>> clocks back to same state for normal operation.
>>>>>>
>>>>>> Acked-by: Thierry Reding <treding@nvidia.com>
>>>>>> Signed-off-by: Sowjanya Komatineni <skomatineni@nvidia.com>
>>>>>> ---
>>>>>>    drivers/clk/tegra/clk-tegra210.c | 115
>>>>>> ++++++++++++++++++++++++++++++++++++++-
>>>>>>    drivers/clk/tegra/clk.c          |  14 +++++
>>>>>>    drivers/clk/tegra/clk.h          |   1 +
>>>>>>    3 files changed, 127 insertions(+), 3 deletions(-)
>>>>>>
>>>>>> diff --git a/drivers/clk/tegra/clk-tegra210.c
>>>>>> b/drivers/clk/tegra/clk-tegra210.c
>>>>>> index 1c08c53482a5..1b839544e086 100644
>>>>>> --- a/drivers/clk/tegra/clk-tegra210.c
>>>>>> +++ b/drivers/clk/tegra/clk-tegra210.c
>>>>>> @@ -9,10 +9,12 @@
>>>>>>    #include <linux/clkdev.h>
>>>>>>    #include <linux/of.h>
>>>>>>    #include <linux/of_address.h>
>>>>>> +#include <linux/of_platform.h>
>>>>>>    #include <linux/delay.h>
>>>>>>    #include <linux/export.h>
>>>>>>    #include <linux/mutex.h>
>>>>>>    #include <linux/clk/tegra.h>
>>>>>> +#include <linux/syscore_ops.h>
>>>>>>    #include <dt-bindings/clock/tegra210-car.h>
>>>>>>    #include <dt-bindings/reset/tegra210-car.h>
>>>>>>    #include <linux/iopoll.h>
>>>>>> @@ -20,6 +22,7 @@
>>>>>>    #include <soc/tegra/pmc.h>
>>>>>>      #include "clk.h"
>>>>>> +#include "clk-dfll.h"
>>>>>>    #include "clk-id.h"
>>>>>>      /*
>>>>>> @@ -225,6 +228,7 @@
>>>>>>      #define CLK_RST_CONTROLLER_RST_DEV_Y_SET 0x2a8
>>>>>>    #define CLK_RST_CONTROLLER_RST_DEV_Y_CLR 0x2ac
>>>>>> +#define CPU_SOFTRST_CTRL 0x380
>>>>>>      #define LVL2_CLK_GATE_OVRA 0xf8
>>>>>>    #define LVL2_CLK_GATE_OVRC 0x3a0
>>>>>> @@ -2820,6 +2824,7 @@ static int tegra210_enable_pllu(void)
>>>>>>        struct tegra_clk_pll_freq_table *fentry;
>>>>>>        struct tegra_clk_pll pllu;
>>>>>>        u32 reg;
>>>>>> +    int ret;
>>>>>>          for (fentry = pll_u_freq_table; fentry->input_rate; 
>>>>>> fentry++) {
>>>>>>            if (fentry->input_rate == pll_ref_freq)
>>>>>> @@ -2847,10 +2852,10 @@ static int tegra210_enable_pllu(void)
>>>>>>        fence_udelay(1, clk_base);
>>>>>>        reg |= PLL_ENABLE;
>>>>>>        writel(reg, clk_base + PLLU_BASE);
>>>>>> +    fence_udelay(1, clk_base);
>>>>>>    -    readl_relaxed_poll_timeout_atomic(clk_base + PLLU_BASE, reg,
>>>>>> -                      reg & PLL_BASE_LOCK, 2, 1000);
>>>>>> -    if (!(reg & PLL_BASE_LOCK)) {
>>>>>> +    ret = tegra210_wait_for_mask(&pllu, PLLU_BASE, PLL_BASE_LOCK);
>>>>>> +    if (ret) {
>>>>>>            pr_err("Timed out waiting for PLL_U to lock\n");
>>>>>>            return -ETIMEDOUT;
>>>>>>        }
>>>>>> @@ -3283,6 +3288,103 @@ static void 
>>>>>> tegra210_disable_cpu_clock(u32 cpu)
>>>>>>    }
>>>>>>      #ifdef CONFIG_PM_SLEEP
>>>>>> +static u32 cpu_softrst_ctx[3];
>>>>>> +static struct platform_device *dfll_pdev;
>>>>>> +#define car_readl(_base, _off) readl_relaxed(clk_base + (_base) +
>>>>>> ((_off) * 4))
>>>>>> +#define car_writel(_val, _base, _off) \
>>>>>> +        writel_relaxed(_val, clk_base + (_base) + ((_off) * 4))
>>>>>> +
>>>>>> +static int tegra210_clk_suspend(void)
>>>>>> +{
>>>>>> +    unsigned int i;
>>>>>> +    struct device_node *node;
>>>>>> +
>>>>>> +    tegra_cclkg_burst_policy_save_context();
>>>>>> +
>>>>>> +    if (!dfll_pdev) {
>>>>>> +        node = of_find_compatible_node(NULL, NULL,
>>>>>> +                           "nvidia,tegra210-dfll");
>>>>>> +        if (node)
>>>>>> +            dfll_pdev = of_find_device_by_node(node);
>>>>>> +
>>>>>> +        of_node_put(node);
>>>>>> +        if (!dfll_pdev)
>>>>>> +            pr_err("dfll node not found. no suspend for dfll\n");
>>>>>> +    }
>>>>>> +
>>>>>> +    if (dfll_pdev)
>>>>>> +        tegra_dfll_suspend(dfll_pdev);
>>>>>> +
>>>>>> +    /* Enable PLLP_OUT_CPU after dfll suspend */
>>>>>> +    tegra_clk_set_pllp_out_cpu(true);
>>>>>> +
>>>>>> +    tegra_sclk_cclklp_burst_policy_save_context();
>>>>>> +
>>>>>> +    clk_save_context();
>>>>>> +
>>>>>> +    for (i = 0; i < ARRAY_SIZE(cpu_softrst_ctx); i++)
>>>>>> +        cpu_softrst_ctx[i] = car_readl(CPU_SOFTRST_CTRL, i);
>>>>>> +
>>>>>> +    return 0;
>>>>>> +}
>>>>>> +
>>>>>> +static void tegra210_clk_resume(void)
>>>>>> +{
>>>>>> +    unsigned int i;
>>>>>> +    struct clk_hw *parent;
>>>>>> +    struct clk *clk;
>>>>>> +
>>>>>> +    /*
>>>>>> +     * clk_restore_context restores clocks as per the clock tree.
>>>>>> +     *
>>>>>> +     * dfllCPU_out is first in the clock tree to get restored 
>>>>>> and it
>>>>>> +     * involves programming DFLL controller along with restoring 
>>>>>> CPUG
>>>>>> +     * clock burst policy.
>>>>>> +     *
>>>>>> +     * DFLL programming needs dfll_ref and dfll_soc peripheral 
>>>>>> clocks
>>>>>> +     * to be restores which are part ofthe peripheral clocks.
>>>                                              ^ white-space
>>>
>>> Please use spellchecker to avoid typos.
>>>
>>>>>> +     * So, peripheral clocks restore should happen prior to dfll 
>>>>>> clock
>>>>>> +     * restore.
>>>>>> +     */
>>>>>> +
>>>>>> +    tegra_clk_osc_resume(clk_base);
>>>>>> +    for (i = 0; i < ARRAY_SIZE(cpu_softrst_ctx); i++)
>>>>>> +        car_writel(cpu_softrst_ctx[i], CPU_SOFTRST_CTRL, i);
>>>>>> +
>>>>>> +    /* restore all plls and peripheral clocks */
>>>>>> +    tegra210_init_pllu();
>>>>>> +    clk_restore_context();
>>>>>> +
>>>>>> +    fence_udelay(5, clk_base);
>>>>>> +
>>>>>> +    /* resume SCLK and CPULP clocks */
>>>>>> +    tegra_sclk_cpulp_burst_policy_restore_context();
>>>>>> +
>>>>>> +    /*
>>>>>> +     * restore CPUG clocks:
>>>>>> +     * - enable DFLL in open loop mode
>>>>>> +     * - switch CPUG to DFLL clock source
>>>>>> +     * - close DFLL loop
>>>>>> +     * - sync PLLX state
>>>>>> +     */
>>>>>> +    if (dfll_pdev)
>>>>>> +        tegra_dfll_resume(dfll_pdev, false);
>>>>>> +
>>>>>> +    tegra_cclkg_burst_policy_restore_context();
>>>>>> +    fence_udelay(2, clk_base);
>>>>>> +
>>>>>> +    if (dfll_pdev)
>>>>>> +        tegra_dfll_resume(dfll_pdev, true);
>>>>>> +
>>>>>> +    parent =
>>>>>> clk_hw_get_parent(__clk_get_hw(clks[TEGRA210_CLK_CCLK_G]));
>>>>>> +    clk = clks[TEGRA210_CLK_PLL_X];
>>>>>> +    if (parent != __clk_get_hw(clk))
>>>>>> +        tegra_clk_sync_state_pll(__clk_get_hw(clk));
>>>>>> +
>>>>>> +    /* Disable PLL_OUT_CPU after DFLL resume */
>>>>>> +    tegra_clk_set_pllp_out_cpu(false);
>>>>>> +}
>>>>>> +
>>>>>>    static void tegra210_cpu_clock_suspend(void)
>>>>>>    {
>>>>>>        /* switch coresite to clk_m, save off original source */
>>>>>> @@ -3298,6 +3400,11 @@ static void tegra210_cpu_clock_resume(void)
>>>>>>    }
>>>>>>    #endif
>>>>>>    +static struct syscore_ops tegra_clk_syscore_ops = {
>>>>>> +    .suspend = tegra210_clk_suspend,
>>>>>> +    .resume = tegra210_clk_resume,
>>>>>> +};
>>>>>> +
>>>>>>    static struct tegra_cpu_car_ops tegra210_cpu_car_ops = {
>>>>>>        .wait_for_reset    = tegra210_wait_cpu_in_reset,
>>>>>>        .disable_clock    = tegra210_disable_cpu_clock,
>>>>>> @@ -3583,5 +3690,7 @@ static void __init tegra210_clock_init(struct
>>>>>> device_node *np)
>>>>>>        tegra210_mbist_clk_init();
>>>>>>          tegra_cpu_car_ops = &tegra210_cpu_car_ops;
>>>>>> +
>>>>>> +    register_syscore_ops(&tegra_clk_syscore_ops);
>>>>>>    }
>>>>> Is it really worthwhile to use syscore_ops for suspend/resume given
>>>>> that drivers for
>>>>> won't resume before the CLK driver anyway? Are there any other 
>>>>> options
>>>>> for CLK
>>>>> suspend/resume?
>>>>>
>>>>> I'm also not sure whether PM runtime API could be used at all in the
>>>>> context of
>>>>> syscore_ops ..
>>>>>
>>>>> Secondly, what about to use generic clk_save_context() /
>>>>> clk_restore_context()
>>>>> helpers for the suspend-resume? It looks to me that some other
>>>>> essential (and proper)
>>>>> platform driver (soc/tegra/? PMC?) should suspend-resume the clocks
>>>>> using the generic
>>>>> CLK Framework API.
>>>> Clock resume should happen very early to restore peripheral and cpu
>>>> clocks very early than peripheral drivers resume happens.
>>> If all peripheral drivers properly requested all of the necessary 
>>> clocks
>>> and CLK driver was a platform driver, then I guess the probe should 
>>> have
>>> been naturally ordered. But that's not very achievable with the
>>> currently available infrastructure in the kernel, so I'm not arguing
>>> that the clocks should be explicitly resumed before the users.
>>>
>>>> this patch series uses clk_save_context and clk_restore_context for
>>>> corresponding divider, pll, pllout.. save and restore context.
>>> Now I see that indeed this API is utilized in this patch, thank you for
>>> the clarification.
>>>
>>>> But as there is dependency on dfll resume and cpu and pllx clocks
>>>> restore, couldnt use clk_save_context and clk_restore_context for 
>>>> dfll.
>>>>
>>>> So implemented recommended dfll resume sequence in main Tegra210 clock
>>>> driver along with invoking clk_save_context/clk_restore_context where
>>>> all other clocks save/restore happens as per clock tree traversal.
>>> Could you please clarify what part of peripherals clocks is required 
>>> for
>>> DFLL's restore? Couldn't DFLL driver be changed to avoid that quirkness
>>> and thus to make DFLL driver suspend/resume the clock?
>>
>> DFLL source ref_clk and soc_clk need to be restored prior to dfll.
>>
>> I see dfllCPU_out parent to CCLK_G first in the clock tree and 
>> dfll_ref and dfll_soc peripheral clocks are not resumed by the time 
>> dfll resume happens first.
>>
>> ref_clk and soc_clk source is from pll_p and clock tree has these 
>> registered under pll_p which happens later.
>>
>> tegra210_clock_init registers in order plls, peripheral clocks, 
>> super_clk init for cclk_g during clock driver probe and dfll probe 
>> and register happens later.
>>
> One more thing, CLDVFS peripheral clock enable is also needed to be 
> enabled to program DFLL Controller and all peripheral clock context is 
> restored only after their PLL sources are restored.
>
> DFLL restore involves dfll source clock resume along with CLDVFS 
> periheral clock enable and reset
>
Will try with dfll_pm_ops instead of dfll suspend/resume in tegra210 
clock driver...

16.07.2019 6:00, Sowjanya Komatineni пишет:
> 
> On 7/15/19 5:35 PM, Sowjanya Komatineni wrote:
>>
>> On 7/14/19 2:41 PM, Dmitry Osipenko wrote:
>>> 13.07.2019 8:54, Sowjanya Komatineni пишет:
>>>> On 6/29/19 8:10 AM, Dmitry Osipenko wrote:
>>>>> 28.06.2019 5:12, Sowjanya Komatineni пишет:
>>>>>> This patch adds system suspend and resume support for Tegra210
>>>>>> clocks.
>>>>>>
>>>>>> All the CAR controller settings are lost on suspend when core power
>>>>>> goes off.
>>>>>>
>>>>>> This patch has implementation for saving and restoring all the PLLs
>>>>>> and clocks context during system suspend and resume to have the
>>>>>> clocks back to same state for normal operation.
>>>>>>
>>>>>> Acked-by: Thierry Reding <treding@nvidia.com>
>>>>>> Signed-off-by: Sowjanya Komatineni <skomatineni@nvidia.com>
>>>>>> ---
>>>>>>    drivers/clk/tegra/clk-tegra210.c | 115
>>>>>> ++++++++++++++++++++++++++++++++++++++-
>>>>>>    drivers/clk/tegra/clk.c          |  14 +++++
>>>>>>    drivers/clk/tegra/clk.h          |   1 +
>>>>>>    3 files changed, 127 insertions(+), 3 deletions(-)
>>>>>>
>>>>>> diff --git a/drivers/clk/tegra/clk-tegra210.c
>>>>>> b/drivers/clk/tegra/clk-tegra210.c
>>>>>> index 1c08c53482a5..1b839544e086 100644
>>>>>> --- a/drivers/clk/tegra/clk-tegra210.c
>>>>>> +++ b/drivers/clk/tegra/clk-tegra210.c
>>>>>> @@ -9,10 +9,12 @@
>>>>>>    #include <linux/clkdev.h>
>>>>>>    #include <linux/of.h>
>>>>>>    #include <linux/of_address.h>
>>>>>> +#include <linux/of_platform.h>
>>>>>>    #include <linux/delay.h>
>>>>>>    #include <linux/export.h>
>>>>>>    #include <linux/mutex.h>
>>>>>>    #include <linux/clk/tegra.h>
>>>>>> +#include <linux/syscore_ops.h>
>>>>>>    #include <dt-bindings/clock/tegra210-car.h>
>>>>>>    #include <dt-bindings/reset/tegra210-car.h>
>>>>>>    #include <linux/iopoll.h>
>>>>>> @@ -20,6 +22,7 @@
>>>>>>    #include <soc/tegra/pmc.h>
>>>>>>      #include "clk.h"
>>>>>> +#include "clk-dfll.h"
>>>>>>    #include "clk-id.h"
>>>>>>      /*
>>>>>> @@ -225,6 +228,7 @@
>>>>>>      #define CLK_RST_CONTROLLER_RST_DEV_Y_SET 0x2a8
>>>>>>    #define CLK_RST_CONTROLLER_RST_DEV_Y_CLR 0x2ac
>>>>>> +#define CPU_SOFTRST_CTRL 0x380
>>>>>>      #define LVL2_CLK_GATE_OVRA 0xf8
>>>>>>    #define LVL2_CLK_GATE_OVRC 0x3a0
>>>>>> @@ -2820,6 +2824,7 @@ static int tegra210_enable_pllu(void)
>>>>>>        struct tegra_clk_pll_freq_table *fentry;
>>>>>>        struct tegra_clk_pll pllu;
>>>>>>        u32 reg;
>>>>>> +    int ret;
>>>>>>          for (fentry = pll_u_freq_table; fentry->input_rate;
>>>>>> fentry++) {
>>>>>>            if (fentry->input_rate == pll_ref_freq)
>>>>>> @@ -2847,10 +2852,10 @@ static int tegra210_enable_pllu(void)
>>>>>>        fence_udelay(1, clk_base);
>>>>>>        reg |= PLL_ENABLE;
>>>>>>        writel(reg, clk_base + PLLU_BASE);
>>>>>> +    fence_udelay(1, clk_base);
>>>>>>    -    readl_relaxed_poll_timeout_atomic(clk_base + PLLU_BASE, reg,
>>>>>> -                      reg & PLL_BASE_LOCK, 2, 1000);
>>>>>> -    if (!(reg & PLL_BASE_LOCK)) {
>>>>>> +    ret = tegra210_wait_for_mask(&pllu, PLLU_BASE, PLL_BASE_LOCK);
>>>>>> +    if (ret) {
>>>>>>            pr_err("Timed out waiting for PLL_U to lock\n");
>>>>>>            return -ETIMEDOUT;
>>>>>>        }
>>>>>> @@ -3283,6 +3288,103 @@ static void tegra210_disable_cpu_clock(u32
>>>>>> cpu)
>>>>>>    }
>>>>>>      #ifdef CONFIG_PM_SLEEP
>>>>>> +static u32 cpu_softrst_ctx[3];
>>>>>> +static struct platform_device *dfll_pdev;
>>>>>> +#define car_readl(_base, _off) readl_relaxed(clk_base + (_base) +
>>>>>> ((_off) * 4))
>>>>>> +#define car_writel(_val, _base, _off) \
>>>>>> +        writel_relaxed(_val, clk_base + (_base) + ((_off) * 4))
>>>>>> +
>>>>>> +static int tegra210_clk_suspend(void)
>>>>>> +{
>>>>>> +    unsigned int i;
>>>>>> +    struct device_node *node;
>>>>>> +
>>>>>> +    tegra_cclkg_burst_policy_save_context();
>>>>>> +
>>>>>> +    if (!dfll_pdev) {
>>>>>> +        node = of_find_compatible_node(NULL, NULL,
>>>>>> +                           "nvidia,tegra210-dfll");
>>>>>> +        if (node)
>>>>>> +            dfll_pdev = of_find_device_by_node(node);
>>>>>> +
>>>>>> +        of_node_put(node);
>>>>>> +        if (!dfll_pdev)
>>>>>> +            pr_err("dfll node not found. no suspend for dfll\n");
>>>>>> +    }
>>>>>> +
>>>>>> +    if (dfll_pdev)
>>>>>> +        tegra_dfll_suspend(dfll_pdev);
>>>>>> +
>>>>>> +    /* Enable PLLP_OUT_CPU after dfll suspend */
>>>>>> +    tegra_clk_set_pllp_out_cpu(true);
>>>>>> +
>>>>>> +    tegra_sclk_cclklp_burst_policy_save_context();
>>>>>> +
>>>>>> +    clk_save_context();
>>>>>> +
>>>>>> +    for (i = 0; i < ARRAY_SIZE(cpu_softrst_ctx); i++)
>>>>>> +        cpu_softrst_ctx[i] = car_readl(CPU_SOFTRST_CTRL, i);
>>>>>> +
>>>>>> +    return 0;
>>>>>> +}
>>>>>> +
>>>>>> +static void tegra210_clk_resume(void)
>>>>>> +{
>>>>>> +    unsigned int i;
>>>>>> +    struct clk_hw *parent;
>>>>>> +    struct clk *clk;
>>>>>> +
>>>>>> +    /*
>>>>>> +     * clk_restore_context restores clocks as per the clock tree.
>>>>>> +     *
>>>>>> +     * dfllCPU_out is first in the clock tree to get restored and it
>>>>>> +     * involves programming DFLL controller along with restoring
>>>>>> CPUG
>>>>>> +     * clock burst policy.
>>>>>> +     *
>>>>>> +     * DFLL programming needs dfll_ref and dfll_soc peripheral
>>>>>> clocks
>>>>>> +     * to be restores which are part ofthe peripheral clocks.
>>>                                              ^ white-space
>>>
>>> Please use spellchecker to avoid typos.
>>>
>>>>>> +     * So, peripheral clocks restore should happen prior to dfll
>>>>>> clock
>>>>>> +     * restore.
>>>>>> +     */
>>>>>> +
>>>>>> +    tegra_clk_osc_resume(clk_base);
>>>>>> +    for (i = 0; i < ARRAY_SIZE(cpu_softrst_ctx); i++)
>>>>>> +        car_writel(cpu_softrst_ctx[i], CPU_SOFTRST_CTRL, i);
>>>>>> +
>>>>>> +    /* restore all plls and peripheral clocks */
>>>>>> +    tegra210_init_pllu();
>>>>>> +    clk_restore_context();
>>>>>> +
>>>>>> +    fence_udelay(5, clk_base);
>>>>>> +
>>>>>> +    /* resume SCLK and CPULP clocks */
>>>>>> +    tegra_sclk_cpulp_burst_policy_restore_context();
>>>>>> +
>>>>>> +    /*
>>>>>> +     * restore CPUG clocks:
>>>>>> +     * - enable DFLL in open loop mode
>>>>>> +     * - switch CPUG to DFLL clock source
>>>>>> +     * - close DFLL loop
>>>>>> +     * - sync PLLX state
>>>>>> +     */
>>>>>> +    if (dfll_pdev)
>>>>>> +        tegra_dfll_resume(dfll_pdev, false);
>>>>>> +
>>>>>> +    tegra_cclkg_burst_policy_restore_context();
>>>>>> +    fence_udelay(2, clk_base);
>>>>>> +
>>>>>> +    if (dfll_pdev)
>>>>>> +        tegra_dfll_resume(dfll_pdev, true);
>>>>>> +
>>>>>> +    parent =
>>>>>> clk_hw_get_parent(__clk_get_hw(clks[TEGRA210_CLK_CCLK_G]));
>>>>>> +    clk = clks[TEGRA210_CLK_PLL_X];
>>>>>> +    if (parent != __clk_get_hw(clk))
>>>>>> +        tegra_clk_sync_state_pll(__clk_get_hw(clk));
>>>>>> +
>>>>>> +    /* Disable PLL_OUT_CPU after DFLL resume */
>>>>>> +    tegra_clk_set_pllp_out_cpu(false);
>>>>>> +}
>>>>>> +
>>>>>>    static void tegra210_cpu_clock_suspend(void)
>>>>>>    {
>>>>>>        /* switch coresite to clk_m, save off original source */
>>>>>> @@ -3298,6 +3400,11 @@ static void tegra210_cpu_clock_resume(void)
>>>>>>    }
>>>>>>    #endif
>>>>>>    +static struct syscore_ops tegra_clk_syscore_ops = {
>>>>>> +    .suspend = tegra210_clk_suspend,
>>>>>> +    .resume = tegra210_clk_resume,
>>>>>> +};
>>>>>> +
>>>>>>    static struct tegra_cpu_car_ops tegra210_cpu_car_ops = {
>>>>>>        .wait_for_reset    = tegra210_wait_cpu_in_reset,
>>>>>>        .disable_clock    = tegra210_disable_cpu_clock,
>>>>>> @@ -3583,5 +3690,7 @@ static void __init tegra210_clock_init(struct
>>>>>> device_node *np)
>>>>>>        tegra210_mbist_clk_init();
>>>>>>          tegra_cpu_car_ops = &tegra210_cpu_car_ops;
>>>>>> +
>>>>>> +    register_syscore_ops(&tegra_clk_syscore_ops);
>>>>>>    }
>>>>> Is it really worthwhile to use syscore_ops for suspend/resume given
>>>>> that drivers for
>>>>> won't resume before the CLK driver anyway? Are there any other options
>>>>> for CLK
>>>>> suspend/resume?
>>>>>
>>>>> I'm also not sure whether PM runtime API could be used at all in the
>>>>> context of
>>>>> syscore_ops ..
>>>>>
>>>>> Secondly, what about to use generic clk_save_context() /
>>>>> clk_restore_context()
>>>>> helpers for the suspend-resume? It looks to me that some other
>>>>> essential (and proper)
>>>>> platform driver (soc/tegra/? PMC?) should suspend-resume the clocks
>>>>> using the generic
>>>>> CLK Framework API.
>>>> Clock resume should happen very early to restore peripheral and cpu
>>>> clocks very early than peripheral drivers resume happens.
>>> If all peripheral drivers properly requested all of the necessary clocks
>>> and CLK driver was a platform driver, then I guess the probe should have
>>> been naturally ordered. But that's not very achievable with the
>>> currently available infrastructure in the kernel, so I'm not arguing
>>> that the clocks should be explicitly resumed before the users.
>>>
>>>> this patch series uses clk_save_context and clk_restore_context for
>>>> corresponding divider, pll, pllout.. save and restore context.
>>> Now I see that indeed this API is utilized in this patch, thank you for
>>> the clarification.
>>>
>>>> But as there is dependency on dfll resume and cpu and pllx clocks
>>>> restore, couldnt use clk_save_context and clk_restore_context for dfll.
>>>>
>>>> So implemented recommended dfll resume sequence in main Tegra210 clock
>>>> driver along with invoking clk_save_context/clk_restore_context where
>>>> all other clocks save/restore happens as per clock tree traversal.
>>> Could you please clarify what part of peripherals clocks is required for
>>> DFLL's restore? Couldn't DFLL driver be changed to avoid that quirkness
>>> and thus to make DFLL driver suspend/resume the clock?
>>
>> DFLL source ref_clk and soc_clk need to be restored prior to dfll.
>>
>> I see dfllCPU_out parent to CCLK_G first in the clock tree and
>> dfll_ref and dfll_soc peripheral clocks are not resumed by the time
>> dfll resume happens first.
>>
>> ref_clk and soc_clk source is from pll_p and clock tree has these
>> registered under pll_p which happens later.
>>
>> tegra210_clock_init registers in order plls, peripheral clocks,
>> super_clk init for cclk_g during clock driver probe and dfll probe and
>> register happens later.
>>
> One more thing, CLDVFS peripheral clock enable is also needed to be
> enabled to program DFLL Controller and all peripheral clock context is
> restored only after their PLL sources are restored.
> 
> DFLL restore involves dfll source clock resume along with CLDVFS
> periheral clock enable and reset
> 

I don't quite see why you can't simply add suspend/resume callbacks to
the CPUFreq driver to:

On suspend:
1. Switch CPU to PLLP (or whatever "safe" parent)
2. Disable/teardown DFLL

On resume:
1. Enable/restore DFLL
2. Switch CPU back to DFLL

On 7/15/19 8:50 PM, Dmitry Osipenko wrote:
> 16.07.2019 6:00, Sowjanya Komatineni пишет:
>> On 7/15/19 5:35 PM, Sowjanya Komatineni wrote:
>>> On 7/14/19 2:41 PM, Dmitry Osipenko wrote:
>>>> 13.07.2019 8:54, Sowjanya Komatineni пишет:
>>>>> On 6/29/19 8:10 AM, Dmitry Osipenko wrote:
>>>>>> 28.06.2019 5:12, Sowjanya Komatineni пишет:
>>>>>>> This patch adds system suspend and resume support for Tegra210
>>>>>>> clocks.
>>>>>>>
>>>>>>> All the CAR controller settings are lost on suspend when core power
>>>>>>> goes off.
>>>>>>>
>>>>>>> This patch has implementation for saving and restoring all the PLLs
>>>>>>> and clocks context during system suspend and resume to have the
>>>>>>> clocks back to same state for normal operation.
>>>>>>>
>>>>>>> Acked-by: Thierry Reding <treding@nvidia.com>
>>>>>>> Signed-off-by: Sowjanya Komatineni <skomatineni@nvidia.com>
>>>>>>> ---
>>>>>>>     drivers/clk/tegra/clk-tegra210.c | 115
>>>>>>> ++++++++++++++++++++++++++++++++++++++-
>>>>>>>     drivers/clk/tegra/clk.c          |  14 +++++
>>>>>>>     drivers/clk/tegra/clk.h          |   1 +
>>>>>>>     3 files changed, 127 insertions(+), 3 deletions(-)
>>>>>>>
>>>>>>> diff --git a/drivers/clk/tegra/clk-tegra210.c
>>>>>>> b/drivers/clk/tegra/clk-tegra210.c
>>>>>>> index 1c08c53482a5..1b839544e086 100644
>>>>>>> --- a/drivers/clk/tegra/clk-tegra210.c
>>>>>>> +++ b/drivers/clk/tegra/clk-tegra210.c
>>>>>>> @@ -9,10 +9,12 @@
>>>>>>>     #include <linux/clkdev.h>
>>>>>>>     #include <linux/of.h>
>>>>>>>     #include <linux/of_address.h>
>>>>>>> +#include <linux/of_platform.h>
>>>>>>>     #include <linux/delay.h>
>>>>>>>     #include <linux/export.h>
>>>>>>>     #include <linux/mutex.h>
>>>>>>>     #include <linux/clk/tegra.h>
>>>>>>> +#include <linux/syscore_ops.h>
>>>>>>>     #include <dt-bindings/clock/tegra210-car.h>
>>>>>>>     #include <dt-bindings/reset/tegra210-car.h>
>>>>>>>     #include <linux/iopoll.h>
>>>>>>> @@ -20,6 +22,7 @@
>>>>>>>     #include <soc/tegra/pmc.h>
>>>>>>>       #include "clk.h"
>>>>>>> +#include "clk-dfll.h"
>>>>>>>     #include "clk-id.h"
>>>>>>>       /*
>>>>>>> @@ -225,6 +228,7 @@
>>>>>>>       #define CLK_RST_CONTROLLER_RST_DEV_Y_SET 0x2a8
>>>>>>>     #define CLK_RST_CONTROLLER_RST_DEV_Y_CLR 0x2ac
>>>>>>> +#define CPU_SOFTRST_CTRL 0x380
>>>>>>>       #define LVL2_CLK_GATE_OVRA 0xf8
>>>>>>>     #define LVL2_CLK_GATE_OVRC 0x3a0
>>>>>>> @@ -2820,6 +2824,7 @@ static int tegra210_enable_pllu(void)
>>>>>>>         struct tegra_clk_pll_freq_table *fentry;
>>>>>>>         struct tegra_clk_pll pllu;
>>>>>>>         u32 reg;
>>>>>>> +    int ret;
>>>>>>>           for (fentry = pll_u_freq_table; fentry->input_rate;
>>>>>>> fentry++) {
>>>>>>>             if (fentry->input_rate == pll_ref_freq)
>>>>>>> @@ -2847,10 +2852,10 @@ static int tegra210_enable_pllu(void)
>>>>>>>         fence_udelay(1, clk_base);
>>>>>>>         reg |= PLL_ENABLE;
>>>>>>>         writel(reg, clk_base + PLLU_BASE);
>>>>>>> +    fence_udelay(1, clk_base);
>>>>>>>     -    readl_relaxed_poll_timeout_atomic(clk_base + PLLU_BASE, reg,
>>>>>>> -                      reg & PLL_BASE_LOCK, 2, 1000);
>>>>>>> -    if (!(reg & PLL_BASE_LOCK)) {
>>>>>>> +    ret = tegra210_wait_for_mask(&pllu, PLLU_BASE, PLL_BASE_LOCK);
>>>>>>> +    if (ret) {
>>>>>>>             pr_err("Timed out waiting for PLL_U to lock\n");
>>>>>>>             return -ETIMEDOUT;
>>>>>>>         }
>>>>>>> @@ -3283,6 +3288,103 @@ static void tegra210_disable_cpu_clock(u32
>>>>>>> cpu)
>>>>>>>     }
>>>>>>>       #ifdef CONFIG_PM_SLEEP
>>>>>>> +static u32 cpu_softrst_ctx[3];
>>>>>>> +static struct platform_device *dfll_pdev;
>>>>>>> +#define car_readl(_base, _off) readl_relaxed(clk_base + (_base) +
>>>>>>> ((_off) * 4))
>>>>>>> +#define car_writel(_val, _base, _off) \
>>>>>>> +        writel_relaxed(_val, clk_base + (_base) + ((_off) * 4))
>>>>>>> +
>>>>>>> +static int tegra210_clk_suspend(void)
>>>>>>> +{
>>>>>>> +    unsigned int i;
>>>>>>> +    struct device_node *node;
>>>>>>> +
>>>>>>> +    tegra_cclkg_burst_policy_save_context();
>>>>>>> +
>>>>>>> +    if (!dfll_pdev) {
>>>>>>> +        node = of_find_compatible_node(NULL, NULL,
>>>>>>> +                           "nvidia,tegra210-dfll");
>>>>>>> +        if (node)
>>>>>>> +            dfll_pdev = of_find_device_by_node(node);
>>>>>>> +
>>>>>>> +        of_node_put(node);
>>>>>>> +        if (!dfll_pdev)
>>>>>>> +            pr_err("dfll node not found. no suspend for dfll\n");
>>>>>>> +    }
>>>>>>> +
>>>>>>> +    if (dfll_pdev)
>>>>>>> +        tegra_dfll_suspend(dfll_pdev);
>>>>>>> +
>>>>>>> +    /* Enable PLLP_OUT_CPU after dfll suspend */
>>>>>>> +    tegra_clk_set_pllp_out_cpu(true);
>>>>>>> +
>>>>>>> +    tegra_sclk_cclklp_burst_policy_save_context();
>>>>>>> +
>>>>>>> +    clk_save_context();
>>>>>>> +
>>>>>>> +    for (i = 0; i < ARRAY_SIZE(cpu_softrst_ctx); i++)
>>>>>>> +        cpu_softrst_ctx[i] = car_readl(CPU_SOFTRST_CTRL, i);
>>>>>>> +
>>>>>>> +    return 0;
>>>>>>> +}
>>>>>>> +
>>>>>>> +static void tegra210_clk_resume(void)
>>>>>>> +{
>>>>>>> +    unsigned int i;
>>>>>>> +    struct clk_hw *parent;
>>>>>>> +    struct clk *clk;
>>>>>>> +
>>>>>>> +    /*
>>>>>>> +     * clk_restore_context restores clocks as per the clock tree.
>>>>>>> +     *
>>>>>>> +     * dfllCPU_out is first in the clock tree to get restored and it
>>>>>>> +     * involves programming DFLL controller along with restoring
>>>>>>> CPUG
>>>>>>> +     * clock burst policy.
>>>>>>> +     *
>>>>>>> +     * DFLL programming needs dfll_ref and dfll_soc peripheral
>>>>>>> clocks
>>>>>>> +     * to be restores which are part ofthe peripheral clocks.
>>>>                                               ^ white-space
>>>>
>>>> Please use spellchecker to avoid typos.
>>>>
>>>>>>> +     * So, peripheral clocks restore should happen prior to dfll
>>>>>>> clock
>>>>>>> +     * restore.
>>>>>>> +     */
>>>>>>> +
>>>>>>> +    tegra_clk_osc_resume(clk_base);
>>>>>>> +    for (i = 0; i < ARRAY_SIZE(cpu_softrst_ctx); i++)
>>>>>>> +        car_writel(cpu_softrst_ctx[i], CPU_SOFTRST_CTRL, i);
>>>>>>> +
>>>>>>> +    /* restore all plls and peripheral clocks */
>>>>>>> +    tegra210_init_pllu();
>>>>>>> +    clk_restore_context();
>>>>>>> +
>>>>>>> +    fence_udelay(5, clk_base);
>>>>>>> +
>>>>>>> +    /* resume SCLK and CPULP clocks */
>>>>>>> +    tegra_sclk_cpulp_burst_policy_restore_context();
>>>>>>> +
>>>>>>> +    /*
>>>>>>> +     * restore CPUG clocks:
>>>>>>> +     * - enable DFLL in open loop mode
>>>>>>> +     * - switch CPUG to DFLL clock source
>>>>>>> +     * - close DFLL loop
>>>>>>> +     * - sync PLLX state
>>>>>>> +     */
>>>>>>> +    if (dfll_pdev)
>>>>>>> +        tegra_dfll_resume(dfll_pdev, false);
>>>>>>> +
>>>>>>> +    tegra_cclkg_burst_policy_restore_context();
>>>>>>> +    fence_udelay(2, clk_base);
>>>>>>> +
>>>>>>> +    if (dfll_pdev)
>>>>>>> +        tegra_dfll_resume(dfll_pdev, true);
>>>>>>> +
>>>>>>> +    parent =
>>>>>>> clk_hw_get_parent(__clk_get_hw(clks[TEGRA210_CLK_CCLK_G]));
>>>>>>> +    clk = clks[TEGRA210_CLK_PLL_X];
>>>>>>> +    if (parent != __clk_get_hw(clk))
>>>>>>> +        tegra_clk_sync_state_pll(__clk_get_hw(clk));
>>>>>>> +
>>>>>>> +    /* Disable PLL_OUT_CPU after DFLL resume */
>>>>>>> +    tegra_clk_set_pllp_out_cpu(false);
>>>>>>> +}
>>>>>>> +
>>>>>>>     static void tegra210_cpu_clock_suspend(void)
>>>>>>>     {
>>>>>>>         /* switch coresite to clk_m, save off original source */
>>>>>>> @@ -3298,6 +3400,11 @@ static void tegra210_cpu_clock_resume(void)
>>>>>>>     }
>>>>>>>     #endif
>>>>>>>     +static struct syscore_ops tegra_clk_syscore_ops = {
>>>>>>> +    .suspend = tegra210_clk_suspend,
>>>>>>> +    .resume = tegra210_clk_resume,
>>>>>>> +};
>>>>>>> +
>>>>>>>     static struct tegra_cpu_car_ops tegra210_cpu_car_ops = {
>>>>>>>         .wait_for_reset    = tegra210_wait_cpu_in_reset,
>>>>>>>         .disable_clock    = tegra210_disable_cpu_clock,
>>>>>>> @@ -3583,5 +3690,7 @@ static void __init tegra210_clock_init(struct
>>>>>>> device_node *np)
>>>>>>>         tegra210_mbist_clk_init();
>>>>>>>           tegra_cpu_car_ops = &tegra210_cpu_car_ops;
>>>>>>> +
>>>>>>> +    register_syscore_ops(&tegra_clk_syscore_ops);
>>>>>>>     }
>>>>>> Is it really worthwhile to use syscore_ops for suspend/resume given
>>>>>> that drivers for
>>>>>> won't resume before the CLK driver anyway? Are there any other options
>>>>>> for CLK
>>>>>> suspend/resume?
>>>>>>
>>>>>> I'm also not sure whether PM runtime API could be used at all in the
>>>>>> context of
>>>>>> syscore_ops ..
>>>>>>
>>>>>> Secondly, what about to use generic clk_save_context() /
>>>>>> clk_restore_context()
>>>>>> helpers for the suspend-resume? It looks to me that some other
>>>>>> essential (and proper)
>>>>>> platform driver (soc/tegra/? PMC?) should suspend-resume the clocks
>>>>>> using the generic
>>>>>> CLK Framework API.
>>>>> Clock resume should happen very early to restore peripheral and cpu
>>>>> clocks very early than peripheral drivers resume happens.
>>>> If all peripheral drivers properly requested all of the necessary clocks
>>>> and CLK driver was a platform driver, then I guess the probe should have
>>>> been naturally ordered. But that's not very achievable with the
>>>> currently available infrastructure in the kernel, so I'm not arguing
>>>> that the clocks should be explicitly resumed before the users.
>>>>
>>>>> this patch series uses clk_save_context and clk_restore_context for
>>>>> corresponding divider, pll, pllout.. save and restore context.
>>>> Now I see that indeed this API is utilized in this patch, thank you for
>>>> the clarification.
>>>>
>>>>> But as there is dependency on dfll resume and cpu and pllx clocks
>>>>> restore, couldnt use clk_save_context and clk_restore_context for dfll.
>>>>>
>>>>> So implemented recommended dfll resume sequence in main Tegra210 clock
>>>>> driver along with invoking clk_save_context/clk_restore_context where
>>>>> all other clocks save/restore happens as per clock tree traversal.
>>>> Could you please clarify what part of peripherals clocks is required for
>>>> DFLL's restore? Couldn't DFLL driver be changed to avoid that quirkness
>>>> and thus to make DFLL driver suspend/resume the clock?
>>> DFLL source ref_clk and soc_clk need to be restored prior to dfll.
>>>
>>> I see dfllCPU_out parent to CCLK_G first in the clock tree and
>>> dfll_ref and dfll_soc peripheral clocks are not resumed by the time
>>> dfll resume happens first.
>>>
>>> ref_clk and soc_clk source is from pll_p and clock tree has these
>>> registered under pll_p which happens later.
>>>
>>> tegra210_clock_init registers in order plls, peripheral clocks,
>>> super_clk init for cclk_g during clock driver probe and dfll probe and
>>> register happens later.
>>>
>> One more thing, CLDVFS peripheral clock enable is also needed to be
>> enabled to program DFLL Controller and all peripheral clock context is
>> restored only after their PLL sources are restored.
>>
>> DFLL restore involves dfll source clock resume along with CLDVFS
>> periheral clock enable and reset
>>
> I don't quite see why you can't simply add suspend/resume callbacks to
> the CPUFreq driver to:
>
> On suspend:
> 1. Switch CPU to PLLP (or whatever "safe" parent)
> 2. Disable/teardown DFLL
>
> On resume:
> 1. Enable/restore DFLL
> 2. Switch CPU back to DFLL

dfll runtime suspend/resume are already part of dfll_pm_ops. Don't we 
want to use it for suspend/resume as well?

currently no APIs are shared b/w clk/tegra driver and CPUFreq driver to 
invoke dfll suspend/resume in CPUFreq driver

В Mon, 15 Jul 2019 21:37:09 -0700
Sowjanya Komatineni <skomatineni@nvidia.com> пишет:

> On 7/15/19 8:50 PM, Dmitry Osipenko wrote:
> > 16.07.2019 6:00, Sowjanya Komatineni пишет:  
> >> On 7/15/19 5:35 PM, Sowjanya Komatineni wrote:  
> >>> On 7/14/19 2:41 PM, Dmitry Osipenko wrote:  
> >>>> 13.07.2019 8:54, Sowjanya Komatineni пишет:  
> >>>>> On 6/29/19 8:10 AM, Dmitry Osipenko wrote:  
> >>>>>> 28.06.2019 5:12, Sowjanya Komatineni пишет:  
> >>>>>>> This patch adds system suspend and resume support for Tegra210
> >>>>>>> clocks.
> >>>>>>>
> >>>>>>> All the CAR controller settings are lost on suspend when core
> >>>>>>> power goes off.
> >>>>>>>
> >>>>>>> This patch has implementation for saving and restoring all
> >>>>>>> the PLLs and clocks context during system suspend and resume
> >>>>>>> to have the clocks back to same state for normal operation.
> >>>>>>>
> >>>>>>> Acked-by: Thierry Reding <treding@nvidia.com>
> >>>>>>> Signed-off-by: Sowjanya Komatineni <skomatineni@nvidia.com>
> >>>>>>> ---
> >>>>>>>     drivers/clk/tegra/clk-tegra210.c | 115
> >>>>>>> ++++++++++++++++++++++++++++++++++++++-
> >>>>>>>     drivers/clk/tegra/clk.c          |  14 +++++
> >>>>>>>     drivers/clk/tegra/clk.h          |   1 +
> >>>>>>>     3 files changed, 127 insertions(+), 3 deletions(-)
> >>>>>>>
> >>>>>>> diff --git a/drivers/clk/tegra/clk-tegra210.c
> >>>>>>> b/drivers/clk/tegra/clk-tegra210.c
> >>>>>>> index 1c08c53482a5..1b839544e086 100644
> >>>>>>> --- a/drivers/clk/tegra/clk-tegra210.c
> >>>>>>> +++ b/drivers/clk/tegra/clk-tegra210.c
> >>>>>>> @@ -9,10 +9,12 @@
> >>>>>>>     #include <linux/clkdev.h>
> >>>>>>>     #include <linux/of.h>
> >>>>>>>     #include <linux/of_address.h>
> >>>>>>> +#include <linux/of_platform.h>
> >>>>>>>     #include <linux/delay.h>
> >>>>>>>     #include <linux/export.h>
> >>>>>>>     #include <linux/mutex.h>
> >>>>>>>     #include <linux/clk/tegra.h>
> >>>>>>> +#include <linux/syscore_ops.h>
> >>>>>>>     #include <dt-bindings/clock/tegra210-car.h>
> >>>>>>>     #include <dt-bindings/reset/tegra210-car.h>
> >>>>>>>     #include <linux/iopoll.h>
> >>>>>>> @@ -20,6 +22,7 @@
> >>>>>>>     #include <soc/tegra/pmc.h>
> >>>>>>>       #include "clk.h"
> >>>>>>> +#include "clk-dfll.h"
> >>>>>>>     #include "clk-id.h"
> >>>>>>>       /*
> >>>>>>> @@ -225,6 +228,7 @@
> >>>>>>>       #define CLK_RST_CONTROLLER_RST_DEV_Y_SET 0x2a8
> >>>>>>>     #define CLK_RST_CONTROLLER_RST_DEV_Y_CLR 0x2ac
> >>>>>>> +#define CPU_SOFTRST_CTRL 0x380
> >>>>>>>       #define LVL2_CLK_GATE_OVRA 0xf8
> >>>>>>>     #define LVL2_CLK_GATE_OVRC 0x3a0
> >>>>>>> @@ -2820,6 +2824,7 @@ static int tegra210_enable_pllu(void)
> >>>>>>>         struct tegra_clk_pll_freq_table *fentry;
> >>>>>>>         struct tegra_clk_pll pllu;
> >>>>>>>         u32 reg;
> >>>>>>> +    int ret;
> >>>>>>>           for (fentry = pll_u_freq_table; fentry->input_rate;
> >>>>>>> fentry++) {
> >>>>>>>             if (fentry->input_rate == pll_ref_freq)
> >>>>>>> @@ -2847,10 +2852,10 @@ static int tegra210_enable_pllu(void)
> >>>>>>>         fence_udelay(1, clk_base);
> >>>>>>>         reg |= PLL_ENABLE;
> >>>>>>>         writel(reg, clk_base + PLLU_BASE);
> >>>>>>> +    fence_udelay(1, clk_base);
> >>>>>>>     -    readl_relaxed_poll_timeout_atomic(clk_base +
> >>>>>>> PLLU_BASE, reg,
> >>>>>>> -                      reg & PLL_BASE_LOCK, 2, 1000);
> >>>>>>> -    if (!(reg & PLL_BASE_LOCK)) {
> >>>>>>> +    ret = tegra210_wait_for_mask(&pllu, PLLU_BASE,
> >>>>>>> PLL_BASE_LOCK);
> >>>>>>> +    if (ret) {
> >>>>>>>             pr_err("Timed out waiting for PLL_U to lock\n");
> >>>>>>>             return -ETIMEDOUT;
> >>>>>>>         }
> >>>>>>> @@ -3283,6 +3288,103 @@ static void
> >>>>>>> tegra210_disable_cpu_clock(u32 cpu)
> >>>>>>>     }
> >>>>>>>       #ifdef CONFIG_PM_SLEEP
> >>>>>>> +static u32 cpu_softrst_ctx[3];
> >>>>>>> +static struct platform_device *dfll_pdev;
> >>>>>>> +#define car_readl(_base, _off) readl_relaxed(clk_base +
> >>>>>>> (_base) + ((_off) * 4))
> >>>>>>> +#define car_writel(_val, _base, _off) \
> >>>>>>> +        writel_relaxed(_val, clk_base + (_base) + ((_off) *
> >>>>>>> 4)) +
> >>>>>>> +static int tegra210_clk_suspend(void)
> >>>>>>> +{
> >>>>>>> +    unsigned int i;
> >>>>>>> +    struct device_node *node;
> >>>>>>> +
> >>>>>>> +    tegra_cclkg_burst_policy_save_context();
> >>>>>>> +
> >>>>>>> +    if (!dfll_pdev) {
> >>>>>>> +        node = of_find_compatible_node(NULL, NULL,
> >>>>>>> +                           "nvidia,tegra210-dfll");
> >>>>>>> +        if (node)
> >>>>>>> +            dfll_pdev = of_find_device_by_node(node);
> >>>>>>> +
> >>>>>>> +        of_node_put(node);
> >>>>>>> +        if (!dfll_pdev)
> >>>>>>> +            pr_err("dfll node not found. no suspend for
> >>>>>>> dfll\n");
> >>>>>>> +    }
> >>>>>>> +
> >>>>>>> +    if (dfll_pdev)
> >>>>>>> +        tegra_dfll_suspend(dfll_pdev);
> >>>>>>> +
> >>>>>>> +    /* Enable PLLP_OUT_CPU after dfll suspend */
> >>>>>>> +    tegra_clk_set_pllp_out_cpu(true);
> >>>>>>> +
> >>>>>>> +    tegra_sclk_cclklp_burst_policy_save_context();
> >>>>>>> +
> >>>>>>> +    clk_save_context();
> >>>>>>> +
> >>>>>>> +    for (i = 0; i < ARRAY_SIZE(cpu_softrst_ctx); i++)
> >>>>>>> +        cpu_softrst_ctx[i] = car_readl(CPU_SOFTRST_CTRL, i);
> >>>>>>> +
> >>>>>>> +    return 0;
> >>>>>>> +}
> >>>>>>> +
> >>>>>>> +static void tegra210_clk_resume(void)
> >>>>>>> +{
> >>>>>>> +    unsigned int i;
> >>>>>>> +    struct clk_hw *parent;
> >>>>>>> +    struct clk *clk;
> >>>>>>> +
> >>>>>>> +    /*
> >>>>>>> +     * clk_restore_context restores clocks as per the clock
> >>>>>>> tree.
> >>>>>>> +     *
> >>>>>>> +     * dfllCPU_out is first in the clock tree to get
> >>>>>>> restored and it
> >>>>>>> +     * involves programming DFLL controller along with
> >>>>>>> restoring CPUG
> >>>>>>> +     * clock burst policy.
> >>>>>>> +     *
> >>>>>>> +     * DFLL programming needs dfll_ref and dfll_soc
> >>>>>>> peripheral clocks
> >>>>>>> +     * to be restores which are part ofthe peripheral
> >>>>>>> clocks.  
> >>>>                                               ^ white-space
> >>>>
> >>>> Please use spellchecker to avoid typos.
> >>>>  
> >>>>>>> +     * So, peripheral clocks restore should happen prior to
> >>>>>>> dfll clock
> >>>>>>> +     * restore.
> >>>>>>> +     */
> >>>>>>> +
> >>>>>>> +    tegra_clk_osc_resume(clk_base);
> >>>>>>> +    for (i = 0; i < ARRAY_SIZE(cpu_softrst_ctx); i++)
> >>>>>>> +        car_writel(cpu_softrst_ctx[i], CPU_SOFTRST_CTRL, i);
> >>>>>>> +
> >>>>>>> +    /* restore all plls and peripheral clocks */
> >>>>>>> +    tegra210_init_pllu();
> >>>>>>> +    clk_restore_context();
> >>>>>>> +
> >>>>>>> +    fence_udelay(5, clk_base);
> >>>>>>> +
> >>>>>>> +    /* resume SCLK and CPULP clocks */
> >>>>>>> +    tegra_sclk_cpulp_burst_policy_restore_context();
> >>>>>>> +
> >>>>>>> +    /*
> >>>>>>> +     * restore CPUG clocks:
> >>>>>>> +     * - enable DFLL in open loop mode
> >>>>>>> +     * - switch CPUG to DFLL clock source
> >>>>>>> +     * - close DFLL loop
> >>>>>>> +     * - sync PLLX state
> >>>>>>> +     */
> >>>>>>> +    if (dfll_pdev)
> >>>>>>> +        tegra_dfll_resume(dfll_pdev, false);
> >>>>>>> +
> >>>>>>> +    tegra_cclkg_burst_policy_restore_context();
> >>>>>>> +    fence_udelay(2, clk_base);
> >>>>>>> +
> >>>>>>> +    if (dfll_pdev)
> >>>>>>> +        tegra_dfll_resume(dfll_pdev, true);
> >>>>>>> +
> >>>>>>> +    parent =
> >>>>>>> clk_hw_get_parent(__clk_get_hw(clks[TEGRA210_CLK_CCLK_G]));
> >>>>>>> +    clk = clks[TEGRA210_CLK_PLL_X];
> >>>>>>> +    if (parent != __clk_get_hw(clk))
> >>>>>>> +        tegra_clk_sync_state_pll(__clk_get_hw(clk));
> >>>>>>> +
> >>>>>>> +    /* Disable PLL_OUT_CPU after DFLL resume */
> >>>>>>> +    tegra_clk_set_pllp_out_cpu(false);
> >>>>>>> +}
> >>>>>>> +
> >>>>>>>     static void tegra210_cpu_clock_suspend(void)
> >>>>>>>     {
> >>>>>>>         /* switch coresite to clk_m, save off original source
> >>>>>>> */ @@ -3298,6 +3400,11 @@ static void
> >>>>>>> tegra210_cpu_clock_resume(void) }
> >>>>>>>     #endif
> >>>>>>>     +static struct syscore_ops tegra_clk_syscore_ops = {
> >>>>>>> +    .suspend = tegra210_clk_suspend,
> >>>>>>> +    .resume = tegra210_clk_resume,
> >>>>>>> +};
> >>>>>>> +
> >>>>>>>     static struct tegra_cpu_car_ops tegra210_cpu_car_ops = {
> >>>>>>>         .wait_for_reset    = tegra210_wait_cpu_in_reset,
> >>>>>>>         .disable_clock    = tegra210_disable_cpu_clock,
> >>>>>>> @@ -3583,5 +3690,7 @@ static void __init
> >>>>>>> tegra210_clock_init(struct device_node *np)
> >>>>>>>         tegra210_mbist_clk_init();
> >>>>>>>           tegra_cpu_car_ops = &tegra210_cpu_car_ops;
> >>>>>>> +
> >>>>>>> +    register_syscore_ops(&tegra_clk_syscore_ops);
> >>>>>>>     }  
> >>>>>> Is it really worthwhile to use syscore_ops for suspend/resume
> >>>>>> given that drivers for
> >>>>>> won't resume before the CLK driver anyway? Are there any other
> >>>>>> options for CLK
> >>>>>> suspend/resume?
> >>>>>>
> >>>>>> I'm also not sure whether PM runtime API could be used at all
> >>>>>> in the context of
> >>>>>> syscore_ops ..
> >>>>>>
> >>>>>> Secondly, what about to use generic clk_save_context() /
> >>>>>> clk_restore_context()
> >>>>>> helpers for the suspend-resume? It looks to me that some other
> >>>>>> essential (and proper)
> >>>>>> platform driver (soc/tegra/? PMC?) should suspend-resume the
> >>>>>> clocks using the generic
> >>>>>> CLK Framework API.  
> >>>>> Clock resume should happen very early to restore peripheral and
> >>>>> cpu clocks very early than peripheral drivers resume happens.  
> >>>> If all peripheral drivers properly requested all of the
> >>>> necessary clocks and CLK driver was a platform driver, then I
> >>>> guess the probe should have been naturally ordered. But that's
> >>>> not very achievable with the currently available infrastructure
> >>>> in the kernel, so I'm not arguing that the clocks should be
> >>>> explicitly resumed before the users. 
> >>>>> this patch series uses clk_save_context and clk_restore_context
> >>>>> for corresponding divider, pll, pllout.. save and restore
> >>>>> context.  
> >>>> Now I see that indeed this API is utilized in this patch, thank
> >>>> you for the clarification.
> >>>>  
> >>>>> But as there is dependency on dfll resume and cpu and pllx
> >>>>> clocks restore, couldnt use clk_save_context and
> >>>>> clk_restore_context for dfll.
> >>>>>
> >>>>> So implemented recommended dfll resume sequence in main
> >>>>> Tegra210 clock driver along with invoking
> >>>>> clk_save_context/clk_restore_context where all other clocks
> >>>>> save/restore happens as per clock tree traversal.  
> >>>> Could you please clarify what part of peripherals clocks is
> >>>> required for DFLL's restore? Couldn't DFLL driver be changed to
> >>>> avoid that quirkness and thus to make DFLL driver suspend/resume
> >>>> the clock?  
> >>> DFLL source ref_clk and soc_clk need to be restored prior to dfll.
> >>>
> >>> I see dfllCPU_out parent to CCLK_G first in the clock tree and
> >>> dfll_ref and dfll_soc peripheral clocks are not resumed by the
> >>> time dfll resume happens first.
> >>>
> >>> ref_clk and soc_clk source is from pll_p and clock tree has these
> >>> registered under pll_p which happens later.
> >>>
> >>> tegra210_clock_init registers in order plls, peripheral clocks,
> >>> super_clk init for cclk_g during clock driver probe and dfll
> >>> probe and register happens later.
> >>>  
> >> One more thing, CLDVFS peripheral clock enable is also needed to be
> >> enabled to program DFLL Controller and all peripheral clock
> >> context is restored only after their PLL sources are restored.
> >>
> >> DFLL restore involves dfll source clock resume along with CLDVFS
> >> periheral clock enable and reset
> >>  
> > I don't quite see why you can't simply add suspend/resume callbacks
> > to the CPUFreq driver to:
> >
> > On suspend:
> > 1. Switch CPU to PLLP (or whatever "safe" parent)
> > 2. Disable/teardown DFLL
> >
> > On resume:
> > 1. Enable/restore DFLL
> > 2. Switch CPU back to DFLL  
> 
> dfll runtime suspend/resume are already part of dfll_pm_ops. Don't we 
> want to use it for suspend/resume as well?

Looks like no. Seems runtime PM of that driver is intended solely for
the DFLL's clk management.

> currently no APIs are shared b/w clk/tegra driver and CPUFreq driver
> to invoke dfll suspend/resume in CPUFreq driver
> 

Just add it. Also, please note that CPUFreq driver is optional and thus
you may need to switch CPU to a safe parent on clk-core suspend as
well in order to resume properly if CPU was running off unsafe parent
during boot and CPUFreq driver is disabled in kernel build (or failed
to load).

The other thing that also need attention is that T124 CPUFreq driver
implicitly relies on DFLL driver to be probed first, which is icky.

В Tue, 16 Jul 2019 08:37:01 +0300
Dmitry Osipenko <digetx@gmail.com> пишет:

> В Mon, 15 Jul 2019 21:37:09 -0700
> Sowjanya Komatineni <skomatineni@nvidia.com> пишет:
> 
> > On 7/15/19 8:50 PM, Dmitry Osipenko wrote:  
> > > 16.07.2019 6:00, Sowjanya Komatineni пишет:    
> > >> On 7/15/19 5:35 PM, Sowjanya Komatineni wrote:    
> > >>> On 7/14/19 2:41 PM, Dmitry Osipenko wrote:    
> > >>>> 13.07.2019 8:54, Sowjanya Komatineni пишет:    
> > >>>>> On 6/29/19 8:10 AM, Dmitry Osipenko wrote:    
> > >>>>>> 28.06.2019 5:12, Sowjanya Komatineni пишет:    
> > >>>>>>> This patch adds system suspend and resume support for
> > >>>>>>> Tegra210 clocks.
> > >>>>>>>
> > >>>>>>> All the CAR controller settings are lost on suspend when
> > >>>>>>> core power goes off.
> > >>>>>>>
> > >>>>>>> This patch has implementation for saving and restoring all
> > >>>>>>> the PLLs and clocks context during system suspend and resume
> > >>>>>>> to have the clocks back to same state for normal operation.
> > >>>>>>>
> > >>>>>>> Acked-by: Thierry Reding <treding@nvidia.com>
> > >>>>>>> Signed-off-by: Sowjanya Komatineni <skomatineni@nvidia.com>
> > >>>>>>> ---
> > >>>>>>>     drivers/clk/tegra/clk-tegra210.c | 115
> > >>>>>>> ++++++++++++++++++++++++++++++++++++++-
> > >>>>>>>     drivers/clk/tegra/clk.c          |  14 +++++
> > >>>>>>>     drivers/clk/tegra/clk.h          |   1 +
> > >>>>>>>     3 files changed, 127 insertions(+), 3 deletions(-)
> > >>>>>>>
> > >>>>>>> diff --git a/drivers/clk/tegra/clk-tegra210.c
> > >>>>>>> b/drivers/clk/tegra/clk-tegra210.c
> > >>>>>>> index 1c08c53482a5..1b839544e086 100644
> > >>>>>>> --- a/drivers/clk/tegra/clk-tegra210.c
> > >>>>>>> +++ b/drivers/clk/tegra/clk-tegra210.c
> > >>>>>>> @@ -9,10 +9,12 @@
> > >>>>>>>     #include <linux/clkdev.h>
> > >>>>>>>     #include <linux/of.h>
> > >>>>>>>     #include <linux/of_address.h>
> > >>>>>>> +#include <linux/of_platform.h>
> > >>>>>>>     #include <linux/delay.h>
> > >>>>>>>     #include <linux/export.h>
> > >>>>>>>     #include <linux/mutex.h>
> > >>>>>>>     #include <linux/clk/tegra.h>
> > >>>>>>> +#include <linux/syscore_ops.h>
> > >>>>>>>     #include <dt-bindings/clock/tegra210-car.h>
> > >>>>>>>     #include <dt-bindings/reset/tegra210-car.h>
> > >>>>>>>     #include <linux/iopoll.h>
> > >>>>>>> @@ -20,6 +22,7 @@
> > >>>>>>>     #include <soc/tegra/pmc.h>
> > >>>>>>>       #include "clk.h"
> > >>>>>>> +#include "clk-dfll.h"
> > >>>>>>>     #include "clk-id.h"
> > >>>>>>>       /*
> > >>>>>>> @@ -225,6 +228,7 @@
> > >>>>>>>       #define CLK_RST_CONTROLLER_RST_DEV_Y_SET 0x2a8
> > >>>>>>>     #define CLK_RST_CONTROLLER_RST_DEV_Y_CLR 0x2ac
> > >>>>>>> +#define CPU_SOFTRST_CTRL 0x380
> > >>>>>>>       #define LVL2_CLK_GATE_OVRA 0xf8
> > >>>>>>>     #define LVL2_CLK_GATE_OVRC 0x3a0
> > >>>>>>> @@ -2820,6 +2824,7 @@ static int tegra210_enable_pllu(void)
> > >>>>>>>         struct tegra_clk_pll_freq_table *fentry;
> > >>>>>>>         struct tegra_clk_pll pllu;
> > >>>>>>>         u32 reg;
> > >>>>>>> +    int ret;
> > >>>>>>>           for (fentry = pll_u_freq_table;
> > >>>>>>> fentry->input_rate; fentry++) {
> > >>>>>>>             if (fentry->input_rate == pll_ref_freq)
> > >>>>>>> @@ -2847,10 +2852,10 @@ static int
> > >>>>>>> tegra210_enable_pllu(void) fence_udelay(1, clk_base);
> > >>>>>>>         reg |= PLL_ENABLE;
> > >>>>>>>         writel(reg, clk_base + PLLU_BASE);
> > >>>>>>> +    fence_udelay(1, clk_base);
> > >>>>>>>     -    readl_relaxed_poll_timeout_atomic(clk_base +
> > >>>>>>> PLLU_BASE, reg,
> > >>>>>>> -                      reg & PLL_BASE_LOCK, 2, 1000);
> > >>>>>>> -    if (!(reg & PLL_BASE_LOCK)) {
> > >>>>>>> +    ret = tegra210_wait_for_mask(&pllu, PLLU_BASE,
> > >>>>>>> PLL_BASE_LOCK);
> > >>>>>>> +    if (ret) {
> > >>>>>>>             pr_err("Timed out waiting for PLL_U to lock\n");
> > >>>>>>>             return -ETIMEDOUT;
> > >>>>>>>         }
> > >>>>>>> @@ -3283,6 +3288,103 @@ static void
> > >>>>>>> tegra210_disable_cpu_clock(u32 cpu)
> > >>>>>>>     }
> > >>>>>>>       #ifdef CONFIG_PM_SLEEP
> > >>>>>>> +static u32 cpu_softrst_ctx[3];
> > >>>>>>> +static struct platform_device *dfll_pdev;
> > >>>>>>> +#define car_readl(_base, _off) readl_relaxed(clk_base +
> > >>>>>>> (_base) + ((_off) * 4))
> > >>>>>>> +#define car_writel(_val, _base, _off) \
> > >>>>>>> +        writel_relaxed(_val, clk_base + (_base) + ((_off) *
> > >>>>>>> 4)) +
> > >>>>>>> +static int tegra210_clk_suspend(void)
> > >>>>>>> +{
> > >>>>>>> +    unsigned int i;
> > >>>>>>> +    struct device_node *node;
> > >>>>>>> +
> > >>>>>>> +    tegra_cclkg_burst_policy_save_context();
> > >>>>>>> +
> > >>>>>>> +    if (!dfll_pdev) {
> > >>>>>>> +        node = of_find_compatible_node(NULL, NULL,
> > >>>>>>> +                           "nvidia,tegra210-dfll");
> > >>>>>>> +        if (node)
> > >>>>>>> +            dfll_pdev = of_find_device_by_node(node);
> > >>>>>>> +
> > >>>>>>> +        of_node_put(node);
> > >>>>>>> +        if (!dfll_pdev)
> > >>>>>>> +            pr_err("dfll node not found. no suspend for
> > >>>>>>> dfll\n");
> > >>>>>>> +    }
> > >>>>>>> +
> > >>>>>>> +    if (dfll_pdev)
> > >>>>>>> +        tegra_dfll_suspend(dfll_pdev);
> > >>>>>>> +
> > >>>>>>> +    /* Enable PLLP_OUT_CPU after dfll suspend */
> > >>>>>>> +    tegra_clk_set_pllp_out_cpu(true);
> > >>>>>>> +
> > >>>>>>> +    tegra_sclk_cclklp_burst_policy_save_context();
> > >>>>>>> +
> > >>>>>>> +    clk_save_context();
> > >>>>>>> +
> > >>>>>>> +    for (i = 0; i < ARRAY_SIZE(cpu_softrst_ctx); i++)
> > >>>>>>> +        cpu_softrst_ctx[i] = car_readl(CPU_SOFTRST_CTRL,
> > >>>>>>> i); +
> > >>>>>>> +    return 0;
> > >>>>>>> +}
> > >>>>>>> +
> > >>>>>>> +static void tegra210_clk_resume(void)
> > >>>>>>> +{
> > >>>>>>> +    unsigned int i;
> > >>>>>>> +    struct clk_hw *parent;
> > >>>>>>> +    struct clk *clk;
> > >>>>>>> +
> > >>>>>>> +    /*
> > >>>>>>> +     * clk_restore_context restores clocks as per the clock
> > >>>>>>> tree.
> > >>>>>>> +     *
> > >>>>>>> +     * dfllCPU_out is first in the clock tree to get
> > >>>>>>> restored and it
> > >>>>>>> +     * involves programming DFLL controller along with
> > >>>>>>> restoring CPUG
> > >>>>>>> +     * clock burst policy.
> > >>>>>>> +     *
> > >>>>>>> +     * DFLL programming needs dfll_ref and dfll_soc
> > >>>>>>> peripheral clocks
> > >>>>>>> +     * to be restores which are part ofthe peripheral
> > >>>>>>> clocks.    
> > >>>>                                               ^ white-space
> > >>>>
> > >>>> Please use spellchecker to avoid typos.
> > >>>>    
> > >>>>>>> +     * So, peripheral clocks restore should happen prior to
> > >>>>>>> dfll clock
> > >>>>>>> +     * restore.
> > >>>>>>> +     */
> > >>>>>>> +
> > >>>>>>> +    tegra_clk_osc_resume(clk_base);
> > >>>>>>> +    for (i = 0; i < ARRAY_SIZE(cpu_softrst_ctx); i++)
> > >>>>>>> +        car_writel(cpu_softrst_ctx[i], CPU_SOFTRST_CTRL,
> > >>>>>>> i); +
> > >>>>>>> +    /* restore all plls and peripheral clocks */
> > >>>>>>> +    tegra210_init_pllu();
> > >>>>>>> +    clk_restore_context();
> > >>>>>>> +
> > >>>>>>> +    fence_udelay(5, clk_base);
> > >>>>>>> +
> > >>>>>>> +    /* resume SCLK and CPULP clocks */
> > >>>>>>> +    tegra_sclk_cpulp_burst_policy_restore_context();
> > >>>>>>> +
> > >>>>>>> +    /*
> > >>>>>>> +     * restore CPUG clocks:
> > >>>>>>> +     * - enable DFLL in open loop mode
> > >>>>>>> +     * - switch CPUG to DFLL clock source
> > >>>>>>> +     * - close DFLL loop
> > >>>>>>> +     * - sync PLLX state
> > >>>>>>> +     */
> > >>>>>>> +    if (dfll_pdev)
> > >>>>>>> +        tegra_dfll_resume(dfll_pdev, false);
> > >>>>>>> +
> > >>>>>>> +    tegra_cclkg_burst_policy_restore_context();
> > >>>>>>> +    fence_udelay(2, clk_base);
> > >>>>>>> +
> > >>>>>>> +    if (dfll_pdev)
> > >>>>>>> +        tegra_dfll_resume(dfll_pdev, true);
> > >>>>>>> +
> > >>>>>>> +    parent =
> > >>>>>>> clk_hw_get_parent(__clk_get_hw(clks[TEGRA210_CLK_CCLK_G]));
> > >>>>>>> +    clk = clks[TEGRA210_CLK_PLL_X];
> > >>>>>>> +    if (parent != __clk_get_hw(clk))
> > >>>>>>> +        tegra_clk_sync_state_pll(__clk_get_hw(clk));
> > >>>>>>> +
> > >>>>>>> +    /* Disable PLL_OUT_CPU after DFLL resume */
> > >>>>>>> +    tegra_clk_set_pllp_out_cpu(false);
> > >>>>>>> +}
> > >>>>>>> +
> > >>>>>>>     static void tegra210_cpu_clock_suspend(void)
> > >>>>>>>     {
> > >>>>>>>         /* switch coresite to clk_m, save off original
> > >>>>>>> source */ @@ -3298,6 +3400,11 @@ static void
> > >>>>>>> tegra210_cpu_clock_resume(void) }
> > >>>>>>>     #endif
> > >>>>>>>     +static struct syscore_ops tegra_clk_syscore_ops = {
> > >>>>>>> +    .suspend = tegra210_clk_suspend,
> > >>>>>>> +    .resume = tegra210_clk_resume,
> > >>>>>>> +};
> > >>>>>>> +
> > >>>>>>>     static struct tegra_cpu_car_ops tegra210_cpu_car_ops = {
> > >>>>>>>         .wait_for_reset    = tegra210_wait_cpu_in_reset,
> > >>>>>>>         .disable_clock    = tegra210_disable_cpu_clock,
> > >>>>>>> @@ -3583,5 +3690,7 @@ static void __init
> > >>>>>>> tegra210_clock_init(struct device_node *np)
> > >>>>>>>         tegra210_mbist_clk_init();
> > >>>>>>>           tegra_cpu_car_ops = &tegra210_cpu_car_ops;
> > >>>>>>> +
> > >>>>>>> +    register_syscore_ops(&tegra_clk_syscore_ops);
> > >>>>>>>     }    
> > >>>>>> Is it really worthwhile to use syscore_ops for suspend/resume
> > >>>>>> given that drivers for
> > >>>>>> won't resume before the CLK driver anyway? Are there any
> > >>>>>> other options for CLK
> > >>>>>> suspend/resume?
> > >>>>>>
> > >>>>>> I'm also not sure whether PM runtime API could be used at all
> > >>>>>> in the context of
> > >>>>>> syscore_ops ..
> > >>>>>>
> > >>>>>> Secondly, what about to use generic clk_save_context() /
> > >>>>>> clk_restore_context()
> > >>>>>> helpers for the suspend-resume? It looks to me that some
> > >>>>>> other essential (and proper)
> > >>>>>> platform driver (soc/tegra/? PMC?) should suspend-resume the
> > >>>>>> clocks using the generic
> > >>>>>> CLK Framework API.    
> > >>>>> Clock resume should happen very early to restore peripheral
> > >>>>> and cpu clocks very early than peripheral drivers resume
> > >>>>> happens.    
> > >>>> If all peripheral drivers properly requested all of the
> > >>>> necessary clocks and CLK driver was a platform driver, then I
> > >>>> guess the probe should have been naturally ordered. But that's
> > >>>> not very achievable with the currently available infrastructure
> > >>>> in the kernel, so I'm not arguing that the clocks should be
> > >>>> explicitly resumed before the users.   
> > >>>>> this patch series uses clk_save_context and
> > >>>>> clk_restore_context for corresponding divider, pll, pllout..
> > >>>>> save and restore context.    
> > >>>> Now I see that indeed this API is utilized in this patch, thank
> > >>>> you for the clarification.
> > >>>>    
> > >>>>> But as there is dependency on dfll resume and cpu and pllx
> > >>>>> clocks restore, couldnt use clk_save_context and
> > >>>>> clk_restore_context for dfll.
> > >>>>>
> > >>>>> So implemented recommended dfll resume sequence in main
> > >>>>> Tegra210 clock driver along with invoking
> > >>>>> clk_save_context/clk_restore_context where all other clocks
> > >>>>> save/restore happens as per clock tree traversal.    
> > >>>> Could you please clarify what part of peripherals clocks is
> > >>>> required for DFLL's restore? Couldn't DFLL driver be changed to
> > >>>> avoid that quirkness and thus to make DFLL driver
> > >>>> suspend/resume the clock?    
> > >>> DFLL source ref_clk and soc_clk need to be restored prior to
> > >>> dfll.
> > >>>
> > >>> I see dfllCPU_out parent to CCLK_G first in the clock tree and
> > >>> dfll_ref and dfll_soc peripheral clocks are not resumed by the
> > >>> time dfll resume happens first.
> > >>>
> > >>> ref_clk and soc_clk source is from pll_p and clock tree has
> > >>> these registered under pll_p which happens later.
> > >>>
> > >>> tegra210_clock_init registers in order plls, peripheral clocks,
> > >>> super_clk init for cclk_g during clock driver probe and dfll
> > >>> probe and register happens later.
> > >>>    
> > >> One more thing, CLDVFS peripheral clock enable is also needed to
> > >> be enabled to program DFLL Controller and all peripheral clock
> > >> context is restored only after their PLL sources are restored.
> > >>
> > >> DFLL restore involves dfll source clock resume along with CLDVFS
> > >> periheral clock enable and reset
> > >>    
> > > I don't quite see why you can't simply add suspend/resume
> > > callbacks to the CPUFreq driver to:
> > >
> > > On suspend:
> > > 1. Switch CPU to PLLP (or whatever "safe" parent)
> > > 2. Disable/teardown DFLL
> > >
> > > On resume:
> > > 1. Enable/restore DFLL
> > > 2. Switch CPU back to DFLL    
> > 
> > dfll runtime suspend/resume are already part of dfll_pm_ops. Don't
> > we want to use it for suspend/resume as well?  
> 
> Looks like no. Seems runtime PM of that driver is intended solely for
> the DFLL's clk management.
> 
> > currently no APIs are shared b/w clk/tegra driver and CPUFreq driver
> > to invoke dfll suspend/resume in CPUFreq driver
> >   
> 
> Just add it. Also, please note that CPUFreq driver is optional and
> thus you may need to switch CPU to a safe parent on clk-core suspend
> as well in order to resume properly if CPU was running off unsafe
> parent during boot and CPUFreq driver is disabled in kernel build (or
> failed to load).

Although, if PLLs are restored before CCLK, then it should be fine
as-is.

> The other thing that also need attention is that T124 CPUFreq driver
> implicitly relies on DFLL driver to be probed first, which is icky.
>

On 7/15/19 10:37 PM, Dmitry Osipenko wrote:
> В Mon, 15 Jul 2019 21:37:09 -0700
> Sowjanya Komatineni <skomatineni@nvidia.com> пишет:
>
>> On 7/15/19 8:50 PM, Dmitry Osipenko wrote:
>>> 16.07.2019 6:00, Sowjanya Komatineni пишет:
>>>> On 7/15/19 5:35 PM, Sowjanya Komatineni wrote:
>>>>> On 7/14/19 2:41 PM, Dmitry Osipenko wrote:
>>>>>> 13.07.2019 8:54, Sowjanya Komatineni пишет:
>>>>>>> On 6/29/19 8:10 AM, Dmitry Osipenko wrote:
>>>>>>>> 28.06.2019 5:12, Sowjanya Komatineni пишет:
>>>>>>>>> This patch adds system suspend and resume support for Tegra210
>>>>>>>>> clocks.
>>>>>>>>>
>>>>>>>>> All the CAR controller settings are lost on suspend when core
>>>>>>>>> power goes off.
>>>>>>>>>
>>>>>>>>> This patch has implementation for saving and restoring all
>>>>>>>>> the PLLs and clocks context during system suspend and resume
>>>>>>>>> to have the clocks back to same state for normal operation.
>>>>>>>>>
>>>>>>>>> Acked-by: Thierry Reding <treding@nvidia.com>
>>>>>>>>> Signed-off-by: Sowjanya Komatineni <skomatineni@nvidia.com>
>>>>>>>>> ---
>>>>>>>>>      drivers/clk/tegra/clk-tegra210.c | 115
>>>>>>>>> ++++++++++++++++++++++++++++++++++++++-
>>>>>>>>>      drivers/clk/tegra/clk.c          |  14 +++++
>>>>>>>>>      drivers/clk/tegra/clk.h          |   1 +
>>>>>>>>>      3 files changed, 127 insertions(+), 3 deletions(-)
>>>>>>>>>
>>>>>>>>> diff --git a/drivers/clk/tegra/clk-tegra210.c
>>>>>>>>> b/drivers/clk/tegra/clk-tegra210.c
>>>>>>>>> index 1c08c53482a5..1b839544e086 100644
>>>>>>>>> --- a/drivers/clk/tegra/clk-tegra210.c
>>>>>>>>> +++ b/drivers/clk/tegra/clk-tegra210.c
>>>>>>>>> @@ -9,10 +9,12 @@
>>>>>>>>>      #include <linux/clkdev.h>
>>>>>>>>>      #include <linux/of.h>
>>>>>>>>>      #include <linux/of_address.h>
>>>>>>>>> +#include <linux/of_platform.h>
>>>>>>>>>      #include <linux/delay.h>
>>>>>>>>>      #include <linux/export.h>
>>>>>>>>>      #include <linux/mutex.h>
>>>>>>>>>      #include <linux/clk/tegra.h>
>>>>>>>>> +#include <linux/syscore_ops.h>
>>>>>>>>>      #include <dt-bindings/clock/tegra210-car.h>
>>>>>>>>>      #include <dt-bindings/reset/tegra210-car.h>
>>>>>>>>>      #include <linux/iopoll.h>
>>>>>>>>> @@ -20,6 +22,7 @@
>>>>>>>>>      #include <soc/tegra/pmc.h>
>>>>>>>>>        #include "clk.h"
>>>>>>>>> +#include "clk-dfll.h"
>>>>>>>>>      #include "clk-id.h"
>>>>>>>>>        /*
>>>>>>>>> @@ -225,6 +228,7 @@
>>>>>>>>>        #define CLK_RST_CONTROLLER_RST_DEV_Y_SET 0x2a8
>>>>>>>>>      #define CLK_RST_CONTROLLER_RST_DEV_Y_CLR 0x2ac
>>>>>>>>> +#define CPU_SOFTRST_CTRL 0x380
>>>>>>>>>        #define LVL2_CLK_GATE_OVRA 0xf8
>>>>>>>>>      #define LVL2_CLK_GATE_OVRC 0x3a0
>>>>>>>>> @@ -2820,6 +2824,7 @@ static int tegra210_enable_pllu(void)
>>>>>>>>>          struct tegra_clk_pll_freq_table *fentry;
>>>>>>>>>          struct tegra_clk_pll pllu;
>>>>>>>>>          u32 reg;
>>>>>>>>> +    int ret;
>>>>>>>>>            for (fentry = pll_u_freq_table; fentry->input_rate;
>>>>>>>>> fentry++) {
>>>>>>>>>              if (fentry->input_rate == pll_ref_freq)
>>>>>>>>> @@ -2847,10 +2852,10 @@ static int tegra210_enable_pllu(void)
>>>>>>>>>          fence_udelay(1, clk_base);
>>>>>>>>>          reg |= PLL_ENABLE;
>>>>>>>>>          writel(reg, clk_base + PLLU_BASE);
>>>>>>>>> +    fence_udelay(1, clk_base);
>>>>>>>>>      -    readl_relaxed_poll_timeout_atomic(clk_base +
>>>>>>>>> PLLU_BASE, reg,
>>>>>>>>> -                      reg & PLL_BASE_LOCK, 2, 1000);
>>>>>>>>> -    if (!(reg & PLL_BASE_LOCK)) {
>>>>>>>>> +    ret = tegra210_wait_for_mask(&pllu, PLLU_BASE,
>>>>>>>>> PLL_BASE_LOCK);
>>>>>>>>> +    if (ret) {
>>>>>>>>>              pr_err("Timed out waiting for PLL_U to lock\n");
>>>>>>>>>              return -ETIMEDOUT;
>>>>>>>>>          }
>>>>>>>>> @@ -3283,6 +3288,103 @@ static void
>>>>>>>>> tegra210_disable_cpu_clock(u32 cpu)
>>>>>>>>>      }
>>>>>>>>>        #ifdef CONFIG_PM_SLEEP
>>>>>>>>> +static u32 cpu_softrst_ctx[3];
>>>>>>>>> +static struct platform_device *dfll_pdev;
>>>>>>>>> +#define car_readl(_base, _off) readl_relaxed(clk_base +
>>>>>>>>> (_base) + ((_off) * 4))
>>>>>>>>> +#define car_writel(_val, _base, _off) \
>>>>>>>>> +        writel_relaxed(_val, clk_base + (_base) + ((_off) *
>>>>>>>>> 4)) +
>>>>>>>>> +static int tegra210_clk_suspend(void)
>>>>>>>>> +{
>>>>>>>>> +    unsigned int i;
>>>>>>>>> +    struct device_node *node;
>>>>>>>>> +
>>>>>>>>> +    tegra_cclkg_burst_policy_save_context();
>>>>>>>>> +
>>>>>>>>> +    if (!dfll_pdev) {
>>>>>>>>> +        node = of_find_compatible_node(NULL, NULL,
>>>>>>>>> +                           "nvidia,tegra210-dfll");
>>>>>>>>> +        if (node)
>>>>>>>>> +            dfll_pdev = of_find_device_by_node(node);
>>>>>>>>> +
>>>>>>>>> +        of_node_put(node);
>>>>>>>>> +        if (!dfll_pdev)
>>>>>>>>> +            pr_err("dfll node not found. no suspend for
>>>>>>>>> dfll\n");
>>>>>>>>> +    }
>>>>>>>>> +
>>>>>>>>> +    if (dfll_pdev)
>>>>>>>>> +        tegra_dfll_suspend(dfll_pdev);
>>>>>>>>> +
>>>>>>>>> +    /* Enable PLLP_OUT_CPU after dfll suspend */
>>>>>>>>> +    tegra_clk_set_pllp_out_cpu(true);
>>>>>>>>> +
>>>>>>>>> +    tegra_sclk_cclklp_burst_policy_save_context();
>>>>>>>>> +
>>>>>>>>> +    clk_save_context();
>>>>>>>>> +
>>>>>>>>> +    for (i = 0; i < ARRAY_SIZE(cpu_softrst_ctx); i++)
>>>>>>>>> +        cpu_softrst_ctx[i] = car_readl(CPU_SOFTRST_CTRL, i);
>>>>>>>>> +
>>>>>>>>> +    return 0;
>>>>>>>>> +}
>>>>>>>>> +
>>>>>>>>> +static void tegra210_clk_resume(void)
>>>>>>>>> +{
>>>>>>>>> +    unsigned int i;
>>>>>>>>> +    struct clk_hw *parent;
>>>>>>>>> +    struct clk *clk;
>>>>>>>>> +
>>>>>>>>> +    /*
>>>>>>>>> +     * clk_restore_context restores clocks as per the clock
>>>>>>>>> tree.
>>>>>>>>> +     *
>>>>>>>>> +     * dfllCPU_out is first in the clock tree to get
>>>>>>>>> restored and it
>>>>>>>>> +     * involves programming DFLL controller along with
>>>>>>>>> restoring CPUG
>>>>>>>>> +     * clock burst policy.
>>>>>>>>> +     *
>>>>>>>>> +     * DFLL programming needs dfll_ref and dfll_soc
>>>>>>>>> peripheral clocks
>>>>>>>>> +     * to be restores which are part ofthe peripheral
>>>>>>>>> clocks.
>>>>>>                                                ^ white-space
>>>>>>
>>>>>> Please use spellchecker to avoid typos.
>>>>>>   
>>>>>>>>> +     * So, peripheral clocks restore should happen prior to
>>>>>>>>> dfll clock
>>>>>>>>> +     * restore.
>>>>>>>>> +     */
>>>>>>>>> +
>>>>>>>>> +    tegra_clk_osc_resume(clk_base);
>>>>>>>>> +    for (i = 0; i < ARRAY_SIZE(cpu_softrst_ctx); i++)
>>>>>>>>> +        car_writel(cpu_softrst_ctx[i], CPU_SOFTRST_CTRL, i);
>>>>>>>>> +
>>>>>>>>> +    /* restore all plls and peripheral clocks */
>>>>>>>>> +    tegra210_init_pllu();
>>>>>>>>> +    clk_restore_context();
>>>>>>>>> +
>>>>>>>>> +    fence_udelay(5, clk_base);
>>>>>>>>> +
>>>>>>>>> +    /* resume SCLK and CPULP clocks */
>>>>>>>>> +    tegra_sclk_cpulp_burst_policy_restore_context();
>>>>>>>>> +
>>>>>>>>> +    /*
>>>>>>>>> +     * restore CPUG clocks:
>>>>>>>>> +     * - enable DFLL in open loop mode
>>>>>>>>> +     * - switch CPUG to DFLL clock source
>>>>>>>>> +     * - close DFLL loop
>>>>>>>>> +     * - sync PLLX state
>>>>>>>>> +     */
>>>>>>>>> +    if (dfll_pdev)
>>>>>>>>> +        tegra_dfll_resume(dfll_pdev, false);
>>>>>>>>> +
>>>>>>>>> +    tegra_cclkg_burst_policy_restore_context();
>>>>>>>>> +    fence_udelay(2, clk_base);
>>>>>>>>> +
>>>>>>>>> +    if (dfll_pdev)
>>>>>>>>> +        tegra_dfll_resume(dfll_pdev, true);
>>>>>>>>> +
>>>>>>>>> +    parent =
>>>>>>>>> clk_hw_get_parent(__clk_get_hw(clks[TEGRA210_CLK_CCLK_G]));
>>>>>>>>> +    clk = clks[TEGRA210_CLK_PLL_X];
>>>>>>>>> +    if (parent != __clk_get_hw(clk))
>>>>>>>>> +        tegra_clk_sync_state_pll(__clk_get_hw(clk));
>>>>>>>>> +
>>>>>>>>> +    /* Disable PLL_OUT_CPU after DFLL resume */
>>>>>>>>> +    tegra_clk_set_pllp_out_cpu(false);
>>>>>>>>> +}
>>>>>>>>> +
>>>>>>>>>      static void tegra210_cpu_clock_suspend(void)
>>>>>>>>>      {
>>>>>>>>>          /* switch coresite to clk_m, save off original source
>>>>>>>>> */ @@ -3298,6 +3400,11 @@ static void
>>>>>>>>> tegra210_cpu_clock_resume(void) }
>>>>>>>>>      #endif
>>>>>>>>>      +static struct syscore_ops tegra_clk_syscore_ops = {
>>>>>>>>> +    .suspend = tegra210_clk_suspend,
>>>>>>>>> +    .resume = tegra210_clk_resume,
>>>>>>>>> +};
>>>>>>>>> +
>>>>>>>>>      static struct tegra_cpu_car_ops tegra210_cpu_car_ops = {
>>>>>>>>>          .wait_for_reset    = tegra210_wait_cpu_in_reset,
>>>>>>>>>          .disable_clock    = tegra210_disable_cpu_clock,
>>>>>>>>> @@ -3583,5 +3690,7 @@ static void __init
>>>>>>>>> tegra210_clock_init(struct device_node *np)
>>>>>>>>>          tegra210_mbist_clk_init();
>>>>>>>>>            tegra_cpu_car_ops = &tegra210_cpu_car_ops;
>>>>>>>>> +
>>>>>>>>> +    register_syscore_ops(&tegra_clk_syscore_ops);
>>>>>>>>>      }
>>>>>>>> Is it really worthwhile to use syscore_ops for suspend/resume
>>>>>>>> given that drivers for
>>>>>>>> won't resume before the CLK driver anyway? Are there any other
>>>>>>>> options for CLK
>>>>>>>> suspend/resume?
>>>>>>>>
>>>>>>>> I'm also not sure whether PM runtime API could be used at all
>>>>>>>> in the context of
>>>>>>>> syscore_ops ..
>>>>>>>>
>>>>>>>> Secondly, what about to use generic clk_save_context() /
>>>>>>>> clk_restore_context()
>>>>>>>> helpers for the suspend-resume? It looks to me that some other
>>>>>>>> essential (and proper)
>>>>>>>> platform driver (soc/tegra/? PMC?) should suspend-resume the
>>>>>>>> clocks using the generic
>>>>>>>> CLK Framework API.
>>>>>>> Clock resume should happen very early to restore peripheral and
>>>>>>> cpu clocks very early than peripheral drivers resume happens.
>>>>>> If all peripheral drivers properly requested all of the
>>>>>> necessary clocks and CLK driver was a platform driver, then I
>>>>>> guess the probe should have been naturally ordered. But that's
>>>>>> not very achievable with the currently available infrastructure
>>>>>> in the kernel, so I'm not arguing that the clocks should be
>>>>>> explicitly resumed before the users.
>>>>>>> this patch series uses clk_save_context and clk_restore_context
>>>>>>> for corresponding divider, pll, pllout.. save and restore
>>>>>>> context.
>>>>>> Now I see that indeed this API is utilized in this patch, thank
>>>>>> you for the clarification.
>>>>>>   
>>>>>>> But as there is dependency on dfll resume and cpu and pllx
>>>>>>> clocks restore, couldnt use clk_save_context and
>>>>>>> clk_restore_context for dfll.
>>>>>>>
>>>>>>> So implemented recommended dfll resume sequence in main
>>>>>>> Tegra210 clock driver along with invoking
>>>>>>> clk_save_context/clk_restore_context where all other clocks
>>>>>>> save/restore happens as per clock tree traversal.
>>>>>> Could you please clarify what part of peripherals clocks is
>>>>>> required for DFLL's restore? Couldn't DFLL driver be changed to
>>>>>> avoid that quirkness and thus to make DFLL driver suspend/resume
>>>>>> the clock?
>>>>> DFLL source ref_clk and soc_clk need to be restored prior to dfll.
>>>>>
>>>>> I see dfllCPU_out parent to CCLK_G first in the clock tree and
>>>>> dfll_ref and dfll_soc peripheral clocks are not resumed by the
>>>>> time dfll resume happens first.
>>>>>
>>>>> ref_clk and soc_clk source is from pll_p and clock tree has these
>>>>> registered under pll_p which happens later.
>>>>>
>>>>> tegra210_clock_init registers in order plls, peripheral clocks,
>>>>> super_clk init for cclk_g during clock driver probe and dfll
>>>>> probe and register happens later.
>>>>>   
>>>> One more thing, CLDVFS peripheral clock enable is also needed to be
>>>> enabled to program DFLL Controller and all peripheral clock
>>>> context is restored only after their PLL sources are restored.
>>>>
>>>> DFLL restore involves dfll source clock resume along with CLDVFS
>>>> periheral clock enable and reset
>>>>   
>>> I don't quite see why you can't simply add suspend/resume callbacks
>>> to the CPUFreq driver to:
>>>
>>> On suspend:
>>> 1. Switch CPU to PLLP (or whatever "safe" parent)
>>> 2. Disable/teardown DFLL
>>>
>>> On resume:
>>> 1. Enable/restore DFLL
>>> 2. Switch CPU back to DFLL
>> dfll runtime suspend/resume are already part of dfll_pm_ops. Don't we
>> want to use it for suspend/resume as well?
> Looks like no. Seems runtime PM of that driver is intended solely for
> the DFLL's clk management.
>
>> currently no APIs are shared b/w clk/tegra driver and CPUFreq driver
>> to invoke dfll suspend/resume in CPUFreq driver
>>
> Just add it. Also, please note that CPUFreq driver is optional and thus
> you may need to switch CPU to a safe parent on clk-core suspend as
> well in order to resume properly if CPU was running off unsafe parent
> during boot and CPUFreq driver is disabled in kernel build (or failed
> to load).
OK, Will add to CPUFreq driver...
>
> The other thing that also need attention is that T124 CPUFreq driver
> implicitly relies on DFLL driver to be probed first, which is icky.
>
Should I add check for successful dfll clk register explicitly in 
CPUFreq driver probe and defer till dfll clk registers?

On 7/16/19 2:35 PM, Sowjanya Komatineni wrote:
> 
> On 7/15/19 10:37 PM, Dmitry Osipenko wrote:
>> В Mon, 15 Jul 2019 21:37:09 -0700
>> Sowjanya Komatineni <skomatineni@nvidia.com> пишет:
>>
>>> On 7/15/19 8:50 PM, Dmitry Osipenko wrote:
>>>> 16.07.2019 6:00, Sowjanya Komatineni пишет:
>>>>> On 7/15/19 5:35 PM, Sowjanya Komatineni wrote:
>>>>>> On 7/14/19 2:41 PM, Dmitry Osipenko wrote:
>>>>>>> 13.07.2019 8:54, Sowjanya Komatineni пишет:
>>>>>>>> On 6/29/19 8:10 AM, Dmitry Osipenko wrote:
>>>>>>>>> 28.06.2019 5:12, Sowjanya Komatineni пишет:
>>>>>>>>>> This patch adds system suspend and resume support for Tegra210
>>>>>>>>>> clocks.
>>>>>>>>>>
>>>>>>>>>> All the CAR controller settings are lost on suspend when core
>>>>>>>>>> power goes off.
>>>>>>>>>>
>>>>>>>>>> This patch has implementation for saving and restoring all
>>>>>>>>>> the PLLs and clocks context during system suspend and resume
>>>>>>>>>> to have the clocks back to same state for normal operation.
>>>>>>>>>>
>>>>>>>>>> Acked-by: Thierry Reding <treding@nvidia.com>
>>>>>>>>>> Signed-off-by: Sowjanya Komatineni <skomatineni@nvidia.com>
>>>>>>>>>> ---
>>>>>>>>>>      drivers/clk/tegra/clk-tegra210.c | 115
>>>>>>>>>> ++++++++++++++++++++++++++++++++++++++-
>>>>>>>>>>      drivers/clk/tegra/clk.c          |  14 +++++
>>>>>>>>>>      drivers/clk/tegra/clk.h          |   1 +
>>>>>>>>>>      3 files changed, 127 insertions(+), 3 deletions(-)
>>>>>>>>>>
>>>>>>>>>> diff --git a/drivers/clk/tegra/clk-tegra210.c
>>>>>>>>>> b/drivers/clk/tegra/clk-tegra210.c
>>>>>>>>>> index 1c08c53482a5..1b839544e086 100644
>>>>>>>>>> --- a/drivers/clk/tegra/clk-tegra210.c
>>>>>>>>>> +++ b/drivers/clk/tegra/clk-tegra210.c
>>>>>>>>>> @@ -9,10 +9,12 @@
>>>>>>>>>>      #include <linux/clkdev.h>
>>>>>>>>>>      #include <linux/of.h>
>>>>>>>>>>      #include <linux/of_address.h>
>>>>>>>>>> +#include <linux/of_platform.h>
>>>>>>>>>>      #include <linux/delay.h>
>>>>>>>>>>      #include <linux/export.h>
>>>>>>>>>>      #include <linux/mutex.h>
>>>>>>>>>>      #include <linux/clk/tegra.h>
>>>>>>>>>> +#include <linux/syscore_ops.h>
>>>>>>>>>>      #include <dt-bindings/clock/tegra210-car.h>
>>>>>>>>>>      #include <dt-bindings/reset/tegra210-car.h>
>>>>>>>>>>      #include <linux/iopoll.h>
>>>>>>>>>> @@ -20,6 +22,7 @@
>>>>>>>>>>      #include <soc/tegra/pmc.h>
>>>>>>>>>>        #include "clk.h"
>>>>>>>>>> +#include "clk-dfll.h"
>>>>>>>>>>      #include "clk-id.h"
>>>>>>>>>>        /*
>>>>>>>>>> @@ -225,6 +228,7 @@
>>>>>>>>>>        #define CLK_RST_CONTROLLER_RST_DEV_Y_SET 0x2a8
>>>>>>>>>>      #define CLK_RST_CONTROLLER_RST_DEV_Y_CLR 0x2ac
>>>>>>>>>> +#define CPU_SOFTRST_CTRL 0x380
>>>>>>>>>>        #define LVL2_CLK_GATE_OVRA 0xf8
>>>>>>>>>>      #define LVL2_CLK_GATE_OVRC 0x3a0
>>>>>>>>>> @@ -2820,6 +2824,7 @@ static int tegra210_enable_pllu(void)
>>>>>>>>>>          struct tegra_clk_pll_freq_table *fentry;
>>>>>>>>>>          struct tegra_clk_pll pllu;
>>>>>>>>>>          u32 reg;
>>>>>>>>>> +    int ret;
>>>>>>>>>>            for (fentry = pll_u_freq_table; fentry->input_rate;
>>>>>>>>>> fentry++) {
>>>>>>>>>>              if (fentry->input_rate == pll_ref_freq)
>>>>>>>>>> @@ -2847,10 +2852,10 @@ static int tegra210_enable_pllu(void)
>>>>>>>>>>          fence_udelay(1, clk_base);
>>>>>>>>>>          reg |= PLL_ENABLE;
>>>>>>>>>>          writel(reg, clk_base + PLLU_BASE);
>>>>>>>>>> +    fence_udelay(1, clk_base);
>>>>>>>>>>      -    readl_relaxed_poll_timeout_atomic(clk_base +
>>>>>>>>>> PLLU_BASE, reg,
>>>>>>>>>> -                      reg & PLL_BASE_LOCK, 2, 1000);
>>>>>>>>>> -    if (!(reg & PLL_BASE_LOCK)) {
>>>>>>>>>> +    ret = tegra210_wait_for_mask(&pllu, PLLU_BASE,
>>>>>>>>>> PLL_BASE_LOCK);
>>>>>>>>>> +    if (ret) {
>>>>>>>>>>              pr_err("Timed out waiting for PLL_U to lock\n");
>>>>>>>>>>              return -ETIMEDOUT;
>>>>>>>>>>          }
>>>>>>>>>> @@ -3283,6 +3288,103 @@ static void
>>>>>>>>>> tegra210_disable_cpu_clock(u32 cpu)
>>>>>>>>>>      }
>>>>>>>>>>        #ifdef CONFIG_PM_SLEEP
>>>>>>>>>> +static u32 cpu_softrst_ctx[3];
>>>>>>>>>> +static struct platform_device *dfll_pdev;
>>>>>>>>>> +#define car_readl(_base, _off) readl_relaxed(clk_base +
>>>>>>>>>> (_base) + ((_off) * 4))
>>>>>>>>>> +#define car_writel(_val, _base, _off) \
>>>>>>>>>> +        writel_relaxed(_val, clk_base + (_base) + ((_off) *
>>>>>>>>>> 4)) +
>>>>>>>>>> +static int tegra210_clk_suspend(void)
>>>>>>>>>> +{
>>>>>>>>>> +    unsigned int i;
>>>>>>>>>> +    struct device_node *node;
>>>>>>>>>> +
>>>>>>>>>> +    tegra_cclkg_burst_policy_save_context();
>>>>>>>>>> +
>>>>>>>>>> +    if (!dfll_pdev) {
>>>>>>>>>> +        node = of_find_compatible_node(NULL, NULL,
>>>>>>>>>> +                           "nvidia,tegra210-dfll");
>>>>>>>>>> +        if (node)
>>>>>>>>>> +            dfll_pdev = of_find_device_by_node(node);
>>>>>>>>>> +
>>>>>>>>>> +        of_node_put(node);
>>>>>>>>>> +        if (!dfll_pdev)
>>>>>>>>>> +            pr_err("dfll node not found. no suspend for
>>>>>>>>>> dfll\n");
>>>>>>>>>> +    }
>>>>>>>>>> +
>>>>>>>>>> +    if (dfll_pdev)
>>>>>>>>>> +        tegra_dfll_suspend(dfll_pdev);
>>>>>>>>>> +
>>>>>>>>>> +    /* Enable PLLP_OUT_CPU after dfll suspend */
>>>>>>>>>> +    tegra_clk_set_pllp_out_cpu(true);
>>>>>>>>>> +
>>>>>>>>>> +    tegra_sclk_cclklp_burst_policy_save_context();
>>>>>>>>>> +
>>>>>>>>>> +    clk_save_context();
>>>>>>>>>> +
>>>>>>>>>> +    for (i = 0; i < ARRAY_SIZE(cpu_softrst_ctx); i++)
>>>>>>>>>> +        cpu_softrst_ctx[i] = car_readl(CPU_SOFTRST_CTRL, i);
>>>>>>>>>> +
>>>>>>>>>> +    return 0;
>>>>>>>>>> +}
>>>>>>>>>> +
>>>>>>>>>> +static void tegra210_clk_resume(void)
>>>>>>>>>> +{
>>>>>>>>>> +    unsigned int i;
>>>>>>>>>> +    struct clk_hw *parent;
>>>>>>>>>> +    struct clk *clk;
>>>>>>>>>> +
>>>>>>>>>> +    /*
>>>>>>>>>> +     * clk_restore_context restores clocks as per the clock
>>>>>>>>>> tree.
>>>>>>>>>> +     *
>>>>>>>>>> +     * dfllCPU_out is first in the clock tree to get
>>>>>>>>>> restored and it
>>>>>>>>>> +     * involves programming DFLL controller along with
>>>>>>>>>> restoring CPUG
>>>>>>>>>> +     * clock burst policy.
>>>>>>>>>> +     *
>>>>>>>>>> +     * DFLL programming needs dfll_ref and dfll_soc
>>>>>>>>>> peripheral clocks
>>>>>>>>>> +     * to be restores which are part ofthe peripheral
>>>>>>>>>> clocks.
>>>>>>>                                                ^ white-space
>>>>>>>
>>>>>>> Please use spellchecker to avoid typos.
>>>>>>>>>> +     * So, peripheral clocks restore should happen prior to
>>>>>>>>>> dfll clock
>>>>>>>>>> +     * restore.
>>>>>>>>>> +     */
>>>>>>>>>> +
>>>>>>>>>> +    tegra_clk_osc_resume(clk_base);
>>>>>>>>>> +    for (i = 0; i < ARRAY_SIZE(cpu_softrst_ctx); i++)
>>>>>>>>>> +        car_writel(cpu_softrst_ctx[i], CPU_SOFTRST_CTRL, i);
>>>>>>>>>> +
>>>>>>>>>> +    /* restore all plls and peripheral clocks */
>>>>>>>>>> +    tegra210_init_pllu();
>>>>>>>>>> +    clk_restore_context();
>>>>>>>>>> +
>>>>>>>>>> +    fence_udelay(5, clk_base);
>>>>>>>>>> +
>>>>>>>>>> +    /* resume SCLK and CPULP clocks */
>>>>>>>>>> +    tegra_sclk_cpulp_burst_policy_restore_context();
>>>>>>>>>> +
>>>>>>>>>> +    /*
>>>>>>>>>> +     * restore CPUG clocks:
>>>>>>>>>> +     * - enable DFLL in open loop mode
>>>>>>>>>> +     * - switch CPUG to DFLL clock source
>>>>>>>>>> +     * - close DFLL loop
>>>>>>>>>> +     * - sync PLLX state
>>>>>>>>>> +     */
>>>>>>>>>> +    if (dfll_pdev)
>>>>>>>>>> +        tegra_dfll_resume(dfll_pdev, false);
>>>>>>>>>> +
>>>>>>>>>> +    tegra_cclkg_burst_policy_restore_context();
>>>>>>>>>> +    fence_udelay(2, clk_base);
>>>>>>>>>> +
>>>>>>>>>> +    if (dfll_pdev)
>>>>>>>>>> +        tegra_dfll_resume(dfll_pdev, true);
>>>>>>>>>> +
>>>>>>>>>> +    parent =
>>>>>>>>>> clk_hw_get_parent(__clk_get_hw(clks[TEGRA210_CLK_CCLK_G]));
>>>>>>>>>> +    clk = clks[TEGRA210_CLK_PLL_X];
>>>>>>>>>> +    if (parent != __clk_get_hw(clk))
>>>>>>>>>> +        tegra_clk_sync_state_pll(__clk_get_hw(clk));
>>>>>>>>>> +
>>>>>>>>>> +    /* Disable PLL_OUT_CPU after DFLL resume */
>>>>>>>>>> +    tegra_clk_set_pllp_out_cpu(false);
>>>>>>>>>> +}
>>>>>>>>>> +
>>>>>>>>>>      static void tegra210_cpu_clock_suspend(void)
>>>>>>>>>>      {
>>>>>>>>>>          /* switch coresite to clk_m, save off original source
>>>>>>>>>> */ @@ -3298,6 +3400,11 @@ static void
>>>>>>>>>> tegra210_cpu_clock_resume(void) }
>>>>>>>>>>      #endif
>>>>>>>>>>      +static struct syscore_ops tegra_clk_syscore_ops = {
>>>>>>>>>> +    .suspend = tegra210_clk_suspend,
>>>>>>>>>> +    .resume = tegra210_clk_resume,
>>>>>>>>>> +};
>>>>>>>>>> +
>>>>>>>>>>      static struct tegra_cpu_car_ops tegra210_cpu_car_ops = {
>>>>>>>>>>          .wait_for_reset    = tegra210_wait_cpu_in_reset,
>>>>>>>>>>          .disable_clock    = tegra210_disable_cpu_clock,
>>>>>>>>>> @@ -3583,5 +3690,7 @@ static void __init
>>>>>>>>>> tegra210_clock_init(struct device_node *np)
>>>>>>>>>>          tegra210_mbist_clk_init();
>>>>>>>>>>            tegra_cpu_car_ops = &tegra210_cpu_car_ops;
>>>>>>>>>> +
>>>>>>>>>> +    register_syscore_ops(&tegra_clk_syscore_ops);
>>>>>>>>>>      }
>>>>>>>>> Is it really worthwhile to use syscore_ops for suspend/resume
>>>>>>>>> given that drivers for
>>>>>>>>> won't resume before the CLK driver anyway? Are there any other
>>>>>>>>> options for CLK
>>>>>>>>> suspend/resume?
>>>>>>>>>
>>>>>>>>> I'm also not sure whether PM runtime API could be used at all
>>>>>>>>> in the context of
>>>>>>>>> syscore_ops ..
>>>>>>>>>
>>>>>>>>> Secondly, what about to use generic clk_save_context() /
>>>>>>>>> clk_restore_context()
>>>>>>>>> helpers for the suspend-resume? It looks to me that some other
>>>>>>>>> essential (and proper)
>>>>>>>>> platform driver (soc/tegra/? PMC?) should suspend-resume the
>>>>>>>>> clocks using the generic
>>>>>>>>> CLK Framework API.
>>>>>>>> Clock resume should happen very early to restore peripheral and
>>>>>>>> cpu clocks very early than peripheral drivers resume happens.
>>>>>>> If all peripheral drivers properly requested all of the
>>>>>>> necessary clocks and CLK driver was a platform driver, then I
>>>>>>> guess the probe should have been naturally ordered. But that's
>>>>>>> not very achievable with the currently available infrastructure
>>>>>>> in the kernel, so I'm not arguing that the clocks should be
>>>>>>> explicitly resumed before the users.
>>>>>>>> this patch series uses clk_save_context and clk_restore_context
>>>>>>>> for corresponding divider, pll, pllout.. save and restore
>>>>>>>> context.
>>>>>>> Now I see that indeed this API is utilized in this patch, thank
>>>>>>> you for the clarification.
>>>>>>>> But as there is dependency on dfll resume and cpu and pllx
>>>>>>>> clocks restore, couldnt use clk_save_context and
>>>>>>>> clk_restore_context for dfll.
>>>>>>>>
>>>>>>>> So implemented recommended dfll resume sequence in main
>>>>>>>> Tegra210 clock driver along with invoking
>>>>>>>> clk_save_context/clk_restore_context where all other clocks
>>>>>>>> save/restore happens as per clock tree traversal.
>>>>>>> Could you please clarify what part of peripherals clocks is
>>>>>>> required for DFLL's restore? Couldn't DFLL driver be changed to
>>>>>>> avoid that quirkness and thus to make DFLL driver suspend/resume
>>>>>>> the clock?
>>>>>> DFLL source ref_clk and soc_clk need to be restored prior to dfll.
>>>>>>
>>>>>> I see dfllCPU_out parent to CCLK_G first in the clock tree and
>>>>>> dfll_ref and dfll_soc peripheral clocks are not resumed by the
>>>>>> time dfll resume happens first.
>>>>>>
>>>>>> ref_clk and soc_clk source is from pll_p and clock tree has these
>>>>>> registered under pll_p which happens later.
>>>>>>
>>>>>> tegra210_clock_init registers in order plls, peripheral clocks,
>>>>>> super_clk init for cclk_g during clock driver probe and dfll
>>>>>> probe and register happens later.
>>>>> One more thing, CLDVFS peripheral clock enable is also needed to be
>>>>> enabled to program DFLL Controller and all peripheral clock
>>>>> context is restored only after their PLL sources are restored.
>>>>>
>>>>> DFLL restore involves dfll source clock resume along with CLDVFS
>>>>> periheral clock enable and reset
>>>> I don't quite see why you can't simply add suspend/resume callbacks
>>>> to the CPUFreq driver to:
>>>>
>>>> On suspend:
>>>> 1. Switch CPU to PLLP (or whatever "safe" parent)
>>>> 2. Disable/teardown DFLL
>>>>
>>>> On resume:
>>>> 1. Enable/restore DFLL
>>>> 2. Switch CPU back to DFLL
>>> dfll runtime suspend/resume are already part of dfll_pm_ops. Don't we
>>> want to use it for suspend/resume as well?
>> Looks like no. Seems runtime PM of that driver is intended solely for
>> the DFLL's clk management.
>>
>>> currently no APIs are shared b/w clk/tegra driver and CPUFreq driver
>>> to invoke dfll suspend/resume in CPUFreq driver
>>>
>> Just add it. Also, please note that CPUFreq driver is optional and thus
>> you may need to switch CPU to a safe parent on clk-core suspend as
>> well in order to resume properly if CPU was running off unsafe parent
>> during boot and CPUFreq driver is disabled in kernel build (or failed
>> to load).
> OK, Will add to CPUFreq driver...
>>
>> The other thing that also need attention is that T124 CPUFreq driver
>> implicitly relies on DFLL driver to be probed first, which is icky.
>>
> Should I add check for successful dfll clk register explicitly in 
> CPUFreq driver probe and defer till dfll clk registers?

Sorry, I didn't follow the mail thread. Just regarding the DFLL part.

As you know it, the DFLL clock is one of the CPU clock sources and 
integrated with DVFS control logic with the regulator. We will not 
switch CPU to other clock sources once we switched to DFLL. Because the 
CPU has been regulated by the DFLL HW with the DVFS table (CVB or OPP 
table you see in the driver.). We shouldn't reparent it to other sources 
with unknew freq/volt pair. That's not guaranteed to work. We allow 
switching to open-loop mode but different sources.

And I don't exactly understand why we need to switch to PLLP in CPU idle 
driver. Just keep it on CL-DVFS mode all the time.

In SC7 entry, the dfll suspend function moves it the open-loop mode. 
That's all. The sc7-entryfirmware will handle the rest of the sequence 
to turn off the CPU power.

In SC7 resume, the warmboot code will handle the sequence to turn on 
regulator and power up the CPU cluster. And leave it on PLL_P. After 
resuming to the kernel, we re-init DFLL, restore the CPU clock policy 
(CPU runs on DFLL open-loop mode) and then moving to close-loop mode.

The DFLL part looks good to me. BTW, change the patch subject to "Add 
suspend-resume support" seems more appropriate to me.

Thanks.

On Tue, Jul 16, 2019 at 03:24:26PM +0800, Joseph Lo wrote:
> > OK, Will add to CPUFreq driver...
> > > 
> > > The other thing that also need attention is that T124 CPUFreq driver
> > > implicitly relies on DFLL driver to be probed first, which is icky.
> > > 
> > Should I add check for successful dfll clk register explicitly in
> > CPUFreq driver probe and defer till dfll clk registers?
> 
> Sorry, I didn't follow the mail thread. Just regarding the DFLL part.
> 
> As you know it, the DFLL clock is one of the CPU clock sources and
> integrated with DVFS control logic with the regulator. We will not switch
> CPU to other clock sources once we switched to DFLL. Because the CPU has
> been regulated by the DFLL HW with the DVFS table (CVB or OPP table you see
> in the driver.). We shouldn't reparent it to other sources with unknew
> freq/volt pair. That's not guaranteed to work. We allow switching to
> open-loop mode but different sources.
> 
> And I don't exactly understand why we need to switch to PLLP in CPU idle
> driver. Just keep it on CL-DVFS mode all the time.
> 
> In SC7 entry, the dfll suspend function moves it the open-loop mode. That's
> all. The sc7-entryfirmware will handle the rest of the sequence to turn off
> the CPU power.
> 
> In SC7 resume, the warmboot code will handle the sequence to turn on
> regulator and power up the CPU cluster. And leave it on PLL_P. After
> resuming to the kernel, we re-init DFLL, restore the CPU clock policy (CPU
> runs on DFLL open-loop mode) and then moving to close-loop mode.
> 
> The DFLL part looks good to me. BTW, change the patch subject to "Add
> suspend-resume support" seems more appropriate to me.
> 

To clarify this, the sequences for DFLL use are as follows (assuming all
required DFLL hw configuration has been done)

Switch to DFLL:
0) Save current parent and frequency
1) Program DFLL to open loop mode
2) Enable DFLL
3) Change cclk_g parent to DFLL
For OVR regulator:
4) Change PWM output pin from tristate to output
5) Enable DFLL PWM output
For I2C regulator:
4) Enable DFLL I2C output
6) Program DFLL to closed loop mode

Switch away from DFLL:
0) Change cclk_g parent to PLLP so the CPU frequency is ok for any vdd_cpu voltage
1) Program DFLL to open loop mode

For OVR regulator:
2) Change PWM output pin from output to tristate: vdd_cpu will go back
   to hardwired boot voltage.
3) Disable DFLL PWM output

For I2C regulator:
2) Program vdd_cpu regulator voltage to the boot voltage
3) Disable DFLL I2C output

4) Reprogram parent saved in step 0 of 'Switch to DFLL' to the saved
   frequency
5) Change cclk_g parent to saved parent
6) Disable DFLL

Peter.

16.07.2019 11:06, Peter De Schrijver пишет:
> On Tue, Jul 16, 2019 at 03:24:26PM +0800, Joseph Lo wrote:
>>> OK, Will add to CPUFreq driver...
>>>>
>>>> The other thing that also need attention is that T124 CPUFreq driver
>>>> implicitly relies on DFLL driver to be probed first, which is icky.
>>>>
>>> Should I add check for successful dfll clk register explicitly in
>>> CPUFreq driver probe and defer till dfll clk registers?

Probably you should use the "device links". See [1][2] for the example.

[1]
https://elixir.bootlin.com/linux/v5.2.1/source/drivers/gpu/drm/tegra/dc.c#L2383

[2] https://www.kernel.org/doc/html/latest/driver-api/device_link.html

Return EPROBE_DEFER instead of EINVAL if device_link_add() fails. And
use of_find_device_by_node() to get the DFLL's device, see [3].

[3]
https://git.kernel.org/pub/scm/linux/kernel/git/next/linux-next.git/tree/drivers/devfreq/tegra20-devfreq.c#n100

>> Sorry, I didn't follow the mail thread. Just regarding the DFLL part.
>>
>> As you know it, the DFLL clock is one of the CPU clock sources and
>> integrated with DVFS control logic with the regulator. We will not switch
>> CPU to other clock sources once we switched to DFLL. Because the CPU has
>> been regulated by the DFLL HW with the DVFS table (CVB or OPP table you see
>> in the driver.). We shouldn't reparent it to other sources with unknew
>> freq/volt pair. That's not guaranteed to work. We allow switching to
>> open-loop mode but different sources.

Okay, then the CPUFreq driver will have to enforce DFLL freq to PLLP's
rate before switching to PLLP in order to have a proper CPU voltage.

>> And I don't exactly understand why we need to switch to PLLP in CPU idle
>> driver. Just keep it on CL-DVFS mode all the time.
>>
>> In SC7 entry, the dfll suspend function moves it the open-loop mode. That's
>> all. The sc7-entryfirmware will handle the rest of the sequence to turn off
>> the CPU power.
>>
>> In SC7 resume, the warmboot code will handle the sequence to turn on
>> regulator and power up the CPU cluster. And leave it on PLL_P. After
>> resuming to the kernel, we re-init DFLL, restore the CPU clock policy (CPU
>> runs on DFLL open-loop mode) and then moving to close-loop mode.

The DFLL is re-inited after switching CCLK to DFLL parent during of the
early clocks-state restoring by CaR driver. Hence instead of having odd
hacks in the CaR driver, it is much nicer to have a proper
suspend-resume sequencing of the device drivers. In this case CPUFreq
driver is the driver that enables DFLL and switches CPU to that clock
source, which means that this driver is also should be responsible for
management of the DFLL's state during of suspend/resume process. If
CPUFreq driver disables DFLL during suspend and re-enables it during
resume, then looks like the CaR driver hacks around DFLL are not needed.

>> The DFLL part looks good to me. BTW, change the patch subject to "Add
>> suspend-resume support" seems more appropriate to me.
>>
> 
> To clarify this, the sequences for DFLL use are as follows (assuming all
> required DFLL hw configuration has been done)
> 
> Switch to DFLL:
> 0) Save current parent and frequency
> 1) Program DFLL to open loop mode
> 2) Enable DFLL
> 3) Change cclk_g parent to DFLL
> For OVR regulator:
> 4) Change PWM output pin from tristate to output
> 5) Enable DFLL PWM output
> For I2C regulator:
> 4) Enable DFLL I2C output
> 6) Program DFLL to closed loop mode
> 
> Switch away from DFLL:
> 0) Change cclk_g parent to PLLP so the CPU frequency is ok for any vdd_cpu voltage
> 1) Program DFLL to open loop mode
> 
> For OVR regulator:
> 2) Change PWM output pin from output to tristate: vdd_cpu will go back
>    to hardwired boot voltage.
> 3) Disable DFLL PWM output
> 
> For I2C regulator:
> 2) Program vdd_cpu regulator voltage to the boot voltage
> 3) Disable DFLL I2C output
> 
> 4) Reprogram parent saved in step 0 of 'Switch to DFLL' to the saved
>    frequency
> 5) Change cclk_g parent to saved parent
> 6) Disable DFLL

Thanks!

On 7/16/19 8:00 AM, Dmitry Osipenko wrote:
> 16.07.2019 11:06, Peter De Schrijver пишет:
>> On Tue, Jul 16, 2019 at 03:24:26PM +0800, Joseph Lo wrote:
>>>> OK, Will add to CPUFreq driver...
>>>>> The other thing that also need attention is that T124 CPUFreq driver
>>>>> implicitly relies on DFLL driver to be probed first, which is icky.
>>>>>
>>>> Should I add check for successful dfll clk register explicitly in
>>>> CPUFreq driver probe and defer till dfll clk registers?
> Probably you should use the "device links". See [1][2] for the example.
>
> [1]
> https://elixir.bootlin.com/linux/v5.2.1/source/drivers/gpu/drm/tegra/dc.c#L2383
>
> [2] https://www.kernel.org/doc/html/latest/driver-api/device_link.html
>
> Return EPROBE_DEFER instead of EINVAL if device_link_add() fails. And
> use of_find_device_by_node() to get the DFLL's device, see [3].
>
> [3]
> https://git.kernel.org/pub/scm/linux/kernel/git/next/linux-next.git/tree/drivers/devfreq/tegra20-devfreq.c#n100
Will go thru and add...
>>> Sorry, I didn't follow the mail thread. Just regarding the DFLL part.
>>>
>>> As you know it, the DFLL clock is one of the CPU clock sources and
>>> integrated with DVFS control logic with the regulator. We will not switch
>>> CPU to other clock sources once we switched to DFLL. Because the CPU has
>>> been regulated by the DFLL HW with the DVFS table (CVB or OPP table you see
>>> in the driver.). We shouldn't reparent it to other sources with unknew
>>> freq/volt pair. That's not guaranteed to work. We allow switching to
>>> open-loop mode but different sources.
> Okay, then the CPUFreq driver will have to enforce DFLL freq to PLLP's
> rate before switching to PLLP in order to have a proper CPU voltage.

PLLP freq is safe to work for any CPU voltage. So no need to enforce 
DFLL freq to PLLP rate before changing CCLK_G source to PLLP during suspend

>>> And I don't exactly understand why we need to switch to PLLP in CPU idle
>>> driver. Just keep it on CL-DVFS mode all the time.
>>>
>>> In SC7 entry, the dfll suspend function moves it the open-loop mode. That's
>>> all. The sc7-entryfirmware will handle the rest of the sequence to turn off
>>> the CPU power.
>>>
>>> In SC7 resume, the warmboot code will handle the sequence to turn on
>>> regulator and power up the CPU cluster. And leave it on PLL_P. After
>>> resuming to the kernel, we re-init DFLL, restore the CPU clock policy (CPU
>>> runs on DFLL open-loop mode) and then moving to close-loop mode.
> The DFLL is re-inited after switching CCLK to DFLL parent during of the
> early clocks-state restoring by CaR driver. Hence instead of having odd
> hacks in the CaR driver, it is much nicer to have a proper
> suspend-resume sequencing of the device drivers. In this case CPUFreq
> driver is the driver that enables DFLL and switches CPU to that clock
> source, which means that this driver is also should be responsible for
> management of the DFLL's state during of suspend/resume process. If
> CPUFreq driver disables DFLL during suspend and re-enables it during
> resume, then looks like the CaR driver hacks around DFLL are not needed.
>
>>> The DFLL part looks good to me. BTW, change the patch subject to "Add
>>> suspend-resume support" seems more appropriate to me.
>>>
>> To clarify this, the sequences for DFLL use are as follows (assuming all
>> required DFLL hw configuration has been done)
>>
>> Switch to DFLL:
>> 0) Save current parent and frequency
>> 1) Program DFLL to open loop mode
>> 2) Enable DFLL
>> 3) Change cclk_g parent to DFLL
>> For OVR regulator:
>> 4) Change PWM output pin from tristate to output
>> 5) Enable DFLL PWM output
>> For I2C regulator:
>> 4) Enable DFLL I2C output
>> 6) Program DFLL to closed loop mode
>>
>> Switch away from DFLL:
>> 0) Change cclk_g parent to PLLP so the CPU frequency is ok for any vdd_cpu voltage
>> 1) Program DFLL to open loop mode
>>
>> For OVR regulator:
>> 2) Change PWM output pin from output to tristate: vdd_cpu will go back
>>     to hardwired boot voltage.
>> 3) Disable DFLL PWM output
>>
>> For I2C regulator:
>> 2) Program vdd_cpu regulator voltage to the boot voltage
>> 3) Disable DFLL I2C output
>>
>> 4) Reprogram parent saved in step 0 of 'Switch to DFLL' to the saved
>>     frequency
>> 5) Change cclk_g parent to saved parent
>> 6) Disable DFLL

This is the same sequence currently implemented. But dfll suspend/resume 
calls are thru Tegra210 clock driver.

Dmitry wants to have dfll suspend/resume along with CCLK_G restore to 
happen from CPUFreq driver pm_ops rather than tegra210 clock driver or 
tegra dfll driver.

Will move it to CPUFreq driver...

> Thanks!

On 7/16/19 9:50 AM, Sowjanya Komatineni wrote:
>
> On 7/16/19 8:00 AM, Dmitry Osipenko wrote:
>> 16.07.2019 11:06, Peter De Schrijver пишет:
>>> On Tue, Jul 16, 2019 at 03:24:26PM +0800, Joseph Lo wrote:
>>>>> OK, Will add to CPUFreq driver...
>>>>>> The other thing that also need attention is that T124 CPUFreq driver
>>>>>> implicitly relies on DFLL driver to be probed first, which is icky.
>>>>>>
>>>>> Should I add check for successful dfll clk register explicitly in
>>>>> CPUFreq driver probe and defer till dfll clk registers?
>> Probably you should use the "device links". See [1][2] for the example.
>>
>> [1]
>> https://elixir.bootlin.com/linux/v5.2.1/source/drivers/gpu/drm/tegra/dc.c#L2383 
>>
>>
>> [2] https://www.kernel.org/doc/html/latest/driver-api/device_link.html
>>
>> Return EPROBE_DEFER instead of EINVAL if device_link_add() fails. And
>> use of_find_device_by_node() to get the DFLL's device, see [3].
>>
>> [3]
>> https://git.kernel.org/pub/scm/linux/kernel/git/next/linux-next.git/tree/drivers/devfreq/tegra20-devfreq.c#n100 
>>
> Will go thru and add...
>>>> Sorry, I didn't follow the mail thread. Just regarding the DFLL part.
>>>>
>>>> As you know it, the DFLL clock is one of the CPU clock sources and
>>>> integrated with DVFS control logic with the regulator. We will not 
>>>> switch
>>>> CPU to other clock sources once we switched to DFLL. Because the 
>>>> CPU has
>>>> been regulated by the DFLL HW with the DVFS table (CVB or OPP table 
>>>> you see
>>>> in the driver.). We shouldn't reparent it to other sources with unknew
>>>> freq/volt pair. That's not guaranteed to work. We allow switching to
>>>> open-loop mode but different sources.
>> Okay, then the CPUFreq driver will have to enforce DFLL freq to PLLP's
>> rate before switching to PLLP in order to have a proper CPU voltage.
>
> PLLP freq is safe to work for any CPU voltage. So no need to enforce 
> DFLL freq to PLLP rate before changing CCLK_G source to PLLP during 
> suspend
>
Sorry, please ignore my above comment. During suspend, need to change 
CCLK_G source to PLLP when dfll is in closed loop mode first and then 
dfll need to be set to open loop.
>>>> And I don't exactly understand why we need to switch to PLLP in CPU 
>>>> idle
>>>> driver. Just keep it on CL-DVFS mode all the time.
>>>>
>>>> In SC7 entry, the dfll suspend function moves it the open-loop 
>>>> mode. That's
>>>> all. The sc7-entryfirmware will handle the rest of the sequence to 
>>>> turn off
>>>> the CPU power.
>>>>
>>>> In SC7 resume, the warmboot code will handle the sequence to turn on
>>>> regulator and power up the CPU cluster. And leave it on PLL_P. After
>>>> resuming to the kernel, we re-init DFLL, restore the CPU clock 
>>>> policy (CPU
>>>> runs on DFLL open-loop mode) and then moving to close-loop mode.
>> The DFLL is re-inited after switching CCLK to DFLL parent during of the
>> early clocks-state restoring by CaR driver. Hence instead of having odd
>> hacks in the CaR driver, it is much nicer to have a proper
>> suspend-resume sequencing of the device drivers. In this case CPUFreq
>> driver is the driver that enables DFLL and switches CPU to that clock
>> source, which means that this driver is also should be responsible for
>> management of the DFLL's state during of suspend/resume process. If
>> CPUFreq driver disables DFLL during suspend and re-enables it during
>> resume, then looks like the CaR driver hacks around DFLL are not needed.
>>
>>>> The DFLL part looks good to me. BTW, change the patch subject to "Add
>>>> suspend-resume support" seems more appropriate to me.
>>>>
>>> To clarify this, the sequences for DFLL use are as follows (assuming 
>>> all
>>> required DFLL hw configuration has been done)
>>>
>>> Switch to DFLL:
>>> 0) Save current parent and frequency
>>> 1) Program DFLL to open loop mode
>>> 2) Enable DFLL
>>> 3) Change cclk_g parent to DFLL
>>> For OVR regulator:
>>> 4) Change PWM output pin from tristate to output
>>> 5) Enable DFLL PWM output
>>> For I2C regulator:
>>> 4) Enable DFLL I2C output
>>> 6) Program DFLL to closed loop mode
>>>
>>> Switch away from DFLL:
>>> 0) Change cclk_g parent to PLLP so the CPU frequency is ok for any 
>>> vdd_cpu voltage
>>> 1) Program DFLL to open loop mode
>>>
I see during switch away from DFLL (suspend), cclk_g parent is not 
changed to PLLP before changing dfll to open loop mode.

Will add this ...

>>> For OVR regulator:
>>> 2) Change PWM output pin from output to tristate: vdd_cpu will go back
>>>     to hardwired boot voltage.
>>> 3) Disable DFLL PWM output
>>>
>>> For I2C regulator:
>>> 2) Program vdd_cpu regulator voltage to the boot voltage
>>> 3) Disable DFLL I2C output
>>>
>>> 4) Reprogram parent saved in step 0 of 'Switch to DFLL' to the saved
>>>     frequency
>>> 5) Change cclk_g parent to saved parent
>>> 6) Disable DFLL
>
> This is the same sequence currently implemented. But dfll 
> suspend/resume calls are thru Tegra210 clock driver.
>
> Dmitry wants to have dfll suspend/resume along with CCLK_G restore to 
> happen from CPUFreq driver pm_ops rather than tegra210 clock driver or 
> tegra dfll driver.
>
> Will move it to CPUFreq driver...
>
Thanks!

16.07.2019 21:19, Sowjanya Komatineni пишет:
> 
> On 7/16/19 9:50 AM, Sowjanya Komatineni wrote:
>>
>> On 7/16/19 8:00 AM, Dmitry Osipenko wrote:
>>> 16.07.2019 11:06, Peter De Schrijver пишет:
>>>> On Tue, Jul 16, 2019 at 03:24:26PM +0800, Joseph Lo wrote:
>>>>>> OK, Will add to CPUFreq driver...
>>>>>>> The other thing that also need attention is that T124 CPUFreq driver
>>>>>>> implicitly relies on DFLL driver to be probed first, which is icky.
>>>>>>>
>>>>>> Should I add check for successful dfll clk register explicitly in
>>>>>> CPUFreq driver probe and defer till dfll clk registers?
>>> Probably you should use the "device links". See [1][2] for the example.
>>>
>>> [1]
>>> https://elixir.bootlin.com/linux/v5.2.1/source/drivers/gpu/drm/tegra/dc.c#L2383
>>>
>>>
>>> [2] https://www.kernel.org/doc/html/latest/driver-api/device_link.html
>>>
>>> Return EPROBE_DEFER instead of EINVAL if device_link_add() fails. And
>>> use of_find_device_by_node() to get the DFLL's device, see [3].
>>>
>>> [3]
>>> https://git.kernel.org/pub/scm/linux/kernel/git/next/linux-next.git/tree/drivers/devfreq/tegra20-devfreq.c#n100
>>>
>> Will go thru and add...

Looks like I initially confused this case with getting orphaned clock.
I'm now seeing that the DFLL driver registers the clock and then
clk_get(dfll) should be returning EPROBE_DEFER until DFLL driver is
probed, hence everything should be fine as-is and there is no real need
for the 'device link'. Sorry for the confusion!

>>>>> Sorry, I didn't follow the mail thread. Just regarding the DFLL part.
>>>>>
>>>>> As you know it, the DFLL clock is one of the CPU clock sources and
>>>>> integrated with DVFS control logic with the regulator. We will not
>>>>> switch
>>>>> CPU to other clock sources once we switched to DFLL. Because the
>>>>> CPU has
>>>>> been regulated by the DFLL HW with the DVFS table (CVB or OPP table
>>>>> you see
>>>>> in the driver.). We shouldn't reparent it to other sources with unknew
>>>>> freq/volt pair. That's not guaranteed to work. We allow switching to
>>>>> open-loop mode but different sources.
>>> Okay, then the CPUFreq driver will have to enforce DFLL freq to PLLP's
>>> rate before switching to PLLP in order to have a proper CPU voltage.
>>
>> PLLP freq is safe to work for any CPU voltage. So no need to enforce
>> DFLL freq to PLLP rate before changing CCLK_G source to PLLP during
>> suspend
>>
> Sorry, please ignore my above comment. During suspend, need to change
> CCLK_G source to PLLP when dfll is in closed loop mode first and then
> dfll need to be set to open loop.

Okay.

>>>>> And I don't exactly understand why we need to switch to PLLP in CPU
>>>>> idle
>>>>> driver. Just keep it on CL-DVFS mode all the time.
>>>>>
>>>>> In SC7 entry, the dfll suspend function moves it the open-loop
>>>>> mode. That's
>>>>> all. The sc7-entryfirmware will handle the rest of the sequence to
>>>>> turn off
>>>>> the CPU power.
>>>>>
>>>>> In SC7 resume, the warmboot code will handle the sequence to turn on
>>>>> regulator and power up the CPU cluster. And leave it on PLL_P. After
>>>>> resuming to the kernel, we re-init DFLL, restore the CPU clock
>>>>> policy (CPU
>>>>> runs on DFLL open-loop mode) and then moving to close-loop mode.
>>> The DFLL is re-inited after switching CCLK to DFLL parent during of the
>>> early clocks-state restoring by CaR driver. Hence instead of having odd
>>> hacks in the CaR driver, it is much nicer to have a proper
>>> suspend-resume sequencing of the device drivers. In this case CPUFreq
>>> driver is the driver that enables DFLL and switches CPU to that clock
>>> source, which means that this driver is also should be responsible for
>>> management of the DFLL's state during of suspend/resume process. If
>>> CPUFreq driver disables DFLL during suspend and re-enables it during
>>> resume, then looks like the CaR driver hacks around DFLL are not needed.
>>>
>>>>> The DFLL part looks good to me. BTW, change the patch subject to "Add
>>>>> suspend-resume support" seems more appropriate to me.
>>>>>
>>>> To clarify this, the sequences for DFLL use are as follows (assuming
>>>> all
>>>> required DFLL hw configuration has been done)
>>>>
>>>> Switch to DFLL:
>>>> 0) Save current parent and frequency
>>>> 1) Program DFLL to open loop mode
>>>> 2) Enable DFLL
>>>> 3) Change cclk_g parent to DFLL
>>>> For OVR regulator:
>>>> 4) Change PWM output pin from tristate to output
>>>> 5) Enable DFLL PWM output
>>>> For I2C regulator:
>>>> 4) Enable DFLL I2C output
>>>> 6) Program DFLL to closed loop mode
>>>>
>>>> Switch away from DFLL:
>>>> 0) Change cclk_g parent to PLLP so the CPU frequency is ok for any
>>>> vdd_cpu voltage
>>>> 1) Program DFLL to open loop mode
>>>>
> I see during switch away from DFLL (suspend), cclk_g parent is not
> changed to PLLP before changing dfll to open loop mode.
> 
> Will add this ...

The CPUFreq driver switches parent to PLLP during the probe, similar
should be done on suspend.

I'm also wondering if it's always safe to switch to PLLP in the probe.
If CPU is running on a lower freq than PLLP, then some other more
appropriate intermediate parent should be selected.

>>>> For OVR regulator:
>>>> 2) Change PWM output pin from output to tristate: vdd_cpu will go back
>>>>     to hardwired boot voltage.
>>>> 3) Disable DFLL PWM output
>>>>
>>>> For I2C regulator:
>>>> 2) Program vdd_cpu regulator voltage to the boot voltage
>>>> 3) Disable DFLL I2C output
>>>>
>>>> 4) Reprogram parent saved in step 0 of 'Switch to DFLL' to the saved
>>>>     frequency
>>>> 5) Change cclk_g parent to saved parent
>>>> 6) Disable DFLL
>>
>> This is the same sequence currently implemented. But dfll
>> suspend/resume calls are thru Tegra210 clock driver.
>>
>> Dmitry wants to have dfll suspend/resume along with CCLK_G restore to
>> happen from CPUFreq driver pm_ops rather than tegra210 clock driver or
>> tegra dfll driver.
>>
>> Will move it to CPUFreq driver...
>>
> Thanks!

On 7/16/19 11:25 AM, Dmitry Osipenko wrote:
> 16.07.2019 21:19, Sowjanya Komatineni пишет:
>> On 7/16/19 9:50 AM, Sowjanya Komatineni wrote:
>>> On 7/16/19 8:00 AM, Dmitry Osipenko wrote:
>>>> 16.07.2019 11:06, Peter De Schrijver пишет:
>>>>> On Tue, Jul 16, 2019 at 03:24:26PM +0800, Joseph Lo wrote:
>>>>>>> OK, Will add to CPUFreq driver...
>>>>>>>> The other thing that also need attention is that T124 CPUFreq driver
>>>>>>>> implicitly relies on DFLL driver to be probed first, which is icky.
>>>>>>>>
>>>>>>> Should I add check for successful dfll clk register explicitly in
>>>>>>> CPUFreq driver probe and defer till dfll clk registers?
>>>> Probably you should use the "device links". See [1][2] for the example.
>>>>
>>>> [1]
>>>> https://elixir.bootlin.com/linux/v5.2.1/source/drivers/gpu/drm/tegra/dc.c#L2383
>>>>
>>>>
>>>> [2] https://www.kernel.org/doc/html/latest/driver-api/device_link.html
>>>>
>>>> Return EPROBE_DEFER instead of EINVAL if device_link_add() fails. And
>>>> use of_find_device_by_node() to get the DFLL's device, see [3].
>>>>
>>>> [3]
>>>> https://git.kernel.org/pub/scm/linux/kernel/git/next/linux-next.git/tree/drivers/devfreq/tegra20-devfreq.c#n100
>>>>
>>> Will go thru and add...
> Looks like I initially confused this case with getting orphaned clock.
> I'm now seeing that the DFLL driver registers the clock and then
> clk_get(dfll) should be returning EPROBE_DEFER until DFLL driver is
> probed, hence everything should be fine as-is and there is no real need
> for the 'device link'. Sorry for the confusion!
>
>>>>>> Sorry, I didn't follow the mail thread. Just regarding the DFLL part.
>>>>>>
>>>>>> As you know it, the DFLL clock is one of the CPU clock sources and
>>>>>> integrated with DVFS control logic with the regulator. We will not
>>>>>> switch
>>>>>> CPU to other clock sources once we switched to DFLL. Because the
>>>>>> CPU has
>>>>>> been regulated by the DFLL HW with the DVFS table (CVB or OPP table
>>>>>> you see
>>>>>> in the driver.). We shouldn't reparent it to other sources with unknew
>>>>>> freq/volt pair. That's not guaranteed to work. We allow switching to
>>>>>> open-loop mode but different sources.
>>>> Okay, then the CPUFreq driver will have to enforce DFLL freq to PLLP's
>>>> rate before switching to PLLP in order to have a proper CPU voltage.
>>> PLLP freq is safe to work for any CPU voltage. So no need to enforce
>>> DFLL freq to PLLP rate before changing CCLK_G source to PLLP during
>>> suspend
>>>
>> Sorry, please ignore my above comment. During suspend, need to change
>> CCLK_G source to PLLP when dfll is in closed loop mode first and then
>> dfll need to be set to open loop.
> Okay.
>
>>>>>> And I don't exactly understand why we need to switch to PLLP in CPU
>>>>>> idle
>>>>>> driver. Just keep it on CL-DVFS mode all the time.
>>>>>>
>>>>>> In SC7 entry, the dfll suspend function moves it the open-loop
>>>>>> mode. That's
>>>>>> all. The sc7-entryfirmware will handle the rest of the sequence to
>>>>>> turn off
>>>>>> the CPU power.
>>>>>>
>>>>>> In SC7 resume, the warmboot code will handle the sequence to turn on
>>>>>> regulator and power up the CPU cluster. And leave it on PLL_P. After
>>>>>> resuming to the kernel, we re-init DFLL, restore the CPU clock
>>>>>> policy (CPU
>>>>>> runs on DFLL open-loop mode) and then moving to close-loop mode.
>>>> The DFLL is re-inited after switching CCLK to DFLL parent during of the
>>>> early clocks-state restoring by CaR driver. Hence instead of having odd
>>>> hacks in the CaR driver, it is much nicer to have a proper
>>>> suspend-resume sequencing of the device drivers. In this case CPUFreq
>>>> driver is the driver that enables DFLL and switches CPU to that clock
>>>> source, which means that this driver is also should be responsible for
>>>> management of the DFLL's state during of suspend/resume process. If
>>>> CPUFreq driver disables DFLL during suspend and re-enables it during
>>>> resume, then looks like the CaR driver hacks around DFLL are not needed.
>>>>
>>>>>> The DFLL part looks good to me. BTW, change the patch subject to "Add
>>>>>> suspend-resume support" seems more appropriate to me.
>>>>>>
>>>>> To clarify this, the sequences for DFLL use are as follows (assuming
>>>>> all
>>>>> required DFLL hw configuration has been done)
>>>>>
>>>>> Switch to DFLL:
>>>>> 0) Save current parent and frequency
>>>>> 1) Program DFLL to open loop mode
>>>>> 2) Enable DFLL
>>>>> 3) Change cclk_g parent to DFLL
>>>>> For OVR regulator:
>>>>> 4) Change PWM output pin from tristate to output
>>>>> 5) Enable DFLL PWM output
>>>>> For I2C regulator:
>>>>> 4) Enable DFLL I2C output
>>>>> 6) Program DFLL to closed loop mode
>>>>>
>>>>> Switch away from DFLL:
>>>>> 0) Change cclk_g parent to PLLP so the CPU frequency is ok for any
>>>>> vdd_cpu voltage
>>>>> 1) Program DFLL to open loop mode
>>>>>
>> I see during switch away from DFLL (suspend), cclk_g parent is not
>> changed to PLLP before changing dfll to open loop mode.
>>
>> Will add this ...
> The CPUFreq driver switches parent to PLLP during the probe, similar
> should be done on suspend.
>
> I'm also wondering if it's always safe to switch to PLLP in the probe.
> If CPU is running on a lower freq than PLLP, then some other more
> appropriate intermediate parent should be selected.
>
CPU parents are PLL_X, PLL_P, and dfll. PLL_X always runs at higher rate 
so switching to PLL_P during CPUFreq probe prior to dfll clock enable 
should be safe.
>>>>> For OVR regulator:
>>>>> 2) Change PWM output pin from output to tristate: vdd_cpu will go back
>>>>>      to hardwired boot voltage.
>>>>> 3) Disable DFLL PWM output
>>>>>
>>>>> For I2C regulator:
>>>>> 2) Program vdd_cpu regulator voltage to the boot voltage
>>>>> 3) Disable DFLL I2C output
>>>>>
>>>>> 4) Reprogram parent saved in step 0 of 'Switch to DFLL' to the saved
>>>>>      frequency
>>>>> 5) Change cclk_g parent to saved parent
>>>>> 6) Disable DFLL
>>> This is the same sequence currently implemented. But dfll
>>> suspend/resume calls are thru Tegra210 clock driver.
>>>
>>> Dmitry wants to have dfll suspend/resume along with CCLK_G restore to
>>> happen from CPUFreq driver pm_ops rather than tegra210 clock driver or
>>> tegra dfll driver.
>>>
>>> Will move it to CPUFreq driver...
>>>
>> Thanks!

16.07.2019 21:30, Sowjanya Komatineni пишет:
> 
> On 7/16/19 11:25 AM, Dmitry Osipenko wrote:
>> 16.07.2019 21:19, Sowjanya Komatineni пишет:
>>> On 7/16/19 9:50 AM, Sowjanya Komatineni wrote:
>>>> On 7/16/19 8:00 AM, Dmitry Osipenko wrote:
>>>>> 16.07.2019 11:06, Peter De Schrijver пишет:
>>>>>> On Tue, Jul 16, 2019 at 03:24:26PM +0800, Joseph Lo wrote:
>>>>>>>> OK, Will add to CPUFreq driver...
>>>>>>>>> The other thing that also need attention is that T124 CPUFreq
>>>>>>>>> driver
>>>>>>>>> implicitly relies on DFLL driver to be probed first, which is
>>>>>>>>> icky.
>>>>>>>>>
>>>>>>>> Should I add check for successful dfll clk register explicitly in
>>>>>>>> CPUFreq driver probe and defer till dfll clk registers?
>>>>> Probably you should use the "device links". See [1][2] for the
>>>>> example.
>>>>>
>>>>> [1]
>>>>> https://elixir.bootlin.com/linux/v5.2.1/source/drivers/gpu/drm/tegra/dc.c#L2383
>>>>>
>>>>>
>>>>>
>>>>> [2] https://www.kernel.org/doc/html/latest/driver-api/device_link.html
>>>>>
>>>>> Return EPROBE_DEFER instead of EINVAL if device_link_add() fails. And
>>>>> use of_find_device_by_node() to get the DFLL's device, see [3].
>>>>>
>>>>> [3]
>>>>> https://git.kernel.org/pub/scm/linux/kernel/git/next/linux-next.git/tree/drivers/devfreq/tegra20-devfreq.c#n100
>>>>>
>>>>>
>>>> Will go thru and add...
>> Looks like I initially confused this case with getting orphaned clock.
>> I'm now seeing that the DFLL driver registers the clock and then
>> clk_get(dfll) should be returning EPROBE_DEFER until DFLL driver is
>> probed, hence everything should be fine as-is and there is no real need
>> for the 'device link'. Sorry for the confusion!
>>
>>>>>>> Sorry, I didn't follow the mail thread. Just regarding the DFLL
>>>>>>> part.
>>>>>>>
>>>>>>> As you know it, the DFLL clock is one of the CPU clock sources and
>>>>>>> integrated with DVFS control logic with the regulator. We will not
>>>>>>> switch
>>>>>>> CPU to other clock sources once we switched to DFLL. Because the
>>>>>>> CPU has
>>>>>>> been regulated by the DFLL HW with the DVFS table (CVB or OPP table
>>>>>>> you see
>>>>>>> in the driver.). We shouldn't reparent it to other sources with
>>>>>>> unknew
>>>>>>> freq/volt pair. That's not guaranteed to work. We allow switching to
>>>>>>> open-loop mode but different sources.
>>>>> Okay, then the CPUFreq driver will have to enforce DFLL freq to PLLP's
>>>>> rate before switching to PLLP in order to have a proper CPU voltage.
>>>> PLLP freq is safe to work for any CPU voltage. So no need to enforce
>>>> DFLL freq to PLLP rate before changing CCLK_G source to PLLP during
>>>> suspend
>>>>
>>> Sorry, please ignore my above comment. During suspend, need to change
>>> CCLK_G source to PLLP when dfll is in closed loop mode first and then
>>> dfll need to be set to open loop.
>> Okay.
>>
>>>>>>> And I don't exactly understand why we need to switch to PLLP in CPU
>>>>>>> idle
>>>>>>> driver. Just keep it on CL-DVFS mode all the time.
>>>>>>>
>>>>>>> In SC7 entry, the dfll suspend function moves it the open-loop
>>>>>>> mode. That's
>>>>>>> all. The sc7-entryfirmware will handle the rest of the sequence to
>>>>>>> turn off
>>>>>>> the CPU power.
>>>>>>>
>>>>>>> In SC7 resume, the warmboot code will handle the sequence to turn on
>>>>>>> regulator and power up the CPU cluster. And leave it on PLL_P. After
>>>>>>> resuming to the kernel, we re-init DFLL, restore the CPU clock
>>>>>>> policy (CPU
>>>>>>> runs on DFLL open-loop mode) and then moving to close-loop mode.
>>>>> The DFLL is re-inited after switching CCLK to DFLL parent during of
>>>>> the
>>>>> early clocks-state restoring by CaR driver. Hence instead of having
>>>>> odd
>>>>> hacks in the CaR driver, it is much nicer to have a proper
>>>>> suspend-resume sequencing of the device drivers. In this case CPUFreq
>>>>> driver is the driver that enables DFLL and switches CPU to that clock
>>>>> source, which means that this driver is also should be responsible for
>>>>> management of the DFLL's state during of suspend/resume process. If
>>>>> CPUFreq driver disables DFLL during suspend and re-enables it during
>>>>> resume, then looks like the CaR driver hacks around DFLL are not
>>>>> needed.
>>>>>
>>>>>>> The DFLL part looks good to me. BTW, change the patch subject to
>>>>>>> "Add
>>>>>>> suspend-resume support" seems more appropriate to me.
>>>>>>>
>>>>>> To clarify this, the sequences for DFLL use are as follows (assuming
>>>>>> all
>>>>>> required DFLL hw configuration has been done)
>>>>>>
>>>>>> Switch to DFLL:
>>>>>> 0) Save current parent and frequency
>>>>>> 1) Program DFLL to open loop mode
>>>>>> 2) Enable DFLL
>>>>>> 3) Change cclk_g parent to DFLL
>>>>>> For OVR regulator:
>>>>>> 4) Change PWM output pin from tristate to output
>>>>>> 5) Enable DFLL PWM output
>>>>>> For I2C regulator:
>>>>>> 4) Enable DFLL I2C output
>>>>>> 6) Program DFLL to closed loop mode
>>>>>>
>>>>>> Switch away from DFLL:
>>>>>> 0) Change cclk_g parent to PLLP so the CPU frequency is ok for any
>>>>>> vdd_cpu voltage
>>>>>> 1) Program DFLL to open loop mode
>>>>>>
>>> I see during switch away from DFLL (suspend), cclk_g parent is not
>>> changed to PLLP before changing dfll to open loop mode.
>>>
>>> Will add this ...
>> The CPUFreq driver switches parent to PLLP during the probe, similar
>> should be done on suspend.
>>
>> I'm also wondering if it's always safe to switch to PLLP in the probe.
>> If CPU is running on a lower freq than PLLP, then some other more
>> appropriate intermediate parent should be selected.
>>
> CPU parents are PLL_X, PLL_P, and dfll. PLL_X always runs at higher rate
> so switching to PLL_P during CPUFreq probe prior to dfll clock enable
> should be safe.

AFAIK, PLLX could run at ~200MHz. There is also a divided output of PLLP
which CCLKG supports, the PLLP_OUT4.

Probably, realistically, CPU is always running off a fast PLLX during
boot, but I'm wondering what may happen on KEXEC. I guess ideally
CPUFreq driver should also have a 'shutdown' callback to teardown DFLL
on a reboot, but likely that there are other clock-related problems as
well that may break KEXEC and thus it is not very important at the moment.

[snip]

On 7/16/19 11:43 AM, Dmitry Osipenko wrote:
> 16.07.2019 21:30, Sowjanya Komatineni пишет:
>> On 7/16/19 11:25 AM, Dmitry Osipenko wrote:
>>> 16.07.2019 21:19, Sowjanya Komatineni пишет:
>>>> On 7/16/19 9:50 AM, Sowjanya Komatineni wrote:
>>>>> On 7/16/19 8:00 AM, Dmitry Osipenko wrote:
>>>>>> 16.07.2019 11:06, Peter De Schrijver пишет:
>>>>>>> On Tue, Jul 16, 2019 at 03:24:26PM +0800, Joseph Lo wrote:
>>>>>>>>> OK, Will add to CPUFreq driver...
>>>>>>>>>> The other thing that also need attention is that T124 CPUFreq
>>>>>>>>>> driver
>>>>>>>>>> implicitly relies on DFLL driver to be probed first, which is
>>>>>>>>>> icky.
>>>>>>>>>>
>>>>>>>>> Should I add check for successful dfll clk register explicitly in
>>>>>>>>> CPUFreq driver probe and defer till dfll clk registers?
>>>>>> Probably you should use the "device links". See [1][2] for the
>>>>>> example.
>>>>>>
>>>>>> [1]
>>>>>> https://elixir.bootlin.com/linux/v5.2.1/source/drivers/gpu/drm/tegra/dc.c#L2383
>>>>>>
>>>>>>
>>>>>>
>>>>>> [2] https://www.kernel.org/doc/html/latest/driver-api/device_link.html
>>>>>>
>>>>>> Return EPROBE_DEFER instead of EINVAL if device_link_add() fails. And
>>>>>> use of_find_device_by_node() to get the DFLL's device, see [3].
>>>>>>
>>>>>> [3]
>>>>>> https://git.kernel.org/pub/scm/linux/kernel/git/next/linux-next.git/tree/drivers/devfreq/tegra20-devfreq.c#n100
>>>>>>
>>>>>>
>>>>> Will go thru and add...
>>> Looks like I initially confused this case with getting orphaned clock.
>>> I'm now seeing that the DFLL driver registers the clock and then
>>> clk_get(dfll) should be returning EPROBE_DEFER until DFLL driver is
>>> probed, hence everything should be fine as-is and there is no real need
>>> for the 'device link'. Sorry for the confusion!
>>>
>>>>>>>> Sorry, I didn't follow the mail thread. Just regarding the DFLL
>>>>>>>> part.
>>>>>>>>
>>>>>>>> As you know it, the DFLL clock is one of the CPU clock sources and
>>>>>>>> integrated with DVFS control logic with the regulator. We will not
>>>>>>>> switch
>>>>>>>> CPU to other clock sources once we switched to DFLL. Because the
>>>>>>>> CPU has
>>>>>>>> been regulated by the DFLL HW with the DVFS table (CVB or OPP table
>>>>>>>> you see
>>>>>>>> in the driver.). We shouldn't reparent it to other sources with
>>>>>>>> unknew
>>>>>>>> freq/volt pair. That's not guaranteed to work. We allow switching to
>>>>>>>> open-loop mode but different sources.
>>>>>> Okay, then the CPUFreq driver will have to enforce DFLL freq to PLLP's
>>>>>> rate before switching to PLLP in order to have a proper CPU voltage.
>>>>> PLLP freq is safe to work for any CPU voltage. So no need to enforce
>>>>> DFLL freq to PLLP rate before changing CCLK_G source to PLLP during
>>>>> suspend
>>>>>
>>>> Sorry, please ignore my above comment. During suspend, need to change
>>>> CCLK_G source to PLLP when dfll is in closed loop mode first and then
>>>> dfll need to be set to open loop.
>>> Okay.
>>>
>>>>>>>> And I don't exactly understand why we need to switch to PLLP in CPU
>>>>>>>> idle
>>>>>>>> driver. Just keep it on CL-DVFS mode all the time.
>>>>>>>>
>>>>>>>> In SC7 entry, the dfll suspend function moves it the open-loop
>>>>>>>> mode. That's
>>>>>>>> all. The sc7-entryfirmware will handle the rest of the sequence to
>>>>>>>> turn off
>>>>>>>> the CPU power.
>>>>>>>>
>>>>>>>> In SC7 resume, the warmboot code will handle the sequence to turn on
>>>>>>>> regulator and power up the CPU cluster. And leave it on PLL_P. After
>>>>>>>> resuming to the kernel, we re-init DFLL, restore the CPU clock
>>>>>>>> policy (CPU
>>>>>>>> runs on DFLL open-loop mode) and then moving to close-loop mode.
>>>>>> The DFLL is re-inited after switching CCLK to DFLL parent during of
>>>>>> the
>>>>>> early clocks-state restoring by CaR driver. Hence instead of having
>>>>>> odd
>>>>>> hacks in the CaR driver, it is much nicer to have a proper
>>>>>> suspend-resume sequencing of the device drivers. In this case CPUFreq
>>>>>> driver is the driver that enables DFLL and switches CPU to that clock
>>>>>> source, which means that this driver is also should be responsible for
>>>>>> management of the DFLL's state during of suspend/resume process. If
>>>>>> CPUFreq driver disables DFLL during suspend and re-enables it during
>>>>>> resume, then looks like the CaR driver hacks around DFLL are not
>>>>>> needed.
>>>>>>
>>>>>>>> The DFLL part looks good to me. BTW, change the patch subject to
>>>>>>>> "Add
>>>>>>>> suspend-resume support" seems more appropriate to me.
>>>>>>>>
>>>>>>> To clarify this, the sequences for DFLL use are as follows (assuming
>>>>>>> all
>>>>>>> required DFLL hw configuration has been done)
>>>>>>>
>>>>>>> Switch to DFLL:
>>>>>>> 0) Save current parent and frequency
>>>>>>> 1) Program DFLL to open loop mode
>>>>>>> 2) Enable DFLL
>>>>>>> 3) Change cclk_g parent to DFLL
>>>>>>> For OVR regulator:
>>>>>>> 4) Change PWM output pin from tristate to output
>>>>>>> 5) Enable DFLL PWM output
>>>>>>> For I2C regulator:
>>>>>>> 4) Enable DFLL I2C output
>>>>>>> 6) Program DFLL to closed loop mode
>>>>>>>
>>>>>>> Switch away from DFLL:
>>>>>>> 0) Change cclk_g parent to PLLP so the CPU frequency is ok for any
>>>>>>> vdd_cpu voltage
>>>>>>> 1) Program DFLL to open loop mode
>>>>>>>
>>>> I see during switch away from DFLL (suspend), cclk_g parent is not
>>>> changed to PLLP before changing dfll to open loop mode.
>>>>
>>>> Will add this ...
>>> The CPUFreq driver switches parent to PLLP during the probe, similar
>>> should be done on suspend.
>>>
>>> I'm also wondering if it's always safe to switch to PLLP in the probe.
>>> If CPU is running on a lower freq than PLLP, then some other more
>>> appropriate intermediate parent should be selected.
>>>
>> CPU parents are PLL_X, PLL_P, and dfll. PLL_X always runs at higher rate
>> so switching to PLL_P during CPUFreq probe prior to dfll clock enable
>> should be safe.
> AFAIK, PLLX could run at ~200MHz. There is also a divided output of PLLP
> which CCLKG supports, the PLLP_OUT4.
>
> Probably, realistically, CPU is always running off a fast PLLX during
> boot, but I'm wondering what may happen on KEXEC. I guess ideally
> CPUFreq driver should also have a 'shutdown' callback to teardown DFLL
> on a reboot, but likely that there are other clock-related problems as
> well that may break KEXEC and thus it is not very important at the moment.
>
> [snip]

During bootup CPUG sources from PLL_X. By PLL_P source above I meant 
PLL_P_OUT4.

As per clock policies, PLL_X is always used for high freq like >800Mhz 
and for low frequency it will be sourced from PLLP.

16.07.2019 22:26, Sowjanya Komatineni пишет:
> 
> On 7/16/19 11:43 AM, Dmitry Osipenko wrote:
>> 16.07.2019 21:30, Sowjanya Komatineni пишет:
>>> On 7/16/19 11:25 AM, Dmitry Osipenko wrote:
>>>> 16.07.2019 21:19, Sowjanya Komatineni пишет:
>>>>> On 7/16/19 9:50 AM, Sowjanya Komatineni wrote:
>>>>>> On 7/16/19 8:00 AM, Dmitry Osipenko wrote:
>>>>>>> 16.07.2019 11:06, Peter De Schrijver пишет:
>>>>>>>> On Tue, Jul 16, 2019 at 03:24:26PM +0800, Joseph Lo wrote:
>>>>>>>>>> OK, Will add to CPUFreq driver...
>>>>>>>>>>> The other thing that also need attention is that T124 CPUFreq
>>>>>>>>>>> driver
>>>>>>>>>>> implicitly relies on DFLL driver to be probed first, which is
>>>>>>>>>>> icky.
>>>>>>>>>>>
>>>>>>>>>> Should I add check for successful dfll clk register explicitly in
>>>>>>>>>> CPUFreq driver probe and defer till dfll clk registers?
>>>>>>> Probably you should use the "device links". See [1][2] for the
>>>>>>> example.
>>>>>>>
>>>>>>> [1]
>>>>>>> https://elixir.bootlin.com/linux/v5.2.1/source/drivers/gpu/drm/tegra/dc.c#L2383
>>>>>>>
>>>>>>>
>>>>>>>
>>>>>>>
>>>>>>> [2]
>>>>>>> https://www.kernel.org/doc/html/latest/driver-api/device_link.html
>>>>>>>
>>>>>>> Return EPROBE_DEFER instead of EINVAL if device_link_add() fails.
>>>>>>> And
>>>>>>> use of_find_device_by_node() to get the DFLL's device, see [3].
>>>>>>>
>>>>>>> [3]
>>>>>>> https://git.kernel.org/pub/scm/linux/kernel/git/next/linux-next.git/tree/drivers/devfreq/tegra20-devfreq.c#n100
>>>>>>>
>>>>>>>
>>>>>>>
>>>>>> Will go thru and add...
>>>> Looks like I initially confused this case with getting orphaned clock.
>>>> I'm now seeing that the DFLL driver registers the clock and then
>>>> clk_get(dfll) should be returning EPROBE_DEFER until DFLL driver is
>>>> probed, hence everything should be fine as-is and there is no real need
>>>> for the 'device link'. Sorry for the confusion!
>>>>
>>>>>>>>> Sorry, I didn't follow the mail thread. Just regarding the DFLL
>>>>>>>>> part.
>>>>>>>>>
>>>>>>>>> As you know it, the DFLL clock is one of the CPU clock sources and
>>>>>>>>> integrated with DVFS control logic with the regulator. We will not
>>>>>>>>> switch
>>>>>>>>> CPU to other clock sources once we switched to DFLL. Because the
>>>>>>>>> CPU has
>>>>>>>>> been regulated by the DFLL HW with the DVFS table (CVB or OPP
>>>>>>>>> table
>>>>>>>>> you see
>>>>>>>>> in the driver.). We shouldn't reparent it to other sources with
>>>>>>>>> unknew
>>>>>>>>> freq/volt pair. That's not guaranteed to work. We allow
>>>>>>>>> switching to
>>>>>>>>> open-loop mode but different sources.
>>>>>>> Okay, then the CPUFreq driver will have to enforce DFLL freq to
>>>>>>> PLLP's
>>>>>>> rate before switching to PLLP in order to have a proper CPU voltage.
>>>>>> PLLP freq is safe to work for any CPU voltage. So no need to enforce
>>>>>> DFLL freq to PLLP rate before changing CCLK_G source to PLLP during
>>>>>> suspend
>>>>>>
>>>>> Sorry, please ignore my above comment. During suspend, need to change
>>>>> CCLK_G source to PLLP when dfll is in closed loop mode first and then
>>>>> dfll need to be set to open loop.
>>>> Okay.
>>>>
>>>>>>>>> And I don't exactly understand why we need to switch to PLLP in
>>>>>>>>> CPU
>>>>>>>>> idle
>>>>>>>>> driver. Just keep it on CL-DVFS mode all the time.
>>>>>>>>>
>>>>>>>>> In SC7 entry, the dfll suspend function moves it the open-loop
>>>>>>>>> mode. That's
>>>>>>>>> all. The sc7-entryfirmware will handle the rest of the sequence to
>>>>>>>>> turn off
>>>>>>>>> the CPU power.
>>>>>>>>>
>>>>>>>>> In SC7 resume, the warmboot code will handle the sequence to
>>>>>>>>> turn on
>>>>>>>>> regulator and power up the CPU cluster. And leave it on PLL_P.
>>>>>>>>> After
>>>>>>>>> resuming to the kernel, we re-init DFLL, restore the CPU clock
>>>>>>>>> policy (CPU
>>>>>>>>> runs on DFLL open-loop mode) and then moving to close-loop mode.
>>>>>>> The DFLL is re-inited after switching CCLK to DFLL parent during of
>>>>>>> the
>>>>>>> early clocks-state restoring by CaR driver. Hence instead of having
>>>>>>> odd
>>>>>>> hacks in the CaR driver, it is much nicer to have a proper
>>>>>>> suspend-resume sequencing of the device drivers. In this case
>>>>>>> CPUFreq
>>>>>>> driver is the driver that enables DFLL and switches CPU to that
>>>>>>> clock
>>>>>>> source, which means that this driver is also should be
>>>>>>> responsible for
>>>>>>> management of the DFLL's state during of suspend/resume process. If
>>>>>>> CPUFreq driver disables DFLL during suspend and re-enables it during
>>>>>>> resume, then looks like the CaR driver hacks around DFLL are not
>>>>>>> needed.
>>>>>>>
>>>>>>>>> The DFLL part looks good to me. BTW, change the patch subject to
>>>>>>>>> "Add
>>>>>>>>> suspend-resume support" seems more appropriate to me.
>>>>>>>>>
>>>>>>>> To clarify this, the sequences for DFLL use are as follows
>>>>>>>> (assuming
>>>>>>>> all
>>>>>>>> required DFLL hw configuration has been done)
>>>>>>>>
>>>>>>>> Switch to DFLL:
>>>>>>>> 0) Save current parent and frequency
>>>>>>>> 1) Program DFLL to open loop mode
>>>>>>>> 2) Enable DFLL
>>>>>>>> 3) Change cclk_g parent to DFLL
>>>>>>>> For OVR regulator:
>>>>>>>> 4) Change PWM output pin from tristate to output
>>>>>>>> 5) Enable DFLL PWM output
>>>>>>>> For I2C regulator:
>>>>>>>> 4) Enable DFLL I2C output
>>>>>>>> 6) Program DFLL to closed loop mode
>>>>>>>>
>>>>>>>> Switch away from DFLL:
>>>>>>>> 0) Change cclk_g parent to PLLP so the CPU frequency is ok for any
>>>>>>>> vdd_cpu voltage
>>>>>>>> 1) Program DFLL to open loop mode
>>>>>>>>
>>>>> I see during switch away from DFLL (suspend), cclk_g parent is not
>>>>> changed to PLLP before changing dfll to open loop mode.
>>>>>
>>>>> Will add this ...
>>>> The CPUFreq driver switches parent to PLLP during the probe, similar
>>>> should be done on suspend.
>>>>
>>>> I'm also wondering if it's always safe to switch to PLLP in the probe.
>>>> If CPU is running on a lower freq than PLLP, then some other more
>>>> appropriate intermediate parent should be selected.
>>>>
>>> CPU parents are PLL_X, PLL_P, and dfll. PLL_X always runs at higher rate
>>> so switching to PLL_P during CPUFreq probe prior to dfll clock enable
>>> should be safe.
>> AFAIK, PLLX could run at ~200MHz. There is also a divided output of PLLP
>> which CCLKG supports, the PLLP_OUT4.
>>
>> Probably, realistically, CPU is always running off a fast PLLX during
>> boot, but I'm wondering what may happen on KEXEC. I guess ideally
>> CPUFreq driver should also have a 'shutdown' callback to teardown DFLL
>> on a reboot, but likely that there are other clock-related problems as
>> well that may break KEXEC and thus it is not very important at the
>> moment.
>>
>> [snip]
> 
> During bootup CPUG sources from PLL_X. By PLL_P source above I meant
> PLL_P_OUT4.
> 
> As per clock policies, PLL_X is always used for high freq like >800Mhz
> and for low frequency it will be sourced from PLLP.

Alright, then please don't forget to pre-initialize PLLP_OUT4 rate to a
reasonable value using tegra_clk_init_table or assigned-clocks.

On 7/16/19 1:47 PM, Dmitry Osipenko wrote:
> 16.07.2019 22:26, Sowjanya Komatineni пишет:
>> On 7/16/19 11:43 AM, Dmitry Osipenko wrote:
>>> 16.07.2019 21:30, Sowjanya Komatineni пишет:
>>>> On 7/16/19 11:25 AM, Dmitry Osipenko wrote:
>>>>> 16.07.2019 21:19, Sowjanya Komatineni пишет:
>>>>>> On 7/16/19 9:50 AM, Sowjanya Komatineni wrote:
>>>>>>> On 7/16/19 8:00 AM, Dmitry Osipenko wrote:
>>>>>>>> 16.07.2019 11:06, Peter De Schrijver пишет:
>>>>>>>>> On Tue, Jul 16, 2019 at 03:24:26PM +0800, Joseph Lo wrote:
>>>>>>>>>>> OK, Will add to CPUFreq driver...
>>>>>>>>>>>> The other thing that also need attention is that T124 CPUFreq
>>>>>>>>>>>> driver
>>>>>>>>>>>> implicitly relies on DFLL driver to be probed first, which is
>>>>>>>>>>>> icky.
>>>>>>>>>>>>
>>>>>>>>>>> Should I add check for successful dfll clk register explicitly in
>>>>>>>>>>> CPUFreq driver probe and defer till dfll clk registers?
>>>>>>>> Probably you should use the "device links". See [1][2] for the
>>>>>>>> example.
>>>>>>>>
>>>>>>>> [1]
>>>>>>>> https://elixir.bootlin.com/linux/v5.2.1/source/drivers/gpu/drm/tegra/dc.c#L2383
>>>>>>>>
>>>>>>>>
>>>>>>>>
>>>>>>>>
>>>>>>>> [2]
>>>>>>>> https://www.kernel.org/doc/html/latest/driver-api/device_link.html
>>>>>>>>
>>>>>>>> Return EPROBE_DEFER instead of EINVAL if device_link_add() fails.
>>>>>>>> And
>>>>>>>> use of_find_device_by_node() to get the DFLL's device, see [3].
>>>>>>>>
>>>>>>>> [3]
>>>>>>>> https://git.kernel.org/pub/scm/linux/kernel/git/next/linux-next.git/tree/drivers/devfreq/tegra20-devfreq.c#n100
>>>>>>>>
>>>>>>>>
>>>>>>>>
>>>>>>> Will go thru and add...
>>>>> Looks like I initially confused this case with getting orphaned clock.
>>>>> I'm now seeing that the DFLL driver registers the clock and then
>>>>> clk_get(dfll) should be returning EPROBE_DEFER until DFLL driver is
>>>>> probed, hence everything should be fine as-is and there is no real need
>>>>> for the 'device link'. Sorry for the confusion!
>>>>>
>>>>>>>>>> Sorry, I didn't follow the mail thread. Just regarding the DFLL
>>>>>>>>>> part.
>>>>>>>>>>
>>>>>>>>>> As you know it, the DFLL clock is one of the CPU clock sources and
>>>>>>>>>> integrated with DVFS control logic with the regulator. We will not
>>>>>>>>>> switch
>>>>>>>>>> CPU to other clock sources once we switched to DFLL. Because the
>>>>>>>>>> CPU has
>>>>>>>>>> been regulated by the DFLL HW with the DVFS table (CVB or OPP
>>>>>>>>>> table
>>>>>>>>>> you see
>>>>>>>>>> in the driver.). We shouldn't reparent it to other sources with
>>>>>>>>>> unknew
>>>>>>>>>> freq/volt pair. That's not guaranteed to work. We allow
>>>>>>>>>> switching to
>>>>>>>>>> open-loop mode but different sources.
>>>>>>>> Okay, then the CPUFreq driver will have to enforce DFLL freq to
>>>>>>>> PLLP's
>>>>>>>> rate before switching to PLLP in order to have a proper CPU voltage.
>>>>>>> PLLP freq is safe to work for any CPU voltage. So no need to enforce
>>>>>>> DFLL freq to PLLP rate before changing CCLK_G source to PLLP during
>>>>>>> suspend
>>>>>>>
>>>>>> Sorry, please ignore my above comment. During suspend, need to change
>>>>>> CCLK_G source to PLLP when dfll is in closed loop mode first and then
>>>>>> dfll need to be set to open loop.
>>>>> Okay.
>>>>>
>>>>>>>>>> And I don't exactly understand why we need to switch to PLLP in
>>>>>>>>>> CPU
>>>>>>>>>> idle
>>>>>>>>>> driver. Just keep it on CL-DVFS mode all the time.
>>>>>>>>>>
>>>>>>>>>> In SC7 entry, the dfll suspend function moves it the open-loop
>>>>>>>>>> mode. That's
>>>>>>>>>> all. The sc7-entryfirmware will handle the rest of the sequence to
>>>>>>>>>> turn off
>>>>>>>>>> the CPU power.
>>>>>>>>>>
>>>>>>>>>> In SC7 resume, the warmboot code will handle the sequence to
>>>>>>>>>> turn on
>>>>>>>>>> regulator and power up the CPU cluster. And leave it on PLL_P.
>>>>>>>>>> After
>>>>>>>>>> resuming to the kernel, we re-init DFLL, restore the CPU clock
>>>>>>>>>> policy (CPU
>>>>>>>>>> runs on DFLL open-loop mode) and then moving to close-loop mode.
>>>>>>>> The DFLL is re-inited after switching CCLK to DFLL parent during of
>>>>>>>> the
>>>>>>>> early clocks-state restoring by CaR driver. Hence instead of having
>>>>>>>> odd
>>>>>>>> hacks in the CaR driver, it is much nicer to have a proper
>>>>>>>> suspend-resume sequencing of the device drivers. In this case
>>>>>>>> CPUFreq
>>>>>>>> driver is the driver that enables DFLL and switches CPU to that
>>>>>>>> clock
>>>>>>>> source, which means that this driver is also should be
>>>>>>>> responsible for
>>>>>>>> management of the DFLL's state during of suspend/resume process. If
>>>>>>>> CPUFreq driver disables DFLL during suspend and re-enables it during
>>>>>>>> resume, then looks like the CaR driver hacks around DFLL are not
>>>>>>>> needed.
>>>>>>>>
>>>>>>>>>> The DFLL part looks good to me. BTW, change the patch subject to
>>>>>>>>>> "Add
>>>>>>>>>> suspend-resume support" seems more appropriate to me.
>>>>>>>>>>
>>>>>>>>> To clarify this, the sequences for DFLL use are as follows
>>>>>>>>> (assuming
>>>>>>>>> all
>>>>>>>>> required DFLL hw configuration has been done)
>>>>>>>>>
>>>>>>>>> Switch to DFLL:
>>>>>>>>> 0) Save current parent and frequency
>>>>>>>>> 1) Program DFLL to open loop mode
>>>>>>>>> 2) Enable DFLL
>>>>>>>>> 3) Change cclk_g parent to DFLL
>>>>>>>>> For OVR regulator:
>>>>>>>>> 4) Change PWM output pin from tristate to output
>>>>>>>>> 5) Enable DFLL PWM output
>>>>>>>>> For I2C regulator:
>>>>>>>>> 4) Enable DFLL I2C output
>>>>>>>>> 6) Program DFLL to closed loop mode
>>>>>>>>>
>>>>>>>>> Switch away from DFLL:
>>>>>>>>> 0) Change cclk_g parent to PLLP so the CPU frequency is ok for any
>>>>>>>>> vdd_cpu voltage
>>>>>>>>> 1) Program DFLL to open loop mode
>>>>>>>>>
>>>>>> I see during switch away from DFLL (suspend), cclk_g parent is not
>>>>>> changed to PLLP before changing dfll to open loop mode.
>>>>>>
>>>>>> Will add this ...
>>>>> The CPUFreq driver switches parent to PLLP during the probe, similar
>>>>> should be done on suspend.
>>>>>
>>>>> I'm also wondering if it's always safe to switch to PLLP in the probe.
>>>>> If CPU is running on a lower freq than PLLP, then some other more
>>>>> appropriate intermediate parent should be selected.
>>>>>
>>>> CPU parents are PLL_X, PLL_P, and dfll. PLL_X always runs at higher rate
>>>> so switching to PLL_P during CPUFreq probe prior to dfll clock enable
>>>> should be safe.
>>> AFAIK, PLLX could run at ~200MHz. There is also a divided output of PLLP
>>> which CCLKG supports, the PLLP_OUT4.
>>>
>>> Probably, realistically, CPU is always running off a fast PLLX during
>>> boot, but I'm wondering what may happen on KEXEC. I guess ideally
>>> CPUFreq driver should also have a 'shutdown' callback to teardown DFLL
>>> on a reboot, but likely that there are other clock-related problems as
>>> well that may break KEXEC and thus it is not very important at the
>>> moment.
>>>
>>> [snip]
>> During bootup CPUG sources from PLL_X. By PLL_P source above I meant
>> PLL_P_OUT4.
>>
>> As per clock policies, PLL_X is always used for high freq like >800Mhz
>> and for low frequency it will be sourced from PLLP.
> Alright, then please don't forget to pre-initialize PLLP_OUT4 rate to a
> reasonable value using tegra_clk_init_table or assigned-clocks.

PLLP_OUT4 rate update is not needed as it is safe to run at 408Mhz 
because it is below fmax @ Vmin

17.07.2019 0:12, Sowjanya Komatineni пишет:
> 
> On 7/16/19 1:47 PM, Dmitry Osipenko wrote:
>> 16.07.2019 22:26, Sowjanya Komatineni пишет:
>>> On 7/16/19 11:43 AM, Dmitry Osipenko wrote:
>>>> 16.07.2019 21:30, Sowjanya Komatineni пишет:
>>>>> On 7/16/19 11:25 AM, Dmitry Osipenko wrote:
>>>>>> 16.07.2019 21:19, Sowjanya Komatineni пишет:
>>>>>>> On 7/16/19 9:50 AM, Sowjanya Komatineni wrote:
>>>>>>>> On 7/16/19 8:00 AM, Dmitry Osipenko wrote:
>>>>>>>>> 16.07.2019 11:06, Peter De Schrijver пишет:
>>>>>>>>>> On Tue, Jul 16, 2019 at 03:24:26PM +0800, Joseph Lo wrote:
>>>>>>>>>>>> OK, Will add to CPUFreq driver...
>>>>>>>>>>>>> The other thing that also need attention is that T124 CPUFreq
>>>>>>>>>>>>> driver
>>>>>>>>>>>>> implicitly relies on DFLL driver to be probed first, which is
>>>>>>>>>>>>> icky.
>>>>>>>>>>>>>
>>>>>>>>>>>> Should I add check for successful dfll clk register
>>>>>>>>>>>> explicitly in
>>>>>>>>>>>> CPUFreq driver probe and defer till dfll clk registers?
>>>>>>>>> Probably you should use the "device links". See [1][2] for the
>>>>>>>>> example.
>>>>>>>>>
>>>>>>>>> [1]
>>>>>>>>> https://elixir.bootlin.com/linux/v5.2.1/source/drivers/gpu/drm/tegra/dc.c#L2383
>>>>>>>>>
>>>>>>>>>
>>>>>>>>>
>>>>>>>>>
>>>>>>>>>
>>>>>>>>> [2]
>>>>>>>>> https://www.kernel.org/doc/html/latest/driver-api/device_link.html
>>>>>>>>>
>>>>>>>>> Return EPROBE_DEFER instead of EINVAL if device_link_add() fails.
>>>>>>>>> And
>>>>>>>>> use of_find_device_by_node() to get the DFLL's device, see [3].
>>>>>>>>>
>>>>>>>>> [3]
>>>>>>>>> https://git.kernel.org/pub/scm/linux/kernel/git/next/linux-next.git/tree/drivers/devfreq/tegra20-devfreq.c#n100
>>>>>>>>>
>>>>>>>>>
>>>>>>>>>
>>>>>>>>>
>>>>>>>> Will go thru and add...
>>>>>> Looks like I initially confused this case with getting orphaned
>>>>>> clock.
>>>>>> I'm now seeing that the DFLL driver registers the clock and then
>>>>>> clk_get(dfll) should be returning EPROBE_DEFER until DFLL driver is
>>>>>> probed, hence everything should be fine as-is and there is no real
>>>>>> need
>>>>>> for the 'device link'. Sorry for the confusion!
>>>>>>
>>>>>>>>>>> Sorry, I didn't follow the mail thread. Just regarding the DFLL
>>>>>>>>>>> part.
>>>>>>>>>>>
>>>>>>>>>>> As you know it, the DFLL clock is one of the CPU clock
>>>>>>>>>>> sources and
>>>>>>>>>>> integrated with DVFS control logic with the regulator. We
>>>>>>>>>>> will not
>>>>>>>>>>> switch
>>>>>>>>>>> CPU to other clock sources once we switched to DFLL. Because the
>>>>>>>>>>> CPU has
>>>>>>>>>>> been regulated by the DFLL HW with the DVFS table (CVB or OPP
>>>>>>>>>>> table
>>>>>>>>>>> you see
>>>>>>>>>>> in the driver.). We shouldn't reparent it to other sources with
>>>>>>>>>>> unknew
>>>>>>>>>>> freq/volt pair. That's not guaranteed to work. We allow
>>>>>>>>>>> switching to
>>>>>>>>>>> open-loop mode but different sources.
>>>>>>>>> Okay, then the CPUFreq driver will have to enforce DFLL freq to
>>>>>>>>> PLLP's
>>>>>>>>> rate before switching to PLLP in order to have a proper CPU
>>>>>>>>> voltage.
>>>>>>>> PLLP freq is safe to work for any CPU voltage. So no need to
>>>>>>>> enforce
>>>>>>>> DFLL freq to PLLP rate before changing CCLK_G source to PLLP during
>>>>>>>> suspend
>>>>>>>>
>>>>>>> Sorry, please ignore my above comment. During suspend, need to
>>>>>>> change
>>>>>>> CCLK_G source to PLLP when dfll is in closed loop mode first and
>>>>>>> then
>>>>>>> dfll need to be set to open loop.
>>>>>> Okay.
>>>>>>
>>>>>>>>>>> And I don't exactly understand why we need to switch to PLLP in
>>>>>>>>>>> CPU
>>>>>>>>>>> idle
>>>>>>>>>>> driver. Just keep it on CL-DVFS mode all the time.
>>>>>>>>>>>
>>>>>>>>>>> In SC7 entry, the dfll suspend function moves it the open-loop
>>>>>>>>>>> mode. That's
>>>>>>>>>>> all. The sc7-entryfirmware will handle the rest of the
>>>>>>>>>>> sequence to
>>>>>>>>>>> turn off
>>>>>>>>>>> the CPU power.
>>>>>>>>>>>
>>>>>>>>>>> In SC7 resume, the warmboot code will handle the sequence to
>>>>>>>>>>> turn on
>>>>>>>>>>> regulator and power up the CPU cluster. And leave it on PLL_P.
>>>>>>>>>>> After
>>>>>>>>>>> resuming to the kernel, we re-init DFLL, restore the CPU clock
>>>>>>>>>>> policy (CPU
>>>>>>>>>>> runs on DFLL open-loop mode) and then moving to close-loop mode.
>>>>>>>>> The DFLL is re-inited after switching CCLK to DFLL parent
>>>>>>>>> during of
>>>>>>>>> the
>>>>>>>>> early clocks-state restoring by CaR driver. Hence instead of
>>>>>>>>> having
>>>>>>>>> odd
>>>>>>>>> hacks in the CaR driver, it is much nicer to have a proper
>>>>>>>>> suspend-resume sequencing of the device drivers. In this case
>>>>>>>>> CPUFreq
>>>>>>>>> driver is the driver that enables DFLL and switches CPU to that
>>>>>>>>> clock
>>>>>>>>> source, which means that this driver is also should be
>>>>>>>>> responsible for
>>>>>>>>> management of the DFLL's state during of suspend/resume
>>>>>>>>> process. If
>>>>>>>>> CPUFreq driver disables DFLL during suspend and re-enables it
>>>>>>>>> during
>>>>>>>>> resume, then looks like the CaR driver hacks around DFLL are not
>>>>>>>>> needed.
>>>>>>>>>
>>>>>>>>>>> The DFLL part looks good to me. BTW, change the patch subject to
>>>>>>>>>>> "Add
>>>>>>>>>>> suspend-resume support" seems more appropriate to me.
>>>>>>>>>>>
>>>>>>>>>> To clarify this, the sequences for DFLL use are as follows
>>>>>>>>>> (assuming
>>>>>>>>>> all
>>>>>>>>>> required DFLL hw configuration has been done)
>>>>>>>>>>
>>>>>>>>>> Switch to DFLL:
>>>>>>>>>> 0) Save current parent and frequency
>>>>>>>>>> 1) Program DFLL to open loop mode
>>>>>>>>>> 2) Enable DFLL
>>>>>>>>>> 3) Change cclk_g parent to DFLL
>>>>>>>>>> For OVR regulator:
>>>>>>>>>> 4) Change PWM output pin from tristate to output
>>>>>>>>>> 5) Enable DFLL PWM output
>>>>>>>>>> For I2C regulator:
>>>>>>>>>> 4) Enable DFLL I2C output
>>>>>>>>>> 6) Program DFLL to closed loop mode
>>>>>>>>>>
>>>>>>>>>> Switch away from DFLL:
>>>>>>>>>> 0) Change cclk_g parent to PLLP so the CPU frequency is ok for
>>>>>>>>>> any
>>>>>>>>>> vdd_cpu voltage
>>>>>>>>>> 1) Program DFLL to open loop mode
>>>>>>>>>>
>>>>>>> I see during switch away from DFLL (suspend), cclk_g parent is not
>>>>>>> changed to PLLP before changing dfll to open loop mode.
>>>>>>>
>>>>>>> Will add this ...
>>>>>> The CPUFreq driver switches parent to PLLP during the probe, similar
>>>>>> should be done on suspend.
>>>>>>
>>>>>> I'm also wondering if it's always safe to switch to PLLP in the
>>>>>> probe.
>>>>>> If CPU is running on a lower freq than PLLP, then some other more
>>>>>> appropriate intermediate parent should be selected.
>>>>>>
>>>>> CPU parents are PLL_X, PLL_P, and dfll. PLL_X always runs at higher
>>>>> rate
>>>>> so switching to PLL_P during CPUFreq probe prior to dfll clock enable
>>>>> should be safe.
>>>> AFAIK, PLLX could run at ~200MHz. There is also a divided output of
>>>> PLLP
>>>> which CCLKG supports, the PLLP_OUT4.
>>>>
>>>> Probably, realistically, CPU is always running off a fast PLLX during
>>>> boot, but I'm wondering what may happen on KEXEC. I guess ideally
>>>> CPUFreq driver should also have a 'shutdown' callback to teardown DFLL
>>>> on a reboot, but likely that there are other clock-related problems as
>>>> well that may break KEXEC and thus it is not very important at the
>>>> moment.
>>>>
>>>> [snip]
>>> During bootup CPUG sources from PLL_X. By PLL_P source above I meant
>>> PLL_P_OUT4.
>>>
>>> As per clock policies, PLL_X is always used for high freq like >800Mhz
>>> and for low frequency it will be sourced from PLLP.
>> Alright, then please don't forget to pre-initialize PLLP_OUT4 rate to a
>> reasonable value using tegra_clk_init_table or assigned-clocks.
> 
> PLLP_OUT4 rate update is not needed as it is safe to run at 408Mhz
> because it is below fmax @ Vmin

So even 204MHz CVB entries are having the same voltage as 408MHz,
correct? It's not instantly obvious to me from the DFLL driver's code
where the fmax @ Vmin is defined, I see that there is the min_millivolts
and frequency entries starting from 204MHZ defined per-table.

On 7/16/19 2:21 PM, Dmitry Osipenko wrote:
> 17.07.2019 0:12, Sowjanya Komatineni пишет:
>> On 7/16/19 1:47 PM, Dmitry Osipenko wrote:
>>> 16.07.2019 22:26, Sowjanya Komatineni пишет:
>>>> On 7/16/19 11:43 AM, Dmitry Osipenko wrote:
>>>>> 16.07.2019 21:30, Sowjanya Komatineni пишет:
>>>>>> On 7/16/19 11:25 AM, Dmitry Osipenko wrote:
>>>>>>> 16.07.2019 21:19, Sowjanya Komatineni пишет:
>>>>>>>> On 7/16/19 9:50 AM, Sowjanya Komatineni wrote:
>>>>>>>>> On 7/16/19 8:00 AM, Dmitry Osipenko wrote:
>>>>>>>>>> 16.07.2019 11:06, Peter De Schrijver пишет:
>>>>>>>>>>> On Tue, Jul 16, 2019 at 03:24:26PM +0800, Joseph Lo wrote:
>>>>>>>>>>>>> OK, Will add to CPUFreq driver...
>>>>>>>>>>>>>> The other thing that also need attention is that T124 CPUFreq
>>>>>>>>>>>>>> driver
>>>>>>>>>>>>>> implicitly relies on DFLL driver to be probed first, which is
>>>>>>>>>>>>>> icky.
>>>>>>>>>>>>>>
>>>>>>>>>>>>> Should I add check for successful dfll clk register
>>>>>>>>>>>>> explicitly in
>>>>>>>>>>>>> CPUFreq driver probe and defer till dfll clk registers?
>>>>>>>>>> Probably you should use the "device links". See [1][2] for the
>>>>>>>>>> example.
>>>>>>>>>>
>>>>>>>>>> [1]
>>>>>>>>>> https://elixir.bootlin.com/linux/v5.2.1/source/drivers/gpu/drm/tegra/dc.c#L2383
>>>>>>>>>>
>>>>>>>>>>
>>>>>>>>>>
>>>>>>>>>>
>>>>>>>>>>
>>>>>>>>>> [2]
>>>>>>>>>> https://www.kernel.org/doc/html/latest/driver-api/device_link.html
>>>>>>>>>>
>>>>>>>>>> Return EPROBE_DEFER instead of EINVAL if device_link_add() fails.
>>>>>>>>>> And
>>>>>>>>>> use of_find_device_by_node() to get the DFLL's device, see [3].
>>>>>>>>>>
>>>>>>>>>> [3]
>>>>>>>>>> https://git.kernel.org/pub/scm/linux/kernel/git/next/linux-next.git/tree/drivers/devfreq/tegra20-devfreq.c#n100
>>>>>>>>>>
>>>>>>>>>>
>>>>>>>>>>
>>>>>>>>>>
>>>>>>>>> Will go thru and add...
>>>>>>> Looks like I initially confused this case with getting orphaned
>>>>>>> clock.
>>>>>>> I'm now seeing that the DFLL driver registers the clock and then
>>>>>>> clk_get(dfll) should be returning EPROBE_DEFER until DFLL driver is
>>>>>>> probed, hence everything should be fine as-is and there is no real
>>>>>>> need
>>>>>>> for the 'device link'. Sorry for the confusion!
>>>>>>>
>>>>>>>>>>>> Sorry, I didn't follow the mail thread. Just regarding the DFLL
>>>>>>>>>>>> part.
>>>>>>>>>>>>
>>>>>>>>>>>> As you know it, the DFLL clock is one of the CPU clock
>>>>>>>>>>>> sources and
>>>>>>>>>>>> integrated with DVFS control logic with the regulator. We
>>>>>>>>>>>> will not
>>>>>>>>>>>> switch
>>>>>>>>>>>> CPU to other clock sources once we switched to DFLL. Because the
>>>>>>>>>>>> CPU has
>>>>>>>>>>>> been regulated by the DFLL HW with the DVFS table (CVB or OPP
>>>>>>>>>>>> table
>>>>>>>>>>>> you see
>>>>>>>>>>>> in the driver.). We shouldn't reparent it to other sources with
>>>>>>>>>>>> unknew
>>>>>>>>>>>> freq/volt pair. That's not guaranteed to work. We allow
>>>>>>>>>>>> switching to
>>>>>>>>>>>> open-loop mode but different sources.
>>>>>>>>>> Okay, then the CPUFreq driver will have to enforce DFLL freq to
>>>>>>>>>> PLLP's
>>>>>>>>>> rate before switching to PLLP in order to have a proper CPU
>>>>>>>>>> voltage.
>>>>>>>>> PLLP freq is safe to work for any CPU voltage. So no need to
>>>>>>>>> enforce
>>>>>>>>> DFLL freq to PLLP rate before changing CCLK_G source to PLLP during
>>>>>>>>> suspend
>>>>>>>>>
>>>>>>>> Sorry, please ignore my above comment. During suspend, need to
>>>>>>>> change
>>>>>>>> CCLK_G source to PLLP when dfll is in closed loop mode first and
>>>>>>>> then
>>>>>>>> dfll need to be set to open loop.
>>>>>>> Okay.
>>>>>>>
>>>>>>>>>>>> And I don't exactly understand why we need to switch to PLLP in
>>>>>>>>>>>> CPU
>>>>>>>>>>>> idle
>>>>>>>>>>>> driver. Just keep it on CL-DVFS mode all the time.
>>>>>>>>>>>>
>>>>>>>>>>>> In SC7 entry, the dfll suspend function moves it the open-loop
>>>>>>>>>>>> mode. That's
>>>>>>>>>>>> all. The sc7-entryfirmware will handle the rest of the
>>>>>>>>>>>> sequence to
>>>>>>>>>>>> turn off
>>>>>>>>>>>> the CPU power.
>>>>>>>>>>>>
>>>>>>>>>>>> In SC7 resume, the warmboot code will handle the sequence to
>>>>>>>>>>>> turn on
>>>>>>>>>>>> regulator and power up the CPU cluster. And leave it on PLL_P.
>>>>>>>>>>>> After
>>>>>>>>>>>> resuming to the kernel, we re-init DFLL, restore the CPU clock
>>>>>>>>>>>> policy (CPU
>>>>>>>>>>>> runs on DFLL open-loop mode) and then moving to close-loop mode.
>>>>>>>>>> The DFLL is re-inited after switching CCLK to DFLL parent
>>>>>>>>>> during of
>>>>>>>>>> the
>>>>>>>>>> early clocks-state restoring by CaR driver. Hence instead of
>>>>>>>>>> having
>>>>>>>>>> odd
>>>>>>>>>> hacks in the CaR driver, it is much nicer to have a proper
>>>>>>>>>> suspend-resume sequencing of the device drivers. In this case
>>>>>>>>>> CPUFreq
>>>>>>>>>> driver is the driver that enables DFLL and switches CPU to that
>>>>>>>>>> clock
>>>>>>>>>> source, which means that this driver is also should be
>>>>>>>>>> responsible for
>>>>>>>>>> management of the DFLL's state during of suspend/resume
>>>>>>>>>> process. If
>>>>>>>>>> CPUFreq driver disables DFLL during suspend and re-enables it
>>>>>>>>>> during
>>>>>>>>>> resume, then looks like the CaR driver hacks around DFLL are not
>>>>>>>>>> needed.
>>>>>>>>>>
>>>>>>>>>>>> The DFLL part looks good to me. BTW, change the patch subject to
>>>>>>>>>>>> "Add
>>>>>>>>>>>> suspend-resume support" seems more appropriate to me.
>>>>>>>>>>>>
>>>>>>>>>>> To clarify this, the sequences for DFLL use are as follows
>>>>>>>>>>> (assuming
>>>>>>>>>>> all
>>>>>>>>>>> required DFLL hw configuration has been done)
>>>>>>>>>>>
>>>>>>>>>>> Switch to DFLL:
>>>>>>>>>>> 0) Save current parent and frequency
>>>>>>>>>>> 1) Program DFLL to open loop mode
>>>>>>>>>>> 2) Enable DFLL
>>>>>>>>>>> 3) Change cclk_g parent to DFLL
>>>>>>>>>>> For OVR regulator:
>>>>>>>>>>> 4) Change PWM output pin from tristate to output
>>>>>>>>>>> 5) Enable DFLL PWM output
>>>>>>>>>>> For I2C regulator:
>>>>>>>>>>> 4) Enable DFLL I2C output
>>>>>>>>>>> 6) Program DFLL to closed loop mode
>>>>>>>>>>>
>>>>>>>>>>> Switch away from DFLL:
>>>>>>>>>>> 0) Change cclk_g parent to PLLP so the CPU frequency is ok for
>>>>>>>>>>> any
>>>>>>>>>>> vdd_cpu voltage
>>>>>>>>>>> 1) Program DFLL to open loop mode
>>>>>>>>>>>
>>>>>>>> I see during switch away from DFLL (suspend), cclk_g parent is not
>>>>>>>> changed to PLLP before changing dfll to open loop mode.
>>>>>>>>
>>>>>>>> Will add this ...
>>>>>>> The CPUFreq driver switches parent to PLLP during the probe, similar
>>>>>>> should be done on suspend.
>>>>>>>
>>>>>>> I'm also wondering if it's always safe to switch to PLLP in the
>>>>>>> probe.
>>>>>>> If CPU is running on a lower freq than PLLP, then some other more
>>>>>>> appropriate intermediate parent should be selected.
>>>>>>>
>>>>>> CPU parents are PLL_X, PLL_P, and dfll. PLL_X always runs at higher
>>>>>> rate
>>>>>> so switching to PLL_P during CPUFreq probe prior to dfll clock enable
>>>>>> should be safe.
>>>>> AFAIK, PLLX could run at ~200MHz. There is also a divided output of
>>>>> PLLP
>>>>> which CCLKG supports, the PLLP_OUT4.
>>>>>
>>>>> Probably, realistically, CPU is always running off a fast PLLX during
>>>>> boot, but I'm wondering what may happen on KEXEC. I guess ideally
>>>>> CPUFreq driver should also have a 'shutdown' callback to teardown DFLL
>>>>> on a reboot, but likely that there are other clock-related problems as
>>>>> well that may break KEXEC and thus it is not very important at the
>>>>> moment.
>>>>>
>>>>> [snip]
>>>> During bootup CPUG sources from PLL_X. By PLL_P source above I meant
>>>> PLL_P_OUT4.
>>>>
>>>> As per clock policies, PLL_X is always used for high freq like >800Mhz
>>>> and for low frequency it will be sourced from PLLP.
>>> Alright, then please don't forget to pre-initialize PLLP_OUT4 rate to a
>>> reasonable value using tegra_clk_init_table or assigned-clocks.
>> PLLP_OUT4 rate update is not needed as it is safe to run at 408Mhz
>> because it is below fmax @ Vmin
> So even 204MHz CVB entries are having the same voltage as 408MHz,
> correct? It's not instantly obvious to me from the DFLL driver's code
> where the fmax @ Vmin is defined, I see that there is the min_millivolts
> and frequency entries starting from 204MHZ defined per-table.
Yes at Vmin CPU Fmax is ~800Mhz. So anything below that will work at 
Vmin voltage and PLLP max is 408Mhz.

17.07.2019 0:35, Sowjanya Komatineni пишет:
> 
> On 7/16/19 2:21 PM, Dmitry Osipenko wrote:
>> 17.07.2019 0:12, Sowjanya Komatineni пишет:
>>> On 7/16/19 1:47 PM, Dmitry Osipenko wrote:
>>>> 16.07.2019 22:26, Sowjanya Komatineni пишет:
>>>>> On 7/16/19 11:43 AM, Dmitry Osipenko wrote:
>>>>>> 16.07.2019 21:30, Sowjanya Komatineni пишет:
>>>>>>> On 7/16/19 11:25 AM, Dmitry Osipenko wrote:
>>>>>>>> 16.07.2019 21:19, Sowjanya Komatineni пишет:
>>>>>>>>> On 7/16/19 9:50 AM, Sowjanya Komatineni wrote:
>>>>>>>>>> On 7/16/19 8:00 AM, Dmitry Osipenko wrote:
>>>>>>>>>>> 16.07.2019 11:06, Peter De Schrijver пишет:
>>>>>>>>>>>> On Tue, Jul 16, 2019 at 03:24:26PM +0800, Joseph Lo wrote:
>>>>>>>>>>>>>> OK, Will add to CPUFreq driver...
>>>>>>>>>>>>>>> The other thing that also need attention is that T124
>>>>>>>>>>>>>>> CPUFreq
>>>>>>>>>>>>>>> driver
>>>>>>>>>>>>>>> implicitly relies on DFLL driver to be probed first,
>>>>>>>>>>>>>>> which is
>>>>>>>>>>>>>>> icky.
>>>>>>>>>>>>>>>
>>>>>>>>>>>>>> Should I add check for successful dfll clk register
>>>>>>>>>>>>>> explicitly in
>>>>>>>>>>>>>> CPUFreq driver probe and defer till dfll clk registers?
>>>>>>>>>>> Probably you should use the "device links". See [1][2] for the
>>>>>>>>>>> example.
>>>>>>>>>>>
>>>>>>>>>>> [1]
>>>>>>>>>>> https://elixir.bootlin.com/linux/v5.2.1/source/drivers/gpu/drm/tegra/dc.c#L2383
>>>>>>>>>>>
>>>>>>>>>>>
>>>>>>>>>>>
>>>>>>>>>>>
>>>>>>>>>>>
>>>>>>>>>>>
>>>>>>>>>>> [2]
>>>>>>>>>>> https://www.kernel.org/doc/html/latest/driver-api/device_link.html
>>>>>>>>>>>
>>>>>>>>>>>
>>>>>>>>>>> Return EPROBE_DEFER instead of EINVAL if device_link_add()
>>>>>>>>>>> fails.
>>>>>>>>>>> And
>>>>>>>>>>> use of_find_device_by_node() to get the DFLL's device, see [3].
>>>>>>>>>>>
>>>>>>>>>>> [3]
>>>>>>>>>>> https://git.kernel.org/pub/scm/linux/kernel/git/next/linux-next.git/tree/drivers/devfreq/tegra20-devfreq.c#n100
>>>>>>>>>>>
>>>>>>>>>>>
>>>>>>>>>>>
>>>>>>>>>>>
>>>>>>>>>>>
>>>>>>>>>> Will go thru and add...
>>>>>>>> Looks like I initially confused this case with getting orphaned
>>>>>>>> clock.
>>>>>>>> I'm now seeing that the DFLL driver registers the clock and then
>>>>>>>> clk_get(dfll) should be returning EPROBE_DEFER until DFLL driver is
>>>>>>>> probed, hence everything should be fine as-is and there is no real
>>>>>>>> need
>>>>>>>> for the 'device link'. Sorry for the confusion!
>>>>>>>>
>>>>>>>>>>>>> Sorry, I didn't follow the mail thread. Just regarding the
>>>>>>>>>>>>> DFLL
>>>>>>>>>>>>> part.
>>>>>>>>>>>>>
>>>>>>>>>>>>> As you know it, the DFLL clock is one of the CPU clock
>>>>>>>>>>>>> sources and
>>>>>>>>>>>>> integrated with DVFS control logic with the regulator. We
>>>>>>>>>>>>> will not
>>>>>>>>>>>>> switch
>>>>>>>>>>>>> CPU to other clock sources once we switched to DFLL.
>>>>>>>>>>>>> Because the
>>>>>>>>>>>>> CPU has
>>>>>>>>>>>>> been regulated by the DFLL HW with the DVFS table (CVB or OPP
>>>>>>>>>>>>> table
>>>>>>>>>>>>> you see
>>>>>>>>>>>>> in the driver.). We shouldn't reparent it to other sources
>>>>>>>>>>>>> with
>>>>>>>>>>>>> unknew
>>>>>>>>>>>>> freq/volt pair. That's not guaranteed to work. We allow
>>>>>>>>>>>>> switching to
>>>>>>>>>>>>> open-loop mode but different sources.
>>>>>>>>>>> Okay, then the CPUFreq driver will have to enforce DFLL freq to
>>>>>>>>>>> PLLP's
>>>>>>>>>>> rate before switching to PLLP in order to have a proper CPU
>>>>>>>>>>> voltage.
>>>>>>>>>> PLLP freq is safe to work for any CPU voltage. So no need to
>>>>>>>>>> enforce
>>>>>>>>>> DFLL freq to PLLP rate before changing CCLK_G source to PLLP
>>>>>>>>>> during
>>>>>>>>>> suspend
>>>>>>>>>>
>>>>>>>>> Sorry, please ignore my above comment. During suspend, need to
>>>>>>>>> change
>>>>>>>>> CCLK_G source to PLLP when dfll is in closed loop mode first and
>>>>>>>>> then
>>>>>>>>> dfll need to be set to open loop.
>>>>>>>> Okay.
>>>>>>>>
>>>>>>>>>>>>> And I don't exactly understand why we need to switch to
>>>>>>>>>>>>> PLLP in
>>>>>>>>>>>>> CPU
>>>>>>>>>>>>> idle
>>>>>>>>>>>>> driver. Just keep it on CL-DVFS mode all the time.
>>>>>>>>>>>>>
>>>>>>>>>>>>> In SC7 entry, the dfll suspend function moves it the open-loop
>>>>>>>>>>>>> mode. That's
>>>>>>>>>>>>> all. The sc7-entryfirmware will handle the rest of the
>>>>>>>>>>>>> sequence to
>>>>>>>>>>>>> turn off
>>>>>>>>>>>>> the CPU power.
>>>>>>>>>>>>>
>>>>>>>>>>>>> In SC7 resume, the warmboot code will handle the sequence to
>>>>>>>>>>>>> turn on
>>>>>>>>>>>>> regulator and power up the CPU cluster. And leave it on PLL_P.
>>>>>>>>>>>>> After
>>>>>>>>>>>>> resuming to the kernel, we re-init DFLL, restore the CPU clock
>>>>>>>>>>>>> policy (CPU
>>>>>>>>>>>>> runs on DFLL open-loop mode) and then moving to close-loop
>>>>>>>>>>>>> mode.
>>>>>>>>>>> The DFLL is re-inited after switching CCLK to DFLL parent
>>>>>>>>>>> during of
>>>>>>>>>>> the
>>>>>>>>>>> early clocks-state restoring by CaR driver. Hence instead of
>>>>>>>>>>> having
>>>>>>>>>>> odd
>>>>>>>>>>> hacks in the CaR driver, it is much nicer to have a proper
>>>>>>>>>>> suspend-resume sequencing of the device drivers. In this case
>>>>>>>>>>> CPUFreq
>>>>>>>>>>> driver is the driver that enables DFLL and switches CPU to that
>>>>>>>>>>> clock
>>>>>>>>>>> source, which means that this driver is also should be
>>>>>>>>>>> responsible for
>>>>>>>>>>> management of the DFLL's state during of suspend/resume
>>>>>>>>>>> process. If
>>>>>>>>>>> CPUFreq driver disables DFLL during suspend and re-enables it
>>>>>>>>>>> during
>>>>>>>>>>> resume, then looks like the CaR driver hacks around DFLL are not
>>>>>>>>>>> needed.
>>>>>>>>>>>
>>>>>>>>>>>>> The DFLL part looks good to me. BTW, change the patch
>>>>>>>>>>>>> subject to
>>>>>>>>>>>>> "Add
>>>>>>>>>>>>> suspend-resume support" seems more appropriate to me.
>>>>>>>>>>>>>
>>>>>>>>>>>> To clarify this, the sequences for DFLL use are as follows
>>>>>>>>>>>> (assuming
>>>>>>>>>>>> all
>>>>>>>>>>>> required DFLL hw configuration has been done)
>>>>>>>>>>>>
>>>>>>>>>>>> Switch to DFLL:
>>>>>>>>>>>> 0) Save current parent and frequency
>>>>>>>>>>>> 1) Program DFLL to open loop mode
>>>>>>>>>>>> 2) Enable DFLL
>>>>>>>>>>>> 3) Change cclk_g parent to DFLL
>>>>>>>>>>>> For OVR regulator:
>>>>>>>>>>>> 4) Change PWM output pin from tristate to output
>>>>>>>>>>>> 5) Enable DFLL PWM output
>>>>>>>>>>>> For I2C regulator:
>>>>>>>>>>>> 4) Enable DFLL I2C output
>>>>>>>>>>>> 6) Program DFLL to closed loop mode
>>>>>>>>>>>>
>>>>>>>>>>>> Switch away from DFLL:
>>>>>>>>>>>> 0) Change cclk_g parent to PLLP so the CPU frequency is ok for
>>>>>>>>>>>> any
>>>>>>>>>>>> vdd_cpu voltage
>>>>>>>>>>>> 1) Program DFLL to open loop mode
>>>>>>>>>>>>
>>>>>>>>> I see during switch away from DFLL (suspend), cclk_g parent is not
>>>>>>>>> changed to PLLP before changing dfll to open loop mode.
>>>>>>>>>
>>>>>>>>> Will add this ...
>>>>>>>> The CPUFreq driver switches parent to PLLP during the probe,
>>>>>>>> similar
>>>>>>>> should be done on suspend.
>>>>>>>>
>>>>>>>> I'm also wondering if it's always safe to switch to PLLP in the
>>>>>>>> probe.
>>>>>>>> If CPU is running on a lower freq than PLLP, then some other more
>>>>>>>> appropriate intermediate parent should be selected.
>>>>>>>>
>>>>>>> CPU parents are PLL_X, PLL_P, and dfll. PLL_X always runs at higher
>>>>>>> rate
>>>>>>> so switching to PLL_P during CPUFreq probe prior to dfll clock
>>>>>>> enable
>>>>>>> should be safe.
>>>>>> AFAIK, PLLX could run at ~200MHz. There is also a divided output of
>>>>>> PLLP
>>>>>> which CCLKG supports, the PLLP_OUT4.
>>>>>>
>>>>>> Probably, realistically, CPU is always running off a fast PLLX during
>>>>>> boot, but I'm wondering what may happen on KEXEC. I guess ideally
>>>>>> CPUFreq driver should also have a 'shutdown' callback to teardown
>>>>>> DFLL
>>>>>> on a reboot, but likely that there are other clock-related
>>>>>> problems as
>>>>>> well that may break KEXEC and thus it is not very important at the
>>>>>> moment.
>>>>>>
>>>>>> [snip]
>>>>> During bootup CPUG sources from PLL_X. By PLL_P source above I meant
>>>>> PLL_P_OUT4.
>>>>>
>>>>> As per clock policies, PLL_X is always used for high freq like >800Mhz
>>>>> and for low frequency it will be sourced from PLLP.
>>>> Alright, then please don't forget to pre-initialize PLLP_OUT4 rate to a
>>>> reasonable value using tegra_clk_init_table or assigned-clocks.
>>> PLLP_OUT4 rate update is not needed as it is safe to run at 408Mhz
>>> because it is below fmax @ Vmin
>> So even 204MHz CVB entries are having the same voltage as 408MHz,
>> correct? It's not instantly obvious to me from the DFLL driver's code
>> where the fmax @ Vmin is defined, I see that there is the min_millivolts
>> and frequency entries starting from 204MHZ defined per-table.
> Yes at Vmin CPU Fmax is ~800Mhz. So anything below that will work at
> Vmin voltage and PLLP max is 408Mhz.

Thank you for the clarification. It would be good to have that commented
in the code as well.

On 7/16/19 3:00 PM, Dmitry Osipenko wrote:
> 17.07.2019 0:35, Sowjanya Komatineni пишет:
>> On 7/16/19 2:21 PM, Dmitry Osipenko wrote:
>>> 17.07.2019 0:12, Sowjanya Komatineni пишет:
>>>> On 7/16/19 1:47 PM, Dmitry Osipenko wrote:
>>>>> 16.07.2019 22:26, Sowjanya Komatineni пишет:
>>>>>> On 7/16/19 11:43 AM, Dmitry Osipenko wrote:
>>>>>>> 16.07.2019 21:30, Sowjanya Komatineni пишет:
>>>>>>>> On 7/16/19 11:25 AM, Dmitry Osipenko wrote:
>>>>>>>>> 16.07.2019 21:19, Sowjanya Komatineni пишет:
>>>>>>>>>> On 7/16/19 9:50 AM, Sowjanya Komatineni wrote:
>>>>>>>>>>> On 7/16/19 8:00 AM, Dmitry Osipenko wrote:
>>>>>>>>>>>> 16.07.2019 11:06, Peter De Schrijver пишет:
>>>>>>>>>>>>> On Tue, Jul 16, 2019 at 03:24:26PM +0800, Joseph Lo wrote:
>>>>>>>>>>>>>>> OK, Will add to CPUFreq driver...
>>>>>>>>>>>>>>>> The other thing that also need attention is that T124
>>>>>>>>>>>>>>>> CPUFreq
>>>>>>>>>>>>>>>> driver
>>>>>>>>>>>>>>>> implicitly relies on DFLL driver to be probed first,
>>>>>>>>>>>>>>>> which is
>>>>>>>>>>>>>>>> icky.
>>>>>>>>>>>>>>>>
>>>>>>>>>>>>>>> Should I add check for successful dfll clk register
>>>>>>>>>>>>>>> explicitly in
>>>>>>>>>>>>>>> CPUFreq driver probe and defer till dfll clk registers?
>>>>>>>>>>>> Probably you should use the "device links". See [1][2] for the
>>>>>>>>>>>> example.
>>>>>>>>>>>>
>>>>>>>>>>>> [1]
>>>>>>>>>>>> https://elixir.bootlin.com/linux/v5.2.1/source/drivers/gpu/drm/tegra/dc.c#L2383
>>>>>>>>>>>>
>>>>>>>>>>>>
>>>>>>>>>>>>
>>>>>>>>>>>>
>>>>>>>>>>>>
>>>>>>>>>>>>
>>>>>>>>>>>> [2]
>>>>>>>>>>>> https://www.kernel.org/doc/html/latest/driver-api/device_link.html
>>>>>>>>>>>>
>>>>>>>>>>>>
>>>>>>>>>>>> Return EPROBE_DEFER instead of EINVAL if device_link_add()
>>>>>>>>>>>> fails.
>>>>>>>>>>>> And
>>>>>>>>>>>> use of_find_device_by_node() to get the DFLL's device, see [3].
>>>>>>>>>>>>
>>>>>>>>>>>> [3]
>>>>>>>>>>>> https://git.kernel.org/pub/scm/linux/kernel/git/next/linux-next.git/tree/drivers/devfreq/tegra20-devfreq.c#n100
>>>>>>>>>>>>
>>>>>>>>>>>>
>>>>>>>>>>>>
>>>>>>>>>>>>
>>>>>>>>>>>>
>>>>>>>>>>> Will go thru and add...
>>>>>>>>> Looks like I initially confused this case with getting orphaned
>>>>>>>>> clock.
>>>>>>>>> I'm now seeing that the DFLL driver registers the clock and then
>>>>>>>>> clk_get(dfll) should be returning EPROBE_DEFER until DFLL driver is
>>>>>>>>> probed, hence everything should be fine as-is and there is no real
>>>>>>>>> need
>>>>>>>>> for the 'device link'. Sorry for the confusion!
>>>>>>>>>
>>>>>>>>>>>>>> Sorry, I didn't follow the mail thread. Just regarding the
>>>>>>>>>>>>>> DFLL
>>>>>>>>>>>>>> part.
>>>>>>>>>>>>>>
>>>>>>>>>>>>>> As you know it, the DFLL clock is one of the CPU clock
>>>>>>>>>>>>>> sources and
>>>>>>>>>>>>>> integrated with DVFS control logic with the regulator. We
>>>>>>>>>>>>>> will not
>>>>>>>>>>>>>> switch
>>>>>>>>>>>>>> CPU to other clock sources once we switched to DFLL.
>>>>>>>>>>>>>> Because the
>>>>>>>>>>>>>> CPU has
>>>>>>>>>>>>>> been regulated by the DFLL HW with the DVFS table (CVB or OPP
>>>>>>>>>>>>>> table
>>>>>>>>>>>>>> you see
>>>>>>>>>>>>>> in the driver.). We shouldn't reparent it to other sources
>>>>>>>>>>>>>> with
>>>>>>>>>>>>>> unknew
>>>>>>>>>>>>>> freq/volt pair. That's not guaranteed to work. We allow
>>>>>>>>>>>>>> switching to
>>>>>>>>>>>>>> open-loop mode but different sources.
>>>>>>>>>>>> Okay, then the CPUFreq driver will have to enforce DFLL freq to
>>>>>>>>>>>> PLLP's
>>>>>>>>>>>> rate before switching to PLLP in order to have a proper CPU
>>>>>>>>>>>> voltage.
>>>>>>>>>>> PLLP freq is safe to work for any CPU voltage. So no need to
>>>>>>>>>>> enforce
>>>>>>>>>>> DFLL freq to PLLP rate before changing CCLK_G source to PLLP
>>>>>>>>>>> during
>>>>>>>>>>> suspend
>>>>>>>>>>>
>>>>>>>>>> Sorry, please ignore my above comment. During suspend, need to
>>>>>>>>>> change
>>>>>>>>>> CCLK_G source to PLLP when dfll is in closed loop mode first and
>>>>>>>>>> then
>>>>>>>>>> dfll need to be set to open loop.
>>>>>>>>> Okay.
>>>>>>>>>
>>>>>>>>>>>>>> And I don't exactly understand why we need to switch to
>>>>>>>>>>>>>> PLLP in
>>>>>>>>>>>>>> CPU
>>>>>>>>>>>>>> idle
>>>>>>>>>>>>>> driver. Just keep it on CL-DVFS mode all the time.
>>>>>>>>>>>>>>
>>>>>>>>>>>>>> In SC7 entry, the dfll suspend function moves it the open-loop
>>>>>>>>>>>>>> mode. That's
>>>>>>>>>>>>>> all. The sc7-entryfirmware will handle the rest of the
>>>>>>>>>>>>>> sequence to
>>>>>>>>>>>>>> turn off
>>>>>>>>>>>>>> the CPU power.
>>>>>>>>>>>>>>
>>>>>>>>>>>>>> In SC7 resume, the warmboot code will handle the sequence to
>>>>>>>>>>>>>> turn on
>>>>>>>>>>>>>> regulator and power up the CPU cluster. And leave it on PLL_P.
>>>>>>>>>>>>>> After
>>>>>>>>>>>>>> resuming to the kernel, we re-init DFLL, restore the CPU clock
>>>>>>>>>>>>>> policy (CPU
>>>>>>>>>>>>>> runs on DFLL open-loop mode) and then moving to close-loop
>>>>>>>>>>>>>> mode.
>>>>>>>>>>>> The DFLL is re-inited after switching CCLK to DFLL parent
>>>>>>>>>>>> during of
>>>>>>>>>>>> the
>>>>>>>>>>>> early clocks-state restoring by CaR driver. Hence instead of
>>>>>>>>>>>> having
>>>>>>>>>>>> odd
>>>>>>>>>>>> hacks in the CaR driver, it is much nicer to have a proper
>>>>>>>>>>>> suspend-resume sequencing of the device drivers. In this case
>>>>>>>>>>>> CPUFreq
>>>>>>>>>>>> driver is the driver that enables DFLL and switches CPU to that
>>>>>>>>>>>> clock
>>>>>>>>>>>> source, which means that this driver is also should be
>>>>>>>>>>>> responsible for
>>>>>>>>>>>> management of the DFLL's state during of suspend/resume
>>>>>>>>>>>> process. If
>>>>>>>>>>>> CPUFreq driver disables DFLL during suspend and re-enables it
>>>>>>>>>>>> during
>>>>>>>>>>>> resume, then looks like the CaR driver hacks around DFLL are not
>>>>>>>>>>>> needed.
>>>>>>>>>>>>
>>>>>>>>>>>>>> The DFLL part looks good to me. BTW, change the patch
>>>>>>>>>>>>>> subject to
>>>>>>>>>>>>>> "Add
>>>>>>>>>>>>>> suspend-resume support" seems more appropriate to me.
>>>>>>>>>>>>>>
>>>>>>>>>>>>> To clarify this, the sequences for DFLL use are as follows
>>>>>>>>>>>>> (assuming
>>>>>>>>>>>>> all
>>>>>>>>>>>>> required DFLL hw configuration has been done)
>>>>>>>>>>>>>
>>>>>>>>>>>>> Switch to DFLL:
>>>>>>>>>>>>> 0) Save current parent and frequency
>>>>>>>>>>>>> 1) Program DFLL to open loop mode
>>>>>>>>>>>>> 2) Enable DFLL
>>>>>>>>>>>>> 3) Change cclk_g parent to DFLL
>>>>>>>>>>>>> For OVR regulator:
>>>>>>>>>>>>> 4) Change PWM output pin from tristate to output
>>>>>>>>>>>>> 5) Enable DFLL PWM output
>>>>>>>>>>>>> For I2C regulator:
>>>>>>>>>>>>> 4) Enable DFLL I2C output
>>>>>>>>>>>>> 6) Program DFLL to closed loop mode
>>>>>>>>>>>>>
>>>>>>>>>>>>> Switch away from DFLL:
>>>>>>>>>>>>> 0) Change cclk_g parent to PLLP so the CPU frequency is ok for
>>>>>>>>>>>>> any
>>>>>>>>>>>>> vdd_cpu voltage
>>>>>>>>>>>>> 1) Program DFLL to open loop mode
>>>>>>>>>>>>>
>>>>>>>>>> I see during switch away from DFLL (suspend), cclk_g parent is not
>>>>>>>>>> changed to PLLP before changing dfll to open loop mode.
>>>>>>>>>>
>>>>>>>>>> Will add this ...
>>>>>>>>> The CPUFreq driver switches parent to PLLP during the probe,
>>>>>>>>> similar
>>>>>>>>> should be done on suspend.
>>>>>>>>>
>>>>>>>>> I'm also wondering if it's always safe to switch to PLLP in the
>>>>>>>>> probe.
>>>>>>>>> If CPU is running on a lower freq than PLLP, then some other more
>>>>>>>>> appropriate intermediate parent should be selected.
>>>>>>>>>
>>>>>>>> CPU parents are PLL_X, PLL_P, and dfll. PLL_X always runs at higher
>>>>>>>> rate
>>>>>>>> so switching to PLL_P during CPUFreq probe prior to dfll clock
>>>>>>>> enable
>>>>>>>> should be safe.
>>>>>>> AFAIK, PLLX could run at ~200MHz. There is also a divided output of
>>>>>>> PLLP
>>>>>>> which CCLKG supports, the PLLP_OUT4.
>>>>>>>
>>>>>>> Probably, realistically, CPU is always running off a fast PLLX during
>>>>>>> boot, but I'm wondering what may happen on KEXEC. I guess ideally
>>>>>>> CPUFreq driver should also have a 'shutdown' callback to teardown
>>>>>>> DFLL
>>>>>>> on a reboot, but likely that there are other clock-related
>>>>>>> problems as
>>>>>>> well that may break KEXEC and thus it is not very important at the
>>>>>>> moment.
>>>>>>>
>>>>>>> [snip]
>>>>>> During bootup CPUG sources from PLL_X. By PLL_P source above I meant
>>>>>> PLL_P_OUT4.
>>>>>>
>>>>>> As per clock policies, PLL_X is always used for high freq like >800Mhz
>>>>>> and for low frequency it will be sourced from PLLP.
>>>>> Alright, then please don't forget to pre-initialize PLLP_OUT4 rate to a
>>>>> reasonable value using tegra_clk_init_table or assigned-clocks.
>>>> PLLP_OUT4 rate update is not needed as it is safe to run at 408Mhz
>>>> because it is below fmax @ Vmin
>>> So even 204MHz CVB entries are having the same voltage as 408MHz,
>>> correct? It's not instantly obvious to me from the DFLL driver's code
>>> where the fmax @ Vmin is defined, I see that there is the min_millivolts
>>> and frequency entries starting from 204MHZ defined per-table.
>> Yes at Vmin CPU Fmax is ~800Mhz. So anything below that will work at
>> Vmin voltage and PLLP max is 408Mhz.
> Thank you for the clarification. It would be good to have that commented
> in the code as well.
OK, Will add...

On 7/16/19 3:06 PM, Sowjanya Komatineni wrote:
>
> On 7/16/19 3:00 PM, Dmitry Osipenko wrote:
>> 17.07.2019 0:35, Sowjanya Komatineni пишет:
>>> On 7/16/19 2:21 PM, Dmitry Osipenko wrote:
>>>> 17.07.2019 0:12, Sowjanya Komatineni пишет:
>>>>> On 7/16/19 1:47 PM, Dmitry Osipenko wrote:
>>>>>> 16.07.2019 22:26, Sowjanya Komatineni пишет:
>>>>>>> On 7/16/19 11:43 AM, Dmitry Osipenko wrote:
>>>>>>>> 16.07.2019 21:30, Sowjanya Komatineni пишет:
>>>>>>>>> On 7/16/19 11:25 AM, Dmitry Osipenko wrote:
>>>>>>>>>> 16.07.2019 21:19, Sowjanya Komatineni пишет:
>>>>>>>>>>> On 7/16/19 9:50 AM, Sowjanya Komatineni wrote:
>>>>>>>>>>>> On 7/16/19 8:00 AM, Dmitry Osipenko wrote:
>>>>>>>>>>>>> 16.07.2019 11:06, Peter De Schrijver пишет:
>>>>>>>>>>>>>> On Tue, Jul 16, 2019 at 03:24:26PM +0800, Joseph Lo wrote:
>>>>>>>>>>>>>>>> OK, Will add to CPUFreq driver...
>>>>>>>>>>>>>>>>> The other thing that also need attention is that T124
>>>>>>>>>>>>>>>>> CPUFreq
>>>>>>>>>>>>>>>>> driver
>>>>>>>>>>>>>>>>> implicitly relies on DFLL driver to be probed first,
>>>>>>>>>>>>>>>>> which is
>>>>>>>>>>>>>>>>> icky.
>>>>>>>>>>>>>>>>>
>>>>>>>>>>>>>>>> Should I add check for successful dfll clk register
>>>>>>>>>>>>>>>> explicitly in
>>>>>>>>>>>>>>>> CPUFreq driver probe and defer till dfll clk registers?
>>>>>>>>>>>>> Probably you should use the "device links". See [1][2] for 
>>>>>>>>>>>>> the
>>>>>>>>>>>>> example.
>>>>>>>>>>>>>
>>>>>>>>>>>>> [1]
>>>>>>>>>>>>> https://elixir.bootlin.com/linux/v5.2.1/source/drivers/gpu/drm/tegra/dc.c#L2383 
>>>>>>>>>>>>>
>>>>>>>>>>>>>
>>>>>>>>>>>>>
>>>>>>>>>>>>>
>>>>>>>>>>>>>
>>>>>>>>>>>>>
>>>>>>>>>>>>>
>>>>>>>>>>>>> [2]
>>>>>>>>>>>>> https://www.kernel.org/doc/html/latest/driver-api/device_link.html 
>>>>>>>>>>>>>
>>>>>>>>>>>>>
>>>>>>>>>>>>>
>>>>>>>>>>>>> Return EPROBE_DEFER instead of EINVAL if device_link_add()
>>>>>>>>>>>>> fails.
>>>>>>>>>>>>> And
>>>>>>>>>>>>> use of_find_device_by_node() to get the DFLL's device, see 
>>>>>>>>>>>>> [3].
>>>>>>>>>>>>>
>>>>>>>>>>>>> [3]
>>>>>>>>>>>>> https://git.kernel.org/pub/scm/linux/kernel/git/next/linux-next.git/tree/drivers/devfreq/tegra20-devfreq.c#n100 
>>>>>>>>>>>>>
>>>>>>>>>>>>>
>>>>>>>>>>>>>
>>>>>>>>>>>>>
>>>>>>>>>>>>>
>>>>>>>>>>>>>
>>>>>>>>>>>> Will go thru and add...
>>>>>>>>>> Looks like I initially confused this case with getting orphaned
>>>>>>>>>> clock.
>>>>>>>>>> I'm now seeing that the DFLL driver registers the clock and then
>>>>>>>>>> clk_get(dfll) should be returning EPROBE_DEFER until DFLL 
>>>>>>>>>> driver is
>>>>>>>>>> probed, hence everything should be fine as-is and there is no 
>>>>>>>>>> real
>>>>>>>>>> need
>>>>>>>>>> for the 'device link'. Sorry for the confusion!
>>>>>>>>>>
>>>>>>>>>>>>>>> Sorry, I didn't follow the mail thread. Just regarding the
>>>>>>>>>>>>>>> DFLL
>>>>>>>>>>>>>>> part.
>>>>>>>>>>>>>>>
>>>>>>>>>>>>>>> As you know it, the DFLL clock is one of the CPU clock
>>>>>>>>>>>>>>> sources and
>>>>>>>>>>>>>>> integrated with DVFS control logic with the regulator. We
>>>>>>>>>>>>>>> will not
>>>>>>>>>>>>>>> switch
>>>>>>>>>>>>>>> CPU to other clock sources once we switched to DFLL.
>>>>>>>>>>>>>>> Because the
>>>>>>>>>>>>>>> CPU has
>>>>>>>>>>>>>>> been regulated by the DFLL HW with the DVFS table (CVB 
>>>>>>>>>>>>>>> or OPP
>>>>>>>>>>>>>>> table
>>>>>>>>>>>>>>> you see
>>>>>>>>>>>>>>> in the driver.). We shouldn't reparent it to other sources
>>>>>>>>>>>>>>> with
>>>>>>>>>>>>>>> unknew
>>>>>>>>>>>>>>> freq/volt pair. That's not guaranteed to work. We allow
>>>>>>>>>>>>>>> switching to
>>>>>>>>>>>>>>> open-loop mode but different sources.
>>>>>>>>>>>>> Okay, then the CPUFreq driver will have to enforce DFLL 
>>>>>>>>>>>>> freq to
>>>>>>>>>>>>> PLLP's
>>>>>>>>>>>>> rate before switching to PLLP in order to have a proper CPU
>>>>>>>>>>>>> voltage.
>>>>>>>>>>>> PLLP freq is safe to work for any CPU voltage. So no need to
>>>>>>>>>>>> enforce
>>>>>>>>>>>> DFLL freq to PLLP rate before changing CCLK_G source to PLLP
>>>>>>>>>>>> during
>>>>>>>>>>>> suspend
>>>>>>>>>>>>
>>>>>>>>>>> Sorry, please ignore my above comment. During suspend, need to
>>>>>>>>>>> change
>>>>>>>>>>> CCLK_G source to PLLP when dfll is in closed loop mode first 
>>>>>>>>>>> and
>>>>>>>>>>> then
>>>>>>>>>>> dfll need to be set to open loop.
>>>>>>>>>> Okay.
>>>>>>>>>>
>>>>>>>>>>>>>>> And I don't exactly understand why we need to switch to
>>>>>>>>>>>>>>> PLLP in
>>>>>>>>>>>>>>> CPU
>>>>>>>>>>>>>>> idle
>>>>>>>>>>>>>>> driver. Just keep it on CL-DVFS mode all the time.
>>>>>>>>>>>>>>>
>>>>>>>>>>>>>>> In SC7 entry, the dfll suspend function moves it the 
>>>>>>>>>>>>>>> open-loop
>>>>>>>>>>>>>>> mode. That's
>>>>>>>>>>>>>>> all. The sc7-entryfirmware will handle the rest of the
>>>>>>>>>>>>>>> sequence to
>>>>>>>>>>>>>>> turn off
>>>>>>>>>>>>>>> the CPU power.
>>>>>>>>>>>>>>>
>>>>>>>>>>>>>>> In SC7 resume, the warmboot code will handle the 
>>>>>>>>>>>>>>> sequence to
>>>>>>>>>>>>>>> turn on
>>>>>>>>>>>>>>> regulator and power up the CPU cluster. And leave it on 
>>>>>>>>>>>>>>> PLL_P.
>>>>>>>>>>>>>>> After
>>>>>>>>>>>>>>> resuming to the kernel, we re-init DFLL, restore the CPU 
>>>>>>>>>>>>>>> clock
>>>>>>>>>>>>>>> policy (CPU
>>>>>>>>>>>>>>> runs on DFLL open-loop mode) and then moving to close-loop
>>>>>>>>>>>>>>> mode.
>>>>>>>>>>>>> The DFLL is re-inited after switching CCLK to DFLL parent
>>>>>>>>>>>>> during of
>>>>>>>>>>>>> the
>>>>>>>>>>>>> early clocks-state restoring by CaR driver. Hence instead of
>>>>>>>>>>>>> having
>>>>>>>>>>>>> odd
>>>>>>>>>>>>> hacks in the CaR driver, it is much nicer to have a proper
>>>>>>>>>>>>> suspend-resume sequencing of the device drivers. In this case
>>>>>>>>>>>>> CPUFreq
>>>>>>>>>>>>> driver is the driver that enables DFLL and switches CPU to 
>>>>>>>>>>>>> that
>>>>>>>>>>>>> clock
>>>>>>>>>>>>> source, which means that this driver is also should be
>>>>>>>>>>>>> responsible for
>>>>>>>>>>>>> management of the DFLL's state during of suspend/resume
>>>>>>>>>>>>> process. If
>>>>>>>>>>>>> CPUFreq driver disables DFLL during suspend and re-enables it
>>>>>>>>>>>>> during
>>>>>>>>>>>>> resume, then looks like the CaR driver hacks around DFLL 
>>>>>>>>>>>>> are not
>>>>>>>>>>>>> needed.
>>>>>>>>>>>>>
>>>>>>>>>>>>>>> The DFLL part looks good to me. BTW, change the patch
>>>>>>>>>>>>>>> subject to
>>>>>>>>>>>>>>> "Add
>>>>>>>>>>>>>>> suspend-resume support" seems more appropriate to me.
>>>>>>>>>>>>>>>
>>>>>>>>>>>>>> To clarify this, the sequences for DFLL use are as follows
>>>>>>>>>>>>>> (assuming
>>>>>>>>>>>>>> all
>>>>>>>>>>>>>> required DFLL hw configuration has been done)
>>>>>>>>>>>>>>
>>>>>>>>>>>>>> Switch to DFLL:
>>>>>>>>>>>>>> 0) Save current parent and frequency
>>>>>>>>>>>>>> 1) Program DFLL to open loop mode
>>>>>>>>>>>>>> 2) Enable DFLL
>>>>>>>>>>>>>> 3) Change cclk_g parent to DFLL
>>>>>>>>>>>>>> For OVR regulator:
>>>>>>>>>>>>>> 4) Change PWM output pin from tristate to output
>>>>>>>>>>>>>> 5) Enable DFLL PWM output
>>>>>>>>>>>>>> For I2C regulator:
>>>>>>>>>>>>>> 4) Enable DFLL I2C output
>>>>>>>>>>>>>> 6) Program DFLL to closed loop mode
>>>>>>>>>>>>>>
>>>>>>>>>>>>>> Switch away from DFLL:
>>>>>>>>>>>>>> 0) Change cclk_g parent to PLLP so the CPU frequency is 
>>>>>>>>>>>>>> ok for
>>>>>>>>>>>>>> any
>>>>>>>>>>>>>> vdd_cpu voltage
>>>>>>>>>>>>>> 1) Program DFLL to open loop mode
>>>>>>>>>>>>>>
>>>>>>>>>>> I see during switch away from DFLL (suspend), cclk_g parent 
>>>>>>>>>>> is not
>>>>>>>>>>> changed to PLLP before changing dfll to open loop mode.
>>>>>>>>>>>
>>>>>>>>>>> Will add this ...
>>>>>>>>>> The CPUFreq driver switches parent to PLLP during the probe,
>>>>>>>>>> similar
>>>>>>>>>> should be done on suspend.
>>>>>>>>>>
>>>>>>>>>> I'm also wondering if it's always safe to switch to PLLP in the
>>>>>>>>>> probe.
>>>>>>>>>> If CPU is running on a lower freq than PLLP, then some other 
>>>>>>>>>> more
>>>>>>>>>> appropriate intermediate parent should be selected.
>>>>>>>>>>
>>>>>>>>> CPU parents are PLL_X, PLL_P, and dfll. PLL_X always runs at 
>>>>>>>>> higher
>>>>>>>>> rate
>>>>>>>>> so switching to PLL_P during CPUFreq probe prior to dfll clock
>>>>>>>>> enable
>>>>>>>>> should be safe.
>>>>>>>> AFAIK, PLLX could run at ~200MHz. There is also a divided 
>>>>>>>> output of
>>>>>>>> PLLP
>>>>>>>> which CCLKG supports, the PLLP_OUT4.
>>>>>>>>
>>>>>>>> Probably, realistically, CPU is always running off a fast PLLX 
>>>>>>>> during
>>>>>>>> boot, but I'm wondering what may happen on KEXEC. I guess ideally
>>>>>>>> CPUFreq driver should also have a 'shutdown' callback to teardown
>>>>>>>> DFLL
>>>>>>>> on a reboot, but likely that there are other clock-related
>>>>>>>> problems as
>>>>>>>> well that may break KEXEC and thus it is not very important at the
>>>>>>>> moment.
>>>>>>>>
>>>>>>>> [snip]
>>>>>>> During bootup CPUG sources from PLL_X. By PLL_P source above I 
>>>>>>> meant
>>>>>>> PLL_P_OUT4.
>>>>>>>
>>>>>>> As per clock policies, PLL_X is always used for high freq like 
>>>>>>> >800Mhz
>>>>>>> and for low frequency it will be sourced from PLLP.
>>>>>> Alright, then please don't forget to pre-initialize PLLP_OUT4 
>>>>>> rate to a
>>>>>> reasonable value using tegra_clk_init_table or assigned-clocks.
>>>>> PLLP_OUT4 rate update is not needed as it is safe to run at 408Mhz
>>>>> because it is below fmax @ Vmin
>>>> So even 204MHz CVB entries are having the same voltage as 408MHz,
>>>> correct? It's not instantly obvious to me from the DFLL driver's code
>>>> where the fmax @ Vmin is defined, I see that there is the 
>>>> min_millivolts
>>>> and frequency entries starting from 204MHZ defined per-table.
>>> Yes at Vmin CPU Fmax is ~800Mhz. So anything below that will work at
>>> Vmin voltage and PLLP max is 408Mhz.
>> Thank you for the clarification. It would be good to have that commented
>> in the code as well.
> OK, Will add...

Regarding, adding suspend/resume to CPUFreq, CPUFreq suspend happens 
very early even before disabling non-boot CPUs and also need to export 
clock driver APIs to CPUFreq.

Was thinking of below way of implementing this...


Clock DFLL driver Suspend:

         - Save CPU clock policy registers, and Perform dfll suspend 
which sets in open loop mode

CPU Freq driver Suspend: does nothing


Clock DFLL driver Resume:

         - Re-init DFLL, Set in Open-Loop mode, restore CPU Clock policy 
registers which actually sets source to DFLL along with other         
CPU Policy register restore.

CPU Freq driver Resume:

         - do clk_prepare_enable which acutally sets DFLL in Closed loop 
mode

On 7/16/19 7:18 PM, Sowjanya Komatineni wrote:
>
> On 7/16/19 3:06 PM, Sowjanya Komatineni wrote:
>>
>> On 7/16/19 3:00 PM, Dmitry Osipenko wrote:
>>> 17.07.2019 0:35, Sowjanya Komatineni пишет:
>>>> On 7/16/19 2:21 PM, Dmitry Osipenko wrote:
>>>>> 17.07.2019 0:12, Sowjanya Komatineni пишет:
>>>>>> On 7/16/19 1:47 PM, Dmitry Osipenko wrote:
>>>>>>> 16.07.2019 22:26, Sowjanya Komatineni пишет:
>>>>>>>> On 7/16/19 11:43 AM, Dmitry Osipenko wrote:
>>>>>>>>> 16.07.2019 21:30, Sowjanya Komatineni пишет:
>>>>>>>>>> On 7/16/19 11:25 AM, Dmitry Osipenko wrote:
>>>>>>>>>>> 16.07.2019 21:19, Sowjanya Komatineni пишет:
>>>>>>>>>>>> On 7/16/19 9:50 AM, Sowjanya Komatineni wrote:
>>>>>>>>>>>>> On 7/16/19 8:00 AM, Dmitry Osipenko wrote:
>>>>>>>>>>>>>> 16.07.2019 11:06, Peter De Schrijver пишет:
>>>>>>>>>>>>>>> On Tue, Jul 16, 2019 at 03:24:26PM +0800, Joseph Lo wrote:
>>>>>>>>>>>>>>>>> OK, Will add to CPUFreq driver...
>>>>>>>>>>>>>>>>>> The other thing that also need attention is that T124
>>>>>>>>>>>>>>>>>> CPUFreq
>>>>>>>>>>>>>>>>>> driver
>>>>>>>>>>>>>>>>>> implicitly relies on DFLL driver to be probed first,
>>>>>>>>>>>>>>>>>> which is
>>>>>>>>>>>>>>>>>> icky.
>>>>>>>>>>>>>>>>>>
>>>>>>>>>>>>>>>>> Should I add check for successful dfll clk register
>>>>>>>>>>>>>>>>> explicitly in
>>>>>>>>>>>>>>>>> CPUFreq driver probe and defer till dfll clk registers?
>>>>>>>>>>>>>> Probably you should use the "device links". See [1][2] 
>>>>>>>>>>>>>> for the
>>>>>>>>>>>>>> example.
>>>>>>>>>>>>>>
>>>>>>>>>>>>>> [1]
>>>>>>>>>>>>>> https://elixir.bootlin.com/linux/v5.2.1/source/drivers/gpu/drm/tegra/dc.c#L2383 
>>>>>>>>>>>>>>
>>>>>>>>>>>>>>
>>>>>>>>>>>>>>
>>>>>>>>>>>>>>
>>>>>>>>>>>>>>
>>>>>>>>>>>>>>
>>>>>>>>>>>>>>
>>>>>>>>>>>>>> [2]
>>>>>>>>>>>>>> https://www.kernel.org/doc/html/latest/driver-api/device_link.html 
>>>>>>>>>>>>>>
>>>>>>>>>>>>>>
>>>>>>>>>>>>>>
>>>>>>>>>>>>>> Return EPROBE_DEFER instead of EINVAL if device_link_add()
>>>>>>>>>>>>>> fails.
>>>>>>>>>>>>>> And
>>>>>>>>>>>>>> use of_find_device_by_node() to get the DFLL's device, 
>>>>>>>>>>>>>> see [3].
>>>>>>>>>>>>>>
>>>>>>>>>>>>>> [3]
>>>>>>>>>>>>>> https://git.kernel.org/pub/scm/linux/kernel/git/next/linux-next.git/tree/drivers/devfreq/tegra20-devfreq.c#n100 
>>>>>>>>>>>>>>
>>>>>>>>>>>>>>
>>>>>>>>>>>>>>
>>>>>>>>>>>>>>
>>>>>>>>>>>>>>
>>>>>>>>>>>>>>
>>>>>>>>>>>>> Will go thru and add...
>>>>>>>>>>> Looks like I initially confused this case with getting orphaned
>>>>>>>>>>> clock.
>>>>>>>>>>> I'm now seeing that the DFLL driver registers the clock and 
>>>>>>>>>>> then
>>>>>>>>>>> clk_get(dfll) should be returning EPROBE_DEFER until DFLL 
>>>>>>>>>>> driver is
>>>>>>>>>>> probed, hence everything should be fine as-is and there is 
>>>>>>>>>>> no real
>>>>>>>>>>> need
>>>>>>>>>>> for the 'device link'. Sorry for the confusion!
>>>>>>>>>>>
>>>>>>>>>>>>>>>> Sorry, I didn't follow the mail thread. Just regarding the
>>>>>>>>>>>>>>>> DFLL
>>>>>>>>>>>>>>>> part.
>>>>>>>>>>>>>>>>
>>>>>>>>>>>>>>>> As you know it, the DFLL clock is one of the CPU clock
>>>>>>>>>>>>>>>> sources and
>>>>>>>>>>>>>>>> integrated with DVFS control logic with the regulator. We
>>>>>>>>>>>>>>>> will not
>>>>>>>>>>>>>>>> switch
>>>>>>>>>>>>>>>> CPU to other clock sources once we switched to DFLL.
>>>>>>>>>>>>>>>> Because the
>>>>>>>>>>>>>>>> CPU has
>>>>>>>>>>>>>>>> been regulated by the DFLL HW with the DVFS table (CVB 
>>>>>>>>>>>>>>>> or OPP
>>>>>>>>>>>>>>>> table
>>>>>>>>>>>>>>>> you see
>>>>>>>>>>>>>>>> in the driver.). We shouldn't reparent it to other sources
>>>>>>>>>>>>>>>> with
>>>>>>>>>>>>>>>> unknew
>>>>>>>>>>>>>>>> freq/volt pair. That's not guaranteed to work. We allow
>>>>>>>>>>>>>>>> switching to
>>>>>>>>>>>>>>>> open-loop mode but different sources.
>>>>>>>>>>>>>> Okay, then the CPUFreq driver will have to enforce DFLL 
>>>>>>>>>>>>>> freq to
>>>>>>>>>>>>>> PLLP's
>>>>>>>>>>>>>> rate before switching to PLLP in order to have a proper CPU
>>>>>>>>>>>>>> voltage.
>>>>>>>>>>>>> PLLP freq is safe to work for any CPU voltage. So no need to
>>>>>>>>>>>>> enforce
>>>>>>>>>>>>> DFLL freq to PLLP rate before changing CCLK_G source to PLLP
>>>>>>>>>>>>> during
>>>>>>>>>>>>> suspend
>>>>>>>>>>>>>
>>>>>>>>>>>> Sorry, please ignore my above comment. During suspend, need to
>>>>>>>>>>>> change
>>>>>>>>>>>> CCLK_G source to PLLP when dfll is in closed loop mode 
>>>>>>>>>>>> first and
>>>>>>>>>>>> then
>>>>>>>>>>>> dfll need to be set to open loop.
>>>>>>>>>>> Okay.
>>>>>>>>>>>
>>>>>>>>>>>>>>>> And I don't exactly understand why we need to switch to
>>>>>>>>>>>>>>>> PLLP in
>>>>>>>>>>>>>>>> CPU
>>>>>>>>>>>>>>>> idle
>>>>>>>>>>>>>>>> driver. Just keep it on CL-DVFS mode all the time.
>>>>>>>>>>>>>>>>
>>>>>>>>>>>>>>>> In SC7 entry, the dfll suspend function moves it the 
>>>>>>>>>>>>>>>> open-loop
>>>>>>>>>>>>>>>> mode. That's
>>>>>>>>>>>>>>>> all. The sc7-entryfirmware will handle the rest of the
>>>>>>>>>>>>>>>> sequence to
>>>>>>>>>>>>>>>> turn off
>>>>>>>>>>>>>>>> the CPU power.
>>>>>>>>>>>>>>>>
>>>>>>>>>>>>>>>> In SC7 resume, the warmboot code will handle the 
>>>>>>>>>>>>>>>> sequence to
>>>>>>>>>>>>>>>> turn on
>>>>>>>>>>>>>>>> regulator and power up the CPU cluster. And leave it on 
>>>>>>>>>>>>>>>> PLL_P.
>>>>>>>>>>>>>>>> After
>>>>>>>>>>>>>>>> resuming to the kernel, we re-init DFLL, restore the 
>>>>>>>>>>>>>>>> CPU clock
>>>>>>>>>>>>>>>> policy (CPU
>>>>>>>>>>>>>>>> runs on DFLL open-loop mode) and then moving to close-loop
>>>>>>>>>>>>>>>> mode.
>>>>>>>>>>>>>> The DFLL is re-inited after switching CCLK to DFLL parent
>>>>>>>>>>>>>> during of
>>>>>>>>>>>>>> the
>>>>>>>>>>>>>> early clocks-state restoring by CaR driver. Hence instead of
>>>>>>>>>>>>>> having
>>>>>>>>>>>>>> odd
>>>>>>>>>>>>>> hacks in the CaR driver, it is much nicer to have a proper
>>>>>>>>>>>>>> suspend-resume sequencing of the device drivers. In this 
>>>>>>>>>>>>>> case
>>>>>>>>>>>>>> CPUFreq
>>>>>>>>>>>>>> driver is the driver that enables DFLL and switches CPU 
>>>>>>>>>>>>>> to that
>>>>>>>>>>>>>> clock
>>>>>>>>>>>>>> source, which means that this driver is also should be
>>>>>>>>>>>>>> responsible for
>>>>>>>>>>>>>> management of the DFLL's state during of suspend/resume
>>>>>>>>>>>>>> process. If
>>>>>>>>>>>>>> CPUFreq driver disables DFLL during suspend and 
>>>>>>>>>>>>>> re-enables it
>>>>>>>>>>>>>> during
>>>>>>>>>>>>>> resume, then looks like the CaR driver hacks around DFLL 
>>>>>>>>>>>>>> are not
>>>>>>>>>>>>>> needed.
>>>>>>>>>>>>>>
>>>>>>>>>>>>>>>> The DFLL part looks good to me. BTW, change the patch
>>>>>>>>>>>>>>>> subject to
>>>>>>>>>>>>>>>> "Add
>>>>>>>>>>>>>>>> suspend-resume support" seems more appropriate to me.
>>>>>>>>>>>>>>>>
>>>>>>>>>>>>>>> To clarify this, the sequences for DFLL use are as follows
>>>>>>>>>>>>>>> (assuming
>>>>>>>>>>>>>>> all
>>>>>>>>>>>>>>> required DFLL hw configuration has been done)
>>>>>>>>>>>>>>>
>>>>>>>>>>>>>>> Switch to DFLL:
>>>>>>>>>>>>>>> 0) Save current parent and frequency
>>>>>>>>>>>>>>> 1) Program DFLL to open loop mode
>>>>>>>>>>>>>>> 2) Enable DFLL
>>>>>>>>>>>>>>> 3) Change cclk_g parent to DFLL
>>>>>>>>>>>>>>> For OVR regulator:
>>>>>>>>>>>>>>> 4) Change PWM output pin from tristate to output
>>>>>>>>>>>>>>> 5) Enable DFLL PWM output
>>>>>>>>>>>>>>> For I2C regulator:
>>>>>>>>>>>>>>> 4) Enable DFLL I2C output
>>>>>>>>>>>>>>> 6) Program DFLL to closed loop mode
>>>>>>>>>>>>>>>
>>>>>>>>>>>>>>> Switch away from DFLL:
>>>>>>>>>>>>>>> 0) Change cclk_g parent to PLLP so the CPU frequency is 
>>>>>>>>>>>>>>> ok for
>>>>>>>>>>>>>>> any
>>>>>>>>>>>>>>> vdd_cpu voltage
>>>>>>>>>>>>>>> 1) Program DFLL to open loop mode
>>>>>>>>>>>>>>>
>>>>>>>>>>>> I see during switch away from DFLL (suspend), cclk_g parent 
>>>>>>>>>>>> is not
>>>>>>>>>>>> changed to PLLP before changing dfll to open loop mode.
>>>>>>>>>>>>
>>>>>>>>>>>> Will add this ...
>>>>>>>>>>> The CPUFreq driver switches parent to PLLP during the probe,
>>>>>>>>>>> similar
>>>>>>>>>>> should be done on suspend.
>>>>>>>>>>>
>>>>>>>>>>> I'm also wondering if it's always safe to switch to PLLP in the
>>>>>>>>>>> probe.
>>>>>>>>>>> If CPU is running on a lower freq than PLLP, then some other 
>>>>>>>>>>> more
>>>>>>>>>>> appropriate intermediate parent should be selected.
>>>>>>>>>>>
>>>>>>>>>> CPU parents are PLL_X, PLL_P, and dfll. PLL_X always runs at 
>>>>>>>>>> higher
>>>>>>>>>> rate
>>>>>>>>>> so switching to PLL_P during CPUFreq probe prior to dfll clock
>>>>>>>>>> enable
>>>>>>>>>> should be safe.
>>>>>>>>> AFAIK, PLLX could run at ~200MHz. There is also a divided 
>>>>>>>>> output of
>>>>>>>>> PLLP
>>>>>>>>> which CCLKG supports, the PLLP_OUT4.
>>>>>>>>>
>>>>>>>>> Probably, realistically, CPU is always running off a fast PLLX 
>>>>>>>>> during
>>>>>>>>> boot, but I'm wondering what may happen on KEXEC. I guess ideally
>>>>>>>>> CPUFreq driver should also have a 'shutdown' callback to teardown
>>>>>>>>> DFLL
>>>>>>>>> on a reboot, but likely that there are other clock-related
>>>>>>>>> problems as
>>>>>>>>> well that may break KEXEC and thus it is not very important at 
>>>>>>>>> the
>>>>>>>>> moment.
>>>>>>>>>
>>>>>>>>> [snip]
>>>>>>>> During bootup CPUG sources from PLL_X. By PLL_P source above I 
>>>>>>>> meant
>>>>>>>> PLL_P_OUT4.
>>>>>>>>
>>>>>>>> As per clock policies, PLL_X is always used for high freq like 
>>>>>>>> >800Mhz
>>>>>>>> and for low frequency it will be sourced from PLLP.
>>>>>>> Alright, then please don't forget to pre-initialize PLLP_OUT4 
>>>>>>> rate to a
>>>>>>> reasonable value using tegra_clk_init_table or assigned-clocks.
>>>>>> PLLP_OUT4 rate update is not needed as it is safe to run at 408Mhz
>>>>>> because it is below fmax @ Vmin
>>>>> So even 204MHz CVB entries are having the same voltage as 408MHz,
>>>>> correct? It's not instantly obvious to me from the DFLL driver's code
>>>>> where the fmax @ Vmin is defined, I see that there is the 
>>>>> min_millivolts
>>>>> and frequency entries starting from 204MHZ defined per-table.
>>>> Yes at Vmin CPU Fmax is ~800Mhz. So anything below that will work at
>>>> Vmin voltage and PLLP max is 408Mhz.
>>> Thank you for the clarification. It would be good to have that 
>>> commented
>>> in the code as well.
>> OK, Will add...
>
> Regarding, adding suspend/resume to CPUFreq, CPUFreq suspend happens 
> very early even before disabling non-boot CPUs and also need to export 
> clock driver APIs to CPUFreq.
>
> Was thinking of below way of implementing this...
>
>
> Clock DFLL driver Suspend:
>
>         - Save CPU clock policy registers, and Perform dfll suspend 
> which sets in open loop mode
>
> CPU Freq driver Suspend: does nothing
>
>
> Clock DFLL driver Resume:
>
>         - Re-init DFLL, Set in Open-Loop mode, restore CPU Clock 
> policy registers which actually sets source to DFLL along with other 
>         CPU Policy register restore.
>
> CPU Freq driver Resume:
>
>         - do clk_prepare_enable which acutally sets DFLL in Closed 
> loop mode
>
>
> Adding one more note: Switching CPU Clock to PLLP is not needed as CPU 
> CLock can be from dfll in open-loop mode as DFLL is not disabled 
> anywhere throught the suspend/resume path and SC7 entry FW and Warm 
> boot code will switch CPU source to PLLP.

В Tue, 16 Jul 2019 19:18:19 -0700
Sowjanya Komatineni <skomatineni@nvidia.com> пишет:

> On 7/16/19 3:06 PM, Sowjanya Komatineni wrote:
> >
> > On 7/16/19 3:00 PM, Dmitry Osipenko wrote:  
> >> 17.07.2019 0:35, Sowjanya Komatineni пишет:  
> >>> On 7/16/19 2:21 PM, Dmitry Osipenko wrote:  
> >>>> 17.07.2019 0:12, Sowjanya Komatineni пишет:  
> >>>>> On 7/16/19 1:47 PM, Dmitry Osipenko wrote:  
> >>>>>> 16.07.2019 22:26, Sowjanya Komatineni пишет:  
> >>>>>>> On 7/16/19 11:43 AM, Dmitry Osipenko wrote:  
> >>>>>>>> 16.07.2019 21:30, Sowjanya Komatineni пишет:  
> >>>>>>>>> On 7/16/19 11:25 AM, Dmitry Osipenko wrote:  
> >>>>>>>>>> 16.07.2019 21:19, Sowjanya Komatineni пишет:  
> >>>>>>>>>>> On 7/16/19 9:50 AM, Sowjanya Komatineni wrote:  
> >>>>>>>>>>>> On 7/16/19 8:00 AM, Dmitry Osipenko wrote:  
> >>>>>>>>>>>>> 16.07.2019 11:06, Peter De Schrijver пишет:  
> >>>>>>>>>>>>>> On Tue, Jul 16, 2019 at 03:24:26PM +0800, Joseph Lo
> >>>>>>>>>>>>>> wrote:  
> >>>>>>>>>>>>>>>> OK, Will add to CPUFreq driver...  
> >>>>>>>>>>>>>>>>> The other thing that also need attention is that
> >>>>>>>>>>>>>>>>> T124 CPUFreq
> >>>>>>>>>>>>>>>>> driver
> >>>>>>>>>>>>>>>>> implicitly relies on DFLL driver to be probed first,
> >>>>>>>>>>>>>>>>> which is
> >>>>>>>>>>>>>>>>> icky.
> >>>>>>>>>>>>>>>>>  
> >>>>>>>>>>>>>>>> Should I add check for successful dfll clk register
> >>>>>>>>>>>>>>>> explicitly in
> >>>>>>>>>>>>>>>> CPUFreq driver probe and defer till dfll clk
> >>>>>>>>>>>>>>>> registers?  
> >>>>>>>>>>>>> Probably you should use the "device links". See [1][2]
> >>>>>>>>>>>>> for the
> >>>>>>>>>>>>> example.
> >>>>>>>>>>>>>
> >>>>>>>>>>>>> [1]
> >>>>>>>>>>>>> https://elixir.bootlin.com/linux/v5.2.1/source/drivers/gpu/drm/tegra/dc.c#L2383 
> >>>>>>>>>>>>>
> >>>>>>>>>>>>>
> >>>>>>>>>>>>>
> >>>>>>>>>>>>>
> >>>>>>>>>>>>>
> >>>>>>>>>>>>>
> >>>>>>>>>>>>>
> >>>>>>>>>>>>> [2]
> >>>>>>>>>>>>> https://www.kernel.org/doc/html/latest/driver-api/device_link.html 
> >>>>>>>>>>>>>
> >>>>>>>>>>>>>
> >>>>>>>>>>>>>
> >>>>>>>>>>>>> Return EPROBE_DEFER instead of EINVAL if
> >>>>>>>>>>>>> device_link_add() fails.
> >>>>>>>>>>>>> And
> >>>>>>>>>>>>> use of_find_device_by_node() to get the DFLL's device,
> >>>>>>>>>>>>> see [3].
> >>>>>>>>>>>>>
> >>>>>>>>>>>>> [3]
> >>>>>>>>>>>>> https://git.kernel.org/pub/scm/linux/kernel/git/next/linux-next.git/tree/drivers/devfreq/tegra20-devfreq.c#n100 
> >>>>>>>>>>>>>
> >>>>>>>>>>>>>
> >>>>>>>>>>>>>
> >>>>>>>>>>>>>
> >>>>>>>>>>>>>
> >>>>>>>>>>>>>  
> >>>>>>>>>>>> Will go thru and add...  
> >>>>>>>>>> Looks like I initially confused this case with getting
> >>>>>>>>>> orphaned clock.
> >>>>>>>>>> I'm now seeing that the DFLL driver registers the clock
> >>>>>>>>>> and then clk_get(dfll) should be returning EPROBE_DEFER
> >>>>>>>>>> until DFLL driver is
> >>>>>>>>>> probed, hence everything should be fine as-is and there is
> >>>>>>>>>> no real
> >>>>>>>>>> need
> >>>>>>>>>> for the 'device link'. Sorry for the confusion!
> >>>>>>>>>>  
> >>>>>>>>>>>>>>> Sorry, I didn't follow the mail thread. Just
> >>>>>>>>>>>>>>> regarding the DFLL
> >>>>>>>>>>>>>>> part.
> >>>>>>>>>>>>>>>
> >>>>>>>>>>>>>>> As you know it, the DFLL clock is one of the CPU clock
> >>>>>>>>>>>>>>> sources and
> >>>>>>>>>>>>>>> integrated with DVFS control logic with the
> >>>>>>>>>>>>>>> regulator. We will not
> >>>>>>>>>>>>>>> switch
> >>>>>>>>>>>>>>> CPU to other clock sources once we switched to DFLL.
> >>>>>>>>>>>>>>> Because the
> >>>>>>>>>>>>>>> CPU has
> >>>>>>>>>>>>>>> been regulated by the DFLL HW with the DVFS table
> >>>>>>>>>>>>>>> (CVB or OPP
> >>>>>>>>>>>>>>> table
> >>>>>>>>>>>>>>> you see
> >>>>>>>>>>>>>>> in the driver.). We shouldn't reparent it to other
> >>>>>>>>>>>>>>> sources with
> >>>>>>>>>>>>>>> unknew
> >>>>>>>>>>>>>>> freq/volt pair. That's not guaranteed to work. We
> >>>>>>>>>>>>>>> allow switching to
> >>>>>>>>>>>>>>> open-loop mode but different sources.  
> >>>>>>>>>>>>> Okay, then the CPUFreq driver will have to enforce DFLL 
> >>>>>>>>>>>>> freq to
> >>>>>>>>>>>>> PLLP's
> >>>>>>>>>>>>> rate before switching to PLLP in order to have a proper
> >>>>>>>>>>>>> CPU voltage.  
> >>>>>>>>>>>> PLLP freq is safe to work for any CPU voltage. So no
> >>>>>>>>>>>> need to enforce
> >>>>>>>>>>>> DFLL freq to PLLP rate before changing CCLK_G source to
> >>>>>>>>>>>> PLLP during
> >>>>>>>>>>>> suspend
> >>>>>>>>>>>>  
> >>>>>>>>>>> Sorry, please ignore my above comment. During suspend,
> >>>>>>>>>>> need to change
> >>>>>>>>>>> CCLK_G source to PLLP when dfll is in closed loop mode
> >>>>>>>>>>> first and
> >>>>>>>>>>> then
> >>>>>>>>>>> dfll need to be set to open loop.  
> >>>>>>>>>> Okay.
> >>>>>>>>>>  
> >>>>>>>>>>>>>>> And I don't exactly understand why we need to switch
> >>>>>>>>>>>>>>> to PLLP in
> >>>>>>>>>>>>>>> CPU
> >>>>>>>>>>>>>>> idle
> >>>>>>>>>>>>>>> driver. Just keep it on CL-DVFS mode all the time.
> >>>>>>>>>>>>>>>
> >>>>>>>>>>>>>>> In SC7 entry, the dfll suspend function moves it the 
> >>>>>>>>>>>>>>> open-loop
> >>>>>>>>>>>>>>> mode. That's
> >>>>>>>>>>>>>>> all. The sc7-entryfirmware will handle the rest of the
> >>>>>>>>>>>>>>> sequence to
> >>>>>>>>>>>>>>> turn off
> >>>>>>>>>>>>>>> the CPU power.
> >>>>>>>>>>>>>>>
> >>>>>>>>>>>>>>> In SC7 resume, the warmboot code will handle the 
> >>>>>>>>>>>>>>> sequence to
> >>>>>>>>>>>>>>> turn on
> >>>>>>>>>>>>>>> regulator and power up the CPU cluster. And leave it
> >>>>>>>>>>>>>>> on PLL_P.
> >>>>>>>>>>>>>>> After
> >>>>>>>>>>>>>>> resuming to the kernel, we re-init DFLL, restore the
> >>>>>>>>>>>>>>> CPU clock
> >>>>>>>>>>>>>>> policy (CPU
> >>>>>>>>>>>>>>> runs on DFLL open-loop mode) and then moving to
> >>>>>>>>>>>>>>> close-loop mode.  
> >>>>>>>>>>>>> The DFLL is re-inited after switching CCLK to DFLL
> >>>>>>>>>>>>> parent during of
> >>>>>>>>>>>>> the
> >>>>>>>>>>>>> early clocks-state restoring by CaR driver. Hence
> >>>>>>>>>>>>> instead of having
> >>>>>>>>>>>>> odd
> >>>>>>>>>>>>> hacks in the CaR driver, it is much nicer to have a
> >>>>>>>>>>>>> proper suspend-resume sequencing of the device drivers.
> >>>>>>>>>>>>> In this case CPUFreq
> >>>>>>>>>>>>> driver is the driver that enables DFLL and switches CPU
> >>>>>>>>>>>>> to that
> >>>>>>>>>>>>> clock
> >>>>>>>>>>>>> source, which means that this driver is also should be
> >>>>>>>>>>>>> responsible for
> >>>>>>>>>>>>> management of the DFLL's state during of suspend/resume
> >>>>>>>>>>>>> process. If
> >>>>>>>>>>>>> CPUFreq driver disables DFLL during suspend and
> >>>>>>>>>>>>> re-enables it during
> >>>>>>>>>>>>> resume, then looks like the CaR driver hacks around
> >>>>>>>>>>>>> DFLL are not
> >>>>>>>>>>>>> needed.
> >>>>>>>>>>>>>  
> >>>>>>>>>>>>>>> The DFLL part looks good to me. BTW, change the patch
> >>>>>>>>>>>>>>> subject to
> >>>>>>>>>>>>>>> "Add
> >>>>>>>>>>>>>>> suspend-resume support" seems more appropriate to me.
> >>>>>>>>>>>>>>>  
> >>>>>>>>>>>>>> To clarify this, the sequences for DFLL use are as
> >>>>>>>>>>>>>> follows (assuming
> >>>>>>>>>>>>>> all
> >>>>>>>>>>>>>> required DFLL hw configuration has been done)
> >>>>>>>>>>>>>>
> >>>>>>>>>>>>>> Switch to DFLL:
> >>>>>>>>>>>>>> 0) Save current parent and frequency
> >>>>>>>>>>>>>> 1) Program DFLL to open loop mode
> >>>>>>>>>>>>>> 2) Enable DFLL
> >>>>>>>>>>>>>> 3) Change cclk_g parent to DFLL
> >>>>>>>>>>>>>> For OVR regulator:
> >>>>>>>>>>>>>> 4) Change PWM output pin from tristate to output
> >>>>>>>>>>>>>> 5) Enable DFLL PWM output
> >>>>>>>>>>>>>> For I2C regulator:
> >>>>>>>>>>>>>> 4) Enable DFLL I2C output
> >>>>>>>>>>>>>> 6) Program DFLL to closed loop mode
> >>>>>>>>>>>>>>
> >>>>>>>>>>>>>> Switch away from DFLL:
> >>>>>>>>>>>>>> 0) Change cclk_g parent to PLLP so the CPU frequency
> >>>>>>>>>>>>>> is ok for
> >>>>>>>>>>>>>> any
> >>>>>>>>>>>>>> vdd_cpu voltage
> >>>>>>>>>>>>>> 1) Program DFLL to open loop mode
> >>>>>>>>>>>>>>  
> >>>>>>>>>>> I see during switch away from DFLL (suspend), cclk_g
> >>>>>>>>>>> parent is not
> >>>>>>>>>>> changed to PLLP before changing dfll to open loop mode.
> >>>>>>>>>>>
> >>>>>>>>>>> Will add this ...  
> >>>>>>>>>> The CPUFreq driver switches parent to PLLP during the
> >>>>>>>>>> probe, similar
> >>>>>>>>>> should be done on suspend.
> >>>>>>>>>>
> >>>>>>>>>> I'm also wondering if it's always safe to switch to PLLP
> >>>>>>>>>> in the probe.
> >>>>>>>>>> If CPU is running on a lower freq than PLLP, then some
> >>>>>>>>>> other more
> >>>>>>>>>> appropriate intermediate parent should be selected.
> >>>>>>>>>>  
> >>>>>>>>> CPU parents are PLL_X, PLL_P, and dfll. PLL_X always runs
> >>>>>>>>> at higher
> >>>>>>>>> rate
> >>>>>>>>> so switching to PLL_P during CPUFreq probe prior to dfll
> >>>>>>>>> clock enable
> >>>>>>>>> should be safe.  
> >>>>>>>> AFAIK, PLLX could run at ~200MHz. There is also a divided 
> >>>>>>>> output of
> >>>>>>>> PLLP
> >>>>>>>> which CCLKG supports, the PLLP_OUT4.
> >>>>>>>>
> >>>>>>>> Probably, realistically, CPU is always running off a fast
> >>>>>>>> PLLX during
> >>>>>>>> boot, but I'm wondering what may happen on KEXEC. I guess
> >>>>>>>> ideally CPUFreq driver should also have a 'shutdown'
> >>>>>>>> callback to teardown DFLL
> >>>>>>>> on a reboot, but likely that there are other clock-related
> >>>>>>>> problems as
> >>>>>>>> well that may break KEXEC and thus it is not very important
> >>>>>>>> at the moment.
> >>>>>>>>
> >>>>>>>> [snip]  
> >>>>>>> During bootup CPUG sources from PLL_X. By PLL_P source above
> >>>>>>> I meant
> >>>>>>> PLL_P_OUT4.
> >>>>>>>
> >>>>>>> As per clock policies, PLL_X is always used for high freq
> >>>>>>> like   
> >>>>>>> >800Mhz  
> >>>>>>> and for low frequency it will be sourced from PLLP.  
> >>>>>> Alright, then please don't forget to pre-initialize PLLP_OUT4 
> >>>>>> rate to a
> >>>>>> reasonable value using tegra_clk_init_table or
> >>>>>> assigned-clocks.  
> >>>>> PLLP_OUT4 rate update is not needed as it is safe to run at
> >>>>> 408Mhz because it is below fmax @ Vmin  
> >>>> So even 204MHz CVB entries are having the same voltage as 408MHz,
> >>>> correct? It's not instantly obvious to me from the DFLL driver's
> >>>> code where the fmax @ Vmin is defined, I see that there is the 
> >>>> min_millivolts
> >>>> and frequency entries starting from 204MHZ defined per-table.  
> >>> Yes at Vmin CPU Fmax is ~800Mhz. So anything below that will work
> >>> at Vmin voltage and PLLP max is 408Mhz.  
> >> Thank you for the clarification. It would be good to have that
> >> commented in the code as well.  
> > OK, Will add...  
> 
> Regarding, adding suspend/resume to CPUFreq, CPUFreq suspend happens 
> very early even before disabling non-boot CPUs and also need to
> export clock driver APIs to CPUFreq.
> 
> Was thinking of below way of implementing this...
> 
> 
> Clock DFLL driver Suspend:
> 
>          - Save CPU clock policy registers, and Perform dfll suspend 
> which sets in open loop mode
> 
> CPU Freq driver Suspend: does nothing
> 
> 
> Clock DFLL driver Resume:
> 
>          - Re-init DFLL, Set in Open-Loop mode, restore CPU Clock
> policy registers which actually sets source to DFLL along with other
> CPU Policy register restore.
> 
> CPU Freq driver Resume:
> 
>          - do clk_prepare_enable which acutally sets DFLL in Closed
> loop mode


It doesn't matter much when CPUFreq driver suspends, it's only
important that it suspends before CaR.

I'm not sure why do you need anything else from DFLL driver other than
what is already exposed via generic CCF API. It looks to me
that switching CPU's parent clock away from DFLL and then disabling
DFLL's clock is enough for suspend, accordingly to what Peter wrote. And
resuming is the same as what's done on CPUFreq's driver probe. The CCLK
policy should be saved and restored by the CaR driver, you don't need to
care about it. The cpufreq-dt driver sets
CPUFREQ_NEED_INITIAL_FREQ_CHECK, hence you don't need to care about
restoring the original CPU freq on resume, IIUC.

On 7/16/19 8:54 PM, Dmitry Osipenko wrote:
> В Tue, 16 Jul 2019 19:18:19 -0700
> Sowjanya Komatineni <skomatineni@nvidia.com> пишет:
>
>> On 7/16/19 3:06 PM, Sowjanya Komatineni wrote:
>>> On 7/16/19 3:00 PM, Dmitry Osipenko wrote:
>>>> 17.07.2019 0:35, Sowjanya Komatineni пишет:
>>>>> On 7/16/19 2:21 PM, Dmitry Osipenko wrote:
>>>>>> 17.07.2019 0:12, Sowjanya Komatineni пишет:
>>>>>>> On 7/16/19 1:47 PM, Dmitry Osipenko wrote:
>>>>>>>> 16.07.2019 22:26, Sowjanya Komatineni пишет:
>>>>>>>>> On 7/16/19 11:43 AM, Dmitry Osipenko wrote:
>>>>>>>>>> 16.07.2019 21:30, Sowjanya Komatineni пишет:
>>>>>>>>>>> On 7/16/19 11:25 AM, Dmitry Osipenko wrote:
>>>>>>>>>>>> 16.07.2019 21:19, Sowjanya Komatineni пишет:
>>>>>>>>>>>>> On 7/16/19 9:50 AM, Sowjanya Komatineni wrote:
>>>>>>>>>>>>>> On 7/16/19 8:00 AM, Dmitry Osipenko wrote:
>>>>>>>>>>>>>>> 16.07.2019 11:06, Peter De Schrijver пишет:
>>>>>>>>>>>>>>>> On Tue, Jul 16, 2019 at 03:24:26PM +0800, Joseph Lo
>>>>>>>>>>>>>>>> wrote:
>>>>>>>>>>>>>>>>>> OK, Will add to CPUFreq driver...
>>>>>>>>>>>>>>>>>>> The other thing that also need attention is that
>>>>>>>>>>>>>>>>>>> T124 CPUFreq
>>>>>>>>>>>>>>>>>>> driver
>>>>>>>>>>>>>>>>>>> implicitly relies on DFLL driver to be probed first,
>>>>>>>>>>>>>>>>>>> which is
>>>>>>>>>>>>>>>>>>> icky.
>>>>>>>>>>>>>>>>>>>   
>>>>>>>>>>>>>>>>>> Should I add check for successful dfll clk register
>>>>>>>>>>>>>>>>>> explicitly in
>>>>>>>>>>>>>>>>>> CPUFreq driver probe and defer till dfll clk
>>>>>>>>>>>>>>>>>> registers?
>>>>>>>>>>>>>>> Probably you should use the "device links". See [1][2]
>>>>>>>>>>>>>>> for the
>>>>>>>>>>>>>>> example.
>>>>>>>>>>>>>>>
>>>>>>>>>>>>>>> [1]
>>>>>>>>>>>>>>> https://elixir.bootlin.com/linux/v5.2.1/source/drivers/gpu/drm/tegra/dc.c#L2383
>>>>>>>>>>>>>>>
>>>>>>>>>>>>>>>
>>>>>>>>>>>>>>>
>>>>>>>>>>>>>>>
>>>>>>>>>>>>>>>
>>>>>>>>>>>>>>>
>>>>>>>>>>>>>>>
>>>>>>>>>>>>>>> [2]
>>>>>>>>>>>>>>> https://www.kernel.org/doc/html/latest/driver-api/device_link.html
>>>>>>>>>>>>>>>
>>>>>>>>>>>>>>>
>>>>>>>>>>>>>>>
>>>>>>>>>>>>>>> Return EPROBE_DEFER instead of EINVAL if
>>>>>>>>>>>>>>> device_link_add() fails.
>>>>>>>>>>>>>>> And
>>>>>>>>>>>>>>> use of_find_device_by_node() to get the DFLL's device,
>>>>>>>>>>>>>>> see [3].
>>>>>>>>>>>>>>>
>>>>>>>>>>>>>>> [3]
>>>>>>>>>>>>>>> https://git.kernel.org/pub/scm/linux/kernel/git/next/linux-next.git/tree/drivers/devfreq/tegra20-devfreq.c#n100
>>>>>>>>>>>>>>>
>>>>>>>>>>>>>>>
>>>>>>>>>>>>>>>
>>>>>>>>>>>>>>>
>>>>>>>>>>>>>>>
>>>>>>>>>>>>>>>   
>>>>>>>>>>>>>> Will go thru and add...
>>>>>>>>>>>> Looks like I initially confused this case with getting
>>>>>>>>>>>> orphaned clock.
>>>>>>>>>>>> I'm now seeing that the DFLL driver registers the clock
>>>>>>>>>>>> and then clk_get(dfll) should be returning EPROBE_DEFER
>>>>>>>>>>>> until DFLL driver is
>>>>>>>>>>>> probed, hence everything should be fine as-is and there is
>>>>>>>>>>>> no real
>>>>>>>>>>>> need
>>>>>>>>>>>> for the 'device link'. Sorry for the confusion!
>>>>>>>>>>>>   
>>>>>>>>>>>>>>>>> Sorry, I didn't follow the mail thread. Just
>>>>>>>>>>>>>>>>> regarding the DFLL
>>>>>>>>>>>>>>>>> part.
>>>>>>>>>>>>>>>>>
>>>>>>>>>>>>>>>>> As you know it, the DFLL clock is one of the CPU clock
>>>>>>>>>>>>>>>>> sources and
>>>>>>>>>>>>>>>>> integrated with DVFS control logic with the
>>>>>>>>>>>>>>>>> regulator. We will not
>>>>>>>>>>>>>>>>> switch
>>>>>>>>>>>>>>>>> CPU to other clock sources once we switched to DFLL.
>>>>>>>>>>>>>>>>> Because the
>>>>>>>>>>>>>>>>> CPU has
>>>>>>>>>>>>>>>>> been regulated by the DFLL HW with the DVFS table
>>>>>>>>>>>>>>>>> (CVB or OPP
>>>>>>>>>>>>>>>>> table
>>>>>>>>>>>>>>>>> you see
>>>>>>>>>>>>>>>>> in the driver.). We shouldn't reparent it to other
>>>>>>>>>>>>>>>>> sources with
>>>>>>>>>>>>>>>>> unknew
>>>>>>>>>>>>>>>>> freq/volt pair. That's not guaranteed to work. We
>>>>>>>>>>>>>>>>> allow switching to
>>>>>>>>>>>>>>>>> open-loop mode but different sources.
>>>>>>>>>>>>>>> Okay, then the CPUFreq driver will have to enforce DFLL
>>>>>>>>>>>>>>> freq to
>>>>>>>>>>>>>>> PLLP's
>>>>>>>>>>>>>>> rate before switching to PLLP in order to have a proper
>>>>>>>>>>>>>>> CPU voltage.
>>>>>>>>>>>>>> PLLP freq is safe to work for any CPU voltage. So no
>>>>>>>>>>>>>> need to enforce
>>>>>>>>>>>>>> DFLL freq to PLLP rate before changing CCLK_G source to
>>>>>>>>>>>>>> PLLP during
>>>>>>>>>>>>>> suspend
>>>>>>>>>>>>>>   
>>>>>>>>>>>>> Sorry, please ignore my above comment. During suspend,
>>>>>>>>>>>>> need to change
>>>>>>>>>>>>> CCLK_G source to PLLP when dfll is in closed loop mode
>>>>>>>>>>>>> first and
>>>>>>>>>>>>> then
>>>>>>>>>>>>> dfll need to be set to open loop.
>>>>>>>>>>>> Okay.
>>>>>>>>>>>>   
>>>>>>>>>>>>>>>>> And I don't exactly understand why we need to switch
>>>>>>>>>>>>>>>>> to PLLP in
>>>>>>>>>>>>>>>>> CPU
>>>>>>>>>>>>>>>>> idle
>>>>>>>>>>>>>>>>> driver. Just keep it on CL-DVFS mode all the time.
>>>>>>>>>>>>>>>>>
>>>>>>>>>>>>>>>>> In SC7 entry, the dfll suspend function moves it the
>>>>>>>>>>>>>>>>> open-loop
>>>>>>>>>>>>>>>>> mode. That's
>>>>>>>>>>>>>>>>> all. The sc7-entryfirmware will handle the rest of the
>>>>>>>>>>>>>>>>> sequence to
>>>>>>>>>>>>>>>>> turn off
>>>>>>>>>>>>>>>>> the CPU power.
>>>>>>>>>>>>>>>>>
>>>>>>>>>>>>>>>>> In SC7 resume, the warmboot code will handle the
>>>>>>>>>>>>>>>>> sequence to
>>>>>>>>>>>>>>>>> turn on
>>>>>>>>>>>>>>>>> regulator and power up the CPU cluster. And leave it
>>>>>>>>>>>>>>>>> on PLL_P.
>>>>>>>>>>>>>>>>> After
>>>>>>>>>>>>>>>>> resuming to the kernel, we re-init DFLL, restore the
>>>>>>>>>>>>>>>>> CPU clock
>>>>>>>>>>>>>>>>> policy (CPU
>>>>>>>>>>>>>>>>> runs on DFLL open-loop mode) and then moving to
>>>>>>>>>>>>>>>>> close-loop mode.
>>>>>>>>>>>>>>> The DFLL is re-inited after switching CCLK to DFLL
>>>>>>>>>>>>>>> parent during of
>>>>>>>>>>>>>>> the
>>>>>>>>>>>>>>> early clocks-state restoring by CaR driver. Hence
>>>>>>>>>>>>>>> instead of having
>>>>>>>>>>>>>>> odd
>>>>>>>>>>>>>>> hacks in the CaR driver, it is much nicer to have a
>>>>>>>>>>>>>>> proper suspend-resume sequencing of the device drivers.
>>>>>>>>>>>>>>> In this case CPUFreq
>>>>>>>>>>>>>>> driver is the driver that enables DFLL and switches CPU
>>>>>>>>>>>>>>> to that
>>>>>>>>>>>>>>> clock
>>>>>>>>>>>>>>> source, which means that this driver is also should be
>>>>>>>>>>>>>>> responsible for
>>>>>>>>>>>>>>> management of the DFLL's state during of suspend/resume
>>>>>>>>>>>>>>> process. If
>>>>>>>>>>>>>>> CPUFreq driver disables DFLL during suspend and
>>>>>>>>>>>>>>> re-enables it during
>>>>>>>>>>>>>>> resume, then looks like the CaR driver hacks around
>>>>>>>>>>>>>>> DFLL are not
>>>>>>>>>>>>>>> needed.
>>>>>>>>>>>>>>>   
>>>>>>>>>>>>>>>>> The DFLL part looks good to me. BTW, change the patch
>>>>>>>>>>>>>>>>> subject to
>>>>>>>>>>>>>>>>> "Add
>>>>>>>>>>>>>>>>> suspend-resume support" seems more appropriate to me.
>>>>>>>>>>>>>>>>>   
>>>>>>>>>>>>>>>> To clarify this, the sequences for DFLL use are as
>>>>>>>>>>>>>>>> follows (assuming
>>>>>>>>>>>>>>>> all
>>>>>>>>>>>>>>>> required DFLL hw configuration has been done)
>>>>>>>>>>>>>>>>
>>>>>>>>>>>>>>>> Switch to DFLL:
>>>>>>>>>>>>>>>> 0) Save current parent and frequency
>>>>>>>>>>>>>>>> 1) Program DFLL to open loop mode
>>>>>>>>>>>>>>>> 2) Enable DFLL
>>>>>>>>>>>>>>>> 3) Change cclk_g parent to DFLL
>>>>>>>>>>>>>>>> For OVR regulator:
>>>>>>>>>>>>>>>> 4) Change PWM output pin from tristate to output
>>>>>>>>>>>>>>>> 5) Enable DFLL PWM output
>>>>>>>>>>>>>>>> For I2C regulator:
>>>>>>>>>>>>>>>> 4) Enable DFLL I2C output
>>>>>>>>>>>>>>>> 6) Program DFLL to closed loop mode
>>>>>>>>>>>>>>>>
>>>>>>>>>>>>>>>> Switch away from DFLL:
>>>>>>>>>>>>>>>> 0) Change cclk_g parent to PLLP so the CPU frequency
>>>>>>>>>>>>>>>> is ok for
>>>>>>>>>>>>>>>> any
>>>>>>>>>>>>>>>> vdd_cpu voltage
>>>>>>>>>>>>>>>> 1) Program DFLL to open loop mode
>>>>>>>>>>>>>>>>   
>>>>>>>>>>>>> I see during switch away from DFLL (suspend), cclk_g
>>>>>>>>>>>>> parent is not
>>>>>>>>>>>>> changed to PLLP before changing dfll to open loop mode.
>>>>>>>>>>>>>
>>>>>>>>>>>>> Will add this ...
>>>>>>>>>>>> The CPUFreq driver switches parent to PLLP during the
>>>>>>>>>>>> probe, similar
>>>>>>>>>>>> should be done on suspend.
>>>>>>>>>>>>
>>>>>>>>>>>> I'm also wondering if it's always safe to switch to PLLP
>>>>>>>>>>>> in the probe.
>>>>>>>>>>>> If CPU is running on a lower freq than PLLP, then some
>>>>>>>>>>>> other more
>>>>>>>>>>>> appropriate intermediate parent should be selected.
>>>>>>>>>>>>   
>>>>>>>>>>> CPU parents are PLL_X, PLL_P, and dfll. PLL_X always runs
>>>>>>>>>>> at higher
>>>>>>>>>>> rate
>>>>>>>>>>> so switching to PLL_P during CPUFreq probe prior to dfll
>>>>>>>>>>> clock enable
>>>>>>>>>>> should be safe.
>>>>>>>>>> AFAIK, PLLX could run at ~200MHz. There is also a divided
>>>>>>>>>> output of
>>>>>>>>>> PLLP
>>>>>>>>>> which CCLKG supports, the PLLP_OUT4.
>>>>>>>>>>
>>>>>>>>>> Probably, realistically, CPU is always running off a fast
>>>>>>>>>> PLLX during
>>>>>>>>>> boot, but I'm wondering what may happen on KEXEC. I guess
>>>>>>>>>> ideally CPUFreq driver should also have a 'shutdown'
>>>>>>>>>> callback to teardown DFLL
>>>>>>>>>> on a reboot, but likely that there are other clock-related
>>>>>>>>>> problems as
>>>>>>>>>> well that may break KEXEC and thus it is not very important
>>>>>>>>>> at the moment.
>>>>>>>>>>
>>>>>>>>>> [snip]
>>>>>>>>> During bootup CPUG sources from PLL_X. By PLL_P source above
>>>>>>>>> I meant
>>>>>>>>> PLL_P_OUT4.
>>>>>>>>>
>>>>>>>>> As per clock policies, PLL_X is always used for high freq
>>>>>>>>> like
>>>>>>>>>> 800Mhz
>>>>>>>>> and for low frequency it will be sourced from PLLP.
>>>>>>>> Alright, then please don't forget to pre-initialize PLLP_OUT4
>>>>>>>> rate to a
>>>>>>>> reasonable value using tegra_clk_init_table or
>>>>>>>> assigned-clocks.
>>>>>>> PLLP_OUT4 rate update is not needed as it is safe to run at
>>>>>>> 408Mhz because it is below fmax @ Vmin
>>>>>> So even 204MHz CVB entries are having the same voltage as 408MHz,
>>>>>> correct? It's not instantly obvious to me from the DFLL driver's
>>>>>> code where the fmax @ Vmin is defined, I see that there is the
>>>>>> min_millivolts
>>>>>> and frequency entries starting from 204MHZ defined per-table.
>>>>> Yes at Vmin CPU Fmax is ~800Mhz. So anything below that will work
>>>>> at Vmin voltage and PLLP max is 408Mhz.
>>>> Thank you for the clarification. It would be good to have that
>>>> commented in the code as well.
>>> OK, Will add...
>> Regarding, adding suspend/resume to CPUFreq, CPUFreq suspend happens
>> very early even before disabling non-boot CPUs and also need to
>> export clock driver APIs to CPUFreq.
>>
>> Was thinking of below way of implementing this...
>>
>>
>> Clock DFLL driver Suspend:
>>
>>           - Save CPU clock policy registers, and Perform dfll suspend
>> which sets in open loop mode
>>
>> CPU Freq driver Suspend: does nothing
>>
>>
>> Clock DFLL driver Resume:
>>
>>           - Re-init DFLL, Set in Open-Loop mode, restore CPU Clock
>> policy registers which actually sets source to DFLL along with other
>> CPU Policy register restore.
>>
>> CPU Freq driver Resume:
>>
>>           - do clk_prepare_enable which acutally sets DFLL in Closed
>> loop mode
>
> It doesn't matter much when CPUFreq driver suspends, it's only
> important that it suspends before CaR.
>
> I'm not sure why do you need anything else from DFLL driver other than
> what is already exposed via generic CCF API. It looks to me
> that switching CPU's parent clock away from DFLL and then disabling
> DFLL's clock is enough for suspend, accordingly to what Peter wrote. And
> resuming is the same as what's done on CPUFreq's driver probe. The CCLK
> policy should be saved and restored by the CaR driver, you don't need to
> care about it. The cpufreq-dt driver sets
> CPUFREQ_NEED_INITIAL_FREQ_CHECK, hence you don't need to care about
> restoring the original CPU freq on resume, IIUC.
>
>
CPU policy save/restore are part of this patch and it wasn't there 
earlier and when I moved suspend/resume to clock-dfll I moved cpu 
restore also to clock-dfll driver.

Thought to restore cpu source to dfll in CPUFreq driver which requires 
exporting API from clock driver. But leaving CPU restore in Tegra210 
clock driver, we can do dfll suspend/resume thru CPUFreq driver pm_ops 
without exporting clock APIs

Will update ...

> With

В Tue, 16 Jul 2019 19:35:49 -0700
Sowjanya Komatineni <skomatineni@nvidia.com> пишет:

> On 7/16/19 7:18 PM, Sowjanya Komatineni wrote:
> >
> > On 7/16/19 3:06 PM, Sowjanya Komatineni wrote:  
> >>
> >> On 7/16/19 3:00 PM, Dmitry Osipenko wrote:  
> >>> 17.07.2019 0:35, Sowjanya Komatineni пишет:  
> >>>> On 7/16/19 2:21 PM, Dmitry Osipenko wrote:  
> >>>>> 17.07.2019 0:12, Sowjanya Komatineni пишет:  
> >>>>>> On 7/16/19 1:47 PM, Dmitry Osipenko wrote:  
> >>>>>>> 16.07.2019 22:26, Sowjanya Komatineni пишет:  
> >>>>>>>> On 7/16/19 11:43 AM, Dmitry Osipenko wrote:  
> >>>>>>>>> 16.07.2019 21:30, Sowjanya Komatineni пишет:  
> >>>>>>>>>> On 7/16/19 11:25 AM, Dmitry Osipenko wrote:  
> >>>>>>>>>>> 16.07.2019 21:19, Sowjanya Komatineni пишет:  
> >>>>>>>>>>>> On 7/16/19 9:50 AM, Sowjanya Komatineni wrote:  
> >>>>>>>>>>>>> On 7/16/19 8:00 AM, Dmitry Osipenko wrote:  
> >>>>>>>>>>>>>> 16.07.2019 11:06, Peter De Schrijver пишет:  
> >>>>>>>>>>>>>>> On Tue, Jul 16, 2019 at 03:24:26PM +0800, Joseph Lo
> >>>>>>>>>>>>>>> wrote:  
> >>>>>>>>>>>>>>>>> OK, Will add to CPUFreq driver...  
> >>>>>>>>>>>>>>>>>> The other thing that also need attention is that
> >>>>>>>>>>>>>>>>>> T124 CPUFreq
> >>>>>>>>>>>>>>>>>> driver
> >>>>>>>>>>>>>>>>>> implicitly relies on DFLL driver to be probed
> >>>>>>>>>>>>>>>>>> first, which is
> >>>>>>>>>>>>>>>>>> icky.
> >>>>>>>>>>>>>>>>>>  
> >>>>>>>>>>>>>>>>> Should I add check for successful dfll clk register
> >>>>>>>>>>>>>>>>> explicitly in
> >>>>>>>>>>>>>>>>> CPUFreq driver probe and defer till dfll clk
> >>>>>>>>>>>>>>>>> registers?  
> >>>>>>>>>>>>>> Probably you should use the "device links". See [1][2] 
> >>>>>>>>>>>>>> for the
> >>>>>>>>>>>>>> example.
> >>>>>>>>>>>>>>
> >>>>>>>>>>>>>> [1]
> >>>>>>>>>>>>>> https://elixir.bootlin.com/linux/v5.2.1/source/drivers/gpu/drm/tegra/dc.c#L2383 
> >>>>>>>>>>>>>>
> >>>>>>>>>>>>>>
> >>>>>>>>>>>>>>
> >>>>>>>>>>>>>>
> >>>>>>>>>>>>>>
> >>>>>>>>>>>>>>
> >>>>>>>>>>>>>>
> >>>>>>>>>>>>>> [2]
> >>>>>>>>>>>>>> https://www.kernel.org/doc/html/latest/driver-api/device_link.html 
> >>>>>>>>>>>>>>
> >>>>>>>>>>>>>>
> >>>>>>>>>>>>>>
> >>>>>>>>>>>>>> Return EPROBE_DEFER instead of EINVAL if
> >>>>>>>>>>>>>> device_link_add() fails.
> >>>>>>>>>>>>>> And
> >>>>>>>>>>>>>> use of_find_device_by_node() to get the DFLL's device, 
> >>>>>>>>>>>>>> see [3].
> >>>>>>>>>>>>>>
> >>>>>>>>>>>>>> [3]
> >>>>>>>>>>>>>> https://git.kernel.org/pub/scm/linux/kernel/git/next/linux-next.git/tree/drivers/devfreq/tegra20-devfreq.c#n100 
> >>>>>>>>>>>>>>
> >>>>>>>>>>>>>>
> >>>>>>>>>>>>>>
> >>>>>>>>>>>>>>
> >>>>>>>>>>>>>>
> >>>>>>>>>>>>>>  
> >>>>>>>>>>>>> Will go thru and add...  
> >>>>>>>>>>> Looks like I initially confused this case with getting
> >>>>>>>>>>> orphaned clock.
> >>>>>>>>>>> I'm now seeing that the DFLL driver registers the clock
> >>>>>>>>>>> and then
> >>>>>>>>>>> clk_get(dfll) should be returning EPROBE_DEFER until DFLL 
> >>>>>>>>>>> driver is
> >>>>>>>>>>> probed, hence everything should be fine as-is and there
> >>>>>>>>>>> is no real
> >>>>>>>>>>> need
> >>>>>>>>>>> for the 'device link'. Sorry for the confusion!
> >>>>>>>>>>>  
> >>>>>>>>>>>>>>>> Sorry, I didn't follow the mail thread. Just
> >>>>>>>>>>>>>>>> regarding the DFLL
> >>>>>>>>>>>>>>>> part.
> >>>>>>>>>>>>>>>>
> >>>>>>>>>>>>>>>> As you know it, the DFLL clock is one of the CPU
> >>>>>>>>>>>>>>>> clock sources and
> >>>>>>>>>>>>>>>> integrated with DVFS control logic with the
> >>>>>>>>>>>>>>>> regulator. We will not
> >>>>>>>>>>>>>>>> switch
> >>>>>>>>>>>>>>>> CPU to other clock sources once we switched to DFLL.
> >>>>>>>>>>>>>>>> Because the
> >>>>>>>>>>>>>>>> CPU has
> >>>>>>>>>>>>>>>> been regulated by the DFLL HW with the DVFS table
> >>>>>>>>>>>>>>>> (CVB or OPP
> >>>>>>>>>>>>>>>> table
> >>>>>>>>>>>>>>>> you see
> >>>>>>>>>>>>>>>> in the driver.). We shouldn't reparent it to other
> >>>>>>>>>>>>>>>> sources with
> >>>>>>>>>>>>>>>> unknew
> >>>>>>>>>>>>>>>> freq/volt pair. That's not guaranteed to work. We
> >>>>>>>>>>>>>>>> allow switching to
> >>>>>>>>>>>>>>>> open-loop mode but different sources.  
> >>>>>>>>>>>>>> Okay, then the CPUFreq driver will have to enforce
> >>>>>>>>>>>>>> DFLL freq to
> >>>>>>>>>>>>>> PLLP's
> >>>>>>>>>>>>>> rate before switching to PLLP in order to have a
> >>>>>>>>>>>>>> proper CPU voltage.  
> >>>>>>>>>>>>> PLLP freq is safe to work for any CPU voltage. So no
> >>>>>>>>>>>>> need to enforce
> >>>>>>>>>>>>> DFLL freq to PLLP rate before changing CCLK_G source to
> >>>>>>>>>>>>> PLLP during
> >>>>>>>>>>>>> suspend
> >>>>>>>>>>>>>  
> >>>>>>>>>>>> Sorry, please ignore my above comment. During suspend,
> >>>>>>>>>>>> need to change
> >>>>>>>>>>>> CCLK_G source to PLLP when dfll is in closed loop mode 
> >>>>>>>>>>>> first and
> >>>>>>>>>>>> then
> >>>>>>>>>>>> dfll need to be set to open loop.  
> >>>>>>>>>>> Okay.
> >>>>>>>>>>>  
> >>>>>>>>>>>>>>>> And I don't exactly understand why we need to switch
> >>>>>>>>>>>>>>>> to PLLP in
> >>>>>>>>>>>>>>>> CPU
> >>>>>>>>>>>>>>>> idle
> >>>>>>>>>>>>>>>> driver. Just keep it on CL-DVFS mode all the time.
> >>>>>>>>>>>>>>>>
> >>>>>>>>>>>>>>>> In SC7 entry, the dfll suspend function moves it the 
> >>>>>>>>>>>>>>>> open-loop
> >>>>>>>>>>>>>>>> mode. That's
> >>>>>>>>>>>>>>>> all. The sc7-entryfirmware will handle the rest of
> >>>>>>>>>>>>>>>> the sequence to
> >>>>>>>>>>>>>>>> turn off
> >>>>>>>>>>>>>>>> the CPU power.
> >>>>>>>>>>>>>>>>
> >>>>>>>>>>>>>>>> In SC7 resume, the warmboot code will handle the 
> >>>>>>>>>>>>>>>> sequence to
> >>>>>>>>>>>>>>>> turn on
> >>>>>>>>>>>>>>>> regulator and power up the CPU cluster. And leave it
> >>>>>>>>>>>>>>>> on PLL_P.
> >>>>>>>>>>>>>>>> After
> >>>>>>>>>>>>>>>> resuming to the kernel, we re-init DFLL, restore the 
> >>>>>>>>>>>>>>>> CPU clock
> >>>>>>>>>>>>>>>> policy (CPU
> >>>>>>>>>>>>>>>> runs on DFLL open-loop mode) and then moving to
> >>>>>>>>>>>>>>>> close-loop mode.  
> >>>>>>>>>>>>>> The DFLL is re-inited after switching CCLK to DFLL
> >>>>>>>>>>>>>> parent during of
> >>>>>>>>>>>>>> the
> >>>>>>>>>>>>>> early clocks-state restoring by CaR driver. Hence
> >>>>>>>>>>>>>> instead of having
> >>>>>>>>>>>>>> odd
> >>>>>>>>>>>>>> hacks in the CaR driver, it is much nicer to have a
> >>>>>>>>>>>>>> proper suspend-resume sequencing of the device
> >>>>>>>>>>>>>> drivers. In this case
> >>>>>>>>>>>>>> CPUFreq
> >>>>>>>>>>>>>> driver is the driver that enables DFLL and switches
> >>>>>>>>>>>>>> CPU to that
> >>>>>>>>>>>>>> clock
> >>>>>>>>>>>>>> source, which means that this driver is also should be
> >>>>>>>>>>>>>> responsible for
> >>>>>>>>>>>>>> management of the DFLL's state during of suspend/resume
> >>>>>>>>>>>>>> process. If
> >>>>>>>>>>>>>> CPUFreq driver disables DFLL during suspend and 
> >>>>>>>>>>>>>> re-enables it
> >>>>>>>>>>>>>> during
> >>>>>>>>>>>>>> resume, then looks like the CaR driver hacks around
> >>>>>>>>>>>>>> DFLL are not
> >>>>>>>>>>>>>> needed.
> >>>>>>>>>>>>>>  
> >>>>>>>>>>>>>>>> The DFLL part looks good to me. BTW, change the patch
> >>>>>>>>>>>>>>>> subject to
> >>>>>>>>>>>>>>>> "Add
> >>>>>>>>>>>>>>>> suspend-resume support" seems more appropriate to me.
> >>>>>>>>>>>>>>>>  
> >>>>>>>>>>>>>>> To clarify this, the sequences for DFLL use are as
> >>>>>>>>>>>>>>> follows (assuming
> >>>>>>>>>>>>>>> all
> >>>>>>>>>>>>>>> required DFLL hw configuration has been done)
> >>>>>>>>>>>>>>>
> >>>>>>>>>>>>>>> Switch to DFLL:
> >>>>>>>>>>>>>>> 0) Save current parent and frequency
> >>>>>>>>>>>>>>> 1) Program DFLL to open loop mode
> >>>>>>>>>>>>>>> 2) Enable DFLL
> >>>>>>>>>>>>>>> 3) Change cclk_g parent to DFLL
> >>>>>>>>>>>>>>> For OVR regulator:
> >>>>>>>>>>>>>>> 4) Change PWM output pin from tristate to output
> >>>>>>>>>>>>>>> 5) Enable DFLL PWM output
> >>>>>>>>>>>>>>> For I2C regulator:
> >>>>>>>>>>>>>>> 4) Enable DFLL I2C output
> >>>>>>>>>>>>>>> 6) Program DFLL to closed loop mode
> >>>>>>>>>>>>>>>
> >>>>>>>>>>>>>>> Switch away from DFLL:
> >>>>>>>>>>>>>>> 0) Change cclk_g parent to PLLP so the CPU frequency
> >>>>>>>>>>>>>>> is ok for
> >>>>>>>>>>>>>>> any
> >>>>>>>>>>>>>>> vdd_cpu voltage
> >>>>>>>>>>>>>>> 1) Program DFLL to open loop mode
> >>>>>>>>>>>>>>>  
> >>>>>>>>>>>> I see during switch away from DFLL (suspend), cclk_g
> >>>>>>>>>>>> parent is not
> >>>>>>>>>>>> changed to PLLP before changing dfll to open loop mode.
> >>>>>>>>>>>>
> >>>>>>>>>>>> Will add this ...  
> >>>>>>>>>>> The CPUFreq driver switches parent to PLLP during the
> >>>>>>>>>>> probe, similar
> >>>>>>>>>>> should be done on suspend.
> >>>>>>>>>>>
> >>>>>>>>>>> I'm also wondering if it's always safe to switch to PLLP
> >>>>>>>>>>> in the probe.
> >>>>>>>>>>> If CPU is running on a lower freq than PLLP, then some
> >>>>>>>>>>> other more
> >>>>>>>>>>> appropriate intermediate parent should be selected.
> >>>>>>>>>>>  
> >>>>>>>>>> CPU parents are PLL_X, PLL_P, and dfll. PLL_X always runs
> >>>>>>>>>> at higher
> >>>>>>>>>> rate
> >>>>>>>>>> so switching to PLL_P during CPUFreq probe prior to dfll
> >>>>>>>>>> clock enable
> >>>>>>>>>> should be safe.  
> >>>>>>>>> AFAIK, PLLX could run at ~200MHz. There is also a divided 
> >>>>>>>>> output of
> >>>>>>>>> PLLP
> >>>>>>>>> which CCLKG supports, the PLLP_OUT4.
> >>>>>>>>>
> >>>>>>>>> Probably, realistically, CPU is always running off a fast
> >>>>>>>>> PLLX during
> >>>>>>>>> boot, but I'm wondering what may happen on KEXEC. I guess
> >>>>>>>>> ideally CPUFreq driver should also have a 'shutdown'
> >>>>>>>>> callback to teardown DFLL
> >>>>>>>>> on a reboot, but likely that there are other clock-related
> >>>>>>>>> problems as
> >>>>>>>>> well that may break KEXEC and thus it is not very important
> >>>>>>>>> at the
> >>>>>>>>> moment.
> >>>>>>>>>
> >>>>>>>>> [snip]  
> >>>>>>>> During bootup CPUG sources from PLL_X. By PLL_P source above
> >>>>>>>> I meant
> >>>>>>>> PLL_P_OUT4.
> >>>>>>>>
> >>>>>>>> As per clock policies, PLL_X is always used for high freq
> >>>>>>>> like   
> >>>>>>>> >800Mhz  
> >>>>>>>> and for low frequency it will be sourced from PLLP.  
> >>>>>>> Alright, then please don't forget to pre-initialize PLLP_OUT4 
> >>>>>>> rate to a
> >>>>>>> reasonable value using tegra_clk_init_table or
> >>>>>>> assigned-clocks.  
> >>>>>> PLLP_OUT4 rate update is not needed as it is safe to run at
> >>>>>> 408Mhz because it is below fmax @ Vmin  
> >>>>> So even 204MHz CVB entries are having the same voltage as
> >>>>> 408MHz, correct? It's not instantly obvious to me from the DFLL
> >>>>> driver's code where the fmax @ Vmin is defined, I see that
> >>>>> there is the min_millivolts
> >>>>> and frequency entries starting from 204MHZ defined per-table.  
> >>>> Yes at Vmin CPU Fmax is ~800Mhz. So anything below that will
> >>>> work at Vmin voltage and PLLP max is 408Mhz.  
> >>> Thank you for the clarification. It would be good to have that 
> >>> commented
> >>> in the code as well.  
> >> OK, Will add...  
> >
> > Regarding, adding suspend/resume to CPUFreq, CPUFreq suspend
> > happens very early even before disabling non-boot CPUs and also
> > need to export clock driver APIs to CPUFreq.
> >
> > Was thinking of below way of implementing this...
> >
> >
> > Clock DFLL driver Suspend:
> >
> >         - Save CPU clock policy registers, and Perform dfll suspend 
> > which sets in open loop mode
> >
> > CPU Freq driver Suspend: does nothing
> >
> >
> > Clock DFLL driver Resume:
> >
> >         - Re-init DFLL, Set in Open-Loop mode, restore CPU Clock 
> > policy registers which actually sets source to DFLL along with
> > other CPU Policy register restore.
> >
> > CPU Freq driver Resume:
> >
> >         - do clk_prepare_enable which acutally sets DFLL in Closed 
> > loop mode
> >
> >
> > Adding one more note: Switching CPU Clock to PLLP is not needed as
> > CPU CLock can be from dfll in open-loop mode as DFLL is not
> > disabled anywhere throught the suspend/resume path and SC7 entry FW
> > and Warm boot code will switch CPU source to PLLP.  
> 

Since CPU resumes on PLLP, it will be cleaner to suspend it on PLLP as
well. And besides, seems that currently disabling DFLL clock will
disable DFLL completely and then you'd want to re-init the DFLL on
resume any ways. So better to just disable DFLL completely on suspend,
which should happen on clk_disable(dfll).

В Tue, 16 Jul 2019 22:25:25 -0700
Sowjanya Komatineni <skomatineni@nvidia.com> пишет:

> On 7/16/19 9:11 PM, Dmitry Osipenko wrote:
> > В Tue, 16 Jul 2019 19:35:49 -0700
> > Sowjanya Komatineni <skomatineni@nvidia.com> пишет:
> >  
> >> On 7/16/19 7:18 PM, Sowjanya Komatineni wrote:  
> >>> On 7/16/19 3:06 PM, Sowjanya Komatineni wrote:  
> >>>> On 7/16/19 3:00 PM, Dmitry Osipenko wrote:  
> >>>>> 17.07.2019 0:35, Sowjanya Komatineni пишет:  
> >>>>>> On 7/16/19 2:21 PM, Dmitry Osipenko wrote:  
> >>>>>>> 17.07.2019 0:12, Sowjanya Komatineni пишет:  
> >>>>>>>> On 7/16/19 1:47 PM, Dmitry Osipenko wrote:  
> >>>>>>>>> 16.07.2019 22:26, Sowjanya Komatineni пишет:  
> >>>>>>>>>> On 7/16/19 11:43 AM, Dmitry Osipenko wrote:  
> >>>>>>>>>>> 16.07.2019 21:30, Sowjanya Komatineni пишет:  
> >>>>>>>>>>>> On 7/16/19 11:25 AM, Dmitry Osipenko wrote:  
> >>>>>>>>>>>>> 16.07.2019 21:19, Sowjanya Komatineni пишет:  
> >>>>>>>>>>>>>> On 7/16/19 9:50 AM, Sowjanya Komatineni wrote:  
> >>>>>>>>>>>>>>> On 7/16/19 8:00 AM, Dmitry Osipenko wrote:  
> >>>>>>>>>>>>>>>> 16.07.2019 11:06, Peter De Schrijver пишет:  
> >>>>>>>>>>>>>>>>> On Tue, Jul 16, 2019 at 03:24:26PM +0800, Joseph Lo
> >>>>>>>>>>>>>>>>> wrote:  
> >>>>>>>>>>>>>>>>>>> OK, Will add to CPUFreq driver...  
> >>>>>>>>>>>>>>>>>>>> The other thing that also need attention is that
> >>>>>>>>>>>>>>>>>>>> T124 CPUFreq
> >>>>>>>>>>>>>>>>>>>> driver
> >>>>>>>>>>>>>>>>>>>> implicitly relies on DFLL driver to be probed
> >>>>>>>>>>>>>>>>>>>> first, which is
> >>>>>>>>>>>>>>>>>>>> icky.
> >>>>>>>>>>>>>>>>>>>>     
> >>>>>>>>>>>>>>>>>>> Should I add check for successful dfll clk
> >>>>>>>>>>>>>>>>>>> register explicitly in
> >>>>>>>>>>>>>>>>>>> CPUFreq driver probe and defer till dfll clk
> >>>>>>>>>>>>>>>>>>> registers?  
> >>>>>>>>>>>>>>>> Probably you should use the "device links". See
> >>>>>>>>>>>>>>>> [1][2] for the
> >>>>>>>>>>>>>>>> example.
> >>>>>>>>>>>>>>>>
> >>>>>>>>>>>>>>>> [1]
> >>>>>>>>>>>>>>>> https://elixir.bootlin.com/linux/v5.2.1/source/drivers/gpu/drm/tegra/dc.c#L2383
> >>>>>>>>>>>>>>>>
> >>>>>>>>>>>>>>>>
> >>>>>>>>>>>>>>>>
> >>>>>>>>>>>>>>>>
> >>>>>>>>>>>>>>>>
> >>>>>>>>>>>>>>>>
> >>>>>>>>>>>>>>>>
> >>>>>>>>>>>>>>>> [2]
> >>>>>>>>>>>>>>>> https://www.kernel.org/doc/html/latest/driver-api/device_link.html
> >>>>>>>>>>>>>>>>
> >>>>>>>>>>>>>>>>
> >>>>>>>>>>>>>>>>
> >>>>>>>>>>>>>>>> Return EPROBE_DEFER instead of EINVAL if
> >>>>>>>>>>>>>>>> device_link_add() fails.
> >>>>>>>>>>>>>>>> And
> >>>>>>>>>>>>>>>> use of_find_device_by_node() to get the DFLL's
> >>>>>>>>>>>>>>>> device, see [3].
> >>>>>>>>>>>>>>>>
> >>>>>>>>>>>>>>>> [3]
> >>>>>>>>>>>>>>>> https://git.kernel.org/pub/scm/linux/kernel/git/next/linux-next.git/tree/drivers/devfreq/tegra20-devfreq.c#n100
> >>>>>>>>>>>>>>>>
> >>>>>>>>>>>>>>>>
> >>>>>>>>>>>>>>>>
> >>>>>>>>>>>>>>>>
> >>>>>>>>>>>>>>>>
> >>>>>>>>>>>>>>>>     
> >>>>>>>>>>>>>>> Will go thru and add...  
> >>>>>>>>>>>>> Looks like I initially confused this case with getting
> >>>>>>>>>>>>> orphaned clock.
> >>>>>>>>>>>>> I'm now seeing that the DFLL driver registers the clock
> >>>>>>>>>>>>> and then
> >>>>>>>>>>>>> clk_get(dfll) should be returning EPROBE_DEFER until
> >>>>>>>>>>>>> DFLL driver is
> >>>>>>>>>>>>> probed, hence everything should be fine as-is and there
> >>>>>>>>>>>>> is no real
> >>>>>>>>>>>>> need
> >>>>>>>>>>>>> for the 'device link'. Sorry for the confusion!
> >>>>>>>>>>>>>     
> >>>>>>>>>>>>>>>>>> Sorry, I didn't follow the mail thread. Just
> >>>>>>>>>>>>>>>>>> regarding the DFLL
> >>>>>>>>>>>>>>>>>> part.
> >>>>>>>>>>>>>>>>>>
> >>>>>>>>>>>>>>>>>> As you know it, the DFLL clock is one of the CPU
> >>>>>>>>>>>>>>>>>> clock sources and
> >>>>>>>>>>>>>>>>>> integrated with DVFS control logic with the
> >>>>>>>>>>>>>>>>>> regulator. We will not
> >>>>>>>>>>>>>>>>>> switch
> >>>>>>>>>>>>>>>>>> CPU to other clock sources once we switched to
> >>>>>>>>>>>>>>>>>> DFLL. Because the
> >>>>>>>>>>>>>>>>>> CPU has
> >>>>>>>>>>>>>>>>>> been regulated by the DFLL HW with the DVFS table
> >>>>>>>>>>>>>>>>>> (CVB or OPP
> >>>>>>>>>>>>>>>>>> table
> >>>>>>>>>>>>>>>>>> you see
> >>>>>>>>>>>>>>>>>> in the driver.). We shouldn't reparent it to other
> >>>>>>>>>>>>>>>>>> sources with
> >>>>>>>>>>>>>>>>>> unknew
> >>>>>>>>>>>>>>>>>> freq/volt pair. That's not guaranteed to work. We
> >>>>>>>>>>>>>>>>>> allow switching to
> >>>>>>>>>>>>>>>>>> open-loop mode but different sources.  
> >>>>>>>>>>>>>>>> Okay, then the CPUFreq driver will have to enforce
> >>>>>>>>>>>>>>>> DFLL freq to
> >>>>>>>>>>>>>>>> PLLP's
> >>>>>>>>>>>>>>>> rate before switching to PLLP in order to have a
> >>>>>>>>>>>>>>>> proper CPU voltage.  
> >>>>>>>>>>>>>>> PLLP freq is safe to work for any CPU voltage. So no
> >>>>>>>>>>>>>>> need to enforce
> >>>>>>>>>>>>>>> DFLL freq to PLLP rate before changing CCLK_G source
> >>>>>>>>>>>>>>> to PLLP during
> >>>>>>>>>>>>>>> suspend
> >>>>>>>>>>>>>>>     
> >>>>>>>>>>>>>> Sorry, please ignore my above comment. During suspend,
> >>>>>>>>>>>>>> need to change
> >>>>>>>>>>>>>> CCLK_G source to PLLP when dfll is in closed loop mode
> >>>>>>>>>>>>>> first and
> >>>>>>>>>>>>>> then
> >>>>>>>>>>>>>> dfll need to be set to open loop.  
> >>>>>>>>>>>>> Okay.
> >>>>>>>>>>>>>     
> >>>>>>>>>>>>>>>>>> And I don't exactly understand why we need to
> >>>>>>>>>>>>>>>>>> switch to PLLP in
> >>>>>>>>>>>>>>>>>> CPU
> >>>>>>>>>>>>>>>>>> idle
> >>>>>>>>>>>>>>>>>> driver. Just keep it on CL-DVFS mode all the time.
> >>>>>>>>>>>>>>>>>>
> >>>>>>>>>>>>>>>>>> In SC7 entry, the dfll suspend function moves it
> >>>>>>>>>>>>>>>>>> the open-loop
> >>>>>>>>>>>>>>>>>> mode. That's
> >>>>>>>>>>>>>>>>>> all. The sc7-entryfirmware will handle the rest of
> >>>>>>>>>>>>>>>>>> the sequence to
> >>>>>>>>>>>>>>>>>> turn off
> >>>>>>>>>>>>>>>>>> the CPU power.
> >>>>>>>>>>>>>>>>>>
> >>>>>>>>>>>>>>>>>> In SC7 resume, the warmboot code will handle the
> >>>>>>>>>>>>>>>>>> sequence to
> >>>>>>>>>>>>>>>>>> turn on
> >>>>>>>>>>>>>>>>>> regulator and power up the CPU cluster. And leave
> >>>>>>>>>>>>>>>>>> it on PLL_P.
> >>>>>>>>>>>>>>>>>> After
> >>>>>>>>>>>>>>>>>> resuming to the kernel, we re-init DFLL, restore
> >>>>>>>>>>>>>>>>>> the CPU clock
> >>>>>>>>>>>>>>>>>> policy (CPU
> >>>>>>>>>>>>>>>>>> runs on DFLL open-loop mode) and then moving to
> >>>>>>>>>>>>>>>>>> close-loop mode.  
> >>>>>>>>>>>>>>>> The DFLL is re-inited after switching CCLK to DFLL
> >>>>>>>>>>>>>>>> parent during of
> >>>>>>>>>>>>>>>> the
> >>>>>>>>>>>>>>>> early clocks-state restoring by CaR driver. Hence
> >>>>>>>>>>>>>>>> instead of having
> >>>>>>>>>>>>>>>> odd
> >>>>>>>>>>>>>>>> hacks in the CaR driver, it is much nicer to have a
> >>>>>>>>>>>>>>>> proper suspend-resume sequencing of the device
> >>>>>>>>>>>>>>>> drivers. In this case
> >>>>>>>>>>>>>>>> CPUFreq
> >>>>>>>>>>>>>>>> driver is the driver that enables DFLL and switches
> >>>>>>>>>>>>>>>> CPU to that
> >>>>>>>>>>>>>>>> clock
> >>>>>>>>>>>>>>>> source, which means that this driver is also should
> >>>>>>>>>>>>>>>> be responsible for
> >>>>>>>>>>>>>>>> management of the DFLL's state during of
> >>>>>>>>>>>>>>>> suspend/resume process. If
> >>>>>>>>>>>>>>>> CPUFreq driver disables DFLL during suspend and
> >>>>>>>>>>>>>>>> re-enables it
> >>>>>>>>>>>>>>>> during
> >>>>>>>>>>>>>>>> resume, then looks like the CaR driver hacks around
> >>>>>>>>>>>>>>>> DFLL are not
> >>>>>>>>>>>>>>>> needed.
> >>>>>>>>>>>>>>>>     
> >>>>>>>>>>>>>>>>>> The DFLL part looks good to me. BTW, change the
> >>>>>>>>>>>>>>>>>> patch subject to
> >>>>>>>>>>>>>>>>>> "Add
> >>>>>>>>>>>>>>>>>> suspend-resume support" seems more appropriate to
> >>>>>>>>>>>>>>>>>> me. 
> >>>>>>>>>>>>>>>>> To clarify this, the sequences for DFLL use are as
> >>>>>>>>>>>>>>>>> follows (assuming
> >>>>>>>>>>>>>>>>> all
> >>>>>>>>>>>>>>>>> required DFLL hw configuration has been done)
> >>>>>>>>>>>>>>>>>
> >>>>>>>>>>>>>>>>> Switch to DFLL:
> >>>>>>>>>>>>>>>>> 0) Save current parent and frequency
> >>>>>>>>>>>>>>>>> 1) Program DFLL to open loop mode
> >>>>>>>>>>>>>>>>> 2) Enable DFLL
> >>>>>>>>>>>>>>>>> 3) Change cclk_g parent to DFLL
> >>>>>>>>>>>>>>>>> For OVR regulator:
> >>>>>>>>>>>>>>>>> 4) Change PWM output pin from tristate to output
> >>>>>>>>>>>>>>>>> 5) Enable DFLL PWM output
> >>>>>>>>>>>>>>>>> For I2C regulator:
> >>>>>>>>>>>>>>>>> 4) Enable DFLL I2C output
> >>>>>>>>>>>>>>>>> 6) Program DFLL to closed loop mode
> >>>>>>>>>>>>>>>>>
> >>>>>>>>>>>>>>>>> Switch away from DFLL:
> >>>>>>>>>>>>>>>>> 0) Change cclk_g parent to PLLP so the CPU frequency
> >>>>>>>>>>>>>>>>> is ok for
> >>>>>>>>>>>>>>>>> any
> >>>>>>>>>>>>>>>>> vdd_cpu voltage
> >>>>>>>>>>>>>>>>> 1) Program DFLL to open loop mode
> >>>>>>>>>>>>>>>>>     
> >>>>>>>>>>>>>> I see during switch away from DFLL (suspend), cclk_g
> >>>>>>>>>>>>>> parent is not
> >>>>>>>>>>>>>> changed to PLLP before changing dfll to open loop mode.
> >>>>>>>>>>>>>>
> >>>>>>>>>>>>>> Will add this ...  
> >>>>>>>>>>>>> The CPUFreq driver switches parent to PLLP during the
> >>>>>>>>>>>>> probe, similar
> >>>>>>>>>>>>> should be done on suspend.
> >>>>>>>>>>>>>
> >>>>>>>>>>>>> I'm also wondering if it's always safe to switch to PLLP
> >>>>>>>>>>>>> in the probe.
> >>>>>>>>>>>>> If CPU is running on a lower freq than PLLP, then some
> >>>>>>>>>>>>> other more
> >>>>>>>>>>>>> appropriate intermediate parent should be selected.
> >>>>>>>>>>>>>     
> >>>>>>>>>>>> CPU parents are PLL_X, PLL_P, and dfll. PLL_X always runs
> >>>>>>>>>>>> at higher
> >>>>>>>>>>>> rate
> >>>>>>>>>>>> so switching to PLL_P during CPUFreq probe prior to dfll
> >>>>>>>>>>>> clock enable
> >>>>>>>>>>>> should be safe.  
> >>>>>>>>>>> AFAIK, PLLX could run at ~200MHz. There is also a divided
> >>>>>>>>>>> output of
> >>>>>>>>>>> PLLP
> >>>>>>>>>>> which CCLKG supports, the PLLP_OUT4.
> >>>>>>>>>>>
> >>>>>>>>>>> Probably, realistically, CPU is always running off a fast
> >>>>>>>>>>> PLLX during
> >>>>>>>>>>> boot, but I'm wondering what may happen on KEXEC. I guess
> >>>>>>>>>>> ideally CPUFreq driver should also have a 'shutdown'
> >>>>>>>>>>> callback to teardown DFLL
> >>>>>>>>>>> on a reboot, but likely that there are other clock-related
> >>>>>>>>>>> problems as
> >>>>>>>>>>> well that may break KEXEC and thus it is not very
> >>>>>>>>>>> important at the
> >>>>>>>>>>> moment.
> >>>>>>>>>>>
> >>>>>>>>>>> [snip]  
> >>>>>>>>>> During bootup CPUG sources from PLL_X. By PLL_P source
> >>>>>>>>>> above I meant
> >>>>>>>>>> PLL_P_OUT4.
> >>>>>>>>>>
> >>>>>>>>>> As per clock policies, PLL_X is always used for high freq
> >>>>>>>>>> like  
> >>>>>>>>>>> 800Mhz  
> >>>>>>>>>> and for low frequency it will be sourced from PLLP.  
> >>>>>>>>> Alright, then please don't forget to pre-initialize
> >>>>>>>>> PLLP_OUT4 rate to a
> >>>>>>>>> reasonable value using tegra_clk_init_table or
> >>>>>>>>> assigned-clocks.  
> >>>>>>>> PLLP_OUT4 rate update is not needed as it is safe to run at
> >>>>>>>> 408Mhz because it is below fmax @ Vmin  
> >>>>>>> So even 204MHz CVB entries are having the same voltage as
> >>>>>>> 408MHz, correct? It's not instantly obvious to me from the
> >>>>>>> DFLL driver's code where the fmax @ Vmin is defined, I see
> >>>>>>> that there is the min_millivolts
> >>>>>>> and frequency entries starting from 204MHZ defined
> >>>>>>> per-table.  
> >>>>>> Yes at Vmin CPU Fmax is ~800Mhz. So anything below that will
> >>>>>> work at Vmin voltage and PLLP max is 408Mhz.  
> >>>>> Thank you for the clarification. It would be good to have that
> >>>>> commented
> >>>>> in the code as well.  
> >>>> OK, Will add...  
> >>> Regarding, adding suspend/resume to CPUFreq, CPUFreq suspend
> >>> happens very early even before disabling non-boot CPUs and also
> >>> need to export clock driver APIs to CPUFreq.
> >>>
> >>> Was thinking of below way of implementing this...
> >>>
> >>>
> >>> Clock DFLL driver Suspend:
> >>>
> >>>          - Save CPU clock policy registers, and Perform dfll
> >>> suspend which sets in open loop mode
> >>>
> >>> CPU Freq driver Suspend: does nothing
> >>>
> >>>
> >>> Clock DFLL driver Resume:
> >>>
> >>>          - Re-init DFLL, Set in Open-Loop mode, restore CPU Clock
> >>> policy registers which actually sets source to DFLL along with
> >>> other CPU Policy register restore.
> >>>
> >>> CPU Freq driver Resume:
> >>>
> >>>          - do clk_prepare_enable which acutally sets DFLL in
> >>> Closed loop mode
> >>>
> >>>
> >>> Adding one more note: Switching CPU Clock to PLLP is not needed as
> >>> CPU CLock can be from dfll in open-loop mode as DFLL is not
> >>> disabled anywhere throught the suspend/resume path and SC7 entry
> >>> FW and Warm boot code will switch CPU source to PLLP.  
> > Since CPU resumes on PLLP, it will be cleaner to suspend it on PLLP
> > as well. And besides, seems that currently disabling DFLL clock will
> > disable DFLL completely and then you'd want to re-init the DFLL on
> > resume any ways. So better to just disable DFLL completely on
> > suspend, which should happen on clk_disable(dfll).  
> 
> Will switch to PLLP during CPUFreq suspend. With decision of using 
> clk_disable during suspend, its mandatory to switch to PLLP as DFLL
> is completely disabled.
> 
> My earlier concern was on restoring CPU policy as we can't do that
> from CPUFreq driver and need export from clock driver.
> 
> Clear now and will do CPU clock policy restore in after dfll re-init.

Why the policy can't be saved/restored by the CaR driver as a context of
any other clock?

> Also I don't see Tegra124 CPU Freq driver using flag 
> CPUFREQ_NEED_INITIAL_FREQ_CHECK.
> 
> Tegra124 CPUFreq driver is not using cpufreq_driver
> 
> 
> <https://elixir.bootlin.com/linux/v5.2.1/ident/cpufreq_driver>
> 

T124 driver is a wrapper around cpufreq-dt driver.

On 7/16/19 10:42 PM, Dmitry Osipenko wrote:
> В Tue, 16 Jul 2019 22:25:25 -0700
> Sowjanya Komatineni <skomatineni@nvidia.com> пишет:
>
>> On 7/16/19 9:11 PM, Dmitry Osipenko wrote:
>>> В Tue, 16 Jul 2019 19:35:49 -0700
>>> Sowjanya Komatineni <skomatineni@nvidia.com> пишет:
>>>   
>>>> On 7/16/19 7:18 PM, Sowjanya Komatineni wrote:
>>>>> On 7/16/19 3:06 PM, Sowjanya Komatineni wrote:
>>>>>> On 7/16/19 3:00 PM, Dmitry Osipenko wrote:
>>>>>>> 17.07.2019 0:35, Sowjanya Komatineni пишет:
>>>>>>>> On 7/16/19 2:21 PM, Dmitry Osipenko wrote:
>>>>>>>>> 17.07.2019 0:12, Sowjanya Komatineni пишет:
>>>>>>>>>> On 7/16/19 1:47 PM, Dmitry Osipenko wrote:
>>>>>>>>>>> 16.07.2019 22:26, Sowjanya Komatineni пишет:
>>>>>>>>>>>> On 7/16/19 11:43 AM, Dmitry Osipenko wrote:
>>>>>>>>>>>>> 16.07.2019 21:30, Sowjanya Komatineni пишет:
>>>>>>>>>>>>>> On 7/16/19 11:25 AM, Dmitry Osipenko wrote:
>>>>>>>>>>>>>>> 16.07.2019 21:19, Sowjanya Komatineni пишет:
>>>>>>>>>>>>>>>> On 7/16/19 9:50 AM, Sowjanya Komatineni wrote:
>>>>>>>>>>>>>>>>> On 7/16/19 8:00 AM, Dmitry Osipenko wrote:
>>>>>>>>>>>>>>>>>> 16.07.2019 11:06, Peter De Schrijver пишет:
>>>>>>>>>>>>>>>>>>> On Tue, Jul 16, 2019 at 03:24:26PM +0800, Joseph Lo
>>>>>>>>>>>>>>>>>>> wrote:
>>>>>>>>>>>>>>>>>>>>> OK, Will add to CPUFreq driver...
>>>>>>>>>>>>>>>>>>>>>> The other thing that also need attention is that
>>>>>>>>>>>>>>>>>>>>>> T124 CPUFreq
>>>>>>>>>>>>>>>>>>>>>> driver
>>>>>>>>>>>>>>>>>>>>>> implicitly relies on DFLL driver to be probed
>>>>>>>>>>>>>>>>>>>>>> first, which is
>>>>>>>>>>>>>>>>>>>>>> icky.
>>>>>>>>>>>>>>>>>>>>>>      
>>>>>>>>>>>>>>>>>>>>> Should I add check for successful dfll clk
>>>>>>>>>>>>>>>>>>>>> register explicitly in
>>>>>>>>>>>>>>>>>>>>> CPUFreq driver probe and defer till dfll clk
>>>>>>>>>>>>>>>>>>>>> registers?
>>>>>>>>>>>>>>>>>> Probably you should use the "device links". See
>>>>>>>>>>>>>>>>>> [1][2] for the
>>>>>>>>>>>>>>>>>> example.
>>>>>>>>>>>>>>>>>>
>>>>>>>>>>>>>>>>>> [1]
>>>>>>>>>>>>>>>>>> https://elixir.bootlin.com/linux/v5.2.1/source/drivers/gpu/drm/tegra/dc.c#L2383
>>>>>>>>>>>>>>>>>>
>>>>>>>>>>>>>>>>>>
>>>>>>>>>>>>>>>>>>
>>>>>>>>>>>>>>>>>>
>>>>>>>>>>>>>>>>>>
>>>>>>>>>>>>>>>>>>
>>>>>>>>>>>>>>>>>>
>>>>>>>>>>>>>>>>>> [2]
>>>>>>>>>>>>>>>>>> https://www.kernel.org/doc/html/latest/driver-api/device_link.html
>>>>>>>>>>>>>>>>>>
>>>>>>>>>>>>>>>>>>
>>>>>>>>>>>>>>>>>>
>>>>>>>>>>>>>>>>>> Return EPROBE_DEFER instead of EINVAL if
>>>>>>>>>>>>>>>>>> device_link_add() fails.
>>>>>>>>>>>>>>>>>> And
>>>>>>>>>>>>>>>>>> use of_find_device_by_node() to get the DFLL's
>>>>>>>>>>>>>>>>>> device, see [3].
>>>>>>>>>>>>>>>>>>
>>>>>>>>>>>>>>>>>> [3]
>>>>>>>>>>>>>>>>>> https://git.kernel.org/pub/scm/linux/kernel/git/next/linux-next.git/tree/drivers/devfreq/tegra20-devfreq.c#n100
>>>>>>>>>>>>>>>>>>
>>>>>>>>>>>>>>>>>>
>>>>>>>>>>>>>>>>>>
>>>>>>>>>>>>>>>>>>
>>>>>>>>>>>>>>>>>>
>>>>>>>>>>>>>>>>>>      
>>>>>>>>>>>>>>>>> Will go thru and add...
>>>>>>>>>>>>>>> Looks like I initially confused this case with getting
>>>>>>>>>>>>>>> orphaned clock.
>>>>>>>>>>>>>>> I'm now seeing that the DFLL driver registers the clock
>>>>>>>>>>>>>>> and then
>>>>>>>>>>>>>>> clk_get(dfll) should be returning EPROBE_DEFER until
>>>>>>>>>>>>>>> DFLL driver is
>>>>>>>>>>>>>>> probed, hence everything should be fine as-is and there
>>>>>>>>>>>>>>> is no real
>>>>>>>>>>>>>>> need
>>>>>>>>>>>>>>> for the 'device link'. Sorry for the confusion!
>>>>>>>>>>>>>>>      
>>>>>>>>>>>>>>>>>>>> Sorry, I didn't follow the mail thread. Just
>>>>>>>>>>>>>>>>>>>> regarding the DFLL
>>>>>>>>>>>>>>>>>>>> part.
>>>>>>>>>>>>>>>>>>>>
>>>>>>>>>>>>>>>>>>>> As you know it, the DFLL clock is one of the CPU
>>>>>>>>>>>>>>>>>>>> clock sources and
>>>>>>>>>>>>>>>>>>>> integrated with DVFS control logic with the
>>>>>>>>>>>>>>>>>>>> regulator. We will not
>>>>>>>>>>>>>>>>>>>> switch
>>>>>>>>>>>>>>>>>>>> CPU to other clock sources once we switched to
>>>>>>>>>>>>>>>>>>>> DFLL. Because the
>>>>>>>>>>>>>>>>>>>> CPU has
>>>>>>>>>>>>>>>>>>>> been regulated by the DFLL HW with the DVFS table
>>>>>>>>>>>>>>>>>>>> (CVB or OPP
>>>>>>>>>>>>>>>>>>>> table
>>>>>>>>>>>>>>>>>>>> you see
>>>>>>>>>>>>>>>>>>>> in the driver.). We shouldn't reparent it to other
>>>>>>>>>>>>>>>>>>>> sources with
>>>>>>>>>>>>>>>>>>>> unknew
>>>>>>>>>>>>>>>>>>>> freq/volt pair. That's not guaranteed to work. We
>>>>>>>>>>>>>>>>>>>> allow switching to
>>>>>>>>>>>>>>>>>>>> open-loop mode but different sources.
>>>>>>>>>>>>>>>>>> Okay, then the CPUFreq driver will have to enforce
>>>>>>>>>>>>>>>>>> DFLL freq to
>>>>>>>>>>>>>>>>>> PLLP's
>>>>>>>>>>>>>>>>>> rate before switching to PLLP in order to have a
>>>>>>>>>>>>>>>>>> proper CPU voltage.
>>>>>>>>>>>>>>>>> PLLP freq is safe to work for any CPU voltage. So no
>>>>>>>>>>>>>>>>> need to enforce
>>>>>>>>>>>>>>>>> DFLL freq to PLLP rate before changing CCLK_G source
>>>>>>>>>>>>>>>>> to PLLP during
>>>>>>>>>>>>>>>>> suspend
>>>>>>>>>>>>>>>>>      
>>>>>>>>>>>>>>>> Sorry, please ignore my above comment. During suspend,
>>>>>>>>>>>>>>>> need to change
>>>>>>>>>>>>>>>> CCLK_G source to PLLP when dfll is in closed loop mode
>>>>>>>>>>>>>>>> first and
>>>>>>>>>>>>>>>> then
>>>>>>>>>>>>>>>> dfll need to be set to open loop.
>>>>>>>>>>>>>>> Okay.
>>>>>>>>>>>>>>>      
>>>>>>>>>>>>>>>>>>>> And I don't exactly understand why we need to
>>>>>>>>>>>>>>>>>>>> switch to PLLP in
>>>>>>>>>>>>>>>>>>>> CPU
>>>>>>>>>>>>>>>>>>>> idle
>>>>>>>>>>>>>>>>>>>> driver. Just keep it on CL-DVFS mode all the time.
>>>>>>>>>>>>>>>>>>>>
>>>>>>>>>>>>>>>>>>>> In SC7 entry, the dfll suspend function moves it
>>>>>>>>>>>>>>>>>>>> the open-loop
>>>>>>>>>>>>>>>>>>>> mode. That's
>>>>>>>>>>>>>>>>>>>> all. The sc7-entryfirmware will handle the rest of
>>>>>>>>>>>>>>>>>>>> the sequence to
>>>>>>>>>>>>>>>>>>>> turn off
>>>>>>>>>>>>>>>>>>>> the CPU power.
>>>>>>>>>>>>>>>>>>>>
>>>>>>>>>>>>>>>>>>>> In SC7 resume, the warmboot code will handle the
>>>>>>>>>>>>>>>>>>>> sequence to
>>>>>>>>>>>>>>>>>>>> turn on
>>>>>>>>>>>>>>>>>>>> regulator and power up the CPU cluster. And leave
>>>>>>>>>>>>>>>>>>>> it on PLL_P.
>>>>>>>>>>>>>>>>>>>> After
>>>>>>>>>>>>>>>>>>>> resuming to the kernel, we re-init DFLL, restore
>>>>>>>>>>>>>>>>>>>> the CPU clock
>>>>>>>>>>>>>>>>>>>> policy (CPU
>>>>>>>>>>>>>>>>>>>> runs on DFLL open-loop mode) and then moving to
>>>>>>>>>>>>>>>>>>>> close-loop mode.
>>>>>>>>>>>>>>>>>> The DFLL is re-inited after switching CCLK to DFLL
>>>>>>>>>>>>>>>>>> parent during of
>>>>>>>>>>>>>>>>>> the
>>>>>>>>>>>>>>>>>> early clocks-state restoring by CaR driver. Hence
>>>>>>>>>>>>>>>>>> instead of having
>>>>>>>>>>>>>>>>>> odd
>>>>>>>>>>>>>>>>>> hacks in the CaR driver, it is much nicer to have a
>>>>>>>>>>>>>>>>>> proper suspend-resume sequencing of the device
>>>>>>>>>>>>>>>>>> drivers. In this case
>>>>>>>>>>>>>>>>>> CPUFreq
>>>>>>>>>>>>>>>>>> driver is the driver that enables DFLL and switches
>>>>>>>>>>>>>>>>>> CPU to that
>>>>>>>>>>>>>>>>>> clock
>>>>>>>>>>>>>>>>>> source, which means that this driver is also should
>>>>>>>>>>>>>>>>>> be responsible for
>>>>>>>>>>>>>>>>>> management of the DFLL's state during of
>>>>>>>>>>>>>>>>>> suspend/resume process. If
>>>>>>>>>>>>>>>>>> CPUFreq driver disables DFLL during suspend and
>>>>>>>>>>>>>>>>>> re-enables it
>>>>>>>>>>>>>>>>>> during
>>>>>>>>>>>>>>>>>> resume, then looks like the CaR driver hacks around
>>>>>>>>>>>>>>>>>> DFLL are not
>>>>>>>>>>>>>>>>>> needed.
>>>>>>>>>>>>>>>>>>      
>>>>>>>>>>>>>>>>>>>> The DFLL part looks good to me. BTW, change the
>>>>>>>>>>>>>>>>>>>> patch subject to
>>>>>>>>>>>>>>>>>>>> "Add
>>>>>>>>>>>>>>>>>>>> suspend-resume support" seems more appropriate to
>>>>>>>>>>>>>>>>>>>> me.
>>>>>>>>>>>>>>>>>>> To clarify this, the sequences for DFLL use are as
>>>>>>>>>>>>>>>>>>> follows (assuming
>>>>>>>>>>>>>>>>>>> all
>>>>>>>>>>>>>>>>>>> required DFLL hw configuration has been done)
>>>>>>>>>>>>>>>>>>>
>>>>>>>>>>>>>>>>>>> Switch to DFLL:
>>>>>>>>>>>>>>>>>>> 0) Save current parent and frequency
>>>>>>>>>>>>>>>>>>> 1) Program DFLL to open loop mode
>>>>>>>>>>>>>>>>>>> 2) Enable DFLL
>>>>>>>>>>>>>>>>>>> 3) Change cclk_g parent to DFLL
>>>>>>>>>>>>>>>>>>> For OVR regulator:
>>>>>>>>>>>>>>>>>>> 4) Change PWM output pin from tristate to output
>>>>>>>>>>>>>>>>>>> 5) Enable DFLL PWM output
>>>>>>>>>>>>>>>>>>> For I2C regulator:
>>>>>>>>>>>>>>>>>>> 4) Enable DFLL I2C output
>>>>>>>>>>>>>>>>>>> 6) Program DFLL to closed loop mode
>>>>>>>>>>>>>>>>>>>
>>>>>>>>>>>>>>>>>>> Switch away from DFLL:
>>>>>>>>>>>>>>>>>>> 0) Change cclk_g parent to PLLP so the CPU frequency
>>>>>>>>>>>>>>>>>>> is ok for
>>>>>>>>>>>>>>>>>>> any
>>>>>>>>>>>>>>>>>>> vdd_cpu voltage
>>>>>>>>>>>>>>>>>>> 1) Program DFLL to open loop mode
>>>>>>>>>>>>>>>>>>>      
>>>>>>>>>>>>>>>> I see during switch away from DFLL (suspend), cclk_g
>>>>>>>>>>>>>>>> parent is not
>>>>>>>>>>>>>>>> changed to PLLP before changing dfll to open loop mode.
>>>>>>>>>>>>>>>>
>>>>>>>>>>>>>>>> Will add this ...
>>>>>>>>>>>>>>> The CPUFreq driver switches parent to PLLP during the
>>>>>>>>>>>>>>> probe, similar
>>>>>>>>>>>>>>> should be done on suspend.
>>>>>>>>>>>>>>>
>>>>>>>>>>>>>>> I'm also wondering if it's always safe to switch to PLLP
>>>>>>>>>>>>>>> in the probe.
>>>>>>>>>>>>>>> If CPU is running on a lower freq than PLLP, then some
>>>>>>>>>>>>>>> other more
>>>>>>>>>>>>>>> appropriate intermediate parent should be selected.
>>>>>>>>>>>>>>>      
>>>>>>>>>>>>>> CPU parents are PLL_X, PLL_P, and dfll. PLL_X always runs
>>>>>>>>>>>>>> at higher
>>>>>>>>>>>>>> rate
>>>>>>>>>>>>>> so switching to PLL_P during CPUFreq probe prior to dfll
>>>>>>>>>>>>>> clock enable
>>>>>>>>>>>>>> should be safe.
>>>>>>>>>>>>> AFAIK, PLLX could run at ~200MHz. There is also a divided
>>>>>>>>>>>>> output of
>>>>>>>>>>>>> PLLP
>>>>>>>>>>>>> which CCLKG supports, the PLLP_OUT4.
>>>>>>>>>>>>>
>>>>>>>>>>>>> Probably, realistically, CPU is always running off a fast
>>>>>>>>>>>>> PLLX during
>>>>>>>>>>>>> boot, but I'm wondering what may happen on KEXEC. I guess
>>>>>>>>>>>>> ideally CPUFreq driver should also have a 'shutdown'
>>>>>>>>>>>>> callback to teardown DFLL
>>>>>>>>>>>>> on a reboot, but likely that there are other clock-related
>>>>>>>>>>>>> problems as
>>>>>>>>>>>>> well that may break KEXEC and thus it is not very
>>>>>>>>>>>>> important at the
>>>>>>>>>>>>> moment.
>>>>>>>>>>>>>
>>>>>>>>>>>>> [snip]
>>>>>>>>>>>> During bootup CPUG sources from PLL_X. By PLL_P source
>>>>>>>>>>>> above I meant
>>>>>>>>>>>> PLL_P_OUT4.
>>>>>>>>>>>>
>>>>>>>>>>>> As per clock policies, PLL_X is always used for high freq
>>>>>>>>>>>> like
>>>>>>>>>>>>> 800Mhz
>>>>>>>>>>>> and for low frequency it will be sourced from PLLP.
>>>>>>>>>>> Alright, then please don't forget to pre-initialize
>>>>>>>>>>> PLLP_OUT4 rate to a
>>>>>>>>>>> reasonable value using tegra_clk_init_table or
>>>>>>>>>>> assigned-clocks.
>>>>>>>>>> PLLP_OUT4 rate update is not needed as it is safe to run at
>>>>>>>>>> 408Mhz because it is below fmax @ Vmin
>>>>>>>>> So even 204MHz CVB entries are having the same voltage as
>>>>>>>>> 408MHz, correct? It's not instantly obvious to me from the
>>>>>>>>> DFLL driver's code where the fmax @ Vmin is defined, I see
>>>>>>>>> that there is the min_millivolts
>>>>>>>>> and frequency entries starting from 204MHZ defined
>>>>>>>>> per-table.
>>>>>>>> Yes at Vmin CPU Fmax is ~800Mhz. So anything below that will
>>>>>>>> work at Vmin voltage and PLLP max is 408Mhz.
>>>>>>> Thank you for the clarification. It would be good to have that
>>>>>>> commented
>>>>>>> in the code as well.
>>>>>> OK, Will add...
>>>>> Regarding, adding suspend/resume to CPUFreq, CPUFreq suspend
>>>>> happens very early even before disabling non-boot CPUs and also
>>>>> need to export clock driver APIs to CPUFreq.
>>>>>
>>>>> Was thinking of below way of implementing this...
>>>>>
>>>>>
>>>>> Clock DFLL driver Suspend:
>>>>>
>>>>>           - Save CPU clock policy registers, and Perform dfll
>>>>> suspend which sets in open loop mode
>>>>>
>>>>> CPU Freq driver Suspend: does nothing
>>>>>
>>>>>
>>>>> Clock DFLL driver Resume:
>>>>>
>>>>>           - Re-init DFLL, Set in Open-Loop mode, restore CPU Clock
>>>>> policy registers which actually sets source to DFLL along with
>>>>> other CPU Policy register restore.
>>>>>
>>>>> CPU Freq driver Resume:
>>>>>
>>>>>           - do clk_prepare_enable which acutally sets DFLL in
>>>>> Closed loop mode
>>>>>
>>>>>
>>>>> Adding one more note: Switching CPU Clock to PLLP is not needed as
>>>>> CPU CLock can be from dfll in open-loop mode as DFLL is not
>>>>> disabled anywhere throught the suspend/resume path and SC7 entry
>>>>> FW and Warm boot code will switch CPU source to PLLP.
>>> Since CPU resumes on PLLP, it will be cleaner to suspend it on PLLP
>>> as well. And besides, seems that currently disabling DFLL clock will
>>> disable DFLL completely and then you'd want to re-init the DFLL on
>>> resume any ways. So better to just disable DFLL completely on
>>> suspend, which should happen on clk_disable(dfll).
>> Will switch to PLLP during CPUFreq suspend. With decision of using
>> clk_disable during suspend, its mandatory to switch to PLLP as DFLL
>> is completely disabled.
>>
>> My earlier concern was on restoring CPU policy as we can't do that
>> from CPUFreq driver and need export from clock driver.
>>
>> Clear now and will do CPU clock policy restore in after dfll re-init.
> Why the policy can't be saved/restored by the CaR driver as a context of
> any other clock?

restoring cpu clock policy involves programming source and 
super_cclkg_divider.

cclk_g is registered as clk_super_mux and it doesn't use frac_div ops to 
do save/restore its divider.

Also, during clock context we cant restore cclk_g as cclk_g source will 
be dfll and dfll will not be resumed/re-initialized by the time 
clk_super_mux save/restore happens.

we can't use save/restore context for dfll clk_ops because dfllCPU_out 
parent to CCLK_G is first in the clock tree and dfll_ref and dfll_soc 
peripheral clocks are not restored by the time dfll restore happens. 
Also dfll peripheral clock enables need to be restored before dfll 
restore happens which involves programming dfll controller for 
re-initialization.

So dfll resume/re-init is done in clk-tegra210 at end of all clocks 
restore in V5 series but instead of in clk-tegra210 driver I moved now 
to dfll-fcpu driver pm_ops as all dfll dependencies will be restored 
thru clk_restore_context by then. This will be in V6.

>> Also I don't see Tegra124 CPU Freq driver using flag
>> CPUFREQ_NEED_INITIAL_FREQ_CHECK.
>>
>> Tegra124 CPUFreq driver is not using cpufreq_driver
>>
>>
>> <https://elixir.bootlin.com/linux/v5.2.1/ident/cpufreq_driver>
>>
> T124 driver is a wrapper around cpufreq-dt driver.

В Tue, 16 Jul 2019 22:55:52 -0700
Sowjanya Komatineni <skomatineni@nvidia.com> пишет:

> On 7/16/19 10:42 PM, Dmitry Osipenko wrote:
> > В Tue, 16 Jul 2019 22:25:25 -0700
> > Sowjanya Komatineni <skomatineni@nvidia.com> пишет:
> >  
> >> On 7/16/19 9:11 PM, Dmitry Osipenko wrote:  
> >>> В Tue, 16 Jul 2019 19:35:49 -0700
> >>> Sowjanya Komatineni <skomatineni@nvidia.com> пишет:
> >>>     
> >>>> On 7/16/19 7:18 PM, Sowjanya Komatineni wrote:  
> >>>>> On 7/16/19 3:06 PM, Sowjanya Komatineni wrote:  
> >>>>>> On 7/16/19 3:00 PM, Dmitry Osipenko wrote:  
> >>>>>>> 17.07.2019 0:35, Sowjanya Komatineni пишет:  
> >>>>>>>> On 7/16/19 2:21 PM, Dmitry Osipenko wrote:  
> >>>>>>>>> 17.07.2019 0:12, Sowjanya Komatineni пишет:  
> >>>>>>>>>> On 7/16/19 1:47 PM, Dmitry Osipenko wrote:  
> >>>>>>>>>>> 16.07.2019 22:26, Sowjanya Komatineni пишет:  
> >>>>>>>>>>>> On 7/16/19 11:43 AM, Dmitry Osipenko wrote:  
> >>>>>>>>>>>>> 16.07.2019 21:30, Sowjanya Komatineni пишет:  
> >>>>>>>>>>>>>> On 7/16/19 11:25 AM, Dmitry Osipenko wrote:  
> >>>>>>>>>>>>>>> 16.07.2019 21:19, Sowjanya Komatineni пишет:  
> >>>>>>>>>>>>>>>> On 7/16/19 9:50 AM, Sowjanya Komatineni wrote:  
> >>>>>>>>>>>>>>>>> On 7/16/19 8:00 AM, Dmitry Osipenko wrote:  
> >>>>>>>>>>>>>>>>>> 16.07.2019 11:06, Peter De Schrijver пишет:  
> >>>>>>>>>>>>>>>>>>> On Tue, Jul 16, 2019 at 03:24:26PM +0800, Joseph
> >>>>>>>>>>>>>>>>>>> Lo wrote:  
> >>>>>>>>>>>>>>>>>>>>> OK, Will add to CPUFreq driver...  
> >>>>>>>>>>>>>>>>>>>>>> The other thing that also need attention is
> >>>>>>>>>>>>>>>>>>>>>> that T124 CPUFreq
> >>>>>>>>>>>>>>>>>>>>>> driver
> >>>>>>>>>>>>>>>>>>>>>> implicitly relies on DFLL driver to be probed
> >>>>>>>>>>>>>>>>>>>>>> first, which is
> >>>>>>>>>>>>>>>>>>>>>> icky.
> >>>>>>>>>>>>>>>>>>>>>>        
> >>>>>>>>>>>>>>>>>>>>> Should I add check for successful dfll clk
> >>>>>>>>>>>>>>>>>>>>> register explicitly in
> >>>>>>>>>>>>>>>>>>>>> CPUFreq driver probe and defer till dfll clk
> >>>>>>>>>>>>>>>>>>>>> registers?  
> >>>>>>>>>>>>>>>>>> Probably you should use the "device links". See
> >>>>>>>>>>>>>>>>>> [1][2] for the
> >>>>>>>>>>>>>>>>>> example.
> >>>>>>>>>>>>>>>>>>
> >>>>>>>>>>>>>>>>>> [1]
> >>>>>>>>>>>>>>>>>> https://elixir.bootlin.com/linux/v5.2.1/source/drivers/gpu/drm/tegra/dc.c#L2383
> >>>>>>>>>>>>>>>>>>
> >>>>>>>>>>>>>>>>>>
> >>>>>>>>>>>>>>>>>>
> >>>>>>>>>>>>>>>>>>
> >>>>>>>>>>>>>>>>>>
> >>>>>>>>>>>>>>>>>>
> >>>>>>>>>>>>>>>>>>
> >>>>>>>>>>>>>>>>>> [2]
> >>>>>>>>>>>>>>>>>> https://www.kernel.org/doc/html/latest/driver-api/device_link.html
> >>>>>>>>>>>>>>>>>>
> >>>>>>>>>>>>>>>>>>
> >>>>>>>>>>>>>>>>>>
> >>>>>>>>>>>>>>>>>> Return EPROBE_DEFER instead of EINVAL if
> >>>>>>>>>>>>>>>>>> device_link_add() fails.
> >>>>>>>>>>>>>>>>>> And
> >>>>>>>>>>>>>>>>>> use of_find_device_by_node() to get the DFLL's
> >>>>>>>>>>>>>>>>>> device, see [3].
> >>>>>>>>>>>>>>>>>>
> >>>>>>>>>>>>>>>>>> [3]
> >>>>>>>>>>>>>>>>>> https://git.kernel.org/pub/scm/linux/kernel/git/next/linux-next.git/tree/drivers/devfreq/tegra20-devfreq.c#n100
> >>>>>>>>>>>>>>>>>>
> >>>>>>>>>>>>>>>>>>
> >>>>>>>>>>>>>>>>>>
> >>>>>>>>>>>>>>>>>>
> >>>>>>>>>>>>>>>>>>
> >>>>>>>>>>>>>>>>>>        
> >>>>>>>>>>>>>>>>> Will go thru and add...  
> >>>>>>>>>>>>>>> Looks like I initially confused this case with getting
> >>>>>>>>>>>>>>> orphaned clock.
> >>>>>>>>>>>>>>> I'm now seeing that the DFLL driver registers the
> >>>>>>>>>>>>>>> clock and then
> >>>>>>>>>>>>>>> clk_get(dfll) should be returning EPROBE_DEFER until
> >>>>>>>>>>>>>>> DFLL driver is
> >>>>>>>>>>>>>>> probed, hence everything should be fine as-is and
> >>>>>>>>>>>>>>> there is no real
> >>>>>>>>>>>>>>> need
> >>>>>>>>>>>>>>> for the 'device link'. Sorry for the confusion!
> >>>>>>>>>>>>>>>        
> >>>>>>>>>>>>>>>>>>>> Sorry, I didn't follow the mail thread. Just
> >>>>>>>>>>>>>>>>>>>> regarding the DFLL
> >>>>>>>>>>>>>>>>>>>> part.
> >>>>>>>>>>>>>>>>>>>>
> >>>>>>>>>>>>>>>>>>>> As you know it, the DFLL clock is one of the CPU
> >>>>>>>>>>>>>>>>>>>> clock sources and
> >>>>>>>>>>>>>>>>>>>> integrated with DVFS control logic with the
> >>>>>>>>>>>>>>>>>>>> regulator. We will not
> >>>>>>>>>>>>>>>>>>>> switch
> >>>>>>>>>>>>>>>>>>>> CPU to other clock sources once we switched to
> >>>>>>>>>>>>>>>>>>>> DFLL. Because the
> >>>>>>>>>>>>>>>>>>>> CPU has
> >>>>>>>>>>>>>>>>>>>> been regulated by the DFLL HW with the DVFS table
> >>>>>>>>>>>>>>>>>>>> (CVB or OPP
> >>>>>>>>>>>>>>>>>>>> table
> >>>>>>>>>>>>>>>>>>>> you see
> >>>>>>>>>>>>>>>>>>>> in the driver.). We shouldn't reparent it to
> >>>>>>>>>>>>>>>>>>>> other sources with
> >>>>>>>>>>>>>>>>>>>> unknew
> >>>>>>>>>>>>>>>>>>>> freq/volt pair. That's not guaranteed to work. We
> >>>>>>>>>>>>>>>>>>>> allow switching to
> >>>>>>>>>>>>>>>>>>>> open-loop mode but different sources.  
> >>>>>>>>>>>>>>>>>> Okay, then the CPUFreq driver will have to enforce
> >>>>>>>>>>>>>>>>>> DFLL freq to
> >>>>>>>>>>>>>>>>>> PLLP's
> >>>>>>>>>>>>>>>>>> rate before switching to PLLP in order to have a
> >>>>>>>>>>>>>>>>>> proper CPU voltage.  
> >>>>>>>>>>>>>>>>> PLLP freq is safe to work for any CPU voltage. So no
> >>>>>>>>>>>>>>>>> need to enforce
> >>>>>>>>>>>>>>>>> DFLL freq to PLLP rate before changing CCLK_G source
> >>>>>>>>>>>>>>>>> to PLLP during
> >>>>>>>>>>>>>>>>> suspend
> >>>>>>>>>>>>>>>>>        
> >>>>>>>>>>>>>>>> Sorry, please ignore my above comment. During
> >>>>>>>>>>>>>>>> suspend, need to change
> >>>>>>>>>>>>>>>> CCLK_G source to PLLP when dfll is in closed loop
> >>>>>>>>>>>>>>>> mode first and
> >>>>>>>>>>>>>>>> then
> >>>>>>>>>>>>>>>> dfll need to be set to open loop.  
> >>>>>>>>>>>>>>> Okay.
> >>>>>>>>>>>>>>>        
> >>>>>>>>>>>>>>>>>>>> And I don't exactly understand why we need to
> >>>>>>>>>>>>>>>>>>>> switch to PLLP in
> >>>>>>>>>>>>>>>>>>>> CPU
> >>>>>>>>>>>>>>>>>>>> idle
> >>>>>>>>>>>>>>>>>>>> driver. Just keep it on CL-DVFS mode all the
> >>>>>>>>>>>>>>>>>>>> time.
> >>>>>>>>>>>>>>>>>>>>
> >>>>>>>>>>>>>>>>>>>> In SC7 entry, the dfll suspend function moves it
> >>>>>>>>>>>>>>>>>>>> the open-loop
> >>>>>>>>>>>>>>>>>>>> mode. That's
> >>>>>>>>>>>>>>>>>>>> all. The sc7-entryfirmware will handle the rest
> >>>>>>>>>>>>>>>>>>>> of the sequence to
> >>>>>>>>>>>>>>>>>>>> turn off
> >>>>>>>>>>>>>>>>>>>> the CPU power.
> >>>>>>>>>>>>>>>>>>>>
> >>>>>>>>>>>>>>>>>>>> In SC7 resume, the warmboot code will handle the
> >>>>>>>>>>>>>>>>>>>> sequence to
> >>>>>>>>>>>>>>>>>>>> turn on
> >>>>>>>>>>>>>>>>>>>> regulator and power up the CPU cluster. And leave
> >>>>>>>>>>>>>>>>>>>> it on PLL_P.
> >>>>>>>>>>>>>>>>>>>> After
> >>>>>>>>>>>>>>>>>>>> resuming to the kernel, we re-init DFLL, restore
> >>>>>>>>>>>>>>>>>>>> the CPU clock
> >>>>>>>>>>>>>>>>>>>> policy (CPU
> >>>>>>>>>>>>>>>>>>>> runs on DFLL open-loop mode) and then moving to
> >>>>>>>>>>>>>>>>>>>> close-loop mode.  
> >>>>>>>>>>>>>>>>>> The DFLL is re-inited after switching CCLK to DFLL
> >>>>>>>>>>>>>>>>>> parent during of
> >>>>>>>>>>>>>>>>>> the
> >>>>>>>>>>>>>>>>>> early clocks-state restoring by CaR driver. Hence
> >>>>>>>>>>>>>>>>>> instead of having
> >>>>>>>>>>>>>>>>>> odd
> >>>>>>>>>>>>>>>>>> hacks in the CaR driver, it is much nicer to have a
> >>>>>>>>>>>>>>>>>> proper suspend-resume sequencing of the device
> >>>>>>>>>>>>>>>>>> drivers. In this case
> >>>>>>>>>>>>>>>>>> CPUFreq
> >>>>>>>>>>>>>>>>>> driver is the driver that enables DFLL and switches
> >>>>>>>>>>>>>>>>>> CPU to that
> >>>>>>>>>>>>>>>>>> clock
> >>>>>>>>>>>>>>>>>> source, which means that this driver is also should
> >>>>>>>>>>>>>>>>>> be responsible for
> >>>>>>>>>>>>>>>>>> management of the DFLL's state during of
> >>>>>>>>>>>>>>>>>> suspend/resume process. If
> >>>>>>>>>>>>>>>>>> CPUFreq driver disables DFLL during suspend and
> >>>>>>>>>>>>>>>>>> re-enables it
> >>>>>>>>>>>>>>>>>> during
> >>>>>>>>>>>>>>>>>> resume, then looks like the CaR driver hacks around
> >>>>>>>>>>>>>>>>>> DFLL are not
> >>>>>>>>>>>>>>>>>> needed.
> >>>>>>>>>>>>>>>>>>        
> >>>>>>>>>>>>>>>>>>>> The DFLL part looks good to me. BTW, change the
> >>>>>>>>>>>>>>>>>>>> patch subject to
> >>>>>>>>>>>>>>>>>>>> "Add
> >>>>>>>>>>>>>>>>>>>> suspend-resume support" seems more appropriate to
> >>>>>>>>>>>>>>>>>>>> me.  
> >>>>>>>>>>>>>>>>>>> To clarify this, the sequences for DFLL use are as
> >>>>>>>>>>>>>>>>>>> follows (assuming
> >>>>>>>>>>>>>>>>>>> all
> >>>>>>>>>>>>>>>>>>> required DFLL hw configuration has been done)
> >>>>>>>>>>>>>>>>>>>
> >>>>>>>>>>>>>>>>>>> Switch to DFLL:
> >>>>>>>>>>>>>>>>>>> 0) Save current parent and frequency
> >>>>>>>>>>>>>>>>>>> 1) Program DFLL to open loop mode
> >>>>>>>>>>>>>>>>>>> 2) Enable DFLL
> >>>>>>>>>>>>>>>>>>> 3) Change cclk_g parent to DFLL
> >>>>>>>>>>>>>>>>>>> For OVR regulator:
> >>>>>>>>>>>>>>>>>>> 4) Change PWM output pin from tristate to output
> >>>>>>>>>>>>>>>>>>> 5) Enable DFLL PWM output
> >>>>>>>>>>>>>>>>>>> For I2C regulator:
> >>>>>>>>>>>>>>>>>>> 4) Enable DFLL I2C output
> >>>>>>>>>>>>>>>>>>> 6) Program DFLL to closed loop mode
> >>>>>>>>>>>>>>>>>>>
> >>>>>>>>>>>>>>>>>>> Switch away from DFLL:
> >>>>>>>>>>>>>>>>>>> 0) Change cclk_g parent to PLLP so the CPU
> >>>>>>>>>>>>>>>>>>> frequency is ok for
> >>>>>>>>>>>>>>>>>>> any
> >>>>>>>>>>>>>>>>>>> vdd_cpu voltage
> >>>>>>>>>>>>>>>>>>> 1) Program DFLL to open loop mode
> >>>>>>>>>>>>>>>>>>>        
> >>>>>>>>>>>>>>>> I see during switch away from DFLL (suspend), cclk_g
> >>>>>>>>>>>>>>>> parent is not
> >>>>>>>>>>>>>>>> changed to PLLP before changing dfll to open loop
> >>>>>>>>>>>>>>>> mode.
> >>>>>>>>>>>>>>>>
> >>>>>>>>>>>>>>>> Will add this ...  
> >>>>>>>>>>>>>>> The CPUFreq driver switches parent to PLLP during the
> >>>>>>>>>>>>>>> probe, similar
> >>>>>>>>>>>>>>> should be done on suspend.
> >>>>>>>>>>>>>>>
> >>>>>>>>>>>>>>> I'm also wondering if it's always safe to switch to
> >>>>>>>>>>>>>>> PLLP in the probe.
> >>>>>>>>>>>>>>> If CPU is running on a lower freq than PLLP, then some
> >>>>>>>>>>>>>>> other more
> >>>>>>>>>>>>>>> appropriate intermediate parent should be selected.
> >>>>>>>>>>>>>>>        
> >>>>>>>>>>>>>> CPU parents are PLL_X, PLL_P, and dfll. PLL_X always
> >>>>>>>>>>>>>> runs at higher
> >>>>>>>>>>>>>> rate
> >>>>>>>>>>>>>> so switching to PLL_P during CPUFreq probe prior to
> >>>>>>>>>>>>>> dfll clock enable
> >>>>>>>>>>>>>> should be safe.  
> >>>>>>>>>>>>> AFAIK, PLLX could run at ~200MHz. There is also a
> >>>>>>>>>>>>> divided output of
> >>>>>>>>>>>>> PLLP
> >>>>>>>>>>>>> which CCLKG supports, the PLLP_OUT4.
> >>>>>>>>>>>>>
> >>>>>>>>>>>>> Probably, realistically, CPU is always running off a
> >>>>>>>>>>>>> fast PLLX during
> >>>>>>>>>>>>> boot, but I'm wondering what may happen on KEXEC. I
> >>>>>>>>>>>>> guess ideally CPUFreq driver should also have a
> >>>>>>>>>>>>> 'shutdown' callback to teardown DFLL
> >>>>>>>>>>>>> on a reboot, but likely that there are other
> >>>>>>>>>>>>> clock-related problems as
> >>>>>>>>>>>>> well that may break KEXEC and thus it is not very
> >>>>>>>>>>>>> important at the
> >>>>>>>>>>>>> moment.
> >>>>>>>>>>>>>
> >>>>>>>>>>>>> [snip]  
> >>>>>>>>>>>> During bootup CPUG sources from PLL_X. By PLL_P source
> >>>>>>>>>>>> above I meant
> >>>>>>>>>>>> PLL_P_OUT4.
> >>>>>>>>>>>>
> >>>>>>>>>>>> As per clock policies, PLL_X is always used for high freq
> >>>>>>>>>>>> like  
> >>>>>>>>>>>>> 800Mhz  
> >>>>>>>>>>>> and for low frequency it will be sourced from PLLP.  
> >>>>>>>>>>> Alright, then please don't forget to pre-initialize
> >>>>>>>>>>> PLLP_OUT4 rate to a
> >>>>>>>>>>> reasonable value using tegra_clk_init_table or
> >>>>>>>>>>> assigned-clocks.  
> >>>>>>>>>> PLLP_OUT4 rate update is not needed as it is safe to run at
> >>>>>>>>>> 408Mhz because it is below fmax @ Vmin  
> >>>>>>>>> So even 204MHz CVB entries are having the same voltage as
> >>>>>>>>> 408MHz, correct? It's not instantly obvious to me from the
> >>>>>>>>> DFLL driver's code where the fmax @ Vmin is defined, I see
> >>>>>>>>> that there is the min_millivolts
> >>>>>>>>> and frequency entries starting from 204MHZ defined
> >>>>>>>>> per-table.  
> >>>>>>>> Yes at Vmin CPU Fmax is ~800Mhz. So anything below that will
> >>>>>>>> work at Vmin voltage and PLLP max is 408Mhz.  
> >>>>>>> Thank you for the clarification. It would be good to have that
> >>>>>>> commented
> >>>>>>> in the code as well.  
> >>>>>> OK, Will add...  
> >>>>> Regarding, adding suspend/resume to CPUFreq, CPUFreq suspend
> >>>>> happens very early even before disabling non-boot CPUs and also
> >>>>> need to export clock driver APIs to CPUFreq.
> >>>>>
> >>>>> Was thinking of below way of implementing this...
> >>>>>
> >>>>>
> >>>>> Clock DFLL driver Suspend:
> >>>>>
> >>>>>           - Save CPU clock policy registers, and Perform dfll
> >>>>> suspend which sets in open loop mode
> >>>>>
> >>>>> CPU Freq driver Suspend: does nothing
> >>>>>
> >>>>>
> >>>>> Clock DFLL driver Resume:
> >>>>>
> >>>>>           - Re-init DFLL, Set in Open-Loop mode, restore CPU
> >>>>> Clock policy registers which actually sets source to DFLL along
> >>>>> with other CPU Policy register restore.
> >>>>>
> >>>>> CPU Freq driver Resume:
> >>>>>
> >>>>>           - do clk_prepare_enable which acutally sets DFLL in
> >>>>> Closed loop mode
> >>>>>
> >>>>>
> >>>>> Adding one more note: Switching CPU Clock to PLLP is not needed
> >>>>> as CPU CLock can be from dfll in open-loop mode as DFLL is not
> >>>>> disabled anywhere throught the suspend/resume path and SC7 entry
> >>>>> FW and Warm boot code will switch CPU source to PLLP.  
> >>> Since CPU resumes on PLLP, it will be cleaner to suspend it on
> >>> PLLP as well. And besides, seems that currently disabling DFLL
> >>> clock will disable DFLL completely and then you'd want to re-init
> >>> the DFLL on resume any ways. So better to just disable DFLL
> >>> completely on suspend, which should happen on clk_disable(dfll).  
> >> Will switch to PLLP during CPUFreq suspend. With decision of using
> >> clk_disable during suspend, its mandatory to switch to PLLP as DFLL
> >> is completely disabled.
> >>
> >> My earlier concern was on restoring CPU policy as we can't do that
> >> from CPUFreq driver and need export from clock driver.
> >>
> >> Clear now and will do CPU clock policy restore in after dfll
> >> re-init.  
> > Why the policy can't be saved/restored by the CaR driver as a
> > context of any other clock?  
> 
> restoring cpu clock policy involves programming source and 
> super_cclkg_divider.
> 
> cclk_g is registered as clk_super_mux and it doesn't use frac_div ops
> to do save/restore its divider.

That can be changed of course and I guess it also could be as simple as
saving and restoring of two raw u32 values of the policy/divider
registers.

> Also, during clock context we cant restore cclk_g as cclk_g source
> will be dfll and dfll will not be resumed/re-initialized by the time 
> clk_super_mux save/restore happens.
> 
> we can't use save/restore context for dfll clk_ops because
> dfllCPU_out parent to CCLK_G is first in the clock tree and dfll_ref
> and dfll_soc peripheral clocks are not restored by the time dfll
> restore happens. Also dfll peripheral clock enables need to be
> restored before dfll restore happens which involves programming dfll
> controller for re-initialization.
> 
> So dfll resume/re-init is done in clk-tegra210 at end of all clocks 
> restore in V5 series but instead of in clk-tegra210 driver I moved
> now to dfll-fcpu driver pm_ops as all dfll dependencies will be
> restored thru clk_restore_context by then. This will be in V6.

Since DFLL is now guaranteed to be disabled across CaR suspend/resume
(hence it has nothing to do in regards to CCLK) and given that PLLs
state is restored before the rest of the clocks, I don't see why not to
implement CCLK save/restore in a generic fasion. CPU policy wull be
restored to either PLLP or PLLX (if CPUFreq driver is disabled).