[v3,3/4] PCI: mediatek: Add system pm support for MT2712 and MT7622

Message ID 1530518264-6125-4-git-send-email-honghui.zhang@mediatek.com
State Superseded
Delegated to: Lorenzo Pieralisi
Headers show
Series
  • PCI: mediatek: fixup find_port, enable_msi and add pm, module support
Related show

Commit Message

Honghui Zhang July 2, 2018, 7:57 a.m.
From: Honghui Zhang <honghui.zhang@mediatek.com>

The MTCMOS of PCIe Host for MT2712 and MT7622 will be off when system
suspend, and all the internal control register will be reset after system
resume. The PCIe link should be re-established and the related control
register values should be re-set after system resume.

Signed-off-by: Honghui Zhang <honghui.zhang@mediatek.com>
Acked-by: Ryder Lee <ryder.lee@mediatek.com>
---
 drivers/pci/controller/pcie-mediatek.c | 67 ++++++++++++++++++++++++++++++++++
 1 file changed, 67 insertions(+)

Comments

Lorenzo Pieralisi July 17, 2018, 5:15 p.m. | #1
[+Rafael, Kevin, Ulf]

On Mon, Jul 02, 2018 at 03:57:43PM +0800, honghui.zhang@mediatek.com wrote:
> From: Honghui Zhang <honghui.zhang@mediatek.com>
> 
> The MTCMOS of PCIe Host for MT2712 and MT7622 will be off when system
> suspend, and all the internal control register will be reset after system
> resume. The PCIe link should be re-established and the related control
> register values should be re-set after system resume.
> 
> Signed-off-by: Honghui Zhang <honghui.zhang@mediatek.com>
> Acked-by: Ryder Lee <ryder.lee@mediatek.com>
> ---
>  drivers/pci/controller/pcie-mediatek.c | 67 ++++++++++++++++++++++++++++++++++
>  1 file changed, 67 insertions(+)
> 
> diff --git a/drivers/pci/controller/pcie-mediatek.c b/drivers/pci/controller/pcie-mediatek.c
> index 86918d4..175d7b6 100644
> --- a/drivers/pci/controller/pcie-mediatek.c
> +++ b/drivers/pci/controller/pcie-mediatek.c
> @@ -134,12 +134,14 @@ struct mtk_pcie_port;
>  /**
>   * struct mtk_pcie_soc - differentiate between host generations
>   * @need_fix_class_id: whether this host's class ID needed to be fixed or not
> + * @pm_support: whether the host's MTCMOS will be off when suspend
>   * @ops: pointer to configuration access functions
>   * @startup: pointer to controller setting functions
>   * @setup_irq: pointer to initialize IRQ functions
>   */
>  struct mtk_pcie_soc {
>  	bool need_fix_class_id;
> +	bool pm_support;
>  	struct pci_ops *ops;
>  	int (*startup)(struct mtk_pcie_port *port);
>  	int (*setup_irq)(struct mtk_pcie_port *port, struct device_node *node);
> @@ -1197,12 +1199,75 @@ static int mtk_pcie_probe(struct platform_device *pdev)
>  	return err;
>  }
>  
> +static int __maybe_unused mtk_pcie_suspend_noirq(struct device *dev)
> +{
> +	struct mtk_pcie *pcie = dev_get_drvdata(dev);
> +	const struct mtk_pcie_soc *soc = pcie->soc;
> +	struct mtk_pcie_port *port;
> +
> +	if (!soc->pm_support)
> +		return 0;
> +
> +	if (list_empty(&pcie->ports))
> +		return 0;
> +
> +	list_for_each_entry(port, &pcie->ports, list) {
> +		clk_disable_unprepare(port->pipe_ck);
> +		clk_disable_unprepare(port->obff_ck);
> +		clk_disable_unprepare(port->axi_ck);
> +		clk_disable_unprepare(port->aux_ck);
> +		clk_disable_unprepare(port->ahb_ck);
> +		clk_disable_unprepare(port->sys_ck);
> +		phy_power_off(port->phy);
> +		phy_exit(port->phy);
> +	}
> +
> +	mtk_pcie_subsys_powerdown(pcie);
> +
> +	return 0;
> +}
> +
> +static int __maybe_unused mtk_pcie_resume_noirq(struct device *dev)
> +{
> +	struct mtk_pcie *pcie = dev_get_drvdata(dev);
> +	const struct mtk_pcie_soc *soc = pcie->soc;
> +	struct mtk_pcie_port *port, *tmp;
> +
> +	if (!soc->pm_support)
> +		return 0;
> +
> +	if (list_empty(&pcie->ports))
> +		return 0;
> +
> +	if (dev->pm_domain) {
> +		pm_runtime_enable(dev);
> +		pm_runtime_get_sync(dev);
> +	}

Are these runtime PM calls needed/abused here ?

Mind explaining the logic ?

There is certainly an asymmetry with the suspend callback which made me
suspicious, I am pretty certain Rafael/Kevin/Ulf can help me clarify so
that we can make progress with this patch.

Lorenzo

> +
> +	clk_prepare_enable(pcie->free_ck);
> +
> +	list_for_each_entry_safe(port, tmp, &pcie->ports, list)
> +		mtk_pcie_enable_port(port);
> +
> +	/* In case of EP was removed while system suspend. */
> +	if (list_empty(&pcie->ports))
> +		mtk_pcie_subsys_powerdown(pcie);
> +
> +	return 0;
> +}
> +
> +static const struct dev_pm_ops mtk_pcie_pm_ops = {
> +	SET_NOIRQ_SYSTEM_SLEEP_PM_OPS(mtk_pcie_suspend_noirq,
> +				      mtk_pcie_resume_noirq)
> +};
> +
>  static const struct mtk_pcie_soc mtk_pcie_soc_v1 = {
>  	.ops = &mtk_pcie_ops,
>  	.startup = mtk_pcie_startup_port,
>  };
>  
>  static const struct mtk_pcie_soc mtk_pcie_soc_mt2712 = {
> +	.pm_support = true,
>  	.ops = &mtk_pcie_ops_v2,
>  	.startup = mtk_pcie_startup_port_v2,
>  	.setup_irq = mtk_pcie_setup_irq,
> @@ -1210,6 +1275,7 @@ static const struct mtk_pcie_soc mtk_pcie_soc_mt2712 = {
>  
>  static const struct mtk_pcie_soc mtk_pcie_soc_mt7622 = {
>  	.need_fix_class_id = true,
> +	.pm_support = true,
>  	.ops = &mtk_pcie_ops_v2,
>  	.startup = mtk_pcie_startup_port_v2,
>  	.setup_irq = mtk_pcie_setup_irq,
> @@ -1229,6 +1295,7 @@ static struct platform_driver mtk_pcie_driver = {
>  		.name = "mtk-pcie",
>  		.of_match_table = mtk_pcie_ids,
>  		.suppress_bind_attrs = true,
> +		.pm = &mtk_pcie_pm_ops,
>  	},
>  };
>  builtin_platform_driver(mtk_pcie_driver);
> -- 
> 2.6.4
>
Honghui Zhang July 18, 2018, 6:02 a.m. | #2
On Tue, 2018-07-17 at 18:15 +0100, Lorenzo Pieralisi wrote:
> [+Rafael, Kevin, Ulf]
> 
> On Mon, Jul 02, 2018 at 03:57:43PM +0800, honghui.zhang@mediatek.com wrote:
> > From: Honghui Zhang <honghui.zhang@mediatek.com>
> > 
> > The MTCMOS of PCIe Host for MT2712 and MT7622 will be off when system
> > suspend, and all the internal control register will be reset after system
> > resume. The PCIe link should be re-established and the related control
> > register values should be re-set after system resume.
> > 
> > Signed-off-by: Honghui Zhang <honghui.zhang@mediatek.com>
> > Acked-by: Ryder Lee <ryder.lee@mediatek.com>
> > ---
> >  drivers/pci/controller/pcie-mediatek.c | 67 ++++++++++++++++++++++++++++++++++
> >  1 file changed, 67 insertions(+)
> > 
> > diff --git a/drivers/pci/controller/pcie-mediatek.c b/drivers/pci/controller/pcie-mediatek.c
> > index 86918d4..175d7b6 100644
> > --- a/drivers/pci/controller/pcie-mediatek.c
> > +++ b/drivers/pci/controller/pcie-mediatek.c
> > @@ -134,12 +134,14 @@ struct mtk_pcie_port;
> >  /**
> >   * struct mtk_pcie_soc - differentiate between host generations
> >   * @need_fix_class_id: whether this host's class ID needed to be fixed or not
> > + * @pm_support: whether the host's MTCMOS will be off when suspend
> >   * @ops: pointer to configuration access functions
> >   * @startup: pointer to controller setting functions
> >   * @setup_irq: pointer to initialize IRQ functions
> >   */
> >  struct mtk_pcie_soc {
> >  	bool need_fix_class_id;
> > +	bool pm_support;
> >  	struct pci_ops *ops;
> >  	int (*startup)(struct mtk_pcie_port *port);
> >  	int (*setup_irq)(struct mtk_pcie_port *port, struct device_node *node);
> > @@ -1197,12 +1199,75 @@ static int mtk_pcie_probe(struct platform_device *pdev)
> >  	return err;
> >  }
> >  
> > +static int __maybe_unused mtk_pcie_suspend_noirq(struct device *dev)
> > +{
> > +	struct mtk_pcie *pcie = dev_get_drvdata(dev);
> > +	const struct mtk_pcie_soc *soc = pcie->soc;
> > +	struct mtk_pcie_port *port;
> > +
> > +	if (!soc->pm_support)
> > +		return 0;
> > +
> > +	if (list_empty(&pcie->ports))
> > +		return 0;
> > +
> > +	list_for_each_entry(port, &pcie->ports, list) {
> > +		clk_disable_unprepare(port->pipe_ck);
> > +		clk_disable_unprepare(port->obff_ck);
> > +		clk_disable_unprepare(port->axi_ck);
> > +		clk_disable_unprepare(port->aux_ck);
> > +		clk_disable_unprepare(port->ahb_ck);
> > +		clk_disable_unprepare(port->sys_ck);
> > +		phy_power_off(port->phy);
> > +		phy_exit(port->phy);
> > +	}
> > +
> > +	mtk_pcie_subsys_powerdown(pcie);
> > +
> > +	return 0;
> > +}
> > +
> > +static int __maybe_unused mtk_pcie_resume_noirq(struct device *dev)
> > +{
> > +	struct mtk_pcie *pcie = dev_get_drvdata(dev);
> > +	const struct mtk_pcie_soc *soc = pcie->soc;
> > +	struct mtk_pcie_port *port, *tmp;
> > +
> > +	if (!soc->pm_support)
> > +		return 0;
> > +
> > +	if (list_empty(&pcie->ports))
> > +		return 0;
> > +
> > +	if (dev->pm_domain) {
> > +		pm_runtime_enable(dev);
> > +		pm_runtime_get_sync(dev);
> > +	}
> 
> Are these runtime PM calls needed/abused here ?
> 
> Mind explaining the logic ?
> 
> There is certainly an asymmetry with the suspend callback which made me
> suspicious, I am pretty certain Rafael/Kevin/Ulf can help me clarify so
> that we can make progress with this patch.
> 
> Lorenzo
> 
Hi Lorenzo, thanks for your comments.
Sorry I don't get you.
I believe that in suspend callbacks the pm_runtime_put_sync and
pm_runtime_disable should be called to gated the CMOS for this module,
while the pm_rumtime_enable and pm_rumtime_get_sync should be called in
resume callback.

That's exactly this patch doing.
But the pm_rumtime_put_sync and pm_runtime_disable functions was wrapped
in the mtk_pcie_subsys_powerdown.

I did not call mtk_pcie_subsys_powerup since it does not just wrapped
pm_rumtime related functions but also do the platform_resource_get,
devm_ioremap, and free_ck clock get which I do not needed in resume
callback.

Do you think it will be much clear if I abstract the
platform_resource_get, devm_ioremap functions from
mtk_pcie_subsys_powerup and put it to a new functions like
mtk_pcie_subsys_resource_get, and then we may call the
mtk_pcie_subsys_powerup in the resume function?

thanks
> > +
> > +	clk_prepare_enable(pcie->free_ck);
> > +
> > +	list_for_each_entry_safe(port, tmp, &pcie->ports, list)
> > +		mtk_pcie_enable_port(port);
> > +
> > +	/* In case of EP was removed while system suspend. */
> > +	if (list_empty(&pcie->ports))
> > +		mtk_pcie_subsys_powerdown(pcie);
> > +
> > +	return 0;
> > +}
> > +
> > +static const struct dev_pm_ops mtk_pcie_pm_ops = {
> > +	SET_NOIRQ_SYSTEM_SLEEP_PM_OPS(mtk_pcie_suspend_noirq,
> > +				      mtk_pcie_resume_noirq)
> > +};
> > +
> >  static const struct mtk_pcie_soc mtk_pcie_soc_v1 = {
> >  	.ops = &mtk_pcie_ops,
> >  	.startup = mtk_pcie_startup_port,
> >  };
> >  
> >  static const struct mtk_pcie_soc mtk_pcie_soc_mt2712 = {
> > +	.pm_support = true,
> >  	.ops = &mtk_pcie_ops_v2,
> >  	.startup = mtk_pcie_startup_port_v2,
> >  	.setup_irq = mtk_pcie_setup_irq,
> > @@ -1210,6 +1275,7 @@ static const struct mtk_pcie_soc mtk_pcie_soc_mt2712 = {
> >  
> >  static const struct mtk_pcie_soc mtk_pcie_soc_mt7622 = {
> >  	.need_fix_class_id = true,
> > +	.pm_support = true,
> >  	.ops = &mtk_pcie_ops_v2,
> >  	.startup = mtk_pcie_startup_port_v2,
> >  	.setup_irq = mtk_pcie_setup_irq,
> > @@ -1229,6 +1295,7 @@ static struct platform_driver mtk_pcie_driver = {
> >  		.name = "mtk-pcie",
> >  		.of_match_table = mtk_pcie_ids,
> >  		.suppress_bind_attrs = true,
> > +		.pm = &mtk_pcie_pm_ops,
> >  	},
> >  };
> >  builtin_platform_driver(mtk_pcie_driver);
> > -- 
> > 2.6.4
> >
Lorenzo Pieralisi July 18, 2018, 9:49 a.m. | #3
On Wed, Jul 18, 2018 at 02:02:41PM +0800, Honghui Zhang wrote:

<snip>

> > > +static int __maybe_unused mtk_pcie_resume_noirq(struct device *dev)
> > > +{
> > > +	struct mtk_pcie *pcie = dev_get_drvdata(dev);
> > > +	const struct mtk_pcie_soc *soc = pcie->soc;
> > > +	struct mtk_pcie_port *port, *tmp;
> > > +
> > > +	if (!soc->pm_support)
> > > +		return 0;
> > > +
> > > +	if (list_empty(&pcie->ports))
> > > +		return 0;
> > > +
> > > +	if (dev->pm_domain) {
> > > +		pm_runtime_enable(dev);
> > > +		pm_runtime_get_sync(dev);
> > > +	}
> > 
> > Are these runtime PM calls needed/abused here ?
> > 
> > Mind explaining the logic ?
> > 
> > There is certainly an asymmetry with the suspend callback which made me
> > suspicious, I am pretty certain Rafael/Kevin/Ulf can help me clarify so
> > that we can make progress with this patch.
> > 
> > Lorenzo
> > 
> Hi Lorenzo, thanks for your comments.
> Sorry I don't get you.
> I believe that in suspend callbacks the pm_runtime_put_sync and
> pm_runtime_disable should be called to gated the CMOS for this module,
> while the pm_rumtime_enable and pm_rumtime_get_sync should be called
> in resume callback.

That's why I CC'ed Rafael, Kevin and Ulf, to answer this question
thoroughly, I am not sure it is needed and that's the right way
of doing it in system suspend callbacks.

> That's exactly this patch doing.
> But the pm_rumtime_put_sync and pm_runtime_disable functions was wrapped
> in the mtk_pcie_subsys_powerdown.

Ah, sorry, I missed that.

> I did not call mtk_pcie_subsys_powerup since it does not just wrapped
> pm_rumtime related functions but also do the platform_resource_get,
> devm_ioremap, and free_ck clock get which I do not needed in resume
> callback.
> 
> Do you think it will be much clear if I abstract the
> platform_resource_get, devm_ioremap functions from
> mtk_pcie_subsys_powerup and put it to a new functions like
> mtk_pcie_subsys_resource_get, and then we may call the
> mtk_pcie_subsys_powerup in the resume function?

I think so but let's wait first for feedback on whether those
runtime PM calls are needed in the first place.

Lorenzo
Honghui Zhang Sept. 7, 2018, 9:43 a.m. | #4
On Wed, 2018-07-18 at 10:49 +0100, Lorenzo Pieralisi wrote:
> On Wed, Jul 18, 2018 at 02:02:41PM +0800, Honghui Zhang wrote:
> 
> <snip>
> 
> > > > +static int __maybe_unused mtk_pcie_resume_noirq(struct device *dev)
> > > > +{
> > > > +	struct mtk_pcie *pcie = dev_get_drvdata(dev);
> > > > +	const struct mtk_pcie_soc *soc = pcie->soc;
> > > > +	struct mtk_pcie_port *port, *tmp;
> > > > +
> > > > +	if (!soc->pm_support)
> > > > +		return 0;
> > > > +
> > > > +	if (list_empty(&pcie->ports))
> > > > +		return 0;
> > > > +
> > > > +	if (dev->pm_domain) {
> > > > +		pm_runtime_enable(dev);
> > > > +		pm_runtime_get_sync(dev);
> > > > +	}
> > > 
> > > Are these runtime PM calls needed/abused here ?
> > > 
> > > Mind explaining the logic ?
> > > 
> > > There is certainly an asymmetry with the suspend callback which made me
> > > suspicious, I am pretty certain Rafael/Kevin/Ulf can help me clarify so
> > > that we can make progress with this patch.
> > > 
> > > Lorenzo
> > > 
> > Hi Lorenzo, thanks for your comments.
> > Sorry I don't get you.
> > I believe that in suspend callbacks the pm_runtime_put_sync and
> > pm_runtime_disable should be called to gated the CMOS for this module,
> > while the pm_rumtime_enable and pm_rumtime_get_sync should be called
> > in resume callback.

> That's why I CC'ed Rafael, Kevin and Ulf, to answer this question
> thoroughly, I am not sure it is needed and that's the right way
> of doing it in system suspend callbacks.
> 

hi, Rafael, Kevin and Ulf,

after reading of the power related documents in Documents/power/, I'm
still confused whether the runtime_pm callbacks should be called in
system suspend callbacks.

I believe that system suspend does not care about the device's CMOS
status. And the device's CMOS status is controlled by runtime pm.
That why I gated the CMOS through runtime pm in the system suspend
callbacks.

But I checked all existing system suspend callbacks and found there's no
runtime pm was executed in system suspend callbacks. Does that means
when system suspend, the system suspend framework does not care about
the power consume? Or does it gated each device's CMOS in somewhere
else?

Or should I just remove the runtime pm callbacks in the system suspend
flow?

Could someone kindly give me some comments?

Thanks in advance.

> > That's exactly this patch doing.
> > But the pm_rumtime_put_sync and pm_runtime_disable functions was wrapped
> > in the mtk_pcie_subsys_powerdown.
> 
> Ah, sorry, I missed that.
> 
> > I did not call mtk_pcie_subsys_powerup since it does not just wrapped
> > pm_rumtime related functions but also do the platform_resource_get,
> > devm_ioremap, and free_ck clock get which I do not needed in resume
> > callback.
> > 
> > Do you think it will be much clear if I abstract the
> > platform_resource_get, devm_ioremap functions from
> > mtk_pcie_subsys_powerup and put it to a new functions like
> > mtk_pcie_subsys_resource_get, and then we may call the
> > mtk_pcie_subsys_powerup in the resume function?
> 
> I think so but let's wait first for feedback on whether those
> runtime PM calls are needed in the first place.
> 
> Lorenzo
Ulf Hansson Sept. 7, 2018, 12:33 p.m. | #5
-trimmed cc-list

On 2 July 2018 at 09:57,  <honghui.zhang@mediatek.com> wrote:
> From: Honghui Zhang <honghui.zhang@mediatek.com>
>
> The MTCMOS of PCIe Host for MT2712 and MT7622 will be off when system
> suspend, and all the internal control register will be reset after system
> resume. The PCIe link should be re-established and the related control
> register values should be re-set after system resume.

Sounds very familiar as what is implemented in
drivers/mmc/host/sdhci-xenon.c. Please have a look, possibly you can
get inspired to use the similar pattern.

A few comments below. However, please note, I am not an expert in PCIe
so I may overlook some obvious things. I am relying on Lorenzo or some
other PCIe expert, to fill in.

>
> Signed-off-by: Honghui Zhang <honghui.zhang@mediatek.com>
> Acked-by: Ryder Lee <ryder.lee@mediatek.com>
> ---
>  drivers/pci/controller/pcie-mediatek.c | 67 ++++++++++++++++++++++++++++++++++
>  1 file changed, 67 insertions(+)
>
> diff --git a/drivers/pci/controller/pcie-mediatek.c b/drivers/pci/controller/pcie-mediatek.c
> index 86918d4..175d7b6 100644
> --- a/drivers/pci/controller/pcie-mediatek.c
> +++ b/drivers/pci/controller/pcie-mediatek.c
> @@ -134,12 +134,14 @@ struct mtk_pcie_port;
>  /**
>   * struct mtk_pcie_soc - differentiate between host generations
>   * @need_fix_class_id: whether this host's class ID needed to be fixed or not
> + * @pm_support: whether the host's MTCMOS will be off when suspend
>   * @ops: pointer to configuration access functions
>   * @startup: pointer to controller setting functions
>   * @setup_irq: pointer to initialize IRQ functions
>   */
>  struct mtk_pcie_soc {
>         bool need_fix_class_id;
> +       bool pm_support;
>         struct pci_ops *ops;
>         int (*startup)(struct mtk_pcie_port *port);
>         int (*setup_irq)(struct mtk_pcie_port *port, struct device_node *node);
> @@ -1197,12 +1199,75 @@ static int mtk_pcie_probe(struct platform_device *pdev)
>         return err;
>  }
>
> +static int __maybe_unused mtk_pcie_suspend_noirq(struct device *dev)

I am not sure I understand why you need to suspend the device in the
"noirq" phase. Isn't it fine to do that in the regular suspend phase?

> +{
> +       struct mtk_pcie *pcie = dev_get_drvdata(dev);
> +       const struct mtk_pcie_soc *soc = pcie->soc;
> +       struct mtk_pcie_port *port;
> +
> +       if (!soc->pm_support)
> +               return 0;
> +
> +       if (list_empty(&pcie->ports))
> +               return 0;
> +
> +       list_for_each_entry(port, &pcie->ports, list) {
> +               clk_disable_unprepare(port->pipe_ck);
> +               clk_disable_unprepare(port->obff_ck);
> +               clk_disable_unprepare(port->axi_ck);
> +               clk_disable_unprepare(port->aux_ck);
> +               clk_disable_unprepare(port->ahb_ck);
> +               clk_disable_unprepare(port->sys_ck);
> +               phy_power_off(port->phy);
> +               phy_exit(port->phy);
> +       }
> +
> +       mtk_pcie_subsys_powerdown(pcie);

Why not gate clocks, unconditionally not depending on if pm_support is
set or not, during system suspend?

I understand that you for some SoCs needs also to restore registers
during system resume, but that's a different thing, isn't it?

> +
> +       return 0;
> +}
> +
> +static int __maybe_unused mtk_pcie_resume_noirq(struct device *dev)
> +{
> +       struct mtk_pcie *pcie = dev_get_drvdata(dev);
> +       const struct mtk_pcie_soc *soc = pcie->soc;
> +       struct mtk_pcie_port *port, *tmp;
> +
> +       if (!soc->pm_support)
> +               return 0;
> +
> +       if (list_empty(&pcie->ports))
> +               return 0;
> +
> +       if (dev->pm_domain) {
> +               pm_runtime_enable(dev);
> +               pm_runtime_get_sync(dev);

I noticed, Lorenzo wanted me to comment on this, so here it goes.

I guess the reason to why you make these pm_runtime_*() calls, is
because you want to restore the actions taken during
mtk_pcie_suspend_noirq() (which calls mtk_pcie_subsys_powerdown ->
pm_runtime_disable|put*())?

If that's the case, I would rather avoid calling pm_runtime_disable()
and pm_runtime_put() via mtk_pcie_suspend_noirq(), simply because it
isn't needed.

Of course, another option is to follow the pattern in
drivers/mmc/host/sdhci-xenon.c.

Overall, I am also wondering why only runtime PM is used when there is
a PM domain attached to the device? That seems to make the logic in
the driver, unnecessary complicated. Why not use runtime
unconditionally and thus enable it during ->probe()?

> +       }
> +
> +       clk_prepare_enable(pcie->free_ck);
> +
> +       list_for_each_entry_safe(port, tmp, &pcie->ports, list)
> +               mtk_pcie_enable_port(port);
> +
> +       /* In case of EP was removed while system suspend. */
> +       if (list_empty(&pcie->ports))
> +               mtk_pcie_subsys_powerdown(pcie);
> +
> +       return 0;
> +}
> +
> +static const struct dev_pm_ops mtk_pcie_pm_ops = {
> +       SET_NOIRQ_SYSTEM_SLEEP_PM_OPS(mtk_pcie_suspend_noirq,
> +                                     mtk_pcie_resume_noirq)
> +};
> +
>  static const struct mtk_pcie_soc mtk_pcie_soc_v1 = {
>         .ops = &mtk_pcie_ops,
>         .startup = mtk_pcie_startup_port,
>  };
>
>  static const struct mtk_pcie_soc mtk_pcie_soc_mt2712 = {
> +       .pm_support = true,
>         .ops = &mtk_pcie_ops_v2,
>         .startup = mtk_pcie_startup_port_v2,
>         .setup_irq = mtk_pcie_setup_irq,
> @@ -1210,6 +1275,7 @@ static const struct mtk_pcie_soc mtk_pcie_soc_mt2712 = {
>
>  static const struct mtk_pcie_soc mtk_pcie_soc_mt7622 = {
>         .need_fix_class_id = true,
> +       .pm_support = true,
>         .ops = &mtk_pcie_ops_v2,
>         .startup = mtk_pcie_startup_port_v2,
>         .setup_irq = mtk_pcie_setup_irq,
> @@ -1229,6 +1295,7 @@ static struct platform_driver mtk_pcie_driver = {
>                 .name = "mtk-pcie",
>                 .of_match_table = mtk_pcie_ids,
>                 .suppress_bind_attrs = true,
> +               .pm = &mtk_pcie_pm_ops,
>         },
>  };
>  builtin_platform_driver(mtk_pcie_driver);
> --
> 2.6.4
>

Kind regards
Uffe
Honghui Zhang Sept. 10, 2018, 9:42 a.m. | #6
On Fri, 2018-09-07 at 14:33 +0200, Ulf Hansson wrote:
> -trimmed cc-list
> 
> On 2 July 2018 at 09:57,  <honghui.zhang@mediatek.com> wrote:
> > From: Honghui Zhang <honghui.zhang@mediatek.com>
> >
> > The MTCMOS of PCIe Host for MT2712 and MT7622 will be off when system
> > suspend, and all the internal control register will be reset after system
> > resume. The PCIe link should be re-established and the related control
> > register values should be re-set after system resume.
> 
> Sounds very familiar as what is implemented in
> drivers/mmc/host/sdhci-xenon.c. Please have a look, possibly you can
> get inspired to use the similar pattern.
> 
> A few comments below. However, please note, I am not an expert in PCIe
> so I may overlook some obvious things. I am relying on Lorenzo or some
> other PCIe expert, to fill in.
> 
Hi, Ulf, thanks very much for your comments, replied below.

> >
> > Signed-off-by: Honghui Zhang <honghui.zhang@mediatek.com>
> > Acked-by: Ryder Lee <ryder.lee@mediatek.com>
> > ---
> >  drivers/pci/controller/pcie-mediatek.c | 67 ++++++++++++++++++++++++++++++++++
> >  1 file changed, 67 insertions(+)
> >
> > diff --git a/drivers/pci/controller/pcie-mediatek.c b/drivers/pci/controller/pcie-mediatek.c
> > index 86918d4..175d7b6 100644
> > --- a/drivers/pci/controller/pcie-mediatek.c
> > +++ b/drivers/pci/controller/pcie-mediatek.c
> > @@ -134,12 +134,14 @@ struct mtk_pcie_port;
> >  /**
> >   * struct mtk_pcie_soc - differentiate between host generations
> >   * @need_fix_class_id: whether this host's class ID needed to be fixed or not
> > + * @pm_support: whether the host's MTCMOS will be off when suspend
> >   * @ops: pointer to configuration access functions
> >   * @startup: pointer to controller setting functions
> >   * @setup_irq: pointer to initialize IRQ functions
> >   */
> >  struct mtk_pcie_soc {
> >         bool need_fix_class_id;
> > +       bool pm_support;
> >         struct pci_ops *ops;
> >         int (*startup)(struct mtk_pcie_port *port);
> >         int (*setup_irq)(struct mtk_pcie_port *port, struct device_node *node);
> > @@ -1197,12 +1199,75 @@ static int mtk_pcie_probe(struct platform_device *pdev)
> >         return err;
> >  }
> >
> > +static int __maybe_unused mtk_pcie_suspend_noirq(struct device *dev)
> 
> I am not sure I understand why you need to suspend the device in the
> "noirq" phase. Isn't it fine to do that in the regular suspend phase?

PCIe device's configuration space values should be saved/restored in
system suspend/resume flow, and PCIe framework will take care of that in
the suspend_noirq phase. Suspend device in the "noirq" phase will spare
device driver from that.

> 
> > +{
> > +       struct mtk_pcie *pcie = dev_get_drvdata(dev);
> > +       const struct mtk_pcie_soc *soc = pcie->soc;
> > +       struct mtk_pcie_port *port;
> > +
> > +       if (!soc->pm_support)
> > +               return 0;
> > +
> > +       if (list_empty(&pcie->ports))
> > +               return 0;
> > +
> > +       list_for_each_entry(port, &pcie->ports, list) {
> > +               clk_disable_unprepare(port->pipe_ck);
> > +               clk_disable_unprepare(port->obff_ck);
> > +               clk_disable_unprepare(port->axi_ck);
> > +               clk_disable_unprepare(port->aux_ck);
> > +               clk_disable_unprepare(port->ahb_ck);
> > +               clk_disable_unprepare(port->sys_ck);
> > +               phy_power_off(port->phy);
> > +               phy_exit(port->phy);
> > +       }
> > +
> > +       mtk_pcie_subsys_powerdown(pcie);
> 
> Why not gate clocks, unconditionally not depending on if pm_support is
> set or not, during system suspend?
> 
> I understand that you for some SoCs needs also to restore registers
> during system resume, but that's a different thing, isn't it?
> 

Well, current host driver support different SoCs like mt7623, mt7622 and
mt2712. mt7623 need quite special take care of when system suspend. This
patch just add the suspend/resume support for mt7622&mt2712, while
mt7623 was excluded. This "pm_support" maybe removed after we have a
solution for mt7623 SoC of the system suspend.

> > +
> > +       return 0;
> > +}
> > +
> > +static int __maybe_unused mtk_pcie_resume_noirq(struct device *dev)
> > +{
> > +       struct mtk_pcie *pcie = dev_get_drvdata(dev);
> > +       const struct mtk_pcie_soc *soc = pcie->soc;
> > +       struct mtk_pcie_port *port, *tmp;
> > +
> > +       if (!soc->pm_support)
> > +               return 0;
> > +
> > +       if (list_empty(&pcie->ports))
> > +               return 0;
> > +
> > +       if (dev->pm_domain) {
> > +               pm_runtime_enable(dev);
> > +               pm_runtime_get_sync(dev);
> 
> I noticed, Lorenzo wanted me to comment on this, so here it goes.
> 
> I guess the reason to why you make these pm_runtime_*() calls, is
> because you want to restore the actions taken during
> mtk_pcie_suspend_noirq() (which calls mtk_pcie_subsys_powerdown ->
> pm_runtime_disable|put*())?

Yes, that's what am I want to do.

> 
> If that's the case, I would rather avoid calling pm_runtime_disable()
> and pm_runtime_put() via mtk_pcie_suspend_noirq(), simply because it
> isn't needed.

> Of course, another option is to follow the pattern in
> drivers/mmc/host/sdhci-xenon.c.
> 

Thanks.
I take a look at the drivers/mmc/sdhci-xenon.c's implement, it has add
system suspend callbacks and set the device status to "PRM_SUSPENDED"
through the pm_runtime_force_suspend() interface.
I noticed that pm_runtime_put_sync will put the device status to
"PRM_SUSPENDED" status in some certain condition.
And I did not found the pci framework setting the status to
"PRM_SUSPENDED". So I add the pm_runtime_put_sync/disable in
suspend_noirq phase.

According to my understanding of PM, the system suspend flow does not
care about the runtime PM status. The runtime pm is responsible for the
CMOS gate, if the device want to gated it's CMOS, it need to call the
runtime pm interface obviously.

But I found that the PCIe framework will still operate with the device
after device driver suspend_noirq executed like save the configuration
space values through pci_save_state, So I guess the cmos could not be
gated at the suspend_noirq phase. I will update the patchset to remove
the pm runtime operations in the suspend_noirq phase.

> Overall, I am also wondering why only runtime PM is used when there is
> a PM domain attached to the device? That seems to make the logic in
> the driver, unnecessary complicated. Why not use runtime
> unconditionally and thus enable it during ->probe()?

I'm not sure, I thought that if there's no "power-domains = " property
in device node, that means the device has no PM domain. Some of the SoCs
does not have independent CMOS domain. So does that means it will leave
the dev->pm_domain as NULL or assign it as the genpd->pm_domain?

I need to do some homework to figure this out and will send a new patch
to fix this if needed.

> 
> > +       }
> > +
> > +       clk_prepare_enable(pcie->free_ck);
> > +
> > +       list_for_each_entry_safe(port, tmp, &pcie->ports, list)
> > +               mtk_pcie_enable_port(port);
> > +
> > +       /* In case of EP was removed while system suspend. */
> > +       if (list_empty(&pcie->ports))
> > +               mtk_pcie_subsys_powerdown(pcie);
> > +
> > +       return 0;
> > +}
> > +
> > +static const struct dev_pm_ops mtk_pcie_pm_ops = {
> > +       SET_NOIRQ_SYSTEM_SLEEP_PM_OPS(mtk_pcie_suspend_noirq,
> > +                                     mtk_pcie_resume_noirq)
> > +};
> > +
> >  static const struct mtk_pcie_soc mtk_pcie_soc_v1 = {
> >         .ops = &mtk_pcie_ops,
> >         .startup = mtk_pcie_startup_port,
> >  };
> >
> >  static const struct mtk_pcie_soc mtk_pcie_soc_mt2712 = {
> > +       .pm_support = true,
> >         .ops = &mtk_pcie_ops_v2,
> >         .startup = mtk_pcie_startup_port_v2,
> >         .setup_irq = mtk_pcie_setup_irq,
> > @@ -1210,6 +1275,7 @@ static const struct mtk_pcie_soc mtk_pcie_soc_mt2712 = {
> >
> >  static const struct mtk_pcie_soc mtk_pcie_soc_mt7622 = {
> >         .need_fix_class_id = true,
> > +       .pm_support = true,
> >         .ops = &mtk_pcie_ops_v2,
> >         .startup = mtk_pcie_startup_port_v2,
> >         .setup_irq = mtk_pcie_setup_irq,
> > @@ -1229,6 +1295,7 @@ static struct platform_driver mtk_pcie_driver = {
> >                 .name = "mtk-pcie",
> >                 .of_match_table = mtk_pcie_ids,
> >                 .suppress_bind_attrs = true,
> > +               .pm = &mtk_pcie_pm_ops,
> >         },
> >  };
> >  builtin_platform_driver(mtk_pcie_driver);
> > --
> > 2.6.4
> >
> 
> Kind regards
> Uffe
Ulf Hansson Sept. 10, 2018, 11:25 a.m. | #7
On 10 September 2018 at 11:42, Honghui Zhang <honghui.zhang@mediatek.com> wrote:
> On Fri, 2018-09-07 at 14:33 +0200, Ulf Hansson wrote:
>> -trimmed cc-list
>>
>> On 2 July 2018 at 09:57,  <honghui.zhang@mediatek.com> wrote:
>> > From: Honghui Zhang <honghui.zhang@mediatek.com>
>> >
>> > The MTCMOS of PCIe Host for MT2712 and MT7622 will be off when system
>> > suspend, and all the internal control register will be reset after system
>> > resume. The PCIe link should be re-established and the related control
>> > register values should be re-set after system resume.
>>
>> Sounds very familiar as what is implemented in
>> drivers/mmc/host/sdhci-xenon.c. Please have a look, possibly you can
>> get inspired to use the similar pattern.
>>
>> A few comments below. However, please note, I am not an expert in PCIe
>> so I may overlook some obvious things. I am relying on Lorenzo or some
>> other PCIe expert, to fill in.
>>
> Hi, Ulf, thanks very much for your comments, replied below.
>
>> >
>> > Signed-off-by: Honghui Zhang <honghui.zhang@mediatek.com>
>> > Acked-by: Ryder Lee <ryder.lee@mediatek.com>
>> > ---
>> >  drivers/pci/controller/pcie-mediatek.c | 67 ++++++++++++++++++++++++++++++++++
>> >  1 file changed, 67 insertions(+)
>> >
>> > diff --git a/drivers/pci/controller/pcie-mediatek.c b/drivers/pci/controller/pcie-mediatek.c
>> > index 86918d4..175d7b6 100644
>> > --- a/drivers/pci/controller/pcie-mediatek.c
>> > +++ b/drivers/pci/controller/pcie-mediatek.c
>> > @@ -134,12 +134,14 @@ struct mtk_pcie_port;
>> >  /**
>> >   * struct mtk_pcie_soc - differentiate between host generations
>> >   * @need_fix_class_id: whether this host's class ID needed to be fixed or not
>> > + * @pm_support: whether the host's MTCMOS will be off when suspend
>> >   * @ops: pointer to configuration access functions
>> >   * @startup: pointer to controller setting functions
>> >   * @setup_irq: pointer to initialize IRQ functions
>> >   */
>> >  struct mtk_pcie_soc {
>> >         bool need_fix_class_id;
>> > +       bool pm_support;
>> >         struct pci_ops *ops;
>> >         int (*startup)(struct mtk_pcie_port *port);
>> >         int (*setup_irq)(struct mtk_pcie_port *port, struct device_node *node);
>> > @@ -1197,12 +1199,75 @@ static int mtk_pcie_probe(struct platform_device *pdev)
>> >         return err;
>> >  }
>> >
>> > +static int __maybe_unused mtk_pcie_suspend_noirq(struct device *dev)
>>
>> I am not sure I understand why you need to suspend the device in the
>> "noirq" phase. Isn't it fine to do that in the regular suspend phase?
>
> PCIe device's configuration space values should be saved/restored in
> system suspend/resume flow, and PCIe framework will take care of that in
> the suspend_noirq phase. Suspend device in the "noirq" phase will spare
> device driver from that.

I see.

>
>>
>> > +{
>> > +       struct mtk_pcie *pcie = dev_get_drvdata(dev);
>> > +       const struct mtk_pcie_soc *soc = pcie->soc;
>> > +       struct mtk_pcie_port *port;
>> > +
>> > +       if (!soc->pm_support)
>> > +               return 0;
>> > +
>> > +       if (list_empty(&pcie->ports))
>> > +               return 0;
>> > +
>> > +       list_for_each_entry(port, &pcie->ports, list) {
>> > +               clk_disable_unprepare(port->pipe_ck);
>> > +               clk_disable_unprepare(port->obff_ck);
>> > +               clk_disable_unprepare(port->axi_ck);
>> > +               clk_disable_unprepare(port->aux_ck);
>> > +               clk_disable_unprepare(port->ahb_ck);
>> > +               clk_disable_unprepare(port->sys_ck);
>> > +               phy_power_off(port->phy);
>> > +               phy_exit(port->phy);
>> > +       }
>> > +
>> > +       mtk_pcie_subsys_powerdown(pcie);
>>
>> Why not gate clocks, unconditionally not depending on if pm_support is
>> set or not, during system suspend?
>>
>> I understand that you for some SoCs needs also to restore registers
>> during system resume, but that's a different thing, isn't it?
>>
>
> Well, current host driver support different SoCs like mt7623, mt7622 and
> mt2712. mt7623 need quite special take care of when system suspend. This
> patch just add the suspend/resume support for mt7622&mt2712, while
> mt7623 was excluded. This "pm_support" maybe removed after we have a
> solution for mt7623 SoC of the system suspend.

Okay. So you simply want to take small steps, that works!

>
>> > +
>> > +       return 0;
>> > +}
>> > +
>> > +static int __maybe_unused mtk_pcie_resume_noirq(struct device *dev)
>> > +{
>> > +       struct mtk_pcie *pcie = dev_get_drvdata(dev);
>> > +       const struct mtk_pcie_soc *soc = pcie->soc;
>> > +       struct mtk_pcie_port *port, *tmp;
>> > +
>> > +       if (!soc->pm_support)
>> > +               return 0;
>> > +
>> > +       if (list_empty(&pcie->ports))
>> > +               return 0;
>> > +
>> > +       if (dev->pm_domain) {
>> > +               pm_runtime_enable(dev);
>> > +               pm_runtime_get_sync(dev);
>>
>> I noticed, Lorenzo wanted me to comment on this, so here it goes.
>>
>> I guess the reason to why you make these pm_runtime_*() calls, is
>> because you want to restore the actions taken during
>> mtk_pcie_suspend_noirq() (which calls mtk_pcie_subsys_powerdown ->
>> pm_runtime_disable|put*())?
>
> Yes, that's what am I want to do.
>
>>
>> If that's the case, I would rather avoid calling pm_runtime_disable()
>> and pm_runtime_put() via mtk_pcie_suspend_noirq(), simply because it
>> isn't needed.
>
>> Of course, another option is to follow the pattern in
>> drivers/mmc/host/sdhci-xenon.c.
>>
>
> Thanks.
> I take a look at the drivers/mmc/sdhci-xenon.c's implement, it has add
> system suspend callbacks and set the device status to "PRM_SUSPENDED"
> through the pm_runtime_force_suspend() interface.
> I noticed that pm_runtime_put_sync will put the device status to
> "PRM_SUSPENDED" status in some certain condition.
> And I did not found the pci framework setting the status to
> "PRM_SUSPENDED". So I add the pm_runtime_put_sync/disable in
> suspend_noirq phase.
>
> According to my understanding of PM, the system suspend flow does not
> care about the runtime PM status. The runtime pm is responsible for the
> CMOS gate, if the device want to gated it's CMOS, it need to call the
> runtime pm interface obviously.

Well, during system suspend, the PM core disables runtime PM for all
devices. See __device_suspend_late().

Additionally, the PM core prevents devices from being runtime
suspended. See device_prepare(), as it calls
pm_runtime_get_noresume().

In other words, drivers can not rely on calling pm_runtime_put_sync()
from a ->suspend_noirq() callback to put its device into low power
state.

>
> But I found that the PCIe framework will still operate with the device
> after device driver suspend_noirq executed like save the configuration
> space values through pci_save_state, So I guess the cmos could not be
> gated at the suspend_noirq phase. I will update the patchset to remove
> the pm runtime operations in the suspend_noirq phase.

That's doesn't sound correct to me. Although, I am not a PCI expert.

>
>> Overall, I am also wondering why only runtime PM is used when there is
>> a PM domain attached to the device? That seems to make the logic in
>> the driver, unnecessary complicated. Why not use runtime
>> unconditionally and thus enable it during ->probe()?
>
> I'm not sure, I thought that if there's no "power-domains = " property
> in device node, that means the device has no PM domain. Some of the SoCs
> does not have independent CMOS domain. So does that means it will leave
> the dev->pm_domain as NULL or assign it as the genpd->pm_domain?

My point is, that no matter if there is a PM domain assigned or not,
the PCIe driver can still enable/use runtime PM. It shouldn't hurt.

>
> I need to do some homework to figure this out and will send a new patch
> to fix this if needed.

I did some more re-search and maybe the easiest thing for you to do is
to follow the pattern in the tegra PCIe controller driver.
drivers/pci/controller/pci-tegra.c.

That should definitely work for your case as well.

[...]

Kind regards
Uffe
Honghui Zhang Sept. 11, 2018, 10:23 a.m. | #8
On Mon, 2018-09-10 at 13:25 +0200, Ulf Hansson wrote:
> On 10 September 2018 at 11:42, Honghui Zhang <honghui.zhang@mediatek.com> wrote:
[...]

> >> > +               clk_disable_unprepare(port->sys_ck);
> >> > +               phy_power_off(port->phy);
> >> > +               phy_exit(port->phy);
> >> > +       }
> >> > +
> >> > +       mtk_pcie_subsys_powerdown(pcie);
> >>
> >> Why not gate clocks, unconditionally not depending on if pm_support is
> >> set or not, during system suspend?
> >>
> >> I understand that you for some SoCs needs also to restore registers
> >> during system resume, but that's a different thing, isn't it?
> >>
> >
> > Well, current host driver support different SoCs like mt7623, mt7622 and
> > mt2712. mt7623 need quite special take care of when system suspend. This
> > patch just add the suspend/resume support for mt7622&mt2712, while
> > mt7623 was excluded. This "pm_support" maybe removed after we have a
> > solution for mt7623 SoC of the system suspend.
> 
> Okay. So you simply want to take small steps, that works!
> 
> >
> >> > +
> >> > +       return 0;
> >> > +}
> >> > +
> >> > +static int __maybe_unused mtk_pcie_resume_noirq(struct device *dev)
> >> > +{
> >> > +       struct mtk_pcie *pcie = dev_get_drvdata(dev);
> >> > +       const struct mtk_pcie_soc *soc = pcie->soc;
> >> > +       struct mtk_pcie_port *port, *tmp;
> >> > +
> >> > +       if (!soc->pm_support)
> >> > +               return 0;
> >> > +
> >> > +       if (list_empty(&pcie->ports))
> >> > +               return 0;
> >> > +
> >> > +       if (dev->pm_domain) {
> >> > +               pm_runtime_enable(dev);
> >> > +               pm_runtime_get_sync(dev);
> >>
> >> I noticed, Lorenzo wanted me to comment on this, so here it goes.
> >>
> >> I guess the reason to why you make these pm_runtime_*() calls, is
> >> because you want to restore the actions taken during
> >> mtk_pcie_suspend_noirq() (which calls mtk_pcie_subsys_powerdown ->
> >> pm_runtime_disable|put*())?
> >
> > Yes, that's what am I want to do.
> >
> >>
> >> If that's the case, I would rather avoid calling pm_runtime_disable()
> >> and pm_runtime_put() via mtk_pcie_suspend_noirq(), simply because it
> >> isn't needed.
> >
> >> Of course, another option is to follow the pattern in
> >> drivers/mmc/host/sdhci-xenon.c.
> >>
> >
> > Thanks.
> > I take a look at the drivers/mmc/sdhci-xenon.c's implement, it has add
> > system suspend callbacks and set the device status to "PRM_SUSPENDED"
> > through the pm_runtime_force_suspend() interface.
> > I noticed that pm_runtime_put_sync will put the device status to
> > "PRM_SUSPENDED" status in some certain condition.
> > And I did not found the pci framework setting the status to
> > "PRM_SUSPENDED". So I add the pm_runtime_put_sync/disable in
> > suspend_noirq phase.
> >
> > According to my understanding of PM, the system suspend flow does not
> > care about the runtime PM status. The runtime pm is responsible for the
> > CMOS gate, if the device want to gated it's CMOS, it need to call the
> > runtime pm interface obviously.
> 
> Well, during system suspend, the PM core disables runtime PM for all
> devices. See __device_suspend_late().
> 
> Additionally, the PM core prevents devices from being runtime
> suspended. See device_prepare(), as it calls
> pm_runtime_get_noresume().
> 
> In other words, drivers can not rely on calling pm_runtime_put_sync()
> from a ->suspend_noirq() callback to put its device into low power
> state.
> 
Thanks for your explain, that's a great help for me with the
understanding the PM logical.

> >
> > But I found that the PCIe framework will still operate with the device
> > after device driver suspend_noirq executed like save the configuration
> > space values through pci_save_state, So I guess the cmos could not be
> > gated at the suspend_noirq phase. I will update the patchset to remove
> > the pm runtime operations in the suspend_noirq phase.
> 
> That's doesn't sound correct to me. Although, I am not a PCI expert.
> 
Never mind, per my understanding, after the driver's suspend_noirq
callback executed, the CMOS should not be gated, since the PCI framework
still got work to do with the device.

> >
> >> Overall, I am also wondering why only runtime PM is used when there is
> >> a PM domain attached to the device? That seems to make the logic in
> >> the driver, unnecessary complicated. Why not use runtime
> >> unconditionally and thus enable it during ->probe()?
> >
> > I'm not sure, I thought that if there's no "power-domains = " property
> > in device node, that means the device has no PM domain. Some of the SoCs
> > does not have independent CMOS domain. So does that means it will leave
> > the dev->pm_domain as NULL or assign it as the genpd->pm_domain?
> 
> My point is, that no matter if there is a PM domain assigned or not,
> the PCIe driver can still enable/use runtime PM. It shouldn't hurt.
> 

Thanks, I will try and propose another patch for this.

> >
> > I need to do some homework to figure this out and will send a new patch
> > to fix this if needed.
> 
> I did some more re-search and maybe the easiest thing for you to do is
> to follow the pattern in the tegra PCIe controller driver.
> drivers/pci/controller/pci-tegra.c.
> 
> That should definitely work for your case as well.

Thanks, most of the PCIe host driver does not set the RUNTIME_PM_OPS for
suspend except pci-tegra.c. So my V4 version patch has followed majority
driver.

> 
> [...]
> 
> Kind regards
> Uffe

Patch

diff --git a/drivers/pci/controller/pcie-mediatek.c b/drivers/pci/controller/pcie-mediatek.c
index 86918d4..175d7b6 100644
--- a/drivers/pci/controller/pcie-mediatek.c
+++ b/drivers/pci/controller/pcie-mediatek.c
@@ -134,12 +134,14 @@  struct mtk_pcie_port;
 /**
  * struct mtk_pcie_soc - differentiate between host generations
  * @need_fix_class_id: whether this host's class ID needed to be fixed or not
+ * @pm_support: whether the host's MTCMOS will be off when suspend
  * @ops: pointer to configuration access functions
  * @startup: pointer to controller setting functions
  * @setup_irq: pointer to initialize IRQ functions
  */
 struct mtk_pcie_soc {
 	bool need_fix_class_id;
+	bool pm_support;
 	struct pci_ops *ops;
 	int (*startup)(struct mtk_pcie_port *port);
 	int (*setup_irq)(struct mtk_pcie_port *port, struct device_node *node);
@@ -1197,12 +1199,75 @@  static int mtk_pcie_probe(struct platform_device *pdev)
 	return err;
 }
 
+static int __maybe_unused mtk_pcie_suspend_noirq(struct device *dev)
+{
+	struct mtk_pcie *pcie = dev_get_drvdata(dev);
+	const struct mtk_pcie_soc *soc = pcie->soc;
+	struct mtk_pcie_port *port;
+
+	if (!soc->pm_support)
+		return 0;
+
+	if (list_empty(&pcie->ports))
+		return 0;
+
+	list_for_each_entry(port, &pcie->ports, list) {
+		clk_disable_unprepare(port->pipe_ck);
+		clk_disable_unprepare(port->obff_ck);
+		clk_disable_unprepare(port->axi_ck);
+		clk_disable_unprepare(port->aux_ck);
+		clk_disable_unprepare(port->ahb_ck);
+		clk_disable_unprepare(port->sys_ck);
+		phy_power_off(port->phy);
+		phy_exit(port->phy);
+	}
+
+	mtk_pcie_subsys_powerdown(pcie);
+
+	return 0;
+}
+
+static int __maybe_unused mtk_pcie_resume_noirq(struct device *dev)
+{
+	struct mtk_pcie *pcie = dev_get_drvdata(dev);
+	const struct mtk_pcie_soc *soc = pcie->soc;
+	struct mtk_pcie_port *port, *tmp;
+
+	if (!soc->pm_support)
+		return 0;
+
+	if (list_empty(&pcie->ports))
+		return 0;
+
+	if (dev->pm_domain) {
+		pm_runtime_enable(dev);
+		pm_runtime_get_sync(dev);
+	}
+
+	clk_prepare_enable(pcie->free_ck);
+
+	list_for_each_entry_safe(port, tmp, &pcie->ports, list)
+		mtk_pcie_enable_port(port);
+
+	/* In case of EP was removed while system suspend. */
+	if (list_empty(&pcie->ports))
+		mtk_pcie_subsys_powerdown(pcie);
+
+	return 0;
+}
+
+static const struct dev_pm_ops mtk_pcie_pm_ops = {
+	SET_NOIRQ_SYSTEM_SLEEP_PM_OPS(mtk_pcie_suspend_noirq,
+				      mtk_pcie_resume_noirq)
+};
+
 static const struct mtk_pcie_soc mtk_pcie_soc_v1 = {
 	.ops = &mtk_pcie_ops,
 	.startup = mtk_pcie_startup_port,
 };
 
 static const struct mtk_pcie_soc mtk_pcie_soc_mt2712 = {
+	.pm_support = true,
 	.ops = &mtk_pcie_ops_v2,
 	.startup = mtk_pcie_startup_port_v2,
 	.setup_irq = mtk_pcie_setup_irq,
@@ -1210,6 +1275,7 @@  static const struct mtk_pcie_soc mtk_pcie_soc_mt2712 = {
 
 static const struct mtk_pcie_soc mtk_pcie_soc_mt7622 = {
 	.need_fix_class_id = true,
+	.pm_support = true,
 	.ops = &mtk_pcie_ops_v2,
 	.startup = mtk_pcie_startup_port_v2,
 	.setup_irq = mtk_pcie_setup_irq,
@@ -1229,6 +1295,7 @@  static struct platform_driver mtk_pcie_driver = {
 		.name = "mtk-pcie",
 		.of_match_table = mtk_pcie_ids,
 		.suppress_bind_attrs = true,
+		.pm = &mtk_pcie_pm_ops,
 	},
 };
 builtin_platform_driver(mtk_pcie_driver);