diff mbox series

[v8,3/7] PCI: qcom: Add ICC bandwidth vote for CPU to PCIe path

Message ID 20240302-opp_support-v8-3-158285b86b10@quicinc.com
State New
Headers show
Series PCI: qcom: Add support for OPP | expand

Commit Message

Krishna chaitanya chundru March 2, 2024, 3:59 a.m. UTC 
To access PCIe registers, PCIe BAR space, config space the CPU-PCIe
ICC (interconnect consumers) path should be voted otherwise it may
lead to NoC (Network on chip) timeout. We are surviving because of
other driver vote for this path.

As there is less access on this path compared to PCIe to mem path
add minimum vote i.e 1KBps bandwidth always.

When suspending, disable this path after register space access
is done.

Reviewed-by: Bryan O'Donoghue <bryan.odonoghue@linaro.org>
Signed-off-by: Krishna chaitanya chundru <quic_krichai@quicinc.com>
---
 drivers/pci/controller/dwc/pcie-qcom.c | 38 ++++++++++++++++++++++++++++++++--
 1 file changed, 36 insertions(+), 2 deletions(-)

Comments

Manivannan Sadhasivam March 4, 2024, 5:41 p.m. UTC | #1 
On Sat, Mar 02, 2024 at 09:29:57AM +0530, Krishna chaitanya chundru wrote:
> To access PCIe registers, PCIe BAR space, config space the CPU-PCIe
> ICC (interconnect consumers) path should be voted otherwise it may
> lead to NoC (Network on chip) timeout. We are surviving because of
> other driver vote for this path.
> 
> As there is less access on this path compared to PCIe to mem path
> add minimum vote i.e 1KBps bandwidth always.

Please add the info that 1KBps is what shared by the HW team.

> 
> When suspending, disable this path after register space access
> is done.
> 
> Reviewed-by: Bryan O'Donoghue <bryan.odonoghue@linaro.org>
> Signed-off-by: Krishna chaitanya chundru <quic_krichai@quicinc.com>
> ---
>  drivers/pci/controller/dwc/pcie-qcom.c | 38 ++++++++++++++++++++++++++++++++--
>  1 file changed, 36 insertions(+), 2 deletions(-)
> 
> diff --git a/drivers/pci/controller/dwc/pcie-qcom.c b/drivers/pci/controller/dwc/pcie-qcom.c
> index 10f2d0bb86be..a0266bfe71f1 100644
> --- a/drivers/pci/controller/dwc/pcie-qcom.c
> +++ b/drivers/pci/controller/dwc/pcie-qcom.c
> @@ -240,6 +240,7 @@ struct qcom_pcie {
>  	struct phy *phy;
>  	struct gpio_desc *reset;
>  	struct icc_path *icc_mem;
> +	struct icc_path *icc_cpu;
>  	const struct qcom_pcie_cfg *cfg;
>  	struct dentry *debugfs;
>  	bool suspended;
> @@ -1372,6 +1373,9 @@ static int qcom_pcie_icc_init(struct qcom_pcie *pcie)
>  	if (IS_ERR(pcie->icc_mem))
>  		return PTR_ERR(pcie->icc_mem);
>  
> +	pcie->icc_cpu = devm_of_icc_get(pci->dev, "cpu-pcie");
> +	if (IS_ERR(pcie->icc_cpu))
> +		return PTR_ERR(pcie->icc_cpu);
>  	/*
>  	 * Some Qualcomm platforms require interconnect bandwidth constraints
>  	 * to be set before enabling interconnect clocks.
> @@ -1381,7 +1385,19 @@ static int qcom_pcie_icc_init(struct qcom_pcie *pcie)
>  	 */
>  	ret = icc_set_bw(pcie->icc_mem, 0, QCOM_PCIE_LINK_SPEED_TO_BW(1));
>  	if (ret) {
> -		dev_err(pci->dev, "failed to set interconnect bandwidth: %d\n",
> +		dev_err(pci->dev, "failed to set interconnect bandwidth for pcie-mem: %d\n",

"PCIe-MEM"

> +			ret);
> +		return ret;
> +	}
> +
> +	/*
> +	 * The config space, BAR space and registers goes through cpu-pcie path
> +	 * Set peak bandwidth to 1KBps as recommended by HW team for this path
> +	 * all the time.

How about,

	"Since the CPU-PCIe path is only used for activities like register
	access, Config/BAR space access, HW team has recommended to use a
	minimal bandwidth of 1KBps just to keep the link active."

> +	 */
> +	ret = icc_set_bw(pcie->icc_cpu, 0, kBps_to_icc(1));
> +	if (ret) {
> +		dev_err(pci->dev, "failed to set interconnect bandwidth for cpu-pcie: %d\n",
>  			ret);
>  		return ret;
>  	}
> @@ -1573,7 +1589,7 @@ static int qcom_pcie_suspend_noirq(struct device *dev)
>  	 */
>  	ret = icc_set_bw(pcie->icc_mem, 0, kBps_to_icc(1));
>  	if (ret) {
> -		dev_err(dev, "Failed to set interconnect bandwidth: %d\n", ret);
> +		dev_err(dev, "Failed to set interconnect bandwidth for pcie-mem: %d\n", ret);

"PCIe-MEM"

>  		return ret;
>  	}
>  
> @@ -1597,6 +1613,18 @@ static int qcom_pcie_suspend_noirq(struct device *dev)
>  		pcie->suspended = true;
>  	}
>  
> +	/* Remove CPU path vote after all the register access is done */

"Remove the vote for CPU-PCIe path now, since at this point onwards, no register
access will be done."

> +	ret = icc_disable(pcie->icc_cpu);
> +	if (ret) {
> +		dev_err(dev, "failed to disable icc path of cpu-pcie: %d\n", ret);

"CPU-PCIe"

> +		if (pcie->suspended) {
> +			qcom_pcie_host_init(&pcie->pci->pp);

Interesting. So if icc_disable() fails, can the IP continue to function?

> +			pcie->suspended = false;
> +		}
> +		qcom_pcie_icc_update(pcie);
> +		return ret;
> +	}
> +
>  	return 0;
>  }
>  
> @@ -1605,6 +1633,12 @@ static int qcom_pcie_resume_noirq(struct device *dev)
>  	struct qcom_pcie *pcie = dev_get_drvdata(dev);
>  	int ret;
>  
> +	ret = icc_enable(pcie->icc_cpu);
> +	if (ret) {
> +		dev_err(dev, "failed to enable icc path of cpu-pcie: %d\n", ret);

"CPU-PCIe"

- Mani
Krishna chaitanya chundru March 5, 2024, 10:53 a.m. UTC | #2 
On 3/4/2024 11:11 PM, Manivannan Sadhasivam wrote:
> On Sat, Mar 02, 2024 at 09:29:57AM +0530, Krishna chaitanya chundru wrote:
>> To access PCIe registers, PCIe BAR space, config space the CPU-PCIe
>> ICC (interconnect consumers) path should be voted otherwise it may
>> lead to NoC (Network on chip) timeout. We are surviving because of
>> other driver vote for this path.
>>
>> As there is less access on this path compared to PCIe to mem path
>> add minimum vote i.e 1KBps bandwidth always.
> 
> Please add the info that 1KBps is what shared by the HW team.
> 
Ack to all the comments
>>
>> When suspending, disable this path after register space access
>> is done.
>>
>> Reviewed-by: Bryan O'Donoghue <bryan.odonoghue@linaro.org>
>> Signed-off-by: Krishna chaitanya chundru <quic_krichai@quicinc.com>
>> ---
>>   drivers/pci/controller/dwc/pcie-qcom.c | 38 ++++++++++++++++++++++++++++++++--
>>   1 file changed, 36 insertions(+), 2 deletions(-)
>>
>> diff --git a/drivers/pci/controller/dwc/pcie-qcom.c b/drivers/pci/controller/dwc/pcie-qcom.c
>> index 10f2d0bb86be..a0266bfe71f1 100644
>> --- a/drivers/pci/controller/dwc/pcie-qcom.c
>> +++ b/drivers/pci/controller/dwc/pcie-qcom.c
>> @@ -240,6 +240,7 @@ struct qcom_pcie {
>>   	struct phy *phy;
>>   	struct gpio_desc *reset;
>>   	struct icc_path *icc_mem;
>> +	struct icc_path *icc_cpu;
>>   	const struct qcom_pcie_cfg *cfg;
>>   	struct dentry *debugfs;
>>   	bool suspended;
>> @@ -1372,6 +1373,9 @@ static int qcom_pcie_icc_init(struct qcom_pcie *pcie)
>>   	if (IS_ERR(pcie->icc_mem))
>>   		return PTR_ERR(pcie->icc_mem);
>>   
>> +	pcie->icc_cpu = devm_of_icc_get(pci->dev, "cpu-pcie");
>> +	if (IS_ERR(pcie->icc_cpu))
>> +		return PTR_ERR(pcie->icc_cpu);
>>   	/*
>>   	 * Some Qualcomm platforms require interconnect bandwidth constraints
>>   	 * to be set before enabling interconnect clocks.
>> @@ -1381,7 +1385,19 @@ static int qcom_pcie_icc_init(struct qcom_pcie *pcie)
>>   	 */
>>   	ret = icc_set_bw(pcie->icc_mem, 0, QCOM_PCIE_LINK_SPEED_TO_BW(1));
>>   	if (ret) {
>> -		dev_err(pci->dev, "failed to set interconnect bandwidth: %d\n",
>> +		dev_err(pci->dev, "failed to set interconnect bandwidth for pcie-mem: %d\n",
> 
> "PCIe-MEM"
> 
>> +			ret);
>> +		return ret;
>> +	}
>> +
>> +	/*
>> +	 * The config space, BAR space and registers goes through cpu-pcie path
>> +	 * Set peak bandwidth to 1KBps as recommended by HW team for this path
>> +	 * all the time.
> 
> How about,
> 
> 	"Since the CPU-PCIe path is only used for activities like register
> 	access, Config/BAR space access, HW team has recommended to use a
> 	minimal bandwidth of 1KBps just to keep the link active."
> 
>> +	 */
>> +	ret = icc_set_bw(pcie->icc_cpu, 0, kBps_to_icc(1));
>> +	if (ret) {
>> +		dev_err(pci->dev, "failed to set interconnect bandwidth for cpu-pcie: %d\n",
>>   			ret);
>>   		return ret;
>>   	}
>> @@ -1573,7 +1589,7 @@ static int qcom_pcie_suspend_noirq(struct device *dev)
>>   	 */
>>   	ret = icc_set_bw(pcie->icc_mem, 0, kBps_to_icc(1));
>>   	if (ret) {
>> -		dev_err(dev, "Failed to set interconnect bandwidth: %d\n", ret);
>> +		dev_err(dev, "Failed to set interconnect bandwidth for pcie-mem: %d\n", ret);
> 
> "PCIe-MEM"
> 
>>   		return ret;
>>   	}
>>   
>> @@ -1597,6 +1613,18 @@ static int qcom_pcie_suspend_noirq(struct device *dev)
>>   		pcie->suspended = true;
>>   	}
>>   
>> +	/* Remove CPU path vote after all the register access is done */
> 
> "Remove the vote for CPU-PCIe path now, since at this point onwards, no register
> access will be done."
> 
>> +	ret = icc_disable(pcie->icc_cpu);
>> +	if (ret) {
>> +		dev_err(dev, "failed to disable icc path of cpu-pcie: %d\n", ret);
> 
> "CPU-PCIe"
> 
>> +		if (pcie->suspended) {
>> +			qcom_pcie_host_init(&pcie->pci->pp);
> 
> Interesting. So if icc_disable() fails, can the IP continue to function?
>
As the ICC already enable before icc_disable() fails, the IP should work.
  - Krishna Chaitanya.
>> +			pcie->suspended = false;
>> +		}
>> +		qcom_pcie_icc_update(pcie);
>> +		return ret;
>> +	}
>> +
>>   	return 0;
>>   }
>>   
>> @@ -1605,6 +1633,12 @@ static int qcom_pcie_resume_noirq(struct device *dev)
>>   	struct qcom_pcie *pcie = dev_get_drvdata(dev);
>>   	int ret;
>>   
>> +	ret = icc_enable(pcie->icc_cpu);
>> +	if (ret) {
>> +		dev_err(dev, "failed to enable icc path of cpu-pcie: %d\n", ret);
> 
> "CPU-PCIe"
> 
> - Mani
>
Manivannan Sadhasivam April 5, 2024, 8:29 a.m. UTC | #3 
On Tue, Mar 05, 2024 at 04:23:21PM +0530, Krishna Chaitanya Chundru wrote:
> 
> 
> On 3/4/2024 11:11 PM, Manivannan Sadhasivam wrote:
> > On Sat, Mar 02, 2024 at 09:29:57AM +0530, Krishna chaitanya chundru wrote:
> > > To access PCIe registers, PCIe BAR space, config space the CPU-PCIe
> > > ICC (interconnect consumers) path should be voted otherwise it may
> > > lead to NoC (Network on chip) timeout. We are surviving because of
> > > other driver vote for this path.
> > > 
> > > As there is less access on this path compared to PCIe to mem path
> > > add minimum vote i.e 1KBps bandwidth always.
> > 
> > Please add the info that 1KBps is what shared by the HW team.
> > 
> Ack to all the comments
> > > 
> > > When suspending, disable this path after register space access
> > > is done.
> > > 
> > > Reviewed-by: Bryan O'Donoghue <bryan.odonoghue@linaro.org>
> > > Signed-off-by: Krishna chaitanya chundru <quic_krichai@quicinc.com>
> > > ---
> > >   drivers/pci/controller/dwc/pcie-qcom.c | 38 ++++++++++++++++++++++++++++++++--
> > >   1 file changed, 36 insertions(+), 2 deletions(-)
> > > 
> > > diff --git a/drivers/pci/controller/dwc/pcie-qcom.c b/drivers/pci/controller/dwc/pcie-qcom.c
> > > index 10f2d0bb86be..a0266bfe71f1 100644
> > > --- a/drivers/pci/controller/dwc/pcie-qcom.c
> > > +++ b/drivers/pci/controller/dwc/pcie-qcom.c

[...]

> > > +	ret = icc_disable(pcie->icc_cpu);
> > > +	if (ret) {
> > > +		dev_err(dev, "failed to disable icc path of cpu-pcie: %d\n", ret);
> > 
> > "CPU-PCIe"
> > 
> > > +		if (pcie->suspended) {
> > > +			qcom_pcie_host_init(&pcie->pci->pp);
> > 
> > Interesting. So if icc_disable() fails, can the IP continue to function?
> > 
> As the ICC already enable before icc_disable() fails, the IP should work.

If icc_disable() fails, then most likely something is wrong with RPMh. How can
the IP continue to work in that case?

- Mani
Krishna chaitanya chundru April 5, 2024, 10:16 a.m. UTC | #4 
On 4/5/2024 1:59 PM, Manivannan Sadhasivam wrote:
> On Tue, Mar 05, 2024 at 04:23:21PM +0530, Krishna Chaitanya Chundru wrote:
>>
>>
>> On 3/4/2024 11:11 PM, Manivannan Sadhasivam wrote:
>>> On Sat, Mar 02, 2024 at 09:29:57AM +0530, Krishna chaitanya chundru wrote:
>>>> To access PCIe registers, PCIe BAR space, config space the CPU-PCIe
>>>> ICC (interconnect consumers) path should be voted otherwise it may
>>>> lead to NoC (Network on chip) timeout. We are surviving because of
>>>> other driver vote for this path.
>>>>
>>>> As there is less access on this path compared to PCIe to mem path
>>>> add minimum vote i.e 1KBps bandwidth always.
>>>
>>> Please add the info that 1KBps is what shared by the HW team.
>>>
>> Ack to all the comments
>>>>
>>>> When suspending, disable this path after register space access
>>>> is done.
>>>>
>>>> Reviewed-by: Bryan O'Donoghue <bryan.odonoghue@linaro.org>
>>>> Signed-off-by: Krishna chaitanya chundru <quic_krichai@quicinc.com>
>>>> ---
>>>>    drivers/pci/controller/dwc/pcie-qcom.c | 38 ++++++++++++++++++++++++++++++++--
>>>>    1 file changed, 36 insertions(+), 2 deletions(-)
>>>>
>>>> diff --git a/drivers/pci/controller/dwc/pcie-qcom.c b/drivers/pci/controller/dwc/pcie-qcom.c
>>>> index 10f2d0bb86be..a0266bfe71f1 100644
>>>> --- a/drivers/pci/controller/dwc/pcie-qcom.c
>>>> +++ b/drivers/pci/controller/dwc/pcie-qcom.c
> 
> [...]
> 
>>>> +	ret = icc_disable(pcie->icc_cpu);
>>>> +	if (ret) {
>>>> +		dev_err(dev, "failed to disable icc path of cpu-pcie: %d\n", ret);
>>>
>>> "CPU-PCIe"
>>>
>>>> +		if (pcie->suspended) {
>>>> +			qcom_pcie_host_init(&pcie->pci->pp);
>>>
>>> Interesting. So if icc_disable() fails, can the IP continue to function?
>>>
>> As the ICC already enable before icc_disable() fails, the IP should work.
> 
> If icc_disable() fails, then most likely something is wrong with RPMh. How can
> the IP continue to work in that case?
>
Ok then I will log the error and return here

- Krishna Chaitanya.

> - Mani
>
diff mbox series

Patch

diff --git a/drivers/pci/controller/dwc/pcie-qcom.c b/drivers/pci/controller/dwc/pcie-qcom.c
index 10f2d0bb86be..a0266bfe71f1 100644
--- a/drivers/pci/controller/dwc/pcie-qcom.c
+++ b/drivers/pci/controller/dwc/pcie-qcom.c
@@ -240,6 +240,7 @@  struct qcom_pcie {
 	struct phy *phy;
 	struct gpio_desc *reset;
 	struct icc_path *icc_mem;
+	struct icc_path *icc_cpu;
 	const struct qcom_pcie_cfg *cfg;
 	struct dentry *debugfs;
 	bool suspended;
@@ -1372,6 +1373,9 @@  static int qcom_pcie_icc_init(struct qcom_pcie *pcie)
 	if (IS_ERR(pcie->icc_mem))
 		return PTR_ERR(pcie->icc_mem);
 
+	pcie->icc_cpu = devm_of_icc_get(pci->dev, "cpu-pcie");
+	if (IS_ERR(pcie->icc_cpu))
+		return PTR_ERR(pcie->icc_cpu);
 	/*
 	 * Some Qualcomm platforms require interconnect bandwidth constraints
 	 * to be set before enabling interconnect clocks.
@@ -1381,7 +1385,19 @@  static int qcom_pcie_icc_init(struct qcom_pcie *pcie)
 	 */
 	ret = icc_set_bw(pcie->icc_mem, 0, QCOM_PCIE_LINK_SPEED_TO_BW(1));
 	if (ret) {
-		dev_err(pci->dev, "failed to set interconnect bandwidth: %d\n",
+		dev_err(pci->dev, "failed to set interconnect bandwidth for pcie-mem: %d\n",
+			ret);
+		return ret;
+	}
+
+	/*
+	 * The config space, BAR space and registers goes through cpu-pcie path
+	 * Set peak bandwidth to 1KBps as recommended by HW team for this path
+	 * all the time.
+	 */
+	ret = icc_set_bw(pcie->icc_cpu, 0, kBps_to_icc(1));
+	if (ret) {
+		dev_err(pci->dev, "failed to set interconnect bandwidth for cpu-pcie: %d\n",
 			ret);
 		return ret;
 	}
@@ -1573,7 +1589,7 @@  static int qcom_pcie_suspend_noirq(struct device *dev)
 	 */
 	ret = icc_set_bw(pcie->icc_mem, 0, kBps_to_icc(1));
 	if (ret) {
-		dev_err(dev, "Failed to set interconnect bandwidth: %d\n", ret);
+		dev_err(dev, "Failed to set interconnect bandwidth for pcie-mem: %d\n", ret);
 		return ret;
 	}
 
@@ -1597,6 +1613,18 @@  static int qcom_pcie_suspend_noirq(struct device *dev)
 		pcie->suspended = true;
 	}
 
+	/* Remove CPU path vote after all the register access is done */
+	ret = icc_disable(pcie->icc_cpu);
+	if (ret) {
+		dev_err(dev, "failed to disable icc path of cpu-pcie: %d\n", ret);
+		if (pcie->suspended) {
+			qcom_pcie_host_init(&pcie->pci->pp);
+			pcie->suspended = false;
+		}
+		qcom_pcie_icc_update(pcie);
+		return ret;
+	}
+
 	return 0;
 }
 
@@ -1605,6 +1633,12 @@  static int qcom_pcie_resume_noirq(struct device *dev)
 	struct qcom_pcie *pcie = dev_get_drvdata(dev);
 	int ret;
 
+	ret = icc_enable(pcie->icc_cpu);
+	if (ret) {
+		dev_err(dev, "failed to enable icc path of cpu-pcie: %d\n", ret);
+		return ret;
+	}
+
 	if (pcie->suspended) {
 		ret = qcom_pcie_host_init(&pcie->pci->pp);
 		if (ret)