[1/2] phb4: set PHB CMPM registers for tunneled operations

Message ID 20171023130830.18818-1-felix@linux.vnet.ibm.com
State New
Headers show
Series
  • [1/2] phb4: set PHB CMPM registers for tunneled operations
Related show

Commit Message

Philippe Bergheaud Oct. 23, 2017, 1:08 p.m.
P9 supports PCI tunneled operations (atomics and as_notify) that require
setting the PHB ASN Compare/Mask register with a 16-bit indication.

This register is currently initialized by enable_capi_mode(). As tunneled
operations may also operate in PCI mode, a new API is required to set the
PHB ASN register without switching to CAPI mode.

This patch provides two new OPAL calls to get/set any of the three PHB
CMPM registers that obey the same logic (ASN, NBW, and CAPI).

This new API aims at letting the operating system choose indications, that
may change in the future. Compatibility with older kernel versions is made
by enable_capi_mode().

Signed-off-by: Philippe Bergheaud <felix@linux.vnet.ibm.com>
---
 core/pci-opal.c                                    |  36 ++++++
 .../opal-pci-get-set-pbcq-tunnel-bar-160-161.rst   |  76 +++++++++++++
 doc/opal-api/opal-pci-get-set-phb-cmpm-158-159.rst | 100 ++++++++++++++++
 hw/phb4.c                                          | 126 +++++++++++++++++++--
 include/opal-api.h                                 |  11 +-
 include/pci.h                                      |   4 +
 include/phb4-regs.h                                |   8 +-
 7 files changed, 345 insertions(+), 16 deletions(-)
 create mode 100644 doc/opal-api/opal-pci-get-set-pbcq-tunnel-bar-160-161.rst
 create mode 100644 doc/opal-api/opal-pci-get-set-phb-cmpm-158-159.rst

Comments

christophe lombard Oct. 24, 2017, 9:20 a.m. | #1
Le 23/10/2017 à 15:08, Philippe Bergheaud a écrit :

> P9 supports PCI tunneled operations (atomics and as_notify) that require
> setting the PHB ASN Compare/Mask register with a 16-bit indication.
>
> This register is currently initialized by enable_capi_mode(). As tunneled
> operations may also operate in PCI mode, a new API is required to set the
> PHB ASN register without switching to CAPI mode.
>
> This patch provides two new OPAL calls to get/set any of the three PHB
> CMPM registers that obey the same logic (ASN, NBW, and CAPI).
>
> This new API aims at letting the operating system choose indications, that
> may change in the future. Compatibility with older kernel versions is made
> by enable_capi_mode().
>
> Signed-off-by: Philippe Bergheaud <felix@linux.vnet.ibm.com>
> ---
>
Sounds good.

Acked-by:  Christophe Lombard<clombard@linux.vnet.ibm.com>

Thanks
Philippe Bergheaud Oct. 26, 2017, 1:43 p.m. | #2
On 23/10/2017 15:08, Philippe Bergheaud wrote:
> P9 supports PCI tunneled operations (atomics and as_notify) that require
> setting the PHB ASN Compare/Mask register with a 16-bit indication.
>
> This register is currently initialized by enable_capi_mode(). As tunneled
> operations may also operate in PCI mode, a new API is required to set the
> PHB ASN register without switching to CAPI mode.
>
> This patch provides two new OPAL calls to get/set any of the three PHB
> CMPM registers that obey the same logic (ASN, NBW, and CAPI).
>
> This new API aims at letting the operating system choose indications, that
> may change in the future. Compatibility with older kernel versions is made
> by enable_capi_mode().
>
> Signed-off-by: Philippe Bergheaud <felix@linux.vnet.ibm.com>
> ---

Linux patch:
https://patchwork.ozlabs.org/patch/830231/

>   core/pci-opal.c                                    |  36 ++++++
>   .../opal-pci-get-set-pbcq-tunnel-bar-160-161.rst   |  76 +++++++++++++
>   doc/opal-api/opal-pci-get-set-phb-cmpm-158-159.rst | 100 ++++++++++++++++
>   hw/phb4.c                                          | 126 +++++++++++++++++++--
>   include/opal-api.h                                 |  11 +-
>   include/pci.h                                      |   4 +
>   include/phb4-regs.h                                |   8 +-
>   7 files changed, 345 insertions(+), 16 deletions(-)
>   create mode 100644 doc/opal-api/opal-pci-get-set-pbcq-tunnel-bar-160-161.rst
>   create mode 100644 doc/opal-api/opal-pci-get-set-phb-cmpm-158-159.rst
>
> diff --git a/core/pci-opal.c b/core/pci-opal.c
> index b8aec941..bbbbcd5d 100644
> --- a/core/pci-opal.c
> +++ b/core/pci-opal.c
> @@ -1016,3 +1016,39 @@ static int64_t opal_pci_set_p2p(uint64_t phbid_init, uint64_t phbid_target,
>   	return OPAL_SUCCESS;
>   }
>   opal_call(OPAL_PCI_SET_P2P, opal_pci_set_p2p, 4);
> +
> +static int64_t opal_pci_get_phb_cmpm(uint64_t phb_id, uint64_t phb_reg,
> +				     uint64_t *ind)
> +{
> +	struct phb *phb = pci_get_phb(phb_id);
> +	int64_t rc;
> +
> +	if (!phb)
> +		return OPAL_PARAMETER;
> +	if (!phb->ops->get_cmpm)
> +		return OPAL_UNSUPPORTED;
> +
> +	phb_lock(phb);
> +	rc = phb->ops->get_cmpm(phb, phb_reg, ind);
> +	phb_unlock(phb);
> +	return rc;
> +}
> +opal_call(OPAL_PCI_GET_PHB_CMPM, opal_pci_get_phb_cmpm, 3);
> +
> +static int64_t opal_pci_set_phb_cmpm(uint64_t phb_id, uint64_t phb_reg,
> +				     uint64_t ind)
> +{
> +	struct phb *phb = pci_get_phb(phb_id);
> +	int64_t rc;
> +
> +	if (!phb)
> +		return OPAL_PARAMETER;
> +	if (!phb->ops->set_cmpm)
> +		return OPAL_UNSUPPORTED;
> +
> +	phb_lock(phb);
> +	rc = phb->ops->set_cmpm(phb, phb_reg, ind);
> +	phb_unlock(phb);
> +	return rc;
> +}
> +opal_call(OPAL_PCI_SET_PHB_CMPM, opal_pci_set_phb_cmpm, 3);
> diff --git a/doc/opal-api/opal-pci-get-set-pbcq-tunnel-bar-160-161.rst b/doc/opal-api/opal-pci-get-set-pbcq-tunnel-bar-160-161.rst
> new file mode 100644
> index 00000000..0af06774
> --- /dev/null
> +++ b/doc/opal-api/opal-pci-get-set-pbcq-tunnel-bar-160-161.rst
> @@ -0,0 +1,76 @@
> +OPAL_PCI_GET_PBCQ_TUNNEL_BAR
> +============================
> +::
> +
> +   #define OPAL_PCI_GET_PBCQ_TUNNEL_BAR 160
> +
> +   int64_t opal_pci_get_pbcq_tunnel_bar(uint64_t phb_id, uint64_t *val)
> +
> +The host calls this function to read the address out of the PBCQ Tunnel
> +Bar register.
> +
> +Parameters
> +----------
> +::
> +
> +   uint64_t phb_id
> +   uint64_t *val
> +
> +``phb_id``
> +  The value from the PHB node ibm,opal-phbid property for the device.
> +
> +``val``
> +  A pointer to the address where the address store in the PBCQ Tunnel Bar
> +  register will be copied.
> +
> +Return Values
> +-------------
> +
> +``OPAL_SUCCESS``
> +  Operation was successful
> +
> +``OPAL_PARAMETER``
> +  Invalid PHB
> +
> +``OPAL_UNSUPPORTED``
> +  Not supported by hardware
> +
> +OPAL_PCI_SET_PBCQ_TUNNEL_BAR
> +============================
> +::
> +
> +   #define OPAL_PCI_SET_PBCQ_TUNNEL_BAR 161
> +
> +   int64_t opal_pci_set_pbcq_tunnel_bar(uint64_t phb_id, uint64_t addr)
> +
> +The host calls this function to set the PBCQ Tunnel Bar register.
> +
> +Parameters
> +----------
> +::
> +
> +   uint64_t phb_id
> +   uint64_t addr
> +
> +``phb_id``
> +  The value from the PHB node ibm,opal-phbid property for the device.
> +
> +``addr``
> +  The value of the address chosen for the PBCQ Tunnel Bar register.
> +  If the address is 0, then the PBCQ Tunnel Bar register will be reset.
> +  It the address is non-zero, then the PBCQ Tunnel Bar register will be
> +  set with ::
> +
> +   Bit[0:42]      Bit[8:50] of the address
> +
> +Return Values
> +-------------
> +
> +``OPAL_SUCCESS``
> +  Operation was successful
> +
> +``OPAL_PARAMETER``
> +  Invalid PHB or addr parameter
> +
> +``OPAL_UNSUPPORTED``
> +  Not supported by hardware
> diff --git a/doc/opal-api/opal-pci-get-set-phb-cmpm-158-159.rst b/doc/opal-api/opal-pci-get-set-phb-cmpm-158-159.rst
> new file mode 100644
> index 00000000..7eabdde3
> --- /dev/null
> +++ b/doc/opal-api/opal-pci-get-set-phb-cmpm-158-159.rst
> @@ -0,0 +1,100 @@
> +OPAL_PCI_GET_PHB_CMPM
> +=====================
> +::
> +
> +   #define OPAL_PCI_GET_PHB_CMPM 158
> +
> +   int64_t opal_pci_get_phb_cmpm(uint64_t phb_id, uint64_t phb_reg,
> +                                 uint64_t *val)
> +
> +The host calls this function to read the indication out of a CMPM register
> +of the PHB.
> +
> +Parameters
> +----------
> +::
> +
> +   uint64_t phb_id
> +   uint64_t phb_reg
> +   uint64_t *val
> +
> +``phb_id``
> +  The value from the PHB node ibm,opal-phbid property for the device.
> +
> +``phb_reg``
> +  Which CMPM register should be accessed, i.e. one of ::
> +
> +   /* PHB Compare/Mask registers */
> +   enum {
> +        OPAL_PHB_ASN_CMPM       = 0,
> +        OPAL_PHB_CAPI_CMPM      = 1,
> +        OPAL_PHB_PBL_NBW_CMPM   = 2,
> +   };
> +
> +``val``
> +  A pointer to the address where the indication of the PHB CMPM register
> +  will be copied.
> +
> +Return Values
> +-------------
> +
> +``OPAL_SUCCESS``
> +  Operation was successful
> +
> +``OPAL_PARAMETER``
> +  Invalid PHB or phb_reg parameter
> +
> +``OPAL_UNSUPPORTED``
> +  Not supported by hardware
> +
> +OPAL_PCI_SET_PHB_CMPM
> +=====================
> +::
> +
> +   #define OPAL_PCI_SET_PHB_CMPM 159
> +
> +   int64_t opal_pci_set_phb_cmpm(uint64_t phb_id, uint64_t phb_reg,
> +                                 uint64_t ind)
> +
> +The host calls this function to set a CMPM register of the PHB.
> +
> +Parameters
> +----------
> +::
> +
> +   uint64_t phb_id
> +   uint64_t phb_reg
> +   uint64_t ind
> +
> +``phb_id``
> +  The value from the PHB node ibm,opal-phbid property for the device.
> +
> +``phb_reg``
> +  Which CMPM register should be accessed, i.e. one of ::
> +
> +   /* PHB Compare/Mask registers */
> +   enum {
> +        OPAL_PHB_ASN_CMPM       = 0,
> +        OPAL_PHB_CAPI_CMPM      = 1,
> +        OPAL_PHB_PBL_NBW_CMPM   = 2,
> +   };
> +
> +``ind``
> +  The value of the indication chosen for the PHB CMPM register.
> +  The PHB CMPM register will be set with ::
> +
> +   Bit[0:15]      the indication value
> +   Bit[16:31]     the (hard-coded) mask value
> +   Bit[63]        the enable bit
> +
> +Return Values
> +-------------
> +
> +``OPAL_SUCCESS``
> +  Operation was successful
> +
> +``OPAL_PARAMETER``
> +  Invalid PHB, phb_reg, or ind parameter
> +
> +``OPAL_UNSUPPORTED``
> +  Not supported by hardware
> diff --git a/hw/phb4.c b/hw/phb4.c
> index c64ded92..1add8a75 100644
> --- a/hw/phb4.c
> +++ b/hw/phb4.c
> @@ -3731,16 +3731,25 @@ static int64_t enable_capi_mode(struct phb4 *p, uint64_t pe_number,
>   	 */
>
>   	/*
> -	 * Bit [0:7] XSL_DSNCTL[capiind]
> -	 * Init_25 - CAPI Compare/Mask
> +	 * Init_26 - CAPI Compare/Mask
> +	 * If unset, then capiind=0x0200.
>   	 */
> -	out_be64(p->regs + PHB_CAPI_CMPM,
> -		 0x0200FE0000000000Ull | PHB_CAPI_CMPM_ENABLE);
> +	reg = in_be64(p->regs + PHB_CAPI_CMPM);
> +	if (!reg) {
> +		reg = 0x0200fe0000000000ull | PHB_CAPI_CMPM_ENABLE;
> +		out_be64(p->regs + PHB_CAPI_CMPM, reg);
> +	}
>
>   	if (!(p->rev == PHB4_REV_NIMBUS_DD10)) {
> -		/* Init_24 - ASN Compare/Mask */
> -		out_be64(p->regs + PHB_PBL_ASN_CMPM,
> -			 0x0400FF0000000000Ull | PHB_PBL_ASN_ENABLE);
> +		/*
> +		 * Init_25 - ASN Compare/Mask
> +		 * If unset, then asnind=0x0400.
> +		 */
> +		reg = in_be64(p->regs + PHB_ASN_CMPM);
> +		if (!reg) {
> +			reg = 0x0400ff0000000000ull | PHB_ASN_CMPM_ENABLE;
> +			out_be64(p->regs + PHB_ASN_CMPM, reg);
> +		}
>
>   		/* PBCQ Tunnel Bar Register
>   		 * Write Tunnel register to match PSL TNR register
> @@ -3990,6 +3999,99 @@ static int64_t phb4_set_capp_recovery(struct phb *phb)
>   	return 0;
>   }
>
> +/*
> + * Return the indication of a PHB CMPM register.
> + */
> +static int64_t phb4_get_cmpm(struct phb *phb, uint64_t phb_reg, uint64_t *ind)
> +{
> +	struct phb4 *p = phb_to_phb4(phb);
> +	uint64_t offset;
> +
> +	switch (phb_reg) {
> +	case OPAL_PHB_ASN_CMPM:
> +		offset = PHB_ASN_CMPM;
> +		break;
> +	case OPAL_PHB_CAPI_CMPM:
> +		offset = PHB_CAPI_CMPM;
> +		break;
> +	case OPAL_PHB_PBL_NBW_CMPM:
> +		offset = PHB_PBL_NBW_CMPM;
> +		break;
> +	default:
> +		return OPAL_UNSUPPORTED;
> +	}
> +
> +	*ind = in_be64(p->regs + offset) >> 48;
> +	return OPAL_SUCCESS;
> +}
> +
> +/*
> + * Set the 16-bit indication of a PHB CMPM register.
> + * Set (hard-coded) mask value, and enable bit.
> + *
> + * This interface aims at letting linux choose the indications,
> + * if they need to be changed in the future.
> + *
> + * Compatibility with older versions of linux, that do not set CMPM
> + * indications with phb4_set_cmpm(), is ensured by enable_capi_mode(),
> + * that will set the default hard-coded values that were assumed then.
> + */
> +static int64_t phb4_set_cmpm(struct phb *phb, uint64_t phb_reg, uint64_t ind)
> +{
> +	struct phb4 *p = phb_to_phb4(phb);
> +	uint64_t mask, offset, enable;
> +
> +	/* Indication is a 16-bit value */
> +	if (ind >> 16)
> +		return OPAL_PARAMETER;
> +	/*
> +	 * The following bits of the PCI address are reserved:
> +	 *   Bit 59 indicates TVE#1
> +	 *   Bit 60 indicates MSI-X
> +	 */
> +	if (ind & 0x1800ull)
> +		return OPAL_PARAMETER;
> +
> +	switch (phb_reg) {
> +	case OPAL_PHB_ASN_CMPM:
> +		/*
> +		 * Init_25 - ASN Compare/Mask
> +		 * matches XSL_DSNCTL[ASB_Addr]
> +		 */
> +		offset = PHB_ASN_CMPM;
> +		enable = PHB_ASN_CMPM_ENABLE;
> +		mask = 0xff00ull;
> +		break;
> +	case OPAL_PHB_CAPI_CMPM:
> +		/*
> +		 * Init_26 - CAPI Compare/Mask
> +		 * matches XSL_DSNCTL[capiind]
> +		 */
> +		offset = PHB_CAPI_CMPM;
> +		enable = PHB_CAPI_CMPM_ENABLE;
> +		mask = 0xfe00ull;
> +		break;
> +	case OPAL_PHB_PBL_NBW_CMPM:
> +		/*
> +		 * Init_124 - NBW Compare/Mask
> +		 * matches XSL_DSNCTL[nbwind]
> +		 */
> +		offset = PHB_PBL_NBW_CMPM;
> +		enable = PHB_PBL_NBW_CMPM_ENABLE;
> +		mask = 0xff00ull;
> +		break;
> +	default:
> +		return OPAL_UNSUPPORTED;
> +	}
> +
> +	/* Masked indication should not be null */
> +	if (!(ind & mask))
> +		return OPAL_PARAMETER;
> +
> +	out_be64(p->regs + offset, (ind << 48) | (mask << 32) | enable);
> +	return OPAL_SUCCESS;
> +}
> +
>   static const struct phb_ops phb4_ops = {
>   	.cfg_read8		= phb4_pcicfg_read8,
>   	.cfg_read16		= phb4_pcicfg_read16,
> @@ -4025,6 +4127,8 @@ static const struct phb_ops phb4_ops = {
>   	.set_capi_mode		= phb4_set_capi_mode,
>   	.set_p2p		= phb4_set_p2p,
>   	.set_capp_recovery	= phb4_set_capp_recovery,
> +	.get_cmpm		= phb4_get_cmpm,
> +	.set_cmpm		= phb4_set_cmpm,
>   };
>
>   static void phb4_init_ioda3(struct phb4 *p)
> @@ -4059,10 +4163,10 @@ static void phb4_init_ioda3(struct phb4 *p)
>   	/* See enable_capi_mode() */
>
>   	/* Init_25 - ASN Compare/Mask */
> -	/* See enable_capi_mode() */
> +	/* See phb4_set_cmpm() and enable_capi_mode() */
>
>   	/* Init_26 - CAPI Compare/Mask */
> -	/* See enable_capi_mode() */
> +	/* See phb4_set_cmpm() and enable_capi_mode() */
>
>   	/* Init_27 - PCIE Outbound upper address */
>   	out_be64(p->regs + PHB_M64_UPPER_BITS, 0);
> @@ -4412,8 +4516,8 @@ static void phb4_init_hw(struct phb4 *p, bool first_init)
>   	 * don't bother as we are doing a PERST soon.
>   	 */
>
> -	/* Init_124 :  NBW. XXX TODO */
> -	/* See enable_capi_mode() */
> +	/* Init_124 - NBW Compare/Mask */
> +	/* See phb4_set_cmpm() and enable_capi_mode() */
>
>   	/* Init_125 : Setup PCI command/status on root complex
>   	 * I don't know why the spec does this now and not earlier, so
> diff --git a/include/opal-api.h b/include/opal-api.h
> index 0bc036ed..ce948136 100644
> --- a/include/opal-api.h
> +++ b/include/opal-api.h
> @@ -214,7 +214,9 @@
>   #define OPAL_SET_POWER_SHIFT_RATIO		155
>   #define OPAL_SENSOR_GROUP_CLEAR			156
>   #define OPAL_PCI_SET_P2P			157
> -#define OPAL_LAST				157
> +#define OPAL_PCI_GET_PHB_CMPM			158
> +#define OPAL_PCI_SET_PHB_CMPM			159
> +#define OPAL_LAST				159
>
>   /* Device tree flags */
>
> @@ -1274,6 +1276,13 @@ enum {
>   	OPAL_PCI_P2P_TARGET	= 1,
>   };
>
> +/* PHB Compare/Mask registers */
> +enum {
> +	OPAL_PHB_ASN_CMPM	= 0,
> +	OPAL_PHB_CAPI_CMPM	= 1,
> +	OPAL_PHB_PBL_NBW_CMPM	= 2,
> +};
> +
>   #endif /* __ASSEMBLY__ */
>
>   #endif /* __OPAL_API_H */
> diff --git a/include/pci.h b/include/pci.h
> index c085b6b8..cdf82ee8 100644
> --- a/include/pci.h
> +++ b/include/pci.h
> @@ -333,6 +333,10 @@ struct phb_ops {
>   	/* PCI peer-to-peer setup */
>   	void (*set_p2p)(struct phb *phb, uint64_t mode, uint64_t flags,
>   			uint16_t pe_number);
> +
> +	/* Get/set PHB Compare/Mask registers */
> +	int64_t (*get_cmpm)(struct phb *phb, uint64_t phb_reg, uint64_t *ind);
> +	int64_t (*set_cmpm)(struct phb *phb, uint64_t phb_reg, uint64_t ind);
>   };
>
>   enum phb_type {
> diff --git a/include/phb4-regs.h b/include/phb4-regs.h
> index e83c8c39..b33185ed 100644
> --- a/include/phb4-regs.h
> +++ b/include/phb4-regs.h
> @@ -71,8 +71,8 @@
>   #define PHB_PEST_BAR			0x1a8
>   #define   PHB_PEST_BAR_ENABLE		PPC_BIT(0)
>   #define   PHB_PEST_BASE_ADDRESS		PPC_BITMASK(8,51)
> -#define PHB_PBL_ASN_CMPM		0x1C0
> -#define   PHB_PBL_ASN_ENABLE		PPC_BIT(63)
> +#define PHB_ASN_CMPM			0x1C0
> +#define   PHB_ASN_CMPM_ENABLE		PPC_BIT(63)
>   #define PHB_CAPI_CMPM			0x1C8
>   #define   PHB_CAPI_CMPM_ENABLE		PPC_BIT(63)
>   #define PHB_M64_UPPER_BITS		0x1f0
> @@ -250,8 +250,8 @@
>   #define PHB_PBL_CONTROL				0x1800
>   #define PHB_PBL_TIMEOUT_CTRL			0x1810
>   #define PHB_PBL_NPTAG_ENABLE			0x1820
> -#define PHB_PBL_NBW_CMP_MASK			0x1830
> -#define   PHB_PBL_NBW_MASK_ENABLE		PPC_BIT(63)
> +#define PHB_PBL_NBW_CMPM			0x1830
> +#define   PHB_PBL_NBW_CMPM_ENABLE		PPC_BIT(63)
>   #define PHB_PBL_SYS_LINK_INIT			0x1838
>   #define PHB_PBL_BUF_STATUS			0x1840
>   #define PHB_PBL_ERR_STATUS			0x1900
Benjamin Herrenschmidt Nov. 2, 2017, 4:03 a.m. | #3
On Thu, 2017-10-26 at 15:43 +0200, Philippe Bergheaud wrote:
> On 23/10/2017 15:08, Philippe Bergheaud wrote:
> > P9 supports PCI tunneled operations (atomics and as_notify) that require
> > setting the PHB ASN Compare/Mask register with a 16-bit indication.
> > 
> > This register is currently initialized by enable_capi_mode(). As tunneled
> > operations may also operate in PCI mode, a new API is required to set the
> > PHB ASN register without switching to CAPI mode.
> > 
> > This patch provides two new OPAL calls to get/set any of the three PHB
> > CMPM registers that obey the same logic (ASN, NBW, and CAPI).
> > 
> > This new API aims at letting the operating system choose indications, that
> > may change in the future. Compatibility with older kernel versions is made
> > by enable_capi_mode().
> > 
> > Signed-off-by: Philippe Bergheaud <felix@linux.vnet.ibm.com>
> > ---
> 
> Linux patch:
> https://patchwork.ozlabs.org/patch/830231/

Any reason why the BAR value is under Linux control ? Can't we just
pick a range that's outside of the MMIO, DMA and MSI windows (some
top bits set) and be done with it ? (ie. just *tell* Linux where we put
it).

> >   core/pci-opal.c                                    |  36 ++++++
> >   .../opal-pci-get-set-pbcq-tunnel-bar-160-161.rst   |  76 +++++++++++++
> >   doc/opal-api/opal-pci-get-set-phb-cmpm-158-159.rst | 100 ++++++++++++++++
> >   hw/phb4.c                                          | 126 +++++++++++++++++++--
> >   include/opal-api.h                                 |  11 +-
> >   include/pci.h                                      |   4 +
> >   include/phb4-regs.h                                |   8 +-
> >   7 files changed, 345 insertions(+), 16 deletions(-)
> >   create mode 100644 doc/opal-api/opal-pci-get-set-pbcq-tunnel-bar-160-161.rst
> >   create mode 100644 doc/opal-api/opal-pci-get-set-phb-cmpm-158-159.rst
> > 
> > diff --git a/core/pci-opal.c b/core/pci-opal.c
> > index b8aec941..bbbbcd5d 100644
> > --- a/core/pci-opal.c
> > +++ b/core/pci-opal.c
> > @@ -1016,3 +1016,39 @@ static int64_t opal_pci_set_p2p(uint64_t phbid_init, uint64_t phbid_target,
> >   	return OPAL_SUCCESS;
> >   }
> >   opal_call(OPAL_PCI_SET_P2P, opal_pci_set_p2p, 4);
> > +
> > +static int64_t opal_pci_get_phb_cmpm(uint64_t phb_id, uint64_t phb_reg,
> > +				     uint64_t *ind)
> > +{
> > +	struct phb *phb = pci_get_phb(phb_id);
> > +	int64_t rc;
> > +
> > +	if (!phb)
> > +		return OPAL_PARAMETER;
> > +	if (!phb->ops->get_cmpm)
> > +		return OPAL_UNSUPPORTED;
> > +
> > +	phb_lock(phb);
> > +	rc = phb->ops->get_cmpm(phb, phb_reg, ind);
> > +	phb_unlock(phb);
> > +	return rc;
> > +}
> > +opal_call(OPAL_PCI_GET_PHB_CMPM, opal_pci_get_phb_cmpm, 3);
> > +
> > +static int64_t opal_pci_set_phb_cmpm(uint64_t phb_id, uint64_t phb_reg,
> > +				     uint64_t ind)
> > +{
> > +	struct phb *phb = pci_get_phb(phb_id);
> > +	int64_t rc;
> > +
> > +	if (!phb)
> > +		return OPAL_PARAMETER;
> > +	if (!phb->ops->set_cmpm)
> > +		return OPAL_UNSUPPORTED;
> > +
> > +	phb_lock(phb);
> > +	rc = phb->ops->set_cmpm(phb, phb_reg, ind);
> > +	phb_unlock(phb);
> > +	return rc;
> > +}
> > +opal_call(OPAL_PCI_SET_PHB_CMPM, opal_pci_set_phb_cmpm, 3);
> > diff --git a/doc/opal-api/opal-pci-get-set-pbcq-tunnel-bar-160-161.rst b/doc/opal-api/opal-pci-get-set-pbcq-tunnel-bar-160-161.rst
> > new file mode 100644
> > index 00000000..0af06774
> > --- /dev/null
> > +++ b/doc/opal-api/opal-pci-get-set-pbcq-tunnel-bar-160-161.rst
> > @@ -0,0 +1,76 @@
> > +OPAL_PCI_GET_PBCQ_TUNNEL_BAR
> > +============================
> > +::
> > +
> > +   #define OPAL_PCI_GET_PBCQ_TUNNEL_BAR 160
> > +
> > +   int64_t opal_pci_get_pbcq_tunnel_bar(uint64_t phb_id, uint64_t *val)
> > +
> > +The host calls this function to read the address out of the PBCQ Tunnel
> > +Bar register.
> > +
> > +Parameters
> > +----------
> > +::
> > +
> > +   uint64_t phb_id
> > +   uint64_t *val
> > +
> > +``phb_id``
> > +  The value from the PHB node ibm,opal-phbid property for the device.
> > +
> > +``val``
> > +  A pointer to the address where the address store in the PBCQ Tunnel Bar
> > +  register will be copied.
> > +
> > +Return Values
> > +-------------
> > +
> > +``OPAL_SUCCESS``
> > +  Operation was successful
> > +
> > +``OPAL_PARAMETER``
> > +  Invalid PHB
> > +
> > +``OPAL_UNSUPPORTED``
> > +  Not supported by hardware
> > +
> > +OPAL_PCI_SET_PBCQ_TUNNEL_BAR
> > +============================
> > +::
> > +
> > +   #define OPAL_PCI_SET_PBCQ_TUNNEL_BAR 161
> > +
> > +   int64_t opal_pci_set_pbcq_tunnel_bar(uint64_t phb_id, uint64_t addr)
> > +
> > +The host calls this function to set the PBCQ Tunnel Bar register.
> > +
> > +Parameters
> > +----------
> > +::
> > +
> > +   uint64_t phb_id
> > +   uint64_t addr
> > +
> > +``phb_id``
> > +  The value from the PHB node ibm,opal-phbid property for the device.
> > +
> > +``addr``
> > +  The value of the address chosen for the PBCQ Tunnel Bar register.
> > +  If the address is 0, then the PBCQ Tunnel Bar register will be reset.
> > +  It the address is non-zero, then the PBCQ Tunnel Bar register will be
> > +  set with ::
> > +
> > +   Bit[0:42]      Bit[8:50] of the address
> > +
> > +Return Values
> > +-------------
> > +
> > +``OPAL_SUCCESS``
> > +  Operation was successful
> > +
> > +``OPAL_PARAMETER``
> > +  Invalid PHB or addr parameter
> > +
> > +``OPAL_UNSUPPORTED``
> > +  Not supported by hardware
> > diff --git a/doc/opal-api/opal-pci-get-set-phb-cmpm-158-159.rst b/doc/opal-api/opal-pci-get-set-phb-cmpm-158-159.rst
> > new file mode 100644
> > index 00000000..7eabdde3
> > --- /dev/null
> > +++ b/doc/opal-api/opal-pci-get-set-phb-cmpm-158-159.rst
> > @@ -0,0 +1,100 @@
> > +OPAL_PCI_GET_PHB_CMPM
> > +=====================
> > +::
> > +
> > +   #define OPAL_PCI_GET_PHB_CMPM 158
> > +
> > +   int64_t opal_pci_get_phb_cmpm(uint64_t phb_id, uint64_t phb_reg,
> > +                                 uint64_t *val)
> > +
> > +The host calls this function to read the indication out of a CMPM register
> > +of the PHB.
> > +
> > +Parameters
> > +----------
> > +::
> > +
> > +   uint64_t phb_id
> > +   uint64_t phb_reg
> > +   uint64_t *val
> > +
> > +``phb_id``
> > +  The value from the PHB node ibm,opal-phbid property for the device.
> > +
> > +``phb_reg``
> > +  Which CMPM register should be accessed, i.e. one of ::
> > +
> > +   /* PHB Compare/Mask registers */
> > +   enum {
> > +        OPAL_PHB_ASN_CMPM       = 0,
> > +        OPAL_PHB_CAPI_CMPM      = 1,
> > +        OPAL_PHB_PBL_NBW_CMPM   = 2,
> > +   };
> > +
> > +``val``
> > +  A pointer to the address where the indication of the PHB CMPM register
> > +  will be copied.
> > +
> > +Return Values
> > +-------------
> > +
> > +``OPAL_SUCCESS``
> > +  Operation was successful
> > +
> > +``OPAL_PARAMETER``
> > +  Invalid PHB or phb_reg parameter
> > +
> > +``OPAL_UNSUPPORTED``
> > +  Not supported by hardware
> > +
> > +OPAL_PCI_SET_PHB_CMPM
> > +=====================
> > +::
> > +
> > +   #define OPAL_PCI_SET_PHB_CMPM 159
> > +
> > +   int64_t opal_pci_set_phb_cmpm(uint64_t phb_id, uint64_t phb_reg,
> > +                                 uint64_t ind)
> > +
> > +The host calls this function to set a CMPM register of the PHB.
> > +
> > +Parameters
> > +----------
> > +::
> > +
> > +   uint64_t phb_id
> > +   uint64_t phb_reg
> > +   uint64_t ind
> > +
> > +``phb_id``
> > +  The value from the PHB node ibm,opal-phbid property for the device.
> > +
> > +``phb_reg``
> > +  Which CMPM register should be accessed, i.e. one of ::
> > +
> > +   /* PHB Compare/Mask registers */
> > +   enum {
> > +        OPAL_PHB_ASN_CMPM       = 0,
> > +        OPAL_PHB_CAPI_CMPM      = 1,
> > +        OPAL_PHB_PBL_NBW_CMPM   = 2,
> > +   };
> > +
> > +``ind``
> > +  The value of the indication chosen for the PHB CMPM register.
> > +  The PHB CMPM register will be set with ::
> > +
> > +   Bit[0:15]      the indication value
> > +   Bit[16:31]     the (hard-coded) mask value
> > +   Bit[63]        the enable bit
> > +
> > +Return Values
> > +-------------
> > +
> > +``OPAL_SUCCESS``
> > +  Operation was successful
> > +
> > +``OPAL_PARAMETER``
> > +  Invalid PHB, phb_reg, or ind parameter
> > +
> > +``OPAL_UNSUPPORTED``
> > +  Not supported by hardware
> > diff --git a/hw/phb4.c b/hw/phb4.c
> > index c64ded92..1add8a75 100644
> > --- a/hw/phb4.c
> > +++ b/hw/phb4.c
> > @@ -3731,16 +3731,25 @@ static int64_t enable_capi_mode(struct phb4 *p, uint64_t pe_number,
> >   	 */
> > 
> >   	/*
> > -	 * Bit [0:7] XSL_DSNCTL[capiind]
> > -	 * Init_25 - CAPI Compare/Mask
> > +	 * Init_26 - CAPI Compare/Mask
> > +	 * If unset, then capiind=0x0200.
> >   	 */
> > -	out_be64(p->regs + PHB_CAPI_CMPM,
> > -		 0x0200FE0000000000Ull | PHB_CAPI_CMPM_ENABLE);
> > +	reg = in_be64(p->regs + PHB_CAPI_CMPM);
> > +	if (!reg) {
> > +		reg = 0x0200fe0000000000ull | PHB_CAPI_CMPM_ENABLE;
> > +		out_be64(p->regs + PHB_CAPI_CMPM, reg);
> > +	}
> > 
> >   	if (!(p->rev == PHB4_REV_NIMBUS_DD10)) {
> > -		/* Init_24 - ASN Compare/Mask */
> > -		out_be64(p->regs + PHB_PBL_ASN_CMPM,
> > -			 0x0400FF0000000000Ull | PHB_PBL_ASN_ENABLE);
> > +		/*
> > +		 * Init_25 - ASN Compare/Mask
> > +		 * If unset, then asnind=0x0400.
> > +		 */
> > +		reg = in_be64(p->regs + PHB_ASN_CMPM);
> > +		if (!reg) {
> > +			reg = 0x0400ff0000000000ull | PHB_ASN_CMPM_ENABLE;
> > +			out_be64(p->regs + PHB_ASN_CMPM, reg);
> > +		}
> > 
> >   		/* PBCQ Tunnel Bar Register
> >   		 * Write Tunnel register to match PSL TNR register
> > @@ -3990,6 +3999,99 @@ static int64_t phb4_set_capp_recovery(struct phb *phb)
> >   	return 0;
> >   }
> > 
> > +/*
> > + * Return the indication of a PHB CMPM register.
> > + */
> > +static int64_t phb4_get_cmpm(struct phb *phb, uint64_t phb_reg, uint64_t *ind)
> > +{
> > +	struct phb4 *p = phb_to_phb4(phb);
> > +	uint64_t offset;
> > +
> > +	switch (phb_reg) {
> > +	case OPAL_PHB_ASN_CMPM:
> > +		offset = PHB_ASN_CMPM;
> > +		break;
> > +	case OPAL_PHB_CAPI_CMPM:
> > +		offset = PHB_CAPI_CMPM;
> > +		break;
> > +	case OPAL_PHB_PBL_NBW_CMPM:
> > +		offset = PHB_PBL_NBW_CMPM;
> > +		break;
> > +	default:
> > +		return OPAL_UNSUPPORTED;
> > +	}
> > +
> > +	*ind = in_be64(p->regs + offset) >> 48;
> > +	return OPAL_SUCCESS;
> > +}
> > +
> > +/*
> > + * Set the 16-bit indication of a PHB CMPM register.
> > + * Set (hard-coded) mask value, and enable bit.
> > + *
> > + * This interface aims at letting linux choose the indications,
> > + * if they need to be changed in the future.
> > + *
> > + * Compatibility with older versions of linux, that do not set CMPM
> > + * indications with phb4_set_cmpm(), is ensured by enable_capi_mode(),
> > + * that will set the default hard-coded values that were assumed then.
> > + */
> > +static int64_t phb4_set_cmpm(struct phb *phb, uint64_t phb_reg, uint64_t ind)
> > +{
> > +	struct phb4 *p = phb_to_phb4(phb);
> > +	uint64_t mask, offset, enable;
> > +
> > +	/* Indication is a 16-bit value */
> > +	if (ind >> 16)
> > +		return OPAL_PARAMETER;
> > +	/*
> > +	 * The following bits of the PCI address are reserved:
> > +	 *   Bit 59 indicates TVE#1
> > +	 *   Bit 60 indicates MSI-X
> > +	 */
> > +	if (ind & 0x1800ull)
> > +		return OPAL_PARAMETER;
> > +
> > +	switch (phb_reg) {
> > +	case OPAL_PHB_ASN_CMPM:
> > +		/*
> > +		 * Init_25 - ASN Compare/Mask
> > +		 * matches XSL_DSNCTL[ASB_Addr]
> > +		 */
> > +		offset = PHB_ASN_CMPM;
> > +		enable = PHB_ASN_CMPM_ENABLE;
> > +		mask = 0xff00ull;
> > +		break;
> > +	case OPAL_PHB_CAPI_CMPM:
> > +		/*
> > +		 * Init_26 - CAPI Compare/Mask
> > +		 * matches XSL_DSNCTL[capiind]
> > +		 */
> > +		offset = PHB_CAPI_CMPM;
> > +		enable = PHB_CAPI_CMPM_ENABLE;
> > +		mask = 0xfe00ull;
> > +		break;
> > +	case OPAL_PHB_PBL_NBW_CMPM:
> > +		/*
> > +		 * Init_124 - NBW Compare/Mask
> > +		 * matches XSL_DSNCTL[nbwind]
> > +		 */
> > +		offset = PHB_PBL_NBW_CMPM;
> > +		enable = PHB_PBL_NBW_CMPM_ENABLE;
> > +		mask = 0xff00ull;
> > +		break;
> > +	default:
> > +		return OPAL_UNSUPPORTED;
> > +	}
> > +
> > +	/* Masked indication should not be null */
> > +	if (!(ind & mask))
> > +		return OPAL_PARAMETER;
> > +
> > +	out_be64(p->regs + offset, (ind << 48) | (mask << 32) | enable);
> > +	return OPAL_SUCCESS;
> > +}
> > +
> >   static const struct phb_ops phb4_ops = {
> >   	.cfg_read8		= phb4_pcicfg_read8,
> >   	.cfg_read16		= phb4_pcicfg_read16,
> > @@ -4025,6 +4127,8 @@ static const struct phb_ops phb4_ops = {
> >   	.set_capi_mode		= phb4_set_capi_mode,
> >   	.set_p2p		= phb4_set_p2p,
> >   	.set_capp_recovery	= phb4_set_capp_recovery,
> > +	.get_cmpm		= phb4_get_cmpm,
> > +	.set_cmpm		= phb4_set_cmpm,
> >   };
> > 
> >   static void phb4_init_ioda3(struct phb4 *p)
> > @@ -4059,10 +4163,10 @@ static void phb4_init_ioda3(struct phb4 *p)
> >   	/* See enable_capi_mode() */
> > 
> >   	/* Init_25 - ASN Compare/Mask */
> > -	/* See enable_capi_mode() */
> > +	/* See phb4_set_cmpm() and enable_capi_mode() */
> > 
> >   	/* Init_26 - CAPI Compare/Mask */
> > -	/* See enable_capi_mode() */
> > +	/* See phb4_set_cmpm() and enable_capi_mode() */
> > 
> >   	/* Init_27 - PCIE Outbound upper address */
> >   	out_be64(p->regs + PHB_M64_UPPER_BITS, 0);
> > @@ -4412,8 +4516,8 @@ static void phb4_init_hw(struct phb4 *p, bool first_init)
> >   	 * don't bother as we are doing a PERST soon.
> >   	 */
> > 
> > -	/* Init_124 :  NBW. XXX TODO */
> > -	/* See enable_capi_mode() */
> > +	/* Init_124 - NBW Compare/Mask */
> > +	/* See phb4_set_cmpm() and enable_capi_mode() */
> > 
> >   	/* Init_125 : Setup PCI command/status on root complex
> >   	 * I don't know why the spec does this now and not earlier, so
> > diff --git a/include/opal-api.h b/include/opal-api.h
> > index 0bc036ed..ce948136 100644
> > --- a/include/opal-api.h
> > +++ b/include/opal-api.h
> > @@ -214,7 +214,9 @@
> >   #define OPAL_SET_POWER_SHIFT_RATIO		155
> >   #define OPAL_SENSOR_GROUP_CLEAR			156
> >   #define OPAL_PCI_SET_P2P			157
> > -#define OPAL_LAST				157
> > +#define OPAL_PCI_GET_PHB_CMPM			158
> > +#define OPAL_PCI_SET_PHB_CMPM			159
> > +#define OPAL_LAST				159
> > 
> >   /* Device tree flags */
> > 
> > @@ -1274,6 +1276,13 @@ enum {
> >   	OPAL_PCI_P2P_TARGET	= 1,
> >   };
> > 
> > +/* PHB Compare/Mask registers */
> > +enum {
> > +	OPAL_PHB_ASN_CMPM	= 0,
> > +	OPAL_PHB_CAPI_CMPM	= 1,
> > +	OPAL_PHB_PBL_NBW_CMPM	= 2,
> > +};
> > +
> >   #endif /* __ASSEMBLY__ */
> > 
> >   #endif /* __OPAL_API_H */
> > diff --git a/include/pci.h b/include/pci.h
> > index c085b6b8..cdf82ee8 100644
> > --- a/include/pci.h
> > +++ b/include/pci.h
> > @@ -333,6 +333,10 @@ struct phb_ops {
> >   	/* PCI peer-to-peer setup */
> >   	void (*set_p2p)(struct phb *phb, uint64_t mode, uint64_t flags,
> >   			uint16_t pe_number);
> > +
> > +	/* Get/set PHB Compare/Mask registers */
> > +	int64_t (*get_cmpm)(struct phb *phb, uint64_t phb_reg, uint64_t *ind);
> > +	int64_t (*set_cmpm)(struct phb *phb, uint64_t phb_reg, uint64_t ind);
> >   };
> > 
> >   enum phb_type {
> > diff --git a/include/phb4-regs.h b/include/phb4-regs.h
> > index e83c8c39..b33185ed 100644
> > --- a/include/phb4-regs.h
> > +++ b/include/phb4-regs.h
> > @@ -71,8 +71,8 @@
> >   #define PHB_PEST_BAR			0x1a8
> >   #define   PHB_PEST_BAR_ENABLE		PPC_BIT(0)
> >   #define   PHB_PEST_BASE_ADDRESS		PPC_BITMASK(8,51)
> > -#define PHB_PBL_ASN_CMPM		0x1C0
> > -#define   PHB_PBL_ASN_ENABLE		PPC_BIT(63)
> > +#define PHB_ASN_CMPM			0x1C0
> > +#define   PHB_ASN_CMPM_ENABLE		PPC_BIT(63)
> >   #define PHB_CAPI_CMPM			0x1C8
> >   #define   PHB_CAPI_CMPM_ENABLE		PPC_BIT(63)
> >   #define PHB_M64_UPPER_BITS		0x1f0
> > @@ -250,8 +250,8 @@
> >   #define PHB_PBL_CONTROL				0x1800
> >   #define PHB_PBL_TIMEOUT_CTRL			0x1810
> >   #define PHB_PBL_NPTAG_ENABLE			0x1820
> > -#define PHB_PBL_NBW_CMP_MASK			0x1830
> > -#define   PHB_PBL_NBW_MASK_ENABLE		PPC_BIT(63)
> > +#define PHB_PBL_NBW_CMPM			0x1830
> > +#define   PHB_PBL_NBW_CMPM_ENABLE		PPC_BIT(63)
> >   #define PHB_PBL_SYS_LINK_INIT			0x1838
> >   #define PHB_PBL_BUF_STATUS			0x1840
> >   #define PHB_PBL_ERR_STATUS			0x1900
> 
> _______________________________________________
> Skiboot mailing list
> Skiboot@lists.ozlabs.org
> https://lists.ozlabs.org/listinfo/skiboot
Philippe Bergheaud Nov. 7, 2017, 1:19 p.m. | #4
On 02/11/2017 05:03, Benjamin Herrenschmidt wrote:
> On Thu, 2017-10-26 at 15:43 +0200, Philippe Bergheaud wrote:
>> On 23/10/2017 15:08, Philippe Bergheaud wrote:
>>> P9 supports PCI tunneled operations (atomics and as_notify) that require
>>> setting the PHB ASN Compare/Mask register with a 16-bit indication.
>>>
>>> This register is currently initialized by enable_capi_mode(). As tunneled
>>> operations may also operate in PCI mode, a new API is required to set the
>>> PHB ASN register without switching to CAPI mode.
>>>
>>> This patch provides two new OPAL calls to get/set any of the three PHB
>>> CMPM registers that obey the same logic (ASN, NBW, and CAPI).
>>>
>>> This new API aims at letting the operating system choose indications, that
>>> may change in the future. Compatibility with older kernel versions is made
>>> by enable_capi_mode().
>>>
>>> Signed-off-by: Philippe Bergheaud <felix@linux.vnet.ibm.com>
>>> ---
>> Linux patch:
>> https://patchwork.ozlabs.org/patch/830231/
> Any reason why the BAR value is under Linux control ? Can't we just
> pick a range that's outside of the MMIO, DMA and MSI windows (some
> top bits set) and be done with it ? (ie. just *tell* Linux where we put
> it).

The ASN/CAPI CMPM indications are currently hard coded in skiboot and
Linux. This patch aims at avoiding a firmware update if ever one of
these values has to be changed in the future. With this patch, a Linux
kernel update would suffice. (Actually, the patch should allow changing
ASN and CAPI CMPM masks also.)

This patch is not mandatory. We can drop it and live with the current
static implementation, if we assume that those values will never change.

>>>    core/pci-opal.c                                    |  36 ++++++
>>>    .../opal-pci-get-set-pbcq-tunnel-bar-160-161.rst   |  76 +++++++++++++
>>>    doc/opal-api/opal-pci-get-set-phb-cmpm-158-159.rst | 100 ++++++++++++++++
>>>    hw/phb4.c                                          | 126 +++++++++++++++++++--
>>>    include/opal-api.h                                 |  11 +-
>>>    include/pci.h                                      |   4 +
>>>    include/phb4-regs.h                                |   8 +-
>>>    7 files changed, 345 insertions(+), 16 deletions(-)
>>>    create mode 100644 doc/opal-api/opal-pci-get-set-pbcq-tunnel-bar-160-161.rst
>>>    create mode 100644 doc/opal-api/opal-pci-get-set-phb-cmpm-158-159.rst
>>>
>>> diff --git a/core/pci-opal.c b/core/pci-opal.c
>>> index b8aec941..bbbbcd5d 100644
>>> --- a/core/pci-opal.c
>>> +++ b/core/pci-opal.c
>>> @@ -1016,3 +1016,39 @@ static int64_t opal_pci_set_p2p(uint64_t phbid_init, uint64_t phbid_target,
>>>    	return OPAL_SUCCESS;
>>>    }
>>>    opal_call(OPAL_PCI_SET_P2P, opal_pci_set_p2p, 4);
>>> +
>>> +static int64_t opal_pci_get_phb_cmpm(uint64_t phb_id, uint64_t phb_reg,
>>> +				     uint64_t *ind)
>>> +{
>>> +	struct phb *phb = pci_get_phb(phb_id);
>>> +	int64_t rc;
>>> +
>>> +	if (!phb)
>>> +		return OPAL_PARAMETER;
>>> +	if (!phb->ops->get_cmpm)
>>> +		return OPAL_UNSUPPORTED;
>>> +
>>> +	phb_lock(phb);
>>> +	rc = phb->ops->get_cmpm(phb, phb_reg, ind);
>>> +	phb_unlock(phb);
>>> +	return rc;
>>> +}
>>> +opal_call(OPAL_PCI_GET_PHB_CMPM, opal_pci_get_phb_cmpm, 3);
>>> +
>>> +static int64_t opal_pci_set_phb_cmpm(uint64_t phb_id, uint64_t phb_reg,
>>> +				     uint64_t ind)
>>> +{
>>> +	struct phb *phb = pci_get_phb(phb_id);
>>> +	int64_t rc;
>>> +
>>> +	if (!phb)
>>> +		return OPAL_PARAMETER;
>>> +	if (!phb->ops->set_cmpm)
>>> +		return OPAL_UNSUPPORTED;
>>> +
>>> +	phb_lock(phb);
>>> +	rc = phb->ops->set_cmpm(phb, phb_reg, ind);
>>> +	phb_unlock(phb);
>>> +	return rc;
>>> +}
>>> +opal_call(OPAL_PCI_SET_PHB_CMPM, opal_pci_set_phb_cmpm, 3);
>>> diff --git a/doc/opal-api/opal-pci-get-set-pbcq-tunnel-bar-160-161.rst b/doc/opal-api/opal-pci-get-set-pbcq-tunnel-bar-160-161.rst
>>> new file mode 100644
>>> index 00000000..0af06774
>>> --- /dev/null
>>> +++ b/doc/opal-api/opal-pci-get-set-pbcq-tunnel-bar-160-161.rst
>>> @@ -0,0 +1,76 @@
>>> +OPAL_PCI_GET_PBCQ_TUNNEL_BAR
>>> +============================
>>> +::
>>> +
>>> +   #define OPAL_PCI_GET_PBCQ_TUNNEL_BAR 160
>>> +
>>> +   int64_t opal_pci_get_pbcq_tunnel_bar(uint64_t phb_id, uint64_t *val)
>>> +
>>> +The host calls this function to read the address out of the PBCQ Tunnel
>>> +Bar register.
>>> +
>>> +Parameters
>>> +----------
>>> +::
>>> +
>>> +   uint64_t phb_id
>>> +   uint64_t *val
>>> +
>>> +``phb_id``
>>> +  The value from the PHB node ibm,opal-phbid property for the device.
>>> +
>>> +``val``
>>> +  A pointer to the address where the address store in the PBCQ Tunnel Bar
>>> +  register will be copied.
>>> +
>>> +Return Values
>>> +-------------
>>> +
>>> +``OPAL_SUCCESS``
>>> +  Operation was successful
>>> +
>>> +``OPAL_PARAMETER``
>>> +  Invalid PHB
>>> +
>>> +``OPAL_UNSUPPORTED``
>>> +  Not supported by hardware
>>> +
>>> +OPAL_PCI_SET_PBCQ_TUNNEL_BAR
>>> +============================
>>> +::
>>> +
>>> +   #define OPAL_PCI_SET_PBCQ_TUNNEL_BAR 161
>>> +
>>> +   int64_t opal_pci_set_pbcq_tunnel_bar(uint64_t phb_id, uint64_t addr)
>>> +
>>> +The host calls this function to set the PBCQ Tunnel Bar register.
>>> +
>>> +Parameters
>>> +----------
>>> +::
>>> +
>>> +   uint64_t phb_id
>>> +   uint64_t addr
>>> +
>>> +``phb_id``
>>> +  The value from the PHB node ibm,opal-phbid property for the device.
>>> +
>>> +``addr``
>>> +  The value of the address chosen for the PBCQ Tunnel Bar register.
>>> +  If the address is 0, then the PBCQ Tunnel Bar register will be reset.
>>> +  It the address is non-zero, then the PBCQ Tunnel Bar register will be
>>> +  set with ::
>>> +
>>> +   Bit[0:42]      Bit[8:50] of the address
>>> +
>>> +Return Values
>>> +-------------
>>> +
>>> +``OPAL_SUCCESS``
>>> +  Operation was successful
>>> +
>>> +``OPAL_PARAMETER``
>>> +  Invalid PHB or addr parameter
>>> +
>>> +``OPAL_UNSUPPORTED``
>>> +  Not supported by hardware
>>> diff --git a/doc/opal-api/opal-pci-get-set-phb-cmpm-158-159.rst b/doc/opal-api/opal-pci-get-set-phb-cmpm-158-159.rst
>>> new file mode 100644
>>> index 00000000..7eabdde3
>>> --- /dev/null
>>> +++ b/doc/opal-api/opal-pci-get-set-phb-cmpm-158-159.rst
>>> @@ -0,0 +1,100 @@
>>> +OPAL_PCI_GET_PHB_CMPM
>>> +=====================
>>> +::
>>> +
>>> +   #define OPAL_PCI_GET_PHB_CMPM 158
>>> +
>>> +   int64_t opal_pci_get_phb_cmpm(uint64_t phb_id, uint64_t phb_reg,
>>> +                                 uint64_t *val)
>>> +
>>> +The host calls this function to read the indication out of a CMPM register
>>> +of the PHB.
>>> +
>>> +Parameters
>>> +----------
>>> +::
>>> +
>>> +   uint64_t phb_id
>>> +   uint64_t phb_reg
>>> +   uint64_t *val
>>> +
>>> +``phb_id``
>>> +  The value from the PHB node ibm,opal-phbid property for the device.
>>> +
>>> +``phb_reg``
>>> +  Which CMPM register should be accessed, i.e. one of ::
>>> +
>>> +   /* PHB Compare/Mask registers */
>>> +   enum {
>>> +        OPAL_PHB_ASN_CMPM       = 0,
>>> +        OPAL_PHB_CAPI_CMPM      = 1,
>>> +        OPAL_PHB_PBL_NBW_CMPM   = 2,
>>> +   };
>>> +
>>> +``val``
>>> +  A pointer to the address where the indication of the PHB CMPM register
>>> +  will be copied.
>>> +
>>> +Return Values
>>> +-------------
>>> +
>>> +``OPAL_SUCCESS``
>>> +  Operation was successful
>>> +
>>> +``OPAL_PARAMETER``
>>> +  Invalid PHB or phb_reg parameter
>>> +
>>> +``OPAL_UNSUPPORTED``
>>> +  Not supported by hardware
>>> +
>>> +OPAL_PCI_SET_PHB_CMPM
>>> +=====================
>>> +::
>>> +
>>> +   #define OPAL_PCI_SET_PHB_CMPM 159
>>> +
>>> +   int64_t opal_pci_set_phb_cmpm(uint64_t phb_id, uint64_t phb_reg,
>>> +                                 uint64_t ind)
>>> +
>>> +The host calls this function to set a CMPM register of the PHB.
>>> +
>>> +Parameters
>>> +----------
>>> +::
>>> +
>>> +   uint64_t phb_id
>>> +   uint64_t phb_reg
>>> +   uint64_t ind
>>> +
>>> +``phb_id``
>>> +  The value from the PHB node ibm,opal-phbid property for the device.
>>> +
>>> +``phb_reg``
>>> +  Which CMPM register should be accessed, i.e. one of ::
>>> +
>>> +   /* PHB Compare/Mask registers */
>>> +   enum {
>>> +        OPAL_PHB_ASN_CMPM       = 0,
>>> +        OPAL_PHB_CAPI_CMPM      = 1,
>>> +        OPAL_PHB_PBL_NBW_CMPM   = 2,
>>> +   };
>>> +
>>> +``ind``
>>> +  The value of the indication chosen for the PHB CMPM register.
>>> +  The PHB CMPM register will be set with ::
>>> +
>>> +   Bit[0:15]      the indication value
>>> +   Bit[16:31]     the (hard-coded) mask value
>>> +   Bit[63]        the enable bit
>>> +
>>> +Return Values
>>> +-------------
>>> +
>>> +``OPAL_SUCCESS``
>>> +  Operation was successful
>>> +
>>> +``OPAL_PARAMETER``
>>> +  Invalid PHB, phb_reg, or ind parameter
>>> +
>>> +``OPAL_UNSUPPORTED``
>>> +  Not supported by hardware
>>> diff --git a/hw/phb4.c b/hw/phb4.c
>>> index c64ded92..1add8a75 100644
>>> --- a/hw/phb4.c
>>> +++ b/hw/phb4.c
>>> @@ -3731,16 +3731,25 @@ static int64_t enable_capi_mode(struct phb4 *p, uint64_t pe_number,
>>>    	 */
>>>
>>>    	/*
>>> -	 * Bit [0:7] XSL_DSNCTL[capiind]
>>> -	 * Init_25 - CAPI Compare/Mask
>>> +	 * Init_26 - CAPI Compare/Mask
>>> +	 * If unset, then capiind=0x0200.
>>>    	 */
>>> -	out_be64(p->regs + PHB_CAPI_CMPM,
>>> -		 0x0200FE0000000000Ull | PHB_CAPI_CMPM_ENABLE);
>>> +	reg = in_be64(p->regs + PHB_CAPI_CMPM);
>>> +	if (!reg) {
>>> +		reg = 0x0200fe0000000000ull | PHB_CAPI_CMPM_ENABLE;
>>> +		out_be64(p->regs + PHB_CAPI_CMPM, reg);
>>> +	}
>>>
>>>    	if (!(p->rev == PHB4_REV_NIMBUS_DD10)) {
>>> -		/* Init_24 - ASN Compare/Mask */
>>> -		out_be64(p->regs + PHB_PBL_ASN_CMPM,
>>> -			 0x0400FF0000000000Ull | PHB_PBL_ASN_ENABLE);
>>> +		/*
>>> +		 * Init_25 - ASN Compare/Mask
>>> +		 * If unset, then asnind=0x0400.
>>> +		 */
>>> +		reg = in_be64(p->regs + PHB_ASN_CMPM);
>>> +		if (!reg) {
>>> +			reg = 0x0400ff0000000000ull | PHB_ASN_CMPM_ENABLE;
>>> +			out_be64(p->regs + PHB_ASN_CMPM, reg);
>>> +		}
>>>
>>>    		/* PBCQ Tunnel Bar Register
>>>    		 * Write Tunnel register to match PSL TNR register
>>> @@ -3990,6 +3999,99 @@ static int64_t phb4_set_capp_recovery(struct phb *phb)
>>>    	return 0;
>>>    }
>>>
>>> +/*
>>> + * Return the indication of a PHB CMPM register.
>>> + */
>>> +static int64_t phb4_get_cmpm(struct phb *phb, uint64_t phb_reg, uint64_t *ind)
>>> +{
>>> +	struct phb4 *p = phb_to_phb4(phb);
>>> +	uint64_t offset;
>>> +
>>> +	switch (phb_reg) {
>>> +	case OPAL_PHB_ASN_CMPM:
>>> +		offset = PHB_ASN_CMPM;
>>> +		break;
>>> +	case OPAL_PHB_CAPI_CMPM:
>>> +		offset = PHB_CAPI_CMPM;
>>> +		break;
>>> +	case OPAL_PHB_PBL_NBW_CMPM:
>>> +		offset = PHB_PBL_NBW_CMPM;
>>> +		break;
>>> +	default:
>>> +		return OPAL_UNSUPPORTED;
>>> +	}
>>> +
>>> +	*ind = in_be64(p->regs + offset) >> 48;
>>> +	return OPAL_SUCCESS;
>>> +}
>>> +
>>> +/*
>>> + * Set the 16-bit indication of a PHB CMPM register.
>>> + * Set (hard-coded) mask value, and enable bit.
>>> + *
>>> + * This interface aims at letting linux choose the indications,
>>> + * if they need to be changed in the future.
>>> + *
>>> + * Compatibility with older versions of linux, that do not set CMPM
>>> + * indications with phb4_set_cmpm(), is ensured by enable_capi_mode(),
>>> + * that will set the default hard-coded values that were assumed then.
>>> + */
>>> +static int64_t phb4_set_cmpm(struct phb *phb, uint64_t phb_reg, uint64_t ind)
>>> +{
>>> +	struct phb4 *p = phb_to_phb4(phb);
>>> +	uint64_t mask, offset, enable;
>>> +
>>> +	/* Indication is a 16-bit value */
>>> +	if (ind >> 16)
>>> +		return OPAL_PARAMETER;
>>> +	/*
>>> +	 * The following bits of the PCI address are reserved:
>>> +	 *   Bit 59 indicates TVE#1
>>> +	 *   Bit 60 indicates MSI-X
>>> +	 */
>>> +	if (ind & 0x1800ull)
>>> +		return OPAL_PARAMETER;
>>> +
>>> +	switch (phb_reg) {
>>> +	case OPAL_PHB_ASN_CMPM:
>>> +		/*
>>> +		 * Init_25 - ASN Compare/Mask
>>> +		 * matches XSL_DSNCTL[ASB_Addr]
>>> +		 */
>>> +		offset = PHB_ASN_CMPM;
>>> +		enable = PHB_ASN_CMPM_ENABLE;
>>> +		mask = 0xff00ull;
>>> +		break;
>>> +	case OPAL_PHB_CAPI_CMPM:
>>> +		/*
>>> +		 * Init_26 - CAPI Compare/Mask
>>> +		 * matches XSL_DSNCTL[capiind]
>>> +		 */
>>> +		offset = PHB_CAPI_CMPM;
>>> +		enable = PHB_CAPI_CMPM_ENABLE;
>>> +		mask = 0xfe00ull;
>>> +		break;
>>> +	case OPAL_PHB_PBL_NBW_CMPM:
>>> +		/*
>>> +		 * Init_124 - NBW Compare/Mask
>>> +		 * matches XSL_DSNCTL[nbwind]
>>> +		 */
>>> +		offset = PHB_PBL_NBW_CMPM;
>>> +		enable = PHB_PBL_NBW_CMPM_ENABLE;
>>> +		mask = 0xff00ull;
>>> +		break;
>>> +	default:
>>> +		return OPAL_UNSUPPORTED;
>>> +	}
>>> +
>>> +	/* Masked indication should not be null */
>>> +	if (!(ind & mask))
>>> +		return OPAL_PARAMETER;
>>> +
>>> +	out_be64(p->regs + offset, (ind << 48) | (mask << 32) | enable);
>>> +	return OPAL_SUCCESS;
>>> +}
>>> +
>>>    static const struct phb_ops phb4_ops = {
>>>    	.cfg_read8		= phb4_pcicfg_read8,
>>>    	.cfg_read16		= phb4_pcicfg_read16,
>>> @@ -4025,6 +4127,8 @@ static const struct phb_ops phb4_ops = {
>>>    	.set_capi_mode		= phb4_set_capi_mode,
>>>    	.set_p2p		= phb4_set_p2p,
>>>    	.set_capp_recovery	= phb4_set_capp_recovery,
>>> +	.get_cmpm		= phb4_get_cmpm,
>>> +	.set_cmpm		= phb4_set_cmpm,
>>>    };
>>>
>>>    static void phb4_init_ioda3(struct phb4 *p)
>>> @@ -4059,10 +4163,10 @@ static void phb4_init_ioda3(struct phb4 *p)
>>>    	/* See enable_capi_mode() */
>>>
>>>    	/* Init_25 - ASN Compare/Mask */
>>> -	/* See enable_capi_mode() */
>>> +	/* See phb4_set_cmpm() and enable_capi_mode() */
>>>
>>>    	/* Init_26 - CAPI Compare/Mask */
>>> -	/* See enable_capi_mode() */
>>> +	/* See phb4_set_cmpm() and enable_capi_mode() */
>>>
>>>    	/* Init_27 - PCIE Outbound upper address */
>>>    	out_be64(p->regs + PHB_M64_UPPER_BITS, 0);
>>> @@ -4412,8 +4516,8 @@ static void phb4_init_hw(struct phb4 *p, bool first_init)
>>>    	 * don't bother as we are doing a PERST soon.
>>>    	 */
>>>
>>> -	/* Init_124 :  NBW. XXX TODO */
>>> -	/* See enable_capi_mode() */
>>> +	/* Init_124 - NBW Compare/Mask */
>>> +	/* See phb4_set_cmpm() and enable_capi_mode() */
>>>
>>>    	/* Init_125 : Setup PCI command/status on root complex
>>>    	 * I don't know why the spec does this now and not earlier, so
>>> diff --git a/include/opal-api.h b/include/opal-api.h
>>> index 0bc036ed..ce948136 100644
>>> --- a/include/opal-api.h
>>> +++ b/include/opal-api.h
>>> @@ -214,7 +214,9 @@
>>>    #define OPAL_SET_POWER_SHIFT_RATIO		155
>>>    #define OPAL_SENSOR_GROUP_CLEAR			156
>>>    #define OPAL_PCI_SET_P2P			157
>>> -#define OPAL_LAST				157
>>> +#define OPAL_PCI_GET_PHB_CMPM			158
>>> +#define OPAL_PCI_SET_PHB_CMPM			159
>>> +#define OPAL_LAST				159
>>>
>>>    /* Device tree flags */
>>>
>>> @@ -1274,6 +1276,13 @@ enum {
>>>    	OPAL_PCI_P2P_TARGET	= 1,
>>>    };
>>>
>>> +/* PHB Compare/Mask registers */
>>> +enum {
>>> +	OPAL_PHB_ASN_CMPM	= 0,
>>> +	OPAL_PHB_CAPI_CMPM	= 1,
>>> +	OPAL_PHB_PBL_NBW_CMPM	= 2,
>>> +};
>>> +
>>>    #endif /* __ASSEMBLY__ */
>>>
>>>    #endif /* __OPAL_API_H */
>>> diff --git a/include/pci.h b/include/pci.h
>>> index c085b6b8..cdf82ee8 100644
>>> --- a/include/pci.h
>>> +++ b/include/pci.h
>>> @@ -333,6 +333,10 @@ struct phb_ops {
>>>    	/* PCI peer-to-peer setup */
>>>    	void (*set_p2p)(struct phb *phb, uint64_t mode, uint64_t flags,
>>>    			uint16_t pe_number);
>>> +
>>> +	/* Get/set PHB Compare/Mask registers */
>>> +	int64_t (*get_cmpm)(struct phb *phb, uint64_t phb_reg, uint64_t *ind);
>>> +	int64_t (*set_cmpm)(struct phb *phb, uint64_t phb_reg, uint64_t ind);
>>>    };
>>>
>>>    enum phb_type {
>>> diff --git a/include/phb4-regs.h b/include/phb4-regs.h
>>> index e83c8c39..b33185ed 100644
>>> --- a/include/phb4-regs.h
>>> +++ b/include/phb4-regs.h
>>> @@ -71,8 +71,8 @@
>>>    #define PHB_PEST_BAR			0x1a8
>>>    #define   PHB_PEST_BAR_ENABLE		PPC_BIT(0)
>>>    #define   PHB_PEST_BASE_ADDRESS		PPC_BITMASK(8,51)
>>> -#define PHB_PBL_ASN_CMPM		0x1C0
>>> -#define   PHB_PBL_ASN_ENABLE		PPC_BIT(63)
>>> +#define PHB_ASN_CMPM			0x1C0
>>> +#define   PHB_ASN_CMPM_ENABLE		PPC_BIT(63)
>>>    #define PHB_CAPI_CMPM			0x1C8
>>>    #define   PHB_CAPI_CMPM_ENABLE		PPC_BIT(63)
>>>    #define PHB_M64_UPPER_BITS		0x1f0
>>> @@ -250,8 +250,8 @@
>>>    #define PHB_PBL_CONTROL				0x1800
>>>    #define PHB_PBL_TIMEOUT_CTRL			0x1810
>>>    #define PHB_PBL_NPTAG_ENABLE			0x1820
>>> -#define PHB_PBL_NBW_CMP_MASK			0x1830
>>> -#define   PHB_PBL_NBW_MASK_ENABLE		PPC_BIT(63)
>>> +#define PHB_PBL_NBW_CMPM			0x1830
>>> +#define   PHB_PBL_NBW_CMPM_ENABLE		PPC_BIT(63)
>>>    #define PHB_PBL_SYS_LINK_INIT			0x1838
>>>    #define PHB_PBL_BUF_STATUS			0x1840
>>>    #define PHB_PBL_ERR_STATUS			0x1900
>> _______________________________________________
>> Skiboot mailing list
>> Skiboot@lists.ozlabs.org
>> https://lists.ozlabs.org/listinfo/skiboot
Benjamin Herrenschmidt Nov. 7, 2017, 8:18 p.m. | #5
On Tue, 2017-11-07 at 14:19 +0100, Philippe Bergheaud wrote:
> The ASN/CAPI CMPM indications are currently hard coded in skiboot and
> Linux. This patch aims at avoiding a firmware update if ever one of
> these values has to be changed in the future. With this patch, a Linux
> kernel update would suffice. (Actually, the patch should allow changing
> ASN and CAPI CMPM masks also.)
> 
> This patch is not mandatory. We can drop it and live with the current
> static implementation, if we assume that those values will never change.

A firmware update is a LOT easier to deploy than a kernel update in
practice. We have more control on the firmware and it's update
schedule, at least for IBM systems, while a kernel update requires
synchronizing with all distros and can be really hard to deploy.

Also, this puts too much knowledge about the HW layout in Linux, which
could be problematic if we have to change things in a future chip as we
are aiming for full backward compatibility.

Cheers,
Ben.
Philippe Bergheaud Nov. 8, 2017, 3:14 p.m. | #6
On 07/11/2017 21:18, Benjamin Herrenschmidt wrote:
> On Tue, 2017-11-07 at 14:19 +0100, Philippe Bergheaud wrote:
>> The ASN/CAPI CMPM indications are currently hard coded in skiboot and
>> Linux. This patch aims at avoiding a firmware update if ever one of
>> these values has to be changed in the future. With this patch, a Linux
>> kernel update would suffice. (Actually, the patch should allow changing
>> ASN and CAPI CMPM masks also.)
>>
>> This patch is not mandatory. We can drop it and live with the current
>> static implementation, if we assume that those values will never change.
> A firmware update is a LOT easier to deploy than a kernel update in
> practice. We have more control on the firmware and it's update
> schedule, at least for IBM systems, while a kernel update requires
> synchronizing with all distros and can be really hard to deploy.
>
> Also, this puts too much knowledge about the HW layout in Linux, which
> could be problematic if we have to change things in a future chip as we
> are aiming for full backward compatibility.

Got it.

One last question, though: should we define CMPM indication and mask
values in skiboot and Linux, or should we add a skiboot API to tell Linux?

Philippe

Patch

diff --git a/core/pci-opal.c b/core/pci-opal.c
index b8aec941..bbbbcd5d 100644
--- a/core/pci-opal.c
+++ b/core/pci-opal.c
@@ -1016,3 +1016,39 @@  static int64_t opal_pci_set_p2p(uint64_t phbid_init, uint64_t phbid_target,
 	return OPAL_SUCCESS;
 }
 opal_call(OPAL_PCI_SET_P2P, opal_pci_set_p2p, 4);
+
+static int64_t opal_pci_get_phb_cmpm(uint64_t phb_id, uint64_t phb_reg,
+				     uint64_t *ind)
+{
+	struct phb *phb = pci_get_phb(phb_id);
+	int64_t rc;
+
+	if (!phb)
+		return OPAL_PARAMETER;
+	if (!phb->ops->get_cmpm)
+		return OPAL_UNSUPPORTED;
+
+	phb_lock(phb);
+	rc = phb->ops->get_cmpm(phb, phb_reg, ind);
+	phb_unlock(phb);
+	return rc;
+}
+opal_call(OPAL_PCI_GET_PHB_CMPM, opal_pci_get_phb_cmpm, 3);
+
+static int64_t opal_pci_set_phb_cmpm(uint64_t phb_id, uint64_t phb_reg,
+				     uint64_t ind)
+{
+	struct phb *phb = pci_get_phb(phb_id);
+	int64_t rc;
+
+	if (!phb)
+		return OPAL_PARAMETER;
+	if (!phb->ops->set_cmpm)
+		return OPAL_UNSUPPORTED;
+
+	phb_lock(phb);
+	rc = phb->ops->set_cmpm(phb, phb_reg, ind);
+	phb_unlock(phb);
+	return rc;
+}
+opal_call(OPAL_PCI_SET_PHB_CMPM, opal_pci_set_phb_cmpm, 3);
diff --git a/doc/opal-api/opal-pci-get-set-pbcq-tunnel-bar-160-161.rst b/doc/opal-api/opal-pci-get-set-pbcq-tunnel-bar-160-161.rst
new file mode 100644
index 00000000..0af06774
--- /dev/null
+++ b/doc/opal-api/opal-pci-get-set-pbcq-tunnel-bar-160-161.rst
@@ -0,0 +1,76 @@ 
+OPAL_PCI_GET_PBCQ_TUNNEL_BAR
+============================
+::
+
+   #define OPAL_PCI_GET_PBCQ_TUNNEL_BAR 160
+
+   int64_t opal_pci_get_pbcq_tunnel_bar(uint64_t phb_id, uint64_t *val)
+
+The host calls this function to read the address out of the PBCQ Tunnel
+Bar register.
+
+Parameters
+----------
+::
+
+   uint64_t phb_id
+   uint64_t *val
+
+``phb_id``
+  The value from the PHB node ibm,opal-phbid property for the device.
+
+``val``
+  A pointer to the address where the address store in the PBCQ Tunnel Bar
+  register will be copied.
+
+Return Values
+-------------
+
+``OPAL_SUCCESS``
+  Operation was successful
+
+``OPAL_PARAMETER``
+  Invalid PHB
+
+``OPAL_UNSUPPORTED``
+  Not supported by hardware
+
+OPAL_PCI_SET_PBCQ_TUNNEL_BAR
+============================
+::
+
+   #define OPAL_PCI_SET_PBCQ_TUNNEL_BAR 161
+
+   int64_t opal_pci_set_pbcq_tunnel_bar(uint64_t phb_id, uint64_t addr)
+
+The host calls this function to set the PBCQ Tunnel Bar register.
+
+Parameters
+----------
+::
+
+   uint64_t phb_id
+   uint64_t addr
+
+``phb_id``
+  The value from the PHB node ibm,opal-phbid property for the device.
+
+``addr``
+  The value of the address chosen for the PBCQ Tunnel Bar register.
+  If the address is 0, then the PBCQ Tunnel Bar register will be reset.
+  It the address is non-zero, then the PBCQ Tunnel Bar register will be
+  set with ::
+
+   Bit[0:42]      Bit[8:50] of the address
+
+Return Values
+-------------
+
+``OPAL_SUCCESS``
+  Operation was successful
+
+``OPAL_PARAMETER``
+  Invalid PHB or addr parameter
+
+``OPAL_UNSUPPORTED``
+  Not supported by hardware
diff --git a/doc/opal-api/opal-pci-get-set-phb-cmpm-158-159.rst b/doc/opal-api/opal-pci-get-set-phb-cmpm-158-159.rst
new file mode 100644
index 00000000..7eabdde3
--- /dev/null
+++ b/doc/opal-api/opal-pci-get-set-phb-cmpm-158-159.rst
@@ -0,0 +1,100 @@ 
+OPAL_PCI_GET_PHB_CMPM
+=====================
+::
+
+   #define OPAL_PCI_GET_PHB_CMPM 158
+
+   int64_t opal_pci_get_phb_cmpm(uint64_t phb_id, uint64_t phb_reg,
+                                 uint64_t *val)
+
+The host calls this function to read the indication out of a CMPM register
+of the PHB.
+
+Parameters
+----------
+::
+
+   uint64_t phb_id
+   uint64_t phb_reg
+   uint64_t *val
+
+``phb_id``
+  The value from the PHB node ibm,opal-phbid property for the device.
+
+``phb_reg``
+  Which CMPM register should be accessed, i.e. one of ::
+
+   /* PHB Compare/Mask registers */
+   enum {
+        OPAL_PHB_ASN_CMPM       = 0,
+        OPAL_PHB_CAPI_CMPM      = 1,
+        OPAL_PHB_PBL_NBW_CMPM   = 2,
+   };
+
+``val``
+  A pointer to the address where the indication of the PHB CMPM register
+  will be copied.
+
+Return Values
+-------------
+
+``OPAL_SUCCESS``
+  Operation was successful
+
+``OPAL_PARAMETER``
+  Invalid PHB or phb_reg parameter
+
+``OPAL_UNSUPPORTED``
+  Not supported by hardware
+
+OPAL_PCI_SET_PHB_CMPM
+=====================
+::
+
+   #define OPAL_PCI_SET_PHB_CMPM 159
+
+   int64_t opal_pci_set_phb_cmpm(uint64_t phb_id, uint64_t phb_reg,
+                                 uint64_t ind)
+
+The host calls this function to set a CMPM register of the PHB.
+
+Parameters
+----------
+::
+
+   uint64_t phb_id
+   uint64_t phb_reg
+   uint64_t ind
+
+``phb_id``
+  The value from the PHB node ibm,opal-phbid property for the device.
+
+``phb_reg``
+  Which CMPM register should be accessed, i.e. one of ::
+
+   /* PHB Compare/Mask registers */
+   enum {
+        OPAL_PHB_ASN_CMPM       = 0,
+        OPAL_PHB_CAPI_CMPM      = 1,
+        OPAL_PHB_PBL_NBW_CMPM   = 2,
+   };
+
+``ind``
+  The value of the indication chosen for the PHB CMPM register.
+  The PHB CMPM register will be set with ::
+
+   Bit[0:15]      the indication value
+   Bit[16:31]     the (hard-coded) mask value
+   Bit[63]        the enable bit
+
+Return Values
+-------------
+
+``OPAL_SUCCESS``
+  Operation was successful
+
+``OPAL_PARAMETER``
+  Invalid PHB, phb_reg, or ind parameter
+
+``OPAL_UNSUPPORTED``
+  Not supported by hardware
diff --git a/hw/phb4.c b/hw/phb4.c
index c64ded92..1add8a75 100644
--- a/hw/phb4.c
+++ b/hw/phb4.c
@@ -3731,16 +3731,25 @@  static int64_t enable_capi_mode(struct phb4 *p, uint64_t pe_number,
 	 */
 
 	/*
-	 * Bit [0:7] XSL_DSNCTL[capiind]
-	 * Init_25 - CAPI Compare/Mask
+	 * Init_26 - CAPI Compare/Mask
+	 * If unset, then capiind=0x0200.
 	 */
-	out_be64(p->regs + PHB_CAPI_CMPM,
-		 0x0200FE0000000000Ull | PHB_CAPI_CMPM_ENABLE);
+	reg = in_be64(p->regs + PHB_CAPI_CMPM);
+	if (!reg) {
+		reg = 0x0200fe0000000000ull | PHB_CAPI_CMPM_ENABLE;
+		out_be64(p->regs + PHB_CAPI_CMPM, reg);
+	}
 
 	if (!(p->rev == PHB4_REV_NIMBUS_DD10)) {
-		/* Init_24 - ASN Compare/Mask */
-		out_be64(p->regs + PHB_PBL_ASN_CMPM,
-			 0x0400FF0000000000Ull | PHB_PBL_ASN_ENABLE);
+		/*
+		 * Init_25 - ASN Compare/Mask
+		 * If unset, then asnind=0x0400.
+		 */
+		reg = in_be64(p->regs + PHB_ASN_CMPM);
+		if (!reg) {
+			reg = 0x0400ff0000000000ull | PHB_ASN_CMPM_ENABLE;
+			out_be64(p->regs + PHB_ASN_CMPM, reg);
+		}
 
 		/* PBCQ Tunnel Bar Register
 		 * Write Tunnel register to match PSL TNR register
@@ -3990,6 +3999,99 @@  static int64_t phb4_set_capp_recovery(struct phb *phb)
 	return 0;
 }
 
+/*
+ * Return the indication of a PHB CMPM register.
+ */
+static int64_t phb4_get_cmpm(struct phb *phb, uint64_t phb_reg, uint64_t *ind)
+{
+	struct phb4 *p = phb_to_phb4(phb);
+	uint64_t offset;
+
+	switch (phb_reg) {
+	case OPAL_PHB_ASN_CMPM:
+		offset = PHB_ASN_CMPM;
+		break;
+	case OPAL_PHB_CAPI_CMPM:
+		offset = PHB_CAPI_CMPM;
+		break;
+	case OPAL_PHB_PBL_NBW_CMPM:
+		offset = PHB_PBL_NBW_CMPM;
+		break;
+	default:
+		return OPAL_UNSUPPORTED;
+	}
+
+	*ind = in_be64(p->regs + offset) >> 48;
+	return OPAL_SUCCESS;
+}
+
+/*
+ * Set the 16-bit indication of a PHB CMPM register.
+ * Set (hard-coded) mask value, and enable bit.
+ *
+ * This interface aims at letting linux choose the indications,
+ * if they need to be changed in the future.
+ *
+ * Compatibility with older versions of linux, that do not set CMPM
+ * indications with phb4_set_cmpm(), is ensured by enable_capi_mode(),
+ * that will set the default hard-coded values that were assumed then.
+ */
+static int64_t phb4_set_cmpm(struct phb *phb, uint64_t phb_reg, uint64_t ind)
+{
+	struct phb4 *p = phb_to_phb4(phb);
+	uint64_t mask, offset, enable;
+
+	/* Indication is a 16-bit value */
+	if (ind >> 16)
+		return OPAL_PARAMETER;
+	/*
+	 * The following bits of the PCI address are reserved:
+	 *   Bit 59 indicates TVE#1
+	 *   Bit 60 indicates MSI-X
+	 */
+	if (ind & 0x1800ull)
+		return OPAL_PARAMETER;
+
+	switch (phb_reg) {
+	case OPAL_PHB_ASN_CMPM:
+		/*
+		 * Init_25 - ASN Compare/Mask
+		 * matches XSL_DSNCTL[ASB_Addr]
+		 */
+		offset = PHB_ASN_CMPM;
+		enable = PHB_ASN_CMPM_ENABLE;
+		mask = 0xff00ull;
+		break;
+	case OPAL_PHB_CAPI_CMPM:
+		/*
+		 * Init_26 - CAPI Compare/Mask
+		 * matches XSL_DSNCTL[capiind]
+		 */
+		offset = PHB_CAPI_CMPM;
+		enable = PHB_CAPI_CMPM_ENABLE;
+		mask = 0xfe00ull;
+		break;
+	case OPAL_PHB_PBL_NBW_CMPM:
+		/*
+		 * Init_124 - NBW Compare/Mask
+		 * matches XSL_DSNCTL[nbwind]
+		 */
+		offset = PHB_PBL_NBW_CMPM;
+		enable = PHB_PBL_NBW_CMPM_ENABLE;
+		mask = 0xff00ull;
+		break;
+	default:
+		return OPAL_UNSUPPORTED;
+	}
+
+	/* Masked indication should not be null */
+	if (!(ind & mask))
+		return OPAL_PARAMETER;
+
+	out_be64(p->regs + offset, (ind << 48) | (mask << 32) | enable);
+	return OPAL_SUCCESS;
+}
+
 static const struct phb_ops phb4_ops = {
 	.cfg_read8		= phb4_pcicfg_read8,
 	.cfg_read16		= phb4_pcicfg_read16,
@@ -4025,6 +4127,8 @@  static const struct phb_ops phb4_ops = {
 	.set_capi_mode		= phb4_set_capi_mode,
 	.set_p2p		= phb4_set_p2p,
 	.set_capp_recovery	= phb4_set_capp_recovery,
+	.get_cmpm		= phb4_get_cmpm,
+	.set_cmpm		= phb4_set_cmpm,
 };
 
 static void phb4_init_ioda3(struct phb4 *p)
@@ -4059,10 +4163,10 @@  static void phb4_init_ioda3(struct phb4 *p)
 	/* See enable_capi_mode() */
 
 	/* Init_25 - ASN Compare/Mask */
-	/* See enable_capi_mode() */
+	/* See phb4_set_cmpm() and enable_capi_mode() */
 
 	/* Init_26 - CAPI Compare/Mask */
-	/* See enable_capi_mode() */
+	/* See phb4_set_cmpm() and enable_capi_mode() */
 
 	/* Init_27 - PCIE Outbound upper address */
 	out_be64(p->regs + PHB_M64_UPPER_BITS, 0);
@@ -4412,8 +4516,8 @@  static void phb4_init_hw(struct phb4 *p, bool first_init)
 	 * don't bother as we are doing a PERST soon.
 	 */
 
-	/* Init_124 :  NBW. XXX TODO */
-	/* See enable_capi_mode() */
+	/* Init_124 - NBW Compare/Mask */
+	/* See phb4_set_cmpm() and enable_capi_mode() */
 
 	/* Init_125 : Setup PCI command/status on root complex
 	 * I don't know why the spec does this now and not earlier, so
diff --git a/include/opal-api.h b/include/opal-api.h
index 0bc036ed..ce948136 100644
--- a/include/opal-api.h
+++ b/include/opal-api.h
@@ -214,7 +214,9 @@ 
 #define OPAL_SET_POWER_SHIFT_RATIO		155
 #define OPAL_SENSOR_GROUP_CLEAR			156
 #define OPAL_PCI_SET_P2P			157
-#define OPAL_LAST				157
+#define OPAL_PCI_GET_PHB_CMPM			158
+#define OPAL_PCI_SET_PHB_CMPM			159
+#define OPAL_LAST				159
 
 /* Device tree flags */
 
@@ -1274,6 +1276,13 @@  enum {
 	OPAL_PCI_P2P_TARGET	= 1,
 };
 
+/* PHB Compare/Mask registers */
+enum {
+	OPAL_PHB_ASN_CMPM	= 0,
+	OPAL_PHB_CAPI_CMPM	= 1,
+	OPAL_PHB_PBL_NBW_CMPM	= 2,
+};
+
 #endif /* __ASSEMBLY__ */
 
 #endif /* __OPAL_API_H */
diff --git a/include/pci.h b/include/pci.h
index c085b6b8..cdf82ee8 100644
--- a/include/pci.h
+++ b/include/pci.h
@@ -333,6 +333,10 @@  struct phb_ops {
 	/* PCI peer-to-peer setup */
 	void (*set_p2p)(struct phb *phb, uint64_t mode, uint64_t flags,
 			uint16_t pe_number);
+
+	/* Get/set PHB Compare/Mask registers */
+	int64_t (*get_cmpm)(struct phb *phb, uint64_t phb_reg, uint64_t *ind);
+	int64_t (*set_cmpm)(struct phb *phb, uint64_t phb_reg, uint64_t ind);
 };
 
 enum phb_type {
diff --git a/include/phb4-regs.h b/include/phb4-regs.h
index e83c8c39..b33185ed 100644
--- a/include/phb4-regs.h
+++ b/include/phb4-regs.h
@@ -71,8 +71,8 @@ 
 #define PHB_PEST_BAR			0x1a8
 #define   PHB_PEST_BAR_ENABLE		PPC_BIT(0)
 #define   PHB_PEST_BASE_ADDRESS		PPC_BITMASK(8,51)
-#define PHB_PBL_ASN_CMPM		0x1C0
-#define   PHB_PBL_ASN_ENABLE		PPC_BIT(63)
+#define PHB_ASN_CMPM			0x1C0
+#define   PHB_ASN_CMPM_ENABLE		PPC_BIT(63)
 #define PHB_CAPI_CMPM			0x1C8
 #define   PHB_CAPI_CMPM_ENABLE		PPC_BIT(63)
 #define PHB_M64_UPPER_BITS		0x1f0
@@ -250,8 +250,8 @@ 
 #define PHB_PBL_CONTROL				0x1800
 #define PHB_PBL_TIMEOUT_CTRL			0x1810
 #define PHB_PBL_NPTAG_ENABLE			0x1820
-#define PHB_PBL_NBW_CMP_MASK			0x1830
-#define   PHB_PBL_NBW_MASK_ENABLE		PPC_BIT(63)
+#define PHB_PBL_NBW_CMPM			0x1830
+#define   PHB_PBL_NBW_CMPM_ENABLE		PPC_BIT(63)
 #define PHB_PBL_SYS_LINK_INIT			0x1838
 #define PHB_PBL_BUF_STATUS			0x1840
 #define PHB_PBL_ERR_STATUS			0x1900