[v4,2/2] phb4: set PBCQ Tunnel BAR for tunneled operations

Message ID 20180110144736.18711-2-felix@linux.vnet.ibm.com
State Changes Requested
Headers show
Series
  • [v4,1/2] phb4: set PHB CMPM registers for tunneled operations
Related show

Commit Message

Philippe Bergheaud Jan. 10, 2018, 2:47 p.m.
P9 supports PCI tunneled operations (atomics and as_notify) that are
initiated by devices.

A subset of the tunneled operations require a response, that must be
sent back from the host to the device. For example, an atomic compare
and swap will return the compare status, as swap will only performed
in case of success.  Similarly, as_notify reports if the target thread
has been woken up or not, because the operation may fail.

To enable tunneled operations, a device driver must tell the host where
it expects tunneled operation responses, by setting the PBCQ Tunnel BAR
Response register with a specific value within the range of its BARs.

This register is currently initialized by enable_capi_mode(). But, as
tunneled operations may also operate in PCI mode, a new API is required
to set the PBCQ Tunnel BAR Response register, without switching to CAPI
mode.

This patch provides two new OPAL calls to get/set the PBCQ Tunnel
BAR Response register.

Note: as there is only one PBCQ Tunnel BAR register, shared between
all the devices connected to the same PHB, only one of these devices
will be able to use tunneled operations, at any time.

Signed-off-by: Philippe Bergheaud <felix@linux.vnet.ibm.com>
---
Changelog:

v2: Adjust new OPAL call numbers.

v3: Rebase on skiboot 5.9.6.
    Document new OPAL calls in doc/opal-api.

v4: opal_pci_get_pbcq_tunnel_bar():
      - check that addr parameter is valid
      - update doc/opal-api
    enable_capi_mode():
      - explain why we do not touch an existing tunnel bar value
      - reuse psl tnr addr value verbatim, and shift it by 8
---
 core/pci-opal.c                                    | 36 ++++++++++
 .../opal-pci-get-set-pbcq-tunnel-bar-159-160.rst   | 76 ++++++++++++++++++++++
 hw/phb4.c                                          | 71 ++++++++++++++++++--
 include/opal-api.h                                 |  4 +-
 include/pci.h                                      |  4 ++
 5 files changed, 185 insertions(+), 6 deletions(-)
 create mode 100644 doc/opal-api/opal-pci-get-set-pbcq-tunnel-bar-159-160.rst

Comments

Frederic Barrat Jan. 10, 2018, 6:56 p.m. | #1
Le 10/01/2018 à 15:47, Philippe Bergheaud a écrit :
> P9 supports PCI tunneled operations (atomics and as_notify) that are
> initiated by devices.
> 
> A subset of the tunneled operations require a response, that must be
> sent back from the host to the device. For example, an atomic compare
> and swap will return the compare status, as swap will only performed
> in case of success.  Similarly, as_notify reports if the target thread
> has been woken up or not, because the operation may fail.
> 
> To enable tunneled operations, a device driver must tell the host where
> it expects tunneled operation responses, by setting the PBCQ Tunnel BAR
> Response register with a specific value within the range of its BARs.
> 
> This register is currently initialized by enable_capi_mode(). But, as
> tunneled operations may also operate in PCI mode, a new API is required
> to set the PBCQ Tunnel BAR Response register, without switching to CAPI
> mode.
> 
> This patch provides two new OPAL calls to get/set the PBCQ Tunnel
> BAR Response register.
> 
> Note: as there is only one PBCQ Tunnel BAR register, shared between
> all the devices connected to the same PHB, only one of these devices
> will be able to use tunneled operations, at any time.
> 
> Signed-off-by: Philippe Bergheaud <felix@linux.vnet.ibm.com>
> ---

Reviewed-by: Frederic Barrat <fbarrat@linux.vnet.ibm.com>


> Changelog:
> 
> v2: Adjust new OPAL call numbers.
> 
> v3: Rebase on skiboot 5.9.6.
>      Document new OPAL calls in doc/opal-api.
> 
> v4: opal_pci_get_pbcq_tunnel_bar():
>        - check that addr parameter is valid
>        - update doc/opal-api
>      enable_capi_mode():
>        - explain why we do not touch an existing tunnel bar value
>        - reuse psl tnr addr value verbatim, and shift it by 8
> ---
>   core/pci-opal.c                                    | 36 ++++++++++
>   .../opal-pci-get-set-pbcq-tunnel-bar-159-160.rst   | 76 ++++++++++++++++++++++
>   hw/phb4.c                                          | 71 ++++++++++++++++++--
>   include/opal-api.h                                 |  4 +-
>   include/pci.h                                      |  4 ++
>   5 files changed, 185 insertions(+), 6 deletions(-)
>   create mode 100644 doc/opal-api/opal-pci-get-set-pbcq-tunnel-bar-159-160.rst
> 
> diff --git a/core/pci-opal.c b/core/pci-opal.c
> index b8aec941..77e965cc 100644
> --- a/core/pci-opal.c
> +++ b/core/pci-opal.c
> @@ -1016,3 +1016,39 @@ static int64_t opal_pci_set_p2p(uint64_t phbid_init, uint64_t phbid_target,
>   	return OPAL_SUCCESS;
>   }
>   opal_call(OPAL_PCI_SET_P2P, opal_pci_set_p2p, 4);
> +
> +static int64_t opal_pci_get_pbcq_tunnel_bar(uint64_t phb_id, uint64_t *addr)
> +{
> +	struct phb *phb = pci_get_phb(phb_id);
> +
> +	if (!opal_addr_valid(addr))
> +		return OPAL_PARAMETER;
> +
> +	if (!phb)
> +		return OPAL_PARAMETER;
> +	if (!phb->ops->get_tunnel_bar)
> +		return OPAL_UNSUPPORTED;
> +
> +	phb_lock(phb);
> +	phb->ops->get_tunnel_bar(phb, addr);
> +	phb_unlock(phb);
> +	return OPAL_SUCCESS;
> +}
> +opal_call(OPAL_PCI_GET_PBCQ_TUNNEL_BAR, opal_pci_get_pbcq_tunnel_bar, 2);
> +
> +static int64_t opal_pci_set_pbcq_tunnel_bar(uint64_t phb_id, uint64_t addr)
> +{
> +	struct phb *phb = pci_get_phb(phb_id);
> +	int64_t rc;
> +
> +	if (!phb)
> +		return OPAL_PARAMETER;
> +	if (!phb->ops->set_tunnel_bar)
> +		return OPAL_UNSUPPORTED;
> +
> +	phb_lock(phb);
> +	rc = phb->ops->set_tunnel_bar(phb, addr);
> +	phb_unlock(phb);
> +	return rc;
> +}
> +opal_call(OPAL_PCI_SET_PBCQ_TUNNEL_BAR, opal_pci_set_pbcq_tunnel_bar, 2);
> diff --git a/doc/opal-api/opal-pci-get-set-pbcq-tunnel-bar-159-160.rst b/doc/opal-api/opal-pci-get-set-pbcq-tunnel-bar-159-160.rst
> new file mode 100644
> index 00000000..6e99d8ac
> --- /dev/null
> +++ b/doc/opal-api/opal-pci-get-set-pbcq-tunnel-bar-159-160.rst
> @@ -0,0 +1,76 @@
> +OPAL_PCI_GET_PBCQ_TUNNEL_BAR
> +============================
> +::
> +
> +   #define OPAL_PCI_GET_PBCQ_TUNNEL_BAR 160
> +
> +   int64_t opal_pci_get_pbcq_tunnel_bar(uint64_t phb_id, uint64_t *addr)
> +
> +The host calls this function to read the address out of the PBCQ Tunnel
> +Bar register.
> +
> +Parameters
> +----------
> +::
> +
> +   uint64_t phb_id
> +   uint64_t *addr
> +
> +``phb_id``
> +  The value from the PHB node ibm,opal-phbid property for the device.
> +
> +``addr``
> +  A pointer to where the address stored in the PBCQ Tunnel Bar register
> +  will be copied.
> +
> +Return Values
> +-------------
> +
> +``OPAL_SUCCESS``
> +  Operation was successful
> +
> +``OPAL_PARAMETER``
> +  Invalid PHB or addr parameter
> +
> +``OPAL_UNSUPPORTED``
> +  Not supported by hardware
> +
> +OPAL_PCI_SET_PBCQ_TUNNEL_BAR
> +============================
> +::
> +
> +   #define OPAL_PCI_SET_PBCQ_TUNNEL_BAR 161
> +
> +   int64_t opal_pci_set_pbcq_tunnel_bar(uint64_t phb_id, uint64_t addr)
> +
> +The host calls this function to set the PBCQ Tunnel Bar register.
> +
> +Parameters
> +----------
> +::
> +
> +   uint64_t phb_id
> +   uint64_t addr
> +
> +``phb_id``
> +  The value from the PHB node ibm,opal-phbid property for the device.
> +
> +``addr``
> +  The value of the address chosen for the PBCQ Tunnel Bar register.
> +  If the address is 0, then the PBCQ Tunnel Bar register will be reset.
> +  It the address is non-zero, then the PBCQ Tunnel Bar register will be
> +  set with ::
> +
> +   Bit[0:42]      Bit[8:50] of the address
> +
> +Return Values
> +-------------
> +
> +``OPAL_SUCCESS``
> +  Operation was successful
> +
> +``OPAL_PARAMETER``
> +  Invalid PHB or addr parameter
> +
> +``OPAL_UNSUPPORTED``
> +  Not supported by hardware
> diff --git a/hw/phb4.c b/hw/phb4.c
> index bbeca73b..86683691 100644
> --- a/hw/phb4.c
> +++ b/hw/phb4.c
> @@ -3799,12 +3799,24 @@ static int64_t enable_capi_mode(struct phb4 *p, uint64_t pe_number,
>   		 ((u64)CAPIMASK << 32) | PHB_CAPI_CMPM_ENABLE);
> 
>   	if (!(p->rev == PHB4_REV_NIMBUS_DD10)) {
> -		/* PBCQ Tunnel Bar Register
> -		 * Write Tunnel register to match PSL TNR register
> +		/*
> +		 * PBCQ Tunnel Bar Register
> +		 *
> +		 * If set, for example by a driver that may already have
> +		 * tweaked the tunnel bar, then we do not touch it when
> +		 * entering capi mode. It's up to the driver to handle it.
> +		 *
> +		 * If unset, then we use the PSL_TNR_ADDR[TNR_Addr] reset
> +		 * value. For fpga/cxl, this code will define the tunnel bar.
>   		 */
> -		xscom_write(p->chip_id,
> -			    p->pe_stk_xscom + XPEC_NEST_STK_TUNNEL_BAR,
> -			    0x020000E000000000);
> +		xscom_read(p->chip_id,
> +			   p->pe_stk_xscom + XPEC_NEST_STK_TUNNEL_BAR, &reg);
> +		if (!reg) {
> +			reg = 0x00020000E0000000ull << 8;
> +			xscom_write(p->chip_id,
> +				    p->pe_stk_xscom + XPEC_NEST_STK_TUNNEL_BAR,
> +				    reg);
> +		}
> 
>   		/* PB AIB Hardware Control Register
>   		 * Wait 32 PCI clocks for a credit to become available
> @@ -4051,6 +4063,53 @@ static int64_t phb4_set_capp_recovery(struct phb *phb)
>   	return 0;
>   }
> 
> +/*
> + * Return the address out of a PBCQ Tunnel Bar register.
> + */
> +static void phb4_get_tunnel_bar(struct phb *phb, uint64_t *addr)
> +{
> +	struct phb4 *p = phb_to_phb4(phb);
> +	uint64_t val;
> +
> +	xscom_read(p->chip_id, p->pe_stk_xscom + XPEC_NEST_STK_TUNNEL_BAR,
> +		   &val);
> +	*addr = val >> 8;
> +}
> +
> +/*
> + * Set PBCQ Tunnel Bar register.
> + * Store addr bits [8:50] in PBCQ Tunnel Bar register bits [0:42].
> + * Note that addr bits [8:50] must also match PSL_TNR_ADDR[8:50].
> + * Reset register if val == 0.
> + *
> + * This interface is required to let device drivers set the Tunnel Bar
> + * value of their choice.
> + *
> + * Compatibility with older versions of linux, that do not set the
> + * Tunnel Bar with phb4_set_tunnel_bar(), is ensured by enable_capi_mode(),
> + * that will set the default value that used to be assumed.
> + */
> +static int64_t phb4_set_tunnel_bar(struct phb *phb, uint64_t addr)
> +{
> +	struct phb4 *p = phb_to_phb4(phb);
> +	uint64_t mask = 0x00FFFFFFFFFFE000ULL;
> +
> +	if (!addr) {
> +		/* Reset register */
> +		xscom_write(p->chip_id,
> +			    p->pe_stk_xscom + XPEC_NEST_STK_TUNNEL_BAR, addr);
> +		return OPAL_SUCCESS;
> +	}
> +	if ((addr & ~mask))
> +		return OPAL_PARAMETER;
> +	if (!(addr & mask))
> +		return OPAL_PARAMETER;
> +
> +	xscom_write(p->chip_id, p->pe_stk_xscom + XPEC_NEST_STK_TUNNEL_BAR,
> +		    (addr & mask) << 8);
> +	return OPAL_SUCCESS;
> +}
> +
>   static const struct phb_ops phb4_ops = {
>   	.cfg_read8		= phb4_pcicfg_read8,
>   	.cfg_read16		= phb4_pcicfg_read16,
> @@ -4086,6 +4145,8 @@ static const struct phb_ops phb4_ops = {
>   	.set_capi_mode		= phb4_set_capi_mode,
>   	.set_p2p		= phb4_set_p2p,
>   	.set_capp_recovery	= phb4_set_capp_recovery,
> +	.get_tunnel_bar         = phb4_get_tunnel_bar,
> +	.set_tunnel_bar         = phb4_set_tunnel_bar,
>   };
> 
>   static void phb4_init_ioda3(struct phb4 *p)
> diff --git a/include/opal-api.h b/include/opal-api.h
> index f0ed5f6c..ac877046 100644
> --- a/include/opal-api.h
> +++ b/include/opal-api.h
> @@ -215,7 +215,9 @@
>   #define OPAL_SENSOR_GROUP_CLEAR			156
>   #define OPAL_PCI_SET_P2P			157
>   #define OPAL_QUIESCE				158
> -#define OPAL_LAST				158
> +#define OPAL_PCI_GET_PBCQ_TUNNEL_BAR		159
> +#define OPAL_PCI_SET_PBCQ_TUNNEL_BAR		160
> +#define OPAL_LAST				160
> 
>   #define QUIESCE_HOLD			1 /* Spin all calls at entry */
>   #define QUIESCE_REJECT			2 /* Fail all calls with OPAL_BUSY */
> diff --git a/include/pci.h b/include/pci.h
> index c085b6b8..97c8dfe4 100644
> --- a/include/pci.h
> +++ b/include/pci.h
> @@ -333,6 +333,10 @@ struct phb_ops {
>   	/* PCI peer-to-peer setup */
>   	void (*set_p2p)(struct phb *phb, uint64_t mode, uint64_t flags,
>   			uint16_t pe_number);
> +
> +	/* Get/set PBCQ Tunnel BAR register */
> +	void (*get_tunnel_bar)(struct phb *phb, uint64_t *addr);
> +	int64_t (*set_tunnel_bar)(struct phb *phb, uint64_t addr);
>   };
> 
>   enum phb_type {
>
christophe lombard Jan. 22, 2018, 10:11 a.m. | #2
Le 10/01/2018 à 15:47, Philippe Bergheaud a écrit :
> P9 supports PCI tunneled operations (atomics and as_notify) that are
> initiated by devices.
> 
> A subset of the tunneled operations require a response, that must be
> sent back from the host to the device. For example, an atomic compare
> and swap will return the compare status, as swap will only performed
> in case of success.  Similarly, as_notify reports if the target thread
> has been woken up or not, because the operation may fail.
> 
> To enable tunneled operations, a device driver must tell the host where
> it expects tunneled operation responses, by setting the PBCQ Tunnel BAR
> Response register with a specific value within the range of its BARs.
> 
> This register is currently initialized by enable_capi_mode(). But, as
> tunneled operations may also operate in PCI mode, a new API is required
> to set the PBCQ Tunnel BAR Response register, without switching to CAPI
> mode.
> 
> This patch provides two new OPAL calls to get/set the PBCQ Tunnel
> BAR Response register.
> 
> Note: as there is only one PBCQ Tunnel BAR register, shared between
> all the devices connected to the same PHB, only one of these devices
> will be able to use tunneled operations, at any time.
> 
> Signed-off-by: Philippe Bergheaud <felix@linux.vnet.ibm.com>
> ---
> Changelog:
> 
> v2: Adjust new OPAL call numbers.
> 
> v3: Rebase on skiboot 5.9.6.
>      Document new OPAL calls in doc/opal-api.
> 
> v4: opal_pci_get_pbcq_tunnel_bar():
>        - check that addr parameter is valid
>        - update doc/opal-api
>      enable_capi_mode():
>        - explain why we do not touch an existing tunnel bar value
>        - reuse psl tnr addr value verbatim, and shift it by 8
> ---
>   core/pci-opal.c                                    | 36 ++++++++++
>   .../opal-pci-get-set-pbcq-tunnel-bar-159-160.rst   | 76 ++++++++++++++++++++++
>   hw/phb4.c                                          | 71 ++++++++++++++++++--
>   include/opal-api.h                                 |  4 +-
>   include/pci.h                                      |  4 ++
>   5 files changed, 185 insertions(+), 6 deletions(-)
>   create mode 100644 doc/opal-api/opal-pci-get-set-pbcq-tunnel-bar-159-160.rst
> 

Reviewed-by: Christophe Lombard clombard@linux.vnet.ibm.com
Philippe Bergheaud Jan. 22, 2018, 1:53 p.m. | #3
On 10/01/2018 15:47, Philippe Bergheaud wrote:
> P9 supports PCI tunneled operations (atomics and as_notify) that are
> initiated by devices.
>
> A subset of the tunneled operations require a response, that must be
> sent back from the host to the device. For example, an atomic compare
> and swap will return the compare status, as swap will only performed
> in case of success.  Similarly, as_notify reports if the target thread
> has been woken up or not, because the operation may fail.
>
> To enable tunneled operations, a device driver must tell the host where
> it expects tunneled operation responses, by setting the PBCQ Tunnel BAR
> Response register with a specific value within the range of its BARs.
>
> This register is currently initialized by enable_capi_mode(). But, as
> tunneled operations may also operate in PCI mode, a new API is required
> to set the PBCQ Tunnel BAR Response register, without switching to CAPI
> mode.
>
> This patch provides two new OPAL calls to get/set the PBCQ Tunnel
> BAR Response register.
>
> Note: as there is only one PBCQ Tunnel BAR register, shared between
> all the devices connected to the same PHB, only one of these devices
> will be able to use tunneled operations, at any time.
>
> Signed-off-by: Philippe Bergheaud <felix@linux.vnet.ibm.com>
> ---
> Changelog:
>
> v2: Adjust new OPAL call numbers.
>
> v3: Rebase on skiboot 5.9.6.
>      Document new OPAL calls in doc/opal-api.
>
> v4: opal_pci_get_pbcq_tunnel_bar():
>        - check that addr parameter is valid
>        - update doc/opal-api
>      enable_capi_mode():
>        - explain why we do not touch an existing tunnel bar value
>        - reuse psl tnr addr value verbatim, and shift it by 8

Linux patches:
https://patchwork.ozlabs.org/project/linuxppc-dev/list/?series=24680

> ---
>   core/pci-opal.c                                    | 36 ++++++++++
>   .../opal-pci-get-set-pbcq-tunnel-bar-159-160.rst   | 76 ++++++++++++++++++++++
>   hw/phb4.c                                          | 71 ++++++++++++++++++--
>   include/opal-api.h                                 |  4 +-
>   include/pci.h                                      |  4 ++
>   5 files changed, 185 insertions(+), 6 deletions(-)
>   create mode 100644 doc/opal-api/opal-pci-get-set-pbcq-tunnel-bar-159-160.rst
>
> diff --git a/core/pci-opal.c b/core/pci-opal.c
> index b8aec941..77e965cc 100644
> --- a/core/pci-opal.c
> +++ b/core/pci-opal.c
> @@ -1016,3 +1016,39 @@ static int64_t opal_pci_set_p2p(uint64_t phbid_init, uint64_t phbid_target,
>   	return OPAL_SUCCESS;
>   }
>   opal_call(OPAL_PCI_SET_P2P, opal_pci_set_p2p, 4);
> +
> +static int64_t opal_pci_get_pbcq_tunnel_bar(uint64_t phb_id, uint64_t *addr)
> +{
> +	struct phb *phb = pci_get_phb(phb_id);
> +
> +	if (!opal_addr_valid(addr))
> +		return OPAL_PARAMETER;
> +
> +	if (!phb)
> +		return OPAL_PARAMETER;
> +	if (!phb->ops->get_tunnel_bar)
> +		return OPAL_UNSUPPORTED;
> +
> +	phb_lock(phb);
> +	phb->ops->get_tunnel_bar(phb, addr);
> +	phb_unlock(phb);
> +	return OPAL_SUCCESS;
> +}
> +opal_call(OPAL_PCI_GET_PBCQ_TUNNEL_BAR, opal_pci_get_pbcq_tunnel_bar, 2);
> +
> +static int64_t opal_pci_set_pbcq_tunnel_bar(uint64_t phb_id, uint64_t addr)
> +{
> +	struct phb *phb = pci_get_phb(phb_id);
> +	int64_t rc;
> +
> +	if (!phb)
> +		return OPAL_PARAMETER;
> +	if (!phb->ops->set_tunnel_bar)
> +		return OPAL_UNSUPPORTED;
> +
> +	phb_lock(phb);
> +	rc = phb->ops->set_tunnel_bar(phb, addr);
> +	phb_unlock(phb);
> +	return rc;
> +}
> +opal_call(OPAL_PCI_SET_PBCQ_TUNNEL_BAR, opal_pci_set_pbcq_tunnel_bar, 2);
> diff --git a/doc/opal-api/opal-pci-get-set-pbcq-tunnel-bar-159-160.rst b/doc/opal-api/opal-pci-get-set-pbcq-tunnel-bar-159-160.rst
> new file mode 100644
> index 00000000..6e99d8ac
> --- /dev/null
> +++ b/doc/opal-api/opal-pci-get-set-pbcq-tunnel-bar-159-160.rst
> @@ -0,0 +1,76 @@
> +OPAL_PCI_GET_PBCQ_TUNNEL_BAR
> +============================
> +::
> +
> +   #define OPAL_PCI_GET_PBCQ_TUNNEL_BAR 160
> +
> +   int64_t opal_pci_get_pbcq_tunnel_bar(uint64_t phb_id, uint64_t *addr)
> +
> +The host calls this function to read the address out of the PBCQ Tunnel
> +Bar register.
> +
> +Parameters
> +----------
> +::
> +
> +   uint64_t phb_id
> +   uint64_t *addr
> +
> +``phb_id``
> +  The value from the PHB node ibm,opal-phbid property for the device.
> +
> +``addr``
> +  A pointer to where the address stored in the PBCQ Tunnel Bar register
> +  will be copied.
> +
> +Return Values
> +-------------
> +
> +``OPAL_SUCCESS``
> +  Operation was successful
> +
> +``OPAL_PARAMETER``
> +  Invalid PHB or addr parameter
> +
> +``OPAL_UNSUPPORTED``
> +  Not supported by hardware
> +
> +OPAL_PCI_SET_PBCQ_TUNNEL_BAR
> +============================
> +::
> +
> +   #define OPAL_PCI_SET_PBCQ_TUNNEL_BAR 161
> +
> +   int64_t opal_pci_set_pbcq_tunnel_bar(uint64_t phb_id, uint64_t addr)
> +
> +The host calls this function to set the PBCQ Tunnel Bar register.
> +
> +Parameters
> +----------
> +::
> +
> +   uint64_t phb_id
> +   uint64_t addr
> +
> +``phb_id``
> +  The value from the PHB node ibm,opal-phbid property for the device.
> +
> +``addr``
> +  The value of the address chosen for the PBCQ Tunnel Bar register.
> +  If the address is 0, then the PBCQ Tunnel Bar register will be reset.
> +  It the address is non-zero, then the PBCQ Tunnel Bar register will be
> +  set with ::
> +
> +   Bit[0:42]      Bit[8:50] of the address
> +
> +Return Values
> +-------------
> +
> +``OPAL_SUCCESS``
> +  Operation was successful
> +
> +``OPAL_PARAMETER``
> +  Invalid PHB or addr parameter
> +
> +``OPAL_UNSUPPORTED``
> +  Not supported by hardware
> diff --git a/hw/phb4.c b/hw/phb4.c
> index bbeca73b..86683691 100644
> --- a/hw/phb4.c
> +++ b/hw/phb4.c
> @@ -3799,12 +3799,24 @@ static int64_t enable_capi_mode(struct phb4 *p, uint64_t pe_number,
>   		 ((u64)CAPIMASK << 32) | PHB_CAPI_CMPM_ENABLE);
>
>   	if (!(p->rev == PHB4_REV_NIMBUS_DD10)) {
> -		/* PBCQ Tunnel Bar Register
> -		 * Write Tunnel register to match PSL TNR register
> +		/*
> +		 * PBCQ Tunnel Bar Register
> +		 *
> +		 * If set, for example by a driver that may already have
> +		 * tweaked the tunnel bar, then we do not touch it when
> +		 * entering capi mode. It's up to the driver to handle it.
> +		 *
> +		 * If unset, then we use the PSL_TNR_ADDR[TNR_Addr] reset
> +		 * value. For fpga/cxl, this code will define the tunnel bar.
>   		 */
> -		xscom_write(p->chip_id,
> -			    p->pe_stk_xscom + XPEC_NEST_STK_TUNNEL_BAR,
> -			    0x020000E000000000);
> +		xscom_read(p->chip_id,
> +			   p->pe_stk_xscom + XPEC_NEST_STK_TUNNEL_BAR, &reg);
> +		if (!reg) {
> +			reg = 0x00020000E0000000ull << 8;
> +			xscom_write(p->chip_id,
> +				    p->pe_stk_xscom + XPEC_NEST_STK_TUNNEL_BAR,
> +				    reg);
> +		}
>
>   		/* PB AIB Hardware Control Register
>   		 * Wait 32 PCI clocks for a credit to become available
> @@ -4051,6 +4063,53 @@ static int64_t phb4_set_capp_recovery(struct phb *phb)
>   	return 0;
>   }
>
> +/*
> + * Return the address out of a PBCQ Tunnel Bar register.
> + */
> +static void phb4_get_tunnel_bar(struct phb *phb, uint64_t *addr)
> +{
> +	struct phb4 *p = phb_to_phb4(phb);
> +	uint64_t val;
> +
> +	xscom_read(p->chip_id, p->pe_stk_xscom + XPEC_NEST_STK_TUNNEL_BAR,
> +		   &val);
> +	*addr = val >> 8;
> +}
> +
> +/*
> + * Set PBCQ Tunnel Bar register.
> + * Store addr bits [8:50] in PBCQ Tunnel Bar register bits [0:42].
> + * Note that addr bits [8:50] must also match PSL_TNR_ADDR[8:50].
> + * Reset register if val == 0.
> + *
> + * This interface is required to let device drivers set the Tunnel Bar
> + * value of their choice.
> + *
> + * Compatibility with older versions of linux, that do not set the
> + * Tunnel Bar with phb4_set_tunnel_bar(), is ensured by enable_capi_mode(),
> + * that will set the default value that used to be assumed.
> + */
> +static int64_t phb4_set_tunnel_bar(struct phb *phb, uint64_t addr)
> +{
> +	struct phb4 *p = phb_to_phb4(phb);
> +	uint64_t mask = 0x00FFFFFFFFFFE000ULL;
> +
> +	if (!addr) {
> +		/* Reset register */
> +		xscom_write(p->chip_id,
> +			    p->pe_stk_xscom + XPEC_NEST_STK_TUNNEL_BAR, addr);
> +		return OPAL_SUCCESS;
> +	}
> +	if ((addr & ~mask))
> +		return OPAL_PARAMETER;
> +	if (!(addr & mask))
> +		return OPAL_PARAMETER;
> +
> +	xscom_write(p->chip_id, p->pe_stk_xscom + XPEC_NEST_STK_TUNNEL_BAR,
> +		    (addr & mask) << 8);
> +	return OPAL_SUCCESS;
> +}
> +
>   static const struct phb_ops phb4_ops = {
>   	.cfg_read8		= phb4_pcicfg_read8,
>   	.cfg_read16		= phb4_pcicfg_read16,
> @@ -4086,6 +4145,8 @@ static const struct phb_ops phb4_ops = {
>   	.set_capi_mode		= phb4_set_capi_mode,
>   	.set_p2p		= phb4_set_p2p,
>   	.set_capp_recovery	= phb4_set_capp_recovery,
> +	.get_tunnel_bar         = phb4_get_tunnel_bar,
> +	.set_tunnel_bar         = phb4_set_tunnel_bar,
>   };
>
>   static void phb4_init_ioda3(struct phb4 *p)
> diff --git a/include/opal-api.h b/include/opal-api.h
> index f0ed5f6c..ac877046 100644
> --- a/include/opal-api.h
> +++ b/include/opal-api.h
> @@ -215,7 +215,9 @@
>   #define OPAL_SENSOR_GROUP_CLEAR			156
>   #define OPAL_PCI_SET_P2P			157
>   #define OPAL_QUIESCE				158
> -#define OPAL_LAST				158
> +#define OPAL_PCI_GET_PBCQ_TUNNEL_BAR		159
> +#define OPAL_PCI_SET_PBCQ_TUNNEL_BAR		160
> +#define OPAL_LAST				160
>
>   #define QUIESCE_HOLD			1 /* Spin all calls at entry */
>   #define QUIESCE_REJECT			2 /* Fail all calls with OPAL_BUSY */
> diff --git a/include/pci.h b/include/pci.h
> index c085b6b8..97c8dfe4 100644
> --- a/include/pci.h
> +++ b/include/pci.h
> @@ -333,6 +333,10 @@ struct phb_ops {
>   	/* PCI peer-to-peer setup */
>   	void (*set_p2p)(struct phb *phb, uint64_t mode, uint64_t flags,
>   			uint16_t pe_number);
> +
> +	/* Get/set PBCQ Tunnel BAR register */
> +	void (*get_tunnel_bar)(struct phb *phb, uint64_t *addr);
> +	int64_t (*set_tunnel_bar)(struct phb *phb, uint64_t addr);
>   };
>
>   enum phb_type {

Patch

diff --git a/core/pci-opal.c b/core/pci-opal.c
index b8aec941..77e965cc 100644
--- a/core/pci-opal.c
+++ b/core/pci-opal.c
@@ -1016,3 +1016,39 @@  static int64_t opal_pci_set_p2p(uint64_t phbid_init, uint64_t phbid_target,
 	return OPAL_SUCCESS;
 }
 opal_call(OPAL_PCI_SET_P2P, opal_pci_set_p2p, 4);
+
+static int64_t opal_pci_get_pbcq_tunnel_bar(uint64_t phb_id, uint64_t *addr)
+{
+	struct phb *phb = pci_get_phb(phb_id);
+
+	if (!opal_addr_valid(addr))
+		return OPAL_PARAMETER;
+
+	if (!phb)
+		return OPAL_PARAMETER;
+	if (!phb->ops->get_tunnel_bar)
+		return OPAL_UNSUPPORTED;
+
+	phb_lock(phb);
+	phb->ops->get_tunnel_bar(phb, addr);
+	phb_unlock(phb);
+	return OPAL_SUCCESS;
+}
+opal_call(OPAL_PCI_GET_PBCQ_TUNNEL_BAR, opal_pci_get_pbcq_tunnel_bar, 2);
+
+static int64_t opal_pci_set_pbcq_tunnel_bar(uint64_t phb_id, uint64_t addr)
+{
+	struct phb *phb = pci_get_phb(phb_id);
+	int64_t rc;
+
+	if (!phb)
+		return OPAL_PARAMETER;
+	if (!phb->ops->set_tunnel_bar)
+		return OPAL_UNSUPPORTED;
+
+	phb_lock(phb);
+	rc = phb->ops->set_tunnel_bar(phb, addr);
+	phb_unlock(phb);
+	return rc;
+}
+opal_call(OPAL_PCI_SET_PBCQ_TUNNEL_BAR, opal_pci_set_pbcq_tunnel_bar, 2);
diff --git a/doc/opal-api/opal-pci-get-set-pbcq-tunnel-bar-159-160.rst b/doc/opal-api/opal-pci-get-set-pbcq-tunnel-bar-159-160.rst
new file mode 100644
index 00000000..6e99d8ac
--- /dev/null
+++ b/doc/opal-api/opal-pci-get-set-pbcq-tunnel-bar-159-160.rst
@@ -0,0 +1,76 @@ 
+OPAL_PCI_GET_PBCQ_TUNNEL_BAR
+============================
+::
+
+   #define OPAL_PCI_GET_PBCQ_TUNNEL_BAR 160
+
+   int64_t opal_pci_get_pbcq_tunnel_bar(uint64_t phb_id, uint64_t *addr)
+
+The host calls this function to read the address out of the PBCQ Tunnel
+Bar register.
+
+Parameters
+----------
+::
+
+   uint64_t phb_id
+   uint64_t *addr
+
+``phb_id``
+  The value from the PHB node ibm,opal-phbid property for the device.
+
+``addr``
+  A pointer to where the address stored in the PBCQ Tunnel Bar register
+  will be copied.
+
+Return Values
+-------------
+
+``OPAL_SUCCESS``
+  Operation was successful
+
+``OPAL_PARAMETER``
+  Invalid PHB or addr parameter
+
+``OPAL_UNSUPPORTED``
+  Not supported by hardware
+
+OPAL_PCI_SET_PBCQ_TUNNEL_BAR
+============================
+::
+
+   #define OPAL_PCI_SET_PBCQ_TUNNEL_BAR 161
+
+   int64_t opal_pci_set_pbcq_tunnel_bar(uint64_t phb_id, uint64_t addr)
+
+The host calls this function to set the PBCQ Tunnel Bar register.
+
+Parameters
+----------
+::
+
+   uint64_t phb_id
+   uint64_t addr
+
+``phb_id``
+  The value from the PHB node ibm,opal-phbid property for the device.
+
+``addr``
+  The value of the address chosen for the PBCQ Tunnel Bar register.
+  If the address is 0, then the PBCQ Tunnel Bar register will be reset.
+  It the address is non-zero, then the PBCQ Tunnel Bar register will be
+  set with ::
+
+   Bit[0:42]      Bit[8:50] of the address
+
+Return Values
+-------------
+
+``OPAL_SUCCESS``
+  Operation was successful
+
+``OPAL_PARAMETER``
+  Invalid PHB or addr parameter
+
+``OPAL_UNSUPPORTED``
+  Not supported by hardware
diff --git a/hw/phb4.c b/hw/phb4.c
index bbeca73b..86683691 100644
--- a/hw/phb4.c
+++ b/hw/phb4.c
@@ -3799,12 +3799,24 @@  static int64_t enable_capi_mode(struct phb4 *p, uint64_t pe_number,
 		 ((u64)CAPIMASK << 32) | PHB_CAPI_CMPM_ENABLE);
 
 	if (!(p->rev == PHB4_REV_NIMBUS_DD10)) {
-		/* PBCQ Tunnel Bar Register
-		 * Write Tunnel register to match PSL TNR register
+		/*
+		 * PBCQ Tunnel Bar Register
+		 *
+		 * If set, for example by a driver that may already have
+		 * tweaked the tunnel bar, then we do not touch it when
+		 * entering capi mode. It's up to the driver to handle it.
+		 *
+		 * If unset, then we use the PSL_TNR_ADDR[TNR_Addr] reset
+		 * value. For fpga/cxl, this code will define the tunnel bar.
 		 */
-		xscom_write(p->chip_id,
-			    p->pe_stk_xscom + XPEC_NEST_STK_TUNNEL_BAR,
-			    0x020000E000000000);
+		xscom_read(p->chip_id,
+			   p->pe_stk_xscom + XPEC_NEST_STK_TUNNEL_BAR, &reg);
+		if (!reg) {
+			reg = 0x00020000E0000000ull << 8;
+			xscom_write(p->chip_id,
+				    p->pe_stk_xscom + XPEC_NEST_STK_TUNNEL_BAR,
+				    reg);
+		}
 
 		/* PB AIB Hardware Control Register
 		 * Wait 32 PCI clocks for a credit to become available
@@ -4051,6 +4063,53 @@  static int64_t phb4_set_capp_recovery(struct phb *phb)
 	return 0;
 }
 
+/*
+ * Return the address out of a PBCQ Tunnel Bar register.
+ */
+static void phb4_get_tunnel_bar(struct phb *phb, uint64_t *addr)
+{
+	struct phb4 *p = phb_to_phb4(phb);
+	uint64_t val;
+
+	xscom_read(p->chip_id, p->pe_stk_xscom + XPEC_NEST_STK_TUNNEL_BAR,
+		   &val);
+	*addr = val >> 8;
+}
+
+/*
+ * Set PBCQ Tunnel Bar register.
+ * Store addr bits [8:50] in PBCQ Tunnel Bar register bits [0:42].
+ * Note that addr bits [8:50] must also match PSL_TNR_ADDR[8:50].
+ * Reset register if val == 0.
+ *
+ * This interface is required to let device drivers set the Tunnel Bar
+ * value of their choice.
+ *
+ * Compatibility with older versions of linux, that do not set the
+ * Tunnel Bar with phb4_set_tunnel_bar(), is ensured by enable_capi_mode(),
+ * that will set the default value that used to be assumed.
+ */
+static int64_t phb4_set_tunnel_bar(struct phb *phb, uint64_t addr)
+{
+	struct phb4 *p = phb_to_phb4(phb);
+	uint64_t mask = 0x00FFFFFFFFFFE000ULL;
+
+	if (!addr) {
+		/* Reset register */
+		xscom_write(p->chip_id,
+			    p->pe_stk_xscom + XPEC_NEST_STK_TUNNEL_BAR, addr);
+		return OPAL_SUCCESS;
+	}
+	if ((addr & ~mask))
+		return OPAL_PARAMETER;
+	if (!(addr & mask))
+		return OPAL_PARAMETER;
+
+	xscom_write(p->chip_id, p->pe_stk_xscom + XPEC_NEST_STK_TUNNEL_BAR,
+		    (addr & mask) << 8);
+	return OPAL_SUCCESS;
+}
+
 static const struct phb_ops phb4_ops = {
 	.cfg_read8		= phb4_pcicfg_read8,
 	.cfg_read16		= phb4_pcicfg_read16,
@@ -4086,6 +4145,8 @@  static const struct phb_ops phb4_ops = {
 	.set_capi_mode		= phb4_set_capi_mode,
 	.set_p2p		= phb4_set_p2p,
 	.set_capp_recovery	= phb4_set_capp_recovery,
+	.get_tunnel_bar         = phb4_get_tunnel_bar,
+	.set_tunnel_bar         = phb4_set_tunnel_bar,
 };
 
 static void phb4_init_ioda3(struct phb4 *p)
diff --git a/include/opal-api.h b/include/opal-api.h
index f0ed5f6c..ac877046 100644
--- a/include/opal-api.h
+++ b/include/opal-api.h
@@ -215,7 +215,9 @@ 
 #define OPAL_SENSOR_GROUP_CLEAR			156
 #define OPAL_PCI_SET_P2P			157
 #define OPAL_QUIESCE				158
-#define OPAL_LAST				158
+#define OPAL_PCI_GET_PBCQ_TUNNEL_BAR		159
+#define OPAL_PCI_SET_PBCQ_TUNNEL_BAR		160
+#define OPAL_LAST				160
 
 #define QUIESCE_HOLD			1 /* Spin all calls at entry */
 #define QUIESCE_REJECT			2 /* Fail all calls with OPAL_BUSY */
diff --git a/include/pci.h b/include/pci.h
index c085b6b8..97c8dfe4 100644
--- a/include/pci.h
+++ b/include/pci.h
@@ -333,6 +333,10 @@  struct phb_ops {
 	/* PCI peer-to-peer setup */
 	void (*set_p2p)(struct phb *phb, uint64_t mode, uint64_t flags,
 			uint16_t pe_number);
+
+	/* Get/set PBCQ Tunnel BAR register */
+	void (*get_tunnel_bar)(struct phb *phb, uint64_t *addr);
+	int64_t (*set_tunnel_bar)(struct phb *phb, uint64_t addr);
 };
 
 enum phb_type {