[10/16,10/16] opencapi5: complete phb ops

Add more PHB interfaces:
- to control pci error type in case of freeze.
- add the addresses of the registers needed by the OS to handle
translation failures.
- to detect the fence state of a specific brick
- to configure BDF (Bus Device Function) and PE (Partitionable Endpoint)
for context identification.

Signed-off-by: Christophe Lombard <clombard@linux.vnet.ibm.com>
---
 hw/pau.c           | 143 +++++++++++++++++++++++++++++++++++++++++++++
 include/pau-regs.h |   9 +++
 2 files changed, 152 insertions(+)

On 20/08/2021 11:45, Christophe Lombard wrote:
> Add more PHB interfaces:
> - to control pci error type in case of freeze.
> - add the addresses of the registers needed by the OS to handle
> translation failures.
> - to detect the fence state of a specific brick
> - to configure BDF (Bus Device Function) and PE (Partitionable Endpoint)
> for context identification.
> 
> Signed-off-by: Christophe Lombard <clombard@linux.vnet.ibm.com>
> ---


Just one comment on a comment below, but other than that:
Reviewed-by: Frederic Barrat <fbarrat@linux.ibm.com>



>   hw/pau.c           | 143 +++++++++++++++++++++++++++++++++++++++++++++
>   include/pau-regs.h |   9 +++
>   2 files changed, 152 insertions(+)
> 
> diff --git a/hw/pau.c b/hw/pau.c
> index 98abe704..68195e48 100644
> --- a/hw/pau.c
> +++ b/hw/pau.c
> @@ -597,6 +597,144 @@ PAU_OPENCAPI_PCI_CFG_WRITE(8, u8)
>   PAU_OPENCAPI_PCI_CFG_WRITE(16, u16)
>   PAU_OPENCAPI_PCI_CFG_WRITE(32, u32)
> 
> +static int64_t pau_opencapi_eeh_freeze_status(struct phb *phb __unused,
> +					      uint64_t pe_num __unused,
> +					      uint8_t *freeze_state,
> +					      uint16_t *pci_error_type,
> +					      uint16_t *severity)
> +{
> +	/*
> +	 * FIXME: When it's called by skiboot PCI config accessor,
> +	 * the PE number is fixed to 0, which is incorrect. We need
> +	 * introduce another PHB callback to translate it. For now,
> +	 * it keeps the skiboot PCI enumeration going.
> +	 */


I don't quite get that comment. PE numbers have no real meaning with 
opencapi. We should discuss it.

   Fred



> +	*freeze_state = OPAL_EEH_STOPPED_NOT_FROZEN;
> +	*pci_error_type = OPAL_EEH_NO_ERROR;
> +
> +	if (severity)
> +		*severity = OPAL_EEH_SEV_NO_ERROR;
> +
> +	return OPAL_SUCCESS;
> +}
> +
> +static int64_t pau_opencapi_ioda_reset(struct phb __unused * phb,
> +				       bool __unused purge)
> +{
> +	/* Not relevant to OpenCAPI - we do this just to silence the error */
> +	return OPAL_SUCCESS;
> +}
> +
> +static int64_t pau_opencapi_next_error(struct phb *phb,
> +				       uint64_t *first_frozen_pe,
> +				       uint16_t *pci_error_type,
> +				       uint16_t *severity)
> +{
> +	struct pau_dev *dev = pau_phb_to_opencapi_dev(phb);
> +	struct pau *pau = dev->pau;
> +	uint32_t pe_num;
> +	uint64_t val;
> +
> +	if (!first_frozen_pe || !pci_error_type || !severity)
> +		return OPAL_PARAMETER;
> +
> +	if (dev->status & PAU_DEV_STATUS_BROKEN) {
> +		val = pau_read(pau, PAU_MISC_BDF2PE_CFG(dev->index));
> +		pe_num = GETFIELD(PAU_MISC_BDF2PE_CFG_PE, val);
> +
> +		PAUDEVDBG(dev, "Reporting device as broken\n");
> +		PAUDEVDBG(dev, "Brick %d fenced! (pe_num: %08x\n",
> +				pau_dev_index(dev, PAU_LINKS_OPENCAPI_PER_PAU),
> +				pe_num);
> +		*first_frozen_pe = pe_num;
> +		*pci_error_type = OPAL_EEH_PHB_ERROR;
> +		*severity = OPAL_EEH_SEV_PHB_DEAD;
> +	} else {
> +		*first_frozen_pe = -1;
> +		*pci_error_type = OPAL_EEH_NO_ERROR;
> +		*severity = OPAL_EEH_SEV_NO_ERROR;
> +	}
> +	return OPAL_SUCCESS;
> +}
> +
> +static uint32_t pau_opencapi_dev_interrupt_level(struct pau_dev *dev)
> +{
> +	/* Interrupt Levels
> +	 * 35: Translation failure for OCAPI link 0
> +	 * 36: Translation failure for OCAPI link 1
> +	 */
> +	const uint32_t level[2] = {35, 36};
> +
> +	return level[dev->index];
> +}
> +
> +static int pau_opencapi_dt_add_interrupts(struct phb *phb,
> +					  struct pci_device *pd,
> +					  void *data __unused)
> +{
> +	struct pau_dev *dev = pau_phb_to_opencapi_dev(phb);
> +	struct pau *pau = dev->pau;
> +	uint64_t dsisr, dar, tfc, handle;
> +	uint32_t irq;
> +
> +	irq = pau->irq_base + pau_opencapi_dev_interrupt_level(dev);
> +
> +	/* When an address translation fail causes the PAU to send an
> +	 * interrupt, information is stored in three registers for use
> +	 * by the interrupt handler. The OS accesses them by mmio.
> +	 */
> +	dsisr  = pau->regs[0] + PAU_OTL_MISC_PSL_DSISR_AN(dev->index);
> +	dar    = pau->regs[0] + PAU_OTL_MISC_PSL_DAR_AN(dev->index);
> +	tfc    = pau->regs[0] + PAU_OTL_MISC_PSL_TFC_AN(dev->index);
> +	handle = pau->regs[0] + PAU_OTL_MISC_PSL_PEHANDLE_AN(dev->index);
> +	dt_add_property_cells(pd->dn, "ibm,opal-xsl-irq", irq);
> +	dt_add_property_cells(pd->dn, "ibm,opal-xsl-mmio",
> +			hi32(dsisr), lo32(dsisr),
> +			hi32(dar), lo32(dar),
> +			hi32(tfc), lo32(tfc),
> +			hi32(handle), lo32(handle));
> +	return 0;
> +}
> +
> +static void pau_opencapi_phb_final_fixup(struct phb *phb)
> +{
> +	pci_walk_dev(phb, NULL, pau_opencapi_dt_add_interrupts, NULL);
> +}
> +
> +static int64_t pau_opencapi_set_pe(struct phb *phb,
> +				   uint64_t pe_num,
> +				   uint64_t bdfn,
> +				   uint8_t bcompare,
> +				   uint8_t dcompare,
> +				   uint8_t fcompare,
> +				   uint8_t action)
> +{
> +	struct pau_dev *dev = pau_phb_to_opencapi_dev(phb);
> +	struct pau *pau = dev->pau;
> +	uint64_t val;
> +
> +	PAUDEVDBG(dev, "Set partitionable endpoint = %08llx, bdfn =  %08llx\n",
> +			pe_num, bdfn);
> +
> +	if (action != OPAL_MAP_PE && action != OPAL_UNMAP_PE)
> +		return OPAL_PARAMETER;
> +
> +	if (pe_num >= PAU_MAX_PE_NUM)
> +		return OPAL_PARAMETER;
> +
> +	if (bcompare != OpalPciBusAll ||
> +	    dcompare != OPAL_COMPARE_RID_DEVICE_NUMBER ||
> +	    fcompare != OPAL_COMPARE_RID_FUNCTION_NUMBER)
> +		return OPAL_UNSUPPORTED;
> +
> +	val = PAU_MISC_BDF2PE_CFG_ENABLE;
> +	val = SETFIELD(PAU_MISC_BDF2PE_CFG_PE, val, pe_num);
> +	val = SETFIELD(PAU_MISC_BDF2PE_CFG_BDF, val, 0);
> +	pau_write(pau, PAU_MISC_BDF2PE_CFG(dev->index), val);
> +
> +	return OPAL_SUCCESS;
> +}
> +
>   static const struct phb_ops pau_opencapi_ops = {
>   	.cfg_read8		= pau_opencapi_pcicfg_read8,
>   	.cfg_read16		= pau_opencapi_pcicfg_read16,
> @@ -604,6 +742,11 @@ static const struct phb_ops pau_opencapi_ops = {
>   	.cfg_write8		= pau_opencapi_pcicfg_write8,
>   	.cfg_write16		= pau_opencapi_pcicfg_write16,
>   	.cfg_write32		= pau_opencapi_pcicfg_write32,
> +	.eeh_freeze_status	= pau_opencapi_eeh_freeze_status,
> +	.next_error		= pau_opencapi_next_error,
> +	.ioda_reset		= pau_opencapi_ioda_reset,
> +	.phb_final_fixup	= pau_opencapi_phb_final_fixup,
> +	.set_pe			= pau_opencapi_set_pe,
>   };
> 
>   static void pau_opencapi_create_phb(struct pau_dev *dev)
> diff --git a/include/pau-regs.h b/include/pau-regs.h
> index d98f435b..19b0b7cd 100644
> --- a/include/pau-regs.h
> +++ b/include/pau-regs.h
> @@ -33,6 +33,7 @@
>   #define PAU_BLOCK_CQ_CTL			PAU_BLOCK(4, 4)
>   #define PAU_BLOCK_CQ_DAT			PAU_BLOCK(4, 5)
>   #define PAU_BLOCK_OTL(brk)			PAU_BLOCK(4, 0xC + (brk))
> +#define PAU_BLOCK_OTL_PSL(brk)			PAU_BLOCK(0, 0xC + (brk))
>   #define PAU_BLOCK_XSL				PAU_BLOCK(4, 0xE)
>   #define PAU_BLOCK_PAU_XTS			PAU_BLOCK(7, 1)
>   #define PAU_BLOCK_PAU_MISC			PAU_BLOCK(7, 2)
> @@ -117,6 +118,10 @@
>   #define   PAU_OTL_MISC_CFG_TX_TEMP2_RATE	PPC_BITMASK(16, 19)
>   #define   PAU_OTL_MISC_CFG_TX_TEMP3_RATE	PPC_BITMASK(20, 23)
>   #define   PAU_OTL_MISC_CFG_TX_CRET_FREQ		PPC_BITMASK(32, 34)
> +#define PAU_OTL_MISC_PSL_DSISR_AN(brk)		(PAU_BLOCK_OTL_PSL(brk) + 0x000)
> +#define PAU_OTL_MISC_PSL_DAR_AN(brk)		(PAU_BLOCK_OTL_PSL(brk) + 0x008)
> +#define PAU_OTL_MISC_PSL_TFC_AN(brk)		(PAU_BLOCK_OTL_PSL(brk) + 0x010)
> +#define PAU_OTL_MISC_PSL_PEHANDLE_AN(brk)	(PAU_BLOCK_OTL_PSL(brk) + 0x018)
> 
>   /* XSL block registers */
>   #define PAU_XSL_WRAP_CFG			(PAU_BLOCK_XSL + 0x100)
> @@ -149,6 +154,10 @@
>   #define PAU_MISC_INT_1_CONFIG			(PAU_BLOCK_PAU_MISC + 0x068)
>   #define PAU_MISC_INT_BAR			(PAU_BLOCK_PAU_MISC + 0x098)
>   #define   PAU_MISC_INT_BAR_ADDR			PPC_BITMASK(0, 39)
> +#define PAU_MISC_BDF2PE_CFG(n)			(PAU_BLOCK_PAU_MISC + 0x100 + (n) * 8)
> +#define   PAU_MISC_BDF2PE_CFG_ENABLE		PPC_BIT(0)
> +#define   PAU_MISC_BDF2PE_CFG_PE		PPC_BITMASK(4, 7)
> +#define   PAU_MISC_BDF2PE_CFG_BDF		PPC_BITMASK(8, 23)
>   #define PAU_MISC_INT_2_CONFIG			(PAU_BLOCK_PAU_MISC + 0x408)
>   #define   PAU_MISC_INT_2_CONFIG_XFAULT_2_5(n)	PPC_BIT(0 + (n))
>   #define   PAU_MISC_INT_2_CONFIG_XFAULT_0_1(n)	PPC_BIT(54 + (n))
>