| Message ID | 20171023130830.18818-2-felix@linux.vnet.ibm.com |
|---|---|
| State | Superseded |
| Headers | show |
| Series | [1/2] phb4: set PHB CMPM registers for tunneled operations | expand |
Le 23/10/2017 à 15:08, Philippe Bergheaud a écrit : > P9 supports PCI tunneled operations (atomics and as_notify) that require > setting the PBCQ Tunnel BAR Response register with an address mask. > > This register is currently initialized by enable_capi_mode(). As tunneled > operations may also operate in PCI mode, a new API is required to set the > PBCQ Tunnel BAR Response register without switching to CAPI mode. > > This patch provides two new OPAL calls to get/set the PBCQ Tunnel BAR > Response register. > > This new API aims at letting devices drivers set the PBCQ Tunnel BAR. > Compatibility with older kernel versions is made by enable_capi_mode(). > > Signed-off-by: Philippe Bergheaud <felix@linux.vnet.ibm.com> > --- Looks ok to me. Reviewed-by: Christophe Lombard<clombard@linux.vnet.ibm.com>
On 23/10/2017 15:08, Philippe Bergheaud wrote: > P9 supports PCI tunneled operations (atomics and as_notify) that require > setting the PBCQ Tunnel BAR Response register with an address mask. > > This register is currently initialized by enable_capi_mode(). As tunneled > operations may also operate in PCI mode, a new API is required to set the > PBCQ Tunnel BAR Response register without switching to CAPI mode. > > This patch provides two new OPAL calls to get/set the PBCQ Tunnel BAR > Response register. > > This new API aims at letting devices drivers set the PBCQ Tunnel BAR. > Compatibility with older kernel versions is made by enable_capi_mode(). > > Signed-off-by: Philippe Bergheaud <felix@linux.vnet.ibm.com> > --- Linux patch: https://patchwork.ozlabs.org/patch/830231/ > core/pci-opal.c | 33 +++++++++++++++++++++++++++ > hw/phb4.c | 65 +++++++++++++++++++++++++++++++++++++++++++++++++----- > include/opal-api.h | 4 +++- > include/pci.h | 4 ++++ > 4 files changed, 100 insertions(+), 6 deletions(-) > > diff --git a/core/pci-opal.c b/core/pci-opal.c > index bbbbcd5d..d7f4230a 100644 > --- a/core/pci-opal.c > +++ b/core/pci-opal.c > @@ -1052,3 +1052,36 @@ static int64_t opal_pci_set_phb_cmpm(uint64_t phb_id, uint64_t phb_reg, > return rc; > } > opal_call(OPAL_PCI_SET_PHB_CMPM, opal_pci_set_phb_cmpm, 3); > + > +static int64_t opal_pci_get_pbcq_tunnel_bar(uint64_t phb_id, uint64_t *addr) > +{ > + struct phb *phb = pci_get_phb(phb_id); > + > + if (!phb) > + return OPAL_PARAMETER; > + if (!phb->ops->get_tunnel_bar) > + return OPAL_UNSUPPORTED; > + > + phb_lock(phb); > + phb->ops->get_tunnel_bar(phb, addr); > + phb_unlock(phb); > + return OPAL_SUCCESS; > +} > +opal_call(OPAL_PCI_GET_PBCQ_TUNNEL_BAR, opal_pci_get_pbcq_tunnel_bar, 2); > + > +static int64_t opal_pci_set_pbcq_tunnel_bar(uint64_t phb_id, uint64_t addr) > +{ > + struct phb *phb = pci_get_phb(phb_id); > + int64_t rc; > + > + if (!phb) > + return OPAL_PARAMETER; > + if (!phb->ops->set_tunnel_bar) > + return OPAL_UNSUPPORTED; > + > + phb_lock(phb); > + rc = phb->ops->set_tunnel_bar(phb, addr); > + phb_unlock(phb); > + return rc; > +} > +opal_call(OPAL_PCI_SET_PBCQ_TUNNEL_BAR, opal_pci_set_pbcq_tunnel_bar, 2); > diff --git a/hw/phb4.c b/hw/phb4.c > index 1add8a75..54dd9fac 100644 > --- a/hw/phb4.c > +++ b/hw/phb4.c > @@ -3751,12 +3751,18 @@ static int64_t enable_capi_mode(struct phb4 *p, uint64_t pe_number, > out_be64(p->regs + PHB_ASN_CMPM, reg); > } > > - /* PBCQ Tunnel Bar Register > - * Write Tunnel register to match PSL TNR register > + /* > + * PBCQ Tunnel Bar Register > + * If unset, then use PSL_TNR_ADDR[TNR_Addr] reset value. > */ > - xscom_write(p->chip_id, > - p->pe_stk_xscom + XPEC_NEST_STK_TUNNEL_BAR, > - 0x020000E000000000); > + xscom_read(p->chip_id, > + p->pe_stk_xscom + XPEC_NEST_STK_TUNNEL_BAR, ®); > + if (!reg) { > + reg = 0x020000e000000000ull; > + xscom_write(p->chip_id, > + p->pe_stk_xscom + XPEC_NEST_STK_TUNNEL_BAR, > + reg); > + } > > /* PB AIB Hardware Control Register > * Wait 32 PCI clocks for a credit to become available > @@ -4092,6 +4098,53 @@ static int64_t phb4_set_cmpm(struct phb *phb, uint64_t phb_reg, uint64_t ind) > return OPAL_SUCCESS; > } > > +/* > + * Return the address out of a PBCQ Tunnel Bar register. > + */ > +static void phb4_get_tunnel_bar(struct phb *phb, uint64_t *addr) > +{ > + struct phb4 *p = phb_to_phb4(phb); > + uint64_t val; > + > + xscom_read(p->chip_id, p->pe_stk_xscom + XPEC_NEST_STK_TUNNEL_BAR, > + &val); > + *addr = val >> 8; > +} > + > +/* > + * Set PBCQ Tunnel Bar register. > + * Store addr bits [8:50] in PBCQ Tunnel Bar register bits [0:42]. > + * Note that addr bits [8:50] must also match PSL_TNR_ADDR[8:50]. > + * Reset register if val == 0. > + * > + * This interface is required to let device drivers set the Tunnel Bar > + * value of their choice. > + * > + * Compatibility with older versions of linux, that do not set the > + * Tunnel Bar with phb4_set_tunnel_bar(), is ensured by enable_capi_mode(), > + * that will set the default value that used to be assumed. > + */ > +static int64_t phb4_set_tunnel_bar(struct phb *phb, uint64_t addr) > +{ > + struct phb4 *p = phb_to_phb4(phb); > + uint64_t mask = 0x00ffffffffffe000ull; > + > + if (! addr) { > + /* Reset register */ > + xscom_write(p->chip_id, > + p->pe_stk_xscom + XPEC_NEST_STK_TUNNEL_BAR, addr); > + return OPAL_SUCCESS; > + } > + if ((addr & ~mask)) > + return OPAL_PARAMETER; > + if (!(addr & mask)) > + return OPAL_PARAMETER; > + > + xscom_write(p->chip_id, p->pe_stk_xscom + XPEC_NEST_STK_TUNNEL_BAR, > + (addr & mask) << 8); > + return OPAL_SUCCESS; > +} > + > static const struct phb_ops phb4_ops = { > .cfg_read8 = phb4_pcicfg_read8, > .cfg_read16 = phb4_pcicfg_read16, > @@ -4129,6 +4182,8 @@ static const struct phb_ops phb4_ops = { > .set_capp_recovery = phb4_set_capp_recovery, > .get_cmpm = phb4_get_cmpm, > .set_cmpm = phb4_set_cmpm, > + .get_tunnel_bar = phb4_get_tunnel_bar, > + .set_tunnel_bar = phb4_set_tunnel_bar, > }; > > static void phb4_init_ioda3(struct phb4 *p) > diff --git a/include/opal-api.h b/include/opal-api.h > index ce948136..1e9715ee 100644 > --- a/include/opal-api.h > +++ b/include/opal-api.h > @@ -216,7 +216,9 @@ > #define OPAL_PCI_SET_P2P 157 > #define OPAL_PCI_GET_PHB_CMPM 158 > #define OPAL_PCI_SET_PHB_CMPM 159 > -#define OPAL_LAST 159 > +#define OPAL_PCI_GET_PBCQ_TUNNEL_BAR 160 > +#define OPAL_PCI_SET_PBCQ_TUNNEL_BAR 161 > +#define OPAL_LAST 161 > > /* Device tree flags */ > > diff --git a/include/pci.h b/include/pci.h > index cdf82ee8..5276b574 100644 > --- a/include/pci.h > +++ b/include/pci.h > @@ -337,6 +337,10 @@ struct phb_ops { > /* Get/set PHB Compare/Mask registers */ > int64_t (*get_cmpm)(struct phb *phb, uint64_t phb_reg, uint64_t *ind); > int64_t (*set_cmpm)(struct phb *phb, uint64_t phb_reg, uint64_t ind); > + > + /* Get/set PBCQ Tunnel BAR register */ > + void (*get_tunnel_bar)(struct phb *phb, uint64_t *addr); > + int64_t (*set_tunnel_bar)(struct phb *phb, uint64_t addr); > }; > > enum phb_type {
On Thu, 2017-10-26 at 15:43 +0200, Philippe Bergheaud wrote: > On 23/10/2017 15:08, Philippe Bergheaud wrote: > > P9 supports PCI tunneled operations (atomics and as_notify) that require > > setting the PBCQ Tunnel BAR Response register with an address mask. > > > > This register is currently initialized by enable_capi_mode(). As tunneled > > operations may also operate in PCI mode, a new API is required to set the > > PBCQ Tunnel BAR Response register without switching to CAPI mode. > > > > This patch provides two new OPAL calls to get/set the PBCQ Tunnel BAR > > Response register. > > > > This new API aims at letting devices drivers set the PBCQ Tunnel BAR. > > Compatibility with older kernel versions is made by enable_capi_mode(). > > > > Signed-off-by: Philippe Bergheaud <felix@linux.vnet.ibm.com> > > --- Same comment as for patch 1.
On 02/11/2017 05:03, Benjamin Herrenschmidt wrote: > On Thu, 2017-10-26 at 15:43 +0200, Philippe Bergheaud wrote: >> On 23/10/2017 15:08, Philippe Bergheaud wrote: >>> P9 supports PCI tunneled operations (atomics and as_notify) that require >>> setting the PBCQ Tunnel BAR Response register with an address mask. >>> >>> This register is currently initialized by enable_capi_mode(). As tunneled >>> operations may also operate in PCI mode, a new API is required to set the >>> PBCQ Tunnel BAR Response register without switching to CAPI mode. >>> >>> This patch provides two new OPAL calls to get/set the PBCQ Tunnel BAR >>> Response register. >>> >>> This new API aims at letting devices drivers set the PBCQ Tunnel BAR. >>> Compatibility with older kernel versions is made by enable_capi_mode(). >>> >>> Signed-off-by: Philippe Bergheaud <felix@linux.vnet.ibm.com> >>> --- > Same comment as for patch 1. > I understand that the Tunnel BAR value is chosen by the device logic, within the regular BARs assigned to the device at power-on (an entire BAR, or a subset). The Tunnel BAR value is fetched by the device driver, that passes it to skiboot, to set the PBCQ Tunnel BAR Response register. I think that this patch is required. I cannot see how to reverse the information flow.
On Tue, 2017-11-07 at 14:19 +0100, Philippe Bergheaud wrote: > > I understand that the Tunnel BAR value is chosen by the device logic, > within the regular BARs assigned to the device at power-on (an entire > BAR, or a subset). The Tunnel BAR value is fetched by the device driver, > that passes it to skiboot, to set the PBCQ Tunnel BAR Response register. > > I think that this patch is required. I cannot see how to reverse the > information flow. Why would it be chosen by the device ? I don't understand... Ben.
On 07/11/2017 21:18, Benjamin Herrenschmidt wrote: > On Tue, 2017-11-07 at 14:19 +0100, Philippe Bergheaud wrote: >> I understand that the Tunnel BAR value is chosen by the device logic, >> within the regular BARs assigned to the device at power-on (an entire >> BAR, or a subset). The Tunnel BAR value is fetched by the device driver, >> that passes it to skiboot, to set the PBCQ Tunnel BAR Response register. >> >> I think that this patch is required. I cannot see how to reverse the >> information flow. > Why would it be chosen by the device ? I don't understand... > There can be more than one device connected to the PHB. But as there is only one PBCQ Tunnel BAR Response register, only one device will be able to use tunneled operations at a certain time. I have chosen a first-come first-serve policy, where a device will: 1. Acquire tunneled operations the by setting the register with its own specific BAR value, chosen within the range of its BARs, where tunneled operation responses will be expected. 2. Release tunneled operations after use by resetting the PBCQ Tunnel BAR response register, authenticating itself by passing its device- specific BAR value again (as required by the API). The register value indicates which device should receive the tunneled operation responses. The value of the register must be dynamically modified by devices to point to their own MMIO range. It cannot be statically set by skiboot. Philippe
On Mon, 2017-11-13 at 11:45 +0100, Philippe Bergheaud wrote: > On 07/11/2017 21:18, Benjamin Herrenschmidt wrote: > > On Tue, 2017-11-07 at 14:19 +0100, Philippe Bergheaud wrote: > > > I understand that the Tunnel BAR value is chosen by the device logic, > > > within the regular BARs assigned to the device at power-on (an entire > > > BAR, or a subset). The Tunnel BAR value is fetched by the device driver, > > > that passes it to skiboot, to set the PBCQ Tunnel BAR Response register. > > > > > > I think that this patch is required. I cannot see how to reverse the > > > information flow. > > > > Why would it be chosen by the device ? I don't understand... > > > > There can be more than one device connected to the PHB. But as there > is only one PBCQ Tunnel BAR Response register, only one device will > be able to use tunneled operations at a certain time. Why ? I don't understand. What is special about those tunneled operations that require that sort of mutual exclusion ? > I have chosen > a first-come first-serve policy, where a device will: > > 1. Acquire tunneled operations the by setting the register with > its own specific BAR value, chosen within the range of its BARs, > where tunneled operation responses will be expected. Ok I obsiously don't understand how tunneled operations work. Can you elaborate a bit ? > 2. Release tunneled operations after use by resetting the PBCQ Tunnel > BAR response register, authenticating itself by passing its device- > specific BAR value again (as required by the API). > > The register value indicates which device should receive the tunneled > operation responses. The value of the register must be dynamically > modified by devices to point to their own MMIO range. It cannot be > statically set by skiboot. Oh so basically we expect a "response" to the device ? That smells like a disgusting HW hack to me ... Ben.
On 13/11/2017 12:19, Benjamin Herrenschmidt wrote: > On Mon, 2017-11-13 at 11:45 +0100, Philippe Bergheaud wrote: >> On 07/11/2017 21:18, Benjamin Herrenschmidt wrote: >>> On Tue, 2017-11-07 at 14:19 +0100, Philippe Bergheaud wrote: >>>> I understand that the Tunnel BAR value is chosen by the device logic, >>>> within the regular BARs assigned to the device at power-on (an entire >>>> BAR, or a subset). The Tunnel BAR value is fetched by the device driver, >>>> that passes it to skiboot, to set the PBCQ Tunnel BAR Response register. >>>> >>>> I think that this patch is required. I cannot see how to reverse the >>>> information flow. >>> Why would it be chosen by the device ? I don't understand... >>> >> There can be more than one device connected to the PHB. But as there >> is only one PBCQ Tunnel BAR Response register, only one device will >> be able to use tunneled operations at a certain time. > Why ? I don't understand. What is special about those tunneled > operations that require that sort of mutual exclusion ? The reason for the mutual exclusion is the unicity of the PBCQ Tunnel BAR Response register, that indicates the address where responses are expected by the device that has initiated the operation. >> I have chosen >> a first-come first-serve policy, where a device will: >> >> 1. Acquire tunneled operations the by setting the register with >> its own specific BAR value, chosen within the range of its BARs, >> where tunneled operation responses will be expected. > Ok I obsiously don't understand how tunneled operations work. Can you > elaborate a bit ? There are two kinds of tunneled operations: AS notify and atomics. AS notify is not guaranted to succeed. The host must tell the device whether the target thread has been woken up or not. Not all atomic operations require a response. But Compare and Swap, for example, does: the device must know the status of the comparison, as swap is only performed in case of success. >> 2. Release tunneled operations after use by resetting the PBCQ Tunnel >> BAR response register, authenticating itself by passing its device- >> specific BAR value again (as required by the API). >> >> The register value indicates which device should receive the tunneled >> operation responses. The value of the register must be dynamically >> modified by devices to point to their own MMIO range. It cannot be >> statically set by skiboot. > Oh so basically we expect a "response" to the device ? That smells like > a disgusting HW hack to me ... > Yes, limiting tunneled operations to one device per PHB seems strange. I have arbitrarily chosen the first-come first-serve policy to cope with this limitation. Philippe
On Tue, 2017-11-14 at 11:29 +0100, Philippe Bergheaud wrote: > > > The register value indicates which device should receive the tunneled > > > operation responses. The value of the register must be dynamically > > > modified by devices to point to their own MMIO range. It cannot be > > > statically set by skiboot. > > > > Oh so basically we expect a "response" to the device ? That smells like > > a disgusting HW hack to me ... > > > > Yes, limiting tunneled operations to one device per PHB seems strange. > I have arbitrarily chosen the first-come first-serve policy to cope > with this limitation. Ok so I was under the incorrect impression that we were configuring the address for the requests. That needs to stay in OPAL. For the responses, I agree, this belongs in the driver, so an OPAL call is appropriate (which the driver calls via some exported support wrapper). Cheers, Ben
diff --git a/core/pci-opal.c b/core/pci-opal.c index bbbbcd5d..d7f4230a 100644 --- a/core/pci-opal.c +++ b/core/pci-opal.c @@ -1052,3 +1052,36 @@ static int64_t opal_pci_set_phb_cmpm(uint64_t phb_id, uint64_t phb_reg, return rc; } opal_call(OPAL_PCI_SET_PHB_CMPM, opal_pci_set_phb_cmpm, 3); + +static int64_t opal_pci_get_pbcq_tunnel_bar(uint64_t phb_id, uint64_t *addr) +{ + struct phb *phb = pci_get_phb(phb_id); + + if (!phb) + return OPAL_PARAMETER; + if (!phb->ops->get_tunnel_bar) + return OPAL_UNSUPPORTED; + + phb_lock(phb); + phb->ops->get_tunnel_bar(phb, addr); + phb_unlock(phb); + return OPAL_SUCCESS; +} +opal_call(OPAL_PCI_GET_PBCQ_TUNNEL_BAR, opal_pci_get_pbcq_tunnel_bar, 2); + +static int64_t opal_pci_set_pbcq_tunnel_bar(uint64_t phb_id, uint64_t addr) +{ + struct phb *phb = pci_get_phb(phb_id); + int64_t rc; + + if (!phb) + return OPAL_PARAMETER; + if (!phb->ops->set_tunnel_bar) + return OPAL_UNSUPPORTED; + + phb_lock(phb); + rc = phb->ops->set_tunnel_bar(phb, addr); + phb_unlock(phb); + return rc; +} +opal_call(OPAL_PCI_SET_PBCQ_TUNNEL_BAR, opal_pci_set_pbcq_tunnel_bar, 2); diff --git a/hw/phb4.c b/hw/phb4.c index 1add8a75..54dd9fac 100644 --- a/hw/phb4.c +++ b/hw/phb4.c @@ -3751,12 +3751,18 @@ static int64_t enable_capi_mode(struct phb4 *p, uint64_t pe_number, out_be64(p->regs + PHB_ASN_CMPM, reg); } - /* PBCQ Tunnel Bar Register - * Write Tunnel register to match PSL TNR register + /* + * PBCQ Tunnel Bar Register + * If unset, then use PSL_TNR_ADDR[TNR_Addr] reset value. */ - xscom_write(p->chip_id, - p->pe_stk_xscom + XPEC_NEST_STK_TUNNEL_BAR, - 0x020000E000000000); + xscom_read(p->chip_id, + p->pe_stk_xscom + XPEC_NEST_STK_TUNNEL_BAR, ®); + if (!reg) { + reg = 0x020000e000000000ull; + xscom_write(p->chip_id, + p->pe_stk_xscom + XPEC_NEST_STK_TUNNEL_BAR, + reg); + } /* PB AIB Hardware Control Register * Wait 32 PCI clocks for a credit to become available @@ -4092,6 +4098,53 @@ static int64_t phb4_set_cmpm(struct phb *phb, uint64_t phb_reg, uint64_t ind) return OPAL_SUCCESS; } +/* + * Return the address out of a PBCQ Tunnel Bar register. + */ +static void phb4_get_tunnel_bar(struct phb *phb, uint64_t *addr) +{ + struct phb4 *p = phb_to_phb4(phb); + uint64_t val; + + xscom_read(p->chip_id, p->pe_stk_xscom + XPEC_NEST_STK_TUNNEL_BAR, + &val); + *addr = val >> 8; +} + +/* + * Set PBCQ Tunnel Bar register. + * Store addr bits [8:50] in PBCQ Tunnel Bar register bits [0:42]. + * Note that addr bits [8:50] must also match PSL_TNR_ADDR[8:50]. + * Reset register if val == 0. + * + * This interface is required to let device drivers set the Tunnel Bar + * value of their choice. + * + * Compatibility with older versions of linux, that do not set the + * Tunnel Bar with phb4_set_tunnel_bar(), is ensured by enable_capi_mode(), + * that will set the default value that used to be assumed. + */ +static int64_t phb4_set_tunnel_bar(struct phb *phb, uint64_t addr) +{ + struct phb4 *p = phb_to_phb4(phb); + uint64_t mask = 0x00ffffffffffe000ull; + + if (! addr) { + /* Reset register */ + xscom_write(p->chip_id, + p->pe_stk_xscom + XPEC_NEST_STK_TUNNEL_BAR, addr); + return OPAL_SUCCESS; + } + if ((addr & ~mask)) + return OPAL_PARAMETER; + if (!(addr & mask)) + return OPAL_PARAMETER; + + xscom_write(p->chip_id, p->pe_stk_xscom + XPEC_NEST_STK_TUNNEL_BAR, + (addr & mask) << 8); + return OPAL_SUCCESS; +} + static const struct phb_ops phb4_ops = { .cfg_read8 = phb4_pcicfg_read8, .cfg_read16 = phb4_pcicfg_read16, @@ -4129,6 +4182,8 @@ static const struct phb_ops phb4_ops = { .set_capp_recovery = phb4_set_capp_recovery, .get_cmpm = phb4_get_cmpm, .set_cmpm = phb4_set_cmpm, + .get_tunnel_bar = phb4_get_tunnel_bar, + .set_tunnel_bar = phb4_set_tunnel_bar, }; static void phb4_init_ioda3(struct phb4 *p) diff --git a/include/opal-api.h b/include/opal-api.h index ce948136..1e9715ee 100644 --- a/include/opal-api.h +++ b/include/opal-api.h @@ -216,7 +216,9 @@ #define OPAL_PCI_SET_P2P 157 #define OPAL_PCI_GET_PHB_CMPM 158 #define OPAL_PCI_SET_PHB_CMPM 159 -#define OPAL_LAST 159 +#define OPAL_PCI_GET_PBCQ_TUNNEL_BAR 160 +#define OPAL_PCI_SET_PBCQ_TUNNEL_BAR 161 +#define OPAL_LAST 161 /* Device tree flags */ diff --git a/include/pci.h b/include/pci.h index cdf82ee8..5276b574 100644 --- a/include/pci.h +++ b/include/pci.h @@ -337,6 +337,10 @@ struct phb_ops { /* Get/set PHB Compare/Mask registers */ int64_t (*get_cmpm)(struct phb *phb, uint64_t phb_reg, uint64_t *ind); int64_t (*set_cmpm)(struct phb *phb, uint64_t phb_reg, uint64_t ind); + + /* Get/set PBCQ Tunnel BAR register */ + void (*get_tunnel_bar)(struct phb *phb, uint64_t *addr); + int64_t (*set_tunnel_bar)(struct phb *phb, uint64_t addr); }; enum phb_type {
P9 supports PCI tunneled operations (atomics and as_notify) that require setting the PBCQ Tunnel BAR Response register with an address mask. This register is currently initialized by enable_capi_mode(). As tunneled operations may also operate in PCI mode, a new API is required to set the PBCQ Tunnel BAR Response register without switching to CAPI mode. This patch provides two new OPAL calls to get/set the PBCQ Tunnel BAR Response register. This new API aims at letting devices drivers set the PBCQ Tunnel BAR. Compatibility with older kernel versions is made by enable_capi_mode(). Signed-off-by: Philippe Bergheaud <felix@linux.vnet.ibm.com> --- core/pci-opal.c | 33 +++++++++++++++++++++++++++ hw/phb4.c | 65 +++++++++++++++++++++++++++++++++++++++++++++++++----- include/opal-api.h | 4 +++- include/pci.h | 4 ++++ 4 files changed, 100 insertions(+), 6 deletions(-)