Message ID | 20200710052340.737567-10-oohall@gmail.com (mailing list archive) |
---|---|
State | Changes Requested |
Headers | show |
Series | [01/15] powernv/pci: Add pci_bus_to_pnvhb() helper | expand |
On 10/07/2020 15:23, Oliver O'Halloran wrote: > The sequence required to use the single PE BAR mode is kinda janky and > requires a little explanation. The API was designed with P7-IOC style > windows where the setup process is something like: > > 1. Configure the window start / end address > 2. Enable the window > 3. Map the segments of each window to the PE > > For Single PE BARs the process is: > > 1. Set the PE for segment zero on a disabled window > 2. Set the range > 3. Enable the window > > Move the OPAL calls into their own helper functions where the quirks can be > contained. > > Signed-off-by: Oliver O'Halloran <oohall@gmail.com> I'd use "segmented" instead of "accordion". Otherwise, Reviewed-by: Alexey Kardashevskiy <aik@ozlabs.ru> > --- > arch/powerpc/platforms/powernv/pci-sriov.c | 132 ++++++++++++++++----- > 1 file changed, 103 insertions(+), 29 deletions(-) > > diff --git a/arch/powerpc/platforms/powernv/pci-sriov.c b/arch/powerpc/platforms/powernv/pci-sriov.c > index e4c65cb49757..d53a85ccb538 100644 > --- a/arch/powerpc/platforms/powernv/pci-sriov.c > +++ b/arch/powerpc/platforms/powernv/pci-sriov.c > @@ -320,6 +320,102 @@ static int pnv_pci_vf_release_m64(struct pci_dev *pdev, u16 num_vfs) > return 0; > } > > + > +/* > + * PHB3 and beyond support "accordion" windows. The window's address range > + * is subdivided into phb->ioda.total_pe_num segments and there's a 1-1 > + * mapping between PEs and segments. > + * > + * They're called that because as the window size changes the segment sizes > + * change with it. Sort of like an accordion, sort of. > + */ > +static int64_t pnv_ioda_map_m64_accordion(struct pnv_phb *phb, > + int window_id, > + resource_size_t start, > + resource_size_t size) > +{ > + int64_t rc; > + > + rc = opal_pci_set_phb_mem_window(phb->opal_id, > + OPAL_M64_WINDOW_TYPE, > + window_id, > + start, > + 0, /* unused */ > + size); > + if (rc) > + goto out; > + > + rc = opal_pci_phb_mmio_enable(phb->opal_id, > + OPAL_M64_WINDOW_TYPE, > + window_id, > + OPAL_ENABLE_M64_SPLIT); > +out: > + if (rc) > + pr_err("Failed to map M64 window #%d: %lld\n", window_id, rc); > + > + return rc; > +} > + > +static int64_t pnv_ioda_map_m64_single(struct pnv_phb *phb, > + int pe_num, > + int window_id, > + resource_size_t start, > + resource_size_t size) > +{ > + int64_t rc; > + > + /* > + * The API for setting up m64 mmio windows seems to have been designed > + * with P7-IOC in mind. For that chip each M64 BAR (window) had a fixed > + * split of 8 equally sized segments each of which could individually > + * assigned to a PE. > + * > + * The problem with this is that the API doesn't have any way to > + * communicate the number of segments we want on a BAR. This wasn't > + * a problem for p7-ioc since you didn't have a choice, but the > + * single PE windows added in PHB3 don't map cleanly to this API. > + * > + * As a result we've got this slightly awkward process where we > + * call opal_pci_map_pe_mmio_window() to put the single in single > + * PE mode, and set the PE for the window before setting the address > + * bounds. We need to do it this way because the single PE windows > + * for PHB3 have different alignment requirements on PHB3. > + */ > + rc = opal_pci_map_pe_mmio_window(phb->opal_id, > + pe_num, > + OPAL_M64_WINDOW_TYPE, > + window_id, > + 0); > + if (rc) > + goto out; > + > + /* > + * NB: In single PE mode the window needs to be aligned to 32MB > + */ > + rc = opal_pci_set_phb_mem_window(phb->opal_id, > + OPAL_M64_WINDOW_TYPE, > + window_id, > + start, > + 0, /* ignored by FW, m64 is 1-1 */ > + size); > + if (rc) > + goto out; > + > + /* > + * Now actually enable it. We specified the BAR should be in "non-split" > + * mode so FW will validate that the BAR is in single PE mode. > + */ > + rc = opal_pci_phb_mmio_enable(phb->opal_id, > + OPAL_M64_WINDOW_TYPE, > + window_id, > + OPAL_ENABLE_M64_NON_SPLIT); > +out: > + if (rc) > + pr_err("Error mapping single PE BAR\n"); > + > + return rc; > +} > + > static int pnv_pci_vf_assign_m64(struct pci_dev *pdev, u16 num_vfs) > { > struct pnv_iov_data *iov; > @@ -330,7 +426,6 @@ static int pnv_pci_vf_assign_m64(struct pci_dev *pdev, u16 num_vfs) > int64_t rc; > int total_vfs; > resource_size_t size, start; > - int pe_num; > int m64_bars; > > phb = pci_bus_to_pnvhb(pdev->bus); > @@ -359,49 +454,28 @@ static int pnv_pci_vf_assign_m64(struct pci_dev *pdev, u16 num_vfs) > } while (test_and_set_bit(win, &phb->ioda.m64_bar_alloc)); > set_bit(win, iov->used_m64_bar_mask); > > + > if (iov->m64_single_mode) { > size = pci_iov_resource_size(pdev, > PCI_IOV_RESOURCES + i); > start = res->start + size * j; > + rc = pnv_ioda_map_m64_single(phb, win, > + iov->pe_num_map[j], > + start, > + size); > } else { > size = resource_size(res); > start = res->start; > - } > > - /* Map the M64 here */ > - if (iov->m64_single_mode) { > - pe_num = iov->pe_num_map[j]; > - rc = opal_pci_map_pe_mmio_window(phb->opal_id, > - pe_num, OPAL_M64_WINDOW_TYPE, > - win, 0); > + rc = pnv_ioda_map_m64_accordion(phb, win, start, > + size); > } > > - rc = opal_pci_set_phb_mem_window(phb->opal_id, > - OPAL_M64_WINDOW_TYPE, > - win, > - start, > - 0, /* unused */ > - size); > - > - > if (rc != OPAL_SUCCESS) { > dev_err(&pdev->dev, "Failed to map M64 window #%d: %lld\n", > win, rc); > goto m64_failed; > } > - > - if (iov->m64_single_mode) > - rc = opal_pci_phb_mmio_enable(phb->opal_id, > - OPAL_M64_WINDOW_TYPE, win, 2); > - else > - rc = opal_pci_phb_mmio_enable(phb->opal_id, > - OPAL_M64_WINDOW_TYPE, win, 1); > - > - if (rc != OPAL_SUCCESS) { > - dev_err(&pdev->dev, "Failed to enable M64 window #%d: %llx\n", > - win, rc); > - goto m64_failed; > - } > } > } > return 0; >
diff --git a/arch/powerpc/platforms/powernv/pci-sriov.c b/arch/powerpc/platforms/powernv/pci-sriov.c index e4c65cb49757..d53a85ccb538 100644 --- a/arch/powerpc/platforms/powernv/pci-sriov.c +++ b/arch/powerpc/platforms/powernv/pci-sriov.c @@ -320,6 +320,102 @@ static int pnv_pci_vf_release_m64(struct pci_dev *pdev, u16 num_vfs) return 0; } + +/* + * PHB3 and beyond support "accordion" windows. The window's address range + * is subdivided into phb->ioda.total_pe_num segments and there's a 1-1 + * mapping between PEs and segments. + * + * They're called that because as the window size changes the segment sizes + * change with it. Sort of like an accordion, sort of. + */ +static int64_t pnv_ioda_map_m64_accordion(struct pnv_phb *phb, + int window_id, + resource_size_t start, + resource_size_t size) +{ + int64_t rc; + + rc = opal_pci_set_phb_mem_window(phb->opal_id, + OPAL_M64_WINDOW_TYPE, + window_id, + start, + 0, /* unused */ + size); + if (rc) + goto out; + + rc = opal_pci_phb_mmio_enable(phb->opal_id, + OPAL_M64_WINDOW_TYPE, + window_id, + OPAL_ENABLE_M64_SPLIT); +out: + if (rc) + pr_err("Failed to map M64 window #%d: %lld\n", window_id, rc); + + return rc; +} + +static int64_t pnv_ioda_map_m64_single(struct pnv_phb *phb, + int pe_num, + int window_id, + resource_size_t start, + resource_size_t size) +{ + int64_t rc; + + /* + * The API for setting up m64 mmio windows seems to have been designed + * with P7-IOC in mind. For that chip each M64 BAR (window) had a fixed + * split of 8 equally sized segments each of which could individually + * assigned to a PE. + * + * The problem with this is that the API doesn't have any way to + * communicate the number of segments we want on a BAR. This wasn't + * a problem for p7-ioc since you didn't have a choice, but the + * single PE windows added in PHB3 don't map cleanly to this API. + * + * As a result we've got this slightly awkward process where we + * call opal_pci_map_pe_mmio_window() to put the single in single + * PE mode, and set the PE for the window before setting the address + * bounds. We need to do it this way because the single PE windows + * for PHB3 have different alignment requirements on PHB3. + */ + rc = opal_pci_map_pe_mmio_window(phb->opal_id, + pe_num, + OPAL_M64_WINDOW_TYPE, + window_id, + 0); + if (rc) + goto out; + + /* + * NB: In single PE mode the window needs to be aligned to 32MB + */ + rc = opal_pci_set_phb_mem_window(phb->opal_id, + OPAL_M64_WINDOW_TYPE, + window_id, + start, + 0, /* ignored by FW, m64 is 1-1 */ + size); + if (rc) + goto out; + + /* + * Now actually enable it. We specified the BAR should be in "non-split" + * mode so FW will validate that the BAR is in single PE mode. + */ + rc = opal_pci_phb_mmio_enable(phb->opal_id, + OPAL_M64_WINDOW_TYPE, + window_id, + OPAL_ENABLE_M64_NON_SPLIT); +out: + if (rc) + pr_err("Error mapping single PE BAR\n"); + + return rc; +} + static int pnv_pci_vf_assign_m64(struct pci_dev *pdev, u16 num_vfs) { struct pnv_iov_data *iov; @@ -330,7 +426,6 @@ static int pnv_pci_vf_assign_m64(struct pci_dev *pdev, u16 num_vfs) int64_t rc; int total_vfs; resource_size_t size, start; - int pe_num; int m64_bars; phb = pci_bus_to_pnvhb(pdev->bus); @@ -359,49 +454,28 @@ static int pnv_pci_vf_assign_m64(struct pci_dev *pdev, u16 num_vfs) } while (test_and_set_bit(win, &phb->ioda.m64_bar_alloc)); set_bit(win, iov->used_m64_bar_mask); + if (iov->m64_single_mode) { size = pci_iov_resource_size(pdev, PCI_IOV_RESOURCES + i); start = res->start + size * j; + rc = pnv_ioda_map_m64_single(phb, win, + iov->pe_num_map[j], + start, + size); } else { size = resource_size(res); start = res->start; - } - /* Map the M64 here */ - if (iov->m64_single_mode) { - pe_num = iov->pe_num_map[j]; - rc = opal_pci_map_pe_mmio_window(phb->opal_id, - pe_num, OPAL_M64_WINDOW_TYPE, - win, 0); + rc = pnv_ioda_map_m64_accordion(phb, win, start, + size); } - rc = opal_pci_set_phb_mem_window(phb->opal_id, - OPAL_M64_WINDOW_TYPE, - win, - start, - 0, /* unused */ - size); - - if (rc != OPAL_SUCCESS) { dev_err(&pdev->dev, "Failed to map M64 window #%d: %lld\n", win, rc); goto m64_failed; } - - if (iov->m64_single_mode) - rc = opal_pci_phb_mmio_enable(phb->opal_id, - OPAL_M64_WINDOW_TYPE, win, 2); - else - rc = opal_pci_phb_mmio_enable(phb->opal_id, - OPAL_M64_WINDOW_TYPE, win, 1); - - if (rc != OPAL_SUCCESS) { - dev_err(&pdev->dev, "Failed to enable M64 window #%d: %llx\n", - win, rc); - goto m64_failed; - } } } return 0;
The sequence required to use the single PE BAR mode is kinda janky and requires a little explanation. The API was designed with P7-IOC style windows where the setup process is something like: 1. Configure the window start / end address 2. Enable the window 3. Map the segments of each window to the PE For Single PE BARs the process is: 1. Set the PE for segment zero on a disabled window 2. Set the range 3. Enable the window Move the OPAL calls into their own helper functions where the quirks can be contained. Signed-off-by: Oliver O'Halloran <oohall@gmail.com> --- arch/powerpc/platforms/powernv/pci-sriov.c | 132 ++++++++++++++++----- 1 file changed, 103 insertions(+), 29 deletions(-)