[v2,04/15] hw/npu2: Move PHY/NTL/GENID BAR assignment to common code

Message ID	b2577f6b922328f4eb5c62d0e3aa18c8c76bb80e.1547168645.git-series.andrew.donnellan@au1.ibm.com
State	Changes Requested
Headers	show Return-Path: <skiboot-bounces+incoming=patchwork.ozlabs.org@lists.ozlabs.org> Gateway: Authorized Use Only! Violators will be prosecuted for <skiboot@lists.ozlabs.org> from <andrew.donnellan@au1.ibm.com>; Fri, 11 Jan 2019 01:09:50 -0000 Gateway: Authorized Use Only! Violators will be prosecuted; (version=TLSv1/SSLv3 cipher=AES256-GCM-SHA384 bits=256/256) Fri, 11 Jan 2019 01:09:47 -0000 From: Andrew Donnellan <andrew.donnellan@au1.ibm.com> To: skiboot@lists.ozlabs.org Date: Fri, 11 Jan 2019 12:09:24 +1100 In-Reply-To: <cover.dd6de53510e344bee896d3b7b417143107a8dff1.1547168645.git-series.andrew.donnellan@au1.ibm.com> References: <cover.dd6de53510e344bee896d3b7b417143107a8dff1.1547168645.git-series.andrew.donnellan@au1.ibm.com> In-Reply-To: <cover.dd6de53510e344bee896d3b7b417143107a8dff1.1547168645.git-series.andrew.donnellan@au1.ibm.com> References: <cover.dd6de53510e344bee896d3b7b417143107a8dff1.1547168645.git-series.andrew.donnellan@au1.ibm.com> Message-Id: <b2577f6b922328f4eb5c62d0e3aa18c8c76bb80e.1547168645.git-series.andrew.donnellan@au1.ibm.com> MIME-Version: 1.0 Subject: [Skiboot] [PATCH v2 04/15] hw/npu2: Move PHY/NTL/GENID BAR assignment to common code Precedence: list Cc: alistair@popple.id.au, arbab@linux.ibm.com Content-Type: text/plain; charset="us-ascii" Content-Transfer-Encoding: 7bit Errors-To: skiboot-bounces+incoming=patchwork.ozlabs.org@lists.ozlabs.org Sender: "Skiboot" <skiboot-bounces+incoming=patchwork.ozlabs.org@lists.ozlabs.org>
Series	Support OpenCAPI and NVLink devices on same NPU on Witherspoon \| expand [v2,00/15] Support OpenCAPI and NVLink devices on same NPU on Witherspoon [v2,01/15] hw/npu2: Simplify BAR structures [v2,02/15] hw/npu2: Merge implementations of BAR accessor functions [v2,03/15] hw/npu2: Replace npu2_dev::bars array with individual BARs [v2,04/15] hw/npu2: Move PHY/NTL/GENID BAR assignment to common code [v2,05/15] hw/npu2: Simplify npu2_write_bar() [v2,06/15] hw/npu2: Don't repopulate NPU devices in NVLink init path [v2,07/15] hw/npu2: Rework npu2_add_interrupt_map() and skip non-NVLink devices [v2,08/15] hw/npu2: Make IRQ setup code common [v2,09/15] platforms/astbmc/witherspoon: Rework NPU presence detection [v2,10/15] hw/npu2: Combined NPU brick configuration procedure [v2,11/15] hw/npu2-opencapi: FIR masking for mixed setups [v2,12/15] hw/npu2: Fix OpenCAPI PE assignment [v2,13/15] hw/npu2: Always check device type when looping through NPU devices [v2,14/15] hw/npu2-common: Allow mixed mode NPU setups [v2,15/15] hw/npu2-opencapi: Remove completed TODO

Context	Check	Description
snowpatch_ozlabs/apply_patch	warning	master/apply_patch Patch failed to apply
snowpatch_ozlabs/apply_patch	fail	Failed to apply to any branch

diff --git a/hw/npu2-common.c b/hw/npu2-common.c index 3446acb45bea..944719d2489a 100644 --- a/hw/npu2-common.c +++ b/hw/npu2-common.c @@ -17,6 +17,7 @@ #include <skiboot.h> #include <xscom.h> #include <pci.h> +#include <pci-cfg.h> #include <npu2.h> #include <npu2-regs.h> #include <bitutils.h> @@ -174,6 +175,81 @@ void npu2_write_bar(struct npu2 *p, struct npu2_bar *bar, uint32_t gcid, } } +static void assign_bars(struct npu2 *npu) +{ + uint32_t i; + struct npu2_bar *bar; + struct npu2_dev *dev; + struct npu2_bar phy_bars[] = { + /* NPU_REGS must be first in this list */ + { .type = NPU_REGS, .index = 0, + .reg = NPU2_REG_OFFSET(NPU2_STACK_STCK_0, 0, NPU2_PHY_BAR), + .enabled = true }, + { .type = NPU_PHY, .index = 0, + .reg = NPU2_REG_OFFSET(NPU2_STACK_STCK_1, 0, NPU2_PHY_BAR), + .enabled = true }, + { .type = NPU_PHY, .index = 1, + .reg = NPU2_REG_OFFSET(NPU2_STACK_STCK_2, 0, NPU2_PHY_BAR), + .enabled = true }, + }; + struct npu2_bar last_bar = + { .type = NPU_GENID, .index = 2, + .reg = NPU2_REG_OFFSET(NPU2_STACK_STCK_2, 0, NPU2_GENID_BAR) }; + + /* Set common PHY BARs */ + for (i = 0; i < ARRAY_SIZE(phy_bars); i++) { + bar = &phy_bars[i]; + npu2_get_bar(npu->chip_id, bar); + npu2_write_bar(npu, bar, npu->chip_id, npu->xscom_base); + } + + /* Device BARs */ + for (i = 0; i < npu->total_devices; i++) { + dev = &npu->devices[i]; + if (dev->type == NPU2_DEV_TYPE_UNKNOWN) + continue; + + /* NTL / OCAPI_MMIO BAR */ + bar = &dev->ntl_bar; + if (dev->type == NPU2_DEV_TYPE_NVLINK) + bar->type = NPU_NTL; + else if (dev->type == NPU2_DEV_TYPE_OPENCAPI) + bar->type = NPU_OCAPI_MMIO; + bar->index = dev->brick_index; + bar->reg = NPU2_REG_OFFSET(NPU2_STACK_STCK_0 + NPU2DEV_STACK(dev), + 0, NPU2DEV_BRICK(dev) == 0 ? + NPU2_NTL0_BAR : NPU2_NTL1_BAR); + bar->flags = PCI_CFG_BAR_TYPE_MEM | PCI_CFG_BAR_MEM64; + npu2_get_bar(npu->chip_id, bar); + npu2_write_bar(npu, bar, npu->chip_id, npu->xscom_base); + + /* GENID BAR */ + bar = &dev->genid_bar; + bar->type = NPU_GENID; + bar->index = NPU2DEV_STACK(dev); + bar->reg = NPU2_REG_OFFSET(NPU2_STACK_STCK_0 + NPU2DEV_STACK(dev), + 0, NPU2_GENID_BAR); + bar->flags = PCI_CFG_BAR_TYPE_MEM | PCI_CFG_BAR_MEM64; + npu2_get_bar(npu->chip_id, bar); + if (dev->type == NPU2_DEV_TYPE_NVLINK) { + bar->size = 0x10000; + if (NPU2DEV_BRICK(dev)) + bar->base += 0x10000; + } + npu2_write_bar(npu, bar, npu->chip_id, npu->xscom_base); + }; + + /* Global MMIO BAR */ + npu->regs = (uint64_t *)phy_bars[0].base; + npu->regs_size = phy_bars[0].size; + + /* NTL and GENID BARs are exposed to kernel via the mm + * window */ + npu2_get_bar(npu->chip_id, &last_bar); + npu->mm_base = phy_bars[2].base + phy_bars[2].size; + npu->mm_size = last_bar.base + last_bar.size - npu->mm_base; +} + static bool _i2c_presence_detect(struct npu2_dev *dev) { uint8_t state, data; @@ -350,6 +426,8 @@ static void setup_devices(struct npu2 *npu) return; } + assign_bars(npu); + if (nvlink_detected) npu2_nvlink_init_npu(npu); else if (ocapi_detected) diff --git a/hw/npu2-opencapi.c b/hw/npu2-opencapi.c index 8dac552adea8..17a16be00eba 100644 --- a/hw/npu2-opencapi.c +++ b/hw/npu2-opencapi.c @@ -730,53 +730,19 @@ static void address_translation_config(uint32_t gcid, uint32_t scom_base, } } -static void setup_global_mmio_bar(uint32_t gcid, uint32_t scom_base, - uint64_t reg[]) -{ - struct npu2_bar *bar; - struct npu2_bar phy_bars[] = { - { .type = NPU_PHY, .index = 0, - .reg = NPU2_REG_OFFSET(NPU2_STACK_STCK_2, 0, NPU2_PHY_BAR), - .enabled = true }, - { .type = NPU_PHY, .index = 1, - .reg = NPU2_REG_OFFSET(NPU2_STACK_STCK_1, 0, NPU2_PHY_BAR), - .enabled = true }, - { .type = NPU_REGS, .index = 0, - .reg = NPU2_REG_OFFSET(NPU2_STACK_STCK_0, 0, NPU2_PHY_BAR), - .enabled = true }, - }; - - for (int i = 0; i < ARRAY_SIZE(phy_bars); i++) { - bar = &phy_bars[i]; - npu2_get_bar(gcid, bar); - npu2_write_bar(NULL, bar, gcid, scom_base); - } - reg[0] = phy_bars[2].base; - reg[1] = phy_bars[2].size; -} - /* Procedure 13.1.3.8 - AFU MMIO Range BARs */ static void setup_afu_mmio_bars(uint32_t gcid, uint32_t scom_base, struct npu2_dev *dev) { uint64_t stack = index_to_stack(dev->brick_index); - uint64_t offset = index_to_block(dev->brick_index) == NPU2_BLOCK_OTL0 ? - NPU2_NTL0_BAR : NPU2_NTL1_BAR; uint64_t pa_offset = index_to_block(dev->brick_index) == NPU2_BLOCK_OTL0 ? NPU2_CQ_CTL_MISC_MMIOPA0_CONFIG : NPU2_CQ_CTL_MISC_MMIOPA1_CONFIG; uint64_t reg; prlog(PR_DEBUG, "OCAPI: %s: Setup AFU MMIO BARs\n", __func__); - dev->ntl_bar.type = NPU_OCAPI_MMIO; - dev->ntl_bar.index = dev->brick_index; - dev->ntl_bar.reg = NPU2_REG_OFFSET(stack, 0, offset); dev->ntl_bar.enabled = true; - npu2_get_bar(gcid, &dev->ntl_bar); - - prlog(PR_DEBUG, "OCAPI: AFU MMIO set to %llx, size %llx\n", - dev->ntl_bar.base, dev->ntl_bar.size); - npu2_write_bar(NULL, &dev->ntl_bar, gcid, scom_base); + npu2_write_bar(dev->npu, &dev->ntl_bar, gcid, scom_base); reg = SETFIELD(NPU2_CQ_CTL_MISC_MMIOPA_ADDR, 0ull, dev->ntl_bar.base >> 16); reg = SETFIELD(NPU2_CQ_CTL_MISC_MMIOPA_SIZE, reg, ilog2(dev->ntl_bar.size >> 16)); @@ -788,20 +754,12 @@ static void setup_afu_mmio_bars(uint32_t gcid, uint32_t scom_base, } /* Procedure 13.1.3.9 - AFU Config BARs */ -static void setup_afu_config_bars(uint32_t gcid, uint32_t scom_base, - struct npu2_dev *dev) +static void setup_afu_config_bars(struct npu2_dev *dev) { - uint64_t stack = index_to_stack(dev->brick_index); - int stack_num = stack - NPU2_STACK_STCK_0; - prlog(PR_DEBUG, "OCAPI: %s: Setup AFU Config BARs\n", __func__); - dev->genid_bar.type = NPU_GENID; - dev->genid_bar.index = stack_num; - dev->genid_bar.reg = NPU2_REG_OFFSET(stack, 0, NPU2_GENID_BAR); dev->genid_bar.enabled = true; - npu2_get_bar(gcid, &dev->genid_bar); - prlog(PR_DEBUG, "OCAPI: Assigning GENID BAR: %016llx\n", dev->genid_bar.base); - npu2_write_bar(NULL, &dev->genid_bar, gcid, scom_base); + npu2_write_bar(dev->npu, &dev->genid_bar, dev->npu->chip_id, + dev->npu->xscom_base); } static void otl_enabletx(uint32_t gcid, uint32_t scom_base, @@ -1580,8 +1538,8 @@ static void setup_device(struct npu2_dev *dev) uint64_t mm_win[2]; /* Populate PHB device node */ - phys_map_get(dev->npu->chip_id, NPU_OCAPI_MMIO, dev->brick_index, &mm_win[0], - &mm_win[1]); + mm_win[0] = dev->ntl_bar.base; + mm_win[1] = dev->ntl_bar.size; prlog(PR_DEBUG, "OCAPI: Setting MMIO window to %016llx + %016llx\n", mm_win[0], mm_win[1]); dn_phb = dt_new_addr(dt_root, "pciex", mm_win[0]); @@ -1627,7 +1585,7 @@ static void setup_device(struct npu2_dev *dev) /* Procedure 13.1.3.8 - AFU MMIO Range BARs */ setup_afu_mmio_bars(dev->npu->chip_id, dev->npu->xscom_base, dev); /* Procedure 13.1.3.9 - AFU Config BARs */ - setup_afu_config_bars(dev->npu->chip_id, dev->npu->xscom_base, dev); + setup_afu_config_bars(dev); set_fence_control(dev->npu->chip_id, dev->npu->xscom_base, dev->brick_index, 0b00); @@ -1668,7 +1626,6 @@ static void read_nvram_training_state(void) int npu2_opencapi_init_npu(struct npu2 *npu) { struct npu2_dev *dev; - uint64_t reg[2]; int rc; assert(platform.ocapi); @@ -1677,10 +1634,6 @@ int npu2_opencapi_init_npu(struct npu2 *npu) /* TODO: Test OpenCAPI with fast reboot and make it work */ disable_fast_reboot("OpenCAPI device enabled"); - setup_global_mmio_bar(npu->chip_id, npu->xscom_base, reg); - - npu->regs = (void *)reg[0]; - for (int i = 0; i < npu->total_devices; i++) { dev = &npu->devices[i]; if (dev->type != NPU2_DEV_TYPE_OPENCAPI) diff --git a/hw/npu2.c b/hw/npu2.c index 60619f381301..b920edef3265 100644 --- a/hw/npu2.c +++ b/hw/npu2.c @@ -139,7 +139,7 @@ static int64_t npu2_cfg_write_cmd(void *dev, * track the enables separately. */ if (NPU2DEV_BRICK(ndev)) - ndev->genid_bar.enabled1 = enabled; + ndev->genid_bar.enabled1 = enabled; // TODO FIXME TABS GAHHHHH else ndev->genid_bar.enabled0 = enabled; @@ -1314,59 +1314,6 @@ static const struct phb_ops npu_ops = { .tce_kill = npu2_tce_kill, }; -static void assign_mmio_bars(uint64_t gcid, uint32_t scom, uint64_t reg[2], uint64_t mm_win[2]) -{ - uint32_t i; - struct npu2_bar *bar; - struct npu2_bar npu2_bars[] = { - /* NPU_REGS must be first in this list */ - { .type = NPU_REGS, .index = 0, - .reg = NPU2_REG_OFFSET(NPU2_STACK_STCK_0, 0, NPU2_PHY_BAR), - .enabled = true }, - { .type = NPU_PHY, .index = 0, - .reg = NPU2_REG_OFFSET(NPU2_STACK_STCK_1, 0, NPU2_PHY_BAR), - .enabled = true }, - { .type = NPU_PHY, .index = 1, - .reg = NPU2_REG_OFFSET(NPU2_STACK_STCK_2, 0, NPU2_PHY_BAR), - .enabled = true }, - { .type = NPU_NTL, .index = 0, - .reg = NPU2_REG_OFFSET(NPU2_STACK_STCK_0, 0, NPU2_NTL0_BAR) }, - { .type = NPU_NTL, .index = 1, - .reg = NPU2_REG_OFFSET(NPU2_STACK_STCK_0, 0, NPU2_NTL1_BAR) }, - { .type = NPU_NTL, .index = 2, - .reg = NPU2_REG_OFFSET(NPU2_STACK_STCK_1, 0, NPU2_NTL0_BAR) }, - { .type = NPU_NTL, .index = 3, - .reg = NPU2_REG_OFFSET(NPU2_STACK_STCK_1, 0, NPU2_NTL1_BAR) }, - { .type = NPU_NTL, .index = 4, - .reg = NPU2_REG_OFFSET(NPU2_STACK_STCK_2, 0, NPU2_NTL0_BAR) }, - { .type = NPU_NTL, .index = 5, - .reg = NPU2_REG_OFFSET(NPU2_STACK_STCK_2, 0, NPU2_NTL1_BAR) }, - { .type = NPU_GENID, .index = 0, - .reg = NPU2_REG_OFFSET(NPU2_STACK_STCK_0, 0, NPU2_GENID_BAR) }, - { .type = NPU_GENID, .index = 1, - .reg = NPU2_REG_OFFSET(NPU2_STACK_STCK_1, 0, NPU2_GENID_BAR) }, - { .type = NPU_GENID, .index = 2, - .reg = NPU2_REG_OFFSET(NPU2_STACK_STCK_2, 0, NPU2_GENID_BAR) }, - }; - - for (i = 0; i < ARRAY_SIZE(npu2_bars); i++) { - bar = &npu2_bars[i]; - npu2_get_bar(gcid, bar); - npu2_write_bar(NULL, bar, gcid, scom); - } - - /* Global MMIO BAR */ - reg[0] = npu2_bars[0].base; - reg[1] = npu2_bars[0].size; - - /* NTL and GENID BARs are exposed to kernel via the mm - * window */ - mm_win[0] = npu2_bars[3].base; - mm_win[1] = npu2_bars[ARRAY_SIZE(npu2_bars) - 1].base + - npu2_bars[ARRAY_SIZE(npu2_bars) - 1].size - - mm_win[0]; -} - /* * Set up NPU for NVLink and create PCI root device node * accordingly. @@ -1428,19 +1375,9 @@ int npu2_nvlink_init_npu(struct npu2 *npu) xscom_write_mask(npu->chip_id, 0x5011469, val, PPC_BITMASK(6,11)); xscom_write_mask(npu->chip_id, 0x5011499, val, PPC_BITMASK(6,11)); - /* Reassign the BARs */ - assign_mmio_bars(npu->chip_id, npu->xscom_base, reg, mm_win); - npu->regs = (void *)reg[0]; - npu->mm_base = mm_win[0]; - npu->mm_size = mm_win[1]; - - if (reg[0] && reg[1]) - prlog(PR_INFO, " Global MMIO BAR: %016llx (%lldMB)\n", - reg[0], reg[1] >> 20); - else - prlog(PR_ERR, " Global MMIO BAR: Disabled\n"); - /* Populate PCI root device node */ + reg[0] = (uint64_t)npu->regs; + reg[1] = npu->regs_size; np = dt_new_addr(dt_root, "pciex", reg[0]); assert(np); dt_add_property_strings(np, @@ -1455,6 +1392,8 @@ int npu2_nvlink_init_npu(struct npu2 *npu) dt_add_property_cells(np, "ibm,xscom-base", npu->xscom_base); dt_add_property_cells(np, "ibm,npcq", npu->dt_node->phandle); dt_add_property_cells(np, "ibm,links", npu->total_devices); + mm_win[0] = npu->mm_base; + mm_win[1] = npu->mm_size; dt_add_property(np, "ibm,mmio-window", mm_win, sizeof(mm_win)); dt_add_property_cells(np, "ibm,phb-diag-data-size", 0); @@ -1679,7 +1618,6 @@ static void npu2_populate_devices(struct npu2 *p, struct npu2_dev *dev; struct dt_node *npu2_dn, *link; uint32_t npu_phandle, index = 0; - int stack; /* * Get the npu node which has the links which we expand here @@ -1713,32 +1651,6 @@ static void npu2_populate_devices(struct npu2 *p, dev->pl_xscom_base = dt_prop_get_u64(link, "ibm,npu-phy"); dev->lane_mask = dt_prop_get_u32(link, "ibm,npu-lane-mask"); - /* Populate BARs. BAR0/1 is the NTL bar. */ - stack = NPU2_STACK_STCK_0 + NPU2DEV_STACK(dev); - dev->ntl_bar.type = NPU_NTL; - dev->ntl_bar.index = dev->brick_index; - dev->ntl_bar.reg = NPU2_REG_OFFSET(stack, 0, - NPU2DEV_BRICK(dev) == 0 ? - NPU2_NTL0_BAR : - NPU2_NTL1_BAR); - npu2_get_bar(p->chip_id, &dev->ntl_bar); - - dev->ntl_bar.flags = PCI_CFG_BAR_TYPE_MEM | PCI_CFG_BAR_MEM64; - - /* BAR2/3 is the GENID bar. */ - dev->genid_bar.type = NPU_GENID; - dev->genid_bar.index = NPU2DEV_STACK(dev); - dev->genid_bar.reg = NPU2_REG_OFFSET(stack, 0, - NPU2_GENID_BAR); - npu2_get_bar(p->chip_id, &dev->genid_bar); - - /* The GENID is a single physical BAR that we split - * for each emulated device */ - dev->genid_bar.size = 0x10000; - if (NPU2DEV_BRICK(dev)) - dev->genid_bar.base += 0x10000; - dev->genid_bar.flags = PCI_CFG_BAR_TYPE_MEM | PCI_CFG_BAR_MEM64; - /* Initialize PCI virtual device */ dev->nvlink.pvd = pci_virt_add_device(&p->phb_nvlink, dev->bdfn, 0x100, dev); if (dev->nvlink.pvd) diff --git a/include/npu2.h b/include/npu2.h index c2ba4370c063..31d9a4a1053f 100644 --- a/include/npu2.h +++ b/include/npu2.h @@ -147,6 +147,7 @@ struct npu2 { uint32_t chip_id; uint64_t xscom_base; void *regs; + uint64_t regs_size; uint64_t mm_base; uint64_t mm_size; uint32_t base_lsi;

[v2,04/15] hw/npu2: Move PHY/NTL/GENID BAR assignment to common code

Checks

Commit Message

Patch