| Message ID | 20200529114641.121332-6-anup.patel@wdc.com |
|---|---|
| State | New |
| Headers | show |
| Series | RISC-V multi-socket support | expand |
On Fri, May 29, 2020 at 4:49 AM Anup Patel <anup.patel@wdc.com> wrote: > > We extend RISC-V virt machine to allow creating a multi-socket > machine. Each RISC-V virt machine socket is a NUMA node having > a set of HARTs, a memory instance, a CLINT instance, and a PLIC > instance. Other devices are shared between all sockets. We also > update the generated device tree accordingly. > > By default, NUMA multi-socket support is disabled for RISC-V virt > machine. To enable it, users can use "-numa" command-line options > of QEMU. > > Example1: For two NUMA nodes with 2 CPUs each, append following > to command-line options: "-smp 4 -numa node -numa node" > > Example2: For two NUMA nodes with 1 and 3 CPUs, append following > to command-line options: > "-smp 4 -numa node -numa node -numa cpu,node-id=0,core-id=0 \ > -numa cpu,node-id=1,core-id=1 -numa cpu,node-id=1,core-id=2 \ > -numa cpu,node-id=1,core-id=3" > > The maximum number of sockets in a RISC-V virt machine is 8 > but this limit can be changed in future. > > Signed-off-by: Anup Patel <anup.patel@wdc.com> > --- > hw/riscv/virt.c | 530 +++++++++++++++++++++++----------------- > include/hw/riscv/virt.h | 9 +- > 2 files changed, 308 insertions(+), 231 deletions(-) > > diff --git a/hw/riscv/virt.c b/hw/riscv/virt.c > index 421815081d..2863b42cea 100644 > --- a/hw/riscv/virt.c > +++ b/hw/riscv/virt.c > @@ -35,6 +35,7 @@ > #include "hw/riscv/sifive_test.h" > #include "hw/riscv/virt.h" > #include "hw/riscv/boot.h" > +#include "hw/riscv/numa.h" > #include "chardev/char.h" > #include "sysemu/arch_init.h" > #include "sysemu/device_tree.h" > @@ -60,7 +61,7 @@ static const struct MemmapEntry { > [VIRT_TEST] = { 0x100000, 0x1000 }, > [VIRT_RTC] = { 0x101000, 0x1000 }, > [VIRT_CLINT] = { 0x2000000, 0x10000 }, > - [VIRT_PLIC] = { 0xc000000, 0x4000000 }, > + [VIRT_PLIC] = { 0xc000000, VIRT_PLIC_SIZE(VIRT_CPUS_MAX * 2) }, > [VIRT_UART0] = { 0x10000000, 0x100 }, > [VIRT_VIRTIO] = { 0x10001000, 0x1000 }, > [VIRT_FLASH] = { 0x20000000, 0x4000000 }, > @@ -182,10 +183,17 @@ static void create_fdt(RISCVVirtState *s, const struct MemmapEntry *memmap, > uint64_t mem_size, const char *cmdline) > { > void *fdt; > - int cpu, i; > - uint32_t *cells; > - char *nodename; > - uint32_t plic_phandle, test_phandle, phandle = 1; > + int i, cpu, socket; > + MachineState *mc = MACHINE(s); > + uint64_t addr, size; > + uint32_t *clint_cells, *plic_cells; > + unsigned long clint_addr, plic_addr; > + uint32_t plic_phandle[MAX_NODES]; > + uint32_t cpu_phandle, intc_phandle, test_phandle; > + uint32_t phandle = 1, plic_mmio_phandle = 1; > + uint32_t plic_pcie_phandle = 1, plic_virtio_phandle = 1; > + char *mem_name, *cpu_name, *core_name, *intc_name; > + char *name, *clint_name, *plic_name, *clust_name; > hwaddr flashsize = virt_memmap[VIRT_FLASH].size / 2; > hwaddr flashbase = virt_memmap[VIRT_FLASH].base; > > @@ -206,231 +214,238 @@ static void create_fdt(RISCVVirtState *s, const struct MemmapEntry *memmap, > qemu_fdt_setprop_cell(fdt, "/soc", "#size-cells", 0x2); > qemu_fdt_setprop_cell(fdt, "/soc", "#address-cells", 0x2); > > - nodename = g_strdup_printf("/memory@%lx", > - (long)memmap[VIRT_DRAM].base); > - qemu_fdt_add_subnode(fdt, nodename); > - qemu_fdt_setprop_cells(fdt, nodename, "reg", > - memmap[VIRT_DRAM].base >> 32, memmap[VIRT_DRAM].base, > - mem_size >> 32, mem_size); > - qemu_fdt_setprop_string(fdt, nodename, "device_type", "memory"); > - g_free(nodename); > - > qemu_fdt_add_subnode(fdt, "/cpus"); > qemu_fdt_setprop_cell(fdt, "/cpus", "timebase-frequency", > SIFIVE_CLINT_TIMEBASE_FREQ); > qemu_fdt_setprop_cell(fdt, "/cpus", "#size-cells", 0x0); > qemu_fdt_setprop_cell(fdt, "/cpus", "#address-cells", 0x1); > + qemu_fdt_add_subnode(fdt, "/cpus/cpu-map"); I'm no expert with cpu-map. Do you mind CCing Atish in the next version and see if he can Ack these DT changes? > + > + for (socket = (riscv_socket_count(mc) - 1); socket >= 0; socket--) { > + clust_name = g_strdup_printf("/cpus/cpu-map/cluster%d", socket); > + qemu_fdt_add_subnode(fdt, clust_name); > + > + plic_cells = g_new0(uint32_t, s->soc[socket].num_harts * 4); > + clint_cells = g_new0(uint32_t, s->soc[socket].num_harts * 4); > + > + for (cpu = s->soc[socket].num_harts - 1; cpu >= 0; cpu--) { > + cpu_phandle = phandle++; > > - for (cpu = s->soc.num_harts - 1; cpu >= 0; cpu--) { > - int cpu_phandle = phandle++; > - int intc_phandle; > - nodename = g_strdup_printf("/cpus/cpu@%d", cpu); > - char *intc = g_strdup_printf("/cpus/cpu@%d/interrupt-controller", cpu); > - char *isa = riscv_isa_string(&s->soc.harts[cpu]); > - qemu_fdt_add_subnode(fdt, nodename); > + cpu_name = g_strdup_printf("/cpus/cpu@%d", > + s->soc[socket].hartid_base + cpu); > + qemu_fdt_add_subnode(fdt, cpu_name); > #if defined(TARGET_RISCV32) > - qemu_fdt_setprop_string(fdt, nodename, "mmu-type", "riscv,sv32"); > + qemu_fdt_setprop_string(fdt, cpu_name, "mmu-type", "riscv,sv32"); > #else > - qemu_fdt_setprop_string(fdt, nodename, "mmu-type", "riscv,sv48"); > + qemu_fdt_setprop_string(fdt, cpu_name, "mmu-type", "riscv,sv48"); > #endif > - qemu_fdt_setprop_string(fdt, nodename, "riscv,isa", isa); > - qemu_fdt_setprop_string(fdt, nodename, "compatible", "riscv"); > - qemu_fdt_setprop_string(fdt, nodename, "status", "okay"); > - qemu_fdt_setprop_cell(fdt, nodename, "reg", cpu); > - qemu_fdt_setprop_string(fdt, nodename, "device_type", "cpu"); > - qemu_fdt_setprop_cell(fdt, nodename, "phandle", cpu_phandle); > - intc_phandle = phandle++; > - qemu_fdt_add_subnode(fdt, intc); > - qemu_fdt_setprop_cell(fdt, intc, "phandle", intc_phandle); > - qemu_fdt_setprop_string(fdt, intc, "compatible", "riscv,cpu-intc"); > - qemu_fdt_setprop(fdt, intc, "interrupt-controller", NULL, 0); > - qemu_fdt_setprop_cell(fdt, intc, "#interrupt-cells", 1); > - g_free(isa); > - g_free(intc); > - g_free(nodename); > - } > + name = riscv_isa_string(&s->soc[socket].harts[cpu]); > + qemu_fdt_setprop_string(fdt, cpu_name, "riscv,isa", name); > + g_free(name); > + qemu_fdt_setprop_string(fdt, cpu_name, "compatible", "riscv"); > + qemu_fdt_setprop_string(fdt, cpu_name, "status", "okay"); > + qemu_fdt_setprop_cell(fdt, cpu_name, "reg", > + s->soc[socket].hartid_base + cpu); > + qemu_fdt_setprop_string(fdt, cpu_name, "device_type", "cpu"); > + riscv_socket_fdt_write_id(mc, fdt, cpu_name, socket); > + qemu_fdt_setprop_cell(fdt, cpu_name, "phandle", cpu_phandle); > + > + intc_name = g_strdup_printf("%s/interrupt-controller", cpu_name); > + qemu_fdt_add_subnode(fdt, intc_name); > + intc_phandle = phandle++; > + qemu_fdt_setprop_cell(fdt, intc_name, "phandle", intc_phandle); > + qemu_fdt_setprop_string(fdt, intc_name, "compatible", > + "riscv,cpu-intc"); > + qemu_fdt_setprop(fdt, intc_name, "interrupt-controller", NULL, 0); > + qemu_fdt_setprop_cell(fdt, intc_name, "#interrupt-cells", 1); > + > + clint_cells[cpu * 4 + 0] = cpu_to_be32(intc_phandle); > + clint_cells[cpu * 4 + 1] = cpu_to_be32(IRQ_M_SOFT); > + clint_cells[cpu * 4 + 2] = cpu_to_be32(intc_phandle); > + clint_cells[cpu * 4 + 3] = cpu_to_be32(IRQ_M_TIMER); > + > + plic_cells[cpu * 4 + 0] = cpu_to_be32(intc_phandle); > + plic_cells[cpu * 4 + 1] = cpu_to_be32(IRQ_M_EXT); > + plic_cells[cpu * 4 + 2] = cpu_to_be32(intc_phandle); > + plic_cells[cpu * 4 + 3] = cpu_to_be32(IRQ_S_EXT); > + > + core_name = g_strdup_printf("%s/core%d", clust_name, cpu); > + qemu_fdt_add_subnode(fdt, core_name); > + qemu_fdt_setprop_cell(fdt, core_name, "cpu", cpu_phandle); > + > + g_free(core_name); > + g_free(intc_name); > + g_free(cpu_name); > + } > > - /* Add cpu-topology node */ > - qemu_fdt_add_subnode(fdt, "/cpus/cpu-map"); > - qemu_fdt_add_subnode(fdt, "/cpus/cpu-map/cluster0"); > - for (cpu = s->soc.num_harts - 1; cpu >= 0; cpu--) { > - char *core_nodename = g_strdup_printf("/cpus/cpu-map/cluster0/core%d", > - cpu); > - char *cpu_nodename = g_strdup_printf("/cpus/cpu@%d", cpu); > - uint32_t intc_phandle = qemu_fdt_get_phandle(fdt, cpu_nodename); > - qemu_fdt_add_subnode(fdt, core_nodename); > - qemu_fdt_setprop_cell(fdt, core_nodename, "cpu", intc_phandle); > - g_free(core_nodename); > - g_free(cpu_nodename); > + addr = memmap[VIRT_DRAM].base + riscv_socket_mem_offset(mc, socket); > + size = riscv_socket_mem_size(mc, socket); > + mem_name = g_strdup_printf("/memory@%lx", (long)addr); > + qemu_fdt_add_subnode(fdt, mem_name); > + qemu_fdt_setprop_cells(fdt, mem_name, "reg", > + addr >> 32, addr, size >> 32, size); > + qemu_fdt_setprop_string(fdt, mem_name, "device_type", "memory"); > + riscv_socket_fdt_write_id(mc, fdt, mem_name, socket); > + g_free(mem_name); > + > + clint_addr = memmap[VIRT_CLINT].base + > + (memmap[VIRT_CLINT].size * socket); > + clint_name = g_strdup_printf("/soc/clint@%lx", clint_addr); > + qemu_fdt_add_subnode(fdt, clint_name); > + qemu_fdt_setprop_string(fdt, clint_name, "compatible", "riscv,clint0"); > + qemu_fdt_setprop_cells(fdt, clint_name, "reg", > + 0x0, clint_addr, 0x0, memmap[VIRT_CLINT].size); > + qemu_fdt_setprop(fdt, clint_name, "interrupts-extended", > + clint_cells, s->soc[socket].num_harts * sizeof(uint32_t) * 4); > + riscv_socket_fdt_write_id(mc, fdt, clint_name, socket); > + g_free(clint_name); > + > + plic_phandle[socket] = phandle++; > + plic_addr = memmap[VIRT_PLIC].base + (memmap[VIRT_PLIC].size * socket); > + plic_name = g_strdup_printf("/soc/plic@%lx", plic_addr); > + qemu_fdt_add_subnode(fdt, plic_name); > + qemu_fdt_setprop_cell(fdt, plic_name, > + "#address-cells", FDT_PLIC_ADDR_CELLS); > + qemu_fdt_setprop_cell(fdt, plic_name, > + "#interrupt-cells", FDT_PLIC_INT_CELLS); > + qemu_fdt_setprop_string(fdt, plic_name, "compatible", "riscv,plic0"); > + qemu_fdt_setprop(fdt, plic_name, "interrupt-controller", NULL, 0); > + qemu_fdt_setprop(fdt, plic_name, "interrupts-extended", > + plic_cells, s->soc[socket].num_harts * sizeof(uint32_t) * 4); > + qemu_fdt_setprop_cells(fdt, plic_name, "reg", > + 0x0, plic_addr, 0x0, memmap[VIRT_PLIC].size); > + qemu_fdt_setprop_cell(fdt, plic_name, "riscv,ndev", VIRTIO_NDEV); > + riscv_socket_fdt_write_id(mc, fdt, plic_name, socket); > + qemu_fdt_setprop_cell(fdt, plic_name, "phandle", plic_phandle[socket]); > + g_free(plic_name); > + > + g_free(clint_cells); > + g_free(plic_cells); > + g_free(clust_name); > } > > - cells = g_new0(uint32_t, s->soc.num_harts * 4); > - for (cpu = 0; cpu < s->soc.num_harts; cpu++) { > - nodename = > - g_strdup_printf("/cpus/cpu@%d/interrupt-controller", cpu); > - uint32_t intc_phandle = qemu_fdt_get_phandle(fdt, nodename); > - cells[cpu * 4 + 0] = cpu_to_be32(intc_phandle); > - cells[cpu * 4 + 1] = cpu_to_be32(IRQ_M_SOFT); > - cells[cpu * 4 + 2] = cpu_to_be32(intc_phandle); > - cells[cpu * 4 + 3] = cpu_to_be32(IRQ_M_TIMER); > - g_free(nodename); > - } > - nodename = g_strdup_printf("/soc/clint@%lx", > - (long)memmap[VIRT_CLINT].base); > - qemu_fdt_add_subnode(fdt, nodename); > - qemu_fdt_setprop_string(fdt, nodename, "compatible", "riscv,clint0"); > - qemu_fdt_setprop_cells(fdt, nodename, "reg", > - 0x0, memmap[VIRT_CLINT].base, > - 0x0, memmap[VIRT_CLINT].size); > - qemu_fdt_setprop(fdt, nodename, "interrupts-extended", > - cells, s->soc.num_harts * sizeof(uint32_t) * 4); > - g_free(cells); > - g_free(nodename); > - > - plic_phandle = phandle++; > - cells = g_new0(uint32_t, s->soc.num_harts * 4); > - for (cpu = 0; cpu < s->soc.num_harts; cpu++) { > - nodename = > - g_strdup_printf("/cpus/cpu@%d/interrupt-controller", cpu); > - uint32_t intc_phandle = qemu_fdt_get_phandle(fdt, nodename); > - cells[cpu * 4 + 0] = cpu_to_be32(intc_phandle); > - cells[cpu * 4 + 1] = cpu_to_be32(IRQ_M_EXT); > - cells[cpu * 4 + 2] = cpu_to_be32(intc_phandle); > - cells[cpu * 4 + 3] = cpu_to_be32(IRQ_S_EXT); > - g_free(nodename); > + for (socket = 0; socket < riscv_socket_count(mc); socket++) { > + if (socket == 0) { > + plic_mmio_phandle = plic_phandle[socket]; > + plic_virtio_phandle = plic_phandle[socket]; > + plic_pcie_phandle = plic_phandle[socket]; > + } > + if (socket == 1) { > + plic_virtio_phandle = plic_phandle[socket]; > + plic_pcie_phandle = plic_phandle[socket]; > + } > + if (socket == 2) { > + plic_pcie_phandle = plic_phandle[socket]; > + } > } > - nodename = g_strdup_printf("/soc/interrupt-controller@%lx", > - (long)memmap[VIRT_PLIC].base); > - qemu_fdt_add_subnode(fdt, nodename); > - qemu_fdt_setprop_cell(fdt, nodename, "#address-cells", > - FDT_PLIC_ADDR_CELLS); > - qemu_fdt_setprop_cell(fdt, nodename, "#interrupt-cells", > - FDT_PLIC_INT_CELLS); > - qemu_fdt_setprop_string(fdt, nodename, "compatible", "riscv,plic0"); > - qemu_fdt_setprop(fdt, nodename, "interrupt-controller", NULL, 0); > - qemu_fdt_setprop(fdt, nodename, "interrupts-extended", > - cells, s->soc.num_harts * sizeof(uint32_t) * 4); > - qemu_fdt_setprop_cells(fdt, nodename, "reg", > - 0x0, memmap[VIRT_PLIC].base, > - 0x0, memmap[VIRT_PLIC].size); > - qemu_fdt_setprop_cell(fdt, nodename, "riscv,ndev", VIRTIO_NDEV); > - qemu_fdt_setprop_cell(fdt, nodename, "phandle", plic_phandle); > - plic_phandle = qemu_fdt_get_phandle(fdt, nodename); > - g_free(cells); > - g_free(nodename); > + > + riscv_socket_fdt_write_distance_matrix(mc, fdt); > > for (i = 0; i < VIRTIO_COUNT; i++) { > - nodename = g_strdup_printf("/virtio_mmio@%lx", > + name = g_strdup_printf("/soc/virtio_mmio@%lx", > (long)(memmap[VIRT_VIRTIO].base + i * memmap[VIRT_VIRTIO].size)); > - qemu_fdt_add_subnode(fdt, nodename); > - qemu_fdt_setprop_string(fdt, nodename, "compatible", "virtio,mmio"); > - qemu_fdt_setprop_cells(fdt, nodename, "reg", > + qemu_fdt_add_subnode(fdt, name); > + qemu_fdt_setprop_string(fdt, name, "compatible", "virtio,mmio"); > + qemu_fdt_setprop_cells(fdt, name, "reg", > 0x0, memmap[VIRT_VIRTIO].base + i * memmap[VIRT_VIRTIO].size, > 0x0, memmap[VIRT_VIRTIO].size); > - qemu_fdt_setprop_cell(fdt, nodename, "interrupt-parent", plic_phandle); > - qemu_fdt_setprop_cell(fdt, nodename, "interrupts", VIRTIO_IRQ + i); > - g_free(nodename); > + qemu_fdt_setprop_cell(fdt, name, "interrupt-parent", > + plic_virtio_phandle); > + qemu_fdt_setprop_cell(fdt, name, "interrupts", VIRTIO_IRQ + i); > + g_free(name); > } > > - nodename = g_strdup_printf("/soc/pci@%lx", > + name = g_strdup_printf("/soc/pci@%lx", > (long) memmap[VIRT_PCIE_ECAM].base); > - qemu_fdt_add_subnode(fdt, nodename); > - qemu_fdt_setprop_cell(fdt, nodename, "#address-cells", > - FDT_PCI_ADDR_CELLS); > - qemu_fdt_setprop_cell(fdt, nodename, "#interrupt-cells", > - FDT_PCI_INT_CELLS); > - qemu_fdt_setprop_cell(fdt, nodename, "#size-cells", 0x2); > - qemu_fdt_setprop_string(fdt, nodename, "compatible", > - "pci-host-ecam-generic"); > - qemu_fdt_setprop_string(fdt, nodename, "device_type", "pci"); > - qemu_fdt_setprop_cell(fdt, nodename, "linux,pci-domain", 0); > - qemu_fdt_setprop_cells(fdt, nodename, "bus-range", 0, > - memmap[VIRT_PCIE_ECAM].size / > - PCIE_MMCFG_SIZE_MIN - 1); > - qemu_fdt_setprop(fdt, nodename, "dma-coherent", NULL, 0); > - qemu_fdt_setprop_cells(fdt, nodename, "reg", 0, memmap[VIRT_PCIE_ECAM].base, > - 0, memmap[VIRT_PCIE_ECAM].size); > - qemu_fdt_setprop_sized_cells(fdt, nodename, "ranges", > + qemu_fdt_add_subnode(fdt, name); > + qemu_fdt_setprop_cell(fdt, name, "#address-cells", FDT_PCI_ADDR_CELLS); > + qemu_fdt_setprop_cell(fdt, name, "#interrupt-cells", FDT_PCI_INT_CELLS); > + qemu_fdt_setprop_cell(fdt, name, "#size-cells", 0x2); > + qemu_fdt_setprop_string(fdt, name, "compatible", "pci-host-ecam-generic"); > + qemu_fdt_setprop_string(fdt, name, "device_type", "pci"); > + qemu_fdt_setprop_cell(fdt, name, "linux,pci-domain", 0); > + qemu_fdt_setprop_cells(fdt, name, "bus-range", 0, > + memmap[VIRT_PCIE_ECAM].size / PCIE_MMCFG_SIZE_MIN - 1); > + qemu_fdt_setprop(fdt, name, "dma-coherent", NULL, 0); > + qemu_fdt_setprop_cells(fdt, name, "reg", 0, > + memmap[VIRT_PCIE_ECAM].base, 0, memmap[VIRT_PCIE_ECAM].size); > + qemu_fdt_setprop_sized_cells(fdt, name, "ranges", > 1, FDT_PCI_RANGE_IOPORT, 2, 0, > 2, memmap[VIRT_PCIE_PIO].base, 2, memmap[VIRT_PCIE_PIO].size, > 1, FDT_PCI_RANGE_MMIO, > 2, memmap[VIRT_PCIE_MMIO].base, > 2, memmap[VIRT_PCIE_MMIO].base, 2, memmap[VIRT_PCIE_MMIO].size); > - create_pcie_irq_map(fdt, nodename, plic_phandle); > - g_free(nodename); > + create_pcie_irq_map(fdt, name, plic_pcie_phandle); > + g_free(name); > > test_phandle = phandle++; > - nodename = g_strdup_printf("/test@%lx", > + name = g_strdup_printf("/soc/test@%lx", > (long)memmap[VIRT_TEST].base); > - qemu_fdt_add_subnode(fdt, nodename); > + qemu_fdt_add_subnode(fdt, name); > { > const char compat[] = "sifive,test1\0sifive,test0\0syscon"; > - qemu_fdt_setprop(fdt, nodename, "compatible", compat, sizeof(compat)); > + qemu_fdt_setprop(fdt, name, "compatible", compat, sizeof(compat)); > } > - qemu_fdt_setprop_cells(fdt, nodename, "reg", > + qemu_fdt_setprop_cells(fdt, name, "reg", > 0x0, memmap[VIRT_TEST].base, > 0x0, memmap[VIRT_TEST].size); > - qemu_fdt_setprop_cell(fdt, nodename, "phandle", test_phandle); > - test_phandle = qemu_fdt_get_phandle(fdt, nodename); > - g_free(nodename); > - > - nodename = g_strdup_printf("/reboot"); > - qemu_fdt_add_subnode(fdt, nodename); > - qemu_fdt_setprop_string(fdt, nodename, "compatible", "syscon-reboot"); > - qemu_fdt_setprop_cell(fdt, nodename, "regmap", test_phandle); > - qemu_fdt_setprop_cell(fdt, nodename, "offset", 0x0); > - qemu_fdt_setprop_cell(fdt, nodename, "value", FINISHER_RESET); > - g_free(nodename); > - > - nodename = g_strdup_printf("/poweroff"); > - qemu_fdt_add_subnode(fdt, nodename); > - qemu_fdt_setprop_string(fdt, nodename, "compatible", "syscon-poweroff"); > - qemu_fdt_setprop_cell(fdt, nodename, "regmap", test_phandle); > - qemu_fdt_setprop_cell(fdt, nodename, "offset", 0x0); > - qemu_fdt_setprop_cell(fdt, nodename, "value", FINISHER_PASS); > - g_free(nodename); > - > - nodename = g_strdup_printf("/uart@%lx", > - (long)memmap[VIRT_UART0].base); > - qemu_fdt_add_subnode(fdt, nodename); > - qemu_fdt_setprop_string(fdt, nodename, "compatible", "ns16550a"); > - qemu_fdt_setprop_cells(fdt, nodename, "reg", > + qemu_fdt_setprop_cell(fdt, name, "phandle", test_phandle); > + test_phandle = qemu_fdt_get_phandle(fdt, name); > + g_free(name); > + > + name = g_strdup_printf("/soc/reboot"); > + qemu_fdt_add_subnode(fdt, name); > + qemu_fdt_setprop_string(fdt, name, "compatible", "syscon-reboot"); > + qemu_fdt_setprop_cell(fdt, name, "regmap", test_phandle); > + qemu_fdt_setprop_cell(fdt, name, "offset", 0x0); > + qemu_fdt_setprop_cell(fdt, name, "value", FINISHER_RESET); > + g_free(name); > + > + name = g_strdup_printf("/soc/poweroff"); > + qemu_fdt_add_subnode(fdt, name); > + qemu_fdt_setprop_string(fdt, name, "compatible", "syscon-poweroff"); > + qemu_fdt_setprop_cell(fdt, name, "regmap", test_phandle); > + qemu_fdt_setprop_cell(fdt, name, "offset", 0x0); > + qemu_fdt_setprop_cell(fdt, name, "value", FINISHER_PASS); > + g_free(name); > + > + name = g_strdup_printf("/soc/uart@%lx", (long)memmap[VIRT_UART0].base); > + qemu_fdt_add_subnode(fdt, name); > + qemu_fdt_setprop_string(fdt, name, "compatible", "ns16550a"); > + qemu_fdt_setprop_cells(fdt, name, "reg", > 0x0, memmap[VIRT_UART0].base, > 0x0, memmap[VIRT_UART0].size); > - qemu_fdt_setprop_cell(fdt, nodename, "clock-frequency", 3686400); > - qemu_fdt_setprop_cell(fdt, nodename, "interrupt-parent", plic_phandle); > - qemu_fdt_setprop_cell(fdt, nodename, "interrupts", UART0_IRQ); > + qemu_fdt_setprop_cell(fdt, name, "clock-frequency", 3686400); > + qemu_fdt_setprop_cell(fdt, name, "interrupt-parent", plic_mmio_phandle); > + qemu_fdt_setprop_cell(fdt, name, "interrupts", UART0_IRQ); > > qemu_fdt_add_subnode(fdt, "/chosen"); > - qemu_fdt_setprop_string(fdt, "/chosen", "stdout-path", nodename); > + qemu_fdt_setprop_string(fdt, "/chosen", "stdout-path", name); > if (cmdline) { > qemu_fdt_setprop_string(fdt, "/chosen", "bootargs", cmdline); > } > - g_free(nodename); > - > - nodename = g_strdup_printf("/rtc@%lx", > - (long)memmap[VIRT_RTC].base); > - qemu_fdt_add_subnode(fdt, nodename); > - qemu_fdt_setprop_string(fdt, nodename, "compatible", > - "google,goldfish-rtc"); > - qemu_fdt_setprop_cells(fdt, nodename, "reg", > + g_free(name); > + > + name = g_strdup_printf("/soc/rtc@%lx", (long)memmap[VIRT_RTC].base); > + qemu_fdt_add_subnode(fdt, name); > + qemu_fdt_setprop_string(fdt, name, "compatible", "google,goldfish-rtc"); > + qemu_fdt_setprop_cells(fdt, name, "reg", > 0x0, memmap[VIRT_RTC].base, > 0x0, memmap[VIRT_RTC].size); > - qemu_fdt_setprop_cell(fdt, nodename, "interrupt-parent", plic_phandle); > - qemu_fdt_setprop_cell(fdt, nodename, "interrupts", RTC_IRQ); > - g_free(nodename); > - > - nodename = g_strdup_printf("/flash@%" PRIx64, flashbase); > - qemu_fdt_add_subnode(s->fdt, nodename); > - qemu_fdt_setprop_string(s->fdt, nodename, "compatible", "cfi-flash"); > - qemu_fdt_setprop_sized_cells(s->fdt, nodename, "reg", > + qemu_fdt_setprop_cell(fdt, name, "interrupt-parent", plic_mmio_phandle); > + qemu_fdt_setprop_cell(fdt, name, "interrupts", RTC_IRQ); > + g_free(name); > + > + name = g_strdup_printf("/soc/flash@%" PRIx64, flashbase); > + qemu_fdt_add_subnode(s->fdt, name); > + qemu_fdt_setprop_string(s->fdt, name, "compatible", "cfi-flash"); > + qemu_fdt_setprop_sized_cells(s->fdt, name, "reg", > 2, flashbase, 2, flashsize, > 2, flashbase + flashsize, 2, flashsize); > - qemu_fdt_setprop_cell(s->fdt, nodename, "bank-width", 4); > - g_free(nodename); > + qemu_fdt_setprop_cell(s->fdt, name, "bank-width", 4); > + g_free(name); > } > > - > static inline DeviceState *gpex_pcie_init(MemoryRegion *sys_mem, > hwaddr ecam_base, hwaddr ecam_size, > hwaddr mmio_base, hwaddr mmio_size, > @@ -478,21 +493,100 @@ static void riscv_virt_board_init(MachineState *machine) > MemoryRegion *system_memory = get_system_memory(); > MemoryRegion *main_mem = g_new(MemoryRegion, 1); > MemoryRegion *mask_rom = g_new(MemoryRegion, 1); > - char *plic_hart_config; > + char *plic_hart_config, *soc_name; > size_t plic_hart_config_len; > target_ulong start_addr = memmap[VIRT_DRAM].base; > - int i; > - unsigned int smp_cpus = machine->smp.cpus; > - > - /* Initialize SOC */ > - object_initialize_child(OBJECT(machine), "soc", &s->soc, sizeof(s->soc), > - TYPE_RISCV_HART_ARRAY, &error_abort, NULL); > - object_property_set_str(OBJECT(&s->soc), machine->cpu_type, "cpu-type", > - &error_abort); > - object_property_set_int(OBJECT(&s->soc), smp_cpus, "num-harts", > - &error_abort); > - object_property_set_bool(OBJECT(&s->soc), true, "realized", > - &error_abort); > + DeviceState *mmio_plic, *virtio_plic, *pcie_plic; > + int i, j, base_hartid, hart_count; > + > + /* Check socket count limit */ > + if (VIRT_SOCKETS_MAX < riscv_socket_count(machine)) { > + error_report("number of sockets/nodes should be less than %d", > + VIRT_SOCKETS_MAX); > + exit(1); > + } > + > + /* Initialize sockets */ > + mmio_plic = virtio_plic = pcie_plic = NULL; > + for (i = 0; i < riscv_socket_count(machine); i++) { > + if (!riscv_socket_check_hartids(machine, i)) { > + error_report("discontinuous hartids in socket%d", i); > + exit(1); > + } > + > + base_hartid = riscv_socket_first_hartid(machine, i); > + if (base_hartid < 0) { > + error_report("can't find hartid base for socket%d", i); > + exit(1); > + } > + > + hart_count = riscv_socket_hart_count(machine, i); > + if (hart_count < 0) { > + error_report("can't find hart count for socket%d", i); > + exit(1); > + } > + > + soc_name = g_strdup_printf("soc%d", i); > + object_initialize_child(OBJECT(machine), soc_name, &s->soc[i], > + sizeof(s->soc[i]), TYPE_RISCV_HART_ARRAY, &error_abort, NULL); > + g_free(soc_name); > + object_property_set_str(OBJECT(&s->soc[i]), > + machine->cpu_type, "cpu-type", &error_abort); > + object_property_set_int(OBJECT(&s->soc[i]), > + base_hartid, "hartid-base", &error_abort); > + object_property_set_int(OBJECT(&s->soc[i]), > + hart_count, "num-harts", &error_abort); > + object_property_set_bool(OBJECT(&s->soc[i]), > + true, "realized", &error_abort); > + > + /* Per-socket CLINT */ > + sifive_clint_create( > + memmap[VIRT_CLINT].base + i * memmap[VIRT_CLINT].size, > + memmap[VIRT_CLINT].size, base_hartid, hart_count, > + SIFIVE_SIP_BASE, SIFIVE_TIMECMP_BASE, SIFIVE_TIME_BASE, true); > + > + /* Per-socket PLIC hart topology configuration string */ > + plic_hart_config_len = > + (strlen(VIRT_PLIC_HART_CONFIG) + 1) * hart_count; > + plic_hart_config = g_malloc0(plic_hart_config_len); > + for (j = 0; j < hart_count; j++) { > + if (j != 0) { > + strncat(plic_hart_config, ",", plic_hart_config_len); > + } > + strncat(plic_hart_config, VIRT_PLIC_HART_CONFIG, > + plic_hart_config_len); > + plic_hart_config_len -= (strlen(VIRT_PLIC_HART_CONFIG) + 1); > + } > + > + /* Per-socket PLIC */ > + s->plic[i] = sifive_plic_create( > + memmap[VIRT_PLIC].base + i * memmap[VIRT_PLIC].size, > + plic_hart_config, base_hartid, > + VIRT_PLIC_NUM_SOURCES, > + VIRT_PLIC_NUM_PRIORITIES, > + VIRT_PLIC_PRIORITY_BASE, > + VIRT_PLIC_PENDING_BASE, > + VIRT_PLIC_ENABLE_BASE, > + VIRT_PLIC_ENABLE_STRIDE, > + VIRT_PLIC_CONTEXT_BASE, > + VIRT_PLIC_CONTEXT_STRIDE, > + memmap[VIRT_PLIC].size); > + g_free(plic_hart_config); > + > + /* Try to use different PLIC instance based device type */ Why do we have different types of PLICs? > + if (i == 0) { > + mmio_plic = s->plic[i]; > + virtio_plic = s->plic[i]; > + pcie_plic = s->plic[i]; > + } > + if (i == 1) { > + virtio_plic = s->plic[i]; > + pcie_plic = s->plic[i]; > + } > + if (i == 2) { > + pcie_plic = s->plic[i]; > + } > + } > > /* register system main memory (actual RAM) */ > memory_region_init_ram(main_mem, NULL, "riscv_virt_board.ram", > @@ -571,38 +665,14 @@ static void riscv_virt_board_init(MachineState *machine) > memmap[VIRT_MROM].base + sizeof(reset_vec), > &address_space_memory); > > - /* create PLIC hart topology configuration string */ > - plic_hart_config_len = (strlen(VIRT_PLIC_HART_CONFIG) + 1) * smp_cpus; > - plic_hart_config = g_malloc0(plic_hart_config_len); > - for (i = 0; i < smp_cpus; i++) { > - if (i != 0) { > - strncat(plic_hart_config, ",", plic_hart_config_len); > - } > - strncat(plic_hart_config, VIRT_PLIC_HART_CONFIG, plic_hart_config_len); > - plic_hart_config_len -= (strlen(VIRT_PLIC_HART_CONFIG) + 1); > - } > - > - /* MMIO */ > - s->plic = sifive_plic_create(memmap[VIRT_PLIC].base, > - plic_hart_config, 0, > - VIRT_PLIC_NUM_SOURCES, > - VIRT_PLIC_NUM_PRIORITIES, > - VIRT_PLIC_PRIORITY_BASE, > - VIRT_PLIC_PENDING_BASE, > - VIRT_PLIC_ENABLE_BASE, > - VIRT_PLIC_ENABLE_STRIDE, > - VIRT_PLIC_CONTEXT_BASE, > - VIRT_PLIC_CONTEXT_STRIDE, > - memmap[VIRT_PLIC].size); > - sifive_clint_create(memmap[VIRT_CLINT].base, > - memmap[VIRT_CLINT].size, 0, smp_cpus, > - SIFIVE_SIP_BASE, SIFIVE_TIMECMP_BASE, SIFIVE_TIME_BASE, true); > + /* SiFive Test MMIO device */ > sifive_test_create(memmap[VIRT_TEST].base); > > + /* VirtIO MMIO devices */ > for (i = 0; i < VIRTIO_COUNT; i++) { > sysbus_create_simple("virtio-mmio", > memmap[VIRT_VIRTIO].base + i * memmap[VIRT_VIRTIO].size, > - qdev_get_gpio_in(DEVICE(s->plic), VIRTIO_IRQ + i)); > + qdev_get_gpio_in(DEVICE(virtio_plic), VIRTIO_IRQ + i)); > } > > gpex_pcie_init(system_memory, > @@ -611,14 +681,14 @@ static void riscv_virt_board_init(MachineState *machine) > memmap[VIRT_PCIE_MMIO].base, > memmap[VIRT_PCIE_MMIO].size, > memmap[VIRT_PCIE_PIO].base, > - DEVICE(s->plic), true); > + DEVICE(pcie_plic), true); > > serial_mm_init(system_memory, memmap[VIRT_UART0].base, > - 0, qdev_get_gpio_in(DEVICE(s->plic), UART0_IRQ), 399193, > + 0, qdev_get_gpio_in(DEVICE(mmio_plic), UART0_IRQ), 399193, > serial_hd(0), DEVICE_LITTLE_ENDIAN); > > sysbus_create_simple("goldfish_rtc", memmap[VIRT_RTC].base, > - qdev_get_gpio_in(DEVICE(s->plic), RTC_IRQ)); > + qdev_get_gpio_in(DEVICE(mmio_plic), RTC_IRQ)); > > virt_flash_create(s); > > @@ -628,8 +698,6 @@ static void riscv_virt_board_init(MachineState *machine) > drive_get(IF_PFLASH, 0, i)); > } > virt_flash_map(s, system_memory); > - > - g_free(plic_hart_config); > } > > static void riscv_virt_machine_instance_init(Object *obj) > @@ -642,9 +710,13 @@ static void riscv_virt_machine_class_init(ObjectClass *oc, void *data) > > mc->desc = "RISC-V VirtIO board"; > mc->init = riscv_virt_board_init; > - mc->max_cpus = 8; > + mc->max_cpus = VIRT_CPUS_MAX; > mc->default_cpu_type = VIRT_CPU; > mc->pci_allow_0_address = true; > + mc->possible_cpu_arch_ids = riscv_numa_possible_cpu_arch_ids; > + mc->cpu_index_to_instance_props = riscv_numa_cpu_index_to_props; > + mc->get_default_cpu_node_id = riscv_numa_get_default_cpu_node_id; > + mc->numa_mem_supported = true; > } > > static const TypeInfo riscv_virt_machine_typeinfo = { > diff --git a/include/hw/riscv/virt.h b/include/hw/riscv/virt.h > index e69355efaf..1beacd7666 100644 > --- a/include/hw/riscv/virt.h > +++ b/include/hw/riscv/virt.h > @@ -23,6 +23,9 @@ > #include "hw/sysbus.h" > #include "hw/block/flash.h" > > +#define VIRT_CPUS_MAX 8 > +#define VIRT_SOCKETS_MAX 8 > + > #define TYPE_RISCV_VIRT_MACHINE MACHINE_TYPE_NAME("virt") > #define RISCV_VIRT_MACHINE(obj) \ > OBJECT_CHECK(RISCVVirtState, (obj), TYPE_RISCV_VIRT_MACHINE) > @@ -32,8 +35,8 @@ typedef struct { > MachineState parent; > > /*< public >*/ > - RISCVHartArrayState soc; > - DeviceState *plic; > + RISCVHartArrayState soc[VIRT_SOCKETS_MAX]; > + DeviceState *plic[VIRT_SOCKETS_MAX]; > PFlashCFI01 *flash[2]; > > void *fdt; > @@ -74,6 +77,8 @@ enum { > #define VIRT_PLIC_ENABLE_STRIDE 0x80 > #define VIRT_PLIC_CONTEXT_BASE 0x200000 > #define VIRT_PLIC_CONTEXT_STRIDE 0x1000 > +#define VIRT_PLIC_SIZE(__num_context) \ > + (VIRT_PLIC_CONTEXT_BASE + (__num_context) * VIRT_PLIC_CONTEXT_STRIDE) > > #define FDT_PCI_ADDR_CELLS 3 > #define FDT_PCI_INT_CELLS 1 > -- > 2.25.1 > >
> -----Original Message----- > From: Alistair Francis <alistair23@gmail.com> > Sent: 11 June 2020 04:55 > To: Anup Patel <Anup.Patel@wdc.com> > Cc: Peter Maydell <peter.maydell@linaro.org>; Palmer Dabbelt > <palmer@dabbelt.com>; Alistair Francis <Alistair.Francis@wdc.com>; Sagar > Karandikar <sagark@eecs.berkeley.edu>; Atish Patra > <Atish.Patra@wdc.com>; open list:RISC-V <qemu-riscv@nongnu.org>; > qemu-devel@nongnu.org Developers <qemu-devel@nongnu.org>; Anup > Patel <anup@brainfault.org> > Subject: Re: [PATCH v5 5/5] hw/riscv: virt: Allow creating multiple NUMA > sockets > > On Fri, May 29, 2020 at 4:49 AM Anup Patel <anup.patel@wdc.com> wrote: > > > > We extend RISC-V virt machine to allow creating a multi-socket > > machine. Each RISC-V virt machine socket is a NUMA node having a set > > of HARTs, a memory instance, a CLINT instance, and a PLIC instance. > > Other devices are shared between all sockets. We also update the > > generated device tree accordingly. > > > > By default, NUMA multi-socket support is disabled for RISC-V virt > > machine. To enable it, users can use "-numa" command-line options of > > QEMU. > > > > Example1: For two NUMA nodes with 2 CPUs each, append following to > > command-line options: "-smp 4 -numa node -numa node" > > > > Example2: For two NUMA nodes with 1 and 3 CPUs, append following to > > command-line options: > > "-smp 4 -numa node -numa node -numa cpu,node-id=0,core-id=0 \ -numa > > cpu,node-id=1,core-id=1 -numa cpu,node-id=1,core-id=2 \ -numa > > cpu,node-id=1,core-id=3" > > > > The maximum number of sockets in a RISC-V virt machine is 8 but this > > limit can be changed in future. > > > > Signed-off-by: Anup Patel <anup.patel@wdc.com> > > --- > > hw/riscv/virt.c | 530 +++++++++++++++++++++++----------------- > > include/hw/riscv/virt.h | 9 +- > > 2 files changed, 308 insertions(+), 231 deletions(-) > > > > diff --git a/hw/riscv/virt.c b/hw/riscv/virt.c index > > 421815081d..2863b42cea 100644 > > --- a/hw/riscv/virt.c > > +++ b/hw/riscv/virt.c > > @@ -35,6 +35,7 @@ > > #include "hw/riscv/sifive_test.h" > > #include "hw/riscv/virt.h" > > #include "hw/riscv/boot.h" > > +#include "hw/riscv/numa.h" > > #include "chardev/char.h" > > #include "sysemu/arch_init.h" > > #include "sysemu/device_tree.h" > > @@ -60,7 +61,7 @@ static const struct MemmapEntry { > > [VIRT_TEST] = { 0x100000, 0x1000 }, > > [VIRT_RTC] = { 0x101000, 0x1000 }, > > [VIRT_CLINT] = { 0x2000000, 0x10000 }, > > - [VIRT_PLIC] = { 0xc000000, 0x4000000 }, > > + [VIRT_PLIC] = { 0xc000000, VIRT_PLIC_SIZE(VIRT_CPUS_MAX * 2) }, > > [VIRT_UART0] = { 0x10000000, 0x100 }, > > [VIRT_VIRTIO] = { 0x10001000, 0x1000 }, > > [VIRT_FLASH] = { 0x20000000, 0x4000000 }, > > @@ -182,10 +183,17 @@ static void create_fdt(RISCVVirtState *s, const > struct MemmapEntry *memmap, > > uint64_t mem_size, const char *cmdline) { > > void *fdt; > > - int cpu, i; > > - uint32_t *cells; > > - char *nodename; > > - uint32_t plic_phandle, test_phandle, phandle = 1; > > + int i, cpu, socket; > > + MachineState *mc = MACHINE(s); > > + uint64_t addr, size; > > + uint32_t *clint_cells, *plic_cells; > > + unsigned long clint_addr, plic_addr; > > + uint32_t plic_phandle[MAX_NODES]; > > + uint32_t cpu_phandle, intc_phandle, test_phandle; > > + uint32_t phandle = 1, plic_mmio_phandle = 1; > > + uint32_t plic_pcie_phandle = 1, plic_virtio_phandle = 1; > > + char *mem_name, *cpu_name, *core_name, *intc_name; > > + char *name, *clint_name, *plic_name, *clust_name; > > hwaddr flashsize = virt_memmap[VIRT_FLASH].size / 2; > > hwaddr flashbase = virt_memmap[VIRT_FLASH].base; > > > > @@ -206,231 +214,238 @@ static void create_fdt(RISCVVirtState *s, const > struct MemmapEntry *memmap, > > qemu_fdt_setprop_cell(fdt, "/soc", "#size-cells", 0x2); > > qemu_fdt_setprop_cell(fdt, "/soc", "#address-cells", 0x2); > > > > - nodename = g_strdup_printf("/memory@%lx", > > - (long)memmap[VIRT_DRAM].base); > > - qemu_fdt_add_subnode(fdt, nodename); > > - qemu_fdt_setprop_cells(fdt, nodename, "reg", > > - memmap[VIRT_DRAM].base >> 32, memmap[VIRT_DRAM].base, > > - mem_size >> 32, mem_size); > > - qemu_fdt_setprop_string(fdt, nodename, "device_type", "memory"); > > - g_free(nodename); > > - > > qemu_fdt_add_subnode(fdt, "/cpus"); > > qemu_fdt_setprop_cell(fdt, "/cpus", "timebase-frequency", > > SIFIVE_CLINT_TIMEBASE_FREQ); > > qemu_fdt_setprop_cell(fdt, "/cpus", "#size-cells", 0x0); > > qemu_fdt_setprop_cell(fdt, "/cpus", "#address-cells", 0x1); > > + qemu_fdt_add_subnode(fdt, "/cpus/cpu-map"); > > I'm no expert with cpu-map. Do you mind CCing Atish in the next version and > see if he can Ack these DT changes? Sure, he is already there is CC. By default, you and Atish are always in CC for all my patches. I apologize if I missed in any of the patches. > > > + > > + for (socket = (riscv_socket_count(mc) - 1); socket >= 0; socket--) { > > + clust_name = g_strdup_printf("/cpus/cpu-map/cluster%d", socket); > > + qemu_fdt_add_subnode(fdt, clust_name); > > + > > + plic_cells = g_new0(uint32_t, s->soc[socket].num_harts * 4); > > + clint_cells = g_new0(uint32_t, s->soc[socket].num_harts * 4); > > + > > + for (cpu = s->soc[socket].num_harts - 1; cpu >= 0; cpu--) { > > + cpu_phandle = phandle++; > > > > - for (cpu = s->soc.num_harts - 1; cpu >= 0; cpu--) { > > - int cpu_phandle = phandle++; > > - int intc_phandle; > > - nodename = g_strdup_printf("/cpus/cpu@%d", cpu); > > - char *intc = g_strdup_printf("/cpus/cpu@%d/interrupt-controller", > cpu); > > - char *isa = riscv_isa_string(&s->soc.harts[cpu]); > > - qemu_fdt_add_subnode(fdt, nodename); > > + cpu_name = g_strdup_printf("/cpus/cpu@%d", > > + s->soc[socket].hartid_base + cpu); > > + qemu_fdt_add_subnode(fdt, cpu_name); > > #if defined(TARGET_RISCV32) > > - qemu_fdt_setprop_string(fdt, nodename, "mmu-type", "riscv,sv32"); > > + qemu_fdt_setprop_string(fdt, cpu_name, "mmu-type", > > + "riscv,sv32"); > > #else > > - qemu_fdt_setprop_string(fdt, nodename, "mmu-type", "riscv,sv48"); > > + qemu_fdt_setprop_string(fdt, cpu_name, "mmu-type", > > + "riscv,sv48"); > > #endif > > - qemu_fdt_setprop_string(fdt, nodename, "riscv,isa", isa); > > - qemu_fdt_setprop_string(fdt, nodename, "compatible", "riscv"); > > - qemu_fdt_setprop_string(fdt, nodename, "status", "okay"); > > - qemu_fdt_setprop_cell(fdt, nodename, "reg", cpu); > > - qemu_fdt_setprop_string(fdt, nodename, "device_type", "cpu"); > > - qemu_fdt_setprop_cell(fdt, nodename, "phandle", cpu_phandle); > > - intc_phandle = phandle++; > > - qemu_fdt_add_subnode(fdt, intc); > > - qemu_fdt_setprop_cell(fdt, intc, "phandle", intc_phandle); > > - qemu_fdt_setprop_string(fdt, intc, "compatible", "riscv,cpu-intc"); > > - qemu_fdt_setprop(fdt, intc, "interrupt-controller", NULL, 0); > > - qemu_fdt_setprop_cell(fdt, intc, "#interrupt-cells", 1); > > - g_free(isa); > > - g_free(intc); > > - g_free(nodename); > > - } > > + name = riscv_isa_string(&s->soc[socket].harts[cpu]); > > + qemu_fdt_setprop_string(fdt, cpu_name, "riscv,isa", name); > > + g_free(name); > > + qemu_fdt_setprop_string(fdt, cpu_name, "compatible", "riscv"); > > + qemu_fdt_setprop_string(fdt, cpu_name, "status", "okay"); > > + qemu_fdt_setprop_cell(fdt, cpu_name, "reg", > > + s->soc[socket].hartid_base + cpu); > > + qemu_fdt_setprop_string(fdt, cpu_name, "device_type", "cpu"); > > + riscv_socket_fdt_write_id(mc, fdt, cpu_name, socket); > > + qemu_fdt_setprop_cell(fdt, cpu_name, "phandle", > > + cpu_phandle); > > + > > + intc_name = g_strdup_printf("%s/interrupt-controller", > cpu_name); > > + qemu_fdt_add_subnode(fdt, intc_name); > > + intc_phandle = phandle++; > > + qemu_fdt_setprop_cell(fdt, intc_name, "phandle", intc_phandle); > > + qemu_fdt_setprop_string(fdt, intc_name, "compatible", > > + "riscv,cpu-intc"); > > + qemu_fdt_setprop(fdt, intc_name, "interrupt-controller", NULL, 0); > > + qemu_fdt_setprop_cell(fdt, intc_name, "#interrupt-cells", > > + 1); > > + > > + clint_cells[cpu * 4 + 0] = cpu_to_be32(intc_phandle); > > + clint_cells[cpu * 4 + 1] = cpu_to_be32(IRQ_M_SOFT); > > + clint_cells[cpu * 4 + 2] = cpu_to_be32(intc_phandle); > > + clint_cells[cpu * 4 + 3] = cpu_to_be32(IRQ_M_TIMER); > > + > > + plic_cells[cpu * 4 + 0] = cpu_to_be32(intc_phandle); > > + plic_cells[cpu * 4 + 1] = cpu_to_be32(IRQ_M_EXT); > > + plic_cells[cpu * 4 + 2] = cpu_to_be32(intc_phandle); > > + plic_cells[cpu * 4 + 3] = cpu_to_be32(IRQ_S_EXT); > > + > > + core_name = g_strdup_printf("%s/core%d", clust_name, cpu); > > + qemu_fdt_add_subnode(fdt, core_name); > > + qemu_fdt_setprop_cell(fdt, core_name, "cpu", > > + cpu_phandle); > > + > > + g_free(core_name); > > + g_free(intc_name); > > + g_free(cpu_name); > > + } > > > > - /* Add cpu-topology node */ > > - qemu_fdt_add_subnode(fdt, "/cpus/cpu-map"); > > - qemu_fdt_add_subnode(fdt, "/cpus/cpu-map/cluster0"); > > - for (cpu = s->soc.num_harts - 1; cpu >= 0; cpu--) { > > - char *core_nodename = g_strdup_printf("/cpus/cpu- > map/cluster0/core%d", > > - cpu); > > - char *cpu_nodename = g_strdup_printf("/cpus/cpu@%d", cpu); > > - uint32_t intc_phandle = qemu_fdt_get_phandle(fdt, > cpu_nodename); > > - qemu_fdt_add_subnode(fdt, core_nodename); > > - qemu_fdt_setprop_cell(fdt, core_nodename, "cpu", intc_phandle); > > - g_free(core_nodename); > > - g_free(cpu_nodename); > > + addr = memmap[VIRT_DRAM].base + riscv_socket_mem_offset(mc, > socket); > > + size = riscv_socket_mem_size(mc, socket); > > + mem_name = g_strdup_printf("/memory@%lx", (long)addr); > > + qemu_fdt_add_subnode(fdt, mem_name); > > + qemu_fdt_setprop_cells(fdt, mem_name, "reg", > > + addr >> 32, addr, size >> 32, size); > > + qemu_fdt_setprop_string(fdt, mem_name, "device_type", > "memory"); > > + riscv_socket_fdt_write_id(mc, fdt, mem_name, socket); > > + g_free(mem_name); > > + > > + clint_addr = memmap[VIRT_CLINT].base + > > + (memmap[VIRT_CLINT].size * socket); > > + clint_name = g_strdup_printf("/soc/clint@%lx", clint_addr); > > + qemu_fdt_add_subnode(fdt, clint_name); > > + qemu_fdt_setprop_string(fdt, clint_name, "compatible", > "riscv,clint0"); > > + qemu_fdt_setprop_cells(fdt, clint_name, "reg", > > + 0x0, clint_addr, 0x0, memmap[VIRT_CLINT].size); > > + qemu_fdt_setprop(fdt, clint_name, "interrupts-extended", > > + clint_cells, s->soc[socket].num_harts * sizeof(uint32_t) * 4); > > + riscv_socket_fdt_write_id(mc, fdt, clint_name, socket); > > + g_free(clint_name); > > + > > + plic_phandle[socket] = phandle++; > > + plic_addr = memmap[VIRT_PLIC].base + (memmap[VIRT_PLIC].size * > socket); > > + plic_name = g_strdup_printf("/soc/plic@%lx", plic_addr); > > + qemu_fdt_add_subnode(fdt, plic_name); > > + qemu_fdt_setprop_cell(fdt, plic_name, > > + "#address-cells", FDT_PLIC_ADDR_CELLS); > > + qemu_fdt_setprop_cell(fdt, plic_name, > > + "#interrupt-cells", FDT_PLIC_INT_CELLS); > > + qemu_fdt_setprop_string(fdt, plic_name, "compatible", "riscv,plic0"); > > + qemu_fdt_setprop(fdt, plic_name, "interrupt-controller", NULL, 0); > > + qemu_fdt_setprop(fdt, plic_name, "interrupts-extended", > > + plic_cells, s->soc[socket].num_harts * sizeof(uint32_t) * 4); > > + qemu_fdt_setprop_cells(fdt, plic_name, "reg", > > + 0x0, plic_addr, 0x0, memmap[VIRT_PLIC].size); > > + qemu_fdt_setprop_cell(fdt, plic_name, "riscv,ndev", VIRTIO_NDEV); > > + riscv_socket_fdt_write_id(mc, fdt, plic_name, socket); > > + qemu_fdt_setprop_cell(fdt, plic_name, "phandle", > plic_phandle[socket]); > > + g_free(plic_name); > > + > > + g_free(clint_cells); > > + g_free(plic_cells); > > + g_free(clust_name); > > } > > > > - cells = g_new0(uint32_t, s->soc.num_harts * 4); > > - for (cpu = 0; cpu < s->soc.num_harts; cpu++) { > > - nodename = > > - g_strdup_printf("/cpus/cpu@%d/interrupt-controller", cpu); > > - uint32_t intc_phandle = qemu_fdt_get_phandle(fdt, nodename); > > - cells[cpu * 4 + 0] = cpu_to_be32(intc_phandle); > > - cells[cpu * 4 + 1] = cpu_to_be32(IRQ_M_SOFT); > > - cells[cpu * 4 + 2] = cpu_to_be32(intc_phandle); > > - cells[cpu * 4 + 3] = cpu_to_be32(IRQ_M_TIMER); > > - g_free(nodename); > > - } > > - nodename = g_strdup_printf("/soc/clint@%lx", > > - (long)memmap[VIRT_CLINT].base); > > - qemu_fdt_add_subnode(fdt, nodename); > > - qemu_fdt_setprop_string(fdt, nodename, "compatible", "riscv,clint0"); > > - qemu_fdt_setprop_cells(fdt, nodename, "reg", > > - 0x0, memmap[VIRT_CLINT].base, > > - 0x0, memmap[VIRT_CLINT].size); > > - qemu_fdt_setprop(fdt, nodename, "interrupts-extended", > > - cells, s->soc.num_harts * sizeof(uint32_t) * 4); > > - g_free(cells); > > - g_free(nodename); > > - > > - plic_phandle = phandle++; > > - cells = g_new0(uint32_t, s->soc.num_harts * 4); > > - for (cpu = 0; cpu < s->soc.num_harts; cpu++) { > > - nodename = > > - g_strdup_printf("/cpus/cpu@%d/interrupt-controller", cpu); > > - uint32_t intc_phandle = qemu_fdt_get_phandle(fdt, nodename); > > - cells[cpu * 4 + 0] = cpu_to_be32(intc_phandle); > > - cells[cpu * 4 + 1] = cpu_to_be32(IRQ_M_EXT); > > - cells[cpu * 4 + 2] = cpu_to_be32(intc_phandle); > > - cells[cpu * 4 + 3] = cpu_to_be32(IRQ_S_EXT); > > - g_free(nodename); > > + for (socket = 0; socket < riscv_socket_count(mc); socket++) { > > + if (socket == 0) { > > + plic_mmio_phandle = plic_phandle[socket]; > > + plic_virtio_phandle = plic_phandle[socket]; > > + plic_pcie_phandle = plic_phandle[socket]; > > + } > > + if (socket == 1) { > > + plic_virtio_phandle = plic_phandle[socket]; > > + plic_pcie_phandle = plic_phandle[socket]; > > + } > > + if (socket == 2) { > > + plic_pcie_phandle = plic_phandle[socket]; > > + } > > } > > - nodename = g_strdup_printf("/soc/interrupt-controller@%lx", > > - (long)memmap[VIRT_PLIC].base); > > - qemu_fdt_add_subnode(fdt, nodename); > > - qemu_fdt_setprop_cell(fdt, nodename, "#address-cells", > > - FDT_PLIC_ADDR_CELLS); > > - qemu_fdt_setprop_cell(fdt, nodename, "#interrupt-cells", > > - FDT_PLIC_INT_CELLS); > > - qemu_fdt_setprop_string(fdt, nodename, "compatible", "riscv,plic0"); > > - qemu_fdt_setprop(fdt, nodename, "interrupt-controller", NULL, 0); > > - qemu_fdt_setprop(fdt, nodename, "interrupts-extended", > > - cells, s->soc.num_harts * sizeof(uint32_t) * 4); > > - qemu_fdt_setprop_cells(fdt, nodename, "reg", > > - 0x0, memmap[VIRT_PLIC].base, > > - 0x0, memmap[VIRT_PLIC].size); > > - qemu_fdt_setprop_cell(fdt, nodename, "riscv,ndev", VIRTIO_NDEV); > > - qemu_fdt_setprop_cell(fdt, nodename, "phandle", plic_phandle); > > - plic_phandle = qemu_fdt_get_phandle(fdt, nodename); > > - g_free(cells); > > - g_free(nodename); > > + > > + riscv_socket_fdt_write_distance_matrix(mc, fdt); > > > > for (i = 0; i < VIRTIO_COUNT; i++) { > > - nodename = g_strdup_printf("/virtio_mmio@%lx", > > + name = g_strdup_printf("/soc/virtio_mmio@%lx", > > (long)(memmap[VIRT_VIRTIO].base + i * > memmap[VIRT_VIRTIO].size)); > > - qemu_fdt_add_subnode(fdt, nodename); > > - qemu_fdt_setprop_string(fdt, nodename, "compatible", > "virtio,mmio"); > > - qemu_fdt_setprop_cells(fdt, nodename, "reg", > > + qemu_fdt_add_subnode(fdt, name); > > + qemu_fdt_setprop_string(fdt, name, "compatible", "virtio,mmio"); > > + qemu_fdt_setprop_cells(fdt, name, "reg", > > 0x0, memmap[VIRT_VIRTIO].base + i * memmap[VIRT_VIRTIO].size, > > 0x0, memmap[VIRT_VIRTIO].size); > > - qemu_fdt_setprop_cell(fdt, nodename, "interrupt-parent", > plic_phandle); > > - qemu_fdt_setprop_cell(fdt, nodename, "interrupts", VIRTIO_IRQ + i); > > - g_free(nodename); > > + qemu_fdt_setprop_cell(fdt, name, "interrupt-parent", > > + plic_virtio_phandle); > > + qemu_fdt_setprop_cell(fdt, name, "interrupts", VIRTIO_IRQ + i); > > + g_free(name); > > } > > > > - nodename = g_strdup_printf("/soc/pci@%lx", > > + name = g_strdup_printf("/soc/pci@%lx", > > (long) memmap[VIRT_PCIE_ECAM].base); > > - qemu_fdt_add_subnode(fdt, nodename); > > - qemu_fdt_setprop_cell(fdt, nodename, "#address-cells", > > - FDT_PCI_ADDR_CELLS); > > - qemu_fdt_setprop_cell(fdt, nodename, "#interrupt-cells", > > - FDT_PCI_INT_CELLS); > > - qemu_fdt_setprop_cell(fdt, nodename, "#size-cells", 0x2); > > - qemu_fdt_setprop_string(fdt, nodename, "compatible", > > - "pci-host-ecam-generic"); > > - qemu_fdt_setprop_string(fdt, nodename, "device_type", "pci"); > > - qemu_fdt_setprop_cell(fdt, nodename, "linux,pci-domain", 0); > > - qemu_fdt_setprop_cells(fdt, nodename, "bus-range", 0, > > - memmap[VIRT_PCIE_ECAM].size / > > - PCIE_MMCFG_SIZE_MIN - 1); > > - qemu_fdt_setprop(fdt, nodename, "dma-coherent", NULL, 0); > > - qemu_fdt_setprop_cells(fdt, nodename, "reg", 0, > memmap[VIRT_PCIE_ECAM].base, > > - 0, memmap[VIRT_PCIE_ECAM].size); > > - qemu_fdt_setprop_sized_cells(fdt, nodename, "ranges", > > + qemu_fdt_add_subnode(fdt, name); > > + qemu_fdt_setprop_cell(fdt, name, "#address-cells", > FDT_PCI_ADDR_CELLS); > > + qemu_fdt_setprop_cell(fdt, name, "#interrupt-cells", > FDT_PCI_INT_CELLS); > > + qemu_fdt_setprop_cell(fdt, name, "#size-cells", 0x2); > > + qemu_fdt_setprop_string(fdt, name, "compatible", "pci-host-ecam- > generic"); > > + qemu_fdt_setprop_string(fdt, name, "device_type", "pci"); > > + qemu_fdt_setprop_cell(fdt, name, "linux,pci-domain", 0); > > + qemu_fdt_setprop_cells(fdt, name, "bus-range", 0, > > + memmap[VIRT_PCIE_ECAM].size / PCIE_MMCFG_SIZE_MIN - 1); > > + qemu_fdt_setprop(fdt, name, "dma-coherent", NULL, 0); > > + qemu_fdt_setprop_cells(fdt, name, "reg", 0, > > + memmap[VIRT_PCIE_ECAM].base, 0, > memmap[VIRT_PCIE_ECAM].size); > > + qemu_fdt_setprop_sized_cells(fdt, name, "ranges", > > 1, FDT_PCI_RANGE_IOPORT, 2, 0, > > 2, memmap[VIRT_PCIE_PIO].base, 2, memmap[VIRT_PCIE_PIO].size, > > 1, FDT_PCI_RANGE_MMIO, > > 2, memmap[VIRT_PCIE_MMIO].base, > > 2, memmap[VIRT_PCIE_MMIO].base, 2, > memmap[VIRT_PCIE_MMIO].size); > > - create_pcie_irq_map(fdt, nodename, plic_phandle); > > - g_free(nodename); > > + create_pcie_irq_map(fdt, name, plic_pcie_phandle); > > + g_free(name); > > > > test_phandle = phandle++; > > - nodename = g_strdup_printf("/test@%lx", > > + name = g_strdup_printf("/soc/test@%lx", > > (long)memmap[VIRT_TEST].base); > > - qemu_fdt_add_subnode(fdt, nodename); > > + qemu_fdt_add_subnode(fdt, name); > > { > > const char compat[] = "sifive,test1\0sifive,test0\0syscon"; > > - qemu_fdt_setprop(fdt, nodename, "compatible", compat, > sizeof(compat)); > > + qemu_fdt_setprop(fdt, name, "compatible", compat, > > + sizeof(compat)); > > } > > - qemu_fdt_setprop_cells(fdt, nodename, "reg", > > + qemu_fdt_setprop_cells(fdt, name, "reg", > > 0x0, memmap[VIRT_TEST].base, > > 0x0, memmap[VIRT_TEST].size); > > - qemu_fdt_setprop_cell(fdt, nodename, "phandle", test_phandle); > > - test_phandle = qemu_fdt_get_phandle(fdt, nodename); > > - g_free(nodename); > > - > > - nodename = g_strdup_printf("/reboot"); > > - qemu_fdt_add_subnode(fdt, nodename); > > - qemu_fdt_setprop_string(fdt, nodename, "compatible", "syscon- > reboot"); > > - qemu_fdt_setprop_cell(fdt, nodename, "regmap", test_phandle); > > - qemu_fdt_setprop_cell(fdt, nodename, "offset", 0x0); > > - qemu_fdt_setprop_cell(fdt, nodename, "value", FINISHER_RESET); > > - g_free(nodename); > > - > > - nodename = g_strdup_printf("/poweroff"); > > - qemu_fdt_add_subnode(fdt, nodename); > > - qemu_fdt_setprop_string(fdt, nodename, "compatible", "syscon- > poweroff"); > > - qemu_fdt_setprop_cell(fdt, nodename, "regmap", test_phandle); > > - qemu_fdt_setprop_cell(fdt, nodename, "offset", 0x0); > > - qemu_fdt_setprop_cell(fdt, nodename, "value", FINISHER_PASS); > > - g_free(nodename); > > - > > - nodename = g_strdup_printf("/uart@%lx", > > - (long)memmap[VIRT_UART0].base); > > - qemu_fdt_add_subnode(fdt, nodename); > > - qemu_fdt_setprop_string(fdt, nodename, "compatible", "ns16550a"); > > - qemu_fdt_setprop_cells(fdt, nodename, "reg", > > + qemu_fdt_setprop_cell(fdt, name, "phandle", test_phandle); > > + test_phandle = qemu_fdt_get_phandle(fdt, name); > > + g_free(name); > > + > > + name = g_strdup_printf("/soc/reboot"); > > + qemu_fdt_add_subnode(fdt, name); > > + qemu_fdt_setprop_string(fdt, name, "compatible", "syscon-reboot"); > > + qemu_fdt_setprop_cell(fdt, name, "regmap", test_phandle); > > + qemu_fdt_setprop_cell(fdt, name, "offset", 0x0); > > + qemu_fdt_setprop_cell(fdt, name, "value", FINISHER_RESET); > > + g_free(name); > > + > > + name = g_strdup_printf("/soc/poweroff"); > > + qemu_fdt_add_subnode(fdt, name); > > + qemu_fdt_setprop_string(fdt, name, "compatible", "syscon- > poweroff"); > > + qemu_fdt_setprop_cell(fdt, name, "regmap", test_phandle); > > + qemu_fdt_setprop_cell(fdt, name, "offset", 0x0); > > + qemu_fdt_setprop_cell(fdt, name, "value", FINISHER_PASS); > > + g_free(name); > > + > > + name = g_strdup_printf("/soc/uart@%lx", > (long)memmap[VIRT_UART0].base); > > + qemu_fdt_add_subnode(fdt, name); > > + qemu_fdt_setprop_string(fdt, name, "compatible", "ns16550a"); > > + qemu_fdt_setprop_cells(fdt, name, "reg", > > 0x0, memmap[VIRT_UART0].base, > > 0x0, memmap[VIRT_UART0].size); > > - qemu_fdt_setprop_cell(fdt, nodename, "clock-frequency", 3686400); > > - qemu_fdt_setprop_cell(fdt, nodename, "interrupt-parent", > plic_phandle); > > - qemu_fdt_setprop_cell(fdt, nodename, "interrupts", UART0_IRQ); > > + qemu_fdt_setprop_cell(fdt, name, "clock-frequency", 3686400); > > + qemu_fdt_setprop_cell(fdt, name, "interrupt-parent", > plic_mmio_phandle); > > + qemu_fdt_setprop_cell(fdt, name, "interrupts", UART0_IRQ); > > > > qemu_fdt_add_subnode(fdt, "/chosen"); > > - qemu_fdt_setprop_string(fdt, "/chosen", "stdout-path", nodename); > > + qemu_fdt_setprop_string(fdt, "/chosen", "stdout-path", name); > > if (cmdline) { > > qemu_fdt_setprop_string(fdt, "/chosen", "bootargs", cmdline); > > } > > - g_free(nodename); > > - > > - nodename = g_strdup_printf("/rtc@%lx", > > - (long)memmap[VIRT_RTC].base); > > - qemu_fdt_add_subnode(fdt, nodename); > > - qemu_fdt_setprop_string(fdt, nodename, "compatible", > > - "google,goldfish-rtc"); > > - qemu_fdt_setprop_cells(fdt, nodename, "reg", > > + g_free(name); > > + > > + name = g_strdup_printf("/soc/rtc@%lx", > (long)memmap[VIRT_RTC].base); > > + qemu_fdt_add_subnode(fdt, name); > > + qemu_fdt_setprop_string(fdt, name, "compatible", "google,goldfish- > rtc"); > > + qemu_fdt_setprop_cells(fdt, name, "reg", > > 0x0, memmap[VIRT_RTC].base, > > 0x0, memmap[VIRT_RTC].size); > > - qemu_fdt_setprop_cell(fdt, nodename, "interrupt-parent", > plic_phandle); > > - qemu_fdt_setprop_cell(fdt, nodename, "interrupts", RTC_IRQ); > > - g_free(nodename); > > - > > - nodename = g_strdup_printf("/flash@%" PRIx64, flashbase); > > - qemu_fdt_add_subnode(s->fdt, nodename); > > - qemu_fdt_setprop_string(s->fdt, nodename, "compatible", "cfi-flash"); > > - qemu_fdt_setprop_sized_cells(s->fdt, nodename, "reg", > > + qemu_fdt_setprop_cell(fdt, name, "interrupt-parent", > plic_mmio_phandle); > > + qemu_fdt_setprop_cell(fdt, name, "interrupts", RTC_IRQ); > > + g_free(name); > > + > > + name = g_strdup_printf("/soc/flash@%" PRIx64, flashbase); > > + qemu_fdt_add_subnode(s->fdt, name); > > + qemu_fdt_setprop_string(s->fdt, name, "compatible", "cfi-flash"); > > + qemu_fdt_setprop_sized_cells(s->fdt, name, "reg", > > 2, flashbase, 2, flashsize, > > 2, flashbase + flashsize, 2, flashsize); > > - qemu_fdt_setprop_cell(s->fdt, nodename, "bank-width", 4); > > - g_free(nodename); > > + qemu_fdt_setprop_cell(s->fdt, name, "bank-width", 4); > > + g_free(name); > > } > > > > - > > static inline DeviceState *gpex_pcie_init(MemoryRegion *sys_mem, > > hwaddr ecam_base, hwaddr ecam_size, > > hwaddr mmio_base, hwaddr > > mmio_size, @@ -478,21 +493,100 @@ static void > riscv_virt_board_init(MachineState *machine) > > MemoryRegion *system_memory = get_system_memory(); > > MemoryRegion *main_mem = g_new(MemoryRegion, 1); > > MemoryRegion *mask_rom = g_new(MemoryRegion, 1); > > - char *plic_hart_config; > > + char *plic_hart_config, *soc_name; > > size_t plic_hart_config_len; > > target_ulong start_addr = memmap[VIRT_DRAM].base; > > - int i; > > - unsigned int smp_cpus = machine->smp.cpus; > > - > > - /* Initialize SOC */ > > - object_initialize_child(OBJECT(machine), "soc", &s->soc, sizeof(s->soc), > > - TYPE_RISCV_HART_ARRAY, &error_abort, NULL); > > - object_property_set_str(OBJECT(&s->soc), machine->cpu_type, "cpu- > type", > > - &error_abort); > > - object_property_set_int(OBJECT(&s->soc), smp_cpus, "num-harts", > > - &error_abort); > > - object_property_set_bool(OBJECT(&s->soc), true, "realized", > > - &error_abort); > > + DeviceState *mmio_plic, *virtio_plic, *pcie_plic; > > + int i, j, base_hartid, hart_count; > > + > > + /* Check socket count limit */ > > + if (VIRT_SOCKETS_MAX < riscv_socket_count(machine)) { > > + error_report("number of sockets/nodes should be less than %d", > > + VIRT_SOCKETS_MAX); > > + exit(1); > > + } > > + > > + /* Initialize sockets */ > > + mmio_plic = virtio_plic = pcie_plic = NULL; > > + for (i = 0; i < riscv_socket_count(machine); i++) { > > + if (!riscv_socket_check_hartids(machine, i)) { > > + error_report("discontinuous hartids in socket%d", i); > > + exit(1); > > + } > > + > > + base_hartid = riscv_socket_first_hartid(machine, i); > > + if (base_hartid < 0) { > > + error_report("can't find hartid base for socket%d", i); > > + exit(1); > > + } > > + > > + hart_count = riscv_socket_hart_count(machine, i); > > + if (hart_count < 0) { > > + error_report("can't find hart count for socket%d", i); > > + exit(1); > > + } > > + > > + soc_name = g_strdup_printf("soc%d", i); > > + object_initialize_child(OBJECT(machine), soc_name, &s->soc[i], > > + sizeof(s->soc[i]), TYPE_RISCV_HART_ARRAY, &error_abort, NULL); > > + g_free(soc_name); > > + object_property_set_str(OBJECT(&s->soc[i]), > > + machine->cpu_type, "cpu-type", &error_abort); > > + object_property_set_int(OBJECT(&s->soc[i]), > > + base_hartid, "hartid-base", &error_abort); > > + object_property_set_int(OBJECT(&s->soc[i]), > > + hart_count, "num-harts", &error_abort); > > + object_property_set_bool(OBJECT(&s->soc[i]), > > + true, "realized", &error_abort); > > + > > + /* Per-socket CLINT */ > > + sifive_clint_create( > > + memmap[VIRT_CLINT].base + i * memmap[VIRT_CLINT].size, > > + memmap[VIRT_CLINT].size, base_hartid, hart_count, > > + SIFIVE_SIP_BASE, SIFIVE_TIMECMP_BASE, SIFIVE_TIME_BASE, > > + true); > > + > > + /* Per-socket PLIC hart topology configuration string */ > > + plic_hart_config_len = > > + (strlen(VIRT_PLIC_HART_CONFIG) + 1) * hart_count; > > + plic_hart_config = g_malloc0(plic_hart_config_len); > > + for (j = 0; j < hart_count; j++) { > > + if (j != 0) { > > + strncat(plic_hart_config, ",", plic_hart_config_len); > > + } > > + strncat(plic_hart_config, VIRT_PLIC_HART_CONFIG, > > + plic_hart_config_len); > > + plic_hart_config_len -= (strlen(VIRT_PLIC_HART_CONFIG) + 1); > > + } > > + > > + /* Per-socket PLIC */ > > + s->plic[i] = sifive_plic_create( > > + memmap[VIRT_PLIC].base + i * memmap[VIRT_PLIC].size, > > + plic_hart_config, base_hartid, > > + VIRT_PLIC_NUM_SOURCES, > > + VIRT_PLIC_NUM_PRIORITIES, > > + VIRT_PLIC_PRIORITY_BASE, > > + VIRT_PLIC_PENDING_BASE, > > + VIRT_PLIC_ENABLE_BASE, > > + VIRT_PLIC_ENABLE_STRIDE, > > + VIRT_PLIC_CONTEXT_BASE, > > + VIRT_PLIC_CONTEXT_STRIDE, > > + memmap[VIRT_PLIC].size); > > + g_free(plic_hart_config); > > + > > + /* Try to use different PLIC instance based device type */ > > Why do we have different types of PLICs? > > > + if (i == 0) { > > + mmio_plic = s->plic[i]; > > + virtio_plic = s->plic[i]; > > + pcie_plic = s->plic[i]; > > + } > > + if (i == 1) { > > + virtio_plic = s->plic[i]; > > + pcie_plic = s->plic[i]; > > + } > > + if (i == 2) { > > + pcie_plic = s->plic[i]; > > + } > > + } > > > > /* register system main memory (actual RAM) */ > > memory_region_init_ram(main_mem, NULL, "riscv_virt_board.ram", > @@ > > -571,38 +665,14 @@ static void riscv_virt_board_init(MachineState > *machine) > > memmap[VIRT_MROM].base + sizeof(reset_vec), > > &address_space_memory); > > > > - /* create PLIC hart topology configuration string */ > > - plic_hart_config_len = (strlen(VIRT_PLIC_HART_CONFIG) + 1) * > smp_cpus; > > - plic_hart_config = g_malloc0(plic_hart_config_len); > > - for (i = 0; i < smp_cpus; i++) { > > - if (i != 0) { > > - strncat(plic_hart_config, ",", plic_hart_config_len); > > - } > > - strncat(plic_hart_config, VIRT_PLIC_HART_CONFIG, > plic_hart_config_len); > > - plic_hart_config_len -= (strlen(VIRT_PLIC_HART_CONFIG) + 1); > > - } > > - > > - /* MMIO */ > > - s->plic = sifive_plic_create(memmap[VIRT_PLIC].base, > > - plic_hart_config, 0, > > - VIRT_PLIC_NUM_SOURCES, > > - VIRT_PLIC_NUM_PRIORITIES, > > - VIRT_PLIC_PRIORITY_BASE, > > - VIRT_PLIC_PENDING_BASE, > > - VIRT_PLIC_ENABLE_BASE, > > - VIRT_PLIC_ENABLE_STRIDE, > > - VIRT_PLIC_CONTEXT_BASE, > > - VIRT_PLIC_CONTEXT_STRIDE, > > - memmap[VIRT_PLIC].size); > > - sifive_clint_create(memmap[VIRT_CLINT].base, > > - memmap[VIRT_CLINT].size, 0, smp_cpus, > > - SIFIVE_SIP_BASE, SIFIVE_TIMECMP_BASE, SIFIVE_TIME_BASE, true); > > + /* SiFive Test MMIO device */ > > sifive_test_create(memmap[VIRT_TEST].base); > > > > + /* VirtIO MMIO devices */ > > for (i = 0; i < VIRTIO_COUNT; i++) { > > sysbus_create_simple("virtio-mmio", > > memmap[VIRT_VIRTIO].base + i * memmap[VIRT_VIRTIO].size, > > - qdev_get_gpio_in(DEVICE(s->plic), VIRTIO_IRQ + i)); > > + qdev_get_gpio_in(DEVICE(virtio_plic), VIRTIO_IRQ + i)); > > } > > > > gpex_pcie_init(system_memory, > > @@ -611,14 +681,14 @@ static void riscv_virt_board_init(MachineState > *machine) > > memmap[VIRT_PCIE_MMIO].base, > > memmap[VIRT_PCIE_MMIO].size, > > memmap[VIRT_PCIE_PIO].base, > > - DEVICE(s->plic), true); > > + DEVICE(pcie_plic), true); > > > > serial_mm_init(system_memory, memmap[VIRT_UART0].base, > > - 0, qdev_get_gpio_in(DEVICE(s->plic), UART0_IRQ), 399193, > > + 0, qdev_get_gpio_in(DEVICE(mmio_plic), UART0_IRQ), 399193, > > serial_hd(0), DEVICE_LITTLE_ENDIAN); > > > > sysbus_create_simple("goldfish_rtc", memmap[VIRT_RTC].base, > > - qdev_get_gpio_in(DEVICE(s->plic), RTC_IRQ)); > > + qdev_get_gpio_in(DEVICE(mmio_plic), RTC_IRQ)); > > > > virt_flash_create(s); > > > > @@ -628,8 +698,6 @@ static void riscv_virt_board_init(MachineState > *machine) > > drive_get(IF_PFLASH, 0, i)); > > } > > virt_flash_map(s, system_memory); > > - > > - g_free(plic_hart_config); > > } > > > > static void riscv_virt_machine_instance_init(Object *obj) @@ -642,9 > > +710,13 @@ static void riscv_virt_machine_class_init(ObjectClass *oc, > > void *data) > > > > mc->desc = "RISC-V VirtIO board"; > > mc->init = riscv_virt_board_init; > > - mc->max_cpus = 8; > > + mc->max_cpus = VIRT_CPUS_MAX; > > mc->default_cpu_type = VIRT_CPU; > > mc->pci_allow_0_address = true; > > + mc->possible_cpu_arch_ids = riscv_numa_possible_cpu_arch_ids; > > + mc->cpu_index_to_instance_props = > riscv_numa_cpu_index_to_props; > > + mc->get_default_cpu_node_id = > riscv_numa_get_default_cpu_node_id; > > + mc->numa_mem_supported = true; > > } > > > > static const TypeInfo riscv_virt_machine_typeinfo = { diff --git > > a/include/hw/riscv/virt.h b/include/hw/riscv/virt.h index > > e69355efaf..1beacd7666 100644 > > --- a/include/hw/riscv/virt.h > > +++ b/include/hw/riscv/virt.h > > @@ -23,6 +23,9 @@ > > #include "hw/sysbus.h" > > #include "hw/block/flash.h" > > > > +#define VIRT_CPUS_MAX 8 > > +#define VIRT_SOCKETS_MAX 8 > > + > > #define TYPE_RISCV_VIRT_MACHINE MACHINE_TYPE_NAME("virt") > #define > > RISCV_VIRT_MACHINE(obj) \ > > OBJECT_CHECK(RISCVVirtState, (obj), TYPE_RISCV_VIRT_MACHINE) @@ > > -32,8 +35,8 @@ typedef struct { > > MachineState parent; > > > > /*< public >*/ > > - RISCVHartArrayState soc; > > - DeviceState *plic; > > + RISCVHartArrayState soc[VIRT_SOCKETS_MAX]; > > + DeviceState *plic[VIRT_SOCKETS_MAX]; > > PFlashCFI01 *flash[2]; > > > > void *fdt; > > @@ -74,6 +77,8 @@ enum { > > #define VIRT_PLIC_ENABLE_STRIDE 0x80 > > #define VIRT_PLIC_CONTEXT_BASE 0x200000 #define > > VIRT_PLIC_CONTEXT_STRIDE 0x1000 > > +#define VIRT_PLIC_SIZE(__num_context) \ > > + (VIRT_PLIC_CONTEXT_BASE + (__num_context) * > > +VIRT_PLIC_CONTEXT_STRIDE) > > > > #define FDT_PCI_ADDR_CELLS 3 > > #define FDT_PCI_INT_CELLS 1 > > -- > > 2.25.1 > > > > Regards, Anup
On Fri, May 29, 2020 at 4:50 AM Anup Patel <anup.patel@wdc.com> wrote: > > We extend RISC-V virt machine to allow creating a multi-socket > machine. Each RISC-V virt machine socket is a NUMA node having > a set of HARTs, a memory instance, a CLINT instance, and a PLIC > instance. Other devices are shared between all sockets. We also > update the generated device tree accordingly. > > By default, NUMA multi-socket support is disabled for RISC-V virt > machine. To enable it, users can use "-numa" command-line options > of QEMU. > > Example1: For two NUMA nodes with 2 CPUs each, append following > to command-line options: "-smp 4 -numa node -numa node" > > Example2: For two NUMA nodes with 1 and 3 CPUs, append following > to command-line options: > "-smp 4 -numa node -numa node -numa cpu,node-id=0,core-id=0 \ > -numa cpu,node-id=1,core-id=1 -numa cpu,node-id=1,core-id=2 \ > -numa cpu,node-id=1,core-id=3" > > The maximum number of sockets in a RISC-V virt machine is 8 > but this limit can be changed in future. > > Signed-off-by: Anup Patel <anup.patel@wdc.com> > --- > hw/riscv/virt.c | 530 +++++++++++++++++++++++----------------- > include/hw/riscv/virt.h | 9 +- > 2 files changed, 308 insertions(+), 231 deletions(-) > > diff --git a/hw/riscv/virt.c b/hw/riscv/virt.c > index 421815081d..2863b42cea 100644 > --- a/hw/riscv/virt.c > +++ b/hw/riscv/virt.c > @@ -35,6 +35,7 @@ > #include "hw/riscv/sifive_test.h" > #include "hw/riscv/virt.h" > #include "hw/riscv/boot.h" > +#include "hw/riscv/numa.h" > #include "chardev/char.h" > #include "sysemu/arch_init.h" > #include "sysemu/device_tree.h" > @@ -60,7 +61,7 @@ static const struct MemmapEntry { > [VIRT_TEST] = { 0x100000, 0x1000 }, > [VIRT_RTC] = { 0x101000, 0x1000 }, > [VIRT_CLINT] = { 0x2000000, 0x10000 }, > - [VIRT_PLIC] = { 0xc000000, 0x4000000 }, > + [VIRT_PLIC] = { 0xc000000, VIRT_PLIC_SIZE(VIRT_CPUS_MAX * 2) }, > [VIRT_UART0] = { 0x10000000, 0x100 }, > [VIRT_VIRTIO] = { 0x10001000, 0x1000 }, > [VIRT_FLASH] = { 0x20000000, 0x4000000 }, > @@ -182,10 +183,17 @@ static void create_fdt(RISCVVirtState *s, const struct MemmapEntry *memmap, > uint64_t mem_size, const char *cmdline) > { > void *fdt; > - int cpu, i; > - uint32_t *cells; > - char *nodename; > - uint32_t plic_phandle, test_phandle, phandle = 1; > + int i, cpu, socket; > + MachineState *mc = MACHINE(s); > + uint64_t addr, size; > + uint32_t *clint_cells, *plic_cells; > + unsigned long clint_addr, plic_addr; > + uint32_t plic_phandle[MAX_NODES]; > + uint32_t cpu_phandle, intc_phandle, test_phandle; > + uint32_t phandle = 1, plic_mmio_phandle = 1; > + uint32_t plic_pcie_phandle = 1, plic_virtio_phandle = 1; > + char *mem_name, *cpu_name, *core_name, *intc_name; > + char *name, *clint_name, *plic_name, *clust_name; > hwaddr flashsize = virt_memmap[VIRT_FLASH].size / 2; > hwaddr flashbase = virt_memmap[VIRT_FLASH].base; > > @@ -206,231 +214,238 @@ static void create_fdt(RISCVVirtState *s, const struct MemmapEntry *memmap, > qemu_fdt_setprop_cell(fdt, "/soc", "#size-cells", 0x2); > qemu_fdt_setprop_cell(fdt, "/soc", "#address-cells", 0x2); > > - nodename = g_strdup_printf("/memory@%lx", > - (long)memmap[VIRT_DRAM].base); > - qemu_fdt_add_subnode(fdt, nodename); > - qemu_fdt_setprop_cells(fdt, nodename, "reg", > - memmap[VIRT_DRAM].base >> 32, memmap[VIRT_DRAM].base, > - mem_size >> 32, mem_size); > - qemu_fdt_setprop_string(fdt, nodename, "device_type", "memory"); > - g_free(nodename); > - > qemu_fdt_add_subnode(fdt, "/cpus"); > qemu_fdt_setprop_cell(fdt, "/cpus", "timebase-frequency", > SIFIVE_CLINT_TIMEBASE_FREQ); > qemu_fdt_setprop_cell(fdt, "/cpus", "#size-cells", 0x0); > qemu_fdt_setprop_cell(fdt, "/cpus", "#address-cells", 0x1); > + qemu_fdt_add_subnode(fdt, "/cpus/cpu-map"); > + > + for (socket = (riscv_socket_count(mc) - 1); socket >= 0; socket--) { > + clust_name = g_strdup_printf("/cpus/cpu-map/cluster%d", socket); > + qemu_fdt_add_subnode(fdt, clust_name); > + > + plic_cells = g_new0(uint32_t, s->soc[socket].num_harts * 4); > + clint_cells = g_new0(uint32_t, s->soc[socket].num_harts * 4); > + > + for (cpu = s->soc[socket].num_harts - 1; cpu >= 0; cpu--) { > + cpu_phandle = phandle++; > > - for (cpu = s->soc.num_harts - 1; cpu >= 0; cpu--) { > - int cpu_phandle = phandle++; > - int intc_phandle; > - nodename = g_strdup_printf("/cpus/cpu@%d", cpu); > - char *intc = g_strdup_printf("/cpus/cpu@%d/interrupt-controller", cpu); > - char *isa = riscv_isa_string(&s->soc.harts[cpu]); > - qemu_fdt_add_subnode(fdt, nodename); > + cpu_name = g_strdup_printf("/cpus/cpu@%d", > + s->soc[socket].hartid_base + cpu); > + qemu_fdt_add_subnode(fdt, cpu_name); > #if defined(TARGET_RISCV32) > - qemu_fdt_setprop_string(fdt, nodename, "mmu-type", "riscv,sv32"); > + qemu_fdt_setprop_string(fdt, cpu_name, "mmu-type", "riscv,sv32"); > #else > - qemu_fdt_setprop_string(fdt, nodename, "mmu-type", "riscv,sv48"); > + qemu_fdt_setprop_string(fdt, cpu_name, "mmu-type", "riscv,sv48"); > #endif > - qemu_fdt_setprop_string(fdt, nodename, "riscv,isa", isa); > - qemu_fdt_setprop_string(fdt, nodename, "compatible", "riscv"); > - qemu_fdt_setprop_string(fdt, nodename, "status", "okay"); > - qemu_fdt_setprop_cell(fdt, nodename, "reg", cpu); > - qemu_fdt_setprop_string(fdt, nodename, "device_type", "cpu"); > - qemu_fdt_setprop_cell(fdt, nodename, "phandle", cpu_phandle); > - intc_phandle = phandle++; > - qemu_fdt_add_subnode(fdt, intc); > - qemu_fdt_setprop_cell(fdt, intc, "phandle", intc_phandle); > - qemu_fdt_setprop_string(fdt, intc, "compatible", "riscv,cpu-intc"); > - qemu_fdt_setprop(fdt, intc, "interrupt-controller", NULL, 0); > - qemu_fdt_setprop_cell(fdt, intc, "#interrupt-cells", 1); > - g_free(isa); > - g_free(intc); > - g_free(nodename); > - } > + name = riscv_isa_string(&s->soc[socket].harts[cpu]); > + qemu_fdt_setprop_string(fdt, cpu_name, "riscv,isa", name); > + g_free(name); > + qemu_fdt_setprop_string(fdt, cpu_name, "compatible", "riscv"); > + qemu_fdt_setprop_string(fdt, cpu_name, "status", "okay"); > + qemu_fdt_setprop_cell(fdt, cpu_name, "reg", > + s->soc[socket].hartid_base + cpu); > + qemu_fdt_setprop_string(fdt, cpu_name, "device_type", "cpu"); > + riscv_socket_fdt_write_id(mc, fdt, cpu_name, socket); > + qemu_fdt_setprop_cell(fdt, cpu_name, "phandle", cpu_phandle); > + > + intc_name = g_strdup_printf("%s/interrupt-controller", cpu_name); > + qemu_fdt_add_subnode(fdt, intc_name); > + intc_phandle = phandle++; > + qemu_fdt_setprop_cell(fdt, intc_name, "phandle", intc_phandle); > + qemu_fdt_setprop_string(fdt, intc_name, "compatible", > + "riscv,cpu-intc"); > + qemu_fdt_setprop(fdt, intc_name, "interrupt-controller", NULL, 0); > + qemu_fdt_setprop_cell(fdt, intc_name, "#interrupt-cells", 1); > + > + clint_cells[cpu * 4 + 0] = cpu_to_be32(intc_phandle); > + clint_cells[cpu * 4 + 1] = cpu_to_be32(IRQ_M_SOFT); > + clint_cells[cpu * 4 + 2] = cpu_to_be32(intc_phandle); > + clint_cells[cpu * 4 + 3] = cpu_to_be32(IRQ_M_TIMER); > + > + plic_cells[cpu * 4 + 0] = cpu_to_be32(intc_phandle); > + plic_cells[cpu * 4 + 1] = cpu_to_be32(IRQ_M_EXT); > + plic_cells[cpu * 4 + 2] = cpu_to_be32(intc_phandle); > + plic_cells[cpu * 4 + 3] = cpu_to_be32(IRQ_S_EXT); > + > + core_name = g_strdup_printf("%s/core%d", clust_name, cpu); > + qemu_fdt_add_subnode(fdt, core_name); > + qemu_fdt_setprop_cell(fdt, core_name, "cpu", cpu_phandle); > + > + g_free(core_name); > + g_free(intc_name); > + g_free(cpu_name); > + } > > - /* Add cpu-topology node */ > - qemu_fdt_add_subnode(fdt, "/cpus/cpu-map"); > - qemu_fdt_add_subnode(fdt, "/cpus/cpu-map/cluster0"); > - for (cpu = s->soc.num_harts - 1; cpu >= 0; cpu--) { > - char *core_nodename = g_strdup_printf("/cpus/cpu-map/cluster0/core%d", > - cpu); > - char *cpu_nodename = g_strdup_printf("/cpus/cpu@%d", cpu); > - uint32_t intc_phandle = qemu_fdt_get_phandle(fdt, cpu_nodename); > - qemu_fdt_add_subnode(fdt, core_nodename); > - qemu_fdt_setprop_cell(fdt, core_nodename, "cpu", intc_phandle); > - g_free(core_nodename); > - g_free(cpu_nodename); > + addr = memmap[VIRT_DRAM].base + riscv_socket_mem_offset(mc, socket); > + size = riscv_socket_mem_size(mc, socket); > + mem_name = g_strdup_printf("/memory@%lx", (long)addr); > + qemu_fdt_add_subnode(fdt, mem_name); > + qemu_fdt_setprop_cells(fdt, mem_name, "reg", > + addr >> 32, addr, size >> 32, size); > + qemu_fdt_setprop_string(fdt, mem_name, "device_type", "memory"); > + riscv_socket_fdt_write_id(mc, fdt, mem_name, socket); > + g_free(mem_name); > + > + clint_addr = memmap[VIRT_CLINT].base + > + (memmap[VIRT_CLINT].size * socket); > + clint_name = g_strdup_printf("/soc/clint@%lx", clint_addr); > + qemu_fdt_add_subnode(fdt, clint_name); > + qemu_fdt_setprop_string(fdt, clint_name, "compatible", "riscv,clint0"); > + qemu_fdt_setprop_cells(fdt, clint_name, "reg", > + 0x0, clint_addr, 0x0, memmap[VIRT_CLINT].size); > + qemu_fdt_setprop(fdt, clint_name, "interrupts-extended", > + clint_cells, s->soc[socket].num_harts * sizeof(uint32_t) * 4); > + riscv_socket_fdt_write_id(mc, fdt, clint_name, socket); > + g_free(clint_name); > + > + plic_phandle[socket] = phandle++; > + plic_addr = memmap[VIRT_PLIC].base + (memmap[VIRT_PLIC].size * socket); > + plic_name = g_strdup_printf("/soc/plic@%lx", plic_addr); > + qemu_fdt_add_subnode(fdt, plic_name); > + qemu_fdt_setprop_cell(fdt, plic_name, > + "#address-cells", FDT_PLIC_ADDR_CELLS); > + qemu_fdt_setprop_cell(fdt, plic_name, > + "#interrupt-cells", FDT_PLIC_INT_CELLS); > + qemu_fdt_setprop_string(fdt, plic_name, "compatible", "riscv,plic0"); > + qemu_fdt_setprop(fdt, plic_name, "interrupt-controller", NULL, 0); > + qemu_fdt_setprop(fdt, plic_name, "interrupts-extended", > + plic_cells, s->soc[socket].num_harts * sizeof(uint32_t) * 4); > + qemu_fdt_setprop_cells(fdt, plic_name, "reg", > + 0x0, plic_addr, 0x0, memmap[VIRT_PLIC].size); > + qemu_fdt_setprop_cell(fdt, plic_name, "riscv,ndev", VIRTIO_NDEV); > + riscv_socket_fdt_write_id(mc, fdt, plic_name, socket); > + qemu_fdt_setprop_cell(fdt, plic_name, "phandle", plic_phandle[socket]); > + g_free(plic_name); > + > + g_free(clint_cells); > + g_free(plic_cells); > + g_free(clust_name); > } > > - cells = g_new0(uint32_t, s->soc.num_harts * 4); > - for (cpu = 0; cpu < s->soc.num_harts; cpu++) { > - nodename = > - g_strdup_printf("/cpus/cpu@%d/interrupt-controller", cpu); > - uint32_t intc_phandle = qemu_fdt_get_phandle(fdt, nodename); > - cells[cpu * 4 + 0] = cpu_to_be32(intc_phandle); > - cells[cpu * 4 + 1] = cpu_to_be32(IRQ_M_SOFT); > - cells[cpu * 4 + 2] = cpu_to_be32(intc_phandle); > - cells[cpu * 4 + 3] = cpu_to_be32(IRQ_M_TIMER); > - g_free(nodename); > - } > - nodename = g_strdup_printf("/soc/clint@%lx", > - (long)memmap[VIRT_CLINT].base); > - qemu_fdt_add_subnode(fdt, nodename); > - qemu_fdt_setprop_string(fdt, nodename, "compatible", "riscv,clint0"); > - qemu_fdt_setprop_cells(fdt, nodename, "reg", > - 0x0, memmap[VIRT_CLINT].base, > - 0x0, memmap[VIRT_CLINT].size); > - qemu_fdt_setprop(fdt, nodename, "interrupts-extended", > - cells, s->soc.num_harts * sizeof(uint32_t) * 4); > - g_free(cells); > - g_free(nodename); > - > - plic_phandle = phandle++; > - cells = g_new0(uint32_t, s->soc.num_harts * 4); > - for (cpu = 0; cpu < s->soc.num_harts; cpu++) { > - nodename = > - g_strdup_printf("/cpus/cpu@%d/interrupt-controller", cpu); > - uint32_t intc_phandle = qemu_fdt_get_phandle(fdt, nodename); > - cells[cpu * 4 + 0] = cpu_to_be32(intc_phandle); > - cells[cpu * 4 + 1] = cpu_to_be32(IRQ_M_EXT); > - cells[cpu * 4 + 2] = cpu_to_be32(intc_phandle); > - cells[cpu * 4 + 3] = cpu_to_be32(IRQ_S_EXT); > - g_free(nodename); > + for (socket = 0; socket < riscv_socket_count(mc); socket++) { > + if (socket == 0) { > + plic_mmio_phandle = plic_phandle[socket]; > + plic_virtio_phandle = plic_phandle[socket]; > + plic_pcie_phandle = plic_phandle[socket]; > + } > + if (socket == 1) { > + plic_virtio_phandle = plic_phandle[socket]; > + plic_pcie_phandle = plic_phandle[socket]; > + } > + if (socket == 2) { > + plic_pcie_phandle = plic_phandle[socket]; > + } > } > - nodename = g_strdup_printf("/soc/interrupt-controller@%lx", > - (long)memmap[VIRT_PLIC].base); > - qemu_fdt_add_subnode(fdt, nodename); > - qemu_fdt_setprop_cell(fdt, nodename, "#address-cells", > - FDT_PLIC_ADDR_CELLS); > - qemu_fdt_setprop_cell(fdt, nodename, "#interrupt-cells", > - FDT_PLIC_INT_CELLS); > - qemu_fdt_setprop_string(fdt, nodename, "compatible", "riscv,plic0"); > - qemu_fdt_setprop(fdt, nodename, "interrupt-controller", NULL, 0); > - qemu_fdt_setprop(fdt, nodename, "interrupts-extended", > - cells, s->soc.num_harts * sizeof(uint32_t) * 4); > - qemu_fdt_setprop_cells(fdt, nodename, "reg", > - 0x0, memmap[VIRT_PLIC].base, > - 0x0, memmap[VIRT_PLIC].size); > - qemu_fdt_setprop_cell(fdt, nodename, "riscv,ndev", VIRTIO_NDEV); > - qemu_fdt_setprop_cell(fdt, nodename, "phandle", plic_phandle); > - plic_phandle = qemu_fdt_get_phandle(fdt, nodename); > - g_free(cells); > - g_free(nodename); > + > + riscv_socket_fdt_write_distance_matrix(mc, fdt); > > for (i = 0; i < VIRTIO_COUNT; i++) { > - nodename = g_strdup_printf("/virtio_mmio@%lx", > + name = g_strdup_printf("/soc/virtio_mmio@%lx", > (long)(memmap[VIRT_VIRTIO].base + i * memmap[VIRT_VIRTIO].size)); > - qemu_fdt_add_subnode(fdt, nodename); > - qemu_fdt_setprop_string(fdt, nodename, "compatible", "virtio,mmio"); > - qemu_fdt_setprop_cells(fdt, nodename, "reg", > + qemu_fdt_add_subnode(fdt, name); > + qemu_fdt_setprop_string(fdt, name, "compatible", "virtio,mmio"); > + qemu_fdt_setprop_cells(fdt, name, "reg", > 0x0, memmap[VIRT_VIRTIO].base + i * memmap[VIRT_VIRTIO].size, > 0x0, memmap[VIRT_VIRTIO].size); > - qemu_fdt_setprop_cell(fdt, nodename, "interrupt-parent", plic_phandle); > - qemu_fdt_setprop_cell(fdt, nodename, "interrupts", VIRTIO_IRQ + i); > - g_free(nodename); > + qemu_fdt_setprop_cell(fdt, name, "interrupt-parent", > + plic_virtio_phandle); > + qemu_fdt_setprop_cell(fdt, name, "interrupts", VIRTIO_IRQ + i); > + g_free(name); > } > > - nodename = g_strdup_printf("/soc/pci@%lx", > + name = g_strdup_printf("/soc/pci@%lx", > (long) memmap[VIRT_PCIE_ECAM].base); > - qemu_fdt_add_subnode(fdt, nodename); > - qemu_fdt_setprop_cell(fdt, nodename, "#address-cells", > - FDT_PCI_ADDR_CELLS); > - qemu_fdt_setprop_cell(fdt, nodename, "#interrupt-cells", > - FDT_PCI_INT_CELLS); > - qemu_fdt_setprop_cell(fdt, nodename, "#size-cells", 0x2); > - qemu_fdt_setprop_string(fdt, nodename, "compatible", > - "pci-host-ecam-generic"); > - qemu_fdt_setprop_string(fdt, nodename, "device_type", "pci"); > - qemu_fdt_setprop_cell(fdt, nodename, "linux,pci-domain", 0); > - qemu_fdt_setprop_cells(fdt, nodename, "bus-range", 0, > - memmap[VIRT_PCIE_ECAM].size / > - PCIE_MMCFG_SIZE_MIN - 1); > - qemu_fdt_setprop(fdt, nodename, "dma-coherent", NULL, 0); > - qemu_fdt_setprop_cells(fdt, nodename, "reg", 0, memmap[VIRT_PCIE_ECAM].base, > - 0, memmap[VIRT_PCIE_ECAM].size); > - qemu_fdt_setprop_sized_cells(fdt, nodename, "ranges", > + qemu_fdt_add_subnode(fdt, name); > + qemu_fdt_setprop_cell(fdt, name, "#address-cells", FDT_PCI_ADDR_CELLS); > + qemu_fdt_setprop_cell(fdt, name, "#interrupt-cells", FDT_PCI_INT_CELLS); > + qemu_fdt_setprop_cell(fdt, name, "#size-cells", 0x2); > + qemu_fdt_setprop_string(fdt, name, "compatible", "pci-host-ecam-generic"); > + qemu_fdt_setprop_string(fdt, name, "device_type", "pci"); > + qemu_fdt_setprop_cell(fdt, name, "linux,pci-domain", 0); > + qemu_fdt_setprop_cells(fdt, name, "bus-range", 0, > + memmap[VIRT_PCIE_ECAM].size / PCIE_MMCFG_SIZE_MIN - 1); > + qemu_fdt_setprop(fdt, name, "dma-coherent", NULL, 0); > + qemu_fdt_setprop_cells(fdt, name, "reg", 0, > + memmap[VIRT_PCIE_ECAM].base, 0, memmap[VIRT_PCIE_ECAM].size); > + qemu_fdt_setprop_sized_cells(fdt, name, "ranges", > 1, FDT_PCI_RANGE_IOPORT, 2, 0, > 2, memmap[VIRT_PCIE_PIO].base, 2, memmap[VIRT_PCIE_PIO].size, > 1, FDT_PCI_RANGE_MMIO, > 2, memmap[VIRT_PCIE_MMIO].base, > 2, memmap[VIRT_PCIE_MMIO].base, 2, memmap[VIRT_PCIE_MMIO].size); > - create_pcie_irq_map(fdt, nodename, plic_phandle); > - g_free(nodename); > + create_pcie_irq_map(fdt, name, plic_pcie_phandle); > + g_free(name); > > test_phandle = phandle++; > - nodename = g_strdup_printf("/test@%lx", > + name = g_strdup_printf("/soc/test@%lx", > (long)memmap[VIRT_TEST].base); > - qemu_fdt_add_subnode(fdt, nodename); > + qemu_fdt_add_subnode(fdt, name); > { > const char compat[] = "sifive,test1\0sifive,test0\0syscon"; > - qemu_fdt_setprop(fdt, nodename, "compatible", compat, sizeof(compat)); > + qemu_fdt_setprop(fdt, name, "compatible", compat, sizeof(compat)); > } > - qemu_fdt_setprop_cells(fdt, nodename, "reg", > + qemu_fdt_setprop_cells(fdt, name, "reg", > 0x0, memmap[VIRT_TEST].base, > 0x0, memmap[VIRT_TEST].size); > - qemu_fdt_setprop_cell(fdt, nodename, "phandle", test_phandle); > - test_phandle = qemu_fdt_get_phandle(fdt, nodename); > - g_free(nodename); > - > - nodename = g_strdup_printf("/reboot"); > - qemu_fdt_add_subnode(fdt, nodename); > - qemu_fdt_setprop_string(fdt, nodename, "compatible", "syscon-reboot"); > - qemu_fdt_setprop_cell(fdt, nodename, "regmap", test_phandle); > - qemu_fdt_setprop_cell(fdt, nodename, "offset", 0x0); > - qemu_fdt_setprop_cell(fdt, nodename, "value", FINISHER_RESET); > - g_free(nodename); > - > - nodename = g_strdup_printf("/poweroff"); > - qemu_fdt_add_subnode(fdt, nodename); > - qemu_fdt_setprop_string(fdt, nodename, "compatible", "syscon-poweroff"); > - qemu_fdt_setprop_cell(fdt, nodename, "regmap", test_phandle); > - qemu_fdt_setprop_cell(fdt, nodename, "offset", 0x0); > - qemu_fdt_setprop_cell(fdt, nodename, "value", FINISHER_PASS); > - g_free(nodename); > - > - nodename = g_strdup_printf("/uart@%lx", > - (long)memmap[VIRT_UART0].base); > - qemu_fdt_add_subnode(fdt, nodename); > - qemu_fdt_setprop_string(fdt, nodename, "compatible", "ns16550a"); > - qemu_fdt_setprop_cells(fdt, nodename, "reg", > + qemu_fdt_setprop_cell(fdt, name, "phandle", test_phandle); > + test_phandle = qemu_fdt_get_phandle(fdt, name); > + g_free(name); > + > + name = g_strdup_printf("/soc/reboot"); > + qemu_fdt_add_subnode(fdt, name); > + qemu_fdt_setprop_string(fdt, name, "compatible", "syscon-reboot"); > + qemu_fdt_setprop_cell(fdt, name, "regmap", test_phandle); > + qemu_fdt_setprop_cell(fdt, name, "offset", 0x0); > + qemu_fdt_setprop_cell(fdt, name, "value", FINISHER_RESET); > + g_free(name); > + > + name = g_strdup_printf("/soc/poweroff"); > + qemu_fdt_add_subnode(fdt, name); > + qemu_fdt_setprop_string(fdt, name, "compatible", "syscon-poweroff"); > + qemu_fdt_setprop_cell(fdt, name, "regmap", test_phandle); > + qemu_fdt_setprop_cell(fdt, name, "offset", 0x0); > + qemu_fdt_setprop_cell(fdt, name, "value", FINISHER_PASS); > + g_free(name); > + > + name = g_strdup_printf("/soc/uart@%lx", (long)memmap[VIRT_UART0].base); > + qemu_fdt_add_subnode(fdt, name); > + qemu_fdt_setprop_string(fdt, name, "compatible", "ns16550a"); > + qemu_fdt_setprop_cells(fdt, name, "reg", > 0x0, memmap[VIRT_UART0].base, > 0x0, memmap[VIRT_UART0].size); > - qemu_fdt_setprop_cell(fdt, nodename, "clock-frequency", 3686400); > - qemu_fdt_setprop_cell(fdt, nodename, "interrupt-parent", plic_phandle); > - qemu_fdt_setprop_cell(fdt, nodename, "interrupts", UART0_IRQ); > + qemu_fdt_setprop_cell(fdt, name, "clock-frequency", 3686400); > + qemu_fdt_setprop_cell(fdt, name, "interrupt-parent", plic_mmio_phandle); > + qemu_fdt_setprop_cell(fdt, name, "interrupts", UART0_IRQ); > > qemu_fdt_add_subnode(fdt, "/chosen"); > - qemu_fdt_setprop_string(fdt, "/chosen", "stdout-path", nodename); > + qemu_fdt_setprop_string(fdt, "/chosen", "stdout-path", name); > if (cmdline) { > qemu_fdt_setprop_string(fdt, "/chosen", "bootargs", cmdline); > } > - g_free(nodename); > - > - nodename = g_strdup_printf("/rtc@%lx", > - (long)memmap[VIRT_RTC].base); > - qemu_fdt_add_subnode(fdt, nodename); > - qemu_fdt_setprop_string(fdt, nodename, "compatible", > - "google,goldfish-rtc"); > - qemu_fdt_setprop_cells(fdt, nodename, "reg", > + g_free(name); > + > + name = g_strdup_printf("/soc/rtc@%lx", (long)memmap[VIRT_RTC].base); > + qemu_fdt_add_subnode(fdt, name); > + qemu_fdt_setprop_string(fdt, name, "compatible", "google,goldfish-rtc"); > + qemu_fdt_setprop_cells(fdt, name, "reg", > 0x0, memmap[VIRT_RTC].base, > 0x0, memmap[VIRT_RTC].size); > - qemu_fdt_setprop_cell(fdt, nodename, "interrupt-parent", plic_phandle); > - qemu_fdt_setprop_cell(fdt, nodename, "interrupts", RTC_IRQ); > - g_free(nodename); > - > - nodename = g_strdup_printf("/flash@%" PRIx64, flashbase); > - qemu_fdt_add_subnode(s->fdt, nodename); > - qemu_fdt_setprop_string(s->fdt, nodename, "compatible", "cfi-flash"); > - qemu_fdt_setprop_sized_cells(s->fdt, nodename, "reg", > + qemu_fdt_setprop_cell(fdt, name, "interrupt-parent", plic_mmio_phandle); > + qemu_fdt_setprop_cell(fdt, name, "interrupts", RTC_IRQ); > + g_free(name); > + > + name = g_strdup_printf("/soc/flash@%" PRIx64, flashbase); > + qemu_fdt_add_subnode(s->fdt, name); > + qemu_fdt_setprop_string(s->fdt, name, "compatible", "cfi-flash"); > + qemu_fdt_setprop_sized_cells(s->fdt, name, "reg", > 2, flashbase, 2, flashsize, > 2, flashbase + flashsize, 2, flashsize); > - qemu_fdt_setprop_cell(s->fdt, nodename, "bank-width", 4); > - g_free(nodename); > + qemu_fdt_setprop_cell(s->fdt, name, "bank-width", 4); > + g_free(name); > } > > - > static inline DeviceState *gpex_pcie_init(MemoryRegion *sys_mem, > hwaddr ecam_base, hwaddr ecam_size, > hwaddr mmio_base, hwaddr mmio_size, > @@ -478,21 +493,100 @@ static void riscv_virt_board_init(MachineState *machine) > MemoryRegion *system_memory = get_system_memory(); > MemoryRegion *main_mem = g_new(MemoryRegion, 1); > MemoryRegion *mask_rom = g_new(MemoryRegion, 1); > - char *plic_hart_config; > + char *plic_hart_config, *soc_name; > size_t plic_hart_config_len; > target_ulong start_addr = memmap[VIRT_DRAM].base; > - int i; > - unsigned int smp_cpus = machine->smp.cpus; > - > - /* Initialize SOC */ > - object_initialize_child(OBJECT(machine), "soc", &s->soc, sizeof(s->soc), > - TYPE_RISCV_HART_ARRAY, &error_abort, NULL); > - object_property_set_str(OBJECT(&s->soc), machine->cpu_type, "cpu-type", > - &error_abort); > - object_property_set_int(OBJECT(&s->soc), smp_cpus, "num-harts", > - &error_abort); > - object_property_set_bool(OBJECT(&s->soc), true, "realized", > - &error_abort); > + DeviceState *mmio_plic, *virtio_plic, *pcie_plic; > + int i, j, base_hartid, hart_count; > + > + /* Check socket count limit */ > + if (VIRT_SOCKETS_MAX < riscv_socket_count(machine)) { > + error_report("number of sockets/nodes should be less than %d", > + VIRT_SOCKETS_MAX); > + exit(1); > + } > + > + /* Initialize sockets */ > + mmio_plic = virtio_plic = pcie_plic = NULL; > + for (i = 0; i < riscv_socket_count(machine); i++) { > + if (!riscv_socket_check_hartids(machine, i)) { > + error_report("discontinuous hartids in socket%d", i); > + exit(1); > + } > + > + base_hartid = riscv_socket_first_hartid(machine, i); > + if (base_hartid < 0) { > + error_report("can't find hartid base for socket%d", i); > + exit(1); > + } > + > + hart_count = riscv_socket_hart_count(machine, i); > + if (hart_count < 0) { > + error_report("can't find hart count for socket%d", i); > + exit(1); > + } > + > + soc_name = g_strdup_printf("soc%d", i); > + object_initialize_child(OBJECT(machine), soc_name, &s->soc[i], > + sizeof(s->soc[i]), TYPE_RISCV_HART_ARRAY, &error_abort, NULL); > + g_free(soc_name); > + object_property_set_str(OBJECT(&s->soc[i]), > + machine->cpu_type, "cpu-type", &error_abort); > + object_property_set_int(OBJECT(&s->soc[i]), > + base_hartid, "hartid-base", &error_abort); > + object_property_set_int(OBJECT(&s->soc[i]), > + hart_count, "num-harts", &error_abort); > + object_property_set_bool(OBJECT(&s->soc[i]), > + true, "realized", &error_abort); > + > + /* Per-socket CLINT */ > + sifive_clint_create( > + memmap[VIRT_CLINT].base + i * memmap[VIRT_CLINT].size, > + memmap[VIRT_CLINT].size, base_hartid, hart_count, > + SIFIVE_SIP_BASE, SIFIVE_TIMECMP_BASE, SIFIVE_TIME_BASE, true); > + > + /* Per-socket PLIC hart topology configuration string */ > + plic_hart_config_len = > + (strlen(VIRT_PLIC_HART_CONFIG) + 1) * hart_count; > + plic_hart_config = g_malloc0(plic_hart_config_len); > + for (j = 0; j < hart_count; j++) { > + if (j != 0) { > + strncat(plic_hart_config, ",", plic_hart_config_len); > + } > + strncat(plic_hart_config, VIRT_PLIC_HART_CONFIG, > + plic_hart_config_len); > + plic_hart_config_len -= (strlen(VIRT_PLIC_HART_CONFIG) + 1); > + } > + > + /* Per-socket PLIC */ > + s->plic[i] = sifive_plic_create( > + memmap[VIRT_PLIC].base + i * memmap[VIRT_PLIC].size, > + plic_hart_config, base_hartid, > + VIRT_PLIC_NUM_SOURCES, > + VIRT_PLIC_NUM_PRIORITIES, > + VIRT_PLIC_PRIORITY_BASE, > + VIRT_PLIC_PENDING_BASE, > + VIRT_PLIC_ENABLE_BASE, > + VIRT_PLIC_ENABLE_STRIDE, > + VIRT_PLIC_CONTEXT_BASE, > + VIRT_PLIC_CONTEXT_STRIDE, > + memmap[VIRT_PLIC].size); > + g_free(plic_hart_config); > + > + /* Try to use different PLIC instance based device type */ > + if (i == 0) { > + mmio_plic = s->plic[i]; > + virtio_plic = s->plic[i]; > + pcie_plic = s->plic[i]; > + } > + if (i == 1) { > + virtio_plic = s->plic[i]; > + pcie_plic = s->plic[i]; > + } > + if (i == 2) { > + pcie_plic = s->plic[i]; > + } > + } > > /* register system main memory (actual RAM) */ > memory_region_init_ram(main_mem, NULL, "riscv_virt_board.ram", > @@ -571,38 +665,14 @@ static void riscv_virt_board_init(MachineState *machine) > memmap[VIRT_MROM].base + sizeof(reset_vec), > &address_space_memory); > > - /* create PLIC hart topology configuration string */ > - plic_hart_config_len = (strlen(VIRT_PLIC_HART_CONFIG) + 1) * smp_cpus; > - plic_hart_config = g_malloc0(plic_hart_config_len); > - for (i = 0; i < smp_cpus; i++) { > - if (i != 0) { > - strncat(plic_hart_config, ",", plic_hart_config_len); > - } > - strncat(plic_hart_config, VIRT_PLIC_HART_CONFIG, plic_hart_config_len); > - plic_hart_config_len -= (strlen(VIRT_PLIC_HART_CONFIG) + 1); > - } > - > - /* MMIO */ > - s->plic = sifive_plic_create(memmap[VIRT_PLIC].base, > - plic_hart_config, 0, > - VIRT_PLIC_NUM_SOURCES, > - VIRT_PLIC_NUM_PRIORITIES, > - VIRT_PLIC_PRIORITY_BASE, > - VIRT_PLIC_PENDING_BASE, > - VIRT_PLIC_ENABLE_BASE, > - VIRT_PLIC_ENABLE_STRIDE, > - VIRT_PLIC_CONTEXT_BASE, > - VIRT_PLIC_CONTEXT_STRIDE, > - memmap[VIRT_PLIC].size); > - sifive_clint_create(memmap[VIRT_CLINT].base, > - memmap[VIRT_CLINT].size, 0, smp_cpus, > - SIFIVE_SIP_BASE, SIFIVE_TIMECMP_BASE, SIFIVE_TIME_BASE, true); > + /* SiFive Test MMIO device */ > sifive_test_create(memmap[VIRT_TEST].base); > > + /* VirtIO MMIO devices */ > for (i = 0; i < VIRTIO_COUNT; i++) { > sysbus_create_simple("virtio-mmio", > memmap[VIRT_VIRTIO].base + i * memmap[VIRT_VIRTIO].size, > - qdev_get_gpio_in(DEVICE(s->plic), VIRTIO_IRQ + i)); > + qdev_get_gpio_in(DEVICE(virtio_plic), VIRTIO_IRQ + i)); > } > > gpex_pcie_init(system_memory, > @@ -611,14 +681,14 @@ static void riscv_virt_board_init(MachineState *machine) > memmap[VIRT_PCIE_MMIO].base, > memmap[VIRT_PCIE_MMIO].size, > memmap[VIRT_PCIE_PIO].base, > - DEVICE(s->plic), true); > + DEVICE(pcie_plic), true); > > serial_mm_init(system_memory, memmap[VIRT_UART0].base, > - 0, qdev_get_gpio_in(DEVICE(s->plic), UART0_IRQ), 399193, > + 0, qdev_get_gpio_in(DEVICE(mmio_plic), UART0_IRQ), 399193, > serial_hd(0), DEVICE_LITTLE_ENDIAN); > > sysbus_create_simple("goldfish_rtc", memmap[VIRT_RTC].base, > - qdev_get_gpio_in(DEVICE(s->plic), RTC_IRQ)); > + qdev_get_gpio_in(DEVICE(mmio_plic), RTC_IRQ)); > > virt_flash_create(s); > > @@ -628,8 +698,6 @@ static void riscv_virt_board_init(MachineState *machine) > drive_get(IF_PFLASH, 0, i)); > } > virt_flash_map(s, system_memory); > - > - g_free(plic_hart_config); > } > > static void riscv_virt_machine_instance_init(Object *obj) > @@ -642,9 +710,13 @@ static void riscv_virt_machine_class_init(ObjectClass *oc, void *data) > > mc->desc = "RISC-V VirtIO board"; > mc->init = riscv_virt_board_init; > - mc->max_cpus = 8; > + mc->max_cpus = VIRT_CPUS_MAX; > mc->default_cpu_type = VIRT_CPU; > mc->pci_allow_0_address = true; > + mc->possible_cpu_arch_ids = riscv_numa_possible_cpu_arch_ids; > + mc->cpu_index_to_instance_props = riscv_numa_cpu_index_to_props; > + mc->get_default_cpu_node_id = riscv_numa_get_default_cpu_node_id; > + mc->numa_mem_supported = true; > } > > static const TypeInfo riscv_virt_machine_typeinfo = { > diff --git a/include/hw/riscv/virt.h b/include/hw/riscv/virt.h > index e69355efaf..1beacd7666 100644 > --- a/include/hw/riscv/virt.h > +++ b/include/hw/riscv/virt.h > @@ -23,6 +23,9 @@ > #include "hw/sysbus.h" > #include "hw/block/flash.h" > > +#define VIRT_CPUS_MAX 8 > +#define VIRT_SOCKETS_MAX 8 > + > #define TYPE_RISCV_VIRT_MACHINE MACHINE_TYPE_NAME("virt") > #define RISCV_VIRT_MACHINE(obj) \ > OBJECT_CHECK(RISCVVirtState, (obj), TYPE_RISCV_VIRT_MACHINE) > @@ -32,8 +35,8 @@ typedef struct { > MachineState parent; > > /*< public >*/ > - RISCVHartArrayState soc; > - DeviceState *plic; > + RISCVHartArrayState soc[VIRT_SOCKETS_MAX]; > + DeviceState *plic[VIRT_SOCKETS_MAX]; > PFlashCFI01 *flash[2]; > > void *fdt; > @@ -74,6 +77,8 @@ enum { > #define VIRT_PLIC_ENABLE_STRIDE 0x80 > #define VIRT_PLIC_CONTEXT_BASE 0x200000 > #define VIRT_PLIC_CONTEXT_STRIDE 0x1000 > +#define VIRT_PLIC_SIZE(__num_context) \ > + (VIRT_PLIC_CONTEXT_BASE + (__num_context) * VIRT_PLIC_CONTEXT_STRIDE) > > #define FDT_PCI_ADDR_CELLS 3 > #define FDT_PCI_INT_CELLS 1 > -- > 2.25.1 > > LGTM. Reviewed-by: Atish Patra <atish.patra@wdc.com>
diff --git a/hw/riscv/virt.c b/hw/riscv/virt.c index 421815081d..2863b42cea 100644 --- a/hw/riscv/virt.c +++ b/hw/riscv/virt.c @@ -35,6 +35,7 @@ #include "hw/riscv/sifive_test.h" #include "hw/riscv/virt.h" #include "hw/riscv/boot.h" +#include "hw/riscv/numa.h" #include "chardev/char.h" #include "sysemu/arch_init.h" #include "sysemu/device_tree.h" @@ -60,7 +61,7 @@ static const struct MemmapEntry { [VIRT_TEST] = { 0x100000, 0x1000 }, [VIRT_RTC] = { 0x101000, 0x1000 }, [VIRT_CLINT] = { 0x2000000, 0x10000 }, - [VIRT_PLIC] = { 0xc000000, 0x4000000 }, + [VIRT_PLIC] = { 0xc000000, VIRT_PLIC_SIZE(VIRT_CPUS_MAX * 2) }, [VIRT_UART0] = { 0x10000000, 0x100 }, [VIRT_VIRTIO] = { 0x10001000, 0x1000 }, [VIRT_FLASH] = { 0x20000000, 0x4000000 }, @@ -182,10 +183,17 @@ static void create_fdt(RISCVVirtState *s, const struct MemmapEntry *memmap, uint64_t mem_size, const char *cmdline) { void *fdt; - int cpu, i; - uint32_t *cells; - char *nodename; - uint32_t plic_phandle, test_phandle, phandle = 1; + int i, cpu, socket; + MachineState *mc = MACHINE(s); + uint64_t addr, size; + uint32_t *clint_cells, *plic_cells; + unsigned long clint_addr, plic_addr; + uint32_t plic_phandle[MAX_NODES]; + uint32_t cpu_phandle, intc_phandle, test_phandle; + uint32_t phandle = 1, plic_mmio_phandle = 1; + uint32_t plic_pcie_phandle = 1, plic_virtio_phandle = 1; + char *mem_name, *cpu_name, *core_name, *intc_name; + char *name, *clint_name, *plic_name, *clust_name; hwaddr flashsize = virt_memmap[VIRT_FLASH].size / 2; hwaddr flashbase = virt_memmap[VIRT_FLASH].base; @@ -206,231 +214,238 @@ static void create_fdt(RISCVVirtState *s, const struct MemmapEntry *memmap, qemu_fdt_setprop_cell(fdt, "/soc", "#size-cells", 0x2); qemu_fdt_setprop_cell(fdt, "/soc", "#address-cells", 0x2); - nodename = g_strdup_printf("/memory@%lx", - (long)memmap[VIRT_DRAM].base); - qemu_fdt_add_subnode(fdt, nodename); - qemu_fdt_setprop_cells(fdt, nodename, "reg", - memmap[VIRT_DRAM].base >> 32, memmap[VIRT_DRAM].base, - mem_size >> 32, mem_size); - qemu_fdt_setprop_string(fdt, nodename, "device_type", "memory"); - g_free(nodename); - qemu_fdt_add_subnode(fdt, "/cpus"); qemu_fdt_setprop_cell(fdt, "/cpus", "timebase-frequency", SIFIVE_CLINT_TIMEBASE_FREQ); qemu_fdt_setprop_cell(fdt, "/cpus", "#size-cells", 0x0); qemu_fdt_setprop_cell(fdt, "/cpus", "#address-cells", 0x1); + qemu_fdt_add_subnode(fdt, "/cpus/cpu-map"); + + for (socket = (riscv_socket_count(mc) - 1); socket >= 0; socket--) { + clust_name = g_strdup_printf("/cpus/cpu-map/cluster%d", socket); + qemu_fdt_add_subnode(fdt, clust_name); + + plic_cells = g_new0(uint32_t, s->soc[socket].num_harts * 4); + clint_cells = g_new0(uint32_t, s->soc[socket].num_harts * 4); + + for (cpu = s->soc[socket].num_harts - 1; cpu >= 0; cpu--) { + cpu_phandle = phandle++; - for (cpu = s->soc.num_harts - 1; cpu >= 0; cpu--) { - int cpu_phandle = phandle++; - int intc_phandle; - nodename = g_strdup_printf("/cpus/cpu@%d", cpu); - char *intc = g_strdup_printf("/cpus/cpu@%d/interrupt-controller", cpu); - char *isa = riscv_isa_string(&s->soc.harts[cpu]); - qemu_fdt_add_subnode(fdt, nodename); + cpu_name = g_strdup_printf("/cpus/cpu@%d", + s->soc[socket].hartid_base + cpu); + qemu_fdt_add_subnode(fdt, cpu_name); #if defined(TARGET_RISCV32) - qemu_fdt_setprop_string(fdt, nodename, "mmu-type", "riscv,sv32"); + qemu_fdt_setprop_string(fdt, cpu_name, "mmu-type", "riscv,sv32"); #else - qemu_fdt_setprop_string(fdt, nodename, "mmu-type", "riscv,sv48"); + qemu_fdt_setprop_string(fdt, cpu_name, "mmu-type", "riscv,sv48"); #endif - qemu_fdt_setprop_string(fdt, nodename, "riscv,isa", isa); - qemu_fdt_setprop_string(fdt, nodename, "compatible", "riscv"); - qemu_fdt_setprop_string(fdt, nodename, "status", "okay"); - qemu_fdt_setprop_cell(fdt, nodename, "reg", cpu); - qemu_fdt_setprop_string(fdt, nodename, "device_type", "cpu"); - qemu_fdt_setprop_cell(fdt, nodename, "phandle", cpu_phandle); - intc_phandle = phandle++; - qemu_fdt_add_subnode(fdt, intc); - qemu_fdt_setprop_cell(fdt, intc, "phandle", intc_phandle); - qemu_fdt_setprop_string(fdt, intc, "compatible", "riscv,cpu-intc"); - qemu_fdt_setprop(fdt, intc, "interrupt-controller", NULL, 0); - qemu_fdt_setprop_cell(fdt, intc, "#interrupt-cells", 1); - g_free(isa); - g_free(intc); - g_free(nodename); - } + name = riscv_isa_string(&s->soc[socket].harts[cpu]); + qemu_fdt_setprop_string(fdt, cpu_name, "riscv,isa", name); + g_free(name); + qemu_fdt_setprop_string(fdt, cpu_name, "compatible", "riscv"); + qemu_fdt_setprop_string(fdt, cpu_name, "status", "okay"); + qemu_fdt_setprop_cell(fdt, cpu_name, "reg", + s->soc[socket].hartid_base + cpu); + qemu_fdt_setprop_string(fdt, cpu_name, "device_type", "cpu"); + riscv_socket_fdt_write_id(mc, fdt, cpu_name, socket); + qemu_fdt_setprop_cell(fdt, cpu_name, "phandle", cpu_phandle); + + intc_name = g_strdup_printf("%s/interrupt-controller", cpu_name); + qemu_fdt_add_subnode(fdt, intc_name); + intc_phandle = phandle++; + qemu_fdt_setprop_cell(fdt, intc_name, "phandle", intc_phandle); + qemu_fdt_setprop_string(fdt, intc_name, "compatible", + "riscv,cpu-intc"); + qemu_fdt_setprop(fdt, intc_name, "interrupt-controller", NULL, 0); + qemu_fdt_setprop_cell(fdt, intc_name, "#interrupt-cells", 1); + + clint_cells[cpu * 4 + 0] = cpu_to_be32(intc_phandle); + clint_cells[cpu * 4 + 1] = cpu_to_be32(IRQ_M_SOFT); + clint_cells[cpu * 4 + 2] = cpu_to_be32(intc_phandle); + clint_cells[cpu * 4 + 3] = cpu_to_be32(IRQ_M_TIMER); + + plic_cells[cpu * 4 + 0] = cpu_to_be32(intc_phandle); + plic_cells[cpu * 4 + 1] = cpu_to_be32(IRQ_M_EXT); + plic_cells[cpu * 4 + 2] = cpu_to_be32(intc_phandle); + plic_cells[cpu * 4 + 3] = cpu_to_be32(IRQ_S_EXT); + + core_name = g_strdup_printf("%s/core%d", clust_name, cpu); + qemu_fdt_add_subnode(fdt, core_name); + qemu_fdt_setprop_cell(fdt, core_name, "cpu", cpu_phandle); + + g_free(core_name); + g_free(intc_name); + g_free(cpu_name); + } - /* Add cpu-topology node */ - qemu_fdt_add_subnode(fdt, "/cpus/cpu-map"); - qemu_fdt_add_subnode(fdt, "/cpus/cpu-map/cluster0"); - for (cpu = s->soc.num_harts - 1; cpu >= 0; cpu--) { - char *core_nodename = g_strdup_printf("/cpus/cpu-map/cluster0/core%d", - cpu); - char *cpu_nodename = g_strdup_printf("/cpus/cpu@%d", cpu); - uint32_t intc_phandle = qemu_fdt_get_phandle(fdt, cpu_nodename); - qemu_fdt_add_subnode(fdt, core_nodename); - qemu_fdt_setprop_cell(fdt, core_nodename, "cpu", intc_phandle); - g_free(core_nodename); - g_free(cpu_nodename); + addr = memmap[VIRT_DRAM].base + riscv_socket_mem_offset(mc, socket); + size = riscv_socket_mem_size(mc, socket); + mem_name = g_strdup_printf("/memory@%lx", (long)addr); + qemu_fdt_add_subnode(fdt, mem_name); + qemu_fdt_setprop_cells(fdt, mem_name, "reg", + addr >> 32, addr, size >> 32, size); + qemu_fdt_setprop_string(fdt, mem_name, "device_type", "memory"); + riscv_socket_fdt_write_id(mc, fdt, mem_name, socket); + g_free(mem_name); + + clint_addr = memmap[VIRT_CLINT].base + + (memmap[VIRT_CLINT].size * socket); + clint_name = g_strdup_printf("/soc/clint@%lx", clint_addr); + qemu_fdt_add_subnode(fdt, clint_name); + qemu_fdt_setprop_string(fdt, clint_name, "compatible", "riscv,clint0"); + qemu_fdt_setprop_cells(fdt, clint_name, "reg", + 0x0, clint_addr, 0x0, memmap[VIRT_CLINT].size); + qemu_fdt_setprop(fdt, clint_name, "interrupts-extended", + clint_cells, s->soc[socket].num_harts * sizeof(uint32_t) * 4); + riscv_socket_fdt_write_id(mc, fdt, clint_name, socket); + g_free(clint_name); + + plic_phandle[socket] = phandle++; + plic_addr = memmap[VIRT_PLIC].base + (memmap[VIRT_PLIC].size * socket); + plic_name = g_strdup_printf("/soc/plic@%lx", plic_addr); + qemu_fdt_add_subnode(fdt, plic_name); + qemu_fdt_setprop_cell(fdt, plic_name, + "#address-cells", FDT_PLIC_ADDR_CELLS); + qemu_fdt_setprop_cell(fdt, plic_name, + "#interrupt-cells", FDT_PLIC_INT_CELLS); + qemu_fdt_setprop_string(fdt, plic_name, "compatible", "riscv,plic0"); + qemu_fdt_setprop(fdt, plic_name, "interrupt-controller", NULL, 0); + qemu_fdt_setprop(fdt, plic_name, "interrupts-extended", + plic_cells, s->soc[socket].num_harts * sizeof(uint32_t) * 4); + qemu_fdt_setprop_cells(fdt, plic_name, "reg", + 0x0, plic_addr, 0x0, memmap[VIRT_PLIC].size); + qemu_fdt_setprop_cell(fdt, plic_name, "riscv,ndev", VIRTIO_NDEV); + riscv_socket_fdt_write_id(mc, fdt, plic_name, socket); + qemu_fdt_setprop_cell(fdt, plic_name, "phandle", plic_phandle[socket]); + g_free(plic_name); + + g_free(clint_cells); + g_free(plic_cells); + g_free(clust_name); } - cells = g_new0(uint32_t, s->soc.num_harts * 4); - for (cpu = 0; cpu < s->soc.num_harts; cpu++) { - nodename = - g_strdup_printf("/cpus/cpu@%d/interrupt-controller", cpu); - uint32_t intc_phandle = qemu_fdt_get_phandle(fdt, nodename); - cells[cpu * 4 + 0] = cpu_to_be32(intc_phandle); - cells[cpu * 4 + 1] = cpu_to_be32(IRQ_M_SOFT); - cells[cpu * 4 + 2] = cpu_to_be32(intc_phandle); - cells[cpu * 4 + 3] = cpu_to_be32(IRQ_M_TIMER); - g_free(nodename); - } - nodename = g_strdup_printf("/soc/clint@%lx", - (long)memmap[VIRT_CLINT].base); - qemu_fdt_add_subnode(fdt, nodename); - qemu_fdt_setprop_string(fdt, nodename, "compatible", "riscv,clint0"); - qemu_fdt_setprop_cells(fdt, nodename, "reg", - 0x0, memmap[VIRT_CLINT].base, - 0x0, memmap[VIRT_CLINT].size); - qemu_fdt_setprop(fdt, nodename, "interrupts-extended", - cells, s->soc.num_harts * sizeof(uint32_t) * 4); - g_free(cells); - g_free(nodename); - - plic_phandle = phandle++; - cells = g_new0(uint32_t, s->soc.num_harts * 4); - for (cpu = 0; cpu < s->soc.num_harts; cpu++) { - nodename = - g_strdup_printf("/cpus/cpu@%d/interrupt-controller", cpu); - uint32_t intc_phandle = qemu_fdt_get_phandle(fdt, nodename); - cells[cpu * 4 + 0] = cpu_to_be32(intc_phandle); - cells[cpu * 4 + 1] = cpu_to_be32(IRQ_M_EXT); - cells[cpu * 4 + 2] = cpu_to_be32(intc_phandle); - cells[cpu * 4 + 3] = cpu_to_be32(IRQ_S_EXT); - g_free(nodename); + for (socket = 0; socket < riscv_socket_count(mc); socket++) { + if (socket == 0) { + plic_mmio_phandle = plic_phandle[socket]; + plic_virtio_phandle = plic_phandle[socket]; + plic_pcie_phandle = plic_phandle[socket]; + } + if (socket == 1) { + plic_virtio_phandle = plic_phandle[socket]; + plic_pcie_phandle = plic_phandle[socket]; + } + if (socket == 2) { + plic_pcie_phandle = plic_phandle[socket]; + } } - nodename = g_strdup_printf("/soc/interrupt-controller@%lx", - (long)memmap[VIRT_PLIC].base); - qemu_fdt_add_subnode(fdt, nodename); - qemu_fdt_setprop_cell(fdt, nodename, "#address-cells", - FDT_PLIC_ADDR_CELLS); - qemu_fdt_setprop_cell(fdt, nodename, "#interrupt-cells", - FDT_PLIC_INT_CELLS); - qemu_fdt_setprop_string(fdt, nodename, "compatible", "riscv,plic0"); - qemu_fdt_setprop(fdt, nodename, "interrupt-controller", NULL, 0); - qemu_fdt_setprop(fdt, nodename, "interrupts-extended", - cells, s->soc.num_harts * sizeof(uint32_t) * 4); - qemu_fdt_setprop_cells(fdt, nodename, "reg", - 0x0, memmap[VIRT_PLIC].base, - 0x0, memmap[VIRT_PLIC].size); - qemu_fdt_setprop_cell(fdt, nodename, "riscv,ndev", VIRTIO_NDEV); - qemu_fdt_setprop_cell(fdt, nodename, "phandle", plic_phandle); - plic_phandle = qemu_fdt_get_phandle(fdt, nodename); - g_free(cells); - g_free(nodename); + + riscv_socket_fdt_write_distance_matrix(mc, fdt); for (i = 0; i < VIRTIO_COUNT; i++) { - nodename = g_strdup_printf("/virtio_mmio@%lx", + name = g_strdup_printf("/soc/virtio_mmio@%lx", (long)(memmap[VIRT_VIRTIO].base + i * memmap[VIRT_VIRTIO].size)); - qemu_fdt_add_subnode(fdt, nodename); - qemu_fdt_setprop_string(fdt, nodename, "compatible", "virtio,mmio"); - qemu_fdt_setprop_cells(fdt, nodename, "reg", + qemu_fdt_add_subnode(fdt, name); + qemu_fdt_setprop_string(fdt, name, "compatible", "virtio,mmio"); + qemu_fdt_setprop_cells(fdt, name, "reg", 0x0, memmap[VIRT_VIRTIO].base + i * memmap[VIRT_VIRTIO].size, 0x0, memmap[VIRT_VIRTIO].size); - qemu_fdt_setprop_cell(fdt, nodename, "interrupt-parent", plic_phandle); - qemu_fdt_setprop_cell(fdt, nodename, "interrupts", VIRTIO_IRQ + i); - g_free(nodename); + qemu_fdt_setprop_cell(fdt, name, "interrupt-parent", + plic_virtio_phandle); + qemu_fdt_setprop_cell(fdt, name, "interrupts", VIRTIO_IRQ + i); + g_free(name); } - nodename = g_strdup_printf("/soc/pci@%lx", + name = g_strdup_printf("/soc/pci@%lx", (long) memmap[VIRT_PCIE_ECAM].base); - qemu_fdt_add_subnode(fdt, nodename); - qemu_fdt_setprop_cell(fdt, nodename, "#address-cells", - FDT_PCI_ADDR_CELLS); - qemu_fdt_setprop_cell(fdt, nodename, "#interrupt-cells", - FDT_PCI_INT_CELLS); - qemu_fdt_setprop_cell(fdt, nodename, "#size-cells", 0x2); - qemu_fdt_setprop_string(fdt, nodename, "compatible", - "pci-host-ecam-generic"); - qemu_fdt_setprop_string(fdt, nodename, "device_type", "pci"); - qemu_fdt_setprop_cell(fdt, nodename, "linux,pci-domain", 0); - qemu_fdt_setprop_cells(fdt, nodename, "bus-range", 0, - memmap[VIRT_PCIE_ECAM].size / - PCIE_MMCFG_SIZE_MIN - 1); - qemu_fdt_setprop(fdt, nodename, "dma-coherent", NULL, 0); - qemu_fdt_setprop_cells(fdt, nodename, "reg", 0, memmap[VIRT_PCIE_ECAM].base, - 0, memmap[VIRT_PCIE_ECAM].size); - qemu_fdt_setprop_sized_cells(fdt, nodename, "ranges", + qemu_fdt_add_subnode(fdt, name); + qemu_fdt_setprop_cell(fdt, name, "#address-cells", FDT_PCI_ADDR_CELLS); + qemu_fdt_setprop_cell(fdt, name, "#interrupt-cells", FDT_PCI_INT_CELLS); + qemu_fdt_setprop_cell(fdt, name, "#size-cells", 0x2); + qemu_fdt_setprop_string(fdt, name, "compatible", "pci-host-ecam-generic"); + qemu_fdt_setprop_string(fdt, name, "device_type", "pci"); + qemu_fdt_setprop_cell(fdt, name, "linux,pci-domain", 0); + qemu_fdt_setprop_cells(fdt, name, "bus-range", 0, + memmap[VIRT_PCIE_ECAM].size / PCIE_MMCFG_SIZE_MIN - 1); + qemu_fdt_setprop(fdt, name, "dma-coherent", NULL, 0); + qemu_fdt_setprop_cells(fdt, name, "reg", 0, + memmap[VIRT_PCIE_ECAM].base, 0, memmap[VIRT_PCIE_ECAM].size); + qemu_fdt_setprop_sized_cells(fdt, name, "ranges", 1, FDT_PCI_RANGE_IOPORT, 2, 0, 2, memmap[VIRT_PCIE_PIO].base, 2, memmap[VIRT_PCIE_PIO].size, 1, FDT_PCI_RANGE_MMIO, 2, memmap[VIRT_PCIE_MMIO].base, 2, memmap[VIRT_PCIE_MMIO].base, 2, memmap[VIRT_PCIE_MMIO].size); - create_pcie_irq_map(fdt, nodename, plic_phandle); - g_free(nodename); + create_pcie_irq_map(fdt, name, plic_pcie_phandle); + g_free(name); test_phandle = phandle++; - nodename = g_strdup_printf("/test@%lx", + name = g_strdup_printf("/soc/test@%lx", (long)memmap[VIRT_TEST].base); - qemu_fdt_add_subnode(fdt, nodename); + qemu_fdt_add_subnode(fdt, name); { const char compat[] = "sifive,test1\0sifive,test0\0syscon"; - qemu_fdt_setprop(fdt, nodename, "compatible", compat, sizeof(compat)); + qemu_fdt_setprop(fdt, name, "compatible", compat, sizeof(compat)); } - qemu_fdt_setprop_cells(fdt, nodename, "reg", + qemu_fdt_setprop_cells(fdt, name, "reg", 0x0, memmap[VIRT_TEST].base, 0x0, memmap[VIRT_TEST].size); - qemu_fdt_setprop_cell(fdt, nodename, "phandle", test_phandle); - test_phandle = qemu_fdt_get_phandle(fdt, nodename); - g_free(nodename); - - nodename = g_strdup_printf("/reboot"); - qemu_fdt_add_subnode(fdt, nodename); - qemu_fdt_setprop_string(fdt, nodename, "compatible", "syscon-reboot"); - qemu_fdt_setprop_cell(fdt, nodename, "regmap", test_phandle); - qemu_fdt_setprop_cell(fdt, nodename, "offset", 0x0); - qemu_fdt_setprop_cell(fdt, nodename, "value", FINISHER_RESET); - g_free(nodename); - - nodename = g_strdup_printf("/poweroff"); - qemu_fdt_add_subnode(fdt, nodename); - qemu_fdt_setprop_string(fdt, nodename, "compatible", "syscon-poweroff"); - qemu_fdt_setprop_cell(fdt, nodename, "regmap", test_phandle); - qemu_fdt_setprop_cell(fdt, nodename, "offset", 0x0); - qemu_fdt_setprop_cell(fdt, nodename, "value", FINISHER_PASS); - g_free(nodename); - - nodename = g_strdup_printf("/uart@%lx", - (long)memmap[VIRT_UART0].base); - qemu_fdt_add_subnode(fdt, nodename); - qemu_fdt_setprop_string(fdt, nodename, "compatible", "ns16550a"); - qemu_fdt_setprop_cells(fdt, nodename, "reg", + qemu_fdt_setprop_cell(fdt, name, "phandle", test_phandle); + test_phandle = qemu_fdt_get_phandle(fdt, name); + g_free(name); + + name = g_strdup_printf("/soc/reboot"); + qemu_fdt_add_subnode(fdt, name); + qemu_fdt_setprop_string(fdt, name, "compatible", "syscon-reboot"); + qemu_fdt_setprop_cell(fdt, name, "regmap", test_phandle); + qemu_fdt_setprop_cell(fdt, name, "offset", 0x0); + qemu_fdt_setprop_cell(fdt, name, "value", FINISHER_RESET); + g_free(name); + + name = g_strdup_printf("/soc/poweroff"); + qemu_fdt_add_subnode(fdt, name); + qemu_fdt_setprop_string(fdt, name, "compatible", "syscon-poweroff"); + qemu_fdt_setprop_cell(fdt, name, "regmap", test_phandle); + qemu_fdt_setprop_cell(fdt, name, "offset", 0x0); + qemu_fdt_setprop_cell(fdt, name, "value", FINISHER_PASS); + g_free(name); + + name = g_strdup_printf("/soc/uart@%lx", (long)memmap[VIRT_UART0].base); + qemu_fdt_add_subnode(fdt, name); + qemu_fdt_setprop_string(fdt, name, "compatible", "ns16550a"); + qemu_fdt_setprop_cells(fdt, name, "reg", 0x0, memmap[VIRT_UART0].base, 0x0, memmap[VIRT_UART0].size); - qemu_fdt_setprop_cell(fdt, nodename, "clock-frequency", 3686400); - qemu_fdt_setprop_cell(fdt, nodename, "interrupt-parent", plic_phandle); - qemu_fdt_setprop_cell(fdt, nodename, "interrupts", UART0_IRQ); + qemu_fdt_setprop_cell(fdt, name, "clock-frequency", 3686400); + qemu_fdt_setprop_cell(fdt, name, "interrupt-parent", plic_mmio_phandle); + qemu_fdt_setprop_cell(fdt, name, "interrupts", UART0_IRQ); qemu_fdt_add_subnode(fdt, "/chosen"); - qemu_fdt_setprop_string(fdt, "/chosen", "stdout-path", nodename); + qemu_fdt_setprop_string(fdt, "/chosen", "stdout-path", name); if (cmdline) { qemu_fdt_setprop_string(fdt, "/chosen", "bootargs", cmdline); } - g_free(nodename); - - nodename = g_strdup_printf("/rtc@%lx", - (long)memmap[VIRT_RTC].base); - qemu_fdt_add_subnode(fdt, nodename); - qemu_fdt_setprop_string(fdt, nodename, "compatible", - "google,goldfish-rtc"); - qemu_fdt_setprop_cells(fdt, nodename, "reg", + g_free(name); + + name = g_strdup_printf("/soc/rtc@%lx", (long)memmap[VIRT_RTC].base); + qemu_fdt_add_subnode(fdt, name); + qemu_fdt_setprop_string(fdt, name, "compatible", "google,goldfish-rtc"); + qemu_fdt_setprop_cells(fdt, name, "reg", 0x0, memmap[VIRT_RTC].base, 0x0, memmap[VIRT_RTC].size); - qemu_fdt_setprop_cell(fdt, nodename, "interrupt-parent", plic_phandle); - qemu_fdt_setprop_cell(fdt, nodename, "interrupts", RTC_IRQ); - g_free(nodename); - - nodename = g_strdup_printf("/flash@%" PRIx64, flashbase); - qemu_fdt_add_subnode(s->fdt, nodename); - qemu_fdt_setprop_string(s->fdt, nodename, "compatible", "cfi-flash"); - qemu_fdt_setprop_sized_cells(s->fdt, nodename, "reg", + qemu_fdt_setprop_cell(fdt, name, "interrupt-parent", plic_mmio_phandle); + qemu_fdt_setprop_cell(fdt, name, "interrupts", RTC_IRQ); + g_free(name); + + name = g_strdup_printf("/soc/flash@%" PRIx64, flashbase); + qemu_fdt_add_subnode(s->fdt, name); + qemu_fdt_setprop_string(s->fdt, name, "compatible", "cfi-flash"); + qemu_fdt_setprop_sized_cells(s->fdt, name, "reg", 2, flashbase, 2, flashsize, 2, flashbase + flashsize, 2, flashsize); - qemu_fdt_setprop_cell(s->fdt, nodename, "bank-width", 4); - g_free(nodename); + qemu_fdt_setprop_cell(s->fdt, name, "bank-width", 4); + g_free(name); } - static inline DeviceState *gpex_pcie_init(MemoryRegion *sys_mem, hwaddr ecam_base, hwaddr ecam_size, hwaddr mmio_base, hwaddr mmio_size, @@ -478,21 +493,100 @@ static void riscv_virt_board_init(MachineState *machine) MemoryRegion *system_memory = get_system_memory(); MemoryRegion *main_mem = g_new(MemoryRegion, 1); MemoryRegion *mask_rom = g_new(MemoryRegion, 1); - char *plic_hart_config; + char *plic_hart_config, *soc_name; size_t plic_hart_config_len; target_ulong start_addr = memmap[VIRT_DRAM].base; - int i; - unsigned int smp_cpus = machine->smp.cpus; - - /* Initialize SOC */ - object_initialize_child(OBJECT(machine), "soc", &s->soc, sizeof(s->soc), - TYPE_RISCV_HART_ARRAY, &error_abort, NULL); - object_property_set_str(OBJECT(&s->soc), machine->cpu_type, "cpu-type", - &error_abort); - object_property_set_int(OBJECT(&s->soc), smp_cpus, "num-harts", - &error_abort); - object_property_set_bool(OBJECT(&s->soc), true, "realized", - &error_abort); + DeviceState *mmio_plic, *virtio_plic, *pcie_plic; + int i, j, base_hartid, hart_count; + + /* Check socket count limit */ + if (VIRT_SOCKETS_MAX < riscv_socket_count(machine)) { + error_report("number of sockets/nodes should be less than %d", + VIRT_SOCKETS_MAX); + exit(1); + } + + /* Initialize sockets */ + mmio_plic = virtio_plic = pcie_plic = NULL; + for (i = 0; i < riscv_socket_count(machine); i++) { + if (!riscv_socket_check_hartids(machine, i)) { + error_report("discontinuous hartids in socket%d", i); + exit(1); + } + + base_hartid = riscv_socket_first_hartid(machine, i); + if (base_hartid < 0) { + error_report("can't find hartid base for socket%d", i); + exit(1); + } + + hart_count = riscv_socket_hart_count(machine, i); + if (hart_count < 0) { + error_report("can't find hart count for socket%d", i); + exit(1); + } + + soc_name = g_strdup_printf("soc%d", i); + object_initialize_child(OBJECT(machine), soc_name, &s->soc[i], + sizeof(s->soc[i]), TYPE_RISCV_HART_ARRAY, &error_abort, NULL); + g_free(soc_name); + object_property_set_str(OBJECT(&s->soc[i]), + machine->cpu_type, "cpu-type", &error_abort); + object_property_set_int(OBJECT(&s->soc[i]), + base_hartid, "hartid-base", &error_abort); + object_property_set_int(OBJECT(&s->soc[i]), + hart_count, "num-harts", &error_abort); + object_property_set_bool(OBJECT(&s->soc[i]), + true, "realized", &error_abort); + + /* Per-socket CLINT */ + sifive_clint_create( + memmap[VIRT_CLINT].base + i * memmap[VIRT_CLINT].size, + memmap[VIRT_CLINT].size, base_hartid, hart_count, + SIFIVE_SIP_BASE, SIFIVE_TIMECMP_BASE, SIFIVE_TIME_BASE, true); + + /* Per-socket PLIC hart topology configuration string */ + plic_hart_config_len = + (strlen(VIRT_PLIC_HART_CONFIG) + 1) * hart_count; + plic_hart_config = g_malloc0(plic_hart_config_len); + for (j = 0; j < hart_count; j++) { + if (j != 0) { + strncat(plic_hart_config, ",", plic_hart_config_len); + } + strncat(plic_hart_config, VIRT_PLIC_HART_CONFIG, + plic_hart_config_len); + plic_hart_config_len -= (strlen(VIRT_PLIC_HART_CONFIG) + 1); + } + + /* Per-socket PLIC */ + s->plic[i] = sifive_plic_create( + memmap[VIRT_PLIC].base + i * memmap[VIRT_PLIC].size, + plic_hart_config, base_hartid, + VIRT_PLIC_NUM_SOURCES, + VIRT_PLIC_NUM_PRIORITIES, + VIRT_PLIC_PRIORITY_BASE, + VIRT_PLIC_PENDING_BASE, + VIRT_PLIC_ENABLE_BASE, + VIRT_PLIC_ENABLE_STRIDE, + VIRT_PLIC_CONTEXT_BASE, + VIRT_PLIC_CONTEXT_STRIDE, + memmap[VIRT_PLIC].size); + g_free(plic_hart_config); + + /* Try to use different PLIC instance based device type */ + if (i == 0) { + mmio_plic = s->plic[i]; + virtio_plic = s->plic[i]; + pcie_plic = s->plic[i]; + } + if (i == 1) { + virtio_plic = s->plic[i]; + pcie_plic = s->plic[i]; + } + if (i == 2) { + pcie_plic = s->plic[i]; + } + } /* register system main memory (actual RAM) */ memory_region_init_ram(main_mem, NULL, "riscv_virt_board.ram", @@ -571,38 +665,14 @@ static void riscv_virt_board_init(MachineState *machine) memmap[VIRT_MROM].base + sizeof(reset_vec), &address_space_memory); - /* create PLIC hart topology configuration string */ - plic_hart_config_len = (strlen(VIRT_PLIC_HART_CONFIG) + 1) * smp_cpus; - plic_hart_config = g_malloc0(plic_hart_config_len); - for (i = 0; i < smp_cpus; i++) { - if (i != 0) { - strncat(plic_hart_config, ",", plic_hart_config_len); - } - strncat(plic_hart_config, VIRT_PLIC_HART_CONFIG, plic_hart_config_len); - plic_hart_config_len -= (strlen(VIRT_PLIC_HART_CONFIG) + 1); - } - - /* MMIO */ - s->plic = sifive_plic_create(memmap[VIRT_PLIC].base, - plic_hart_config, 0, - VIRT_PLIC_NUM_SOURCES, - VIRT_PLIC_NUM_PRIORITIES, - VIRT_PLIC_PRIORITY_BASE, - VIRT_PLIC_PENDING_BASE, - VIRT_PLIC_ENABLE_BASE, - VIRT_PLIC_ENABLE_STRIDE, - VIRT_PLIC_CONTEXT_BASE, - VIRT_PLIC_CONTEXT_STRIDE, - memmap[VIRT_PLIC].size); - sifive_clint_create(memmap[VIRT_CLINT].base, - memmap[VIRT_CLINT].size, 0, smp_cpus, - SIFIVE_SIP_BASE, SIFIVE_TIMECMP_BASE, SIFIVE_TIME_BASE, true); + /* SiFive Test MMIO device */ sifive_test_create(memmap[VIRT_TEST].base); + /* VirtIO MMIO devices */ for (i = 0; i < VIRTIO_COUNT; i++) { sysbus_create_simple("virtio-mmio", memmap[VIRT_VIRTIO].base + i * memmap[VIRT_VIRTIO].size, - qdev_get_gpio_in(DEVICE(s->plic), VIRTIO_IRQ + i)); + qdev_get_gpio_in(DEVICE(virtio_plic), VIRTIO_IRQ + i)); } gpex_pcie_init(system_memory, @@ -611,14 +681,14 @@ static void riscv_virt_board_init(MachineState *machine) memmap[VIRT_PCIE_MMIO].base, memmap[VIRT_PCIE_MMIO].size, memmap[VIRT_PCIE_PIO].base, - DEVICE(s->plic), true); + DEVICE(pcie_plic), true); serial_mm_init(system_memory, memmap[VIRT_UART0].base, - 0, qdev_get_gpio_in(DEVICE(s->plic), UART0_IRQ), 399193, + 0, qdev_get_gpio_in(DEVICE(mmio_plic), UART0_IRQ), 399193, serial_hd(0), DEVICE_LITTLE_ENDIAN); sysbus_create_simple("goldfish_rtc", memmap[VIRT_RTC].base, - qdev_get_gpio_in(DEVICE(s->plic), RTC_IRQ)); + qdev_get_gpio_in(DEVICE(mmio_plic), RTC_IRQ)); virt_flash_create(s); @@ -628,8 +698,6 @@ static void riscv_virt_board_init(MachineState *machine) drive_get(IF_PFLASH, 0, i)); } virt_flash_map(s, system_memory); - - g_free(plic_hart_config); } static void riscv_virt_machine_instance_init(Object *obj) @@ -642,9 +710,13 @@ static void riscv_virt_machine_class_init(ObjectClass *oc, void *data) mc->desc = "RISC-V VirtIO board"; mc->init = riscv_virt_board_init; - mc->max_cpus = 8; + mc->max_cpus = VIRT_CPUS_MAX; mc->default_cpu_type = VIRT_CPU; mc->pci_allow_0_address = true; + mc->possible_cpu_arch_ids = riscv_numa_possible_cpu_arch_ids; + mc->cpu_index_to_instance_props = riscv_numa_cpu_index_to_props; + mc->get_default_cpu_node_id = riscv_numa_get_default_cpu_node_id; + mc->numa_mem_supported = true; } static const TypeInfo riscv_virt_machine_typeinfo = { diff --git a/include/hw/riscv/virt.h b/include/hw/riscv/virt.h index e69355efaf..1beacd7666 100644 --- a/include/hw/riscv/virt.h +++ b/include/hw/riscv/virt.h @@ -23,6 +23,9 @@ #include "hw/sysbus.h" #include "hw/block/flash.h" +#define VIRT_CPUS_MAX 8 +#define VIRT_SOCKETS_MAX 8 + #define TYPE_RISCV_VIRT_MACHINE MACHINE_TYPE_NAME("virt") #define RISCV_VIRT_MACHINE(obj) \ OBJECT_CHECK(RISCVVirtState, (obj), TYPE_RISCV_VIRT_MACHINE) @@ -32,8 +35,8 @@ typedef struct { MachineState parent; /*< public >*/ - RISCVHartArrayState soc; - DeviceState *plic; + RISCVHartArrayState soc[VIRT_SOCKETS_MAX]; + DeviceState *plic[VIRT_SOCKETS_MAX]; PFlashCFI01 *flash[2]; void *fdt; @@ -74,6 +77,8 @@ enum { #define VIRT_PLIC_ENABLE_STRIDE 0x80 #define VIRT_PLIC_CONTEXT_BASE 0x200000 #define VIRT_PLIC_CONTEXT_STRIDE 0x1000 +#define VIRT_PLIC_SIZE(__num_context) \ + (VIRT_PLIC_CONTEXT_BASE + (__num_context) * VIRT_PLIC_CONTEXT_STRIDE) #define FDT_PCI_ADDR_CELLS 3 #define FDT_PCI_INT_CELLS 1
We extend RISC-V virt machine to allow creating a multi-socket machine. Each RISC-V virt machine socket is a NUMA node having a set of HARTs, a memory instance, a CLINT instance, and a PLIC instance. Other devices are shared between all sockets. We also update the generated device tree accordingly. By default, NUMA multi-socket support is disabled for RISC-V virt machine. To enable it, users can use "-numa" command-line options of QEMU. Example1: For two NUMA nodes with 2 CPUs each, append following to command-line options: "-smp 4 -numa node -numa node" Example2: For two NUMA nodes with 1 and 3 CPUs, append following to command-line options: "-smp 4 -numa node -numa node -numa cpu,node-id=0,core-id=0 \ -numa cpu,node-id=1,core-id=1 -numa cpu,node-id=1,core-id=2 \ -numa cpu,node-id=1,core-id=3" The maximum number of sockets in a RISC-V virt machine is 8 but this limit can be changed in future. Signed-off-by: Anup Patel <anup.patel@wdc.com> --- hw/riscv/virt.c | 530 +++++++++++++++++++++++----------------- include/hw/riscv/virt.h | 9 +- 2 files changed, 308 insertions(+), 231 deletions(-)