| Message ID | 1527680741-2725-6-git-send-email-eric.auger@redhat.com |
|---|---|
| State | New |
| Headers | show |
| Series | KVM/ARM: Relax the max 123 vcpus limitation along with KVM GICv3 | expand |
On Wed, 30 May 2018 13:45:38 +0200 Eric Auger <eric.auger@redhat.com> wrote: > This patch allows the creation of a GICv3 node with 1 or 2 > redistributor regions depending on the number of smu_cpus. > The second redistributor region is located just after the > existing RAM region, at 256GB and contains up to up to 512 vcpus. > > Please refer to kernel documentation for further node details: > Documentation/devicetree/bindings/interrupt-controller/arm,gic-v3.txt > > Signed-off-by: Eric Auger <eric.auger@redhat.com> > > --- > > v2 -> v3: > - VIRT_GIC_REDIST2 is now 64MB large, ie. 512 redistributor capacity > - virt_gicv3_redist_region_count does not test kvm_irqchip_in_kernel > anymore > --- > hw/arm/virt.c | 29 ++++++++++++++++++++++++----- > include/hw/arm/virt.h | 14 ++++++++++++++ > 2 files changed, 38 insertions(+), 5 deletions(-) > > diff --git a/hw/arm/virt.c b/hw/arm/virt.c > index ed79460..3018ec2 100644 > --- a/hw/arm/virt.c > +++ b/hw/arm/virt.c > @@ -149,6 +149,8 @@ static const MemMapEntry a15memmap[] = { > [VIRT_PCIE_PIO] = { 0x3eff0000, 0x00010000 }, > [VIRT_PCIE_ECAM] = { 0x3f000000, 0x01000000 }, > [VIRT_MEM] = { 0x40000000, RAMLIMIT_BYTES }, > + /* Additional 64 MB redist region (can contain up to 512 redistributors) */ > + [VIRT_GIC_REDIST2] = { 0x4000000000ULL, 0x4000000ULL }, could it be placed after VIRT_PCIE_MMIO_HIGH, so we would have some space here to increase RAM without creating the second RAM region upto 512GB boundary? > /* Second PCIe window, 512GB wide at the 512GB boundary */ > [VIRT_PCIE_MMIO_HIGH] = { 0x8000000000ULL, 0x8000000000ULL }, > }; > @@ -402,13 +404,30 @@ static void fdt_add_gic_node(VirtMachineState *vms) > qemu_fdt_setprop_cell(vms->fdt, "/intc", "#size-cells", 0x2); > qemu_fdt_setprop(vms->fdt, "/intc", "ranges", NULL, 0); > if (vms->gic_version == 3) { > + int nb_redist_regions = virt_gicv3_redist_region_count(vms); > + > qemu_fdt_setprop_string(vms->fdt, "/intc", "compatible", > "arm,gic-v3"); > - qemu_fdt_setprop_sized_cells(vms->fdt, "/intc", "reg", > - 2, vms->memmap[VIRT_GIC_DIST].base, > - 2, vms->memmap[VIRT_GIC_DIST].size, > - 2, vms->memmap[VIRT_GIC_REDIST].base, > - 2, vms->memmap[VIRT_GIC_REDIST].size); > + > + qemu_fdt_setprop_cell(vms->fdt, "/intc", > + "#redistributor-regions", nb_redist_regions); > + > + if (nb_redist_regions == 1) { > + qemu_fdt_setprop_sized_cells(vms->fdt, "/intc", "reg", > + 2, vms->memmap[VIRT_GIC_DIST].base, > + 2, vms->memmap[VIRT_GIC_DIST].size, > + 2, vms->memmap[VIRT_GIC_REDIST].base, > + 2, vms->memmap[VIRT_GIC_REDIST].size); > + } else { > + qemu_fdt_setprop_sized_cells(vms->fdt, "/intc", "reg", > + 2, vms->memmap[VIRT_GIC_DIST].base, > + 2, vms->memmap[VIRT_GIC_DIST].size, > + 2, vms->memmap[VIRT_GIC_REDIST].base, > + 2, vms->memmap[VIRT_GIC_REDIST].size, > + 2, vms->memmap[VIRT_GIC_REDIST2].base, > + 2, vms->memmap[VIRT_GIC_REDIST2].size); > + } > + > if (vms->virt) { > qemu_fdt_setprop_cells(vms->fdt, "/intc", "interrupts", > GIC_FDT_IRQ_TYPE_PPI, ARCH_GICV3_MAINT_IRQ, > diff --git a/include/hw/arm/virt.h b/include/hw/arm/virt.h > index 4ac7ef6..308156f 100644 > --- a/include/hw/arm/virt.h > +++ b/include/hw/arm/virt.h > @@ -35,6 +35,8 @@ > #include "qemu/notify.h" > #include "hw/boards.h" > #include "hw/arm/arm.h" > +#include "sysemu/kvm.h" > +#include "hw/intc/arm_gicv3_common.h" > > #define NUM_GICV2M_SPIS 64 > #define NUM_VIRTIO_TRANSPORTS 32 > @@ -60,6 +62,7 @@ enum { > VIRT_GIC_V2M, > VIRT_GIC_ITS, > VIRT_GIC_REDIST, > + VIRT_GIC_REDIST2, > VIRT_SMMU, > VIRT_UART, > VIRT_MMIO, > @@ -130,4 +133,15 @@ typedef struct { > > void virt_acpi_setup(VirtMachineState *vms); > > +/* Return the number of used redistributor regions */ > +static inline int virt_gicv3_redist_region_count(VirtMachineState *vms) > +{ > + uint32_t redist0_capacity = > + vms->memmap[VIRT_GIC_REDIST].size / GICV3_REDIST_SIZE; > + > + assert(vms->gic_version == 3); > + > + return vms->smp_cpus > redist0_capacity ? 2 : 1; > +} > + > #endif /* QEMU_ARM_VIRT_H */
Hi Igor, On 05/30/2018 02:13 PM, Igor Mammedov wrote: > On Wed, 30 May 2018 13:45:38 +0200 > Eric Auger <eric.auger@redhat.com> wrote: > >> This patch allows the creation of a GICv3 node with 1 or 2 >> redistributor regions depending on the number of smu_cpus. >> The second redistributor region is located just after the >> existing RAM region, at 256GB and contains up to up to 512 vcpus. >> >> Please refer to kernel documentation for further node details: >> Documentation/devicetree/bindings/interrupt-controller/arm,gic-v3.txt >> >> Signed-off-by: Eric Auger <eric.auger@redhat.com> >> >> --- >> >> v2 -> v3: >> - VIRT_GIC_REDIST2 is now 64MB large, ie. 512 redistributor capacity >> - virt_gicv3_redist_region_count does not test kvm_irqchip_in_kernel >> anymore >> --- >> hw/arm/virt.c | 29 ++++++++++++++++++++++++----- >> include/hw/arm/virt.h | 14 ++++++++++++++ >> 2 files changed, 38 insertions(+), 5 deletions(-) >> >> diff --git a/hw/arm/virt.c b/hw/arm/virt.c >> index ed79460..3018ec2 100644 >> --- a/hw/arm/virt.c >> +++ b/hw/arm/virt.c >> @@ -149,6 +149,8 @@ static const MemMapEntry a15memmap[] = { >> [VIRT_PCIE_PIO] = { 0x3eff0000, 0x00010000 }, >> [VIRT_PCIE_ECAM] = { 0x3f000000, 0x01000000 }, >> [VIRT_MEM] = { 0x40000000, RAMLIMIT_BYTES }, >> + /* Additional 64 MB redist region (can contain up to 512 redistributors) */ >> + [VIRT_GIC_REDIST2] = { 0x4000000000ULL, 0x4000000ULL }, > could it be placed after VIRT_PCIE_MMIO_HIGH, > so we would have some space here to increase RAM without > creating the second RAM region upto 512GB boundary? Personally I don't have any objection but see my reply to Shannon's similar query in http://lists.gnu.org/archive/html/qemu-arm/2018-03/msg00465.html * If we need to provide more RAM to VMs in the future then we need to: * * allocate a second bank of RAM starting at 2TB and working up * * fix the DT and ACPI table generation code in QEMU to correctly * report two split lumps of RAM to the guest * * fix KVM in the host kernel to allow guests with >40 bit address spaces * (We don't want to fill all the way up to 512GB with RAM because * we might want it for non-RAM purposes later. Thanks Eric > >> /* Second PCIe window, 512GB wide at the 512GB boundary */ >> [VIRT_PCIE_MMIO_HIGH] = { 0x8000000000ULL, 0x8000000000ULL }, >> }; >> @@ -402,13 +404,30 @@ static void fdt_add_gic_node(VirtMachineState *vms) >> qemu_fdt_setprop_cell(vms->fdt, "/intc", "#size-cells", 0x2); >> qemu_fdt_setprop(vms->fdt, "/intc", "ranges", NULL, 0); >> if (vms->gic_version == 3) { >> + int nb_redist_regions = virt_gicv3_redist_region_count(vms); >> + >> qemu_fdt_setprop_string(vms->fdt, "/intc", "compatible", >> "arm,gic-v3"); >> - qemu_fdt_setprop_sized_cells(vms->fdt, "/intc", "reg", >> - 2, vms->memmap[VIRT_GIC_DIST].base, >> - 2, vms->memmap[VIRT_GIC_DIST].size, >> - 2, vms->memmap[VIRT_GIC_REDIST].base, >> - 2, vms->memmap[VIRT_GIC_REDIST].size); >> + >> + qemu_fdt_setprop_cell(vms->fdt, "/intc", >> + "#redistributor-regions", nb_redist_regions); >> + >> + if (nb_redist_regions == 1) { >> + qemu_fdt_setprop_sized_cells(vms->fdt, "/intc", "reg", >> + 2, vms->memmap[VIRT_GIC_DIST].base, >> + 2, vms->memmap[VIRT_GIC_DIST].size, >> + 2, vms->memmap[VIRT_GIC_REDIST].base, >> + 2, vms->memmap[VIRT_GIC_REDIST].size); >> + } else { >> + qemu_fdt_setprop_sized_cells(vms->fdt, "/intc", "reg", >> + 2, vms->memmap[VIRT_GIC_DIST].base, >> + 2, vms->memmap[VIRT_GIC_DIST].size, >> + 2, vms->memmap[VIRT_GIC_REDIST].base, >> + 2, vms->memmap[VIRT_GIC_REDIST].size, >> + 2, vms->memmap[VIRT_GIC_REDIST2].base, >> + 2, vms->memmap[VIRT_GIC_REDIST2].size); >> + } >> + >> if (vms->virt) { >> qemu_fdt_setprop_cells(vms->fdt, "/intc", "interrupts", >> GIC_FDT_IRQ_TYPE_PPI, ARCH_GICV3_MAINT_IRQ, >> diff --git a/include/hw/arm/virt.h b/include/hw/arm/virt.h >> index 4ac7ef6..308156f 100644 >> --- a/include/hw/arm/virt.h >> +++ b/include/hw/arm/virt.h >> @@ -35,6 +35,8 @@ >> #include "qemu/notify.h" >> #include "hw/boards.h" >> #include "hw/arm/arm.h" >> +#include "sysemu/kvm.h" >> +#include "hw/intc/arm_gicv3_common.h" >> >> #define NUM_GICV2M_SPIS 64 >> #define NUM_VIRTIO_TRANSPORTS 32 >> @@ -60,6 +62,7 @@ enum { >> VIRT_GIC_V2M, >> VIRT_GIC_ITS, >> VIRT_GIC_REDIST, >> + VIRT_GIC_REDIST2, >> VIRT_SMMU, >> VIRT_UART, >> VIRT_MMIO, >> @@ -130,4 +133,15 @@ typedef struct { >> >> void virt_acpi_setup(VirtMachineState *vms); >> >> +/* Return the number of used redistributor regions */ >> +static inline int virt_gicv3_redist_region_count(VirtMachineState *vms) >> +{ >> + uint32_t redist0_capacity = >> + vms->memmap[VIRT_GIC_REDIST].size / GICV3_REDIST_SIZE; >> + >> + assert(vms->gic_version == 3); >> + >> + return vms->smp_cpus > redist0_capacity ? 2 : 1; >> +} >> + >> #endif /* QEMU_ARM_VIRT_H */ >
diff --git a/hw/arm/virt.c b/hw/arm/virt.c index ed79460..3018ec2 100644 --- a/hw/arm/virt.c +++ b/hw/arm/virt.c @@ -149,6 +149,8 @@ static const MemMapEntry a15memmap[] = { [VIRT_PCIE_PIO] = { 0x3eff0000, 0x00010000 }, [VIRT_PCIE_ECAM] = { 0x3f000000, 0x01000000 }, [VIRT_MEM] = { 0x40000000, RAMLIMIT_BYTES }, + /* Additional 64 MB redist region (can contain up to 512 redistributors) */ + [VIRT_GIC_REDIST2] = { 0x4000000000ULL, 0x4000000ULL }, /* Second PCIe window, 512GB wide at the 512GB boundary */ [VIRT_PCIE_MMIO_HIGH] = { 0x8000000000ULL, 0x8000000000ULL }, }; @@ -402,13 +404,30 @@ static void fdt_add_gic_node(VirtMachineState *vms) qemu_fdt_setprop_cell(vms->fdt, "/intc", "#size-cells", 0x2); qemu_fdt_setprop(vms->fdt, "/intc", "ranges", NULL, 0); if (vms->gic_version == 3) { + int nb_redist_regions = virt_gicv3_redist_region_count(vms); + qemu_fdt_setprop_string(vms->fdt, "/intc", "compatible", "arm,gic-v3"); - qemu_fdt_setprop_sized_cells(vms->fdt, "/intc", "reg", - 2, vms->memmap[VIRT_GIC_DIST].base, - 2, vms->memmap[VIRT_GIC_DIST].size, - 2, vms->memmap[VIRT_GIC_REDIST].base, - 2, vms->memmap[VIRT_GIC_REDIST].size); + + qemu_fdt_setprop_cell(vms->fdt, "/intc", + "#redistributor-regions", nb_redist_regions); + + if (nb_redist_regions == 1) { + qemu_fdt_setprop_sized_cells(vms->fdt, "/intc", "reg", + 2, vms->memmap[VIRT_GIC_DIST].base, + 2, vms->memmap[VIRT_GIC_DIST].size, + 2, vms->memmap[VIRT_GIC_REDIST].base, + 2, vms->memmap[VIRT_GIC_REDIST].size); + } else { + qemu_fdt_setprop_sized_cells(vms->fdt, "/intc", "reg", + 2, vms->memmap[VIRT_GIC_DIST].base, + 2, vms->memmap[VIRT_GIC_DIST].size, + 2, vms->memmap[VIRT_GIC_REDIST].base, + 2, vms->memmap[VIRT_GIC_REDIST].size, + 2, vms->memmap[VIRT_GIC_REDIST2].base, + 2, vms->memmap[VIRT_GIC_REDIST2].size); + } + if (vms->virt) { qemu_fdt_setprop_cells(vms->fdt, "/intc", "interrupts", GIC_FDT_IRQ_TYPE_PPI, ARCH_GICV3_MAINT_IRQ, diff --git a/include/hw/arm/virt.h b/include/hw/arm/virt.h index 4ac7ef6..308156f 100644 --- a/include/hw/arm/virt.h +++ b/include/hw/arm/virt.h @@ -35,6 +35,8 @@ #include "qemu/notify.h" #include "hw/boards.h" #include "hw/arm/arm.h" +#include "sysemu/kvm.h" +#include "hw/intc/arm_gicv3_common.h" #define NUM_GICV2M_SPIS 64 #define NUM_VIRTIO_TRANSPORTS 32 @@ -60,6 +62,7 @@ enum { VIRT_GIC_V2M, VIRT_GIC_ITS, VIRT_GIC_REDIST, + VIRT_GIC_REDIST2, VIRT_SMMU, VIRT_UART, VIRT_MMIO, @@ -130,4 +133,15 @@ typedef struct { void virt_acpi_setup(VirtMachineState *vms); +/* Return the number of used redistributor regions */ +static inline int virt_gicv3_redist_region_count(VirtMachineState *vms) +{ + uint32_t redist0_capacity = + vms->memmap[VIRT_GIC_REDIST].size / GICV3_REDIST_SIZE; + + assert(vms->gic_version == 3); + + return vms->smp_cpus > redist0_capacity ? 2 : 1; +} + #endif /* QEMU_ARM_VIRT_H */
This patch allows the creation of a GICv3 node with 1 or 2 redistributor regions depending on the number of smu_cpus. The second redistributor region is located just after the existing RAM region, at 256GB and contains up to up to 512 vcpus. Please refer to kernel documentation for further node details: Documentation/devicetree/bindings/interrupt-controller/arm,gic-v3.txt Signed-off-by: Eric Auger <eric.auger@redhat.com> --- v2 -> v3: - VIRT_GIC_REDIST2 is now 64MB large, ie. 512 redistributor capacity - virt_gicv3_redist_region_count does not test kvm_irqchip_in_kernel anymore --- hw/arm/virt.c | 29 ++++++++++++++++++++++++----- include/hw/arm/virt.h | 14 ++++++++++++++ 2 files changed, 38 insertions(+), 5 deletions(-)