| Message ID | 1427117764-23008-20-git-send-email-bharata@linux.vnet.ibm.com |
|---|---|
| State | New |
| Headers | show |
On Mon, Mar 23, 2015 at 07:06:00PM +0530, Bharata B Rao wrote: > Support hot removal of CPU for sPAPR guests by sending the hot > unplug notification to the guest via EPOW interrupt. > > Signed-off-by: Bharata B Rao <bharata@linux.vnet.ibm.com> > --- > hw/ppc/spapr.c | 78 ++++++++++++++++++++++++++++++++++++++++++++++- > linux-headers/linux/kvm.h | 1 + > target-ppc/kvm.c | 7 +++++ > target-ppc/kvm_ppc.h | 6 ++++ > 4 files changed, 91 insertions(+), 1 deletion(-) > > diff --git a/hw/ppc/spapr.c b/hw/ppc/spapr.c > index b48994b..7b8784d 100644 > --- a/hw/ppc/spapr.c > +++ b/hw/ppc/spapr.c > @@ -1468,6 +1468,12 @@ static void spapr_cpu_init(PowerPCCPU *cpu) > qemu_register_reset(spapr_cpu_reset, cpu); > } > > +static void spapr_cpu_destroy(PowerPCCPU *cpu) > +{ > + xics_cpu_destroy(spapr->icp, cpu); > + qemu_unregister_reset(spapr_cpu_reset, cpu); > +} > + > /* pSeries LPAR / sPAPR hardware init */ > static void ppc_spapr_init(MachineState *machine) > { > @@ -1880,6 +1886,18 @@ static void spapr_cpu_hotplug_add(DeviceState *dev, CPUState *cs, Error **errp) > } > } > > +static void spapr_cpu_hotplug_remove(DeviceState *dev, CPUState *cs, > + Error **errp) > +{ > + PowerPCCPU *cpu = POWERPC_CPU(cs); > + int id = ppc_get_vcpu_dt_id(cpu); > + sPAPRDRConnector *drc = > + spapr_dr_connector_by_id(SPAPR_DR_CONNECTOR_TYPE_CPU, id); > + sPAPRDRConnectorClass *drck = SPAPR_DR_CONNECTOR_GET_CLASS(drc); > + > + drck->detach(drc, dev, NULL, NULL, errp); > +} > + > static void spapr_cpu_plug(HotplugHandler *hotplug_dev, DeviceState *dev, > Error **errp) > { > @@ -1911,6 +1929,51 @@ static void spapr_cpu_plug(HotplugHandler *hotplug_dev, DeviceState *dev, > return; > } > > +static int spapr_cpu_unplug(Object *obj, void *opaque) > +{ > + Error **errp = opaque; > + DeviceState *dev = DEVICE(obj); > + CPUState *cs = CPU(dev); > + PowerPCCPU *cpu = POWERPC_CPU(cs); > + int id = ppc_get_vcpu_dt_id(cpu); > + int smt = kvmppc_smt_threads(); > + sPAPRDRConnector *drc = > + spapr_dr_connector_by_id(SPAPR_DR_CONNECTOR_TYPE_CPU, id); > + > + spapr_cpu_destroy(cpu); I may be misunderstanding something here, but don't you need to signal the removal to the guest (and get its ack) before you "physically" remove the cpu? > + /* > + * SMT threads return from here, only main thread (core) will > + * continue and signal hot unplug event to the guest. > + */ > + if ((id % smt) != 0) { > + return 0; > + } > + g_assert(drc); > + > + spapr_cpu_hotplug_remove(dev, cs, errp); > + if (*errp) { > + return -1; > + } > + spapr_hotplug_req_remove_event(drc); > + > + return 0; > +} > + > +static int spapr_cpu_core_unplug(Object *obj, void *opaque) > +{ > + Error **errp = opaque; > + > + object_child_foreach(obj, spapr_cpu_unplug, errp); > + return 0; > +} > + > +static void spapr_cpu_socket_unplug(HotplugHandler *hotplug_dev, > + DeviceState *dev, Error **errp) > +{ > + object_child_foreach(OBJECT(dev), spapr_cpu_core_unplug, errp); > +} > + > static void spapr_machine_device_plug(HotplugHandler *hotplug_dev, > DeviceState *dev, Error **errp) > { > @@ -1926,10 +1989,21 @@ static void spapr_machine_device_plug(HotplugHandler *hotplug_dev, > } > } > > +static void spapr_machine_device_unplug(HotplugHandler *hotplug_dev, > + DeviceState *dev, Error **errp) > +{ > + if (object_dynamic_cast(OBJECT(dev), TYPE_CPU_SOCKET)) { > + if (dev->hotplugged && spapr->dr_cpu_enabled) { > + spapr_cpu_socket_unplug(hotplug_dev, dev, errp); > + } > + } > +} > + > static HotplugHandler *spapr_get_hotpug_handler(MachineState *machine, > DeviceState *dev) > { > - if (object_dynamic_cast(OBJECT(dev), TYPE_CPU)) { > + if (object_dynamic_cast(OBJECT(dev), TYPE_CPU) || > + object_dynamic_cast(OBJECT(dev), TYPE_CPU_SOCKET)) { > return HOTPLUG_HANDLER(machine); > } > return NULL; > @@ -1953,6 +2027,8 @@ static void spapr_machine_class_init(ObjectClass *oc, void *data) > mc->has_dynamic_sysbus = true; > mc->get_hotplug_handler = spapr_get_hotpug_handler; > hc->plug = spapr_machine_device_plug; > + hc->unplug = spapr_machine_device_unplug; > + > smc->dr_phb_enabled = false; > smc->dr_cpu_enabled = false; > smc->dr_lmb_enabled = false; > diff --git a/linux-headers/linux/kvm.h b/linux-headers/linux/kvm.h > index 12045a1..0c1be07 100644 > --- a/linux-headers/linux/kvm.h > +++ b/linux-headers/linux/kvm.h > @@ -761,6 +761,7 @@ struct kvm_ppc_smmu_info { > #define KVM_CAP_PPC_FIXUP_HCALL 103 > #define KVM_CAP_PPC_ENABLE_HCALL 104 > #define KVM_CAP_CHECK_EXTENSION_VM 105 > +#define KVM_CAP_SPAPR_REUSE_VCPU 107 Updates to linux-headers/ are usually put into their own patch for safety (along with an indication of what kernel commit the additions came from). > #ifdef KVM_CAP_IRQ_ROUTING > > diff --git a/target-ppc/kvm.c b/target-ppc/kvm.c > index 1edf2b5..ee23bf6 100644 > --- a/target-ppc/kvm.c > +++ b/target-ppc/kvm.c > @@ -72,6 +72,7 @@ static int cap_ppc_watchdog; > static int cap_papr; > static int cap_htab_fd; > static int cap_fixup_hcalls; > +static int cap_spapr_reuse_vcpu; AFAICT nothing in this patch checks this variable. Does it belong in this patch? > static uint32_t debug_inst_opcode; > > @@ -114,6 +115,7 @@ int kvm_arch_init(KVMState *s) > * only activated after this by kvmppc_set_papr() */ > cap_htab_fd = kvm_check_extension(s, KVM_CAP_PPC_HTAB_FD); > cap_fixup_hcalls = kvm_check_extension(s, KVM_CAP_PPC_FIXUP_HCALL); > + cap_spapr_reuse_vcpu = kvm_check_extension(s, KVM_CAP_SPAPR_REUSE_VCPU); > > if (!cap_interrupt_level) { > fprintf(stderr, "KVM: Couldn't find level irq capability. Expect the " > @@ -2408,3 +2410,8 @@ int kvm_arch_fixup_msi_route(struct kvm_irq_routing_entry *route, > { > return 0; > } > + > +bool kvmppc_spapr_reuse_vcpu(void) > +{ > + return cap_spapr_reuse_vcpu; > +} > diff --git a/target-ppc/kvm_ppc.h b/target-ppc/kvm_ppc.h > index 2e0224c..c831229 100644 > --- a/target-ppc/kvm_ppc.h > +++ b/target-ppc/kvm_ppc.h > @@ -40,6 +40,7 @@ void *kvmppc_create_spapr_tce(uint32_t liobn, uint32_t window_size, int *pfd, > int kvmppc_remove_spapr_tce(void *table, int pfd, uint32_t window_size); > int kvmppc_reset_htab(int shift_hint); > uint64_t kvmppc_rma_size(uint64_t current_size, unsigned int hash_shift); > +bool kvmppc_spapr_reuse_vcpu(void); > #endif /* !CONFIG_USER_ONLY */ > bool kvmppc_has_cap_epr(void); > int kvmppc_define_rtas_kernel_token(uint32_t token, const char *function); > @@ -185,6 +186,11 @@ static inline int kvmppc_update_sdr1(CPUPPCState *env) > return 0; > } > > +static inline bool kvmppc_spapr_reuse_vcpu(void) > +{ > + return false; > +} > + > #endif /* !CONFIG_USER_ONLY */ > > static inline bool kvmppc_has_cap_epr(void)
On Wed, Mar 25, 2015 at 04:44:48PM +1100, David Gibson wrote: > On Mon, Mar 23, 2015 at 07:06:00PM +0530, Bharata B Rao wrote: > > Support hot removal of CPU for sPAPR guests by sending the hot > > unplug notification to the guest via EPOW interrupt. > > > > Signed-off-by: Bharata B Rao <bharata@linux.vnet.ibm.com> > > --- > > hw/ppc/spapr.c | 78 ++++++++++++++++++++++++++++++++++++++++++++++- > > linux-headers/linux/kvm.h | 1 + > > target-ppc/kvm.c | 7 +++++ > > target-ppc/kvm_ppc.h | 6 ++++ > > 4 files changed, 91 insertions(+), 1 deletion(-) > > > > diff --git a/hw/ppc/spapr.c b/hw/ppc/spapr.c > > index b48994b..7b8784d 100644 > > --- a/hw/ppc/spapr.c > > +++ b/hw/ppc/spapr.c > > @@ -1468,6 +1468,12 @@ static void spapr_cpu_init(PowerPCCPU *cpu) > > qemu_register_reset(spapr_cpu_reset, cpu); > > } > > > > +static void spapr_cpu_destroy(PowerPCCPU *cpu) > > +{ > > + xics_cpu_destroy(spapr->icp, cpu); > > + qemu_unregister_reset(spapr_cpu_reset, cpu); > > +} > > + > > /* pSeries LPAR / sPAPR hardware init */ > > static void ppc_spapr_init(MachineState *machine) > > { > > @@ -1880,6 +1886,18 @@ static void spapr_cpu_hotplug_add(DeviceState *dev, CPUState *cs, Error **errp) > > } > > } > > > > +static void spapr_cpu_hotplug_remove(DeviceState *dev, CPUState *cs, > > + Error **errp) > > +{ > > + PowerPCCPU *cpu = POWERPC_CPU(cs); > > + int id = ppc_get_vcpu_dt_id(cpu); > > + sPAPRDRConnector *drc = > > + spapr_dr_connector_by_id(SPAPR_DR_CONNECTOR_TYPE_CPU, id); > > + sPAPRDRConnectorClass *drck = SPAPR_DR_CONNECTOR_GET_CLASS(drc); > > + > > + drck->detach(drc, dev, NULL, NULL, errp); > > +} > > + > > static void spapr_cpu_plug(HotplugHandler *hotplug_dev, DeviceState *dev, > > Error **errp) > > { > > @@ -1911,6 +1929,51 @@ static void spapr_cpu_plug(HotplugHandler *hotplug_dev, DeviceState *dev, > > return; > > } > > > > +static int spapr_cpu_unplug(Object *obj, void *opaque) > > +{ > > + Error **errp = opaque; > > + DeviceState *dev = DEVICE(obj); > > + CPUState *cs = CPU(dev); > > + PowerPCCPU *cpu = POWERPC_CPU(cs); > > + int id = ppc_get_vcpu_dt_id(cpu); > > + int smt = kvmppc_smt_threads(); > > + sPAPRDRConnector *drc = > > + spapr_dr_connector_by_id(SPAPR_DR_CONNECTOR_TYPE_CPU, id); > > + > > + spapr_cpu_destroy(cpu); > > I may be misunderstanding something here, but don't you need to signal > the removal to the guest (and get its ack) before you "physically" > remove the cpu? Oh yes, I should move this call to spapr_cpu_release(). > > > + /* > > + * SMT threads return from here, only main thread (core) will > > + * continue and signal hot unplug event to the guest. > > + */ > > + if ((id % smt) != 0) { > > + return 0; > > + } > > + g_assert(drc); > > + > > + spapr_cpu_hotplug_remove(dev, cs, errp); > > + if (*errp) { > > + return -1; > > + } > > + spapr_hotplug_req_remove_event(drc); > > + > > + return 0; > > +} > > + > > +static int spapr_cpu_core_unplug(Object *obj, void *opaque) > > +{ > > + Error **errp = opaque; > > + > > + object_child_foreach(obj, spapr_cpu_unplug, errp); > > + return 0; > > +} > > + > > +static void spapr_cpu_socket_unplug(HotplugHandler *hotplug_dev, > > + DeviceState *dev, Error **errp) > > +{ > > + object_child_foreach(OBJECT(dev), spapr_cpu_core_unplug, errp); > > +} > > + > > static void spapr_machine_device_plug(HotplugHandler *hotplug_dev, > > DeviceState *dev, Error **errp) > > { > > @@ -1926,10 +1989,21 @@ static void spapr_machine_device_plug(HotplugHandler *hotplug_dev, > > } > > } > > > > +static void spapr_machine_device_unplug(HotplugHandler *hotplug_dev, > > + DeviceState *dev, Error **errp) > > +{ > > + if (object_dynamic_cast(OBJECT(dev), TYPE_CPU_SOCKET)) { > > + if (dev->hotplugged && spapr->dr_cpu_enabled) { > > + spapr_cpu_socket_unplug(hotplug_dev, dev, errp); > > + } > > + } > > +} > > + > > static HotplugHandler *spapr_get_hotpug_handler(MachineState *machine, > > DeviceState *dev) > > { > > - if (object_dynamic_cast(OBJECT(dev), TYPE_CPU)) { > > + if (object_dynamic_cast(OBJECT(dev), TYPE_CPU) || > > + object_dynamic_cast(OBJECT(dev), TYPE_CPU_SOCKET)) { > > return HOTPLUG_HANDLER(machine); > > } > > return NULL; > > @@ -1953,6 +2027,8 @@ static void spapr_machine_class_init(ObjectClass *oc, void *data) > > mc->has_dynamic_sysbus = true; > > mc->get_hotplug_handler = spapr_get_hotpug_handler; > > hc->plug = spapr_machine_device_plug; > > + hc->unplug = spapr_machine_device_unplug; > > + > > smc->dr_phb_enabled = false; > > smc->dr_cpu_enabled = false; > > smc->dr_lmb_enabled = false; > > diff --git a/linux-headers/linux/kvm.h b/linux-headers/linux/kvm.h > > index 12045a1..0c1be07 100644 > > --- a/linux-headers/linux/kvm.h > > +++ b/linux-headers/linux/kvm.h > > @@ -761,6 +761,7 @@ struct kvm_ppc_smmu_info { > > #define KVM_CAP_PPC_FIXUP_HCALL 103 > > #define KVM_CAP_PPC_ENABLE_HCALL 104 > > #define KVM_CAP_CHECK_EXTENSION_VM 105 > > +#define KVM_CAP_SPAPR_REUSE_VCPU 107 > > Updates to linux-headers/ are usually put into their own patch for > safety (along with an indication of what kernel commit the additions > came from). > > > #ifdef KVM_CAP_IRQ_ROUTING > > > > diff --git a/target-ppc/kvm.c b/target-ppc/kvm.c > > index 1edf2b5..ee23bf6 100644 > > --- a/target-ppc/kvm.c > > +++ b/target-ppc/kvm.c > > @@ -72,6 +72,7 @@ static int cap_ppc_watchdog; > > static int cap_papr; > > static int cap_htab_fd; > > static int cap_fixup_hcalls; > > +static int cap_spapr_reuse_vcpu; > > AFAICT nothing in this patch checks this variable. Does it belong in > this patch? This is some remnant of an old approach I was trying. It got left out by mistake and sorry that you had to review this part. I will clean this up in the next post. Regards, Bharata.
On 03/24/2015 12:36 AM, Bharata B Rao wrote: > Support hot removal of CPU for sPAPR guests by sending the hot > unplug notification to the guest via EPOW interrupt. > > Signed-off-by: Bharata B Rao <bharata@linux.vnet.ibm.com> > --- > hw/ppc/spapr.c | 78 ++++++++++++++++++++++++++++++++++++++++++++++- > linux-headers/linux/kvm.h | 1 + > target-ppc/kvm.c | 7 +++++ > target-ppc/kvm_ppc.h | 6 ++++ > 4 files changed, 91 insertions(+), 1 deletion(-) > > diff --git a/hw/ppc/spapr.c b/hw/ppc/spapr.c > index b48994b..7b8784d 100644 > --- a/hw/ppc/spapr.c > +++ b/hw/ppc/spapr.c > @@ -1468,6 +1468,12 @@ static void spapr_cpu_init(PowerPCCPU *cpu) > qemu_register_reset(spapr_cpu_reset, cpu); > } > > +static void spapr_cpu_destroy(PowerPCCPU *cpu) > +{ > + xics_cpu_destroy(spapr->icp, cpu); > + qemu_unregister_reset(spapr_cpu_reset, cpu); > +} > + > /* pSeries LPAR / sPAPR hardware init */ > static void ppc_spapr_init(MachineState *machine) > { > @@ -1880,6 +1886,18 @@ static void spapr_cpu_hotplug_add(DeviceState *dev, CPUState *cs, Error **errp) > } > } > > +static void spapr_cpu_hotplug_remove(DeviceState *dev, CPUState *cs, > + Error **errp) > +{ > + PowerPCCPU *cpu = POWERPC_CPU(cs); > + int id = ppc_get_vcpu_dt_id(cpu); > + sPAPRDRConnector *drc = > + spapr_dr_connector_by_id(SPAPR_DR_CONNECTOR_TYPE_CPU, id); > + sPAPRDRConnectorClass *drck = SPAPR_DR_CONNECTOR_GET_CLASS(drc); > + > + drck->detach(drc, dev, NULL, NULL, errp); > +} > + > static void spapr_cpu_plug(HotplugHandler *hotplug_dev, DeviceState *dev, > Error **errp) > { > @@ -1911,6 +1929,51 @@ static void spapr_cpu_plug(HotplugHandler *hotplug_dev, DeviceState *dev, > return; > } > > +static int spapr_cpu_unplug(Object *obj, void *opaque) > +{ > + Error **errp = opaque; > + DeviceState *dev = DEVICE(obj); > + CPUState *cs = CPU(dev); > + PowerPCCPU *cpu = POWERPC_CPU(cs); > + int id = ppc_get_vcpu_dt_id(cpu); > + int smt = kvmppc_smt_threads(); > + sPAPRDRConnector *drc = > + spapr_dr_connector_by_id(SPAPR_DR_CONNECTOR_TYPE_CPU, id); > + > + spapr_cpu_destroy(cpu); > + > + /* > + * SMT threads return from here, only main thread (core) will > + * continue and signal hot unplug event to the guest. > + */ > + if ((id % smt) != 0) { > + return 0; > + } > + g_assert(drc); > + > + spapr_cpu_hotplug_remove(dev, cs, errp); > + if (*errp) { > + return -1; > + } > + spapr_hotplug_req_remove_event(drc); > + > + return 0; > +} > + > +static int spapr_cpu_core_unplug(Object *obj, void *opaque) > +{ > + Error **errp = opaque; > + > + object_child_foreach(obj, spapr_cpu_unplug, errp); > + return 0; > +} > + > +static void spapr_cpu_socket_unplug(HotplugHandler *hotplug_dev, > + DeviceState *dev, Error **errp) > +{ > + object_child_foreach(OBJECT(dev), spapr_cpu_core_unplug, errp); > +} > + > static void spapr_machine_device_plug(HotplugHandler *hotplug_dev, > DeviceState *dev, Error **errp) > { > @@ -1926,10 +1989,21 @@ static void spapr_machine_device_plug(HotplugHandler *hotplug_dev, > } > } > > +static void spapr_machine_device_unplug(HotplugHandler *hotplug_dev, > + DeviceState *dev, Error **errp) > +{ > + if (object_dynamic_cast(OBJECT(dev), TYPE_CPU_SOCKET)) { > + if (dev->hotplugged && spapr->dr_cpu_enabled) { > + spapr_cpu_socket_unplug(hotplug_dev, dev, errp); > + } > + } > +} > + > static HotplugHandler *spapr_get_hotpug_handler(MachineState *machine, > DeviceState *dev) > { > - if (object_dynamic_cast(OBJECT(dev), TYPE_CPU)) { > + if (object_dynamic_cast(OBJECT(dev), TYPE_CPU) || > + object_dynamic_cast(OBJECT(dev), TYPE_CPU_SOCKET)) { What is this change for? I mean why is not it always socket-only? Commit log would not hurt here... > return HOTPLUG_HANDLER(machine); > } > return NULL; > @@ -1953,6 +2027,8 @@ static void spapr_machine_class_init(ObjectClass *oc, void *data) > mc->has_dynamic_sysbus = true; > mc->get_hotplug_handler = spapr_get_hotpug_handler; > hc->plug = spapr_machine_device_plug; > + hc->unplug = spapr_machine_device_unplug; > + > smc->dr_phb_enabled = false; > smc->dr_cpu_enabled = false; > smc->dr_lmb_enabled = false; > diff --git a/linux-headers/linux/kvm.h b/linux-headers/linux/kvm.h > index 12045a1..0c1be07 100644 > --- a/linux-headers/linux/kvm.h > +++ b/linux-headers/linux/kvm.h > @@ -761,6 +761,7 @@ struct kvm_ppc_smmu_info { > #define KVM_CAP_PPC_FIXUP_HCALL 103 > #define KVM_CAP_PPC_ENABLE_HCALL 104 > #define KVM_CAP_CHECK_EXTENSION_VM 105 > +#define KVM_CAP_SPAPR_REUSE_VCPU 107 > > #ifdef KVM_CAP_IRQ_ROUTING > > diff --git a/target-ppc/kvm.c b/target-ppc/kvm.c > index 1edf2b5..ee23bf6 100644 > --- a/target-ppc/kvm.c > +++ b/target-ppc/kvm.c > @@ -72,6 +72,7 @@ static int cap_ppc_watchdog; > static int cap_papr; > static int cap_htab_fd; > static int cap_fixup_hcalls; > +static int cap_spapr_reuse_vcpu; > > static uint32_t debug_inst_opcode; > > @@ -114,6 +115,7 @@ int kvm_arch_init(KVMState *s) > * only activated after this by kvmppc_set_papr() */ > cap_htab_fd = kvm_check_extension(s, KVM_CAP_PPC_HTAB_FD); > cap_fixup_hcalls = kvm_check_extension(s, KVM_CAP_PPC_FIXUP_HCALL); > + cap_spapr_reuse_vcpu = kvm_check_extension(s, KVM_CAP_SPAPR_REUSE_VCPU); > > if (!cap_interrupt_level) { > fprintf(stderr, "KVM: Couldn't find level irq capability. Expect the " > @@ -2408,3 +2410,8 @@ int kvm_arch_fixup_msi_route(struct kvm_irq_routing_entry *route, > { > return 0; > } > + > +bool kvmppc_spapr_reuse_vcpu(void) > +{ > + return cap_spapr_reuse_vcpu; > +} > diff --git a/target-ppc/kvm_ppc.h b/target-ppc/kvm_ppc.h > index 2e0224c..c831229 100644 > --- a/target-ppc/kvm_ppc.h > +++ b/target-ppc/kvm_ppc.h > @@ -40,6 +40,7 @@ void *kvmppc_create_spapr_tce(uint32_t liobn, uint32_t window_size, int *pfd, > int kvmppc_remove_spapr_tce(void *table, int pfd, uint32_t window_size); > int kvmppc_reset_htab(int shift_hint); > uint64_t kvmppc_rma_size(uint64_t current_size, unsigned int hash_shift); > +bool kvmppc_spapr_reuse_vcpu(void); > #endif /* !CONFIG_USER_ONLY */ > bool kvmppc_has_cap_epr(void); > int kvmppc_define_rtas_kernel_token(uint32_t token, const char *function); > @@ -185,6 +186,11 @@ static inline int kvmppc_update_sdr1(CPUPPCState *env) > return 0; > } > > +static inline bool kvmppc_spapr_reuse_vcpu(void) > +{ > + return false; > +} > + > #endif /* !CONFIG_USER_ONLY */ > > static inline bool kvmppc_has_cap_epr(void) >
On Tue, Apr 07, 2015 at 04:45:17PM +1000, Alexey Kardashevskiy wrote: > On 03/24/2015 12:36 AM, Bharata B Rao wrote: > >Support hot removal of CPU for sPAPR guests by sending the hot > >unplug notification to the guest via EPOW interrupt. > > > >Signed-off-by: Bharata B Rao <bharata@linux.vnet.ibm.com> > >--- > > hw/ppc/spapr.c | 78 ++++++++++++++++++++++++++++++++++++++++++++++- > > linux-headers/linux/kvm.h | 1 + > > target-ppc/kvm.c | 7 +++++ > > target-ppc/kvm_ppc.h | 6 ++++ > > 4 files changed, 91 insertions(+), 1 deletion(-) > > > >diff --git a/hw/ppc/spapr.c b/hw/ppc/spapr.c > >index b48994b..7b8784d 100644 > >--- a/hw/ppc/spapr.c > >+++ b/hw/ppc/spapr.c > >@@ -1468,6 +1468,12 @@ static void spapr_cpu_init(PowerPCCPU *cpu) > > qemu_register_reset(spapr_cpu_reset, cpu); > > } > > > >+static void spapr_cpu_destroy(PowerPCCPU *cpu) > >+{ > >+ xics_cpu_destroy(spapr->icp, cpu); > >+ qemu_unregister_reset(spapr_cpu_reset, cpu); > >+} > >+ > > /* pSeries LPAR / sPAPR hardware init */ > > static void ppc_spapr_init(MachineState *machine) > > { > >@@ -1880,6 +1886,18 @@ static void spapr_cpu_hotplug_add(DeviceState *dev, CPUState *cs, Error **errp) > > } > > } > > > >+static void spapr_cpu_hotplug_remove(DeviceState *dev, CPUState *cs, > >+ Error **errp) > >+{ > >+ PowerPCCPU *cpu = POWERPC_CPU(cs); > >+ int id = ppc_get_vcpu_dt_id(cpu); > >+ sPAPRDRConnector *drc = > >+ spapr_dr_connector_by_id(SPAPR_DR_CONNECTOR_TYPE_CPU, id); > >+ sPAPRDRConnectorClass *drck = SPAPR_DR_CONNECTOR_GET_CLASS(drc); > >+ > >+ drck->detach(drc, dev, NULL, NULL, errp); > >+} > >+ > > static void spapr_cpu_plug(HotplugHandler *hotplug_dev, DeviceState *dev, > > Error **errp) > > { > >@@ -1911,6 +1929,51 @@ static void spapr_cpu_plug(HotplugHandler *hotplug_dev, DeviceState *dev, > > return; > > } > > > >+static int spapr_cpu_unplug(Object *obj, void *opaque) > >+{ > >+ Error **errp = opaque; > >+ DeviceState *dev = DEVICE(obj); > >+ CPUState *cs = CPU(dev); > >+ PowerPCCPU *cpu = POWERPC_CPU(cs); > >+ int id = ppc_get_vcpu_dt_id(cpu); > >+ int smt = kvmppc_smt_threads(); > >+ sPAPRDRConnector *drc = > >+ spapr_dr_connector_by_id(SPAPR_DR_CONNECTOR_TYPE_CPU, id); > >+ > >+ spapr_cpu_destroy(cpu); > >+ > >+ /* > >+ * SMT threads return from here, only main thread (core) will > >+ * continue and signal hot unplug event to the guest. > >+ */ > >+ if ((id % smt) != 0) { > >+ return 0; > >+ } > >+ g_assert(drc); > >+ > >+ spapr_cpu_hotplug_remove(dev, cs, errp); > >+ if (*errp) { > >+ return -1; > >+ } > >+ spapr_hotplug_req_remove_event(drc); > >+ > >+ return 0; > >+} > >+ > >+static int spapr_cpu_core_unplug(Object *obj, void *opaque) > >+{ > >+ Error **errp = opaque; > >+ > >+ object_child_foreach(obj, spapr_cpu_unplug, errp); > >+ return 0; > >+} > >+ > >+static void spapr_cpu_socket_unplug(HotplugHandler *hotplug_dev, > >+ DeviceState *dev, Error **errp) > >+{ > >+ object_child_foreach(OBJECT(dev), spapr_cpu_core_unplug, errp); > >+} > >+ > > static void spapr_machine_device_plug(HotplugHandler *hotplug_dev, > > DeviceState *dev, Error **errp) > > { > >@@ -1926,10 +1989,21 @@ static void spapr_machine_device_plug(HotplugHandler *hotplug_dev, > > } > > } > > > >+static void spapr_machine_device_unplug(HotplugHandler *hotplug_dev, > >+ DeviceState *dev, Error **errp) > >+{ > >+ if (object_dynamic_cast(OBJECT(dev), TYPE_CPU_SOCKET)) { > >+ if (dev->hotplugged && spapr->dr_cpu_enabled) { > >+ spapr_cpu_socket_unplug(hotplug_dev, dev, errp); > >+ } > >+ } > >+} > >+ > > static HotplugHandler *spapr_get_hotpug_handler(MachineState *machine, > > DeviceState *dev) > > { > >- if (object_dynamic_cast(OBJECT(dev), TYPE_CPU)) { > >+ if (object_dynamic_cast(OBJECT(dev), TYPE_CPU) || > >+ object_dynamic_cast(OBJECT(dev), TYPE_CPU_SOCKET)) { > > > What is this change for? I mean why is not it always socket-only? Commit log > would not hurt here... In the hot add case (do_device_add), the CPU socket device is realized first which will realize the CPU core devices. Core devices will realize the CPU thread devices. So the ->plug() operation happens as part of CPU thread devices and hence hotplug_handler is returned only for TYPE_CPU. However in case of hot remove, qdev_unplug() directly does ->unplug() and hence I need to return the hotplug_handler for TYPE_CPU_SOCKET also. This ensures that ->unplug() gets called for socket object where I take care of recursively walking down the core and thread objects and unplugging the CPU thread object eventually. Regards, Bharata.
diff --git a/hw/ppc/spapr.c b/hw/ppc/spapr.c index b48994b..7b8784d 100644 --- a/hw/ppc/spapr.c +++ b/hw/ppc/spapr.c @@ -1468,6 +1468,12 @@ static void spapr_cpu_init(PowerPCCPU *cpu) qemu_register_reset(spapr_cpu_reset, cpu); } +static void spapr_cpu_destroy(PowerPCCPU *cpu) +{ + xics_cpu_destroy(spapr->icp, cpu); + qemu_unregister_reset(spapr_cpu_reset, cpu); +} + /* pSeries LPAR / sPAPR hardware init */ static void ppc_spapr_init(MachineState *machine) { @@ -1880,6 +1886,18 @@ static void spapr_cpu_hotplug_add(DeviceState *dev, CPUState *cs, Error **errp) } } +static void spapr_cpu_hotplug_remove(DeviceState *dev, CPUState *cs, + Error **errp) +{ + PowerPCCPU *cpu = POWERPC_CPU(cs); + int id = ppc_get_vcpu_dt_id(cpu); + sPAPRDRConnector *drc = + spapr_dr_connector_by_id(SPAPR_DR_CONNECTOR_TYPE_CPU, id); + sPAPRDRConnectorClass *drck = SPAPR_DR_CONNECTOR_GET_CLASS(drc); + + drck->detach(drc, dev, NULL, NULL, errp); +} + static void spapr_cpu_plug(HotplugHandler *hotplug_dev, DeviceState *dev, Error **errp) { @@ -1911,6 +1929,51 @@ static void spapr_cpu_plug(HotplugHandler *hotplug_dev, DeviceState *dev, return; } +static int spapr_cpu_unplug(Object *obj, void *opaque) +{ + Error **errp = opaque; + DeviceState *dev = DEVICE(obj); + CPUState *cs = CPU(dev); + PowerPCCPU *cpu = POWERPC_CPU(cs); + int id = ppc_get_vcpu_dt_id(cpu); + int smt = kvmppc_smt_threads(); + sPAPRDRConnector *drc = + spapr_dr_connector_by_id(SPAPR_DR_CONNECTOR_TYPE_CPU, id); + + spapr_cpu_destroy(cpu); + + /* + * SMT threads return from here, only main thread (core) will + * continue and signal hot unplug event to the guest. + */ + if ((id % smt) != 0) { + return 0; + } + g_assert(drc); + + spapr_cpu_hotplug_remove(dev, cs, errp); + if (*errp) { + return -1; + } + spapr_hotplug_req_remove_event(drc); + + return 0; +} + +static int spapr_cpu_core_unplug(Object *obj, void *opaque) +{ + Error **errp = opaque; + + object_child_foreach(obj, spapr_cpu_unplug, errp); + return 0; +} + +static void spapr_cpu_socket_unplug(HotplugHandler *hotplug_dev, + DeviceState *dev, Error **errp) +{ + object_child_foreach(OBJECT(dev), spapr_cpu_core_unplug, errp); +} + static void spapr_machine_device_plug(HotplugHandler *hotplug_dev, DeviceState *dev, Error **errp) { @@ -1926,10 +1989,21 @@ static void spapr_machine_device_plug(HotplugHandler *hotplug_dev, } } +static void spapr_machine_device_unplug(HotplugHandler *hotplug_dev, + DeviceState *dev, Error **errp) +{ + if (object_dynamic_cast(OBJECT(dev), TYPE_CPU_SOCKET)) { + if (dev->hotplugged && spapr->dr_cpu_enabled) { + spapr_cpu_socket_unplug(hotplug_dev, dev, errp); + } + } +} + static HotplugHandler *spapr_get_hotpug_handler(MachineState *machine, DeviceState *dev) { - if (object_dynamic_cast(OBJECT(dev), TYPE_CPU)) { + if (object_dynamic_cast(OBJECT(dev), TYPE_CPU) || + object_dynamic_cast(OBJECT(dev), TYPE_CPU_SOCKET)) { return HOTPLUG_HANDLER(machine); } return NULL; @@ -1953,6 +2027,8 @@ static void spapr_machine_class_init(ObjectClass *oc, void *data) mc->has_dynamic_sysbus = true; mc->get_hotplug_handler = spapr_get_hotpug_handler; hc->plug = spapr_machine_device_plug; + hc->unplug = spapr_machine_device_unplug; + smc->dr_phb_enabled = false; smc->dr_cpu_enabled = false; smc->dr_lmb_enabled = false; diff --git a/linux-headers/linux/kvm.h b/linux-headers/linux/kvm.h index 12045a1..0c1be07 100644 --- a/linux-headers/linux/kvm.h +++ b/linux-headers/linux/kvm.h @@ -761,6 +761,7 @@ struct kvm_ppc_smmu_info { #define KVM_CAP_PPC_FIXUP_HCALL 103 #define KVM_CAP_PPC_ENABLE_HCALL 104 #define KVM_CAP_CHECK_EXTENSION_VM 105 +#define KVM_CAP_SPAPR_REUSE_VCPU 107 #ifdef KVM_CAP_IRQ_ROUTING diff --git a/target-ppc/kvm.c b/target-ppc/kvm.c index 1edf2b5..ee23bf6 100644 --- a/target-ppc/kvm.c +++ b/target-ppc/kvm.c @@ -72,6 +72,7 @@ static int cap_ppc_watchdog; static int cap_papr; static int cap_htab_fd; static int cap_fixup_hcalls; +static int cap_spapr_reuse_vcpu; static uint32_t debug_inst_opcode; @@ -114,6 +115,7 @@ int kvm_arch_init(KVMState *s) * only activated after this by kvmppc_set_papr() */ cap_htab_fd = kvm_check_extension(s, KVM_CAP_PPC_HTAB_FD); cap_fixup_hcalls = kvm_check_extension(s, KVM_CAP_PPC_FIXUP_HCALL); + cap_spapr_reuse_vcpu = kvm_check_extension(s, KVM_CAP_SPAPR_REUSE_VCPU); if (!cap_interrupt_level) { fprintf(stderr, "KVM: Couldn't find level irq capability. Expect the " @@ -2408,3 +2410,8 @@ int kvm_arch_fixup_msi_route(struct kvm_irq_routing_entry *route, { return 0; } + +bool kvmppc_spapr_reuse_vcpu(void) +{ + return cap_spapr_reuse_vcpu; +} diff --git a/target-ppc/kvm_ppc.h b/target-ppc/kvm_ppc.h index 2e0224c..c831229 100644 --- a/target-ppc/kvm_ppc.h +++ b/target-ppc/kvm_ppc.h @@ -40,6 +40,7 @@ void *kvmppc_create_spapr_tce(uint32_t liobn, uint32_t window_size, int *pfd, int kvmppc_remove_spapr_tce(void *table, int pfd, uint32_t window_size); int kvmppc_reset_htab(int shift_hint); uint64_t kvmppc_rma_size(uint64_t current_size, unsigned int hash_shift); +bool kvmppc_spapr_reuse_vcpu(void); #endif /* !CONFIG_USER_ONLY */ bool kvmppc_has_cap_epr(void); int kvmppc_define_rtas_kernel_token(uint32_t token, const char *function); @@ -185,6 +186,11 @@ static inline int kvmppc_update_sdr1(CPUPPCState *env) return 0; } +static inline bool kvmppc_spapr_reuse_vcpu(void) +{ + return false; +} + #endif /* !CONFIG_USER_ONLY */ static inline bool kvmppc_has_cap_epr(void)
Support hot removal of CPU for sPAPR guests by sending the hot unplug notification to the guest via EPOW interrupt. Signed-off-by: Bharata B Rao <bharata@linux.vnet.ibm.com> --- hw/ppc/spapr.c | 78 ++++++++++++++++++++++++++++++++++++++++++++++- linux-headers/linux/kvm.h | 1 + target-ppc/kvm.c | 7 +++++ target-ppc/kvm_ppc.h | 6 ++++ 4 files changed, 91 insertions(+), 1 deletion(-)