[RFC,v2.1,12/12] spapr: CPU hot unplug support

Remove the CPU core device by removing the underlying CPU thread devices.
Hot removal of CPU for sPAPR guests is achieved by sending the hot unplug
notification to the guest. Release the vCPU object after CPU hot unplug so
that vCPU fd can be parked and reused.

Signed-off-by: Bharata B Rao <bharata@linux.vnet.ibm.com>
---
 hw/ppc/spapr.c                  | 16 ++++++++
 hw/ppc/spapr_cpu_core.c         | 86 +++++++++++++++++++++++++++++++++++++++++
 include/hw/ppc/spapr.h          |  1 +
 include/hw/ppc/spapr_cpu_core.h | 11 ++++++
 4 files changed, 114 insertions(+)

On Thu, Mar 31, 2016 at 02:09:21PM +0530, Bharata B Rao wrote:
> Remove the CPU core device by removing the underlying CPU thread devices.
> Hot removal of CPU for sPAPR guests is achieved by sending the hot unplug
> notification to the guest. Release the vCPU object after CPU hot unplug so
> that vCPU fd can be parked and reused.
> 
> Signed-off-by: Bharata B Rao <bharata@linux.vnet.ibm.com>
> ---
>  hw/ppc/spapr.c                  | 16 ++++++++
>  hw/ppc/spapr_cpu_core.c         | 86 +++++++++++++++++++++++++++++++++++++++++
>  include/hw/ppc/spapr.h          |  1 +
>  include/hw/ppc/spapr_cpu_core.h | 11 ++++++
>  4 files changed, 114 insertions(+)
> 
> diff --git a/hw/ppc/spapr.c b/hw/ppc/spapr.c
> index 1a5dbd9..74cdcf2 100644
> --- a/hw/ppc/spapr.c
> +++ b/hw/ppc/spapr.c
> @@ -2348,11 +2348,27 @@ static void spapr_machine_device_plug(HotplugHandler *hotplug_dev,
>      }
>  }
>  
> +void spapr_cpu_destroy(PowerPCCPU *cpu)
> +{
> +    sPAPRMachineState *spapr = SPAPR_MACHINE(qdev_get_machine());
> +
> +    xics_cpu_destroy(spapr->icp, cpu);
> +    qemu_unregister_reset(spapr_cpu_reset, cpu);
> +}
> +
>  static void spapr_machine_device_unplug(HotplugHandler *hotplug_dev,
>                                        DeviceState *dev, Error **errp)
>  {
> +    sPAPRMachineClass *smc = SPAPR_MACHINE_GET_CLASS(qdev_get_machine());
> +
>      if (object_dynamic_cast(OBJECT(dev), TYPE_PC_DIMM)) {
>          error_setg(errp, "Memory hot unplug not supported by sPAPR");
> +    } else if (object_dynamic_cast(OBJECT(dev), TYPE_SPAPR_CPU_CORE)) {
> +        if (!smc->dr_cpu_enabled) {
> +            error_setg(errp, "CPU hot unplug not supported on this machine");
> +            return;
> +        }
> +        spapr_core_unplug(hotplug_dev, dev, errp);
>      }
>  }
>  
> diff --git a/hw/ppc/spapr_cpu_core.c b/hw/ppc/spapr_cpu_core.c
> index a9ba843..09a592e 100644
> --- a/hw/ppc/spapr_cpu_core.c
> +++ b/hw/ppc/spapr_cpu_core.c
> @@ -119,6 +119,92 @@ void spapr_core_plug(HotplugHandler *hotplug_dev, DeviceState *dev,
>      }
>  }
>  
> +static void spapr_cpu_core_cleanup(struct sPAPRCPUUnplugList *unplug_list)
> +{
> +    sPAPRCPUUnplug *unplug, *next;
> +    Object *cpu;
> +
> +    QLIST_FOREACH_SAFE(unplug, unplug_list, node, next) {
> +        cpu = unplug->cpu;
> +        object_unparent(cpu);

Is there any danger in the fact that the cpu object is still in the
QOM tree until unparented here?  My usual expectation would be that
you'd remove the object from the tree immediately, but defer the
actual free.  But I'm a bit unclear on how QOM removals are supposed
to work.

> +        QLIST_REMOVE(unplug, node);
> +        g_free(unplug);
> +    }
> +}
> +
> +static void spapr_add_cpu_to_unplug_list(Object *cpu,
> +                                         struct sPAPRCPUUnplugList *unplug_list)
> +{
> +    sPAPRCPUUnplug *unplug = g_malloc(sizeof(*unplug));
> +
> +    unplug->cpu = cpu;
> +    QLIST_INSERT_HEAD(unplug_list, unplug, node);
> +}
> +
> +static int spapr_cpu_release(Object *obj, void *opaque)
> +{
> +    DeviceState *dev = DEVICE(obj);
> +    CPUState *cs = CPU(dev);
> +    PowerPCCPU *cpu = POWERPC_CPU(cs);
> +    struct sPAPRCPUUnplugList *unplug_list = opaque;
> +
> +    spapr_cpu_destroy(cpu);
> +    cpu_remove_sync(cs);
> +
> +    /*
> +     * We are still walking the core object's children list, and
> +     * hence can't cleanup this CPU thread object just yet. Put
> +     * it on a list for later removal.
> +     */
> +    spapr_add_cpu_to_unplug_list(obj, unplug_list);
> +    return 0;
> +}
> +
> +static void spapr_core_release(DeviceState *dev)
> +{
> +    struct sPAPRCPUUnplugList unplug_list;
> +    sPAPRMachineState *spapr = SPAPR_MACHINE(qdev_get_machine());
> +    sPAPRCPUCore *core = SPAPR_CPU_CORE(OBJECT(dev));
> +    CPUCore *cc = CPU_CORE(dev);
> +    int smt = kvmppc_smt_threads();
> +
> +    QLIST_INIT(&unplug_list);
> +    object_child_foreach(OBJECT(dev), spapr_cpu_release, &unplug_list);
> +    spapr_cpu_core_cleanup(&unplug_list);
> +    spapr->cores[cc->core / smt] = NULL;
> +
> +    g_free(core->threads);
> +}
> +
> +static void spapr_core_release_unparent(DeviceState *dev, void *opaque)
> +{
> +    spapr_core_release(dev);
> +    object_unparent(OBJECT(dev));
> +}
> +
> +void spapr_core_unplug(HotplugHandler *hotplug_dev, DeviceState *dev,
> +                       Error **errp)
> +{
> +    sPAPRCPUCore *core = SPAPR_CPU_CORE(OBJECT(dev));
> +    PowerPCCPU *cpu = &core->threads[0];
> +    int id = ppc_get_vcpu_dt_id(cpu);
> +    sPAPRDRConnector *drc =
> +        spapr_dr_connector_by_id(SPAPR_DR_CONNECTOR_TYPE_CPU, id);
> +    sPAPRDRConnectorClass *drck;
> +    Error *local_err = NULL;
> +
> +    g_assert(drc);
> +
> +    drck = SPAPR_DR_CONNECTOR_GET_CLASS(drc);
> +    drck->detach(drc, dev, spapr_core_release_unparent, NULL, &local_err);
> +    if (local_err) {
> +        error_propagate(errp, local_err);
> +        return;
> +    }
> +
> +    spapr_hotplug_req_remove_by_index(drc);
> +}
> +
>  static const TypeInfo spapr_cpu_core_type_info = {
>      .name = TYPE_SPAPR_CPU_CORE,
>      .parent = TYPE_CPU_CORE,
> diff --git a/include/hw/ppc/spapr.h b/include/hw/ppc/spapr.h
> index 619db98..c5a4a15 100644
> --- a/include/hw/ppc/spapr.h
> +++ b/include/hw/ppc/spapr.h
> @@ -590,6 +590,7 @@ void spapr_hotplug_req_remove_by_count(sPAPRDRConnectorType drc_type,
>  void spapr_cpu_init(sPAPRMachineState *spapr, PowerPCCPU *cpu, Error **errp);
>  void *spapr_populate_hotplug_cpu_dt(CPUState *cs, int *fdt_offset,
>                                      sPAPRMachineState *spapr);
> +void spapr_cpu_destroy(PowerPCCPU *cpu);
>  
>  /* rtas-configure-connector state */
>  struct sPAPRConfigureConnectorState {
> diff --git a/include/hw/ppc/spapr_cpu_core.h b/include/hw/ppc/spapr_cpu_core.h
> index 165af7c..9bc2e01 100644
> --- a/include/hw/ppc/spapr_cpu_core.h
> +++ b/include/hw/ppc/spapr_cpu_core.h
> @@ -65,4 +65,15 @@ void spapr_core_pre_plug(HotplugHandler *hotplug_dev, DeviceState *dev,
>  char *spapr_get_cpu_core_type(const char *model);
>  void spapr_core_plug(HotplugHandler *hotplug_dev, DeviceState *dev,
>                       Error **errp);
> +void spapr_core_unplug(HotplugHandler *hotplug_dev, DeviceState *dev,
> +                       Error **errp);
> +
> +/* List to store unplugged CPU objects for cleanup during unplug */
> +typedef struct sPAPRCPUUnplug {
> +    Object *cpu;
> +    QLIST_ENTRY(sPAPRCPUUnplug) node;
> +} sPAPRCPUUnplug;
> +
> +QLIST_HEAD(sPAPRCPUUnplugList, sPAPRCPUUnplug);
> +
>  #endif

On Mon, Apr 04, 2016 at 02:27:24PM +1000, David Gibson wrote:
> On Thu, Mar 31, 2016 at 02:09:21PM +0530, Bharata B Rao wrote:
> > Remove the CPU core device by removing the underlying CPU thread devices.
> > Hot removal of CPU for sPAPR guests is achieved by sending the hot unplug
> > notification to the guest. Release the vCPU object after CPU hot unplug so
> > that vCPU fd can be parked and reused.
> > 
> > Signed-off-by: Bharata B Rao <bharata@linux.vnet.ibm.com>
> > ---
> >  hw/ppc/spapr.c                  | 16 ++++++++
> >  hw/ppc/spapr_cpu_core.c         | 86 +++++++++++++++++++++++++++++++++++++++++
> >  include/hw/ppc/spapr.h          |  1 +
> >  include/hw/ppc/spapr_cpu_core.h | 11 ++++++
> >  4 files changed, 114 insertions(+)
> > 
> > diff --git a/hw/ppc/spapr.c b/hw/ppc/spapr.c
> > index 1a5dbd9..74cdcf2 100644
> > --- a/hw/ppc/spapr.c
> > +++ b/hw/ppc/spapr.c
> > @@ -2348,11 +2348,27 @@ static void spapr_machine_device_plug(HotplugHandler *hotplug_dev,
> >      }
> >  }
> >  
> > +void spapr_cpu_destroy(PowerPCCPU *cpu)
> > +{
> > +    sPAPRMachineState *spapr = SPAPR_MACHINE(qdev_get_machine());
> > +
> > +    xics_cpu_destroy(spapr->icp, cpu);
> > +    qemu_unregister_reset(spapr_cpu_reset, cpu);
> > +}
> > +
> >  static void spapr_machine_device_unplug(HotplugHandler *hotplug_dev,
> >                                        DeviceState *dev, Error **errp)
> >  {
> > +    sPAPRMachineClass *smc = SPAPR_MACHINE_GET_CLASS(qdev_get_machine());
> > +
> >      if (object_dynamic_cast(OBJECT(dev), TYPE_PC_DIMM)) {
> >          error_setg(errp, "Memory hot unplug not supported by sPAPR");
> > +    } else if (object_dynamic_cast(OBJECT(dev), TYPE_SPAPR_CPU_CORE)) {
> > +        if (!smc->dr_cpu_enabled) {
> > +            error_setg(errp, "CPU hot unplug not supported on this machine");
> > +            return;
> > +        }
> > +        spapr_core_unplug(hotplug_dev, dev, errp);
> >      }
> >  }
> >  
> > diff --git a/hw/ppc/spapr_cpu_core.c b/hw/ppc/spapr_cpu_core.c
> > index a9ba843..09a592e 100644
> > --- a/hw/ppc/spapr_cpu_core.c
> > +++ b/hw/ppc/spapr_cpu_core.c
> > @@ -119,6 +119,92 @@ void spapr_core_plug(HotplugHandler *hotplug_dev, DeviceState *dev,
> >      }
> >  }
> >  
> > +static void spapr_cpu_core_cleanup(struct sPAPRCPUUnplugList *unplug_list)
> > +{
> > +    sPAPRCPUUnplug *unplug, *next;
> > +    Object *cpu;
> > +
> > +    QLIST_FOREACH_SAFE(unplug, unplug_list, node, next) {
> > +        cpu = unplug->cpu;
> > +        object_unparent(cpu);
> 
> Is there any danger in the fact that the cpu object is still in the
> QOM tree until unparented here?  My usual expectation would be that
> you'd remove the object from the tree immediately, but defer the
> actual free.  But I'm a bit unclear on how QOM removals are supposed
> to work.

As per my understanding, object_unparent() removes the object from
its parent and finalizes the child too.

The reason I defer unparenting of each CPU thread object like this is because
from the parent core object's detach_cb routine (spapr_core_release), we are
still walking the parent core's child list and can't immediately unparent the
child thread objects.

Regards,
Bharata.