| Message ID | 158326542572.40452.15331466265190906874.stgit@naples-babu.amd.com |
|---|---|
| State | New |
| Headers | show |
| Series | APIC ID fixes for AMD EPYC CPU model | expand |
On Tue, 03 Mar 2020 13:57:05 -0600 Babu Moger <babu.moger@amd.com> wrote: > This is an effort to re-arrange few data structure for better readability. > > 1. Add X86CPUTopoInfo which will have all the topology informations > required to build the cpu topology. There is no functional changes. > 2. Introduce init_topo_info to initialize X86CPUTopoInfo members from > X86MachineState. > > There is no functional changes. > > Signed-off-by: Babu Moger <babu.moger@amd.com> Reviewed-by: Igor Mammedov <imammedo@redhat.com> > --- > hw/i386/pc.c | 12 ++++++------ > hw/i386/x86.c | 32 ++++++++++++++++++++++++-------- > include/hw/i386/topology.h | 38 ++++++++++++++++++++++++-------------- > include/hw/i386/x86.h | 3 +++ > 4 files changed, 57 insertions(+), 28 deletions(-) > > diff --git a/hw/i386/pc.c b/hw/i386/pc.c > index 715f79f58c..ef23ae2af5 100644 > --- a/hw/i386/pc.c > +++ b/hw/i386/pc.c > @@ -1514,6 +1514,7 @@ static void pc_cpu_pre_plug(HotplugHandler *hotplug_dev, > X86MachineState *x86ms = X86_MACHINE(pcms); > unsigned int smp_cores = ms->smp.cores; > unsigned int smp_threads = ms->smp.threads; > + X86CPUTopoInfo topo_info; > > if(!object_dynamic_cast(OBJECT(cpu), ms->cpu_type)) { > error_setg(errp, "Invalid CPU type, expected cpu type: '%s'", > @@ -1521,6 +1522,8 @@ static void pc_cpu_pre_plug(HotplugHandler *hotplug_dev, > return; > } > > + init_topo_info(&topo_info, x86ms); > + > env->nr_dies = x86ms->smp_dies; > > /* > @@ -1576,16 +1579,14 @@ static void pc_cpu_pre_plug(HotplugHandler *hotplug_dev, > topo_ids.die_id = cpu->die_id; > topo_ids.core_id = cpu->core_id; > topo_ids.smt_id = cpu->thread_id; > - cpu->apic_id = apicid_from_topo_ids(x86ms->smp_dies, smp_cores, > - smp_threads, &topo_ids); > + cpu->apic_id = apicid_from_topo_ids(&topo_info, &topo_ids); > } > > cpu_slot = pc_find_cpu_slot(MACHINE(pcms), cpu->apic_id, &idx); > if (!cpu_slot) { > MachineState *ms = MACHINE(pcms); > > - x86_topo_ids_from_apicid(cpu->apic_id, x86ms->smp_dies, > - smp_cores, smp_threads, &topo_ids); > + x86_topo_ids_from_apicid(cpu->apic_id, &topo_info, &topo_ids); > error_setg(errp, > "Invalid CPU [socket: %u, die: %u, core: %u, thread: %u] with" > " APIC ID %" PRIu32 ", valid index range 0:%d", > @@ -1606,8 +1607,7 @@ static void pc_cpu_pre_plug(HotplugHandler *hotplug_dev, > /* TODO: move socket_id/core_id/thread_id checks into x86_cpu_realizefn() > * once -smp refactoring is complete and there will be CPU private > * CPUState::nr_cores and CPUState::nr_threads fields instead of globals */ > - x86_topo_ids_from_apicid(cpu->apic_id, x86ms->smp_dies, > - smp_cores, smp_threads, &topo_ids); > + x86_topo_ids_from_apicid(cpu->apic_id, &topo_info, &topo_ids); > if (cpu->socket_id != -1 && cpu->socket_id != topo_ids.pkg_id) { > error_setg(errp, "property socket-id: %u doesn't match set apic-id:" > " 0x%x (socket-id: %u)", cpu->socket_id, cpu->apic_id, > diff --git a/hw/i386/x86.c b/hw/i386/x86.c > index 322fb6abbc..03b8962c98 100644 > --- a/hw/i386/x86.c > +++ b/hw/i386/x86.c > @@ -57,6 +57,16 @@ > /* Physical Address of PVH entry point read from kernel ELF NOTE */ > static size_t pvh_start_addr; > > +inline void init_topo_info(X86CPUTopoInfo *topo_info, > + const X86MachineState *x86ms) > +{ > + MachineState *ms = MACHINE(x86ms); > + > + topo_info->dies_per_pkg = x86ms->smp_dies; > + topo_info->cores_per_die = ms->smp.cores; > + topo_info->threads_per_core = ms->smp.threads; > +} > + > /* > * Calculates initial APIC ID for a specific CPU index > * > @@ -68,13 +78,14 @@ static size_t pvh_start_addr; > uint32_t x86_cpu_apic_id_from_index(X86MachineState *x86ms, > unsigned int cpu_index) > { > - MachineState *ms = MACHINE(x86ms); > X86MachineClass *x86mc = X86_MACHINE_GET_CLASS(x86ms); > + X86CPUTopoInfo topo_info; > uint32_t correct_id; > static bool warned; > > - correct_id = x86_apicid_from_cpu_idx(x86ms->smp_dies, ms->smp.cores, > - ms->smp.threads, cpu_index); > + init_topo_info(&topo_info, x86ms); > + > + correct_id = x86_apicid_from_cpu_idx(&topo_info, cpu_index); > if (x86mc->compat_apic_id_mode) { > if (cpu_index != correct_id && !warned && !qtest_enabled()) { > error_report("APIC IDs set in compatibility mode, " > @@ -145,19 +156,22 @@ int64_t x86_get_default_cpu_node_id(const MachineState *ms, int idx) > { > X86CPUTopoIDs topo_ids; > X86MachineState *x86ms = X86_MACHINE(ms); > + X86CPUTopoInfo topo_info; > + > + init_topo_info(&topo_info, x86ms); > > assert(idx < ms->possible_cpus->len); > x86_topo_ids_from_apicid(ms->possible_cpus->cpus[idx].arch_id, > - x86ms->smp_dies, ms->smp.cores, > - ms->smp.threads, &topo_ids); > + &topo_info, &topo_ids); > return topo_ids.pkg_id % ms->numa_state->num_nodes; > } > > const CPUArchIdList *x86_possible_cpu_arch_ids(MachineState *ms) > { > X86MachineState *x86ms = X86_MACHINE(ms); > - int i; > unsigned int max_cpus = ms->smp.max_cpus; > + X86CPUTopoInfo topo_info; > + int i; > > if (ms->possible_cpus) { > /* > @@ -171,6 +185,9 @@ const CPUArchIdList *x86_possible_cpu_arch_ids(MachineState *ms) > ms->possible_cpus = g_malloc0(sizeof(CPUArchIdList) + > sizeof(CPUArchId) * max_cpus); > ms->possible_cpus->len = max_cpus; > + > + init_topo_info(&topo_info, x86ms); > + > for (i = 0; i < ms->possible_cpus->len; i++) { > X86CPUTopoIDs topo_ids; > > @@ -179,8 +196,7 @@ const CPUArchIdList *x86_possible_cpu_arch_ids(MachineState *ms) > ms->possible_cpus->cpus[i].arch_id = > x86_cpu_apic_id_from_index(x86ms, i); > x86_topo_ids_from_apicid(ms->possible_cpus->cpus[i].arch_id, > - x86ms->smp_dies, ms->smp.cores, > - ms->smp.threads, &topo_ids); > + &topo_info, &topo_ids); > ms->possible_cpus->cpus[i].props.has_socket_id = true; > ms->possible_cpus->cpus[i].props.socket_id = topo_ids.pkg_id; > if (x86ms->smp_dies > 1) { > diff --git a/include/hw/i386/topology.h b/include/hw/i386/topology.h > index 52def68610..7ea507f376 100644 > --- a/include/hw/i386/topology.h > +++ b/include/hw/i386/topology.h > @@ -52,6 +52,12 @@ typedef struct X86CPUTopoIDs { > unsigned smt_id; > } X86CPUTopoIDs; > > +typedef struct X86CPUTopoInfo { > + unsigned dies_per_pkg; > + unsigned cores_per_die; > + unsigned threads_per_core; > +} X86CPUTopoInfo; > + > /* Return the bit width needed for 'count' IDs > */ > static unsigned apicid_bitwidth_for_count(unsigned count) > @@ -119,11 +125,13 @@ static inline unsigned apicid_pkg_offset(unsigned nr_dies, > * > * The caller must make sure core_id < nr_cores and smt_id < nr_threads. > */ > -static inline apic_id_t apicid_from_topo_ids(unsigned nr_dies, > - unsigned nr_cores, > - unsigned nr_threads, > +static inline apic_id_t apicid_from_topo_ids(X86CPUTopoInfo *topo_info, > const X86CPUTopoIDs *topo_ids) > { > + unsigned nr_dies = topo_info->dies_per_pkg; > + unsigned nr_cores = topo_info->cores_per_die; > + unsigned nr_threads = topo_info->threads_per_core; > + > return (topo_ids->pkg_id << > apicid_pkg_offset(nr_dies, nr_cores, nr_threads)) | > (topo_ids->die_id << > @@ -136,12 +144,14 @@ static inline apic_id_t apicid_from_topo_ids(unsigned nr_dies, > /* Calculate thread/core/package IDs for a specific topology, > * based on (contiguous) CPU index > */ > -static inline void x86_topo_ids_from_idx(unsigned nr_dies, > - unsigned nr_cores, > - unsigned nr_threads, > +static inline void x86_topo_ids_from_idx(X86CPUTopoInfo *topo_info, > unsigned cpu_index, > X86CPUTopoIDs *topo_ids) > { > + unsigned nr_dies = topo_info->dies_per_pkg; > + unsigned nr_cores = topo_info->cores_per_die; > + unsigned nr_threads = topo_info->threads_per_core; > + > topo_ids->pkg_id = cpu_index / (nr_dies * nr_cores * nr_threads); > topo_ids->die_id = cpu_index / (nr_cores * nr_threads) % nr_dies; > topo_ids->core_id = cpu_index / nr_threads % nr_cores; > @@ -152,11 +162,13 @@ static inline void x86_topo_ids_from_idx(unsigned nr_dies, > * based on APIC ID > */ > static inline void x86_topo_ids_from_apicid(apic_id_t apicid, > - unsigned nr_dies, > - unsigned nr_cores, > - unsigned nr_threads, > + X86CPUTopoInfo *topo_info, > X86CPUTopoIDs *topo_ids) > { > + unsigned nr_dies = topo_info->dies_per_pkg; > + unsigned nr_cores = topo_info->cores_per_die; > + unsigned nr_threads = topo_info->threads_per_core; > + > topo_ids->smt_id = apicid & > ~(0xFFFFFFFFUL << apicid_smt_width(nr_dies, nr_cores, nr_threads)); > topo_ids->core_id = > @@ -173,14 +185,12 @@ static inline void x86_topo_ids_from_apicid(apic_id_t apicid, > * > * 'cpu_index' is a sequential, contiguous ID for the CPU. > */ > -static inline apic_id_t x86_apicid_from_cpu_idx(unsigned nr_dies, > - unsigned nr_cores, > - unsigned nr_threads, > +static inline apic_id_t x86_apicid_from_cpu_idx(X86CPUTopoInfo *topo_info, > unsigned cpu_index) > { > X86CPUTopoIDs topo_ids; > - x86_topo_ids_from_idx(nr_dies, nr_cores, nr_threads, cpu_index, &topo_ids); > - return apicid_from_topo_ids(nr_dies, nr_cores, nr_threads, &topo_ids); > + x86_topo_ids_from_idx(topo_info, cpu_index, &topo_ids); > + return apicid_from_topo_ids(topo_info, &topo_ids); > } > > #endif /* HW_I386_TOPOLOGY_H */ > diff --git a/include/hw/i386/x86.h b/include/hw/i386/x86.h > index 41fe37b8a3..22babcb3bb 100644 > --- a/include/hw/i386/x86.h > +++ b/include/hw/i386/x86.h > @@ -21,6 +21,7 @@ > #include "exec/hwaddr.h" > #include "qemu/notify.h" > > +#include "hw/i386/topology.h" > #include "hw/boards.h" > #include "hw/nmi.h" > #include "hw/isa/isa.h" > @@ -82,6 +83,8 @@ typedef struct { > #define X86_MACHINE_CLASS(class) \ > OBJECT_CLASS_CHECK(X86MachineClass, class, TYPE_X86_MACHINE) > > +void init_topo_info(X86CPUTopoInfo *topo_info, const X86MachineState *x86ms); > + > uint32_t x86_cpu_apic_id_from_index(X86MachineState *pcms, > unsigned int cpu_index); > >
On Tue, Mar 03, 2020 at 01:57:05PM -0600, Babu Moger wrote: > This is an effort to re-arrange few data structure for better readability. > > 1. Add X86CPUTopoInfo which will have all the topology informations > required to build the cpu topology. There is no functional changes. > 2. Introduce init_topo_info to initialize X86CPUTopoInfo members from > X86MachineState. > > There is no functional changes. > > Signed-off-by: Babu Moger <babu.moger@amd.com> I was queueing this one, but I had to dequeue. You forgot to tests/test-x86-cpuid.c to use the new X86CPUTopoInfo argument.
diff --git a/hw/i386/pc.c b/hw/i386/pc.c index 715f79f58c..ef23ae2af5 100644 --- a/hw/i386/pc.c +++ b/hw/i386/pc.c @@ -1514,6 +1514,7 @@ static void pc_cpu_pre_plug(HotplugHandler *hotplug_dev, X86MachineState *x86ms = X86_MACHINE(pcms); unsigned int smp_cores = ms->smp.cores; unsigned int smp_threads = ms->smp.threads; + X86CPUTopoInfo topo_info; if(!object_dynamic_cast(OBJECT(cpu), ms->cpu_type)) { error_setg(errp, "Invalid CPU type, expected cpu type: '%s'", @@ -1521,6 +1522,8 @@ static void pc_cpu_pre_plug(HotplugHandler *hotplug_dev, return; } + init_topo_info(&topo_info, x86ms); + env->nr_dies = x86ms->smp_dies; /* @@ -1576,16 +1579,14 @@ static void pc_cpu_pre_plug(HotplugHandler *hotplug_dev, topo_ids.die_id = cpu->die_id; topo_ids.core_id = cpu->core_id; topo_ids.smt_id = cpu->thread_id; - cpu->apic_id = apicid_from_topo_ids(x86ms->smp_dies, smp_cores, - smp_threads, &topo_ids); + cpu->apic_id = apicid_from_topo_ids(&topo_info, &topo_ids); } cpu_slot = pc_find_cpu_slot(MACHINE(pcms), cpu->apic_id, &idx); if (!cpu_slot) { MachineState *ms = MACHINE(pcms); - x86_topo_ids_from_apicid(cpu->apic_id, x86ms->smp_dies, - smp_cores, smp_threads, &topo_ids); + x86_topo_ids_from_apicid(cpu->apic_id, &topo_info, &topo_ids); error_setg(errp, "Invalid CPU [socket: %u, die: %u, core: %u, thread: %u] with" " APIC ID %" PRIu32 ", valid index range 0:%d", @@ -1606,8 +1607,7 @@ static void pc_cpu_pre_plug(HotplugHandler *hotplug_dev, /* TODO: move socket_id/core_id/thread_id checks into x86_cpu_realizefn() * once -smp refactoring is complete and there will be CPU private * CPUState::nr_cores and CPUState::nr_threads fields instead of globals */ - x86_topo_ids_from_apicid(cpu->apic_id, x86ms->smp_dies, - smp_cores, smp_threads, &topo_ids); + x86_topo_ids_from_apicid(cpu->apic_id, &topo_info, &topo_ids); if (cpu->socket_id != -1 && cpu->socket_id != topo_ids.pkg_id) { error_setg(errp, "property socket-id: %u doesn't match set apic-id:" " 0x%x (socket-id: %u)", cpu->socket_id, cpu->apic_id, diff --git a/hw/i386/x86.c b/hw/i386/x86.c index 322fb6abbc..03b8962c98 100644 --- a/hw/i386/x86.c +++ b/hw/i386/x86.c @@ -57,6 +57,16 @@ /* Physical Address of PVH entry point read from kernel ELF NOTE */ static size_t pvh_start_addr; +inline void init_topo_info(X86CPUTopoInfo *topo_info, + const X86MachineState *x86ms) +{ + MachineState *ms = MACHINE(x86ms); + + topo_info->dies_per_pkg = x86ms->smp_dies; + topo_info->cores_per_die = ms->smp.cores; + topo_info->threads_per_core = ms->smp.threads; +} + /* * Calculates initial APIC ID for a specific CPU index * @@ -68,13 +78,14 @@ static size_t pvh_start_addr; uint32_t x86_cpu_apic_id_from_index(X86MachineState *x86ms, unsigned int cpu_index) { - MachineState *ms = MACHINE(x86ms); X86MachineClass *x86mc = X86_MACHINE_GET_CLASS(x86ms); + X86CPUTopoInfo topo_info; uint32_t correct_id; static bool warned; - correct_id = x86_apicid_from_cpu_idx(x86ms->smp_dies, ms->smp.cores, - ms->smp.threads, cpu_index); + init_topo_info(&topo_info, x86ms); + + correct_id = x86_apicid_from_cpu_idx(&topo_info, cpu_index); if (x86mc->compat_apic_id_mode) { if (cpu_index != correct_id && !warned && !qtest_enabled()) { error_report("APIC IDs set in compatibility mode, " @@ -145,19 +156,22 @@ int64_t x86_get_default_cpu_node_id(const MachineState *ms, int idx) { X86CPUTopoIDs topo_ids; X86MachineState *x86ms = X86_MACHINE(ms); + X86CPUTopoInfo topo_info; + + init_topo_info(&topo_info, x86ms); assert(idx < ms->possible_cpus->len); x86_topo_ids_from_apicid(ms->possible_cpus->cpus[idx].arch_id, - x86ms->smp_dies, ms->smp.cores, - ms->smp.threads, &topo_ids); + &topo_info, &topo_ids); return topo_ids.pkg_id % ms->numa_state->num_nodes; } const CPUArchIdList *x86_possible_cpu_arch_ids(MachineState *ms) { X86MachineState *x86ms = X86_MACHINE(ms); - int i; unsigned int max_cpus = ms->smp.max_cpus; + X86CPUTopoInfo topo_info; + int i; if (ms->possible_cpus) { /* @@ -171,6 +185,9 @@ const CPUArchIdList *x86_possible_cpu_arch_ids(MachineState *ms) ms->possible_cpus = g_malloc0(sizeof(CPUArchIdList) + sizeof(CPUArchId) * max_cpus); ms->possible_cpus->len = max_cpus; + + init_topo_info(&topo_info, x86ms); + for (i = 0; i < ms->possible_cpus->len; i++) { X86CPUTopoIDs topo_ids; @@ -179,8 +196,7 @@ const CPUArchIdList *x86_possible_cpu_arch_ids(MachineState *ms) ms->possible_cpus->cpus[i].arch_id = x86_cpu_apic_id_from_index(x86ms, i); x86_topo_ids_from_apicid(ms->possible_cpus->cpus[i].arch_id, - x86ms->smp_dies, ms->smp.cores, - ms->smp.threads, &topo_ids); + &topo_info, &topo_ids); ms->possible_cpus->cpus[i].props.has_socket_id = true; ms->possible_cpus->cpus[i].props.socket_id = topo_ids.pkg_id; if (x86ms->smp_dies > 1) { diff --git a/include/hw/i386/topology.h b/include/hw/i386/topology.h index 52def68610..7ea507f376 100644 --- a/include/hw/i386/topology.h +++ b/include/hw/i386/topology.h @@ -52,6 +52,12 @@ typedef struct X86CPUTopoIDs { unsigned smt_id; } X86CPUTopoIDs; +typedef struct X86CPUTopoInfo { + unsigned dies_per_pkg; + unsigned cores_per_die; + unsigned threads_per_core; +} X86CPUTopoInfo; + /* Return the bit width needed for 'count' IDs */ static unsigned apicid_bitwidth_for_count(unsigned count) @@ -119,11 +125,13 @@ static inline unsigned apicid_pkg_offset(unsigned nr_dies, * * The caller must make sure core_id < nr_cores and smt_id < nr_threads. */ -static inline apic_id_t apicid_from_topo_ids(unsigned nr_dies, - unsigned nr_cores, - unsigned nr_threads, +static inline apic_id_t apicid_from_topo_ids(X86CPUTopoInfo *topo_info, const X86CPUTopoIDs *topo_ids) { + unsigned nr_dies = topo_info->dies_per_pkg; + unsigned nr_cores = topo_info->cores_per_die; + unsigned nr_threads = topo_info->threads_per_core; + return (topo_ids->pkg_id << apicid_pkg_offset(nr_dies, nr_cores, nr_threads)) | (topo_ids->die_id << @@ -136,12 +144,14 @@ static inline apic_id_t apicid_from_topo_ids(unsigned nr_dies, /* Calculate thread/core/package IDs for a specific topology, * based on (contiguous) CPU index */ -static inline void x86_topo_ids_from_idx(unsigned nr_dies, - unsigned nr_cores, - unsigned nr_threads, +static inline void x86_topo_ids_from_idx(X86CPUTopoInfo *topo_info, unsigned cpu_index, X86CPUTopoIDs *topo_ids) { + unsigned nr_dies = topo_info->dies_per_pkg; + unsigned nr_cores = topo_info->cores_per_die; + unsigned nr_threads = topo_info->threads_per_core; + topo_ids->pkg_id = cpu_index / (nr_dies * nr_cores * nr_threads); topo_ids->die_id = cpu_index / (nr_cores * nr_threads) % nr_dies; topo_ids->core_id = cpu_index / nr_threads % nr_cores; @@ -152,11 +162,13 @@ static inline void x86_topo_ids_from_idx(unsigned nr_dies, * based on APIC ID */ static inline void x86_topo_ids_from_apicid(apic_id_t apicid, - unsigned nr_dies, - unsigned nr_cores, - unsigned nr_threads, + X86CPUTopoInfo *topo_info, X86CPUTopoIDs *topo_ids) { + unsigned nr_dies = topo_info->dies_per_pkg; + unsigned nr_cores = topo_info->cores_per_die; + unsigned nr_threads = topo_info->threads_per_core; + topo_ids->smt_id = apicid & ~(0xFFFFFFFFUL << apicid_smt_width(nr_dies, nr_cores, nr_threads)); topo_ids->core_id = @@ -173,14 +185,12 @@ static inline void x86_topo_ids_from_apicid(apic_id_t apicid, * * 'cpu_index' is a sequential, contiguous ID for the CPU. */ -static inline apic_id_t x86_apicid_from_cpu_idx(unsigned nr_dies, - unsigned nr_cores, - unsigned nr_threads, +static inline apic_id_t x86_apicid_from_cpu_idx(X86CPUTopoInfo *topo_info, unsigned cpu_index) { X86CPUTopoIDs topo_ids; - x86_topo_ids_from_idx(nr_dies, nr_cores, nr_threads, cpu_index, &topo_ids); - return apicid_from_topo_ids(nr_dies, nr_cores, nr_threads, &topo_ids); + x86_topo_ids_from_idx(topo_info, cpu_index, &topo_ids); + return apicid_from_topo_ids(topo_info, &topo_ids); } #endif /* HW_I386_TOPOLOGY_H */ diff --git a/include/hw/i386/x86.h b/include/hw/i386/x86.h index 41fe37b8a3..22babcb3bb 100644 --- a/include/hw/i386/x86.h +++ b/include/hw/i386/x86.h @@ -21,6 +21,7 @@ #include "exec/hwaddr.h" #include "qemu/notify.h" +#include "hw/i386/topology.h" #include "hw/boards.h" #include "hw/nmi.h" #include "hw/isa/isa.h" @@ -82,6 +83,8 @@ typedef struct { #define X86_MACHINE_CLASS(class) \ OBJECT_CLASS_CHECK(X86MachineClass, class, TYPE_X86_MACHINE) +void init_topo_info(X86CPUTopoInfo *topo_info, const X86MachineState *x86ms); + uint32_t x86_cpu_apic_id_from_index(X86MachineState *pcms, unsigned int cpu_index);
This is an effort to re-arrange few data structure for better readability. 1. Add X86CPUTopoInfo which will have all the topology informations required to build the cpu topology. There is no functional changes. 2. Introduce init_topo_info to initialize X86CPUTopoInfo members from X86MachineState. There is no functional changes. Signed-off-by: Babu Moger <babu.moger@amd.com> --- hw/i386/pc.c | 12 ++++++------ hw/i386/x86.c | 32 ++++++++++++++++++++++++-------- include/hw/i386/topology.h | 38 ++++++++++++++++++++++++-------------- include/hw/i386/x86.h | 3 +++ 4 files changed, 57 insertions(+), 28 deletions(-)