| Message ID | 20191119095251.21263-1-ike.pan@canonical.com |
|---|---|
| State | New |
| Headers | show |
| Series | [1/2,Bionic] Revert "arm64: capabilities: Change scope of VHE to Boot CPU feature" | expand |
Hello Ike, as this is just reverting one commit, I think [PATCH 1/2] should be just [PATCH], right? Thanks On Tue, Nov 19, 2019 at 5:53 PM Ike Panhc <ike.pan@canonical.com> wrote: > > BugLink: http://bugs.launchpad.net/bugs/1852723 > > The 4.15.0-71.80 generic kernel can not boot on several arm64 machines. > Bisect patches and find out we can solve this problem by reverting > a single patch which comes from stable kernel update. > > This reverts commit 628859e8621cbb105cfbfa2138a68ddbd72958a7. > > Signed-off-by: Ike Panhc <ike.pan@canonical.com> > --- > arch/arm64/include/asm/cpufeature.h | 6 ----- > arch/arm64/include/asm/virt.h | 6 +++++ > arch/arm64/kernel/cpufeature.c | 5 ++-- > arch/arm64/kernel/smp.c | 38 +++++++++++++++++++++++++++++ > 4 files changed, 46 insertions(+), 9 deletions(-) > > diff --git a/arch/arm64/include/asm/cpufeature.h b/arch/arm64/include/asm/cpufeature.h > index f3aa281b97cb..513570cd3e6f 100644 > --- a/arch/arm64/include/asm/cpufeature.h > +++ b/arch/arm64/include/asm/cpufeature.h > @@ -289,12 +289,6 @@ extern struct arm64_ftr_reg arm64_ftr_reg_ctrel0; > (ARM64_CPUCAP_SCOPE_LOCAL_CPU | \ > ARM64_CPUCAP_OPTIONAL_FOR_LATE_CPU) > > -/* > - * CPU feature used early in the boot based on the boot CPU. All secondary > - * CPUs must match the state of the capability as detected by the boot CPU. > - */ > -#define ARM64_CPUCAP_STRICT_BOOT_CPU_FEATURE ARM64_CPUCAP_SCOPE_BOOT_CPU > - > struct arm64_cpu_capabilities { > const char *desc; > u16 capability; > diff --git a/arch/arm64/include/asm/virt.h b/arch/arm64/include/asm/virt.h > index 9d1e24e030b3..c5f89442785c 100644 > --- a/arch/arm64/include/asm/virt.h > +++ b/arch/arm64/include/asm/virt.h > @@ -102,6 +102,12 @@ static inline bool has_vhe(void) > return false; > } > > +#ifdef CONFIG_ARM64_VHE > +extern void verify_cpu_run_el(void); > +#else > +static inline void verify_cpu_run_el(void) {} > +#endif > + > #endif /* __ASSEMBLY__ */ > > #endif /* ! __ASM__VIRT_H */ > diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c > index d19a7b8bb6fb..672443860341 100644 > --- a/arch/arm64/kernel/cpufeature.c > +++ b/arch/arm64/kernel/cpufeature.c > @@ -1113,15 +1113,13 @@ static const struct arm64_cpu_capabilities arm64_features[] = { > .matches = cpufeature_pan_not_uao, > }, > #endif /* CONFIG_ARM64_PAN */ > -#ifdef CONFIG_ARM64_VHE > { > .desc = "Virtualization Host Extensions", > .capability = ARM64_HAS_VIRT_HOST_EXTN, > - .type = ARM64_CPUCAP_STRICT_BOOT_CPU_FEATURE, > + .type = ARM64_CPUCAP_SYSTEM_FEATURE, > .matches = runs_at_el2, > .cpu_enable = cpu_copy_el2regs, > }, > -#endif /* CONFIG_ARM64_VHE */ > { > .desc = "32-bit EL0 Support", > .capability = ARM64_HAS_32BIT_EL0, > @@ -1491,6 +1489,7 @@ static bool verify_local_cpu_caps(u16 scope_mask) > */ > static void check_early_cpu_features(void) > { > + verify_cpu_run_el(); > verify_cpu_asid_bits(); > /* > * Early features are used by the kernel already. If there > diff --git a/arch/arm64/kernel/smp.c b/arch/arm64/kernel/smp.c > index 2faa9863d2e5..a077c3d2fd73 100644 > --- a/arch/arm64/kernel/smp.c > +++ b/arch/arm64/kernel/smp.c > @@ -85,6 +85,43 @@ enum ipi_msg_type { > IPI_WAKEUP > }; > > +#ifdef CONFIG_ARM64_VHE > + > +/* Whether the boot CPU is running in HYP mode or not*/ > +static bool boot_cpu_hyp_mode; > + > +static inline void save_boot_cpu_run_el(void) > +{ > + boot_cpu_hyp_mode = is_kernel_in_hyp_mode(); > +} > + > +static inline bool is_boot_cpu_in_hyp_mode(void) > +{ > + return boot_cpu_hyp_mode; > +} > + > +/* > + * Verify that a secondary CPU is running the kernel at the same > + * EL as that of the boot CPU. > + */ > +void verify_cpu_run_el(void) > +{ > + bool in_el2 = is_kernel_in_hyp_mode(); > + bool boot_cpu_el2 = is_boot_cpu_in_hyp_mode(); > + > + if (in_el2 ^ boot_cpu_el2) { > + pr_crit("CPU%d: mismatched Exception Level(EL%d) with boot CPU(EL%d)\n", > + smp_processor_id(), > + in_el2 ? 2 : 1, > + boot_cpu_el2 ? 2 : 1); > + cpu_panic_kernel(); > + } > +} > + > +#else > +static inline void save_boot_cpu_run_el(void) {} > +#endif > + > #ifdef CONFIG_HOTPLUG_CPU > static int op_cpu_kill(unsigned int cpu); > #else > @@ -410,6 +447,7 @@ void __init smp_prepare_boot_cpu(void) > */ > jump_label_init(); > cpuinfo_store_boot_cpu(); > + save_boot_cpu_run_el(); > } > > static u64 __init of_get_cpu_mpidr(struct device_node *dn) > -- > 2.17.1 > > > -- > kernel-team mailing list > kernel-team@lists.ubuntu.com > https://lists.ubuntu.com/mailman/listinfo/kernel-team
Yes, I forget to edit subject before sending it out. On 11/21/19 10:05 AM, Po-Hsu Lin wrote: > Hello Ike, > as this is just reverting one commit, I think [PATCH 1/2] should be > just [PATCH], right? > Thanks > > On Tue, Nov 19, 2019 at 5:53 PM Ike Panhc <ike.pan@canonical.com> wrote: >> >> BugLink: http://bugs.launchpad.net/bugs/1852723 >> >> The 4.15.0-71.80 generic kernel can not boot on several arm64 machines. >> Bisect patches and find out we can solve this problem by reverting >> a single patch which comes from stable kernel update. >> >> This reverts commit 628859e8621cbb105cfbfa2138a68ddbd72958a7. >> >> Signed-off-by: Ike Panhc <ike.pan@canonical.com> >> --- >> arch/arm64/include/asm/cpufeature.h | 6 ----- >> arch/arm64/include/asm/virt.h | 6 +++++ >> arch/arm64/kernel/cpufeature.c | 5 ++-- >> arch/arm64/kernel/smp.c | 38 +++++++++++++++++++++++++++++ >> 4 files changed, 46 insertions(+), 9 deletions(-) >> >> diff --git a/arch/arm64/include/asm/cpufeature.h b/arch/arm64/include/asm/cpufeature.h >> index f3aa281b97cb..513570cd3e6f 100644 >> --- a/arch/arm64/include/asm/cpufeature.h >> +++ b/arch/arm64/include/asm/cpufeature.h >> @@ -289,12 +289,6 @@ extern struct arm64_ftr_reg arm64_ftr_reg_ctrel0; >> (ARM64_CPUCAP_SCOPE_LOCAL_CPU | \ >> ARM64_CPUCAP_OPTIONAL_FOR_LATE_CPU) >> >> -/* >> - * CPU feature used early in the boot based on the boot CPU. All secondary >> - * CPUs must match the state of the capability as detected by the boot CPU. >> - */ >> -#define ARM64_CPUCAP_STRICT_BOOT_CPU_FEATURE ARM64_CPUCAP_SCOPE_BOOT_CPU >> - >> struct arm64_cpu_capabilities { >> const char *desc; >> u16 capability; >> diff --git a/arch/arm64/include/asm/virt.h b/arch/arm64/include/asm/virt.h >> index 9d1e24e030b3..c5f89442785c 100644 >> --- a/arch/arm64/include/asm/virt.h >> +++ b/arch/arm64/include/asm/virt.h >> @@ -102,6 +102,12 @@ static inline bool has_vhe(void) >> return false; >> } >> >> +#ifdef CONFIG_ARM64_VHE >> +extern void verify_cpu_run_el(void); >> +#else >> +static inline void verify_cpu_run_el(void) {} >> +#endif >> + >> #endif /* __ASSEMBLY__ */ >> >> #endif /* ! __ASM__VIRT_H */ >> diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c >> index d19a7b8bb6fb..672443860341 100644 >> --- a/arch/arm64/kernel/cpufeature.c >> +++ b/arch/arm64/kernel/cpufeature.c >> @@ -1113,15 +1113,13 @@ static const struct arm64_cpu_capabilities arm64_features[] = { >> .matches = cpufeature_pan_not_uao, >> }, >> #endif /* CONFIG_ARM64_PAN */ >> -#ifdef CONFIG_ARM64_VHE >> { >> .desc = "Virtualization Host Extensions", >> .capability = ARM64_HAS_VIRT_HOST_EXTN, >> - .type = ARM64_CPUCAP_STRICT_BOOT_CPU_FEATURE, >> + .type = ARM64_CPUCAP_SYSTEM_FEATURE, >> .matches = runs_at_el2, >> .cpu_enable = cpu_copy_el2regs, >> }, >> -#endif /* CONFIG_ARM64_VHE */ >> { >> .desc = "32-bit EL0 Support", >> .capability = ARM64_HAS_32BIT_EL0, >> @@ -1491,6 +1489,7 @@ static bool verify_local_cpu_caps(u16 scope_mask) >> */ >> static void check_early_cpu_features(void) >> { >> + verify_cpu_run_el(); >> verify_cpu_asid_bits(); >> /* >> * Early features are used by the kernel already. If there >> diff --git a/arch/arm64/kernel/smp.c b/arch/arm64/kernel/smp.c >> index 2faa9863d2e5..a077c3d2fd73 100644 >> --- a/arch/arm64/kernel/smp.c >> +++ b/arch/arm64/kernel/smp.c >> @@ -85,6 +85,43 @@ enum ipi_msg_type { >> IPI_WAKEUP >> }; >> >> +#ifdef CONFIG_ARM64_VHE >> + >> +/* Whether the boot CPU is running in HYP mode or not*/ >> +static bool boot_cpu_hyp_mode; >> + >> +static inline void save_boot_cpu_run_el(void) >> +{ >> + boot_cpu_hyp_mode = is_kernel_in_hyp_mode(); >> +} >> + >> +static inline bool is_boot_cpu_in_hyp_mode(void) >> +{ >> + return boot_cpu_hyp_mode; >> +} >> + >> +/* >> + * Verify that a secondary CPU is running the kernel at the same >> + * EL as that of the boot CPU. >> + */ >> +void verify_cpu_run_el(void) >> +{ >> + bool in_el2 = is_kernel_in_hyp_mode(); >> + bool boot_cpu_el2 = is_boot_cpu_in_hyp_mode(); >> + >> + if (in_el2 ^ boot_cpu_el2) { >> + pr_crit("CPU%d: mismatched Exception Level(EL%d) with boot CPU(EL%d)\n", >> + smp_processor_id(), >> + in_el2 ? 2 : 1, >> + boot_cpu_el2 ? 2 : 1); >> + cpu_panic_kernel(); >> + } >> +} >> + >> +#else >> +static inline void save_boot_cpu_run_el(void) {} >> +#endif >> + >> #ifdef CONFIG_HOTPLUG_CPU >> static int op_cpu_kill(unsigned int cpu); >> #else >> @@ -410,6 +447,7 @@ void __init smp_prepare_boot_cpu(void) >> */ >> jump_label_init(); >> cpuinfo_store_boot_cpu(); >> + save_boot_cpu_run_el(); >> } >> >> static u64 __init of_get_cpu_mpidr(struct device_node *dn) >> -- >> 2.17.1 >> >> >> -- >> kernel-team mailing list >> kernel-team@lists.ubuntu.com >> https://lists.ubuntu.com/mailman/listinfo/kernel-team >
This SAUCE patch is better then reverting. https://lists.ubuntu.com/archives/kernel-team/2019-November/105749.html On 11/21/19 10:35 AM, Ike Panhc wrote: > Yes, I forget to edit subject before sending it out. > > On 11/21/19 10:05 AM, Po-Hsu Lin wrote: >> Hello Ike, >> as this is just reverting one commit, I think [PATCH 1/2] should be >> just [PATCH], right? >> Thanks >> >> On Tue, Nov 19, 2019 at 5:53 PM Ike Panhc <ike.pan@canonical.com> wrote: >>> >>> BugLink: http://bugs.launchpad.net/bugs/1852723 >>> >>> The 4.15.0-71.80 generic kernel can not boot on several arm64 machines. >>> Bisect patches and find out we can solve this problem by reverting >>> a single patch which comes from stable kernel update. >>> >>> This reverts commit 628859e8621cbb105cfbfa2138a68ddbd72958a7. >>> >>> Signed-off-by: Ike Panhc <ike.pan@canonical.com> >>> --- >>> arch/arm64/include/asm/cpufeature.h | 6 ----- >>> arch/arm64/include/asm/virt.h | 6 +++++ >>> arch/arm64/kernel/cpufeature.c | 5 ++-- >>> arch/arm64/kernel/smp.c | 38 +++++++++++++++++++++++++++++ >>> 4 files changed, 46 insertions(+), 9 deletions(-) >>> >>> diff --git a/arch/arm64/include/asm/cpufeature.h b/arch/arm64/include/asm/cpufeature.h >>> index f3aa281b97cb..513570cd3e6f 100644 >>> --- a/arch/arm64/include/asm/cpufeature.h >>> +++ b/arch/arm64/include/asm/cpufeature.h >>> @@ -289,12 +289,6 @@ extern struct arm64_ftr_reg arm64_ftr_reg_ctrel0; >>> (ARM64_CPUCAP_SCOPE_LOCAL_CPU | \ >>> ARM64_CPUCAP_OPTIONAL_FOR_LATE_CPU) >>> >>> -/* >>> - * CPU feature used early in the boot based on the boot CPU. All secondary >>> - * CPUs must match the state of the capability as detected by the boot CPU. >>> - */ >>> -#define ARM64_CPUCAP_STRICT_BOOT_CPU_FEATURE ARM64_CPUCAP_SCOPE_BOOT_CPU >>> - >>> struct arm64_cpu_capabilities { >>> const char *desc; >>> u16 capability; >>> diff --git a/arch/arm64/include/asm/virt.h b/arch/arm64/include/asm/virt.h >>> index 9d1e24e030b3..c5f89442785c 100644 >>> --- a/arch/arm64/include/asm/virt.h >>> +++ b/arch/arm64/include/asm/virt.h >>> @@ -102,6 +102,12 @@ static inline bool has_vhe(void) >>> return false; >>> } >>> >>> +#ifdef CONFIG_ARM64_VHE >>> +extern void verify_cpu_run_el(void); >>> +#else >>> +static inline void verify_cpu_run_el(void) {} >>> +#endif >>> + >>> #endif /* __ASSEMBLY__ */ >>> >>> #endif /* ! __ASM__VIRT_H */ >>> diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c >>> index d19a7b8bb6fb..672443860341 100644 >>> --- a/arch/arm64/kernel/cpufeature.c >>> +++ b/arch/arm64/kernel/cpufeature.c >>> @@ -1113,15 +1113,13 @@ static const struct arm64_cpu_capabilities arm64_features[] = { >>> .matches = cpufeature_pan_not_uao, >>> }, >>> #endif /* CONFIG_ARM64_PAN */ >>> -#ifdef CONFIG_ARM64_VHE >>> { >>> .desc = "Virtualization Host Extensions", >>> .capability = ARM64_HAS_VIRT_HOST_EXTN, >>> - .type = ARM64_CPUCAP_STRICT_BOOT_CPU_FEATURE, >>> + .type = ARM64_CPUCAP_SYSTEM_FEATURE, >>> .matches = runs_at_el2, >>> .cpu_enable = cpu_copy_el2regs, >>> }, >>> -#endif /* CONFIG_ARM64_VHE */ >>> { >>> .desc = "32-bit EL0 Support", >>> .capability = ARM64_HAS_32BIT_EL0, >>> @@ -1491,6 +1489,7 @@ static bool verify_local_cpu_caps(u16 scope_mask) >>> */ >>> static void check_early_cpu_features(void) >>> { >>> + verify_cpu_run_el(); >>> verify_cpu_asid_bits(); >>> /* >>> * Early features are used by the kernel already. If there >>> diff --git a/arch/arm64/kernel/smp.c b/arch/arm64/kernel/smp.c >>> index 2faa9863d2e5..a077c3d2fd73 100644 >>> --- a/arch/arm64/kernel/smp.c >>> +++ b/arch/arm64/kernel/smp.c >>> @@ -85,6 +85,43 @@ enum ipi_msg_type { >>> IPI_WAKEUP >>> }; >>> >>> +#ifdef CONFIG_ARM64_VHE >>> + >>> +/* Whether the boot CPU is running in HYP mode or not*/ >>> +static bool boot_cpu_hyp_mode; >>> + >>> +static inline void save_boot_cpu_run_el(void) >>> +{ >>> + boot_cpu_hyp_mode = is_kernel_in_hyp_mode(); >>> +} >>> + >>> +static inline bool is_boot_cpu_in_hyp_mode(void) >>> +{ >>> + return boot_cpu_hyp_mode; >>> +} >>> + >>> +/* >>> + * Verify that a secondary CPU is running the kernel at the same >>> + * EL as that of the boot CPU. >>> + */ >>> +void verify_cpu_run_el(void) >>> +{ >>> + bool in_el2 = is_kernel_in_hyp_mode(); >>> + bool boot_cpu_el2 = is_boot_cpu_in_hyp_mode(); >>> + >>> + if (in_el2 ^ boot_cpu_el2) { >>> + pr_crit("CPU%d: mismatched Exception Level(EL%d) with boot CPU(EL%d)\n", >>> + smp_processor_id(), >>> + in_el2 ? 2 : 1, >>> + boot_cpu_el2 ? 2 : 1); >>> + cpu_panic_kernel(); >>> + } >>> +} >>> + >>> +#else >>> +static inline void save_boot_cpu_run_el(void) {} >>> +#endif >>> + >>> #ifdef CONFIG_HOTPLUG_CPU >>> static int op_cpu_kill(unsigned int cpu); >>> #else >>> @@ -410,6 +447,7 @@ void __init smp_prepare_boot_cpu(void) >>> */ >>> jump_label_init(); >>> cpuinfo_store_boot_cpu(); >>> + save_boot_cpu_run_el(); >>> } >>> >>> static u64 __init of_get_cpu_mpidr(struct device_node *dn) >>> -- >>> 2.17.1 >>> >>> >>> -- >>> kernel-team mailing list >>> kernel-team@lists.ubuntu.com >>> https://lists.ubuntu.com/mailman/listinfo/kernel-team >> > >
diff --git a/arch/arm64/include/asm/cpufeature.h b/arch/arm64/include/asm/cpufeature.h index f3aa281b97cb..513570cd3e6f 100644 --- a/arch/arm64/include/asm/cpufeature.h +++ b/arch/arm64/include/asm/cpufeature.h @@ -289,12 +289,6 @@ extern struct arm64_ftr_reg arm64_ftr_reg_ctrel0; (ARM64_CPUCAP_SCOPE_LOCAL_CPU | \ ARM64_CPUCAP_OPTIONAL_FOR_LATE_CPU) -/* - * CPU feature used early in the boot based on the boot CPU. All secondary - * CPUs must match the state of the capability as detected by the boot CPU. - */ -#define ARM64_CPUCAP_STRICT_BOOT_CPU_FEATURE ARM64_CPUCAP_SCOPE_BOOT_CPU - struct arm64_cpu_capabilities { const char *desc; u16 capability; diff --git a/arch/arm64/include/asm/virt.h b/arch/arm64/include/asm/virt.h index 9d1e24e030b3..c5f89442785c 100644 --- a/arch/arm64/include/asm/virt.h +++ b/arch/arm64/include/asm/virt.h @@ -102,6 +102,12 @@ static inline bool has_vhe(void) return false; } +#ifdef CONFIG_ARM64_VHE +extern void verify_cpu_run_el(void); +#else +static inline void verify_cpu_run_el(void) {} +#endif + #endif /* __ASSEMBLY__ */ #endif /* ! __ASM__VIRT_H */ diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c index d19a7b8bb6fb..672443860341 100644 --- a/arch/arm64/kernel/cpufeature.c +++ b/arch/arm64/kernel/cpufeature.c @@ -1113,15 +1113,13 @@ static const struct arm64_cpu_capabilities arm64_features[] = { .matches = cpufeature_pan_not_uao, }, #endif /* CONFIG_ARM64_PAN */ -#ifdef CONFIG_ARM64_VHE { .desc = "Virtualization Host Extensions", .capability = ARM64_HAS_VIRT_HOST_EXTN, - .type = ARM64_CPUCAP_STRICT_BOOT_CPU_FEATURE, + .type = ARM64_CPUCAP_SYSTEM_FEATURE, .matches = runs_at_el2, .cpu_enable = cpu_copy_el2regs, }, -#endif /* CONFIG_ARM64_VHE */ { .desc = "32-bit EL0 Support", .capability = ARM64_HAS_32BIT_EL0, @@ -1491,6 +1489,7 @@ static bool verify_local_cpu_caps(u16 scope_mask) */ static void check_early_cpu_features(void) { + verify_cpu_run_el(); verify_cpu_asid_bits(); /* * Early features are used by the kernel already. If there diff --git a/arch/arm64/kernel/smp.c b/arch/arm64/kernel/smp.c index 2faa9863d2e5..a077c3d2fd73 100644 --- a/arch/arm64/kernel/smp.c +++ b/arch/arm64/kernel/smp.c @@ -85,6 +85,43 @@ enum ipi_msg_type { IPI_WAKEUP }; +#ifdef CONFIG_ARM64_VHE + +/* Whether the boot CPU is running in HYP mode or not*/ +static bool boot_cpu_hyp_mode; + +static inline void save_boot_cpu_run_el(void) +{ + boot_cpu_hyp_mode = is_kernel_in_hyp_mode(); +} + +static inline bool is_boot_cpu_in_hyp_mode(void) +{ + return boot_cpu_hyp_mode; +} + +/* + * Verify that a secondary CPU is running the kernel at the same + * EL as that of the boot CPU. + */ +void verify_cpu_run_el(void) +{ + bool in_el2 = is_kernel_in_hyp_mode(); + bool boot_cpu_el2 = is_boot_cpu_in_hyp_mode(); + + if (in_el2 ^ boot_cpu_el2) { + pr_crit("CPU%d: mismatched Exception Level(EL%d) with boot CPU(EL%d)\n", + smp_processor_id(), + in_el2 ? 2 : 1, + boot_cpu_el2 ? 2 : 1); + cpu_panic_kernel(); + } +} + +#else +static inline void save_boot_cpu_run_el(void) {} +#endif + #ifdef CONFIG_HOTPLUG_CPU static int op_cpu_kill(unsigned int cpu); #else @@ -410,6 +447,7 @@ void __init smp_prepare_boot_cpu(void) */ jump_label_init(); cpuinfo_store_boot_cpu(); + save_boot_cpu_run_el(); } static u64 __init of_get_cpu_mpidr(struct device_node *dn)
BugLink: http://bugs.launchpad.net/bugs/1852723 The 4.15.0-71.80 generic kernel can not boot on several arm64 machines. Bisect patches and find out we can solve this problem by reverting a single patch which comes from stable kernel update. This reverts commit 628859e8621cbb105cfbfa2138a68ddbd72958a7. Signed-off-by: Ike Panhc <ike.pan@canonical.com> --- arch/arm64/include/asm/cpufeature.h | 6 ----- arch/arm64/include/asm/virt.h | 6 +++++ arch/arm64/kernel/cpufeature.c | 5 ++-- arch/arm64/kernel/smp.c | 38 +++++++++++++++++++++++++++++ 4 files changed, 46 insertions(+), 9 deletions(-)