Message ID | 20180502130729.24077-3-mpe@ellerman.id.au (mailing list archive) |
---|---|
State | Accepted |
Commit | 5cc05910f26e6fd6da15f052f86f6150e4b91664 |
Headers | show |
Series | [RFC,1/4] powerpc/64: Save stack pointer when we hard disable interrupts | expand |
Hi Michael, It looks like this commit generates the following error: stacktrace.c:(.text+0x1b0): undefined reference to `.smp_send_safe_nmi_ipi' make[1]: *** [vmlinux] Error 1 make: *** [sub-make] Error 2 See http://kisskb.ellerman.id.au/kisskb/buildresult/13395345/ for details Seems like that function only exists when CONFIG_NMI_IPI is defined. Christophe Le 02/05/2018 à 15:07, Michael Ellerman a écrit : > This allows eg. the RCU stall detector, or the soft/hardlockup > detectors to trigger a backtrace on all CPUs. > > We implement this by sending a "safe" NMI, which will actually only > send an IPI. Unfortunately the generic code prints "NMI", so that's a > little confusing but we can probably live with it. > > If one of the CPUs doesn't respond to the IPI, we then print some info > from it's paca and do a backtrace based on its saved_r1. > > Example output: > > INFO: rcu_sched detected stalls on CPUs/tasks: > 2-...0: (0 ticks this GP) idle=1be/1/4611686018427387904 softirq=1055/1055 fqs=25735 > (detected by 4, t=58847 jiffies, g=58, c=57, q=1258) > Sending NMI from CPU 4 to CPUs 2: > CPU 2 didn't respond to backtrace IPI, inspecting paca. > irq_soft_mask: 0x01 in_mce: 0 in_nmi: 0 current: 3623 (bash) > Back trace of paca->saved_r1 (0xc0000000e1c83ba0) (possibly stale): > Call Trace: > [c0000000e1c83ba0] [0000000000000014] 0x14 (unreliable) > [c0000000e1c83bc0] [c000000000765798] lkdtm_do_action+0x48/0x80 > [c0000000e1c83bf0] [c000000000765a40] direct_entry+0x110/0x1b0 > [c0000000e1c83c90] [c00000000058e650] full_proxy_write+0x90/0xe0 > [c0000000e1c83ce0] [c0000000003aae3c] __vfs_write+0x6c/0x1f0 > [c0000000e1c83d80] [c0000000003ab214] vfs_write+0xd4/0x240 > [c0000000e1c83dd0] [c0000000003ab5cc] ksys_write+0x6c/0x110 > [c0000000e1c83e30] [c00000000000b860] system_call+0x58/0x6c > > Signed-off-by: Michael Ellerman <mpe@ellerman.id.au> > --- > arch/powerpc/include/asm/nmi.h | 4 ++++ > arch/powerpc/kernel/stacktrace.c | 51 ++++++++++++++++++++++++++++++++++++++++ > 2 files changed, 55 insertions(+) > > diff --git a/arch/powerpc/include/asm/nmi.h b/arch/powerpc/include/asm/nmi.h > index 9c80939b4d14..e97f58689ca7 100644 > --- a/arch/powerpc/include/asm/nmi.h > +++ b/arch/powerpc/include/asm/nmi.h > @@ -4,6 +4,10 @@ > > #ifdef CONFIG_PPC_WATCHDOG > extern void arch_touch_nmi_watchdog(void); > +extern void arch_trigger_cpumask_backtrace(const cpumask_t *mask, > + bool exclude_self); > +#define arch_trigger_cpumask_backtrace arch_trigger_cpumask_backtrace > + > #else > static inline void arch_touch_nmi_watchdog(void) {} > #endif > diff --git a/arch/powerpc/kernel/stacktrace.c b/arch/powerpc/kernel/stacktrace.c > index d534ed901538..cf4652d5df80 100644 > --- a/arch/powerpc/kernel/stacktrace.c > +++ b/arch/powerpc/kernel/stacktrace.c > @@ -11,12 +11,15 @@ > */ > > #include <linux/export.h> > +#include <linux/nmi.h> > #include <linux/sched.h> > #include <linux/sched/debug.h> > #include <linux/stacktrace.h> > #include <asm/ptrace.h> > #include <asm/processor.h> > > +#include <asm/paca.h> > + > /* > * Save stack-backtrace addresses into a stack_trace buffer. > */ > @@ -76,3 +79,51 @@ save_stack_trace_regs(struct pt_regs *regs, struct stack_trace *trace) > save_context_stack(trace, regs->gpr[1], current, 0); > } > EXPORT_SYMBOL_GPL(save_stack_trace_regs); > + > +#ifdef CONFIG_PPC_BOOK3S_64 > +static void handle_backtrace_ipi(struct pt_regs *regs) > +{ > + nmi_cpu_backtrace(regs); > +} > + > +static void raise_backtrace_ipi(cpumask_t *mask) > +{ > + unsigned int cpu; > + > + for_each_cpu(cpu, mask) { > + if (cpu == smp_processor_id()) > + handle_backtrace_ipi(NULL); > + else > + smp_send_safe_nmi_ipi(cpu, handle_backtrace_ipi, 5 * USEC_PER_SEC); > + } > + > + for_each_cpu(cpu, mask) { > + struct paca_struct *p = paca_ptrs[cpu]; > + > + cpumask_clear_cpu(cpu, mask); > + > + pr_warn("CPU %d didn't respond to backtrace IPI, inspecting paca.\n", cpu); > + if (!virt_addr_valid(p)) { > + pr_warn("paca pointer appears corrupt? (%px)\n", p); > + continue; > + } > + > + pr_warn("irq_soft_mask: 0x%02x in_mce: %d in_nmi: %d", > + p->irq_soft_mask, p->in_mce, p->in_nmi); > + > + if (virt_addr_valid(p->__current)) > + pr_cont(" current: %d (%s)\n", p->__current->pid, > + p->__current->comm); > + else > + pr_cont(" current pointer corrupt? (%px)\n", p->__current); > + > + pr_warn("Back trace of paca->saved_r1 (0x%016llx) (possibly stale):\n", p->saved_r1); > + show_stack(p->__current, (unsigned long *)p->saved_r1); > + } > +} > + > +void arch_trigger_cpumask_backtrace(const cpumask_t *mask, bool exclude_self) > +{ > + nmi_trigger_cpumask_backtrace(mask, exclude_self, raise_backtrace_ipi); > +} > +#endif /* CONFIG_PPC64 */ >
diff --git a/arch/powerpc/include/asm/nmi.h b/arch/powerpc/include/asm/nmi.h index 9c80939b4d14..e97f58689ca7 100644 --- a/arch/powerpc/include/asm/nmi.h +++ b/arch/powerpc/include/asm/nmi.h @@ -4,6 +4,10 @@ #ifdef CONFIG_PPC_WATCHDOG extern void arch_touch_nmi_watchdog(void); +extern void arch_trigger_cpumask_backtrace(const cpumask_t *mask, + bool exclude_self); +#define arch_trigger_cpumask_backtrace arch_trigger_cpumask_backtrace + #else static inline void arch_touch_nmi_watchdog(void) {} #endif diff --git a/arch/powerpc/kernel/stacktrace.c b/arch/powerpc/kernel/stacktrace.c index d534ed901538..cf4652d5df80 100644 --- a/arch/powerpc/kernel/stacktrace.c +++ b/arch/powerpc/kernel/stacktrace.c @@ -11,12 +11,15 @@ */ #include <linux/export.h> +#include <linux/nmi.h> #include <linux/sched.h> #include <linux/sched/debug.h> #include <linux/stacktrace.h> #include <asm/ptrace.h> #include <asm/processor.h> +#include <asm/paca.h> + /* * Save stack-backtrace addresses into a stack_trace buffer. */ @@ -76,3 +79,51 @@ save_stack_trace_regs(struct pt_regs *regs, struct stack_trace *trace) save_context_stack(trace, regs->gpr[1], current, 0); } EXPORT_SYMBOL_GPL(save_stack_trace_regs); + +#ifdef CONFIG_PPC_BOOK3S_64 +static void handle_backtrace_ipi(struct pt_regs *regs) +{ + nmi_cpu_backtrace(regs); +} + +static void raise_backtrace_ipi(cpumask_t *mask) +{ + unsigned int cpu; + + for_each_cpu(cpu, mask) { + if (cpu == smp_processor_id()) + handle_backtrace_ipi(NULL); + else + smp_send_safe_nmi_ipi(cpu, handle_backtrace_ipi, 5 * USEC_PER_SEC); + } + + for_each_cpu(cpu, mask) { + struct paca_struct *p = paca_ptrs[cpu]; + + cpumask_clear_cpu(cpu, mask); + + pr_warn("CPU %d didn't respond to backtrace IPI, inspecting paca.\n", cpu); + if (!virt_addr_valid(p)) { + pr_warn("paca pointer appears corrupt? (%px)\n", p); + continue; + } + + pr_warn("irq_soft_mask: 0x%02x in_mce: %d in_nmi: %d", + p->irq_soft_mask, p->in_mce, p->in_nmi); + + if (virt_addr_valid(p->__current)) + pr_cont(" current: %d (%s)\n", p->__current->pid, + p->__current->comm); + else + pr_cont(" current pointer corrupt? (%px)\n", p->__current); + + pr_warn("Back trace of paca->saved_r1 (0x%016llx) (possibly stale):\n", p->saved_r1); + show_stack(p->__current, (unsigned long *)p->saved_r1); + } +} + +void arch_trigger_cpumask_backtrace(const cpumask_t *mask, bool exclude_self) +{ + nmi_trigger_cpumask_backtrace(mask, exclude_self, raise_backtrace_ipi); +} +#endif /* CONFIG_PPC64 */
This allows eg. the RCU stall detector, or the soft/hardlockup detectors to trigger a backtrace on all CPUs. We implement this by sending a "safe" NMI, which will actually only send an IPI. Unfortunately the generic code prints "NMI", so that's a little confusing but we can probably live with it. If one of the CPUs doesn't respond to the IPI, we then print some info from it's paca and do a backtrace based on its saved_r1. Example output: INFO: rcu_sched detected stalls on CPUs/tasks: 2-...0: (0 ticks this GP) idle=1be/1/4611686018427387904 softirq=1055/1055 fqs=25735 (detected by 4, t=58847 jiffies, g=58, c=57, q=1258) Sending NMI from CPU 4 to CPUs 2: CPU 2 didn't respond to backtrace IPI, inspecting paca. irq_soft_mask: 0x01 in_mce: 0 in_nmi: 0 current: 3623 (bash) Back trace of paca->saved_r1 (0xc0000000e1c83ba0) (possibly stale): Call Trace: [c0000000e1c83ba0] [0000000000000014] 0x14 (unreliable) [c0000000e1c83bc0] [c000000000765798] lkdtm_do_action+0x48/0x80 [c0000000e1c83bf0] [c000000000765a40] direct_entry+0x110/0x1b0 [c0000000e1c83c90] [c00000000058e650] full_proxy_write+0x90/0xe0 [c0000000e1c83ce0] [c0000000003aae3c] __vfs_write+0x6c/0x1f0 [c0000000e1c83d80] [c0000000003ab214] vfs_write+0xd4/0x240 [c0000000e1c83dd0] [c0000000003ab5cc] ksys_write+0x6c/0x110 [c0000000e1c83e30] [c00000000000b860] system_call+0x58/0x6c Signed-off-by: Michael Ellerman <mpe@ellerman.id.au> --- arch/powerpc/include/asm/nmi.h | 4 ++++ arch/powerpc/kernel/stacktrace.c | 51 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 55 insertions(+)