diff mbox

Next April 28: boot failure on PowerPC with SLQB

Message ID 20090430041146.GB23746@wotan.suse.de (mailing list archive)
State Not Applicable, archived
Headers show

Commit Message

Nick Piggin April 30, 2009, 4:11 a.m. UTC 
On Wed, Apr 29, 2009 at 09:56:19PM +0530, Sachin Sant wrote:
> Nick Piggin wrote:
> >Does this help?
> >---
> With the patch the machine boots past the failure point, but panics
> immediately with the following trace...

OK good, that solves one problem.

 
> Unable to handle kernel paging request for data at address 0x00000010
> Faulting instruction address: 0xc0000000007d03ec
> Oops: Kernel access of bad area, sig: 11 [#1]
> SMP NR_CPUS=1024 DEBUG_PAGEALLOC NUMA pSeries
> Modules linked in:
> NIP: c0000000007d03ec LR: c0000000007b0bbc CTR: 0000000000136f8c
> REGS: c000000000a23bd0 TRAP: 0300   Not tainted  (2.6.30-rc3-next-20090429)
> MSR: 8000000000009032 <EE,ME,IR,DR>  CR: 28000084  XER: 00000010
> DAR: 0000000000000010, DSISR: 0000000040000000
> TASK = c000000000955fc0[0] 'swapper' THREAD: c000000000a20000 CPU: 0
> GPR00: 0000000000000001 c000000000a23e50 c000000000a17650 000000000000001f
> GPR04: 0000000000000000 ffffffffffffffff 000000000077a4b9 800000000c9b2cc0
> GPR08: 0000000000000000 0000000000000010 0000000000000000 c00000000095b0f8
> GPR12: 0000000028000082 c000000000af2400 c0000000007f3200 c000000000705c32
> GPR16: 00000000014f3138 0000000000000000 c0000000007f3138 0000000002f1fc90
> GPR20: c0000000007f3150 c000000000725d11 00000000007bb8e4 0000000002f1fc90
> GPR24: 0000000002f1fc90 c0000000007f31f0 0000000000d00000 c000000000b73b10
> GPR28: c0000000007f0440 c00000000095db00 c00000000098d5f0 0000000003c90000
> NIP [c0000000007d03ec] .pidmap_init+0x28/0x88

Well kmalloc is failing. It should not be though, even if the
current node is offline, it should be able to fall back to other
nodes. Stephen's trace indicates the same thing.

Could you try the following patch please, and capture the output
it generates?

Thanks,
Nick
---
 mm/slqb.c |   40 ++++++++++++++++++++++++++++++++++------
 1 file changed, 34 insertions(+), 6 deletions(-)

Comments

Sachin P. Sant April 30, 2009, 5:36 a.m. UTC | #1 
Nick Piggin wrote:
> Well kmalloc is failing. It should not be though, even if the
> current node is offline, it should be able to fall back to other
> nodes. Stephen's trace indicates the same thing.
>
> Could you try the following patch please, and capture the output
> it generates?
With this patch i don't get any extra information other that what is 
already reported.
Have attached the boot log captured using loglevel=8 mminit_loglevel=4 
options.

Thanks
-Sachin
Nick Piggin April 30, 2009, 6:03 a.m. UTC | #2 
On Thu, Apr 30, 2009 at 11:06:36AM +0530, Sachin Sant wrote:
> Nick Piggin wrote:
> >Well kmalloc is failing. It should not be though, even if the
> >current node is offline, it should be able to fall back to other
> >nodes. Stephen's trace indicates the same thing.
> >
> >Could you try the following patch please, and capture the output
> >it generates?
> With this patch i don't get any extra information other that what is 
> already reported.
> Have attached the boot log captured using loglevel=8 mminit_loglevel=4 
> options.

Hmm, OK. Can you post the disassembly of your pidmap_init please?
Sachin P. Sant April 30, 2009, 6:42 a.m. UTC | #3 
Nick Piggin wrote:
> On Thu, Apr 30, 2009 at 11:06:36AM +0530, Sachin Sant wrote:
>   
> Hmm, OK. Can you post the disassembly of your pidmap_init please?
>   
Here is the information.

Unable to handle kernel paging request for data at address 0x00000010
Faulting instruction address: 0xc0000000007d0404
cpu 0x0: Vector: 300 (Data Access) at [c000000000a23bd0]
    pc: c0000000007d0404: .pidmap_init+0x28/0x88
    lr: c0000000007b0bbc: .start_kernel+0x458/0x51c
    sp: c000000000a23e50
   msr: 8000000000009032
   dar: 10
 dsisr: 40000000
  current = 0xc000000000955fc0
  paca    = 0xc000000000af2400
    pid   = 0, comm = swapper
enter ? for help
[c000000000a23ee0] c0000000007b0bbc .start_kernel+0x458/0x51c
[c000000000a23f90] c0000000000083d8 .start_here_common+0x1c/0x44
0:mon> ls .pidmap_init
.pidmap_init: c0000000007d03dc
0:mon> di c0000000007d03dc
c0000000007d03dc  fbc1fff0      std     r30,-16(r1)
c0000000007d03e0  ebc2b1a8      ld      r30,-20056(r2)
c0000000007d03e4  39200010      li      r9,16
c0000000007d03e8  7c0802a6      mflr    r0
c0000000007d03ec  fba1ffe8      std     r29,-24(r1)
c0000000007d03f0  f8010010      std     r0,16(r1)
c0000000007d03f4  38000001      li      r0,1
c0000000007d03f8  ebbe8008      ld      r29,-32760(r30)
c0000000007d03fc  f821ff71      stdu    r1,-144(r1)
c0000000007d0400  f93d0010      std     r9,16(r29)
c0000000007d0404  7d6048a8      ldarx   r11,0,r9
c0000000007d0408  7d6b0378      or      r11,r11,r0
c0000000007d040c  7d6049ad      stdcx.  r11,0,r9
c0000000007d0410  40c2fff4      bne-    c0000000007d0404        # .pidmap_init+0x28/0x88
c0000000007d0414  381d0008      addi    r0,r29,8
c0000000007d0418  7d200028      lwarx   r9,0,r0
0:mon>
c0000000007d041c  3129ffff      addic   r9,r9,-1
c0000000007d0420  7d20012d      stwcx.  r9,0,r0
c0000000007d0424  40c2fff4      bne-    c0000000007d0418        # .pidmap_init+0x3c/0x88
c0000000007d0428  3cc00004      lis     r6,4
c0000000007d042c  e87e8010      ld      r3,-32752(r30)
c0000000007d0430  38800050      li      r4,80
c0000000007d0434  60c62000      ori     r6,r6,8192
c0000000007d0438  38a00008      li      r5,8
c0000000007d043c  38e00000      li      r7,0
c0000000007d0440  4b9716d9      bl      c000000000141b18        # .kmem_cache_create+0x0/0x2a0
c0000000007d0444  60000000      nop
c0000000007d0448  38210090      addi    r1,r1,144
c0000000007d044c  f87d0098      std     r3,152(r29)
c0000000007d0450  e8010010      ld      r0,16(r1)
c0000000007d0454  eba1ffe8      ld      r29,-24(r1)
c0000000007d0458  7c0803a6      mtlr    r0
0:mon> r
R00 = 0000000000000001   R16 = 00000000014f3150
R01 = c000000000a23e50   R17 = 0000000000000000
R02 = c000000000a17690   R18 = c0000000007f3150
R03 = 000000000000001f   R19 = 0000000002f1fc90
R04 = 0000000000000000   R20 = c0000000007f3168
R05 = ffffffffffffffff   R21 = c000000000725e2f
R06 = 00000000007974d4   R22 = 00000000007bb977
R07 = 800000000c9b2cc0   R23 = 0000000002f1fc90
R08 = 0000000000000000   R24 = 0000000002f1fc90
R09 = 0000000000000010   R25 = c0000000007f3208
R10 = 0000000000000000   R26 = 0000000000d00000
R11 = c00000000095b0f8   R27 = c000000000b73b10
R12 = 0000000028000082   R28 = c0000000007f0458
R13 = c000000000af2400   R29 = c00000000095db00
R14 = c0000000007f3218   R30 = c00000000098d5f0
R15 = c000000000705c32   R31 = 0000000003c90000
pc  = c0000000007d0404 .pidmap_init+0x28/0x88
lr  = c0000000007b0bbc .start_kernel+0x458/0x51c
msr = 8000000000009032   cr  = 28000084
ctr = 0000000000136f8c   xer = 0000000000000010   trap =  300
dar = 0000000000000010   dsisr = 40000000
0:mon> di %pc
c0000000007d0404  7d6048a8      ldarx   r11,0,r9
c0000000007d0408  7d6b0378      or      r11,r11,r0
c0000000007d040c  7d6049ad      stdcx.  r11,0,r9
c0000000007d0410  40c2fff4      bne-    c0000000007d0404        # .pidmap_init+0x28/0x88
c0000000007d0414  381d0008      addi    r0,r29,8
c0000000007d0418  7d200028      lwarx   r9,0,r0
c0000000007d041c  3129ffff      addic   r9,r9,-1
c0000000007d0420  7d20012d      stwcx.  r9,0,r0
c0000000007d0424  40c2fff4      bne-    c0000000007d0418        # .pidmap_init+0x3c/0x88
c0000000007d0428  3cc00004      lis     r6,4
c0000000007d042c  e87e8010      ld      r3,-32752(r30)
c0000000007d0430  38800050      li      r4,80
c0000000007d0434  60c62000      ori     r6,r6,8192
c0000000007d0438  38a00008      li      r5,8
c0000000007d043c  38e00000      li      r7,0
c0000000007d0440  4b9716d9      bl      c000000000141b18        # .kmem_cache_create+0x0/0x2a0
0:mon>

Thanks
-Sachin
diff mbox

Patch

Index: linux-2.6/mm/slqb.c
===================================================================
--- linux-2.6.orig/mm/slqb.c
+++ linux-2.6/mm/slqb.c
@@ -1456,7 +1456,7 @@  static void *__remote_slab_alloc_node(st
 }
 
 static noinline void *__remote_slab_alloc(struct kmem_cache *s,
-				gfp_t gfpflags, int node)
+				gfp_t gfpflags, int node, int trace)
 {
 	void *object;
 	struct zonelist *zonelist;
@@ -1465,19 +1465,32 @@  static noinline void *__remote_slab_allo
 	enum zone_type high_zoneidx = gfp_zone(gfpflags);
 
 	object = __remote_slab_alloc_node(s, gfpflags, node);
+	if (trace && !object)
+		printk("__remote_slab_alloc_node(node:%d) failed\n", node);
 	if (likely(object || (gfpflags & __GFP_THISNODE)))
 		return object;
 
-	zonelist = node_zonelist(slab_node(current->mempolicy), gfpflags);
+	node = slab_node(current->mempolicy);
+	if (trace)
+		printk("slab_node(current->mempolicy) = %d\n", node);
+
+	zonelist = node_zonelist(node, gfpflags);
 	for_each_zone_zonelist(zone, z, zonelist, high_zoneidx) {
-		if (!cpuset_zone_allowed_hardwall(zone, gfpflags))
+		if (!cpuset_zone_allowed_hardwall(zone, gfpflags)) {
+			if (trace)
+				printk("cpuset not allowed node:%d\n", zone_to_nid(zone));
 			continue;
+		}
 
 		node = zone_to_nid(zone);
 		object = __remote_slab_alloc_node(s, gfpflags, node);
 		if (likely(object))
 			return object;
+		if (trace)
+			printk("__remote_slab_alloc_node(node:%d) failed\n", node);
 	}
+	if (trace)
+		printk("__remote_slab_alloc failed\n");
 	return NULL;
 }
 #endif
@@ -1488,7 +1501,7 @@  static noinline void *__remote_slab_allo
  * Must be called with interrupts disabled.
  */
 static __always_inline void *__slab_alloc(struct kmem_cache *s,
-				gfp_t gfpflags, int node)
+				gfp_t gfpflags, int node, int trace)
 {
 	void *object;
 	struct kmem_cache_cpu *c;
@@ -1497,7 +1510,7 @@  static __always_inline void *__slab_allo
 #ifdef CONFIG_NUMA
 	if (unlikely(node != -1) && unlikely(node != numa_node_id())) {
 try_remote:
-		return __remote_slab_alloc(s, gfpflags, node);
+		return __remote_slab_alloc(s, gfpflags, node, trace);
 	}
 #endif
 
@@ -1509,6 +1522,8 @@  try_remote:
 		object = cache_list_get_page(s, l);
 		if (unlikely(!object)) {
 			object = __slab_alloc_page(s, gfpflags, node);
+			if (trace && !object)
+				printk("__slab_alloc_page(node:%d) failed\n", node);
 #ifdef CONFIG_NUMA
 			if (unlikely(!object)) {
 				node = numa_node_id();
@@ -1532,10 +1547,11 @@  static __always_inline void *slab_alloc(
 {
 	void *object;
 	unsigned long flags;
+	int trace = 0;
 
 again:
 	local_irq_save(flags);
-	object = __slab_alloc(s, gfpflags, node);
+	object = __slab_alloc(s, gfpflags, node, trace);
 	local_irq_restore(flags);
 
 	if (unlikely(slab_debug(s)) && likely(object)) {
@@ -1546,6 +1562,18 @@  again:
 	if (unlikely(gfpflags & __GFP_ZERO) && likely(object))
 		memset(object, 0, s->objsize);
 
+	if (!object && !trace) {
+		trace = 1;
+		dump_stack();
+		printk("slab_alloc allocation failed\n");
+		printk("slab:%s flags:%x node:%d\n", s->name, gfpflags, node);
+		goto again;
+	}
+	if (trace) {
+		if (object)
+			printk("slab_alloc allocation worked when being traced, bugger\n");
+	}
+
 	return object;
 }