@@ -1191,8 +1191,29 @@ _GLOBAL(ftrace_caller)
2: // Here we have our proper TOC ptr in R2,
// and the one we need to restore on return in r0.
- ld r12, 16(r1) // get caller's adress
+ ld r12, LRSAVE(r1) // get caller's adress
+ // Is there a TOC restore insn?
+ std r11,-8(r1)
+ std r12,-16(r1)
+ lwz r12,0(r12)
+ lis r11,0xe841 // "ld r2,TOCSAVE(r1)"
+ li r11,TOCSAVE
+ cmpd r12,r11
+ ld r11,-8(r1)
+ ld r12,-16(r1)
+ beq 4f
+ // TODO: test if it's really a NOP. What if not?
+
+ // If no TOC restore insn, then it was a local call and
+ // the callee's TOC needs to be restored as caller's TOC
+ // in case of live patching.
+ std r0,TOCSAVE(r1)
+ stdu r1,-32(r1) // open new mini stack frame
+ LOAD_REG_IMMEDIATE(r11,KLP_return_helper)
+ std r11,LRSAVE(r1)
+ ld r11,24(r1) // restore again, from -8(r1+32)
+4:
stdu r1,-SWITCH_FRAME_SIZE(r1)
std r12, _LINK(r1)
@@ -1265,6 +1286,23 @@ _GLOBAL(ftrace_stub)
nop
.localentry ftrace_stub,.-ftrace_stub
blr
+
+/* Helper functions for local calls that are becoming global
+ due to live patching.
+ We can't simply patch the NOP after the original call,
+ because, depending on the consistency model, some kernel
+ threads may still have called the original, local function
+ *without* saving their TOC in the respective stack frame slot,
+ so the decision is made per-thread during function return by
+ maybe inserting a KLP_return_helper frame or not.
+*/
+KLP_return_helper:
+ ld r0,LRSAVE(r1) // get the real return address
+ addi r1, r1, 32 // destroy this stack frame
+ ld r2,TOCSAVE(r1) // restore TOC
+ mtlr r0
+ blr
+
#else
_GLOBAL_TOC(_mcount)
/* Taken from output of objdump from lib64/glibc */
Here is my approach to handle ABIv2 local calls that naturally become global when passing through KLP. On ppc64le a call is either local within the same module, and the TOC pointer is neither saved nor restored by the caller, or it is a global call that passes over a TOC-saving trampoline and the next instruction after the bl restores it. The problem I'm struggling with is the case where KLP redirects a local call into the newly-loaded module, making it global. I have considered and discarded some solutions, but have not detected a flaw in this one so far ;-) GCC adds a NOP after every function call that can be patched by the module linker to restore the TOC; the module linker also generates the matching trampolines. So, if I see the restore instruction, as generated by the module linker, I assume the call already is global and takes care of the caller's TOC. KLP can simply change the call into the new functions' global entry. Should it be a local call that passes through ftrace_caller (where klp_ftrace_handler registers itself), I add a small stack frame that restores the TOC on return. This way it can be done on a per- thread basis to support various consistency models. Please skim over this patch and tell me if you see any problems. (This is on top of my ftrace with regs series, which probably needs another iteration) Signed-off-by: Torsten Duwe <duwe@suse.de>