From: Christophe Leroy Date: Mon, 20 Dec 2021 16:38:44 +0000 (+0000) Subject: powerpc/ftrace: Remove ftrace_32.S X-Git-Url: http://git.maquefel.me/?a=commitdiff_plain;h=4ee83a2cfbc46c13f2a08fe6d48dbcede53cdbf8;p=linux.git powerpc/ftrace: Remove ftrace_32.S Functions in ftrace_32.S are common with PPC64. Reuse the ones defined for PPC64 with slight modification when required. Signed-off-by: Christophe Leroy [mpe: Squash in fixup diff from Christophe] Signed-off-by: Michael Ellerman Link: https://lore.kernel.org/r/5e837fc190504c4ef834272e70d60ae33f175d49.1640017960.git.christophe.leroy@csgroup.eu --- diff --git a/arch/powerpc/kernel/trace/Makefile b/arch/powerpc/kernel/trace/Makefile index 858503775c583..542aa7a8b2b42 100644 --- a/arch/powerpc/kernel/trace/Makefile +++ b/arch/powerpc/kernel/trace/Makefile @@ -8,13 +8,13 @@ ifdef CONFIG_FUNCTION_TRACER CFLAGS_REMOVE_ftrace.o = $(CC_FLAGS_FTRACE) endif -obj32-$(CONFIG_FUNCTION_TRACER) += ftrace_32.o -obj64-$(CONFIG_FUNCTION_TRACER) += ftrace_64.o +obj32-$(CONFIG_FUNCTION_TRACER) += ftrace_mprofile.o ifdef CONFIG_MPROFILE_KERNEL -obj64-$(CONFIG_FUNCTION_TRACER) += ftrace_64_mprofile.o +obj64-$(CONFIG_FUNCTION_TRACER) += ftrace_mprofile.o else obj64-$(CONFIG_FUNCTION_TRACER) += ftrace_64_pg.o endif +obj-$(CONFIG_FUNCTION_TRACER) += ftrace_low.o obj-$(CONFIG_DYNAMIC_FTRACE) += ftrace.o obj-$(CONFIG_FUNCTION_GRAPH_TRACER) += ftrace.o obj-$(CONFIG_FTRACE_SYSCALLS) += ftrace.o diff --git a/arch/powerpc/kernel/trace/ftrace_32.S b/arch/powerpc/kernel/trace/ftrace_32.S deleted file mode 100644 index 2b425da97a6bb..0000000000000 --- a/arch/powerpc/kernel/trace/ftrace_32.S +++ /dev/null @@ -1,152 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-or-later */ -/* - * Split from entry_32.S - */ - -#include -#include -#include -#include -#include -#include -#include - -_GLOBAL(mcount) -_GLOBAL(_mcount) - /* - * It is required that _mcount on PPC32 must preserve the - * link register. But we have r12 to play with. We use r12 - * to push the return address back to the caller of mcount - * into the ctr register, restore the link register and - * then jump back using the ctr register. - */ - mflr r12 - mtctr r12 - mtlr r0 - bctr -EXPORT_SYMBOL(_mcount) - -_GLOBAL(ftrace_caller) - stwu r1, -INT_FRAME_SIZE(r1) - - SAVE_GPRS(3, 10, r1) - - addi r8, r1, INT_FRAME_SIZE - stw r8, GPR1(r1) - - mflr r3 - stw r3, _NIP(r1) - subi r3, r3, MCOUNT_INSN_SIZE - - stw r0, _LINK(r1) - mr r4, r0 - - lis r5,function_trace_op@ha - lwz r5,function_trace_op@l(r5) - - addi r6, r1, STACK_FRAME_OVERHEAD -.globl ftrace_call -ftrace_call: - bl ftrace_stub - nop - - lwz r3, _NIP(r1) - mtctr r3 - - REST_GPRS(3, 10, r1) - - lwz r0, _LINK(r1) - mtlr r0 - - addi r1, r1, INT_FRAME_SIZE - /* old link register ends up in ctr reg */ - bctr - - -_GLOBAL(ftrace_stub) - blr - -_GLOBAL(ftrace_regs_caller) - /* Create our stack frame + pt_regs */ - stwu r1,-INT_FRAME_SIZE(r1) - - /* Save all gprs to pt_regs */ - stw r0, GPR0(r1) - stmw r2, GPR2(r1) - - /* Save previous stack pointer (r1) */ - addi r8, r1, INT_FRAME_SIZE - stw r8, GPR1(r1) - - /* Load special regs for save below */ - mfmsr r8 - mfctr r9 - mfxer r10 - mfcr r11 - - /* Get the _mcount() call site out of LR */ - mflr r7 - /* Save it as pt_regs->nip */ - stw r7, _NIP(r1) - /* Save the read LR in pt_regs->link */ - stw r0, _LINK(r1) - - lis r3,function_trace_op@ha - lwz r5,function_trace_op@l(r3) - - /* Calculate ip from nip-4 into r3 for call below */ - subi r3, r7, MCOUNT_INSN_SIZE - - /* Put the original return address in r4 as parent_ip */ - mr r4, r0 - - /* Save special regs */ - stw r8, _MSR(r1) - stw r9, _CTR(r1) - stw r10, _XER(r1) - stw r11, _CCR(r1) - - /* Load &pt_regs in r6 for call below */ - addi r6, r1, STACK_FRAME_OVERHEAD - - /* ftrace_call(r3, r4, r5, r6) */ -.globl ftrace_regs_call -ftrace_regs_call: - bl ftrace_stub - nop - - /* Load ctr with the possibly modified NIP */ - lwz r3, _NIP(r1) - mtctr r3 - - /* Restore gprs */ - lmw r2, GPR2(r1) - - /* Restore possibly modified LR */ - lwz r0, _LINK(r1) - mtlr r0 - - /* Pop our stack frame */ - addi r1, r1, INT_FRAME_SIZE - /* old link register ends up in ctr reg */ - bctr - -#ifdef CONFIG_FUNCTION_GRAPH_TRACER -_GLOBAL(return_to_handler) - /* need to save return values */ - stwu r1, -16(r1) - stw r3, 8(r1) - stw r4, 12(r1) - - bl ftrace_return_to_handler - - /* return value has real return address */ - mtlr r3 - - lwz r3, 8(r1) - lwz r4, 12(r1) - addi r1, r1, 16 - - /* Jump back to real return address */ - blr -#endif /* CONFIG_FUNCTION_GRAPH_TRACER */ diff --git a/arch/powerpc/kernel/trace/ftrace_64.S b/arch/powerpc/kernel/trace/ftrace_64.S deleted file mode 100644 index 25e5b9e47c06d..0000000000000 --- a/arch/powerpc/kernel/trace/ftrace_64.S +++ /dev/null @@ -1,64 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-or-later */ -/* - * Split from entry_64.S - */ - -#include -#include -#include -#include -#include -#include - -.pushsection ".tramp.ftrace.text","aw",@progbits; -.globl ftrace_tramp_text -ftrace_tramp_text: - .space 64 -.popsection - -.pushsection ".tramp.ftrace.init","aw",@progbits; -.globl ftrace_tramp_init -ftrace_tramp_init: - .space 64 -.popsection - -_GLOBAL(mcount) -_GLOBAL(_mcount) -EXPORT_SYMBOL(_mcount) - mflr r12 - mtctr r12 - mtlr r0 - bctr - -#ifdef CONFIG_FUNCTION_GRAPH_TRACER -_GLOBAL(return_to_handler) - /* need to save return values */ - std r4, -32(r1) - std r3, -24(r1) - /* save TOC */ - std r2, -16(r1) - std r31, -8(r1) - mr r31, r1 - stdu r1, -112(r1) - - /* - * We might be called from a module. - * Switch to our TOC to run inside the core kernel. - */ - ld r2, PACATOC(r13) - - bl ftrace_return_to_handler - nop - - /* return value has real return address */ - mtlr r3 - - ld r1, 0(r1) - ld r4, -32(r1) - ld r3, -24(r1) - ld r2, -16(r1) - ld r31, -8(r1) - - /* Jump back to real return address */ - blr -#endif /* CONFIG_FUNCTION_GRAPH_TRACER */ diff --git a/arch/powerpc/kernel/trace/ftrace_64_mprofile.S b/arch/powerpc/kernel/trace/ftrace_64_mprofile.S deleted file mode 100644 index 56da60e98327a..0000000000000 --- a/arch/powerpc/kernel/trace/ftrace_64_mprofile.S +++ /dev/null @@ -1,307 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-or-later */ -/* - * Split from ftrace_64.S - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -/* - * - * ftrace_caller()/ftrace_regs_caller() is the function that replaces _mcount() - * when ftrace is active. - * - * We arrive here after a function A calls function B, and we are the trace - * function for B. When we enter r1 points to A's stack frame, B has not yet - * had a chance to allocate one yet. - * - * Additionally r2 may point either to the TOC for A, or B, depending on - * whether B did a TOC setup sequence before calling us. - * - * On entry the LR points back to the _mcount() call site, and r0 holds the - * saved LR as it was on entry to B, ie. the original return address at the - * call site in A. - * - * Our job is to save the register state into a struct pt_regs (on the stack) - * and then arrange for the ftrace function to be called. - */ -_GLOBAL(ftrace_regs_caller) - /* Save the original return address in A's stack frame */ -#ifdef CONFIG_MPROFILE_KERNEL - PPC_STL r0,LRSAVE(r1) -#endif - - /* Create our stack frame + pt_regs */ - PPC_STLU r1,-SWITCH_FRAME_SIZE(r1) - - /* Save all gprs to pt_regs */ - SAVE_GPR(0, r1) -#ifdef CONFIG_PPC64 - SAVE_GPRS(2, 11, r1) - - /* Ok to continue? */ - lbz r3, PACA_FTRACE_ENABLED(r13) - cmpdi r3, 0 - beq ftrace_no_trace - - SAVE_GPRS(12, 31, r1) -#else - stmw r2, GPR2(r1) -#endif - - /* Save previous stack pointer (r1) */ - addi r8, r1, SWITCH_FRAME_SIZE - PPC_STL r8, GPR1(r1) - - /* Load special regs for save below */ - mfmsr r8 - mfctr r9 - mfxer r10 - mfcr r11 - - /* Get the _mcount() call site out of LR */ - mflr r7 - /* Save it as pt_regs->nip */ - PPC_STL r7, _NIP(r1) - /* Save the read LR in pt_regs->link */ - PPC_STL r0, _LINK(r1) - -#ifdef CONFIG_PPC64 - /* Save callee's TOC in the ABI compliant location */ - std r2, 24(r1) - ld r2,PACATOC(r13) /* get kernel TOC in r2 */ - - addis r3,r2,function_trace_op@toc@ha - addi r3,r3,function_trace_op@toc@l - ld r5,0(r3) -#else - lis r3,function_trace_op@ha - lwz r5,function_trace_op@l(r3) -#endif - -#ifdef CONFIG_LIVEPATCH_64 - mr r14,r7 /* remember old NIP */ -#endif - /* Calculate ip from nip-4 into r3 for call below */ - subi r3, r7, MCOUNT_INSN_SIZE - - /* Put the original return address in r4 as parent_ip */ - mr r4, r0 - - /* Save special regs */ - PPC_STL r8, _MSR(r1) - PPC_STL r9, _CTR(r1) - PPC_STL r10, _XER(r1) - PPC_STL r11, _CCR(r1) - - /* Load &pt_regs in r6 for call below */ - addi r6, r1 ,STACK_FRAME_OVERHEAD - - /* ftrace_call(r3, r4, r5, r6) */ -.globl ftrace_regs_call -ftrace_regs_call: - bl ftrace_stub - nop - - /* Load ctr with the possibly modified NIP */ - PPC_LL r3, _NIP(r1) - mtctr r3 -#ifdef CONFIG_LIVEPATCH_64 - cmpd r14, r3 /* has NIP been altered? */ -#endif - - /* Restore gprs */ -#ifdef CONFIG_PPC64 - REST_GPRS(2, 31, r1) -#else - lmw r2, GPR2(r1) -#endif - - /* Restore possibly modified LR */ - PPC_LL r0, _LINK(r1) - mtlr r0 - -#ifdef CONFIG_PPC64 - /* Restore callee's TOC */ - ld r2, 24(r1) -#endif - - /* Pop our stack frame */ - addi r1, r1, SWITCH_FRAME_SIZE - -#ifdef CONFIG_LIVEPATCH_64 - /* Based on the cmpd above, if the NIP was altered handle livepatch */ - bne- livepatch_handler -#endif - bctr /* jump after _mcount site */ - -_GLOBAL(ftrace_stub) - blr - -#ifdef CONFIG_PPC64 -ftrace_no_trace: - mflr r3 - mtctr r3 - REST_GPR(3, r1) - addi r1, r1, SWITCH_FRAME_SIZE - mtlr r0 - bctr -#endif - -_GLOBAL(ftrace_caller) - /* Save the original return address in A's stack frame */ -#ifdef CONFIG_MPROFILE_KERNEL - PPC_STL r0, LRSAVE(r1) -#endif - - /* Create our stack frame + pt_regs */ - PPC_STLU r1, -SWITCH_FRAME_SIZE(r1) - - /* Save all gprs to pt_regs */ - SAVE_GPRS(3, 10, r1) - -#ifdef CONFIG_PPC64 - lbz r3, PACA_FTRACE_ENABLED(r13) - cmpdi r3, 0 - beq ftrace_no_trace -#endif - - /* Get the _mcount() call site out of LR */ - mflr r7 - PPC_STL r7, _NIP(r1) - -#ifdef CONFIG_PPC64 - /* Save callee's TOC in the ABI compliant location */ - std r2, 24(r1) - ld r2, PACATOC(r13) /* get kernel TOC in r2 */ - - addis r3, r2, function_trace_op@toc@ha - addi r3, r3, function_trace_op@toc@l - ld r5, 0(r3) -#else - lis r3,function_trace_op@ha - lwz r5,function_trace_op@l(r3) -#endif - -#ifdef CONFIG_LIVEPATCH_64 - SAVE_GPR(14, r1) - mr r14,r7 /* remember old NIP */ -#endif - /* Calculate ip from nip-4 into r3 for call below */ - subi r3, r7, MCOUNT_INSN_SIZE - - /* Put the original return address in r4 as parent_ip */ - PPC_STL r0, _LINK(r1) - mr r4, r0 - - /* Load &pt_regs in r6 for call below */ - addi r6, r1 ,STACK_FRAME_OVERHEAD - - /* ftrace_call(r3, r4, r5, r6) */ -.globl ftrace_call -ftrace_call: - bl ftrace_stub - nop - - PPC_LL r3, _NIP(r1) - mtctr r3 -#ifdef CONFIG_LIVEPATCH_64 - cmpd r14, r3 /* has NIP been altered? */ - REST_GPR(14, r1) -#endif - - /* Restore gprs */ - REST_GPRS(3, 10, r1) - -#ifdef CONFIG_PPC64 - /* Restore callee's TOC */ - ld r2, 24(r1) -#endif - - /* Restore possibly modified LR */ - PPC_LL r0, _LINK(r1) - mtlr r0 - - /* Pop our stack frame */ - addi r1, r1, SWITCH_FRAME_SIZE - -#ifdef CONFIG_LIVEPATCH_64 - /* Based on the cmpd above, if the NIP was altered handle livepatch */ - bne- livepatch_handler -#endif - bctr /* jump after _mcount site */ - -#ifdef CONFIG_LIVEPATCH_64 - /* - * This function runs in the mcount context, between two functions. As - * such it can only clobber registers which are volatile and used in - * function linkage. - * - * We get here when a function A, calls another function B, but B has - * been live patched with a new function C. - * - * On entry: - * - we have no stack frame and can not allocate one - * - LR points back to the original caller (in A) - * - CTR holds the new NIP in C - * - r0, r11 & r12 are free - */ -livepatch_handler: - ld r12, PACA_THREAD_INFO(r13) - - /* Allocate 3 x 8 bytes */ - ld r11, TI_livepatch_sp(r12) - addi r11, r11, 24 - std r11, TI_livepatch_sp(r12) - - /* Save toc & real LR on livepatch stack */ - std r2, -24(r11) - mflr r12 - std r12, -16(r11) - - /* Store stack end marker */ - lis r12, STACK_END_MAGIC@h - ori r12, r12, STACK_END_MAGIC@l - std r12, -8(r11) - - /* Put ctr in r12 for global entry and branch there */ - mfctr r12 - bctrl - - /* - * Now we are returning from the patched function to the original - * caller A. We are free to use r11, r12 and we can use r2 until we - * restore it. - */ - - ld r12, PACA_THREAD_INFO(r13) - - ld r11, TI_livepatch_sp(r12) - - /* Check stack marker hasn't been trashed */ - lis r2, STACK_END_MAGIC@h - ori r2, r2, STACK_END_MAGIC@l - ld r12, -8(r11) -1: tdne r12, r2 - EMIT_BUG_ENTRY 1b, __FILE__, __LINE__ - 1, 0 - - /* Restore LR & toc from livepatch stack */ - ld r12, -16(r11) - mtlr r12 - ld r2, -24(r11) - - /* Pop livepatch stack frame */ - ld r12, PACA_THREAD_INFO(r13) - subi r11, r11, 24 - std r11, TI_livepatch_sp(r12) - - /* Return to original caller of live patched function */ - blr -#endif /* CONFIG_LIVEPATCH */ diff --git a/arch/powerpc/kernel/trace/ftrace_low.S b/arch/powerpc/kernel/trace/ftrace_low.S new file mode 100644 index 0000000000000..0bddf1fa66361 --- /dev/null +++ b/arch/powerpc/kernel/trace/ftrace_low.S @@ -0,0 +1,78 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Split from entry_64.S + */ + +#include +#include +#include +#include +#include +#include + +#ifdef CONFIG_PPC64 +.pushsection ".tramp.ftrace.text","aw",@progbits; +.globl ftrace_tramp_text +ftrace_tramp_text: + .space 64 +.popsection + +.pushsection ".tramp.ftrace.init","aw",@progbits; +.globl ftrace_tramp_init +ftrace_tramp_init: + .space 64 +.popsection +#endif + +_GLOBAL(mcount) +_GLOBAL(_mcount) +EXPORT_SYMBOL(_mcount) + mflr r12 + mtctr r12 + mtlr r0 + bctr + +#ifdef CONFIG_FUNCTION_GRAPH_TRACER +_GLOBAL(return_to_handler) + /* need to save return values */ +#ifdef CONFIG_PPC64 + std r4, -32(r1) + std r3, -24(r1) + /* save TOC */ + std r2, -16(r1) + std r31, -8(r1) + mr r31, r1 + stdu r1, -112(r1) + + /* + * We might be called from a module. + * Switch to our TOC to run inside the core kernel. + */ + ld r2, PACATOC(r13) +#else + stwu r1, -16(r1) + stw r3, 8(r1) + stw r4, 12(r1) +#endif + + bl ftrace_return_to_handler + nop + + /* return value has real return address */ + mtlr r3 + +#ifdef CONFIG_PPC64 + ld r1, 0(r1) + ld r4, -32(r1) + ld r3, -24(r1) + ld r2, -16(r1) + ld r31, -8(r1) +#else + lwz r3, 8(r1) + lwz r4, 12(r1) + addi r1, r1, 16 +#endif + + /* Jump back to real return address */ + blr +#endif /* CONFIG_FUNCTION_GRAPH_TRACER */ diff --git a/arch/powerpc/kernel/trace/ftrace_mprofile.S b/arch/powerpc/kernel/trace/ftrace_mprofile.S new file mode 100644 index 0000000000000..56da60e98327a --- /dev/null +++ b/arch/powerpc/kernel/trace/ftrace_mprofile.S @@ -0,0 +1,307 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Split from ftrace_64.S + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* + * + * ftrace_caller()/ftrace_regs_caller() is the function that replaces _mcount() + * when ftrace is active. + * + * We arrive here after a function A calls function B, and we are the trace + * function for B. When we enter r1 points to A's stack frame, B has not yet + * had a chance to allocate one yet. + * + * Additionally r2 may point either to the TOC for A, or B, depending on + * whether B did a TOC setup sequence before calling us. + * + * On entry the LR points back to the _mcount() call site, and r0 holds the + * saved LR as it was on entry to B, ie. the original return address at the + * call site in A. + * + * Our job is to save the register state into a struct pt_regs (on the stack) + * and then arrange for the ftrace function to be called. + */ +_GLOBAL(ftrace_regs_caller) + /* Save the original return address in A's stack frame */ +#ifdef CONFIG_MPROFILE_KERNEL + PPC_STL r0,LRSAVE(r1) +#endif + + /* Create our stack frame + pt_regs */ + PPC_STLU r1,-SWITCH_FRAME_SIZE(r1) + + /* Save all gprs to pt_regs */ + SAVE_GPR(0, r1) +#ifdef CONFIG_PPC64 + SAVE_GPRS(2, 11, r1) + + /* Ok to continue? */ + lbz r3, PACA_FTRACE_ENABLED(r13) + cmpdi r3, 0 + beq ftrace_no_trace + + SAVE_GPRS(12, 31, r1) +#else + stmw r2, GPR2(r1) +#endif + + /* Save previous stack pointer (r1) */ + addi r8, r1, SWITCH_FRAME_SIZE + PPC_STL r8, GPR1(r1) + + /* Load special regs for save below */ + mfmsr r8 + mfctr r9 + mfxer r10 + mfcr r11 + + /* Get the _mcount() call site out of LR */ + mflr r7 + /* Save it as pt_regs->nip */ + PPC_STL r7, _NIP(r1) + /* Save the read LR in pt_regs->link */ + PPC_STL r0, _LINK(r1) + +#ifdef CONFIG_PPC64 + /* Save callee's TOC in the ABI compliant location */ + std r2, 24(r1) + ld r2,PACATOC(r13) /* get kernel TOC in r2 */ + + addis r3,r2,function_trace_op@toc@ha + addi r3,r3,function_trace_op@toc@l + ld r5,0(r3) +#else + lis r3,function_trace_op@ha + lwz r5,function_trace_op@l(r3) +#endif + +#ifdef CONFIG_LIVEPATCH_64 + mr r14,r7 /* remember old NIP */ +#endif + /* Calculate ip from nip-4 into r3 for call below */ + subi r3, r7, MCOUNT_INSN_SIZE + + /* Put the original return address in r4 as parent_ip */ + mr r4, r0 + + /* Save special regs */ + PPC_STL r8, _MSR(r1) + PPC_STL r9, _CTR(r1) + PPC_STL r10, _XER(r1) + PPC_STL r11, _CCR(r1) + + /* Load &pt_regs in r6 for call below */ + addi r6, r1 ,STACK_FRAME_OVERHEAD + + /* ftrace_call(r3, r4, r5, r6) */ +.globl ftrace_regs_call +ftrace_regs_call: + bl ftrace_stub + nop + + /* Load ctr with the possibly modified NIP */ + PPC_LL r3, _NIP(r1) + mtctr r3 +#ifdef CONFIG_LIVEPATCH_64 + cmpd r14, r3 /* has NIP been altered? */ +#endif + + /* Restore gprs */ +#ifdef CONFIG_PPC64 + REST_GPRS(2, 31, r1) +#else + lmw r2, GPR2(r1) +#endif + + /* Restore possibly modified LR */ + PPC_LL r0, _LINK(r1) + mtlr r0 + +#ifdef CONFIG_PPC64 + /* Restore callee's TOC */ + ld r2, 24(r1) +#endif + + /* Pop our stack frame */ + addi r1, r1, SWITCH_FRAME_SIZE + +#ifdef CONFIG_LIVEPATCH_64 + /* Based on the cmpd above, if the NIP was altered handle livepatch */ + bne- livepatch_handler +#endif + bctr /* jump after _mcount site */ + +_GLOBAL(ftrace_stub) + blr + +#ifdef CONFIG_PPC64 +ftrace_no_trace: + mflr r3 + mtctr r3 + REST_GPR(3, r1) + addi r1, r1, SWITCH_FRAME_SIZE + mtlr r0 + bctr +#endif + +_GLOBAL(ftrace_caller) + /* Save the original return address in A's stack frame */ +#ifdef CONFIG_MPROFILE_KERNEL + PPC_STL r0, LRSAVE(r1) +#endif + + /* Create our stack frame + pt_regs */ + PPC_STLU r1, -SWITCH_FRAME_SIZE(r1) + + /* Save all gprs to pt_regs */ + SAVE_GPRS(3, 10, r1) + +#ifdef CONFIG_PPC64 + lbz r3, PACA_FTRACE_ENABLED(r13) + cmpdi r3, 0 + beq ftrace_no_trace +#endif + + /* Get the _mcount() call site out of LR */ + mflr r7 + PPC_STL r7, _NIP(r1) + +#ifdef CONFIG_PPC64 + /* Save callee's TOC in the ABI compliant location */ + std r2, 24(r1) + ld r2, PACATOC(r13) /* get kernel TOC in r2 */ + + addis r3, r2, function_trace_op@toc@ha + addi r3, r3, function_trace_op@toc@l + ld r5, 0(r3) +#else + lis r3,function_trace_op@ha + lwz r5,function_trace_op@l(r3) +#endif + +#ifdef CONFIG_LIVEPATCH_64 + SAVE_GPR(14, r1) + mr r14,r7 /* remember old NIP */ +#endif + /* Calculate ip from nip-4 into r3 for call below */ + subi r3, r7, MCOUNT_INSN_SIZE + + /* Put the original return address in r4 as parent_ip */ + PPC_STL r0, _LINK(r1) + mr r4, r0 + + /* Load &pt_regs in r6 for call below */ + addi r6, r1 ,STACK_FRAME_OVERHEAD + + /* ftrace_call(r3, r4, r5, r6) */ +.globl ftrace_call +ftrace_call: + bl ftrace_stub + nop + + PPC_LL r3, _NIP(r1) + mtctr r3 +#ifdef CONFIG_LIVEPATCH_64 + cmpd r14, r3 /* has NIP been altered? */ + REST_GPR(14, r1) +#endif + + /* Restore gprs */ + REST_GPRS(3, 10, r1) + +#ifdef CONFIG_PPC64 + /* Restore callee's TOC */ + ld r2, 24(r1) +#endif + + /* Restore possibly modified LR */ + PPC_LL r0, _LINK(r1) + mtlr r0 + + /* Pop our stack frame */ + addi r1, r1, SWITCH_FRAME_SIZE + +#ifdef CONFIG_LIVEPATCH_64 + /* Based on the cmpd above, if the NIP was altered handle livepatch */ + bne- livepatch_handler +#endif + bctr /* jump after _mcount site */ + +#ifdef CONFIG_LIVEPATCH_64 + /* + * This function runs in the mcount context, between two functions. As + * such it can only clobber registers which are volatile and used in + * function linkage. + * + * We get here when a function A, calls another function B, but B has + * been live patched with a new function C. + * + * On entry: + * - we have no stack frame and can not allocate one + * - LR points back to the original caller (in A) + * - CTR holds the new NIP in C + * - r0, r11 & r12 are free + */ +livepatch_handler: + ld r12, PACA_THREAD_INFO(r13) + + /* Allocate 3 x 8 bytes */ + ld r11, TI_livepatch_sp(r12) + addi r11, r11, 24 + std r11, TI_livepatch_sp(r12) + + /* Save toc & real LR on livepatch stack */ + std r2, -24(r11) + mflr r12 + std r12, -16(r11) + + /* Store stack end marker */ + lis r12, STACK_END_MAGIC@h + ori r12, r12, STACK_END_MAGIC@l + std r12, -8(r11) + + /* Put ctr in r12 for global entry and branch there */ + mfctr r12 + bctrl + + /* + * Now we are returning from the patched function to the original + * caller A. We are free to use r11, r12 and we can use r2 until we + * restore it. + */ + + ld r12, PACA_THREAD_INFO(r13) + + ld r11, TI_livepatch_sp(r12) + + /* Check stack marker hasn't been trashed */ + lis r2, STACK_END_MAGIC@h + ori r2, r2, STACK_END_MAGIC@l + ld r12, -8(r11) +1: tdne r12, r2 + EMIT_BUG_ENTRY 1b, __FILE__, __LINE__ - 1, 0 + + /* Restore LR & toc from livepatch stack */ + ld r12, -16(r11) + mtlr r12 + ld r2, -24(r11) + + /* Pop livepatch stack frame */ + ld r12, PACA_THREAD_INFO(r13) + subi r11, r11, 24 + std r11, TI_livepatch_sp(r12) + + /* Return to original caller of live patched function */ + blr +#endif /* CONFIG_LIVEPATCH */