ARM: ftrace: enable the graph tracer with the EABI unwinder

Enable the function graph tracer in combination with the EABI unwinder,
so that Thumb2 builds or Clang ARM builds can make use of it.

This involves using the unwinder to locate the return address of an
instrumented function on the stack, so that it can be overridden and
made to refer to the ftrace handling routines that need to be called at
function return.

Given that for these builds, it is not guaranteed that the value of the
link register is stored on the stack, fall back to the stack slot that
will be used by the ftrace exit code to restore LR in the instrumented
function's execution context.

Signed-off-by: Ard Biesheuvel <ardb@kernel.org>
Reviewed-by: Steven Rostedt (Google) <rostedt@goodmis.org>

diff --git a/arch/arm/Kconfig b/arch/arm/Kconfig
index 70ab8d807032f873dcfd959f93e4cbd073186b14..ec98387367d0a7a4cabf001a5f0f58894ac4ed6e 100644 (file)
--- a/arch/arm/Kconfig
+++ b/arch/arm/Kconfig
@@ -91,7 +91,7 @@ config ARM
        select HAVE_EXIT_THREAD
        select HAVE_FAST_GUP if ARM_LPAE
        select HAVE_FTRACE_MCOUNT_RECORD if !XIP_KERNEL
-       select HAVE_FUNCTION_GRAPH_TRACER if !THUMB2_KERNEL && !CC_IS_CLANG
+       select HAVE_FUNCTION_GRAPH_TRACER
        select HAVE_FUNCTION_TRACER if !XIP_KERNEL && !(THUMB2_KERNEL && CC_IS_CLANG)
        select HAVE_FUTEX_CMPXCHG if FUTEX
        select HAVE_GCC_PLUGINS

diff --git a/arch/arm/Kconfig.debug b/arch/arm/Kconfig.debug
index 98436702e0c7ebe63bfbff066b38d8aac9682932..b79dc7fa89bf4caf884990d7401797fae3e5b37f 100644 (file)
--- a/arch/arm/Kconfig.debug
+++ b/arch/arm/Kconfig.debug
@@ -65,7 +65,7 @@ config UNWINDER_FRAME_POINTER
 
 config UNWINDER_ARM
        bool "ARM EABI stack unwinder"
-       depends on AEABI && !FUNCTION_GRAPH_TRACER
+       depends on AEABI
        # https://github.com/ClangBuiltLinux/linux/issues/732
        depends on !LD_IS_LLD || LLD_VERSION >= 110000
        select ARM_UNWIND

diff --git a/arch/arm/include/asm/ftrace.h b/arch/arm/include/asm/ftrace.h
index b4f5fab6b04e4ddca998c1c57feef2c1fd8c5d4f..5358aad6783128dd16a240ddf3872578886a9350 100644 (file)
--- a/arch/arm/include/asm/ftrace.h
+++ b/arch/arm/include/asm/ftrace.h
@@ -35,26 +35,8 @@ static inline unsigned long ftrace_call_adjust(unsigned long addr)
 
 #ifndef __ASSEMBLY__
 
-#if defined(CONFIG_FRAME_POINTER) && !defined(CONFIG_ARM_UNWIND)
-/*
- * return_address uses walk_stackframe to do it's work.  If both
- * CONFIG_FRAME_POINTER=y and CONFIG_ARM_UNWIND=y walk_stackframe uses unwind
- * information.  For this to work in the function tracer many functions would
- * have to be marked with __notrace.  So for now just depend on
- * !CONFIG_ARM_UNWIND.
- */
-
 void *return_address(unsigned int);
 
-#else
-
-static inline void *return_address(unsigned int level)
-{
-       return NULL;
-}
-
-#endif
-
 #define ftrace_return_address(n) return_address(n)
 
 #define ARCH_HAS_SYSCALL_MATCH_SYM_NAME

diff --git a/arch/arm/kernel/Makefile b/arch/arm/kernel/Makefile
index 56511856ff9d2a90c160f9b97b93bdf5fc608148..5cebb8d5a1d691286f3545267dc2487a8edd3fb8 100644 (file)
--- a/arch/arm/kernel/Makefile
+++ b/arch/arm/kernel/Makefile
@@ -25,10 +25,7 @@ obj-y                := elf.o entry-common.o irq.o opcodes.o \
 KASAN_SANITIZE_stacktrace.o := n
 KASAN_SANITIZE_traps.o := n
 
-ifneq ($(CONFIG_ARM_UNWIND),y)
-obj-$(CONFIG_FRAME_POINTER)    += return_address.o
-endif
-
+obj-y                          += return_address.o
 obj-$(CONFIG_ATAGS)            += atags_parse.o
 obj-$(CONFIG_ATAGS_PROC)       += atags_proc.o
 obj-$(CONFIG_DEPRECATED_PARAM_STRUCT) += atags_compat.o

diff --git a/arch/arm/kernel/entry-ftrace.S b/arch/arm/kernel/entry-ftrace.S
index bbfa0954c385ea96a1f45950f92e1ed38a1fd429..3e7bcaca5e07d7db0e5a384bc0e4838475a0b638 100644 (file)
--- a/arch/arm/kernel/entry-ftrace.S
+++ b/arch/arm/kernel/entry-ftrace.S
@@ -100,7 +100,8 @@ ftrace_regs_call:
 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
        .globl ftrace_graph_regs_call
 ftrace_graph_regs_call:
-       mov     r0, r0
+ARM(   mov     r0, r0  )
+THUMB( nop.w           )
 #endif
 
        @ pop saved regs
@@ -112,13 +113,18 @@ ftrace_graph_regs_call:
 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
 .macro __ftrace_graph_regs_caller
 
-       sub     r0, fp, #4              @ lr of instrumented routine (parent)
+#ifdef CONFIG_UNWINDER_FRAME_POINTER
+       sub     r0, fp, #4              @ lr of instrumented routine (parent)
+#else
+       add     r0, sp, #S_LR
+#endif
 
        @ called from __ftrace_regs_caller
-       ldr     r1, [sp, #S_PC]         @ instrumented routine (func)
+       ldr     r1, [sp, #S_PC]         @ instrumented routine (func)
        mcount_adjust_addr      r1, r1
 
-       mov     r2, fp                  @ frame pointer
+       mov     r2, fpreg               @ frame pointer
+       add     r3, sp, #PT_REGS_SIZE
        bl      prepare_ftrace_return
 
        @ pop registers saved in ftrace_regs_caller
@@ -149,14 +155,19 @@ ftrace_call\suffix:
 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
        .globl ftrace_graph_call\suffix
 ftrace_graph_call\suffix:
-       mov     r0, r0
+ARM(   mov     r0, r0  )
+THUMB( nop.w           )
 #endif
 
        mcount_exit
 .endm
 
 .macro __ftrace_graph_caller
+#ifdef CONFIG_UNWINDER_FRAME_POINTER
        sub     r0, fp, #4              @ &lr of instrumented routine (&parent)
+#else
+       add     r0, sp, #20
+#endif
 #ifdef CONFIG_DYNAMIC_FTRACE
        @ called from __ftrace_caller, saved in mcount_enter
        ldr     r1, [sp, #16]           @ instrumented routine (func)
@@ -165,7 +176,8 @@ ftrace_graph_call\suffix:
        @ called from __mcount, untouched in lr
        mcount_adjust_addr      r1, lr  @ instrumented routine (func)
 #endif
-       mov     r2, fp                  @ frame pointer
+       mov     r2, fpreg               @ frame pointer
+       add     r3, sp, #24
        bl      prepare_ftrace_return
        mcount_exit
 .endm
@@ -244,14 +256,14 @@ ENDPROC(ftrace_graph_regs_caller)
 .purgem mcount_exit
 
 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
-       .globl return_to_handler
-return_to_handler:
+ENTRY(return_to_handler)
        stmdb   sp!, {r0-r3}
        add     r0, sp, #16             @ sp at exit of instrumented routine
        bl      ftrace_return_to_handler
        mov     lr, r0                  @ r0 has real ret addr
        ldmia   sp!, {r0-r3}
        ret     lr
+ENDPROC(return_to_handler)
 #endif
 
 ENTRY(ftrace_stub)

diff --git a/arch/arm/kernel/ftrace.c b/arch/arm/kernel/ftrace.c
index ea2396900c7dcbddbb13041508ab3972f2afcf84..83cc068586bc5c25dbb2e63cc2378229f43c801f 100644 (file)
--- a/arch/arm/kernel/ftrace.c
+++ b/arch/arm/kernel/ftrace.c
@@ -22,6 +22,7 @@
 #include <asm/ftrace.h>
 #include <asm/insn.h>
 #include <asm/set_memory.h>
+#include <asm/stacktrace.h>
 #include <asm/patch.h>
 
 /*
@@ -224,8 +225,10 @@ int ftrace_make_nop(struct module *mod,
 #endif /* CONFIG_DYNAMIC_FTRACE */
 
 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
+asmlinkage
 void prepare_ftrace_return(unsigned long *parent, unsigned long self_addr,
-                          unsigned long frame_pointer)
+                          unsigned long frame_pointer,
+                          unsigned long stack_pointer)
 {
        unsigned long return_hooker = (unsigned long) &return_to_handler;
        unsigned long old;
@@ -236,6 +239,18 @@ void prepare_ftrace_return(unsigned long *parent, unsigned long self_addr,
        if (IS_ENABLED(CONFIG_UNWINDER_FRAME_POINTER)) {
                /* FP points one word below parent's top of stack */
                frame_pointer += 4;
+       } else {
+               struct stackframe frame = {
+                       .fp = frame_pointer,
+                       .sp = stack_pointer,
+                       .lr = self_addr,
+                       .pc = self_addr,
+               };
+               if (unwind_frame(&frame) < 0)
+                       return;
+               if (frame.lr != self_addr)
+                       parent = frame.lr_addr;
+               frame_pointer = frame.sp;
        }
 
        old = *parent;
@@ -258,7 +273,7 @@ static int __ftrace_modify_caller(unsigned long *callsite,
        unsigned long caller_fn = (unsigned long) func;
        unsigned long pc = (unsigned long) callsite;
        unsigned long branch = arm_gen_branch(pc, caller_fn);
-       unsigned long nop = 0xe1a00000; /* mov r0, r0 */
+       unsigned long nop = arm_gen_nop();
        unsigned long old = enable ? nop : branch;
        unsigned long new = enable ? branch : nop;