ftrace/x86: Allow for arguments to be passed in to ftrace_regs by default

Currently, the only way to get access to the registers of a function via a
ftrace callback is to set the "FL_SAVE_REGS" bit in the ftrace_ops. But as this
saves all regs as if a breakpoint were to trigger (for use with kprobes), it
is expensive.

The regs are already saved on the stack for the default ftrace callbacks, as
that is required otherwise a function being traced will get the wrong
arguments and possibly crash. And on x86, the arguments are already stored
where they would be on a pt_regs structure to use that code for both the
regs version of a callback, it makes sense to pass that information always
to all functions.

If an architecture does this (as x86_64 now does), it is to set
HAVE_DYNAMIC_FTRACE_WITH_ARGS, and this will let the generic code that it
could have access to arguments without having to set the flags.

This also includes having the stack pointer being saved, which could be used
for accessing arguments on the stack, as well as having the function graph
tracer not require its own trampoline!

Acked-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Signed-off-by: Steven Rostedt (VMware) <rostedt@goodmis.org>

diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig
index f6946b81f74a95da92bce9f483dcbd6ca6aa7cff..478526aabe5dd36db8148e1b855c518265e18f93 100644 (file)
--- a/arch/x86/Kconfig
+++ b/arch/x86/Kconfig
@@ -167,6 +167,7 @@ config X86
        select HAVE_DMA_CONTIGUOUS
        select HAVE_DYNAMIC_FTRACE
        select HAVE_DYNAMIC_FTRACE_WITH_REGS
+       select HAVE_DYNAMIC_FTRACE_WITH_ARGS    if X86_64
        select HAVE_DYNAMIC_FTRACE_WITH_DIRECT_CALLS
        select HAVE_EBPF_JIT
        select HAVE_EFFICIENT_UNALIGNED_ACCESS

diff --git a/arch/x86/include/asm/ftrace.h b/arch/x86/include/asm/ftrace.h
index 84b9449be0801b168d7038c87c105e93d48a8d96..e00fe88146e0b7443396caf2ea0fd0b41c78eff6 100644 (file)
--- a/arch/x86/include/asm/ftrace.h
+++ b/arch/x86/include/asm/ftrace.h
@@ -41,6 +41,21 @@ static inline void arch_ftrace_set_direct_caller(struct pt_regs *regs, unsigned
        regs->orig_ax = addr;
 }
 
+#ifdef CONFIG_HAVE_DYNAMIC_FTRACE_WITH_ARGS
+struct ftrace_regs {
+       struct pt_regs          regs;
+};
+
+static __always_inline struct pt_regs *
+arch_ftrace_get_regs(struct ftrace_regs *fregs)
+{
+       /* Only when FL_SAVE_REGS is set, cs will be non zero */
+       if (!fregs->regs.cs)
+               return NULL;
+       return &fregs->regs;
+}
+#endif
+
 #ifdef CONFIG_DYNAMIC_FTRACE
 
 struct dyn_arch_ftrace {

diff --git a/arch/x86/kernel/ftrace_64.S b/arch/x86/kernel/ftrace_64.S
index ac3d5f22fe64b9900dbe00117173bd5276b6fd76..60e3b64f5ea678d9bfaa89f944a22ff35d73a212 100644 (file)
--- a/arch/x86/kernel/ftrace_64.S
+++ b/arch/x86/kernel/ftrace_64.S
@@ -140,12 +140,19 @@ SYM_FUNC_START(ftrace_caller)
        /* save_mcount_regs fills in first two parameters */
        save_mcount_regs
 
+       /* Stack - skipping return address of ftrace_caller */
+       leaq MCOUNT_REG_SIZE+8(%rsp), %rcx
+       movq %rcx, RSP(%rsp)
+
 SYM_INNER_LABEL(ftrace_caller_op_ptr, SYM_L_GLOBAL)
        /* Load the ftrace_ops into the 3rd parameter */
        movq function_trace_op(%rip), %rdx
 
-       /* regs go into 4th parameter (but make it NULL) */
-       movq $0, %rcx
+       /* regs go into 4th parameter */
+       leaq (%rsp), %rcx
+
+       /* Only ops with REGS flag set should have CS register set */
+       movq $0, CS(%rsp)
 
 SYM_INNER_LABEL(ftrace_call, SYM_L_GLOBAL)
        call ftrace_stub

diff --git a/include/linux/ftrace.h b/include/linux/ftrace.h
index 24e1fa52337d9e9eeca8959c34b9259fa80dd16a..588ea7023a7a3db5cd509509667c5b4cf702c6fc 100644 (file)
--- a/include/linux/ftrace.h
+++ b/include/linux/ftrace.h
@@ -90,16 +90,21 @@ ftrace_enable_sysctl(struct ctl_table *table, int write,
 
 struct ftrace_ops;
 
+#ifndef CONFIG_HAVE_DYNAMIC_FTRACE_WITH_ARGS
+
 struct ftrace_regs {
        struct pt_regs          regs;
 };
+#define arch_ftrace_get_regs(fregs) (&(fregs)->regs)
+
+#endif /* CONFIG_HAVE_DYNAMIC_FTRACE_WITH_ARGS */
 
 static __always_inline struct pt_regs *ftrace_get_regs(struct ftrace_regs *fregs)
 {
        if (!fregs)
                return NULL;
 
-       return &fregs->regs;
+       return arch_ftrace_get_regs(fregs);
 }
 
 typedef void (*ftrace_func_t)(unsigned long ip, unsigned long parent_ip,

diff --git a/kernel/trace/Kconfig b/kernel/trace/Kconfig
index 6aa36ec73ccb29348dec0ab2c04291c0dde6f09f..c9b64dea121628549d1b2de1e5665e54d474a107 100644 (file)
--- a/kernel/trace/Kconfig
+++ b/kernel/trace/Kconfig
@@ -31,6 +31,15 @@ config HAVE_DYNAMIC_FTRACE_WITH_REGS
 config HAVE_DYNAMIC_FTRACE_WITH_DIRECT_CALLS
        bool
 
+config HAVE_DYNAMIC_FTRACE_WITH_ARGS
+       bool
+       help
+        If this is set, then arguments and stack can be found from
+        the pt_regs passed into the function callback regs parameter
+        by default, even without setting the REGS flag in the ftrace_ops.
+        This allows for use of regs_get_kernel_argument() and
+        kernel_stack_pointer().
+
 config HAVE_FTRACE_MCOUNT_RECORD
        bool
        help