ftrace: Store direct called addresses in their ops

All direct calls are now registered using the register_ftrace_direct API
so each ops can jump to only one direct-called trampoline.

By storing the direct called trampoline address directly in the ops we
can save one hashmap lookup in the direct call ops and implement arm64
direct calls on top of call ops.

Link: https://lkml.kernel.org/r/20230321140424.345218-6-revest@chromium.org
Signed-off-by: Florent Revest <revest@chromium.org>
Acked-by: Jiri Olsa <jolsa@kernel.org>
Signed-off-by: Steven Rostedt (Google) <rostedt@goodmis.org>

diff --git a/include/linux/ftrace.h b/include/linux/ftrace.h
index abee60865fc78ba4e0dd179e83761e38286d2ad0..6a532dd6789e6f5747877b7dca31b8f2f15cc92b 100644 (file)
--- a/include/linux/ftrace.h
+++ b/include/linux/ftrace.h
@@ -321,6 +321,9 @@ struct ftrace_ops {
        unsigned long                   trampoline_size;
        struct list_head                list;
        ftrace_ops_func_t               ops_func;
+#ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS
+       unsigned long                   direct_call;
+#endif
 #endif
 };
 

diff --git a/kernel/trace/ftrace.c b/kernel/trace/ftrace.c
index 845c4012630fecea97f81d40e2e2f59e0f55af63..3bef2abc037a785b2b85482d52834fcc2864b08a 100644 (file)
--- a/kernel/trace/ftrace.c
+++ b/kernel/trace/ftrace.c
@@ -2583,9 +2583,8 @@ ftrace_add_rec_direct(unsigned long ip, unsigned long addr,
 static void call_direct_funcs(unsigned long ip, unsigned long pip,
                              struct ftrace_ops *ops, struct ftrace_regs *fregs)
 {
-       unsigned long addr;
+       unsigned long addr = READ_ONCE(ops->direct_call);
 
-       addr = ftrace_find_rec_direct(ip);
        if (!addr)
                return;
 
@@ -5381,6 +5380,7 @@ int register_ftrace_direct(struct ftrace_ops *ops, unsigned long addr)
        ops->func = call_direct_funcs;
        ops->flags = MULTI_FLAGS;
        ops->trampoline = FTRACE_REGS_ADDR;
+       ops->direct_call = addr;
 
        err = register_ftrace_function_nolock(ops);
 
@@ -5455,6 +5455,7 @@ __modify_ftrace_direct(struct ftrace_ops *ops, unsigned long addr)
        /* Enable the tmp_ops to have the same functions as the direct ops */
        ftrace_ops_init(&tmp_ops);
        tmp_ops.func_hash = ops->func_hash;
+       tmp_ops.direct_call = addr;
 
        err = register_ftrace_function_nolock(&tmp_ops);
        if (err)
@@ -5476,6 +5477,8 @@ __modify_ftrace_direct(struct ftrace_ops *ops, unsigned long addr)
                        entry->direct = addr;
                }
        }
+       /* Prevent store tearing if a trampoline concurrently accesses the value */
+       WRITE_ONCE(ops->direct_call, addr);
 
        mutex_unlock(&ftrace_lock);