x86/alternatives: Introduce int3_emulate_jcc()

Move the kprobe Jcc emulation into int3_emulate_jcc() so it can be
used by more code -- specifically static_call() will need this.

Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Signed-off-by: Ingo Molnar <mingo@kernel.org>
Reviewed-by: Masami Hiramatsu (Google) <mhiramat@kernel.org>
Link: https://lore.kernel.org/r/20230123210607.057678245@infradead.org

diff --git a/arch/x86/include/asm/text-patching.h b/arch/x86/include/asm/text-patching.h
index f4b87f08f5c50d6083977f22d0043c318def3002..29832c338cdc5a2b144126117a28483cd55c5b04 100644 (file)
--- a/arch/x86/include/asm/text-patching.h
+++ b/arch/x86/include/asm/text-patching.h
@@ -184,6 +184,37 @@ void int3_emulate_ret(struct pt_regs *regs)
        unsigned long ip = int3_emulate_pop(regs);
        int3_emulate_jmp(regs, ip);
 }
+
+static __always_inline
+void int3_emulate_jcc(struct pt_regs *regs, u8 cc, unsigned long ip, unsigned long disp)
+{
+       static const unsigned long jcc_mask[6] = {
+               [0] = X86_EFLAGS_OF,
+               [1] = X86_EFLAGS_CF,
+               [2] = X86_EFLAGS_ZF,
+               [3] = X86_EFLAGS_CF | X86_EFLAGS_ZF,
+               [4] = X86_EFLAGS_SF,
+               [5] = X86_EFLAGS_PF,
+       };
+
+       bool invert = cc & 1;
+       bool match;
+
+       if (cc < 0xc) {
+               match = regs->flags & jcc_mask[cc >> 1];
+       } else {
+               match = ((regs->flags & X86_EFLAGS_SF) >> X86_EFLAGS_SF_BIT) ^
+                       ((regs->flags & X86_EFLAGS_OF) >> X86_EFLAGS_OF_BIT);
+               if (cc >= 0xe)
+                       match = match || (regs->flags & X86_EFLAGS_ZF);
+       }
+
+       if ((match && !invert) || (!match && invert))
+               ip += disp;
+
+       int3_emulate_jmp(regs, ip);
+}
+
 #endif /* !CONFIG_UML_X86 */
 
 #endif /* _ASM_X86_TEXT_PATCHING_H */

diff --git a/arch/x86/kernel/kprobes/core.c b/arch/x86/kernel/kprobes/core.c
index 66299682b6b7e24305160f6282ff6c40f07f82d4..d48638cc71ebe0e4ec7671386ee7fb4b39c9d24f 100644 (file)
--- a/arch/x86/kernel/kprobes/core.c
+++ b/arch/x86/kernel/kprobes/core.c
@@ -460,50 +460,26 @@ static void kprobe_emulate_call(struct kprobe *p, struct pt_regs *regs)
 }
 NOKPROBE_SYMBOL(kprobe_emulate_call);
 
-static nokprobe_inline
-void __kprobe_emulate_jmp(struct kprobe *p, struct pt_regs *regs, bool cond)
+static void kprobe_emulate_jmp(struct kprobe *p, struct pt_regs *regs)
 {
        unsigned long ip = regs->ip - INT3_INSN_SIZE + p->ainsn.size;
 
-       if (cond)
-               ip += p->ainsn.rel32;
+       ip += p->ainsn.rel32;
        int3_emulate_jmp(regs, ip);
 }
-
-static void kprobe_emulate_jmp(struct kprobe *p, struct pt_regs *regs)
-{
-       __kprobe_emulate_jmp(p, regs, true);
-}
 NOKPROBE_SYMBOL(kprobe_emulate_jmp);
 
-static const unsigned long jcc_mask[6] = {
-       [0] = X86_EFLAGS_OF,
-       [1] = X86_EFLAGS_CF,
-       [2] = X86_EFLAGS_ZF,
-       [3] = X86_EFLAGS_CF | X86_EFLAGS_ZF,
-       [4] = X86_EFLAGS_SF,
-       [5] = X86_EFLAGS_PF,
-};
-
 static void kprobe_emulate_jcc(struct kprobe *p, struct pt_regs *regs)
 {
-       bool invert = p->ainsn.jcc.type & 1;
-       bool match;
+       unsigned long ip = regs->ip - INT3_INSN_SIZE + p->ainsn.size;
 
-       if (p->ainsn.jcc.type < 0xc) {
-               match = regs->flags & jcc_mask[p->ainsn.jcc.type >> 1];
-       } else {
-               match = ((regs->flags & X86_EFLAGS_SF) >> X86_EFLAGS_SF_BIT) ^
-                       ((regs->flags & X86_EFLAGS_OF) >> X86_EFLAGS_OF_BIT);
-               if (p->ainsn.jcc.type >= 0xe)
-                       match = match || (regs->flags & X86_EFLAGS_ZF);
-       }
-       __kprobe_emulate_jmp(p, regs, (match && !invert) || (!match && invert));
+       int3_emulate_jcc(regs, p->ainsn.jcc.type, ip, p->ainsn.rel32);
 }
 NOKPROBE_SYMBOL(kprobe_emulate_jcc);
 
 static void kprobe_emulate_loop(struct kprobe *p, struct pt_regs *regs)
 {
+       unsigned long ip = regs->ip - INT3_INSN_SIZE + p->ainsn.size;
        bool match;
 
        if (p->ainsn.loop.type != 3) {  /* LOOP* */
@@ -531,7 +507,9 @@ static void kprobe_emulate_loop(struct kprobe *p, struct pt_regs *regs)
        else if (p->ainsn.loop.type == 1)       /* LOOPE */
                match = match && (regs->flags & X86_EFLAGS_ZF);
 
-       __kprobe_emulate_jmp(p, regs, match);
+       if (match)
+               ip += p->ainsn.rel32;
+       int3_emulate_jmp(regs, ip);
 }
 NOKPROBE_SYMBOL(kprobe_emulate_loop);