x86/ibt,entry: Sprinkle ENDBR dust

Kernel entry points should be having ENDBR on for IBT configs.

The SYSCALL entry points are found through taking their respective
address in order to program them in the MSRs, while the exception
entry points are found through UNWIND_HINT_IRET_REGS.

The rule is that any UNWIND_HINT_IRET_REGS at sym+0 should have an
ENDBR, see the later objtool ibt validation patch.

Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Acked-by: Josh Poimboeuf <jpoimboe@redhat.com>
Link: https://lore.kernel.org/r/20220308154317.933157479@infradead.org

diff --git a/arch/x86/entry/entry_64.S b/arch/x86/entry/entry_64.S
index 3fd38286302d345e1631bd3e422c1f846ba0f697..50b61186f8b225a589e4d5237a4f88708170fd01 100644 (file)
--- a/arch/x86/entry/entry_64.S
+++ b/arch/x86/entry/entry_64.S
@@ -86,6 +86,7 @@
 
 SYM_CODE_START(entry_SYSCALL_64)
        UNWIND_HINT_EMPTY
+       ENDBR
 
        swapgs
        /* tss.sp2 is scratch space. */
@@ -350,6 +351,7 @@ SYM_CODE_END(ret_from_fork)
 .macro idtentry vector asmsym cfunc has_error_code:req
 SYM_CODE_START(\asmsym)
        UNWIND_HINT_IRET_REGS offset=\has_error_code*8
+       ENDBR
        ASM_CLAC
 
        .if \has_error_code == 0
@@ -417,6 +419,7 @@ SYM_CODE_END(\asmsym)
 .macro idtentry_mce_db vector asmsym cfunc
 SYM_CODE_START(\asmsym)
        UNWIND_HINT_IRET_REGS
+       ENDBR
        ASM_CLAC
 
        pushq   $-1                     /* ORIG_RAX: no syscall to restart */
@@ -472,6 +475,7 @@ SYM_CODE_END(\asmsym)
 .macro idtentry_vc vector asmsym cfunc
 SYM_CODE_START(\asmsym)
        UNWIND_HINT_IRET_REGS
+       ENDBR
        ASM_CLAC
 
        /*
@@ -533,6 +537,7 @@ SYM_CODE_END(\asmsym)
 .macro idtentry_df vector asmsym cfunc
 SYM_CODE_START(\asmsym)
        UNWIND_HINT_IRET_REGS offset=8
+       ENDBR
        ASM_CLAC
 
        /* paranoid_entry returns GS information for paranoid_exit in EBX. */
@@ -1069,6 +1074,7 @@ SYM_CODE_END(error_return)
  */
 SYM_CODE_START(asm_exc_nmi)
        UNWIND_HINT_IRET_REGS
+       ENDBR
 
        /*
         * We allow breakpoints in NMIs. If a breakpoint occurs, then
@@ -1427,6 +1433,7 @@ SYM_CODE_END(asm_exc_nmi)
  */
 SYM_CODE_START(ignore_sysret)
        UNWIND_HINT_EMPTY
+       ENDBR
        mov     $-ENOSYS, %eax
        sysretl
 SYM_CODE_END(ignore_sysret)

diff --git a/arch/x86/entry/entry_64_compat.S b/arch/x86/entry/entry_64_compat.S
index 0051cf5c792d1aeccb7800ba4c02b354c24f885c..35a0e69cf3873fe2ce6d432ade7ae50537cfd686 100644 (file)
--- a/arch/x86/entry/entry_64_compat.S
+++ b/arch/x86/entry/entry_64_compat.S
@@ -48,6 +48,7 @@
  */
 SYM_CODE_START(entry_SYSENTER_compat)
        UNWIND_HINT_EMPTY
+       ENDBR
        /* Interrupts are off on entry. */
        SWAPGS
 
@@ -198,6 +199,7 @@ SYM_CODE_END(entry_SYSENTER_compat)
  */
 SYM_CODE_START(entry_SYSCALL_compat)
        UNWIND_HINT_EMPTY
+       ENDBR
        /* Interrupts are off on entry. */
        swapgs
 
@@ -340,6 +342,7 @@ SYM_CODE_END(entry_SYSCALL_compat)
  */
 SYM_CODE_START(entry_INT80_compat)
        UNWIND_HINT_EMPTY
+       ENDBR
        /*
         * Interrupts are off on entry.
         */

diff --git a/arch/x86/include/asm/idtentry.h b/arch/x86/include/asm/idtentry.h
index 1345088e99025b6a92ab556ca1a3660e4a14b850..f84280ab213cf152e094d843bf9f44ecb3d97104 100644 (file)
--- a/arch/x86/include/asm/idtentry.h
+++ b/arch/x86/include/asm/idtentry.h
@@ -5,6 +5,8 @@
 /* Interrupts/Exceptions */
 #include <asm/trapnr.h>
 
+#define IDT_ALIGN      (8 * (1 + HAS_KERNEL_IBT))
+
 #ifndef __ASSEMBLY__
 #include <linux/entry-common.h>
 #include <linux/hardirq.h>
@@ -480,7 +482,7 @@ __visible noinstr void func(struct pt_regs *regs,                   \
 
 /*
  * ASM code to emit the common vector entry stubs where each stub is
- * packed into 8 bytes.
+ * packed into IDT_ALIGN bytes.
  *
  * Note, that the 'pushq imm8' is emitted via '.byte 0x6a, vector' because
  * GCC treats the local vector variable as unsigned int and would expand
@@ -492,33 +494,33 @@ __visible noinstr void func(struct pt_regs *regs,                 \
  * point is to mask off the bits above bit 7 because the push is sign
  * extending.
  */
-       .align 8
+       .align IDT_ALIGN
 SYM_CODE_START(irq_entries_start)
     vector=FIRST_EXTERNAL_VECTOR
     .rept NR_EXTERNAL_VECTORS
        UNWIND_HINT_IRET_REGS
 0 :
+       ENDBR
        .byte   0x6a, vector
        jmp     asm_common_interrupt
-       nop
-       /* Ensure that the above is 8 bytes max */
-       . = 0b + 8
+       /* Ensure that the above is IDT_ALIGN bytes max */
+       .fill 0b + IDT_ALIGN - ., 1, 0xcc
        vector = vector+1
     .endr
 SYM_CODE_END(irq_entries_start)
 
 #ifdef CONFIG_X86_LOCAL_APIC
-       .align 8
+       .align IDT_ALIGN
 SYM_CODE_START(spurious_entries_start)
     vector=FIRST_SYSTEM_VECTOR
     .rept NR_SYSTEM_VECTORS
        UNWIND_HINT_IRET_REGS
 0 :
+       ENDBR
        .byte   0x6a, vector
        jmp     asm_spurious_interrupt
-       nop
-       /* Ensure that the above is 8 bytes max */
-       . = 0b + 8
+       /* Ensure that the above is IDT_ALIGN bytes max */
+       .fill 0b + IDT_ALIGN - ., 1, 0xcc
        vector = vector+1
     .endr
 SYM_CODE_END(spurious_entries_start)

diff --git a/arch/x86/include/asm/segment.h b/arch/x86/include/asm/segment.h
index 3a31d4ea61ea448c2fbeaa3f2b8b75ce062347ab..656ed6531d035d20d9aed5aaaddc79a32b065619 100644 (file)
--- a/arch/x86/include/asm/segment.h
+++ b/arch/x86/include/asm/segment.h
@@ -4,6 +4,7 @@
 
 #include <linux/const.h>
 #include <asm/alternative.h>
+#include <asm/ibt.h>
 
 /*
  * Constructor for a conventional segment GDT (or LDT) entry.
@@ -275,7 +276,7 @@ static inline void vdso_read_cpunode(unsigned *cpu, unsigned *node)
  * vector has no error code (two bytes), a 'push $vector_number' (two
  * bytes), and a jump to the common entry code (up to five bytes).
  */
-#define EARLY_IDT_HANDLER_SIZE 9
+#define EARLY_IDT_HANDLER_SIZE (9 + ENDBR_INSN_SIZE)
 
 /*
  * xen_early_idt_handler_array is for Xen pv guests: for each entry in

diff --git a/arch/x86/kernel/head_64.S b/arch/x86/kernel/head_64.S
index 990960a8bdb447ef89f31a8a623fa780c8c63b06..9b6fa760e1dfb4c188ed59545c361d40bbf8c9d0 100644 (file)
--- a/arch/x86/kernel/head_64.S
+++ b/arch/x86/kernel/head_64.S
@@ -371,9 +371,11 @@ SYM_CODE_START(early_idt_handler_array)
        .rept NUM_EXCEPTION_VECTORS
        .if ((EXCEPTION_ERRCODE_MASK >> i) & 1) == 0
                UNWIND_HINT_IRET_REGS
+               ENDBR
                pushq $0        # Dummy error code, to make stack frame uniform
        .else
                UNWIND_HINT_IRET_REGS offset=8
+               ENDBR
        .endif
        pushq $i                # 72(%rsp) Vector number
        jmp early_idt_handler_common
@@ -381,11 +383,11 @@ SYM_CODE_START(early_idt_handler_array)
        i = i + 1
        .fill early_idt_handler_array + i*EARLY_IDT_HANDLER_SIZE - ., 1, 0xcc
        .endr
-       UNWIND_HINT_IRET_REGS offset=16
 SYM_CODE_END(early_idt_handler_array)
        ANNOTATE_NOENDBR // early_idt_handler_array[NUM_EXCEPTION_VECTORS]
 
 SYM_CODE_START_LOCAL(early_idt_handler_common)
+       UNWIND_HINT_IRET_REGS offset=16
        /*
         * The stack is the hardware frame, an error code or zero, and the
         * vector number.

diff --git a/arch/x86/kernel/idt.c b/arch/x86/kernel/idt.c
index df0fa695bb09cb1b9b9b9660749b3909ed376263..7676e3444c83c0b5c20bb03f803bc9803e69274d 100644 (file)
--- a/arch/x86/kernel/idt.c
+++ b/arch/x86/kernel/idt.c
@@ -10,6 +10,7 @@
 #include <asm/proto.h>
 #include <asm/desc.h>
 #include <asm/hw_irq.h>
+#include <asm/idtentry.h>
 
 #define DPL0           0x0
 #define DPL3           0x3
@@ -272,7 +273,7 @@ void __init idt_setup_apic_and_irq_gates(void)
        idt_setup_from_table(idt_table, apic_idts, ARRAY_SIZE(apic_idts), true);
 
        for_each_clear_bit_from(i, system_vectors, FIRST_SYSTEM_VECTOR) {
-               entry = irq_entries_start + 8 * (i - FIRST_EXTERNAL_VECTOR);
+               entry = irq_entries_start + IDT_ALIGN * (i - FIRST_EXTERNAL_VECTOR);
                set_intr_gate(i, entry);
        }
 
@@ -283,7 +284,7 @@ void __init idt_setup_apic_and_irq_gates(void)
                 * system_vectors bitmap. Otherwise they show up in
                 * /proc/interrupts.
                 */
-               entry = spurious_entries_start + 8 * (i - FIRST_SYSTEM_VECTOR);
+               entry = spurious_entries_start + IDT_ALIGN * (i - FIRST_SYSTEM_VECTOR);
                set_intr_gate(i, entry);
        }
 #endif