x86/ibt,xen: Sprinkle the ENDBR

Even though Xen currently doesn't advertise IBT, prepare for when it
will eventually do so and sprinkle the ENDBR dust accordingly.

Even though most of the entry points are IRET like, the CPL0
Hypervisor can set WAIT-FOR-ENDBR and demand ENDBR at these sites.

Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Acked-by: Josh Poimboeuf <jpoimboe@redhat.com>
Link: https://lore.kernel.org/r/20220308154317.873919996@infradead.org

diff --git a/arch/x86/entry/entry_64.S b/arch/x86/entry/entry_64.S
index f731393b5af4d9685bdfe7a13adec95720130aa1..3fd38286302d345e1631bd3e422c1f846ba0f697 100644 (file)
--- a/arch/x86/entry/entry_64.S
+++ b/arch/x86/entry/entry_64.S
@@ -809,6 +809,7 @@ SYM_CODE_END(exc_xen_hypervisor_callback)
  */
 SYM_CODE_START(xen_failsafe_callback)
        UNWIND_HINT_EMPTY
+       ENDBR
        movl    %ds, %ecx
        cmpw    %cx, 0x10(%rsp)
        jne     1f

diff --git a/arch/x86/include/asm/segment.h b/arch/x86/include/asm/segment.h
index b228c9d44ee77c6ac82953da994d3f402ab96074..3a31d4ea61ea448c2fbeaa3f2b8b75ce062347ab 100644 (file)
--- a/arch/x86/include/asm/segment.h
+++ b/arch/x86/include/asm/segment.h
@@ -283,7 +283,7 @@ static inline void vdso_read_cpunode(unsigned *cpu, unsigned *node)
  * pop %rcx; pop %r11; jmp early_idt_handler_array[i]; summing up to
  * max 8 bytes.
  */
-#define XEN_EARLY_IDT_HANDLER_SIZE 8
+#define XEN_EARLY_IDT_HANDLER_SIZE (8 + ENDBR_INSN_SIZE)
 
 #ifndef __ASSEMBLY__
 

diff --git a/arch/x86/kernel/head_64.S b/arch/x86/kernel/head_64.S
index 023761cd690358e4843e60617cd97c6dfddc77d8..990960a8bdb447ef89f31a8a623fa780c8c63b06 100644 (file)
--- a/arch/x86/kernel/head_64.S
+++ b/arch/x86/kernel/head_64.S
@@ -383,6 +383,7 @@ SYM_CODE_START(early_idt_handler_array)
        .endr
        UNWIND_HINT_IRET_REGS offset=16
 SYM_CODE_END(early_idt_handler_array)
+       ANNOTATE_NOENDBR // early_idt_handler_array[NUM_EXCEPTION_VECTORS]
 
 SYM_CODE_START_LOCAL(early_idt_handler_common)
        /*

diff --git a/arch/x86/xen/enlighten_pv.c b/arch/x86/xen/enlighten_pv.c
index 3c7a55c43f139060967cb12f3c5518493faeeb85..5038edb79ad518750a6f665b61ec0d2a28d6e21b 100644 (file)
--- a/arch/x86/xen/enlighten_pv.c
+++ b/arch/x86/xen/enlighten_pv.c
@@ -624,6 +624,9 @@ static struct trap_array_entry trap_array[] = {
        TRAP_ENTRY(exc_coprocessor_error,               false ),
        TRAP_ENTRY(exc_alignment_check,                 false ),
        TRAP_ENTRY(exc_simd_coprocessor_error,          false ),
+#ifdef CONFIG_X86_KERNEL_IBT
+       TRAP_ENTRY(exc_control_protection,              false ),
+#endif
 };
 
 static bool __ref get_trap_addr(void **addr, unsigned int ist)

diff --git a/arch/x86/xen/xen-asm.S b/arch/x86/xen/xen-asm.S
index ee17b94e3fc0b95a76d645560054d3ec9bb14cfb..caa9bc2fa100897cff0d5fa79ec4fb27a249871d 100644 (file)
--- a/arch/x86/xen/xen-asm.S
+++ b/arch/x86/xen/xen-asm.S
@@ -122,6 +122,7 @@ SYM_FUNC_END(xen_read_cr2_direct);
 .macro xen_pv_trap name
 SYM_CODE_START(xen_\name)
        UNWIND_HINT_EMPTY
+       ENDBR
        pop %rcx
        pop %r11
        jmp  \name
@@ -147,6 +148,9 @@ xen_pv_trap asm_exc_page_fault
 xen_pv_trap asm_exc_spurious_interrupt_bug
 xen_pv_trap asm_exc_coprocessor_error
 xen_pv_trap asm_exc_alignment_check
+#ifdef CONFIG_X86_KERNEL_IBT
+xen_pv_trap asm_exc_control_protection
+#endif
 #ifdef CONFIG_X86_MCE
 xen_pv_trap asm_xenpv_exc_machine_check
 #endif /* CONFIG_X86_MCE */
@@ -162,6 +166,7 @@ SYM_CODE_START(xen_early_idt_handler_array)
        i = 0
        .rept NUM_EXCEPTION_VECTORS
        UNWIND_HINT_EMPTY
+       ENDBR
        pop %rcx
        pop %r11
        jmp early_idt_handler_array + i*EARLY_IDT_HANDLER_SIZE
@@ -231,6 +236,7 @@ SYM_CODE_END(xenpv_restore_regs_and_return_to_usermode)
 /* Normal 64-bit system call target */
 SYM_CODE_START(xen_syscall_target)
        UNWIND_HINT_EMPTY
+       ENDBR
        popq %rcx
        popq %r11
 
@@ -250,6 +256,7 @@ SYM_CODE_END(xen_syscall_target)
 /* 32-bit compat syscall target */
 SYM_CODE_START(xen_syscall32_target)
        UNWIND_HINT_EMPTY
+       ENDBR
        popq %rcx
        popq %r11
 
@@ -267,6 +274,7 @@ SYM_CODE_END(xen_syscall32_target)
 /* 32-bit compat sysenter target */
 SYM_CODE_START(xen_sysenter_target)
        UNWIND_HINT_EMPTY
+       ENDBR
        /*
         * NB: Xen is polite and clears TF from EFLAGS for us.  This means
         * that we don't need to guard against single step exceptions here.
@@ -290,6 +298,7 @@ SYM_CODE_END(xen_sysenter_target)
 SYM_CODE_START(xen_syscall32_target)
 SYM_CODE_START(xen_sysenter_target)
        UNWIND_HINT_EMPTY
+       ENDBR
        lea 16(%rsp), %rsp      /* strip %rcx, %r11 */
        mov $-ENOSYS, %rax
        pushq $0

diff --git a/arch/x86/xen/xen-head.S b/arch/x86/xen/xen-head.S
index 11d286529fe542b18091e961ab1a7d70f1b8bcd4..ac17196e2518775eaac5a1c69d7001203de90885 100644 (file)
--- a/arch/x86/xen/xen-head.S
+++ b/arch/x86/xen/xen-head.S
@@ -25,8 +25,12 @@
 SYM_CODE_START(hypercall_page)
        .rept (PAGE_SIZE / 32)
                UNWIND_HINT_FUNC
-               .skip 31, 0x90
-               RET
+               ANNOTATE_NOENDBR
+               ret
+               /*
+                * Xen will write the hypercall page, and sort out ENDBR.
+                */
+               .skip 31, 0xcc
        .endr
 
 #define HYPERCALL(n) \
@@ -74,6 +78,7 @@ SYM_CODE_END(startup_xen)
 .pushsection .text
 SYM_CODE_START(asm_cpu_bringup_and_idle)
        UNWIND_HINT_EMPTY
+       ENDBR
 
        call cpu_bringup_and_idle
 SYM_CODE_END(asm_cpu_bringup_and_idle)