x86/entry_32: Remove .fixup usage

[ Upstream commit aa93e2ad7464ffb90155a5ffdde963816f86d5dc ]

Where possible, push the .fixup into code, at the tail of functions.

This is hard for macros since they're used in multiple functions,
therefore introduce a new extable handler to pop zeros.

Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Reviewed-by: Josh Poimboeuf <jpoimboe@redhat.com>
Link: https://lore.kernel.org/r/20211110101325.245184699@infradead.org
Signed-off-by: Sasha Levin <sashal@kernel.org>

diff --git a/arch/x86/entry/entry_32.S b/arch/x86/entry/entry_32.S
index 5bd3baf36d874b23c198cb8a98f1300a3f9e2bde..2cba70f9753bc0918161ea56d563bfd3df3c0708 100644 (file)
--- a/arch/x86/entry/entry_32.S
+++ b/arch/x86/entry/entry_32.S
@@ -270,17 +270,9 @@
 3:     popl    %fs
        addl    $(4 + \pop), %esp       /* pop the unused "gs" slot */
        IRET_FRAME
-.pushsection .fixup, "ax"
-4:     movl    $0, (%esp)
-       jmp     1b
-5:     movl    $0, (%esp)
-       jmp     2b
-6:     movl    $0, (%esp)
-       jmp     3b
-.popsection
-       _ASM_EXTABLE(1b, 4b)
-       _ASM_EXTABLE(2b, 5b)
-       _ASM_EXTABLE(3b, 6b)
+       _ASM_EXTABLE_TYPE(1b, 1b, EX_TYPE_POP_ZERO)
+       _ASM_EXTABLE_TYPE(2b, 2b, EX_TYPE_POP_ZERO)
+       _ASM_EXTABLE_TYPE(3b, 3b, EX_TYPE_POP_ZERO)
 .endm
 
 .macro RESTORE_ALL_NMI cr3_reg:req pop=0
@@ -923,10 +915,8 @@ SYM_FUNC_START(entry_SYSENTER_32)
        sti
        sysexit
 
-.pushsection .fixup, "ax"
-2:     movl    $0, PT_FS(%esp)
-       jmp     1b
-.popsection
+2:     movl    $0, PT_FS(%esp)
+       jmp     1b
        _ASM_EXTABLE(1b, 2b)
 
 .Lsysenter_fix_flags:
@@ -994,8 +984,7 @@ restore_all_switch_stack:
         */
        iret
 
-.section .fixup, "ax"
-SYM_CODE_START(asm_iret_error)
+.Lasm_iret_error:
        pushl   $0                              # no error code
        pushl   $iret_error
 
@@ -1012,9 +1001,8 @@ SYM_CODE_START(asm_iret_error)
 #endif
 
        jmp     handle_exception
-SYM_CODE_END(asm_iret_error)
-.previous
-       _ASM_EXTABLE(.Lirq_return, asm_iret_error)
+
+       _ASM_EXTABLE(.Lirq_return, .Lasm_iret_error)
 SYM_FUNC_END(entry_INT80_32)
 
 .macro FIXUP_ESPFIX_STACK

diff --git a/arch/x86/include/asm/extable_fixup_types.h b/arch/x86/include/asm/extable_fixup_types.h
index 409524d5d2eb1f4b14507f2c221bfe640f78c5cd..4d709a2768bb881ff2926db0b737d118ae0e9cf6 100644 (file)
--- a/arch/x86/include/asm/extable_fixup_types.h
+++ b/arch/x86/include/asm/extable_fixup_types.h
@@ -19,4 +19,6 @@
 #define        EX_TYPE_DEFAULT_MCE_SAFE        12
 #define        EX_TYPE_FAULT_MCE_SAFE          13
 
+#define        EX_TYPE_POP_ZERO                14
+
 #endif

diff --git a/arch/x86/mm/extable.c b/arch/x86/mm/extable.c
index f37e290e6d0ab4a19dfeb5a533f5e337a99a38f2..f59a4d017070a23e4c0b096a1021c62d7e993cc3 100644 (file)
--- a/arch/x86/mm/extable.c
+++ b/arch/x86/mm/extable.c
@@ -99,6 +99,18 @@ static bool ex_handler_clear_fs(const struct exception_table_entry *fixup,
        return ex_handler_default(fixup, regs);
 }
 
+static bool ex_handler_pop_zero(const struct exception_table_entry *fixup,
+                               struct pt_regs *regs)
+{
+       /*
+        * Typically used for when "pop %seg" traps, in which case we'll clear
+        * the stack slot and re-try the instruction, which will then succeed
+        * to pop zero.
+        */
+       *((unsigned long *)regs->sp) = 0;
+       return ex_handler_default(fixup, regs);
+}
+
 int ex_get_fixup_type(unsigned long ip)
 {
        const struct exception_table_entry *e = search_exception_tables(ip);
@@ -156,6 +168,8 @@ int fixup_exception(struct pt_regs *regs, int trapnr, unsigned long error_code,
        case EX_TYPE_WRMSR_IN_MCE:
                ex_handler_msr_mce(regs, true);
                break;
+       case EX_TYPE_POP_ZERO:
+               return ex_handler_pop_zero(e, regs);
        }
        BUG();
 }