x86/entry: Switch the stack after error_entry() returns

commit 520a7e80c96d655fbe4650d9cc985bd9d0443389 upstream.

error_entry() calls fixup_bad_iret() before sync_regs() if it is a fault
from a bad IRET, to copy pt_regs to the kernel stack. It switches to the
kernel stack directly after sync_regs().

But error_entry() itself is also a function call, so it has to stash
the address it is going to return to, in %r12 which is unnecessarily
complicated.

Move the stack switching after error_entry() and get rid of the need to
handle the return address.

  [ bp: Massage commit message. ]

Signed-off-by: Lai Jiangshan <jiangshan.ljs@antgroup.com>
Signed-off-by: Borislav Petkov <bp@suse.de>
Link: https://lore.kernel.org/r/20220503032107.680190-3-jiangshanlai@gmail.com
Signed-off-by: Thadeu Lima de Souza Cascardo <cascardo@canonical.com>
Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>

diff --git a/arch/x86/entry/entry_64.S b/arch/x86/entry/entry_64.S
index 1b4796af97a05d5c455fd78090815393d61c53cd..ce1b436416e01967a30fa4a5771bb23f2e988b5b 100644 (file)
--- a/arch/x86/entry/entry_64.S
+++ b/arch/x86/entry/entry_64.S
@@ -323,6 +323,8 @@ SYM_CODE_END(ret_from_fork)
 .macro idtentry_body cfunc has_error_code:req
 
        call    error_entry
+       movq    %rax, %rsp                      /* switch to the task stack if from userspace */
+       ENCODE_FRAME_POINTER
        UNWIND_HINT_REGS
 
        movq    %rsp, %rdi                      /* pt_regs pointer into 1st argument*/
@@ -982,14 +984,10 @@ SYM_CODE_START_LOCAL(error_entry)
        /* We have user CR3.  Change to kernel CR3. */
        SWITCH_TO_KERNEL_CR3 scratch_reg=%rax
 
+       leaq    8(%rsp), %rdi                   /* arg0 = pt_regs pointer */
 .Lerror_entry_from_usermode_after_swapgs:
        /* Put us onto the real thread stack. */
-       popq    %r12                            /* save return addr in %12 */
-       movq    %rsp, %rdi                      /* arg0 = pt_regs pointer */
        call    sync_regs
-       movq    %rax, %rsp                      /* switch stack */
-       ENCODE_FRAME_POINTER
-       pushq   %r12
        RET
 
        /*
@@ -1021,6 +1019,7 @@ SYM_CODE_START_LOCAL(error_entry)
         */
 .Lerror_entry_done_lfence:
        FENCE_SWAPGS_KERNEL_ENTRY
+       leaq    8(%rsp), %rax                   /* return pt_regs pointer */
        RET
 
 .Lbstep_iret:
@@ -1041,12 +1040,9 @@ SYM_CODE_START_LOCAL(error_entry)
         * Pretend that the exception came from user mode: set up pt_regs
         * as if we faulted immediately after IRET.
         */
-       popq    %r12                            /* save return addr in %12 */
-       movq    %rsp, %rdi                      /* arg0 = pt_regs pointer */
+       leaq    8(%rsp), %rdi                   /* arg0 = pt_regs pointer */
        call    fixup_bad_iret
-       mov     %rax, %rsp
-       ENCODE_FRAME_POINTER
-       pushq   %r12
+       mov     %rax, %rdi
        jmp     .Lerror_entry_from_usermode_after_swapgs
 SYM_CODE_END(error_entry)