x86/paravirt: Convert simple paravirt functions to asm

All functions referenced via __PV_IS_CALLEE_SAVE() need to be assembler
functions, as those functions calls are hidden from the compiler.

In case the kernel is compiled with "-fzero-call-used-regs" the compiler
will clobber caller-saved registers at the end of C functions, which
will result in unexpectedly zeroed registers at the call site of the
related paravirt functions.

Replace the C functions with DEFINE_PARAVIRT_ASM() constructs using
the same instructions as the related paravirt calls in the
PVOP_ALT_[V]CALLEE*() macros. And since they're not C functions visible
to the compiler anymore, latter won't do the callee-clobbered zeroing
invoked by -fzero-call-used-regs and thus won't corrupt registers.

  [ bp: Extend commit message. ]

Signed-off-by: Juergen Gross <jgross@suse.com>
Signed-off-by: Borislav Petkov (AMD) <bp@alien8.de>
Link: https://lore.kernel.org/r/20230317063325.361-1-jgross@suse.com

diff --git a/arch/x86/include/asm/paravirt_types.h b/arch/x86/include/asm/paravirt_types.h
index 71bf64b963dfa384fc740fffc01e73048e2357d6..4acbcddddc29f21fc9f21117b653c842430d943c 100644 (file)
--- a/arch/x86/include/asm/paravirt_types.h
+++ b/arch/x86/include/asm/paravirt_types.h
@@ -559,8 +559,14 @@ void paravirt_flush_lazy_mmu(void);
 
 void _paravirt_nop(void);
 void paravirt_BUG(void);
-u64 _paravirt_ident_64(u64);
 unsigned long paravirt_ret0(void);
+#ifdef CONFIG_PARAVIRT_XXL
+u64 _paravirt_ident_64(u64);
+unsigned long pv_native_save_fl(void);
+void pv_native_irq_disable(void);
+void pv_native_irq_enable(void);
+unsigned long pv_native_read_cr2(void);
+#endif
 
 #define paravirt_nop   ((void *)_paravirt_nop)
 

diff --git a/arch/x86/kernel/paravirt.c b/arch/x86/kernel/paravirt.c
index 0e68a31be7c121da744c025e069fe91a352bc5cc..ac10b46c5832bf9f57ababa615903e877bd8f02a 100644 (file)
--- a/arch/x86/kernel/paravirt.c
+++ b/arch/x86/kernel/paravirt.c
@@ -64,11 +64,11 @@ static unsigned paravirt_patch_call(void *insn_buff, const void *target,
 }
 
 #ifdef CONFIG_PARAVIRT_XXL
-/* identity function, which can be inlined */
-u64 notrace _paravirt_ident_64(u64 x)
-{
-       return x;
-}
+DEFINE_PARAVIRT_ASM(_paravirt_ident_64, "mov %rdi, %rax", .text);
+DEFINE_PARAVIRT_ASM(pv_native_save_fl, "pushf; pop %rax", .noinstr.text);
+DEFINE_PARAVIRT_ASM(pv_native_irq_disable, "cli", .noinstr.text);
+DEFINE_PARAVIRT_ASM(pv_native_irq_enable, "sti", .noinstr.text);
+DEFINE_PARAVIRT_ASM(pv_native_read_cr2, "mov %cr2, %rax", .noinstr.text);
 #endif
 
 DEFINE_STATIC_KEY_TRUE(virt_spin_lock_key);
@@ -197,11 +197,6 @@ void paravirt_end_context_switch(struct task_struct *next)
                arch_enter_lazy_mmu_mode();
 }
 
-static noinstr unsigned long pv_native_read_cr2(void)
-{
-       return native_read_cr2();
-}
-
 static noinstr void pv_native_write_cr2(unsigned long val)
 {
        native_write_cr2(val);
@@ -222,16 +217,6 @@ noinstr void pv_native_wbinvd(void)
        native_wbinvd();
 }
 
-static noinstr void pv_native_irq_enable(void)
-{
-       native_irq_enable();
-}
-
-static noinstr void pv_native_irq_disable(void)
-{
-       native_irq_disable();
-}
-
 static noinstr void pv_native_safe_halt(void)
 {
        native_safe_halt();
@@ -298,7 +283,7 @@ struct paravirt_patch_template pv_ops = {
        .cpu.end_context_switch         = paravirt_nop,
 
        /* Irq ops. */
-       .irq.save_fl            = __PV_IS_CALLEE_SAVE(native_save_fl),
+       .irq.save_fl            = __PV_IS_CALLEE_SAVE(pv_native_save_fl),
        .irq.irq_disable        = __PV_IS_CALLEE_SAVE(pv_native_irq_disable),
        .irq.irq_enable         = __PV_IS_CALLEE_SAVE(pv_native_irq_enable),
        .irq.safe_halt          = pv_native_safe_halt,