KVM: arm64: Split hyp/sysreg-sr.c to VHE/nVHE

sysreg-sr.c contains KVM's code for saving/restoring system registers, with
some code shared between VHE/nVHE. These common routines are moved to
a header file, VHE-specific code is moved to vhe/sysreg-sr.c and nVHE-specific
code to nvhe/sysreg-sr.c.

Signed-off-by: David Brazdil <dbrazdil@google.com>
Signed-off-by: Marc Zyngier <maz@kernel.org>
Link: https://lore.kernel.org/r/20200625131420.71444-12-dbrazdil@google.com

diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h
index 49d1a5cd8f8f495cda1f85a6c34b1e1c005ab115..e0920df1d0c144c8c23acc7ddbaa3fe4e4f84ad9 100644 (file)
--- a/arch/arm64/include/asm/kvm_host.h
+++ b/arch/arm64/include/asm/kvm_host.h
@@ -338,7 +338,7 @@ struct kvm_vcpu_arch {
        struct vcpu_reset_state reset_state;
 
        /* True when deferrable sysregs are loaded on the physical CPU,
-        * see kvm_vcpu_load_sysregs and kvm_vcpu_put_sysregs. */
+        * see kvm_vcpu_load_sysregs_vhe and kvm_vcpu_put_sysregs_vhe. */
        bool sysregs_loaded_on_cpu;
 
        /* Guest PV state */
@@ -639,8 +639,8 @@ static inline int kvm_arm_have_ssbd(void)
        }
 }
 
-void kvm_vcpu_load_sysregs(struct kvm_vcpu *vcpu);
-void kvm_vcpu_put_sysregs(struct kvm_vcpu *vcpu);
+void kvm_vcpu_load_sysregs_vhe(struct kvm_vcpu *vcpu);
+void kvm_vcpu_put_sysregs_vhe(struct kvm_vcpu *vcpu);
 
 int kvm_set_ipa_limit(void);
 

diff --git a/arch/arm64/include/asm/kvm_hyp.h b/arch/arm64/include/asm/kvm_hyp.h
index 82fa05d15b8bd566c93017d4219d37bcb9816040..997c5bda1ac79b304bb0db1269a534ed07f4bd18 100644 (file)
--- a/arch/arm64/include/asm/kvm_hyp.h
+++ b/arch/arm64/include/asm/kvm_hyp.h
@@ -66,14 +66,15 @@ int __vgic_v3_perform_cpuif_access(struct kvm_vcpu *vcpu);
 void __timer_enable_traps(struct kvm_vcpu *vcpu);
 void __timer_disable_traps(struct kvm_vcpu *vcpu);
 
+#ifdef __KVM_NVHE_HYPERVISOR__
 void __sysreg_save_state_nvhe(struct kvm_cpu_context *ctxt);
 void __sysreg_restore_state_nvhe(struct kvm_cpu_context *ctxt);
+#else
 void sysreg_save_host_state_vhe(struct kvm_cpu_context *ctxt);
 void sysreg_restore_host_state_vhe(struct kvm_cpu_context *ctxt);
 void sysreg_save_guest_state_vhe(struct kvm_cpu_context *ctxt);
 void sysreg_restore_guest_state_vhe(struct kvm_cpu_context *ctxt);
-void __sysreg32_save_state(struct kvm_vcpu *vcpu);
-void __sysreg32_restore_state(struct kvm_vcpu *vcpu);
+#endif
 
 void __debug_switch_to_guest(struct kvm_vcpu *vcpu);
 void __debug_switch_to_host(struct kvm_vcpu *vcpu);

diff --git a/arch/arm64/kernel/image-vars.h b/arch/arm64/kernel/image-vars.h
index c44ca4e7dd663c2fcf94ede6252db8aca43cc12d..59eb55893eaf64c77a2ddd883b01cfcdba8cd15b 100644 (file)
--- a/arch/arm64/kernel/image-vars.h
+++ b/arch/arm64/kernel/image-vars.h
@@ -76,13 +76,6 @@ KVM_NVHE_ALIAS(abort_guest_exit_start);
 KVM_NVHE_ALIAS(__fpsimd_restore_state);
 KVM_NVHE_ALIAS(__fpsimd_save_state);
 
-/* Symbols defined in sysreg-sr.c (not yet compiled with nVHE build rules). */
-KVM_NVHE_ALIAS(__kvm_enable_ssbs);
-KVM_NVHE_ALIAS(__sysreg32_restore_state);
-KVM_NVHE_ALIAS(__sysreg32_save_state);
-KVM_NVHE_ALIAS(__sysreg_restore_state_nvhe);
-KVM_NVHE_ALIAS(__sysreg_save_state_nvhe);
-
 /* Symbols defined in timer-sr.c (not yet compiled with nVHE build rules). */
 KVM_NVHE_ALIAS(__kvm_timer_set_cntvoff);
 KVM_NVHE_ALIAS(__timer_disable_traps);

diff --git a/arch/arm64/kvm/arm.c b/arch/arm64/kvm/arm.c
index 26780b78a523fd9b980f16cff46093b723fc81c0..0bf2cf5614c6b1b02acbd2e3999e8206556b86c2 100644 (file)
--- a/arch/arm64/kvm/arm.c
+++ b/arch/arm64/kvm/arm.c
@@ -351,7 +351,8 @@ void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
 
        kvm_vgic_load(vcpu);
        kvm_timer_vcpu_load(vcpu);
-       kvm_vcpu_load_sysregs(vcpu);
+       if (has_vhe())
+               kvm_vcpu_load_sysregs_vhe(vcpu);
        kvm_arch_vcpu_load_fp(vcpu);
        kvm_vcpu_pmu_restore_guest(vcpu);
        if (kvm_arm_is_pvtime_enabled(&vcpu->arch))
@@ -369,7 +370,8 @@ void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
 void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
 {
        kvm_arch_vcpu_put_fp(vcpu);
-       kvm_vcpu_put_sysregs(vcpu);
+       if (has_vhe())
+               kvm_vcpu_put_sysregs_vhe(vcpu);
        kvm_timer_vcpu_put(vcpu);
        kvm_vgic_put(vcpu);
        kvm_vcpu_pmu_restore_host(vcpu);
@@ -1302,7 +1304,7 @@ static void cpu_init_hyp_mode(void)
         */
        if (this_cpu_has_cap(ARM64_SSBS) &&
            arm64_get_ssbd_state() == ARM64_SSBD_FORCE_DISABLE) {
-               kvm_call_hyp(__kvm_enable_ssbs);
+               kvm_call_hyp_nvhe(__kvm_enable_ssbs);
        }
 }
 

diff --git a/arch/arm64/kvm/hyp/Makefile b/arch/arm64/kvm/hyp/Makefile
index fc09025d2e97d7d6ffd0d401c359a002c182c95d..f49797237818316522a7d3e5285232fc2f550c8c 100644 (file)
--- a/arch/arm64/kvm/hyp/Makefile
+++ b/arch/arm64/kvm/hyp/Makefile
@@ -13,8 +13,8 @@ subdir-ccflags-y := -I$(incdir)                               \
 obj-$(CONFIG_KVM) += hyp.o vhe/ nvhe/
 obj-$(CONFIG_KVM_INDIRECT_VECTORS) += smccc_wa.o
 
-hyp-y := vgic-v3-sr.o timer-sr.o aarch32.o vgic-v2-cpuif-proxy.o sysreg-sr.o \
-        entry.o fpsimd.o
+hyp-y := vgic-v3-sr.o timer-sr.o aarch32.o vgic-v2-cpuif-proxy.o entry.o \
+        fpsimd.o
 
 # KVM code is run at a different exception code with a different map, so
 # compiler instrumentation that inserts callbacks or checks into the code may

diff --git a/arch/arm64/kvm/hyp/include/hyp/sysreg-sr.h b/arch/arm64/kvm/hyp/include/hyp/sysreg-sr.h
new file mode 100644 (file)
index 0000000..3e0585f
--- /dev/null
+++ b/arch/arm64/kvm/hyp/include/hyp/sysreg-sr.h
@@ -0,0 +1,203 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (C) 2012-2015 - ARM Ltd
+ * Author: Marc Zyngier <marc.zyngier@arm.com>
+ */
+
+#ifndef __ARM64_KVM_HYP_SYSREG_SR_H__
+#define __ARM64_KVM_HYP_SYSREG_SR_H__
+
+#include <linux/compiler.h>
+#include <linux/kvm_host.h>
+
+#include <asm/kprobes.h>
+#include <asm/kvm_asm.h>
+#include <asm/kvm_emulate.h>
+#include <asm/kvm_hyp.h>
+
+static inline void __hyp_text __sysreg_save_common_state(struct kvm_cpu_context *ctxt)
+{
+       ctxt->sys_regs[MDSCR_EL1]       = read_sysreg(mdscr_el1);
+}
+
+static inline void __hyp_text __sysreg_save_user_state(struct kvm_cpu_context *ctxt)
+{
+       ctxt->sys_regs[TPIDR_EL0]       = read_sysreg(tpidr_el0);
+       ctxt->sys_regs[TPIDRRO_EL0]     = read_sysreg(tpidrro_el0);
+}
+
+static inline void __hyp_text __sysreg_save_el1_state(struct kvm_cpu_context *ctxt)
+{
+       ctxt->sys_regs[CSSELR_EL1]      = read_sysreg(csselr_el1);
+       ctxt->sys_regs[SCTLR_EL1]       = read_sysreg_el1(SYS_SCTLR);
+       ctxt->sys_regs[CPACR_EL1]       = read_sysreg_el1(SYS_CPACR);
+       ctxt->sys_regs[TTBR0_EL1]       = read_sysreg_el1(SYS_TTBR0);
+       ctxt->sys_regs[TTBR1_EL1]       = read_sysreg_el1(SYS_TTBR1);
+       ctxt->sys_regs[TCR_EL1]         = read_sysreg_el1(SYS_TCR);
+       ctxt->sys_regs[ESR_EL1]         = read_sysreg_el1(SYS_ESR);
+       ctxt->sys_regs[AFSR0_EL1]       = read_sysreg_el1(SYS_AFSR0);
+       ctxt->sys_regs[AFSR1_EL1]       = read_sysreg_el1(SYS_AFSR1);
+       ctxt->sys_regs[FAR_EL1]         = read_sysreg_el1(SYS_FAR);
+       ctxt->sys_regs[MAIR_EL1]        = read_sysreg_el1(SYS_MAIR);
+       ctxt->sys_regs[VBAR_EL1]        = read_sysreg_el1(SYS_VBAR);
+       ctxt->sys_regs[CONTEXTIDR_EL1]  = read_sysreg_el1(SYS_CONTEXTIDR);
+       ctxt->sys_regs[AMAIR_EL1]       = read_sysreg_el1(SYS_AMAIR);
+       ctxt->sys_regs[CNTKCTL_EL1]     = read_sysreg_el1(SYS_CNTKCTL);
+       ctxt->sys_regs[PAR_EL1]         = read_sysreg(par_el1);
+       ctxt->sys_regs[TPIDR_EL1]       = read_sysreg(tpidr_el1);
+
+       ctxt->gp_regs.sp_el1            = read_sysreg(sp_el1);
+       ctxt->gp_regs.elr_el1           = read_sysreg_el1(SYS_ELR);
+       ctxt->gp_regs.spsr[KVM_SPSR_EL1]= read_sysreg_el1(SYS_SPSR);
+}
+
+static inline void __hyp_text __sysreg_save_el2_return_state(struct kvm_cpu_context *ctxt)
+{
+       ctxt->gp_regs.regs.pc           = read_sysreg_el2(SYS_ELR);
+       ctxt->gp_regs.regs.pstate       = read_sysreg_el2(SYS_SPSR);
+
+       if (cpus_have_final_cap(ARM64_HAS_RAS_EXTN))
+               ctxt->sys_regs[DISR_EL1] = read_sysreg_s(SYS_VDISR_EL2);
+}
+
+static inline void __hyp_text __sysreg_restore_common_state(struct kvm_cpu_context *ctxt)
+{
+       write_sysreg(ctxt->sys_regs[MDSCR_EL1],   mdscr_el1);
+}
+
+static inline void __hyp_text __sysreg_restore_user_state(struct kvm_cpu_context *ctxt)
+{
+       write_sysreg(ctxt->sys_regs[TPIDR_EL0],         tpidr_el0);
+       write_sysreg(ctxt->sys_regs[TPIDRRO_EL0],       tpidrro_el0);
+}
+
+static inline void __hyp_text __sysreg_restore_el1_state(struct kvm_cpu_context *ctxt)
+{
+       write_sysreg(ctxt->sys_regs[MPIDR_EL1],         vmpidr_el2);
+       write_sysreg(ctxt->sys_regs[CSSELR_EL1],        csselr_el1);
+
+       if (has_vhe() ||
+           !cpus_have_final_cap(ARM64_WORKAROUND_SPECULATIVE_AT)) {
+               write_sysreg_el1(ctxt->sys_regs[SCTLR_EL1],     SYS_SCTLR);
+               write_sysreg_el1(ctxt->sys_regs[TCR_EL1],       SYS_TCR);
+       } else  if (!ctxt->__hyp_running_vcpu) {
+               /*
+                * Must only be done for guest registers, hence the context
+                * test. We're coming from the host, so SCTLR.M is already
+                * set. Pairs with nVHE's __activate_traps().
+                */
+               write_sysreg_el1((ctxt->sys_regs[TCR_EL1] |
+                                 TCR_EPD1_MASK | TCR_EPD0_MASK),
+                                SYS_TCR);
+               isb();
+       }
+
+       write_sysreg_el1(ctxt->sys_regs[CPACR_EL1],     SYS_CPACR);
+       write_sysreg_el1(ctxt->sys_regs[TTBR0_EL1],     SYS_TTBR0);
+       write_sysreg_el1(ctxt->sys_regs[TTBR1_EL1],     SYS_TTBR1);
+       write_sysreg_el1(ctxt->sys_regs[ESR_EL1],       SYS_ESR);
+       write_sysreg_el1(ctxt->sys_regs[AFSR0_EL1],     SYS_AFSR0);
+       write_sysreg_el1(ctxt->sys_regs[AFSR1_EL1],     SYS_AFSR1);
+       write_sysreg_el1(ctxt->sys_regs[FAR_EL1],       SYS_FAR);
+       write_sysreg_el1(ctxt->sys_regs[MAIR_EL1],      SYS_MAIR);
+       write_sysreg_el1(ctxt->sys_regs[VBAR_EL1],      SYS_VBAR);
+       write_sysreg_el1(ctxt->sys_regs[CONTEXTIDR_EL1],SYS_CONTEXTIDR);
+       write_sysreg_el1(ctxt->sys_regs[AMAIR_EL1],     SYS_AMAIR);
+       write_sysreg_el1(ctxt->sys_regs[CNTKCTL_EL1],   SYS_CNTKCTL);
+       write_sysreg(ctxt->sys_regs[PAR_EL1],           par_el1);
+       write_sysreg(ctxt->sys_regs[TPIDR_EL1],         tpidr_el1);
+
+       if (!has_vhe() &&
+           cpus_have_final_cap(ARM64_WORKAROUND_SPECULATIVE_AT) &&
+           ctxt->__hyp_running_vcpu) {
+               /*
+                * Must only be done for host registers, hence the context
+                * test. Pairs with nVHE's __deactivate_traps().
+                */
+               isb();
+               /*
+                * At this stage, and thanks to the above isb(), S2 is
+                * deconfigured and disabled. We can now restore the host's
+                * S1 configuration: SCTLR, and only then TCR.
+                */
+               write_sysreg_el1(ctxt->sys_regs[SCTLR_EL1],     SYS_SCTLR);
+               isb();
+               write_sysreg_el1(ctxt->sys_regs[TCR_EL1],       SYS_TCR);
+       }
+
+       write_sysreg(ctxt->gp_regs.sp_el1,              sp_el1);
+       write_sysreg_el1(ctxt->gp_regs.elr_el1,         SYS_ELR);
+       write_sysreg_el1(ctxt->gp_regs.spsr[KVM_SPSR_EL1],SYS_SPSR);
+}
+
+static inline void __hyp_text __sysreg_restore_el2_return_state(struct kvm_cpu_context *ctxt)
+{
+       u64 pstate = ctxt->gp_regs.regs.pstate;
+       u64 mode = pstate & PSR_AA32_MODE_MASK;
+
+       /*
+        * Safety check to ensure we're setting the CPU up to enter the guest
+        * in a less privileged mode.
+        *
+        * If we are attempting a return to EL2 or higher in AArch64 state,
+        * program SPSR_EL2 with M=EL2h and the IL bit set which ensures that
+        * we'll take an illegal exception state exception immediately after
+        * the ERET to the guest.  Attempts to return to AArch32 Hyp will
+        * result in an illegal exception return because EL2's execution state
+        * is determined by SCR_EL3.RW.
+        */
+       if (!(mode & PSR_MODE32_BIT) && mode >= PSR_MODE_EL2t)
+               pstate = PSR_MODE_EL2h | PSR_IL_BIT;
+
+       write_sysreg_el2(ctxt->gp_regs.regs.pc,         SYS_ELR);
+       write_sysreg_el2(pstate,                        SYS_SPSR);
+
+       if (cpus_have_final_cap(ARM64_HAS_RAS_EXTN))
+               write_sysreg_s(ctxt->sys_regs[DISR_EL1], SYS_VDISR_EL2);
+}
+
+static inline void __hyp_text __sysreg32_save_state(struct kvm_vcpu *vcpu)
+{
+       u64 *spsr, *sysreg;
+
+       if (!vcpu_el1_is_32bit(vcpu))
+               return;
+
+       spsr = vcpu->arch.ctxt.gp_regs.spsr;
+       sysreg = vcpu->arch.ctxt.sys_regs;
+
+       spsr[KVM_SPSR_ABT] = read_sysreg(spsr_abt);
+       spsr[KVM_SPSR_UND] = read_sysreg(spsr_und);
+       spsr[KVM_SPSR_IRQ] = read_sysreg(spsr_irq);
+       spsr[KVM_SPSR_FIQ] = read_sysreg(spsr_fiq);
+
+       sysreg[DACR32_EL2] = read_sysreg(dacr32_el2);
+       sysreg[IFSR32_EL2] = read_sysreg(ifsr32_el2);
+
+       if (has_vhe() || vcpu->arch.flags & KVM_ARM64_DEBUG_DIRTY)
+               sysreg[DBGVCR32_EL2] = read_sysreg(dbgvcr32_el2);
+}
+
+static inline void __hyp_text __sysreg32_restore_state(struct kvm_vcpu *vcpu)
+{
+       u64 *spsr, *sysreg;
+
+       if (!vcpu_el1_is_32bit(vcpu))
+               return;
+
+       spsr = vcpu->arch.ctxt.gp_regs.spsr;
+       sysreg = vcpu->arch.ctxt.sys_regs;
+
+       write_sysreg(spsr[KVM_SPSR_ABT], spsr_abt);
+       write_sysreg(spsr[KVM_SPSR_UND], spsr_und);
+       write_sysreg(spsr[KVM_SPSR_IRQ], spsr_irq);
+       write_sysreg(spsr[KVM_SPSR_FIQ], spsr_fiq);
+
+       write_sysreg(sysreg[DACR32_EL2], dacr32_el2);
+       write_sysreg(sysreg[IFSR32_EL2], ifsr32_el2);
+
+       if (has_vhe() || vcpu->arch.flags & KVM_ARM64_DEBUG_DIRTY)
+               write_sysreg(sysreg[DBGVCR32_EL2], dbgvcr32_el2);
+}
+
+#endif /* __ARM64_KVM_HYP_SYSREG_SR_H__ */

diff --git a/arch/arm64/kvm/hyp/nvhe/Makefile b/arch/arm64/kvm/hyp/nvhe/Makefile
index b3cb67b63d671d9324a6dabe9e08aaee1ea439da..61a8160f0dd948f16006a7a8d809c35dd647a2f2 100644 (file)
--- a/arch/arm64/kvm/hyp/nvhe/Makefile
+++ b/arch/arm64/kvm/hyp/nvhe/Makefile
@@ -6,7 +6,7 @@
 asflags-y := -D__KVM_NVHE_HYPERVISOR__
 ccflags-y := -D__KVM_NVHE_HYPERVISOR__
 
-obj-y := debug-sr.o switch.o tlb.o hyp-init.o ../hyp-entry.o
+obj-y := sysreg-sr.o debug-sr.o switch.o tlb.o hyp-init.o ../hyp-entry.o
 
 obj-y := $(patsubst %.o,%.hyp.o,$(obj-y))
 extra-y := $(patsubst %.hyp.o,%.hyp.tmp.o,$(obj-y))

diff --git a/arch/arm64/kvm/hyp/nvhe/switch.c b/arch/arm64/kvm/hyp/nvhe/switch.c
index 7f6b8d3dc637a40e905e11f97234e77fe3dbac47..f08bfb951df6018aa438e98f6517938cff5b3e05 100644 (file)
--- a/arch/arm64/kvm/hyp/nvhe/switch.c
+++ b/arch/arm64/kvm/hyp/nvhe/switch.c
@@ -5,6 +5,7 @@
  */
 
 #include <hyp/switch.h>
+#include <hyp/sysreg-sr.h>
 
 #include <linux/arm-smccc.h>
 #include <linux/kvm_host.h>

diff --git a/arch/arm64/kvm/hyp/nvhe/sysreg-sr.c b/arch/arm64/kvm/hyp/nvhe/sysreg-sr.c
new file mode 100644 (file)
index 0000000..710cf28
--- /dev/null
+++ b/arch/arm64/kvm/hyp/nvhe/sysreg-sr.c
@@ -0,0 +1,46 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (C) 2012-2015 - ARM Ltd
+ * Author: Marc Zyngier <marc.zyngier@arm.com>
+ */
+
+#include <hyp/sysreg-sr.h>
+
+#include <linux/compiler.h>
+#include <linux/kvm_host.h>
+
+#include <asm/kprobes.h>
+#include <asm/kvm_asm.h>
+#include <asm/kvm_emulate.h>
+#include <asm/kvm_hyp.h>
+
+/*
+ * Non-VHE: Both host and guest must save everything.
+ */
+
+void __hyp_text __sysreg_save_state_nvhe(struct kvm_cpu_context *ctxt)
+{
+       __sysreg_save_el1_state(ctxt);
+       __sysreg_save_common_state(ctxt);
+       __sysreg_save_user_state(ctxt);
+       __sysreg_save_el2_return_state(ctxt);
+}
+
+void __hyp_text __sysreg_restore_state_nvhe(struct kvm_cpu_context *ctxt)
+{
+       __sysreg_restore_el1_state(ctxt);
+       __sysreg_restore_common_state(ctxt);
+       __sysreg_restore_user_state(ctxt);
+       __sysreg_restore_el2_return_state(ctxt);
+}
+
+void __hyp_text __kvm_enable_ssbs(void)
+{
+       u64 tmp;
+
+       asm volatile(
+       "mrs    %0, sctlr_el2\n"
+       "orr    %0, %0, %1\n"
+       "msr    sctlr_el2, %0"
+       : "=&r" (tmp) : "L" (SCTLR_ELx_DSSBS));
+}

diff --git a/arch/arm64/kvm/hyp/sysreg-sr.c b/arch/arm64/kvm/hyp/sysreg-sr.c
deleted file mode 100644 (file)
index 2493439..0000000
--- a/arch/arm64/kvm/hyp/sysreg-sr.c
+++ /dev/null
@@ -1,333 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-only
-/*
- * Copyright (C) 2012-2015 - ARM Ltd
- * Author: Marc Zyngier <marc.zyngier@arm.com>
- */
-
-#include <linux/compiler.h>
-#include <linux/kvm_host.h>
-
-#include <asm/kprobes.h>
-#include <asm/kvm_asm.h>
-#include <asm/kvm_emulate.h>
-#include <asm/kvm_hyp.h>
-
-/*
- * Non-VHE: Both host and guest must save everything.
- *
- * VHE: Host and guest must save mdscr_el1 and sp_el0 (and the PC and
- * pstate, which are handled as part of the el2 return state) on every
- * switch (sp_el0 is being dealt with in the assembly code).
- * tpidr_el0 and tpidrro_el0 only need to be switched when going
- * to host userspace or a different VCPU.  EL1 registers only need to be
- * switched when potentially going to run a different VCPU.  The latter two
- * classes are handled as part of kvm_arch_vcpu_load and kvm_arch_vcpu_put.
- */
-
-static void __hyp_text __sysreg_save_common_state(struct kvm_cpu_context *ctxt)
-{
-       ctxt->sys_regs[MDSCR_EL1]       = read_sysreg(mdscr_el1);
-}
-
-static void __hyp_text __sysreg_save_user_state(struct kvm_cpu_context *ctxt)
-{
-       ctxt->sys_regs[TPIDR_EL0]       = read_sysreg(tpidr_el0);
-       ctxt->sys_regs[TPIDRRO_EL0]     = read_sysreg(tpidrro_el0);
-}
-
-static void __hyp_text __sysreg_save_el1_state(struct kvm_cpu_context *ctxt)
-{
-       ctxt->sys_regs[CSSELR_EL1]      = read_sysreg(csselr_el1);
-       ctxt->sys_regs[SCTLR_EL1]       = read_sysreg_el1(SYS_SCTLR);
-       ctxt->sys_regs[CPACR_EL1]       = read_sysreg_el1(SYS_CPACR);
-       ctxt->sys_regs[TTBR0_EL1]       = read_sysreg_el1(SYS_TTBR0);
-       ctxt->sys_regs[TTBR1_EL1]       = read_sysreg_el1(SYS_TTBR1);
-       ctxt->sys_regs[TCR_EL1]         = read_sysreg_el1(SYS_TCR);
-       ctxt->sys_regs[ESR_EL1]         = read_sysreg_el1(SYS_ESR);
-       ctxt->sys_regs[AFSR0_EL1]       = read_sysreg_el1(SYS_AFSR0);
-       ctxt->sys_regs[AFSR1_EL1]       = read_sysreg_el1(SYS_AFSR1);
-       ctxt->sys_regs[FAR_EL1]         = read_sysreg_el1(SYS_FAR);
-       ctxt->sys_regs[MAIR_EL1]        = read_sysreg_el1(SYS_MAIR);
-       ctxt->sys_regs[VBAR_EL1]        = read_sysreg_el1(SYS_VBAR);
-       ctxt->sys_regs[CONTEXTIDR_EL1]  = read_sysreg_el1(SYS_CONTEXTIDR);
-       ctxt->sys_regs[AMAIR_EL1]       = read_sysreg_el1(SYS_AMAIR);
-       ctxt->sys_regs[CNTKCTL_EL1]     = read_sysreg_el1(SYS_CNTKCTL);
-       ctxt->sys_regs[PAR_EL1]         = read_sysreg(par_el1);
-       ctxt->sys_regs[TPIDR_EL1]       = read_sysreg(tpidr_el1);
-
-       ctxt->gp_regs.sp_el1            = read_sysreg(sp_el1);
-       ctxt->gp_regs.elr_el1           = read_sysreg_el1(SYS_ELR);
-       ctxt->gp_regs.spsr[KVM_SPSR_EL1]= read_sysreg_el1(SYS_SPSR);
-}
-
-static void __hyp_text __sysreg_save_el2_return_state(struct kvm_cpu_context *ctxt)
-{
-       ctxt->gp_regs.regs.pc           = read_sysreg_el2(SYS_ELR);
-       ctxt->gp_regs.regs.pstate       = read_sysreg_el2(SYS_SPSR);
-
-       if (cpus_have_final_cap(ARM64_HAS_RAS_EXTN))
-               ctxt->sys_regs[DISR_EL1] = read_sysreg_s(SYS_VDISR_EL2);
-}
-
-void __hyp_text __sysreg_save_state_nvhe(struct kvm_cpu_context *ctxt)
-{
-       __sysreg_save_el1_state(ctxt);
-       __sysreg_save_common_state(ctxt);
-       __sysreg_save_user_state(ctxt);
-       __sysreg_save_el2_return_state(ctxt);
-}
-
-void sysreg_save_host_state_vhe(struct kvm_cpu_context *ctxt)
-{
-       __sysreg_save_common_state(ctxt);
-}
-NOKPROBE_SYMBOL(sysreg_save_host_state_vhe);
-
-void sysreg_save_guest_state_vhe(struct kvm_cpu_context *ctxt)
-{
-       __sysreg_save_common_state(ctxt);
-       __sysreg_save_el2_return_state(ctxt);
-}
-NOKPROBE_SYMBOL(sysreg_save_guest_state_vhe);
-
-static void __hyp_text __sysreg_restore_common_state(struct kvm_cpu_context *ctxt)
-{
-       write_sysreg(ctxt->sys_regs[MDSCR_EL1],   mdscr_el1);
-}
-
-static void __hyp_text __sysreg_restore_user_state(struct kvm_cpu_context *ctxt)
-{
-       write_sysreg(ctxt->sys_regs[TPIDR_EL0],         tpidr_el0);
-       write_sysreg(ctxt->sys_regs[TPIDRRO_EL0],       tpidrro_el0);
-}
-
-static void __hyp_text __sysreg_restore_el1_state(struct kvm_cpu_context *ctxt)
-{
-       write_sysreg(ctxt->sys_regs[MPIDR_EL1],         vmpidr_el2);
-       write_sysreg(ctxt->sys_regs[CSSELR_EL1],        csselr_el1);
-
-       if (has_vhe() ||
-           !cpus_have_final_cap(ARM64_WORKAROUND_SPECULATIVE_AT)) {
-               write_sysreg_el1(ctxt->sys_regs[SCTLR_EL1],     SYS_SCTLR);
-               write_sysreg_el1(ctxt->sys_regs[TCR_EL1],       SYS_TCR);
-       } else  if (!ctxt->__hyp_running_vcpu) {
-               /*
-                * Must only be done for guest registers, hence the context
-                * test. We're coming from the host, so SCTLR.M is already
-                * set. Pairs with nVHE's __activate_traps().
-                */
-               write_sysreg_el1((ctxt->sys_regs[TCR_EL1] |
-                                 TCR_EPD1_MASK | TCR_EPD0_MASK),
-                                SYS_TCR);
-               isb();
-       }
-
-       write_sysreg_el1(ctxt->sys_regs[CPACR_EL1],     SYS_CPACR);
-       write_sysreg_el1(ctxt->sys_regs[TTBR0_EL1],     SYS_TTBR0);
-       write_sysreg_el1(ctxt->sys_regs[TTBR1_EL1],     SYS_TTBR1);
-       write_sysreg_el1(ctxt->sys_regs[ESR_EL1],       SYS_ESR);
-       write_sysreg_el1(ctxt->sys_regs[AFSR0_EL1],     SYS_AFSR0);
-       write_sysreg_el1(ctxt->sys_regs[AFSR1_EL1],     SYS_AFSR1);
-       write_sysreg_el1(ctxt->sys_regs[FAR_EL1],       SYS_FAR);
-       write_sysreg_el1(ctxt->sys_regs[MAIR_EL1],      SYS_MAIR);
-       write_sysreg_el1(ctxt->sys_regs[VBAR_EL1],      SYS_VBAR);
-       write_sysreg_el1(ctxt->sys_regs[CONTEXTIDR_EL1],SYS_CONTEXTIDR);
-       write_sysreg_el1(ctxt->sys_regs[AMAIR_EL1],     SYS_AMAIR);
-       write_sysreg_el1(ctxt->sys_regs[CNTKCTL_EL1],   SYS_CNTKCTL);
-       write_sysreg(ctxt->sys_regs[PAR_EL1],           par_el1);
-       write_sysreg(ctxt->sys_regs[TPIDR_EL1],         tpidr_el1);
-
-       if (!has_vhe() &&
-           cpus_have_final_cap(ARM64_WORKAROUND_SPECULATIVE_AT) &&
-           ctxt->__hyp_running_vcpu) {
-               /*
-                * Must only be done for host registers, hence the context
-                * test. Pairs with nVHE's __deactivate_traps().
-                */
-               isb();
-               /*
-                * At this stage, and thanks to the above isb(), S2 is
-                * deconfigured and disabled. We can now restore the host's
-                * S1 configuration: SCTLR, and only then TCR.
-                */
-               write_sysreg_el1(ctxt->sys_regs[SCTLR_EL1],     SYS_SCTLR);
-               isb();
-               write_sysreg_el1(ctxt->sys_regs[TCR_EL1],       SYS_TCR);
-       }
-
-       write_sysreg(ctxt->gp_regs.sp_el1,              sp_el1);
-       write_sysreg_el1(ctxt->gp_regs.elr_el1,         SYS_ELR);
-       write_sysreg_el1(ctxt->gp_regs.spsr[KVM_SPSR_EL1],SYS_SPSR);
-}
-
-static void __hyp_text
-__sysreg_restore_el2_return_state(struct kvm_cpu_context *ctxt)
-{
-       u64 pstate = ctxt->gp_regs.regs.pstate;
-       u64 mode = pstate & PSR_AA32_MODE_MASK;
-
-       /*
-        * Safety check to ensure we're setting the CPU up to enter the guest
-        * in a less privileged mode.
-        *
-        * If we are attempting a return to EL2 or higher in AArch64 state,
-        * program SPSR_EL2 with M=EL2h and the IL bit set which ensures that
-        * we'll take an illegal exception state exception immediately after
-        * the ERET to the guest.  Attempts to return to AArch32 Hyp will
-        * result in an illegal exception return because EL2's execution state
-        * is determined by SCR_EL3.RW.
-        */
-       if (!(mode & PSR_MODE32_BIT) && mode >= PSR_MODE_EL2t)
-               pstate = PSR_MODE_EL2h | PSR_IL_BIT;
-
-       write_sysreg_el2(ctxt->gp_regs.regs.pc,         SYS_ELR);
-       write_sysreg_el2(pstate,                        SYS_SPSR);
-
-       if (cpus_have_final_cap(ARM64_HAS_RAS_EXTN))
-               write_sysreg_s(ctxt->sys_regs[DISR_EL1], SYS_VDISR_EL2);
-}
-
-void __hyp_text __sysreg_restore_state_nvhe(struct kvm_cpu_context *ctxt)
-{
-       __sysreg_restore_el1_state(ctxt);
-       __sysreg_restore_common_state(ctxt);
-       __sysreg_restore_user_state(ctxt);
-       __sysreg_restore_el2_return_state(ctxt);
-}
-
-void sysreg_restore_host_state_vhe(struct kvm_cpu_context *ctxt)
-{
-       __sysreg_restore_common_state(ctxt);
-}
-NOKPROBE_SYMBOL(sysreg_restore_host_state_vhe);
-
-void sysreg_restore_guest_state_vhe(struct kvm_cpu_context *ctxt)
-{
-       __sysreg_restore_common_state(ctxt);
-       __sysreg_restore_el2_return_state(ctxt);
-}
-NOKPROBE_SYMBOL(sysreg_restore_guest_state_vhe);
-
-void __hyp_text __sysreg32_save_state(struct kvm_vcpu *vcpu)
-{
-       u64 *spsr, *sysreg;
-
-       if (!vcpu_el1_is_32bit(vcpu))
-               return;
-
-       spsr = vcpu->arch.ctxt.gp_regs.spsr;
-       sysreg = vcpu->arch.ctxt.sys_regs;
-
-       spsr[KVM_SPSR_ABT] = read_sysreg(spsr_abt);
-       spsr[KVM_SPSR_UND] = read_sysreg(spsr_und);
-       spsr[KVM_SPSR_IRQ] = read_sysreg(spsr_irq);
-       spsr[KVM_SPSR_FIQ] = read_sysreg(spsr_fiq);
-
-       sysreg[DACR32_EL2] = read_sysreg(dacr32_el2);
-       sysreg[IFSR32_EL2] = read_sysreg(ifsr32_el2);
-
-       if (has_vhe() || vcpu->arch.flags & KVM_ARM64_DEBUG_DIRTY)
-               sysreg[DBGVCR32_EL2] = read_sysreg(dbgvcr32_el2);
-}
-
-void __hyp_text __sysreg32_restore_state(struct kvm_vcpu *vcpu)
-{
-       u64 *spsr, *sysreg;
-
-       if (!vcpu_el1_is_32bit(vcpu))
-               return;
-
-       spsr = vcpu->arch.ctxt.gp_regs.spsr;
-       sysreg = vcpu->arch.ctxt.sys_regs;
-
-       write_sysreg(spsr[KVM_SPSR_ABT], spsr_abt);
-       write_sysreg(spsr[KVM_SPSR_UND], spsr_und);
-       write_sysreg(spsr[KVM_SPSR_IRQ], spsr_irq);
-       write_sysreg(spsr[KVM_SPSR_FIQ], spsr_fiq);
-
-       write_sysreg(sysreg[DACR32_EL2], dacr32_el2);
-       write_sysreg(sysreg[IFSR32_EL2], ifsr32_el2);
-
-       if (has_vhe() || vcpu->arch.flags & KVM_ARM64_DEBUG_DIRTY)
-               write_sysreg(sysreg[DBGVCR32_EL2], dbgvcr32_el2);
-}
-
-/**
- * kvm_vcpu_load_sysregs - Load guest system registers to the physical CPU
- *
- * @vcpu: The VCPU pointer
- *
- * Load system registers that do not affect the host's execution, for
- * example EL1 system registers on a VHE system where the host kernel
- * runs at EL2.  This function is called from KVM's vcpu_load() function
- * and loading system register state early avoids having to load them on
- * every entry to the VM.
- */
-void kvm_vcpu_load_sysregs(struct kvm_vcpu *vcpu)
-{
-       struct kvm_cpu_context *guest_ctxt = &vcpu->arch.ctxt;
-       struct kvm_cpu_context *host_ctxt;
-
-       if (!has_vhe())
-               return;
-
-       host_ctxt = &__hyp_this_cpu_ptr(kvm_host_data)->host_ctxt;
-       __sysreg_save_user_state(host_ctxt);
-
-       /*
-        * Load guest EL1 and user state
-        *
-        * We must restore the 32-bit state before the sysregs, thanks
-        * to erratum #852523 (Cortex-A57) or #853709 (Cortex-A72).
-        */
-       __sysreg32_restore_state(vcpu);
-       __sysreg_restore_user_state(guest_ctxt);
-       __sysreg_restore_el1_state(guest_ctxt);
-
-       vcpu->arch.sysregs_loaded_on_cpu = true;
-
-       activate_traps_vhe_load(vcpu);
-}
-
-/**
- * kvm_vcpu_put_sysregs - Restore host system registers to the physical CPU
- *
- * @vcpu: The VCPU pointer
- *
- * Save guest system registers that do not affect the host's execution, for
- * example EL1 system registers on a VHE system where the host kernel
- * runs at EL2.  This function is called from KVM's vcpu_put() function
- * and deferring saving system register state until we're no longer running the
- * VCPU avoids having to save them on every exit from the VM.
- */
-void kvm_vcpu_put_sysregs(struct kvm_vcpu *vcpu)
-{
-       struct kvm_cpu_context *guest_ctxt = &vcpu->arch.ctxt;
-       struct kvm_cpu_context *host_ctxt;
-
-       if (!has_vhe())
-               return;
-
-       host_ctxt = &__hyp_this_cpu_ptr(kvm_host_data)->host_ctxt;
-       deactivate_traps_vhe_put();
-
-       __sysreg_save_el1_state(guest_ctxt);
-       __sysreg_save_user_state(guest_ctxt);
-       __sysreg32_save_state(vcpu);
-
-       /* Restore host user state */
-       __sysreg_restore_user_state(host_ctxt);
-
-       vcpu->arch.sysregs_loaded_on_cpu = false;
-}
-
-void __hyp_text __kvm_enable_ssbs(void)
-{
-       u64 tmp;
-
-       asm volatile(
-       "mrs    %0, sctlr_el2\n"
-       "orr    %0, %0, %1\n"
-       "msr    sctlr_el2, %0"
-       : "=&r" (tmp) : "L" (SCTLR_ELx_DSSBS));
-}

diff --git a/arch/arm64/kvm/hyp/vhe/Makefile b/arch/arm64/kvm/hyp/vhe/Makefile
index 62bdaf272b03a49c543a31dac803b19a1c12fe3e..2801582a739a5843872c1bd44242cb3e61859d32 100644 (file)
--- a/arch/arm64/kvm/hyp/vhe/Makefile
+++ b/arch/arm64/kvm/hyp/vhe/Makefile
@@ -6,7 +6,7 @@
 asflags-y := -D__KVM_VHE_HYPERVISOR__
 ccflags-y := -D__KVM_VHE_HYPERVISOR__
 
-obj-y := debug-sr.o switch.o tlb.o ../hyp-entry.o
+obj-y := sysreg-sr.o debug-sr.o switch.o tlb.o ../hyp-entry.o
 
 # KVM code is run at a different exception code with a different map, so
 # compiler instrumentation that inserts callbacks or checks into the code may

diff --git a/arch/arm64/kvm/hyp/vhe/switch.c b/arch/arm64/kvm/hyp/vhe/switch.c
index e8d76cab44e43969ef0036bd2eb4483bcc2320b3..c0d33deba77e6759daa8545edabbf87c0a4e11af 100644 (file)
--- a/arch/arm64/kvm/hyp/vhe/switch.c
+++ b/arch/arm64/kvm/hyp/vhe/switch.c
@@ -120,7 +120,7 @@ static int __kvm_vcpu_run_vhe(struct kvm_vcpu *vcpu)
         * HCR_EL2.TGE.
         *
         * We have already configured the guest's stage 1 translation in
-        * kvm_vcpu_load_sysregs above.  We must now call __activate_vm
+        * kvm_vcpu_load_sysregs_vhe above.  We must now call __activate_vm
         * before __activate_traps, because __activate_vm configures
         * stage 2 translation, and __activate_traps clear HCR_EL2.TGE
         * (among other things).

diff --git a/arch/arm64/kvm/hyp/vhe/sysreg-sr.c b/arch/arm64/kvm/hyp/vhe/sysreg-sr.c
new file mode 100644 (file)
index 0000000..996471e
--- /dev/null
+++ b/arch/arm64/kvm/hyp/vhe/sysreg-sr.c
@@ -0,0 +1,114 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (C) 2012-2015 - ARM Ltd
+ * Author: Marc Zyngier <marc.zyngier@arm.com>
+ */
+
+#include <hyp/sysreg-sr.h>
+
+#include <linux/compiler.h>
+#include <linux/kvm_host.h>
+
+#include <asm/kprobes.h>
+#include <asm/kvm_asm.h>
+#include <asm/kvm_emulate.h>
+#include <asm/kvm_hyp.h>
+
+/*
+ * VHE: Host and guest must save mdscr_el1 and sp_el0 (and the PC and
+ * pstate, which are handled as part of the el2 return state) on every
+ * switch (sp_el0 is being dealt with in the assembly code).
+ * tpidr_el0 and tpidrro_el0 only need to be switched when going
+ * to host userspace or a different VCPU.  EL1 registers only need to be
+ * switched when potentially going to run a different VCPU.  The latter two
+ * classes are handled as part of kvm_arch_vcpu_load and kvm_arch_vcpu_put.
+ */
+
+void sysreg_save_host_state_vhe(struct kvm_cpu_context *ctxt)
+{
+       __sysreg_save_common_state(ctxt);
+}
+NOKPROBE_SYMBOL(sysreg_save_host_state_vhe);
+
+void sysreg_save_guest_state_vhe(struct kvm_cpu_context *ctxt)
+{
+       __sysreg_save_common_state(ctxt);
+       __sysreg_save_el2_return_state(ctxt);
+}
+NOKPROBE_SYMBOL(sysreg_save_guest_state_vhe);
+
+void sysreg_restore_host_state_vhe(struct kvm_cpu_context *ctxt)
+{
+       __sysreg_restore_common_state(ctxt);
+}
+NOKPROBE_SYMBOL(sysreg_restore_host_state_vhe);
+
+void sysreg_restore_guest_state_vhe(struct kvm_cpu_context *ctxt)
+{
+       __sysreg_restore_common_state(ctxt);
+       __sysreg_restore_el2_return_state(ctxt);
+}
+NOKPROBE_SYMBOL(sysreg_restore_guest_state_vhe);
+
+/**
+ * kvm_vcpu_load_sysregs_vhe - Load guest system registers to the physical CPU
+ *
+ * @vcpu: The VCPU pointer
+ *
+ * Load system registers that do not affect the host's execution, for
+ * example EL1 system registers on a VHE system where the host kernel
+ * runs at EL2.  This function is called from KVM's vcpu_load() function
+ * and loading system register state early avoids having to load them on
+ * every entry to the VM.
+ */
+void kvm_vcpu_load_sysregs_vhe(struct kvm_vcpu *vcpu)
+{
+       struct kvm_cpu_context *guest_ctxt = &vcpu->arch.ctxt;
+       struct kvm_cpu_context *host_ctxt;
+
+       host_ctxt = &__hyp_this_cpu_ptr(kvm_host_data)->host_ctxt;
+       __sysreg_save_user_state(host_ctxt);
+
+       /*
+        * Load guest EL1 and user state
+        *
+        * We must restore the 32-bit state before the sysregs, thanks
+        * to erratum #852523 (Cortex-A57) or #853709 (Cortex-A72).
+        */
+       __sysreg32_restore_state(vcpu);
+       __sysreg_restore_user_state(guest_ctxt);
+       __sysreg_restore_el1_state(guest_ctxt);
+
+       vcpu->arch.sysregs_loaded_on_cpu = true;
+
+       activate_traps_vhe_load(vcpu);
+}
+
+/**
+ * kvm_vcpu_put_sysregs_vhe - Restore host system registers to the physical CPU
+ *
+ * @vcpu: The VCPU pointer
+ *
+ * Save guest system registers that do not affect the host's execution, for
+ * example EL1 system registers on a VHE system where the host kernel
+ * runs at EL2.  This function is called from KVM's vcpu_put() function
+ * and deferring saving system register state until we're no longer running the
+ * VCPU avoids having to save them on every exit from the VM.
+ */
+void kvm_vcpu_put_sysregs_vhe(struct kvm_vcpu *vcpu)
+{
+       struct kvm_cpu_context *guest_ctxt = &vcpu->arch.ctxt;
+       struct kvm_cpu_context *host_ctxt;
+
+       host_ctxt = &__hyp_this_cpu_ptr(kvm_host_data)->host_ctxt;
+       deactivate_traps_vhe_put();
+
+       __sysreg_save_el1_state(guest_ctxt);
+       __sysreg_save_user_state(guest_ctxt);
+       __sysreg32_save_state(vcpu);
+
+       /* Restore host user state */
+       __sysreg_restore_user_state(host_ctxt);
+
+       vcpu->arch.sysregs_loaded_on_cpu = false;
+}