arm64: Split kpti_install_ng_mappings()

The arm64_cpu_capabilities::cpu_enable callbacks are intended for
cpu-local feature enablement (e.g. poking system registers). These get
called for each online CPU when boot/system cpucaps get finalized and
enabled, and get called whenever a CPU is subsequently onlined.

For KPTI with the ARM64_UNMAP_KERNEL_AT_EL0 cpucap, we use the
kpti_install_ng_mappings() function as the cpu_enable callback. This
does a mixture of cpu-local configuration (setting VBAR_EL1 to the
appropriate trampoline vectors) and some global configuration (rewriting
the swapper page tables to sue non-glboal mappings) that must happen at
most once.

This patch splits kpti_install_ng_mappings() into a cpu-local
cpu_enable_kpti() initialization function and a system-wide
kpti_install_ng_mappings() function. The cpu_enable_kpti() function is
responsible for selecting the necessary cpu-local vectors each time a
CPU is onlined, and the kpti_install_ng_mappings() function performs the
one-time rewrite of the translation tables too use non-global mappings.
Splitting the two makes the code a bit easier to follow and also allows
the page table rewriting code to be marked as __init such that it can be
freed after use.

Signed-off-by: Mark Rutland <mark.rutland@arm.com>
Cc: Suzuki K Poulose <suzuki.poulose@arm.com>
Cc: Will Deacon <will@kernel.org>
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>

diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c
index 46508399796f96b153deacf946061af42f5516ed..5add7d06469d8913787ac0fff3fefd799781957b 100644 (file)
--- a/arch/arm64/kernel/cpufeature.c
+++ b/arch/arm64/kernel/cpufeature.c
@@ -1754,16 +1754,15 @@ void create_kpti_ng_temp_pgd(pgd_t *pgdir, phys_addr_t phys, unsigned long virt,
                             phys_addr_t size, pgprot_t prot,
                             phys_addr_t (*pgtable_alloc)(int), int flags);
 
-static phys_addr_t kpti_ng_temp_alloc;
+static phys_addr_t __initdata kpti_ng_temp_alloc;
 
-static phys_addr_t kpti_ng_pgd_alloc(int shift)
+static phys_addr_t __init kpti_ng_pgd_alloc(int shift)
 {
        kpti_ng_temp_alloc -= PAGE_SIZE;
        return kpti_ng_temp_alloc;
 }
 
-static void
-kpti_install_ng_mappings(const struct arm64_cpu_capabilities *__unused)
+static int __init __kpti_install_ng_mappings(void *__unused)
 {
        typedef void (kpti_remap_fn)(int, int, phys_addr_t, unsigned long);
        extern kpti_remap_fn idmap_kpti_install_ng_mappings;
@@ -1776,20 +1775,6 @@ kpti_install_ng_mappings(const struct arm64_cpu_capabilities *__unused)
        pgd_t *kpti_ng_temp_pgd;
        u64 alloc = 0;
 
-       if (__this_cpu_read(this_cpu_vector) == vectors) {
-               const char *v = arm64_get_bp_hardening_vector(EL1_VECTOR_KPTI);
-
-               __this_cpu_write(this_cpu_vector, v);
-       }
-
-       /*
-        * We don't need to rewrite the page-tables if either we've done
-        * it already or we have KASLR enabled and therefore have not
-        * created any global mappings at all.
-        */
-       if (arm64_use_ng_mappings)
-               return;
-
        remap_fn = (void *)__pa_symbol(idmap_kpti_install_ng_mappings);
 
        if (!cpu) {
@@ -1826,14 +1811,39 @@ kpti_install_ng_mappings(const struct arm64_cpu_capabilities *__unused)
                free_pages(alloc, order);
                arm64_use_ng_mappings = true;
        }
+
+       return 0;
+}
+
+static void __init kpti_install_ng_mappings(void)
+{
+       /*
+        * We don't need to rewrite the page-tables if either we've done
+        * it already or we have KASLR enabled and therefore have not
+        * created any global mappings at all.
+        */
+       if (arm64_use_ng_mappings)
+               return;
+
+       stop_machine(__kpti_install_ng_mappings, NULL, cpu_online_mask);
 }
+
 #else
-static void
-kpti_install_ng_mappings(const struct arm64_cpu_capabilities *__unused)
+static inline void kpti_install_ng_mappings(void)
 {
 }
 #endif /* CONFIG_UNMAP_KERNEL_AT_EL0 */
 
+static void cpu_enable_kpti(struct arm64_cpu_capabilities const *cap)
+{
+       if (__this_cpu_read(this_cpu_vector) == vectors) {
+               const char *v = arm64_get_bp_hardening_vector(EL1_VECTOR_KPTI);
+
+               __this_cpu_write(this_cpu_vector, v);
+       }
+
+}
+
 static int __init parse_kpti(char *str)
 {
        bool enabled;
@@ -2362,7 +2372,7 @@ static const struct arm64_cpu_capabilities arm64_features[] = {
                .desc = "Kernel page table isolation (KPTI)",
                .capability = ARM64_UNMAP_KERNEL_AT_EL0,
                .type = ARM64_CPUCAP_BOOT_RESTRICTED_CPU_LOCAL_FEATURE,
-               .cpu_enable = kpti_install_ng_mappings,
+               .cpu_enable = cpu_enable_kpti,
                .matches = unmap_kernel_at_el0,
                /*
                 * The ID feature fields below are used to indicate that
@@ -3355,6 +3365,8 @@ void __init setup_system_features(void)
         */
        enable_cpu_capabilities(SCOPE_ALL & ~SCOPE_BOOT_CPU);
 
+       kpti_install_ng_mappings();
+
        sve_setup();
        sme_setup();