arm64: mm: add LPA2 and 5 level paging support to G-to-nG conversion

Add support for 5 level paging in the G-to-nG routine that creates its
own temporary page tables to traverse the swapper page tables. Also add
support for running the 5 level configuration with the top level folded
at runtime, to support CPUs that do not implement the LPA2 extension.

While at it, wire up the level skipping logic so it will also trigger on
4 level configurations with LPA2 enabled at build time but not active at
runtime, as we'll fall back to 3 level paging in that case.

Signed-off-by: Ard Biesheuvel <ardb@kernel.org>
Link: https://lore.kernel.org/r/20240214122845.2033971-77-ardb+git@google.com
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>

diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c
index ed9670d8360cfb3bef261842b4c002bfa3f55cf9..bc5e4e5698644fb9711c8053172cf0e283691f82 100644 (file)
--- a/arch/arm64/kernel/cpufeature.c
+++ b/arch/arm64/kernel/cpufeature.c
@@ -1765,6 +1765,9 @@ static int __init __kpti_install_ng_mappings(void *__unused)
        pgd_t *kpti_ng_temp_pgd;
        u64 alloc = 0;
 
+       if (levels == 5 && !pgtable_l5_enabled())
+               levels = 4;
+
        remap_fn = (void *)__pa_symbol(idmap_kpti_install_ng_mappings);
 
        if (!cpu) {
@@ -1778,9 +1781,9 @@ static int __init __kpti_install_ng_mappings(void *__unused)
                //
                // The physical pages are laid out as follows:
                //
-               // +--------+-/-------+-/------ +-\\--------+
-               // :  PTE[] : | PMD[] : | PUD[] : || PGD[]  :
-               // +--------+-\-------+-\------ +-//--------+
+               // +--------+-/-------+-/------ +-/------ +-\\\--------+
+               // :  PTE[] : | PMD[] : | PUD[] : | P4D[] : ||| PGD[]  :
+               // +--------+-\-------+-\------ +-\------ +-///--------+
                //      ^
                // The first page is mapped into this hierarchy at a PMD_SHIFT
                // aligned virtual address, so that we can manipulate the PTE

diff --git a/arch/arm64/mm/proc.S b/arch/arm64/mm/proc.S
index d03434b7bca5b994305dd1d85afaef94dc285321..fa0d7c63f8d2b91a67f020aad0efe338342ffbb1 100644 (file)
--- a/arch/arm64/mm/proc.S
+++ b/arch/arm64/mm/proc.S
@@ -216,16 +216,15 @@ SYM_FUNC_ALIAS(__pi_idmap_cpu_replace_ttbr1, idmap_cpu_replace_ttbr1)
        .macro  kpti_mk_tbl_ng, type, num_entries
        add     end_\type\()p, cur_\type\()p, #\num_entries * 8
 .Ldo_\type:
-       ldr     \type, [cur_\type\()p]          // Load the entry
+       ldr     \type, [cur_\type\()p], #8      // Load the entry and advance
        tbz     \type, #0, .Lnext_\type         // Skip invalid and
        tbnz    \type, #11, .Lnext_\type        // non-global entries
        orr     \type, \type, #PTE_NG           // Same bit for blocks and pages
-       str     \type, [cur_\type\()p]          // Update the entry
+       str     \type, [cur_\type\()p, #-8]     // Update the entry
        .ifnc   \type, pte
        tbnz    \type, #1, .Lderef_\type
        .endif
 .Lnext_\type:
-       add     cur_\type\()p, cur_\type\()p, #8
        cmp     cur_\type\()p, end_\type\()p
        b.ne    .Ldo_\type
        .endm
@@ -235,18 +234,18 @@ SYM_FUNC_ALIAS(__pi_idmap_cpu_replace_ttbr1, idmap_cpu_replace_ttbr1)
         * fixmap slot associated with the current level.
         */
        .macro  kpti_map_pgtbl, type, level
-       str     xzr, [temp_pte, #8 * (\level + 1)]      // break before make
+       str     xzr, [temp_pte, #8 * (\level + 2)]      // break before make
        dsb     nshst
-       add     pte, temp_pte, #PAGE_SIZE * (\level + 1)
+       add     pte, temp_pte, #PAGE_SIZE * (\level + 2)
        lsr     pte, pte, #12
        tlbi    vaae1, pte
        dsb     nsh
        isb
 
        phys_to_pte pte, cur_\type\()p
-       add     cur_\type\()p, temp_pte, #PAGE_SIZE * (\level + 1)
+       add     cur_\type\()p, temp_pte, #PAGE_SIZE * (\level + 2)
        orr     pte, pte, pte_flags
-       str     pte, [temp_pte, #8 * (\level + 1)]
+       str     pte, [temp_pte, #8 * (\level + 2)]
        dsb     nshst
        .endm
 
@@ -279,6 +278,8 @@ SYM_TYPED_FUNC_START(idmap_kpti_install_ng_mappings)
        end_ptep        .req    x15
        pte             .req    x16
        valid           .req    x17
+       cur_p4dp        .req    x19
+       end_p4dp        .req    x20
 
        mov     x5, x3                          // preserve temp_pte arg
        mrs     swapper_ttb, ttbr1_el1
@@ -286,6 +287,12 @@ SYM_TYPED_FUNC_START(idmap_kpti_install_ng_mappings)
 
        cbnz    cpu, __idmap_kpti_secondary
 
+#if CONFIG_PGTABLE_LEVELS > 4
+       stp     x29, x30, [sp, #-32]!
+       mov     x29, sp
+       stp     x19, x20, [sp, #16]
+#endif
+
        /* We're the boot CPU. Wait for the others to catch up */
        sevl
 1:     wfe
@@ -303,9 +310,32 @@ SYM_TYPED_FUNC_START(idmap_kpti_install_ng_mappings)
        mov_q   pte_flags, KPTI_NG_PTE_FLAGS
 
        /* Everybody is enjoying the idmap, so we can rewrite swapper. */
+
+#ifdef CONFIG_ARM64_LPA2
+       /*
+        * If LPA2 support is configured, but 52-bit virtual addressing is not
+        * enabled at runtime, we will fall back to one level of paging less,
+        * and so we have to walk swapper_pg_dir as if we dereferenced its
+        * address from a PGD level entry, and terminate the PGD level loop
+        * right after.
+        */
+       adrp    pgd, swapper_pg_dir     // walk &swapper_pg_dir at the next level
+       mov     cur_pgdp, end_pgdp      // must be equal to terminate the PGD loop
+alternative_if_not ARM64_HAS_VA52
+       b       .Lderef_pgd             // skip to the next level
+alternative_else_nop_endif
+       /*
+        * LPA2 based 52-bit virtual addressing requires 52-bit physical
+        * addressing to be enabled as well. In this case, the shareability
+        * bits are repurposed as physical address bits, and should not be
+        * set in pte_flags.
+        */
+       bic     pte_flags, pte_flags, #PTE_SHARED
+#endif
+
        /* PGD */
        adrp            cur_pgdp, swapper_pg_dir
-       kpti_map_pgtbl  pgd, 0
+       kpti_map_pgtbl  pgd, -1
        kpti_mk_tbl_ng  pgd, PTRS_PER_PGD
 
        /* Ensure all the updated entries are visible to secondary CPUs */
@@ -318,16 +348,33 @@ SYM_TYPED_FUNC_START(idmap_kpti_install_ng_mappings)
 
        /* Set the flag to zero to indicate that we're all done */
        str     wzr, [flag_ptr]
+#if CONFIG_PGTABLE_LEVELS > 4
+       ldp     x19, x20, [sp, #16]
+       ldp     x29, x30, [sp], #32
+#endif
        ret
 
 .Lderef_pgd:
+       /* P4D */
+       .if             CONFIG_PGTABLE_LEVELS > 4
+       p4d             .req    x30
+       pte_to_phys     cur_p4dp, pgd
+       kpti_map_pgtbl  p4d, 0
+       kpti_mk_tbl_ng  p4d, PTRS_PER_P4D
+       b               .Lnext_pgd
+       .else           /* CONFIG_PGTABLE_LEVELS <= 4 */
+       p4d             .req    pgd
+       .set            .Lnext_p4d, .Lnext_pgd
+       .endif
+
+.Lderef_p4d:
        /* PUD */
        .if             CONFIG_PGTABLE_LEVELS > 3
        pud             .req    x10
-       pte_to_phys     cur_pudp, pgd
+       pte_to_phys     cur_pudp, p4d
        kpti_map_pgtbl  pud, 1
        kpti_mk_tbl_ng  pud, PTRS_PER_PUD
-       b               .Lnext_pgd
+       b               .Lnext_p4d
        .else           /* CONFIG_PGTABLE_LEVELS <= 3 */
        pud             .req    pgd
        .set            .Lnext_pud, .Lnext_pgd
@@ -371,6 +418,9 @@ SYM_TYPED_FUNC_START(idmap_kpti_install_ng_mappings)
        .unreq  end_ptep
        .unreq  pte
        .unreq  valid
+       .unreq  cur_p4dp
+       .unreq  end_p4dp
+       .unreq  p4d
 
        /* Secondary CPUs end up here */
 __idmap_kpti_secondary: