arm64: mm: Add feature override support for LVA

Add support for overriding the VARange field of the MMFR2 CPU ID
register. This permits the associated LVA feature to be overridden early
enough for the boot code that creates the kernel mapping to take it into
account.

Given that LPA2 implies LVA, disabling the latter should disable the
former as well. So override the ID_AA64MMFR0.TGran field of the current
page size as well if it advertises support for 52-bit addressing.

Signed-off-by: Ard Biesheuvel <ardb@kernel.org>
Link: https://lore.kernel.org/r/20240214122845.2033971-71-ardb+git@google.com
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>

diff --git a/arch/arm64/include/asm/assembler.h b/arch/arm64/include/asm/assembler.h
index 6a467c6940397dd2e78e5dc6700851c7e477acdb..68a99b116256ba9dd520891c4908fb46af32a283 100644 (file)
--- a/arch/arm64/include/asm/assembler.h
+++ b/arch/arm64/include/asm/assembler.h
@@ -576,18 +576,21 @@ alternative_endif
        .endm
 
 /*
- * Offset ttbr1 to allow for 48-bit kernel VAs set with 52-bit PTRS_PER_PGD.
+ * If the kernel is built for 52-bit virtual addressing but the hardware only
+ * supports 48 bits, we cannot program the pgdir address into TTBR1 directly,
+ * but we have to add an offset so that the TTBR1 address corresponds with the
+ * pgdir entry that covers the lowest 48-bit addressable VA.
+ *
  * orr is used as it can cover the immediate value (and is idempotent).
- * In future this may be nop'ed out when dealing with 52-bit kernel VAs.
  *     ttbr: Value of ttbr to set, modified.
  */
        .macro  offset_ttbr1, ttbr, tmp
 #ifdef CONFIG_ARM64_VA_BITS_52
-       mrs_s   \tmp, SYS_ID_AA64MMFR2_EL1
-       and     \tmp, \tmp, #(0xf << ID_AA64MMFR2_EL1_VARange_SHIFT)
-       cbnz    \tmp, .Lskipoffs_\@
-       orr     \ttbr, \ttbr, #TTBR1_BADDR_4852_OFFSET
-.Lskipoffs_\@ :
+       mrs     \tmp, tcr_el1
+       and     \tmp, \tmp, #TCR_T1SZ_MASK
+       cmp     \tmp, #TCR_T1SZ(VA_BITS_MIN)
+       orr     \tmp, \ttbr, #TTBR1_BADDR_4852_OFFSET
+       csel    \ttbr, \tmp, \ttbr, eq
 #endif
        .endm
 

diff --git a/arch/arm64/include/asm/cpufeature.h b/arch/arm64/include/asm/cpufeature.h
index 4f4dc5496ee3a745e7860f641ffd6883dceda290..a2ac31aecdd966c6ed879216dddc1adcc089a124 100644 (file)
--- a/arch/arm64/include/asm/cpufeature.h
+++ b/arch/arm64/include/asm/cpufeature.h
@@ -906,7 +906,9 @@ static inline unsigned int get_vmid_bits(u64 mmfr1)
 s64 arm64_ftr_safe_value(const struct arm64_ftr_bits *ftrp, s64 new, s64 cur);
 struct arm64_ftr_reg *get_arm64_ftr_reg(u32 sys_id);
 
+extern struct arm64_ftr_override id_aa64mmfr0_override;
 extern struct arm64_ftr_override id_aa64mmfr1_override;
+extern struct arm64_ftr_override id_aa64mmfr2_override;
 extern struct arm64_ftr_override id_aa64pfr0_override;
 extern struct arm64_ftr_override id_aa64pfr1_override;
 extern struct arm64_ftr_override id_aa64zfr0_override;
@@ -1000,6 +1002,8 @@ static inline bool cpu_has_lva(void)
        u64 mmfr2;
 
        mmfr2 = read_sysreg_s(SYS_ID_AA64MMFR2_EL1);
+       mmfr2 &= ~id_aa64mmfr2_override.mask;
+       mmfr2 |= id_aa64mmfr2_override.val;
        return cpuid_feature_extract_unsigned_field(mmfr2,
                                                    ID_AA64MMFR2_EL1_VARange_SHIFT);
 }

diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c
index 8eb8c7f7b317b0bb21b2de5832134372c2f465f9..ed9670d8360cfb3bef261842b4c002bfa3f55cf9 100644 (file)
--- a/arch/arm64/kernel/cpufeature.c
+++ b/arch/arm64/kernel/cpufeature.c
@@ -655,7 +655,9 @@ static const struct arm64_ftr_bits ftr_raz[] = {
 #define ARM64_FTR_REG(id, table)               \
        __ARM64_FTR_REG_OVERRIDE(#id, id, table, &no_override)
 
+struct arm64_ftr_override id_aa64mmfr0_override;
 struct arm64_ftr_override id_aa64mmfr1_override;
+struct arm64_ftr_override id_aa64mmfr2_override;
 struct arm64_ftr_override id_aa64pfr0_override;
 struct arm64_ftr_override id_aa64pfr1_override;
 struct arm64_ftr_override id_aa64zfr0_override;
@@ -719,10 +721,12 @@ static const struct __ftr_reg_entry {
                               &id_aa64isar2_override),
 
        /* Op1 = 0, CRn = 0, CRm = 7 */
-       ARM64_FTR_REG(SYS_ID_AA64MMFR0_EL1, ftr_id_aa64mmfr0),
+       ARM64_FTR_REG_OVERRIDE(SYS_ID_AA64MMFR0_EL1, ftr_id_aa64mmfr0,
+                              &id_aa64mmfr0_override),
        ARM64_FTR_REG_OVERRIDE(SYS_ID_AA64MMFR1_EL1, ftr_id_aa64mmfr1,
                               &id_aa64mmfr1_override),
-       ARM64_FTR_REG(SYS_ID_AA64MMFR2_EL1, ftr_id_aa64mmfr2),
+       ARM64_FTR_REG_OVERRIDE(SYS_ID_AA64MMFR2_EL1, ftr_id_aa64mmfr2,
+                              &id_aa64mmfr2_override),
        ARM64_FTR_REG(SYS_ID_AA64MMFR3_EL1, ftr_id_aa64mmfr3),
 
        /* Op1 = 1, CRn = 0, CRm = 0 */

diff --git a/arch/arm64/kernel/image-vars.h b/arch/arm64/kernel/image-vars.h
index 2b9d702abe0f8b05e698e727257191acd60a8ac6..ff81f809a240dbdd69d8b10e7808f8c28f1ee712 100644 (file)
--- a/arch/arm64/kernel/image-vars.h
+++ b/arch/arm64/kernel/image-vars.h
@@ -38,7 +38,9 @@ PROVIDE(__pi___memset                 = __pi_memset);
 
 PROVIDE(__pi_id_aa64isar1_override     = id_aa64isar1_override);
 PROVIDE(__pi_id_aa64isar2_override     = id_aa64isar2_override);
+PROVIDE(__pi_id_aa64mmfr0_override     = id_aa64mmfr0_override);
 PROVIDE(__pi_id_aa64mmfr1_override     = id_aa64mmfr1_override);
+PROVIDE(__pi_id_aa64mmfr2_override     = id_aa64mmfr2_override);
 PROVIDE(__pi_id_aa64pfr0_override      = id_aa64pfr0_override);
 PROVIDE(__pi_id_aa64pfr1_override      = id_aa64pfr1_override);
 PROVIDE(__pi_id_aa64smfr0_override     = id_aa64smfr0_override);

diff --git a/arch/arm64/kernel/pi/idreg-override.c b/arch/arm64/kernel/pi/idreg-override.c
index 1884bd936c0d9d6951fb2bdfe8045f47363ee1e9..aad399796e812180aa4e8e244d30a97fb29950d9 100644 (file)
--- a/arch/arm64/kernel/pi/idreg-override.c
+++ b/arch/arm64/kernel/pi/idreg-override.c
@@ -59,6 +59,35 @@ static const struct ftr_set_desc mmfr1 __prel64_initconst = {
        },
 };
 
+
+static bool __init mmfr2_varange_filter(u64 val)
+{
+       int __maybe_unused feat;
+
+       if (val)
+               return false;
+
+#ifdef CONFIG_ARM64_LPA2
+       feat = cpuid_feature_extract_signed_field(read_sysreg(id_aa64mmfr0_el1),
+                                                 ID_AA64MMFR0_EL1_TGRAN_SHIFT);
+       if (feat >= ID_AA64MMFR0_EL1_TGRAN_LPA2) {
+               id_aa64mmfr0_override.val |=
+                       (ID_AA64MMFR0_EL1_TGRAN_LPA2 - 1) << ID_AA64MMFR0_EL1_TGRAN_SHIFT;
+               id_aa64mmfr0_override.mask |= 0xfU << ID_AA64MMFR0_EL1_TGRAN_SHIFT;
+       }
+#endif
+       return true;
+}
+
+static const struct ftr_set_desc mmfr2 __prel64_initconst = {
+       .name           = "id_aa64mmfr2",
+       .override       = &id_aa64mmfr2_override,
+       .fields         = {
+               FIELD("varange", ID_AA64MMFR2_EL1_VARange_SHIFT, mmfr2_varange_filter),
+               {}
+       },
+};
+
 static bool __init pfr0_sve_filter(u64 val)
 {
        /*
@@ -167,6 +196,7 @@ static const struct ftr_set_desc sw_features __prel64_initconst = {
 static const
 PREL64(const struct ftr_set_desc, reg) regs[] __prel64_initconst = {
        { &mmfr1        },
+       { &mmfr2        },
        { &pfr0         },
        { &pfr1         },
        { &isar1        },
@@ -192,6 +222,7 @@ static const struct {
        { "arm64.nomte",                "id_aa64pfr1.mte=0" },
        { "nokaslr",                    "arm64_sw.nokaslr=1" },
        { "rodata=off",                 "arm64_sw.rodataoff=1" },
+       { "arm64.nolva",                "id_aa64mmfr2.varange=0" },
 };
 
 static int __init parse_hexdigit(const char *p, u64 *v)