arm64: mte: Add asymmetric mode support

MTE provides an asymmetric mode for detecting tag exceptions. In
particular, when such a mode is present, the CPU triggers a fault
on a tag mismatch during a load operation and asynchronously updates
a register when a tag mismatch is detected during a store operation.

Add support for MTE asymmetric mode.

Note: If the CPU does not support MTE asymmetric mode the kernel falls
back on synchronous mode which is the default for kasan=on.

Cc: Will Deacon <will@kernel.org>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: Andrey Konovalov <andreyknvl@gmail.com>
Signed-off-by: Vincenzo Frascino <vincenzo.frascino@arm.com>
Reviewed-by: Catalin Marinas <catalin.marinas@arm.com>
Acked-by: Andrey Konovalov <andreyknvl@gmail.com>
Link: https://lore.kernel.org/r/20211006154751.4463-5-vincenzo.frascino@arm.com
Signed-off-by: Will Deacon <will@kernel.org>

diff --git a/arch/arm64/include/asm/memory.h b/arch/arm64/include/asm/memory.h
index f1745a8434144a42c9d5a2a40253e27ca134e4b8..1b9a1e2426127329778ae42570676fb13b588c9e 100644 (file)
--- a/arch/arm64/include/asm/memory.h
+++ b/arch/arm64/include/asm/memory.h
@@ -243,6 +243,7 @@ static inline const void *__tag_set(const void *addr, u8 tag)
 #ifdef CONFIG_KASAN_HW_TAGS
 #define arch_enable_tagging_sync()             mte_enable_kernel_sync()
 #define arch_enable_tagging_async()            mte_enable_kernel_async()
+#define arch_enable_tagging_asymm()            mte_enable_kernel_asymm()
 #define arch_force_async_tag_fault()           mte_check_tfsr_exit()
 #define arch_get_random_tag()                  mte_get_random_tag()
 #define arch_get_mem_tag(addr)                 mte_get_mem_tag(addr)

diff --git a/arch/arm64/include/asm/mte-kasan.h b/arch/arm64/include/asm/mte-kasan.h
index 22420e1f8c0379285512dca2d8d6841d53996db9..478b9bcf69ad19be1a04ae8287187524b2eb910f 100644 (file)
--- a/arch/arm64/include/asm/mte-kasan.h
+++ b/arch/arm64/include/asm/mte-kasan.h
@@ -130,6 +130,7 @@ static inline void mte_set_mem_tag_range(void *addr, size_t size, u8 tag,
 
 void mte_enable_kernel_sync(void);
 void mte_enable_kernel_async(void);
+void mte_enable_kernel_asymm(void);
 
 #else /* CONFIG_ARM64_MTE */
 
@@ -161,6 +162,10 @@ static inline void mte_enable_kernel_async(void)
 {
 }
 
+static inline void mte_enable_kernel_asymm(void)
+{
+}
+
 #endif /* CONFIG_ARM64_MTE */
 
 #endif /* __ASSEMBLY__ */

diff --git a/arch/arm64/include/asm/mte.h b/arch/arm64/include/asm/mte.h
index 02511650cffe52188a9696a22ea7cd589421b676..075539f5f1c88f42298ae4c6b824cb34a9728a7e 100644 (file)
--- a/arch/arm64/include/asm/mte.h
+++ b/arch/arm64/include/asm/mte.h
@@ -88,11 +88,11 @@ static inline int mte_ptrace_copy_tags(struct task_struct *child,
 
 #ifdef CONFIG_KASAN_HW_TAGS
 /* Whether the MTE asynchronous mode is enabled. */
-DECLARE_STATIC_KEY_FALSE(mte_async_mode);
+DECLARE_STATIC_KEY_FALSE(mte_async_or_asymm_mode);
 
-static inline bool system_uses_mte_async_mode(void)
+static inline bool system_uses_mte_async_or_asymm_mode(void)
 {
-       return static_branch_unlikely(&mte_async_mode);
+       return static_branch_unlikely(&mte_async_or_asymm_mode);
 }
 
 void mte_check_tfsr_el1(void);
@@ -121,7 +121,7 @@ static inline void mte_check_tfsr_exit(void)
        mte_check_tfsr_el1();
 }
 #else
-static inline bool system_uses_mte_async_mode(void)
+static inline bool system_uses_mte_async_or_asymm_mode(void)
 {
        return false;
 }

diff --git a/arch/arm64/include/asm/uaccess.h b/arch/arm64/include/asm/uaccess.h
index 190b494e22ab91d9988a914bbb05fe1c687a1b65..315354047d69387f61ddea9c3d5a5f12300513eb 100644 (file)
--- a/arch/arm64/include/asm/uaccess.h
+++ b/arch/arm64/include/asm/uaccess.h
@@ -196,13 +196,13 @@ static inline void __uaccess_enable_tco(void)
  */
 static inline void __uaccess_disable_tco_async(void)
 {
-       if (system_uses_mte_async_mode())
+       if (system_uses_mte_async_or_asymm_mode())
                 __uaccess_disable_tco();
 }
 
 static inline void __uaccess_enable_tco_async(void)
 {
-       if (system_uses_mte_async_mode())
+       if (system_uses_mte_async_or_asymm_mode())
                __uaccess_enable_tco();
 }
 

diff --git a/arch/arm64/kernel/mte.c b/arch/arm64/kernel/mte.c
index 0cdae086966e82c48f39d944de1bc2c71694db09..f418ebc65f9509f079dff585c2ac714019fffbe9 100644 (file)
--- a/arch/arm64/kernel/mte.c
+++ b/arch/arm64/kernel/mte.c
@@ -26,9 +26,12 @@
 static DEFINE_PER_CPU_READ_MOSTLY(u64, mte_tcf_preferred);
 
 #ifdef CONFIG_KASAN_HW_TAGS
-/* Whether the MTE asynchronous mode is enabled. */
-DEFINE_STATIC_KEY_FALSE(mte_async_mode);
-EXPORT_SYMBOL_GPL(mte_async_mode);
+/*
+ * The asynchronous and asymmetric MTE modes have the same behavior for
+ * store operations. This flag is set when either of these modes is enabled.
+ */
+DEFINE_STATIC_KEY_FALSE(mte_async_or_asymm_mode);
+EXPORT_SYMBOL_GPL(mte_async_or_asymm_mode);
 #endif
 
 static void mte_sync_page_tags(struct page *page, pte_t old_pte,
@@ -116,7 +119,7 @@ void mte_enable_kernel_sync(void)
         * Make sure we enter this function when no PE has set
         * async mode previously.
         */
-       WARN_ONCE(system_uses_mte_async_mode(),
+       WARN_ONCE(system_uses_mte_async_or_asymm_mode(),
                        "MTE async mode enabled system wide!");
 
        __mte_enable_kernel("synchronous", SCTLR_ELx_TCF_SYNC);
@@ -134,8 +137,34 @@ void mte_enable_kernel_async(void)
         * mode in between sync and async, this strategy needs
         * to be reviewed.
         */
-       if (!system_uses_mte_async_mode())
-               static_branch_enable(&mte_async_mode);
+       if (!system_uses_mte_async_or_asymm_mode())
+               static_branch_enable(&mte_async_or_asymm_mode);
+}
+
+void mte_enable_kernel_asymm(void)
+{
+       if (cpus_have_cap(ARM64_MTE_ASYMM)) {
+               __mte_enable_kernel("asymmetric", SCTLR_ELx_TCF_ASYMM);
+
+               /*
+                * MTE asymm mode behaves as async mode for store
+                * operations. The mode is set system wide by the
+                * first PE that executes this function.
+                *
+                * Note: If in future KASAN acquires a runtime switching
+                * mode in between sync and async, this strategy needs
+                * to be reviewed.
+                */
+               if (!system_uses_mte_async_or_asymm_mode())
+                       static_branch_enable(&mte_async_or_asymm_mode);
+       } else {
+               /*
+                * If the CPU does not support MTE asymmetric mode the
+                * kernel falls back on synchronous mode which is the
+                * default for kasan=on.
+                */
+               mte_enable_kernel_sync();
+       }
 }
 #endif