ARM: 9384/2: mm: Make tlbflush routines CFI safe

Instead of avoiding CFI entirely on the TLB flush helpers, reorganize
the code so that the CFI machinery can deal with it. The important
things to take into account are:
- functions in asm called indirectly from C need to be defined using
  SYM_TYPED_FUNC_START()
- a reference to the asm function needs to be visible to the compiler,
  in order to get it to emit the typeid symbol.

The latter means that defining the cpu_tlb_fns structs is best done from
C code, so that the references in the static initializers will be
visible to the compiler.

Signed-off-by: Ard Biesheuvel <ardb@kernel.org>
Tested-by: Kees Cook <keescook@chromium.org>
Reviewed-by: Sami Tolvanen <samitolvanen@google.com>
Signed-off-by: Linus Walleij <linus.walleij@linaro.org>
Signed-off-by: Russell King (Oracle) <rmk+kernel@armlinux.org.uk>

diff --git a/arch/arm/mm/Makefile b/arch/arm/mm/Makefile
index 71b858c9b10c47cf2fd5ab1227fa74e159a51256..cc8255fdf56e2b007a4ca67b507dd55b0aeec9c6 100644 (file)
--- a/arch/arm/mm/Makefile
+++ b/arch/arm/mm/Makefile
@@ -62,6 +62,7 @@ obj-$(CONFIG_CPU_TLB_FEROCEON)        += tlb-v4wbi.o  # reuse v4wbi TLB functions
 obj-$(CONFIG_CPU_TLB_V6)       += tlb-v6.o
 obj-$(CONFIG_CPU_TLB_V7)       += tlb-v7.o
 obj-$(CONFIG_CPU_TLB_FA)       += tlb-fa.o
+obj-y                          += tlb.o
 
 obj-$(CONFIG_CPU_ARM7TDMI)     += proc-arm7tdmi.o
 obj-$(CONFIG_CPU_ARM720T)      += proc-arm720.o

diff --git a/arch/arm/mm/proc-macros.S b/arch/arm/mm/proc-macros.S
index e43f6d716b4b877bff461385a940cff9957a3a6f..c0acfeac3e84550a9e5cef1edfbab61dee7770d9 100644 (file)
--- a/arch/arm/mm/proc-macros.S
+++ b/arch/arm/mm/proc-macros.S
@@ -338,21 +338,6 @@ ENTRY(\name\()_cache_fns)
        .size   \name\()_cache_fns, . - \name\()_cache_fns
 .endm
 
-.macro define_tlb_functions name:req, flags_up:req, flags_smp
-       .type   \name\()_tlb_fns, #object
-       .align 2
-ENTRY(\name\()_tlb_fns)
-       .long   \name\()_flush_user_tlb_range
-       .long   \name\()_flush_kern_tlb_range
-       .ifnb \flags_smp
-               ALT_SMP(.long   \flags_smp )
-               ALT_UP(.long    \flags_up )
-       .else
-               .long   \flags_up
-       .endif
-       .size   \name\()_tlb_fns, . - \name\()_tlb_fns
-.endm
-
 .macro globl_equ x, y
        .globl  \x
        .equ    \x, \y

diff --git a/arch/arm/mm/tlb-fa.S b/arch/arm/mm/tlb-fa.S
index def6161ec4523d5579ba9f99e1be3a1589af0081..85a6fe766b21f2c4981e04203d57f298221f9729 100644 (file)
--- a/arch/arm/mm/tlb-fa.S
+++ b/arch/arm/mm/tlb-fa.S
@@ -15,6 +15,7 @@
  */
 #include <linux/linkage.h>
 #include <linux/init.h>
+#include <linux/cfi_types.h>
 #include <asm/assembler.h>
 #include <asm/asm-offsets.h>
 #include <asm/tlbflush.h>
@@ -31,7 +32,7 @@
  *     - mm    - mm_struct describing address space
  */
        .align  4
-ENTRY(fa_flush_user_tlb_range)
+SYM_TYPED_FUNC_START(fa_flush_user_tlb_range)
        vma_vm_mm ip, r2
        act_mm  r3                              @ get current->active_mm
        eors    r3, ip, r3                      @ == mm ?
@@ -46,9 +47,10 @@ ENTRY(fa_flush_user_tlb_range)
        blo     1b
        mcr     p15, 0, r3, c7, c10, 4          @ data write barrier
        ret     lr
+SYM_FUNC_END(fa_flush_user_tlb_range)
 
 
-ENTRY(fa_flush_kern_tlb_range)
+SYM_TYPED_FUNC_START(fa_flush_kern_tlb_range)
        mov     r3, #0
        mcr     p15, 0, r3, c7, c10, 4          @ drain WB
        bic     r0, r0, #0x0ff
@@ -60,8 +62,4 @@ ENTRY(fa_flush_kern_tlb_range)
        mcr     p15, 0, r3, c7, c10, 4          @ data write barrier
        mcr     p15, 0, r3, c7, c5, 4           @ prefetch flush (isb)
        ret     lr
-
-       __INITDATA
-
-       /* define struct cpu_tlb_fns (see <asm/tlbflush.h> and proc-macros.S) */
-       define_tlb_functions fa, fa_tlb_flags
+SYM_FUNC_END(fa_flush_kern_tlb_range)

diff --git a/arch/arm/mm/tlb-v4.S b/arch/arm/mm/tlb-v4.S
index b962b4e751584f7080c6ca4e55b61ed8b39dd622..09ff69008d94d278a67ca1ddee5d06b16f97feea 100644 (file)
--- a/arch/arm/mm/tlb-v4.S
+++ b/arch/arm/mm/tlb-v4.S
@@ -11,6 +11,7 @@
  */
 #include <linux/linkage.h>
 #include <linux/init.h>
+#include <linux/cfi_types.h>
 #include <asm/assembler.h>
 #include <asm/asm-offsets.h>
 #include <asm/tlbflush.h>
@@ -27,7 +28,7 @@
  *     - mm    - mm_struct describing address space
  */
        .align  5
-ENTRY(v4_flush_user_tlb_range)
+SYM_TYPED_FUNC_START(v4_flush_user_tlb_range)
        vma_vm_mm ip, r2
        act_mm  r3                              @ get current->active_mm
        eors    r3, ip, r3                              @ == mm ?
@@ -40,6 +41,7 @@ ENTRY(v4_flush_user_tlb_range)
        cmp     r0, r1
        blo     1b
        ret     lr
+SYM_FUNC_END(v4_flush_user_tlb_range)
 
 /*
  *     v4_flush_kern_tlb_range(start, end)
@@ -50,10 +52,11 @@ ENTRY(v4_flush_user_tlb_range)
  *     - start - virtual address (may not be aligned)
  *     - end   - virtual address (may not be aligned)
  */
+#ifdef CONFIG_CFI_CLANG
+SYM_TYPED_FUNC_START(v4_flush_kern_tlb_range)
+       b       .v4_flush_kern_tlb_range
+SYM_FUNC_END(v4_flush_kern_tlb_range)
+#else
 .globl v4_flush_kern_tlb_range
 .equ v4_flush_kern_tlb_range, .v4_flush_kern_tlb_range
-
-       __INITDATA
-
-       /* define struct cpu_tlb_fns (see <asm/tlbflush.h> and proc-macros.S) */
-       define_tlb_functions v4, v4_tlb_flags
+#endif

diff --git a/arch/arm/mm/tlb-v4wb.S b/arch/arm/mm/tlb-v4wb.S
index 9348bba7586a0e6e8e256aa9ae511a6446548c67..04e46c359e75c715e29015377c1da5377cbb5502 100644 (file)
--- a/arch/arm/mm/tlb-v4wb.S
+++ b/arch/arm/mm/tlb-v4wb.S
@@ -11,6 +11,7 @@
  */
 #include <linux/linkage.h>
 #include <linux/init.h>
+#include <linux/cfi_types.h>
 #include <asm/assembler.h>
 #include <asm/asm-offsets.h>
 #include <asm/tlbflush.h>
@@ -27,7 +28,7 @@
  *     - mm    - mm_struct describing address space
  */
        .align  5
-ENTRY(v4wb_flush_user_tlb_range)
+SYM_TYPED_FUNC_START(v4wb_flush_user_tlb_range)
        vma_vm_mm ip, r2
        act_mm  r3                              @ get current->active_mm
        eors    r3, ip, r3                              @ == mm ?
@@ -43,6 +44,7 @@ ENTRY(v4wb_flush_user_tlb_range)
        cmp     r0, r1
        blo     1b
        ret     lr
+SYM_FUNC_END(v4wb_flush_user_tlb_range)
 
 /*
  *     v4_flush_kern_tlb_range(start, end)
@@ -53,7 +55,7 @@ ENTRY(v4wb_flush_user_tlb_range)
  *     - start - virtual address (may not be aligned)
  *     - end   - virtual address (may not be aligned)
  */
-ENTRY(v4wb_flush_kern_tlb_range)
+SYM_TYPED_FUNC_START(v4wb_flush_kern_tlb_range)
        mov     r3, #0
        mcr     p15, 0, r3, c7, c10, 4          @ drain WB
        bic     r0, r0, #0x0ff
@@ -64,8 +66,4 @@ ENTRY(v4wb_flush_kern_tlb_range)
        cmp     r0, r1
        blo     1b
        ret     lr
-
-       __INITDATA
-
-       /* define struct cpu_tlb_fns (see <asm/tlbflush.h> and proc-macros.S) */
-       define_tlb_functions v4wb, v4wb_tlb_flags
+SYM_FUNC_END(v4wb_flush_kern_tlb_range)

diff --git a/arch/arm/mm/tlb-v4wbi.S b/arch/arm/mm/tlb-v4wbi.S
index d4f9040a4111c180861de6feb7db275ad8a0a417..502dfe5628a370d31638fc84c45ef0d7cf956bb8 100644 (file)
--- a/arch/arm/mm/tlb-v4wbi.S
+++ b/arch/arm/mm/tlb-v4wbi.S
@@ -11,6 +11,7 @@
  */
 #include <linux/linkage.h>
 #include <linux/init.h>
+#include <linux/cfi_types.h>
 #include <asm/assembler.h>
 #include <asm/asm-offsets.h>
 #include <asm/tlbflush.h>
@@ -26,7 +27,7 @@
  *     - mm    - mm_struct describing address space
  */
        .align  5
-ENTRY(v4wbi_flush_user_tlb_range)
+SYM_TYPED_FUNC_START(v4wbi_flush_user_tlb_range)
        vma_vm_mm ip, r2
        act_mm  r3                              @ get current->active_mm
        eors    r3, ip, r3                      @ == mm ?
@@ -43,8 +44,9 @@ ENTRY(v4wbi_flush_user_tlb_range)
        cmp     r0, r1
        blo     1b
        ret     lr
+SYM_FUNC_END(v4wbi_flush_user_tlb_range)
 
-ENTRY(v4wbi_flush_kern_tlb_range)
+SYM_TYPED_FUNC_START(v4wbi_flush_kern_tlb_range)
        mov     r3, #0
        mcr     p15, 0, r3, c7, c10, 4          @ drain WB
        bic     r0, r0, #0x0ff
@@ -55,8 +57,4 @@ ENTRY(v4wbi_flush_kern_tlb_range)
        cmp     r0, r1
        blo     1b
        ret     lr
-
-       __INITDATA
-
-       /* define struct cpu_tlb_fns (see <asm/tlbflush.h> and proc-macros.S) */
-       define_tlb_functions v4wbi, v4wbi_tlb_flags
+SYM_FUNC_END(v4wbi_flush_kern_tlb_range)

diff --git a/arch/arm/mm/tlb-v6.S b/arch/arm/mm/tlb-v6.S
index 1d91e49b2c2d6653a4037a0d6e4f9cac487a0063..8256a67ac6542fdbea036bb19c4399b46312b245 100644 (file)
--- a/arch/arm/mm/tlb-v6.S
+++ b/arch/arm/mm/tlb-v6.S
@@ -9,6 +9,7 @@
  */
 #include <linux/init.h>
 #include <linux/linkage.h>
+#include <linux/cfi_types.h>
 #include <asm/asm-offsets.h>
 #include <asm/assembler.h>
 #include <asm/page.h>
@@ -32,7 +33,7 @@
  *     - the "Invalidate single entry" instruction will invalidate
  *       both the I and the D TLBs on Harvard-style TLBs
  */
-ENTRY(v6wbi_flush_user_tlb_range)
+SYM_TYPED_FUNC_START(v6wbi_flush_user_tlb_range)
        vma_vm_mm r3, r2                        @ get vma->vm_mm
        mov     ip, #0
        mmid    r3, r3                          @ get vm_mm->context.id
@@ -56,6 +57,7 @@ ENTRY(v6wbi_flush_user_tlb_range)
        blo     1b
        mcr     p15, 0, ip, c7, c10, 4          @ data synchronization barrier
        ret     lr
+SYM_FUNC_END(v6wbi_flush_user_tlb_range)
 
 /*
  *     v6wbi_flush_kern_tlb_range(start,end)
@@ -65,7 +67,7 @@ ENTRY(v6wbi_flush_user_tlb_range)
  *     - start - start address (may not be aligned)
  *     - end   - end address (exclusive, may not be aligned)
  */
-ENTRY(v6wbi_flush_kern_tlb_range)
+SYM_TYPED_FUNC_START(v6wbi_flush_kern_tlb_range)
        mov     r2, #0
        mcr     p15, 0, r2, c7, c10, 4          @ drain write buffer
        mov     r0, r0, lsr #PAGE_SHIFT         @ align address
@@ -85,8 +87,4 @@ ENTRY(v6wbi_flush_kern_tlb_range)
        mcr     p15, 0, r2, c7, c10, 4          @ data synchronization barrier
        mcr     p15, 0, r2, c7, c5, 4           @ prefetch flush (isb)
        ret     lr
-
-       __INIT
-
-       /* define struct cpu_tlb_fns (see <asm/tlbflush.h> and proc-macros.S) */
-       define_tlb_functions v6wbi, v6wbi_tlb_flags
+SYM_FUNC_END(v6wbi_flush_kern_tlb_range)

diff --git a/arch/arm/mm/tlb-v7.S b/arch/arm/mm/tlb-v7.S
index 35fd6d4f0d0384e2f82306124650e1d74f08aba7..f1aa0764a2ccc160e89ac074197a5e73dfb340a0 100644 (file)
--- a/arch/arm/mm/tlb-v7.S
+++ b/arch/arm/mm/tlb-v7.S
@@ -10,6 +10,7 @@
  */
 #include <linux/init.h>
 #include <linux/linkage.h>
+#include <linux/cfi_types.h>
 #include <asm/assembler.h>
 #include <asm/asm-offsets.h>
 #include <asm/page.h>
@@ -31,7 +32,7 @@
  *     - the "Invalidate single entry" instruction will invalidate
  *       both the I and the D TLBs on Harvard-style TLBs
  */
-ENTRY(v7wbi_flush_user_tlb_range)
+SYM_TYPED_FUNC_START(v7wbi_flush_user_tlb_range)
        vma_vm_mm r3, r2                        @ get vma->vm_mm
        mmid    r3, r3                          @ get vm_mm->context.id
        dsb     ish
@@ -57,7 +58,7 @@ ENTRY(v7wbi_flush_user_tlb_range)
        blo     1b
        dsb     ish
        ret     lr
-ENDPROC(v7wbi_flush_user_tlb_range)
+SYM_FUNC_END(v7wbi_flush_user_tlb_range)
 
 /*
  *     v7wbi_flush_kern_tlb_range(start,end)
@@ -67,7 +68,7 @@ ENDPROC(v7wbi_flush_user_tlb_range)
  *     - start - start address (may not be aligned)
  *     - end   - end address (exclusive, may not be aligned)
  */
-ENTRY(v7wbi_flush_kern_tlb_range)
+SYM_TYPED_FUNC_START(v7wbi_flush_kern_tlb_range)
        dsb     ish
        mov     r0, r0, lsr #PAGE_SHIFT         @ align address
        mov     r1, r1, lsr #PAGE_SHIFT
@@ -86,9 +87,4 @@ ENTRY(v7wbi_flush_kern_tlb_range)
        dsb     ish
        isb
        ret     lr
-ENDPROC(v7wbi_flush_kern_tlb_range)
-
-       __INIT
-
-       /* define struct cpu_tlb_fns (see <asm/tlbflush.h> and proc-macros.S) */
-       define_tlb_functions v7wbi, v7wbi_tlb_flags_up, flags_smp=v7wbi_tlb_flags_smp
+SYM_FUNC_END(v7wbi_flush_kern_tlb_range)

diff --git a/arch/arm/mm/tlb.c b/arch/arm/mm/tlb.c
new file mode 100644 (file)
index 0000000..4235979
--- /dev/null
+++ b/arch/arm/mm/tlb.c
@@ -0,0 +1,84 @@
+// SPDX-License-Identifier: GPL-2.0-only
+// Copyright 2024 Google LLC
+// Author: Ard Biesheuvel <ardb@google.com>
+
+#include <linux/types.h>
+#include <asm/tlbflush.h>
+
+#ifdef CONFIG_CPU_TLB_V4WT
+void v4_flush_user_tlb_range(unsigned long, unsigned long, struct vm_area_struct *);
+void v4_flush_kern_tlb_range(unsigned long, unsigned long);
+
+struct cpu_tlb_fns v4_tlb_fns __initconst = {
+       .flush_user_range       = v4_flush_user_tlb_range,
+       .flush_kern_range       = v4_flush_kern_tlb_range,
+       .tlb_flags              = v4_tlb_flags,
+};
+#endif
+
+#ifdef CONFIG_CPU_TLB_V4WB
+void v4wb_flush_user_tlb_range(unsigned long, unsigned long, struct vm_area_struct *);
+void v4wb_flush_kern_tlb_range(unsigned long, unsigned long);
+
+struct cpu_tlb_fns v4wb_tlb_fns __initconst = {
+       .flush_user_range       = v4wb_flush_user_tlb_range,
+       .flush_kern_range       = v4wb_flush_kern_tlb_range,
+       .tlb_flags              = v4wb_tlb_flags,
+};
+#endif
+
+#if defined(CONFIG_CPU_TLB_V4WBI) || defined(CONFIG_CPU_TLB_FEROCEON)
+void v4wbi_flush_user_tlb_range(unsigned long, unsigned long, struct vm_area_struct *);
+void v4wbi_flush_kern_tlb_range(unsigned long, unsigned long);
+
+struct cpu_tlb_fns v4wbi_tlb_fns __initconst = {
+       .flush_user_range       = v4wbi_flush_user_tlb_range,
+       .flush_kern_range       = v4wbi_flush_kern_tlb_range,
+       .tlb_flags              = v4wbi_tlb_flags,
+};
+#endif
+
+#ifdef CONFIG_CPU_TLB_V6
+void v6wbi_flush_user_tlb_range(unsigned long, unsigned long, struct vm_area_struct *);
+void v6wbi_flush_kern_tlb_range(unsigned long, unsigned long);
+
+struct cpu_tlb_fns v6wbi_tlb_fns __initconst = {
+       .flush_user_range       = v6wbi_flush_user_tlb_range,
+       .flush_kern_range       = v6wbi_flush_kern_tlb_range,
+       .tlb_flags              = v6wbi_tlb_flags,
+};
+#endif
+
+#ifdef CONFIG_CPU_TLB_V7
+void v7wbi_flush_user_tlb_range(unsigned long, unsigned long, struct vm_area_struct *);
+void v7wbi_flush_kern_tlb_range(unsigned long, unsigned long);
+
+struct cpu_tlb_fns v7wbi_tlb_fns __initconst = {
+       .flush_user_range       = v7wbi_flush_user_tlb_range,
+       .flush_kern_range       = v7wbi_flush_kern_tlb_range,
+       .tlb_flags              = IS_ENABLED(CONFIG_SMP) ? v7wbi_tlb_flags_smp
+                                                        : v7wbi_tlb_flags_up,
+};
+
+#ifdef CONFIG_SMP_ON_UP
+/* This will be run-time patched so the offset better be right */
+static_assert(offsetof(struct cpu_tlb_fns, tlb_flags) == 8);
+
+asm("  .pushsection    \".alt.smp.init\", \"a\"                \n" \
+    "  .align          2                                       \n" \
+    "  .long           v7wbi_tlb_fns + 8 - .                   \n" \
+    "  .long "         __stringify(v7wbi_tlb_flags_up) "       \n" \
+    "  .popsection                                             \n");
+#endif
+#endif
+
+#ifdef CONFIG_CPU_TLB_FA
+void fa_flush_user_tlb_range(unsigned long, unsigned long, struct vm_area_struct *);
+void fa_flush_kern_tlb_range(unsigned long, unsigned long);
+
+struct cpu_tlb_fns fa_tlb_fns __initconst = {
+       .flush_user_range       = fa_flush_user_tlb_range,
+       .flush_kern_range       = fa_flush_kern_tlb_range,
+       .tlb_flags              = fa_tlb_flags,
+};
+#endif