powerpc/32s: Save content of sr0 to avoid 'mfsr'

Calling 'mfsr' to get the content of segment registers is heavy,
in addition it requires clearing of the 'reserved' bits.

In order to avoid this operation, save it in mm context and in
thread struct.

The saved sr0 is the one used by kernel, this means that on
locking entry it can be used as is.

For unlocking, the only thing to do is to clear SR_NX.

This improves null_syscall selftest by 12 cycles, ie 4%.

Signed-off-by: Christophe Leroy <christophe.leroy@csgroup.eu>
Signed-off-by: Michael Ellerman <mpe@ellerman.id.au>
Link: https://lore.kernel.org/r/b02baf2ed8f09bad910dfaeeb7353b2ae6830525.1634627931.git.christophe.leroy@csgroup.eu

diff --git a/arch/powerpc/include/asm/book3s/32/mmu-hash.h b/arch/powerpc/include/asm/book3s/32/mmu-hash.h
index e2f7ccc13edb373f257f8aa5bbd6c2ff6d7f8ae4..7be27862329fc4245e55c33828423b7d87b90f59 100644 (file)
--- a/arch/powerpc/include/asm/book3s/32/mmu-hash.h
+++ b/arch/powerpc/include/asm/book3s/32/mmu-hash.h
@@ -175,9 +175,14 @@ struct hash_pte {
 
 typedef struct {
        unsigned long id;
+       unsigned long sr0;
        void __user *vdso;
 } mm_context_t;
 
+#ifdef CONFIG_PPC_KUEP
+#define INIT_MM_CONTEXT(mm) .context.sr0 = SR_NX
+#endif
+
 void update_bats(void);
 static inline void cleanup_cpu_mmu_context(void) { }
 

diff --git a/arch/powerpc/include/asm/processor.h b/arch/powerpc/include/asm/processor.h
index 978a803084666d82c94e3db4757100c8a2b28a32..fe1ef1d7523bb7bdb8623a14f6e1e82446c3c9bb 100644 (file)
--- a/arch/powerpc/include/asm/processor.h
+++ b/arch/powerpc/include/asm/processor.h
@@ -157,6 +157,7 @@ struct thread_struct {
 #ifdef CONFIG_PPC_BOOK3S_32
        unsigned long   r0, r3, r4, r5, r6, r8, r9, r11;
        unsigned long   lr, ctr;
+       unsigned long   sr0;
 #endif
 #endif /* CONFIG_PPC32 */
        /* Debug Registers */
@@ -278,6 +279,12 @@ struct thread_struct {
 #define SPEFSCR_INIT
 #endif
 
+#ifdef CONFIG_PPC_BOOK3S_32
+#define SR0_INIT       .sr0 = IS_ENABLED(CONFIG_PPC_KUEP) ? SR_NX : 0,
+#else
+#define SR0_INIT
+#endif
+
 #if defined(CONFIG_PPC_BOOK3S_32) && defined(CONFIG_PPC_KUAP)
 #define INIT_THREAD { \
        .ksp = INIT_SP, \
@@ -285,6 +292,7 @@ struct thread_struct {
        .kuap = ~0UL, /* KUAP_NONE */ \
        .fpexc_mode = MSR_FE0 | MSR_FE1, \
        SPEFSCR_INIT \
+       SR0_INIT \
 }
 #elif defined(CONFIG_PPC32)
 #define INIT_THREAD { \
@@ -292,6 +300,7 @@ struct thread_struct {
        .pgdir = swapper_pg_dir, \
        .fpexc_mode = MSR_FE0 | MSR_FE1, \
        SPEFSCR_INIT \
+       SR0_INIT \
 }
 #else
 #define INIT_THREAD  { \

diff --git a/arch/powerpc/kernel/asm-offsets.c b/arch/powerpc/kernel/asm-offsets.c
index b823f484c6402f06ba20867df841511e0a0e9acc..cf3436b7b166e72fd1bafe985cfb13d0258b02aa 100644 (file)
--- a/arch/powerpc/kernel/asm-offsets.c
+++ b/arch/powerpc/kernel/asm-offsets.c
@@ -139,6 +139,7 @@ int main(void)
        OFFSET(THR11, thread_struct, r11);
        OFFSET(THLR, thread_struct, lr);
        OFFSET(THCTR, thread_struct, ctr);
+       OFFSET(THSR0, thread_struct, sr0);
 #endif
 #ifdef CONFIG_SPE
        OFFSET(THREAD_EVR0, thread_struct, evr[0]);

diff --git a/arch/powerpc/kernel/entry_32.S b/arch/powerpc/kernel/entry_32.S
index 0756829b2f7fa023483b82a94a13c883d6e97cf5..035bf4f3eb5d84101d5b81dbeee6e40249053e56 100644 (file)
--- a/arch/powerpc/kernel/entry_32.S
+++ b/arch/powerpc/kernel/entry_32.S
@@ -76,15 +76,13 @@ _ASM_NOKPROBE_SYMBOL(prepare_transfer_to_handler)
 #if defined(CONFIG_PPC_KUEP) && defined(CONFIG_PPC_BOOK3S_32)
        .globl  __kuep_lock
 __kuep_lock:
-       mfsr    r9,0
-       rlwinm  r9,r9,0,8,3
-       oris    r9,r9,SR_NX@h
+       lwz     r9, THREAD+THSR0(r2)
        update_user_segments_by_4 r9, r10, r11, r12
        blr
 
 __kuep_unlock:
-       mfsr    r9,0
-       rlwinm  r9,r9,0,8,2
+       lwz     r9, THREAD+THSR0(r2)
+       rlwinm  r9,r9,0,~SR_NX
        update_user_segments_by_4 r9, r10, r11, r12
        blr
 

diff --git a/arch/powerpc/mm/book3s32/kuap.c b/arch/powerpc/mm/book3s32/kuap.c
index 0f920f09af57b3ee21bceb2323a0244f16d1ef23..28676cabb005deccbf3bb3ee341048bbdc8b4000 100644 (file)
--- a/arch/powerpc/mm/book3s32/kuap.c
+++ b/arch/powerpc/mm/book3s32/kuap.c
@@ -20,8 +20,11 @@ EXPORT_SYMBOL(kuap_unlock_all_ool);
 
 void setup_kuap(bool disabled)
 {
-       if (!disabled)
+       if (!disabled) {
                kuap_lock_all_ool();
+               init_mm.context.sr0 |= SR_KS;
+               current->thread.sr0 |= SR_KS;
+       }
 
        if (smp_processor_id() != boot_cpuid)
                return;

diff --git a/arch/powerpc/mm/book3s32/kuep.c b/arch/powerpc/mm/book3s32/kuep.c
index bac1420d028b67fbe5cd4b4b9c27444e832ceb36..78fc48eee510e5f099222f5ce0dc6fe18997b8d4 100644 (file)
--- a/arch/powerpc/mm/book3s32/kuep.c
+++ b/arch/powerpc/mm/book3s32/kuep.c
@@ -1,5 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0-or-later
 
+#include <asm/code-patching.h>
 #include <asm/kup.h>
 #include <asm/smp.h>
 

diff --git a/arch/powerpc/mm/book3s32/mmu_context.c b/arch/powerpc/mm/book3s32/mmu_context.c
index e2708e387dc39351aa4ef0ff07cf70cf08f6ee02..269a3eb25a733fbeb5ce851cff14d84286027312 100644 (file)
--- a/arch/powerpc/mm/book3s32/mmu_context.c
+++ b/arch/powerpc/mm/book3s32/mmu_context.c
@@ -69,6 +69,12 @@ EXPORT_SYMBOL_GPL(__init_new_context);
 int init_new_context(struct task_struct *t, struct mm_struct *mm)
 {
        mm->context.id = __init_new_context();
+       mm->context.sr0 = CTX_TO_VSID(mm->context.id, 0);
+
+       if (!kuep_is_disabled())
+               mm->context.sr0 |= SR_NX;
+       if (!kuap_is_disabled())
+               mm->context.sr0 |= SR_KS;
 
        return 0;
 }
@@ -108,20 +114,13 @@ void __init mmu_context_init(void)
 void switch_mmu_context(struct mm_struct *prev, struct mm_struct *next, struct task_struct *tsk)
 {
        long id = next->context.id;
-       unsigned long val;
 
        if (id < 0)
                panic("mm_struct %p has no context ID", next);
 
        isync();
 
-       val = CTX_TO_VSID(id, 0);
-       if (!kuep_is_disabled())
-               val |= SR_NX;
-       if (!kuap_is_disabled())
-               val |= SR_KS;
-
-       update_user_segments(val);
+       update_user_segments(next->context.sr0);
 
        if (IS_ENABLED(CONFIG_BDI_SWITCH))
                abatron_pteptrs[1] = next->pgd;

diff --git a/arch/powerpc/mm/mmu_context.c b/arch/powerpc/mm/mmu_context.c
index 74246536b83263514b110d0b6d3ba5e8472a51fc..e618d5442a28ee07493abda193d73067e110db49 100644 (file)
--- a/arch/powerpc/mm/mmu_context.c
+++ b/arch/powerpc/mm/mmu_context.c
@@ -18,6 +18,9 @@ static inline void switch_mm_pgdir(struct task_struct *tsk,
 {
        /* 32-bit keeps track of the current PGDIR in the thread struct */
        tsk->thread.pgdir = mm->pgd;
+#ifdef CONFIG_PPC_BOOK3S_32
+       tsk->thread.sr0 = mm->context.sr0;
+#endif
 }
 #elif defined(CONFIG_PPC_BOOK3E_64)
 static inline void switch_mm_pgdir(struct task_struct *tsk,