powerpc/64s: make HPTE lock and native_tlbie_lock irq-safe

With kfence enabled, there are several cases where HPTE and TLBIE locks
are called from softirq context, for example:

  WARNING: inconsistent lock state
  6.0.0-11845-g0cbbc95b12ac #1 Tainted: G                 N
  --------------------------------
  inconsistent {IN-SOFTIRQ-W} -> {SOFTIRQ-ON-W} usage.
  swapper/0/1 [HC0[0]:SC0[0]:HE1:SE1] takes:
  c000000002734de8 (native_tlbie_lock){+.?.}-{2:2}, at: .native_hpte_updateboltedpp+0x1a4/0x600
  {IN-SOFTIRQ-W} state was registered at:
    .lock_acquire+0x20c/0x520
    ._raw_spin_lock+0x4c/0x70
    .native_hpte_invalidate+0x62c/0x840
    .hash__kernel_map_pages+0x450/0x640
    .kfence_protect+0x58/0xc0
    .kfence_guarded_free+0x374/0x5a0
    .__slab_free+0x3d0/0x630
    .put_cred_rcu+0xcc/0x120
    .rcu_core+0x3c4/0x14e0
    .__do_softirq+0x1dc/0x7dc
    .do_softirq_own_stack+0x40/0x60

Fix this by consistently disabling irqs while taking either of these
locks. Don't just disable bh because several of the more common cases
already disable irqs, so this just makes the locks always irq-safe.

Reported-by: Guenter Roeck <linux@roeck-us.net>
Signed-off-by: Nicholas Piggin <npiggin@gmail.com>
Signed-off-by: Michael Ellerman <mpe@ellerman.id.au>
Link: https://lore.kernel.org/r/20221013230710.1987253-2-npiggin@gmail.com

diff --git a/arch/powerpc/mm/book3s64/hash_native.c b/arch/powerpc/mm/book3s64/hash_native.c
index 7a640e610ea35affd3ae1df67861c23539e4ddae..9342e79870dfd408bb8220ed1865c8f9e1959f9e 100644 (file)
--- a/arch/powerpc/mm/book3s64/hash_native.c
+++ b/arch/powerpc/mm/book3s64/hash_native.c
@@ -268,8 +268,11 @@ static long native_hpte_insert(unsigned long hpte_group, unsigned long vpn,
 {
        struct hash_pte *hptep = htab_address + hpte_group;
        unsigned long hpte_v, hpte_r;
+       unsigned long flags;
        int i;
 
+       local_irq_save(flags);
+
        if (!(vflags & HPTE_V_BOLTED)) {
                DBG_LOW("    insert(group=%lx, vpn=%016lx, pa=%016lx,"
                        " rflags=%lx, vflags=%lx, psize=%d)\n",
@@ -288,8 +291,10 @@ static long native_hpte_insert(unsigned long hpte_group, unsigned long vpn,
                hptep++;
        }
 
-       if (i == HPTES_PER_GROUP)
+       if (i == HPTES_PER_GROUP) {
+               local_irq_restore(flags);
                return -1;
+       }
 
        hpte_v = hpte_encode_v(vpn, psize, apsize, ssize) | vflags | HPTE_V_VALID;
        hpte_r = hpte_encode_r(pa, psize, apsize) | rflags;
@@ -304,7 +309,6 @@ static long native_hpte_insert(unsigned long hpte_group, unsigned long vpn,
                hpte_v = hpte_old_to_new_v(hpte_v);
        }
 
-       release_hpte_lock();
        hptep->r = cpu_to_be64(hpte_r);
        /* Guarantee the second dword is visible before the valid bit */
        eieio();
@@ -312,10 +316,13 @@ static long native_hpte_insert(unsigned long hpte_group, unsigned long vpn,
         * Now set the first dword including the valid bit
         * NOTE: this also unlocks the hpte
         */
+       release_hpte_lock();
        hptep->v = cpu_to_be64(hpte_v);
 
        __asm__ __volatile__ ("ptesync" : : : "memory");
 
+       local_irq_restore(flags);
+
        return i | (!!(vflags & HPTE_V_SECONDARY) << 3);
 }
 
@@ -366,6 +373,9 @@ static long native_hpte_updatepp(unsigned long slot, unsigned long newpp,
        struct hash_pte *hptep = htab_address + slot;
        unsigned long hpte_v, want_v;
        int ret = 0, local = 0;
+       unsigned long irqflags;
+
+       local_irq_save(irqflags);
 
        want_v = hpte_encode_avpn(vpn, bpsize, ssize);
 
@@ -409,6 +419,8 @@ static long native_hpte_updatepp(unsigned long slot, unsigned long newpp,
        if (!(flags & HPTE_NOHPTE_UPDATE))
                tlbie(vpn, bpsize, apsize, ssize, local);
 
+       local_irq_restore(irqflags);
+
        return ret;
 }
 
@@ -472,6 +484,9 @@ static void native_hpte_updateboltedpp(unsigned long newpp, unsigned long ea,
        unsigned long vsid;
        long slot;
        struct hash_pte *hptep;
+       unsigned long flags;
+
+       local_irq_save(flags);
 
        vsid = get_kernel_vsid(ea, ssize);
        vpn = hpt_vpn(ea, vsid, ssize);
@@ -490,6 +505,8 @@ static void native_hpte_updateboltedpp(unsigned long newpp, unsigned long ea,
         * actual page size will be same.
         */
        tlbie(vpn, psize, psize, ssize, 0);
+
+       local_irq_restore(flags);
 }
 
 /*
@@ -503,6 +520,9 @@ static int native_hpte_removebolted(unsigned long ea, int psize, int ssize)
        unsigned long vsid;
        long slot;
        struct hash_pte *hptep;
+       unsigned long flags;
+
+       local_irq_save(flags);
 
        vsid = get_kernel_vsid(ea, ssize);
        vpn = hpt_vpn(ea, vsid, ssize);
@@ -520,6 +540,9 @@ static int native_hpte_removebolted(unsigned long ea, int psize, int ssize)
 
        /* Invalidate the TLB */
        tlbie(vpn, psize, psize, ssize, 0);
+
+       local_irq_restore(flags);
+
        return 0;
 }