powerc/mm: try VMA lock-based page fault handling first

Attempt VMA lock-based page fault handling first, and fall back to the
existing mmap_lock-based handling if that fails.  Copied from "x86/mm: try
VMA lock-based page fault handling first"

[ldufour@linux.ibm.com: powerpc/mm: fix mmap_lock bad unlock]
Link: https://lkml.kernel.org/r/20230306154244.17560-1-ldufour@linux.ibm.com
Link: https://lore.kernel.org/linux-mm/842502FB-F99C-417C-9648-A37D0ECDC9CE@linux.ibm.com
Link: https://lkml.kernel.org/r/20230227173632.3292573-32-surenb@google.com
Signed-off-by: Laurent Dufour <ldufour@linux.ibm.com>
Signed-off-by: Suren Baghdasaryan <surenb@google.com>
Cc: Sachin Sant <sachinp@linux.ibm.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>

diff --git a/arch/powerpc/mm/fault.c b/arch/powerpc/mm/fault.c
index af46aa88422bf950cd1868dd4715c6adf2f6c5bb..531177a4ee0881b412c2b8ded4b62d9902107f1e 100644 (file)
--- a/arch/powerpc/mm/fault.c
+++ b/arch/powerpc/mm/fault.c
@@ -474,6 +474,40 @@ static int ___do_page_fault(struct pt_regs *regs, unsigned long address,
        if (is_exec)
                flags |= FAULT_FLAG_INSTRUCTION;
 
+#ifdef CONFIG_PER_VMA_LOCK
+       if (!(flags & FAULT_FLAG_USER))
+               goto lock_mmap;
+
+       vma = lock_vma_under_rcu(mm, address);
+       if (!vma)
+               goto lock_mmap;
+
+       if (unlikely(access_pkey_error(is_write, is_exec,
+                                      (error_code & DSISR_KEYFAULT), vma))) {
+               vma_end_read(vma);
+               goto lock_mmap;
+       }
+
+       if (unlikely(access_error(is_write, is_exec, vma))) {
+               vma_end_read(vma);
+               goto lock_mmap;
+       }
+
+       fault = handle_mm_fault(vma, address, flags | FAULT_FLAG_VMA_LOCK, regs);
+       vma_end_read(vma);
+
+       if (!(fault & VM_FAULT_RETRY)) {
+               count_vm_vma_lock_event(VMA_LOCK_SUCCESS);
+               goto done;
+       }
+       count_vm_vma_lock_event(VMA_LOCK_RETRY);
+
+       if (fault_signal_pending(fault, regs))
+               return user_mode(regs) ? 0 : SIGBUS;
+
+lock_mmap:
+#endif /* CONFIG_PER_VMA_LOCK */
+
        /* When running in the kernel we expect faults to occur only to
         * addresses in user space.  All other faults represent errors in the
         * kernel and should generate an OOPS.  Unfortunately, in the case of an
@@ -550,6 +584,9 @@ retry:
 
        mmap_read_unlock(current->mm);
 
+#ifdef CONFIG_PER_VMA_LOCK
+done:
+#endif
        if (unlikely(fault & VM_FAULT_ERROR))
                return mm_fault_error(regs, address, fault);
 

diff --git a/arch/powerpc/platforms/powernv/Kconfig b/arch/powerpc/platforms/powernv/Kconfig
index ae248a161b435528317ea54dd82c29781fa5e1e2..70a46acc70d63d422d61039bd6f3d66ad6d65ed5 100644 (file)
--- a/arch/powerpc/platforms/powernv/Kconfig
+++ b/arch/powerpc/platforms/powernv/Kconfig
@@ -16,6 +16,7 @@ config PPC_POWERNV
        select PPC_DOORBELL
        select MMU_NOTIFIER
        select FORCE_SMP
+       select ARCH_SUPPORTS_PER_VMA_LOCK
        default y
 
 config OPAL_PRD

diff --git a/arch/powerpc/platforms/pseries/Kconfig b/arch/powerpc/platforms/pseries/Kconfig
index 21b22bf16ce66b8e374f731d2f377104eb0071fc..4ebf2ef2845d02b4e9804471e4aca98a9cb23adb 100644 (file)
--- a/arch/powerpc/platforms/pseries/Kconfig
+++ b/arch/powerpc/platforms/pseries/Kconfig
@@ -22,6 +22,7 @@ config PPC_PSERIES
        select HOTPLUG_CPU
        select FORCE_SMP
        select SWIOTLB
+       select ARCH_SUPPORTS_PER_VMA_LOCK
        default y
 
 config PARAVIRT