entry: Provide generic syscall exit function

Like syscall entry all architectures have similar and pointlessly different
code to handle pending work before returning from a syscall to user space.

  1) One-time syscall exit work:
      - rseq syscall exit
      - audit
      - syscall tracing
      - tracehook (single stepping)

  2) Preparatory work
      - Exit to user mode loop (common TIF handling).
      - Architecture specific one time work arch_exit_to_user_mode_prepare()
      - Address limit and lockdep checks

  3) Final transition (lockdep, tracing, context tracking, RCU). Invokes
     arch_exit_to_user_mode() to handle e.g. speculation mitigations

Provide a generic version based on the x86 code which has all the RCU and
instrumentation protections right.

Provide a variant for interrupt return to user mode as well which shares
the above #2 and #3 work items.

After syscall_exit_to_user_mode() and irqentry_exit_to_user_mode() the
architecture code just has to return to user space. The code after
returning from these functions must not be instrumented.

Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Reviewed-by: Kees Cook <keescook@chromium.org>
Link: https://lkml.kernel.org/r/20200722220519.613977173@linutronix.de

diff --git a/include/linux/entry-common.h b/include/linux/entry-common.h
index 42fc8e4632bb3779b91a677966abadefcd54a512..c4a57be9cde408ce53a3ec73f7cc2bb5518c8c8f 100644 (file)
--- a/include/linux/entry-common.h
+++ b/include/linux/entry-common.h
@@ -29,6 +29,14 @@
 # define _TIF_SYSCALL_AUDIT            (0)
 #endif
 
+#ifndef _TIF_PATCH_PENDING
+# define _TIF_PATCH_PENDING            (0)
+#endif
+
+#ifndef _TIF_UPROBE
+# define _TIF_UPROBE                   (0)
+#endif
+
 /*
  * TIF flags handled in syscall_enter_from_usermode()
  */
@@ -41,6 +49,29 @@
         _TIF_SYSCALL_TRACEPOINT | _TIF_SYSCALL_EMU |                   \
         ARCH_SYSCALL_ENTER_WORK)
 
+/*
+ * TIF flags handled in syscall_exit_to_user_mode()
+ */
+#ifndef ARCH_SYSCALL_EXIT_WORK
+# define ARCH_SYSCALL_EXIT_WORK                (0)
+#endif
+
+#define SYSCALL_EXIT_WORK                                              \
+       (_TIF_SYSCALL_TRACE | _TIF_SYSCALL_AUDIT |                      \
+        _TIF_SYSCALL_TRACEPOINT | ARCH_SYSCALL_EXIT_WORK)
+
+/*
+ * TIF flags handled in exit_to_user_mode_loop()
+ */
+#ifndef ARCH_EXIT_TO_USER_MODE_WORK
+# define ARCH_EXIT_TO_USER_MODE_WORK           (0)
+#endif
+
+#define EXIT_TO_USER_MODE_WORK                                         \
+       (_TIF_SIGPENDING | _TIF_NOTIFY_RESUME | _TIF_UPROBE |           \
+        _TIF_NEED_RESCHED | _TIF_PATCH_PENDING |                       \
+        ARCH_EXIT_TO_USER_MODE_WORK)
+
 /**
  * arch_check_user_regs - Architecture specific sanity check for user mode regs
  * @regs:      Pointer to currents pt_regs
@@ -105,6 +136,149 @@ static inline __must_check int arch_syscall_enter_tracehook(struct pt_regs *regs
  */
 long syscall_enter_from_user_mode(struct pt_regs *regs, long syscall);
 
+/**
+ * local_irq_enable_exit_to_user - Exit to user variant of local_irq_enable()
+ * @ti_work:   Cached TIF flags gathered with interrupts disabled
+ *
+ * Defaults to local_irq_enable(). Can be supplied by architecture specific
+ * code.
+ */
+static inline void local_irq_enable_exit_to_user(unsigned long ti_work);
+
+#ifndef local_irq_enable_exit_to_user
+static inline void local_irq_enable_exit_to_user(unsigned long ti_work)
+{
+       local_irq_enable();
+}
+#endif
+
+/**
+ * local_irq_disable_exit_to_user - Exit to user variant of local_irq_disable()
+ *
+ * Defaults to local_irq_disable(). Can be supplied by architecture specific
+ * code.
+ */
+static inline void local_irq_disable_exit_to_user(void);
+
+#ifndef local_irq_disable_exit_to_user
+static inline void local_irq_disable_exit_to_user(void)
+{
+       local_irq_disable();
+}
+#endif
+
+/**
+ * arch_exit_to_user_mode_work - Architecture specific TIF work for exit
+ *                              to user mode.
+ * @regs:      Pointer to currents pt_regs
+ * @ti_work:   Cached TIF flags gathered with interrupts disabled
+ *
+ * Invoked from exit_to_user_mode_loop() with interrupt enabled
+ *
+ * Defaults to NOOP. Can be supplied by architecture specific code.
+ */
+static inline void arch_exit_to_user_mode_work(struct pt_regs *regs,
+                                              unsigned long ti_work);
+
+#ifndef arch_exit_to_user_mode_work
+static inline void arch_exit_to_user_mode_work(struct pt_regs *regs,
+                                              unsigned long ti_work)
+{
+}
+#endif
+
+/**
+ * arch_exit_to_user_mode_prepare - Architecture specific preparation for
+ *                                 exit to user mode.
+ * @regs:      Pointer to currents pt_regs
+ * @ti_work:   Cached TIF flags gathered with interrupts disabled
+ *
+ * Invoked from exit_to_user_mode_prepare() with interrupt disabled as the last
+ * function before return. Defaults to NOOP.
+ */
+static inline void arch_exit_to_user_mode_prepare(struct pt_regs *regs,
+                                                 unsigned long ti_work);
+
+#ifndef arch_exit_to_user_mode_prepare
+static inline void arch_exit_to_user_mode_prepare(struct pt_regs *regs,
+                                                 unsigned long ti_work)
+{
+}
+#endif
+
+/**
+ * arch_exit_to_user_mode - Architecture specific final work before
+ *                         exit to user mode.
+ *
+ * Invoked from exit_to_user_mode() with interrupt disabled as the last
+ * function before return. Defaults to NOOP.
+ *
+ * This needs to be __always_inline because it is non-instrumentable code
+ * invoked after context tracking switched to user mode.
+ *
+ * An architecture implementation must not do anything complex, no locking
+ * etc. The main purpose is for speculation mitigations.
+ */
+static __always_inline void arch_exit_to_user_mode(void);
+
+#ifndef arch_exit_to_user_mode
+static __always_inline void arch_exit_to_user_mode(void) { }
+#endif
+
+/**
+ * arch_do_signal -  Architecture specific signal delivery function
+ * @regs:      Pointer to currents pt_regs
+ *
+ * Invoked from exit_to_user_mode_loop().
+ */
+void arch_do_signal(struct pt_regs *regs);
+
+/**
+ * arch_syscall_exit_tracehook - Wrapper around tracehook_report_syscall_exit()
+ * @regs:      Pointer to currents pt_regs
+ * @step:      Indicator for single step
+ *
+ * Defaults to tracehook_report_syscall_exit(). Can be replaced by
+ * architecture specific code.
+ *
+ * Invoked from syscall_exit_to_user_mode()
+ */
+static inline void arch_syscall_exit_tracehook(struct pt_regs *regs, bool step);
+
+#ifndef arch_syscall_exit_tracehook
+static inline void arch_syscall_exit_tracehook(struct pt_regs *regs, bool step)
+{
+       tracehook_report_syscall_exit(regs, step);
+}
+#endif
+
+/**
+ * syscall_exit_to_user_mode - Handle work before returning to user mode
+ * @regs:      Pointer to currents pt_regs
+ *
+ * Invoked with interrupts enabled and fully valid regs. Returns with all
+ * work handled, interrupts disabled such that the caller can immediately
+ * switch to user mode. Called from architecture specific syscall and ret
+ * from fork code.
+ *
+ * The call order is:
+ *  1) One-time syscall exit work:
+ *     - rseq syscall exit
+ *      - audit
+ *     - syscall tracing
+ *     - tracehook (single stepping)
+ *
+ *  2) Preparatory work
+ *     - Exit to user mode loop (common TIF handling). Invokes
+ *       arch_exit_to_user_mode_work() for architecture specific TIF work
+ *     - Architecture specific one time work arch_exit_to_user_mode_prepare()
+ *     - Address limit and lockdep checks
+ *
+ *  3) Final transition (lockdep, tracing, context tracking, RCU). Invokes
+ *     arch_exit_to_user_mode() to handle e.g. speculation mitigations
+ */
+void syscall_exit_to_user_mode(struct pt_regs *regs);
+
 /**
  * irqentry_enter_from_user_mode - Establish state before invoking the irq handler
  * @regs:      Pointer to currents pt_regs
@@ -118,4 +292,19 @@ long syscall_enter_from_user_mode(struct pt_regs *regs, long syscall);
  */
 void irqentry_enter_from_user_mode(struct pt_regs *regs);
 
+/**
+ * irqentry_exit_to_user_mode - Interrupt exit work
+ * @regs:      Pointer to current's pt_regs
+ *
+ * Invoked with interrupts disbled and fully valid regs. Returns with all
+ * work handled, interrupts disabled such that the caller can immediately
+ * switch to user mode. Called from architecture specific interrupt
+ * handling code.
+ *
+ * The call order is #2 and #3 as described in syscall_exit_to_user_mode().
+ * Interrupt exit is not invoking #1 which is the syscall specific one time
+ * work.
+ */
+void irqentry_exit_to_user_mode(struct pt_regs *regs);
+
 #endif

diff --git a/kernel/entry/common.c b/kernel/entry/common.c
index 1d636dee2fec347d1bf7772c534f144c67c30c66..0a051bb91a56cfc42012c34203c16f29f109e9d9 100644 (file)
--- a/kernel/entry/common.c
+++ b/kernel/entry/common.c
@@ -2,6 +2,8 @@
 
 #include <linux/context_tracking.h>
 #include <linux/entry-common.h>
+#include <linux/livepatch.h>
+#include <linux/audit.h>
 
 #define CREATE_TRACE_POINTS
 #include <trace/events/syscalls.h>
@@ -82,7 +84,174 @@ noinstr long syscall_enter_from_user_mode(struct pt_regs *regs, long syscall)
        return syscall;
 }
 
+/**
+ * exit_to_user_mode - Fixup state when exiting to user mode
+ *
+ * Syscall/interupt exit enables interrupts, but the kernel state is
+ * interrupts disabled when this is invoked. Also tell RCU about it.
+ *
+ * 1) Trace interrupts on state
+ * 2) Invoke context tracking if enabled to adjust RCU state
+ * 3) Invoke architecture specific last minute exit code, e.g. speculation
+ *    mitigations, etc.
+ * 4) Tell lockdep that interrupts are enabled
+ */
+static __always_inline void exit_to_user_mode(void)
+{
+       instrumentation_begin();
+       trace_hardirqs_on_prepare();
+       lockdep_hardirqs_on_prepare(CALLER_ADDR0);
+       instrumentation_end();
+
+       user_enter_irqoff();
+       arch_exit_to_user_mode();
+       lockdep_hardirqs_on(CALLER_ADDR0);
+}
+
+/* Workaround to allow gradual conversion of architecture code */
+void __weak arch_do_signal(struct pt_regs *regs) { }
+
+static unsigned long exit_to_user_mode_loop(struct pt_regs *regs,
+                                           unsigned long ti_work)
+{
+       /*
+        * Before returning to user space ensure that all pending work
+        * items have been completed.
+        */
+       while (ti_work & EXIT_TO_USER_MODE_WORK) {
+
+               local_irq_enable_exit_to_user(ti_work);
+
+               if (ti_work & _TIF_NEED_RESCHED)
+                       schedule();
+
+               if (ti_work & _TIF_UPROBE)
+                       uprobe_notify_resume(regs);
+
+               if (ti_work & _TIF_PATCH_PENDING)
+                       klp_update_patch_state(current);
+
+               if (ti_work & _TIF_SIGPENDING)
+                       arch_do_signal(regs);
+
+               if (ti_work & _TIF_NOTIFY_RESUME) {
+                       clear_thread_flag(TIF_NOTIFY_RESUME);
+                       tracehook_notify_resume(regs);
+                       rseq_handle_notify_resume(NULL, regs);
+               }
+
+               /* Architecture specific TIF work */
+               arch_exit_to_user_mode_work(regs, ti_work);
+
+               /*
+                * Disable interrupts and reevaluate the work flags as they
+                * might have changed while interrupts and preemption was
+                * enabled above.
+                */
+               local_irq_disable_exit_to_user();
+               ti_work = READ_ONCE(current_thread_info()->flags);
+       }
+
+       /* Return the latest work state for arch_exit_to_user_mode() */
+       return ti_work;
+}
+
+static void exit_to_user_mode_prepare(struct pt_regs *regs)
+{
+       unsigned long ti_work = READ_ONCE(current_thread_info()->flags);
+
+       lockdep_assert_irqs_disabled();
+
+       if (unlikely(ti_work & EXIT_TO_USER_MODE_WORK))
+               ti_work = exit_to_user_mode_loop(regs, ti_work);
+
+       arch_exit_to_user_mode_prepare(regs, ti_work);
+
+       /* Ensure that the address limit is intact and no locks are held */
+       addr_limit_user_check();
+       lockdep_assert_irqs_disabled();
+       lockdep_sys_exit();
+}
+
+#ifndef _TIF_SINGLESTEP
+static inline bool report_single_step(unsigned long ti_work)
+{
+       return false;
+}
+#else
+/*
+ * If TIF_SYSCALL_EMU is set, then the only reason to report is when
+ * TIF_SINGLESTEP is set (i.e. PTRACE_SYSEMU_SINGLESTEP).  This syscall
+ * instruction has been already reported in syscall_enter_from_usermode().
+ */
+#define SYSEMU_STEP    (_TIF_SINGLESTEP | _TIF_SYSCALL_EMU)
+
+static inline bool report_single_step(unsigned long ti_work)
+{
+       return (ti_work & SYSEMU_STEP) == _TIF_SINGLESTEP;
+}
+#endif
+
+static void syscall_exit_work(struct pt_regs *regs, unsigned long ti_work)
+{
+       bool step;
+
+       audit_syscall_exit(regs);
+
+       if (ti_work & _TIF_SYSCALL_TRACEPOINT)
+               trace_sys_exit(regs, syscall_get_return_value(current, regs));
+
+       step = report_single_step(ti_work);
+       if (step || ti_work & _TIF_SYSCALL_TRACE)
+               arch_syscall_exit_tracehook(regs, step);
+}
+
+/*
+ * Syscall specific exit to user mode preparation. Runs with interrupts
+ * enabled.
+ */
+static void syscall_exit_to_user_mode_prepare(struct pt_regs *regs)
+{
+       u32 cached_flags = READ_ONCE(current_thread_info()->flags);
+       unsigned long nr = syscall_get_nr(current, regs);
+
+       CT_WARN_ON(ct_state() != CONTEXT_KERNEL);
+
+       if (IS_ENABLED(CONFIG_PROVE_LOCKING)) {
+               if (WARN(irqs_disabled(), "syscall %lu left IRQs disabled", nr))
+                       local_irq_enable();
+       }
+
+       rseq_syscall(regs);
+
+       /*
+        * Do one-time syscall specific work. If these work items are
+        * enabled, we want to run them exactly once per syscall exit with
+        * interrupts enabled.
+        */
+       if (unlikely(cached_flags & SYSCALL_EXIT_WORK))
+               syscall_exit_work(regs, cached_flags);
+}
+
+__visible noinstr void syscall_exit_to_user_mode(struct pt_regs *regs)
+{
+       instrumentation_begin();
+       syscall_exit_to_user_mode_prepare(regs);
+       local_irq_disable_exit_to_user();
+       exit_to_user_mode_prepare(regs);
+       instrumentation_end();
+       exit_to_user_mode();
+}
+
 noinstr void irqentry_enter_from_user_mode(struct pt_regs *regs)
 {
        enter_from_user_mode(regs);
 }
+
+noinstr void irqentry_exit_to_user_mode(struct pt_regs *regs)
+{
+       instrumentation_begin();
+       exit_to_user_mode_prepare(regs);
+       instrumentation_end();
+       exit_to_user_mode();
+}