locking/rwsem: Disable preemption for spinning region

[ Upstream commit 7cdacc5f52d68a9370f182c844b5b3e6cc975cc1 ]

The spinning region rwsem_spin_on_owner() should not be preempted,
however the rwsem_down_write_slowpath() invokes it and don't disable
preemption. Fix it by adding a pair of preempt_disable/enable().

Signed-off-by: Yanfei Xu <yanfei.xu@windriver.com>
[peterz: Fix CONFIG_RWSEM_SPIN_ON_OWNER=n build]
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Acked-by: Waiman Long <longman@redhat.com>
Link: https://lore.kernel.org/r/20211013134154.1085649-3-yanfei.xu@windriver.com
Signed-off-by: Sasha Levin <sashal@kernel.org>

diff --git a/kernel/locking/rwsem.c b/kernel/locking/rwsem.c
index 000e8d5a288419ba41a570ed846f8d81550fbcc7..29eea50a3e6782b38a3674a49a7200cf5e0b6d03 100644 (file)
--- a/kernel/locking/rwsem.c
+++ b/kernel/locking/rwsem.c
@@ -577,6 +577,24 @@ static inline bool rwsem_try_write_lock(struct rw_semaphore *sem,
        return true;
 }
 
+/*
+ * The rwsem_spin_on_owner() function returns the following 4 values
+ * depending on the lock owner state.
+ *   OWNER_NULL  : owner is currently NULL
+ *   OWNER_WRITER: when owner changes and is a writer
+ *   OWNER_READER: when owner changes and the new owner may be a reader.
+ *   OWNER_NONSPINNABLE:
+ *                when optimistic spinning has to stop because either the
+ *                owner stops running, is unknown, or its timeslice has
+ *                been used up.
+ */
+enum owner_state {
+       OWNER_NULL              = 1 << 0,
+       OWNER_WRITER            = 1 << 1,
+       OWNER_READER            = 1 << 2,
+       OWNER_NONSPINNABLE      = 1 << 3,
+};
+
 #ifdef CONFIG_RWSEM_SPIN_ON_OWNER
 /*
  * Try to acquire write lock before the writer has been put on wait queue.
@@ -632,23 +650,6 @@ static inline bool rwsem_can_spin_on_owner(struct rw_semaphore *sem)
        return ret;
 }
 
-/*
- * The rwsem_spin_on_owner() function returns the following 4 values
- * depending on the lock owner state.
- *   OWNER_NULL  : owner is currently NULL
- *   OWNER_WRITER: when owner changes and is a writer
- *   OWNER_READER: when owner changes and the new owner may be a reader.
- *   OWNER_NONSPINNABLE:
- *                when optimistic spinning has to stop because either the
- *                owner stops running, is unknown, or its timeslice has
- *                been used up.
- */
-enum owner_state {
-       OWNER_NULL              = 1 << 0,
-       OWNER_WRITER            = 1 << 1,
-       OWNER_READER            = 1 << 2,
-       OWNER_NONSPINNABLE      = 1 << 3,
-};
 #define OWNER_SPINNABLE                (OWNER_NULL | OWNER_WRITER | OWNER_READER)
 
 static inline enum owner_state
@@ -878,12 +879,11 @@ static inline bool rwsem_optimistic_spin(struct rw_semaphore *sem)
 
 static inline void clear_nonspinnable(struct rw_semaphore *sem) { }
 
-static inline int
+static inline enum owner_state
 rwsem_spin_on_owner(struct rw_semaphore *sem)
 {
-       return 0;
+       return OWNER_NONSPINNABLE;
 }
-#define OWNER_NULL     1
 #endif
 
 /*
@@ -1095,9 +1095,16 @@ wait:
                 * In this case, we attempt to acquire the lock again
                 * without sleeping.
                 */
-               if (wstate == WRITER_HANDOFF &&
-                   rwsem_spin_on_owner(sem) == OWNER_NULL)
-                       goto trylock_again;
+               if (wstate == WRITER_HANDOFF) {
+                       enum owner_state owner_state;
+
+                       preempt_disable();
+                       owner_state = rwsem_spin_on_owner(sem);
+                       preempt_enable();
+
+                       if (owner_state == OWNER_NULL)
+                               goto trylock_again;
+               }
 
                /* Block until there are no active lockers. */
                for (;;) {