six locks: Split out seq, use atomic_t instead of atomic64_t

Signed-off-by: Kent Overstreet <kent.overstreet@linux.dev>

diff --git a/fs/bcachefs/six.c b/fs/bcachefs/six.c
index ff5d0506662e8b03e314012fc845da811e621d11..54e4aa35a350e08891fd1dadb023c8020555b229 100644 (file)
--- a/fs/bcachefs/six.c
+++ b/fs/bcachefs/six.c
@@ -32,25 +32,18 @@ static void do_six_unlock_type(struct six_lock *lock, enum six_lock_type type);
  * bits 29-30          has read waiters
  * bits 30-31          has intent waiters
  * bits 31-32          has write waiters
- * bits 32-64          sequence number: incremented on every write lock or
- *                     unlock, thus bit 33 (sequence number odd) indicates
- *                     lock is currently held for write
  */
 
 #define SIX_STATE_READ_OFFSET          0
 #define SIX_STATE_READ_BITS            26
 
-#define SIX_STATE_READ_LOCK            ~(~0ULL << 26)
-#define SIX_STATE_INTENT_HELD          (1ULL << 26)
-#define SIX_STATE_WRITE_LOCK           (1ULL << 27)
-#define SIX_STATE_NOSPIN               (1ULL << 28)
-#define SIX_STATE_WAITING_READ         (1ULL << (29 + SIX_LOCK_read))
-#define SIX_STATE_WAITING_INTENT       (1ULL << (29 + SIX_LOCK_intent))
-#define SIX_STATE_WAITING_WRITE                (1ULL << (29 + SIX_LOCK_write))
-
-#define SIX_STATE_SEQ_OFFSET           32
-#define SIX_STATE_SEQ_BITS             32
-#define SIX_STATE_SEQ                  (~0ULL << 32)
+#define SIX_STATE_READ_LOCK            ~(~0U << 26)
+#define SIX_STATE_INTENT_HELD          (1U << 26)
+#define SIX_STATE_WRITE_LOCK           (1U << 27)
+#define SIX_STATE_NOSPIN               (1U << 28)
+#define SIX_STATE_WAITING_READ         (1U << (29 + SIX_LOCK_read))
+#define SIX_STATE_WAITING_INTENT       (1U << (29 + SIX_LOCK_intent))
+#define SIX_STATE_WAITING_WRITE                (1U << (29 + SIX_LOCK_write))
 
 #define SIX_LOCK_HELD_read             SIX_STATE_READ_LOCK
 #define SIX_LOCK_HELD_intent           SIX_STATE_INTENT_HELD
@@ -58,13 +51,13 @@ static void do_six_unlock_type(struct six_lock *lock, enum six_lock_type type);
 
 struct six_lock_vals {
        /* Value we add to the lock in order to take the lock: */
-       u64                     lock_val;
+       u32                     lock_val;
 
        /* If the lock has this value (used as a mask), taking the lock fails: */
-       u64                     lock_fail;
+       u32                     lock_fail;
 
        /* Mask that indicates lock is held for this type: */
-       u64                     held_mask;
+       u32                     held_mask;
 
        /* Waitlist we wakeup when releasing the lock: */
        enum six_lock_type      unlock_wakeup;
@@ -72,7 +65,7 @@ struct six_lock_vals {
 
 static const struct six_lock_vals l[] = {
        [SIX_LOCK_read] = {
-               .lock_val       = 1ULL << SIX_STATE_READ_OFFSET,
+               .lock_val       = 1U << SIX_STATE_READ_OFFSET,
                .lock_fail      = SIX_LOCK_HELD_write,
                .held_mask      = SIX_LOCK_HELD_read,
                .unlock_wakeup  = SIX_LOCK_write,
@@ -91,25 +84,20 @@ static const struct six_lock_vals l[] = {
        },
 };
 
-static inline u32 six_state_seq(u64 state)
-{
-       return state >> SIX_STATE_SEQ_OFFSET;
-}
-
-static inline void six_set_bitmask(struct six_lock *lock, u64 mask)
+static inline void six_set_bitmask(struct six_lock *lock, u32 mask)
 {
-       if ((atomic64_read(&lock->state) & mask) != mask)
-               atomic64_or(mask, &lock->state);
+       if ((atomic_read(&lock->state) & mask) != mask)
+               atomic_or(mask, &lock->state);
 }
 
-static inline void six_clear_bitmask(struct six_lock *lock, u64 mask)
+static inline void six_clear_bitmask(struct six_lock *lock, u32 mask)
 {
-       if (atomic64_read(&lock->state) & mask)
-               atomic64_and(~mask, &lock->state);
+       if (atomic_read(&lock->state) & mask)
+               atomic_and(~mask, &lock->state);
 }
 
 static inline void six_set_owner(struct six_lock *lock, enum six_lock_type type,
-                                u64 old, struct task_struct *owner)
+                                u32 old, struct task_struct *owner)
 {
        if (type != SIX_LOCK_intent)
                return;
@@ -145,13 +133,11 @@ static int __do_six_trylock(struct six_lock *lock, enum six_lock_type type,
                            struct task_struct *task, bool try)
 {
        int ret;
-       u64 old, new, v;
+       u32 old, new, v;
 
        EBUG_ON(type == SIX_LOCK_write && lock->owner != task);
        EBUG_ON(type == SIX_LOCK_write &&
-               (try != !(atomic64_read(&lock->state) & SIX_LOCK_HELD_write)));
-       EBUG_ON(type == SIX_LOCK_write &&
-               (try != !(atomic64_read(&lock->state) & SIX_STATE_WRITE_LOCK)));
+               (try != !(atomic_read(&lock->state) & SIX_STATE_WRITE_LOCK)));
 
        /*
         * Percpu reader mode:
@@ -186,7 +172,7 @@ static int __do_six_trylock(struct six_lock *lock, enum six_lock_type type,
 
                smp_mb();
 
-               old = atomic64_read(&lock->state);
+               old = atomic_read(&lock->state);
                ret = !(old & l[type].lock_fail);
 
                this_cpu_sub(*lock->readers, !ret);
@@ -196,19 +182,19 @@ static int __do_six_trylock(struct six_lock *lock, enum six_lock_type type,
                        ret = -1 - SIX_LOCK_write;
        } else if (type == SIX_LOCK_write && lock->readers) {
                if (try) {
-                       atomic64_add(SIX_STATE_WRITE_LOCK, &lock->state);
+                       atomic_add(SIX_STATE_WRITE_LOCK, &lock->state);
                        smp_mb__after_atomic();
                }
 
                ret = !pcpu_read_count(lock);
 
                if (try && !ret) {
-                       old = atomic64_sub_return(SIX_STATE_WRITE_LOCK, &lock->state);
+                       old = atomic_sub_return(SIX_STATE_WRITE_LOCK, &lock->state);
                        if (old & SIX_STATE_WAITING_READ)
                                ret = -1 - SIX_LOCK_read;
                }
        } else {
-               v = atomic64_read(&lock->state);
+               v = atomic_read(&lock->state);
                do {
                        new = old = v;
 
@@ -218,16 +204,16 @@ static int __do_six_trylock(struct six_lock *lock, enum six_lock_type type,
                                break;
 
                        new += l[type].lock_val;
-               } while ((v = atomic64_cmpxchg_acquire(&lock->state, old, new)) != old);
+               } while ((v = atomic_cmpxchg_acquire(&lock->state, old, new)) != old);
 
-               EBUG_ON(ret && !(atomic64_read(&lock->state) & l[type].held_mask));
+               EBUG_ON(ret && !(atomic_read(&lock->state) & l[type].held_mask));
        }
 
        if (ret > 0)
                six_set_owner(lock, type, old, task);
 
        EBUG_ON(type == SIX_LOCK_write && try && ret <= 0 &&
-               (atomic64_read(&lock->state) & SIX_STATE_WRITE_LOCK));
+               (atomic_read(&lock->state) & SIX_STATE_WRITE_LOCK));
 
        return ret;
 }
@@ -277,7 +263,7 @@ unlock:
 }
 
 __always_inline
-static void six_lock_wakeup(struct six_lock *lock, u64 state,
+static void six_lock_wakeup(struct six_lock *lock, u32 state,
                            enum six_lock_type lock_type)
 {
        if (lock_type == SIX_LOCK_write && (state & SIX_LOCK_HELD_read))
@@ -317,7 +303,8 @@ bool six_trylock_ip(struct six_lock *lock, enum six_lock_type type, unsigned lon
        if (type != SIX_LOCK_write)
                six_acquire(&lock->dep_map, 1, type == SIX_LOCK_read, ip);
        else
-               atomic64_add(1ULL << SIX_STATE_SEQ_OFFSET, &lock->state);
+               lock->seq++;
+
        return true;
 }
 EXPORT_SYMBOL_GPL(six_trylock_ip);
@@ -482,12 +469,12 @@ static int six_lock_slowpath(struct six_lock *lock, enum six_lock_type type,
                             six_lock_should_sleep_fn should_sleep_fn, void *p,
                             unsigned long ip)
 {
-       u64 old;
+       u32 old;
        int ret = 0;
 
        if (type == SIX_LOCK_write) {
-               EBUG_ON(atomic64_read(&lock->state) & SIX_STATE_WRITE_LOCK);
-               atomic64_add(SIX_STATE_WRITE_LOCK, &lock->state);
+               EBUG_ON(atomic_read(&lock->state) & SIX_STATE_WRITE_LOCK);
+               atomic_add(SIX_STATE_WRITE_LOCK, &lock->state);
                smp_mb__after_atomic();
        }
 
@@ -609,8 +596,7 @@ int six_lock_ip_waiter(struct six_lock *lock, enum six_lock_type type,
        ret = do_six_trylock(lock, type, true) ? 0
                : six_lock_slowpath(lock, type, wait, should_sleep_fn, p, ip);
 
-       if (!ret && type == SIX_LOCK_write)
-               atomic64_add(1ULL << SIX_STATE_SEQ_OFFSET, &lock->state);
+       lock->seq += !ret && type == SIX_LOCK_write;
 
        if (ret && type != SIX_LOCK_write)
                six_release(&lock->dep_map, ip);
@@ -624,7 +610,7 @@ EXPORT_SYMBOL_GPL(six_lock_ip_waiter);
 __always_inline
 static void do_six_unlock_type(struct six_lock *lock, enum six_lock_type type)
 {
-       u64 state;
+       u32 state;
 
        if (type == SIX_LOCK_intent)
                lock->owner = NULL;
@@ -634,15 +620,15 @@ static void do_six_unlock_type(struct six_lock *lock, enum six_lock_type type)
                smp_mb(); /* unlock barrier */
                this_cpu_dec(*lock->readers);
                smp_mb(); /* between unlocking and checking for waiters */
-               state = atomic64_read(&lock->state);
+               state = atomic_read(&lock->state);
        } else {
-               u64 v = l[type].lock_val;
+               u32 v = l[type].lock_val;
 
                if (type != SIX_LOCK_read)
-                       v += atomic64_read(&lock->state) & SIX_STATE_NOSPIN;
+                       v += atomic_read(&lock->state) & SIX_STATE_NOSPIN;
 
-               EBUG_ON(!(atomic64_read(&lock->state) & l[type].held_mask));
-               state = atomic64_sub_return_release(v, &lock->state);
+               EBUG_ON(!(atomic_read(&lock->state) & l[type].held_mask));
+               state = atomic_sub_return_release(v, &lock->state);
        }
 
        six_lock_wakeup(lock, state, l[type].unlock_wakeup);
@@ -666,7 +652,7 @@ static void do_six_unlock_type(struct six_lock *lock, enum six_lock_type type)
 void six_unlock_ip(struct six_lock *lock, enum six_lock_type type, unsigned long ip)
 {
        EBUG_ON(type == SIX_LOCK_write &&
-               !(atomic64_read(&lock->state) & SIX_LOCK_HELD_intent));
+               !(atomic_read(&lock->state) & SIX_LOCK_HELD_intent));
        EBUG_ON((type == SIX_LOCK_write ||
                 type == SIX_LOCK_intent) &&
                lock->owner != current);
@@ -674,7 +660,7 @@ void six_unlock_ip(struct six_lock *lock, enum six_lock_type type, unsigned long
        if (type != SIX_LOCK_write)
                six_release(&lock->dep_map, ip);
        else
-               atomic64_add(1ULL << SIX_STATE_SEQ_OFFSET, &lock->state);
+               lock->seq++;
 
        if (type == SIX_LOCK_intent &&
            lock->intent_lock_recurse) {
@@ -710,7 +696,7 @@ EXPORT_SYMBOL_GPL(six_lock_downgrade);
  */
 bool six_lock_tryupgrade(struct six_lock *lock)
 {
-       u64 old, new, v = atomic64_read(&lock->state);
+       u32 old, new, v = atomic_read(&lock->state);
 
        do {
                new = old = v;
@@ -724,7 +710,7 @@ bool six_lock_tryupgrade(struct six_lock *lock)
                }
 
                new |= SIX_LOCK_HELD_intent;
-       } while ((v = atomic64_cmpxchg_acquire(&lock->state, old, new)) != old);
+       } while ((v = atomic_cmpxchg_acquire(&lock->state, old, new)) != old);
 
        if (lock->readers)
                this_cpu_dec(*lock->readers);
@@ -786,14 +772,14 @@ void six_lock_increment(struct six_lock *lock, enum six_lock_type type)
                if (lock->readers) {
                        this_cpu_inc(*lock->readers);
                } else {
-                       EBUG_ON(!(atomic64_read(&lock->state) &
+                       EBUG_ON(!(atomic_read(&lock->state) &
                                  (SIX_LOCK_HELD_read|
                                   SIX_LOCK_HELD_intent)));
-                       atomic64_add(l[type].lock_val, &lock->state);
+                       atomic_add(l[type].lock_val, &lock->state);
                }
                break;
        case SIX_LOCK_intent:
-               EBUG_ON(!(atomic64_read(&lock->state) & SIX_LOCK_HELD_intent));
+               EBUG_ON(!(atomic_read(&lock->state) & SIX_LOCK_HELD_intent));
                lock->intent_lock_recurse++;
                break;
        case SIX_LOCK_write:
@@ -815,7 +801,7 @@ EXPORT_SYMBOL_GPL(six_lock_increment);
  */
 void six_lock_wakeup_all(struct six_lock *lock)
 {
-       u64 state = atomic64_read(&lock->state);
+       u32 state = atomic_read(&lock->state);
        struct six_lock_waiter *w;
 
        six_lock_wakeup(lock, state, SIX_LOCK_read);
@@ -840,11 +826,11 @@ struct six_lock_count six_lock_counts(struct six_lock *lock)
        struct six_lock_count ret;
 
        ret.n[SIX_LOCK_read]    = !lock->readers
-               ? atomic64_read(&lock->state) & SIX_STATE_READ_LOCK
+               ? atomic_read(&lock->state) & SIX_STATE_READ_LOCK
                : pcpu_read_count(lock);
-       ret.n[SIX_LOCK_intent]  = !!(atomic64_read(&lock->state) & SIX_LOCK_HELD_intent) +
+       ret.n[SIX_LOCK_intent]  = !!(atomic_read(&lock->state) & SIX_LOCK_HELD_intent) +
                lock->intent_lock_recurse;
-       ret.n[SIX_LOCK_write]   = !!(atomic64_read(&lock->state) & SIX_LOCK_HELD_write);
+       ret.n[SIX_LOCK_write]   = !!(atomic_read(&lock->state) & SIX_LOCK_HELD_write);
 
        return ret;
 }
@@ -875,9 +861,9 @@ void six_lock_readers_add(struct six_lock *lock, int nr)
        if (lock->readers) {
                this_cpu_add(*lock->readers, nr);
        } else {
-               EBUG_ON((int) (atomic64_read(&lock->state) & SIX_STATE_READ_LOCK) + nr < 0);
+               EBUG_ON((int) (atomic_read(&lock->state) & SIX_STATE_READ_LOCK) + nr < 0);
                /* reader count starts at bit 0 */
-               atomic64_add(nr, &lock->state);
+               atomic_add(nr, &lock->state);
        }
 }
 EXPORT_SYMBOL_GPL(six_lock_readers_add);
@@ -892,7 +878,7 @@ EXPORT_SYMBOL_GPL(six_lock_readers_add);
 void six_lock_exit(struct six_lock *lock)
 {
        WARN_ON(lock->readers && pcpu_read_count(lock));
-       WARN_ON(atomic64_read(&lock->state) & SIX_LOCK_HELD_read);
+       WARN_ON(atomic_read(&lock->state) & SIX_LOCK_HELD_read);
 
        free_percpu(lock->readers);
        lock->readers = NULL;
@@ -902,7 +888,7 @@ EXPORT_SYMBOL_GPL(six_lock_exit);
 void __six_lock_init(struct six_lock *lock, const char *name,
                     struct lock_class_key *key, enum six_lock_init_flags flags)
 {
-       atomic64_set(&lock->state, 0);
+       atomic_set(&lock->state, 0);
        raw_spin_lock_init(&lock->wait_lock);
        INIT_LIST_HEAD(&lock->wait_list);
 #ifdef CONFIG_DEBUG_LOCK_ALLOC

diff --git a/fs/bcachefs/six.h b/fs/bcachefs/six.h
index 82bf9de7249075dc3e6a2f507d8ef24d2dff4c2f..4c268b0b83162cc314c9ff810261a6c6362e6b97 100644 (file)
--- a/fs/bcachefs/six.h
+++ b/fs/bcachefs/six.h
@@ -138,7 +138,8 @@ enum six_lock_type {
 };
 
 struct six_lock {
-       atomic64_t              state;
+       atomic_t                state;
+       u32                     seq;
        unsigned                intent_lock_recurse;
        struct task_struct      *owner;
        unsigned __percpu       *readers;
@@ -196,7 +197,7 @@ do {                                                                        \
  */
 static inline u32 six_lock_seq(const struct six_lock *lock)
 {
-       return atomic64_read(&lock->state) >> 32;
+       return lock->seq;
 }
 
 bool six_trylock_ip(struct six_lock *lock, enum six_lock_type type, unsigned long ip);