bcachefs: Factor out two_state_shared_lock

We have a unique lock used for controlling adding to the pagecache: the
lock has two states, where both states are shared - the lock may be held
multiple times for either state - but not both states at the same time.

This is exactly what we need for nocow mode locking, so this patch pulls
it out of fs.c into its own file.

Signed-off-by: Kent Overstreet <kent.overstreet@linux.dev>

diff --git a/fs/bcachefs/Makefile b/fs/bcachefs/Makefile
index 444e79c62b5095cce12b2158fa543719991d4a04..966c9b9a74fca6d1e306423c0cb744f8e842f8fb 100644 (file)
--- a/fs/bcachefs/Makefile
+++ b/fs/bcachefs/Makefile
@@ -65,6 +65,7 @@ bcachefs-y            :=      \
        sysfs.o                 \
        tests.o                 \
        trace.o                 \
+       two_state_shared_lock.o \
        util.o                  \
        varint.o                \
        xattr.o

diff --git a/fs/bcachefs/fs-io.c b/fs/bcachefs/fs-io.c
index 3c3fa95215ac533fab4c6624599cd77e632b14a3..ab5b4e086e0a14e7a7f22d3a4caab8c97e994aed 100644 (file)
--- a/fs/bcachefs/fs-io.c
+++ b/fs/bcachefs/fs-io.c
@@ -751,25 +751,25 @@ vm_fault_t bch2_page_fault(struct vm_fault *vmf)
        if (fdm > mapping) {
                struct bch_inode_info *fdm_host = to_bch_ei(fdm->host);
 
-               if (bch2_pagecache_add_tryget(&inode->ei_pagecache_lock))
+               if (bch2_pagecache_add_tryget(inode))
                        goto got_lock;
 
-               bch2_pagecache_block_put(&fdm_host->ei_pagecache_lock);
+               bch2_pagecache_block_put(fdm_host);
 
-               bch2_pagecache_add_get(&inode->ei_pagecache_lock);
-               bch2_pagecache_add_put(&inode->ei_pagecache_lock);
+               bch2_pagecache_add_get(inode);
+               bch2_pagecache_add_put(inode);
 
-               bch2_pagecache_block_get(&fdm_host->ei_pagecache_lock);
+               bch2_pagecache_block_get(fdm_host);
 
                /* Signal that lock has been dropped: */
                set_fdm_dropped_locks();
                return VM_FAULT_SIGBUS;
        }
 
-       bch2_pagecache_add_get(&inode->ei_pagecache_lock);
+       bch2_pagecache_add_get(inode);
 got_lock:
        ret = filemap_fault(vmf);
-       bch2_pagecache_add_put(&inode->ei_pagecache_lock);
+       bch2_pagecache_add_put(inode);
 
        return ret;
 }
@@ -797,7 +797,7 @@ vm_fault_t bch2_page_mkwrite(struct vm_fault *vmf)
         * a write_invalidate_inode_pages_range() that works without dropping
         * page lock before invalidating page
         */
-       bch2_pagecache_add_get(&inode->ei_pagecache_lock);
+       bch2_pagecache_add_get(inode);
 
        lock_page(page);
        isize = i_size_read(&inode->v);
@@ -830,7 +830,7 @@ vm_fault_t bch2_page_mkwrite(struct vm_fault *vmf)
        wait_for_stable_page(page);
        ret = VM_FAULT_LOCKED;
 out:
-       bch2_pagecache_add_put(&inode->ei_pagecache_lock);
+       bch2_pagecache_add_put(inode);
        sb_end_pagefault(inode->v.i_sb);
 
        return ret;
@@ -1098,7 +1098,7 @@ void bch2_readahead(struct readahead_control *ractl)
 
        bch2_trans_init(&trans, c, 0, 0);
 
-       bch2_pagecache_add_get(&inode->ei_pagecache_lock);
+       bch2_pagecache_add_get(inode);
 
        while ((page = readpage_iter_next(&readpages_iter))) {
                pgoff_t index = readpages_iter.offset + readpages_iter.idx;
@@ -1121,7 +1121,7 @@ void bch2_readahead(struct readahead_control *ractl)
                           &readpages_iter);
        }
 
-       bch2_pagecache_add_put(&inode->ei_pagecache_lock);
+       bch2_pagecache_add_put(inode);
 
        bch2_trans_exit(&trans);
        kfree(readpages_iter.pages);
@@ -1483,7 +1483,7 @@ int bch2_write_begin(struct file *file, struct address_space *mapping,
        bch2_page_reservation_init(c, inode, res);
        *fsdata = res;
 
-       bch2_pagecache_add_get(&inode->ei_pagecache_lock);
+       bch2_pagecache_add_get(inode);
 
        page = grab_cache_page_write_begin(mapping, index);
        if (!page)
@@ -1540,7 +1540,7 @@ err:
        put_page(page);
        *pagep = NULL;
 err_unlock:
-       bch2_pagecache_add_put(&inode->ei_pagecache_lock);
+       bch2_pagecache_add_put(inode);
        kfree(res);
        *fsdata = NULL;
        return bch2_err_class(ret);
@@ -1584,7 +1584,7 @@ int bch2_write_end(struct file *file, struct address_space *mapping,
 
        unlock_page(page);
        put_page(page);
-       bch2_pagecache_add_put(&inode->ei_pagecache_lock);
+       bch2_pagecache_add_put(inode);
 
        bch2_page_reservation_put(c, inode, res);
        kfree(res);
@@ -1753,7 +1753,7 @@ static ssize_t bch2_buffered_write(struct kiocb *iocb, struct iov_iter *iter)
        ssize_t written = 0;
        int ret = 0;
 
-       bch2_pagecache_add_get(&inode->ei_pagecache_lock);
+       bch2_pagecache_add_get(inode);
 
        do {
                unsigned offset = pos & (PAGE_SIZE - 1);
@@ -1811,7 +1811,7 @@ again:
                balance_dirty_pages_ratelimited(mapping);
        } while (iov_iter_count(iter));
 
-       bch2_pagecache_add_put(&inode->ei_pagecache_lock);
+       bch2_pagecache_add_put(inode);
 
        return written ? written : ret;
 }
@@ -1991,9 +1991,9 @@ ssize_t bch2_read_iter(struct kiocb *iocb, struct iov_iter *iter)
                if (ret >= 0)
                        iocb->ki_pos += ret;
        } else {
-               bch2_pagecache_add_get(&inode->ei_pagecache_lock);
+               bch2_pagecache_add_get(inode);
                ret = generic_file_read_iter(iocb, iter);
-               bch2_pagecache_add_put(&inode->ei_pagecache_lock);
+               bch2_pagecache_add_put(inode);
        }
 out:
        return bch2_err_class(ret);
@@ -2149,7 +2149,7 @@ static __always_inline long bch2_dio_write_done(struct dio_write *dio)
                        return -EIOCBQUEUED;
        }
 
-       bch2_pagecache_block_put(&inode->ei_pagecache_lock);
+       bch2_pagecache_block_put(inode);
        bch2_quota_reservation_put(c, inode, &dio->quota_res);
 
        if (dio->free_iov)
@@ -2357,7 +2357,7 @@ ssize_t bch2_direct_write(struct kiocb *req, struct iov_iter *iter)
                goto err;
 
        inode_dio_begin(&inode->v);
-       bch2_pagecache_block_get(&inode->ei_pagecache_lock);
+       bch2_pagecache_block_get(inode);
 
        extending = req->ki_pos + iter->count > inode->v.i_size;
        if (!extending) {
@@ -2403,7 +2403,7 @@ err:
                inode_unlock(&inode->v);
        return ret;
 err_put_bio:
-       bch2_pagecache_block_put(&inode->ei_pagecache_lock);
+       bch2_pagecache_block_put(inode);
        bch2_quota_reservation_put(c, inode, &dio->quota_res);
        bio_put(bio);
        inode_dio_end(&inode->v);
@@ -2704,7 +2704,7 @@ int bch2_truncate(struct mnt_idmap *idmap,
        }
 
        inode_dio_wait(&inode->v);
-       bch2_pagecache_block_get(&inode->ei_pagecache_lock);
+       bch2_pagecache_block_get(inode);
 
        ret = bch2_inode_find_by_inum(c, inode_inum(inode), &inode_u);
        if (ret)
@@ -2783,7 +2783,7 @@ int bch2_truncate(struct mnt_idmap *idmap,
 
        ret = bch2_setattr_nonsize(idmap, inode, iattr);
 err:
-       bch2_pagecache_block_put(&inode->ei_pagecache_lock);
+       bch2_pagecache_block_put(inode);
        return bch2_err_class(ret);
 }
 
@@ -3195,7 +3195,7 @@ long bch2_fallocate_dispatch(struct file *file, int mode,
 
        inode_lock(&inode->v);
        inode_dio_wait(&inode->v);
-       bch2_pagecache_block_get(&inode->ei_pagecache_lock);
+       bch2_pagecache_block_get(inode);
 
        ret = file_modified(file);
        if (ret)
@@ -3212,7 +3212,7 @@ long bch2_fallocate_dispatch(struct file *file, int mode,
        else
                ret = -EOPNOTSUPP;
 err:
-       bch2_pagecache_block_put(&inode->ei_pagecache_lock);
+       bch2_pagecache_block_put(inode);
        inode_unlock(&inode->v);
        percpu_ref_put(&c->writes);
 

diff --git a/fs/bcachefs/fs.c b/fs/bcachefs/fs.c
index 485cb9cbcd512c21a7f526b5cd66fc603aee6b5a..90297cfc79348ace03d8a808bee728d1e9a75b1e 100644 (file)
--- a/fs/bcachefs/fs.c
+++ b/fs/bcachefs/fs.c
@@ -43,58 +43,6 @@ static void bch2_vfs_inode_init(struct btree_trans *, subvol_inum,
                                struct bch_inode_unpacked *,
                                struct bch_subvolume *);
 
-static void __pagecache_lock_put(struct pagecache_lock *lock, long i)
-{
-       BUG_ON(atomic_long_read(&lock->v) == 0);
-
-       if (atomic_long_sub_return_release(i, &lock->v) == 0)
-               wake_up_all(&lock->wait);
-}
-
-static bool __pagecache_lock_tryget(struct pagecache_lock *lock, long i)
-{
-       long v = atomic_long_read(&lock->v), old;
-
-       do {
-               old = v;
-
-               if (i > 0 ? v < 0 : v > 0)
-                       return false;
-       } while ((v = atomic_long_cmpxchg_acquire(&lock->v,
-                                       old, old + i)) != old);
-       return true;
-}
-
-static void __pagecache_lock_get(struct pagecache_lock *lock, long i)
-{
-       wait_event(lock->wait, __pagecache_lock_tryget(lock, i));
-}
-
-void bch2_pagecache_add_put(struct pagecache_lock *lock)
-{
-       __pagecache_lock_put(lock, 1);
-}
-
-bool bch2_pagecache_add_tryget(struct pagecache_lock *lock)
-{
-       return __pagecache_lock_tryget(lock, 1);
-}
-
-void bch2_pagecache_add_get(struct pagecache_lock *lock)
-{
-       __pagecache_lock_get(lock, 1);
-}
-
-void bch2_pagecache_block_put(struct pagecache_lock *lock)
-{
-       __pagecache_lock_put(lock, -1);
-}
-
-void bch2_pagecache_block_get(struct pagecache_lock *lock)
-{
-       __pagecache_lock_get(lock, -1);
-}
-
 void bch2_inode_update_after_write(struct btree_trans *trans,
                                   struct bch_inode_info *inode,
                                   struct bch_inode_unpacked *bi,
@@ -1410,7 +1358,7 @@ static struct inode *bch2_alloc_inode(struct super_block *sb)
 
        inode_init_once(&inode->v);
        mutex_init(&inode->ei_update_lock);
-       pagecache_lock_init(&inode->ei_pagecache_lock);
+       two_state_lock_init(&inode->ei_pagecache_lock);
        mutex_init(&inode->ei_quota_lock);
 
        return &inode->v;

diff --git a/fs/bcachefs/fs.h b/fs/bcachefs/fs.h
index 73b96d0b5d836bb673f92c2c0d886829d0f412e3..4164d0669d7001892e264d3ae0a071cfa0e67906 100644 (file)
--- a/fs/bcachefs/fs.h
+++ b/fs/bcachefs/fs.h
@@ -6,31 +6,11 @@
 #include "opts.h"
 #include "str_hash.h"
 #include "quota_types.h"
+#include "two_state_shared_lock.h"
 
 #include <linux/seqlock.h>
 #include <linux/stat.h>
 
-/*
- * Two-state lock - can be taken for add or block - both states are shared,
- * like read side of rwsem, but conflict with other state:
- */
-struct pagecache_lock {
-       atomic_long_t           v;
-       wait_queue_head_t       wait;
-};
-
-static inline void pagecache_lock_init(struct pagecache_lock *lock)
-{
-       atomic_long_set(&lock->v, 0);
-       init_waitqueue_head(&lock->wait);
-}
-
-void bch2_pagecache_add_put(struct pagecache_lock *);
-bool bch2_pagecache_add_tryget(struct pagecache_lock *);
-void bch2_pagecache_add_get(struct pagecache_lock *);
-void bch2_pagecache_block_put(struct pagecache_lock *);
-void bch2_pagecache_block_get(struct pagecache_lock *);
-
 struct bch_inode_info {
        struct inode            v;
        unsigned long           ei_flags;
@@ -38,7 +18,7 @@ struct bch_inode_info {
        struct mutex            ei_update_lock;
        u64                     ei_quota_reserved;
        unsigned long           ei_last_dirtied;
-       struct pagecache_lock   ei_pagecache_lock;
+       two_state_lock_t        ei_pagecache_lock;
 
        struct mutex            ei_quota_lock;
        struct bch_qid          ei_qid;
@@ -49,6 +29,13 @@ struct bch_inode_info {
        struct bch_inode_unpacked ei_inode;
 };
 
+#define bch2_pagecache_add_put(i)      bch2_two_state_unlock(&i->ei_pagecache_lock, 0)
+#define bch2_pagecache_add_tryget(i)   bch2_two_state_trylock(&i->ei_pagecache_lock, 0)
+#define bch2_pagecache_add_get(i)      bch2_two_state_lock(&i->ei_pagecache_lock, 0)
+
+#define bch2_pagecache_block_put(i)    bch2_two_state_unlock(&i->ei_pagecache_lock, 1)
+#define bch2_pagecache_block_get(i)    bch2_two_state_lock(&i->ei_pagecache_lock, 1)
+
 static inline subvol_inum inode_inum(struct bch_inode_info *inode)
 {
        return (subvol_inum) {
@@ -95,7 +82,7 @@ do {                                                                  \
                        if ((_locks) & INODE_LOCK)                      \
                                down_write_nested(&a[i]->v.i_rwsem, i); \
                        if ((_locks) & INODE_PAGECACHE_BLOCK)           \
-                               bch2_pagecache_block_get(&a[i]->ei_pagecache_lock);\
+                               bch2_pagecache_block_get(a[i]);\
                        if ((_locks) & INODE_UPDATE_LOCK)                       \
                                mutex_lock_nested(&a[i]->ei_update_lock, i);\
                }                                                       \
@@ -113,7 +100,7 @@ do {                                                                        \
                        if ((_locks) & INODE_LOCK)                      \
                                up_write(&a[i]->v.i_rwsem);             \
                        if ((_locks) & INODE_PAGECACHE_BLOCK)           \
-                               bch2_pagecache_block_put(&a[i]->ei_pagecache_lock);\
+                               bch2_pagecache_block_put(a[i]);\
                        if ((_locks) & INODE_UPDATE_LOCK)                       \
                                mutex_unlock(&a[i]->ei_update_lock);    \
                }                                                       \

diff --git a/fs/bcachefs/two_state_shared_lock.c b/fs/bcachefs/two_state_shared_lock.c
new file mode 100644 (file)
index 0000000..dc508d5
--- /dev/null
+++ b/fs/bcachefs/two_state_shared_lock.c
@@ -0,0 +1,33 @@
+// SPDX-License-Identifier: GPL-2.0
+
+#include "two_state_shared_lock.h"
+
+void bch2_two_state_unlock(two_state_lock_t *lock, int s)
+{
+       long i = s ? 1 : -1;
+
+       BUG_ON(atomic_long_read(&lock->v) == 0);
+
+       if (atomic_long_sub_return_release(i, &lock->v) == 0)
+               wake_up_all(&lock->wait);
+}
+
+bool bch2_two_state_trylock(two_state_lock_t *lock, int s)
+{
+       long i = s ? 1 : -1;
+       long v = atomic_long_read(&lock->v), old;
+
+       do {
+               old = v;
+
+               if (i > 0 ? v < 0 : v > 0)
+                       return false;
+       } while ((v = atomic_long_cmpxchg_acquire(&lock->v,
+                                       old, old + i)) != old);
+       return true;
+}
+
+void bch2_two_state_lock(two_state_lock_t *lock, int s)
+{
+       wait_event(lock->wait, bch2_two_state_trylock(lock, s));
+}

diff --git a/fs/bcachefs/two_state_shared_lock.h b/fs/bcachefs/two_state_shared_lock.h
new file mode 100644 (file)
index 0000000..1b4f108
--- /dev/null
+++ b/fs/bcachefs/two_state_shared_lock.h
@@ -0,0 +1,28 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef _BCACHEFS_TWO_STATE_LOCK_H
+#define _BCACHEFS_TWO_STATE_LOCK_H
+
+#include <linux/atomic.h>
+#include <linux/sched.h>
+#include <linux/wait.h>
+
+/*
+ * Two-state lock - can be taken for add or block - both states are shared,
+ * like read side of rwsem, but conflict with other state:
+ */
+typedef struct {
+       atomic_long_t           v;
+       wait_queue_head_t       wait;
+} two_state_lock_t;
+
+static inline void two_state_lock_init(two_state_lock_t *lock)
+{
+       atomic_long_set(&lock->v, 0);
+       init_waitqueue_head(&lock->wait);
+}
+
+void bch2_two_state_unlock(two_state_lock_t *, int);
+bool bch2_two_state_trylock(two_state_lock_t *, int);
+void bch2_two_state_lock(two_state_lock_t *, int);
+
+#endif /* _BCACHEFS_TWO_STATE_LOCK_H */