xfs: active perag reference counting

We need to be able to dynamically remove instantiated AGs from
memory safely, either for shrinking the filesystem or paging AG
state in and out of memory (e.g. supporting millions of AGs). This
means we need to be able to safely exclude operations from accessing
perags while dynamic removal is in progress.

To do this, introduce the concept of active and passive references.
Active references are required for high level operations that make
use of an AG for a given operation (e.g. allocation) and pin the
perag in memory for the duration of the operation that is operating
on the perag (e.g. transaction scope). This means we can fail to get
an active reference to an AG, hence callers of the new active
reference API must be able to handle lookup failure gracefully.

Passive references are used in low level code, where we might need
to access the perag structure for the purposes of completing high
level operations. For example, buffers need to use passive
references because:
- we need to be able to do metadata IO during operations like grow
  and shrink transactions where high level active references to the
  AG have already been blocked
- buffers need to pin the perag until they are reclaimed from
  memory, something that high level code has no direct control over.
- unused cached buffers should not prevent a shrink from being
  started.

Hence we have active references that will form exclusion barriers
for operations to be performed on an AG, and passive references that
will prevent reclaim of the perag until all objects with passive
references have been reclaimed themselves.

This patch introduce xfs_perag_grab()/xfs_perag_rele() as the API
for active AG reference functionality. We also need to convert the
for_each_perag*() iterators to use active references, which will
start the process of converting high level code over to using active
references. Conversion of non-iterator based code to active
references will be done in followup patches.

Note that the implementation using reference counting is really just
a development vehicle for the API to ensure we don't have any leaks
in the callers. Once we need to remove perag structures from memory
dyanmically, we will need a much more robust per-ag state transition
mechanism for preventing new references from being taken while we
wait for existing references to drain before removal from memory can
occur....

Signed-off-by: Dave Chinner <dchinner@redhat.com>
Reviewed-by: Allison Henderson <allison.henderson@oracle.com>
Reviewed-by: Darrick J. Wong <djwong@kernel.org>

diff --git a/fs/xfs/libxfs/xfs_ag.c b/fs/xfs/libxfs/xfs_ag.c
index bb0c700afe3cb19d3daac50d274af8d0941fb8a0..46e25c682bf412bb419a3923349ae7491b8b50a4 100644 (file)
--- a/fs/xfs/libxfs/xfs_ag.c
+++ b/fs/xfs/libxfs/xfs_ag.c
@@ -94,6 +94,68 @@ xfs_perag_put(
        trace_xfs_perag_put(pag->pag_mount, pag->pag_agno, ref, _RET_IP_);
 }
 
+/*
+ * Active references for perag structures. This is for short term access to the
+ * per ag structures for walking trees or accessing state. If an AG is being
+ * shrunk or is offline, then this will fail to find that AG and return NULL
+ * instead.
+ */
+struct xfs_perag *
+xfs_perag_grab(
+       struct xfs_mount        *mp,
+       xfs_agnumber_t          agno)
+{
+       struct xfs_perag        *pag;
+
+       rcu_read_lock();
+       pag = radix_tree_lookup(&mp->m_perag_tree, agno);
+       if (pag) {
+               trace_xfs_perag_grab(mp, pag->pag_agno,
+                               atomic_read(&pag->pag_active_ref), _RET_IP_);
+               if (!atomic_inc_not_zero(&pag->pag_active_ref))
+                       pag = NULL;
+       }
+       rcu_read_unlock();
+       return pag;
+}
+
+/*
+ * search from @first to find the next perag with the given tag set.
+ */
+struct xfs_perag *
+xfs_perag_grab_tag(
+       struct xfs_mount        *mp,
+       xfs_agnumber_t          first,
+       int                     tag)
+{
+       struct xfs_perag        *pag;
+       int                     found;
+
+       rcu_read_lock();
+       found = radix_tree_gang_lookup_tag(&mp->m_perag_tree,
+                                       (void **)&pag, first, 1, tag);
+       if (found <= 0) {
+               rcu_read_unlock();
+               return NULL;
+       }
+       trace_xfs_perag_grab_tag(mp, pag->pag_agno,
+                       atomic_read(&pag->pag_active_ref), _RET_IP_);
+       if (!atomic_inc_not_zero(&pag->pag_active_ref))
+               pag = NULL;
+       rcu_read_unlock();
+       return pag;
+}
+
+void
+xfs_perag_rele(
+       struct xfs_perag        *pag)
+{
+       trace_xfs_perag_rele(pag->pag_mount, pag->pag_agno,
+                       atomic_read(&pag->pag_active_ref), _RET_IP_);
+       if (atomic_dec_and_test(&pag->pag_active_ref))
+               wake_up(&pag->pag_active_wq);
+}
+
 /*
  * xfs_initialize_perag_data
  *
@@ -196,6 +258,10 @@ xfs_free_perag(
                cancel_delayed_work_sync(&pag->pag_blockgc_work);
                xfs_buf_hash_destroy(pag);
 
+               /* drop the mount's active reference */
+               xfs_perag_rele(pag);
+               XFS_IS_CORRUPT(pag->pag_mount,
+                               atomic_read(&pag->pag_active_ref) != 0);
                call_rcu(&pag->rcu_head, __xfs_free_perag);
        }
 }
@@ -314,6 +380,7 @@ xfs_initialize_perag(
                INIT_DELAYED_WORK(&pag->pag_blockgc_work, xfs_blockgc_worker);
                INIT_RADIX_TREE(&pag->pag_ici_root, GFP_ATOMIC);
                init_waitqueue_head(&pag->pagb_wait);
+               init_waitqueue_head(&pag->pag_active_wq);
                pag->pagb_count = 0;
                pag->pagb_tree = RB_ROOT;
 #endif /* __KERNEL__ */
@@ -322,6 +389,9 @@ xfs_initialize_perag(
                if (error)
                        goto out_remove_pag;
 
+               /* Active ref owned by mount indicates AG is online. */
+               atomic_set(&pag->pag_active_ref, 1);
+
                /* first new pag is fully initialized */
                if (first_initialised == NULLAGNUMBER)
                        first_initialised = index;

diff --git a/fs/xfs/libxfs/xfs_ag.h b/fs/xfs/libxfs/xfs_ag.h
index 191b22b9a35bfd67981de4f05c97703ad2371f7f..aeb21c8df201eb7f66d5f2191232932ee6eddfb2 100644 (file)
--- a/fs/xfs/libxfs/xfs_ag.h
+++ b/fs/xfs/libxfs/xfs_ag.h
@@ -32,7 +32,9 @@ struct xfs_ag_resv {
 struct xfs_perag {
        struct xfs_mount *pag_mount;    /* owner filesystem */
        xfs_agnumber_t  pag_agno;       /* AG this structure belongs to */
-       atomic_t        pag_ref;        /* perag reference count */
+       atomic_t        pag_ref;        /* passive reference count */
+       atomic_t        pag_active_ref; /* active reference count */
+       wait_queue_head_t pag_active_wq;/* woken active_ref falls to zero */
        char            pagf_init;      /* this agf's entry is initialized */
        char            pagi_init;      /* this agi's entry is initialized */
        char            pagf_metadata;  /* the agf is preferred to be metadata */
@@ -111,11 +113,18 @@ int xfs_initialize_perag(struct xfs_mount *mp, xfs_agnumber_t agcount,
 int xfs_initialize_perag_data(struct xfs_mount *mp, xfs_agnumber_t agno);
 void xfs_free_perag(struct xfs_mount *mp);
 
+/* Passive AG references */
 struct xfs_perag *xfs_perag_get(struct xfs_mount *mp, xfs_agnumber_t agno);
 struct xfs_perag *xfs_perag_get_tag(struct xfs_mount *mp, xfs_agnumber_t agno,
                unsigned int tag);
 void xfs_perag_put(struct xfs_perag *pag);
 
+/* Active AG references */
+struct xfs_perag *xfs_perag_grab(struct xfs_mount *, xfs_agnumber_t);
+struct xfs_perag *xfs_perag_grab_tag(struct xfs_mount *, xfs_agnumber_t,
+                                  int tag);
+void xfs_perag_rele(struct xfs_perag *pag);
+
 /*
  * Per-ag geometry infomation and validation
  */
@@ -193,14 +202,18 @@ xfs_perag_next(
        struct xfs_mount        *mp = pag->pag_mount;
 
        *agno = pag->pag_agno + 1;
-       xfs_perag_put(pag);
-       if (*agno > end_agno)
-               return NULL;
-       return xfs_perag_get(mp, *agno);
+       xfs_perag_rele(pag);
+       while (*agno <= end_agno) {
+               pag = xfs_perag_grab(mp, *agno);
+               if (pag)
+                       return pag;
+               (*agno)++;
+       }
+       return NULL;
 }
 
 #define for_each_perag_range(mp, agno, end_agno, pag) \
-       for ((pag) = xfs_perag_get((mp), (agno)); \
+       for ((pag) = xfs_perag_grab((mp), (agno)); \
                (pag) != NULL; \
                (pag) = xfs_perag_next((pag), &(agno), (end_agno)))
 
@@ -213,11 +226,11 @@ xfs_perag_next(
        for_each_perag_from((mp), (agno), (pag))
 
 #define for_each_perag_tag(mp, agno, pag, tag) \
-       for ((agno) = 0, (pag) = xfs_perag_get_tag((mp), 0, (tag)); \
+       for ((agno) = 0, (pag) = xfs_perag_grab_tag((mp), 0, (tag)); \
                (pag) != NULL; \
                (agno) = (pag)->pag_agno + 1, \
-               xfs_perag_put(pag), \
-               (pag) = xfs_perag_get_tag((mp), (agno), (tag)))
+               xfs_perag_rele(pag), \
+               (pag) = xfs_perag_grab_tag((mp), (agno), (tag)))
 
 struct aghdr_init_data {
        /* per ag data */

diff --git a/fs/xfs/scrub/bmap.c b/fs/xfs/scrub/bmap.c
index d50d0eab196abfe0640bad8385d67e7abed92539..dbbc7037074c4ce7e3a7a1fead738a993cf34930 100644 (file)
--- a/fs/xfs/scrub/bmap.c
+++ b/fs/xfs/scrub/bmap.c
@@ -662,7 +662,7 @@ xchk_bmap_check_rmaps(
                error = xchk_bmap_check_ag_rmaps(sc, whichfork, pag);
                if (error ||
                    (sc->sm->sm_flags & XFS_SCRUB_OFLAG_CORRUPT)) {
-                       xfs_perag_put(pag);
+                       xfs_perag_rele(pag);
                        return error;
                }
        }

diff --git a/fs/xfs/scrub/fscounters.c b/fs/xfs/scrub/fscounters.c
index 4777e7b89fdc69527fbb31d65801acf693e4af4f..ef97670970c311799d342afbec895dfb4d086587 100644 (file)
--- a/fs/xfs/scrub/fscounters.c
+++ b/fs/xfs/scrub/fscounters.c
@@ -117,7 +117,7 @@ xchk_fscount_warmup(
        if (agi_bp)
                xfs_buf_relse(agi_bp);
        if (pag)
-               xfs_perag_put(pag);
+               xfs_perag_rele(pag);
        return error;
 }
 
@@ -249,7 +249,7 @@ retry:
 
        }
        if (pag)
-               xfs_perag_put(pag);
+               xfs_perag_rele(pag);
        if (error) {
                xchk_set_incomplete(sc);
                return error;

diff --git a/fs/xfs/xfs_fsmap.c b/fs/xfs/xfs_fsmap.c
index 88a88506ffffe4aaeaba676e2b734bd0bce6e7cf..120d284a03fe4b607df60101bb3a661dad59f913 100644 (file)
--- a/fs/xfs/xfs_fsmap.c
+++ b/fs/xfs/xfs_fsmap.c
@@ -688,11 +688,11 @@ __xfs_getfsmap_datadev(
                info->agf_bp = NULL;
        }
        if (info->pag) {
-               xfs_perag_put(info->pag);
+               xfs_perag_rele(info->pag);
                info->pag = NULL;
        } else if (pag) {
                /* loop termination case */
-               xfs_perag_put(pag);
+               xfs_perag_rele(pag);
        }
 
        return error;

diff --git a/fs/xfs/xfs_icache.c b/fs/xfs/xfs_icache.c
index ddeaccc04aec94a3362efffb617121ca6157e0ba..0f4a014dded3d8990c5777baa6b830bcc6a3a64e 100644 (file)
--- a/fs/xfs/xfs_icache.c
+++ b/fs/xfs/xfs_icache.c
@@ -1767,7 +1767,7 @@ xfs_icwalk(
                if (error) {
                        last_error = error;
                        if (error == -EFSCORRUPTED) {
-                               xfs_perag_put(pag);
+                               xfs_perag_rele(pag);
                                break;
                        }
                }

diff --git a/fs/xfs/xfs_iwalk.c b/fs/xfs/xfs_iwalk.c
index 7558486f49371c861e3d38a0d803ef2537fa4e06..c31857d903a4532b8d531fd0bec605a1d7533094 100644 (file)
--- a/fs/xfs/xfs_iwalk.c
+++ b/fs/xfs/xfs_iwalk.c
@@ -591,7 +591,7 @@ xfs_iwalk(
        }
 
        if (iwag.pag)
-               xfs_perag_put(pag);
+               xfs_perag_rele(pag);
        xfs_iwalk_free(&iwag);
        return error;
 }
@@ -683,7 +683,7 @@ xfs_iwalk_threaded(
                        break;
        }
        if (pag)
-               xfs_perag_put(pag);
+               xfs_perag_rele(pag);
        if (polled)
                xfs_pwork_poll(&pctl);
        return xfs_pwork_destroy(&pctl);
@@ -776,7 +776,7 @@ xfs_inobt_walk(
        }
 
        if (iwag.pag)
-               xfs_perag_put(pag);
+               xfs_perag_rele(pag);
        xfs_iwalk_free(&iwag);
        return error;
 }

diff --git a/fs/xfs/xfs_reflink.c b/fs/xfs/xfs_reflink.c
index 57bf59ff48543eb9b5a596d311bc9f90e1c0060f..f5dc46ce980311676910dd622c329add19ce8ad4 100644 (file)
--- a/fs/xfs/xfs_reflink.c
+++ b/fs/xfs/xfs_reflink.c
@@ -927,7 +927,7 @@ xfs_reflink_recover_cow(
        for_each_perag(mp, agno, pag) {
                error = xfs_refcount_recover_cow_leftovers(mp, pag);
                if (error) {
-                       xfs_perag_put(pag);
+                       xfs_perag_rele(pag);
                        break;
                }
        }

diff --git a/fs/xfs/xfs_trace.h b/fs/xfs/xfs_trace.h
index 2ac98d8ddbfde99c243b64c23602fc356c5b3521..eb5e49d44f13e5b029050dfe054919248a8a51f8 100644 (file)
--- a/fs/xfs/xfs_trace.h
+++ b/fs/xfs/xfs_trace.h
@@ -189,6 +189,9 @@ DEFINE_EVENT(xfs_perag_class, name, \
 DEFINE_PERAG_REF_EVENT(xfs_perag_get);
 DEFINE_PERAG_REF_EVENT(xfs_perag_get_tag);
 DEFINE_PERAG_REF_EVENT(xfs_perag_put);
+DEFINE_PERAG_REF_EVENT(xfs_perag_grab);
+DEFINE_PERAG_REF_EVENT(xfs_perag_grab_tag);
+DEFINE_PERAG_REF_EVENT(xfs_perag_rele);
 DEFINE_PERAG_REF_EVENT(xfs_perag_set_inode_tag);
 DEFINE_PERAG_REF_EVENT(xfs_perag_clear_inode_tag);