lib/generic-radix-tree.c: Add peek_prev()

This patch adds genradix_peek_prev(), genradix_iter_rewind(), and
genradix_for_each_reverse(), for iterating backwards over a generic
radix tree.

Signed-off-by: Kent Overstreet <kent.overstreet@linux.dev>

diff --git a/include/linux/generic-radix-tree.h b/include/linux/generic-radix-tree.h
index f6cd0f909d9fb9fd9c6fa59c26051e05383707c0..8474131647388b954812515930c81b0c83315f3a 100644 (file)
--- a/include/linux/generic-radix-tree.h
+++ b/include/linux/generic-radix-tree.h
@@ -117,6 +117,11 @@ static inline size_t __idx_to_offset(size_t idx, size_t obj_size)
 
 #define __genradix_cast(_radix)                (typeof((_radix)->type[0]) *)
 #define __genradix_obj_size(_radix)    sizeof((_radix)->type[0])
+#define __genradix_objs_per_page(_radix)                       \
+       (PAGE_SIZE / sizeof((_radix)->type[0]))
+#define __genradix_page_remainder(_radix)                      \
+       (PAGE_SIZE % sizeof((_radix)->type[0]))
+
 #define __genradix_idx_to_offset(_radix, _idx)                 \
        __idx_to_offset(_idx, __genradix_obj_size(_radix))
 
@@ -180,7 +185,25 @@ void *__genradix_iter_peek(struct genradix_iter *, struct __genradix *, size_t);
 #define genradix_iter_peek(_iter, _radix)                      \
        (__genradix_cast(_radix)                                \
         __genradix_iter_peek(_iter, &(_radix)->tree,           \
-                             PAGE_SIZE / __genradix_obj_size(_radix)))
+                       __genradix_objs_per_page(_radix)))
+
+void *__genradix_iter_peek_prev(struct genradix_iter *, struct __genradix *,
+                               size_t, size_t);
+
+/**
+ * genradix_iter_peek_prev - get first entry at or below iterator's current
+ *                          position
+ * @_iter:     a genradix_iter
+ * @_radix:    genradix being iterated over
+ *
+ * If no more entries exist at or below @_iter's current position, returns NULL
+ */
+#define genradix_iter_peek_prev(_iter, _radix)                 \
+       (__genradix_cast(_radix)                                \
+        __genradix_iter_peek_prev(_iter, &(_radix)->tree,      \
+                       __genradix_objs_per_page(_radix),       \
+                       __genradix_obj_size(_radix) +           \
+                       __genradix_page_remainder(_radix)))
 
 static inline void __genradix_iter_advance(struct genradix_iter *iter,
                                           size_t obj_size)
@@ -203,6 +226,25 @@ static inline void __genradix_iter_advance(struct genradix_iter *iter,
 #define genradix_iter_advance(_iter, _radix)                   \
        __genradix_iter_advance(_iter, __genradix_obj_size(_radix))
 
+static inline void __genradix_iter_rewind(struct genradix_iter *iter,
+                                         size_t obj_size)
+{
+       if (iter->offset == 0 ||
+           iter->offset == SIZE_MAX) {
+               iter->offset = SIZE_MAX;
+               return;
+       }
+
+       if ((iter->offset & (PAGE_SIZE - 1)) == 0)
+               iter->offset -= PAGE_SIZE % obj_size;
+
+       iter->offset -= obj_size;
+       iter->pos--;
+}
+
+#define genradix_iter_rewind(_iter, _radix)                    \
+       __genradix_iter_rewind(_iter, __genradix_obj_size(_radix))
+
 #define genradix_for_each_from(_radix, _iter, _p, _start)      \
        for (_iter = genradix_iter_init(_radix, _start);        \
             (_p = genradix_iter_peek(&_iter, _radix)) != NULL; \
@@ -220,6 +262,23 @@ static inline void __genradix_iter_advance(struct genradix_iter *iter,
 #define genradix_for_each(_radix, _iter, _p)                   \
        genradix_for_each_from(_radix, _iter, _p, 0)
 
+#define genradix_last_pos(_radix)                              \
+       (SIZE_MAX / PAGE_SIZE * __genradix_objs_per_page(_radix) - 1)
+
+/**
+ * genradix_for_each_reverse - iterate over entry in a genradix, reverse order
+ * @_radix:    genradix to iterate over
+ * @_iter:     a genradix_iter to track current position
+ * @_p:                pointer to genradix entry type
+ *
+ * On every iteration, @_p will point to the current entry, and @_iter.pos
+ * will be the current entry's index.
+ */
+#define genradix_for_each_reverse(_radix, _iter, _p)           \
+       for (_iter = genradix_iter_init(_radix, genradix_last_pos(_radix));\
+            (_p = genradix_iter_peek_prev(&_iter, _radix)) != NULL;\
+            genradix_iter_rewind(&_iter, _radix))
+
 int __genradix_prealloc(struct __genradix *, size_t, gfp_t);
 
 /**

diff --git a/lib/generic-radix-tree.c b/lib/generic-radix-tree.c
index 7dfa88282b006a408c7a6e77f69c17353b333301..41f1bcdc44886c1cef8f25c080c006d22a45339d 100644 (file)
--- a/lib/generic-radix-tree.c
+++ b/lib/generic-radix-tree.c
@@ -1,4 +1,5 @@
 
+#include <linux/atomic.h>
 #include <linux/export.h>
 #include <linux/generic-radix-tree.h>
 #include <linux/gfp.h>
@@ -212,6 +213,64 @@ restart:
 }
 EXPORT_SYMBOL(__genradix_iter_peek);
 
+void *__genradix_iter_peek_prev(struct genradix_iter *iter,
+                               struct __genradix *radix,
+                               size_t objs_per_page,
+                               size_t obj_size_plus_page_remainder)
+{
+       struct genradix_root *r;
+       struct genradix_node *n;
+       unsigned level, i;
+
+       if (iter->offset == SIZE_MAX)
+               return NULL;
+
+restart:
+       r = READ_ONCE(radix->root);
+       if (!r)
+               return NULL;
+
+       n       = genradix_root_to_node(r);
+       level   = genradix_root_to_depth(r);
+
+       if (ilog2(iter->offset) >= genradix_depth_shift(level)) {
+               iter->offset = genradix_depth_size(level);
+               iter->pos = (iter->offset >> PAGE_SHIFT) * objs_per_page;
+
+               iter->offset -= obj_size_plus_page_remainder;
+               iter->pos--;
+       }
+
+       while (level) {
+               level--;
+
+               i = (iter->offset >> genradix_depth_shift(level)) &
+                       (GENRADIX_ARY - 1);
+
+               while (!n->children[i]) {
+                       size_t objs_per_ptr = genradix_depth_size(level);
+
+                       iter->offset = round_down(iter->offset, objs_per_ptr);
+                       iter->pos = (iter->offset >> PAGE_SHIFT) * objs_per_page;
+
+                       if (!iter->offset)
+                               return NULL;
+
+                       iter->offset -= obj_size_plus_page_remainder;
+                       iter->pos--;
+
+                       if (!i)
+                               goto restart;
+                       --i;
+               }
+
+               n = n->children[i];
+       }
+
+       return &n->data[iter->offset & (PAGE_SIZE - 1)];
+}
+EXPORT_SYMBOL(__genradix_iter_peek_prev);
+
 static void genradix_free_recurse(struct genradix_node *n, unsigned level)
 {
        if (level) {