io-wq: add io_wq_work_node based stack

Apart from just using lists (i.e. io_wq_work_list), we also want to have
stacks, which are a bit faster, and have some interoperability between
them. Add a stack implementation based on io_wq_work_node and some
helpers.

Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
Link: https://lore.kernel.org/r/5d3a412a5ac0d47e0f0499d70d2207d70a68925e.1632516769.git.asml.silence@gmail.com
Signed-off-by: Jens Axboe <axboe@kernel.dk>

diff --git a/fs/io-wq.h b/fs/io-wq.h
index bf5c4c533760574f87d0cab165be2f4c60ecbff3..c870062105d1b8fe9eeacf307740db71fc1d9533 100644 (file)
--- a/fs/io-wq.h
+++ b/fs/io-wq.h
@@ -29,6 +29,15 @@ struct io_wq_work_list {
        struct io_wq_work_node *last;
 };
 
+#define wq_list_for_each(pos, prv, head)                       \
+       for (pos = (head)->first, prv = NULL; pos; prv = pos, pos = (pos)->next)
+
+#define wq_list_empty(list)    (READ_ONCE((list)->first) == NULL)
+#define INIT_WQ_LIST(list)     do {                            \
+       (list)->first = NULL;                                   \
+       (list)->last = NULL;                                    \
+} while (0)
+
 static inline void wq_list_add_after(struct io_wq_work_node *node,
                                     struct io_wq_work_node *pos,
                                     struct io_wq_work_list *list)
@@ -54,6 +63,15 @@ static inline void wq_list_add_tail(struct io_wq_work_node *node,
        }
 }
 
+static inline void wq_list_add_head(struct io_wq_work_node *node,
+                                   struct io_wq_work_list *list)
+{
+       node->next = list->first;
+       if (!node->next)
+               list->last = node;
+       WRITE_ONCE(list->first, node);
+}
+
 static inline void wq_list_cut(struct io_wq_work_list *list,
                               struct io_wq_work_node *last,
                               struct io_wq_work_node *prev)
@@ -69,6 +87,31 @@ static inline void wq_list_cut(struct io_wq_work_list *list,
        last->next = NULL;
 }
 
+static inline void __wq_list_splice(struct io_wq_work_list *list,
+                                   struct io_wq_work_node *to)
+{
+       list->last->next = to->next;
+       to->next = list->first;
+       INIT_WQ_LIST(list);
+}
+
+static inline bool wq_list_splice(struct io_wq_work_list *list,
+                                 struct io_wq_work_node *to)
+{
+       if (!wq_list_empty(list)) {
+               __wq_list_splice(list, to);
+               return true;
+       }
+       return false;
+}
+
+static inline void wq_stack_add_head(struct io_wq_work_node *node,
+                                    struct io_wq_work_node *stack)
+{
+       node->next = stack->next;
+       stack->next = node;
+}
+
 static inline void wq_list_del(struct io_wq_work_list *list,
                               struct io_wq_work_node *node,
                               struct io_wq_work_node *prev)
@@ -76,14 +119,14 @@ static inline void wq_list_del(struct io_wq_work_list *list,
        wq_list_cut(list, node, prev);
 }
 
-#define wq_list_for_each(pos, prv, head)                       \
-       for (pos = (head)->first, prv = NULL; pos; prv = pos, pos = (pos)->next)
+static inline
+struct io_wq_work_node *wq_stack_extract(struct io_wq_work_node *stack)
+{
+       struct io_wq_work_node *node = stack->next;
 
-#define wq_list_empty(list)    (READ_ONCE((list)->first) == NULL)
-#define INIT_WQ_LIST(list)     do {                            \
-       (list)->first = NULL;                                   \
-       (list)->last = NULL;                                    \
-} while (0)
+       stack->next = node->next;
+       return node;
+}
 
 struct io_wq_work {
        struct io_wq_work_node list;