maple_tree: add end of node tracking to the maple state

Analysis of the mas_for_each() iteration showed that there is a
significant time spent finding the end of a node.  This time can be
greatly reduced if the end of the node is cached in the maple state.  Care
must be taken to update & invalidate as necessary.

Link: https://lkml.kernel.org/r/20231101171629.3612299-5-Liam.Howlett@oracle.com
Signed-off-by: Liam R. Howlett <Liam.Howlett@oracle.com>
Cc: Peng Zhang <zhangpeng.00@bytedance.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>

diff --git a/include/linux/maple_tree.h b/include/linux/maple_tree.h
index b5d5992578c91c3b3c060cb4f7ee0b0fa1b6acd1..0b82efe0cf1eaa550cb811c259bfe7ca8c4c990a 100644 (file)
--- a/include/linux/maple_tree.h
+++ b/include/linux/maple_tree.h
@@ -393,6 +393,7 @@ struct ma_state {
        unsigned char depth;            /* depth of tree descent during write */
        unsigned char offset;
        unsigned char mas_flags;
+       unsigned char end;              /* The end of the node */
 };
 
 struct ma_wr_state {

diff --git a/lib/maple_tree.c b/lib/maple_tree.c
index 8d379d34ea0a764f6cfccba66e7d8966cd9b95bb..ea0a36341fed3e957bdf68109107039ed6f8ae91 100644 (file)
--- a/lib/maple_tree.c
+++ b/lib/maple_tree.c
@@ -2841,6 +2841,7 @@ next:
                        goto dead_node;
        } while (!ma_is_leaf(type));
 
+       mas->end = end;
        mas->offset = offset;
        mas->index = min;
        mas->last = max;
@@ -3507,6 +3508,7 @@ static noinline_for_kasan int mas_commit_b_node(struct ma_wr_state *wr_mas,
        mas_replace_node(wr_mas->mas, old_enode);
 reuse_node:
        mas_update_gap(wr_mas->mas);
+       wr_mas->mas->end = b_end;
        return 1;
 }
 
@@ -4010,6 +4012,7 @@ done:
        }
        trace_ma_write(__func__, mas, 0, wr_mas->entry);
        mas_update_gap(mas);
+       mas->end = new_end;
        return true;
 }
 
@@ -4190,6 +4193,7 @@ static inline bool mas_wr_append(struct ma_wr_state *wr_mas,
        if (!wr_mas->content || !wr_mas->entry)
                mas_update_gap(mas);
 
+       mas->end = new_end;
        trace_ma_write(__func__, mas, new_end, wr_mas->entry);
        return  true;
 }
@@ -4428,6 +4432,7 @@ static inline int mas_prev_node(struct ma_state *mas, unsigned long min)
        if (unlikely(mte_dead_node(mas->node)))
                return 1;
 
+       mas->end = mas->offset;
        return 0;
 
 no_entry:
@@ -5074,6 +5079,7 @@ int mas_empty_area(struct ma_state *mas, unsigned long min,
        if (mas->index < min)
                mas->index = min;
        mas->last = mas->index + size - 1;
+       mas->end = mas_data_end(mas);
        return 0;
 }
 EXPORT_SYMBOL_GPL(mas_empty_area);
@@ -5134,6 +5140,7 @@ int mas_empty_area_rev(struct ma_state *mas, unsigned long min,
                mas->last = max;
 
        mas->index = mas->last - size + 1;
+       mas->end = mas_data_end(mas);
        return 0;
 }
 EXPORT_SYMBOL_GPL(mas_empty_area_rev);

diff --git a/tools/testing/radix-tree/maple.c b/tools/testing/radix-tree/maple.c
index cb53586745213a6cbe8a5fc4cd7d8574c8ee8e96..7095fb0ec026a2e8deb351dbfd799db36474eecc 100644 (file)
--- a/tools/testing/radix-tree/maple.c
+++ b/tools/testing/radix-tree/maple.c
@@ -945,6 +945,7 @@ retry:
                goto retry;
        }
 
+       mas->end = mas_data_end(mas);
        return ret;
 
 not_found: