xarray: Step through an XArray

The xas_next and xas_prev functions move the xas index by one position,
and adjust the rest of the iterator state to match it.  This is more
efficient than calling xas_set() as it keeps the iterator at the leaves
of the tree instead of walking the iterator from the root each time.

Signed-off-by: Matthew Wilcox <willy@infradead.org>

diff --git a/include/linux/xarray.h b/include/linux/xarray.h
index 0a758fa3ed2c783ebe86cd7a8bc58af49feafde2..381f94a667625bf3d29e066a817b17984a425004 100644 (file)
--- a/include/linux/xarray.h
+++ b/include/linux/xarray.h
@@ -828,6 +828,12 @@ static inline bool xas_not_node(struct xa_node *node)
        return ((unsigned long)node & 3) || !node;
 }
 
+/* True if the node represents RESTART or an error */
+static inline bool xas_frozen(struct xa_node *node)
+{
+       return (unsigned long)node & 2;
+}
+
 /* True if the node represents head-of-tree, RESTART or BOUNDS */
 static inline bool xas_top(struct xa_node *node)
 {
@@ -1082,4 +1088,65 @@ enum {
        for (entry = xas_find_marked(xas, max, mark); entry; \
             entry = xas_next_marked(xas, max, mark))
 
+void *__xas_next(struct xa_state *);
+void *__xas_prev(struct xa_state *);
+
+/**
+ * xas_prev() - Move iterator to previous index.
+ * @xas: XArray operation state.
+ *
+ * If the @xas was in an error state, it will remain in an error state
+ * and this function will return %NULL.  If the @xas has never been walked,
+ * it will have the effect of calling xas_load().  Otherwise one will be
+ * subtracted from the index and the state will be walked to the correct
+ * location in the array for the next operation.
+ *
+ * If the iterator was referencing index 0, this function wraps
+ * around to %ULONG_MAX.
+ *
+ * Return: The entry at the new index.  This may be %NULL or an internal
+ * entry.
+ */
+static inline void *xas_prev(struct xa_state *xas)
+{
+       struct xa_node *node = xas->xa_node;
+
+       if (unlikely(xas_not_node(node) || node->shift ||
+                               xas->xa_offset == 0))
+               return __xas_prev(xas);
+
+       xas->xa_index--;
+       xas->xa_offset--;
+       return xa_entry(xas->xa, node, xas->xa_offset);
+}
+
+/**
+ * xas_next() - Move state to next index.
+ * @xas: XArray operation state.
+ *
+ * If the @xas was in an error state, it will remain in an error state
+ * and this function will return %NULL.  If the @xas has never been walked,
+ * it will have the effect of calling xas_load().  Otherwise one will be
+ * added to the index and the state will be walked to the correct
+ * location in the array for the next operation.
+ *
+ * If the iterator was referencing index %ULONG_MAX, this function wraps
+ * around to 0.
+ *
+ * Return: The entry at the new index.  This may be %NULL or an internal
+ * entry.
+ */
+static inline void *xas_next(struct xa_state *xas)
+{
+       struct xa_node *node = xas->xa_node;
+
+       if (unlikely(xas_not_node(node) || node->shift ||
+                               xas->xa_offset == XA_CHUNK_MASK))
+               return __xas_next(xas);
+
+       xas->xa_index++;
+       xas->xa_offset++;
+       return xa_entry(xas->xa, node, xas->xa_offset);
+}
+
 #endif /* _LINUX_XARRAY_H */

diff --git a/lib/test_xarray.c b/lib/test_xarray.c
index a96f67caa1c2e94d95c86c4809e38a8904834246..85ef1882dd3ce8295542e36a930aaa983c5fb08a 100644 (file)
--- a/lib/test_xarray.c
+++ b/lib/test_xarray.c
@@ -465,6 +465,120 @@ static noinline void check_find(struct xarray *xa)
        check_multi_find_2(xa);
 }
 
+static noinline void check_move_small(struct xarray *xa, unsigned long idx)
+{
+       XA_STATE(xas, xa, 0);
+       unsigned long i;
+
+       xa_store_index(xa, 0, GFP_KERNEL);
+       xa_store_index(xa, idx, GFP_KERNEL);
+
+       rcu_read_lock();
+       for (i = 0; i < idx * 4; i++) {
+               void *entry = xas_next(&xas);
+               if (i <= idx)
+                       XA_BUG_ON(xa, xas.xa_node == XAS_RESTART);
+               XA_BUG_ON(xa, xas.xa_index != i);
+               if (i == 0 || i == idx)
+                       XA_BUG_ON(xa, entry != xa_mk_value(i));
+               else
+                       XA_BUG_ON(xa, entry != NULL);
+       }
+       xas_next(&xas);
+       XA_BUG_ON(xa, xas.xa_index != i);
+
+       do {
+               void *entry = xas_prev(&xas);
+               i--;
+               if (i <= idx)
+                       XA_BUG_ON(xa, xas.xa_node == XAS_RESTART);
+               XA_BUG_ON(xa, xas.xa_index != i);
+               if (i == 0 || i == idx)
+                       XA_BUG_ON(xa, entry != xa_mk_value(i));
+               else
+                       XA_BUG_ON(xa, entry != NULL);
+       } while (i > 0);
+
+       xas_set(&xas, ULONG_MAX);
+       XA_BUG_ON(xa, xas_next(&xas) != NULL);
+       XA_BUG_ON(xa, xas.xa_index != ULONG_MAX);
+       XA_BUG_ON(xa, xas_next(&xas) != xa_mk_value(0));
+       XA_BUG_ON(xa, xas.xa_index != 0);
+       XA_BUG_ON(xa, xas_prev(&xas) != NULL);
+       XA_BUG_ON(xa, xas.xa_index != ULONG_MAX);
+       rcu_read_unlock();
+
+       xa_erase_index(xa, 0);
+       xa_erase_index(xa, idx);
+       XA_BUG_ON(xa, !xa_empty(xa));
+}
+
+static noinline void check_move(struct xarray *xa)
+{
+       XA_STATE(xas, xa, (1 << 16) - 1);
+       unsigned long i;
+
+       for (i = 0; i < (1 << 16); i++)
+               XA_BUG_ON(xa, xa_store_index(xa, i, GFP_KERNEL) != NULL);
+
+       rcu_read_lock();
+       do {
+               void *entry = xas_prev(&xas);
+               i--;
+               XA_BUG_ON(xa, entry != xa_mk_value(i));
+               XA_BUG_ON(xa, i != xas.xa_index);
+       } while (i != 0);
+
+       XA_BUG_ON(xa, xas_prev(&xas) != NULL);
+       XA_BUG_ON(xa, xas.xa_index != ULONG_MAX);
+
+       do {
+               void *entry = xas_next(&xas);
+               XA_BUG_ON(xa, entry != xa_mk_value(i));
+               XA_BUG_ON(xa, i != xas.xa_index);
+               i++;
+       } while (i < (1 << 16));
+       rcu_read_unlock();
+
+       for (i = (1 << 8); i < (1 << 15); i++)
+               xa_erase_index(xa, i);
+
+       i = xas.xa_index;
+
+       rcu_read_lock();
+       do {
+               void *entry = xas_prev(&xas);
+               i--;
+               if ((i < (1 << 8)) || (i >= (1 << 15)))
+                       XA_BUG_ON(xa, entry != xa_mk_value(i));
+               else
+                       XA_BUG_ON(xa, entry != NULL);
+               XA_BUG_ON(xa, i != xas.xa_index);
+       } while (i != 0);
+
+       XA_BUG_ON(xa, xas_prev(&xas) != NULL);
+       XA_BUG_ON(xa, xas.xa_index != ULONG_MAX);
+
+       do {
+               void *entry = xas_next(&xas);
+               if ((i < (1 << 8)) || (i >= (1 << 15)))
+                       XA_BUG_ON(xa, entry != xa_mk_value(i));
+               else
+                       XA_BUG_ON(xa, entry != NULL);
+               XA_BUG_ON(xa, i != xas.xa_index);
+               i++;
+       } while (i < (1 << 16));
+       rcu_read_unlock();
+
+       xa_destroy(xa);
+
+       for (i = 0; i < 16; i++)
+               check_move_small(xa, 1UL << i);
+
+       for (i = 2; i < 16; i++)
+               check_move_small(xa, (1UL << i) - 1);
+}
+
 static noinline void check_destroy(struct xarray *xa)
 {
        unsigned long index;
@@ -512,6 +626,7 @@ static int xarray_checks(void)
        check_multi_store(&array);
        check_find(&array);
        check_destroy(&array);
+       check_move(&array);
 
        printk("XArray: %u of %u tests passed\n", tests_passed, tests_run);
        return (tests_run == tests_passed) ? 0 : -EINVAL;

diff --git a/lib/xarray.c b/lib/xarray.c
index 057dbe0d3bf61ed18b9e1cc019f4e61e3fcc51ef..303c46579598d7719d98f4333c294d58340a3947 100644 (file)
--- a/lib/xarray.c
+++ b/lib/xarray.c
@@ -876,6 +876,80 @@ void xas_pause(struct xa_state *xas)
 }
 EXPORT_SYMBOL_GPL(xas_pause);
 
+/*
+ * __xas_prev() - Find the previous entry in the XArray.
+ * @xas: XArray operation state.
+ *
+ * Helper function for xas_prev() which handles all the complex cases
+ * out of line.
+ */
+void *__xas_prev(struct xa_state *xas)
+{
+       void *entry;
+
+       if (!xas_frozen(xas->xa_node))
+               xas->xa_index--;
+       if (xas_not_node(xas->xa_node))
+               return xas_load(xas);
+
+       if (xas->xa_offset != get_offset(xas->xa_index, xas->xa_node))
+               xas->xa_offset--;
+
+       while (xas->xa_offset == 255) {
+               xas->xa_offset = xas->xa_node->offset - 1;
+               xas->xa_node = xa_parent(xas->xa, xas->xa_node);
+               if (!xas->xa_node)
+                       return set_bounds(xas);
+       }
+
+       for (;;) {
+               entry = xa_entry(xas->xa, xas->xa_node, xas->xa_offset);
+               if (!xa_is_node(entry))
+                       return entry;
+
+               xas->xa_node = xa_to_node(entry);
+               xas_set_offset(xas);
+       }
+}
+EXPORT_SYMBOL_GPL(__xas_prev);
+
+/*
+ * __xas_next() - Find the next entry in the XArray.
+ * @xas: XArray operation state.
+ *
+ * Helper function for xas_next() which handles all the complex cases
+ * out of line.
+ */
+void *__xas_next(struct xa_state *xas)
+{
+       void *entry;
+
+       if (!xas_frozen(xas->xa_node))
+               xas->xa_index++;
+       if (xas_not_node(xas->xa_node))
+               return xas_load(xas);
+
+       if (xas->xa_offset != get_offset(xas->xa_index, xas->xa_node))
+               xas->xa_offset++;
+
+       while (xas->xa_offset == XA_CHUNK_SIZE) {
+               xas->xa_offset = xas->xa_node->offset + 1;
+               xas->xa_node = xa_parent(xas->xa, xas->xa_node);
+               if (!xas->xa_node)
+                       return set_bounds(xas);
+       }
+
+       for (;;) {
+               entry = xa_entry(xas->xa, xas->xa_node, xas->xa_offset);
+               if (!xa_is_node(entry))
+                       return entry;
+
+               xas->xa_node = xa_to_node(entry);
+               xas_set_offset(xas);
+       }
+}
+EXPORT_SYMBOL_GPL(__xas_next);
+
 /**
  * xas_find() - Find the next present entry in the XArray.
  * @xas: XArray operation state.