abd: add page iterator

[mirror_zfs.git] / include / sys / abd_impl.h

diff --git a/include/sys/abd_impl.h b/include/sys/abd_impl.h
index 5aee772b1e04b55f08b643efc238848fa0be6d1b..f88ea25e245d2319b5913462e2f1a22b350113e9 100644 (file)
--- a/include/sys/abd_impl.h
+++ b/include/sys/abd_impl.h
@@ -6,7 +6,7 @@
  * You may not use this file except in compliance with the License.
  *
  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
- * or http://www.opensolaris.org/os/licensing.
+ * or https://opensource.org/licenses/CDDL-1.0.
  * See the License for the specific language governing permissions
  * and limitations under the License.
  *
@@ -21,60 +21,48 @@
 /*
  * Copyright (c) 2014 by Chunwei Chen. All rights reserved.
  * Copyright (c) 2016, 2019 by Delphix. All rights reserved.
+ * Copyright (c) 2023, 2024, Klara Inc.
  */
 
 #ifndef _ABD_IMPL_H
 #define        _ABD_IMPL_H
 
 #include <sys/abd.h>
+#include <sys/wmsum.h>
 
 #ifdef __cplusplus
 extern "C" {
 #endif
 
-typedef enum abd_flags {
-       ABD_FLAG_LINEAR         = 1 << 0, /* is buffer linear (or scattered)? */
-       ABD_FLAG_OWNER          = 1 << 1, /* does it own its data buffers? */
-       ABD_FLAG_META           = 1 << 2, /* does this represent FS metadata? */
-       ABD_FLAG_MULTI_ZONE     = 1 << 3, /* pages split over memory zones */
-       ABD_FLAG_MULTI_CHUNK    = 1 << 4, /* pages split over multiple chunks */
-       ABD_FLAG_LINEAR_PAGE    = 1 << 5, /* linear but allocd from page */
-} abd_flags_t;
-
 typedef enum abd_stats_op {
        ABDSTAT_INCR, /* Increase abdstat values */
        ABDSTAT_DECR  /* Decrease abdstat values */
 } abd_stats_op_t;
 
-struct abd {
-       abd_flags_t     abd_flags;
-       uint_t          abd_size;       /* excludes scattered abd_offset */
-       struct abd      *abd_parent;
-       zfs_refcount_t  abd_children;
-       union {
-               struct abd_scatter {
-                       uint_t          abd_offset;
-#if defined(__FreeBSD__) && defined(_KERNEL)
-                       uint_t  abd_chunk_size;
-                       void    *abd_chunks[];
-#else
-                       uint_t          abd_nents;
-                       struct scatterlist *abd_sgl;
-#endif
-               } abd_scatter;
-               struct abd_linear {
-                       void            *abd_buf;
-                       struct scatterlist *abd_sgl; /* for LINEAR_PAGE */
-               } abd_linear;
-       } abd_u;
-};
-
-struct scatterlist; /* forward declaration */
+/* forward declarations */
+struct scatterlist;
+struct page;
 
 struct abd_iter {
        /* public interface */
-       void            *iter_mapaddr;  /* addr corresponding to iter_pos */
-       size_t          iter_mapsize;   /* length of data valid at mapaddr */
+       union {
+               /* for abd_iter_map()/abd_iter_unmap() */
+               struct {
+                       /* addr corresponding to iter_pos */
+                       void            *iter_mapaddr;
+                       /* length of data valid at mapaddr */
+                       size_t          iter_mapsize;
+               };
+               /* for abd_iter_page() */
+               struct {
+                       /* current page */
+                       struct page     *iter_page;
+                       /* offset of data in page */
+                       size_t          iter_page_doff;
+                       /* size of data in page */
+                       size_t          iter_page_dsize;
+               };
+       };
 
        /* private */
        abd_t           *iter_abd;      /* ABD being iterated through */
@@ -84,16 +72,21 @@ struct abd_iter {
        struct scatterlist *iter_sg;    /* current sg */
 };
 
+extern abd_t *abd_zero_scatter;
+
+abd_t *abd_gang_get_offset(abd_t *, size_t *);
+abd_t *abd_alloc_struct(size_t);
+void abd_free_struct(abd_t *);
+
 /*
  * OS specific functions
  */
 
-abd_t *abd_alloc_struct(size_t);
-abd_t *abd_get_offset_scatter(abd_t *, size_t);
-void abd_free_struct(abd_t *);
+abd_t *abd_alloc_struct_impl(size_t);
+abd_t *abd_get_offset_scatter(abd_t *, abd_t *, size_t, size_t);
+void abd_free_struct_impl(abd_t *);
 void abd_alloc_chunks(abd_t *, size_t);
 void abd_free_chunks(abd_t *);
-boolean_t abd_size_alloc_linear(size_t);
 void abd_update_scatter_stats(abd_t *, abd_stats_op_t);
 void abd_update_linear_stats(abd_t *, abd_stats_op_t);
 void abd_verify_scatter(abd_t *);
@@ -104,18 +97,19 @@ boolean_t abd_iter_at_end(struct abd_iter *);
 void abd_iter_advance(struct abd_iter *, size_t);
 void abd_iter_map(struct abd_iter *);
 void abd_iter_unmap(struct abd_iter *);
+void abd_iter_page(struct abd_iter *);
 
 /*
  * Helper macros
  */
-#define        ABDSTAT(stat)           (abd_stats.stat.value.ui64)
 #define        ABDSTAT_INCR(stat, val) \
-       atomic_add_64(&abd_stats.stat.value.ui64, (val))
+       wmsum_add(&abd_sums.stat, (val))
 #define        ABDSTAT_BUMP(stat)      ABDSTAT_INCR(stat, 1)
 #define        ABDSTAT_BUMPDOWN(stat)  ABDSTAT_INCR(stat, -1)
 
 #define        ABD_SCATTER(abd)        (abd->abd_u.abd_scatter)
 #define        ABD_LINEAR_BUF(abd)     (abd->abd_u.abd_linear.abd_buf)
+#define        ABD_GANG(abd)           (abd->abd_u.abd_gang)
 
 #if defined(_KERNEL)
 #if defined(__FreeBSD__)