* You may not use this file except in compliance with the License.
*
* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
- * or http://www.opensolaris.org/os/licensing.
+ * or https://opensource.org/licenses/CDDL-1.0.
* See the License for the specific language governing permissions
* and limitations under the License.
*
void
abd_verify(abd_t *abd)
{
- ASSERT3U(abd->abd_size, >, 0);
+#ifdef ZFS_DEBUG
ASSERT3U(abd->abd_size, <=, SPA_MAXBLOCKSIZE);
ASSERT3U(abd->abd_flags, ==, abd->abd_flags & (ABD_FLAG_LINEAR |
ABD_FLAG_OWNER | ABD_FLAG_META | ABD_FLAG_MULTI_ZONE |
IMPLY(abd->abd_parent != NULL, !(abd->abd_flags & ABD_FLAG_OWNER));
IMPLY(abd->abd_flags & ABD_FLAG_META, abd->abd_flags & ABD_FLAG_OWNER);
if (abd_is_linear(abd)) {
+ ASSERT3U(abd->abd_size, >, 0);
ASSERT3P(ABD_LINEAR_BUF(abd), !=, NULL);
} else if (abd_is_gang(abd)) {
uint_t child_sizes = 0;
}
ASSERT3U(abd->abd_size, ==, child_sizes);
} else {
+ ASSERT3U(abd->abd_size, >, 0);
abd_verify_scatter(abd);
}
+#endif
}
static void
{
list_link_init(&abd->abd_gang_link);
mutex_init(&abd->abd_mtx, NULL, MUTEX_DEFAULT, NULL);
- zfs_refcount_create(&abd->abd_children);
abd->abd_flags = 0;
+#ifdef ZFS_DEBUG
+ zfs_refcount_create(&abd->abd_children);
abd->abd_parent = NULL;
+#endif
abd->abd_size = 0;
}
{
mutex_destroy(&abd->abd_mtx);
ASSERT(!list_link_active(&abd->abd_gang_link));
+#ifdef ZFS_DEBUG
zfs_refcount_destroy(&abd->abd_children);
+#endif
}
abd_t *
abd_t *
abd_alloc(size_t size, boolean_t is_metadata)
{
- if (!zfs_abd_scatter_enabled || abd_size_alloc_linear(size))
+ if (abd_size_alloc_linear(size))
return (abd_alloc_linear(size, is_metadata));
VERIFY3U(size, <=, SPA_MAXBLOCKSIZE);
return;
abd_verify(abd);
+#ifdef ZFS_DEBUG
IMPLY(abd->abd_flags & ABD_FLAG_OWNER, abd->abd_parent == NULL);
+#endif
if (abd_is_gang(abd)) {
abd_free_gang(abd);
abd_free_scatter(abd);
}
+#ifdef ZFS_DEBUG
if (abd->abd_parent != NULL) {
(void) zfs_refcount_remove_many(&abd->abd_parent->abd_children,
abd->abd_size, abd);
}
+#endif
abd_fini_struct(abd);
if (abd->abd_flags & ABD_FLAG_ALLOCD)
* will retain all the free_on_free settings after being
* added to the parents list.
*/
+#ifdef ZFS_DEBUG
+ /*
+ * If cabd had abd_parent, we have to drop it here. We can't
+ * transfer it to pabd, nor we can clear abd_size leaving it.
+ */
+ if (cabd->abd_parent != NULL) {
+ (void) zfs_refcount_remove_many(
+ &cabd->abd_parent->abd_children,
+ cabd->abd_size, cabd);
+ cabd->abd_parent = NULL;
+ }
+#endif
pabd->abd_size += cabd->abd_size;
+ cabd->abd_size = 0;
list_move_tail(&ABD_GANG(pabd).abd_gang_chain,
&ABD_GANG(cabd).abd_gang_chain);
ASSERT(list_is_empty(&ABD_GANG(cabd).abd_gang_chain));
child = list_next(&ABD_GANG(cabd).abd_gang_chain, child)) {
/*
* We always pass B_FALSE for free_on_free as it is the
- * original child gang ABDs responsibilty to determine
+ * original child gang ABDs responsibility to determine
* if any of its child ABDs should be free'd on the call
* to abd_free().
*/
*/
if (abd_is_gang(cabd)) {
ASSERT(!list_link_active(&cabd->abd_gang_link));
- ASSERT(!list_is_empty(&ABD_GANG(cabd).abd_gang_chain));
return (abd_gang_add_gang(pabd, cabd, free_on_free));
}
ASSERT(!abd_is_gang(cabd));
}
ASSERT3U(left, ==, 0);
} else {
- abd = abd_get_offset_scatter(abd, sabd, off);
+ abd = abd_get_offset_scatter(abd, sabd, off, size);
}
ASSERT3P(abd, !=, NULL);
abd->abd_size = size;
+#ifdef ZFS_DEBUG
abd->abd_parent = sabd;
(void) zfs_refcount_add_many(&sabd->abd_children, abd->abd_size, abd);
+#endif
return (abd);
}
abd_t *
abd_get_offset_struct(abd_t *abd, abd_t *sabd, size_t off, size_t size)
{
+ abd_t *result;
abd_init_struct(abd);
- return (abd_get_offset_impl(abd, sabd, off, size));
+ result = abd_get_offset_impl(abd, sabd, off, size);
+ if (result != abd)
+ abd_fini_struct(abd);
+ return (result);
}
abd_t *
} else {
buf = zio_buf_alloc(n);
}
+#ifdef ZFS_DEBUG
(void) zfs_refcount_add_many(&abd->abd_children, n, buf);
+#endif
return (buf);
}
{
abd_verify(abd);
ASSERT3U(abd->abd_size, >=, n);
+#ifdef ZFS_DEBUG
+ (void) zfs_refcount_remove_many(&abd->abd_children, n, buf);
+#endif
if (abd_is_linear(abd)) {
ASSERT3P(buf, ==, abd_to_buf(abd));
} else {
ASSERT0(abd_cmp_buf(abd, buf, n));
zio_buf_free(buf, n);
}
- (void) zfs_refcount_remove_many(&abd->abd_children, n, buf);
}
void
abd_verify(abd);
ASSERT3U(off + size, <=, abd->abd_size);
- boolean_t gang = abd_is_gang(abd);
abd_t *c_abd = abd_init_abd_iter(abd, &aiter, off);
while (size > 0) {
- /* If we are at the end of the gang ABD we are done */
- if (gang && !c_abd)
- break;
+ IMPLY(abd_is_gang(abd), c_abd != NULL);
abd_iter_map(&aiter);
return (ret);
}
+#if defined(__linux__) && defined(_KERNEL)
+int
+abd_iterate_page_func(abd_t *abd, size_t off, size_t size,
+ abd_iter_page_func_t *func, void *private)
+{
+ struct abd_iter aiter;
+ int ret = 0;
+
+ if (size == 0)
+ return (0);
+
+ abd_verify(abd);
+ ASSERT3U(off + size, <=, abd->abd_size);
+
+ abd_t *c_abd = abd_init_abd_iter(abd, &aiter, off);
+
+ while (size > 0) {
+ IMPLY(abd_is_gang(abd), c_abd != NULL);
+
+ abd_iter_page(&aiter);
+
+ size_t len = MIN(aiter.iter_page_dsize, size);
+ ASSERT3U(len, >, 0);
+
+ ret = func(aiter.iter_page, aiter.iter_page_doff,
+ len, private);
+
+ aiter.iter_page = NULL;
+ aiter.iter_page_doff = 0;
+ aiter.iter_page_dsize = 0;
+
+ if (ret != 0)
+ break;
+
+ size -= len;
+ c_abd = abd_advance_abd_iter(abd, c_abd, &aiter, len);
+ }
+
+ return (ret);
+}
+#endif
+
struct buf_arg {
void *arg_buf;
};
&ba_ptr);
}
-/*ARGSUSED*/
static int
abd_zero_off_cb(void *buf, size_t size, void *private)
{
+ (void) private;
(void) memset(buf, 0, size);
return (0);
}
{
int ret = 0;
struct abd_iter daiter, saiter;
- boolean_t dabd_is_gang_abd, sabd_is_gang_abd;
abd_t *c_dabd, *c_sabd;
if (size == 0)
ASSERT3U(doff + size, <=, dabd->abd_size);
ASSERT3U(soff + size, <=, sabd->abd_size);
- dabd_is_gang_abd = abd_is_gang(dabd);
- sabd_is_gang_abd = abd_is_gang(sabd);
c_dabd = abd_init_abd_iter(dabd, &daiter, doff);
c_sabd = abd_init_abd_iter(sabd, &saiter, soff);
while (size > 0) {
- /* if we are at the end of the gang ABD we are done */
- if ((dabd_is_gang_abd && !c_dabd) ||
- (sabd_is_gang_abd && !c_sabd))
- break;
+ IMPLY(abd_is_gang(dabd), c_dabd != NULL);
+ IMPLY(abd_is_gang(sabd), c_sabd != NULL);
abd_iter_map(&daiter);
abd_iter_map(&saiter);
return (ret);
}
-/*ARGSUSED*/
static int
abd_copy_off_cb(void *dbuf, void *sbuf, size_t size, void *private)
{
+ (void) private;
(void) memcpy(dbuf, sbuf, size);
return (0);
}
abd_copy_off_cb, NULL);
}
-/*ARGSUSED*/
static int
abd_cmp_cb(void *bufa, void *bufb, size_t size, void *private)
{
+ (void) private;
return (memcmp(bufa, bufb, size));
}
* is the same when taking linear and when taking scatter
*/
void
-abd_raidz_gen_iterate(abd_t **cabds, abd_t *dabd,
- ssize_t csize, ssize_t dsize, const unsigned parity,
+abd_raidz_gen_iterate(abd_t **cabds, abd_t *dabd, size_t off,
+ size_t csize, size_t dsize, const unsigned parity,
void (*func_raidz_gen)(void **, const void *, size_t, size_t))
{
int i;
- ssize_t len, dlen;
+ size_t len, dlen;
struct abd_iter caiters[3];
- struct abd_iter daiter = {0};
- void *caddrs[3];
+ struct abd_iter daiter;
+ void *caddrs[3], *daddr;
unsigned long flags __maybe_unused = 0;
abd_t *c_cabds[3];
abd_t *c_dabd = NULL;
- boolean_t cabds_is_gang_abd[3];
- boolean_t dabd_is_gang_abd = B_FALSE;
ASSERT3U(parity, <=, 3);
-
for (i = 0; i < parity; i++) {
- cabds_is_gang_abd[i] = abd_is_gang(cabds[i]);
- c_cabds[i] = abd_init_abd_iter(cabds[i], &caiters[i], 0);
+ abd_verify(cabds[i]);
+ ASSERT3U(off + csize, <=, cabds[i]->abd_size);
+ c_cabds[i] = abd_init_abd_iter(cabds[i], &caiters[i], off);
}
- if (dabd) {
- dabd_is_gang_abd = abd_is_gang(dabd);
- c_dabd = abd_init_abd_iter(dabd, &daiter, 0);
+ if (dsize > 0) {
+ ASSERT(dabd);
+ abd_verify(dabd);
+ ASSERT3U(off + dsize, <=, dabd->abd_size);
+ c_dabd = abd_init_abd_iter(dabd, &daiter, off);
}
- ASSERT3S(dsize, >=, 0);
-
abd_enter_critical(flags);
while (csize > 0) {
- /* if we are at the end of the gang ABD we are done */
- if (dabd_is_gang_abd && !c_dabd)
- break;
-
+ len = csize;
for (i = 0; i < parity; i++) {
- /*
- * If we are at the end of the gang ABD we are
- * done.
- */
- if (cabds_is_gang_abd[i] && !c_cabds[i])
- break;
+ IMPLY(abd_is_gang(cabds[i]), c_cabds[i] != NULL);
abd_iter_map(&caiters[i]);
caddrs[i] = caiters[i].iter_mapaddr;
+ len = MIN(caiters[i].iter_mapsize, len);
}
- len = csize;
-
- if (dabd && dsize > 0)
+ if (dsize > 0) {
+ IMPLY(abd_is_gang(dabd), c_dabd != NULL);
abd_iter_map(&daiter);
-
- switch (parity) {
- case 3:
- len = MIN(caiters[2].iter_mapsize, len);
- /* falls through */
- case 2:
- len = MIN(caiters[1].iter_mapsize, len);
- /* falls through */
- case 1:
- len = MIN(caiters[0].iter_mapsize, len);
- }
-
- /* must be progressive */
- ASSERT3S(len, >, 0);
-
- if (dabd && dsize > 0) {
- /* this needs precise iter.length */
+ daddr = daiter.iter_mapaddr;
len = MIN(daiter.iter_mapsize, len);
dlen = len;
- } else
+ } else {
+ daddr = NULL;
dlen = 0;
+ }
/* must be progressive */
- ASSERT3S(len, >, 0);
+ ASSERT3U(len, >, 0);
/*
* The iterated function likely will not do well if each
* segment except the last one is not multiple of 512 (raidz).
*/
ASSERT3U(((uint64_t)len & 511ULL), ==, 0);
- func_raidz_gen(caddrs, daiter.iter_mapaddr, len, dlen);
+ func_raidz_gen(caddrs, daddr, len, dlen);
for (i = parity-1; i >= 0; i--) {
abd_iter_unmap(&caiters[i]);
&caiters[i], len);
}
- if (dabd && dsize > 0) {
+ if (dsize > 0) {
abd_iter_unmap(&daiter);
c_dabd =
abd_advance_abd_iter(dabd, c_dabd, &daiter,
}
csize -= len;
-
- ASSERT3S(dsize, >=, 0);
- ASSERT3S(csize, >=, 0);
}
abd_exit_critical(flags);
}
*/
void
abd_raidz_rec_iterate(abd_t **cabds, abd_t **tabds,
- ssize_t tsize, const unsigned parity,
+ size_t tsize, const unsigned parity,
void (*func_raidz_rec)(void **t, const size_t tsize, void **c,
const unsigned *mul),
const unsigned *mul)
{
int i;
- ssize_t len;
+ size_t len;
struct abd_iter citers[3];
struct abd_iter xiters[3];
void *caddrs[3], *xaddrs[3];
unsigned long flags __maybe_unused = 0;
- boolean_t cabds_is_gang_abd[3];
- boolean_t tabds_is_gang_abd[3];
abd_t *c_cabds[3];
abd_t *c_tabds[3];
ASSERT3U(parity, <=, 3);
for (i = 0; i < parity; i++) {
- cabds_is_gang_abd[i] = abd_is_gang(cabds[i]);
- tabds_is_gang_abd[i] = abd_is_gang(tabds[i]);
+ abd_verify(cabds[i]);
+ abd_verify(tabds[i]);
+ ASSERT3U(tsize, <=, cabds[i]->abd_size);
+ ASSERT3U(tsize, <=, tabds[i]->abd_size);
c_cabds[i] =
abd_init_abd_iter(cabds[i], &citers[i], 0);
c_tabds[i] =
abd_enter_critical(flags);
while (tsize > 0) {
-
+ len = tsize;
for (i = 0; i < parity; i++) {
- /*
- * If we are at the end of the gang ABD we
- * are done.
- */
- if (cabds_is_gang_abd[i] && !c_cabds[i])
- break;
- if (tabds_is_gang_abd[i] && !c_tabds[i])
- break;
+ IMPLY(abd_is_gang(cabds[i]), c_cabds[i] != NULL);
+ IMPLY(abd_is_gang(tabds[i]), c_tabds[i] != NULL);
abd_iter_map(&citers[i]);
abd_iter_map(&xiters[i]);
caddrs[i] = citers[i].iter_mapaddr;
xaddrs[i] = xiters[i].iter_mapaddr;
+ len = MIN(citers[i].iter_mapsize, len);
+ len = MIN(xiters[i].iter_mapsize, len);
}
- len = tsize;
- switch (parity) {
- case 3:
- len = MIN(xiters[2].iter_mapsize, len);
- len = MIN(citers[2].iter_mapsize, len);
- /* falls through */
- case 2:
- len = MIN(xiters[1].iter_mapsize, len);
- len = MIN(citers[1].iter_mapsize, len);
- /* falls through */
- case 1:
- len = MIN(xiters[0].iter_mapsize, len);
- len = MIN(citers[0].iter_mapsize, len);
- }
/* must be progressive */
ASSERT3S(len, >, 0);
/*
projects / mirror_zfs.git / blobdiff