Rename range_tree_verify to range_tree_verify_not_present

[mirror_zfs.git] / module / zfs / range_tree.c

/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */
/*
 * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */
/*
 * Copyright (c) 2013, 2017 by Delphix. All rights reserved.
 */

#include <sys/zfs_context.h>
#include <sys/spa.h>
#include <sys/dmu.h>
#include <sys/dnode.h>
#include <sys/zio.h>
#include <sys/range_tree.h>

/*
 * Range trees are tree-based data structures that can be used to
 * track free space or generally any space allocation information.
 * A range tree keeps track of individual segments and automatically
 * provides facilities such as adjacent extent merging and extent
 * splitting in response to range add/remove requests.
 *
 * A range tree starts out completely empty, with no segments in it.
 * Adding an allocation via range_tree_add to the range tree can either:
 * 1) create a new extent
 * 2) extend an adjacent extent
 * 3) merge two adjacent extents
 * Conversely, removing an allocation via range_tree_remove can:
 * 1) completely remove an extent
 * 2) shorten an extent (if the allocation was near one of its ends)
 * 3) split an extent into two extents, in effect punching a hole
 *
 * A range tree is also capable of 'bridging' gaps when adding
 * allocations. This is useful for cases when close proximity of
 * allocations is an important detail that needs to be represented
 * in the range tree. See range_tree_set_gap(). The default behavior
 * is not to bridge gaps (i.e. the maximum allowed gap size is 0).
 *
 * In order to traverse a range tree, use either the range_tree_walk()
 * or range_tree_vacate() functions.
 *
 * To obtain more accurate information on individual segment
 * operations that the range tree performs "under the hood", you can
 * specify a set of callbacks by passing a range_tree_ops_t structure
 * to the range_tree_create function. Any callbacks that are non-NULL
 * are then called at the appropriate times.
 *
 * The range tree code also supports a special variant of range trees
 * that can bridge small gaps between segments. This kind of tree is used
 * by the dsl scanning code to group I/Os into mostly sequential chunks to
 * optimize disk performance. The code here attempts to do this with as
 * little memory and computational overhead as possible. One limitation of
 * this implementation is that segments of range trees with gaps can only
 * support removing complete segments.
 */

kmem_cache_t *range_seg_cache;

/* Generic ops for managing an AVL tree alongside a range tree */
struct range_tree_ops rt_avl_ops = {
        .rtop_create = rt_avl_create,
        .rtop_destroy = rt_avl_destroy,
        .rtop_add = rt_avl_add,
        .rtop_remove = rt_avl_remove,
        .rtop_vacate = rt_avl_vacate,
};

void
range_tree_init(void)
{
        ASSERT(range_seg_cache == NULL);
        range_seg_cache = kmem_cache_create("range_seg_cache",
            sizeof (range_seg_t), 0, NULL, NULL, NULL, NULL, NULL, 0);
}

void
range_tree_fini(void)
{
        kmem_cache_destroy(range_seg_cache);
        range_seg_cache = NULL;
}

void
range_tree_stat_verify(range_tree_t *rt)
{
        range_seg_t *rs;
        uint64_t hist[RANGE_TREE_HISTOGRAM_SIZE] = { 0 };
        int i;

        for (rs = avl_first(&rt->rt_root); rs != NULL;
            rs = AVL_NEXT(&rt->rt_root, rs)) {
                uint64_t size = rs->rs_end - rs->rs_start;
                int idx = highbit64(size) - 1;

                hist[idx]++;
                ASSERT3U(hist[idx], !=, 0);
        }

        for (i = 0; i < RANGE_TREE_HISTOGRAM_SIZE; i++) {
                if (hist[i] != rt->rt_histogram[i]) {
                        zfs_dbgmsg("i=%d, hist=%p, hist=%llu, rt_hist=%llu",
                            i, hist, hist[i], rt->rt_histogram[i]);
                }
                VERIFY3U(hist[i], ==, rt->rt_histogram[i]);
        }
}

static void
range_tree_stat_incr(range_tree_t *rt, range_seg_t *rs)
{
        uint64_t size = rs->rs_end - rs->rs_start;
        int idx = highbit64(size) - 1;

        ASSERT(size != 0);
        ASSERT3U(idx, <,
            sizeof (rt->rt_histogram) / sizeof (*rt->rt_histogram));

        rt->rt_histogram[idx]++;
        ASSERT3U(rt->rt_histogram[idx], !=, 0);
}

static void
range_tree_stat_decr(range_tree_t *rt, range_seg_t *rs)
{
        uint64_t size = rs->rs_end - rs->rs_start;
        int idx = highbit64(size) - 1;

        ASSERT(size != 0);
        ASSERT3U(idx, <,
            sizeof (rt->rt_histogram) / sizeof (*rt->rt_histogram));

        ASSERT3U(rt->rt_histogram[idx], !=, 0);
        rt->rt_histogram[idx]--;
}

/*
 * NOTE: caller is responsible for all locking.
 */
static int
range_tree_seg_compare(const void *x1, const void *x2)
{
        const range_seg_t *r1 = (const range_seg_t *)x1;
        const range_seg_t *r2 = (const range_seg_t *)x2;

        ASSERT3U(r1->rs_start, <=, r1->rs_end);
        ASSERT3U(r2->rs_start, <=, r2->rs_end);

        return ((r1->rs_start >= r2->rs_end) - (r1->rs_end <= r2->rs_start));
}

range_tree_t *
range_tree_create_impl(range_tree_ops_t *ops, void *arg,
    int (*avl_compare) (const void *, const void *), uint64_t gap)
{
        range_tree_t *rt = kmem_zalloc(sizeof (range_tree_t), KM_SLEEP);

        avl_create(&rt->rt_root, range_tree_seg_compare,
            sizeof (range_seg_t), offsetof(range_seg_t, rs_node));

        rt->rt_ops = ops;
        rt->rt_gap = gap;
        rt->rt_arg = arg;
        rt->rt_avl_compare = avl_compare;

        if (rt->rt_ops != NULL && rt->rt_ops->rtop_create != NULL)
                rt->rt_ops->rtop_create(rt, rt->rt_arg);

        return (rt);
}

range_tree_t *
range_tree_create(range_tree_ops_t *ops, void *arg)
{
        return (range_tree_create_impl(ops, arg, NULL, 0));
}

void
range_tree_destroy(range_tree_t *rt)
{
        VERIFY0(rt->rt_space);

        if (rt->rt_ops != NULL && rt->rt_ops->rtop_destroy != NULL)
                rt->rt_ops->rtop_destroy(rt, rt->rt_arg);

        avl_destroy(&rt->rt_root);
        kmem_free(rt, sizeof (*rt));
}

void
range_tree_adjust_fill(range_tree_t *rt, range_seg_t *rs, int64_t delta)
{
        ASSERT3U(rs->rs_fill + delta, !=, 0);
        ASSERT3U(rs->rs_fill + delta, <=, rs->rs_end - rs->rs_start);

        if (rt->rt_ops != NULL && rt->rt_ops->rtop_remove != NULL)
                rt->rt_ops->rtop_remove(rt, rs, rt->rt_arg);
        rs->rs_fill += delta;
        if (rt->rt_ops != NULL && rt->rt_ops->rtop_add != NULL)
                rt->rt_ops->rtop_add(rt, rs, rt->rt_arg);
}

static void
range_tree_add_impl(void *arg, uint64_t start, uint64_t size, uint64_t fill)
{
        range_tree_t *rt = arg;
        avl_index_t where;
        range_seg_t rsearch, *rs_before, *rs_after, *rs;
        uint64_t end = start + size, gap = rt->rt_gap;
        uint64_t bridge_size = 0;
        boolean_t merge_before, merge_after;

        ASSERT3U(size, !=, 0);
        ASSERT3U(fill, <=, size);

        rsearch.rs_start = start;
        rsearch.rs_end = end;
        rs = avl_find(&rt->rt_root, &rsearch, &where);

        if (gap == 0 && rs != NULL &&
            rs->rs_start <= start && rs->rs_end >= end) {
                zfs_panic_recover("zfs: allocating allocated segment"
                    "(offset=%llu size=%llu) of (offset=%llu size=%llu)\n",
                    (longlong_t)start, (longlong_t)size,
                    (longlong_t)rs->rs_start,
                    (longlong_t)rs->rs_end - rs->rs_start);
                return;
        }

        /*
         * If this is a gap-supporting range tree, it is possible that we
         * are inserting into an existing segment. In this case simply
         * bump the fill count and call the remove / add callbacks. If the
         * new range will extend an existing segment, we remove the
         * existing one, apply the new extent to it and re-insert it using
         * the normal code paths.
         */
        if (rs != NULL) {
                ASSERT3U(gap, !=, 0);
                if (rs->rs_start <= start && rs->rs_end >= end) {
                        range_tree_adjust_fill(rt, rs, fill);
                        return;
                }

                avl_remove(&rt->rt_root, rs);
                if (rt->rt_ops != NULL && rt->rt_ops->rtop_remove != NULL)
                        rt->rt_ops->rtop_remove(rt, rs, rt->rt_arg);

                range_tree_stat_decr(rt, rs);
                rt->rt_space -= rs->rs_end - rs->rs_start;

                fill += rs->rs_fill;
                start = MIN(start, rs->rs_start);
                end = MAX(end, rs->rs_end);
                size = end - start;

                range_tree_add_impl(rt, start, size, fill);

                kmem_cache_free(range_seg_cache, rs);
                return;
        }

        ASSERT3P(rs, ==, NULL);

        /*
         * Determine whether or not we will have to merge with our neighbors.
         * If gap != 0, we might need to merge with our neighbors even if we
         * aren't directly touching.
         */
        rs_before = avl_nearest(&rt->rt_root, where, AVL_BEFORE);
        rs_after = avl_nearest(&rt->rt_root, where, AVL_AFTER);

        merge_before = (rs_before != NULL && rs_before->rs_end >= start - gap);
        merge_after = (rs_after != NULL && rs_after->rs_start <= end + gap);

        if (merge_before && gap != 0)
                bridge_size += start - rs_before->rs_end;
        if (merge_after && gap != 0)
                bridge_size += rs_after->rs_start - end;

        if (merge_before && merge_after) {
                avl_remove(&rt->rt_root, rs_before);
                if (rt->rt_ops != NULL && rt->rt_ops->rtop_remove != NULL) {
                        rt->rt_ops->rtop_remove(rt, rs_before, rt->rt_arg);
                        rt->rt_ops->rtop_remove(rt, rs_after, rt->rt_arg);
                }

                range_tree_stat_decr(rt, rs_before);
                range_tree_stat_decr(rt, rs_after);

                rs_after->rs_fill += rs_before->rs_fill + fill;
                rs_after->rs_start = rs_before->rs_start;
                kmem_cache_free(range_seg_cache, rs_before);
                rs = rs_after;
        } else if (merge_before) {
                if (rt->rt_ops != NULL && rt->rt_ops->rtop_remove != NULL)
                        rt->rt_ops->rtop_remove(rt, rs_before, rt->rt_arg);

                range_tree_stat_decr(rt, rs_before);

                rs_before->rs_fill += fill;
                rs_before->rs_end = end;
                rs = rs_before;
        } else if (merge_after) {
                if (rt->rt_ops != NULL && rt->rt_ops->rtop_remove != NULL)
                        rt->rt_ops->rtop_remove(rt, rs_after, rt->rt_arg);

                range_tree_stat_decr(rt, rs_after);

                rs_after->rs_fill += fill;
                rs_after->rs_start = start;
                rs = rs_after;
        } else {
                rs = kmem_cache_alloc(range_seg_cache, KM_SLEEP);

                rs->rs_fill = fill;
                rs->rs_start = start;
                rs->rs_end = end;
                avl_insert(&rt->rt_root, rs, where);
        }

        if (gap != 0)
                ASSERT3U(rs->rs_fill, <=, rs->rs_end - rs->rs_start);
        else
                ASSERT3U(rs->rs_fill, ==, rs->rs_end - rs->rs_start);

        if (rt->rt_ops != NULL && rt->rt_ops->rtop_add != NULL)
                rt->rt_ops->rtop_add(rt, rs, rt->rt_arg);

        range_tree_stat_incr(rt, rs);
        rt->rt_space += size + bridge_size;
}

void
range_tree_add(void *arg, uint64_t start, uint64_t size)
{
        range_tree_add_impl(arg, start, size, size);
}

static void
range_tree_remove_impl(range_tree_t *rt, uint64_t start, uint64_t size,
    boolean_t do_fill)
{
        avl_index_t where;
        range_seg_t rsearch, *rs, *newseg;
        uint64_t end = start + size;
        boolean_t left_over, right_over;

        VERIFY3U(size, !=, 0);
        VERIFY3U(size, <=, rt->rt_space);

        rsearch.rs_start = start;
        rsearch.rs_end = end;
        rs = avl_find(&rt->rt_root, &rsearch, &where);

        /* Make sure we completely overlap with someone */
        if (rs == NULL) {
                zfs_panic_recover("zfs: freeing free segment "
                    "(offset=%llu size=%llu)",
                    (longlong_t)start, (longlong_t)size);
                return;
        }

        /*
         * Range trees with gap support must only remove complete segments
         * from the tree. This allows us to maintain accurate fill accounting
         * and to ensure that bridged sections are not leaked. If we need to
         * remove less than the full segment, we can only adjust the fill count.
         */
        if (rt->rt_gap != 0) {
                if (do_fill) {
                        if (rs->rs_fill == size) {
                                start = rs->rs_start;
                                end = rs->rs_end;
                                size = end - start;
                        } else {
                                range_tree_adjust_fill(rt, rs, -size);
                                return;
                        }
                } else if (rs->rs_start != start || rs->rs_end != end) {
                        zfs_panic_recover("zfs: freeing partial segment of "
                            "gap tree (offset=%llu size=%llu) of "
                            "(offset=%llu size=%llu)",
                            (longlong_t)start, (longlong_t)size,
                            (longlong_t)rs->rs_start,
                            (longlong_t)rs->rs_end - rs->rs_start);
                        return;
                }
        }

        VERIFY3U(rs->rs_start, <=, start);
        VERIFY3U(rs->rs_end, >=, end);

        left_over = (rs->rs_start != start);
        right_over = (rs->rs_end != end);

        range_tree_stat_decr(rt, rs);

        if (rt->rt_ops != NULL && rt->rt_ops->rtop_remove != NULL)
                rt->rt_ops->rtop_remove(rt, rs, rt->rt_arg);

        if (left_over && right_over) {
                newseg = kmem_cache_alloc(range_seg_cache, KM_SLEEP);
                newseg->rs_start = end;
                newseg->rs_end = rs->rs_end;
                newseg->rs_fill = newseg->rs_end - newseg->rs_start;
                range_tree_stat_incr(rt, newseg);

                rs->rs_end = start;

                avl_insert_here(&rt->rt_root, newseg, rs, AVL_AFTER);
                if (rt->rt_ops != NULL && rt->rt_ops->rtop_add != NULL)
                        rt->rt_ops->rtop_add(rt, newseg, rt->rt_arg);
        } else if (left_over) {
                rs->rs_end = start;
        } else if (right_over) {
                rs->rs_start = end;
        } else {
                avl_remove(&rt->rt_root, rs);
                kmem_cache_free(range_seg_cache, rs);
                rs = NULL;
        }

        if (rs != NULL) {
                /*
                 * The fill of the leftover segment will always be equal to
                 * the size, since we do not support removing partial segments
                 * of range trees with gaps.
                 */
                rs->rs_fill = rs->rs_end - rs->rs_start;
                range_tree_stat_incr(rt, rs);

                if (rt->rt_ops != NULL && rt->rt_ops->rtop_add != NULL)
                        rt->rt_ops->rtop_add(rt, rs, rt->rt_arg);
        }

        rt->rt_space -= size;
}

void
range_tree_remove(void *arg, uint64_t start, uint64_t size)
{
        range_tree_remove_impl(arg, start, size, B_FALSE);
}

void
range_tree_remove_fill(range_tree_t *rt, uint64_t start, uint64_t size)
{
        range_tree_remove_impl(rt, start, size, B_TRUE);
}

void
range_tree_resize_segment(range_tree_t *rt, range_seg_t *rs,
    uint64_t newstart, uint64_t newsize)
{
        int64_t delta = newsize - (rs->rs_end - rs->rs_start);

        range_tree_stat_decr(rt, rs);
        if (rt->rt_ops != NULL && rt->rt_ops->rtop_remove != NULL)
                rt->rt_ops->rtop_remove(rt, rs, rt->rt_arg);

        rs->rs_start = newstart;
        rs->rs_end = newstart + newsize;

        range_tree_stat_incr(rt, rs);
        if (rt->rt_ops != NULL && rt->rt_ops->rtop_add != NULL)
                rt->rt_ops->rtop_add(rt, rs, rt->rt_arg);

        rt->rt_space += delta;
}

static range_seg_t *
range_tree_find_impl(range_tree_t *rt, uint64_t start, uint64_t size)
{
        range_seg_t rsearch;
        uint64_t end = start + size;

        VERIFY(size != 0);

        rsearch.rs_start = start;
        rsearch.rs_end = end;
        return (avl_find(&rt->rt_root, &rsearch, NULL));
}

range_seg_t *
range_tree_find(range_tree_t *rt, uint64_t start, uint64_t size)
{
        range_seg_t *rs = range_tree_find_impl(rt, start, size);
        if (rs != NULL && rs->rs_start <= start && rs->rs_end >= start + size)
                return (rs);
        return (NULL);
}

void
range_tree_verify_not_present(range_tree_t *rt, uint64_t off, uint64_t size)
{
        range_seg_t *rs = range_tree_find(rt, off, size);
        if (rs != NULL)
                panic("segment already in tree; rs=%p", (void *)rs);
}

boolean_t
range_tree_contains(range_tree_t *rt, uint64_t start, uint64_t size)
{
        return (range_tree_find(rt, start, size) != NULL);
}

/*
 * Ensure that this range is not in the tree, regardless of whether
 * it is currently in the tree.
 */
void
range_tree_clear(range_tree_t *rt, uint64_t start, uint64_t size)
{
        range_seg_t *rs;

        if (size == 0)
                return;

        while ((rs = range_tree_find_impl(rt, start, size)) != NULL) {
                uint64_t free_start = MAX(rs->rs_start, start);
                uint64_t free_end = MIN(rs->rs_end, start + size);
                range_tree_remove(rt, free_start, free_end - free_start);
        }
}

void
range_tree_swap(range_tree_t **rtsrc, range_tree_t **rtdst)
{
        range_tree_t *rt;

        ASSERT0(range_tree_space(*rtdst));
        ASSERT0(avl_numnodes(&(*rtdst)->rt_root));

        rt = *rtsrc;
        *rtsrc = *rtdst;
        *rtdst = rt;
}

void
range_tree_vacate(range_tree_t *rt, range_tree_func_t *func, void *arg)
{
        range_seg_t *rs;
        void *cookie = NULL;

        if (rt->rt_ops != NULL && rt->rt_ops->rtop_vacate != NULL)
                rt->rt_ops->rtop_vacate(rt, rt->rt_arg);

        while ((rs = avl_destroy_nodes(&rt->rt_root, &cookie)) != NULL) {
                if (func != NULL)
                        func(arg, rs->rs_start, rs->rs_end - rs->rs_start);
                kmem_cache_free(range_seg_cache, rs);
        }

        bzero(rt->rt_histogram, sizeof (rt->rt_histogram));
        rt->rt_space = 0;
}

void
range_tree_walk(range_tree_t *rt, range_tree_func_t *func, void *arg)
{
        range_seg_t *rs;

        for (rs = avl_first(&rt->rt_root); rs; rs = AVL_NEXT(&rt->rt_root, rs))
                func(arg, rs->rs_start, rs->rs_end - rs->rs_start);
}

range_seg_t *
range_tree_first(range_tree_t *rt)
{
        return (avl_first(&rt->rt_root));
}

uint64_t
range_tree_space(range_tree_t *rt)
{
        return (rt->rt_space);
}

boolean_t
range_tree_is_empty(range_tree_t *rt)
{
        ASSERT(rt != NULL);
        return (range_tree_space(rt) == 0);
}

/* Generic range tree functions for maintaining segments in an AVL tree. */
void
rt_avl_create(range_tree_t *rt, void *arg)
{
        avl_tree_t *tree = arg;

        avl_create(tree, rt->rt_avl_compare, sizeof (range_seg_t),
            offsetof(range_seg_t, rs_pp_node));
}

void
rt_avl_destroy(range_tree_t *rt, void *arg)
{
        avl_tree_t *tree = arg;

        ASSERT0(avl_numnodes(tree));
        avl_destroy(tree);
}

void
rt_avl_add(range_tree_t *rt, range_seg_t *rs, void *arg)
{
        avl_tree_t *tree = arg;
        avl_add(tree, rs);
}

void
rt_avl_remove(range_tree_t *rt, range_seg_t *rs, void *arg)
{
        avl_tree_t *tree = arg;
        avl_remove(tree, rs);
}

void
rt_avl_vacate(range_tree_t *rt, void *arg)
{
        /*
         * Normally one would walk the tree freeing nodes along the way.
         * Since the nodes are shared with the range trees we can avoid
         * walking all nodes and just reinitialize the avl tree. The nodes
         * will be freed by the range tree, so we don't want to free them here.
         */
        rt_avl_create(rt, arg);
}

uint64_t
range_tree_min(range_tree_t *rt)
{
        range_seg_t *rs = avl_first(&rt->rt_root);
        return (rs != NULL ? rs->rs_start : 0);
}

uint64_t
range_tree_max(range_tree_t *rt)
{
        range_seg_t *rs = avl_last(&rt->rt_root);
        return (rs != NULL ? rs->rs_end : 0);
}

uint64_t
range_tree_span(range_tree_t *rt)
{
        return (range_tree_max(rt) - range_tree_min(rt));
}