import 15.2.0 Octopus source

[ceph.git] / ceph / src / spdk / lib / env_dpdk / memory.c

/*-
 *   BSD LICENSE
 *
 *   Copyright (c) Intel Corporation.
 *   All rights reserved.
 *
 *   Redistribution and use in source and binary forms, with or without
 *   modification, are permitted provided that the following conditions
 *   are met:
 *
 *     * Redistributions of source code must retain the above copyright
 *       notice, this list of conditions and the following disclaimer.
 *     * Redistributions in binary form must reproduce the above copyright
 *       notice, this list of conditions and the following disclaimer in
 *       the documentation and/or other materials provided with the
 *       distribution.
 *     * Neither the name of Intel Corporation nor the names of its
 *       contributors may be used to endorse or promote products derived
 *       from this software without specific prior written permission.
 *
 *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
 *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
 *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
 *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */

#include "spdk/stdinc.h"

#include "env_internal.h"

#include <rte_config.h>
#include <rte_eal_memconfig.h>

#include "spdk_internal/assert.h"
#include "spdk_internal/memory.h"

#include "spdk/assert.h"
#include "spdk/likely.h"
#include "spdk/queue.h"
#include "spdk/util.h"
#include "spdk/env_dpdk.h"

#ifdef __FreeBSD__
#define SPDK_VFIO_ENABLED 0
#else
#include <linux/version.h>
#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 6, 0)
#define SPDK_VFIO_ENABLED 1
#include <linux/vfio.h>
#include <rte_vfio.h>

struct spdk_vfio_dma_map {
        struct vfio_iommu_type1_dma_map map;
        struct vfio_iommu_type1_dma_unmap unmap;
        TAILQ_ENTRY(spdk_vfio_dma_map) tailq;
};

struct vfio_cfg {
        int fd;
        bool enabled;
        bool noiommu_enabled;
        unsigned device_ref;
        TAILQ_HEAD(, spdk_vfio_dma_map) maps;
        pthread_mutex_t mutex;
};

static struct vfio_cfg g_vfio = {
        .fd = -1,
        .enabled = false,
        .noiommu_enabled = false,
        .device_ref = 0,
        .maps = TAILQ_HEAD_INITIALIZER(g_vfio.maps),
        .mutex = PTHREAD_MUTEX_INITIALIZER
};

#else
#define SPDK_VFIO_ENABLED 0
#endif
#endif

#if DEBUG
#define DEBUG_PRINT(...) fprintf(stderr, __VA_ARGS__)
#else
#define DEBUG_PRINT(...)
#endif

#define FN_2MB_TO_4KB(fn)       (fn << (SHIFT_2MB - SHIFT_4KB))
#define FN_4KB_TO_2MB(fn)       (fn >> (SHIFT_2MB - SHIFT_4KB))

#define MAP_256TB_IDX(vfn_2mb)  ((vfn_2mb) >> (SHIFT_1GB - SHIFT_2MB))
#define MAP_1GB_IDX(vfn_2mb)    ((vfn_2mb) & ((1ULL << (SHIFT_1GB - SHIFT_2MB)) - 1))

/* Page is registered */
#define REG_MAP_REGISTERED      (1ULL << 62)

/* A notification region barrier. The 2MB translation entry that's marked
 * with this flag must be unregistered separately. This allows contiguous
 * regions to be unregistered in the same chunks they were registered.
 */
#define REG_MAP_NOTIFY_START    (1ULL << 63)

/* Translation of a single 2MB page. */
struct map_2mb {
        uint64_t translation_2mb;
};

/* Second-level map table indexed by bits [21..29] of the virtual address.
 * Each entry contains the address translation or error for entries that haven't
 * been retrieved yet.
 */
struct map_1gb {
        struct map_2mb map[1ULL << (SHIFT_1GB - SHIFT_2MB)];
};

/* Top-level map table indexed by bits [30..47] of the virtual address.
 * Each entry points to a second-level map table or NULL.
 */
struct map_256tb {
        struct map_1gb *map[1ULL << (SHIFT_256TB - SHIFT_1GB)];
};

/* Page-granularity memory address translation */
struct spdk_mem_map {
        struct map_256tb map_256tb;
        pthread_mutex_t mutex;
        uint64_t default_translation;
        struct spdk_mem_map_ops ops;
        void *cb_ctx;
        TAILQ_ENTRY(spdk_mem_map) tailq;
};

/* Registrations map. The 64 bit translations are bit fields with the
 * following layout (starting with the low bits):
 *    0 - 61 : reserved
 *   62 - 63 : flags
 */
static struct spdk_mem_map *g_mem_reg_map;
static TAILQ_HEAD(, spdk_mem_map) g_spdk_mem_maps = TAILQ_HEAD_INITIALIZER(g_spdk_mem_maps);
static pthread_mutex_t g_spdk_mem_map_mutex = PTHREAD_MUTEX_INITIALIZER;

/*
 * Walk the currently registered memory via the main memory registration map
 * and call the new map's notify callback for each virtually contiguous region.
 */
static int
spdk_mem_map_notify_walk(struct spdk_mem_map *map, enum spdk_mem_map_notify_action action)
{
        size_t idx_256tb;
        uint64_t idx_1gb;
        uint64_t contig_start = UINT64_MAX;
        uint64_t contig_end = UINT64_MAX;
        struct map_1gb *map_1gb;
        int rc;

        if (!g_mem_reg_map) {
                return -EINVAL;
        }

        /* Hold the memory registration map mutex so no new registrations can be added while we are looping. */
        pthread_mutex_lock(&g_mem_reg_map->mutex);

        for (idx_256tb = 0;
             idx_256tb < sizeof(g_mem_reg_map->map_256tb.map) / sizeof(g_mem_reg_map->map_256tb.map[0]);
             idx_256tb++) {
                map_1gb = g_mem_reg_map->map_256tb.map[idx_256tb];

                if (!map_1gb) {
                        if (contig_start != UINT64_MAX) {
                                /* End of of a virtually contiguous range */
                                rc = map->ops.notify_cb(map->cb_ctx, map, action,
                                                        (void *)contig_start,
                                                        contig_end - contig_start + VALUE_2MB);
                                /* Don't bother handling unregister failures. It can't be any worse */
                                if (rc != 0 && action == SPDK_MEM_MAP_NOTIFY_REGISTER) {
                                        goto err_unregister;
                                }
                        }
                        contig_start = UINT64_MAX;
                        continue;
                }

                for (idx_1gb = 0; idx_1gb < sizeof(map_1gb->map) / sizeof(map_1gb->map[0]); idx_1gb++) {
                        if ((map_1gb->map[idx_1gb].translation_2mb & REG_MAP_REGISTERED) &&
                            (contig_start == UINT64_MAX ||
                             (map_1gb->map[idx_1gb].translation_2mb & REG_MAP_NOTIFY_START) == 0)) {
                                /* Rebuild the virtual address from the indexes */
                                uint64_t vaddr = (idx_256tb << SHIFT_1GB) | (idx_1gb << SHIFT_2MB);

                                if (contig_start == UINT64_MAX) {
                                        contig_start = vaddr;
                                }

                                contig_end = vaddr;
                        } else {
                                if (contig_start != UINT64_MAX) {
                                        /* End of of a virtually contiguous range */
                                        rc = map->ops.notify_cb(map->cb_ctx, map, action,
                                                                (void *)contig_start,
                                                                contig_end - contig_start + VALUE_2MB);
                                        /* Don't bother handling unregister failures. It can't be any worse */
                                        if (rc != 0 && action == SPDK_MEM_MAP_NOTIFY_REGISTER) {
                                                goto err_unregister;
                                        }

                                        /* This page might be a part of a neighbour region, so process
                                         * it again. The idx_1gb will be incremented immediately.
                                         */
                                        idx_1gb--;
                                }
                                contig_start = UINT64_MAX;
                        }
                }
        }

        pthread_mutex_unlock(&g_mem_reg_map->mutex);
        return 0;

err_unregister:
        /* Unwind to the first empty translation so we don't unregister
         * a region that just failed to register.
         */
        idx_256tb = MAP_256TB_IDX((contig_start >> SHIFT_2MB) - 1);
        idx_1gb = MAP_1GB_IDX((contig_start >> SHIFT_2MB) - 1);
        contig_start = UINT64_MAX;
        contig_end = UINT64_MAX;

        /* Unregister any memory we managed to register before the failure */
        for (; idx_256tb < SIZE_MAX; idx_256tb--) {
                map_1gb = g_mem_reg_map->map_256tb.map[idx_256tb];

                if (!map_1gb) {
                        if (contig_end != UINT64_MAX) {
                                /* End of of a virtually contiguous range */
                                map->ops.notify_cb(map->cb_ctx, map,
                                                   SPDK_MEM_MAP_NOTIFY_UNREGISTER,
                                                   (void *)contig_start,
                                                   contig_end - contig_start + VALUE_2MB);
                        }
                        contig_end = UINT64_MAX;
                        continue;
                }

                for (; idx_1gb < UINT64_MAX; idx_1gb--) {
                        if ((map_1gb->map[idx_1gb].translation_2mb & REG_MAP_REGISTERED) &&
                            (contig_end == UINT64_MAX || (map_1gb->map[idx_1gb].translation_2mb & REG_MAP_NOTIFY_START) == 0)) {
                                /* Rebuild the virtual address from the indexes */
                                uint64_t vaddr = (idx_256tb << SHIFT_1GB) | (idx_1gb << SHIFT_2MB);

                                if (contig_end == UINT64_MAX) {
                                        contig_end = vaddr;
                                }
                                contig_start = vaddr;
                        } else {
                                if (contig_end != UINT64_MAX) {
                                        /* End of of a virtually contiguous range */
                                        map->ops.notify_cb(map->cb_ctx, map,
                                                           SPDK_MEM_MAP_NOTIFY_UNREGISTER,
                                                           (void *)contig_start,
                                                           contig_end - contig_start + VALUE_2MB);
                                        idx_1gb++;
                                }
                                contig_end = UINT64_MAX;
                        }
                }
                idx_1gb = sizeof(map_1gb->map) / sizeof(map_1gb->map[0]) - 1;
        }

        pthread_mutex_unlock(&g_mem_reg_map->mutex);
        return rc;
}

struct spdk_mem_map *
spdk_mem_map_alloc(uint64_t default_translation, const struct spdk_mem_map_ops *ops, void *cb_ctx)
{
        struct spdk_mem_map *map;
        int rc;

        map = calloc(1, sizeof(*map));
        if (map == NULL) {
                return NULL;
        }

        if (pthread_mutex_init(&map->mutex, NULL)) {
                free(map);
                return NULL;
        }

        map->default_translation = default_translation;
        map->cb_ctx = cb_ctx;
        if (ops) {
                map->ops = *ops;
        }

        if (ops && ops->notify_cb) {
                pthread_mutex_lock(&g_spdk_mem_map_mutex);
                rc = spdk_mem_map_notify_walk(map, SPDK_MEM_MAP_NOTIFY_REGISTER);
                if (rc != 0) {
                        pthread_mutex_unlock(&g_spdk_mem_map_mutex);
                        DEBUG_PRINT("Initial mem_map notify failed\n");
                        pthread_mutex_destroy(&map->mutex);
                        free(map);
                        return NULL;
                }
                TAILQ_INSERT_TAIL(&g_spdk_mem_maps, map, tailq);
                pthread_mutex_unlock(&g_spdk_mem_map_mutex);
        }

        return map;
}

void
spdk_mem_map_free(struct spdk_mem_map **pmap)
{
        struct spdk_mem_map *map;
        size_t i;

        if (!pmap) {
                return;
        }

        map = *pmap;

        if (!map) {
                return;
        }

        if (map->ops.notify_cb) {
                pthread_mutex_lock(&g_spdk_mem_map_mutex);
                spdk_mem_map_notify_walk(map, SPDK_MEM_MAP_NOTIFY_UNREGISTER);
                TAILQ_REMOVE(&g_spdk_mem_maps, map, tailq);
                pthread_mutex_unlock(&g_spdk_mem_map_mutex);
        }

        for (i = 0; i < sizeof(map->map_256tb.map) / sizeof(map->map_256tb.map[0]); i++) {
                free(map->map_256tb.map[i]);
        }

        pthread_mutex_destroy(&map->mutex);

        free(map);
        *pmap = NULL;
}

int
spdk_mem_register(void *vaddr, size_t len)
{
        struct spdk_mem_map *map;
        int rc;
        void *seg_vaddr;
        size_t seg_len;
        uint64_t reg;

        if ((uintptr_t)vaddr & ~MASK_256TB) {
                DEBUG_PRINT("invalid usermode virtual address %p\n", vaddr);
                return -EINVAL;
        }

        if (((uintptr_t)vaddr & MASK_2MB) || (len & MASK_2MB)) {
                DEBUG_PRINT("invalid %s parameters, vaddr=%p len=%ju\n",
                            __func__, vaddr, len);
                return -EINVAL;
        }

        if (len == 0) {
                return 0;
        }

        pthread_mutex_lock(&g_spdk_mem_map_mutex);

        seg_vaddr = vaddr;
        seg_len = len;
        while (seg_len > 0) {
                reg = spdk_mem_map_translate(g_mem_reg_map, (uint64_t)seg_vaddr, NULL);
                if (reg & REG_MAP_REGISTERED) {
                        pthread_mutex_unlock(&g_spdk_mem_map_mutex);
                        return -EBUSY;
                }
                seg_vaddr += VALUE_2MB;
                seg_len -= VALUE_2MB;
        }

        seg_vaddr = vaddr;
        seg_len = 0;
        while (len > 0) {
                spdk_mem_map_set_translation(g_mem_reg_map, (uint64_t)vaddr, VALUE_2MB,
                                             seg_len == 0 ? REG_MAP_REGISTERED | REG_MAP_NOTIFY_START : REG_MAP_REGISTERED);
                seg_len += VALUE_2MB;
                vaddr += VALUE_2MB;
                len -= VALUE_2MB;
        }

        TAILQ_FOREACH(map, &g_spdk_mem_maps, tailq) {
                rc = map->ops.notify_cb(map->cb_ctx, map, SPDK_MEM_MAP_NOTIFY_REGISTER, seg_vaddr, seg_len);
                if (rc != 0) {
                        pthread_mutex_unlock(&g_spdk_mem_map_mutex);
                        return rc;
                }
        }

        pthread_mutex_unlock(&g_spdk_mem_map_mutex);
        return 0;
}

int
spdk_mem_unregister(void *vaddr, size_t len)
{
        struct spdk_mem_map *map;
        int rc;
        void *seg_vaddr;
        size_t seg_len;
        uint64_t reg, newreg;

        if ((uintptr_t)vaddr & ~MASK_256TB) {
                DEBUG_PRINT("invalid usermode virtual address %p\n", vaddr);
                return -EINVAL;
        }

        if (((uintptr_t)vaddr & MASK_2MB) || (len & MASK_2MB)) {
                DEBUG_PRINT("invalid %s parameters, vaddr=%p len=%ju\n",
                            __func__, vaddr, len);
                return -EINVAL;
        }

        pthread_mutex_lock(&g_spdk_mem_map_mutex);

        /* The first page must be a start of a region. Also check if it's
         * registered to make sure we don't return -ERANGE for non-registered
         * regions.
         */
        reg = spdk_mem_map_translate(g_mem_reg_map, (uint64_t)vaddr, NULL);
        if ((reg & REG_MAP_REGISTERED) && (reg & REG_MAP_NOTIFY_START) == 0) {
                pthread_mutex_unlock(&g_spdk_mem_map_mutex);
                return -ERANGE;
        }

        seg_vaddr = vaddr;
        seg_len = len;
        while (seg_len > 0) {
                reg = spdk_mem_map_translate(g_mem_reg_map, (uint64_t)seg_vaddr, NULL);
                if ((reg & REG_MAP_REGISTERED) == 0) {
                        pthread_mutex_unlock(&g_spdk_mem_map_mutex);
                        return -EINVAL;
                }
                seg_vaddr += VALUE_2MB;
                seg_len -= VALUE_2MB;
        }

        newreg = spdk_mem_map_translate(g_mem_reg_map, (uint64_t)seg_vaddr, NULL);
        /* If the next page is registered, it must be a start of a region as well,
         * otherwise we'd be unregistering only a part of a region.
         */
        if ((newreg & REG_MAP_NOTIFY_START) == 0 && (newreg & REG_MAP_REGISTERED)) {
                pthread_mutex_unlock(&g_spdk_mem_map_mutex);
                return -ERANGE;
        }
        seg_vaddr = vaddr;
        seg_len = 0;

        while (len > 0) {
                reg = spdk_mem_map_translate(g_mem_reg_map, (uint64_t)vaddr, NULL);
                spdk_mem_map_set_translation(g_mem_reg_map, (uint64_t)vaddr, VALUE_2MB, 0);

                if (seg_len > 0 && (reg & REG_MAP_NOTIFY_START)) {
                        TAILQ_FOREACH(map, &g_spdk_mem_maps, tailq) {
                                rc = map->ops.notify_cb(map->cb_ctx, map, SPDK_MEM_MAP_NOTIFY_UNREGISTER, seg_vaddr, seg_len);
                                if (rc != 0) {
                                        pthread_mutex_unlock(&g_spdk_mem_map_mutex);
                                        return rc;
                                }
                        }

                        seg_vaddr = vaddr;
                        seg_len = VALUE_2MB;
                } else {
                        seg_len += VALUE_2MB;
                }

                vaddr += VALUE_2MB;
                len -= VALUE_2MB;
        }

        if (seg_len > 0) {
                TAILQ_FOREACH(map, &g_spdk_mem_maps, tailq) {
                        rc = map->ops.notify_cb(map->cb_ctx, map, SPDK_MEM_MAP_NOTIFY_UNREGISTER, seg_vaddr, seg_len);
                        if (rc != 0) {
                                pthread_mutex_unlock(&g_spdk_mem_map_mutex);
                                return rc;
                        }
                }
        }

        pthread_mutex_unlock(&g_spdk_mem_map_mutex);
        return 0;
}

static struct map_1gb *
spdk_mem_map_get_map_1gb(struct spdk_mem_map *map, uint64_t vfn_2mb)
{
        struct map_1gb *map_1gb;
        uint64_t idx_256tb = MAP_256TB_IDX(vfn_2mb);
        size_t i;

        if (spdk_unlikely(idx_256tb >= SPDK_COUNTOF(map->map_256tb.map))) {
                return NULL;
        }

        map_1gb = map->map_256tb.map[idx_256tb];

        if (!map_1gb) {
                pthread_mutex_lock(&map->mutex);

                /* Recheck to make sure nobody else got the mutex first. */
                map_1gb = map->map_256tb.map[idx_256tb];
                if (!map_1gb) {
                        map_1gb = malloc(sizeof(struct map_1gb));
                        if (map_1gb) {
                                /* initialize all entries to default translation */
                                for (i = 0; i < SPDK_COUNTOF(map_1gb->map); i++) {
                                        map_1gb->map[i].translation_2mb = map->default_translation;
                                }
                                map->map_256tb.map[idx_256tb] = map_1gb;
                        }
                }

                pthread_mutex_unlock(&map->mutex);

                if (!map_1gb) {
                        DEBUG_PRINT("allocation failed\n");
                        return NULL;
                }
        }

        return map_1gb;
}

int
spdk_mem_map_set_translation(struct spdk_mem_map *map, uint64_t vaddr, uint64_t size,
                             uint64_t translation)
{
        uint64_t vfn_2mb;
        struct map_1gb *map_1gb;
        uint64_t idx_1gb;
        struct map_2mb *map_2mb;

        if ((uintptr_t)vaddr & ~MASK_256TB) {
                DEBUG_PRINT("invalid usermode virtual address %lu\n", vaddr);
                return -EINVAL;
        }

        /* For now, only 2 MB-aligned registrations are supported */
        if (((uintptr_t)vaddr & MASK_2MB) || (size & MASK_2MB)) {
                DEBUG_PRINT("invalid %s parameters, vaddr=%lu len=%ju\n",
                            __func__, vaddr, size);
                return -EINVAL;
        }

        vfn_2mb = vaddr >> SHIFT_2MB;

        while (size) {
                map_1gb = spdk_mem_map_get_map_1gb(map, vfn_2mb);
                if (!map_1gb) {
                        DEBUG_PRINT("could not get %p map\n", (void *)vaddr);
                        return -ENOMEM;
                }

                idx_1gb = MAP_1GB_IDX(vfn_2mb);
                map_2mb = &map_1gb->map[idx_1gb];
                map_2mb->translation_2mb = translation;

                size -= VALUE_2MB;
                vfn_2mb++;
        }

        return 0;
}

int
spdk_mem_map_clear_translation(struct spdk_mem_map *map, uint64_t vaddr, uint64_t size)
{
        uint64_t vfn_2mb;
        struct map_1gb *map_1gb;
        uint64_t idx_1gb;
        struct map_2mb *map_2mb;

        if ((uintptr_t)vaddr & ~MASK_256TB) {
                DEBUG_PRINT("invalid usermode virtual address %lu\n", vaddr);
                return -EINVAL;
        }

        /* For now, only 2 MB-aligned registrations are supported */
        if (((uintptr_t)vaddr & MASK_2MB) || (size & MASK_2MB)) {
                DEBUG_PRINT("invalid %s parameters, vaddr=%lu len=%ju\n",
                            __func__, vaddr, size);
                return -EINVAL;
        }

        vfn_2mb = vaddr >> SHIFT_2MB;

        while (size) {
                map_1gb = spdk_mem_map_get_map_1gb(map, vfn_2mb);
                if (!map_1gb) {
                        DEBUG_PRINT("could not get %p map\n", (void *)vaddr);
                        return -ENOMEM;
                }

                idx_1gb = MAP_1GB_IDX(vfn_2mb);
                map_2mb = &map_1gb->map[idx_1gb];
                map_2mb->translation_2mb = map->default_translation;

                size -= VALUE_2MB;
                vfn_2mb++;
        }

        return 0;
}

inline uint64_t
spdk_mem_map_translate(const struct spdk_mem_map *map, uint64_t vaddr, uint64_t *size)
{
        const struct map_1gb *map_1gb;
        const struct map_2mb *map_2mb;
        uint64_t idx_256tb;
        uint64_t idx_1gb;
        uint64_t vfn_2mb;
        uint64_t cur_size;
        uint64_t prev_translation;
        uint64_t orig_translation;

        if (spdk_unlikely(vaddr & ~MASK_256TB)) {
                DEBUG_PRINT("invalid usermode virtual address %p\n", (void *)vaddr);
                return map->default_translation;
        }

        vfn_2mb = vaddr >> SHIFT_2MB;
        idx_256tb = MAP_256TB_IDX(vfn_2mb);
        idx_1gb = MAP_1GB_IDX(vfn_2mb);

        map_1gb = map->map_256tb.map[idx_256tb];
        if (spdk_unlikely(!map_1gb)) {
                return map->default_translation;
        }

        cur_size = VALUE_2MB - _2MB_OFFSET(vaddr);
        map_2mb = &map_1gb->map[idx_1gb];
        if (size == NULL || map->ops.are_contiguous == NULL ||
            map_2mb->translation_2mb == map->default_translation) {
                if (size != NULL) {
                        *size = spdk_min(*size, cur_size);
                }
                return map_2mb->translation_2mb;
        }

        orig_translation = map_2mb->translation_2mb;
        prev_translation = orig_translation;
        while (cur_size < *size) {
                vfn_2mb++;
                idx_256tb = MAP_256TB_IDX(vfn_2mb);
                idx_1gb = MAP_1GB_IDX(vfn_2mb);

                map_1gb = map->map_256tb.map[idx_256tb];
                if (spdk_unlikely(!map_1gb)) {
                        break;
                }

                map_2mb = &map_1gb->map[idx_1gb];
                if (!map->ops.are_contiguous(prev_translation, map_2mb->translation_2mb)) {
                        break;
                }

                cur_size += VALUE_2MB;
                prev_translation = map_2mb->translation_2mb;
        }

        *size = spdk_min(*size, cur_size);
        return orig_translation;
}

#if RTE_VERSION >= RTE_VERSION_NUM(18, 05, 0, 0)
static void
memory_hotplug_cb(enum rte_mem_event event_type,
                  const void *addr, size_t len, void *arg)
{
        if (event_type == RTE_MEM_EVENT_ALLOC) {
                spdk_mem_register((void *)addr, len);

#if RTE_VERSION >= RTE_VERSION_NUM(19, 02, 0, 0)
                if (!spdk_env_dpdk_external_init()) {
                        return;
                }
#endif

                /* Prior to DPDK 19.02, we have to worry about DPDK
                 * freeing memory in different units than it was allocated.
                 * That doesn't work with things like RDMA MRs.  So for
                 * those versions of DPDK, mark each segment so that DPDK
                 * won't later free it.  That ensures we don't have to deal
                 * with that scenario.
                 *
                 * DPDK 19.02 added the --match-allocations RTE flag to
                 * avoid this condition.
                 *
                 * Note: if the user initialized DPDK separately, we can't
                 * be sure that --match-allocations was specified, so need
                 * to still mark the segments so they aren't freed.
                 */
                while (len > 0) {
                        struct rte_memseg *seg;

                        seg = rte_mem_virt2memseg(addr, NULL);
                        assert(seg != NULL);
                        seg->flags |= RTE_MEMSEG_FLAG_DO_NOT_FREE;
                        addr = (void *)((uintptr_t)addr + seg->hugepage_sz);
                        len -= seg->hugepage_sz;
                }
        } else if (event_type == RTE_MEM_EVENT_FREE) {
                spdk_mem_unregister((void *)addr, len);
        }
}

static int
memory_iter_cb(const struct rte_memseg_list *msl,
               const struct rte_memseg *ms, size_t len, void *arg)
{
        return spdk_mem_register(ms->addr, len);
}
#endif

int
spdk_mem_map_init(void)
{
        g_mem_reg_map = spdk_mem_map_alloc(0, NULL, NULL);
        if (g_mem_reg_map == NULL) {
                DEBUG_PRINT("memory registration map allocation failed\n");
                return -1;
        }

        /*
         * Walk all DPDK memory segments and register them
         * with the master memory map
         */
#if RTE_VERSION >= RTE_VERSION_NUM(18, 05, 0, 0)
        rte_mem_event_callback_register("spdk", memory_hotplug_cb, NULL);
        rte_memseg_contig_walk(memory_iter_cb, NULL);
#else
        struct rte_mem_config *mcfg;
        size_t seg_idx;

        mcfg = rte_eal_get_configuration()->mem_config;
        for (seg_idx = 0; seg_idx < RTE_MAX_MEMSEG; seg_idx++) {
                struct rte_memseg *seg = &mcfg->memseg[seg_idx];

                if (seg->addr == NULL) {
                        break;
                }

                spdk_mem_register(seg->addr, seg->len);
        }
#endif
        return 0;
}

bool
spdk_iommu_is_enabled(void)
{
#if SPDK_VFIO_ENABLED
        return g_vfio.enabled && !g_vfio.noiommu_enabled;
#else
        return false;
#endif
}

struct spdk_vtophys_pci_device {
        struct rte_pci_device *pci_device;
        TAILQ_ENTRY(spdk_vtophys_pci_device) tailq;
};

static pthread_mutex_t g_vtophys_pci_devices_mutex = PTHREAD_MUTEX_INITIALIZER;
static TAILQ_HEAD(, spdk_vtophys_pci_device) g_vtophys_pci_devices =
        TAILQ_HEAD_INITIALIZER(g_vtophys_pci_devices);

static struct spdk_mem_map *g_vtophys_map;

#if SPDK_VFIO_ENABLED
static int
vtophys_iommu_map_dma(uint64_t vaddr, uint64_t iova, uint64_t size)
{
        struct spdk_vfio_dma_map *dma_map;
        int ret;

        dma_map = calloc(1, sizeof(*dma_map));
        if (dma_map == NULL) {
                return -ENOMEM;
        }

        dma_map->map.argsz = sizeof(dma_map->map);
        dma_map->map.flags = VFIO_DMA_MAP_FLAG_READ | VFIO_DMA_MAP_FLAG_WRITE;
        dma_map->map.vaddr = vaddr;
        dma_map->map.iova = iova;
        dma_map->map.size = size;

        dma_map->unmap.argsz = sizeof(dma_map->unmap);
        dma_map->unmap.flags = 0;
        dma_map->unmap.iova = iova;
        dma_map->unmap.size = size;

        pthread_mutex_lock(&g_vfio.mutex);
        if (g_vfio.device_ref == 0) {
                /* VFIO requires at least one device (IOMMU group) to be added to
                 * a VFIO container before it is possible to perform any IOMMU
                 * operations on that container. This memory will be mapped once
                 * the first device (IOMMU group) is hotplugged.
                 *
                 * Since the vfio container is managed internally by DPDK, it is
                 * also possible that some device is already in that container, but
                 * it's not managed by SPDK -  e.g. an NIC attached internally
                 * inside DPDK. We could map the memory straight away in such
                 * scenario, but there's no need to do it. DPDK devices clearly
                 * don't need our mappings and hence we defer the mapping
                 * unconditionally until the first SPDK-managed device is
                 * hotplugged.
                 */
                goto out_insert;
        }

        ret = ioctl(g_vfio.fd, VFIO_IOMMU_MAP_DMA, &dma_map->map);
        if (ret) {
                DEBUG_PRINT("Cannot set up DMA mapping, error %d\n", errno);
                pthread_mutex_unlock(&g_vfio.mutex);
                free(dma_map);
                return ret;
        }

out_insert:
        TAILQ_INSERT_TAIL(&g_vfio.maps, dma_map, tailq);
        pthread_mutex_unlock(&g_vfio.mutex);
        return 0;
}

static int
vtophys_iommu_unmap_dma(uint64_t iova, uint64_t size)
{
        struct spdk_vfio_dma_map *dma_map;
        int ret;

        pthread_mutex_lock(&g_vfio.mutex);
        TAILQ_FOREACH(dma_map, &g_vfio.maps, tailq) {
                if (dma_map->map.iova == iova) {
                        break;
                }
        }

        if (dma_map == NULL) {
                DEBUG_PRINT("Cannot clear DMA mapping for IOVA %"PRIx64" - it's not mapped\n", iova);
                pthread_mutex_unlock(&g_vfio.mutex);
                return -ENXIO;
        }

        /** don't support partial or multiple-page unmap for now */
        assert(dma_map->map.size == size);

        if (g_vfio.device_ref == 0) {
                /* Memory is not mapped anymore, just remove it's references */
                goto out_remove;
        }


        ret = ioctl(g_vfio.fd, VFIO_IOMMU_UNMAP_DMA, &dma_map->unmap);
        if (ret) {
                DEBUG_PRINT("Cannot clear DMA mapping, error %d\n", errno);
                pthread_mutex_unlock(&g_vfio.mutex);
                return ret;
        }

out_remove:
        TAILQ_REMOVE(&g_vfio.maps, dma_map, tailq);
        pthread_mutex_unlock(&g_vfio.mutex);
        free(dma_map);
        return 0;
}
#endif

static uint64_t
vtophys_get_paddr_memseg(uint64_t vaddr)
{
        uintptr_t paddr;
        struct rte_memseg *seg;

#if RTE_VERSION >= RTE_VERSION_NUM(18, 05, 0, 0)
        seg = rte_mem_virt2memseg((void *)(uintptr_t)vaddr, NULL);
        if (seg != NULL) {
                paddr = seg->phys_addr;
                if (paddr == RTE_BAD_IOVA) {
                        return SPDK_VTOPHYS_ERROR;
                }
                paddr += (vaddr - (uintptr_t)seg->addr);
                return paddr;
        }
#else
        struct rte_mem_config *mcfg;
        uint32_t seg_idx;

        mcfg = rte_eal_get_configuration()->mem_config;
        for (seg_idx = 0; seg_idx < RTE_MAX_MEMSEG; seg_idx++) {
                seg = &mcfg->memseg[seg_idx];
                if (seg->addr == NULL) {
                        break;
                }

                if (vaddr >= (uintptr_t)seg->addr &&
                    vaddr < ((uintptr_t)seg->addr + seg->len)) {
                        paddr = seg->phys_addr;
                        if (paddr == RTE_BAD_IOVA) {
                                return SPDK_VTOPHYS_ERROR;
                        }
                        paddr += (vaddr - (uintptr_t)seg->addr);
                        return paddr;
                }
        }
#endif

        return SPDK_VTOPHYS_ERROR;
}

/* Try to get the paddr from /proc/self/pagemap */
static uint64_t
vtophys_get_paddr_pagemap(uint64_t vaddr)
{
        uintptr_t paddr;

        paddr = rte_mem_virt2iova((void *)vaddr);
        if (paddr == RTE_BAD_IOVA) {
                /*
                 * The vaddr may be valid but doesn't have a backing page
                 * assigned yet.  Touch the page to ensure a backing page
                 * gets assigned, then try to translate again.
                 */
                rte_atomic64_read((rte_atomic64_t *)vaddr);
                paddr = rte_mem_virt2iova((void *)vaddr);
        }
        if (paddr == RTE_BAD_IOVA) {
                /* Unable to get to the physical address. */
                return SPDK_VTOPHYS_ERROR;
        }

        return paddr;
}

/* Try to get the paddr from pci devices */
static uint64_t
vtophys_get_paddr_pci(uint64_t vaddr)
{
        struct spdk_vtophys_pci_device *vtophys_dev;
        uintptr_t paddr;
        struct rte_pci_device   *dev;
        struct rte_mem_resource *res;
        unsigned r;

        pthread_mutex_lock(&g_vtophys_pci_devices_mutex);
        TAILQ_FOREACH(vtophys_dev, &g_vtophys_pci_devices, tailq) {
                dev = vtophys_dev->pci_device;

                for (r = 0; r < PCI_MAX_RESOURCE; r++) {
                        res = &dev->mem_resource[r];
                        if (res->phys_addr && vaddr >= (uint64_t)res->addr &&
                            vaddr < (uint64_t)res->addr + res->len) {
                                paddr = res->phys_addr + (vaddr - (uint64_t)res->addr);
                                DEBUG_PRINT("%s: %p -> %p\n", __func__, (void *)vaddr,
                                            (void *)paddr);
                                pthread_mutex_unlock(&g_vtophys_pci_devices_mutex);
                                return paddr;
                        }
                }
        }
        pthread_mutex_unlock(&g_vtophys_pci_devices_mutex);

        return  SPDK_VTOPHYS_ERROR;
}

static int
spdk_vtophys_notify(void *cb_ctx, struct spdk_mem_map *map,
                    enum spdk_mem_map_notify_action action,
                    void *vaddr, size_t len)
{
        int rc = 0, pci_phys = 0;
        uint64_t paddr;

        if ((uintptr_t)vaddr & ~MASK_256TB) {
                DEBUG_PRINT("invalid usermode virtual address %p\n", vaddr);
                return -EINVAL;
        }

        if (((uintptr_t)vaddr & MASK_2MB) || (len & MASK_2MB)) {
                DEBUG_PRINT("invalid %s parameters, vaddr=%p len=%ju\n",
                            __func__, vaddr, len);
                return -EINVAL;
        }

        while (len > 0) {
                /* Get the physical address from the DPDK memsegs */
                paddr = vtophys_get_paddr_memseg((uint64_t)vaddr);

                switch (action) {
                case SPDK_MEM_MAP_NOTIFY_REGISTER:
                        if (paddr == SPDK_VTOPHYS_ERROR) {
                                /* This is not an address that DPDK is managing. */
#if SPDK_VFIO_ENABLED
                                if (spdk_iommu_is_enabled()) {
                                        /* We'll use the virtual address as the iova. DPDK
                                         * currently uses physical addresses as the iovas (or counts
                                         * up from 0 if it can't get physical addresses), so
                                         * the range of user space virtual addresses and physical
                                         * addresses will never overlap.
                                         */
                                        paddr = (uint64_t)vaddr;
                                        rc = vtophys_iommu_map_dma((uint64_t)vaddr, paddr, VALUE_2MB);
                                        if (rc) {
                                                return -EFAULT;
                                        }
                                } else
#endif
                                {
                                        /* Get the physical address from /proc/self/pagemap. */
                                        paddr = vtophys_get_paddr_pagemap((uint64_t)vaddr);
                                        if (paddr == SPDK_VTOPHYS_ERROR) {
                                                /* Get the physical address from PCI devices */
                                                paddr = vtophys_get_paddr_pci((uint64_t)vaddr);
                                                if (paddr == SPDK_VTOPHYS_ERROR) {
                                                        DEBUG_PRINT("could not get phys addr for %p\n", vaddr);
                                                        return -EFAULT;
                                                }
                                                pci_phys = 1;
                                        }
                                }
                        }
                        /* Since PCI paddr can break the 2MiB physical alignment skip this check for that. */
                        if (!pci_phys && (paddr & MASK_2MB)) {
                                DEBUG_PRINT("invalid paddr 0x%" PRIx64 " - must be 2MB aligned\n", paddr);
                                return -EINVAL;
                        }

                        rc = spdk_mem_map_set_translation(map, (uint64_t)vaddr, VALUE_2MB, paddr);
                        break;
                case SPDK_MEM_MAP_NOTIFY_UNREGISTER:
#if SPDK_VFIO_ENABLED
                        if (paddr == SPDK_VTOPHYS_ERROR) {
                                /*
                                 * This is not an address that DPDK is managing. If vfio is enabled,
                                 * we need to unmap the range from the IOMMU
                                 */
                                if (spdk_iommu_is_enabled()) {
                                        uint64_t buffer_len = VALUE_2MB;
                                        paddr = spdk_mem_map_translate(map, (uint64_t)vaddr, &buffer_len);
                                        if (buffer_len != VALUE_2MB) {
                                                return -EINVAL;
                                        }
                                        rc = vtophys_iommu_unmap_dma(paddr, VALUE_2MB);
                                        if (rc) {
                                                return -EFAULT;
                                        }
                                }
                        }
#endif
                        rc = spdk_mem_map_clear_translation(map, (uint64_t)vaddr, VALUE_2MB);
                        break;
                default:
                        SPDK_UNREACHABLE();
                }

                if (rc != 0) {
                        return rc;
                }
                vaddr += VALUE_2MB;
                len -= VALUE_2MB;
        }

        return rc;
}

#if SPDK_VFIO_ENABLED

static bool
spdk_vfio_enabled(void)
{
        return rte_vfio_is_enabled("vfio_pci");
}

/* Check if IOMMU is enabled on the system */
static bool
has_iommu_groups(void)
{
        struct dirent *d;
        int count = 0;
        DIR *dir = opendir("/sys/kernel/iommu_groups");

        if (dir == NULL) {
                return false;
        }

        while (count < 3 && (d = readdir(dir)) != NULL) {
                count++;
        }

        closedir(dir);
        /* there will always be ./ and ../ entries */
        return count > 2;
}

static bool
spdk_vfio_noiommu_enabled(void)
{
        return rte_vfio_noiommu_is_enabled();
}

static void
spdk_vtophys_iommu_init(void)
{
        char proc_fd_path[PATH_MAX + 1];
        char link_path[PATH_MAX + 1];
        const char vfio_path[] = "/dev/vfio/vfio";
        DIR *dir;
        struct dirent *d;

        if (!spdk_vfio_enabled()) {
                return;
        }

        if (spdk_vfio_noiommu_enabled()) {
                g_vfio.noiommu_enabled = true;
        } else if (!has_iommu_groups()) {
                return;
        }

        dir = opendir("/proc/self/fd");
        if (!dir) {
                DEBUG_PRINT("Failed to open /proc/self/fd (%d)\n", errno);
                return;
        }

        while ((d = readdir(dir)) != NULL) {
                if (d->d_type != DT_LNK) {
                        continue;
                }

                snprintf(proc_fd_path, sizeof(proc_fd_path), "/proc/self/fd/%s", d->d_name);
                if (readlink(proc_fd_path, link_path, sizeof(link_path)) != (sizeof(vfio_path) - 1)) {
                        continue;
                }

                if (memcmp(link_path, vfio_path, sizeof(vfio_path) - 1) == 0) {
                        sscanf(d->d_name, "%d", &g_vfio.fd);
                        break;
                }
        }

        closedir(dir);

        if (g_vfio.fd < 0) {
                DEBUG_PRINT("Failed to discover DPDK VFIO container fd.\n");
                return;
        }

        g_vfio.enabled = true;

        return;
}
#endif

void
spdk_vtophys_pci_device_added(struct rte_pci_device *pci_device)
{
        struct spdk_vtophys_pci_device *vtophys_dev;

        pthread_mutex_lock(&g_vtophys_pci_devices_mutex);

        vtophys_dev = calloc(1, sizeof(*vtophys_dev));
        if (vtophys_dev) {
                vtophys_dev->pci_device = pci_device;
                TAILQ_INSERT_TAIL(&g_vtophys_pci_devices, vtophys_dev, tailq);
        } else {
                DEBUG_PRINT("Memory allocation error\n");
        }
        pthread_mutex_unlock(&g_vtophys_pci_devices_mutex);

#if SPDK_VFIO_ENABLED
        struct spdk_vfio_dma_map *dma_map;
        int ret;

        if (!g_vfio.enabled) {
                return;
        }

        pthread_mutex_lock(&g_vfio.mutex);
        g_vfio.device_ref++;
        if (g_vfio.device_ref > 1) {
                pthread_mutex_unlock(&g_vfio.mutex);
                return;
        }

        /* This is the first SPDK device using DPDK vfio. This means that the first
         * IOMMU group might have been just been added to the DPDK vfio container.
         * From this point it is certain that the memory can be mapped now.
         */
        TAILQ_FOREACH(dma_map, &g_vfio.maps, tailq) {
                ret = ioctl(g_vfio.fd, VFIO_IOMMU_MAP_DMA, &dma_map->map);
                if (ret) {
                        DEBUG_PRINT("Cannot update DMA mapping, error %d\n", errno);
                        break;
                }
        }
        pthread_mutex_unlock(&g_vfio.mutex);
#endif
}

void
spdk_vtophys_pci_device_removed(struct rte_pci_device *pci_device)
{
        struct spdk_vtophys_pci_device *vtophys_dev;

        pthread_mutex_lock(&g_vtophys_pci_devices_mutex);
        TAILQ_FOREACH(vtophys_dev, &g_vtophys_pci_devices, tailq) {
                if (vtophys_dev->pci_device == pci_device) {
                        TAILQ_REMOVE(&g_vtophys_pci_devices, vtophys_dev, tailq);
                        free(vtophys_dev);
                        break;
                }
        }
        pthread_mutex_unlock(&g_vtophys_pci_devices_mutex);

#if SPDK_VFIO_ENABLED
        struct spdk_vfio_dma_map *dma_map;
        int ret;

        if (!g_vfio.enabled) {
                return;
        }

        pthread_mutex_lock(&g_vfio.mutex);
        assert(g_vfio.device_ref > 0);
        g_vfio.device_ref--;
        if (g_vfio.device_ref > 0) {
                pthread_mutex_unlock(&g_vfio.mutex);
                return;
        }

        /* This is the last SPDK device using DPDK vfio. If DPDK doesn't have
         * any additional devices using it's vfio container, all the mappings
         * will be automatically removed by the Linux vfio driver. We unmap
         * the memory manually to be able to easily re-map it later regardless
         * of other, external factors.
         */
        TAILQ_FOREACH(dma_map, &g_vfio.maps, tailq) {
                ret = ioctl(g_vfio.fd, VFIO_IOMMU_UNMAP_DMA, &dma_map->unmap);
                if (ret) {
                        DEBUG_PRINT("Cannot unmap DMA memory, error %d\n", errno);
                        break;
                }
        }
        pthread_mutex_unlock(&g_vfio.mutex);
#endif
}

int
spdk_vtophys_init(void)
{
        const struct spdk_mem_map_ops vtophys_map_ops = {
                .notify_cb = spdk_vtophys_notify,
                .are_contiguous = NULL
        };

#if SPDK_VFIO_ENABLED
        spdk_vtophys_iommu_init();
#endif

        g_vtophys_map = spdk_mem_map_alloc(SPDK_VTOPHYS_ERROR, &vtophys_map_ops, NULL);
        if (g_vtophys_map == NULL) {
                DEBUG_PRINT("vtophys map allocation failed\n");
                return -1;
        }
        return 0;
}

uint64_t
spdk_vtophys(void *buf, uint64_t *size)
{
        uint64_t vaddr, paddr_2mb;

        vaddr = (uint64_t)buf;
        paddr_2mb = spdk_mem_map_translate(g_vtophys_map, vaddr, size);

        /*
         * SPDK_VTOPHYS_ERROR has all bits set, so if the lookup returned SPDK_VTOPHYS_ERROR,
         * we will still bitwise-or it with the buf offset below, but the result will still be
         * SPDK_VTOPHYS_ERROR. However now that we do + rather than | (due to PCI vtophys being
         * unaligned) we must now check the return value before addition.
         */
        SPDK_STATIC_ASSERT(SPDK_VTOPHYS_ERROR == UINT64_C(-1), "SPDK_VTOPHYS_ERROR should be all 1s");
        if (paddr_2mb == SPDK_VTOPHYS_ERROR) {
                return SPDK_VTOPHYS_ERROR;
        } else {
                return paddr_2mb + (vaddr & MASK_2MB);
        }
}