ceph/src/dpdk/drivers/net/xenvirt/rte_mempool_gntalloc.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) 2010-2014 Intel Corporation. All rights reserved.
   5  *   All rights reserved.
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *     * Redistributions of source code must retain the above copyright
  12  *       notice, this list of conditions and the following disclaimer.
  13  *     * Redistributions in binary form must reproduce the above copyright
  14  *       notice, this list of conditions and the following disclaimer in
  15  *       the documentation and/or other materials provided with the
  16  *       distribution.
  17  *     * Neither the name of Intel Corporation nor the names of its
  18  *       contributors may be used to endorse or promote products derived
  19  *       from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33
  34 #include <stdint.h>
  35 #include <unistd.h>
  36 #include <stdlib.h>
  37 #include <sys/mman.h>
  38 #include <sys/ioctl.h>
  39 #include <string.h>
  40 #include <xen/sys/gntalloc.h>
  41
  42 #include <rte_common.h>
  43 #include <rte_mempool.h>
  44 #include <rte_memory.h>
  45 #include <rte_errno.h>
  46
  47 #include "rte_xen_lib.h"
  48 #include "rte_eth_xenvirt.h"
  49
  50 struct _gntarr {
  51         uint32_t gref;
  52         phys_addr_t pa;
  53         uint64_t index;
  54         void *va;
  55 };
  56
  57 struct _mempool_gntalloc_info {
  58         struct rte_mempool *mp;
  59         uint32_t pg_num;
  60         uint32_t *gref_arr;
  61         phys_addr_t *pa_arr;
  62         void *va;
  63         uint32_t mempool_idx;
  64         uint64_t start_index;
  65 };
  66
  67
  68 static rte_atomic32_t global_xenvirt_mempool_idx = RTE_ATOMIC32_INIT(-1);
  69
  70 static int
  71 compare(const void *p1, const void *p2)
  72 {
  73         return ((const struct _gntarr *)p1)->pa  - ((const struct _gntarr *)p2)->pa;
  74 }
  75
  76
  77 static struct _mempool_gntalloc_info
  78 _create_mempool(const char *name, unsigned elt_num, unsigned elt_size,
  79                    unsigned cache_size, unsigned private_data_size,
  80                    rte_mempool_ctor_t *mp_init, void *mp_init_arg,
  81                    rte_mempool_obj_cb_t *obj_init, void *obj_init_arg,
  82                    int socket_id, unsigned flags)
  83 {
  84         struct _mempool_gntalloc_info mgi;
  85         struct rte_mempool *mp = NULL;
  86         struct rte_mempool_objsz  objsz;
  87         uint32_t pg_num, rpg_num, pg_shift, pg_sz;
  88         char *va, *orig_va, *uv; /* uv: from which, the pages could be freed */
  89         ssize_t sz, usz; /* usz: unused size */
  90         /*
  91          * for each page allocated through xen_gntalloc driver,
  92          * gref_arr:stores grant references,
  93          * pa_arr: stores physical address,
  94          * gnt_arr: stores all meta dat
  95          */
  96         uint32_t *gref_arr = NULL;
  97         phys_addr_t *pa_arr = NULL;
  98         struct _gntarr *gnt_arr = NULL;
  99         /* start index of the grant referances, used for dealloc*/
 100         uint64_t start_index;
 101         uint32_t i, j;
 102         int rv = 0;
 103         struct ioctl_gntalloc_dealloc_gref arg;
 104
 105         mgi.mp = NULL;
 106         va = orig_va = uv = NULL;
 107         pg_num = rpg_num = 0;
 108         sz = 0;
 109
 110         pg_sz = getpagesize();
 111         if (rte_is_power_of_2(pg_sz) == 0) {
 112                 goto out;
 113         }
 114         pg_shift = rte_bsf32(pg_sz);
 115
 116         rte_mempool_calc_obj_size(elt_size, flags, &objsz);
 117         sz = rte_mempool_xmem_size(elt_num, objsz.total_size, pg_shift);
 118         pg_num = sz >> pg_shift;
 119
 120         pa_arr = calloc(pg_num, sizeof(pa_arr[0]));
 121         gref_arr = calloc(pg_num, sizeof(gref_arr[0]));
 122         gnt_arr  = calloc(pg_num, sizeof(gnt_arr[0]));
 123         if ((gnt_arr == NULL) || (gref_arr == NULL) || (pa_arr == NULL))
 124                 goto out;
 125
 126         /* grant index is continuous in ascending order */
 127         orig_va = gntalloc(sz, gref_arr, &start_index);
 128         if (orig_va == NULL)
 129                 goto out;
 130
 131         get_phys_map(orig_va, pa_arr, pg_num, pg_sz);
 132         for (i = 0; i < pg_num; i++) {
 133                 gnt_arr[i].index = start_index + i * pg_sz;
 134                 gnt_arr[i].gref = gref_arr[i];
 135                 gnt_arr[i].pa = pa_arr[i];
 136                 gnt_arr[i].va  = RTE_PTR_ADD(orig_va, i * pg_sz);
 137         }
 138         qsort(gnt_arr, pg_num, sizeof(struct _gntarr), compare);
 139
 140         va = get_xen_virtual(sz, pg_sz);
 141         if (va == NULL) {
 142                 goto out;
 143         }
 144
 145         /*
 146          * map one by one, as index isn't continuous now.
 147          * pg_num VMAs, doesn't linux has a limitation on this?
 148          */
 149         for (i = 0; i < pg_num; i++) {
 150         /* update gref_arr and pa_arr after sort */
 151                 gref_arr[i] = gnt_arr[i].gref;
 152                 pa_arr[i]   = gnt_arr[i].pa;
 153                 gnt_arr[i].va = mmap(va + i * pg_sz, pg_sz, PROT_READ | PROT_WRITE,
 154                         MAP_SHARED | MAP_FIXED, gntalloc_fd, gnt_arr[i].index);
 155                 if ((gnt_arr[i].va == MAP_FAILED) || (gnt_arr[i].va != (va + i * pg_sz))) {
 156                         RTE_LOG(ERR, PMD, "failed to map %d pages\n", i);
 157                         goto mmap_failed;
 158                 }
 159         }
 160
 161         /*
 162          * Check that allocated size is big enough to hold elt_num
 163          * objects and a calcualte how many bytes are actually required.
 164          */
 165         usz = rte_mempool_xmem_usage(va, elt_num, objsz.total_size, pa_arr, pg_num, pg_shift);
 166         if (usz < 0) {
 167                 mp = NULL;
 168                 i = pg_num;
 169                 goto mmap_failed;
 170         } else {
 171                 /* unmap unused pages if any */
 172                 uv = RTE_PTR_ADD(va, usz);
 173                 if ((usz = va + sz - uv) > 0) {
 174
 175                         RTE_LOG(ERR, PMD,
 176                                 "%s(%s): unmap unused %zu of %zu "
 177                                 "mmaped bytes @%p orig:%p\n",
 178                                 __func__, name, usz, sz, uv, va);
 179                         munmap(uv, usz);
 180                         i = (sz - usz) / pg_sz;
 181                         for (; i < pg_num; i++) {
 182                                 arg.count = 1;
 183                                 arg.index = gnt_arr[i].index;
 184                                 rv = ioctl(gntalloc_fd, IOCTL_GNTALLOC_DEALLOC_GREF, &arg);
 185                                 if (rv) {
 186                                         /* shouldn't fail here */
 187                                         RTE_LOG(ERR, PMD, "va=%p pa=%"PRIu64"x index=%"PRIu64" %s\n",
 188                                                 gnt_arr[i].va,
 189                                                 gnt_arr[i].pa,
 190                                                 arg.index, strerror(errno));
 191                                         rte_panic("gntdealloc failed when freeing pages\n");
 192                                 }
 193                         }
 194
 195                         rpg_num = (sz - usz) >> pg_shift;
 196                 } else
 197                         rpg_num = pg_num;
 198
 199                 mp = rte_mempool_xmem_create(name, elt_num, elt_size,
 200                                 cache_size, private_data_size,
 201                                 mp_init, mp_init_arg,
 202                                 obj_init, obj_init_arg,
 203                                 socket_id, flags, va, pa_arr, rpg_num, pg_shift);
 204
 205                 RTE_ASSERT(elt_num == mp->size);
 206         }
 207         mgi.mp = mp;
 208         mgi.pg_num = rpg_num;
 209         mgi.gref_arr = gref_arr;
 210         mgi.pa_arr = pa_arr;
 211         if (mp)
 212                 mgi.mempool_idx = rte_atomic32_add_return(&global_xenvirt_mempool_idx, 1);
 213         mgi.start_index = start_index;
 214         mgi.va = va;
 215
 216         if (mp == NULL) {
 217                 i = pg_num;
 218                 goto mmap_failed;
 219         }
 220
 221 /*
 222  * unmap only, without deallocate grant reference.
 223  * unused pages have already been unmaped,
 224  * unmap twice will fail, but it is safe.
 225  */
 226 mmap_failed:
 227         for (j = 0; j < i; j++) {
 228                 if (gnt_arr[i].va)
 229                         munmap(gnt_arr[i].va, pg_sz);
 230         }
 231 out:
 232         free(gnt_arr);
 233         if (orig_va)
 234                 munmap(orig_va, sz);
 235         if (mp == NULL) {
 236                 free(gref_arr);
 237                 free(pa_arr);
 238
 239                 /* some gref has already been de-allocated from the list in the driver,
 240                  * so dealloc one by one, and it is safe to deallocate twice
 241                  */
 242                 if (orig_va) {
 243                         for (i = 0; i < pg_num; i++) {
 244                                 arg.index = start_index + i * pg_sz;
 245                                 rv = ioctl(gntalloc_fd, IOCTL_GNTALLOC_DEALLOC_GREF, arg);
 246                         }
 247                 }
 248         }
 249         return mgi;
 250 }
 251
 252 struct rte_mempool *
 253 rte_mempool_gntalloc_create(const char *name, unsigned elt_num, unsigned elt_size,
 254                    unsigned cache_size, unsigned private_data_size,
 255                    rte_mempool_ctor_t *mp_init, void *mp_init_arg,
 256                    rte_mempool_obj_cb_t *obj_init, void *obj_init_arg,
 257                    int socket_id, unsigned flags)
 258 {
 259         int rv;
 260         uint32_t i;
 261         struct _mempool_gntalloc_info mgi;
 262         struct ioctl_gntalloc_dealloc_gref arg;
 263         int pg_sz = getpagesize();
 264
 265         mgi = _create_mempool(name, elt_num, elt_size,
 266                         cache_size, private_data_size,
 267                         mp_init, mp_init_arg,
 268                         obj_init, obj_init_arg,
 269                         socket_id, flags);
 270         if (mgi.mp) {
 271                 rv = grant_gntalloc_mbuf_pool(mgi.mp,
 272                         mgi.pg_num,
 273                         mgi.gref_arr,
 274                         mgi.pa_arr,
 275                         mgi.mempool_idx);
 276                 free(mgi.gref_arr);
 277                 free(mgi.pa_arr);
 278                 if (rv == 0)
 279                         return mgi.mp;
 280                 /*
 281                  * in _create_mempool, unused pages have already been unmapped, deallocagted
 282                  * unmap and dealloc the remained ones here.
 283                  */
 284                 munmap(mgi.va, pg_sz * mgi.pg_num);
 285                 for (i = 0; i < mgi.pg_num; i++) {
 286                         arg.index = mgi.start_index + i * pg_sz;
 287                         rv = ioctl(gntalloc_fd, IOCTL_GNTALLOC_DEALLOC_GREF, arg);
 288                 }
 289                 return NULL;
 290         }
 291         return NULL;
 292
 293
 294
 295 }