ceph/src/seastar/dpdk/drivers/net/i40e/base/i40e_hmc.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2001-2018
   3  */
   4
   5 #include "i40e_osdep.h"
   6 #include "i40e_register.h"
   7 #include "i40e_status.h"
   8 #include "i40e_alloc.h"
   9 #include "i40e_hmc.h"
  10 #include "i40e_type.h"
  11
  12 /**
  13  * i40e_add_sd_table_entry - Adds a segment descriptor to the table
  14  * @hw: pointer to our hw struct
  15  * @hmc_info: pointer to the HMC configuration information struct
  16  * @sd_index: segment descriptor index to manipulate
  17  * @type: what type of segment descriptor we're manipulating
  18  * @direct_mode_sz: size to alloc in direct mode
  19  **/
  20 enum i40e_status_code i40e_add_sd_table_entry(struct i40e_hw *hw,
  21                                               struct i40e_hmc_info *hmc_info,
  22                                               u32 sd_index,
  23                                               enum i40e_sd_entry_type type,
  24                                               u64 direct_mode_sz)
  25 {
  26         enum i40e_status_code ret_code = I40E_SUCCESS;
  27         struct i40e_hmc_sd_entry *sd_entry;
  28         enum   i40e_memory_type mem_type;
  29         bool dma_mem_alloc_done = false;
  30         struct i40e_dma_mem mem;
  31         u64 alloc_len;
  32
  33         if (NULL == hmc_info->sd_table.sd_entry) {
  34                 ret_code = I40E_ERR_BAD_PTR;
  35                 DEBUGOUT("i40e_add_sd_table_entry: bad sd_entry\n");
  36                 goto exit;
  37         }
  38
  39         if (sd_index >= hmc_info->sd_table.sd_cnt) {
  40                 ret_code = I40E_ERR_INVALID_SD_INDEX;
  41                 DEBUGOUT("i40e_add_sd_table_entry: bad sd_index\n");
  42                 goto exit;
  43         }
  44
  45         sd_entry = &hmc_info->sd_table.sd_entry[sd_index];
  46         if (!sd_entry->valid) {
  47                 if (I40E_SD_TYPE_PAGED == type) {
  48                         mem_type = i40e_mem_pd;
  49                         alloc_len = I40E_HMC_PAGED_BP_SIZE;
  50                 } else {
  51                         mem_type = i40e_mem_bp_jumbo;
  52                         alloc_len = direct_mode_sz;
  53                 }
  54
  55                 /* allocate a 4K pd page or 2M backing page */
  56                 ret_code = i40e_allocate_dma_mem(hw, &mem, mem_type, alloc_len,
  57                                                  I40E_HMC_PD_BP_BUF_ALIGNMENT);
  58                 if (ret_code)
  59                         goto exit;
  60                 dma_mem_alloc_done = true;
  61                 if (I40E_SD_TYPE_PAGED == type) {
  62                         ret_code = i40e_allocate_virt_mem(hw,
  63                                         &sd_entry->u.pd_table.pd_entry_virt_mem,
  64                                         sizeof(struct i40e_hmc_pd_entry) * 512);
  65                         if (ret_code)
  66                                 goto exit;
  67                         sd_entry->u.pd_table.pd_entry =
  68                                 (struct i40e_hmc_pd_entry *)
  69                                 sd_entry->u.pd_table.pd_entry_virt_mem.va;
  70                         i40e_memcpy(&sd_entry->u.pd_table.pd_page_addr,
  71                                     &mem, sizeof(struct i40e_dma_mem),
  72                                     I40E_NONDMA_TO_NONDMA);
  73                 } else {
  74                         i40e_memcpy(&sd_entry->u.bp.addr,
  75                                     &mem, sizeof(struct i40e_dma_mem),
  76                                     I40E_NONDMA_TO_NONDMA);
  77                         sd_entry->u.bp.sd_pd_index = sd_index;
  78                 }
  79                 /* initialize the sd entry */
  80                 hmc_info->sd_table.sd_entry[sd_index].entry_type = type;
  81
  82                 /* increment the ref count */
  83                 I40E_INC_SD_REFCNT(&hmc_info->sd_table);
  84         }
  85         /* Increment backing page reference count */
  86         if (I40E_SD_TYPE_DIRECT == sd_entry->entry_type)
  87                 I40E_INC_BP_REFCNT(&sd_entry->u.bp);
  88 exit:
  89         if (I40E_SUCCESS != ret_code)
  90                 if (dma_mem_alloc_done)
  91                         i40e_free_dma_mem(hw, &mem);
  92
  93         return ret_code;
  94 }
  95
  96 /**
  97  * i40e_add_pd_table_entry - Adds page descriptor to the specified table
  98  * @hw: pointer to our HW structure
  99  * @hmc_info: pointer to the HMC configuration information structure
 100  * @pd_index: which page descriptor index to manipulate
 101  * @rsrc_pg: if not NULL, use preallocated page instead of allocating new one.
 102  *
 103  * This function:
 104  *      1. Initializes the pd entry
 105  *      2. Adds pd_entry in the pd_table
 106  *      3. Mark the entry valid in i40e_hmc_pd_entry structure
 107  *      4. Initializes the pd_entry's ref count to 1
 108  * assumptions:
 109  *      1. The memory for pd should be pinned down, physically contiguous and
 110  *         aligned on 4K boundary and zeroed memory.
 111  *      2. It should be 4K in size.
 112  **/
 113 enum i40e_status_code i40e_add_pd_table_entry(struct i40e_hw *hw,
 114                                               struct i40e_hmc_info *hmc_info,
 115                                               u32 pd_index,
 116                                               struct i40e_dma_mem *rsrc_pg)
 117 {
 118         enum i40e_status_code ret_code = I40E_SUCCESS;
 119         struct i40e_hmc_pd_table *pd_table;
 120         struct i40e_hmc_pd_entry *pd_entry;
 121         struct i40e_dma_mem mem;
 122         struct i40e_dma_mem *page = &mem;
 123         u32 sd_idx, rel_pd_idx;
 124         u64 *pd_addr;
 125         u64 page_desc;
 126
 127         if (pd_index / I40E_HMC_PD_CNT_IN_SD >= hmc_info->sd_table.sd_cnt) {
 128                 ret_code = I40E_ERR_INVALID_PAGE_DESC_INDEX;
 129                 DEBUGOUT("i40e_add_pd_table_entry: bad pd_index\n");
 130                 goto exit;
 131         }
 132
 133         /* find corresponding sd */
 134         sd_idx = (pd_index / I40E_HMC_PD_CNT_IN_SD);
 135         if (I40E_SD_TYPE_PAGED !=
 136             hmc_info->sd_table.sd_entry[sd_idx].entry_type)
 137                 goto exit;
 138
 139         rel_pd_idx = (pd_index % I40E_HMC_PD_CNT_IN_SD);
 140         pd_table = &hmc_info->sd_table.sd_entry[sd_idx].u.pd_table;
 141         pd_entry = &pd_table->pd_entry[rel_pd_idx];
 142         if (!pd_entry->valid) {
 143                 if (rsrc_pg) {
 144                         pd_entry->rsrc_pg = true;
 145                         page = rsrc_pg;
 146                 } else {
 147                         /* allocate a 4K backing page */
 148                         ret_code = i40e_allocate_dma_mem(hw, page, i40e_mem_bp,
 149                                                 I40E_HMC_PAGED_BP_SIZE,
 150                                                 I40E_HMC_PD_BP_BUF_ALIGNMENT);
 151                         if (ret_code)
 152                                 goto exit;
 153                         pd_entry->rsrc_pg = false;
 154                 }
 155
 156                 i40e_memcpy(&pd_entry->bp.addr, page,
 157                             sizeof(struct i40e_dma_mem), I40E_NONDMA_TO_NONDMA);
 158                 pd_entry->bp.sd_pd_index = pd_index;
 159                 pd_entry->bp.entry_type = I40E_SD_TYPE_PAGED;
 160                 /* Set page address and valid bit */
 161                 page_desc = page->pa | 0x1;
 162
 163                 pd_addr = (u64 *)pd_table->pd_page_addr.va;
 164                 pd_addr += rel_pd_idx;
 165
 166                 /* Add the backing page physical address in the pd entry */
 167                 i40e_memcpy(pd_addr, &page_desc, sizeof(u64),
 168                             I40E_NONDMA_TO_DMA);
 169
 170                 pd_entry->sd_index = sd_idx;
 171                 pd_entry->valid = true;
 172                 I40E_INC_PD_REFCNT(pd_table);
 173         }
 174         I40E_INC_BP_REFCNT(&pd_entry->bp);
 175 exit:
 176         return ret_code;
 177 }
 178
 179 /**
 180  * i40e_remove_pd_bp - remove a backing page from a page descriptor
 181  * @hw: pointer to our HW structure
 182  * @hmc_info: pointer to the HMC configuration information structure
 183  * @idx: the page index
 184  *
 185  * This function:
 186  *      1. Marks the entry in pd tabe (for paged address mode) or in sd table
 187  *         (for direct address mode) invalid.
 188  *      2. Write to register PMPDINV to invalidate the backing page in FV cache
 189  *      3. Decrement the ref count for the pd _entry
 190  * assumptions:
 191  *      1. Caller can deallocate the memory used by backing storage after this
 192  *         function returns.
 193  **/
 194 enum i40e_status_code i40e_remove_pd_bp(struct i40e_hw *hw,
 195                                         struct i40e_hmc_info *hmc_info,
 196                                         u32 idx)
 197 {
 198         enum i40e_status_code ret_code = I40E_SUCCESS;
 199         struct i40e_hmc_pd_entry *pd_entry;
 200         struct i40e_hmc_pd_table *pd_table;
 201         struct i40e_hmc_sd_entry *sd_entry;
 202         u32 sd_idx, rel_pd_idx;
 203         u64 *pd_addr;
 204
 205         /* calculate index */
 206         sd_idx = idx / I40E_HMC_PD_CNT_IN_SD;
 207         rel_pd_idx = idx % I40E_HMC_PD_CNT_IN_SD;
 208         if (sd_idx >= hmc_info->sd_table.sd_cnt) {
 209                 ret_code = I40E_ERR_INVALID_PAGE_DESC_INDEX;
 210                 DEBUGOUT("i40e_remove_pd_bp: bad idx\n");
 211                 goto exit;
 212         }
 213         sd_entry = &hmc_info->sd_table.sd_entry[sd_idx];
 214         if (I40E_SD_TYPE_PAGED != sd_entry->entry_type) {
 215                 ret_code = I40E_ERR_INVALID_SD_TYPE;
 216                 DEBUGOUT("i40e_remove_pd_bp: wrong sd_entry type\n");
 217                 goto exit;
 218         }
 219         /* get the entry and decrease its ref counter */
 220         pd_table = &hmc_info->sd_table.sd_entry[sd_idx].u.pd_table;
 221         pd_entry = &pd_table->pd_entry[rel_pd_idx];
 222         I40E_DEC_BP_REFCNT(&pd_entry->bp);
 223         if (pd_entry->bp.ref_cnt)
 224                 goto exit;
 225
 226         /* mark the entry invalid */
 227         pd_entry->valid = false;
 228         I40E_DEC_PD_REFCNT(pd_table);
 229         pd_addr = (u64 *)pd_table->pd_page_addr.va;
 230         pd_addr += rel_pd_idx;
 231         i40e_memset(pd_addr, 0, sizeof(u64), I40E_DMA_MEM);
 232         I40E_INVALIDATE_PF_HMC_PD(hw, sd_idx, idx);
 233
 234         /* free memory here */
 235         if (!pd_entry->rsrc_pg)
 236                 ret_code = i40e_free_dma_mem(hw, &(pd_entry->bp.addr));
 237         if (I40E_SUCCESS != ret_code)
 238                 goto exit;
 239         if (!pd_table->ref_cnt)
 240                 i40e_free_virt_mem(hw, &pd_table->pd_entry_virt_mem);
 241 exit:
 242         return ret_code;
 243 }
 244
 245 /**
 246  * i40e_prep_remove_sd_bp - Prepares to remove a backing page from a sd entry
 247  * @hmc_info: pointer to the HMC configuration information structure
 248  * @idx: the page index
 249  **/
 250 enum i40e_status_code i40e_prep_remove_sd_bp(struct i40e_hmc_info *hmc_info,
 251                                              u32 idx)
 252 {
 253         enum i40e_status_code ret_code = I40E_SUCCESS;
 254         struct i40e_hmc_sd_entry *sd_entry;
 255
 256         /* get the entry and decrease its ref counter */
 257         sd_entry = &hmc_info->sd_table.sd_entry[idx];
 258         I40E_DEC_BP_REFCNT(&sd_entry->u.bp);
 259         if (sd_entry->u.bp.ref_cnt) {
 260                 ret_code = I40E_ERR_NOT_READY;
 261                 goto exit;
 262         }
 263         I40E_DEC_SD_REFCNT(&hmc_info->sd_table);
 264
 265         /* mark the entry invalid */
 266         sd_entry->valid = false;
 267 exit:
 268         return ret_code;
 269 }
 270
 271 /**
 272  * i40e_remove_sd_bp_new - Removes a backing page from a segment descriptor
 273  * @hw: pointer to our hw struct
 274  * @hmc_info: pointer to the HMC configuration information structure
 275  * @idx: the page index
 276  * @is_pf: used to distinguish between VF and PF
 277  **/
 278 enum i40e_status_code i40e_remove_sd_bp_new(struct i40e_hw *hw,
 279                                             struct i40e_hmc_info *hmc_info,
 280                                             u32 idx, bool is_pf)
 281 {
 282         struct i40e_hmc_sd_entry *sd_entry;
 283
 284         if (!is_pf)
 285                 return I40E_NOT_SUPPORTED;
 286
 287         /* get the entry and decrease its ref counter */
 288         sd_entry = &hmc_info->sd_table.sd_entry[idx];
 289         I40E_CLEAR_PF_SD_ENTRY(hw, idx, I40E_SD_TYPE_DIRECT);
 290
 291         return i40e_free_dma_mem(hw, &(sd_entry->u.bp.addr));
 292 }
 293
 294 /**
 295  * i40e_prep_remove_pd_page - Prepares to remove a PD page from sd entry.
 296  * @hmc_info: pointer to the HMC configuration information structure
 297  * @idx: segment descriptor index to find the relevant page descriptor
 298  **/
 299 enum i40e_status_code i40e_prep_remove_pd_page(struct i40e_hmc_info *hmc_info,
 300                                                u32 idx)
 301 {
 302         enum i40e_status_code ret_code = I40E_SUCCESS;
 303         struct i40e_hmc_sd_entry *sd_entry;
 304
 305         sd_entry = &hmc_info->sd_table.sd_entry[idx];
 306
 307         if (sd_entry->u.pd_table.ref_cnt) {
 308                 ret_code = I40E_ERR_NOT_READY;
 309                 goto exit;
 310         }
 311
 312         /* mark the entry invalid */
 313         sd_entry->valid = false;
 314
 315         I40E_DEC_SD_REFCNT(&hmc_info->sd_table);
 316 exit:
 317         return ret_code;
 318 }
 319
 320 /**
 321  * i40e_remove_pd_page_new - Removes a PD page from sd entry.
 322  * @hw: pointer to our hw struct
 323  * @hmc_info: pointer to the HMC configuration information structure
 324  * @idx: segment descriptor index to find the relevant page descriptor
 325  * @is_pf: used to distinguish between VF and PF
 326  **/
 327 enum i40e_status_code i40e_remove_pd_page_new(struct i40e_hw *hw,
 328                                               struct i40e_hmc_info *hmc_info,
 329                                               u32 idx, bool is_pf)
 330 {
 331         struct i40e_hmc_sd_entry *sd_entry;
 332
 333         if (!is_pf)
 334                 return I40E_NOT_SUPPORTED;
 335
 336         sd_entry = &hmc_info->sd_table.sd_entry[idx];
 337         I40E_CLEAR_PF_SD_ENTRY(hw, idx, I40E_SD_TYPE_PAGED);
 338
 339         return i40e_free_dma_mem(hw, &(sd_entry->u.pd_table.pd_page_addr));
 340 }