ceph/src/spdk/ocf/src/engine/engine_rd.c

   1 /*
   2  * Copyright(c) 2012-2018 Intel Corporation
   3  * SPDX-License-Identifier: BSD-3-Clause-Clear
   4  */
   5
   6 #include "ocf/ocf.h"
   7 #include "../ocf_cache_priv.h"
   8 #include "engine_rd.h"
   9 #include "engine_pt.h"
  10 #include "engine_inv.h"
  11 #include "engine_bf.h"
  12 #include "engine_common.h"
  13 #include "cache_engine.h"
  14 #include "../concurrency/ocf_concurrency.h"
  15 #include "../utils/utils_io.h"
  16 #include "../utils/utils_req.h"
  17 #include "../utils/utils_cache_line.h"
  18 #include "../utils/utils_part.h"
  19 #include "../metadata/metadata.h"
  20 #include "../ocf_def_priv.h"
  21
  22 #define OCF_ENGINE_DEBUG_IO_NAME "rd"
  23 #include "engine_debug.h"
  24
  25 static void _ocf_read_generic_hit_complete(struct ocf_request *req, int error)
  26 {
  27         if (error)
  28                 req->error |= error;
  29
  30         if (req->error)
  31                 inc_fallback_pt_error_counter(req->cache);
  32
  33         /* Handle callback-caller race to let only one of the two complete the
  34          * request. Also, complete original request only if this is the last
  35          * sub-request to complete
  36          */
  37         if (env_atomic_dec_return(&req->req_remaining) == 0) {
  38                 OCF_DEBUG_RQ(req, "HIT completion");
  39
  40                 if (req->error) {
  41                         env_atomic_inc(&req->cache->core[req->core_id].
  42                                 counters->cache_errors.read);
  43                         ocf_engine_push_req_front_pt(req);
  44                 } else {
  45
  46                         ocf_req_unlock(req);
  47
  48                         /* Complete request */
  49                         req->complete(req, req->error);
  50
  51                         /* Free the request at the last point
  52                          * of the completion path
  53                          */
  54                         ocf_req_put(req);
  55                 }
  56         }
  57 }
  58
  59 static void _ocf_read_generic_miss_complete(struct ocf_request *req, int error)
  60 {
  61         struct ocf_cache *cache = req->cache;
  62
  63         if (error)
  64                 req->error = error;
  65
  66         /* Handle callback-caller race to let only one of the two complete the
  67          * request. Also, complete original request only if this is the last
  68          * sub-request to complete
  69          */
  70         if (env_atomic_dec_return(&req->req_remaining) == 0) {
  71                 OCF_DEBUG_RQ(req, "MISS completion");
  72
  73                 if (req->error) {
  74                         /*
  75                          * --- Do not submit this request to write-back-thread.
  76                          * Stop it here ---
  77                          */
  78                         req->complete(req, req->error);
  79
  80                         req->info.core_error = 1;
  81                         env_atomic_inc(&cache->core[req->core_id].
  82                                         counters->core_errors.read);
  83
  84                         ctx_data_free(cache->owner, req->cp_data);
  85                         req->cp_data = NULL;
  86
  87                         /* Invalidate metadata */
  88                         ocf_engine_invalidate(req);
  89
  90                         return;
  91                 }
  92
  93                 /* Copy pages to copy vec, since this is the one needed
  94                  * by the above layer
  95                  */
  96                 ctx_data_cpy(cache->owner, req->cp_data, req->data, 0, 0,
  97                                 req->byte_length);
  98
  99                 /* Complete request */
 100                 req->complete(req, req->error);
 101
 102                 ocf_engine_backfill(req);
 103         }
 104 }
 105
 106 static inline void _ocf_read_generic_submit_hit(struct ocf_request *req)
 107 {
 108         env_atomic_set(&req->req_remaining, ocf_engine_io_count(req));
 109
 110         ocf_submit_cache_reqs(req->cache, req->map, req, OCF_READ,
 111                 ocf_engine_io_count(req), _ocf_read_generic_hit_complete);
 112 }
 113
 114 static inline void _ocf_read_generic_submit_miss(struct ocf_request *req)
 115 {
 116         struct ocf_cache *cache = req->cache;
 117         int ret;
 118
 119         env_atomic_set(&req->req_remaining, 1);
 120
 121         req->cp_data = ctx_data_alloc(cache->owner,
 122                         BYTES_TO_PAGES(req->byte_length));
 123         if (!req->cp_data)
 124                 goto err_alloc;
 125
 126         ret = ctx_data_mlock(cache->owner, req->cp_data);
 127         if (ret)
 128                 goto err_alloc;
 129
 130         /* Submit read request to core device. */
 131         ocf_submit_volume_req(&cache->core[req->core_id].volume, req,
 132                         _ocf_read_generic_miss_complete);
 133
 134         return;
 135
 136 err_alloc:
 137         _ocf_read_generic_miss_complete(req, -ENOMEM);
 138 }
 139
 140 static int _ocf_read_generic_do(struct ocf_request *req)
 141 {
 142         struct ocf_cache *cache = req->cache;
 143
 144         if (ocf_engine_is_miss(req) && req->map->rd_locked) {
 145                 /* Miss can be handled only on write locks.
 146                  * Need to switch to PT
 147                  */
 148                 OCF_DEBUG_RQ(req, "Switching to PT");
 149                 ocf_read_pt_do(req);
 150                 return 0;
 151         }
 152
 153         /* Get OCF request - increase reference counter */
 154         ocf_req_get(req);
 155
 156         if (ocf_engine_is_miss(req)) {
 157                 if (req->info.dirty_any) {
 158                         OCF_METADATA_LOCK_RD();
 159
 160                         /* Request is dirty need to clean request */
 161                         ocf_engine_clean(req);
 162
 163                         OCF_METADATA_UNLOCK_RD();
 164
 165                         /* We need to clean request before processing, return */
 166                         ocf_req_put(req);
 167
 168                         return 0;
 169                 }
 170
 171                 OCF_METADATA_LOCK_RD();
 172
 173                 /* Set valid status bits map */
 174                 ocf_set_valid_map_info(req);
 175
 176                 OCF_METADATA_UNLOCK_RD();
 177         }
 178
 179         if (req->info.re_part) {
 180                 OCF_DEBUG_RQ(req, "Re-Part");
 181
 182                 OCF_METADATA_LOCK_WR();
 183
 184                 /* Probably some cache lines are assigned into wrong
 185                  * partition. Need to move it to new one
 186                  */
 187                 ocf_part_move(req);
 188
 189                 OCF_METADATA_UNLOCK_WR();
 190         }
 191
 192         OCF_DEBUG_RQ(req, "Submit");
 193
 194         /* Submit IO */
 195         if (ocf_engine_is_hit(req))
 196                 _ocf_read_generic_submit_hit(req);
 197         else
 198                 _ocf_read_generic_submit_miss(req);
 199
 200         /* Updata statistics */
 201         ocf_engine_update_request_stats(req);
 202         ocf_engine_update_block_stats(req);
 203
 204         /* Put OCF request - decrease reference counter */
 205         ocf_req_put(req);
 206
 207         return 0;
 208 }
 209
 210 static const struct ocf_io_if _io_if_read_generic_resume = {
 211                 .read = _ocf_read_generic_do,
 212                 .write = _ocf_read_generic_do,
 213 };
 214
 215 int ocf_read_generic(struct ocf_request *req)
 216 {
 217         bool mapped;
 218         int lock = OCF_LOCK_NOT_ACQUIRED;
 219         struct ocf_cache *cache = req->cache;
 220
 221         ocf_io_start(req->io);
 222
 223         if (env_atomic_read(&cache->pending_read_misses_list_blocked)) {
 224                 /* There are conditions to bypass IO */
 225                 ocf_get_io_if(ocf_cache_mode_pt)->read(req);
 226                 return 0;
 227         }
 228
 229         /* Get OCF request - increase reference counter */
 230         ocf_req_get(req);
 231
 232         /* Set resume call backs */
 233         req->resume = ocf_engine_on_resume;
 234         req->io_if = &_io_if_read_generic_resume;
 235
 236         /*- Metadata RD access -----------------------------------------------*/
 237
 238         OCF_METADATA_LOCK_RD();
 239
 240         /* Traverse request to cache if there is hit */
 241         ocf_engine_traverse(req);
 242
 243         mapped = ocf_engine_is_mapped(req);
 244         if (mapped) {
 245                 /* Request is fully mapped, no need to call eviction */
 246                 if (ocf_engine_is_hit(req)) {
 247                         /* There is a hit, lock request for READ access */
 248                         lock = ocf_req_trylock_rd(req);
 249                 } else {
 250                         /* All cache line mapped, but some sectors are not valid
 251                          * and cache insert will be performed - lock for
 252                          * WRITE is required
 253                          */
 254                         lock = ocf_req_trylock_wr(req);
 255                 }
 256         }
 257
 258         OCF_METADATA_UNLOCK_RD();
 259
 260         /*- END Metadata RD access -------------------------------------------*/
 261
 262         if (!mapped) {
 263
 264                 /*- Metadata WR access ---------------------------------------*/
 265
 266                 OCF_METADATA_LOCK_WR();
 267
 268                 /* Now there is exclusive access for metadata. May traverse once
 269                  * again. If there are misses need to call eviction. This
 270                  * process is called 'mapping'.
 271                  */
 272                 ocf_engine_map(req);
 273
 274                 if (!req->info.eviction_error) {
 275                         if (ocf_engine_is_hit(req)) {
 276                                 /* After mapping turns out there is hit,
 277                                  * so lock OCF request for read access
 278                                  */
 279                                 lock = ocf_req_trylock_rd(req);
 280                         } else {
 281                                 /* Miss, new cache lines were mapped,
 282                                  * need to lock OCF request for write access
 283                                  */
 284                                 lock = ocf_req_trylock_wr(req);
 285                         }
 286                 }
 287                 OCF_METADATA_UNLOCK_WR();
 288
 289                 /*- END Metadata WR access -----------------------------------*/
 290         }
 291
 292         if (!req->info.eviction_error) {
 293                 if (lock >= 0) {
 294                         if (lock != OCF_LOCK_ACQUIRED) {
 295                                 /* Lock was not acquired, need to wait for resume */
 296                                 OCF_DEBUG_RQ(req, "NO LOCK");
 297                         } else {
 298                                 /* Lock was acquired can perform IO */
 299                                 _ocf_read_generic_do(req);
 300                         }
 301                 } else {
 302                         OCF_DEBUG_RQ(req, "LOCK ERROR %d", lock);
 303                         req->complete(req, lock);
 304                         ocf_req_put(req);
 305                 }
 306         } else {
 307                 ocf_req_clear(req);
 308                 ocf_get_io_if(ocf_cache_mode_pt)->read(req);
 309         }
 310
 311
 312         /* Put OCF request - decrease reference counter */
 313         ocf_req_put(req);
 314
 315         return 0;
 316 }