src/cache.c

   1 /*
   2  * Copyright (C) the libgit2 contributors. All rights reserved.
   3  *
   4  * This file is part of libgit2, distributed under the GNU GPL v2 with
   5  * a Linking Exception. For full terms see the included COPYING file.
   6  */
   7
   8 #include "common.h"
   9 #include "repository.h"
  10 #include "commit.h"
  11 #include "thread-utils.h"
  12 #include "util.h"
  13 #include "cache.h"
  14 #include "odb.h"
  15 #include "object.h"
  16 #include "git2/oid.h"
  17
  18 GIT__USE_OIDMAP
  19
  20 bool git_cache__enabled = true;
  21 ssize_t git_cache__max_storage = (256 * 1024 * 1024);
  22 git_atomic_ssize git_cache__current_storage = {0};
  23
  24 static size_t git_cache__max_object_size[8] = {
  25         0,     /* GIT_OBJ__EXT1 */
  26         4096,  /* GIT_OBJ_COMMIT */
  27         4096,  /* GIT_OBJ_TREE */
  28         0,     /* GIT_OBJ_BLOB */
  29         4096,  /* GIT_OBJ_TAG */
  30         0,     /* GIT_OBJ__EXT2 */
  31         0,     /* GIT_OBJ_OFS_DELTA */
  32         0      /* GIT_OBJ_REF_DELTA */
  33 };
  34
  35 int git_cache_set_max_object_size(git_otype type, size_t size)
  36 {
  37         if (type < 0 || (size_t)type >= ARRAY_SIZE(git_cache__max_object_size)) {
  38                 giterr_set(GITERR_INVALID, "type out of range");
  39                 return -1;
  40         }
  41
  42         git_cache__max_object_size[type] = size;
  43         return 0;
  44 }
  45
  46 void git_cache_dump_stats(git_cache *cache)
  47 {
  48         git_cached_obj *object;
  49
  50         if (kh_size(cache->map) == 0)
  51                 return;
  52
  53         printf("Cache %p: %d items cached, %"PRIdZ" bytes\n",
  54                 cache, kh_size(cache->map), cache->used_memory);
  55
  56         kh_foreach_value(cache->map, object, {
  57                 char oid_str[9];
  58                 printf(" %s%c %s (%"PRIuZ")\n",
  59                         git_object_type2string(object->type),
  60                         object->flags == GIT_CACHE_STORE_PARSED ? '*' : ' ',
  61                         git_oid_tostr(oid_str, sizeof(oid_str), &object->oid),
  62                         object->size
  63                 );
  64         });
  65 }
  66
  67 int git_cache_init(git_cache *cache)
  68 {
  69         memset(cache, 0, sizeof(*cache));
  70         cache->map = git_oidmap_alloc();
  71         GITERR_CHECK_ALLOC(cache->map);
  72         if (git_rwlock_init(&cache->lock)) {
  73                 giterr_set(GITERR_OS, "Failed to initialize cache rwlock");
  74                 return -1;
  75         }
  76         return 0;
  77 }
  78
  79 /* called with lock */
  80 static void clear_cache(git_cache *cache)
  81 {
  82         git_cached_obj *evict = NULL;
  83
  84         if (kh_size(cache->map) == 0)
  85                 return;
  86
  87         kh_foreach_value(cache->map, evict, {
  88                 git_cached_obj_decref(evict);
  89         });
  90
  91         kh_clear(oid, cache->map);
  92         git_atomic_ssize_add(&git_cache__current_storage, -cache->used_memory);
  93         cache->used_memory = 0;
  94 }
  95
  96 void git_cache_clear(git_cache *cache)
  97 {
  98         if (git_rwlock_wrlock(&cache->lock) < 0)
  99                 return;
 100
 101         clear_cache(cache);
 102
 103         git_rwlock_wrunlock(&cache->lock);
 104 }
 105
 106 void git_cache_free(git_cache *cache)
 107 {
 108         git_cache_clear(cache);
 109         git_oidmap_free(cache->map);
 110         git_rwlock_free(&cache->lock);
 111         git__memzero(cache, sizeof(*cache));
 112 }
 113
 114 /* Called with lock */
 115 static void cache_evict_entries(git_cache *cache)
 116 {
 117         uint32_t seed = rand();
 118         size_t evict_count = 8;
 119         ssize_t evicted_memory = 0;
 120
 121         /* do not infinite loop if there's not enough entries to evict  */
 122         if (evict_count > kh_size(cache->map)) {
 123                 clear_cache(cache);
 124                 return;
 125         }
 126
 127         while (evict_count > 0) {
 128                 khiter_t pos = seed++ % kh_end(cache->map);
 129
 130                 if (kh_exist(cache->map, pos)) {
 131                         git_cached_obj *evict = kh_val(cache->map, pos);
 132
 133                         evict_count--;
 134                         evicted_memory += evict->size;
 135                         git_cached_obj_decref(evict);
 136
 137                         kh_del(oid, cache->map, pos);
 138                 }
 139         }
 140
 141         cache->used_memory -= evicted_memory;
 142         git_atomic_ssize_add(&git_cache__current_storage, -evicted_memory);
 143 }
 144
 145 static bool cache_should_store(git_otype object_type, size_t object_size)
 146 {
 147         size_t max_size = git_cache__max_object_size[object_type];
 148         return git_cache__enabled && object_size < max_size;
 149 }
 150
 151 static void *cache_get(git_cache *cache, const git_oid *oid, unsigned int flags)
 152 {
 153         khiter_t pos;
 154         git_cached_obj *entry = NULL;
 155
 156         if (!git_cache__enabled || git_rwlock_rdlock(&cache->lock) < 0)
 157                 return NULL;
 158
 159         pos = kh_get(oid, cache->map, oid);
 160         if (pos != kh_end(cache->map)) {
 161                 entry = kh_val(cache->map, pos);
 162
 163                 if (flags && entry->flags != flags) {
 164                         entry = NULL;
 165                 } else {
 166                         git_cached_obj_incref(entry);
 167                 }
 168         }
 169
 170         git_rwlock_rdunlock(&cache->lock);
 171
 172         return entry;
 173 }
 174
 175 static void *cache_store(git_cache *cache, git_cached_obj *entry)
 176 {
 177         khiter_t pos;
 178
 179         git_cached_obj_incref(entry);
 180
 181         if (!git_cache__enabled && cache->used_memory > 0) {
 182                 git_cache_clear(cache);
 183                 return entry;
 184         }
 185
 186         if (!cache_should_store(entry->type, entry->size))
 187                 return entry;
 188
 189         if (git_rwlock_wrlock(&cache->lock) < 0)
 190                 return entry;
 191
 192         /* soften the load on the cache */
 193         if (git_cache__current_storage.val > git_cache__max_storage)
 194                 cache_evict_entries(cache);
 195
 196         pos = kh_get(oid, cache->map, &entry->oid);
 197
 198         /* not found */
 199         if (pos == kh_end(cache->map)) {
 200                 int rval;
 201
 202                 pos = kh_put(oid, cache->map, &entry->oid, &rval);
 203                 if (rval >= 0) {
 204                         kh_key(cache->map, pos) = &entry->oid;
 205                         kh_val(cache->map, pos) = entry;
 206                         git_cached_obj_incref(entry);
 207                         cache->used_memory += entry->size;
 208                         git_atomic_ssize_add(&git_cache__current_storage, (ssize_t)entry->size);
 209                 }
 210         }
 211         /* found */
 212         else {
 213                 git_cached_obj *stored_entry = kh_val(cache->map, pos);
 214
 215                 if (stored_entry->flags == entry->flags) {
 216                         git_cached_obj_decref(entry);
 217                         git_cached_obj_incref(stored_entry);
 218                         entry = stored_entry;
 219                 } else if (stored_entry->flags == GIT_CACHE_STORE_RAW &&
 220                         entry->flags == GIT_CACHE_STORE_PARSED) {
 221                         git_cached_obj_decref(stored_entry);
 222                         git_cached_obj_incref(entry);
 223
 224                         kh_key(cache->map, pos) = &entry->oid;
 225                         kh_val(cache->map, pos) = entry;
 226                 } else {
 227                         /* NO OP */
 228                 }
 229         }
 230
 231         git_rwlock_wrunlock(&cache->lock);
 232         return entry;
 233 }
 234
 235 void *git_cache_store_raw(git_cache *cache, git_odb_object *entry)
 236 {
 237         entry->cached.flags = GIT_CACHE_STORE_RAW;
 238         return cache_store(cache, (git_cached_obj *)entry);
 239 }
 240
 241 void *git_cache_store_parsed(git_cache *cache, git_object *entry)
 242 {
 243         entry->cached.flags = GIT_CACHE_STORE_PARSED;
 244         return cache_store(cache, (git_cached_obj *)entry);
 245 }
 246
 247 git_odb_object *git_cache_get_raw(git_cache *cache, const git_oid *oid)
 248 {
 249         return cache_get(cache, oid, GIT_CACHE_STORE_RAW);
 250 }
 251
 252 git_object *git_cache_get_parsed(git_cache *cache, const git_oid *oid)
 253 {
 254         return cache_get(cache, oid, GIT_CACHE_STORE_PARSED);
 255 }
 256
 257 void *git_cache_get_any(git_cache *cache, const git_oid *oid)
 258 {
 259         return cache_get(cache, oid, GIT_CACHE_STORE_ANY);
 260 }
 261
 262 void git_cached_obj_decref(void *_obj)
 263 {
 264         git_cached_obj *obj = _obj;
 265
 266         if (git_atomic_dec(&obj->refcount) == 0) {
 267                 switch (obj->flags) {
 268                 case GIT_CACHE_STORE_RAW:
 269                         git_odb_object__free(_obj);
 270                         break;
 271
 272                 case GIT_CACHE_STORE_PARSED:
 273                         git_object__free(_obj);
 274                         break;
 275
 276                 default:
 277                         git__free(_obj);
 278                         break;
 279                 }
 280         }
 281 }