src/cache.c

   1 /*
   2  * Copyright (C) the libgit2 contributors. All rights reserved.
   3  *
   4  * This file is part of libgit2, distributed under the GNU GPL v2 with
   5  * a Linking Exception. For full terms see the included COPYING file.
   6  */
   7
   8 #include "common.h"
   9 #include "repository.h"
  10 #include "commit.h"
  11 #include "thread-utils.h"
  12 #include "util.h"
  13 #include "cache.h"
  14 #include "odb.h"
  15 #include "object.h"
  16 #include "git2/oid.h"
  17
  18 GIT__USE_OIDMAP
  19
  20 bool git_cache__enabled = true;
  21 ssize_t git_cache__max_storage = (256 * 1024 * 1024);
  22 git_atomic_ssize git_cache__current_storage = {0};
  23
  24 static size_t git_cache__max_object_size[8] = {
  25         0,     /* GIT_OBJ__EXT1 */
  26         4096,  /* GIT_OBJ_COMMIT */
  27         4096,  /* GIT_OBJ_TREE */
  28         0,     /* GIT_OBJ_BLOB */
  29         4096,  /* GIT_OBJ_TAG */
  30         0,     /* GIT_OBJ__EXT2 */
  31         0,     /* GIT_OBJ_OFS_DELTA */
  32         0      /* GIT_OBJ_REF_DELTA */
  33 };
  34
  35 int git_cache_set_max_object_size(git_otype type, size_t size)
  36 {
  37         if (type < 0 || (size_t)type >= ARRAY_SIZE(git_cache__max_object_size)) {
  38                 giterr_set(GITERR_INVALID, "type out of range");
  39                 return -1;
  40         }
  41
  42         git_cache__max_object_size[type] = size;
  43         return 0;
  44 }
  45
  46 void git_cache_dump_stats(git_cache *cache)
  47 {
  48         git_cached_obj *object;
  49
  50         if (kh_size(cache->map) == 0)
  51                 return;
  52
  53         printf("Cache %p: %d items cached, %d bytes\n",
  54                 cache, kh_size(cache->map), (int)cache->used_memory);
  55
  56         kh_foreach_value(cache->map, object, {
  57                 char oid_str[9];
  58                 printf(" %s%c %s (%d)\n",
  59                         git_object_type2string(object->type),
  60                         object->flags == GIT_CACHE_STORE_PARSED ? '*' : ' ',
  61                         git_oid_tostr(oid_str, sizeof(oid_str), &object->oid),
  62                         (int)object->size
  63                 );
  64         });
  65 }
  66
  67 int git_cache_init(git_cache *cache)
  68 {
  69         memset(cache, 0, sizeof(*cache));
  70         cache->map = git_oidmap_alloc();
  71         if (git_rwlock_init(&cache->lock)) {
  72                 giterr_set(GITERR_OS, "Failed to initialize cache rwlock");
  73                 return -1;
  74         }
  75         return 0;
  76 }
  77
  78 /* called with lock */
  79 static void clear_cache(git_cache *cache)
  80 {
  81         git_cached_obj *evict = NULL;
  82
  83         if (kh_size(cache->map) == 0)
  84                 return;
  85
  86         kh_foreach_value(cache->map, evict, {
  87                 git_cached_obj_decref(evict);
  88         });
  89
  90         kh_clear(oid, cache->map);
  91         git_atomic_ssize_add(&git_cache__current_storage, -cache->used_memory);
  92         cache->used_memory = 0;
  93 }
  94
  95 void git_cache_clear(git_cache *cache)
  96 {
  97         if (git_rwlock_wrlock(&cache->lock) < 0)
  98                 return;
  99
 100         clear_cache(cache);
 101
 102         git_rwlock_wrunlock(&cache->lock);
 103 }
 104
 105 void git_cache_free(git_cache *cache)
 106 {
 107         git_cache_clear(cache);
 108         git_oidmap_free(cache->map);
 109         git_rwlock_free(&cache->lock);
 110         git__memzero(cache, sizeof(*cache));
 111 }
 112
 113 /* Called with lock */
 114 static void cache_evict_entries(git_cache *cache)
 115 {
 116         uint32_t seed = rand();
 117         size_t evict_count = 8;
 118         ssize_t evicted_memory = 0;
 119
 120         /* do not infinite loop if there's not enough entries to evict  */
 121         if (evict_count > kh_size(cache->map)) {
 122                 clear_cache(cache);
 123                 return;
 124         }
 125
 126         while (evict_count > 0) {
 127                 khiter_t pos = seed++ % kh_end(cache->map);
 128
 129                 if (kh_exist(cache->map, pos)) {
 130                         git_cached_obj *evict = kh_val(cache->map, pos);
 131
 132                         evict_count--;
 133                         evicted_memory += evict->size;
 134                         git_cached_obj_decref(evict);
 135
 136                         kh_del(oid, cache->map, pos);
 137                 }
 138         }
 139
 140         cache->used_memory -= evicted_memory;
 141         git_atomic_ssize_add(&git_cache__current_storage, -evicted_memory);
 142 }
 143
 144 static bool cache_should_store(git_otype object_type, size_t object_size)
 145 {
 146         size_t max_size = git_cache__max_object_size[object_type];
 147         return git_cache__enabled && object_size < max_size;
 148 }
 149
 150 static void *cache_get(git_cache *cache, const git_oid *oid, unsigned int flags)
 151 {
 152         khiter_t pos;
 153         git_cached_obj *entry = NULL;
 154
 155         if (!git_cache__enabled || git_rwlock_rdlock(&cache->lock) < 0)
 156                 return NULL;
 157
 158         pos = kh_get(oid, cache->map, oid);
 159         if (pos != kh_end(cache->map)) {
 160                 entry = kh_val(cache->map, pos);
 161
 162                 if (flags && entry->flags != flags) {
 163                         entry = NULL;
 164                 } else {
 165                         git_cached_obj_incref(entry);
 166                 }
 167         }
 168
 169         git_rwlock_rdunlock(&cache->lock);
 170
 171         return entry;
 172 }
 173
 174 static void *cache_store(git_cache *cache, git_cached_obj *entry)
 175 {
 176         khiter_t pos;
 177
 178         git_cached_obj_incref(entry);
 179
 180         if (!git_cache__enabled && cache->used_memory > 0) {
 181                 git_cache_clear(cache);
 182                 return entry;
 183         }
 184
 185         if (!cache_should_store(entry->type, entry->size))
 186                 return entry;
 187
 188         if (git_rwlock_wrlock(&cache->lock) < 0)
 189                 return entry;
 190
 191         /* soften the load on the cache */
 192         if (git_cache__current_storage.val > git_cache__max_storage)
 193                 cache_evict_entries(cache);
 194
 195         pos = kh_get(oid, cache->map, &entry->oid);
 196
 197         /* not found */
 198         if (pos == kh_end(cache->map)) {
 199                 int rval;
 200
 201                 pos = kh_put(oid, cache->map, &entry->oid, &rval);
 202                 if (rval >= 0) {
 203                         kh_key(cache->map, pos) = &entry->oid;
 204                         kh_val(cache->map, pos) = entry;
 205                         git_cached_obj_incref(entry);
 206                         cache->used_memory += entry->size;
 207                         git_atomic_ssize_add(&git_cache__current_storage, (ssize_t)entry->size);
 208                 }
 209         }
 210         /* found */
 211         else {
 212                 git_cached_obj *stored_entry = kh_val(cache->map, pos);
 213
 214                 if (stored_entry->flags == entry->flags) {
 215                         git_cached_obj_decref(entry);
 216                         git_cached_obj_incref(stored_entry);
 217                         entry = stored_entry;
 218                 } else if (stored_entry->flags == GIT_CACHE_STORE_RAW &&
 219                         entry->flags == GIT_CACHE_STORE_PARSED) {
 220                         git_cached_obj_decref(stored_entry);
 221                         git_cached_obj_incref(entry);
 222
 223                         kh_key(cache->map, pos) = &entry->oid;
 224                         kh_val(cache->map, pos) = entry;
 225                 } else {
 226                         /* NO OP */
 227                 }
 228         }
 229
 230         git_rwlock_wrunlock(&cache->lock);
 231         return entry;
 232 }
 233
 234 void *git_cache_store_raw(git_cache *cache, git_odb_object *entry)
 235 {
 236         entry->cached.flags = GIT_CACHE_STORE_RAW;
 237         return cache_store(cache, (git_cached_obj *)entry);
 238 }
 239
 240 void *git_cache_store_parsed(git_cache *cache, git_object *entry)
 241 {
 242         entry->cached.flags = GIT_CACHE_STORE_PARSED;
 243         return cache_store(cache, (git_cached_obj *)entry);
 244 }
 245
 246 git_odb_object *git_cache_get_raw(git_cache *cache, const git_oid *oid)
 247 {
 248         return cache_get(cache, oid, GIT_CACHE_STORE_RAW);
 249 }
 250
 251 git_object *git_cache_get_parsed(git_cache *cache, const git_oid *oid)
 252 {
 253         return cache_get(cache, oid, GIT_CACHE_STORE_PARSED);
 254 }
 255
 256 void *git_cache_get_any(git_cache *cache, const git_oid *oid)
 257 {
 258         return cache_get(cache, oid, GIT_CACHE_STORE_ANY);
 259 }
 260
 261 void git_cached_obj_decref(void *_obj)
 262 {
 263         git_cached_obj *obj = _obj;
 264
 265         if (git_atomic_dec(&obj->refcount) == 0) {
 266                 switch (obj->flags) {
 267                 case GIT_CACHE_STORE_RAW:
 268                         git_odb_object__free(_obj);
 269                         break;
 270
 271                 case GIT_CACHE_STORE_PARSED:
 272                         git_object__free(_obj);
 273                         break;
 274
 275                 default:
 276                         git__free(_obj);
 277                         break;
 278                 }
 279         }
 280 }