src/odb.c

   1 /*
   2  * This file is free software; you can redistribute it and/or modify
   3  * it under the terms of the GNU General Public License, version 2,
   4  * as published by the Free Software Foundation.
   5  *
   6  * In addition to the permissions in the GNU General Public License,
   7  * the authors give you unlimited permission to link the compiled
   8  * version of this file into combinations with other programs,
   9  * and to distribute those combinations without any restriction
  10  * coming from the use of this file.  (The General Public License
  11  * restrictions do apply in other respects; for example, they cover
  12  * modification of the file, and distribution when not linked into
  13  * a combined executable.)
  14  *
  15  * This file is distributed in the hope that it will be useful, but
  16  * WITHOUT ANY WARRANTY; without even the implied warranty of
  17  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  18  * General Public License for more details.
  19  *
  20  * You should have received a copy of the GNU General Public License
  21  * along with this program; see the file COPYING.  If not, write to
  22  * the Free Software Foundation, 51 Franklin Street, Fifth Floor,
  23  * Boston, MA 02110-1301, USA.
  24  */
  25
  26 #include "common.h"
  27 #include "git2/zlib.h"
  28 #include "git2/object.h"
  29 #include "fileops.h"
  30 #include "hash.h"
  31 #include "odb.h"
  32 #include "delta-apply.h"
  33
  34 #include "git2/odb_backend.h"
  35
  36 #define GIT_ALTERNATES_FILE "info/alternates"
  37
  38 /* TODO: is this correct? */
  39 #define GIT_LOOSE_PRIORITY 2
  40 #define GIT_PACKED_PRIORITY 1
  41
  42 typedef struct
  43 {
  44         git_odb_backend *backend;
  45         int priority;
  46         int is_alternate;
  47 } backend_internal;
  48
  49 static int format_object_header(char *hdr, size_t n, git_rawobj *obj)
  50 {
  51         const char *type_str = git_object_type2string(obj->type);
  52         int len = snprintf(hdr, n, "%s %"PRIuZ, type_str, obj->len);
  53
  54         assert(len > 0);             /* otherwise snprintf() is broken  */
  55         assert(((size_t) len) < n);  /* otherwise the caller is broken! */
  56
  57         if (len < 0 || ((size_t) len) >= n)
  58                 return git__throw(GIT_ERROR, "Cannot format object header. Length is out of bounds");
  59         return len+1;
  60 }
  61
  62 int git_odb__hash_obj(git_oid *id, char *hdr, size_t n, int *len, git_rawobj *obj)
  63 {
  64         git_buf_vec vec[2];
  65         int  hdrlen;
  66
  67         assert(id && hdr && len && obj);
  68
  69         if (!git_object_typeisloose(obj->type))
  70                 return git__throw(GIT_ERROR, "Failed to hash object. Wrong object type");
  71
  72         if (!obj->data && obj->len != 0)
  73                 return git__throw(GIT_ERROR, "Failed to hash object. No data given");
  74
  75         if ((hdrlen = format_object_header(hdr, n, obj)) < 0)
  76                 return git__rethrow(hdrlen, "Failed to hash object");
  77
  78         *len = hdrlen;
  79
  80         vec[0].data = hdr;
  81         vec[0].len  = hdrlen;
  82         vec[1].data = obj->data;
  83         vec[1].len  = obj->len;
  84
  85         git_hash_vec(id, vec, 2);
  86
  87         return GIT_SUCCESS;
  88 }
  89
  90
  91 static git_odb_object *new_odb_object(const git_oid *oid, git_rawobj *source)
  92 {
  93         git_odb_object *object = git__malloc(sizeof(git_odb_object));
  94         memset(object, 0x0, sizeof(git_odb_object));
  95
  96         git_oid_cpy(&object->cached.oid, oid);
  97         memcpy(&object->raw, source, sizeof(git_rawobj));
  98
  99         return object;
 100 }
 101
 102 static void free_odb_object(void *o)
 103 {
 104         git_odb_object *object = (git_odb_object *)o;
 105
 106         if (object != NULL) {
 107                 free(object->raw.data);
 108                 free(object);
 109         }
 110 }
 111
 112 const git_oid *git_odb_object_id(git_odb_object *object)
 113 {
 114         return &object->cached.oid;
 115 }
 116
 117 const void *git_odb_object_data(git_odb_object *object)
 118 {
 119         return object->raw.data;
 120 }
 121
 122 size_t git_odb_object_size(git_odb_object *object)
 123 {
 124         return object->raw.len;
 125 }
 126
 127 git_otype git_odb_object_type(git_odb_object *object)
 128 {
 129         return object->raw.type;
 130 }
 131
 132 void git_odb_object_close(git_odb_object *object)
 133 {
 134         git_cached_obj_decref((git_cached_obj *)object, &free_odb_object);
 135 }
 136
 137 int git_odb_hash(git_oid *id, const void *data, size_t len, git_otype type)
 138 {
 139         char hdr[64];
 140         int  hdrlen;
 141         git_rawobj raw;
 142
 143         assert(id);
 144
 145         raw.data = (void *)data;
 146         raw.len = len;
 147         raw.type = type;
 148
 149         return git_odb__hash_obj(id, hdr, sizeof(hdr), &hdrlen, &raw);
 150 }
 151
 152 /**
 153  * FAKE WSTREAM
 154  */
 155
 156 typedef struct {
 157         git_odb_stream stream;
 158         char *buffer;
 159         size_t size, written;
 160         git_otype type;
 161 } fake_wstream;
 162
 163 static int fake_wstream__fwrite(git_oid *oid, git_odb_stream *_stream)
 164 {
 165         fake_wstream *stream = (fake_wstream *)_stream;
 166         return _stream->backend->write(oid, _stream->backend, stream->buffer, stream->size, stream->type);
 167 }
 168
 169 static int fake_wstream__write(git_odb_stream *_stream, const char *data, size_t len)
 170 {
 171         fake_wstream *stream = (fake_wstream *)_stream;
 172
 173         if (stream->written + len > stream->size)
 174                 return GIT_ENOMEM;
 175
 176         memcpy(stream->buffer + stream->written, data, len);
 177         stream->written += len;
 178         return GIT_SUCCESS;
 179 }
 180
 181 static void fake_wstream__free(git_odb_stream *_stream)
 182 {
 183         fake_wstream *stream = (fake_wstream *)_stream;
 184
 185         free(stream->buffer);
 186         free(stream);
 187 }
 188
 189 static int init_fake_wstream(git_odb_stream **stream_p, git_odb_backend *backend, size_t size, git_otype type)
 190 {
 191         fake_wstream *stream;
 192
 193         stream = git__calloc(1, sizeof(fake_wstream));
 194         if (stream == NULL)
 195                 return GIT_ENOMEM;
 196
 197         stream->size = size;
 198         stream->type = type;
 199         stream->buffer = git__malloc(size);
 200         if (stream->buffer == NULL) {
 201                 free(stream);
 202                 return GIT_ENOMEM;
 203         }
 204
 205         stream->stream.backend = backend;
 206         stream->stream.read = NULL; /* read only */
 207         stream->stream.write = &fake_wstream__write;
 208         stream->stream.finalize_write = &fake_wstream__fwrite;
 209         stream->stream.free = &fake_wstream__free;
 210         stream->stream.mode = GIT_STREAM_WRONLY;
 211
 212         *stream_p = (git_odb_stream *)stream;
 213         return GIT_SUCCESS;
 214 }
 215
 216 /***********************************************************
 217  *
 218  * OBJECT DATABASE PUBLIC API
 219  *
 220  * Public calls for the ODB functionality
 221  *
 222  ***********************************************************/
 223
 224 static int backend_sort_cmp(const void *a, const void *b)
 225 {
 226         const backend_internal *backend_a = *(const backend_internal **)(a);
 227         const backend_internal *backend_b = *(const backend_internal **)(b);
 228
 229         if (backend_a->is_alternate == backend_b->is_alternate)
 230                 return (backend_b->priority - backend_a->priority);
 231
 232         return backend_a->is_alternate ? 1 : -1;
 233 }
 234
 235 int git_odb_new(git_odb **out)
 236 {
 237         int error;
 238
 239         git_odb *db = git__calloc(1, sizeof(*db));
 240         if (!db)
 241                 return GIT_ENOMEM;
 242
 243         error = git_cache_init(&db->cache, GIT_DEFAULT_CACHE_SIZE, &free_odb_object);
 244         if (error < GIT_SUCCESS) {
 245                 free(db);
 246                 return git__rethrow(error, "Failed to create object database");
 247         }
 248
 249         if ((error = git_vector_init(&db->backends, 4, backend_sort_cmp)) < GIT_SUCCESS) {
 250                 free(db);
 251                 return git__rethrow(error, "Failed to create object database");
 252         }
 253
 254         *out = db;
 255         return GIT_SUCCESS;
 256 }
 257
 258 static int add_backend_internal(git_odb *odb, git_odb_backend *backend, int priority, int is_alternate)
 259 {
 260         backend_internal *internal;
 261
 262         assert(odb && backend);
 263
 264         if (backend->odb != NULL && backend->odb != odb)
 265                 return git__throw(GIT_EBUSY, "The backend is already owned by another ODB");
 266
 267         internal = git__malloc(sizeof(backend_internal));
 268         if (internal == NULL)
 269                 return GIT_ENOMEM;
 270
 271         internal->backend = backend;
 272         internal->priority = priority;
 273         internal->is_alternate = is_alternate;
 274
 275         if (git_vector_insert(&odb->backends, internal) < 0) {
 276                 free(internal);
 277                 return GIT_ENOMEM;
 278         }
 279
 280         git_vector_sort(&odb->backends);
 281         internal->backend->odb = odb;
 282         return GIT_SUCCESS;
 283 }
 284
 285 int git_odb_add_backend(git_odb *odb, git_odb_backend *backend, int priority)
 286 {
 287         return add_backend_internal(odb, backend, priority, 0);
 288 }
 289
 290 int git_odb_add_alternate(git_odb *odb, git_odb_backend *backend, int priority)
 291 {
 292         return add_backend_internal(odb, backend, priority, 1);
 293 }
 294
 295 static int add_default_backends(git_odb *db, const char *objects_dir, int as_alternates)
 296 {
 297         git_odb_backend *loose, *packed;
 298         int error;
 299
 300         /* add the loose object backend */
 301         error = git_odb_backend_loose(&loose, objects_dir);
 302         if (error < GIT_SUCCESS)
 303                 return error;
 304
 305         error = add_backend_internal(db, loose, GIT_LOOSE_PRIORITY, as_alternates);
 306         if (error < GIT_SUCCESS)
 307                 return git__rethrow(error, "Failed to add backend");
 308
 309         /* add the packed file backend */
 310         error = git_odb_backend_pack(&packed, objects_dir);
 311         if (error < GIT_SUCCESS)
 312                 return error;
 313
 314         error = add_backend_internal(db, packed, GIT_PACKED_PRIORITY, as_alternates);
 315         if (error < GIT_SUCCESS)
 316                 return git__rethrow(error, "Failed to add backend");
 317
 318         return GIT_SUCCESS;
 319 }
 320
 321 static int load_alternates(git_odb *odb, const char *objects_dir)
 322 {
 323         char alternates_path[GIT_PATH_MAX];
 324         char *buffer, *alternate;
 325
 326         gitfo_buf alternates_buf = GITFO_BUF_INIT;
 327         int error;
 328
 329         git__joinpath(alternates_path, objects_dir, GIT_ALTERNATES_FILE);
 330
 331         if (gitfo_exists(alternates_path) < GIT_SUCCESS)
 332                 return GIT_SUCCESS;
 333
 334         if (gitfo_read_file(&alternates_buf, alternates_path) < GIT_SUCCESS)
 335                 return git__throw(GIT_EOSERR, "Failed to add backend. Can't read alternates");
 336
 337         buffer = (char *)alternates_buf.data;
 338         error = GIT_SUCCESS;
 339
 340         /* add each alternate as a new backend; one alternate per line */
 341         while ((alternate = git__strtok(&buffer, "\r\n")) != NULL) {
 342                 char full_path[GIT_PATH_MAX];
 343
 344                 if (*alternate == '\0' || *alternate == '#')
 345                         continue;
 346
 347                 /* relative path: build based on the current `objects` folder */
 348                 if (*alternate == '.') {
 349                         git__joinpath(full_path, objects_dir, alternate);
 350                         alternate = full_path;
 351                 }
 352
 353                 if ((error = add_default_backends(odb, alternate, 1)) < GIT_SUCCESS)
 354                         break;
 355         }
 356
 357         gitfo_free_buf(&alternates_buf);
 358         if (error < GIT_SUCCESS)
 359                 return git__rethrow(error, "Failed to load alternates");
 360         return error;
 361 }
 362
 363 int git_odb_open(git_odb **out, const char *objects_dir)
 364 {
 365         git_odb *db;
 366         int error;
 367
 368         assert(out && objects_dir);
 369
 370         *out = NULL;
 371
 372         if ((error = git_odb_new(&db)) < 0)
 373                 return git__rethrow(error, "Failed to open ODB");
 374
 375         if ((error = add_default_backends(db, objects_dir, 0)) < GIT_SUCCESS)
 376                 goto cleanup;
 377
 378         if ((error = load_alternates(db, objects_dir)) < GIT_SUCCESS)
 379                 goto cleanup;
 380
 381         *out = db;
 382         return GIT_SUCCESS;
 383
 384 cleanup:
 385         git_odb_close(db);
 386         return error; /* error already set - pass through */
 387 }
 388
 389 void git_odb_close(git_odb *db)
 390 {
 391         unsigned int i;
 392
 393         if (db == NULL)
 394                 return;
 395
 396         for (i = 0; i < db->backends.length; ++i) {
 397                 backend_internal *internal = git_vector_get(&db->backends, i);
 398                 git_odb_backend *backend = internal->backend;
 399
 400                 if (backend->free) backend->free(backend);
 401                 else free(backend);
 402
 403                 free(internal);
 404         }
 405
 406         git_vector_free(&db->backends);
 407         git_cache_free(&db->cache);
 408         free(db);
 409 }
 410
 411 int git_odb_exists(git_odb *db, const git_oid *id)
 412 {
 413         git_odb_object *object;
 414         unsigned int i;
 415         int found = 0;
 416
 417         assert(db && id);
 418
 419         if ((object = git_cache_get(&db->cache, id)) != NULL) {
 420                 git_odb_object_close(object);
 421                 return 1;
 422         }
 423
 424         for (i = 0; i < db->backends.length && !found; ++i) {
 425                 backend_internal *internal = git_vector_get(&db->backends, i);
 426                 git_odb_backend *b = internal->backend;
 427
 428                 if (b->exists != NULL)
 429                         found = b->exists(b, id);
 430         }
 431
 432         return found;
 433 }
 434
 435 int git_odb_read_header(size_t *len_p, git_otype *type_p, git_odb *db, const git_oid *id)
 436 {
 437         unsigned int i;
 438         int error = GIT_ENOTFOUND;
 439         git_odb_object *object;
 440
 441         assert(db && id);
 442
 443         if ((object = git_cache_get(&db->cache, id)) != NULL) {
 444                 *len_p = object->raw.len;
 445                 *type_p = object->raw.type;
 446                 git_odb_object_close(object);
 447                 return GIT_SUCCESS;
 448         }
 449
 450         for (i = 0; i < db->backends.length && error < 0; ++i) {
 451                 backend_internal *internal = git_vector_get(&db->backends, i);
 452                 git_odb_backend *b = internal->backend;
 453
 454                 if (b->read_header != NULL)
 455                         error = b->read_header(len_p, type_p, b, id);
 456         }
 457
 458         if (error == GIT_EPASSTHROUGH)
 459                 return GIT_SUCCESS;
 460
 461         /*
 462          * no backend could read only the header.
 463          * try reading the whole object and freeing the contents
 464          */
 465         if (error < 0) {
 466                 if ((error = git_odb_read(&object, db, id)) < GIT_SUCCESS)
 467                         return error; /* error already set - pass through */
 468
 469                 *len_p = object->raw.len;
 470                 *type_p = object->raw.type;
 471                 git_odb_object_close(object);
 472         }
 473
 474         return GIT_SUCCESS;
 475 }
 476
 477 int git_odb_read(git_odb_object **out, git_odb *db, const git_oid *id)
 478 {
 479         unsigned int i;
 480         int error = GIT_ENOTFOUND;
 481         git_rawobj raw;
 482
 483         assert(out && db && id);
 484
 485         *out = git_cache_get(&db->cache, id);
 486         if (*out != NULL)
 487                 return GIT_SUCCESS;
 488
 489         for (i = 0; i < db->backends.length && error < 0; ++i) {
 490                 backend_internal *internal = git_vector_get(&db->backends, i);
 491                 git_odb_backend *b = internal->backend;
 492
 493                 if (b->read != NULL)
 494                         error = b->read(&raw.data, &raw.len, &raw.type, b, id);
 495         }
 496
 497         if (error == GIT_EPASSTHROUGH || error == GIT_SUCCESS) {
 498                 *out = git_cache_try_store(&db->cache, new_odb_object(id, &raw));
 499                 return GIT_SUCCESS;
 500         }
 501
 502         return git__rethrow(error, "Failed to read object");
 503 }
 504
 505 int git_odb_read_prefix(git_odb_object **out, git_odb *db, const git_oid *short_id, unsigned int len)
 506 {
 507         unsigned int i;
 508         int error = GIT_ENOTFOUND;
 509         git_oid full_oid;
 510         git_rawobj raw;
 511         int found = 0;
 512
 513         assert(out && db);
 514
 515         if (len < GIT_OID_MINPREFIXLEN)
 516                 return git__throw(GIT_EAMBIGUOUSOIDPREFIX, "Failed to lookup object. Prefix length is lower than %d.", GIT_OID_MINPREFIXLEN);
 517
 518         if (len > GIT_OID_HEXSZ)
 519                 len = GIT_OID_HEXSZ;
 520
 521         if (len == GIT_OID_HEXSZ) {
 522                 *out = git_cache_get(&db->cache, short_id);
 523                 if (*out != NULL)
 524                         return GIT_SUCCESS;
 525         }
 526
 527         for (i = 0; i < db->backends.length && found < 2; ++i) {
 528                 backend_internal *internal = git_vector_get(&db->backends, i);
 529                 git_odb_backend *b = internal->backend;
 530
 531                 if (b->read != NULL) {
 532                         error = b->read_prefix(&full_oid, &raw.data, &raw.len, &raw.type, b, short_id, len);
 533                         switch (error) {
 534                         case GIT_SUCCESS:
 535                                 found++;
 536                                 break;
 537                         case GIT_ENOTFOUND:
 538                         case GIT_EPASSTHROUGH:
 539                                 break;
 540                         case GIT_EAMBIGUOUSOIDPREFIX:
 541                                 return git__rethrow(error, "Failed to read object. Ambiguous sha1 prefix");
 542                         default:
 543                                 return git__rethrow(error, "Failed to read object");
 544                         }
 545                 }
 546         }
 547
 548         if (found == 1) {
 549                 *out = git_cache_try_store(&db->cache, new_odb_object(&full_oid, &raw));
 550         } else if (found > 1) {
 551                 return git__throw(GIT_EAMBIGUOUSOIDPREFIX, "Failed to read object. Ambiguous sha1 prefix");
 552         } else {
 553                 return git__throw(GIT_ENOTFOUND, "Failed to read object. Object not found");
 554         }
 555
 556         return GIT_SUCCESS;
 557 }
 558
 559 int git_odb_write(git_oid *oid, git_odb *db, const void *data, size_t len, git_otype type)
 560 {
 561         unsigned int i;
 562         int error = GIT_ERROR;
 563         git_odb_stream *stream;
 564
 565         assert(oid && db);
 566
 567         for (i = 0; i < db->backends.length && error < 0; ++i) {
 568                 backend_internal *internal = git_vector_get(&db->backends, i);
 569                 git_odb_backend *b = internal->backend;
 570
 571                 /* we don't write in alternates! */
 572                 if (internal->is_alternate)
 573                         continue;
 574
 575                 if (b->write != NULL)
 576                         error = b->write(oid, b, data, len, type);
 577         }
 578
 579         if (error == GIT_EPASSTHROUGH || error == GIT_SUCCESS)
 580                 return GIT_SUCCESS;
 581
 582         /* if no backends were able to write the object directly, we try a streaming
 583          * write to the backends; just write the whole object into the stream in one
 584          * push */
 585
 586         if ((error = git_odb_open_wstream(&stream, db, len, type)) == GIT_SUCCESS) {
 587                 stream->write(stream, data, len);
 588                 error = stream->finalize_write(oid, stream);
 589                 stream->free(stream);
 590                 return GIT_SUCCESS;
 591         }
 592
 593         return git__rethrow(error, "Failed to write object");
 594 }
 595
 596 int git_odb_open_wstream(git_odb_stream **stream, git_odb *db, size_t size, git_otype type)
 597 {
 598         unsigned int i;
 599         int error = GIT_ERROR;
 600
 601         assert(stream && db);
 602
 603         for (i = 0; i < db->backends.length && error < 0; ++i) {
 604                 backend_internal *internal = git_vector_get(&db->backends, i);
 605                 git_odb_backend *b = internal->backend;
 606
 607                 /* we don't write in alternates! */
 608                 if (internal->is_alternate)
 609                         continue;
 610
 611                 if (b->writestream != NULL)
 612                         error = b->writestream(stream, b, size, type);
 613                 else if (b->write != NULL)
 614                         error = init_fake_wstream(stream, b, size, type);
 615         }
 616
 617         if (error == GIT_EPASSTHROUGH || error == GIT_SUCCESS)
 618                 return GIT_SUCCESS;
 619
 620         return git__rethrow(error, "Failed to open write stream");
 621 }
 622
 623 int git_odb_open_rstream(git_odb_stream **stream, git_odb *db, const git_oid *oid)
 624 {
 625         unsigned int i;
 626         int error = GIT_ERROR;
 627
 628         assert(stream && db);
 629
 630         for (i = 0; i < db->backends.length && error < 0; ++i) {
 631                 backend_internal *internal = git_vector_get(&db->backends, i);
 632                 git_odb_backend *b = internal->backend;
 633
 634                 if (b->readstream != NULL)
 635                         error = b->readstream(stream, b, oid);
 636         }
 637
 638         if (error == GIT_EPASSTHROUGH || error == GIT_SUCCESS)
 639                 return GIT_SUCCESS;
 640
 641         return git__rethrow(error, "Failed to open read stream");
 642 }
 643