src/odb.c

   1 /*
   2  * Copyright (C) 2009-2012 the libgit2 contributors
   3  *
   4  * This file is part of libgit2, distributed under the GNU GPL v2 with
   5  * a Linking Exception. For full terms see the included COPYING file.
   6  */
   7
   8 #include "common.h"
   9 #include <zlib.h>
  10 #include "git2/object.h"
  11 #include "fileops.h"
  12 #include "hash.h"
  13 #include "odb.h"
  14 #include "delta-apply.h"
  15
  16 #include "git2/odb_backend.h"
  17 #include "git2/oid.h"
  18
  19 #define GIT_ALTERNATES_FILE "info/alternates"
  20
  21 /* TODO: is this correct? */
  22 #define GIT_LOOSE_PRIORITY 2
  23 #define GIT_PACKED_PRIORITY 1
  24
  25 typedef struct
  26 {
  27         git_odb_backend *backend;
  28         int priority;
  29         int is_alternate;
  30 } backend_internal;
  31
  32 static int format_object_header(char *hdr, size_t n, size_t obj_len, git_otype obj_type)
  33 {
  34         const char *type_str = git_object_type2string(obj_type);
  35         int len = p_snprintf(hdr, n, "%s %"PRIuZ, type_str, obj_len);
  36         assert(len > 0 && len <= (int)n);
  37         return len+1;
  38 }
  39
  40 int git_odb__hashobj(git_oid *id, git_rawobj *obj)
  41 {
  42         git_buf_vec vec[2];
  43         char header[64];
  44         int hdrlen;
  45
  46         assert(id && obj);
  47
  48         if (!git_object_typeisloose(obj->type))
  49                 return -1;
  50         if (!obj->data && obj->len != 0)
  51                 return -1;
  52
  53         hdrlen = format_object_header(header, sizeof(header), obj->len, obj->type);
  54
  55         vec[0].data = header;
  56         vec[0].len = hdrlen;
  57         vec[1].data = obj->data;
  58         vec[1].len = obj->len;
  59
  60         git_hash_vec(id, vec, 2);
  61
  62         return 0;
  63 }
  64
  65
  66 static git_odb_object *new_odb_object(const git_oid *oid, git_rawobj *source)
  67 {
  68         git_odb_object *object = git__malloc(sizeof(git_odb_object));
  69         memset(object, 0x0, sizeof(git_odb_object));
  70
  71         git_oid_cpy(&object->cached.oid, oid);
  72         memcpy(&object->raw, source, sizeof(git_rawobj));
  73
  74         return object;
  75 }
  76
  77 static void free_odb_object(void *o)
  78 {
  79         git_odb_object *object = (git_odb_object *)o;
  80
  81         if (object != NULL) {
  82                 git__free(object->raw.data);
  83                 git__free(object);
  84         }
  85 }
  86
  87 const git_oid *git_odb_object_id(git_odb_object *object)
  88 {
  89         return &object->cached.oid;
  90 }
  91
  92 const void *git_odb_object_data(git_odb_object *object)
  93 {
  94         return object->raw.data;
  95 }
  96
  97 size_t git_odb_object_size(git_odb_object *object)
  98 {
  99         return object->raw.len;
 100 }
 101
 102 git_otype git_odb_object_type(git_odb_object *object)
 103 {
 104         return object->raw.type;
 105 }
 106
 107 void git_odb_object_free(git_odb_object *object)
 108 {
 109         git_cached_obj_decref((git_cached_obj *)object, &free_odb_object);
 110 }
 111
 112 int git_odb__hashfd(git_oid *out, git_file fd, size_t size, git_otype type)
 113 {
 114         int hdr_len;
 115         char hdr[64], buffer[2048];
 116         git_hash_ctx *ctx;
 117
 118         hdr_len = format_object_header(hdr, sizeof(hdr), size, type);
 119
 120         ctx = git_hash_new_ctx();
 121
 122         git_hash_update(ctx, hdr, hdr_len);
 123
 124         while (size > 0) {
 125                 ssize_t read_len = read(fd, buffer, sizeof(buffer));
 126
 127                 if (read_len < 0) {
 128                         git_hash_free_ctx(ctx);
 129                         giterr_set(GITERR_OS, "Error reading file");
 130                         return -1;
 131                 }
 132
 133                 git_hash_update(ctx, buffer, read_len);
 134                 size -= read_len;
 135         }
 136
 137         git_hash_final(out, ctx);
 138         git_hash_free_ctx(ctx);
 139
 140         return 0;
 141 }
 142
 143 int git_odb__hashlink(git_oid *out, const char *path)
 144 {
 145         struct stat st;
 146         git_off_t size;
 147         int result;
 148
 149         if (git_path_lstat(path, &st) < 0)
 150                 return -1;
 151
 152         size = st.st_size;
 153
 154         if (!git__is_sizet(size)) {
 155                 giterr_set(GITERR_OS, "File size overflow for 32-bit systems");
 156                 return -1;
 157         }
 158
 159         if (S_ISLNK(st.st_mode)) {
 160                 char *link_data;
 161                 ssize_t read_len;
 162
 163                 link_data = git__malloc((size_t)size);
 164                 GITERR_CHECK_ALLOC(link_data);
 165
 166                 read_len = p_readlink(path, link_data, (size_t)(size + 1));
 167                 if (read_len != (ssize_t)size) {
 168                         giterr_set(GITERR_OS, "Failed to read symlink data for '%s'", path);
 169                         return -1;
 170                 }
 171
 172                 result = git_odb_hash(out, link_data, (size_t)size, GIT_OBJ_BLOB);
 173                 git__free(link_data);
 174         } else {
 175                 int fd = git_futils_open_ro(path);
 176                 if (fd < 0)
 177                         return -1;
 178                 result = git_odb__hashfd(out, fd, (size_t)size, GIT_OBJ_BLOB);
 179                 p_close(fd);
 180         }
 181
 182         return result;
 183 }
 184
 185 int git_odb_hashfile(git_oid *out, const char *path, git_otype type)
 186 {
 187         git_off_t size;
 188         int result, fd = git_futils_open_ro(path);
 189         if (fd < 0)
 190                 return fd;
 191
 192         if ((size = git_futils_filesize(fd)) < 0 || !git__is_sizet(size)) {
 193                 giterr_set(GITERR_OS, "File size overflow for 32-bit systems");
 194                 p_close(fd);
 195                 return -1;
 196         }
 197
 198         result = git_odb__hashfd(out, fd, (size_t)size, type);
 199         p_close(fd);
 200         return result;
 201 }
 202
 203 int git_odb_hash(git_oid *id, const void *data, size_t len, git_otype type)
 204 {
 205         git_rawobj raw;
 206
 207         assert(id);
 208
 209         raw.data = (void *)data;
 210         raw.len = len;
 211         raw.type = type;
 212
 213         return git_odb__hashobj(id, &raw);
 214 }
 215
 216 /**
 217  * FAKE WSTREAM
 218  */
 219
 220 typedef struct {
 221         git_odb_stream stream;
 222         char *buffer;
 223         size_t size, written;
 224         git_otype type;
 225 } fake_wstream;
 226
 227 static int fake_wstream__fwrite(git_oid *oid, git_odb_stream *_stream)
 228 {
 229         fake_wstream *stream = (fake_wstream *)_stream;
 230         return _stream->backend->write(oid, _stream->backend, stream->buffer, stream->size, stream->type);
 231 }
 232
 233 static int fake_wstream__write(git_odb_stream *_stream, const char *data, size_t len)
 234 {
 235         fake_wstream *stream = (fake_wstream *)_stream;
 236
 237         if (stream->written + len > stream->size)
 238                 return -1;
 239
 240         memcpy(stream->buffer + stream->written, data, len);
 241         stream->written += len;
 242         return 0;
 243 }
 244
 245 static void fake_wstream__free(git_odb_stream *_stream)
 246 {
 247         fake_wstream *stream = (fake_wstream *)_stream;
 248
 249         git__free(stream->buffer);
 250         git__free(stream);
 251 }
 252
 253 static int init_fake_wstream(git_odb_stream **stream_p, git_odb_backend *backend, size_t size, git_otype type)
 254 {
 255         fake_wstream *stream;
 256
 257         stream = git__calloc(1, sizeof(fake_wstream));
 258         GITERR_CHECK_ALLOC(stream);
 259
 260         stream->size = size;
 261         stream->type = type;
 262         stream->buffer = git__malloc(size);
 263         if (stream->buffer == NULL) {
 264                 git__free(stream);
 265                 return -1;
 266         }
 267
 268         stream->stream.backend = backend;
 269         stream->stream.read = NULL; /* read only */
 270         stream->stream.write = &fake_wstream__write;
 271         stream->stream.finalize_write = &fake_wstream__fwrite;
 272         stream->stream.free = &fake_wstream__free;
 273         stream->stream.mode = GIT_STREAM_WRONLY;
 274
 275         *stream_p = (git_odb_stream *)stream;
 276         return 0;
 277 }
 278
 279 /***********************************************************
 280  *
 281  * OBJECT DATABASE PUBLIC API
 282  *
 283  * Public calls for the ODB functionality
 284  *
 285  ***********************************************************/
 286
 287 static int backend_sort_cmp(const void *a, const void *b)
 288 {
 289         const backend_internal *backend_a = (const backend_internal *)(a);
 290         const backend_internal *backend_b = (const backend_internal *)(b);
 291
 292         if (backend_a->is_alternate == backend_b->is_alternate)
 293                 return (backend_b->priority - backend_a->priority);
 294
 295         return backend_a->is_alternate ? 1 : -1;
 296 }
 297
 298 int git_odb_new(git_odb **out)
 299 {
 300         git_odb *db = git__calloc(1, sizeof(*db));
 301         GITERR_CHECK_ALLOC(db);
 302
 303         if (git_cache_init(&db->cache, GIT_DEFAULT_CACHE_SIZE, &free_odb_object) < 0 ||
 304                 git_vector_init(&db->backends, 4, backend_sort_cmp) < 0)
 305         {
 306                 git__free(db);
 307                 return -1;
 308         }
 309
 310         *out = db;
 311         GIT_REFCOUNT_INC(db);
 312         return 0;
 313 }
 314
 315 static int add_backend_internal(git_odb *odb, git_odb_backend *backend, int priority, int is_alternate)
 316 {
 317         backend_internal *internal;
 318
 319         assert(odb && backend);
 320
 321         /* Check if the backend is already owned by another ODB */
 322         assert(!backend->odb || backend->odb == odb);
 323
 324         internal = git__malloc(sizeof(backend_internal));
 325         GITERR_CHECK_ALLOC(internal);
 326
 327         internal->backend = backend;
 328         internal->priority = priority;
 329         internal->is_alternate = is_alternate;
 330
 331         if (git_vector_insert(&odb->backends, internal) < 0) {
 332                 git__free(internal);
 333                 return -1;
 334         }
 335
 336         git_vector_sort(&odb->backends);
 337         internal->backend->odb = odb;
 338         return 0;
 339 }
 340
 341 int git_odb_add_backend(git_odb *odb, git_odb_backend *backend, int priority)
 342 {
 343         return add_backend_internal(odb, backend, priority, 0);
 344 }
 345
 346 int git_odb_add_alternate(git_odb *odb, git_odb_backend *backend, int priority)
 347 {
 348         return add_backend_internal(odb, backend, priority, 1);
 349 }
 350
 351 static int add_default_backends(git_odb *db, const char *objects_dir, int as_alternates)
 352 {
 353         git_odb_backend *loose, *packed;
 354
 355         /* add the loose object backend */
 356         if (git_odb_backend_loose(&loose, objects_dir, -1, 0) < 0 ||
 357                 add_backend_internal(db, loose, GIT_LOOSE_PRIORITY, as_alternates) < 0)
 358                 return -1;
 359
 360         /* add the packed file backend */
 361         if (git_odb_backend_pack(&packed, objects_dir) < 0 ||
 362                 add_backend_internal(db, packed, GIT_PACKED_PRIORITY, as_alternates) < 0)
 363                 return -1;
 364
 365         return 0;
 366 }
 367
 368 static int load_alternates(git_odb *odb, const char *objects_dir)
 369 {
 370         git_buf alternates_path = GIT_BUF_INIT;
 371         git_buf alternates_buf = GIT_BUF_INIT;
 372         char *buffer;
 373         const char *alternate;
 374         int result = 0;
 375
 376         if (git_buf_joinpath(&alternates_path, objects_dir, GIT_ALTERNATES_FILE) < 0)
 377                 return -1;
 378
 379         if (git_path_exists(alternates_path.ptr) == false) {
 380                 git_buf_free(&alternates_path);
 381                 return 0;
 382         }
 383
 384         if (git_futils_readbuffer(&alternates_buf, alternates_path.ptr) < 0) {
 385                 git_buf_free(&alternates_path);
 386                 return -1;
 387         }
 388
 389         buffer = (char *)alternates_buf.ptr;
 390
 391         /* add each alternate as a new backend; one alternate per line */
 392         while ((alternate = git__strtok(&buffer, "\r\n")) != NULL) {
 393                 if (*alternate == '\0' || *alternate == '#')
 394                         continue;
 395
 396                 /* relative path: build based on the current `objects` folder */
 397                 if (*alternate == '.') {
 398                         if ((result = git_buf_joinpath(&alternates_path, objects_dir, alternate)) < 0)
 399                                 break;
 400                         alternate = git_buf_cstr(&alternates_path);
 401                 }
 402
 403                 if ((result = add_default_backends(odb, alternate, 1)) < 0)
 404                         break;
 405         }
 406
 407         git_buf_free(&alternates_path);
 408         git_buf_free(&alternates_buf);
 409
 410         return result;
 411 }
 412
 413 int git_odb_open(git_odb **out, const char *objects_dir)
 414 {
 415         git_odb *db;
 416
 417         assert(out && objects_dir);
 418
 419         *out = NULL;
 420
 421         if (git_odb_new(&db) < 0)
 422                 return -1;
 423
 424         if (add_default_backends(db, objects_dir, 0) < 0 ||
 425                 load_alternates(db, objects_dir) < 0)
 426         {
 427                 git_odb_free(db);
 428                 return -1;
 429         }
 430
 431         *out = db;
 432         return 0;
 433 }
 434
 435 static void odb_free(git_odb *db)
 436 {
 437         unsigned int i;
 438
 439         for (i = 0; i < db->backends.length; ++i) {
 440                 backend_internal *internal = git_vector_get(&db->backends, i);
 441                 git_odb_backend *backend = internal->backend;
 442
 443                 if (backend->free) backend->free(backend);
 444                 else git__free(backend);
 445
 446                 git__free(internal);
 447         }
 448
 449         git_vector_free(&db->backends);
 450         git_cache_free(&db->cache);
 451         git__free(db);
 452 }
 453
 454 void git_odb_free(git_odb *db)
 455 {
 456         if (db == NULL)
 457                 return;
 458
 459         GIT_REFCOUNT_DEC(db, odb_free);
 460 }
 461
 462 int git_odb_exists(git_odb *db, const git_oid *id)
 463 {
 464         git_odb_object *object;
 465         unsigned int i;
 466         bool found = false;
 467
 468         assert(db && id);
 469
 470         if ((object = git_cache_get(&db->cache, id)) != NULL) {
 471                 git_odb_object_free(object);
 472                 return (int)true;
 473         }
 474
 475         for (i = 0; i < db->backends.length && !found; ++i) {
 476                 backend_internal *internal = git_vector_get(&db->backends, i);
 477                 git_odb_backend *b = internal->backend;
 478
 479                 if (b->exists != NULL)
 480                         found = b->exists(b, id);
 481         }
 482
 483         return (int)found;
 484 }
 485
 486 int git_odb_read_header(size_t *len_p, git_otype *type_p, git_odb *db, const git_oid *id)
 487 {
 488         unsigned int i;
 489         int error = GIT_ENOTFOUND;
 490         git_odb_object *object;
 491
 492         assert(db && id);
 493
 494         if ((object = git_cache_get(&db->cache, id)) != NULL) {
 495                 *len_p = object->raw.len;
 496                 *type_p = object->raw.type;
 497                 git_odb_object_free(object);
 498                 return 0;
 499         }
 500
 501         for (i = 0; i < db->backends.length && error < 0; ++i) {
 502                 backend_internal *internal = git_vector_get(&db->backends, i);
 503                 git_odb_backend *b = internal->backend;
 504
 505                 if (b->read_header != NULL)
 506                         error = b->read_header(len_p, type_p, b, id);
 507         }
 508
 509         if (!error || error == GIT_PASSTHROUGH)
 510                 return 0;
 511
 512         /*
 513          * no backend could read only the header.
 514          * try reading the whole object and freeing the contents
 515          */
 516         if ((error = git_odb_read(&object, db, id)) < 0)
 517                 return error; /* error already set - pass along */
 518
 519         *len_p = object->raw.len;
 520         *type_p = object->raw.type;
 521         git_odb_object_free(object);
 522         return 0;
 523 }
 524
 525 int git_odb_read(git_odb_object **out, git_odb *db, const git_oid *id)
 526 {
 527         unsigned int i;
 528         int error = GIT_ENOTFOUND;
 529         git_rawobj raw;
 530
 531         assert(out && db && id);
 532
 533         *out = git_cache_get(&db->cache, id);
 534         if (*out != NULL)
 535                 return 0;
 536
 537         for (i = 0; i < db->backends.length && error < 0; ++i) {
 538                 backend_internal *internal = git_vector_get(&db->backends, i);
 539                 git_odb_backend *b = internal->backend;
 540
 541                 if (b->read != NULL)
 542                         error = b->read(&raw.data, &raw.len, &raw.type, b, id);
 543         }
 544
 545         /* TODO: If no backends are configured, this returns GIT_ENOTFOUND but
 546          * will never have called giterr_set().
 547          */
 548
 549         if (error && error != GIT_PASSTHROUGH)
 550                 return error;
 551
 552         *out = git_cache_try_store(&db->cache, new_odb_object(id, &raw));
 553         return 0;
 554 }
 555
 556 int git_odb_read_prefix(
 557         git_odb_object **out, git_odb *db, const git_oid *short_id, size_t len)
 558 {
 559         unsigned int i;
 560         int error = GIT_ENOTFOUND;
 561         git_oid found_full_oid = {{0}};
 562         git_rawobj raw;
 563         void *data = NULL;
 564         bool found = false;
 565
 566         assert(out && db);
 567
 568         if (len < GIT_OID_MINPREFIXLEN)
 569                 return git_odb__error_ambiguous("prefix length too short");
 570
 571         if (len > GIT_OID_HEXSZ)
 572                 len = GIT_OID_HEXSZ;
 573
 574         if (len == GIT_OID_HEXSZ) {
 575                 *out = git_cache_get(&db->cache, short_id);
 576                 if (*out != NULL)
 577                         return 0;
 578         }
 579
 580         for (i = 0; i < db->backends.length; ++i) {
 581                 backend_internal *internal = git_vector_get(&db->backends, i);
 582                 git_odb_backend *b = internal->backend;
 583
 584                 if (b->read != NULL) {
 585                         git_oid full_oid;
 586                         error = b->read_prefix(&full_oid, &raw.data, &raw.len, &raw.type, b, short_id, len);
 587                         if (error == GIT_ENOTFOUND || error == GIT_PASSTHROUGH)
 588                                 continue;
 589
 590                         if (error)
 591                                 return error;
 592
 593                         git__free(data);
 594                         data = raw.data;
 595                         if (found && git_oid_cmp(&full_oid, &found_full_oid))
 596                                 return git_odb__error_ambiguous("multiple matches for prefix");
 597                         found_full_oid = full_oid;
 598                         found = true;
 599                 }
 600         }
 601
 602         if (!found)
 603                 return git_odb__error_notfound("no match for prefix", short_id);
 604
 605         *out = git_cache_try_store(&db->cache, new_odb_object(&found_full_oid, &raw));
 606         return 0;
 607 }
 608
 609 int git_odb_foreach(git_odb *db, int (*cb)(git_oid *oid, void *data), void *data)
 610 {
 611         unsigned int i;
 612         backend_internal *internal;
 613
 614         git_vector_foreach(&db->backends, i, internal) {
 615                 git_odb_backend *b = internal->backend;
 616                 int error = b->foreach(b, cb, data);
 617                 if (error < 0)
 618                         return error;
 619         }
 620
 621         return 0;
 622 }
 623
 624 int git_odb_write(
 625         git_oid *oid, git_odb *db, const void *data, size_t len, git_otype type)
 626 {
 627         unsigned int i;
 628         int error = GIT_ERROR;
 629         git_odb_stream *stream;
 630
 631         assert(oid && db);
 632
 633         for (i = 0; i < db->backends.length && error < 0; ++i) {
 634                 backend_internal *internal = git_vector_get(&db->backends, i);
 635                 git_odb_backend *b = internal->backend;
 636
 637                 /* we don't write in alternates! */
 638                 if (internal->is_alternate)
 639                         continue;
 640
 641                 if (b->write != NULL)
 642                         error = b->write(oid, b, data, len, type);
 643         }
 644
 645         if (!error || error == GIT_PASSTHROUGH)
 646                 return 0;
 647
 648         /* if no backends were able to write the object directly, we try a streaming
 649          * write to the backends; just write the whole object into the stream in one
 650          * push */
 651
 652         if ((error = git_odb_open_wstream(&stream, db, len, type)) != 0)
 653                 return error;
 654
 655         stream->write(stream, data, len);
 656         error = stream->finalize_write(oid, stream);
 657         stream->free(stream);
 658
 659         return error;
 660 }
 661
 662 int git_odb_open_wstream(
 663         git_odb_stream **stream, git_odb *db, size_t size, git_otype type)
 664 {
 665         unsigned int i;
 666         int error = GIT_ERROR;
 667
 668         assert(stream && db);
 669
 670         for (i = 0; i < db->backends.length && error < 0; ++i) {
 671                 backend_internal *internal = git_vector_get(&db->backends, i);
 672                 git_odb_backend *b = internal->backend;
 673
 674                 /* we don't write in alternates! */
 675                 if (internal->is_alternate)
 676                         continue;
 677
 678                 if (b->writestream != NULL)
 679                         error = b->writestream(stream, b, size, type);
 680                 else if (b->write != NULL)
 681                         error = init_fake_wstream(stream, b, size, type);
 682         }
 683
 684         if (error == GIT_PASSTHROUGH)
 685                 error = 0;
 686
 687         return error;
 688 }
 689
 690 int git_odb_open_rstream(git_odb_stream **stream, git_odb *db, const git_oid *oid)
 691 {
 692         unsigned int i;
 693         int error = GIT_ERROR;
 694
 695         assert(stream && db);
 696
 697         for (i = 0; i < db->backends.length && error < 0; ++i) {
 698                 backend_internal *internal = git_vector_get(&db->backends, i);
 699                 git_odb_backend *b = internal->backend;
 700
 701                 if (b->readstream != NULL)
 702                         error = b->readstream(stream, b, oid);
 703         }
 704
 705         if (error == GIT_PASSTHROUGH)
 706                 error = 0;
 707
 708         return error;
 709 }
 710
 711 void * git_odb_backend_malloc(git_odb_backend *backend, size_t len)
 712 {
 713         GIT_UNUSED(backend);
 714         return git__malloc(len);
 715 }
 716
 717 int git_odb__error_notfound(const char *message, const git_oid *oid)
 718 {
 719         if (oid != NULL) {
 720                 char oid_str[GIT_OID_HEXSZ + 1];
 721                 git_oid_tostr(oid_str, sizeof(oid_str), oid);
 722                 giterr_set(GITERR_ODB, "Object not found - %s (%s)", message, oid_str);
 723         } else
 724                 giterr_set(GITERR_ODB, "Object not found - %s", message);
 725
 726         return GIT_ENOTFOUND;
 727 }
 728
 729 int git_odb__error_ambiguous(const char *message)
 730 {
 731         giterr_set(GITERR_ODB, "Ambiguous SHA1 prefix - %s", message);
 732         return GIT_EAMBIGUOUS;
 733 }
 734