src/odb.c

   1 /*
   2  * Copyright (C) 2009-2012 the libgit2 contributors
   3  *
   4  * This file is part of libgit2, distributed under the GNU GPL v2 with
   5  * a Linking Exception. For full terms see the included COPYING file.
   6  */
   7
   8 #include "common.h"
   9 #include <zlib.h>
  10 #include "git2/object.h"
  11 #include "fileops.h"
  12 #include "hash.h"
  13 #include "odb.h"
  14 #include "delta-apply.h"
  15
  16 #include "git2/odb_backend.h"
  17
  18 #define GIT_ALTERNATES_FILE "info/alternates"
  19
  20 /* TODO: is this correct? */
  21 #define GIT_LOOSE_PRIORITY 2
  22 #define GIT_PACKED_PRIORITY 1
  23
  24 typedef struct
  25 {
  26         git_odb_backend *backend;
  27         int priority;
  28         int is_alternate;
  29 } backend_internal;
  30
  31 static int format_object_header(char *hdr, size_t n, size_t obj_len, git_otype obj_type)
  32 {
  33         const char *type_str = git_object_type2string(obj_type);
  34         int len = p_snprintf(hdr, n, "%s %"PRIuZ, type_str, obj_len);
  35         assert(len > 0 && len <= (int)n);
  36         return len+1;
  37 }
  38
  39 int git_odb__hashobj(git_oid *id, git_rawobj *obj)
  40 {
  41         git_buf_vec vec[2];
  42         char header[64];
  43         int hdrlen;
  44
  45         assert(id && obj);
  46
  47         if (!git_object_typeisloose(obj->type))
  48                 return -1;
  49         if (!obj->data && obj->len != 0)
  50                 return -1;
  51
  52         hdrlen = format_object_header(header, sizeof(header), obj->len, obj->type);
  53
  54         vec[0].data = header;
  55         vec[0].len = hdrlen;
  56         vec[1].data = obj->data;
  57         vec[1].len = obj->len;
  58
  59         git_hash_vec(id, vec, 2);
  60
  61         return 0;
  62 }
  63
  64
  65 static git_odb_object *new_odb_object(const git_oid *oid, git_rawobj *source)
  66 {
  67         git_odb_object *object = git__malloc(sizeof(git_odb_object));
  68         memset(object, 0x0, sizeof(git_odb_object));
  69
  70         git_oid_cpy(&object->cached.oid, oid);
  71         memcpy(&object->raw, source, sizeof(git_rawobj));
  72
  73         return object;
  74 }
  75
  76 static void free_odb_object(void *o)
  77 {
  78         git_odb_object *object = (git_odb_object *)o;
  79
  80         if (object != NULL) {
  81                 git__free(object->raw.data);
  82                 git__free(object);
  83         }
  84 }
  85
  86 const git_oid *git_odb_object_id(git_odb_object *object)
  87 {
  88         return &object->cached.oid;
  89 }
  90
  91 const void *git_odb_object_data(git_odb_object *object)
  92 {
  93         return object->raw.data;
  94 }
  95
  96 size_t git_odb_object_size(git_odb_object *object)
  97 {
  98         return object->raw.len;
  99 }
 100
 101 git_otype git_odb_object_type(git_odb_object *object)
 102 {
 103         return object->raw.type;
 104 }
 105
 106 void git_odb_object_free(git_odb_object *object)
 107 {
 108         git_cached_obj_decref((git_cached_obj *)object, &free_odb_object);
 109 }
 110
 111 int git_odb__hashfd(git_oid *out, git_file fd, size_t size, git_otype type)
 112 {
 113         int hdr_len;
 114         char hdr[64], buffer[2048];
 115         git_hash_ctx *ctx;
 116
 117         hdr_len = format_object_header(hdr, sizeof(hdr), size, type);
 118
 119         ctx = git_hash_new_ctx();
 120
 121         git_hash_update(ctx, hdr, hdr_len);
 122
 123         while (size > 0) {
 124                 ssize_t read_len = read(fd, buffer, sizeof(buffer));
 125
 126                 if (read_len < 0) {
 127                         git_hash_free_ctx(ctx);
 128                         giterr_set(GITERR_OS, "Error reading file");
 129                         return -1;
 130                 }
 131
 132                 git_hash_update(ctx, buffer, read_len);
 133                 size -= read_len;
 134         }
 135
 136         git_hash_final(out, ctx);
 137         git_hash_free_ctx(ctx);
 138
 139         return 0;
 140 }
 141
 142 int git_odb__hashlink(git_oid *out, const char *path)
 143 {
 144         struct stat st;
 145         git_off_t size;
 146         int result;
 147
 148         if (git_path_lstat(path, &st) < 0)
 149                 return -1;
 150
 151         size = st.st_size;
 152
 153         if (!git__is_sizet(size)) {
 154                 giterr_set(GITERR_OS, "File size overflow for 32-bit systems");
 155                 return -1;
 156         }
 157
 158         if (S_ISLNK(st.st_mode)) {
 159                 char *link_data;
 160                 ssize_t read_len;
 161
 162                 link_data = git__malloc((size_t)size);
 163                 GITERR_CHECK_ALLOC(link_data);
 164
 165                 read_len = p_readlink(path, link_data, (size_t)(size + 1));
 166                 if (read_len != (ssize_t)size) {
 167                         giterr_set(GITERR_OS, "Failed to read symlink data for '%s'", path);
 168                         return -1;
 169                 }
 170
 171                 result = git_odb_hash(out, link_data, (size_t)size, GIT_OBJ_BLOB);
 172                 git__free(link_data);
 173         } else {
 174                 int fd = git_futils_open_ro(path);
 175                 if (fd < 0)
 176                         return -1;
 177                 result = git_odb__hashfd(out, fd, (size_t)size, GIT_OBJ_BLOB);
 178                 p_close(fd);
 179         }
 180
 181         return result;
 182 }
 183
 184 int git_odb_hashfile(git_oid *out, const char *path, git_otype type)
 185 {
 186         git_off_t size;
 187         int result, fd = git_futils_open_ro(path);
 188         if (fd < 0)
 189                 return fd;
 190
 191         if ((size = git_futils_filesize(fd)) < 0 || !git__is_sizet(size)) {
 192                 giterr_set(GITERR_OS, "File size overflow for 32-bit systems");
 193                 p_close(fd);
 194                 return -1;
 195         }
 196
 197         result = git_odb__hashfd(out, fd, (size_t)size, type);
 198         p_close(fd);
 199         return result;
 200 }
 201
 202 int git_odb_hash(git_oid *id, const void *data, size_t len, git_otype type)
 203 {
 204         git_rawobj raw;
 205
 206         assert(id);
 207
 208         raw.data = (void *)data;
 209         raw.len = len;
 210         raw.type = type;
 211
 212         return git_odb__hashobj(id, &raw);
 213 }
 214
 215 /**
 216  * FAKE WSTREAM
 217  */
 218
 219 typedef struct {
 220         git_odb_stream stream;
 221         char *buffer;
 222         size_t size, written;
 223         git_otype type;
 224 } fake_wstream;
 225
 226 static int fake_wstream__fwrite(git_oid *oid, git_odb_stream *_stream)
 227 {
 228         fake_wstream *stream = (fake_wstream *)_stream;
 229         return _stream->backend->write(oid, _stream->backend, stream->buffer, stream->size, stream->type);
 230 }
 231
 232 static int fake_wstream__write(git_odb_stream *_stream, const char *data, size_t len)
 233 {
 234         fake_wstream *stream = (fake_wstream *)_stream;
 235
 236         if (stream->written + len > stream->size)
 237                 return -1;
 238
 239         memcpy(stream->buffer + stream->written, data, len);
 240         stream->written += len;
 241         return 0;
 242 }
 243
 244 static void fake_wstream__free(git_odb_stream *_stream)
 245 {
 246         fake_wstream *stream = (fake_wstream *)_stream;
 247
 248         git__free(stream->buffer);
 249         git__free(stream);
 250 }
 251
 252 static int init_fake_wstream(git_odb_stream **stream_p, git_odb_backend *backend, size_t size, git_otype type)
 253 {
 254         fake_wstream *stream;
 255
 256         stream = git__calloc(1, sizeof(fake_wstream));
 257         GITERR_CHECK_ALLOC(stream);
 258
 259         stream->size = size;
 260         stream->type = type;
 261         stream->buffer = git__malloc(size);
 262         if (stream->buffer == NULL) {
 263                 git__free(stream);
 264                 return -1;
 265         }
 266
 267         stream->stream.backend = backend;
 268         stream->stream.read = NULL; /* read only */
 269         stream->stream.write = &fake_wstream__write;
 270         stream->stream.finalize_write = &fake_wstream__fwrite;
 271         stream->stream.free = &fake_wstream__free;
 272         stream->stream.mode = GIT_STREAM_WRONLY;
 273
 274         *stream_p = (git_odb_stream *)stream;
 275         return 0;
 276 }
 277
 278 /***********************************************************
 279  *
 280  * OBJECT DATABASE PUBLIC API
 281  *
 282  * Public calls for the ODB functionality
 283  *
 284  ***********************************************************/
 285
 286 static int backend_sort_cmp(const void *a, const void *b)
 287 {
 288         const backend_internal *backend_a = (const backend_internal *)(a);
 289         const backend_internal *backend_b = (const backend_internal *)(b);
 290
 291         if (backend_a->is_alternate == backend_b->is_alternate)
 292                 return (backend_b->priority - backend_a->priority);
 293
 294         return backend_a->is_alternate ? 1 : -1;
 295 }
 296
 297 int git_odb_new(git_odb **out)
 298 {
 299         git_odb *db = git__calloc(1, sizeof(*db));
 300         GITERR_CHECK_ALLOC(db);
 301
 302         if (git_cache_init(&db->cache, GIT_DEFAULT_CACHE_SIZE, &free_odb_object) < 0 ||
 303                 git_vector_init(&db->backends, 4, backend_sort_cmp) < 0)
 304         {
 305                 git__free(db);
 306                 return -1;
 307         }
 308
 309         *out = db;
 310         GIT_REFCOUNT_INC(db);
 311         return 0;
 312 }
 313
 314 static int add_backend_internal(git_odb *odb, git_odb_backend *backend, int priority, int is_alternate)
 315 {
 316         backend_internal *internal;
 317
 318         assert(odb && backend);
 319
 320         /* Check if the backend is already owned by another ODB */
 321         assert(!backend->odb || backend->odb == odb);
 322
 323         internal = git__malloc(sizeof(backend_internal));
 324         GITERR_CHECK_ALLOC(internal);
 325
 326         internal->backend = backend;
 327         internal->priority = priority;
 328         internal->is_alternate = is_alternate;
 329
 330         if (git_vector_insert(&odb->backends, internal) < 0) {
 331                 git__free(internal);
 332                 return -1;
 333         }
 334
 335         git_vector_sort(&odb->backends);
 336         internal->backend->odb = odb;
 337         return 0;
 338 }
 339
 340 int git_odb_add_backend(git_odb *odb, git_odb_backend *backend, int priority)
 341 {
 342         return add_backend_internal(odb, backend, priority, 0);
 343 }
 344
 345 int git_odb_add_alternate(git_odb *odb, git_odb_backend *backend, int priority)
 346 {
 347         return add_backend_internal(odb, backend, priority, 1);
 348 }
 349
 350 static int add_default_backends(git_odb *db, const char *objects_dir, int as_alternates)
 351 {
 352         git_odb_backend *loose, *packed;
 353
 354         /* add the loose object backend */
 355         if (git_odb_backend_loose(&loose, objects_dir, -1, 0) < 0 ||
 356                 add_backend_internal(db, loose, GIT_LOOSE_PRIORITY, as_alternates) < 0)
 357                 return -1;
 358
 359         /* add the packed file backend */
 360         if (git_odb_backend_pack(&packed, objects_dir) < 0 ||
 361                 add_backend_internal(db, packed, GIT_PACKED_PRIORITY, as_alternates) < 0)
 362                 return -1;
 363
 364         return 0;
 365 }
 366
 367 static int load_alternates(git_odb *odb, const char *objects_dir)
 368 {
 369         git_buf alternates_path = GIT_BUF_INIT;
 370         git_buf alternates_buf = GIT_BUF_INIT;
 371         char *buffer;
 372         const char *alternate;
 373         int result = 0;
 374
 375         if (git_buf_joinpath(&alternates_path, objects_dir, GIT_ALTERNATES_FILE) < 0)
 376                 return -1;
 377
 378         if (git_path_exists(alternates_path.ptr) == false) {
 379                 git_buf_free(&alternates_path);
 380                 return 0;
 381         }
 382
 383         if (git_futils_readbuffer(&alternates_buf, alternates_path.ptr) < 0) {
 384                 git_buf_free(&alternates_path);
 385                 return -1;
 386         }
 387
 388         buffer = (char *)alternates_buf.ptr;
 389
 390         /* add each alternate as a new backend; one alternate per line */
 391         while ((alternate = git__strtok(&buffer, "\r\n")) != NULL) {
 392                 if (*alternate == '\0' || *alternate == '#')
 393                         continue;
 394
 395                 /* relative path: build based on the current `objects` folder */
 396                 if (*alternate == '.') {
 397                         if ((result = git_buf_joinpath(&alternates_path, objects_dir, alternate)) < 0)
 398                                 break;
 399                         alternate = git_buf_cstr(&alternates_path);
 400                 }
 401
 402                 if ((result = add_default_backends(odb, alternate, 1)) < 0)
 403                         break;
 404         }
 405
 406         git_buf_free(&alternates_path);
 407         git_buf_free(&alternates_buf);
 408
 409         return result;
 410 }
 411
 412 int git_odb_open(git_odb **out, const char *objects_dir)
 413 {
 414         git_odb *db;
 415
 416         assert(out && objects_dir);
 417
 418         *out = NULL;
 419
 420         if (git_odb_new(&db) < 0)
 421                 return -1;
 422
 423         if (add_default_backends(db, objects_dir, 0) < 0 ||
 424                 load_alternates(db, objects_dir) < 0)
 425         {
 426                 git_odb_free(db);
 427                 return -1;
 428         }
 429
 430         *out = db;
 431         return 0;
 432 }
 433
 434 static void odb_free(git_odb *db)
 435 {
 436         unsigned int i;
 437
 438         for (i = 0; i < db->backends.length; ++i) {
 439                 backend_internal *internal = git_vector_get(&db->backends, i);
 440                 git_odb_backend *backend = internal->backend;
 441
 442                 if (backend->free) backend->free(backend);
 443                 else git__free(backend);
 444
 445                 git__free(internal);
 446         }
 447
 448         git_vector_free(&db->backends);
 449         git_cache_free(&db->cache);
 450         git__free(db);
 451 }
 452
 453 void git_odb_free(git_odb *db)
 454 {
 455         if (db == NULL)
 456                 return;
 457
 458         GIT_REFCOUNT_DEC(db, odb_free);
 459 }
 460
 461 int git_odb_exists(git_odb *db, const git_oid *id)
 462 {
 463         git_odb_object *object;
 464         unsigned int i;
 465         bool found = false;
 466
 467         assert(db && id);
 468
 469         if ((object = git_cache_get(&db->cache, id)) != NULL) {
 470                 git_odb_object_free(object);
 471                 return (int)true;
 472         }
 473
 474         for (i = 0; i < db->backends.length && !found; ++i) {
 475                 backend_internal *internal = git_vector_get(&db->backends, i);
 476                 git_odb_backend *b = internal->backend;
 477
 478                 if (b->exists != NULL)
 479                         found = b->exists(b, id);
 480         }
 481
 482         return (int)found;
 483 }
 484
 485 int git_odb_read_header(size_t *len_p, git_otype *type_p, git_odb *db, const git_oid *id)
 486 {
 487         unsigned int i;
 488         int error = GIT_ENOTFOUND;
 489         git_odb_object *object;
 490
 491         assert(db && id);
 492
 493         if ((object = git_cache_get(&db->cache, id)) != NULL) {
 494                 *len_p = object->raw.len;
 495                 *type_p = object->raw.type;
 496                 git_odb_object_free(object);
 497                 return 0;
 498         }
 499
 500         for (i = 0; i < db->backends.length && error < 0; ++i) {
 501                 backend_internal *internal = git_vector_get(&db->backends, i);
 502                 git_odb_backend *b = internal->backend;
 503
 504                 if (b->read_header != NULL)
 505                         error = b->read_header(len_p, type_p, b, id);
 506         }
 507
 508         if (!error || error == GIT_EPASSTHROUGH)
 509                 return 0;
 510
 511         /*
 512          * no backend could read only the header.
 513          * try reading the whole object and freeing the contents
 514          */
 515         if ((error = git_odb_read(&object, db, id)) < 0)
 516                 return error; /* error already set - pass along */
 517
 518         *len_p = object->raw.len;
 519         *type_p = object->raw.type;
 520         git_odb_object_free(object);
 521         return 0;
 522 }
 523
 524 int git_odb_read(git_odb_object **out, git_odb *db, const git_oid *id)
 525 {
 526         unsigned int i;
 527         int error = GIT_ENOTFOUND;
 528         git_rawobj raw;
 529
 530         assert(out && db && id);
 531
 532         *out = git_cache_get(&db->cache, id);
 533         if (*out != NULL)
 534                 return 0;
 535
 536         for (i = 0; i < db->backends.length && error < 0; ++i) {
 537                 backend_internal *internal = git_vector_get(&db->backends, i);
 538                 git_odb_backend *b = internal->backend;
 539
 540                 if (b->read != NULL)
 541                         error = b->read(&raw.data, &raw.len, &raw.type, b, id);
 542         }
 543
 544         /* TODO: If no backends are configured, this returns GIT_ENOTFOUND but
 545          * will never have called giterr_set().
 546          */
 547
 548         if (error && error != GIT_EPASSTHROUGH)
 549                 return error;
 550
 551         *out = git_cache_try_store(&db->cache, new_odb_object(id, &raw));
 552         return 0;
 553 }
 554
 555 int git_odb_read_prefix(
 556         git_odb_object **out, git_odb *db, const git_oid *short_id, unsigned int len)
 557 {
 558         unsigned int i;
 559         int error = GIT_ENOTFOUND;
 560         git_oid full_oid;
 561         git_rawobj raw;
 562         int found = 0;
 563
 564         assert(out && db);
 565
 566         if (len < GIT_OID_MINPREFIXLEN)
 567                 return git_odb__error_ambiguous("prefix length too short");
 568
 569         if (len > GIT_OID_HEXSZ)
 570                 len = GIT_OID_HEXSZ;
 571
 572         if (len == GIT_OID_HEXSZ) {
 573                 *out = git_cache_get(&db->cache, short_id);
 574                 if (*out != NULL)
 575                         return 0;
 576         }
 577
 578         for (i = 0; i < db->backends.length && found < 2; ++i) {
 579                 backend_internal *internal = git_vector_get(&db->backends, i);
 580                 git_odb_backend *b = internal->backend;
 581
 582                 if (b->read != NULL) {
 583                         error = b->read_prefix(&full_oid, &raw.data, &raw.len, &raw.type, b, short_id, len);
 584                         if (!error)
 585                                 found++;
 586                         else if (error != GIT_ENOTFOUND && error != GIT_EPASSTHROUGH)
 587                                 return error;
 588                 }
 589         }
 590
 591         if (found == 0)
 592                 return git_odb__error_notfound("no match for prefix");
 593         if (found > 1)
 594                 return git_odb__error_ambiguous("multiple matches for prefix");
 595
 596         *out = git_cache_try_store(&db->cache, new_odb_object(&full_oid, &raw));
 597         return 0;
 598 }
 599
 600 int git_odb_write(
 601         git_oid *oid, git_odb *db, const void *data, size_t len, git_otype type)
 602 {
 603         unsigned int i;
 604         int error = GIT_ERROR;
 605         git_odb_stream *stream;
 606
 607         assert(oid && db);
 608
 609         for (i = 0; i < db->backends.length && error < 0; ++i) {
 610                 backend_internal *internal = git_vector_get(&db->backends, i);
 611                 git_odb_backend *b = internal->backend;
 612
 613                 /* we don't write in alternates! */
 614                 if (internal->is_alternate)
 615                         continue;
 616
 617                 if (b->write != NULL)
 618                         error = b->write(oid, b, data, len, type);
 619         }
 620
 621         if (!error || error == GIT_EPASSTHROUGH)
 622                 return 0;
 623
 624         /* if no backends were able to write the object directly, we try a streaming
 625          * write to the backends; just write the whole object into the stream in one
 626          * push */
 627
 628         if ((error = git_odb_open_wstream(&stream, db, len, type)) != 0)
 629                 return error;
 630
 631         stream->write(stream, data, len);
 632         error = stream->finalize_write(oid, stream);
 633         stream->free(stream);
 634
 635         return error;
 636 }
 637
 638 int git_odb_open_wstream(
 639         git_odb_stream **stream, git_odb *db, size_t size, git_otype type)
 640 {
 641         unsigned int i;
 642         int error = GIT_ERROR;
 643
 644         assert(stream && db);
 645
 646         for (i = 0; i < db->backends.length && error < 0; ++i) {
 647                 backend_internal *internal = git_vector_get(&db->backends, i);
 648                 git_odb_backend *b = internal->backend;
 649
 650                 /* we don't write in alternates! */
 651                 if (internal->is_alternate)
 652                         continue;
 653
 654                 if (b->writestream != NULL)
 655                         error = b->writestream(stream, b, size, type);
 656                 else if (b->write != NULL)
 657                         error = init_fake_wstream(stream, b, size, type);
 658         }
 659
 660         if (error == GIT_EPASSTHROUGH)
 661                 error = 0;
 662
 663         return error;
 664 }
 665
 666 int git_odb_open_rstream(git_odb_stream **stream, git_odb *db, const git_oid *oid)
 667 {
 668         unsigned int i;
 669         int error = GIT_ERROR;
 670
 671         assert(stream && db);
 672
 673         for (i = 0; i < db->backends.length && error < 0; ++i) {
 674                 backend_internal *internal = git_vector_get(&db->backends, i);
 675                 git_odb_backend *b = internal->backend;
 676
 677                 if (b->readstream != NULL)
 678                         error = b->readstream(stream, b, oid);
 679         }
 680
 681         if (error == GIT_EPASSTHROUGH)
 682                 error = 0;
 683
 684         return error;
 685 }
 686
 687 int git_odb__error_notfound(const char *message)
 688 {
 689         giterr_set(GITERR_ODB, "Object not found - %s", message);
 690         return GIT_ENOTFOUND;
 691 }
 692
 693 int git_odb__error_ambiguous(const char *message)
 694 {
 695         giterr_set(GITERR_ODB, "Ambiguous SHA1 prefix - %s", message);
 696         return GIT_EAMBIGUOUS;
 697 }
 698