src/odb.c

   1 /*
   2  * Copyright (C) 2009-2011 the libgit2 contributors
   3  *
   4  * This file is part of libgit2, distributed under the GNU GPL v2 with
   5  * a Linking Exception. For full terms see the included COPYING file.
   6  */
   7
   8 #include "common.h"
   9 #include "git2/zlib.h"
  10 #include "git2/object.h"
  11 #include "fileops.h"
  12 #include "hash.h"
  13 #include "odb.h"
  14 #include "delta-apply.h"
  15
  16 #include "git2/odb_backend.h"
  17
  18 #define GIT_ALTERNATES_FILE "info/alternates"
  19
  20 /* TODO: is this correct? */
  21 #define GIT_LOOSE_PRIORITY 2
  22 #define GIT_PACKED_PRIORITY 1
  23
  24 typedef struct
  25 {
  26         git_odb_backend *backend;
  27         int priority;
  28         int is_alternate;
  29 } backend_internal;
  30
  31 static int format_object_header(char *hdr, size_t n, size_t obj_len, git_otype obj_type)
  32 {
  33         const char *type_str = git_object_type2string(obj_type);
  34         int len = p_snprintf(hdr, n, "%s %"PRIuZ, type_str, obj_len);
  35
  36         if (len < 0 || ((size_t) len) >= n)
  37                 return git__throw(GIT_ERROR, "Cannot format object header. Length is out of bounds");
  38
  39         return len+1;
  40 }
  41
  42 int git_odb__hash_obj(git_oid *id, git_rawobj *obj)
  43 {
  44         git_buf_vec vec[2];
  45         char header[64];
  46         int hdrlen;
  47
  48         assert(id && obj);
  49
  50         if (!git_object_typeisloose(obj->type))
  51                 return git__throw(GIT_ERROR, "Failed to hash object. Wrong object type");
  52
  53         if (!obj->data && obj->len != 0)
  54                 return git__throw(GIT_ERROR, "Failed to hash object. No data given");
  55
  56         if ((hdrlen = format_object_header(header, sizeof(header), obj->len, obj->type)) < 0)
  57                 return git__rethrow(hdrlen, "Failed to hash object");
  58
  59         vec[0].data = header;
  60         vec[0].len  = hdrlen;
  61         vec[1].data = obj->data;
  62         vec[1].len  = obj->len;
  63
  64         git_hash_vec(id, vec, 2);
  65
  66         return GIT_SUCCESS;
  67 }
  68
  69
  70 static git_odb_object *new_odb_object(const git_oid *oid, git_rawobj *source)
  71 {
  72         git_odb_object *object = git__malloc(sizeof(git_odb_object));
  73         memset(object, 0x0, sizeof(git_odb_object));
  74
  75         git_oid_cpy(&object->cached.oid, oid);
  76         memcpy(&object->raw, source, sizeof(git_rawobj));
  77
  78         return object;
  79 }
  80
  81 static void free_odb_object(void *o)
  82 {
  83         git_odb_object *object = (git_odb_object *)o;
  84
  85         if (object != NULL) {
  86                 free(object->raw.data);
  87                 free(object);
  88         }
  89 }
  90
  91 const git_oid *git_odb_object_id(git_odb_object *object)
  92 {
  93         return &object->cached.oid;
  94 }
  95
  96 const void *git_odb_object_data(git_odb_object *object)
  97 {
  98         return object->raw.data;
  99 }
 100
 101 size_t git_odb_object_size(git_odb_object *object)
 102 {
 103         return object->raw.len;
 104 }
 105
 106 git_otype git_odb_object_type(git_odb_object *object)
 107 {
 108         return object->raw.type;
 109 }
 110
 111 void git_odb_object_close(git_odb_object *object)
 112 {
 113         git_cached_obj_decref((git_cached_obj *)object, &free_odb_object);
 114 }
 115
 116 int git_odb_hashfile(git_oid *out, const char *path, git_otype type)
 117 {
 118         int fd, hdr_len;
 119         char hdr[64], buffer[2048];
 120         git_off_t size;
 121         git_hash_ctx *ctx;
 122
 123         if ((fd = p_open(path, O_RDONLY)) < 0)
 124                 return git__throw(GIT_ENOTFOUND, "Could not open '%s'", path);
 125
 126         if ((size = git_futils_filesize(fd)) < 0 || !git__is_sizet(size)) {
 127                 p_close(fd);
 128                 return git__throw(GIT_EOSERR, "'%s' appears to be corrupted", path);
 129         }
 130
 131         hdr_len = format_object_header(hdr, sizeof(hdr), (size_t)size, type);
 132         if (hdr_len < 0)
 133                 return git__throw(GIT_ERROR, "Failed to format blob header. Length is out of bounds");
 134
 135         ctx = git_hash_new_ctx();
 136
 137         git_hash_update(ctx, hdr, hdr_len);
 138
 139         while (size > 0) {
 140                 ssize_t read_len;
 141
 142                 read_len = read(fd, buffer, sizeof(buffer));
 143
 144                 if (read_len < 0) {
 145                         p_close(fd);
 146                         git_hash_free_ctx(ctx);
 147                         return git__throw(GIT_EOSERR, "Can't read full file '%s'", path);
 148                 }
 149
 150                 git_hash_update(ctx, buffer, read_len);
 151                 size -= read_len;
 152         }
 153
 154         p_close(fd);
 155
 156         git_hash_final(out, ctx);
 157         git_hash_free_ctx(ctx);
 158
 159         return GIT_SUCCESS;
 160 }
 161
 162 int git_odb_hash(git_oid *id, const void *data, size_t len, git_otype type)
 163 {
 164         git_rawobj raw;
 165
 166         assert(id);
 167
 168         raw.data = (void *)data;
 169         raw.len = len;
 170         raw.type = type;
 171
 172         return git_odb__hash_obj(id, &raw);
 173 }
 174
 175 /**
 176  * FAKE WSTREAM
 177  */
 178
 179 typedef struct {
 180         git_odb_stream stream;
 181         char *buffer;
 182         size_t size, written;
 183         git_otype type;
 184 } fake_wstream;
 185
 186 static int fake_wstream__fwrite(git_oid *oid, git_odb_stream *_stream)
 187 {
 188         fake_wstream *stream = (fake_wstream *)_stream;
 189         return _stream->backend->write(oid, _stream->backend, stream->buffer, stream->size, stream->type);
 190 }
 191
 192 static int fake_wstream__write(git_odb_stream *_stream, const char *data, size_t len)
 193 {
 194         fake_wstream *stream = (fake_wstream *)_stream;
 195
 196         if (stream->written + len > stream->size)
 197                 return GIT_ENOMEM;
 198
 199         memcpy(stream->buffer + stream->written, data, len);
 200         stream->written += len;
 201         return GIT_SUCCESS;
 202 }
 203
 204 static void fake_wstream__free(git_odb_stream *_stream)
 205 {
 206         fake_wstream *stream = (fake_wstream *)_stream;
 207
 208         free(stream->buffer);
 209         free(stream);
 210 }
 211
 212 static int init_fake_wstream(git_odb_stream **stream_p, git_odb_backend *backend, size_t size, git_otype type)
 213 {
 214         fake_wstream *stream;
 215
 216         stream = git__calloc(1, sizeof(fake_wstream));
 217         if (stream == NULL)
 218                 return GIT_ENOMEM;
 219
 220         stream->size = size;
 221         stream->type = type;
 222         stream->buffer = git__malloc(size);
 223         if (stream->buffer == NULL) {
 224                 free(stream);
 225                 return GIT_ENOMEM;
 226         }
 227
 228         stream->stream.backend = backend;
 229         stream->stream.read = NULL; /* read only */
 230         stream->stream.write = &fake_wstream__write;
 231         stream->stream.finalize_write = &fake_wstream__fwrite;
 232         stream->stream.free = &fake_wstream__free;
 233         stream->stream.mode = GIT_STREAM_WRONLY;
 234
 235         *stream_p = (git_odb_stream *)stream;
 236         return GIT_SUCCESS;
 237 }
 238
 239 /***********************************************************
 240  *
 241  * OBJECT DATABASE PUBLIC API
 242  *
 243  * Public calls for the ODB functionality
 244  *
 245  ***********************************************************/
 246
 247 static int backend_sort_cmp(const void *a, const void *b)
 248 {
 249         const backend_internal *backend_a = (const backend_internal *)(a);
 250         const backend_internal *backend_b = (const backend_internal *)(b);
 251
 252         if (backend_a->is_alternate == backend_b->is_alternate)
 253                 return (backend_b->priority - backend_a->priority);
 254
 255         return backend_a->is_alternate ? 1 : -1;
 256 }
 257
 258 int git_odb_new(git_odb **out)
 259 {
 260         int error;
 261
 262         git_odb *db = git__calloc(1, sizeof(*db));
 263         if (!db)
 264                 return GIT_ENOMEM;
 265
 266         error = git_cache_init(&db->cache, GIT_DEFAULT_CACHE_SIZE, &free_odb_object);
 267         if (error < GIT_SUCCESS) {
 268                 free(db);
 269                 return git__rethrow(error, "Failed to create object database");
 270         }
 271
 272         if ((error = git_vector_init(&db->backends, 4, backend_sort_cmp)) < GIT_SUCCESS) {
 273                 free(db);
 274                 return git__rethrow(error, "Failed to create object database");
 275         }
 276
 277         *out = db;
 278         return GIT_SUCCESS;
 279 }
 280
 281 static int add_backend_internal(git_odb *odb, git_odb_backend *backend, int priority, int is_alternate)
 282 {
 283         backend_internal *internal;
 284
 285         assert(odb && backend);
 286
 287         if (backend->odb != NULL && backend->odb != odb)
 288                 return git__throw(GIT_EBUSY, "The backend is already owned by another ODB");
 289
 290         internal = git__malloc(sizeof(backend_internal));
 291         if (internal == NULL)
 292                 return GIT_ENOMEM;
 293
 294         internal->backend = backend;
 295         internal->priority = priority;
 296         internal->is_alternate = is_alternate;
 297
 298         if (git_vector_insert(&odb->backends, internal) < 0) {
 299                 free(internal);
 300                 return GIT_ENOMEM;
 301         }
 302
 303         git_vector_sort(&odb->backends);
 304         internal->backend->odb = odb;
 305         return GIT_SUCCESS;
 306 }
 307
 308 int git_odb_add_backend(git_odb *odb, git_odb_backend *backend, int priority)
 309 {
 310         return add_backend_internal(odb, backend, priority, 0);
 311 }
 312
 313 int git_odb_add_alternate(git_odb *odb, git_odb_backend *backend, int priority)
 314 {
 315         return add_backend_internal(odb, backend, priority, 1);
 316 }
 317
 318 static int add_default_backends(git_odb *db, const char *objects_dir, int as_alternates)
 319 {
 320         git_odb_backend *loose, *packed;
 321         int error;
 322
 323         /* add the loose object backend */
 324         error = git_odb_backend_loose(&loose, objects_dir);
 325         if (error < GIT_SUCCESS)
 326                 return error;
 327
 328         error = add_backend_internal(db, loose, GIT_LOOSE_PRIORITY, as_alternates);
 329         if (error < GIT_SUCCESS)
 330                 return git__rethrow(error, "Failed to add backend");
 331
 332         /* add the packed file backend */
 333         error = git_odb_backend_pack(&packed, objects_dir);
 334         if (error < GIT_SUCCESS)
 335                 return error;
 336
 337         error = add_backend_internal(db, packed, GIT_PACKED_PRIORITY, as_alternates);
 338         if (error < GIT_SUCCESS)
 339                 return git__rethrow(error, "Failed to add backend");
 340
 341         return GIT_SUCCESS;
 342 }
 343
 344 static int load_alternates(git_odb *odb, const char *objects_dir)
 345 {
 346         char alternates_path[GIT_PATH_MAX];
 347         char *buffer, *alternate;
 348
 349         git_fbuffer alternates_buf = GIT_FBUFFER_INIT;
 350         int error;
 351
 352         git_path_join(alternates_path, objects_dir, GIT_ALTERNATES_FILE);
 353
 354         if (git_futils_exists(alternates_path) < GIT_SUCCESS)
 355                 return GIT_SUCCESS;
 356
 357         if (git_futils_readbuffer(&alternates_buf, alternates_path) < GIT_SUCCESS)
 358                 return git__throw(GIT_EOSERR, "Failed to add backend. Can't read alternates");
 359
 360         buffer = (char *)alternates_buf.data;
 361         error = GIT_SUCCESS;
 362
 363         /* add each alternate as a new backend; one alternate per line */
 364         while ((alternate = git__strtok(&buffer, "\r\n")) != NULL) {
 365                 char full_path[GIT_PATH_MAX];
 366
 367                 if (*alternate == '\0' || *alternate == '#')
 368                         continue;
 369
 370                 /* relative path: build based on the current `objects` folder */
 371                 if (*alternate == '.') {
 372                         git_path_join(full_path, objects_dir, alternate);
 373                         alternate = full_path;
 374                 }
 375
 376                 if ((error = add_default_backends(odb, alternate, 1)) < GIT_SUCCESS)
 377                         break;
 378         }
 379
 380         git_futils_freebuffer(&alternates_buf);
 381         if (error < GIT_SUCCESS)
 382                 return git__rethrow(error, "Failed to load alternates");
 383         return error;
 384 }
 385
 386 int git_odb_open(git_odb **out, const char *objects_dir)
 387 {
 388         git_odb *db;
 389         int error;
 390
 391         assert(out && objects_dir);
 392
 393         *out = NULL;
 394
 395         if ((error = git_odb_new(&db)) < 0)
 396                 return git__rethrow(error, "Failed to open ODB");
 397
 398         if ((error = add_default_backends(db, objects_dir, 0)) < GIT_SUCCESS)
 399                 goto cleanup;
 400
 401         if ((error = load_alternates(db, objects_dir)) < GIT_SUCCESS)
 402                 goto cleanup;
 403
 404         *out = db;
 405         return GIT_SUCCESS;
 406
 407 cleanup:
 408         git_odb_close(db);
 409         return error; /* error already set - pass through */
 410 }
 411
 412 void git_odb_close(git_odb *db)
 413 {
 414         unsigned int i;
 415
 416         if (db == NULL)
 417                 return;
 418
 419         for (i = 0; i < db->backends.length; ++i) {
 420                 backend_internal *internal = git_vector_get(&db->backends, i);
 421                 git_odb_backend *backend = internal->backend;
 422
 423                 if (backend->free) backend->free(backend);
 424                 else free(backend);
 425
 426                 free(internal);
 427         }
 428
 429         git_vector_free(&db->backends);
 430         git_cache_free(&db->cache);
 431         free(db);
 432 }
 433
 434 int git_odb_exists(git_odb *db, const git_oid *id)
 435 {
 436         git_odb_object *object;
 437         unsigned int i;
 438         int found = 0;
 439
 440         assert(db && id);
 441
 442         if ((object = git_cache_get(&db->cache, id)) != NULL) {
 443                 git_odb_object_close(object);
 444                 return 1;
 445         }
 446
 447         for (i = 0; i < db->backends.length && !found; ++i) {
 448                 backend_internal *internal = git_vector_get(&db->backends, i);
 449                 git_odb_backend *b = internal->backend;
 450
 451                 if (b->exists != NULL)
 452                         found = b->exists(b, id);
 453         }
 454
 455         return found;
 456 }
 457
 458 int git_odb_read_header(size_t *len_p, git_otype *type_p, git_odb *db, const git_oid *id)
 459 {
 460         unsigned int i;
 461         int error = GIT_ENOTFOUND;
 462         git_odb_object *object;
 463
 464         assert(db && id);
 465
 466         if ((object = git_cache_get(&db->cache, id)) != NULL) {
 467                 *len_p = object->raw.len;
 468                 *type_p = object->raw.type;
 469                 git_odb_object_close(object);
 470                 return GIT_SUCCESS;
 471         }
 472
 473         for (i = 0; i < db->backends.length && error < 0; ++i) {
 474                 backend_internal *internal = git_vector_get(&db->backends, i);
 475                 git_odb_backend *b = internal->backend;
 476
 477                 if (b->read_header != NULL)
 478                         error = b->read_header(len_p, type_p, b, id);
 479         }
 480
 481         if (error == GIT_EPASSTHROUGH)
 482                 return GIT_SUCCESS;
 483
 484         /*
 485          * no backend could read only the header.
 486          * try reading the whole object and freeing the contents
 487          */
 488         if (error < 0) {
 489                 if ((error = git_odb_read(&object, db, id)) < GIT_SUCCESS)
 490                         return error; /* error already set - pass through */
 491
 492                 *len_p = object->raw.len;
 493                 *type_p = object->raw.type;
 494                 git_odb_object_close(object);
 495         }
 496
 497         return GIT_SUCCESS;
 498 }
 499
 500 int git_odb_read(git_odb_object **out, git_odb *db, const git_oid *id)
 501 {
 502         unsigned int i;
 503         int error = GIT_ENOTFOUND;
 504         git_rawobj raw;
 505
 506         assert(out && db && id);
 507
 508         *out = git_cache_get(&db->cache, id);
 509         if (*out != NULL)
 510                 return GIT_SUCCESS;
 511
 512         for (i = 0; i < db->backends.length && error < 0; ++i) {
 513                 backend_internal *internal = git_vector_get(&db->backends, i);
 514                 git_odb_backend *b = internal->backend;
 515
 516                 if (b->read != NULL)
 517                         error = b->read(&raw.data, &raw.len, &raw.type, b, id);
 518         }
 519
 520         if (error == GIT_EPASSTHROUGH || error == GIT_SUCCESS) {
 521                 *out = git_cache_try_store(&db->cache, new_odb_object(id, &raw));
 522                 return GIT_SUCCESS;
 523         }
 524
 525         return git__rethrow(error, "Failed to read object");
 526 }
 527
 528 int git_odb_read_prefix(git_odb_object **out, git_odb *db, const git_oid *short_id, unsigned int len)
 529 {
 530         unsigned int i;
 531         int error = GIT_ENOTFOUND;
 532         git_oid full_oid;
 533         git_rawobj raw;
 534         int found = 0;
 535
 536         assert(out && db);
 537
 538         if (len < GIT_OID_MINPREFIXLEN)
 539                 return git__throw(GIT_EAMBIGUOUSOIDPREFIX, "Failed to lookup object. Prefix length is lower than %d.", GIT_OID_MINPREFIXLEN);
 540
 541         if (len > GIT_OID_HEXSZ)
 542                 len = GIT_OID_HEXSZ;
 543
 544         if (len == GIT_OID_HEXSZ) {
 545                 *out = git_cache_get(&db->cache, short_id);
 546                 if (*out != NULL)
 547                         return GIT_SUCCESS;
 548         }
 549
 550         for (i = 0; i < db->backends.length && found < 2; ++i) {
 551                 backend_internal *internal = git_vector_get(&db->backends, i);
 552                 git_odb_backend *b = internal->backend;
 553
 554                 if (b->read != NULL) {
 555                         error = b->read_prefix(&full_oid, &raw.data, &raw.len, &raw.type, b, short_id, len);
 556                         switch (error) {
 557                         case GIT_SUCCESS:
 558                                 found++;
 559                                 break;
 560                         case GIT_ENOTFOUND:
 561                         case GIT_EPASSTHROUGH:
 562                                 break;
 563                         case GIT_EAMBIGUOUSOIDPREFIX:
 564                                 return git__rethrow(error, "Failed to read object. Ambiguous sha1 prefix");
 565                         default:
 566                                 return git__rethrow(error, "Failed to read object");
 567                         }
 568                 }
 569         }
 570
 571         if (found == 1) {
 572                 *out = git_cache_try_store(&db->cache, new_odb_object(&full_oid, &raw));
 573         } else if (found > 1) {
 574                 return git__throw(GIT_EAMBIGUOUSOIDPREFIX, "Failed to read object. Ambiguous sha1 prefix");
 575         } else {
 576                 return git__throw(GIT_ENOTFOUND, "Failed to read object. Object not found");
 577         }
 578
 579         return GIT_SUCCESS;
 580 }
 581
 582 int git_odb_write(git_oid *oid, git_odb *db, const void *data, size_t len, git_otype type)
 583 {
 584         unsigned int i;
 585         int error = GIT_ERROR;
 586         git_odb_stream *stream;
 587
 588         assert(oid && db);
 589
 590         for (i = 0; i < db->backends.length && error < 0; ++i) {
 591                 backend_internal *internal = git_vector_get(&db->backends, i);
 592                 git_odb_backend *b = internal->backend;
 593
 594                 /* we don't write in alternates! */
 595                 if (internal->is_alternate)
 596                         continue;
 597
 598                 if (b->write != NULL)
 599                         error = b->write(oid, b, data, len, type);
 600         }
 601
 602         if (error == GIT_EPASSTHROUGH || error == GIT_SUCCESS)
 603                 return GIT_SUCCESS;
 604
 605         /* if no backends were able to write the object directly, we try a streaming
 606          * write to the backends; just write the whole object into the stream in one
 607          * push */
 608
 609         if ((error = git_odb_open_wstream(&stream, db, len, type)) == GIT_SUCCESS) {
 610                 stream->write(stream, data, len);
 611                 error = stream->finalize_write(oid, stream);
 612                 stream->free(stream);
 613                 return GIT_SUCCESS;
 614         }
 615
 616         return git__rethrow(error, "Failed to write object");
 617 }
 618
 619 int git_odb_open_wstream(git_odb_stream **stream, git_odb *db, size_t size, git_otype type)
 620 {
 621         unsigned int i;
 622         int error = GIT_ERROR;
 623
 624         assert(stream && db);
 625
 626         for (i = 0; i < db->backends.length && error < 0; ++i) {
 627                 backend_internal *internal = git_vector_get(&db->backends, i);
 628                 git_odb_backend *b = internal->backend;
 629
 630                 /* we don't write in alternates! */
 631                 if (internal->is_alternate)
 632                         continue;
 633
 634                 if (b->writestream != NULL)
 635                         error = b->writestream(stream, b, size, type);
 636                 else if (b->write != NULL)
 637                         error = init_fake_wstream(stream, b, size, type);
 638         }
 639
 640         if (error == GIT_EPASSTHROUGH || error == GIT_SUCCESS)
 641                 return GIT_SUCCESS;
 642
 643         return git__rethrow(error, "Failed to open write stream");
 644 }
 645
 646 int git_odb_open_rstream(git_odb_stream **stream, git_odb *db, const git_oid *oid)
 647 {
 648         unsigned int i;
 649         int error = GIT_ERROR;
 650
 651         assert(stream && db);
 652
 653         for (i = 0; i < db->backends.length && error < 0; ++i) {
 654                 backend_internal *internal = git_vector_get(&db->backends, i);
 655                 git_odb_backend *b = internal->backend;
 656
 657                 if (b->readstream != NULL)
 658                         error = b->readstream(stream, b, oid);
 659         }
 660
 661         if (error == GIT_EPASSTHROUGH || error == GIT_SUCCESS)
 662                 return GIT_SUCCESS;
 663
 664         return git__rethrow(error, "Failed to open read stream");
 665 }
 666