2 * Copyright (C) the libgit2 contributors. All rights reserved.
4 * This file is part of libgit2, distributed under the GNU GPL v2 with
5 * a Linking Exception. For full terms see the included COPYING file.
8 #include "git2/indexer.h"
9 #include "git2/object.h"
22 extern git_mutex git__mwindow_mutex
;
24 #define UINT31_MAX (0x7FFFFFFF)
34 unsigned int parsed_header
:1,
39 struct git_pack_header hdr
;
40 struct git_pack_file
*pack
;
43 git_off_t entry_start
;
44 git_packfile_stream stream
;
48 unsigned int fanout
[256];
49 git_hash_ctx hash_ctx
;
51 git_transfer_progress_cb progress_cb
;
52 void *progress_payload
;
55 /* Needed to look up objects which we want to inject to fix a thin pack */
58 /* Fields for calculating the packfile trailer (hash of everything before it) */
59 char inbuf
[GIT_OID_RAWSZ
];
68 const git_oid
*git_indexer_hash(const git_indexer
*idx
)
73 static int parse_header(struct git_pack_header
*hdr
, struct git_pack_file
*pack
)
78 if ((error
= p_mmap(&map
, sizeof(*hdr
), GIT_PROT_READ
, GIT_MAP_SHARED
, pack
->mwf
.fd
, 0)) < 0)
81 memcpy(hdr
, map
.data
, sizeof(*hdr
));
84 /* Verify we recognize this pack file format. */
85 if (hdr
->hdr_signature
!= ntohl(PACK_SIGNATURE
)) {
86 giterr_set(GITERR_INDEXER
, "wrong pack signature");
90 if (!pack_version_ok(hdr
->hdr_version
)) {
91 giterr_set(GITERR_INDEXER
, "wrong pack version");
98 static int objects_cmp(const void *a
, const void *b
)
100 const struct entry
*entrya
= a
;
101 const struct entry
*entryb
= b
;
103 return git_oid__cmp(&entrya
->oid
, &entryb
->oid
);
111 git_transfer_progress_cb progress_cb
,
112 void *progress_payload
)
115 git_buf path
= GIT_BUF_INIT
, tmp_path
= GIT_BUF_INIT
;
116 static const char suff
[] = "/pack";
119 idx
= git__calloc(1, sizeof(git_indexer
));
120 GITERR_CHECK_ALLOC(idx
);
122 idx
->progress_cb
= progress_cb
;
123 idx
->progress_payload
= progress_payload
;
124 idx
->mode
= mode
? mode
: GIT_PACK_FILE_MODE
;
125 git_hash_ctx_init(&idx
->hash_ctx
);
126 git_hash_ctx_init(&idx
->trailer
);
128 if (git_repository__fsync_gitdir
)
131 error
= git_buf_joinpath(&path
, prefix
, suff
);
135 fd
= git_futils_mktmp(&tmp_path
, git_buf_cstr(&path
), idx
->mode
);
140 error
= git_packfile_alloc(&idx
->pack
, git_buf_cstr(&tmp_path
));
141 git_buf_free(&tmp_path
);
146 idx
->pack
->mwf
.fd
= fd
;
147 if ((error
= git_mwindow_file_register(&idx
->pack
->mwf
)) < 0)
157 if (git_buf_len(&tmp_path
) > 0)
158 p_unlink(git_buf_cstr(&tmp_path
));
160 if (idx
->pack
!= NULL
)
161 p_unlink(idx
->pack
->pack_name
);
164 git_buf_free(&tmp_path
);
169 void git_indexer__set_fsync(git_indexer
*idx
, int do_fsync
)
171 idx
->do_fsync
= !!do_fsync
;
174 /* Try to store the delta so we can try to resolve it later */
175 static int store_delta(git_indexer
*idx
)
177 struct delta_info
*delta
;
179 delta
= git__calloc(1, sizeof(struct delta_info
));
180 GITERR_CHECK_ALLOC(delta
);
181 delta
->delta_off
= idx
->entry_start
;
183 if (git_vector_insert(&idx
->deltas
, delta
) < 0)
189 static void hash_header(git_hash_ctx
*ctx
, git_off_t len
, git_otype type
)
194 hdrlen
= git_odb__format_object_header(buffer
, sizeof(buffer
), (size_t)len
, type
);
195 git_hash_update(ctx
, buffer
, hdrlen
);
198 static int hash_object_stream(git_indexer
*idx
, git_packfile_stream
*stream
)
202 assert(idx
&& stream
);
205 if ((read
= git_packfile_stream_read(stream
, idx
->objbuf
, sizeof(idx
->objbuf
))) < 0)
208 git_hash_update(&idx
->hash_ctx
, idx
->objbuf
, read
);
217 /* In order to create the packfile stream, we need to skip over the delta base description */
218 static int advance_delta_offset(git_indexer
*idx
, git_otype type
)
220 git_mwindow
*w
= NULL
;
222 assert(type
== GIT_OBJ_REF_DELTA
|| type
== GIT_OBJ_OFS_DELTA
);
224 if (type
== GIT_OBJ_REF_DELTA
) {
225 idx
->off
+= GIT_OID_RAWSZ
;
227 git_off_t base_off
= get_delta_base(idx
->pack
, &w
, &idx
->off
, type
, idx
->entry_start
);
228 git_mwindow_close(&w
);
230 return (int)base_off
;
236 /* Read from the stream and discard any output */
237 static int read_object_stream(git_indexer
*idx
, git_packfile_stream
*stream
)
244 read
= git_packfile_stream_read(stream
, idx
->objbuf
, sizeof(idx
->objbuf
));
253 static int crc_object(uint32_t *crc_out
, git_mwindow_file
*mwf
, git_off_t start
, git_off_t size
)
257 unsigned int left
, len
;
258 git_mwindow
*w
= NULL
;
260 crc
= crc32(0L, Z_NULL
, 0);
262 ptr
= git_mwindow_open(mwf
, &w
, start
, (size_t)size
, &left
);
266 len
= min(left
, (unsigned int)size
);
267 crc
= crc32(crc
, ptr
, len
);
270 git_mwindow_close(&w
);
273 *crc_out
= htonl(crc
);
277 static int store_object(git_indexer
*idx
)
283 git_off_t entry_size
;
284 struct git_pack_entry
*pentry
;
285 git_off_t entry_start
= idx
->entry_start
;
287 entry
= git__calloc(1, sizeof(*entry
));
288 GITERR_CHECK_ALLOC(entry
);
290 pentry
= git__calloc(1, sizeof(struct git_pack_entry
));
291 GITERR_CHECK_ALLOC(pentry
);
293 git_hash_final(&oid
, &idx
->hash_ctx
);
294 entry_size
= idx
->off
- entry_start
;
295 if (entry_start
> UINT31_MAX
) {
296 entry
->offset
= UINT32_MAX
;
297 entry
->offset_long
= entry_start
;
299 entry
->offset
= (uint32_t)entry_start
;
302 git_oid_cpy(&pentry
->sha1
, &oid
);
303 pentry
->offset
= entry_start
;
305 k
= git_oidmap_put(idx
->pack
->idx_cache
, &pentry
->sha1
, &error
);
313 giterr_set(GITERR_INDEXER
, "duplicate object %s found in pack", git_oid_tostr_s(&pentry
->sha1
));
319 git_oidmap_set_value_at(idx
->pack
->idx_cache
, k
, pentry
);
321 git_oid_cpy(&entry
->oid
, &oid
);
323 if (crc_object(&entry
->crc
, &idx
->pack
->mwf
, entry_start
, entry_size
) < 0)
326 /* Add the object to the list */
327 if (git_vector_insert(&idx
->objects
, entry
) < 0)
330 for (i
= oid
.id
[0]; i
< 256; ++i
) {
342 GIT_INLINE(bool) has_entry(git_indexer
*idx
, git_oid
*id
)
344 return git_oidmap_exists(idx
->pack
->idx_cache
, id
);
347 static int save_entry(git_indexer
*idx
, struct entry
*entry
, struct git_pack_entry
*pentry
, git_off_t entry_start
)
352 if (entry_start
> UINT31_MAX
) {
353 entry
->offset
= UINT32_MAX
;
354 entry
->offset_long
= entry_start
;
356 entry
->offset
= (uint32_t)entry_start
;
359 pentry
->offset
= entry_start
;
360 k
= git_oidmap_put(idx
->pack
->idx_cache
, &pentry
->sha1
, &error
);
363 giterr_set(GITERR_INDEXER
, "cannot insert object into pack");
367 git_oidmap_set_value_at(idx
->pack
->idx_cache
, k
, pentry
);
369 /* Add the object to the list */
370 if (git_vector_insert(&idx
->objects
, entry
) < 0)
373 for (i
= entry
->oid
.id
[0]; i
< 256; ++i
) {
380 static int hash_and_save(git_indexer
*idx
, git_rawobj
*obj
, git_off_t entry_start
)
385 struct git_pack_entry
*pentry
= NULL
;
387 entry
= git__calloc(1, sizeof(*entry
));
388 GITERR_CHECK_ALLOC(entry
);
390 if (git_odb__hashobj(&oid
, obj
) < 0) {
391 giterr_set(GITERR_INDEXER
, "failed to hash object");
395 pentry
= git__calloc(1, sizeof(struct git_pack_entry
));
396 GITERR_CHECK_ALLOC(pentry
);
398 git_oid_cpy(&pentry
->sha1
, &oid
);
399 git_oid_cpy(&entry
->oid
, &oid
);
400 entry
->crc
= crc32(0L, Z_NULL
, 0);
402 entry_size
= (size_t)(idx
->off
- entry_start
);
403 if (crc_object(&entry
->crc
, &idx
->pack
->mwf
, entry_start
, entry_size
) < 0)
406 return save_entry(idx
, entry
, pentry
, entry_start
);
411 git__free(obj
->data
);
415 static int do_progress_callback(git_indexer
*idx
, git_transfer_progress
*stats
)
417 if (idx
->progress_cb
)
418 return giterr_set_after_callback_function(
419 idx
->progress_cb(stats
, idx
->progress_payload
),
424 /* Hash everything but the last 20B of input */
425 static void hash_partially(git_indexer
*idx
, const uint8_t *data
, size_t size
)
427 size_t to_expell
, to_keep
;
432 /* Easy case, dump the buffer and the data minus the last 20 bytes */
433 if (size
>= GIT_OID_RAWSZ
) {
434 git_hash_update(&idx
->trailer
, idx
->inbuf
, idx
->inbuf_len
);
435 git_hash_update(&idx
->trailer
, data
, size
- GIT_OID_RAWSZ
);
437 data
+= size
- GIT_OID_RAWSZ
;
438 memcpy(idx
->inbuf
, data
, GIT_OID_RAWSZ
);
439 idx
->inbuf_len
= GIT_OID_RAWSZ
;
443 /* We can just append */
444 if (idx
->inbuf_len
+ size
<= GIT_OID_RAWSZ
) {
445 memcpy(idx
->inbuf
+ idx
->inbuf_len
, data
, size
);
446 idx
->inbuf_len
+= size
;
450 /* We need to partially drain the buffer and then append */
451 to_keep
= GIT_OID_RAWSZ
- size
;
452 to_expell
= idx
->inbuf_len
- to_keep
;
454 git_hash_update(&idx
->trailer
, idx
->inbuf
, to_expell
);
456 memmove(idx
->inbuf
, idx
->inbuf
+ to_expell
, to_keep
);
457 memcpy(idx
->inbuf
+ to_keep
, data
, size
);
458 idx
->inbuf_len
+= size
- to_expell
;
461 static int write_at(git_indexer
*idx
, const void *data
, git_off_t offset
, size_t size
)
463 git_file fd
= idx
->pack
->mwf
.fd
;
464 size_t mmap_alignment
;
466 git_off_t page_start
;
467 unsigned char *map_data
;
471 assert(data
&& size
);
473 if ((error
= git__mmap_alignment(&mmap_alignment
)) < 0)
476 /* the offset needs to be at the mmap boundary for the platform */
477 page_offset
= offset
% mmap_alignment
;
478 page_start
= offset
- page_offset
;
480 if ((error
= p_mmap(&map
, page_offset
+ size
, GIT_PROT_WRITE
, GIT_MAP_SHARED
, fd
, page_start
)) < 0)
483 map_data
= (unsigned char *)map
.data
;
484 memcpy(map_data
+ page_offset
, data
, size
);
490 static int append_to_pack(git_indexer
*idx
, const void *data
, size_t size
)
493 size_t mmap_alignment
;
495 git_off_t page_start
;
496 git_off_t current_size
= idx
->pack
->mwf
.size
;
497 int fd
= idx
->pack
->mwf
.fd
;
503 if ((error
= git__mmap_alignment(&mmap_alignment
)) < 0)
506 /* Write a single byte to force the file system to allocate space now or
507 * report an error, since we can't report errors when writing using mmap.
508 * Round the size up to the nearest page so that we only need to perform file
509 * I/O when we add a page, instead of whenever we write even a single byte. */
510 new_size
= current_size
+ size
;
511 page_offset
= new_size
% mmap_alignment
;
512 page_start
= new_size
- page_offset
;
514 if (p_lseek(fd
, page_start
+ mmap_alignment
- 1, SEEK_SET
) < 0 ||
515 p_write(idx
->pack
->mwf
.fd
, data
, 1) < 0) {
516 giterr_set(GITERR_OS
, "cannot extend packfile '%s'", idx
->pack
->pack_name
);
520 return write_at(idx
, data
, idx
->pack
->mwf
.size
, size
);
523 int git_indexer_append(git_indexer
*idx
, const void *data
, size_t size
, git_transfer_progress
*stats
)
527 struct git_pack_header
*hdr
= &idx
->hdr
;
528 git_mwindow_file
*mwf
= &idx
->pack
->mwf
;
530 assert(idx
&& data
&& stats
);
532 processed
= stats
->indexed_objects
;
534 if ((error
= append_to_pack(idx
, data
, size
)) < 0)
537 hash_partially(idx
, data
, (int)size
);
539 /* Make sure we set the new size of the pack */
540 idx
->pack
->mwf
.size
+= size
;
542 if (!idx
->parsed_header
) {
543 unsigned int total_objects
;
545 if ((unsigned)idx
->pack
->mwf
.size
< sizeof(struct git_pack_header
))
548 if ((error
= parse_header(&idx
->hdr
, idx
->pack
)) < 0)
551 idx
->parsed_header
= 1;
552 idx
->nr_objects
= ntohl(hdr
->hdr_entries
);
553 idx
->off
= sizeof(struct git_pack_header
);
555 /* for now, limit to 2^32 objects */
556 assert(idx
->nr_objects
== (size_t)((unsigned int)idx
->nr_objects
));
557 if (idx
->nr_objects
== (size_t)((unsigned int)idx
->nr_objects
))
558 total_objects
= (unsigned int)idx
->nr_objects
;
560 total_objects
= UINT_MAX
;
562 idx
->pack
->idx_cache
= git_oidmap_alloc();
563 GITERR_CHECK_ALLOC(idx
->pack
->idx_cache
);
565 idx
->pack
->has_cache
= 1;
566 if (git_vector_init(&idx
->objects
, total_objects
, objects_cmp
) < 0)
569 if (git_vector_init(&idx
->deltas
, total_objects
/ 2, NULL
) < 0)
572 stats
->received_objects
= 0;
573 stats
->local_objects
= 0;
574 stats
->total_deltas
= 0;
575 stats
->indexed_deltas
= 0;
576 processed
= stats
->indexed_objects
= 0;
577 stats
->total_objects
= total_objects
;
579 if ((error
= do_progress_callback(idx
, stats
)) != 0)
583 /* Now that we have data in the pack, let's try to parse it */
585 /* As the file grows any windows we try to use will be out of date */
586 git_mwindow_free_all(mwf
);
588 while (processed
< idx
->nr_objects
) {
589 git_packfile_stream
*stream
= &idx
->stream
;
590 git_off_t entry_start
= idx
->off
;
593 git_mwindow
*w
= NULL
;
595 if (idx
->pack
->mwf
.size
<= idx
->off
+ 20)
598 if (!idx
->have_stream
) {
599 error
= git_packfile_unpack_header(&entry_size
, &type
, mwf
, &w
, &idx
->off
);
600 if (error
== GIT_EBUFS
) {
601 idx
->off
= entry_start
;
607 git_mwindow_close(&w
);
608 idx
->entry_start
= entry_start
;
609 git_hash_init(&idx
->hash_ctx
);
611 if (type
== GIT_OBJ_REF_DELTA
|| type
== GIT_OBJ_OFS_DELTA
) {
612 error
= advance_delta_offset(idx
, type
);
613 if (error
== GIT_EBUFS
) {
614 idx
->off
= entry_start
;
623 hash_header(&idx
->hash_ctx
, entry_size
, type
);
626 idx
->have_stream
= 1;
628 error
= git_packfile_stream_open(stream
, idx
->pack
, idx
->off
);
633 if (idx
->have_delta
) {
634 error
= read_object_stream(idx
, stream
);
636 error
= hash_object_stream(idx
, stream
);
639 idx
->off
= stream
->curpos
;
640 if (error
== GIT_EBUFS
)
643 /* We want to free the stream reasorces no matter what here */
644 idx
->have_stream
= 0;
645 git_packfile_stream_free(stream
);
650 if (idx
->have_delta
) {
651 error
= store_delta(idx
);
653 error
= store_object(idx
);
659 if (!idx
->have_delta
) {
660 stats
->indexed_objects
= (unsigned int)++processed
;
662 stats
->received_objects
++;
664 if ((error
= do_progress_callback(idx
, stats
)) != 0)
671 git_mwindow_free_all(mwf
);
675 static int index_path(git_buf
*path
, git_indexer
*idx
, const char *suffix
)
677 const char prefix
[] = "pack-";
678 size_t slash
= (size_t)path
->size
;
680 /* search backwards for '/' */
681 while (slash
> 0 && path
->ptr
[slash
- 1] != '/')
684 if (git_buf_grow(path
, slash
+ 1 + strlen(prefix
) +
685 GIT_OID_HEXSZ
+ strlen(suffix
) + 1) < 0)
688 git_buf_truncate(path
, slash
);
689 git_buf_puts(path
, prefix
);
690 git_oid_fmt(path
->ptr
+ git_buf_len(path
), &idx
->hash
);
691 path
->size
+= GIT_OID_HEXSZ
;
692 git_buf_puts(path
, suffix
);
694 return git_buf_oom(path
) ? -1 : 0;
698 * Rewind the packfile by the trailer, as we might need to fix the
699 * packfile by injecting objects at the tail and must overwrite it.
701 static void seek_back_trailer(git_indexer
*idx
)
703 idx
->pack
->mwf
.size
-= GIT_OID_RAWSZ
;
704 git_mwindow_free_all(&idx
->pack
->mwf
);
707 static int inject_object(git_indexer
*idx
, git_oid
*id
)
711 struct git_pack_entry
*pentry
= NULL
;
713 unsigned char hdr
[64];
714 git_buf buf
= GIT_BUF_INIT
;
715 git_off_t entry_start
;
720 seek_back_trailer(idx
);
721 entry_start
= idx
->pack
->mwf
.size
;
723 if (git_odb_read(&obj
, idx
->odb
, id
) < 0) {
724 giterr_set(GITERR_INDEXER
, "missing delta bases");
728 data
= git_odb_object_data(obj
);
729 len
= git_odb_object_size(obj
);
731 entry
= git__calloc(1, sizeof(*entry
));
732 GITERR_CHECK_ALLOC(entry
);
734 entry
->crc
= crc32(0L, Z_NULL
, 0);
736 /* Write out the object header */
737 hdr_len
= git_packfile__object_header(hdr
, len
, git_odb_object_type(obj
));
738 if ((error
= append_to_pack(idx
, hdr
, hdr_len
)) < 0)
741 idx
->pack
->mwf
.size
+= hdr_len
;
742 entry
->crc
= crc32(entry
->crc
, hdr
, (uInt
)hdr_len
);
744 if ((error
= git_zstream_deflatebuf(&buf
, data
, len
)) < 0)
747 /* And then the compressed object */
748 if ((error
= append_to_pack(idx
, buf
.ptr
, buf
.size
)) < 0)
751 idx
->pack
->mwf
.size
+= buf
.size
;
752 entry
->crc
= htonl(crc32(entry
->crc
, (unsigned char *)buf
.ptr
, (uInt
)buf
.size
));
755 /* Write a fake trailer so the pack functions play ball */
757 if ((error
= append_to_pack(idx
, &foo
, GIT_OID_RAWSZ
)) < 0)
760 idx
->pack
->mwf
.size
+= GIT_OID_RAWSZ
;
762 pentry
= git__calloc(1, sizeof(struct git_pack_entry
));
763 GITERR_CHECK_ALLOC(pentry
);
765 git_oid_cpy(&pentry
->sha1
, id
);
766 git_oid_cpy(&entry
->oid
, id
);
767 idx
->off
= entry_start
+ hdr_len
+ len
;
769 error
= save_entry(idx
, entry
, pentry
, entry_start
);
777 git_odb_object_free(obj
);
781 static int fix_thin_pack(git_indexer
*idx
, git_transfer_progress
*stats
)
783 int error
, found_ref_delta
= 0;
785 struct delta_info
*delta
;
788 git_mwindow
*w
= NULL
;
789 git_off_t curpos
= 0;
790 unsigned char *base_info
;
791 unsigned int left
= 0;
794 assert(git_vector_length(&idx
->deltas
) > 0);
796 if (idx
->odb
== NULL
) {
797 giterr_set(GITERR_INDEXER
, "cannot fix a thin pack without an ODB");
801 /* Loop until we find the first REF delta */
802 git_vector_foreach(&idx
->deltas
, i
, delta
) {
806 curpos
= delta
->delta_off
;
807 error
= git_packfile_unpack_header(&size
, &type
, &idx
->pack
->mwf
, &w
, &curpos
);
811 if (type
== GIT_OBJ_REF_DELTA
) {
817 if (!found_ref_delta
) {
818 giterr_set(GITERR_INDEXER
, "no REF_DELTA found, cannot inject object");
822 /* curpos now points to the base information, which is an OID */
823 base_info
= git_mwindow_open(&idx
->pack
->mwf
, &w
, curpos
, GIT_OID_RAWSZ
, &left
);
824 if (base_info
== NULL
) {
825 giterr_set(GITERR_INDEXER
, "failed to map delta information");
829 git_oid_fromraw(&base
, base_info
);
830 git_mwindow_close(&w
);
832 if (has_entry(idx
, &base
))
835 if (inject_object(idx
, &base
) < 0)
838 stats
->local_objects
++;
843 static int resolve_deltas(git_indexer
*idx
, git_transfer_progress
*stats
)
846 struct delta_info
*delta
;
847 int progressed
= 0, non_null
= 0, progress_cb_result
;
849 while (idx
->deltas
.length
> 0) {
852 git_vector_foreach(&idx
->deltas
, i
, delta
) {
853 git_rawobj obj
= {NULL
};
859 idx
->off
= delta
->delta_off
;
860 if (git_packfile_unpack(&obj
, idx
->pack
, &idx
->off
) < 0)
863 if (hash_and_save(idx
, &obj
, delta
->delta_off
) < 0)
867 stats
->indexed_objects
++;
868 stats
->indexed_deltas
++;
870 if ((progress_cb_result
= do_progress_callback(idx
, stats
)) < 0)
871 return progress_cb_result
;
873 /* remove from the list */
874 git_vector_set(NULL
, &idx
->deltas
, i
, NULL
);
878 /* if none were actually set, we're done */
882 if (!progressed
&& (fix_thin_pack(idx
, stats
) < 0)) {
890 static int update_header_and_rehash(git_indexer
*idx
, git_transfer_progress
*stats
)
893 size_t chunk
= 1024*1024;
894 git_off_t hashed
= 0;
895 git_mwindow
*w
= NULL
;
896 git_mwindow_file
*mwf
;
899 mwf
= &idx
->pack
->mwf
;
901 git_hash_init(&idx
->trailer
);
904 /* Update the header to include the numer of local objects we injected */
905 idx
->hdr
.hdr_entries
= htonl(stats
->total_objects
+ stats
->local_objects
);
906 if (write_at(idx
, &idx
->hdr
, 0, sizeof(struct git_pack_header
)) < 0)
910 * We now use the same technique as before to determine the
911 * hash. We keep reading up to the end and let
912 * hash_partially() keep the existing trailer out of the
915 git_mwindow_free_all(mwf
);
917 while (hashed
< mwf
->size
) {
918 ptr
= git_mwindow_open(mwf
, &w
, hashed
, chunk
, &left
);
922 hash_partially(idx
, ptr
, left
);
925 git_mwindow_close(&w
);
931 int git_indexer_commit(git_indexer
*idx
, git_transfer_progress
*stats
)
933 git_mwindow
*w
= NULL
;
934 unsigned int i
, long_offsets
= 0, left
;
936 struct git_pack_idx_header hdr
;
937 git_buf filename
= GIT_BUF_INIT
;
939 git_oid trailer_hash
, file_hash
;
941 git_filebuf index_file
= {0};
942 void *packfile_trailer
;
944 if (!idx
->parsed_header
) {
945 giterr_set(GITERR_INDEXER
, "incomplete pack header");
949 if (git_hash_ctx_init(&ctx
) < 0)
952 /* Test for this before resolve_deltas(), as it plays with idx->off */
953 if (idx
->off
+ 20 < idx
->pack
->mwf
.size
) {
954 giterr_set(GITERR_INDEXER
, "unexpected data at the end of the pack");
958 packfile_trailer
= git_mwindow_open(&idx
->pack
->mwf
, &w
, idx
->pack
->mwf
.size
- GIT_OID_RAWSZ
, GIT_OID_RAWSZ
, &left
);
959 if (packfile_trailer
== NULL
) {
960 git_mwindow_close(&w
);
964 /* Compare the packfile trailer as it was sent to us and what we calculated */
965 git_oid_fromraw(&file_hash
, packfile_trailer
);
966 git_mwindow_close(&w
);
968 git_hash_final(&trailer_hash
, &idx
->trailer
);
969 if (git_oid_cmp(&file_hash
, &trailer_hash
)) {
970 giterr_set(GITERR_INDEXER
, "packfile trailer mismatch");
974 /* Freeze the number of deltas */
975 stats
->total_deltas
= stats
->total_objects
- stats
->indexed_objects
;
977 if ((error
= resolve_deltas(idx
, stats
)) < 0)
980 if (stats
->indexed_objects
!= stats
->total_objects
) {
981 giterr_set(GITERR_INDEXER
, "early EOF");
985 if (stats
->local_objects
> 0) {
986 if (update_header_and_rehash(idx
, stats
) < 0)
989 git_hash_final(&trailer_hash
, &idx
->trailer
);
990 write_at(idx
, &trailer_hash
, idx
->pack
->mwf
.size
- GIT_OID_RAWSZ
, GIT_OID_RAWSZ
);
993 git_vector_sort(&idx
->objects
);
995 git_buf_sets(&filename
, idx
->pack
->pack_name
);
996 git_buf_shorten(&filename
, strlen("pack"));
997 git_buf_puts(&filename
, "idx");
998 if (git_buf_oom(&filename
))
1001 if (git_filebuf_open(&index_file
, filename
.ptr
,
1002 GIT_FILEBUF_HASH_CONTENTS
|
1003 (idx
->do_fsync
? GIT_FILEBUF_FSYNC
: 0),
1007 /* Write out the header */
1008 hdr
.idx_signature
= htonl(PACK_IDX_SIGNATURE
);
1009 hdr
.idx_version
= htonl(2);
1010 git_filebuf_write(&index_file
, &hdr
, sizeof(hdr
));
1012 /* Write out the fanout table */
1013 for (i
= 0; i
< 256; ++i
) {
1014 uint32_t n
= htonl(idx
->fanout
[i
]);
1015 git_filebuf_write(&index_file
, &n
, sizeof(n
));
1018 /* Write out the object names (SHA-1 hashes) */
1019 git_vector_foreach(&idx
->objects
, i
, entry
) {
1020 git_filebuf_write(&index_file
, &entry
->oid
, sizeof(git_oid
));
1021 git_hash_update(&ctx
, &entry
->oid
, GIT_OID_RAWSZ
);
1023 git_hash_final(&idx
->hash
, &ctx
);
1025 /* Write out the CRC32 values */
1026 git_vector_foreach(&idx
->objects
, i
, entry
) {
1027 git_filebuf_write(&index_file
, &entry
->crc
, sizeof(uint32_t));
1030 /* Write out the offsets */
1031 git_vector_foreach(&idx
->objects
, i
, entry
) {
1034 if (entry
->offset
== UINT32_MAX
)
1035 n
= htonl(0x80000000 | long_offsets
++);
1037 n
= htonl(entry
->offset
);
1039 git_filebuf_write(&index_file
, &n
, sizeof(uint32_t));
1042 /* Write out the long offsets */
1043 git_vector_foreach(&idx
->objects
, i
, entry
) {
1046 if (entry
->offset
!= UINT32_MAX
)
1049 split
[0] = htonl(entry
->offset_long
>> 32);
1050 split
[1] = htonl(entry
->offset_long
& 0xffffffff);
1052 git_filebuf_write(&index_file
, &split
, sizeof(uint32_t) * 2);
1055 /* Write out the packfile trailer to the index */
1056 if (git_filebuf_write(&index_file
, &trailer_hash
, GIT_OID_RAWSZ
) < 0)
1059 /* Write out the hash of the idx */
1060 if (git_filebuf_hash(&trailer_hash
, &index_file
) < 0)
1063 git_filebuf_write(&index_file
, &trailer_hash
, sizeof(git_oid
));
1065 /* Figure out what the final name should be */
1066 if (index_path(&filename
, idx
, ".idx") < 0)
1070 if (git_filebuf_commit_at(&index_file
, filename
.ptr
) < 0)
1073 git_mwindow_free_all(&idx
->pack
->mwf
);
1075 /* Truncate file to undo rounding up to next page_size in append_to_pack */
1076 if (p_ftruncate(idx
->pack
->mwf
.fd
, idx
->pack
->mwf
.size
) < 0) {
1077 giterr_set(GITERR_OS
, "failed to truncate pack file '%s'", idx
->pack
->pack_name
);
1081 if (idx
->do_fsync
&& p_fsync(idx
->pack
->mwf
.fd
) < 0) {
1082 giterr_set(GITERR_OS
, "failed to fsync packfile");
1086 /* We need to close the descriptor here so Windows doesn't choke on commit_at */
1087 if (p_close(idx
->pack
->mwf
.fd
) < 0) {
1088 giterr_set(GITERR_OS
, "failed to close packfile");
1092 idx
->pack
->mwf
.fd
= -1;
1094 if (index_path(&filename
, idx
, ".pack") < 0)
1097 /* And don't forget to rename the packfile to its new place. */
1098 if (p_rename(idx
->pack
->pack_name
, git_buf_cstr(&filename
)) < 0)
1101 /* And fsync the parent directory if we're asked to. */
1102 if (idx
->do_fsync
&&
1103 git_futils_fsync_parent(git_buf_cstr(&filename
)) < 0)
1106 idx
->pack_committed
= 1;
1108 git_buf_free(&filename
);
1109 git_hash_ctx_cleanup(&ctx
);
1113 git_mwindow_free_all(&idx
->pack
->mwf
);
1114 git_filebuf_cleanup(&index_file
);
1115 git_buf_free(&filename
);
1116 git_hash_ctx_cleanup(&ctx
);
1120 void git_indexer_free(git_indexer
*idx
)
1125 git_vector_free_deep(&idx
->objects
);
1127 if (idx
->pack
->idx_cache
) {
1128 struct git_pack_entry
*pentry
;
1129 git_oidmap_foreach_value(idx
->pack
->idx_cache
, pentry
, {
1133 git_oidmap_free(idx
->pack
->idx_cache
);
1136 git_vector_free_deep(&idx
->deltas
);
1138 if (!git_mutex_lock(&git__mwindow_mutex
)) {
1139 if (!idx
->pack_committed
)
1140 git_packfile_close(idx
->pack
, true);
1142 git_packfile_free(idx
->pack
);
1143 git_mutex_unlock(&git__mwindow_mutex
);
1146 git_hash_ctx_cleanup(&idx
->trailer
);
1147 git_hash_ctx_cleanup(&idx
->hash_ctx
);