tests/pack/indexer.c

   1 #include "clar_libgit2.h"
   2 #include <git2.h>
   3 #include "futils.h"
   4 #include "hash.h"
   5 #include "iterator.h"
   6 #include "vector.h"
   7 #include "posix.h"
   8
   9
  10 /*
  11  * This is a packfile with three objects. The second is a delta which
  12  * depends on the third, which is also a delta.
  13  */
  14 static const unsigned char out_of_order_pack[] = {
  15   0x50, 0x41, 0x43, 0x4b, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x03,
  16   0x32, 0x78, 0x9c, 0x63, 0x67, 0x00, 0x00, 0x00, 0x10, 0x00, 0x08, 0x76,
  17   0xe6, 0x8f, 0xe8, 0x12, 0x9b, 0x54, 0x6b, 0x10, 0x1a, 0xee, 0x95, 0x10,
  18   0xc5, 0x32, 0x8e, 0x7f, 0x21, 0xca, 0x1d, 0x18, 0x78, 0x9c, 0x63, 0x62,
  19   0x66, 0x4e, 0xcb, 0xcf, 0x07, 0x00, 0x02, 0xac, 0x01, 0x4d, 0x75, 0x01,
  20   0xd7, 0x71, 0x36, 0x66, 0xf4, 0xde, 0x82, 0x27, 0x76, 0xc7, 0x62, 0x2c,
  21   0x10, 0xf1, 0xb0, 0x7d, 0xe2, 0x80, 0xdc, 0x78, 0x9c, 0x63, 0x62, 0x62,
  22   0x62, 0xb7, 0x03, 0x00, 0x00, 0x69, 0x00, 0x4c, 0xde, 0x7d, 0xaa, 0xe4,
  23   0x19, 0x87, 0x58, 0x80, 0x61, 0x09, 0x9a, 0x33, 0xca, 0x7a, 0x31, 0x92,
  24   0x6f, 0xae, 0x66, 0x75
  25 };
  26 static const unsigned int out_of_order_pack_len = 112;
  27
  28 /*
  29  * Packfile with two objects. The second is a delta against an object
  30  * which is not in the packfile
  31  */
  32 static const unsigned char thin_pack[] = {
  33   0x50, 0x41, 0x43, 0x4b, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x02,
  34   0x32, 0x78, 0x9c, 0x63, 0x67, 0x00, 0x00, 0x00, 0x10, 0x00, 0x08, 0x76,
  35   0xe6, 0x8f, 0xe8, 0x12, 0x9b, 0x54, 0x6b, 0x10, 0x1a, 0xee, 0x95, 0x10,
  36   0xc5, 0x32, 0x8e, 0x7f, 0x21, 0xca, 0x1d, 0x18, 0x78, 0x9c, 0x63, 0x62,
  37   0x66, 0x4e, 0xcb, 0xcf, 0x07, 0x00, 0x02, 0xac, 0x01, 0x4d, 0x42, 0x52,
  38   0x3a, 0x6f, 0x39, 0xd1, 0xfe, 0x66, 0x68, 0x6b, 0xa5, 0xe5, 0xe2, 0x97,
  39   0xac, 0x94, 0x6c, 0x76, 0x0b, 0x04
  40 };
  41 static const unsigned int thin_pack_len = 78;
  42
  43 /*
  44  * Packfile with one object. It references an object which is not in the
  45  * packfile and has a corrupt length (states the deltified stream is 1 byte
  46  * long, where it is actually 6).
  47  */
  48 static const unsigned char corrupt_thin_pack[] = {
  49   0x50, 0x41, 0x43, 0x4b, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x01,
  50   0x71, 0xe6, 0x8f, 0xe8, 0x12, 0x9b, 0x54, 0x6b, 0x10, 0x1a, 0xee, 0x95,
  51   0x10, 0xc5, 0x32, 0x8e, 0x7f, 0x21, 0xca, 0x1d, 0x18, 0x78, 0x9c, 0x63,
  52   0x62, 0x66, 0x4e, 0xcb, 0xcf, 0x07, 0x00, 0x02, 0xac, 0x01, 0x4d, 0x07,
  53   0x67, 0x03, 0xc5, 0x40, 0x99, 0x49, 0xb1, 0x3b, 0x7d, 0xae, 0x9b, 0x0e,
  54   0xdd, 0xde, 0xc6, 0x76, 0x43, 0x24, 0x64
  55 };
  56 static const unsigned int corrupt_thin_pack_len = 67;
  57
  58 /*
  59  * Packfile with a missing trailer.
  60  */
  61 static const unsigned char missing_trailer_pack[] = {
  62   0x50, 0x41, 0x43, 0x4b, 0x00, 0x00, 0x00, 0x03, 0x00, 0x50, 0xf4, 0x3b,
  63 };
  64 static const unsigned int missing_trailer_pack_len = 12;
  65
  66 /*
  67  * Packfile that causes the packfile stream to open in a way in which it leaks
  68  * the stream reader.
  69  */
  70 static const unsigned char leaky_pack[] = {
  71         0x50, 0x41, 0x43, 0x4b, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x03,
  72         0xf4, 0xbd, 0x51, 0x51, 0x51, 0x51, 0x51, 0x72, 0x65, 0x41, 0x4b, 0x63,
  73         0x5f, 0x64, 0x65, 0x70, 0x74, 0x68, 0xbd, 0x41, 0x4b
  74 };
  75 static const unsigned int leaky_pack_len = 33;
  76
  77 /*
  78  * Packfile with a three objects. The first one is a tree referencing two blobs,
  79  * the second object is one of those blobs. The second blob is missing.
  80  */
  81 unsigned char incomplete_pack[] = {
  82   0x50, 0x41, 0x43, 0x4b, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x02,
  83   0xae, 0x03, 0x78, 0x9c, 0x33, 0x34, 0x30, 0x30, 0x33, 0x31, 0x51, 0x48,
  84   0x4a, 0x2c, 0x62, 0x08, 0x17, 0x3b, 0x15, 0xd9, 0x7e, 0xfa, 0x67, 0x6d,
  85   0xf6, 0x56, 0x4f, 0x85, 0x7d, 0xcb, 0xd6, 0xde, 0x53, 0xd1, 0x6d, 0x7f,
  86   0x66, 0x08, 0x91, 0x4e, 0xcb, 0xcf, 0x67, 0x50, 0xad, 0x39, 0x9a, 0xa2,
  87   0xb3, 0x71, 0x41, 0xc8, 0x87, 0x9e, 0x13, 0xf6, 0xba, 0x53, 0xec, 0xc2,
  88   0xfe, 0xda, 0xed, 0x9b, 0x09, 0x00, 0xe8, 0xc8, 0x19, 0xab, 0x34, 0x78,
  89   0x9c, 0x4b, 0x4a, 0x2c, 0xe2, 0x02, 0x00, 0x03, 0x9d, 0x01, 0x40, 0x4b,
  90   0x72, 0xa2, 0x6f, 0xb6, 0x88, 0x2d, 0x6c, 0xa5, 0x07, 0xb2, 0xa5, 0x45,
  91   0xe8, 0xdb, 0xe6, 0x53, 0xb3, 0x52, 0xe2
  92 };
  93 unsigned int incomplete_pack_len = 115;
  94
  95 static const unsigned char base_obj[] = { 07, 076 };
  96 static const unsigned int base_obj_len = 2;
  97
  98 void test_pack_indexer__out_of_order(void)
  99 {
 100         git_indexer *idx = 0;
 101         git_indexer_progress stats = { 0 };
 102
 103         cl_git_pass(git_indexer_new(&idx, ".", 0, NULL, NULL));
 104         cl_git_pass(git_indexer_append(
 105                 idx, out_of_order_pack, out_of_order_pack_len, &stats));
 106         cl_git_pass(git_indexer_commit(idx, &stats));
 107
 108         cl_assert_equal_i(stats.total_objects, 3);
 109         cl_assert_equal_i(stats.received_objects, 3);
 110         cl_assert_equal_i(stats.indexed_objects, 3);
 111
 112         git_indexer_free(idx);
 113 }
 114
 115 void test_pack_indexer__missing_trailer(void)
 116 {
 117         git_indexer *idx = 0;
 118         git_indexer_progress stats = { 0 };
 119
 120         cl_git_pass(git_indexer_new(&idx, ".", 0, NULL, NULL));
 121         cl_git_pass(git_indexer_append(
 122                 idx, missing_trailer_pack, missing_trailer_pack_len, &stats));
 123         cl_git_fail(git_indexer_commit(idx, &stats));
 124
 125         cl_assert(git_error_last() != NULL);
 126         cl_assert_equal_i(git_error_last()->klass, GIT_ERROR_INDEXER);
 127
 128         git_indexer_free(idx);
 129 }
 130
 131 void test_pack_indexer__leaky(void)
 132 {
 133         git_indexer *idx = 0;
 134         git_indexer_progress stats = { 0 };
 135
 136         cl_git_pass(git_indexer_new(&idx, ".", 0, NULL, NULL));
 137         cl_git_pass(git_indexer_append(
 138                 idx, leaky_pack, leaky_pack_len, &stats));
 139         cl_git_fail(git_indexer_commit(idx, &stats));
 140
 141         cl_assert(git_error_last() != NULL);
 142         cl_assert_equal_i(git_error_last()->klass, GIT_ERROR_INDEXER);
 143
 144         git_indexer_free(idx);
 145 }
 146
 147 void test_pack_indexer__fix_thin(void)
 148 {
 149         git_indexer *idx = NULL;
 150         git_indexer_progress stats = { 0 };
 151         git_repository *repo;
 152         git_odb *odb;
 153         git_oid id, should_id;
 154
 155         cl_git_pass(git_repository_init(&repo, "thin.git", true));
 156         cl_git_pass(git_repository_odb(&odb, repo));
 157
 158         /* Store the missing base into your ODB so the indexer can fix the pack */
 159         cl_git_pass(git_odb_write(&id, odb, base_obj, base_obj_len, GIT_OBJECT_BLOB));
 160         git_oid_fromstr(&should_id, "e68fe8129b546b101aee9510c5328e7f21ca1d18");
 161         cl_assert_equal_oid(&should_id, &id);
 162
 163         cl_git_pass(git_indexer_new(&idx, ".", 0, odb, NULL));
 164         cl_git_pass(git_indexer_append(idx, thin_pack, thin_pack_len, &stats));
 165         cl_git_pass(git_indexer_commit(idx, &stats));
 166
 167         cl_assert_equal_i(stats.total_objects, 2);
 168         cl_assert_equal_i(stats.received_objects, 2);
 169         cl_assert_equal_i(stats.indexed_objects, 2);
 170         cl_assert_equal_i(stats.local_objects, 1);
 171
 172         cl_assert_equal_s("fefdb2d740a3a6b6c03a0c7d6ce431c6d5810e13", git_indexer_name(idx));
 173
 174         git_indexer_free(idx);
 175         git_odb_free(odb);
 176         git_repository_free(repo);
 177
 178         /*
 179          * The pack's name/hash only tells us what objects there are,
 180          * so we need to go through the packfile again in order to
 181          * figure out whether we calculated the trailer correctly.
 182          */
 183         {
 184                 unsigned char buffer[128];
 185                 int fd;
 186                 ssize_t read;
 187                 struct stat st;
 188                 const char *name = "pack-fefdb2d740a3a6b6c03a0c7d6ce431c6d5810e13.pack";
 189
 190                 fd = p_open(name, O_RDONLY);
 191                 cl_assert(fd != -1);
 192
 193                 cl_git_pass(p_stat(name, &st));
 194
 195                 cl_git_pass(git_indexer_new(&idx, ".", 0, NULL, NULL));
 196                 read = p_read(fd, buffer, sizeof(buffer));
 197                 cl_assert(read != -1);
 198                 p_close(fd);
 199
 200                 cl_git_pass(git_indexer_append(idx, buffer, read, &stats));
 201                 cl_git_pass(git_indexer_commit(idx, &stats));
 202
 203                 cl_assert_equal_i(stats.total_objects, 3);
 204                 cl_assert_equal_i(stats.received_objects, 3);
 205                 cl_assert_equal_i(stats.indexed_objects, 3);
 206                 cl_assert_equal_i(stats.local_objects, 0);
 207
 208                 git_indexer_free(idx);
 209         }
 210 }
 211
 212 void test_pack_indexer__corrupt_length(void)
 213 {
 214         git_indexer *idx = NULL;
 215         git_indexer_progress stats = { 0 };
 216         git_repository *repo;
 217         git_odb *odb;
 218         git_oid id, should_id;
 219
 220         cl_git_pass(git_repository_init(&repo, "thin.git", true));
 221         cl_git_pass(git_repository_odb(&odb, repo));
 222
 223         /* Store the missing base into your ODB so the indexer can fix the pack */
 224         cl_git_pass(git_odb_write(&id, odb, base_obj, base_obj_len, GIT_OBJECT_BLOB));
 225         git_oid_fromstr(&should_id, "e68fe8129b546b101aee9510c5328e7f21ca1d18");
 226         cl_assert_equal_oid(&should_id, &id);
 227
 228         cl_git_pass(git_indexer_new(&idx, ".", 0, odb, NULL));
 229         cl_git_pass(git_indexer_append(
 230                 idx, corrupt_thin_pack, corrupt_thin_pack_len, &stats));
 231         cl_git_fail(git_indexer_commit(idx, &stats));
 232
 233         cl_assert(git_error_last() != NULL);
 234         cl_assert_equal_i(git_error_last()->klass, GIT_ERROR_ZLIB);
 235
 236         git_indexer_free(idx);
 237         git_odb_free(odb);
 238         git_repository_free(repo);
 239 }
 240
 241 void test_pack_indexer__incomplete_pack_fails_with_strict(void)
 242 {
 243         git_indexer_options opts = GIT_INDEXER_OPTIONS_INIT;
 244         git_indexer *idx = 0;
 245         git_indexer_progress stats = { 0 };
 246
 247         opts.verify = 1;
 248
 249         cl_git_pass(git_indexer_new(&idx, ".", 0, NULL, &opts));
 250         cl_git_pass(git_indexer_append(
 251                 idx, incomplete_pack, incomplete_pack_len, &stats));
 252         cl_git_fail(git_indexer_commit(idx, &stats));
 253
 254         cl_assert_equal_i(stats.total_objects, 2);
 255         cl_assert_equal_i(stats.received_objects, 2);
 256         cl_assert_equal_i(stats.indexed_objects, 2);
 257
 258         git_indexer_free(idx);
 259 }
 260
 261 void test_pack_indexer__out_of_order_with_connectivity_checks(void)
 262 {
 263         git_indexer_options opts = GIT_INDEXER_OPTIONS_INIT;
 264         git_indexer *idx = 0;
 265         git_indexer_progress stats = { 0 };
 266
 267         opts.verify = 1;
 268
 269         cl_git_pass(git_indexer_new(&idx, ".", 0, NULL, &opts));
 270         cl_git_pass(git_indexer_append(
 271                 idx, out_of_order_pack, out_of_order_pack_len, &stats));
 272         cl_git_pass(git_indexer_commit(idx, &stats));
 273
 274         cl_assert_equal_i(stats.total_objects, 3);
 275         cl_assert_equal_i(stats.received_objects, 3);
 276         cl_assert_equal_i(stats.indexed_objects, 3);
 277
 278         git_indexer_free(idx);
 279 }
 280
 281 static int find_tmp_file_recurs(void *opaque, git_str *path)
 282 {
 283         int error = 0;
 284         git_str *first_tmp_file = opaque;
 285         struct stat st;
 286
 287         if ((error = p_lstat_posixly(path->ptr, &st)) < 0)
 288                 return error;
 289
 290         if (S_ISDIR(st.st_mode))
 291                 return git_fs_path_direach(path, 0, find_tmp_file_recurs, opaque);
 292
 293         /* This is the template that's used in git_futils_mktmp. */
 294         if (strstr(git_str_cstr(path), "_git2_") != NULL)
 295                 return git_str_sets(first_tmp_file, git_str_cstr(path));
 296
 297         return 0;
 298 }
 299
 300 void test_pack_indexer__no_tmp_files(void)
 301 {
 302         git_indexer *idx = NULL;
 303         git_str path = GIT_STR_INIT;
 304         git_str first_tmp_file = GIT_STR_INIT;
 305
 306         /* Precondition: there are no temporary files. */
 307         cl_git_pass(git_str_sets(&path, clar_sandbox_path()));
 308         cl_git_pass(find_tmp_file_recurs(&first_tmp_file, &path));
 309         git_str_dispose(&path);
 310         cl_assert(git_str_len(&first_tmp_file) == 0);
 311
 312         cl_git_pass(git_indexer_new(&idx, ".", 0, NULL, NULL));
 313         git_indexer_free(idx);
 314
 315         cl_git_pass(git_str_sets(&path, clar_sandbox_path()));
 316         cl_git_pass(find_tmp_file_recurs(&first_tmp_file, &path));
 317         git_str_dispose(&path);
 318         cl_assert(git_str_len(&first_tmp_file) == 0);
 319         git_str_dispose(&first_tmp_file);
 320 }