src/libgit2/pack.h

   1 /*
   2  * Copyright (C) the libgit2 contributors. All rights reserved.
   3  *
   4  * This file is part of libgit2, distributed under the GNU GPL v2 with
   5  * a Linking Exception. For full terms see the included COPYING file.
   6  */
   7
   8 #ifndef INCLUDE_pack_h__
   9 #define INCLUDE_pack_h__
  10
  11 #include "common.h"
  12
  13 #include "git2/oid.h"
  14
  15 #include "array.h"
  16 #include "map.h"
  17 #include "mwindow.h"
  18 #include "odb.h"
  19 #include "offmap.h"
  20 #include "oidmap.h"
  21 #include "zstream.h"
  22 #include "oid.h"
  23
  24 /**
  25  * Function type for callbacks from git_pack_foreach_entry_offset.
  26  */
  27 typedef int git_pack_foreach_entry_offset_cb(
  28                 const git_oid *id,
  29                 off64_t offset,
  30                 void *payload);
  31
  32 #define GIT_PACK_FILE_MODE 0444
  33
  34 #define PACK_SIGNATURE 0x5041434b       /* "PACK" */
  35 #define PACK_VERSION 2
  36 #define pack_version_ok(v) ((v) == htonl(2))
  37 struct git_pack_header {
  38         uint32_t hdr_signature;
  39         uint32_t hdr_version;
  40         uint32_t hdr_entries;
  41 };
  42
  43 /*
  44  * The first four bytes of index formats later than version 1 should
  45  * start with this signature, as all older git binaries would find this
  46  * value illegal and abort reading the file.
  47  *
  48  * This is the case because the number of objects in a packfile
  49  * cannot exceed 1,431,660,000 as every object would need at least
  50  * 3 bytes of data and the overall packfile cannot exceed 4 GiB with
  51  * version 1 of the index file due to the offsets limited to 32 bits.
  52  * Clearly the signature exceeds this maximum.
  53  *
  54  * Very old git binaries will also compare the first 4 bytes to the
  55  * next 4 bytes in the index and abort with a "non-monotonic index"
  56  * error if the second 4 byte word is smaller than the first 4
  57  * byte word. This would be true in the proposed future index
  58  * format as idx_signature would be greater than idx_version.
  59  */
  60
  61 #define PACK_IDX_SIGNATURE 0xff744f63   /* "\377tOc" */
  62
  63 struct git_pack_idx_header {
  64         uint32_t idx_signature;
  65         uint32_t idx_version;
  66 };
  67
  68 typedef struct git_pack_cache_entry {
  69         size_t last_usage; /* enough? */
  70         git_atomic32 refcount;
  71         git_rawobj raw;
  72 } git_pack_cache_entry;
  73
  74 struct pack_chain_elem {
  75         off64_t base_key;
  76         off64_t offset;
  77         size_t size;
  78         git_object_t type;
  79 };
  80
  81 typedef git_array_t(struct pack_chain_elem) git_dependency_chain;
  82
  83 #define GIT_PACK_CACHE_MEMORY_LIMIT 16 * 1024 * 1024
  84 #define GIT_PACK_CACHE_SIZE_LIMIT 1024 * 1024 /* don't bother caching anything over 1MB */
  85
  86 typedef struct {
  87         size_t memory_used;
  88         size_t memory_limit;
  89         size_t use_ctr;
  90         git_mutex lock;
  91         git_offmap *entries;
  92 } git_pack_cache;
  93
  94 struct git_pack_file {
  95         git_mwindow_file mwf;
  96         git_map index_map;
  97         git_mutex lock; /* protect updates to index_map */
  98         git_atomic32 refcount;
  99
 100         uint32_t num_objects;
 101         uint32_t num_bad_objects;
 102         git_oid *bad_object_sha1; /* array of git_oid */
 103
 104         int index_version;
 105         git_time_t mtime;
 106         unsigned pack_local:1, pack_keep:1, has_cache:1;
 107         git_oidmap *idx_cache;
 108         unsigned char **oids;
 109
 110         git_pack_cache bases; /* delta base cache */
 111
 112         time_t last_freshen; /* last time the packfile was freshened */
 113
 114         /* something like ".git/objects/pack/xxxxx.pack" */
 115         char pack_name[GIT_FLEX_ARRAY]; /* more */
 116 };
 117
 118 /**
 119  * Return the position where an OID (or a prefix) would be inserted within the
 120  * OID Lookup Table of an .idx file. This performs binary search between the lo
 121  * and hi indices.
 122  *
 123  * The stride parameter is provided because .idx files version 1 store the OIDs
 124  * interleaved with the 4-byte file offsets of the objects within the .pack
 125  * file (stride = 24), whereas files with version 2 store them in a contiguous
 126  * flat array (stride = 20).
 127  */
 128 int git_pack__lookup_sha1(const void *oid_lookup_table, size_t stride, unsigned lo,
 129                 unsigned hi, const unsigned char *oid_prefix);
 130
 131 struct git_pack_entry {
 132         off64_t offset;
 133         git_oid sha1;
 134         struct git_pack_file *p;
 135 };
 136
 137 typedef struct git_packfile_stream {
 138         off64_t curpos;
 139         int done;
 140         git_zstream zstream;
 141         struct git_pack_file *p;
 142         git_mwindow *mw;
 143 } git_packfile_stream;
 144
 145 int git_packfile__object_header(size_t *out, unsigned char *hdr, size_t size, git_object_t type);
 146
 147 int git_packfile__name(char **out, const char *path);
 148
 149 int git_packfile_unpack_header(
 150                 size_t *size_p,
 151                 git_object_t *type_p,
 152                 struct git_pack_file *p,
 153                 git_mwindow **w_curs,
 154                 off64_t *curpos);
 155
 156 int git_packfile_resolve_header(
 157                 size_t *size_p,
 158                 git_object_t *type_p,
 159                 struct git_pack_file *p,
 160                 off64_t offset);
 161
 162 int git_packfile_unpack(git_rawobj *obj, struct git_pack_file *p, off64_t *obj_offset);
 163
 164 int git_packfile_stream_open(git_packfile_stream *obj, struct git_pack_file *p, off64_t curpos);
 165 ssize_t git_packfile_stream_read(git_packfile_stream *obj, void *buffer, size_t len);
 166 void git_packfile_stream_dispose(git_packfile_stream *obj);
 167
 168 int get_delta_base(
 169                 off64_t *delta_base_out,
 170                 struct git_pack_file *p,
 171                 git_mwindow **w_curs,
 172                 off64_t *curpos,
 173                 git_object_t type,
 174                 off64_t delta_obj_offset);
 175
 176 void git_packfile_free(struct git_pack_file *p, bool unlink_packfile);
 177 int git_packfile_alloc(struct git_pack_file **pack_out, const char *path);
 178
 179 int git_pack_entry_find(
 180                 struct git_pack_entry *e,
 181                 struct git_pack_file *p,
 182                 const git_oid *short_oid,
 183                 size_t len);
 184 int git_pack_foreach_entry(
 185                 struct git_pack_file *p,
 186                 git_odb_foreach_cb cb,
 187                 void *data);
 188 /**
 189  * Similar to git_pack_foreach_entry, but:
 190  * - It also provides the offset of the object within the
 191  *   packfile.
 192  * - It does not sort the objects in any order.
 193  * - It retains the lock while invoking the callback.
 194  */
 195 int git_pack_foreach_entry_offset(
 196                 struct git_pack_file *p,
 197                 git_pack_foreach_entry_offset_cb cb,
 198                 void *data);
 199
 200 #endif