2 * This file is free software; you can redistribute it and/or modify
3 * it under the terms of the GNU General Public License, version 2,
4 * as published by the Free Software Foundation.
6 * In addition to the permissions in the GNU General Public License,
7 * the authors give you unlimited permission to link the compiled
8 * version of this file into combinations with other programs,
9 * and to distribute those combinations without any restriction
10 * coming from the use of this file. (The General Public License
11 * restrictions do apply in other respects; for example, they cover
12 * modification of the file, and distribution when not linked into
13 * a combined executable.)
15 * This file is distributed in the hope that it will be useful, but
16 * WITHOUT ANY WARRANTY; without even the implied warranty of
17 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
18 * General Public License for more details.
20 * You should have received a copy of the GNU General Public License
21 * along with this program; see the file COPYING. If not, write to
22 * the Free Software Foundation, 51 Franklin Street, Fifth Floor,
23 * Boston, MA 02110-1301, USA.
27 #include "git2/zlib.h"
28 #include "git2/object.h"
33 #include "delta-apply.h"
36 #include "git2/odb_backend.h"
37 #include "git2/types.h"
39 typedef struct { /* object header data */
40 git_otype type
; /* object type */
41 size_t size
; /* object size */
45 git_odb_stream stream
;
50 typedef struct loose_backend
{
51 git_odb_backend parent
;
53 int object_zlib_level
; /** loose object zlib compression level. */
54 int fsync_object_files
; /** loose object file fsync flag. */
58 /* State structure for exploring directories,
59 * in order to locate objects matching a short oid.
63 unsigned char short_oid
[GIT_OID_HEXSZ
]; /* hex formatted oid to match */
64 unsigned int short_oid_len
;
65 int found
; /* number of matching
66 * objects already found */
67 unsigned char res_oid
[GIT_OID_HEXSZ
]; /* hex formatted oid of
69 } loose_locate_object_state
;
73 /***********************************************************
75 * MISCELANEOUS HELPER FUNCTIONS
77 ***********************************************************/
79 static size_t object_file_name(char *name
, size_t n
, char *dir
, const git_oid
*id
)
81 size_t len
= strlen(dir
);
83 /* check length: 43 = 40 hex sha1 chars + 2 * '/' + '\0' */
87 /* the object dir: eg $GIT_DIR/objects */
89 if (name
[len
-1] != '/')
92 /* loose object filename: aa/aaa... (41 bytes) */
93 git_oid_pathfmt(&name
[len
], id
);
100 static size_t get_binary_object_header(obj_hdr
*hdr
, git_fbuffer
*obj
)
103 unsigned char *data
= obj
->data
;
104 size_t shift
, size
, used
= 0;
110 hdr
->type
= (c
>> 4) & 7;
115 if (obj
->len
<= used
)
117 if (sizeof(size_t) * 8 <= shift
)
120 size
+= (c
& 0x7f) << shift
;
128 static size_t get_object_header(obj_hdr
*hdr
, unsigned char *data
)
130 char c
, typename
[10];
131 size_t size
, used
= 0;
134 * type name string followed by space.
136 while ((c
= data
[used
]) != ' ') {
137 typename
[used
++] = c
;
138 if (used
>= sizeof(typename
))
144 hdr
->type
= git_object_string2type(typename
);
145 used
++; /* consume the space */
148 * length follows immediately in decimal (without
151 size
= data
[used
++] - '0';
155 while ((c
= data
[used
]) != '\0') {
160 size
= size
* 10 + d
;
166 * the length must be followed by a zero byte
168 if (data
[used
++] != '\0')
176 /***********************************************************
178 * ZLIB RELATED FUNCTIONS
180 ***********************************************************/
182 static void init_stream(z_stream
*s
, void *out
, size_t len
)
184 memset(s
, 0, sizeof(*s
));
189 static void set_stream_input(z_stream
*s
, void *in
, size_t len
)
195 static void set_stream_output(z_stream
*s
, void *out
, size_t len
)
202 static int start_inflate(z_stream
*s
, git_fbuffer
*obj
, void *out
, size_t len
)
206 init_stream(s
, out
, len
);
207 set_stream_input(s
, obj
->data
, obj
->len
);
209 if ((status
= inflateInit(s
)) < Z_OK
)
212 return inflate(s
, 0);
215 static int finish_inflate(z_stream
*s
)
219 while (status
== Z_OK
)
220 status
= inflate(s
, Z_FINISH
);
224 if ((status
!= Z_STREAM_END
) || (s
->avail_in
!= 0))
225 return git__throw(GIT_ERROR
, "Failed to finish inflation. Stream aborted prematurely");
230 static int is_zlib_compressed_data(unsigned char *data
)
234 w
= ((unsigned int)(data
[0]) << 8) + data
[1];
235 return data
[0] == 0x78 && !(w
% 31);
238 static int inflate_buffer(void *in
, size_t inlen
, void *out
, size_t outlen
)
243 memset(&zs
, 0x0, sizeof(zs
));
246 zs
.avail_out
= outlen
;
251 if (inflateInit(&zs
) < Z_OK
)
252 return git__throw(GIT_ERROR
, "Failed to inflate buffer");
254 while (status
== Z_OK
)
255 status
= inflate(&zs
, Z_FINISH
);
259 if ((status
!= Z_STREAM_END
) /*|| (zs.avail_in != 0) */)
260 return git__throw(GIT_ERROR
, "Failed to inflate buffer. Stream aborted prematurely");
262 if (zs
.total_out
!= outlen
)
263 return git__throw(GIT_ERROR
, "Failed to inflate buffer. Stream aborted prematurely");
268 static void *inflate_tail(z_stream
*s
, void *hb
, size_t used
, obj_hdr
*hdr
)
270 unsigned char *buf
, *head
= hb
;
274 * allocate a buffer to hold the inflated data and copy the
275 * initial sequence of inflated data from the tail of the
276 * head buffer, if any.
278 if ((buf
= git__malloc(hdr
->size
+ 1)) == NULL
) {
282 tail
= s
->total_out
- used
;
283 if (used
> 0 && tail
> 0) {
284 if (tail
> hdr
->size
)
286 memcpy(buf
, head
+ used
, tail
);
291 * inflate the remainder of the object data, if any
293 if (hdr
->size
< used
)
296 set_stream_output(s
, buf
+ used
, hdr
->size
- used
);
297 if (finish_inflate(s
)) {
307 * At one point, there was a loose object format that was intended to
308 * mimic the format used in pack-files. This was to allow easy copying
309 * of loose object data into packs. This format is no longer used, but
310 * we must still read it.
312 static int inflate_packlike_loose_disk_obj(git_rawobj
*out
, git_fbuffer
*obj
)
314 unsigned char *in
, *buf
;
319 * read the object header, which is an (uncompressed)
320 * binary encoding of the object type and size.
322 if ((used
= get_binary_object_header(&hdr
, obj
)) == 0)
323 return git__throw(GIT_ERROR
, "Failed to inflate loose object. Object has no header");
325 if (!git_object_typeisloose(hdr
.type
))
326 return git__throw(GIT_ERROR
, "Failed to inflate loose object. Wrong object type");
329 * allocate a buffer and inflate the data into it
331 buf
= git__malloc(hdr
.size
+ 1);
335 in
= ((unsigned char *)obj
->data
) + used
;
336 len
= obj
->len
- used
;
337 if (inflate_buffer(in
, len
, buf
, hdr
.size
)) {
339 return git__throw(GIT_ERROR
, "Failed to inflate loose object. Could not inflate buffer");
341 buf
[hdr
.size
] = '\0';
345 out
->type
= hdr
.type
;
350 static int inflate_disk_obj(git_rawobj
*out
, git_fbuffer
*obj
)
352 unsigned char head
[64], *buf
;
358 * check for a pack-like loose object
360 if (!is_zlib_compressed_data(obj
->data
))
361 return inflate_packlike_loose_disk_obj(out
, obj
);
364 * inflate the initial part of the io buffer in order
365 * to parse the object header (type and size).
367 if (start_inflate(&zs
, obj
, head
, sizeof(head
)) < Z_OK
)
368 return git__throw(GIT_ERROR
, "Failed to inflate disk object. Could not inflate buffer");
370 if ((used
= get_object_header(&hdr
, head
)) == 0)
371 return git__throw(GIT_ERROR
, "Failed to inflate disk object. Object has no header");
373 if (!git_object_typeisloose(hdr
.type
))
374 return git__throw(GIT_ERROR
, "Failed to inflate disk object. Wrong object type");
377 * allocate a buffer and inflate the object data into it
378 * (including the initial sequence in the head buffer).
380 if ((buf
= inflate_tail(&zs
, head
, used
, &hdr
)) == NULL
)
382 buf
[hdr
.size
] = '\0';
386 out
->type
= hdr
.type
;
396 /***********************************************************
398 * ODB OBJECT READING & WRITING
400 * Backend for the public API; read headers and full objects
401 * from the ODB. Write raw data to the ODB.
403 ***********************************************************/
405 static int read_loose(git_rawobj
*out
, const char *loc
)
408 git_fbuffer obj
= GIT_FBUFFER_INIT
;
414 out
->type
= GIT_OBJ_BAD
;
416 if (git_futils_readbuffer(&obj
, loc
) < 0)
417 return git__throw(GIT_ENOTFOUND
, "Failed to read loose object. File not found");
419 error
= inflate_disk_obj(out
, &obj
);
420 git_futils_freebuffer(&obj
);
422 return error
== GIT_SUCCESS
? GIT_SUCCESS
: git__rethrow(error
, "Failed to read loose object");
425 static int read_header_loose(git_rawobj
*out
, const char *loc
)
427 int error
= GIT_SUCCESS
, z_return
= Z_ERRNO
, read_bytes
;
431 unsigned char raw_buffer
[16], inflated_buffer
[64];
437 if ((fd
= p_open(loc
, O_RDONLY
)) < 0)
438 return git__throw(GIT_ENOTFOUND
, "Failed to read loose object header. File not found");
440 init_stream(&zs
, inflated_buffer
, sizeof(inflated_buffer
));
442 if (inflateInit(&zs
) < Z_OK
) {
448 if ((read_bytes
= read(fd
, raw_buffer
, sizeof(raw_buffer
))) > 0) {
449 set_stream_input(&zs
, raw_buffer
, read_bytes
);
450 z_return
= inflate(&zs
, 0);
452 z_return
= Z_STREAM_END
;
455 } while (z_return
== Z_OK
);
457 if ((z_return
!= Z_STREAM_END
&& z_return
!= Z_BUF_ERROR
)
458 || get_object_header(&header_obj
, inflated_buffer
) == 0
459 || git_object_typeisloose(header_obj
.type
) == 0) {
460 error
= GIT_EOBJCORRUPTED
;
464 out
->len
= header_obj
.size
;
465 out
->type
= header_obj
.type
;
471 if (error
< GIT_SUCCESS
)
472 return git__throw(error
, "Failed to read loose object header. Header is corrupted");
477 static int locate_object(char *object_location
, loose_backend
*backend
, const git_oid
*oid
)
479 object_file_name(object_location
, GIT_PATH_MAX
, backend
->objects_dir
, oid
);
480 return git_futils_exists(object_location
);
483 /* Explore an entry of a directory and see if it matches a short oid */
484 int fn_locate_object_short_oid(void *state
, char *pathbuf
) {
485 loose_locate_object_state
*sstate
= (loose_locate_object_state
*)state
;
487 size_t pathbuf_len
= strlen(pathbuf
);
488 if (pathbuf_len
- sstate
->dir_len
!= GIT_OID_HEXSZ
- 2) {
489 /* Entry cannot be an object. Continue to next entry */
493 if (!git_futils_exists(pathbuf
) && git_futils_isdir(pathbuf
)) {
494 /* We are already in the directory matching the 2 first hex characters,
495 * compare the first ncmp characters of the oids */
496 if (!memcmp(sstate
->short_oid
+ 2,
497 (unsigned char *)pathbuf
+ sstate
->dir_len
,
498 sstate
->short_oid_len
- 2)) {
500 if (!sstate
->found
) {
501 sstate
->res_oid
[0] = sstate
->short_oid
[0];
502 sstate
->res_oid
[1] = sstate
->short_oid
[1];
503 memcpy(sstate
->res_oid
+2, pathbuf
+sstate
->dir_len
, GIT_OID_HEXSZ
-2);
508 if (sstate
->found
> 1)
509 return git__throw(GIT_EAMBIGUOUSOIDPREFIX
, "Ambiguous sha1 prefix within loose objects");
514 /* Locate an object matching a given short oid */
515 static int locate_object_short_oid(char *object_location
, git_oid
*res_oid
, loose_backend
*backend
, const git_oid
*short_oid
, unsigned int len
)
517 char *objects_dir
= backend
->objects_dir
;
518 size_t dir_len
= strlen(objects_dir
);
519 loose_locate_object_state state
;
522 if (dir_len
+43 > GIT_PATH_MAX
)
523 return git__throw(GIT_ERROR
, "Failed to locate object from short oid. Object path too long");
525 strcpy(object_location
, objects_dir
);
527 /* Add a separator if not already there */
528 if (object_location
[dir_len
-1] != '/')
529 object_location
[dir_len
++] = '/';
531 /* Convert raw oid to hex formatted oid */
532 git_oid_fmt((char *)state
.short_oid
, short_oid
);
533 /* Explore OBJ_DIR/xx/ where xx is the beginning of hex formatted short oid */
534 sprintf(object_location
+dir_len
, "%.2s/", state
.short_oid
);
536 /* Check that directory exists */
537 if (git_futils_exists(object_location
) || git_futils_isdir(object_location
))
538 return git__throw(GIT_ENOTFOUND
, "Failed to locate object from short oid. Object not found");
540 state
.dir_len
= dir_len
+3;
541 state
.short_oid_len
= len
;
543 /* Explore directory to find a unique object matching short_oid */
544 error
= git_futils_direach(object_location
, GIT_PATH_MAX
, fn_locate_object_short_oid
, &state
);
546 return git__rethrow(error
, "Failed to locate object from short oid");
549 return git__throw(GIT_ENOTFOUND
, "Failed to locate object from short oid. Object not found");
552 /* Convert obtained hex formatted oid to raw */
553 error
= git_oid_fromstr(res_oid
, (char *)state
.res_oid
);
555 return git__rethrow(error
, "Failed to locate object from short oid");
558 /* Update the location according to the oid obtained */
559 git_oid_pathfmt(object_location
+dir_len
, res_oid
);
572 /***********************************************************
574 * LOOSE BACKEND PUBLIC API
576 * Implement the git_odb_backend API calls
578 ***********************************************************/
580 int loose_backend__read_header(size_t *len_p
, git_otype
*type_p
, git_odb_backend
*backend
, const git_oid
*oid
)
582 char object_path
[GIT_PATH_MAX
];
586 assert(backend
&& oid
);
589 raw
.type
= GIT_OBJ_BAD
;
591 if (locate_object(object_path
, (loose_backend
*)backend
, oid
) < 0)
592 return git__throw(GIT_ENOTFOUND
, "Failed to read loose backend header. Object not found");
594 if ((error
= read_header_loose(&raw
, object_path
)) < GIT_SUCCESS
)
602 int loose_backend__read(void **buffer_p
, size_t *len_p
, git_otype
*type_p
, git_odb_backend
*backend
, const git_oid
*oid
)
604 char object_path
[GIT_PATH_MAX
];
608 assert(backend
&& oid
);
610 if (locate_object(object_path
, (loose_backend
*)backend
, oid
) < 0)
611 return git__throw(GIT_ENOTFOUND
, "Failed to read loose backend. Object not found");
613 if ((error
= read_loose(&raw
, object_path
)) < GIT_SUCCESS
)
614 return git__rethrow(error
, "Failed to read loose backend");
616 *buffer_p
= raw
.data
;
623 int loose_backend__read_prefix(
628 git_odb_backend
*backend
,
629 const git_oid
*short_oid
,
632 if (len
< GIT_OID_MINPREFIXLEN
)
633 return git__throw(GIT_EAMBIGUOUSOIDPREFIX
, "Failed to read loose backend. Prefix length is lower than %d.", GIT_OID_MINPREFIXLEN
);
635 if (len
>= GIT_OID_HEXSZ
) {
636 /* We can fall back to regular read method */
637 int error
= loose_backend__read(buffer_p
, len_p
, type_p
, backend
, short_oid
);
638 if (error
== GIT_SUCCESS
)
639 git_oid_cpy(out_oid
, short_oid
);
643 char object_path
[GIT_PATH_MAX
];
647 assert(backend
&& short_oid
);
649 if ((error
= locate_object_short_oid(object_path
, out_oid
, (loose_backend
*)backend
, short_oid
, len
)) < 0) {
650 return git__rethrow(error
, "Failed to read loose backend");
653 if ((error
= read_loose(&raw
, object_path
)) < GIT_SUCCESS
)
654 return git__rethrow(error
, "Failed to read loose backend");
656 *buffer_p
= raw
.data
;
664 int loose_backend__exists(git_odb_backend
*backend
, const git_oid
*oid
)
666 char object_path
[GIT_PATH_MAX
];
668 assert(backend
&& oid
);
670 return locate_object(object_path
, (loose_backend
*)backend
, oid
) == GIT_SUCCESS
;
673 int loose_backend__stream_fwrite(git_oid
*oid
, git_odb_stream
*_stream
)
675 loose_writestream
*stream
= (loose_writestream
*)_stream
;
676 loose_backend
*backend
= (loose_backend
*)_stream
->backend
;
679 char final_path
[GIT_PATH_MAX
];
681 if ((error
= git_filebuf_hash(oid
, &stream
->fbuf
)) < GIT_SUCCESS
)
682 return git__rethrow(error
, "Failed to write loose backend");
684 if (object_file_name(final_path
, sizeof(final_path
), backend
->objects_dir
, oid
))
687 if ((error
= git_futils_mkpath2file(final_path
)) < GIT_SUCCESS
)
688 return git__rethrow(error
, "Failed to write loose backend");
690 stream
->finished
= 1;
691 return git_filebuf_commit_at(&stream
->fbuf
, final_path
);
694 int loose_backend__stream_write(git_odb_stream
*_stream
, const char *data
, size_t len
)
696 loose_writestream
*stream
= (loose_writestream
*)_stream
;
697 return git_filebuf_write(&stream
->fbuf
, data
, len
);
700 void loose_backend__stream_free(git_odb_stream
*_stream
)
702 loose_writestream
*stream
= (loose_writestream
*)_stream
;
704 if (!stream
->finished
)
705 git_filebuf_cleanup(&stream
->fbuf
);
710 static int format_object_header(char *hdr
, size_t n
, size_t obj_len
, git_otype obj_type
)
712 const char *type_str
= git_object_type2string(obj_type
);
713 int len
= snprintf(hdr
, n
, "%s %"PRIuZ
, type_str
, obj_len
);
715 assert(len
> 0); /* otherwise snprintf() is broken */
716 assert(((size_t) len
) < n
); /* otherwise the caller is broken! */
718 if (len
< 0 || ((size_t) len
) >= n
)
719 return git__throw(GIT_ERROR
, "Failed to format object header. Length is out of bounds");
723 int loose_backend__stream(git_odb_stream
**stream_out
, git_odb_backend
*_backend
, size_t length
, git_otype type
)
725 loose_backend
*backend
;
726 loose_writestream
*stream
;
728 char hdr
[64], tmp_path
[GIT_PATH_MAX
];
734 backend
= (loose_backend
*)_backend
;
737 hdrlen
= format_object_header(hdr
, sizeof(hdr
), length
, type
);
738 if (hdrlen
< GIT_SUCCESS
)
739 return git__throw(GIT_EOBJCORRUPTED
, "Failed to create loose backend stream. Object is corrupted");
741 stream
= git__calloc(1, sizeof(loose_writestream
));
745 stream
->stream
.backend
= _backend
;
746 stream
->stream
.read
= NULL
; /* read only */
747 stream
->stream
.write
= &loose_backend__stream_write
;
748 stream
->stream
.finalize_write
= &loose_backend__stream_fwrite
;
749 stream
->stream
.free
= &loose_backend__stream_free
;
750 stream
->stream
.mode
= GIT_STREAM_WRONLY
;
752 git_path_join(tmp_path
, backend
->objects_dir
, "tmp_object");
754 error
= git_filebuf_open(&stream
->fbuf
, tmp_path
,
755 GIT_FILEBUF_HASH_CONTENTS
|
756 GIT_FILEBUF_DEFLATE_CONTENTS
|
757 GIT_FILEBUF_TEMPORARY
);
759 if (error
< GIT_SUCCESS
) {
761 return git__rethrow(error
, "Failed to create loose backend stream");
764 error
= stream
->stream
.write((git_odb_stream
*)stream
, hdr
, hdrlen
);
765 if (error
< GIT_SUCCESS
) {
766 git_filebuf_cleanup(&stream
->fbuf
);
768 return git__rethrow(error
, "Failed to create loose backend stream");
771 *stream_out
= (git_odb_stream
*)stream
;
775 int loose_backend__write(git_oid
*oid
, git_odb_backend
*_backend
, const void *data
, size_t len
, git_otype type
)
777 int error
, header_len
;
778 char final_path
[GIT_PATH_MAX
], header
[64];
780 loose_backend
*backend
;
782 backend
= (loose_backend
*)_backend
;
784 /* prepare the header for the file */
786 header_len
= format_object_header(header
, sizeof(header
), len
, type
);
787 if (header_len
< GIT_SUCCESS
)
788 return GIT_EOBJCORRUPTED
;
791 git_path_join(final_path
, backend
->objects_dir
, "tmp_object");
793 error
= git_filebuf_open(&fbuf
, final_path
,
794 GIT_FILEBUF_HASH_CONTENTS
|
795 GIT_FILEBUF_DEFLATE_CONTENTS
|
796 GIT_FILEBUF_TEMPORARY
);
798 if (error
< GIT_SUCCESS
)
801 git_filebuf_write(&fbuf
, header
, header_len
);
802 git_filebuf_write(&fbuf
, data
, len
);
803 git_filebuf_hash(oid
, &fbuf
);
805 if ((error
= object_file_name(final_path
, sizeof(final_path
), backend
->objects_dir
, oid
)) < GIT_SUCCESS
)
808 if ((error
= git_futils_mkpath2file(final_path
)) < GIT_SUCCESS
)
811 return git_filebuf_commit_at(&fbuf
, final_path
);
814 git_filebuf_cleanup(&fbuf
);
818 void loose_backend__free(git_odb_backend
*_backend
)
820 loose_backend
*backend
;
822 backend
= (loose_backend
*)_backend
;
824 free(backend
->objects_dir
);
828 int git_odb_backend_loose(git_odb_backend
**backend_out
, const char *objects_dir
)
830 loose_backend
*backend
;
832 backend
= git__calloc(1, sizeof(loose_backend
));
836 backend
->objects_dir
= git__strdup(objects_dir
);
837 if (backend
->objects_dir
== NULL
) {
842 backend
->object_zlib_level
= Z_BEST_SPEED
;
843 backend
->fsync_object_files
= 0;
845 backend
->parent
.read
= &loose_backend__read
;
846 backend
->parent
.write
= &loose_backend__write
;
847 backend
->parent
.read_prefix
= &loose_backend__read_prefix
;
848 backend
->parent
.read_header
= &loose_backend__read_header
;
849 backend
->parent
.writestream
= &loose_backend__stream
;
850 backend
->parent
.exists
= &loose_backend__exists
;
851 backend
->parent
.free
= &loose_backend__free
;
853 *backend_out
= (git_odb_backend
*)backend
;