odb: make the writestream's size a git_off_t

Restricting files to size_t is a silly limitation. The loose backend
writes to a file directly, so there is no issue in using 63 bits for the
size.

We still assume that the header is going to fit in 64 bytes, which does
mean quite a bit smaller files due to the run-length encoding, but it's
still a much larger size than you would want Git to handle.

diff --git a/include/git2/odb.h b/include/git2/odb.h
index 114f6b31764dbff199680f8c99aff25238022bc9..4f1e18bc1a58e74c749317c10a86132832bfab81 100644 (file)
--- a/include/git2/odb.h
+++ b/include/git2/odb.h
@@ -247,7 +247,7 @@ GIT_EXTERN(int) git_odb_write(git_oid *out, git_odb *odb, const void *data, size
  * @param type type of the object that will be written
  * @return 0 if the stream was created; error code otherwise
  */
-GIT_EXTERN(int) git_odb_open_wstream(git_odb_stream **out, git_odb *db, size_t size, git_otype type);
+GIT_EXTERN(int) git_odb_open_wstream(git_odb_stream **out, git_odb *db, git_off_t size, git_otype type);
 
 /**
  * Write to an odb stream

diff --git a/include/git2/odb_backend.h b/include/git2/odb_backend.h
index 4d772cab92391ddb9bb120c274c0dec523469b07..b17cfd8bab823986b42db79ca51f65d5ff86bc97 100644 (file)
--- a/include/git2/odb_backend.h
+++ b/include/git2/odb_backend.h
@@ -86,8 +86,8 @@ struct git_odb_stream {
        unsigned int mode;
        void *hash_ctx;
 
-       size_t declared_size;
-       size_t received_bytes;
+       git_off_t declared_size;
+       git_off_t received_bytes;
 
        /**
         * Write at most `len` bytes into `buffer` and advance the stream.

diff --git a/include/git2/sys/odb_backend.h b/include/git2/sys/odb_backend.h
index 1fc3c3159200cb322d6b8a4c48032ccdc93cf61d..0a51c6dba4aa44f33260232240ae8b3cf10b06ba 100644 (file)
--- a/include/git2/sys/odb_backend.h
+++ b/include/git2/sys/odb_backend.h
@@ -53,7 +53,7 @@ struct git_odb_backend {
                git_odb_backend *, const git_oid *, const void *, size_t, git_otype);
 
        int (* writestream)(
-               git_odb_stream **, git_odb_backend *, size_t, git_otype);
+               git_odb_stream **, git_odb_backend *, git_off_t, git_otype);
 
        int (* readstream)(
                git_odb_stream **, git_odb_backend *, const git_oid *);

diff --git a/src/blob.c b/src/blob.c
index 47216507b2c3077ece2f71b607f910591fa39c07..07c4d92c84b826772f6f1b00e982594384410f2c 100644 (file)
--- a/src/blob.c
+++ b/src/blob.c
@@ -76,10 +76,11 @@ static int write_file_stream(
        int fd, error;
        char buffer[FILEIO_BUFSIZE];
        git_odb_stream *stream = NULL;
-       ssize_t read_len = -1, written = 0;
+       ssize_t read_len = -1;
+       git_off_t written = 0;
 
        if ((error = git_odb_open_wstream(
-                       &stream, odb, (size_t)file_size, GIT_OBJ_BLOB)) < 0)
+                       &stream, odb, file_size, GIT_OBJ_BLOB)) < 0)
                return error;
 
        if ((fd = git_futils_open_ro(path)) < 0) {

diff --git a/src/odb.c b/src/odb.c
index c3ae15a6af24dd0b571da7af2a446b0ab212ae3b..d0bad5331b5bf4bebad5d22cacf1f825e6a349ff 100644 (file)
--- a/src/odb.c
+++ b/src/odb.c
@@ -47,7 +47,7 @@ static git_cache *odb_cache(git_odb *odb)
 
 static int load_alternates(git_odb *odb, const char *objects_dir, int alternate_depth);
 
-int git_odb__format_object_header(char *hdr, size_t n, size_t obj_len, git_otype obj_type)
+int git_odb__format_object_header(char *hdr, size_t n, git_off_t obj_len, git_otype obj_type)
 {
        const char *type_str = git_object_type2string(obj_type);
        int len = p_snprintf(hdr, n, "%s %"PRIuZ, type_str, obj_len);
@@ -327,10 +327,15 @@ static void fake_wstream__free(git_odb_stream *_stream)
        git__free(stream);
 }
 
-static int init_fake_wstream(git_odb_stream **stream_p, git_odb_backend *backend, size_t size, git_otype type)
+static int init_fake_wstream(git_odb_stream **stream_p, git_odb_backend *backend, git_off_t size, git_otype type)
 {
        fake_wstream *stream;
 
+       if (!git__is_ssizet(size)) {
+               giterr_set(GITERR_ODB, "object size too large to keep in memory");
+               return -1;
+       }
+
        stream = git__calloc(1, sizeof(fake_wstream));
        GITERR_CHECK_ALLOC(stream);
 
@@ -937,7 +942,7 @@ int git_odb_write(
        return error;
 }
 
-static void hash_header(git_hash_ctx *ctx, size_t size, git_otype type)
+static void hash_header(git_hash_ctx *ctx, git_off_t size, git_otype type)
 {
        char header[64];
        int hdrlen;
@@ -947,7 +952,7 @@ static void hash_header(git_hash_ctx *ctx, size_t size, git_otype type)
 }
 
 int git_odb_open_wstream(
-       git_odb_stream **stream, git_odb *db, size_t size, git_otype type)
+       git_odb_stream **stream, git_odb *db, git_off_t size, git_otype type)
 {
        size_t i, writes = 0;
        int error = GIT_ERROR;

diff --git a/src/odb.h b/src/odb.h
index 61dd9a7fdd6f61ffb11982b70051670097ae64bb..281bd3a4d4a8c3bd935b359711d2a91ef88e236e 100644 (file)
--- a/src/odb.h
+++ b/src/odb.h
@@ -49,7 +49,7 @@ int git_odb__hashobj(git_oid *id, git_rawobj *obj);
 /*
  * Format the object header such as it would appear in the on-disk object
  */
-int git_odb__format_object_header(char *hdr, size_t n, size_t obj_len, git_otype obj_type);
+int git_odb__format_object_header(char *hdr, size_t n, git_off_t obj_len, git_otype obj_type);
 /*
  * Hash an open file descriptor.
  * This is a performance call when the contents of a fd need to be hashed,

diff --git a/src/odb_loose.c b/src/odb_loose.c
index bfd95588b9ada20fc17e04d6aa7321d8f272e701..99b8f7c917b8257927afccb20f2ffbf0829cb31b 100644 (file)
--- a/src/odb_loose.c
+++ b/src/odb_loose.c
@@ -834,7 +834,7 @@ static void loose_backend__stream_free(git_odb_stream *_stream)
        git__free(stream);
 }
 
-static int loose_backend__stream(git_odb_stream **stream_out, git_odb_backend *_backend, size_t length, git_otype type)
+static int loose_backend__stream(git_odb_stream **stream_out, git_odb_backend *_backend, git_off_t length, git_otype type)
 {
        loose_backend *backend;
        loose_writestream *stream = NULL;
@@ -842,7 +842,7 @@ static int loose_backend__stream(git_odb_stream **stream_out, git_odb_backend *_
        git_buf tmp_path = GIT_BUF_INIT;
        int hdrlen;
 
-       assert(_backend);
+       assert(_backend && length >= 0);
 
        backend = (loose_backend *)_backend;
        *stream_out = NULL;