]> git.proxmox.com Git - libgit2.git/blobdiff - src/iterator.c
Improvements to ignore performance on Windows.
[libgit2.git] / src / iterator.c
index bdc98d22be0cb12c656cfa07a2684fafaf8d2062..8bab1aab0ac7426f126e72d113be1c16e8409175 100644 (file)
@@ -10,7 +10,7 @@
 #include "index.h"
 #include "ignore.h"
 #include "buffer.h"
-#include "git2/submodule.h"
+#include "submodule.h"
 #include <ctype.h>
 
 #define ITERATOR_SET_CB(P,NAME_LC) do { \
@@ -336,7 +336,7 @@ static int tree_iterator__push_frame(tree_iterator *ti)
 {
        int error = 0;
        tree_iterator_frame *head = ti->head, *tf = NULL;
-       size_t i, n_entries = 0;
+       size_t i, n_entries = 0, alloclen;
 
        if (head->current >= head->n_entries || !head->entries[head->current]->tree)
                return GIT_ITEROVER;
@@ -344,8 +344,10 @@ static int tree_iterator__push_frame(tree_iterator *ti)
        for (i = head->current; i < head->next; ++i)
                n_entries += git_tree_entrycount(head->entries[i]->tree);
 
-       tf = git__calloc(sizeof(tree_iterator_frame) +
-               n_entries * sizeof(tree_iterator_entry *), 1);
+       GITERR_CHECK_ALLOC_MULTIPLY(&alloclen, sizeof(tree_iterator_entry *), n_entries);
+       GITERR_CHECK_ALLOC_ADD(&alloclen, alloclen, sizeof(tree_iterator_frame));
+
+       tf = git__calloc(1, alloclen);
        GITERR_CHECK_ALLOC(tf);
 
        tf->n_entries = n_entries;
@@ -447,7 +449,7 @@ static int tree_iterator__update_entry(tree_iterator *ti)
     te = tf->entries[tf->current]->te;
 
        ti->entry.mode = te->attr;
-       git_oid_cpy(&ti->entry.oid, &te->oid);
+       git_oid_cpy(&ti->entry.id, &te->oid);
 
        ti->entry.path = tree_iterator__current_filename(ti, te);
        GITERR_CHECK_ALLOC(ti->entry.path);
@@ -644,6 +646,8 @@ typedef struct {
        git_iterator base;
        git_iterator_callbacks cb;
        git_index *index;
+       git_vector entries;
+       git_vector_cmp entry_srch;
        size_t current;
        /* when not in autoexpand mode, use these to represent "tree" state */
        git_buf partial;
@@ -654,10 +658,10 @@ typedef struct {
 
 static const git_index_entry *index_iterator__index_entry(index_iterator *ii)
 {
-       const git_index_entry *ie = git_index_get_byindex(ii->index, ii->current);
+       const git_index_entry *ie = git_vector_get(&ii->entries, ii->current);
 
        if (ie != NULL && iterator__past_end(ii, ie->path)) {
-               ii->current = git_index_entrycount(ii->index);
+               ii->current = git_vector_length(&ii->entries);
                ie = NULL;
        }
 
@@ -726,7 +730,7 @@ static int index_iterator__current(
        const git_index_entry **entry, git_iterator *self)
 {
        index_iterator *ii = (index_iterator *)self;
-       const git_index_entry *ie = git_index_get_byindex(ii->index, ii->current);
+       const git_index_entry *ie = git_vector_get(&ii->entries, ii->current);
 
        if (ie != NULL && index_iterator__at_tree(ii)) {
                ii->tree_entry.path = ii->partial.ptr;
@@ -744,14 +748,14 @@ static int index_iterator__current(
 static int index_iterator__at_end(git_iterator *self)
 {
        index_iterator *ii = (index_iterator *)self;
-       return (ii->current >= git_index_entrycount(ii->index));
+       return (ii->current >= git_vector_length(&ii->entries));
 }
 
 static int index_iterator__advance(
        const git_index_entry **entry, git_iterator *self)
 {
        index_iterator *ii = (index_iterator *)self;
-       size_t entrycount = git_index_entrycount(ii->index);
+       size_t entrycount = git_vector_length(&ii->entries);
        const git_index_entry *ie;
 
        if (!iterator__has_been_accessed(ii))
@@ -766,7 +770,7 @@ static int index_iterator__advance(
                        while (ii->current < entrycount) {
                                ii->current++;
 
-                               if (!(ie = git_index_get_byindex(ii->index, ii->current)) ||
+                               if (!(ie = git_vector_get(&ii->entries, ii->current)) ||
                                        ii->base.prefixcomp(ie->path, ii->partial.ptr) != 0)
                                        break;
                        }
@@ -789,7 +793,7 @@ static int index_iterator__advance_into(
        const git_index_entry **entry, git_iterator *self)
 {
        index_iterator *ii = (index_iterator *)self;
-       const git_index_entry *ie = git_index_get_byindex(ii->index, ii->current);
+       const git_index_entry *ie = git_vector_get(&ii->entries, ii->current);
 
        if (ie != NULL && index_iterator__at_tree(ii)) {
                if (ii->restore_terminator)
@@ -815,8 +819,11 @@ static int index_iterator__reset(
        if (iterator__reset_range(self, start, end) < 0)
                return -1;
 
-       ii->current = ii->base.start ?
-               git_index__prefix_position(ii->index, ii->base.start) : 0;
+       ii->current = 0;
+
+       if (ii->base.start)
+               git_index_snapshot_find(
+                       &ii->current, &ii->entries, ii->entry_srch, ii->base.start, 0, 0);
 
        if ((ie = index_iterator__skip_conflicts(ii)) == NULL)
                return 0;
@@ -841,9 +848,8 @@ static int index_iterator__reset(
 static void index_iterator__free(git_iterator *self)
 {
        index_iterator *ii = (index_iterator *)self;
-       git_index_free(ii->index);
+       git_index_snapshot_release(&ii->entries, ii->index);
        ii->index = NULL;
-
        git_buf_free(&ii->partial);
 }
 
@@ -854,18 +860,29 @@ int git_iterator_for_index(
        const char *start,
        const char *end)
 {
+       int error = 0;
        index_iterator *ii = git__calloc(1, sizeof(index_iterator));
        GITERR_CHECK_ALLOC(ii);
 
+       if ((error = git_index_snapshot_new(&ii->entries, index)) < 0) {
+               git__free(ii);
+               return error;
+       }
+       ii->index = index;
+
        ITERATOR_BASE_INIT(ii, index, INDEX, git_index_owner(index));
 
-       if (index->ignore_case) {
-               ii->base.flags |= GIT_ITERATOR_IGNORE_CASE;
-               ii->base.prefixcomp = git__prefixcmp_icase;
+       if ((error = iterator__update_ignore_case((git_iterator *)ii, flags)) < 0) {
+               git_iterator_free((git_iterator *)ii);
+               return error;
        }
 
-       ii->index = index;
-       GIT_REFCOUNT_INC(index);
+       ii->entry_srch = iterator__ignore_case(ii) ?
+               git_index_entry_isrch : git_index_entry_srch;
+
+       git_vector_set_cmp(&ii->entries, iterator__ignore_case(ii) ?
+               git_index_entry_icmp : git_index_entry_cmp);
+       git_vector_sort(&ii->entries);
 
        git_buf_init(&ii->partial, 0);
        ii->tree_entry.mode = GIT_FILEMODE_TREE;
@@ -873,7 +890,6 @@ int git_iterator_for_index(
        index_iterator__reset((git_iterator *)ii, NULL, NULL);
 
        *iter = (git_iterator *)ii;
-
        return 0;
 }
 
@@ -883,6 +899,7 @@ struct fs_iterator_frame {
        fs_iterator_frame *next;
        git_vector entries;
        size_t index;
+       int is_ignored;
 };
 
 typedef struct fs_iterator fs_iterator;
@@ -893,6 +910,7 @@ struct fs_iterator {
        git_index_entry entry;
        git_buf path;
        size_t root_len;
+       uint32_t dirload_flags;
        int depth;
 
        int (*enter_dir_cb)(fs_iterator *self);
@@ -919,12 +937,7 @@ static fs_iterator_frame *fs_iterator__alloc_frame(fs_iterator *fi)
 
 static void fs_iterator__free_frame(fs_iterator_frame *ff)
 {
-       size_t i;
-       git_path_with_stat *path;
-
-       git_vector_foreach(&ff->entries, i, path)
-               git__free(path);
-       git_vector_free(&ff->entries);
+       git_vector_free_deep(&ff->entries);
        git__free(ff);
 }
 
@@ -986,19 +999,27 @@ static int fs_iterator__expand_dir(fs_iterator *fi)
        GITERR_CHECK_ALLOC(ff);
 
        error = git_path_dirload_with_stat(
-               fi->path.ptr, fi->root_len, iterator__ignore_case(fi),
+               fi->path.ptr, fi->root_len, fi->dirload_flags,
                fi->base.start, fi->base.end, &ff->entries);
 
        if (error < 0) {
+               git_error_state last_error = { 0 };
+               giterr_capture(&last_error, error);
+
+               /* these callbacks may clear the error message */
                fs_iterator__free_frame(ff);
                fs_iterator__advance_over(NULL, (git_iterator *)fi);
-               return error;
+               /* next time return value we skipped to */
+               fi->base.flags &= ~GIT_ITERATOR_FIRST_ACCESS;
+
+               return giterr_restore(&last_error);
        }
 
        if (ff->entries.length == 0) {
                fs_iterator__free_frame(ff);
                return GIT_ENOTFOUND;
        }
+       fi->base.stat_calls += ff->entries.length;
 
        fs_iterator__seek_frame_start(fi, ff);
 
@@ -1174,7 +1195,7 @@ static int fs_iterator__update_entry(fs_iterator *fi)
                return GIT_ITEROVER;
 
        fi->entry.path = ps->path;
-       git_index_entry__init_from_stat(&fi->entry, &ps->st);
+       git_index_entry__init_from_stat(&fi->entry, &ps->st, true);
 
        /* need different mode here to keep directories during iteration */
        fi->entry.mode = git_futils_canonical_mode(ps->st.st_mode);
@@ -1207,6 +1228,11 @@ static int fs_iterator__initialize(
        }
        fi->root_len = fi->path.size;
 
+       fi->dirload_flags =
+               (iterator__ignore_case(fi) ? GIT_PATH_DIR_IGNORE_CASE : 0) |
+               (iterator__flag(fi, PRECOMPOSE_UNICODE) ?
+                       GIT_PATH_DIR_PRECOMPOSE_UNICODE : 0);
+
        if ((error = fs_iterator__expand_dir(fi)) < 0) {
                if (error == GIT_ENOTFOUND || error == GIT_ITEROVER) {
                        giterr_clear();
@@ -1244,6 +1270,16 @@ typedef struct {
        fs_iterator fi;
        git_ignores ignores;
        int is_ignored;
+
+       /*
+        * We may have a tree or the index+snapshot to compare against
+        * when checking for submodules.
+        */
+       git_tree *tree;
+       git_index *index;
+       git_vector index_snapshot;
+       git_vector_cmp entry_srch;
+
 } workdir_iterator;
 
 GIT_INLINE(bool) workdir_path_is_dotgit(const git_buf *path)
@@ -1265,16 +1301,107 @@ GIT_INLINE(bool) workdir_path_is_dotgit(const git_buf *path)
        return (len == 4 || path->ptr[len - 5] == '/');
 }
 
+/**
+ * Figure out if an entry is a submodule.
+ *
+ * We consider it a submodule if the path is listed as a submodule in
+ * either the tree or the index.
+ */
+static int is_submodule(workdir_iterator *wi, git_path_with_stat *ie)
+{
+       int error, is_submodule = 0;
+
+       if (wi->tree) {
+               git_tree_entry *e;
+
+               /* remove the trailing slash for finding */
+               ie->path[ie->path_len-1] = '\0';
+               error = git_tree_entry_bypath(&e, wi->tree, ie->path);
+               ie->path[ie->path_len-1] = '/';
+               if (error < 0 && error != GIT_ENOTFOUND)
+                       return 0;
+               if (!error) {
+                       is_submodule = e->attr == GIT_FILEMODE_COMMIT;
+                       git_tree_entry_free(e);
+               }
+       }
+
+       if (!is_submodule && wi->index) {
+               git_index_entry *e;
+               size_t pos;
+
+               error = git_index_snapshot_find(&pos, &wi->index_snapshot, wi->entry_srch, ie->path, ie->path_len-1, 0);
+               if (error < 0 && error != GIT_ENOTFOUND)
+                       return 0;
+
+               if (!error) {
+                       e = git_vector_get(&wi->index_snapshot, pos);
+
+                       is_submodule = e->mode == GIT_FILEMODE_COMMIT;
+               }
+       }
+
+       return is_submodule;
+}
+
+GIT_INLINE(git_dir_flag) git_entry__dir_flag(git_index_entry *entry) {
+#if defined(GIT_WIN32) && !defined(__MINGW32__)
+       return (entry && entry->mode)
+               ? S_ISDIR(entry->mode) ? GIT_DIR_FLAG_TRUE : GIT_DIR_FLAG_FALSE
+               : GIT_DIR_FLAG_UNKNOWN;
+#else
+       return GIT_DIR_FLAG_UNKNOWN;
+#endif
+}
+
 static int workdir_iterator__enter_dir(fs_iterator *fi)
 {
-       /* only push new ignores if this is not top level directory */
-       if (fi->stack->next != NULL) {
-               workdir_iterator *wi = (workdir_iterator *)fi;
+       workdir_iterator *wi = (workdir_iterator *)fi;
+       fs_iterator_frame *ff = fi->stack;
+       size_t pos;
+       git_path_with_stat *entry;
+       bool found_submodules = false;
+
+       git_dir_flag dir_flag = git_entry__dir_flag(&fi->entry);
+
+       /* check if this directory is ignored */
+       if (git_ignore__lookup(&ff->is_ignored, &wi->ignores, fi->path.ptr + fi->root_len, dir_flag) < 0) {
+               giterr_clear();
+               ff->is_ignored = GIT_IGNORE_NOTFOUND;
+       }
+
+       /* if this is not the top level directory... */
+       if (ff->next != NULL) {
                ssize_t slash_pos = git_buf_rfind_next(&fi->path, '/');
 
+               /* inherit ignored from parent if no rule specified */
+               if (ff->is_ignored <= GIT_IGNORE_NOTFOUND)
+                       ff->is_ignored = ff->next->is_ignored;
+
+               /* push new ignores for files in this directory */
                (void)git_ignore__push_dir(&wi->ignores, &fi->path.ptr[slash_pos + 1]);
        }
 
+       /* convert submodules to GITLINK and remove trailing slashes */
+       git_vector_foreach(&ff->entries, pos, entry) {
+               if (!S_ISDIR(entry->st.st_mode) || !strcmp(GIT_DIR, entry->path))
+                       continue;
+
+               if (is_submodule(wi, entry)) {
+                       entry->st.st_mode = GIT_FILEMODE_COMMIT;
+                       entry->path_len--;
+                       entry->path[entry->path_len] = '\0';
+                       found_submodules = true;
+               }
+       }
+
+       /* if we renamed submodules, re-sort and re-seek to start */
+       if (found_submodules) {
+               git_vector_set_sorted(&ff->entries, 0);
+               git_vector_sort(&ff->entries);
+               fs_iterator__seek_frame_start(fi, ff);
+       }
+
        return 0;
 }
 
@@ -1287,7 +1414,6 @@ static int workdir_iterator__leave_dir(fs_iterator *fi)
 
 static int workdir_iterator__update_entry(fs_iterator *fi)
 {
-       int error = 0;
        workdir_iterator *wi = (workdir_iterator *)fi;
 
        /* skip over .git entries */
@@ -1295,21 +1421,7 @@ static int workdir_iterator__update_entry(fs_iterator *fi)
                return GIT_ENOTFOUND;
 
        /* reset is_ignored since we haven't checked yet */
-       wi->is_ignored = -1;
-
-       /* check if apparent tree entries are actually submodules */
-       if (fi->entry.mode != GIT_FILEMODE_TREE)
-               return 0;
-
-       error = git_submodule_lookup(NULL, fi->base.repo, fi->entry.path);
-       if (error < 0)
-               giterr_clear();
-
-       /* mark submodule (or any dir with .git) as GITLINK and remove slash */
-       if (!error || error == GIT_EEXISTS) {
-               fi->entry.mode = S_IFGITLINK;
-               fi->entry.path[strlen(fi->entry.path) - 1] = '\0';
-       }
+       wi->is_ignored = GIT_IGNORE_UNCHECKED;
 
        return 0;
 }
@@ -1317,6 +1429,9 @@ static int workdir_iterator__update_entry(fs_iterator *fi)
 static void workdir_iterator__free(git_iterator *self)
 {
        workdir_iterator *wi = (workdir_iterator *)self;
+       if (wi->index)
+               git_index_snapshot_release(&wi->index_snapshot, wi->index);
+       git_tree_free(wi->tree);
        fs_iterator__free(self);
        git_ignore__free(&wi->ignores);
 }
@@ -1325,11 +1440,13 @@ int git_iterator_for_workdir_ext(
        git_iterator **out,
        git_repository *repo,
        const char *repo_workdir,
+       git_index *index,
+       git_tree *tree,
        git_iterator_flag_t flags,
        const char *start,
        const char *end)
 {
-       int error;
+       int error, precompose = 0;
        workdir_iterator *wi;
 
        if (!repo_workdir) {
@@ -1356,10 +1473,27 @@ int git_iterator_for_workdir_ext(
                return error;
        }
 
+       if (tree && (error = git_object_dup((git_object **)&wi->tree, (git_object *)tree)) < 0)
+               return error;
+
+       wi->index = index;
+       if (index && (error = git_index_snapshot_new(&wi->index_snapshot, index)) < 0) {
+               git_iterator_free((git_iterator *)wi);
+               return error;
+       }
+       wi->entry_srch = iterator__ignore_case(wi) ?
+               git_index_entry_isrch : git_index_entry_srch;
+
+
+       /* try to look up precompose and set flag if appropriate */
+       if (git_repository__cvar(&precompose, repo, GIT_CVAR_PRECOMPOSE) < 0)
+               giterr_clear();
+       else if (precompose)
+               wi->fi.base.flags |= GIT_ITERATOR_PRECOMPOSE_UNICODE;
+
        return fs_iterator__initialize(out, &wi->fi, repo_workdir);
 }
 
-
 void git_iterator_free(git_iterator *iter)
 {
        if (iter == NULL)
@@ -1448,6 +1582,20 @@ int git_iterator_current_parent_tree(
        return 0;
 }
 
+static void workdir_iterator_update_is_ignored(workdir_iterator *wi)
+{
+       git_dir_flag dir_flag = git_entry__dir_flag(&wi->fi.entry);
+
+       if (git_ignore__lookup(&wi->is_ignored, &wi->ignores, wi->fi.entry.path, dir_flag) < 0) {
+               giterr_clear();
+               wi->is_ignored = GIT_IGNORE_NOTFOUND;
+       }
+
+       /* use ignore from containing frame stack */
+       if (wi->is_ignored <= GIT_IGNORE_NOTFOUND)
+               wi->is_ignored = wi->fi.stack->is_ignored;
+}
+
 bool git_iterator_current_is_ignored(git_iterator *iter)
 {
        workdir_iterator *wi = (workdir_iterator *)iter;
@@ -1455,14 +1603,22 @@ bool git_iterator_current_is_ignored(git_iterator *iter)
        if (iter->type != GIT_ITERATOR_TYPE_WORKDIR)
                return false;
 
-       if (wi->is_ignored != -1)
-               return (bool)(wi->is_ignored != 0);
+       if (wi->is_ignored != GIT_IGNORE_UNCHECKED)
+               return (bool)(wi->is_ignored == GIT_IGNORE_TRUE);
+
+       workdir_iterator_update_is_ignored(wi);
+
+       return (bool)(wi->is_ignored == GIT_IGNORE_TRUE);
+}
+
+bool git_iterator_current_tree_is_ignored(git_iterator *iter)
+{
+       workdir_iterator *wi = (workdir_iterator *)iter;
 
-       if (git_ignore__lookup(
-                       &wi->ignores, wi->fi.entry.path, &wi->is_ignored) < 0)
-               wi->is_ignored = true;
+       if (iter->type != GIT_ITERATOR_TYPE_WORKDIR)
+               return false;
 
-       return (bool)wi->is_ignored;
+       return (bool)(wi->fi.stack->is_ignored == GIT_IGNORE_TRUE);
 }
 
 int git_iterator_cmp(git_iterator *iter, const char *path_prefix)
@@ -1491,3 +1647,73 @@ int git_iterator_current_workdir_path(git_buf **path, git_iterator *iter)
 
        return 0;
 }
+
+int git_iterator_advance_over_with_status(
+       const git_index_entry **entryptr,
+       git_iterator_status_t *status,
+       git_iterator *iter)
+{
+       int error = 0;
+       workdir_iterator *wi = (workdir_iterator *)iter;
+       char *base = NULL;
+       const git_index_entry *entry;
+
+       *status = GIT_ITERATOR_STATUS_NORMAL;
+
+       if (iter->type != GIT_ITERATOR_TYPE_WORKDIR)
+               return git_iterator_advance(entryptr, iter);
+       if ((error = git_iterator_current(&entry, iter)) < 0)
+               return error;
+
+       if (!S_ISDIR(entry->mode)) {
+               workdir_iterator_update_is_ignored(wi);
+               if (wi->is_ignored == GIT_IGNORE_TRUE)
+                       *status = GIT_ITERATOR_STATUS_IGNORED;
+               return git_iterator_advance(entryptr, iter);
+       }
+
+       *status = GIT_ITERATOR_STATUS_EMPTY;
+
+       base = git__strdup(entry->path);
+       GITERR_CHECK_ALLOC(base);
+
+       /* scan inside directory looking for a non-ignored item */
+       while (entry && !iter->prefixcomp(entry->path, base)) {
+               workdir_iterator_update_is_ignored(wi);
+
+               /* if we found an explicitly ignored item, then update from
+                * EMPTY to IGNORED
+                */
+               if (wi->is_ignored == GIT_IGNORE_TRUE)
+                       *status = GIT_ITERATOR_STATUS_IGNORED;
+               else if (S_ISDIR(entry->mode)) {
+                       error = git_iterator_advance_into(&entry, iter);
+
+                       if (!error)
+                               continue;
+                       else if (error == GIT_ENOTFOUND) {
+                               error = 0;
+                               wi->is_ignored = GIT_IGNORE_TRUE; /* mark empty dirs ignored */
+                       } else
+                               break; /* real error, stop here */
+               } else {
+                       /* we found a non-ignored item, treat parent as untracked */
+                       *status = GIT_ITERATOR_STATUS_NORMAL;
+                       break;
+               }
+
+               if ((error = git_iterator_advance(&entry, iter)) < 0)
+                       break;
+       }
+
+       /* wrap up scan back to base directory */
+       while (entry && !iter->prefixcomp(entry->path, base))
+               if ((error = git_iterator_advance(&entry, iter)) < 0)
+                       break;
+
+       *entryptr = entry;
+       git__free(base);
+
+       return error;
+}
+