src/fetch.c

   1 /*
   2  * Copyright (C) 2009-2012 the libgit2 contributors
   3  *
   4  * This file is part of libgit2, distributed under the GNU GPL v2 with
   5  * a Linking Exception. For full terms see the included COPYING file.
   6  */
   7
   8 #include "git2/oid.h"
   9 #include "git2/refs.h"
  10 #include "git2/revwalk.h"
  11 #include "git2/indexer.h"
  12
  13 #include "common.h"
  14 #include "transport.h"
  15 #include "remote.h"
  16 #include "refspec.h"
  17 #include "pack.h"
  18 #include "fetch.h"
  19 #include "netops.h"
  20 #include "pkt.h"
  21
  22 struct filter_payload {
  23         git_remote *remote;
  24         const git_refspec *spec, *tagspec;
  25         git_odb *odb;
  26         int found_head;
  27 };
  28
  29 static int filter_ref__cb(git_remote_head *head, void *payload)
  30 {
  31         struct filter_payload *p = payload;
  32         int match = 0;
  33
  34         if (!git_reference_is_valid_name(head->name))
  35                 return 0;
  36
  37         if (!p->found_head && strcmp(head->name, GIT_HEAD_FILE) == 0)
  38                 p->found_head = 1;
  39         else if (git_refspec_src_matches(p->spec, head->name))
  40                         match = 1;
  41         else if (p->remote->download_tags == GIT_REMOTE_DOWNLOAD_TAGS_ALL &&
  42                  git_refspec_src_matches(p->tagspec, head->name))
  43                         match = 1;
  44
  45         if (!match)
  46                 return 0;
  47
  48         /* If we have the object, mark it so we don't ask for it */
  49         if (git_odb_exists(p->odb, &head->oid))
  50                 head->local = 1;
  51         else
  52                 p->remote->need_pack = 1;
  53
  54         return git_vector_insert(&p->remote->refs, head);
  55 }
  56
  57 static int filter_wants(git_remote *remote)
  58 {
  59         struct filter_payload p;
  60         git_refspec tagspec;
  61
  62         git_vector_clear(&remote->refs);
  63         if (git_refspec__parse(&tagspec, GIT_REFSPEC_TAGS, true) < 0)
  64                 return -1;
  65
  66         /*
  67          * The fetch refspec can be NULL, and what this means is that the
  68          * user didn't specify one. This is fine, as it means that we're
  69          * not interested in any particular branch but just the remote's
  70          * HEAD, which will be stored in FETCH_HEAD after the fetch.
  71          */
  72         p.spec = git_remote_fetchspec(remote);
  73         p.tagspec = &tagspec;
  74         p.found_head = 0;
  75         p.remote = remote;
  76
  77         if (git_repository_odb__weakptr(&p.odb, remote->repo) < 0)
  78                 return -1;
  79
  80         return git_remote_ls(remote, filter_ref__cb, &p);
  81 }
  82
  83 /* Wait until we get an ack from the */
  84 static int recv_pkt(git_pkt **out, gitno_buffer *buf)
  85 {
  86         const char *ptr = buf->data, *line_end = ptr;
  87         git_pkt *pkt;
  88         int pkt_type, error = 0, ret;
  89
  90         do {
  91                 if (buf->offset > 0)
  92                         error = git_pkt_parse_line(&pkt, ptr, &line_end, buf->offset);
  93                 else
  94                         error = GIT_EBUFS;
  95
  96                 if (error == 0)
  97                         break; /* return the pkt */
  98
  99                 if (error < 0 && error != GIT_EBUFS)
 100                         return -1;
 101
 102                 if ((ret = gitno_recv(buf)) < 0)
 103                         return -1;
 104         } while (error);
 105
 106         gitno_consume(buf, line_end);
 107         pkt_type = pkt->type;
 108         if (out != NULL)
 109                 *out = pkt;
 110         else
 111                 git__free(pkt);
 112
 113         return pkt_type;
 114 }
 115
 116 static int store_common(git_transport *t)
 117 {
 118         git_pkt *pkt = NULL;
 119         gitno_buffer *buf = &t->buffer;
 120
 121         do {
 122                 if (recv_pkt(&pkt, buf) < 0)
 123                         return -1;
 124
 125                 if (pkt->type == GIT_PKT_ACK) {
 126                         if (git_vector_insert(&t->common, pkt) < 0)
 127                                 return -1;
 128                 } else {
 129                         git__free(pkt);
 130                         return 0;
 131                 }
 132
 133         } while (1);
 134
 135         return 0;
 136 }
 137
 138 /*
 139  * In this first version, we push all our refs in and start sending
 140  * them out. When we get an ACK we hide that commit and continue
 141  * traversing until we're done
 142  */
 143 int git_fetch_negotiate(git_remote *remote)
 144 {
 145         git_transport *t = remote->transport;
 146         gitno_buffer *buf = &t->buffer;
 147         git_buf data = GIT_BUF_INIT;
 148         git_revwalk *walk = NULL;
 149         int error, pkt_type;
 150         unsigned int i;
 151         git_oid oid;
 152
 153         if (filter_wants(remote) < 0) {
 154                 giterr_set(GITERR_NET, "Failed to filter the reference list for wants");
 155                 return -1;
 156         }
 157
 158         /* Don't try to negotiate when we don't want anything */
 159         if (remote->refs.length == 0 || !remote->need_pack)
 160                 return 0;
 161
 162         /*
 163          * Now we have everything set up so we can start tell the
 164          * server what we want and what we have. Call the function if
 165          * the transport has its own logic. This is transitional and
 166          * will be removed once this function can support git and http.
 167          */
 168         if (t->own_logic)
 169                 return t->negotiate_fetch(t, remote->repo, &remote->refs);
 170
 171         /* No own logic, do our thing */
 172         if (git_pkt_buffer_wants(&remote->refs, &t->caps, &data) < 0)
 173                 return -1;
 174
 175         if (git_fetch_setup_walk(&walk, remote->repo) < 0)
 176                 goto on_error;
 177         /*
 178          * We don't support any kind of ACK extensions, so the negotiation
 179          * boils down to sending what we have and listening for an ACK
 180          * every once in a while.
 181          */
 182         i = 0;
 183         while ((error = git_revwalk_next(&oid, walk)) == 0) {
 184                 git_pkt_buffer_have(&oid, &data);
 185                 i++;
 186                 if (i % 20 == 0) {
 187                         git_pkt_buffer_flush(&data);
 188                         if (git_buf_oom(&data))
 189                                 goto on_error;
 190
 191                         if (t->negotiation_step(t, data.ptr, data.size) < 0)
 192                                 goto on_error;
 193
 194                         git_buf_clear(&data);
 195                         if (t->caps.multi_ack) {
 196                                 if (store_common(t) < 0)
 197                                         goto on_error;
 198                         } else {
 199                                 pkt_type = recv_pkt(NULL, buf);
 200
 201                                 if (pkt_type == GIT_PKT_ACK) {
 202                                         break;
 203                                 } else if (pkt_type == GIT_PKT_NAK) {
 204                                         continue;
 205                                 } else {
 206                                         giterr_set(GITERR_NET, "Unexpected pkt type");
 207                                         goto on_error;
 208                                 }
 209                         }
 210                 }
 211
 212                 if (t->common.length > 0)
 213                         break;
 214
 215                 if (i % 20 == 0 && t->rpc) {
 216                         git_pkt_ack *pkt;
 217                         unsigned int i;
 218
 219                         if (git_pkt_buffer_wants(&remote->refs, &t->caps, &data) < 0)
 220                                 goto on_error;
 221
 222                         git_vector_foreach(&t->common, i, pkt) {
 223                                 git_pkt_buffer_have(&pkt->oid, &data);
 224                         }
 225
 226                         if (git_buf_oom(&data))
 227                                 goto on_error;
 228                 }
 229         }
 230
 231         if (error < 0 && error != GIT_ITEROVER)
 232                 goto on_error;
 233
 234         /* Tell the other end that we're done negotiating */
 235         if (t->rpc && t->common.length > 0) {
 236                 git_pkt_ack *pkt;
 237                 unsigned int i;
 238
 239                 if (git_pkt_buffer_wants(&remote->refs, &t->caps, &data) < 0)
 240                         goto on_error;
 241
 242                 git_vector_foreach(&t->common, i, pkt) {
 243                         git_pkt_buffer_have(&pkt->oid, &data);
 244                 }
 245
 246                 if (git_buf_oom(&data))
 247                         goto on_error;
 248         }
 249
 250         git_pkt_buffer_done(&data);
 251         if (t->negotiation_step(t, data.ptr, data.size) < 0)
 252                 goto on_error;
 253
 254         git_buf_free(&data);
 255         git_revwalk_free(walk);
 256
 257         /* Now let's eat up whatever the server gives us */
 258         if (!t->caps.multi_ack) {
 259                 pkt_type = recv_pkt(NULL, buf);
 260                 if (pkt_type != GIT_PKT_ACK && pkt_type != GIT_PKT_NAK) {
 261                         giterr_set(GITERR_NET, "Unexpected pkt type");
 262                         return -1;
 263                 }
 264         } else {
 265                 git_pkt_ack *pkt;
 266                 do {
 267                         if (recv_pkt((git_pkt **)&pkt, buf) < 0)
 268                                 return -1;
 269
 270                         if (pkt->type == GIT_PKT_NAK ||
 271                             (pkt->type == GIT_PKT_ACK && pkt->status != GIT_ACK_CONTINUE)) {
 272                                 git__free(pkt);
 273                                 break;
 274                         }
 275
 276                         git__free(pkt);
 277                 } while (1);
 278         }
 279
 280         return 0;
 281
 282 on_error:
 283         git_revwalk_free(walk);
 284         git_buf_free(&data);
 285         return -1;
 286 }
 287
 288 int git_fetch_download_pack(git_remote *remote, git_off_t *bytes, git_indexer_stats *stats)
 289 {
 290         git_transport *t = remote->transport;
 291
 292         if(!remote->need_pack)
 293                 return 0;
 294
 295         if (t->own_logic)
 296                 return t->download_pack(t, remote->repo, bytes, stats);
 297
 298         return git_fetch__download_pack(t, remote->repo, bytes, stats);
 299
 300 }
 301
 302 static int no_sideband(git_indexer_stream *idx, gitno_buffer *buf, git_off_t *bytes, git_indexer_stats *stats)
 303 {
 304         int recvd;
 305
 306         do {
 307                 if (git_indexer_stream_add(idx, buf->data, buf->offset, stats) < 0)
 308                         return -1;
 309
 310                 gitno_consume_n(buf, buf->offset);
 311
 312                 if ((recvd = gitno_recv(buf)) < 0)
 313                         return -1;
 314
 315                 *bytes += recvd;
 316         } while(recvd > 0);
 317
 318         if (git_indexer_stream_finalize(idx, stats))
 319                 return -1;
 320
 321         return 0;
 322 }
 323
 324 /* Receiving data from a socket and storing it is pretty much the same for git and HTTP */
 325 int git_fetch__download_pack(
 326         git_transport *t,
 327         git_repository *repo,
 328         git_off_t *bytes,
 329         git_indexer_stats *stats)
 330 {
 331         git_buf path = GIT_BUF_INIT;
 332         gitno_buffer *buf = &t->buffer;
 333         git_indexer_stream *idx = NULL;
 334
 335         if (git_buf_joinpath(&path, git_repository_path(repo), "objects/pack") < 0)
 336                 return -1;
 337
 338         if (git_indexer_stream_new(&idx, git_buf_cstr(&path)) < 0)
 339                 goto on_error;
 340
 341         git_buf_free(&path);
 342         memset(stats, 0, sizeof(git_indexer_stats));
 343         *bytes = 0;
 344
 345         /*
 346          * If the remote doesn't support the side-band, we can feed
 347          * the data directly to the indexer. Otherwise, we need to
 348          * check which one belongs there.
 349          */
 350         if (!t->caps.side_band && !t->caps.side_band_64k) {
 351                 if (no_sideband(idx, buf, bytes, stats) < 0)
 352                         goto on_error;
 353
 354                 git_indexer_stream_free(idx);
 355                 return 0;
 356         }
 357
 358         do {
 359                 git_pkt *pkt;
 360                 if (recv_pkt(&pkt, buf) < 0)
 361                         goto on_error;
 362
 363                 if (pkt->type == GIT_PKT_PROGRESS) {
 364                         if (t->progress_cb) {
 365                                 git_pkt_progress *p = (git_pkt_progress *) pkt;
 366                                 t->progress_cb(p->data, p->len, t->cb_data);
 367                         }
 368                         git__free(pkt);
 369                 } else if (pkt->type == GIT_PKT_DATA) {
 370                         git_pkt_data *p = (git_pkt_data *) pkt;
 371                         *bytes += p->len;
 372                         if (git_indexer_stream_add(idx, p->data, p->len, stats) < 0)
 373                                 goto on_error;
 374
 375                         git__free(pkt);
 376                 } else if (pkt->type == GIT_PKT_FLUSH) {
 377                         /* A flush indicates the end of the packfile */
 378                         git__free(pkt);
 379                         break;
 380                 }
 381         } while (1);
 382
 383         if (git_indexer_stream_finalize(idx, stats) < 0)
 384                 goto on_error;
 385
 386         git_indexer_stream_free(idx);
 387         return 0;
 388
 389 on_error:
 390         git_buf_free(&path);
 391         git_indexer_stream_free(idx);
 392         return -1;
 393 }
 394
 395 int git_fetch_setup_walk(git_revwalk **out, git_repository *repo)
 396 {
 397         git_revwalk *walk;
 398         git_strarray refs;
 399         unsigned int i;
 400         git_reference *ref;
 401
 402         if (git_reference_list(&refs, repo, GIT_REF_LISTALL) < 0)
 403                 return -1;
 404
 405         if (git_revwalk_new(&walk, repo) < 0)
 406                 return -1;
 407
 408         git_revwalk_sorting(walk, GIT_SORT_TIME);
 409
 410         for (i = 0; i < refs.count; ++i) {
 411                 /* No tags */
 412                 if (!git__prefixcmp(refs.strings[i], GIT_REFS_TAGS_DIR))
 413                         continue;
 414
 415                 if (git_reference_lookup(&ref, repo, refs.strings[i]) < 0)
 416                         goto on_error;
 417
 418                 if (git_reference_type(ref) == GIT_REF_SYMBOLIC)
 419                         continue;
 420                 if (git_revwalk_push(walk, git_reference_oid(ref)) < 0)
 421                         goto on_error;
 422
 423                 git_reference_free(ref);
 424         }
 425
 426         git_strarray_free(&refs);
 427         *out = walk;
 428         return 0;
 429
 430 on_error:
 431         git_reference_free(ref);
 432         git_strarray_free(&refs);
 433         return -1;
 434 }