src/fetch.c

   1 /*
   2  * Copyright (C) 2009-2012 the libgit2 contributors
   3  *
   4  * This file is part of libgit2, distributed under the GNU GPL v2 with
   5  * a Linking Exception. For full terms see the included COPYING file.
   6  */
   7
   8 #include "git2/oid.h"
   9 #include "git2/refs.h"
  10 #include "git2/revwalk.h"
  11 #include "git2/indexer.h"
  12
  13 #include "common.h"
  14 #include "transport.h"
  15 #include "remote.h"
  16 #include "refspec.h"
  17 #include "pack.h"
  18 #include "fetch.h"
  19 #include "netops.h"
  20 #include "pkt.h"
  21
  22 struct filter_payload {
  23         git_remote *remote;
  24         const git_refspec *spec, *tagspec;
  25         git_odb *odb;
  26         int found_head;
  27 };
  28
  29 static int filter_ref__cb(git_remote_head *head, void *payload)
  30 {
  31         struct filter_payload *p = payload;
  32         int match = 0;
  33
  34         if (!git_reference_is_valid_name(head->name))
  35                 return 0;
  36
  37         if (!p->found_head && strcmp(head->name, GIT_HEAD_FILE) == 0)
  38                 p->found_head = 1;
  39         else if (git_refspec_src_matches(p->spec, head->name))
  40                         match = 1;
  41         else if (p->remote->download_tags == GIT_REMOTE_DOWNLOAD_TAGS_ALL &&
  42                  git_refspec_src_matches(p->tagspec, head->name))
  43                         match = 1;
  44
  45         if (!match)
  46                 return 0;
  47
  48         /* If we have the object, mark it so we don't ask for it */
  49         if (git_odb_exists(p->odb, &head->oid))
  50                 head->local = 1;
  51         else
  52                 p->remote->need_pack = 1;
  53
  54         return git_vector_insert(&p->remote->refs, head);
  55 }
  56
  57 static int filter_wants(git_remote *remote)
  58 {
  59         struct filter_payload p;
  60         git_refspec tagspec;
  61         int error = -1;
  62
  63         git_vector_clear(&remote->refs);
  64         if (git_refspec__parse(&tagspec, GIT_REFSPEC_TAGS, true) < 0)
  65                 return error;
  66
  67         /*
  68          * The fetch refspec can be NULL, and what this means is that the
  69          * user didn't specify one. This is fine, as it means that we're
  70          * not interested in any particular branch but just the remote's
  71          * HEAD, which will be stored in FETCH_HEAD after the fetch.
  72          */
  73         p.spec = git_remote_fetchspec(remote);
  74         p.tagspec = &tagspec;
  75         p.found_head = 0;
  76         p.remote = remote;
  77
  78         if (git_repository_odb__weakptr(&p.odb, remote->repo) < 0)
  79                 goto cleanup;
  80
  81         error = git_remote_ls(remote, filter_ref__cb, &p);
  82
  83 cleanup:
  84         git_refspec__free(&tagspec);
  85
  86         return error;
  87 }
  88
  89 /* Wait until we get an ack from the */
  90 static int recv_pkt(git_pkt **out, gitno_buffer *buf)
  91 {
  92         const char *ptr = buf->data, *line_end = ptr;
  93         git_pkt *pkt;
  94         int pkt_type, error = 0, ret;
  95
  96         do {
  97                 if (buf->offset > 0)
  98                         error = git_pkt_parse_line(&pkt, ptr, &line_end, buf->offset);
  99                 else
 100                         error = GIT_EBUFS;
 101
 102                 if (error == 0)
 103                         break; /* return the pkt */
 104
 105                 if (error < 0 && error != GIT_EBUFS)
 106                         return -1;
 107
 108                 if ((ret = gitno_recv(buf)) < 0)
 109                         return -1;
 110         } while (error);
 111
 112         gitno_consume(buf, line_end);
 113         pkt_type = pkt->type;
 114         if (out != NULL)
 115                 *out = pkt;
 116         else
 117                 git__free(pkt);
 118
 119         return pkt_type;
 120 }
 121
 122 static int store_common(git_transport *t)
 123 {
 124         git_pkt *pkt = NULL;
 125         gitno_buffer *buf = &t->buffer;
 126
 127         do {
 128                 if (recv_pkt(&pkt, buf) < 0)
 129                         return -1;
 130
 131                 if (pkt->type == GIT_PKT_ACK) {
 132                         if (git_vector_insert(&t->common, pkt) < 0)
 133                                 return -1;
 134                 } else {
 135                         git__free(pkt);
 136                         return 0;
 137                 }
 138
 139         } while (1);
 140
 141         return 0;
 142 }
 143
 144 /*
 145  * In this first version, we push all our refs in and start sending
 146  * them out. When we get an ACK we hide that commit and continue
 147  * traversing until we're done
 148  */
 149 int git_fetch_negotiate(git_remote *remote)
 150 {
 151         git_transport *t = remote->transport;
 152         gitno_buffer *buf = &t->buffer;
 153         git_buf data = GIT_BUF_INIT;
 154         git_revwalk *walk = NULL;
 155         int error = -1, pkt_type;
 156         unsigned int i;
 157         git_oid oid;
 158
 159         if (filter_wants(remote) < 0) {
 160                 giterr_set(GITERR_NET, "Failed to filter the reference list for wants");
 161                 return -1;
 162         }
 163
 164         /* Don't try to negotiate when we don't want anything */
 165         if (remote->refs.length == 0 || !remote->need_pack)
 166                 return 0;
 167
 168         /*
 169          * Now we have everything set up so we can start tell the
 170          * server what we want and what we have. Call the function if
 171          * the transport has its own logic. This is transitional and
 172          * will be removed once this function can support git and http.
 173          */
 174         if (t->own_logic)
 175                 return t->negotiate_fetch(t, remote->repo, &remote->refs);
 176
 177         /* No own logic, do our thing */
 178         if (git_pkt_buffer_wants(&remote->refs, &t->caps, &data) < 0)
 179                 return -1;
 180
 181         if (git_fetch_setup_walk(&walk, remote->repo) < 0)
 182                 goto on_error;
 183         /*
 184          * We don't support any kind of ACK extensions, so the negotiation
 185          * boils down to sending what we have and listening for an ACK
 186          * every once in a while.
 187          */
 188         i = 0;
 189         while ((error = git_revwalk_next(&oid, walk)) == 0) {
 190                 git_pkt_buffer_have(&oid, &data);
 191                 i++;
 192                 if (i % 20 == 0) {
 193                         if (t->cancel.val) {
 194                                 giterr_set(GITERR_NET, "The fetch was cancelled by the user");
 195                                         error = GIT_EUSER;
 196                                         goto on_error;
 197                         }
 198
 199                         git_pkt_buffer_flush(&data);
 200                         if (git_buf_oom(&data))
 201                                 goto on_error;
 202
 203                         if (t->negotiation_step(t, data.ptr, data.size) < 0)
 204                                 goto on_error;
 205
 206                         git_buf_clear(&data);
 207                         if (t->caps.multi_ack) {
 208                                 if (store_common(t) < 0)
 209                                         goto on_error;
 210                         } else {
 211                                 pkt_type = recv_pkt(NULL, buf);
 212
 213                                 if (pkt_type == GIT_PKT_ACK) {
 214                                         break;
 215                                 } else if (pkt_type == GIT_PKT_NAK) {
 216                                         continue;
 217                                 } else {
 218                                         giterr_set(GITERR_NET, "Unexpected pkt type");
 219                                         goto on_error;
 220                                 }
 221                         }
 222                 }
 223
 224                 if (t->common.length > 0)
 225                         break;
 226
 227                 if (i % 20 == 0 && t->rpc) {
 228                         git_pkt_ack *pkt;
 229                         unsigned int i;
 230
 231                         if (git_pkt_buffer_wants(&remote->refs, &t->caps, &data) < 0)
 232                                 goto on_error;
 233
 234                         git_vector_foreach(&t->common, i, pkt) {
 235                                 git_pkt_buffer_have(&pkt->oid, &data);
 236                         }
 237
 238                         if (git_buf_oom(&data))
 239                                 goto on_error;
 240                 }
 241         }
 242
 243         if (error < 0 && error != GIT_ITEROVER)
 244                 goto on_error;
 245
 246         /* Tell the other end that we're done negotiating */
 247         if (t->rpc && t->common.length > 0) {
 248                 git_pkt_ack *pkt;
 249                 unsigned int i;
 250
 251                 if (git_pkt_buffer_wants(&remote->refs, &t->caps, &data) < 0)
 252                         goto on_error;
 253
 254                 git_vector_foreach(&t->common, i, pkt) {
 255                         git_pkt_buffer_have(&pkt->oid, &data);
 256                 }
 257
 258                 if (git_buf_oom(&data))
 259                         goto on_error;
 260         }
 261
 262         git_pkt_buffer_done(&data);
 263         if (t->cancel.val) {
 264                 giterr_set(GITERR_NET, "The fetch was cancelled by the user");
 265                 error = GIT_EUSER;
 266                 goto on_error;
 267         }
 268         if (t->negotiation_step(t, data.ptr, data.size) < 0)
 269                 goto on_error;
 270
 271         git_buf_free(&data);
 272         git_revwalk_free(walk);
 273
 274         /* Now let's eat up whatever the server gives us */
 275         if (!t->caps.multi_ack) {
 276                 pkt_type = recv_pkt(NULL, buf);
 277                 if (pkt_type != GIT_PKT_ACK && pkt_type != GIT_PKT_NAK) {
 278                         giterr_set(GITERR_NET, "Unexpected pkt type");
 279                         return -1;
 280                 }
 281         } else {
 282                 git_pkt_ack *pkt;
 283                 do {
 284                         if (recv_pkt((git_pkt **)&pkt, buf) < 0)
 285                                 return -1;
 286
 287                         if (pkt->type == GIT_PKT_NAK ||
 288                             (pkt->type == GIT_PKT_ACK && pkt->status != GIT_ACK_CONTINUE)) {
 289                                 git__free(pkt);
 290                                 break;
 291                         }
 292
 293                         git__free(pkt);
 294                 } while (1);
 295         }
 296
 297         return 0;
 298
 299 on_error:
 300         git_revwalk_free(walk);
 301         git_buf_free(&data);
 302         return error;
 303 }
 304
 305 int git_fetch_download_pack(git_remote *remote, git_off_t *bytes)
 306 {
 307         git_transport *t = remote->transport;
 308
 309         if(!remote->need_pack)
 310                 return 0;
 311
 312         if (t->own_logic)
 313                 return t->download_pack(t, remote->repo, bytes, &remote->stats);
 314
 315         return git_fetch__download_pack(t, remote->repo, bytes, &remote->stats);
 316
 317 }
 318
 319 static int no_sideband(git_transport *t, git_indexer_stream *idx, gitno_buffer *buf, git_off_t *bytes, git_indexer_stats *stats)
 320 {
 321         int recvd;
 322
 323         do {
 324                 if (t->cancel.val) {
 325                         giterr_set(GITERR_NET, "The fetch was cancelled by the user");
 326                         return GIT_EUSER;
 327                 }
 328
 329                 if (git_indexer_stream_add(idx, buf->data, buf->offset, stats) < 0)
 330                         return -1;
 331
 332                 gitno_consume_n(buf, buf->offset);
 333
 334                 if ((recvd = gitno_recv(buf)) < 0)
 335                         return -1;
 336
 337                 *bytes += recvd;
 338         } while(recvd > 0);
 339
 340         if (git_indexer_stream_finalize(idx, stats))
 341                 return -1;
 342
 343         return 0;
 344 }
 345
 346 /* Receiving data from a socket and storing it is pretty much the same for git and HTTP */
 347 int git_fetch__download_pack(
 348         git_transport *t,
 349         git_repository *repo,
 350         git_off_t *bytes,
 351         git_indexer_stats *stats)
 352 {
 353         git_buf path = GIT_BUF_INIT;
 354         gitno_buffer *buf = &t->buffer;
 355         git_indexer_stream *idx = NULL;
 356         int error = -1;
 357
 358         if (git_buf_joinpath(&path, git_repository_path(repo), "objects/pack") < 0)
 359                 return -1;
 360
 361         if (git_indexer_stream_new(&idx, git_buf_cstr(&path)) < 0)
 362                 goto on_error;
 363
 364         git_buf_free(&path);
 365         memset(stats, 0, sizeof(git_indexer_stats));
 366         *bytes = 0;
 367
 368         /*
 369          * If the remote doesn't support the side-band, we can feed
 370          * the data directly to the indexer. Otherwise, we need to
 371          * check which one belongs there.
 372          */
 373         if (!t->caps.side_band && !t->caps.side_band_64k) {
 374                 if (no_sideband(t, idx, buf, bytes, stats) < 0)
 375                         goto on_error;
 376
 377                 git_indexer_stream_free(idx);
 378                 return 0;
 379         }
 380
 381         do {
 382                 git_pkt *pkt;
 383
 384                 if (t->cancel.val) {
 385                         giterr_set(GITERR_NET, "The fetch was cancelled by the user");
 386                         error = GIT_EUSER;
 387                         goto on_error;
 388                 }
 389
 390                 if (recv_pkt(&pkt, buf) < 0)
 391                         goto on_error;
 392
 393                 if (pkt->type == GIT_PKT_PROGRESS) {
 394                         if (t->progress_cb) {
 395                                 git_pkt_progress *p = (git_pkt_progress *) pkt;
 396                                 t->progress_cb(p->data, p->len, t->cb_data);
 397                         }
 398                         git__free(pkt);
 399                 } else if (pkt->type == GIT_PKT_DATA) {
 400                         git_pkt_data *p = (git_pkt_data *) pkt;
 401                         *bytes += p->len;
 402                         if (git_indexer_stream_add(idx, p->data, p->len, stats) < 0)
 403                                 goto on_error;
 404
 405                         git__free(pkt);
 406                 } else if (pkt->type == GIT_PKT_FLUSH) {
 407                         /* A flush indicates the end of the packfile */
 408                         git__free(pkt);
 409                         break;
 410                 }
 411         } while (1);
 412
 413         if (git_indexer_stream_finalize(idx, stats) < 0)
 414                 goto on_error;
 415
 416         git_indexer_stream_free(idx);
 417         return 0;
 418
 419 on_error:
 420         git_buf_free(&path);
 421         git_indexer_stream_free(idx);
 422         return error;
 423 }
 424
 425 int git_fetch_setup_walk(git_revwalk **out, git_repository *repo)
 426 {
 427         git_revwalk *walk;
 428         git_strarray refs;
 429         unsigned int i;
 430         git_reference *ref;
 431
 432         if (git_reference_list(&refs, repo, GIT_REF_LISTALL) < 0)
 433                 return -1;
 434
 435         if (git_revwalk_new(&walk, repo) < 0)
 436                 return -1;
 437
 438         git_revwalk_sorting(walk, GIT_SORT_TIME);
 439
 440         for (i = 0; i < refs.count; ++i) {
 441                 /* No tags */
 442                 if (!git__prefixcmp(refs.strings[i], GIT_REFS_TAGS_DIR))
 443                         continue;
 444
 445                 if (git_reference_lookup(&ref, repo, refs.strings[i]) < 0)
 446                         goto on_error;
 447
 448                 if (git_reference_type(ref) == GIT_REF_SYMBOLIC)
 449                         continue;
 450                 if (git_revwalk_push(walk, git_reference_oid(ref)) < 0)
 451                         goto on_error;
 452
 453                 git_reference_free(ref);
 454         }
 455
 456         git_strarray_free(&refs);
 457         *out = walk;
 458         return 0;
 459
 460 on_error:
 461         git_reference_free(ref);
 462         git_strarray_free(&refs);
 463         return -1;
 464 }