src/fetch.c

   1 /*
   2  * Copyright (C) 2009-2012 the libgit2 contributors
   3  *
   4  * This file is part of libgit2, distributed under the GNU GPL v2 with
   5  * a Linking Exception. For full terms see the included COPYING file.
   6  */
   7
   8 #include "git2/oid.h"
   9 #include "git2/refs.h"
  10 #include "git2/revwalk.h"
  11 #include "git2/indexer.h"
  12
  13 #include "common.h"
  14 #include "transport.h"
  15 #include "remote.h"
  16 #include "refspec.h"
  17 #include "pack.h"
  18 #include "fetch.h"
  19 #include "netops.h"
  20 #include "pkt.h"
  21
  22 #define NETWORK_XFER_THRESHOLD (100*1024)
  23
  24 struct filter_payload {
  25         git_remote *remote;
  26         const git_refspec *spec, *tagspec;
  27         git_odb *odb;
  28         int found_head;
  29 };
  30
  31 static int filter_ref__cb(git_remote_head *head, void *payload)
  32 {
  33         struct filter_payload *p = payload;
  34         int match = 0;
  35
  36         if (!git_reference_is_valid_name(head->name))
  37                 return 0;
  38
  39         if (!p->found_head && strcmp(head->name, GIT_HEAD_FILE) == 0)
  40                 p->found_head = 1;
  41         else if (git_refspec_src_matches(p->spec, head->name))
  42                         match = 1;
  43         else if (p->remote->download_tags == GIT_REMOTE_DOWNLOAD_TAGS_ALL &&
  44                  git_refspec_src_matches(p->tagspec, head->name))
  45                         match = 1;
  46
  47         if (!match)
  48                 return 0;
  49
  50         /* If we have the object, mark it so we don't ask for it */
  51         if (git_odb_exists(p->odb, &head->oid))
  52                 head->local = 1;
  53         else
  54                 p->remote->need_pack = 1;
  55
  56         return git_vector_insert(&p->remote->refs, head);
  57 }
  58
  59 static int filter_wants(git_remote *remote)
  60 {
  61         struct filter_payload p;
  62         git_refspec tagspec;
  63         int error = -1;
  64
  65         git_vector_clear(&remote->refs);
  66         if (git_refspec__parse(&tagspec, GIT_REFSPEC_TAGS, true) < 0)
  67                 return error;
  68
  69         /*
  70          * The fetch refspec can be NULL, and what this means is that the
  71          * user didn't specify one. This is fine, as it means that we're
  72          * not interested in any particular branch but just the remote's
  73          * HEAD, which will be stored in FETCH_HEAD after the fetch.
  74          */
  75         p.spec = git_remote_fetchspec(remote);
  76         p.tagspec = &tagspec;
  77         p.found_head = 0;
  78         p.remote = remote;
  79
  80         if (git_repository_odb__weakptr(&p.odb, remote->repo) < 0)
  81                 goto cleanup;
  82
  83         error = git_remote_ls(remote, filter_ref__cb, &p);
  84
  85 cleanup:
  86         git_refspec__free(&tagspec);
  87
  88         return error;
  89 }
  90
  91 /* Wait until we get an ack from the */
  92 static int recv_pkt(git_pkt **out, gitno_buffer *buf)
  93 {
  94         const char *ptr = buf->data, *line_end = ptr;
  95         git_pkt *pkt;
  96         int pkt_type, error = 0, ret;
  97
  98         do {
  99                 if (buf->offset > 0)
 100                         error = git_pkt_parse_line(&pkt, ptr, &line_end, buf->offset);
 101                 else
 102                         error = GIT_EBUFS;
 103
 104                 if (error == 0)
 105                         break; /* return the pkt */
 106
 107                 if (error < 0 && error != GIT_EBUFS)
 108                         return -1;
 109
 110                 if ((ret = gitno_recv(buf)) < 0)
 111                         return -1;
 112         } while (error);
 113
 114         gitno_consume(buf, line_end);
 115         pkt_type = pkt->type;
 116         if (out != NULL)
 117                 *out = pkt;
 118         else
 119                 git__free(pkt);
 120
 121         return pkt_type;
 122 }
 123
 124 static int store_common(git_transport *t)
 125 {
 126         git_pkt *pkt = NULL;
 127         gitno_buffer *buf = &t->buffer;
 128
 129         do {
 130                 if (recv_pkt(&pkt, buf) < 0)
 131                         return -1;
 132
 133                 if (pkt->type == GIT_PKT_ACK) {
 134                         if (git_vector_insert(&t->common, pkt) < 0)
 135                                 return -1;
 136                 } else {
 137                         git__free(pkt);
 138                         return 0;
 139                 }
 140
 141         } while (1);
 142
 143         return 0;
 144 }
 145
 146 /*
 147  * In this first version, we push all our refs in and start sending
 148  * them out. When we get an ACK we hide that commit and continue
 149  * traversing until we're done
 150  */
 151 int git_fetch_negotiate(git_remote *remote)
 152 {
 153         git_transport *t = remote->transport;
 154         gitno_buffer *buf = &t->buffer;
 155         git_buf data = GIT_BUF_INIT;
 156         git_revwalk *walk = NULL;
 157         int error = -1, pkt_type;
 158         unsigned int i;
 159         git_oid oid;
 160
 161         if (filter_wants(remote) < 0) {
 162                 giterr_set(GITERR_NET, "Failed to filter the reference list for wants");
 163                 return -1;
 164         }
 165
 166         /* Don't try to negotiate when we don't want anything */
 167         if (remote->refs.length == 0 || !remote->need_pack)
 168                 return 0;
 169
 170         /*
 171          * Now we have everything set up so we can start tell the
 172          * server what we want and what we have. Call the function if
 173          * the transport has its own logic. This is transitional and
 174          * will be removed once this function can support git and http.
 175          */
 176         if (t->own_logic)
 177                 return t->negotiate_fetch(t, remote->repo, &remote->refs);
 178
 179         /* No own logic, do our thing */
 180         if (git_pkt_buffer_wants(&remote->refs, &t->caps, &data) < 0)
 181                 return -1;
 182
 183         if (git_fetch_setup_walk(&walk, remote->repo) < 0)
 184                 goto on_error;
 185         /*
 186          * We don't support any kind of ACK extensions, so the negotiation
 187          * boils down to sending what we have and listening for an ACK
 188          * every once in a while.
 189          */
 190         i = 0;
 191         while ((error = git_revwalk_next(&oid, walk)) == 0) {
 192                 git_pkt_buffer_have(&oid, &data);
 193                 i++;
 194                 if (i % 20 == 0) {
 195                         if (t->cancel.val) {
 196                                 giterr_set(GITERR_NET, "The fetch was cancelled by the user");
 197                                         error = GIT_EUSER;
 198                                         goto on_error;
 199                         }
 200
 201                         git_pkt_buffer_flush(&data);
 202                         if (git_buf_oom(&data))
 203                                 goto on_error;
 204
 205                         if (t->negotiation_step(t, data.ptr, data.size) < 0)
 206                                 goto on_error;
 207
 208                         git_buf_clear(&data);
 209                         if (t->caps.multi_ack) {
 210                                 if (store_common(t) < 0)
 211                                         goto on_error;
 212                         } else {
 213                                 pkt_type = recv_pkt(NULL, buf);
 214
 215                                 if (pkt_type == GIT_PKT_ACK) {
 216                                         break;
 217                                 } else if (pkt_type == GIT_PKT_NAK) {
 218                                         continue;
 219                                 } else {
 220                                         giterr_set(GITERR_NET, "Unexpected pkt type");
 221                                         goto on_error;
 222                                 }
 223                         }
 224                 }
 225
 226                 if (t->common.length > 0)
 227                         break;
 228
 229                 if (i % 20 == 0 && t->rpc) {
 230                         git_pkt_ack *pkt;
 231                         unsigned int i;
 232
 233                         if (git_pkt_buffer_wants(&remote->refs, &t->caps, &data) < 0)
 234                                 goto on_error;
 235
 236                         git_vector_foreach(&t->common, i, pkt) {
 237                                 git_pkt_buffer_have(&pkt->oid, &data);
 238                         }
 239
 240                         if (git_buf_oom(&data))
 241                                 goto on_error;
 242                 }
 243         }
 244
 245         if (error < 0 && error != GIT_ITEROVER)
 246                 goto on_error;
 247
 248         /* Tell the other end that we're done negotiating */
 249         if (t->rpc && t->common.length > 0) {
 250                 git_pkt_ack *pkt;
 251                 unsigned int i;
 252
 253                 if (git_pkt_buffer_wants(&remote->refs, &t->caps, &data) < 0)
 254                         goto on_error;
 255
 256                 git_vector_foreach(&t->common, i, pkt) {
 257                         git_pkt_buffer_have(&pkt->oid, &data);
 258                 }
 259
 260                 if (git_buf_oom(&data))
 261                         goto on_error;
 262         }
 263
 264         git_pkt_buffer_done(&data);
 265         if (t->cancel.val) {
 266                 giterr_set(GITERR_NET, "The fetch was cancelled by the user");
 267                 error = GIT_EUSER;
 268                 goto on_error;
 269         }
 270         if (t->negotiation_step(t, data.ptr, data.size) < 0)
 271                 goto on_error;
 272
 273         git_buf_free(&data);
 274         git_revwalk_free(walk);
 275
 276         /* Now let's eat up whatever the server gives us */
 277         if (!t->caps.multi_ack) {
 278                 pkt_type = recv_pkt(NULL, buf);
 279                 if (pkt_type != GIT_PKT_ACK && pkt_type != GIT_PKT_NAK) {
 280                         giterr_set(GITERR_NET, "Unexpected pkt type");
 281                         return -1;
 282                 }
 283         } else {
 284                 git_pkt_ack *pkt;
 285                 do {
 286                         if (recv_pkt((git_pkt **)&pkt, buf) < 0)
 287                                 return -1;
 288
 289                         if (pkt->type == GIT_PKT_NAK ||
 290                             (pkt->type == GIT_PKT_ACK && pkt->status != GIT_ACK_CONTINUE)) {
 291                                 git__free(pkt);
 292                                 break;
 293                         }
 294
 295                         git__free(pkt);
 296                 } while (1);
 297         }
 298
 299         return 0;
 300
 301 on_error:
 302         git_revwalk_free(walk);
 303         git_buf_free(&data);
 304         return error;
 305 }
 306
 307 int git_fetch_download_pack(
 308                 git_remote *remote,
 309                 git_transfer_progress_callback progress_cb,
 310                 void *progress_payload)
 311 {
 312         git_transport *t = remote->transport;
 313
 314         if(!remote->need_pack)
 315                 return 0;
 316
 317         if (t->own_logic)
 318                 return t->download_pack(t, remote->repo, &remote->stats);
 319
 320         return git_fetch__download_pack(t, remote->repo, &remote->stats,
 321                         progress_cb, progress_payload);
 322
 323 }
 324
 325 static int no_sideband(git_transport *t, git_indexer_stream *idx, gitno_buffer *buf, git_transfer_progress *stats)
 326 {
 327         int recvd;
 328
 329         do {
 330                 if (t->cancel.val) {
 331                         giterr_set(GITERR_NET, "The fetch was cancelled by the user");
 332                         return GIT_EUSER;
 333                 }
 334
 335                 if (git_indexer_stream_add(idx, buf->data, buf->offset, stats) < 0)
 336                         return -1;
 337
 338                 gitno_consume_n(buf, buf->offset);
 339
 340                 if ((recvd = gitno_recv(buf)) < 0)
 341                         return -1;
 342         } while(recvd > 0);
 343
 344         if (git_indexer_stream_finalize(idx, stats))
 345                 return -1;
 346
 347         return 0;
 348 }
 349
 350 struct network_packetsize_payload
 351 {
 352         git_transfer_progress_callback callback;
 353         void *payload;
 354         git_transfer_progress *stats;
 355         git_off_t last_fired_bytes;
 356 };
 357
 358 static void network_packetsize(int received, void *payload)
 359 {
 360         struct network_packetsize_payload *npp = (struct network_packetsize_payload*)payload;
 361
 362         /* Accumulate bytes */
 363         npp->stats->received_bytes += received;
 364
 365         /* Fire notification if the threshold is reached */
 366         if ((npp->stats->received_bytes - npp->last_fired_bytes) > NETWORK_XFER_THRESHOLD) {
 367                 npp->last_fired_bytes = npp->stats->received_bytes;
 368                 npp->callback(npp->stats, npp->payload);
 369         }
 370 }
 371
 372 /* Receiving data from a socket and storing it is pretty much the same for git and HTTP */
 373 int git_fetch__download_pack(
 374         git_transport *t,
 375         git_repository *repo,
 376         git_transfer_progress *stats,
 377         git_transfer_progress_callback progress_cb,
 378         void *progress_payload)
 379 {
 380         git_buf path = GIT_BUF_INIT;
 381         gitno_buffer *buf = &t->buffer;
 382         git_indexer_stream *idx = NULL;
 383         int error = -1;
 384         struct network_packetsize_payload npp = {0};
 385
 386         if (progress_cb) {
 387                 npp.callback = progress_cb;
 388                 npp.payload = progress_payload;
 389                 npp.stats = stats;
 390                 buf->packetsize_cb = &network_packetsize;
 391                 buf->packetsize_payload = &npp;
 392         }
 393
 394         if (git_buf_joinpath(&path, git_repository_path(repo), "objects/pack") < 0)
 395                 return -1;
 396
 397         if (git_indexer_stream_new(&idx, git_buf_cstr(&path), progress_cb, progress_payload) < 0)
 398                 goto on_error;
 399
 400         git_buf_free(&path);
 401         memset(stats, 0, sizeof(git_transfer_progress));
 402
 403         /*
 404          * If the remote doesn't support the side-band, we can feed
 405          * the data directly to the indexer. Otherwise, we need to
 406          * check which one belongs there.
 407          */
 408         if (!t->caps.side_band && !t->caps.side_band_64k) {
 409                 if (no_sideband(t, idx, buf, stats) < 0)
 410                         goto on_error;
 411
 412                 git_indexer_stream_free(idx);
 413                 return 0;
 414         }
 415
 416         do {
 417                 git_pkt *pkt;
 418
 419                 if (t->cancel.val) {
 420                         giterr_set(GITERR_NET, "The fetch was cancelled by the user");
 421                         error = GIT_EUSER;
 422                         goto on_error;
 423                 }
 424
 425                 if (recv_pkt(&pkt, buf) < 0)
 426                         goto on_error;
 427
 428                 if (pkt->type == GIT_PKT_PROGRESS) {
 429                         if (t->progress_cb) {
 430                                 git_pkt_progress *p = (git_pkt_progress *) pkt;
 431                                 t->progress_cb(p->data, p->len, t->cb_data);
 432                         }
 433                         git__free(pkt);
 434                 } else if (pkt->type == GIT_PKT_DATA) {
 435                         git_pkt_data *p = (git_pkt_data *) pkt;
 436                         if (git_indexer_stream_add(idx, p->data, p->len, stats) < 0)
 437                                 goto on_error;
 438
 439                         git__free(pkt);
 440                 } else if (pkt->type == GIT_PKT_FLUSH) {
 441                         /* A flush indicates the end of the packfile */
 442                         git__free(pkt);
 443                         break;
 444                 }
 445         } while (1);
 446
 447         if (git_indexer_stream_finalize(idx, stats) < 0)
 448                 goto on_error;
 449
 450         git_indexer_stream_free(idx);
 451         return 0;
 452
 453 on_error:
 454         git_buf_free(&path);
 455         git_indexer_stream_free(idx);
 456         return error;
 457 }
 458
 459 int git_fetch_setup_walk(git_revwalk **out, git_repository *repo)
 460 {
 461         git_revwalk *walk;
 462         git_strarray refs;
 463         unsigned int i;
 464         git_reference *ref;
 465
 466         if (git_reference_list(&refs, repo, GIT_REF_LISTALL) < 0)
 467                 return -1;
 468
 469         if (git_revwalk_new(&walk, repo) < 0)
 470                 return -1;
 471
 472         git_revwalk_sorting(walk, GIT_SORT_TIME);
 473
 474         for (i = 0; i < refs.count; ++i) {
 475                 /* No tags */
 476                 if (!git__prefixcmp(refs.strings[i], GIT_REFS_TAGS_DIR))
 477                         continue;
 478
 479                 if (git_reference_lookup(&ref, repo, refs.strings[i]) < 0)
 480                         goto on_error;
 481
 482                 if (git_reference_type(ref) == GIT_REF_SYMBOLIC)
 483                         continue;
 484                 if (git_revwalk_push(walk, git_reference_oid(ref)) < 0)
 485                         goto on_error;
 486
 487                 git_reference_free(ref);
 488         }
 489
 490         git_strarray_free(&refs);
 491         *out = walk;
 492         return 0;
 493
 494 on_error:
 495         git_reference_free(ref);
 496         git_strarray_free(&refs);
 497         return -1;
 498 }