src/transports/smart_protocol.c

   1 /*
   2  * Copyright (C) the libgit2 contributors. All rights reserved.
   3  *
   4  * This file is part of libgit2, distributed under the GNU GPL v2 with
   5  * a Linking Exception. For full terms see the included COPYING file.
   6  */
   7
   8 #include "common.h"
   9
  10 #include "git2.h"
  11 #include "git2/odb_backend.h"
  12
  13 #include "smart.h"
  14 #include "refs.h"
  15 #include "repository.h"
  16 #include "push.h"
  17 #include "pack-objects.h"
  18 #include "remote.h"
  19 #include "util.h"
  20 #include "revwalk.h"
  21
  22 #define NETWORK_XFER_THRESHOLD (100*1024)
  23 /* The minimal interval between progress updates (in seconds). */
  24 #define MIN_PROGRESS_UPDATE_INTERVAL 0.5
  25
  26 bool git_smart__ofs_delta_enabled = true;
  27
  28 int git_smart__store_refs(transport_smart *t, int flushes)
  29 {
  30         gitno_buffer *buf = &t->buffer;
  31         git_vector *refs = &t->refs;
  32         int error, flush = 0, recvd;
  33         const char *line_end = NULL;
  34         git_pkt *pkt = NULL;
  35         size_t i;
  36
  37         /* Clear existing refs in case git_remote_connect() is called again
  38          * after git_remote_disconnect().
  39          */
  40         git_vector_foreach(refs, i, pkt) {
  41                 git_pkt_free(pkt);
  42         }
  43         git_vector_clear(refs);
  44         pkt = NULL;
  45
  46         do {
  47                 if (buf->offset > 0)
  48                         error = git_pkt_parse_line(&pkt, &line_end, buf->data, buf->offset);
  49                 else
  50                         error = GIT_EBUFS;
  51
  52                 if (error < 0 && error != GIT_EBUFS)
  53                         return error;
  54
  55                 if (error == GIT_EBUFS) {
  56                         if ((recvd = gitno_recv(buf)) < 0)
  57                                 return recvd;
  58
  59                         if (recvd == 0) {
  60                                 git_error_set(GIT_ERROR_NET, "early EOF");
  61                                 return GIT_EEOF;
  62                         }
  63
  64                         continue;
  65                 }
  66
  67                 if (gitno_consume(buf, line_end) < 0)
  68                         return -1;
  69
  70                 if (pkt->type == GIT_PKT_ERR) {
  71                         git_error_set(GIT_ERROR_NET, "remote error: %s", ((git_pkt_err *)pkt)->error);
  72                         git__free(pkt);
  73                         return -1;
  74                 }
  75
  76                 if (pkt->type != GIT_PKT_FLUSH && git_vector_insert(refs, pkt) < 0)
  77                         return -1;
  78
  79                 if (pkt->type == GIT_PKT_FLUSH) {
  80                         flush++;
  81                         git_pkt_free(pkt);
  82                 }
  83         } while (flush < flushes);
  84
  85         return flush;
  86 }
  87
  88 static int append_symref(const char **out, git_vector *symrefs, const char *ptr)
  89 {
  90         int error;
  91         const char *end;
  92         git_str buf = GIT_STR_INIT;
  93         git_refspec *mapping = NULL;
  94
  95         ptr += strlen(GIT_CAP_SYMREF);
  96         if (*ptr != '=')
  97                 goto on_invalid;
  98
  99         ptr++;
 100         if (!(end = strchr(ptr, ' ')) &&
 101             !(end = strchr(ptr, '\0')))
 102                 goto on_invalid;
 103
 104         if ((error = git_str_put(&buf, ptr, end - ptr)) < 0)
 105                 return error;
 106
 107         /* symref mapping has refspec format */
 108         mapping = git__calloc(1, sizeof(git_refspec));
 109         GIT_ERROR_CHECK_ALLOC(mapping);
 110
 111         error = git_refspec__parse(mapping, git_str_cstr(&buf), true);
 112         git_str_dispose(&buf);
 113
 114         /* if the error isn't OOM, then it's a parse error; let's use a nicer message */
 115         if (error < 0) {
 116                 if (git_error_last()->klass != GIT_ERROR_NOMEMORY)
 117                         goto on_invalid;
 118
 119                 git__free(mapping);
 120                 return error;
 121         }
 122
 123         if ((error = git_vector_insert(symrefs, mapping)) < 0)
 124                 return error;
 125
 126         *out = end;
 127         return 0;
 128
 129 on_invalid:
 130         git_error_set(GIT_ERROR_NET, "remote sent invalid symref");
 131         git_refspec__dispose(mapping);
 132         git__free(mapping);
 133         return -1;
 134 }
 135
 136 int git_smart__detect_caps(git_pkt_ref *pkt, transport_smart_caps *caps, git_vector *symrefs)
 137 {
 138         const char *ptr;
 139
 140         /* No refs or capabilities, odd but not a problem */
 141         if (pkt == NULL || pkt->capabilities == NULL)
 142                 return GIT_ENOTFOUND;
 143
 144         ptr = pkt->capabilities;
 145         while (ptr != NULL && *ptr != '\0') {
 146                 if (*ptr == ' ')
 147                         ptr++;
 148
 149                 if (git_smart__ofs_delta_enabled && !git__prefixcmp(ptr, GIT_CAP_OFS_DELTA)) {
 150                         caps->common = caps->ofs_delta = 1;
 151                         ptr += strlen(GIT_CAP_OFS_DELTA);
 152                         continue;
 153                 }
 154
 155                 /* Keep multi_ack_detailed before multi_ack */
 156                 if (!git__prefixcmp(ptr, GIT_CAP_MULTI_ACK_DETAILED)) {
 157                         caps->common = caps->multi_ack_detailed = 1;
 158                         ptr += strlen(GIT_CAP_MULTI_ACK_DETAILED);
 159                         continue;
 160                 }
 161
 162                 if (!git__prefixcmp(ptr, GIT_CAP_MULTI_ACK)) {
 163                         caps->common = caps->multi_ack = 1;
 164                         ptr += strlen(GIT_CAP_MULTI_ACK);
 165                         continue;
 166                 }
 167
 168                 if (!git__prefixcmp(ptr, GIT_CAP_INCLUDE_TAG)) {
 169                         caps->common = caps->include_tag = 1;
 170                         ptr += strlen(GIT_CAP_INCLUDE_TAG);
 171                         continue;
 172                 }
 173
 174                 /* Keep side-band check after side-band-64k */
 175                 if (!git__prefixcmp(ptr, GIT_CAP_SIDE_BAND_64K)) {
 176                         caps->common = caps->side_band_64k = 1;
 177                         ptr += strlen(GIT_CAP_SIDE_BAND_64K);
 178                         continue;
 179                 }
 180
 181                 if (!git__prefixcmp(ptr, GIT_CAP_SIDE_BAND)) {
 182                         caps->common = caps->side_band = 1;
 183                         ptr += strlen(GIT_CAP_SIDE_BAND);
 184                         continue;
 185                 }
 186
 187                 if (!git__prefixcmp(ptr, GIT_CAP_DELETE_REFS)) {
 188                         caps->common = caps->delete_refs = 1;
 189                         ptr += strlen(GIT_CAP_DELETE_REFS);
 190                         continue;
 191                 }
 192
 193                 if (!git__prefixcmp(ptr, GIT_CAP_THIN_PACK)) {
 194                         caps->common = caps->thin_pack = 1;
 195                         ptr += strlen(GIT_CAP_THIN_PACK);
 196                         continue;
 197                 }
 198
 199                 if (!git__prefixcmp(ptr, GIT_CAP_SYMREF)) {
 200                         int error;
 201
 202                         if ((error = append_symref(&ptr, symrefs, ptr)) < 0)
 203                                 return error;
 204
 205                         continue;
 206                 }
 207
 208                 if (!git__prefixcmp(ptr, GIT_CAP_WANT_TIP_SHA1)) {
 209                         caps->common = caps->want_tip_sha1 = 1;
 210                         ptr += strlen(GIT_CAP_DELETE_REFS);
 211                         continue;
 212                 }
 213
 214                 if (!git__prefixcmp(ptr, GIT_CAP_WANT_REACHABLE_SHA1)) {
 215                         caps->common = caps->want_reachable_sha1 = 1;
 216                         ptr += strlen(GIT_CAP_DELETE_REFS);
 217                         continue;
 218                 }
 219
 220                 /* We don't know this capability, so skip it */
 221                 ptr = strchr(ptr, ' ');
 222         }
 223
 224         return 0;
 225 }
 226
 227 static int recv_pkt(git_pkt **out_pkt, git_pkt_type *out_type, gitno_buffer *buf)
 228 {
 229         const char *ptr = buf->data, *line_end = ptr;
 230         git_pkt *pkt = NULL;
 231         int error = 0, ret;
 232
 233         do {
 234                 if (buf->offset > 0)
 235                         error = git_pkt_parse_line(&pkt, &line_end, ptr, buf->offset);
 236                 else
 237                         error = GIT_EBUFS;
 238
 239                 if (error == 0)
 240                         break; /* return the pkt */
 241
 242                 if (error < 0 && error != GIT_EBUFS)
 243                         return error;
 244
 245                 if ((ret = gitno_recv(buf)) < 0) {
 246                         return ret;
 247                 } else if (ret == 0) {
 248                         git_error_set(GIT_ERROR_NET, "early EOF");
 249                         return GIT_EEOF;
 250                 }
 251         } while (error);
 252
 253         if (gitno_consume(buf, line_end) < 0)
 254                 return -1;
 255
 256         if (out_type != NULL)
 257                 *out_type = pkt->type;
 258         if (out_pkt != NULL)
 259                 *out_pkt = pkt;
 260         else
 261                 git__free(pkt);
 262
 263         return error;
 264 }
 265
 266 static int store_common(transport_smart *t)
 267 {
 268         git_pkt *pkt = NULL;
 269         gitno_buffer *buf = &t->buffer;
 270         int error;
 271
 272         do {
 273                 if ((error = recv_pkt(&pkt, NULL, buf)) < 0)
 274                         return error;
 275
 276                 if (pkt->type != GIT_PKT_ACK) {
 277                         git__free(pkt);
 278                         return 0;
 279                 }
 280
 281                 if (git_vector_insert(&t->common, pkt) < 0) {
 282                         git__free(pkt);
 283                         return -1;
 284                 }
 285         } while (1);
 286
 287         return 0;
 288 }
 289
 290 static int wait_while_ack(gitno_buffer *buf)
 291 {
 292         int error;
 293         git_pkt *pkt = NULL;
 294         git_pkt_ack *ack = NULL;
 295
 296         while (1) {
 297                 git_pkt_free(pkt);
 298
 299                 if ((error = recv_pkt(&pkt, NULL, buf)) < 0)
 300                         return error;
 301
 302                 if (pkt->type == GIT_PKT_NAK)
 303                         break;
 304                 if (pkt->type != GIT_PKT_ACK)
 305                         continue;
 306
 307                 ack = (git_pkt_ack*)pkt;
 308
 309                 if (ack->status != GIT_ACK_CONTINUE &&
 310                     ack->status != GIT_ACK_COMMON &&
 311                     ack->status != GIT_ACK_READY) {
 312                         break;
 313                 }
 314         }
 315
 316         git_pkt_free(pkt);
 317         return 0;
 318 }
 319
 320 int git_smart__negotiate_fetch(git_transport *transport, git_repository *repo, const git_remote_head * const *wants, size_t count)
 321 {
 322         transport_smart *t = (transport_smart *)transport;
 323         git_revwalk__push_options opts = GIT_REVWALK__PUSH_OPTIONS_INIT;
 324         gitno_buffer *buf = &t->buffer;
 325         git_str data = GIT_STR_INIT;
 326         git_revwalk *walk = NULL;
 327         int error = -1;
 328         git_pkt_type pkt_type;
 329         unsigned int i;
 330         git_oid oid;
 331
 332         if ((error = git_pkt_buffer_wants(wants, count, &t->caps, &data)) < 0)
 333                 return error;
 334
 335         if ((error = git_revwalk_new(&walk, repo)) < 0)
 336                 goto on_error;
 337
 338         opts.insert_by_date = 1;
 339         if ((error = git_revwalk__push_glob(walk, "refs/*", &opts)) < 0)
 340                 goto on_error;
 341
 342         /*
 343          * Our support for ACK extensions is simply to parse them. On
 344          * the first ACK we will accept that as enough common
 345          * objects. We give up if we haven't found an answer in the
 346          * first 256 we send.
 347          */
 348         i = 0;
 349         while (i < 256) {
 350                 error = git_revwalk_next(&oid, walk);
 351
 352                 if (error < 0) {
 353                         if (GIT_ITEROVER == error)
 354                                 break;
 355
 356                         goto on_error;
 357                 }
 358
 359                 git_pkt_buffer_have(&oid, &data);
 360                 i++;
 361                 if (i % 20 == 0) {
 362                         if (t->cancelled.val) {
 363                                 git_error_set(GIT_ERROR_NET, "The fetch was cancelled by the user");
 364                                 error = GIT_EUSER;
 365                                 goto on_error;
 366                         }
 367
 368                         git_pkt_buffer_flush(&data);
 369                         if (git_str_oom(&data)) {
 370                                 error = -1;
 371                                 goto on_error;
 372                         }
 373
 374                         if ((error = git_smart__negotiation_step(&t->parent, data.ptr, data.size)) < 0)
 375                                 goto on_error;
 376
 377                         git_str_clear(&data);
 378                         if (t->caps.multi_ack || t->caps.multi_ack_detailed) {
 379                                 if ((error = store_common(t)) < 0)
 380                                         goto on_error;
 381                         } else {
 382                                 if ((error = recv_pkt(NULL, &pkt_type, buf)) < 0)
 383                                         goto on_error;
 384
 385                                 if (pkt_type == GIT_PKT_ACK) {
 386                                         break;
 387                                 } else if (pkt_type == GIT_PKT_NAK) {
 388                                         continue;
 389                                 } else {
 390                                         git_error_set(GIT_ERROR_NET, "unexpected pkt type");
 391                                         error = -1;
 392                                         goto on_error;
 393                                 }
 394                         }
 395                 }
 396
 397                 if (t->common.length > 0)
 398                         break;
 399
 400                 if (i % 20 == 0 && t->rpc) {
 401                         git_pkt_ack *pkt;
 402                         unsigned int j;
 403
 404                         if ((error = git_pkt_buffer_wants(wants, count, &t->caps, &data)) < 0)
 405                                 goto on_error;
 406
 407                         git_vector_foreach(&t->common, j, pkt) {
 408                                 if ((error = git_pkt_buffer_have(&pkt->oid, &data)) < 0)
 409                                         goto on_error;
 410                         }
 411
 412                         if (git_str_oom(&data)) {
 413                                 error = -1;
 414                                 goto on_error;
 415                         }
 416                 }
 417         }
 418
 419         /* Tell the other end that we're done negotiating */
 420         if (t->rpc && t->common.length > 0) {
 421                 git_pkt_ack *pkt;
 422                 unsigned int j;
 423
 424                 if ((error = git_pkt_buffer_wants(wants, count, &t->caps, &data)) < 0)
 425                         goto on_error;
 426
 427                 git_vector_foreach(&t->common, j, pkt) {
 428                         if ((error = git_pkt_buffer_have(&pkt->oid, &data)) < 0)
 429                                 goto on_error;
 430                 }
 431
 432                 if (git_str_oom(&data)) {
 433                         error = -1;
 434                         goto on_error;
 435                 }
 436         }
 437
 438         if ((error = git_pkt_buffer_done(&data)) < 0)
 439                 goto on_error;
 440
 441         if (t->cancelled.val) {
 442                 git_error_set(GIT_ERROR_NET, "The fetch was cancelled by the user");
 443                 error = GIT_EUSER;
 444                 goto on_error;
 445         }
 446         if ((error = git_smart__negotiation_step(&t->parent, data.ptr, data.size)) < 0)
 447                 goto on_error;
 448
 449         git_str_dispose(&data);
 450         git_revwalk_free(walk);
 451
 452         /* Now let's eat up whatever the server gives us */
 453         if (!t->caps.multi_ack && !t->caps.multi_ack_detailed) {
 454                 if ((error = recv_pkt(NULL, &pkt_type, buf)) < 0)
 455                         return error;
 456
 457                 if (pkt_type != GIT_PKT_ACK && pkt_type != GIT_PKT_NAK) {
 458                         git_error_set(GIT_ERROR_NET, "unexpected pkt type");
 459                         return -1;
 460                 }
 461         } else {
 462                 error = wait_while_ack(buf);
 463         }
 464
 465         return error;
 466
 467 on_error:
 468         git_revwalk_free(walk);
 469         git_str_dispose(&data);
 470         return error;
 471 }
 472
 473 static int no_sideband(transport_smart *t, struct git_odb_writepack *writepack, gitno_buffer *buf, git_indexer_progress *stats)
 474 {
 475         int recvd;
 476
 477         do {
 478                 if (t->cancelled.val) {
 479                         git_error_set(GIT_ERROR_NET, "the fetch was cancelled by the user");
 480                         return GIT_EUSER;
 481                 }
 482
 483                 if (writepack->append(writepack, buf->data, buf->offset, stats) < 0)
 484                         return -1;
 485
 486                 gitno_consume_n(buf, buf->offset);
 487
 488                 if ((recvd = gitno_recv(buf)) < 0)
 489                         return recvd;
 490         } while(recvd > 0);
 491
 492         if (writepack->commit(writepack, stats) < 0)
 493                 return -1;
 494
 495         return 0;
 496 }
 497
 498 struct network_packetsize_payload
 499 {
 500         git_indexer_progress_cb callback;
 501         void *payload;
 502         git_indexer_progress *stats;
 503         size_t last_fired_bytes;
 504 };
 505
 506 static int network_packetsize(size_t received, void *payload)
 507 {
 508         struct network_packetsize_payload *npp = (struct network_packetsize_payload*)payload;
 509
 510         /* Accumulate bytes */
 511         npp->stats->received_bytes += received;
 512
 513         /* Fire notification if the threshold is reached */
 514         if ((npp->stats->received_bytes - npp->last_fired_bytes) > NETWORK_XFER_THRESHOLD) {
 515                 npp->last_fired_bytes = npp->stats->received_bytes;
 516
 517                 if (npp->callback(npp->stats, npp->payload))
 518                         return GIT_EUSER;
 519         }
 520
 521         return 0;
 522 }
 523
 524 int git_smart__download_pack(
 525         git_transport *transport,
 526         git_repository *repo,
 527         git_indexer_progress *stats)
 528 {
 529         transport_smart *t = (transport_smart *)transport;
 530         gitno_buffer *buf = &t->buffer;
 531         git_odb *odb;
 532         struct git_odb_writepack *writepack = NULL;
 533         int error = 0;
 534         struct network_packetsize_payload npp = {0};
 535
 536         git_indexer_progress_cb progress_cb = t->connect_opts.callbacks.transfer_progress;
 537         void *progress_payload = t->connect_opts.callbacks.payload;
 538
 539         memset(stats, 0, sizeof(git_indexer_progress));
 540
 541         if (progress_cb) {
 542                 npp.callback = progress_cb;
 543                 npp.payload = progress_payload;
 544                 npp.stats = stats;
 545                 t->packetsize_cb = &network_packetsize;
 546                 t->packetsize_payload = &npp;
 547
 548                 /* We might have something in the buffer already from negotiate_fetch */
 549                 if (t->buffer.offset > 0 && !t->cancelled.val)
 550                         if (t->packetsize_cb(t->buffer.offset, t->packetsize_payload))
 551                                 git_atomic32_set(&t->cancelled, 1);
 552         }
 553
 554         if ((error = git_repository_odb__weakptr(&odb, repo)) < 0 ||
 555                 ((error = git_odb_write_pack(&writepack, odb, progress_cb, progress_payload)) != 0))
 556                 goto done;
 557
 558         /*
 559          * If the remote doesn't support the side-band, we can feed
 560          * the data directly to the pack writer. Otherwise, we need to
 561          * check which one belongs there.
 562          */
 563         if (!t->caps.side_band && !t->caps.side_band_64k) {
 564                 error = no_sideband(t, writepack, buf, stats);
 565                 goto done;
 566         }
 567
 568         do {
 569                 git_pkt *pkt = NULL;
 570
 571                 /* Check cancellation before network call */
 572                 if (t->cancelled.val) {
 573                         git_error_clear();
 574                         error = GIT_EUSER;
 575                         goto done;
 576                 }
 577
 578                 if ((error = recv_pkt(&pkt, NULL, buf)) >= 0) {
 579                         /* Check cancellation after network call */
 580                         if (t->cancelled.val) {
 581                                 git_error_clear();
 582                                 error = GIT_EUSER;
 583                         } else if (pkt->type == GIT_PKT_PROGRESS) {
 584                                 if (t->connect_opts.callbacks.sideband_progress) {
 585                                         git_pkt_progress *p = (git_pkt_progress *) pkt;
 586
 587                                         if (p->len > INT_MAX) {
 588                                                 git_error_set(GIT_ERROR_NET, "oversized progress message");
 589                                                 error = GIT_ERROR;
 590                                                 goto done;
 591                                         }
 592
 593                                         error = t->connect_opts.callbacks.sideband_progress(p->data, (int)p->len, t->connect_opts.callbacks.payload);
 594                                 }
 595                         } else if (pkt->type == GIT_PKT_DATA) {
 596                                 git_pkt_data *p = (git_pkt_data *) pkt;
 597
 598                                 if (p->len)
 599                                         error = writepack->append(writepack, p->data, p->len, stats);
 600                         } else if (pkt->type == GIT_PKT_FLUSH) {
 601                                 /* A flush indicates the end of the packfile */
 602                                 git__free(pkt);
 603                                 break;
 604                         }
 605                 }
 606
 607                 git_pkt_free(pkt);
 608
 609                 if (error < 0)
 610                         goto done;
 611
 612         } while (1);
 613
 614         /*
 615          * Trailing execution of progress_cb, if necessary...
 616          * Only the callback through the npp datastructure currently
 617          * updates the last_fired_bytes value. It is possible that
 618          * progress has already been reported with the correct
 619          * "received_bytes" value, but until (if?) this is unified
 620          * then we will report progress again to be sure that the
 621          * correct last received_bytes value is reported.
 622          */
 623         if (npp.callback && npp.stats->received_bytes > npp.last_fired_bytes) {
 624                 error = npp.callback(npp.stats, npp.payload);
 625                 if (error != 0)
 626                         goto done;
 627         }
 628
 629         error = writepack->commit(writepack, stats);
 630
 631 done:
 632         if (writepack)
 633                 writepack->free(writepack);
 634         if (progress_cb) {
 635                 t->packetsize_cb = NULL;
 636                 t->packetsize_payload = NULL;
 637         }
 638
 639         return error;
 640 }
 641
 642 static int gen_pktline(git_str *buf, git_push *push)
 643 {
 644         push_spec *spec;
 645         size_t i, len;
 646         char old_id[GIT_OID_HEXSZ+1], new_id[GIT_OID_HEXSZ+1];
 647
 648         old_id[GIT_OID_HEXSZ] = '\0'; new_id[GIT_OID_HEXSZ] = '\0';
 649
 650         git_vector_foreach(&push->specs, i, spec) {
 651                 len = 2*GIT_OID_HEXSZ + 7 + strlen(spec->refspec.dst);
 652
 653                 if (i == 0) {
 654                         ++len; /* '\0' */
 655                         if (push->report_status)
 656                                 len += strlen(GIT_CAP_REPORT_STATUS) + 1;
 657                         len += strlen(GIT_CAP_SIDE_BAND_64K) + 1;
 658                 }
 659
 660                 git_oid_fmt(old_id, &spec->roid);
 661                 git_oid_fmt(new_id, &spec->loid);
 662
 663                 git_str_printf(buf, "%04"PRIxZ"%s %s %s", len, old_id, new_id, spec->refspec.dst);
 664
 665                 if (i == 0) {
 666                         git_str_putc(buf, '\0');
 667                         /* Core git always starts their capabilities string with a space */
 668                         if (push->report_status) {
 669                                 git_str_putc(buf, ' ');
 670                                 git_str_printf(buf, GIT_CAP_REPORT_STATUS);
 671                         }
 672                         git_str_putc(buf, ' ');
 673                         git_str_printf(buf, GIT_CAP_SIDE_BAND_64K);
 674                 }
 675
 676                 git_str_putc(buf, '\n');
 677         }
 678
 679         git_str_puts(buf, "0000");
 680         return git_str_oom(buf) ? -1 : 0;
 681 }
 682
 683 static int add_push_report_pkt(git_push *push, git_pkt *pkt)
 684 {
 685         push_status *status;
 686
 687         switch (pkt->type) {
 688                 case GIT_PKT_OK:
 689                         status = git__calloc(1, sizeof(push_status));
 690                         GIT_ERROR_CHECK_ALLOC(status);
 691                         status->msg = NULL;
 692                         status->ref = git__strdup(((git_pkt_ok *)pkt)->ref);
 693                         if (!status->ref ||
 694                                 git_vector_insert(&push->status, status) < 0) {
 695                                 git_push_status_free(status);
 696                                 return -1;
 697                         }
 698                         break;
 699                 case GIT_PKT_NG:
 700                         status = git__calloc(1, sizeof(push_status));
 701                         GIT_ERROR_CHECK_ALLOC(status);
 702                         status->ref = git__strdup(((git_pkt_ng *)pkt)->ref);
 703                         status->msg = git__strdup(((git_pkt_ng *)pkt)->msg);
 704                         if (!status->ref || !status->msg ||
 705                                 git_vector_insert(&push->status, status) < 0) {
 706                                 git_push_status_free(status);
 707                                 return -1;
 708                         }
 709                         break;
 710                 case GIT_PKT_UNPACK:
 711                         push->unpack_ok = ((git_pkt_unpack *)pkt)->unpack_ok;
 712                         break;
 713                 case GIT_PKT_FLUSH:
 714                         return GIT_ITEROVER;
 715                 default:
 716                         git_error_set(GIT_ERROR_NET, "report-status: protocol error");
 717                         return -1;
 718         }
 719
 720         return 0;
 721 }
 722
 723 static int add_push_report_sideband_pkt(git_push *push, git_pkt_data *data_pkt, git_str *data_pkt_buf)
 724 {
 725         git_pkt *pkt;
 726         const char *line, *line_end = NULL;
 727         size_t line_len;
 728         int error;
 729         int reading_from_buf = data_pkt_buf->size > 0;
 730
 731         if (reading_from_buf) {
 732                 /* We had an existing partial packet, so add the new
 733                  * packet to the buffer and parse the whole thing */
 734                 git_str_put(data_pkt_buf, data_pkt->data, data_pkt->len);
 735                 line = data_pkt_buf->ptr;
 736                 line_len = data_pkt_buf->size;
 737         }
 738         else {
 739                 line = data_pkt->data;
 740                 line_len = data_pkt->len;
 741         }
 742
 743         while (line_len > 0) {
 744                 error = git_pkt_parse_line(&pkt, &line_end, line, line_len);
 745
 746                 if (error == GIT_EBUFS) {
 747                         /* Buffer the data when the inner packet is split
 748                          * across multiple sideband packets */
 749                         if (!reading_from_buf)
 750                                 git_str_put(data_pkt_buf, line, line_len);
 751                         error = 0;
 752                         goto done;
 753                 }
 754                 else if (error < 0)
 755                         goto done;
 756
 757                 /* Advance in the buffer */
 758                 line_len -= (line_end - line);
 759                 line = line_end;
 760
 761                 error = add_push_report_pkt(push, pkt);
 762
 763                 git_pkt_free(pkt);
 764
 765                 if (error < 0 && error != GIT_ITEROVER)
 766                         goto done;
 767         }
 768
 769         error = 0;
 770
 771 done:
 772         if (reading_from_buf)
 773                 git_str_consume(data_pkt_buf, line_end);
 774         return error;
 775 }
 776
 777 static int parse_report(transport_smart *transport, git_push *push)
 778 {
 779         git_pkt *pkt = NULL;
 780         const char *line_end = NULL;
 781         gitno_buffer *buf = &transport->buffer;
 782         int error, recvd;
 783         git_str data_pkt_buf = GIT_STR_INIT;
 784
 785         for (;;) {
 786                 if (buf->offset > 0)
 787                         error = git_pkt_parse_line(&pkt, &line_end,
 788                                                    buf->data, buf->offset);
 789                 else
 790                         error = GIT_EBUFS;
 791
 792                 if (error < 0 && error != GIT_EBUFS) {
 793                         error = -1;
 794                         goto done;
 795                 }
 796
 797                 if (error == GIT_EBUFS) {
 798                         if ((recvd = gitno_recv(buf)) < 0) {
 799                                 error = recvd;
 800                                 goto done;
 801                         }
 802
 803                         if (recvd == 0) {
 804                                 git_error_set(GIT_ERROR_NET, "early EOF");
 805                                 error = GIT_EEOF;
 806                                 goto done;
 807                         }
 808                         continue;
 809                 }
 810
 811                 if (gitno_consume(buf, line_end) < 0)
 812                         return -1;
 813
 814                 error = 0;
 815
 816                 switch (pkt->type) {
 817                         case GIT_PKT_DATA:
 818                                 /* This is a sideband packet which contains other packets */
 819                                 error = add_push_report_sideband_pkt(push, (git_pkt_data *)pkt, &data_pkt_buf);
 820                                 break;
 821                         case GIT_PKT_ERR:
 822                                 git_error_set(GIT_ERROR_NET, "report-status: Error reported: %s",
 823                                         ((git_pkt_err *)pkt)->error);
 824                                 error = -1;
 825                                 break;
 826                         case GIT_PKT_PROGRESS:
 827                                 if (transport->connect_opts.callbacks.sideband_progress) {
 828                                         git_pkt_progress *p = (git_pkt_progress *) pkt;
 829
 830                                         if (p->len > INT_MAX) {
 831                                                 git_error_set(GIT_ERROR_NET, "oversized progress message");
 832                                                 error = GIT_ERROR;
 833                                                 goto done;
 834                                         }
 835
 836                                         error = transport->connect_opts.callbacks.sideband_progress(p->data, (int)p->len, transport->connect_opts.callbacks.payload);
 837                                 }
 838                                 break;
 839                         default:
 840                                 error = add_push_report_pkt(push, pkt);
 841                                 break;
 842                 }
 843
 844                 git_pkt_free(pkt);
 845
 846                 /* add_push_report_pkt returns GIT_ITEROVER when it receives a flush */
 847                 if (error == GIT_ITEROVER) {
 848                         error = 0;
 849                         if (data_pkt_buf.size > 0) {
 850                                 /* If there was data remaining in the pack data buffer,
 851                                  * then the server sent a partial pkt-line */
 852                                 git_error_set(GIT_ERROR_NET, "incomplete pack data pkt-line");
 853                                 error = GIT_ERROR;
 854                         }
 855                         goto done;
 856                 }
 857
 858                 if (error < 0) {
 859                         goto done;
 860                 }
 861         }
 862 done:
 863         git_str_dispose(&data_pkt_buf);
 864         return error;
 865 }
 866
 867 static int add_ref_from_push_spec(git_vector *refs, push_spec *push_spec)
 868 {
 869         git_pkt_ref *added = git__calloc(1, sizeof(git_pkt_ref));
 870         GIT_ERROR_CHECK_ALLOC(added);
 871
 872         added->type = GIT_PKT_REF;
 873         git_oid_cpy(&added->head.oid, &push_spec->loid);
 874         added->head.name = git__strdup(push_spec->refspec.dst);
 875
 876         if (!added->head.name ||
 877                 git_vector_insert(refs, added) < 0) {
 878                 git_pkt_free((git_pkt *)added);
 879                 return -1;
 880         }
 881
 882         return 0;
 883 }
 884
 885 static int update_refs_from_report(
 886         git_vector *refs,
 887         git_vector *push_specs,
 888         git_vector *push_report)
 889 {
 890         git_pkt_ref *ref;
 891         push_spec *push_spec;
 892         push_status *push_status;
 893         size_t i, j, refs_len;
 894         int cmp;
 895
 896         /* For each push spec we sent to the server, we should have
 897          * gotten back a status packet in the push report */
 898         if (push_specs->length != push_report->length) {
 899                 git_error_set(GIT_ERROR_NET, "report-status: protocol error");
 900                 return -1;
 901         }
 902
 903         /* We require that push_specs be sorted with push_spec_rref_cmp,
 904          * and that push_report be sorted with push_status_ref_cmp */
 905         git_vector_sort(push_specs);
 906         git_vector_sort(push_report);
 907
 908         git_vector_foreach(push_specs, i, push_spec) {
 909                 push_status = git_vector_get(push_report, i);
 910
 911                 /* For each push spec we sent to the server, we should have
 912                  * gotten back a status packet in the push report which matches */
 913                 if (strcmp(push_spec->refspec.dst, push_status->ref)) {
 914                         git_error_set(GIT_ERROR_NET, "report-status: protocol error");
 915                         return -1;
 916                 }
 917         }
 918
 919         /* We require that refs be sorted with ref_name_cmp */
 920         git_vector_sort(refs);
 921         i = j = 0;
 922         refs_len = refs->length;
 923
 924         /* Merge join push_specs with refs */
 925         while (i < push_specs->length && j < refs_len) {
 926                 push_spec = git_vector_get(push_specs, i);
 927                 push_status = git_vector_get(push_report, i);
 928                 ref = git_vector_get(refs, j);
 929
 930                 cmp = strcmp(push_spec->refspec.dst, ref->head.name);
 931
 932                 /* Iterate appropriately */
 933                 if (cmp <= 0) i++;
 934                 if (cmp >= 0) j++;
 935
 936                 /* Add case */
 937                 if (cmp < 0 &&
 938                         !push_status->msg &&
 939                         add_ref_from_push_spec(refs, push_spec) < 0)
 940                         return -1;
 941
 942                 /* Update case, delete case */
 943                 if (cmp == 0 &&
 944                         !push_status->msg)
 945                         git_oid_cpy(&ref->head.oid, &push_spec->loid);
 946         }
 947
 948         for (; i < push_specs->length; i++) {
 949                 push_spec = git_vector_get(push_specs, i);
 950                 push_status = git_vector_get(push_report, i);
 951
 952                 /* Add case */
 953                 if (!push_status->msg &&
 954                         add_ref_from_push_spec(refs, push_spec) < 0)
 955                         return -1;
 956         }
 957
 958         /* Remove any refs which we updated to have a zero OID. */
 959         git_vector_rforeach(refs, i, ref) {
 960                 if (git_oid_is_zero(&ref->head.oid)) {
 961                         git_vector_remove(refs, i);
 962                         git_pkt_free((git_pkt *)ref);
 963                 }
 964         }
 965
 966         git_vector_sort(refs);
 967
 968         return 0;
 969 }
 970
 971 struct push_packbuilder_payload
 972 {
 973         git_smart_subtransport_stream *stream;
 974         git_packbuilder *pb;
 975         git_push_transfer_progress_cb cb;
 976         void *cb_payload;
 977         size_t last_bytes;
 978         double last_progress_report_time;
 979 };
 980
 981 static int stream_thunk(void *buf, size_t size, void *data)
 982 {
 983         int error = 0;
 984         struct push_packbuilder_payload *payload = data;
 985
 986         if ((error = payload->stream->write(payload->stream, (const char *)buf, size)) < 0)
 987                 return error;
 988
 989         if (payload->cb) {
 990                 double current_time = git__timer();
 991                 double elapsed = current_time - payload->last_progress_report_time;
 992                 payload->last_bytes += size;
 993
 994                 if (elapsed < 0 || elapsed >= MIN_PROGRESS_UPDATE_INTERVAL) {
 995                         payload->last_progress_report_time = current_time;
 996                         error = payload->cb(payload->pb->nr_written, payload->pb->nr_objects, payload->last_bytes, payload->cb_payload);
 997                 }
 998         }
 999
1000         return error;
1001 }
1002
1003 int git_smart__push(git_transport *transport, git_push *push)
1004 {
1005         transport_smart *t = (transport_smart *)transport;
1006         git_remote_callbacks *cbs = &t->connect_opts.callbacks;
1007         struct push_packbuilder_payload packbuilder_payload = {0};
1008         git_str pktline = GIT_STR_INIT;
1009         int error = 0, need_pack = 0;
1010         push_spec *spec;
1011         unsigned int i;
1012
1013         packbuilder_payload.pb = push->pb;
1014
1015         if (cbs && cbs->push_transfer_progress) {
1016                 packbuilder_payload.cb = cbs->push_transfer_progress;
1017                 packbuilder_payload.cb_payload = cbs->payload;
1018         }
1019
1020 #ifdef PUSH_DEBUG
1021 {
1022         git_remote_head *head;
1023         char hex[GIT_OID_HEXSZ+1]; hex[GIT_OID_HEXSZ] = '\0';
1024
1025         git_vector_foreach(&push->remote->refs, i, head) {
1026                 git_oid_fmt(hex, &head->oid);
1027                 fprintf(stderr, "%s (%s)\n", hex, head->name);
1028         }
1029
1030         git_vector_foreach(&push->specs, i, spec) {
1031                 git_oid_fmt(hex, &spec->roid);
1032                 fprintf(stderr, "%s (%s) -> ", hex, spec->lref);
1033                 git_oid_fmt(hex, &spec->loid);
1034                 fprintf(stderr, "%s (%s)\n", hex, spec->rref ?
1035                         spec->rref : spec->lref);
1036         }
1037 }
1038 #endif
1039
1040         /*
1041          * Figure out if we need to send a packfile; which is in all
1042          * cases except when we only send delete commands
1043          */
1044         git_vector_foreach(&push->specs, i, spec) {
1045                 if (spec->refspec.src && spec->refspec.src[0] != '\0') {
1046                         need_pack = 1;
1047                         break;
1048                 }
1049         }
1050
1051         /* prepare pack before sending pack header to avoid timeouts */
1052         if (need_pack && ((error = git_packbuilder__prepare(push->pb))) < 0)
1053                 goto done;
1054
1055         if ((error = git_smart__get_push_stream(t, &packbuilder_payload.stream)) < 0 ||
1056                 (error = gen_pktline(&pktline, push)) < 0 ||
1057                 (error = packbuilder_payload.stream->write(packbuilder_payload.stream, git_str_cstr(&pktline), git_str_len(&pktline))) < 0)
1058                 goto done;
1059
1060         if (need_pack &&
1061                 (error = git_packbuilder_foreach(push->pb, &stream_thunk, &packbuilder_payload)) < 0)
1062                 goto done;
1063
1064         /* If we sent nothing or the server doesn't support report-status, then
1065          * we consider the pack to have been unpacked successfully */
1066         if (!push->specs.length || !push->report_status)
1067                 push->unpack_ok = 1;
1068         else if ((error = parse_report(t, push)) < 0)
1069                 goto done;
1070
1071         /* If progress is being reported write the final report */
1072         if (cbs && cbs->push_transfer_progress) {
1073                 error = cbs->push_transfer_progress(
1074                                         push->pb->nr_written,
1075                                         push->pb->nr_objects,
1076                                         packbuilder_payload.last_bytes,
1077                                         cbs->payload);
1078
1079                 if (error < 0)
1080                         goto done;
1081         }
1082
1083         if (push->status.length) {
1084                 error = update_refs_from_report(&t->refs, &push->specs, &push->status);
1085                 if (error < 0)
1086                         goto done;
1087
1088                 error = git_smart__update_heads(t, NULL);
1089         }
1090
1091 done:
1092         git_str_dispose(&pktline);
1093         return error;
1094 }