src/transports/smart_protocol.c

   1 /*
   2  * Copyright (C) the libgit2 contributors. All rights reserved.
   3  *
   4  * This file is part of libgit2, distributed under the GNU GPL v2 with
   5  * a Linking Exception. For full terms see the included COPYING file.
   6  */
   7
   8 #include "common.h"
   9
  10 #include "git2.h"
  11 #include "git2/odb_backend.h"
  12
  13 #include "smart.h"
  14 #include "refs.h"
  15 #include "repository.h"
  16 #include "push.h"
  17 #include "pack-objects.h"
  18 #include "remote.h"
  19 #include "util.h"
  20 #include "revwalk.h"
  21
  22 #define NETWORK_XFER_THRESHOLD (100*1024)
  23 /* The minimal interval between progress updates (in seconds). */
  24 #define MIN_PROGRESS_UPDATE_INTERVAL 0.5
  25
  26 bool git_smart__ofs_delta_enabled = true;
  27
  28 int git_smart__store_refs(transport_smart *t, int flushes)
  29 {
  30         gitno_buffer *buf = &t->buffer;
  31         git_vector *refs = &t->refs;
  32         int error, flush = 0, recvd;
  33         const char *line_end = NULL;
  34         git_pkt *pkt = NULL;
  35         size_t i;
  36
  37         /* Clear existing refs in case git_remote_connect() is called again
  38          * after git_remote_disconnect().
  39          */
  40         git_vector_foreach(refs, i, pkt) {
  41                 git_pkt_free(pkt);
  42         }
  43         git_vector_clear(refs);
  44         pkt = NULL;
  45
  46         do {
  47                 if (buf->offset > 0)
  48                         error = git_pkt_parse_line(&pkt, &line_end, buf->data, buf->offset);
  49                 else
  50                         error = GIT_EBUFS;
  51
  52                 if (error < 0 && error != GIT_EBUFS)
  53                         return error;
  54
  55                 if (error == GIT_EBUFS) {
  56                         if ((recvd = gitno_recv(buf)) < 0)
  57                                 return recvd;
  58
  59                         if (recvd == 0) {
  60                                 git_error_set(GIT_ERROR_NET, "early EOF");
  61                                 return GIT_EEOF;
  62                         }
  63
  64                         continue;
  65                 }
  66
  67                 if (gitno_consume(buf, line_end) < 0)
  68                         return -1;
  69
  70                 if (pkt->type == GIT_PKT_ERR) {
  71                         git_error_set(GIT_ERROR_NET, "remote error: %s", ((git_pkt_err *)pkt)->error);
  72                         git__free(pkt);
  73                         return -1;
  74                 }
  75
  76                 if (pkt->type != GIT_PKT_FLUSH && git_vector_insert(refs, pkt) < 0)
  77                         return -1;
  78
  79                 if (pkt->type == GIT_PKT_FLUSH) {
  80                         flush++;
  81                         git_pkt_free(pkt);
  82                 }
  83         } while (flush < flushes);
  84
  85         return flush;
  86 }
  87
  88 static int append_symref(const char **out, git_vector *symrefs, const char *ptr)
  89 {
  90         int error;
  91         const char *end;
  92         git_buf buf = GIT_BUF_INIT;
  93         git_refspec *mapping = NULL;
  94
  95         ptr += strlen(GIT_CAP_SYMREF);
  96         if (*ptr != '=')
  97                 goto on_invalid;
  98
  99         ptr++;
 100         if (!(end = strchr(ptr, ' ')) &&
 101             !(end = strchr(ptr, '\0')))
 102                 goto on_invalid;
 103
 104         if ((error = git_buf_put(&buf, ptr, end - ptr)) < 0)
 105                 return error;
 106
 107         /* symref mapping has refspec format */
 108         mapping = git__calloc(1, sizeof(git_refspec));
 109         GIT_ERROR_CHECK_ALLOC(mapping);
 110
 111         error = git_refspec__parse(mapping, git_buf_cstr(&buf), true);
 112         git_buf_dispose(&buf);
 113
 114         /* if the error isn't OOM, then it's a parse error; let's use a nicer message */
 115         if (error < 0) {
 116                 if (git_error_last()->klass != GIT_ERROR_NOMEMORY)
 117                         goto on_invalid;
 118
 119                 git__free(mapping);
 120                 return error;
 121         }
 122
 123         if ((error = git_vector_insert(symrefs, mapping)) < 0)
 124                 return error;
 125
 126         *out = end;
 127         return 0;
 128
 129 on_invalid:
 130         git_error_set(GIT_ERROR_NET, "remote sent invalid symref");
 131         git_refspec__dispose(mapping);
 132         git__free(mapping);
 133         return -1;
 134 }
 135
 136 int git_smart__detect_caps(git_pkt_ref *pkt, transport_smart_caps *caps, git_vector *symrefs)
 137 {
 138         const char *ptr;
 139
 140         /* No refs or capabilites, odd but not a problem */
 141         if (pkt == NULL || pkt->capabilities == NULL)
 142                 return GIT_ENOTFOUND;
 143
 144         ptr = pkt->capabilities;
 145         while (ptr != NULL && *ptr != '\0') {
 146                 if (*ptr == ' ')
 147                         ptr++;
 148
 149                 if (git_smart__ofs_delta_enabled && !git__prefixcmp(ptr, GIT_CAP_OFS_DELTA)) {
 150                         caps->common = caps->ofs_delta = 1;
 151                         ptr += strlen(GIT_CAP_OFS_DELTA);
 152                         continue;
 153                 }
 154
 155                 /* Keep multi_ack_detailed before multi_ack */
 156                 if (!git__prefixcmp(ptr, GIT_CAP_MULTI_ACK_DETAILED)) {
 157                         caps->common = caps->multi_ack_detailed = 1;
 158                         ptr += strlen(GIT_CAP_MULTI_ACK_DETAILED);
 159                         continue;
 160                 }
 161
 162                 if (!git__prefixcmp(ptr, GIT_CAP_MULTI_ACK)) {
 163                         caps->common = caps->multi_ack = 1;
 164                         ptr += strlen(GIT_CAP_MULTI_ACK);
 165                         continue;
 166                 }
 167
 168                 if (!git__prefixcmp(ptr, GIT_CAP_INCLUDE_TAG)) {
 169                         caps->common = caps->include_tag = 1;
 170                         ptr += strlen(GIT_CAP_INCLUDE_TAG);
 171                         continue;
 172                 }
 173
 174                 /* Keep side-band check after side-band-64k */
 175                 if (!git__prefixcmp(ptr, GIT_CAP_SIDE_BAND_64K)) {
 176                         caps->common = caps->side_band_64k = 1;
 177                         ptr += strlen(GIT_CAP_SIDE_BAND_64K);
 178                         continue;
 179                 }
 180
 181                 if (!git__prefixcmp(ptr, GIT_CAP_SIDE_BAND)) {
 182                         caps->common = caps->side_band = 1;
 183                         ptr += strlen(GIT_CAP_SIDE_BAND);
 184                         continue;
 185                 }
 186
 187                 if (!git__prefixcmp(ptr, GIT_CAP_DELETE_REFS)) {
 188                         caps->common = caps->delete_refs = 1;
 189                         ptr += strlen(GIT_CAP_DELETE_REFS);
 190                         continue;
 191                 }
 192
 193                 if (!git__prefixcmp(ptr, GIT_CAP_THIN_PACK)) {
 194                         caps->common = caps->thin_pack = 1;
 195                         ptr += strlen(GIT_CAP_THIN_PACK);
 196                         continue;
 197                 }
 198
 199                 if (!git__prefixcmp(ptr, GIT_CAP_SYMREF)) {
 200                         int error;
 201
 202                         if ((error = append_symref(&ptr, symrefs, ptr)) < 0)
 203                                 return error;
 204
 205                         continue;
 206                 }
 207
 208                 /* We don't know this capability, so skip it */
 209                 ptr = strchr(ptr, ' ');
 210         }
 211
 212         return 0;
 213 }
 214
 215 static int recv_pkt(git_pkt **out_pkt, git_pkt_type *out_type, gitno_buffer *buf)
 216 {
 217         const char *ptr = buf->data, *line_end = ptr;
 218         git_pkt *pkt = NULL;
 219         int error = 0, ret;
 220
 221         do {
 222                 if (buf->offset > 0)
 223                         error = git_pkt_parse_line(&pkt, &line_end, ptr, buf->offset);
 224                 else
 225                         error = GIT_EBUFS;
 226
 227                 if (error == 0)
 228                         break; /* return the pkt */
 229
 230                 if (error < 0 && error != GIT_EBUFS)
 231                         return error;
 232
 233                 if ((ret = gitno_recv(buf)) < 0) {
 234                         return ret;
 235                 } else if (ret == 0) {
 236                         git_error_set(GIT_ERROR_NET, "early EOF");
 237                         return GIT_EEOF;
 238                 }
 239         } while (error);
 240
 241         if (gitno_consume(buf, line_end) < 0)
 242                 return -1;
 243
 244         if (out_type != NULL)
 245                 *out_type = pkt->type;
 246         if (out_pkt != NULL)
 247                 *out_pkt = pkt;
 248         else
 249                 git__free(pkt);
 250
 251         return error;
 252 }
 253
 254 static int store_common(transport_smart *t)
 255 {
 256         git_pkt *pkt = NULL;
 257         gitno_buffer *buf = &t->buffer;
 258         int error;
 259
 260         do {
 261                 if ((error = recv_pkt(&pkt, NULL, buf)) < 0)
 262                         return error;
 263
 264                 if (pkt->type != GIT_PKT_ACK) {
 265                         git__free(pkt);
 266                         return 0;
 267                 }
 268
 269                 if (git_vector_insert(&t->common, pkt) < 0) {
 270                         git__free(pkt);
 271                         return -1;
 272                 }
 273         } while (1);
 274
 275         return 0;
 276 }
 277
 278 static int wait_while_ack(gitno_buffer *buf)
 279 {
 280         int error;
 281         git_pkt *pkt = NULL;
 282         git_pkt_ack *ack = NULL;
 283
 284         while (1) {
 285                 git_pkt_free(pkt);
 286
 287                 if ((error = recv_pkt(&pkt, NULL, buf)) < 0)
 288                         return error;
 289
 290                 if (pkt->type == GIT_PKT_NAK)
 291                         break;
 292                 if (pkt->type != GIT_PKT_ACK)
 293                         continue;
 294
 295                 ack = (git_pkt_ack*)pkt;
 296
 297                 if (ack->status != GIT_ACK_CONTINUE &&
 298                     ack->status != GIT_ACK_COMMON &&
 299                     ack->status != GIT_ACK_READY) {
 300                         break;
 301                 }
 302         }
 303
 304         git_pkt_free(pkt);
 305         return 0;
 306 }
 307
 308 int git_smart__negotiate_fetch(git_transport *transport, git_repository *repo, const git_remote_head * const *wants, size_t count)
 309 {
 310         transport_smart *t = (transport_smart *)transport;
 311         git_revwalk__push_options opts = GIT_REVWALK__PUSH_OPTIONS_INIT;
 312         gitno_buffer *buf = &t->buffer;
 313         git_buf data = GIT_BUF_INIT;
 314         git_revwalk *walk = NULL;
 315         int error = -1;
 316         git_pkt_type pkt_type;
 317         unsigned int i;
 318         git_oid oid;
 319
 320         if ((error = git_pkt_buffer_wants(wants, count, &t->caps, &data)) < 0)
 321                 return error;
 322
 323         if ((error = git_revwalk_new(&walk, repo)) < 0)
 324                 goto on_error;
 325
 326         opts.insert_by_date = 1;
 327         if ((error = git_revwalk__push_glob(walk, "refs/*", &opts)) < 0)
 328                 goto on_error;
 329
 330         /*
 331          * Our support for ACK extensions is simply to parse them. On
 332          * the first ACK we will accept that as enough common
 333          * objects. We give up if we haven't found an answer in the
 334          * first 256 we send.
 335          */
 336         i = 0;
 337         while (i < 256) {
 338                 error = git_revwalk_next(&oid, walk);
 339
 340                 if (error < 0) {
 341                         if (GIT_ITEROVER == error)
 342                                 break;
 343
 344                         goto on_error;
 345                 }
 346
 347                 git_pkt_buffer_have(&oid, &data);
 348                 i++;
 349                 if (i % 20 == 0) {
 350                         if (t->cancelled.val) {
 351                                 git_error_set(GIT_ERROR_NET, "The fetch was cancelled by the user");
 352                                 error = GIT_EUSER;
 353                                 goto on_error;
 354                         }
 355
 356                         git_pkt_buffer_flush(&data);
 357                         if (git_buf_oom(&data)) {
 358                                 error = -1;
 359                                 goto on_error;
 360                         }
 361
 362                         if ((error = git_smart__negotiation_step(&t->parent, data.ptr, data.size)) < 0)
 363                                 goto on_error;
 364
 365                         git_buf_clear(&data);
 366                         if (t->caps.multi_ack || t->caps.multi_ack_detailed) {
 367                                 if ((error = store_common(t)) < 0)
 368                                         goto on_error;
 369                         } else {
 370                                 if ((error = recv_pkt(NULL, &pkt_type, buf)) < 0)
 371                                         goto on_error;
 372
 373                                 if (pkt_type == GIT_PKT_ACK) {
 374                                         break;
 375                                 } else if (pkt_type == GIT_PKT_NAK) {
 376                                         continue;
 377                                 } else {
 378                                         git_error_set(GIT_ERROR_NET, "unexpected pkt type");
 379                                         error = -1;
 380                                         goto on_error;
 381                                 }
 382                         }
 383                 }
 384
 385                 if (t->common.length > 0)
 386                         break;
 387
 388                 if (i % 20 == 0 && t->rpc) {
 389                         git_pkt_ack *pkt;
 390                         unsigned int j;
 391
 392                         if ((error = git_pkt_buffer_wants(wants, count, &t->caps, &data)) < 0)
 393                                 goto on_error;
 394
 395                         git_vector_foreach(&t->common, j, pkt) {
 396                                 if ((error = git_pkt_buffer_have(&pkt->oid, &data)) < 0)
 397                                         goto on_error;
 398                         }
 399
 400                         if (git_buf_oom(&data)) {
 401                                 error = -1;
 402                                 goto on_error;
 403                         }
 404                 }
 405         }
 406
 407         /* Tell the other end that we're done negotiating */
 408         if (t->rpc && t->common.length > 0) {
 409                 git_pkt_ack *pkt;
 410                 unsigned int j;
 411
 412                 if ((error = git_pkt_buffer_wants(wants, count, &t->caps, &data)) < 0)
 413                         goto on_error;
 414
 415                 git_vector_foreach(&t->common, j, pkt) {
 416                         if ((error = git_pkt_buffer_have(&pkt->oid, &data)) < 0)
 417                                 goto on_error;
 418                 }
 419
 420                 if (git_buf_oom(&data)) {
 421                         error = -1;
 422                         goto on_error;
 423                 }
 424         }
 425
 426         if ((error = git_pkt_buffer_done(&data)) < 0)
 427                 goto on_error;
 428
 429         if (t->cancelled.val) {
 430                 git_error_set(GIT_ERROR_NET, "The fetch was cancelled by the user");
 431                 error = GIT_EUSER;
 432                 goto on_error;
 433         }
 434         if ((error = git_smart__negotiation_step(&t->parent, data.ptr, data.size)) < 0)
 435                 goto on_error;
 436
 437         git_buf_dispose(&data);
 438         git_revwalk_free(walk);
 439
 440         /* Now let's eat up whatever the server gives us */
 441         if (!t->caps.multi_ack && !t->caps.multi_ack_detailed) {
 442                 if ((error = recv_pkt(NULL, &pkt_type, buf)) < 0)
 443                         return error;
 444
 445                 if (pkt_type != GIT_PKT_ACK && pkt_type != GIT_PKT_NAK) {
 446                         git_error_set(GIT_ERROR_NET, "unexpected pkt type");
 447                         return -1;
 448                 }
 449         } else {
 450                 error = wait_while_ack(buf);
 451         }
 452
 453         return error;
 454
 455 on_error:
 456         git_revwalk_free(walk);
 457         git_buf_dispose(&data);
 458         return error;
 459 }
 460
 461 static int no_sideband(transport_smart *t, struct git_odb_writepack *writepack, gitno_buffer *buf, git_indexer_progress *stats)
 462 {
 463         int recvd;
 464
 465         do {
 466                 if (t->cancelled.val) {
 467                         git_error_set(GIT_ERROR_NET, "the fetch was cancelled by the user");
 468                         return GIT_EUSER;
 469                 }
 470
 471                 if (writepack->append(writepack, buf->data, buf->offset, stats) < 0)
 472                         return -1;
 473
 474                 gitno_consume_n(buf, buf->offset);
 475
 476                 if ((recvd = gitno_recv(buf)) < 0)
 477                         return recvd;
 478         } while(recvd > 0);
 479
 480         if (writepack->commit(writepack, stats) < 0)
 481                 return -1;
 482
 483         return 0;
 484 }
 485
 486 struct network_packetsize_payload
 487 {
 488         git_indexer_progress_cb callback;
 489         void *payload;
 490         git_indexer_progress *stats;
 491         size_t last_fired_bytes;
 492 };
 493
 494 static int network_packetsize(size_t received, void *payload)
 495 {
 496         struct network_packetsize_payload *npp = (struct network_packetsize_payload*)payload;
 497
 498         /* Accumulate bytes */
 499         npp->stats->received_bytes += received;
 500
 501         /* Fire notification if the threshold is reached */
 502         if ((npp->stats->received_bytes - npp->last_fired_bytes) > NETWORK_XFER_THRESHOLD) {
 503                 npp->last_fired_bytes = npp->stats->received_bytes;
 504
 505                 if (npp->callback(npp->stats, npp->payload))
 506                         return GIT_EUSER;
 507         }
 508
 509         return 0;
 510 }
 511
 512 int git_smart__download_pack(
 513         git_transport *transport,
 514         git_repository *repo,
 515         git_indexer_progress *stats,
 516         git_indexer_progress_cb progress_cb,
 517         void *progress_payload)
 518 {
 519         transport_smart *t = (transport_smart *)transport;
 520         gitno_buffer *buf = &t->buffer;
 521         git_odb *odb;
 522         struct git_odb_writepack *writepack = NULL;
 523         int error = 0;
 524         struct network_packetsize_payload npp = {0};
 525
 526         memset(stats, 0, sizeof(git_indexer_progress));
 527
 528         if (progress_cb) {
 529                 npp.callback = progress_cb;
 530                 npp.payload = progress_payload;
 531                 npp.stats = stats;
 532                 t->packetsize_cb = &network_packetsize;
 533                 t->packetsize_payload = &npp;
 534
 535                 /* We might have something in the buffer already from negotiate_fetch */
 536                 if (t->buffer.offset > 0 && !t->cancelled.val)
 537                         if (t->packetsize_cb(t->buffer.offset, t->packetsize_payload))
 538                                 git_atomic32_set(&t->cancelled, 1);
 539         }
 540
 541         if ((error = git_repository_odb__weakptr(&odb, repo)) < 0 ||
 542                 ((error = git_odb_write_pack(&writepack, odb, progress_cb, progress_payload)) != 0))
 543                 goto done;
 544
 545         /*
 546          * If the remote doesn't support the side-band, we can feed
 547          * the data directly to the pack writer. Otherwise, we need to
 548          * check which one belongs there.
 549          */
 550         if (!t->caps.side_band && !t->caps.side_band_64k) {
 551                 error = no_sideband(t, writepack, buf, stats);
 552                 goto done;
 553         }
 554
 555         do {
 556                 git_pkt *pkt = NULL;
 557
 558                 /* Check cancellation before network call */
 559                 if (t->cancelled.val) {
 560                         git_error_clear();
 561                         error = GIT_EUSER;
 562                         goto done;
 563                 }
 564
 565                 if ((error = recv_pkt(&pkt, NULL, buf)) >= 0) {
 566                         /* Check cancellation after network call */
 567                         if (t->cancelled.val) {
 568                                 git_error_clear();
 569                                 error = GIT_EUSER;
 570                         } else if (pkt->type == GIT_PKT_PROGRESS) {
 571                                 if (t->progress_cb) {
 572                                         git_pkt_progress *p = (git_pkt_progress *) pkt;
 573
 574                                         if (p->len > INT_MAX) {
 575                                                 git_error_set(GIT_ERROR_NET, "oversized progress message");
 576                                                 error = GIT_ERROR;
 577                                                 goto done;
 578                                         }
 579
 580                                         error = t->progress_cb(p->data, (int)p->len, t->message_cb_payload);
 581                                 }
 582                         } else if (pkt->type == GIT_PKT_DATA) {
 583                                 git_pkt_data *p = (git_pkt_data *) pkt;
 584
 585                                 if (p->len)
 586                                         error = writepack->append(writepack, p->data, p->len, stats);
 587                         } else if (pkt->type == GIT_PKT_FLUSH) {
 588                                 /* A flush indicates the end of the packfile */
 589                                 git__free(pkt);
 590                                 break;
 591                         }
 592                 }
 593
 594                 git_pkt_free(pkt);
 595
 596                 if (error < 0)
 597                         goto done;
 598
 599         } while (1);
 600
 601         /*
 602          * Trailing execution of progress_cb, if necessary...
 603          * Only the callback through the npp datastructure currently
 604          * updates the last_fired_bytes value. It is possible that
 605          * progress has already been reported with the correct
 606          * "received_bytes" value, but until (if?) this is unified
 607          * then we will report progress again to be sure that the
 608          * correct last received_bytes value is reported.
 609          */
 610         if (npp.callback && npp.stats->received_bytes > npp.last_fired_bytes) {
 611                 error = npp.callback(npp.stats, npp.payload);
 612                 if (error != 0)
 613                         goto done;
 614         }
 615
 616         error = writepack->commit(writepack, stats);
 617
 618 done:
 619         if (writepack)
 620                 writepack->free(writepack);
 621         if (progress_cb) {
 622                 t->packetsize_cb = NULL;
 623                 t->packetsize_payload = NULL;
 624         }
 625
 626         return error;
 627 }
 628
 629 static int gen_pktline(git_buf *buf, git_push *push)
 630 {
 631         push_spec *spec;
 632         size_t i, len;
 633         char old_id[GIT_OID_HEXSZ+1], new_id[GIT_OID_HEXSZ+1];
 634
 635         old_id[GIT_OID_HEXSZ] = '\0'; new_id[GIT_OID_HEXSZ] = '\0';
 636
 637         git_vector_foreach(&push->specs, i, spec) {
 638                 len = 2*GIT_OID_HEXSZ + 7 + strlen(spec->refspec.dst);
 639
 640                 if (i == 0) {
 641                         ++len; /* '\0' */
 642                         if (push->report_status)
 643                                 len += strlen(GIT_CAP_REPORT_STATUS) + 1;
 644                         len += strlen(GIT_CAP_SIDE_BAND_64K) + 1;
 645                 }
 646
 647                 git_oid_fmt(old_id, &spec->roid);
 648                 git_oid_fmt(new_id, &spec->loid);
 649
 650                 git_buf_printf(buf, "%04"PRIxZ"%s %s %s", len, old_id, new_id, spec->refspec.dst);
 651
 652                 if (i == 0) {
 653                         git_buf_putc(buf, '\0');
 654                         /* Core git always starts their capabilities string with a space */
 655                         if (push->report_status) {
 656                                 git_buf_putc(buf, ' ');
 657                                 git_buf_printf(buf, GIT_CAP_REPORT_STATUS);
 658                         }
 659                         git_buf_putc(buf, ' ');
 660                         git_buf_printf(buf, GIT_CAP_SIDE_BAND_64K);
 661                 }
 662
 663                 git_buf_putc(buf, '\n');
 664         }
 665
 666         git_buf_puts(buf, "0000");
 667         return git_buf_oom(buf) ? -1 : 0;
 668 }
 669
 670 static int add_push_report_pkt(git_push *push, git_pkt *pkt)
 671 {
 672         push_status *status;
 673
 674         switch (pkt->type) {
 675                 case GIT_PKT_OK:
 676                         status = git__calloc(1, sizeof(push_status));
 677                         GIT_ERROR_CHECK_ALLOC(status);
 678                         status->msg = NULL;
 679                         status->ref = git__strdup(((git_pkt_ok *)pkt)->ref);
 680                         if (!status->ref ||
 681                                 git_vector_insert(&push->status, status) < 0) {
 682                                 git_push_status_free(status);
 683                                 return -1;
 684                         }
 685                         break;
 686                 case GIT_PKT_NG:
 687                         status = git__calloc(1, sizeof(push_status));
 688                         GIT_ERROR_CHECK_ALLOC(status);
 689                         status->ref = git__strdup(((git_pkt_ng *)pkt)->ref);
 690                         status->msg = git__strdup(((git_pkt_ng *)pkt)->msg);
 691                         if (!status->ref || !status->msg ||
 692                                 git_vector_insert(&push->status, status) < 0) {
 693                                 git_push_status_free(status);
 694                                 return -1;
 695                         }
 696                         break;
 697                 case GIT_PKT_UNPACK:
 698                         push->unpack_ok = ((git_pkt_unpack *)pkt)->unpack_ok;
 699                         break;
 700                 case GIT_PKT_FLUSH:
 701                         return GIT_ITEROVER;
 702                 default:
 703                         git_error_set(GIT_ERROR_NET, "report-status: protocol error");
 704                         return -1;
 705         }
 706
 707         return 0;
 708 }
 709
 710 static int add_push_report_sideband_pkt(git_push *push, git_pkt_data *data_pkt, git_buf *data_pkt_buf)
 711 {
 712         git_pkt *pkt;
 713         const char *line, *line_end = NULL;
 714         size_t line_len;
 715         int error;
 716         int reading_from_buf = data_pkt_buf->size > 0;
 717
 718         if (reading_from_buf) {
 719                 /* We had an existing partial packet, so add the new
 720                  * packet to the buffer and parse the whole thing */
 721                 git_buf_put(data_pkt_buf, data_pkt->data, data_pkt->len);
 722                 line = data_pkt_buf->ptr;
 723                 line_len = data_pkt_buf->size;
 724         }
 725         else {
 726                 line = data_pkt->data;
 727                 line_len = data_pkt->len;
 728         }
 729
 730         while (line_len > 0) {
 731                 error = git_pkt_parse_line(&pkt, &line_end, line, line_len);
 732
 733                 if (error == GIT_EBUFS) {
 734                         /* Buffer the data when the inner packet is split
 735                          * across multiple sideband packets */
 736                         if (!reading_from_buf)
 737                                 git_buf_put(data_pkt_buf, line, line_len);
 738                         error = 0;
 739                         goto done;
 740                 }
 741                 else if (error < 0)
 742                         goto done;
 743
 744                 /* Advance in the buffer */
 745                 line_len -= (line_end - line);
 746                 line = line_end;
 747
 748                 error = add_push_report_pkt(push, pkt);
 749
 750                 git_pkt_free(pkt);
 751
 752                 if (error < 0 && error != GIT_ITEROVER)
 753                         goto done;
 754         }
 755
 756         error = 0;
 757
 758 done:
 759         if (reading_from_buf)
 760                 git_buf_consume(data_pkt_buf, line_end);
 761         return error;
 762 }
 763
 764 static int parse_report(transport_smart *transport, git_push *push)
 765 {
 766         git_pkt *pkt = NULL;
 767         const char *line_end = NULL;
 768         gitno_buffer *buf = &transport->buffer;
 769         int error, recvd;
 770         git_buf data_pkt_buf = GIT_BUF_INIT;
 771
 772         for (;;) {
 773                 if (buf->offset > 0)
 774                         error = git_pkt_parse_line(&pkt, &line_end,
 775                                                    buf->data, buf->offset);
 776                 else
 777                         error = GIT_EBUFS;
 778
 779                 if (error < 0 && error != GIT_EBUFS) {
 780                         error = -1;
 781                         goto done;
 782                 }
 783
 784                 if (error == GIT_EBUFS) {
 785                         if ((recvd = gitno_recv(buf)) < 0) {
 786                                 error = recvd;
 787                                 goto done;
 788                         }
 789
 790                         if (recvd == 0) {
 791                                 git_error_set(GIT_ERROR_NET, "early EOF");
 792                                 error = GIT_EEOF;
 793                                 goto done;
 794                         }
 795                         continue;
 796                 }
 797
 798                 if (gitno_consume(buf, line_end) < 0)
 799                         return -1;
 800
 801                 error = 0;
 802
 803                 switch (pkt->type) {
 804                         case GIT_PKT_DATA:
 805                                 /* This is a sideband packet which contains other packets */
 806                                 error = add_push_report_sideband_pkt(push, (git_pkt_data *)pkt, &data_pkt_buf);
 807                                 break;
 808                         case GIT_PKT_ERR:
 809                                 git_error_set(GIT_ERROR_NET, "report-status: Error reported: %s",
 810                                         ((git_pkt_err *)pkt)->error);
 811                                 error = -1;
 812                                 break;
 813                         case GIT_PKT_PROGRESS:
 814                                 if (transport->progress_cb) {
 815                                         git_pkt_progress *p = (git_pkt_progress *) pkt;
 816
 817                                         if (p->len > INT_MAX) {
 818                                                 git_error_set(GIT_ERROR_NET, "oversized progress message");
 819                                                 error = GIT_ERROR;
 820                                                 goto done;
 821                                         }
 822
 823                                         error = transport->progress_cb(p->data, (int)p->len, transport->message_cb_payload);
 824                                 }
 825                                 break;
 826                         default:
 827                                 error = add_push_report_pkt(push, pkt);
 828                                 break;
 829                 }
 830
 831                 git_pkt_free(pkt);
 832
 833                 /* add_push_report_pkt returns GIT_ITEROVER when it receives a flush */
 834                 if (error == GIT_ITEROVER) {
 835                         error = 0;
 836                         if (data_pkt_buf.size > 0) {
 837                                 /* If there was data remaining in the pack data buffer,
 838                                  * then the server sent a partial pkt-line */
 839                                 git_error_set(GIT_ERROR_NET, "incomplete pack data pkt-line");
 840                                 error = GIT_ERROR;
 841                         }
 842                         goto done;
 843                 }
 844
 845                 if (error < 0) {
 846                         goto done;
 847                 }
 848         }
 849 done:
 850         git_buf_dispose(&data_pkt_buf);
 851         return error;
 852 }
 853
 854 static int add_ref_from_push_spec(git_vector *refs, push_spec *push_spec)
 855 {
 856         git_pkt_ref *added = git__calloc(1, sizeof(git_pkt_ref));
 857         GIT_ERROR_CHECK_ALLOC(added);
 858
 859         added->type = GIT_PKT_REF;
 860         git_oid_cpy(&added->head.oid, &push_spec->loid);
 861         added->head.name = git__strdup(push_spec->refspec.dst);
 862
 863         if (!added->head.name ||
 864                 git_vector_insert(refs, added) < 0) {
 865                 git_pkt_free((git_pkt *)added);
 866                 return -1;
 867         }
 868
 869         return 0;
 870 }
 871
 872 static int update_refs_from_report(
 873         git_vector *refs,
 874         git_vector *push_specs,
 875         git_vector *push_report)
 876 {
 877         git_pkt_ref *ref;
 878         push_spec *push_spec;
 879         push_status *push_status;
 880         size_t i, j, refs_len;
 881         int cmp;
 882
 883         /* For each push spec we sent to the server, we should have
 884          * gotten back a status packet in the push report */
 885         if (push_specs->length != push_report->length) {
 886                 git_error_set(GIT_ERROR_NET, "report-status: protocol error");
 887                 return -1;
 888         }
 889
 890         /* We require that push_specs be sorted with push_spec_rref_cmp,
 891          * and that push_report be sorted with push_status_ref_cmp */
 892         git_vector_sort(push_specs);
 893         git_vector_sort(push_report);
 894
 895         git_vector_foreach(push_specs, i, push_spec) {
 896                 push_status = git_vector_get(push_report, i);
 897
 898                 /* For each push spec we sent to the server, we should have
 899                  * gotten back a status packet in the push report which matches */
 900                 if (strcmp(push_spec->refspec.dst, push_status->ref)) {
 901                         git_error_set(GIT_ERROR_NET, "report-status: protocol error");
 902                         return -1;
 903                 }
 904         }
 905
 906         /* We require that refs be sorted with ref_name_cmp */
 907         git_vector_sort(refs);
 908         i = j = 0;
 909         refs_len = refs->length;
 910
 911         /* Merge join push_specs with refs */
 912         while (i < push_specs->length && j < refs_len) {
 913                 push_spec = git_vector_get(push_specs, i);
 914                 push_status = git_vector_get(push_report, i);
 915                 ref = git_vector_get(refs, j);
 916
 917                 cmp = strcmp(push_spec->refspec.dst, ref->head.name);
 918
 919                 /* Iterate appropriately */
 920                 if (cmp <= 0) i++;
 921                 if (cmp >= 0) j++;
 922
 923                 /* Add case */
 924                 if (cmp < 0 &&
 925                         !push_status->msg &&
 926                         add_ref_from_push_spec(refs, push_spec) < 0)
 927                         return -1;
 928
 929                 /* Update case, delete case */
 930                 if (cmp == 0 &&
 931                         !push_status->msg)
 932                         git_oid_cpy(&ref->head.oid, &push_spec->loid);
 933         }
 934
 935         for (; i < push_specs->length; i++) {
 936                 push_spec = git_vector_get(push_specs, i);
 937                 push_status = git_vector_get(push_report, i);
 938
 939                 /* Add case */
 940                 if (!push_status->msg &&
 941                         add_ref_from_push_spec(refs, push_spec) < 0)
 942                         return -1;
 943         }
 944
 945         /* Remove any refs which we updated to have a zero OID. */
 946         git_vector_rforeach(refs, i, ref) {
 947                 if (git_oid_is_zero(&ref->head.oid)) {
 948                         git_vector_remove(refs, i);
 949                         git_pkt_free((git_pkt *)ref);
 950                 }
 951         }
 952
 953         git_vector_sort(refs);
 954
 955         return 0;
 956 }
 957
 958 struct push_packbuilder_payload
 959 {
 960         git_smart_subtransport_stream *stream;
 961         git_packbuilder *pb;
 962         git_push_transfer_progress_cb cb;
 963         void *cb_payload;
 964         size_t last_bytes;
 965         double last_progress_report_time;
 966 };
 967
 968 static int stream_thunk(void *buf, size_t size, void *data)
 969 {
 970         int error = 0;
 971         struct push_packbuilder_payload *payload = data;
 972
 973         if ((error = payload->stream->write(payload->stream, (const char *)buf, size)) < 0)
 974                 return error;
 975
 976         if (payload->cb) {
 977                 double current_time = git__timer();
 978                 double elapsed = current_time - payload->last_progress_report_time;
 979                 payload->last_bytes += size;
 980
 981                 if (elapsed < 0 || elapsed >= MIN_PROGRESS_UPDATE_INTERVAL) {
 982                         payload->last_progress_report_time = current_time;
 983                         error = payload->cb(payload->pb->nr_written, payload->pb->nr_objects, payload->last_bytes, payload->cb_payload);
 984                 }
 985         }
 986
 987         return error;
 988 }
 989
 990 int git_smart__push(git_transport *transport, git_push *push, const git_remote_callbacks *cbs)
 991 {
 992         transport_smart *t = (transport_smart *)transport;
 993         struct push_packbuilder_payload packbuilder_payload = {0};
 994         git_buf pktline = GIT_BUF_INIT;
 995         int error = 0, need_pack = 0;
 996         push_spec *spec;
 997         unsigned int i;
 998
 999         packbuilder_payload.pb = push->pb;
1000
1001         if (cbs && cbs->push_transfer_progress) {
1002                 packbuilder_payload.cb = cbs->push_transfer_progress;
1003                 packbuilder_payload.cb_payload = cbs->payload;
1004         }
1005
1006 #ifdef PUSH_DEBUG
1007 {
1008         git_remote_head *head;
1009         char hex[GIT_OID_HEXSZ+1]; hex[GIT_OID_HEXSZ] = '\0';
1010
1011         git_vector_foreach(&push->remote->refs, i, head) {
1012                 git_oid_fmt(hex, &head->oid);
1013                 fprintf(stderr, "%s (%s)\n", hex, head->name);
1014         }
1015
1016         git_vector_foreach(&push->specs, i, spec) {
1017                 git_oid_fmt(hex, &spec->roid);
1018                 fprintf(stderr, "%s (%s) -> ", hex, spec->lref);
1019                 git_oid_fmt(hex, &spec->loid);
1020                 fprintf(stderr, "%s (%s)\n", hex, spec->rref ?
1021                         spec->rref : spec->lref);
1022         }
1023 }
1024 #endif
1025
1026         /*
1027          * Figure out if we need to send a packfile; which is in all
1028          * cases except when we only send delete commands
1029          */
1030         git_vector_foreach(&push->specs, i, spec) {
1031                 if (spec->refspec.src && spec->refspec.src[0] != '\0') {
1032                         need_pack = 1;
1033                         break;
1034                 }
1035         }
1036
1037         if ((error = git_smart__get_push_stream(t, &packbuilder_payload.stream)) < 0 ||
1038                 (error = gen_pktline(&pktline, push)) < 0 ||
1039                 (error = packbuilder_payload.stream->write(packbuilder_payload.stream, git_buf_cstr(&pktline), git_buf_len(&pktline))) < 0)
1040                 goto done;
1041
1042         if (need_pack &&
1043                 (error = git_packbuilder_foreach(push->pb, &stream_thunk, &packbuilder_payload)) < 0)
1044                 goto done;
1045
1046         /* If we sent nothing or the server doesn't support report-status, then
1047          * we consider the pack to have been unpacked successfully */
1048         if (!push->specs.length || !push->report_status)
1049                 push->unpack_ok = 1;
1050         else if ((error = parse_report(t, push)) < 0)
1051                 goto done;
1052
1053         /* If progress is being reported write the final report */
1054         if (cbs && cbs->push_transfer_progress) {
1055                 error = cbs->push_transfer_progress(
1056                                         push->pb->nr_written,
1057                                         push->pb->nr_objects,
1058                                         packbuilder_payload.last_bytes,
1059                                         cbs->payload);
1060
1061                 if (error < 0)
1062                         goto done;
1063         }
1064
1065         if (push->status.length) {
1066                 error = update_refs_from_report(&t->refs, &push->specs, &push->status);
1067                 if (error < 0)
1068                         goto done;
1069
1070                 error = git_smart__update_heads(t, NULL);
1071         }
1072
1073 done:
1074         git_buf_dispose(&pktline);
1075         return error;
1076 }