vmtar.c

   1 /*
   2     Copyright (C) 2007-2012 Proxmox Server Solutions GmbH
   3
   4     Copyright: vzdump is under GNU GPL, the GNU General Public License.
   5
   6     This program is free software; you can redistribute it and/or modify
   7     it under the terms of the GNU General Public License as published by
   8     the Free Software Foundation; version 2 dated June, 1991.
   9
  10     This program is distributed in the hope that it will be useful,
  11     but WITHOUT ANY WARRANTY; without even the implied warranty of
  12     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13     GNU General Public License for more details.
  14
  15     You should have received a copy of the GNU General Public License
  16     along with this program; if not, write to the
  17     Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston,
  18     MA 02110-1301, USA.
  19
  20     Author: Dietmar Maurer <dietmar@proxmox.com>
  21
  22     NOTE: the tar specific code is copied from the GNU tar package (just
  23     slighly modified to fit our needs).
  24 */
  25
  26 #define _GNU_SOURCE
  27
  28 #include <sys/types.h>
  29 #include <sys/stat.h>
  30 #include <fcntl.h>
  31 #include <stdio.h>
  32 #include <stdlib.h>
  33 #include <unistd.h>
  34 #include <errno.h>
  35 #include <string.h>
  36 #include <time.h>
  37 #include <stdint.h>
  38 #include <getopt.h>
  39 #include <signal.h>
  40
  41 #include "utils.c"
  42
  43
  44 #define BLOCKSIZE 512
  45 #define BUFFER_BLOCKS 32
  46
  47 static char *outname;
  48
  49 struct writebuffer
  50 {
  51   int fd;
  52   char buffer[BUFFER_BLOCKS*BLOCKSIZE];
  53   size_t bpos;
  54   size_t total;
  55 };
  56
  57 /* OLDGNU_MAGIC uses both magic and version fields, which are contiguous. */
  58 #define OLDGNU_MAGIC "ustar  "  /* 7 chars and a null */
  59
  60 struct posix_header
  61 {                               /* byte offset */
  62   char name[100];               /*   0 */
  63   char mode[8];                 /* 100 */
  64   char uid[8];                  /* 108 */
  65   char gid[8];                  /* 116 */
  66   char size[12];                /* 124 */
  67   char mtime[12];               /* 136 */
  68   char chksum[8];               /* 148 */
  69   char typeflag;                /* 156 */
  70   char linkname[100];           /* 157 */
  71   char magic[6];                /* 257 */
  72   char version[2];              /* 263 */
  73   char uname[32];               /* 265 */
  74   char gname[32];               /* 297 */
  75   char devmajor[8];             /* 329 */
  76   char devminor[8];             /* 337 */
  77   char prefix[155];             /* 345 */
  78                                 /* 500 */
  79 };
  80
  81 struct sparse
  82 {                               /* byte offset */
  83   char offset[12];              /*   0 */
  84   char numbytes[12];            /*  12 */
  85                                 /*  24 */
  86 };
  87
  88 struct oldgnu_header
  89 {                               /* byte offset */
  90   char unused_pad1[345];        /*   0 */
  91   char atime[12];               /* 345 Incr. archive: atime of the file */
  92   char ctime[12];               /* 357 Incr. archive: ctime of the file */
  93   char offset[12];              /* 369 Multivolume archive: the offset of
  94                                    the start of this volume */
  95   char longnames[4];            /* 381 Not used */
  96   char unused_pad2;             /* 385 */
  97   struct sparse sp[4];
  98                                 /* 386 */
  99   char isextended;              /* 482 Sparse file: Extension sparse header
 100                                    follows */
 101   char realsize[12];            /* 483 Sparse file: Real size*/
 102                                 /* 495 */
 103 };
 104
 105 struct sparse_header
 106 {                               /* byte offset */
 107   struct sparse sp[21];         /*   0 */
 108   char isextended;              /* 504 */
 109                                 /* 505 */
 110 };
 111
 112 union block
 113 {
 114   char buffer[BLOCKSIZE];
 115   struct posix_header header;
 116   struct oldgnu_header oldgnu_header;
 117   struct sparse_header sparse_header;
 118 };
 119
 120
 121 struct sp_entry
 122 {
 123   off_t offset;
 124   size_t bytes;
 125 };
 126
 127 struct sp_array {
 128   size_t real_size;
 129   size_t effective_size;
 130   size_t avail;
 131   size_t size;
 132   struct sp_entry *map;
 133 };
 134
 135 static void
 136 cleanup (void)
 137 {
 138   if (outname)
 139     unlink (outname);
 140 }
 141
 142 void term_handler()
 143 {
 144   fprintf (stderr, "received signal - terminate process\n");
 145   exit(-1);
 146 }
 147
 148 struct sp_array*
 149 sparray_new (void) {
 150   struct sp_array *ma = malloc (sizeof (struct sp_array));
 151   if (!ma) {
 152     fprintf (stderr, "ERROR: memory allocation failure\n");
 153     exit (-1);
 154   }
 155   ma->real_size = 0;
 156   ma->effective_size = 0;
 157   ma->avail = 0;
 158   ma->size = 1024;
 159   ma->map = malloc (ma->size * sizeof (struct sp_entry));
 160   if (!ma->map) {
 161     fprintf (stderr, "ERROR: memory allocation failure\n");
 162     exit (-1);
 163   }
 164   return ma;
 165 }
 166
 167 void
 168 sparray_resize (struct sp_array *ma)
 169 {
 170   ma->size += 1024;
 171   if (!(ma->map = realloc (ma->map, ma->size * sizeof (struct sp_entry)))) {
 172     fprintf (stderr, "ERROR: memory allocation failure\n");
 173     exit (-1);
 174   }
 175 }
 176
 177 void
 178 sparray_add (struct sp_array *ma, off_t offset, size_t bytes)
 179 {
 180
 181   if (ma->avail == ma->size) {
 182     sparray_resize(ma);
 183   }
 184   ma->map[ma->avail].offset = offset;
 185   ma->map[ma->avail].bytes = bytes;
 186   ma->avail++;
 187 }
 188
 189 static void
 190 to_base256 (uintmax_t value, char *where, size_t size)
 191 {
 192   uintmax_t v = value;
 193   size_t i = size - 1;
 194
 195   where[0] = 1 << 7;
 196
 197   do {
 198     where[i--] = v & ((1 << 8) - 1);
 199     v >>= 8;
 200   } while (i);
 201 }
 202
 203 static void
 204 to_octal (uintmax_t value, char *where, size_t size)
 205 {
 206   uintmax_t v = value;
 207   size_t i = size - 1;
 208
 209   where[i] = '\0';
 210   do {
 211     where[--i] = '0' + (v & ((1 << 3) - 1));
 212     v >>= 3;
 213   } while (i);
 214 }
 215
 216 /* The maximum uintmax_t value that can be represented with DIGITS digits,
 217    assuming that each digit is BITS_PER_DIGIT wide.  */
 218 #define MAX_VAL_WITH_DIGITS(digits, bits_per_digit) \
 219    ((digits) * (bits_per_digit) < sizeof (uintmax_t) * 8 \
 220     ? ((uintmax_t) 1 << ((digits) * (bits_per_digit))) - 1 \
 221     : (uintmax_t) -1)
 222
 223 /* The maximum uintmax_t value that can be represented with octal
 224    digits and a trailing NUL in BUFFER.  */
 225 #define MAX_OCTAL_VAL(buffer) MAX_VAL_WITH_DIGITS (sizeof (buffer) - 1, 3)
 226
 227 void
 228 off12_to_chars (char *p, off_t v)
 229 {
 230   if (v < 0) {
 231     fprintf (stderr, "ERROR: internal error - got negative offset\n");
 232     exit (-1);
 233   }
 234
 235   uintmax_t value = (uintmax_t) v;
 236
 237   if (value <= MAX_VAL_WITH_DIGITS (11, 3)) {
 238     to_octal (value, p, 12);
 239   } else {
 240     to_base256 (value, p, 12);
 241   }
 242 }
 243
 244 char *
 245 buffer_block(struct writebuffer *wbuf)
 246 {
 247   size_t space = sizeof (wbuf->buffer) - wbuf->bpos;
 248   char *blk;
 249
 250   if (space >= BLOCKSIZE) {
 251     blk = wbuf->buffer + wbuf->bpos;
 252     wbuf->bpos += BLOCKSIZE;
 253   } else {
 254     full_write (wbuf->fd, wbuf->buffer, wbuf->bpos);
 255     wbuf->total += wbuf->bpos;
 256     wbuf->bpos = BLOCKSIZE;
 257     blk = wbuf->buffer;
 258   }
 259   return blk;
 260 }
 261
 262 struct writebuffer*
 263 buffer_new(int fd)
 264 {
 265   struct writebuffer *wbuf = calloc (1, sizeof (struct writebuffer));
 266
 267   if (!wbuf) {
 268     fprintf (stderr, "ERROR: memory allocation failure\n");
 269     exit (-1);
 270   }
 271
 272   wbuf->fd = fd;
 273
 274   return wbuf;
 275 }
 276
 277 void
 278 buffer_flush(struct writebuffer *wbuf)
 279 {
 280   full_write (wbuf->fd, wbuf->buffer, wbuf->bpos);
 281   wbuf->total += wbuf->bpos;
 282   wbuf->bpos = 0;
 283 }
 284
 285 void
 286 dump_header (struct writebuffer *wbuf, const char *filename, time_t mtime, struct sp_array *ma)
 287 {
 288   union block *blk = (union block *)buffer_block (wbuf);
 289   memset (blk->buffer, 0, BLOCKSIZE);
 290
 291   if (strlen(filename)>98) {
 292     fprintf (stderr, "ERROR: filename '%s' too long\n", filename);
 293     exit (-1);
 294   }
 295
 296   strncpy (blk->header.name, filename, 100);
 297
 298   sprintf (blk->header.mode, "%07o", 0644);
 299   sprintf (blk->header.uid, "%07o", 0);
 300   sprintf (blk->header.gid, "%07o", 0);
 301   off12_to_chars (blk->header.mtime, mtime);
 302
 303   memcpy (blk->header.chksum, "        ", 8);
 304
 305   blk->header.typeflag = ma->avail ? 'S' : '0';
 306
 307   sprintf (blk->header.magic, "%s", OLDGNU_MAGIC);
 308
 309   sprintf (blk->header.uname, "%s", "root");
 310   sprintf (blk->header.gname, "%s", "root");
 311
 312   size_t ind = 0;
 313   if (ind < ma->avail) {
 314     size_t i;
 315     for (i = 0;i < 4 && ind < ma->avail; i++, ind++) {
 316       off12_to_chars (blk->oldgnu_header.sp[i].offset, ma->map[ind].offset);
 317       off12_to_chars (blk->oldgnu_header.sp[i].numbytes, ma->map[ind].bytes);
 318     }
 319   }
 320
 321   if (ma->avail > 4)
 322     blk->oldgnu_header.isextended = 1;
 323
 324   off12_to_chars (blk->header.size, ma->effective_size);
 325   off12_to_chars (blk->oldgnu_header.realsize, ma->real_size);
 326
 327   int sum = 0;
 328   char *p = blk->buffer;
 329   int i;
 330   for (i = BLOCKSIZE; i-- != 0; )
 331     sum += 0xFF & *p++;
 332
 333   sprintf (blk->header.chksum, "%6o", sum);
 334
 335   while (ind < ma->avail) {
 336     blk = (union block *)buffer_block (wbuf);
 337     memset (blk->buffer, 0, BLOCKSIZE);
 338     size_t i;
 339     for (i = 0;i < 21 && ind < ma->avail; i++, ind++) {
 340       off12_to_chars (blk->sparse_header.sp[i].offset, ma->map[ind].offset);
 341       off12_to_chars (blk->sparse_header.sp[i].numbytes, ma->map[ind].bytes);
 342     }
 343     if (ind < ma->avail)
 344       blk->sparse_header.isextended = 1;
 345
 346   }
 347 }
 348
 349 int
 350 scan_sparse_file (int fd, struct sp_array *ma)
 351 {
 352   char buffer[BLOCKSIZE];
 353   size_t count;
 354   off_t offset = 0;
 355   off_t file_size = 0;
 356   size_t sp_bytes = 0;
 357   off_t sp_offset = 0;
 358
 359   if (lseek (fd, 0, SEEK_SET) < 0)
 360     return 0;
 361
 362   while ((count = full_read (fd, buffer, sizeof (buffer))) > 0) {
 363     if (block_is_zero (buffer, count)) {
 364       if (sp_bytes) {
 365         sparray_add (ma, sp_offset, sp_bytes);
 366         sp_bytes = 0;
 367       }
 368     } else {
 369       file_size += count;
 370       if (!sp_bytes)
 371         sp_offset = offset;
 372       sp_bytes += count;
 373     }
 374     offset += count;
 375   }
 376
 377   if (sp_bytes == 0)
 378     sp_offset = offset;
 379
 380   sparray_add (ma, sp_offset, sp_bytes);
 381
 382   ma->real_size = offset;
 383   ma->effective_size = file_size;
 384
 385   return 1;
 386 }
 387
 388 int
 389 dump_sparse_file (int fd, struct writebuffer *wbuf, struct sp_array *ma)
 390 {
 391   if (lseek (fd, 0, SEEK_SET) < 0)
 392     return 0;
 393
 394   int i;
 395   size_t dumped_size = 0;
 396   for (i = 0; i < ma->avail; i++) {
 397     struct sp_entry *e = &ma->map[i];
 398     if (lseek (fd, e->offset, SEEK_SET) < 0)
 399       return 0;
 400
 401     off_t bytes_left = e->bytes;
 402
 403     while (bytes_left > 0) {
 404       size_t bufsize = (bytes_left > BLOCKSIZE) ? BLOCKSIZE : bytes_left;
 405       ssize_t bytes_read;
 406
 407       char *blkbuf = buffer_block (wbuf);
 408       if ((bytes_read = full_read (fd, blkbuf, bufsize)) < 0) {
 409         return 0;
 410       }
 411
 412       if (!bytes_read) {
 413         fprintf (stderr, "ERROR: got unexpected EOF\n");
 414         return 0;
 415       }
 416
 417       memset (blkbuf + bytes_read, 0, BLOCKSIZE - bytes_read);
 418
 419       dumped_size += bytes_read;
 420
 421       bytes_left -= bytes_read;
 422     }
 423   }
 424
 425   return 1;
 426 }
 427
 428 int
 429 main (int argc, char **argv)
 430 {
 431   struct sigaction sa;
 432   int sparse = 0;
 433
 434   while (1) {
 435     int option_index = 0;
 436     static struct option long_options[] = {
 437       {"sparse", 0, 0, 's'},
 438       {"output", 1, 0, 'o'},
 439       {0, 0, 0, 0}
 440     };
 441
 442     char c = getopt_long (argc, argv, "so:", long_options, &option_index);
 443     if (c == -1)
 444       break;
 445
 446     switch (c) {
 447     case 's':
 448       sparse = 1;
 449       break;
 450     case 'o':
 451       outname = optarg;
 452       break;
 453     default:
 454       fprintf (stderr, "?? getopt returned character code 0%o ??\n", c);
 455       exit (-1);
 456     }
 457   }
 458
 459   int numargs = argc - optind;
 460   if (numargs <= 0 || (numargs % 2)) {
 461       fprintf (stderr, "wrong number of arguments\n");
 462       exit (-1);
 463   }
 464
 465   time_t starttime = time(NULL);
 466
 467   int outfd;
 468
 469   if (outname) {
 470     if ((outfd = open(outname, O_WRONLY|O_CREAT|O_TRUNC, 0644)) == -1) {
 471       fprintf (stderr, "unable to open archive '%s' - %s\n",
 472                outname, strerror (errno));
 473       exit (-1);
 474     }
 475     atexit(cleanup);
 476   } else {
 477     outfd = fileno (stdout);
 478   }
 479
 480   setsig(&sa, SIGINT, term_handler, SA_RESTART);
 481   setsig(&sa, SIGQUIT, term_handler, SA_RESTART);
 482   setsig(&sa, SIGTERM, term_handler, SA_RESTART);
 483   setsig(&sa, SIGPIPE, term_handler, SA_RESTART);
 484
 485   int saved_optind = optind;
 486   while (optind < argc) {
 487     char *source = argv[optind];
 488     optind += 2;
 489     struct stat fs;
 490
 491     if (stat (source, &fs) != 0) {
 492       fprintf (stderr, "unable to read '%s' - %s\n",
 493                source, strerror (errno));
 494       exit (-1);
 495     }
 496
 497     if (!(S_ISREG(fs.st_mode) || S_ISBLK(fs.st_mode))) {
 498       fprintf (stderr, "unable to read '%s' - not a file or block device\n",
 499                source);
 500       exit (-1);
 501     }
 502   }
 503
 504   optind = saved_optind;
 505
 506   struct writebuffer *wbuf = buffer_new (outfd);
 507
 508   while (optind < argc) {
 509     char *source = argv[optind++];
 510     char *archivename = argv[optind++];
 511
 512     int fd;
 513
 514     fprintf (stderr, "adding '%s' to archive ('%s')\n", source, archivename);
 515
 516     if ((fd = open(source, O_RDONLY)) == -1) {
 517       fprintf (stderr, "unable to open '%s' - %s\n",
 518                source, strerror (errno));
 519       exit (-1);
 520     }
 521
 522     struct stat fs;
 523
 524     if (fstat (fd, &fs) != 0) {
 525       fprintf (stderr, "unable to stat '%s' - %s\n",
 526                source, strerror (errno));
 527       exit (-1);
 528     }
 529
 530     time_t ctime = fs.st_mtime;
 531
 532     struct sp_array *ma = sparray_new();
 533     if (sparse && !S_ISBLK(fs.st_mode)) {
 534       if (!scan_sparse_file (fd, ma)) {
 535         fprintf (stderr, "scanning '%s' failed\n", source);
 536         exit (-1);
 537       }
 538     } else {
 539       off_t file_size = lseek(fd, 0, SEEK_END);
 540       if (file_size < 0) {
 541         fprintf (stderr, "unable to get file size of '%s'\n", source);
 542         exit (-1);
 543       }
 544       sparray_add (ma, 0, file_size);
 545       ma->real_size = file_size;
 546       ma->effective_size = file_size;
 547     }
 548
 549     dump_header (wbuf, archivename, ctime, ma);
 550
 551     if (!dump_sparse_file (fd, wbuf, ma)) {
 552       fprintf (stderr, "writing '%s' to archive failed\n", source);
 553       exit (-1);
 554     }
 555
 556     free (ma);
 557
 558     close (fd);
 559
 560   }
 561
 562   // write tar end
 563   char *buf = buffer_block (wbuf);
 564   memset (buf, 0, BLOCKSIZE);
 565   buf = buffer_block (wbuf);
 566   memset (buf, 0, BLOCKSIZE);
 567
 568   buffer_flush (wbuf);
 569
 570   close (outfd);
 571
 572   time_t delay = time(NULL) - starttime;
 573   if (delay <= 0) delay = 1;
 574
 575   fprintf (stderr, "Total bytes written: %zu (%.2f MiB/s)\n", wbuf->total,
 576            (wbuf->total/(1024*1024))/(float)delay);
 577
 578   outname = NULL;
 579
 580   exit (0);
 581 }