ceph/src/tools/rbd/action/Import.cc

   1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
   2 // vim: ts=8 sw=2 smarttab
   3
   4 #include "tools/rbd/ArgumentTypes.h"
   5 #include "tools/rbd/Shell.h"
   6 #include "tools/rbd/Utils.h"
   7 #include "include/Context.h"
   8 #include "common/blkdev.h"
   9 #include "common/debug.h"
  10 #include "common/errno.h"
  11 #include "common/Throttle.h"
  12 #include "include/compat.h"
  13 #include "include/encoding.h"
  14 #include "common/debug.h"
  15 #include "common/errno.h"
  16 #include "common/safe_io.h"
  17 #include <iostream>
  18 #include <boost/program_options.hpp>
  19 #include <boost/scoped_ptr.hpp>
  20 #include "include/assert.h"
  21
  22 #define dout_context g_ceph_context
  23 #define dout_subsys ceph_subsys_rbd
  24
  25 namespace rbd {
  26 namespace action {
  27 namespace import {
  28
  29 struct ImportDiffContext {
  30   librbd::Image *image;
  31   int fd;
  32   size_t size;
  33   utils::ProgressContext pc;
  34   OrderedThrottle throttle;
  35   uint64_t last_offset;
  36
  37   ImportDiffContext(librbd::Image *image, int fd, size_t size, bool no_progress)
  38     : image(image), fd(fd), size(size), pc("Importing image diff", no_progress),
  39       throttle((fd == STDIN_FILENO) ? 1 :
  40                   g_conf->get_val<int64_t>("rbd_concurrent_management_ops"),
  41                false),
  42       last_offset(0) {
  43   }
  44
  45   void update_size(size_t new_size)
  46   {
  47     if (fd == STDIN_FILENO) {
  48       size = new_size;
  49     }
  50   }
  51
  52   void update_progress(uint64_t off)
  53   {
  54     if (size) {
  55       pc.update_progress(off, size);
  56       last_offset = off;
  57     }
  58   }
  59
  60   void update_progress()
  61   {
  62     uint64_t off = last_offset;
  63     if (fd != STDIN_FILENO) {
  64       off = lseek(fd, 0, SEEK_CUR);
  65     }
  66
  67     update_progress(off);
  68   }
  69
  70   void finish(int r)
  71   {
  72     if (r < 0) {
  73       pc.fail();
  74     } else {
  75       pc.finish();
  76     }
  77   }
  78 };
  79
  80 class C_ImportDiff : public Context {
  81 public:
  82   C_ImportDiff(ImportDiffContext *idiffctx, bufferlist data, uint64_t offset,
  83                uint64_t length, bool discard)
  84     : m_idiffctx(idiffctx), m_data(data), m_offset(offset), m_length(length),
  85       m_discard(discard) {
  86     // use block offset (stdin) or import file position to report
  87     // progress.
  88     if (m_idiffctx->fd == STDIN_FILENO) {
  89       m_prog_offset = offset;
  90     } else {
  91       m_prog_offset  = lseek(m_idiffctx->fd, 0, SEEK_CUR);
  92     }
  93   }
  94
  95   int send()
  96   {
  97     if (m_idiffctx->throttle.pending_error()) {
  98       return m_idiffctx->throttle.wait_for_ret();
  99     }
 100
 101     C_OrderedThrottle *ctx = m_idiffctx->throttle.start_op(this);
 102     librbd::RBD::AioCompletion *aio_completion =
 103       new librbd::RBD::AioCompletion(ctx, &utils::aio_context_callback);
 104
 105     int r;
 106     if (m_discard) {
 107       r = m_idiffctx->image->aio_discard(m_offset, m_length, aio_completion);
 108     } else {
 109       r = m_idiffctx->image->aio_write2(m_offset, m_length, m_data,
 110                                         aio_completion, LIBRADOS_OP_FLAG_FADVISE_NOCACHE);
 111     }
 112
 113     if (r < 0) {
 114       aio_completion->release();
 115       ctx->complete(r);
 116     }
 117
 118     return r;
 119   }
 120
 121   void finish(int r) override
 122   {
 123     m_idiffctx->update_progress(m_prog_offset);
 124     m_idiffctx->throttle.end_op(r);
 125   }
 126
 127 private:
 128   ImportDiffContext *m_idiffctx;
 129   bufferlist m_data;
 130   uint64_t m_offset;
 131   uint64_t m_length;
 132   bool m_discard;
 133   uint64_t m_prog_offset;
 134 };
 135
 136 static int do_image_snap_from(ImportDiffContext *idiffctx)
 137 {
 138   int r;
 139   string from;
 140   r = utils::read_string(idiffctx->fd, 4096, &from);   // 4k limit to make sure we don't get a garbage string
 141   if (r < 0) {
 142     std::cerr << "rbd: failed to decode start snap name" << std::endl;
 143     return r;
 144   }
 145
 146   bool exists;
 147   r = idiffctx->image->snap_exists2(from.c_str(), &exists);
 148   if (r < 0) {
 149     std::cerr << "rbd: failed to query start snap state" << std::endl;
 150     return r;
 151   }
 152
 153   if (!exists) {
 154     std::cerr << "start snapshot '" << from
 155               << "' does not exist in the image, aborting" << std::endl;
 156     return -EINVAL;
 157   }
 158
 159   idiffctx->update_progress();
 160   return 0;
 161 }
 162
 163 static int do_image_snap_to(ImportDiffContext *idiffctx, std::string *tosnap)
 164 {
 165   int r;
 166   string to;
 167   r = utils::read_string(idiffctx->fd, 4096, &to);   // 4k limit to make sure we don't get a garbage string
 168   if (r < 0) {
 169     std::cerr << "rbd: failed to decode end snap name" << std::endl;
 170     return r;
 171   }
 172
 173   bool exists;
 174   r = idiffctx->image->snap_exists2(to.c_str(), &exists);
 175   if (r < 0) {
 176     std::cerr << "rbd: failed to query end snap state" << std::endl;
 177     return r;
 178   }
 179
 180   if (exists) {
 181     std::cerr << "end snapshot '" << to << "' already exists, aborting"
 182               << std::endl;
 183     return -EEXIST;
 184   }
 185
 186   *tosnap = to;
 187   idiffctx->update_progress();
 188
 189   return 0;
 190 }
 191
 192 static int get_snap_protection_status(ImportDiffContext *idiffctx,
 193                                       bool *is_protected)
 194 {
 195   int r;
 196   char buf[sizeof(__u8)];
 197   r = safe_read_exact(idiffctx->fd, buf, sizeof(buf));
 198   if (r < 0) {
 199     std::cerr << "rbd: failed to decode snap protection status" << std::endl;
 200     return r;
 201   }
 202
 203   *is_protected = (buf[0] != 0);
 204   idiffctx->update_progress();
 205
 206   return 0;
 207 }
 208
 209 static int do_image_resize(ImportDiffContext *idiffctx)
 210 {
 211   int r;
 212   char buf[sizeof(uint64_t)];
 213   uint64_t end_size;
 214   r = safe_read_exact(idiffctx->fd, buf, sizeof(buf));
 215   if (r < 0) {
 216     std::cerr << "rbd: failed to decode image size" << std::endl;
 217     return r;
 218   }
 219
 220   bufferlist bl;
 221   bl.append(buf, sizeof(buf));
 222   bufferlist::iterator p = bl.begin();
 223   ::decode(end_size, p);
 224
 225   uint64_t cur_size;
 226   idiffctx->image->size(&cur_size);
 227   if (cur_size != end_size) {
 228     idiffctx->image->resize(end_size);
 229   }
 230
 231   idiffctx->update_size(end_size);
 232   idiffctx->update_progress();
 233   return 0;
 234 }
 235
 236 static int do_image_io(ImportDiffContext *idiffctx, bool discard, size_t sparse_size)
 237 {
 238   int r;
 239   char buf[16];
 240   r = safe_read_exact(idiffctx->fd, buf, sizeof(buf));
 241   if (r < 0) {
 242     std::cerr << "rbd: failed to decode IO length" << std::endl;
 243     return r;
 244   }
 245
 246   bufferlist bl;
 247   bl.append(buf, sizeof(buf));
 248   bufferlist::iterator p = bl.begin();
 249
 250   uint64_t image_offset, buffer_length;
 251   ::decode(image_offset, p);
 252   ::decode(buffer_length, p);
 253
 254   if (!discard) {
 255     bufferptr bp = buffer::create(buffer_length);
 256     r = safe_read_exact(idiffctx->fd, bp.c_str(), buffer_length);
 257     if (r < 0) {
 258       std::cerr << "rbd: failed to decode write data" << std::endl;
 259       return r;
 260     }
 261
 262     size_t buffer_offset = 0;
 263     while (buffer_offset < buffer_length) {
 264       size_t write_length = 0;
 265       bool zeroed = false;
 266       utils::calc_sparse_extent(bp, sparse_size, buffer_offset, buffer_length,
 267                                 &write_length, &zeroed);
 268       assert(write_length > 0);
 269
 270       bufferlist write_bl;
 271       if (!zeroed) {
 272         bufferptr write_ptr(bp, buffer_offset, write_length);
 273         write_bl.push_back(write_ptr);
 274         assert(write_bl.length() == write_length);
 275       }
 276
 277       C_ImportDiff *ctx = new C_ImportDiff(idiffctx, write_bl,
 278                                            image_offset + buffer_offset,
 279                                            write_length, zeroed);
 280       r = ctx->send();
 281       if (r < 0) {
 282         return r;
 283       }
 284
 285       buffer_offset += write_length;
 286     }
 287   } else {
 288     bufferlist data;
 289     C_ImportDiff *ctx = new C_ImportDiff(idiffctx, data, image_offset,
 290                                          buffer_length, true);
 291     return ctx->send();
 292   }
 293   return r;
 294 }
 295
 296 static int validate_banner(int fd, std::string banner)
 297 {
 298   int r;
 299   char buf[banner.size() + 1];
 300   memset(buf, 0, sizeof(buf));
 301   r = safe_read_exact(fd, buf, banner.size());
 302   if (r < 0) {
 303     std::cerr << "rbd: failed to decode diff banner" << std::endl;
 304     return r;
 305   }
 306
 307   buf[banner.size()] = '\0';
 308   if (strcmp(buf, banner.c_str())) {
 309     std::cerr << "rbd: invalid or unexpected diff banner" << std::endl;
 310     return -EINVAL;
 311   }
 312
 313   return 0;
 314 }
 315
 316 static int skip_tag(int fd, uint64_t length)
 317 {
 318   int r;
 319
 320   if (fd == STDIN_FILENO) {
 321     // read the appending data out to skip this tag.
 322     char buf[4096];
 323     uint64_t len = min<uint64_t>(length, sizeof(buf));
 324     while (len > 0) {
 325       r = safe_read_exact(fd, buf, len);
 326       if (r < 0) {
 327         std::cerr << "rbd: failed to decode skipped tag data" << std::endl;
 328         return r;
 329       }
 330       length -= len;
 331       len = min<uint64_t>(length, sizeof(buf));
 332     }
 333   } else {
 334     // lseek to skip this tag
 335     off64_t offs = lseek64(fd, length, SEEK_CUR);
 336     if (offs < 0) {
 337       return -errno;
 338     }
 339   }
 340
 341   return 0;
 342 }
 343
 344 static int read_tag(int fd, __u8 end_tag, int format, __u8 *tag, uint64_t *readlen)
 345 {
 346   int r;
 347   __u8 read_tag;
 348
 349   r = safe_read_exact(fd, &read_tag, sizeof(read_tag));
 350   if (r < 0) {
 351     std::cerr << "rbd: failed to decode tag" << std::endl;
 352     return r;
 353   }
 354
 355   *tag = read_tag;
 356   if (read_tag != end_tag && format == 2) {
 357     char buf[sizeof(uint64_t)];
 358     r = safe_read_exact(fd, buf, sizeof(buf));
 359     if (r < 0) {
 360       std::cerr << "rbd: failed to decode tag length" << std::endl;
 361       return r;
 362     }
 363
 364     bufferlist bl;
 365     bl.append(buf, sizeof(buf));
 366     bufferlist::iterator p = bl.begin();
 367     ::decode(*readlen, p);
 368   }
 369
 370   return 0;
 371 }
 372
 373 int do_import_diff_fd(librados::Rados &rados, librbd::Image &image, int fd,
 374                       bool no_progress, int format, size_t sparse_size)
 375 {
 376   int r;
 377
 378   uint64_t size = 0;
 379   bool from_stdin = (fd == STDIN_FILENO);
 380   if (!from_stdin) {
 381     struct stat stat_buf;
 382     r = ::fstat(fd, &stat_buf);
 383     if (r < 0) {
 384       std::cerr << "rbd: failed to stat specified diff file" << std::endl;
 385       return r;
 386     }
 387     size = (uint64_t)stat_buf.st_size;
 388   }
 389
 390   r = validate_banner(fd, (format == 1 ? utils::RBD_DIFF_BANNER :
 391                            utils::RBD_DIFF_BANNER_V2));
 392   if (r < 0) {
 393     return r;
 394   }
 395
 396   std::string skip_partial_discard;
 397   r = rados.conf_get("rbd_skip_partial_discard", skip_partial_discard);
 398   if (r < 0 || skip_partial_discard != "false") {
 399     dout(1) << "disabling sparse import" << dendl;
 400     sparse_size = 0;
 401     r = 0;
 402   }
 403
 404   // begin image import
 405   std::string tosnap;
 406   bool is_protected = false;
 407   ImportDiffContext idiffctx(&image, fd, size, no_progress);
 408   while (r == 0) {
 409     __u8 tag;
 410     uint64_t length = 0;
 411
 412     r = read_tag(fd, RBD_DIFF_END, format, &tag, &length);
 413     if (r < 0 || tag == RBD_DIFF_END) {
 414       break;
 415     }
 416
 417     if (tag == RBD_DIFF_FROM_SNAP) {
 418       r = do_image_snap_from(&idiffctx);
 419     } else if (tag == RBD_DIFF_TO_SNAP) {
 420       r = do_image_snap_to(&idiffctx, &tosnap);
 421     } else if (tag == RBD_SNAP_PROTECTION_STATUS) {
 422       r = get_snap_protection_status(&idiffctx, &is_protected);
 423     } else if (tag == RBD_DIFF_IMAGE_SIZE) {
 424       r = do_image_resize(&idiffctx);
 425     } else if (tag == RBD_DIFF_WRITE || tag == RBD_DIFF_ZERO) {
 426       r = do_image_io(&idiffctx, (tag == RBD_DIFF_ZERO), sparse_size);
 427     } else {
 428       std::cerr << "unrecognized tag byte " << (int)tag << " in stream; skipping"
 429                 << std::endl;
 430       r = skip_tag(fd, length);
 431     }
 432   }
 433
 434   int temp_r = idiffctx.throttle.wait_for_ret();
 435   r = (r < 0) ? r : temp_r; // preserve original error
 436   if (r == 0 && tosnap.length()) {
 437     r = idiffctx.image->snap_create(tosnap.c_str());
 438     if (r == 0 && is_protected) {
 439       r = idiffctx.image->snap_protect(tosnap.c_str());
 440     }
 441   }
 442
 443   idiffctx.finish(r);
 444   return r;
 445 }
 446
 447 int do_import_diff(librados::Rados &rados, librbd::Image &image,
 448                    const char *path, bool no_progress, size_t sparse_size)
 449 {
 450   int r;
 451   int fd;
 452
 453   if (strcmp(path, "-") == 0) {
 454     fd = STDIN_FILENO;
 455   } else {
 456     fd = open(path, O_RDONLY);
 457     if (fd < 0) {
 458       r = -errno;
 459       std::cerr << "rbd: error opening " << path << std::endl;
 460       return r;
 461     }
 462   }
 463   r = do_import_diff_fd(rados, image, fd, no_progress, 1, sparse_size);
 464
 465   if (fd != 0)
 466     close(fd);
 467   return r;
 468 }
 469
 470 namespace at = argument_types;
 471 namespace po = boost::program_options;
 472
 473 void get_arguments_diff(po::options_description *positional,
 474                    po::options_description *options) {
 475   at::add_path_options(positional, options,
 476                        "import file (or '-' for stdin)");
 477   at::add_image_spec_options(positional, options, at::ARGUMENT_MODIFIER_NONE);
 478   at::add_sparse_size_option(options);
 479   at::add_no_progress_option(options);
 480 }
 481
 482 int execute_diff(const po::variables_map &vm) {
 483   std::string path;
 484   int r = utils::get_path(vm, utils::get_positional_argument(vm, 0), &path);
 485   if (r < 0) {
 486     return r;
 487   }
 488
 489   size_t arg_index = 1;
 490   std::string pool_name;
 491   std::string image_name;
 492   std::string snap_name;
 493   r = utils::get_pool_image_snapshot_names(
 494     vm, at::ARGUMENT_MODIFIER_NONE, &arg_index, &pool_name, &image_name,
 495     &snap_name, utils::SNAPSHOT_PRESENCE_NONE, utils::SPEC_VALIDATION_NONE);
 496   if (r < 0) {
 497     return r;
 498   }
 499
 500   size_t sparse_size = utils::RBD_DEFAULT_SPARSE_SIZE;
 501   if (vm.count(at::IMAGE_SPARSE_SIZE)) {
 502     sparse_size = vm[at::IMAGE_SPARSE_SIZE].as<size_t>();
 503   }
 504
 505   librados::Rados rados;
 506   librados::IoCtx io_ctx;
 507   librbd::Image image;
 508   r = utils::init_and_open_image(pool_name, image_name, "", "", false,
 509                                  &rados, &io_ctx, &image);
 510   if (r < 0) {
 511     return r;
 512   }
 513
 514   r = do_import_diff(rados, image, path.c_str(),
 515                      vm[at::NO_PROGRESS].as<bool>(), sparse_size);
 516   if (r == -EDOM) {
 517     r = -EBADMSG;
 518   }
 519   if (r < 0) {
 520     cerr << "rbd: import-diff failed: " << cpp_strerror(r) << std::endl;
 521     return r;
 522   }
 523   return 0;
 524 }
 525
 526 Shell::Action action_diff(
 527   {"import-diff"}, {}, "Import an incremental diff.", "", &get_arguments_diff,
 528   &execute_diff);
 529
 530 class C_Import : public Context {
 531 public:
 532   C_Import(SimpleThrottle &simple_throttle, librbd::Image &image,
 533            bufferlist &bl, uint64_t offset)
 534     : m_throttle(simple_throttle), m_image(image),
 535       m_aio_completion(
 536         new librbd::RBD::AioCompletion(this, &utils::aio_context_callback)),
 537       m_bufferlist(bl), m_offset(offset)
 538   {
 539   }
 540
 541   void send()
 542   {
 543     m_throttle.start_op();
 544
 545     int op_flags = LIBRADOS_OP_FLAG_FADVISE_SEQUENTIAL |
 546                    LIBRADOS_OP_FLAG_FADVISE_NOCACHE;
 547     int r = m_image.aio_write2(m_offset, m_bufferlist.length(), m_bufferlist,
 548                                m_aio_completion, op_flags);
 549     if (r < 0) {
 550       std::cerr << "rbd: error requesting write to destination image"
 551                 << std::endl;
 552       m_aio_completion->release();
 553       m_throttle.end_op(r);
 554     }
 555   }
 556
 557   void finish(int r) override
 558   {
 559     if (r < 0) {
 560       std::cerr << "rbd: error writing to destination image at offset "
 561                 << m_offset << ": " << cpp_strerror(r) << std::endl;
 562     }
 563     m_throttle.end_op(r);
 564   }
 565
 566 private:
 567   SimpleThrottle &m_throttle;
 568   librbd::Image &m_image;
 569   librbd::RBD::AioCompletion *m_aio_completion;
 570   bufferlist m_bufferlist;
 571   uint64_t m_offset;
 572 };
 573
 574 static int decode_and_set_image_option(int fd, uint64_t imageopt, librbd::ImageOptions& opts)
 575 {
 576   int r;
 577   char buf[sizeof(uint64_t)];
 578
 579   r = safe_read_exact(fd, buf, sizeof(buf));
 580   if (r < 0) {
 581     std::cerr << "rbd: failed to decode image option" << std::endl;
 582     return r;
 583   }
 584
 585   bufferlist bl;
 586   bl.append(buf, sizeof(buf));
 587   bufferlist::iterator it;
 588   it = bl.begin();
 589
 590   uint64_t val;
 591   ::decode(val, it);
 592
 593   if (opts.get(imageopt, &val) != 0) {
 594     opts.set(imageopt, val);
 595   }
 596
 597   return 0;
 598 }
 599
 600 static int do_import_metadata(int import_format, librbd::Image& image,
 601                               const std::map<std::string, std::string> &imagemetas)
 602 {
 603   int r = 0;
 604
 605   //v1 format
 606   if (import_format == 1) {
 607     return 0;
 608   }
 609
 610   for (std::map<std::string, std::string>::const_iterator it = imagemetas.begin();
 611        it != imagemetas.end(); ++it) {
 612     r = image.metadata_set(it->first, it->second);
 613     if (r < 0)
 614       return r;
 615   }
 616
 617   return 0;
 618 }
 619
 620 static int decode_imagemeta(int fd, uint64_t length, std::map<std::string, std::string>* imagemetas)
 621 {
 622   int r;
 623   string key;
 624   string value;
 625
 626   r = utils::read_string(fd, length, &key);
 627   if (r < 0) {
 628     std::cerr << "rbd: failed to decode metadata key" << std::endl;
 629     return r;
 630   }
 631
 632   r = utils::read_string(fd, length, &value);
 633   if (r < 0) {
 634     std::cerr << "rbd: failed to decode metadata value" << std::endl;
 635     return r;
 636   }
 637
 638   (*imagemetas)[key] = value;
 639   return 0;
 640 }
 641
 642 static int do_import_header(int fd, int import_format, uint64_t &size, librbd::ImageOptions& opts,
 643                             std::map<std::string, std::string>* imagemetas)
 644 {
 645   // There is no header in v1 image.
 646   if (import_format == 1) {
 647     return 0;
 648   }
 649
 650   if (fd == STDIN_FILENO || size < utils::RBD_IMAGE_BANNER_V2.size()) {
 651     return -EINVAL;
 652   }
 653
 654   int r;
 655   r = validate_banner(fd, utils::RBD_IMAGE_BANNER_V2);
 656   if (r < 0) {
 657     return r;
 658   }
 659
 660   // As V1 format for image is already deprecated, import image in V2 by default.
 661   uint64_t image_format = 2;
 662   if (opts.get(RBD_IMAGE_OPTION_FORMAT, &image_format) != 0) {
 663     opts.set(RBD_IMAGE_OPTION_FORMAT, image_format);
 664   }
 665
 666   while (r == 0) {
 667     __u8 tag;
 668     uint64_t length;
 669     r = read_tag(fd, RBD_EXPORT_IMAGE_END, image_format, &tag, &length);
 670     if (r < 0 || tag == RBD_EXPORT_IMAGE_END) {
 671       break;
 672     }
 673
 674     if (tag == RBD_EXPORT_IMAGE_ORDER) {
 675       r = decode_and_set_image_option(fd, RBD_IMAGE_OPTION_ORDER, opts);
 676     } else if (tag == RBD_EXPORT_IMAGE_FEATURES) {
 677       r = decode_and_set_image_option(fd, RBD_IMAGE_OPTION_FEATURES, opts);
 678     } else if (tag == RBD_EXPORT_IMAGE_STRIPE_UNIT) {
 679       r = decode_and_set_image_option(fd, RBD_IMAGE_OPTION_STRIPE_UNIT, opts);
 680     } else if (tag == RBD_EXPORT_IMAGE_STRIPE_COUNT) {
 681       r = decode_and_set_image_option(fd, RBD_IMAGE_OPTION_STRIPE_COUNT, opts);
 682     } else if (tag == RBD_EXPORT_IMAGE_META) {
 683       r = decode_imagemeta(fd, length, imagemetas);
 684     } else {
 685       std::cerr << "rbd: invalid tag in image properties zone: " << tag << "Skip it."
 686                 << std::endl;
 687       r = skip_tag(fd, length);
 688     }
 689   }
 690
 691   return r;
 692 }
 693
 694 static int do_import_v2(librados::Rados &rados, int fd, librbd::Image &image,
 695                         uint64_t size, size_t imgblklen,
 696                         utils::ProgressContext &pc, size_t sparse_size)
 697 {
 698   int r = 0;
 699   r = validate_banner(fd, utils::RBD_IMAGE_DIFFS_BANNER_V2);
 700   if (r < 0) {
 701     return r;
 702   }
 703
 704   char buf[sizeof(uint64_t)];
 705   r = safe_read_exact(fd, buf, sizeof(buf));
 706   if (r < 0) {
 707     std::cerr << "rbd: failed to decode diff count" << std::endl;
 708     return r;
 709   }
 710   bufferlist bl;
 711   bl.append(buf, sizeof(buf));
 712   bufferlist::iterator p = bl.begin();
 713   uint64_t diff_num;
 714   ::decode(diff_num, p);
 715
 716   for (size_t i = 0; i < diff_num; i++) {
 717     r = do_import_diff_fd(rados, image, fd, true, 2, sparse_size);
 718     if (r < 0) {
 719       pc.fail();
 720       std::cerr << "rbd: import-diff failed: " << cpp_strerror(r) << std::endl;
 721       return r;
 722     }
 723     pc.update_progress(i + 1, diff_num);
 724   }
 725
 726   return r;
 727 }
 728
 729 static int do_import_v1(int fd, librbd::Image &image, uint64_t size,
 730                         size_t imgblklen, utils::ProgressContext &pc,
 731                         size_t sparse_size)
 732 {
 733   int r = 0;
 734   size_t reqlen = imgblklen;    // amount requested from read
 735   ssize_t readlen;              // amount received from one read
 736   size_t blklen = 0;            // amount accumulated from reads to fill blk
 737   char *p = new char[imgblklen];
 738   uint64_t image_pos = 0;
 739   bool from_stdin = (fd == STDIN_FILENO);
 740   boost::scoped_ptr<SimpleThrottle> throttle;
 741
 742   if (from_stdin) {
 743     throttle.reset(new SimpleThrottle(1, false));
 744   } else {
 745     throttle.reset(new SimpleThrottle(
 746       g_conf->get_val<int64_t>("rbd_concurrent_management_ops"), false));
 747   }
 748
 749   reqlen = min<uint64_t>(reqlen, size);
 750   // loop body handles 0 return, as we may have a block to flush
 751   while ((readlen = ::read(fd, p + blklen, reqlen)) >= 0) {
 752     if (throttle->pending_error()) {
 753       break;
 754     }
 755
 756     blklen += readlen;
 757     // if read was short, try again to fill the block before writing
 758     if (readlen && ((size_t)readlen < reqlen)) {
 759       reqlen -= readlen;
 760       continue;
 761     }
 762     if (!from_stdin)
 763       pc.update_progress(image_pos, size);
 764
 765     bufferptr blkptr(p, blklen);
 766     // resize output image by binary expansion as we go for stdin
 767     if (from_stdin && (image_pos + (size_t)blklen) > size) {
 768       size *= 2;
 769       r = image.resize(size);
 770       if (r < 0) {
 771         std::cerr << "rbd: can't resize image during import" << std::endl;
 772         goto out;
 773       }
 774     }
 775
 776     // write as much as we got; perhaps less than imgblklen
 777     // but skip writing zeros to create sparse images
 778     size_t buffer_offset = 0;
 779     while (buffer_offset < blklen) {
 780       size_t write_length = 0;
 781       bool zeroed = false;
 782       utils::calc_sparse_extent(blkptr, sparse_size, buffer_offset, blklen,
 783                                 &write_length, &zeroed);
 784
 785       if (!zeroed) {
 786         bufferlist write_bl;
 787         bufferptr write_ptr(blkptr, buffer_offset, write_length);
 788         write_bl.push_back(write_ptr);
 789         assert(write_bl.length() == write_length);
 790
 791         C_Import *ctx = new C_Import(*throttle, image, write_bl,
 792                                      image_pos + buffer_offset);
 793         ctx->send();
 794       }
 795
 796       buffer_offset += write_length;
 797     }
 798
 799     // done with whole block, whether written or not
 800     image_pos += blklen;
 801     if (!from_stdin && image_pos >= size)
 802       break;
 803     // if read had returned 0, we're at EOF and should quit
 804     if (readlen == 0)
 805       break;
 806     blklen = 0;
 807     reqlen = imgblklen;
 808   }
 809   r = throttle->wait_for_ret();
 810   if (r < 0) {
 811     goto out;
 812   }
 813
 814   if (fd == STDIN_FILENO) {
 815     r = image.resize(image_pos);
 816     if (r < 0) {
 817       std::cerr << "rbd: final image resize failed" << std::endl;
 818       goto out;
 819     }
 820   }
 821 out:
 822   delete[] p;
 823   return r;
 824 }
 825
 826 static int do_import(librados::Rados &rados, librbd::RBD &rbd,
 827                      librados::IoCtx& io_ctx, const char *imgname,
 828                      const char *path, librbd::ImageOptions& opts,
 829                      bool no_progress, int import_format, size_t sparse_size)
 830 {
 831   int fd, r;
 832   struct stat stat_buf;
 833   utils::ProgressContext pc("Importing image", no_progress);
 834   std::map<std::string, std::string> imagemetas;
 835
 836   assert(imgname);
 837
 838   uint64_t order;
 839   if (opts.get(RBD_IMAGE_OPTION_ORDER, &order) != 0) {
 840     order = g_conf->get_val<int64_t>("rbd_default_order");
 841   }
 842
 843   // try to fill whole imgblklen blocks for sparsification
 844   size_t imgblklen = 1 << order;
 845   librbd::Image image;
 846   uint64_t size = 0;
 847
 848   bool from_stdin = !strcmp(path, "-");
 849   if (from_stdin) {
 850     fd = STDIN_FILENO;
 851     size = 1ULL << order;
 852   } else {
 853     if ((fd = open(path, O_RDONLY)) < 0) {
 854       r = -errno;
 855       std::cerr << "rbd: error opening " << path << std::endl;
 856       goto done2;
 857     }
 858
 859     if ((fstat(fd, &stat_buf)) < 0) {
 860       r = -errno;
 861       std::cerr << "rbd: stat error " << path << std::endl;
 862       goto done;
 863     }
 864     if (S_ISDIR(stat_buf.st_mode)) {
 865       r = -EISDIR;
 866       std::cerr << "rbd: cannot import a directory" << std::endl;
 867       goto done;
 868     }
 869     if (stat_buf.st_size)
 870       size = (uint64_t)stat_buf.st_size;
 871
 872     if (!size) {
 873       int64_t bdev_size = 0;
 874       r = get_block_device_size(fd, &bdev_size);
 875       if (r < 0) {
 876         std::cerr << "rbd: unable to get size of file/block device"
 877                   << std::endl;
 878         goto done;
 879       }
 880       assert(bdev_size >= 0);
 881       size = (uint64_t) bdev_size;
 882     }
 883 #ifdef HAVE_POSIX_FADVISE
 884     posix_fadvise(fd, 0, 0, POSIX_FADV_SEQUENTIAL);
 885 #endif
 886   }
 887
 888   r = do_import_header(fd, import_format, size, opts, &imagemetas);
 889   if (r < 0) {
 890     std::cerr << "rbd: import header failed." << std::endl;
 891     goto done;
 892   }
 893
 894   r = rbd.create4(io_ctx, imgname, size, opts);
 895   if (r < 0) {
 896     std::cerr << "rbd: image creation failed" << std::endl;
 897     goto done;
 898   }
 899
 900   r = rbd.open(io_ctx, image, imgname);
 901   if (r < 0) {
 902     std::cerr << "rbd: failed to open image" << std::endl;
 903     goto err;
 904   }
 905
 906   r = do_import_metadata(import_format, image, imagemetas);
 907   if (r < 0) {
 908     std::cerr << "rbd: failed to import image-meta" << std::endl;
 909     goto err;
 910   }
 911
 912   if (import_format == 1) {
 913     r = do_import_v1(fd, image, size, imgblklen, pc, sparse_size);
 914   } else {
 915     r = do_import_v2(rados, fd, image, size, imgblklen, pc, sparse_size);
 916   }
 917   if (r < 0) {
 918     std::cerr << "rbd: failed to import image" << std::endl;
 919     image.close();
 920     goto err;
 921   }
 922
 923   r = image.close();
 924 err:
 925   if (r < 0)
 926     rbd.remove(io_ctx, imgname);
 927 done:
 928   if (r < 0)
 929     pc.fail();
 930   else
 931     pc.finish();
 932   if (!from_stdin)
 933     close(fd);
 934 done2:
 935   return r;
 936 }
 937
 938 void get_arguments(po::options_description *positional,
 939                    po::options_description *options) {
 940   at::add_path_options(positional, options,
 941                        "import file (or '-' for stdin)");
 942   at::add_image_spec_options(positional, options, at::ARGUMENT_MODIFIER_DEST);
 943   at::add_create_image_options(options, true);
 944   at::add_sparse_size_option(options);
 945   at::add_no_progress_option(options);
 946   at::add_export_format_option(options);
 947
 948   // TODO legacy rbd allowed import to accept both 'image'/'dest' and
 949   //      'pool'/'dest-pool'
 950   at::add_pool_option(options, at::ARGUMENT_MODIFIER_NONE, " (deprecated)");
 951   at::add_image_option(options, at::ARGUMENT_MODIFIER_NONE, " (deprecated)");
 952 }
 953
 954 int execute(const po::variables_map &vm) {
 955   std::string path;
 956   int r = utils::get_path(vm, utils::get_positional_argument(vm, 0), &path);
 957   if (r < 0) {
 958     return r;
 959   }
 960
 961   // odd check to support legacy / deprecated behavior of import
 962   std::string deprecated_pool_name;
 963   if (vm.count(at::POOL_NAME)) {
 964     deprecated_pool_name = vm[at::POOL_NAME].as<std::string>();
 965     std::cerr << "rbd: --pool is deprecated for import, use --dest-pool"
 966               << std::endl;
 967   }
 968
 969   std::string deprecated_image_name;
 970   if (vm.count(at::IMAGE_NAME)) {
 971     deprecated_image_name = vm[at::IMAGE_NAME].as<std::string>();
 972     std::cerr << "rbd: --image is deprecated for import, use --dest"
 973               << std::endl;
 974   } else {
 975     deprecated_image_name = path.substr(path.find_last_of("/") + 1);
 976   }
 977
 978   std::string deprecated_snap_name;
 979   r = utils::extract_spec(deprecated_image_name, &deprecated_pool_name,
 980                           &deprecated_image_name, &deprecated_snap_name,
 981                           utils::SPEC_VALIDATION_FULL);
 982   if (r < 0) {
 983     return r;
 984   }
 985
 986   size_t sparse_size = utils::RBD_DEFAULT_SPARSE_SIZE;
 987   if (vm.count(at::IMAGE_SPARSE_SIZE)) {
 988     sparse_size = vm[at::IMAGE_SPARSE_SIZE].as<size_t>();
 989   }
 990
 991   size_t arg_index = 1;
 992   std::string pool_name = deprecated_pool_name;
 993   std::string image_name;
 994   std::string snap_name = deprecated_snap_name;
 995   r = utils::get_pool_image_snapshot_names(
 996     vm, at::ARGUMENT_MODIFIER_DEST, &arg_index, &pool_name, &image_name,
 997     &snap_name, utils::SNAPSHOT_PRESENCE_NONE, utils::SPEC_VALIDATION_FULL,
 998     false);
 999   if (r < 0) {
1000     return r;
1001   }
1002
1003   if (image_name.empty()) {
1004     image_name = deprecated_image_name;
1005   }
1006
1007   librbd::ImageOptions opts;
1008   r = utils::get_image_options(vm, true, &opts);
1009   if (r < 0) {
1010     return r;
1011   }
1012
1013   librados::Rados rados;
1014   librados::IoCtx io_ctx;
1015   r = utils::init(pool_name, &rados, &io_ctx);
1016   if (r < 0) {
1017     return r;
1018   }
1019
1020   int format = 1;
1021   if (vm.count("export-format"))
1022     format = vm["export-format"].as<uint64_t>();
1023
1024   librbd::RBD rbd;
1025   r = do_import(rados, rbd, io_ctx, image_name.c_str(), path.c_str(),
1026                 opts, vm[at::NO_PROGRESS].as<bool>(), format, sparse_size);
1027   if (r < 0) {
1028     std::cerr << "rbd: import failed: " << cpp_strerror(r) << std::endl;
1029     return r;
1030   }
1031
1032   return 0;
1033 }
1034
1035 Shell::Action action(
1036   {"import"}, {}, "Import image from file.", at::get_long_features_help(),
1037   &get_arguments, &execute);
1038
1039 } // namespace import
1040 } // namespace action
1041 } // namespace rbd