ceph/src/tools/rbd/action/Import.cc

   1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
   2 // vim: ts=8 sw=2 smarttab
   3
   4 #include "tools/rbd/ArgumentTypes.h"
   5 #include "tools/rbd/Shell.h"
   6 #include "tools/rbd/Utils.h"
   7 #include "include/Context.h"
   8 #include "common/blkdev.h"
   9 #include "common/debug.h"
  10 #include "common/errno.h"
  11 #include "common/Throttle.h"
  12 #include "include/compat.h"
  13 #include "include/encoding.h"
  14 #include "common/debug.h"
  15 #include "common/errno.h"
  16 #include "common/safe_io.h"
  17 #include <iostream>
  18 #include <boost/program_options.hpp>
  19 #include <boost/scoped_ptr.hpp>
  20 #include "include/ceph_assert.h"
  21
  22 #define dout_context g_ceph_context
  23 #define dout_subsys ceph_subsys_rbd
  24
  25 namespace rbd {
  26 namespace action {
  27 namespace import {
  28
  29 struct ImportDiffContext {
  30   librbd::Image *image;
  31   int fd;
  32   size_t size;
  33   utils::ProgressContext pc;
  34   OrderedThrottle throttle;
  35   uint64_t last_offset;
  36
  37   ImportDiffContext(librbd::Image *image, int fd, size_t size, bool no_progress)
  38     : image(image), fd(fd), size(size), pc("Importing image diff", no_progress),
  39       throttle((fd == STDIN_FILENO) ? 1 :
  40                   g_conf().get_val<uint64_t>("rbd_concurrent_management_ops"),
  41                false),
  42       last_offset(0) {
  43   }
  44
  45   void update_size(size_t new_size)
  46   {
  47     if (fd == STDIN_FILENO) {
  48       size = new_size;
  49     }
  50   }
  51
  52   void update_progress(uint64_t off)
  53   {
  54     if (size) {
  55       pc.update_progress(off, size);
  56       last_offset = off;
  57     }
  58   }
  59
  60   void update_progress()
  61   {
  62     uint64_t off = last_offset;
  63     if (fd != STDIN_FILENO) {
  64       off = lseek(fd, 0, SEEK_CUR);
  65     }
  66
  67     update_progress(off);
  68   }
  69
  70   void finish(int r)
  71   {
  72     if (r < 0) {
  73       pc.fail();
  74     } else {
  75       pc.finish();
  76     }
  77   }
  78 };
  79
  80 class C_ImportDiff : public Context {
  81 public:
  82   C_ImportDiff(ImportDiffContext *idiffctx, bufferlist data, uint64_t offset,
  83                uint64_t length, bool write_zeroes)
  84     : m_idiffctx(idiffctx), m_data(data), m_offset(offset), m_length(length),
  85       m_write_zeroes(write_zeroes) {
  86     // use block offset (stdin) or import file position to report
  87     // progress.
  88     if (m_idiffctx->fd == STDIN_FILENO) {
  89       m_prog_offset = offset;
  90     } else {
  91       m_prog_offset  = lseek(m_idiffctx->fd, 0, SEEK_CUR);
  92     }
  93   }
  94
  95   int send()
  96   {
  97     if (m_idiffctx->throttle.pending_error()) {
  98       return m_idiffctx->throttle.wait_for_ret();
  99     }
 100
 101     C_OrderedThrottle *ctx = m_idiffctx->throttle.start_op(this);
 102     librbd::RBD::AioCompletion *aio_completion =
 103       new librbd::RBD::AioCompletion(ctx, &utils::aio_context_callback);
 104
 105     int r;
 106     if (m_write_zeroes) {
 107       r = m_idiffctx->image->aio_write_zeroes(m_offset, m_length,
 108                                               aio_completion, 0U,
 109                                               LIBRADOS_OP_FLAG_FADVISE_NOCACHE);
 110     } else {
 111       r = m_idiffctx->image->aio_write2(m_offset, m_length, m_data,
 112                                         aio_completion,
 113                                         LIBRADOS_OP_FLAG_FADVISE_NOCACHE);
 114     }
 115
 116     if (r < 0) {
 117       aio_completion->release();
 118       ctx->complete(r);
 119     }
 120
 121     return r;
 122   }
 123
 124   void finish(int r) override
 125   {
 126     m_idiffctx->update_progress(m_prog_offset);
 127     m_idiffctx->throttle.end_op(r);
 128   }
 129
 130 private:
 131   ImportDiffContext *m_idiffctx;
 132   bufferlist m_data;
 133   uint64_t m_offset;
 134   uint64_t m_length;
 135   bool m_write_zeroes;
 136   uint64_t m_prog_offset;
 137 };
 138
 139 static int do_image_snap_from(ImportDiffContext *idiffctx)
 140 {
 141   int r;
 142   string from;
 143   r = utils::read_string(idiffctx->fd, 4096, &from);   // 4k limit to make sure we don't get a garbage string
 144   if (r < 0) {
 145     std::cerr << "rbd: failed to decode start snap name" << std::endl;
 146     return r;
 147   }
 148
 149   bool exists;
 150   r = idiffctx->image->snap_exists2(from.c_str(), &exists);
 151   if (r < 0) {
 152     std::cerr << "rbd: failed to query start snap state" << std::endl;
 153     return r;
 154   }
 155
 156   if (!exists) {
 157     std::cerr << "start snapshot '" << from
 158               << "' does not exist in the image, aborting" << std::endl;
 159     return -EINVAL;
 160   }
 161
 162   idiffctx->update_progress();
 163   return 0;
 164 }
 165
 166 static int do_image_snap_to(ImportDiffContext *idiffctx, std::string *tosnap)
 167 {
 168   int r;
 169   string to;
 170   r = utils::read_string(idiffctx->fd, 4096, &to);   // 4k limit to make sure we don't get a garbage string
 171   if (r < 0) {
 172     std::cerr << "rbd: failed to decode end snap name" << std::endl;
 173     return r;
 174   }
 175
 176   bool exists;
 177   r = idiffctx->image->snap_exists2(to.c_str(), &exists);
 178   if (r < 0) {
 179     std::cerr << "rbd: failed to query end snap state" << std::endl;
 180     return r;
 181   }
 182
 183   if (exists) {
 184     std::cerr << "end snapshot '" << to << "' already exists, aborting"
 185               << std::endl;
 186     return -EEXIST;
 187   }
 188
 189   *tosnap = to;
 190   idiffctx->update_progress();
 191
 192   return 0;
 193 }
 194
 195 static int get_snap_protection_status(ImportDiffContext *idiffctx,
 196                                       bool *is_protected)
 197 {
 198   int r;
 199   char buf[sizeof(__u8)];
 200   r = safe_read_exact(idiffctx->fd, buf, sizeof(buf));
 201   if (r < 0) {
 202     std::cerr << "rbd: failed to decode snap protection status" << std::endl;
 203     return r;
 204   }
 205
 206   *is_protected = (buf[0] != 0);
 207   idiffctx->update_progress();
 208
 209   return 0;
 210 }
 211
 212 static int do_image_resize(ImportDiffContext *idiffctx)
 213 {
 214   int r;
 215   char buf[sizeof(uint64_t)];
 216   uint64_t end_size;
 217   r = safe_read_exact(idiffctx->fd, buf, sizeof(buf));
 218   if (r < 0) {
 219     std::cerr << "rbd: failed to decode image size" << std::endl;
 220     return r;
 221   }
 222
 223   bufferlist bl;
 224   bl.append(buf, sizeof(buf));
 225   auto p = bl.cbegin();
 226   decode(end_size, p);
 227
 228   uint64_t cur_size;
 229   idiffctx->image->size(&cur_size);
 230   if (cur_size != end_size) {
 231     idiffctx->image->resize(end_size);
 232   }
 233
 234   idiffctx->update_size(end_size);
 235   idiffctx->update_progress();
 236   return 0;
 237 }
 238
 239 static int do_image_io(ImportDiffContext *idiffctx, bool write_zeroes,
 240                        size_t sparse_size)
 241 {
 242   int r;
 243   char buf[16];
 244   r = safe_read_exact(idiffctx->fd, buf, sizeof(buf));
 245   if (r < 0) {
 246     std::cerr << "rbd: failed to decode IO length" << std::endl;
 247     return r;
 248   }
 249
 250   bufferlist bl;
 251   bl.append(buf, sizeof(buf));
 252   auto p = bl.cbegin();
 253
 254   uint64_t image_offset, buffer_length;
 255   decode(image_offset, p);
 256   decode(buffer_length, p);
 257
 258   if (!write_zeroes) {
 259     bufferptr bp = buffer::create(buffer_length);
 260     r = safe_read_exact(idiffctx->fd, bp.c_str(), buffer_length);
 261     if (r < 0) {
 262       std::cerr << "rbd: failed to decode write data" << std::endl;
 263       return r;
 264     }
 265
 266     size_t buffer_offset = 0;
 267     while (buffer_offset < buffer_length) {
 268       size_t write_length = 0;
 269       bool zeroed = false;
 270       utils::calc_sparse_extent(bp, sparse_size, buffer_offset, buffer_length,
 271                                 &write_length, &zeroed);
 272       ceph_assert(write_length > 0);
 273
 274       bufferlist write_bl;
 275       if (!zeroed) {
 276         bufferptr write_ptr(bp, buffer_offset, write_length);
 277         write_bl.push_back(write_ptr);
 278         ceph_assert(write_bl.length() == write_length);
 279       }
 280
 281       C_ImportDiff *ctx = new C_ImportDiff(idiffctx, write_bl,
 282                                            image_offset + buffer_offset,
 283                                            write_length, zeroed);
 284       r = ctx->send();
 285       if (r < 0) {
 286         return r;
 287       }
 288
 289       buffer_offset += write_length;
 290     }
 291   } else {
 292     bufferlist data;
 293     C_ImportDiff *ctx = new C_ImportDiff(idiffctx, data, image_offset,
 294                                          buffer_length, true);
 295     return ctx->send();
 296   }
 297   return r;
 298 }
 299
 300 static int validate_banner(int fd, std::string banner)
 301 {
 302   int r;
 303   char buf[banner.size() + 1];
 304   memset(buf, 0, sizeof(buf));
 305   r = safe_read_exact(fd, buf, banner.size());
 306   if (r < 0) {
 307     std::cerr << "rbd: failed to decode diff banner" << std::endl;
 308     return r;
 309   }
 310
 311   buf[banner.size()] = '\0';
 312   if (strcmp(buf, banner.c_str())) {
 313     std::cerr << "rbd: invalid or unexpected diff banner" << std::endl;
 314     return -EINVAL;
 315   }
 316
 317   return 0;
 318 }
 319
 320 static int skip_tag(int fd, uint64_t length)
 321 {
 322   int r;
 323
 324   if (fd == STDIN_FILENO) {
 325     // read the appending data out to skip this tag.
 326     char buf[4096];
 327     uint64_t len = min<uint64_t>(length, sizeof(buf));
 328     while (len > 0) {
 329       r = safe_read_exact(fd, buf, len);
 330       if (r < 0) {
 331         std::cerr << "rbd: failed to decode skipped tag data" << std::endl;
 332         return r;
 333       }
 334       length -= len;
 335       len = min<uint64_t>(length, sizeof(buf));
 336     }
 337   } else {
 338     // lseek to skip this tag
 339     off64_t offs = lseek64(fd, length, SEEK_CUR);
 340     if (offs < 0) {
 341       return -errno;
 342     }
 343   }
 344
 345   return 0;
 346 }
 347
 348 static int read_tag(int fd, __u8 end_tag, int format, __u8 *tag, uint64_t *readlen)
 349 {
 350   int r;
 351   __u8 read_tag;
 352
 353   r = safe_read_exact(fd, &read_tag, sizeof(read_tag));
 354   if (r < 0) {
 355     std::cerr << "rbd: failed to decode tag" << std::endl;
 356     return r;
 357   }
 358
 359   *tag = read_tag;
 360   if (read_tag != end_tag && format == 2) {
 361     char buf[sizeof(uint64_t)];
 362     r = safe_read_exact(fd, buf, sizeof(buf));
 363     if (r < 0) {
 364       std::cerr << "rbd: failed to decode tag length" << std::endl;
 365       return r;
 366     }
 367
 368     bufferlist bl;
 369     bl.append(buf, sizeof(buf));
 370     auto p = bl.cbegin();
 371     decode(*readlen, p);
 372   }
 373
 374   return 0;
 375 }
 376
 377 int do_import_diff_fd(librados::Rados &rados, librbd::Image &image, int fd,
 378                       bool no_progress, int format, size_t sparse_size)
 379 {
 380   int r;
 381
 382   uint64_t size = 0;
 383   bool from_stdin = (fd == STDIN_FILENO);
 384   if (!from_stdin) {
 385     struct stat stat_buf;
 386     r = ::fstat(fd, &stat_buf);
 387     if (r < 0) {
 388       std::cerr << "rbd: failed to stat specified diff file" << std::endl;
 389       return r;
 390     }
 391     size = (uint64_t)stat_buf.st_size;
 392   }
 393
 394   r = validate_banner(fd, (format == 1 ? utils::RBD_DIFF_BANNER :
 395                            utils::RBD_DIFF_BANNER_V2));
 396   if (r < 0) {
 397     return r;
 398   }
 399
 400   // begin image import
 401   std::string tosnap;
 402   bool is_protected = false;
 403   ImportDiffContext idiffctx(&image, fd, size, no_progress);
 404   while (r == 0) {
 405     __u8 tag;
 406     uint64_t length = 0;
 407
 408     r = read_tag(fd, RBD_DIFF_END, format, &tag, &length);
 409     if (r < 0 || tag == RBD_DIFF_END) {
 410       break;
 411     }
 412
 413     if (tag == RBD_DIFF_FROM_SNAP) {
 414       r = do_image_snap_from(&idiffctx);
 415     } else if (tag == RBD_DIFF_TO_SNAP) {
 416       r = do_image_snap_to(&idiffctx, &tosnap);
 417     } else if (tag == RBD_SNAP_PROTECTION_STATUS) {
 418       r = get_snap_protection_status(&idiffctx, &is_protected);
 419     } else if (tag == RBD_DIFF_IMAGE_SIZE) {
 420       r = do_image_resize(&idiffctx);
 421     } else if (tag == RBD_DIFF_WRITE || tag == RBD_DIFF_ZERO) {
 422       r = do_image_io(&idiffctx, (tag == RBD_DIFF_ZERO), sparse_size);
 423     } else {
 424       std::cerr << "unrecognized tag byte " << (int)tag << " in stream; skipping"
 425                 << std::endl;
 426       r = skip_tag(fd, length);
 427     }
 428   }
 429
 430   int temp_r = idiffctx.throttle.wait_for_ret();
 431   r = (r < 0) ? r : temp_r; // preserve original error
 432   if (r == 0 && tosnap.length()) {
 433     r = idiffctx.image->snap_create(tosnap.c_str());
 434     if (r == 0 && is_protected) {
 435       r = idiffctx.image->snap_protect(tosnap.c_str());
 436     }
 437   }
 438
 439   idiffctx.finish(r);
 440   return r;
 441 }
 442
 443 int do_import_diff(librados::Rados &rados, librbd::Image &image,
 444                    const char *path, bool no_progress, size_t sparse_size)
 445 {
 446   int r;
 447   int fd;
 448
 449   if (strcmp(path, "-") == 0) {
 450     fd = STDIN_FILENO;
 451   } else {
 452     fd = open(path, O_RDONLY|O_BINARY);
 453     if (fd < 0) {
 454       r = -errno;
 455       std::cerr << "rbd: error opening " << path << std::endl;
 456       return r;
 457     }
 458   }
 459   r = do_import_diff_fd(rados, image, fd, no_progress, 1, sparse_size);
 460
 461   if (fd != 0)
 462     close(fd);
 463   return r;
 464 }
 465
 466 namespace at = argument_types;
 467 namespace po = boost::program_options;
 468
 469 void get_arguments_diff(po::options_description *positional,
 470                    po::options_description *options) {
 471   at::add_path_options(positional, options,
 472                        "import file (or '-' for stdin)");
 473   at::add_image_spec_options(positional, options, at::ARGUMENT_MODIFIER_NONE);
 474   at::add_sparse_size_option(options);
 475   at::add_no_progress_option(options);
 476 }
 477
 478 int execute_diff(const po::variables_map &vm,
 479                  const std::vector<std::string> &ceph_global_init_args) {
 480   std::string path;
 481   size_t arg_index = 0;
 482   int r = utils::get_path(vm, &arg_index, &path);
 483   if (r < 0) {
 484     return r;
 485   }
 486
 487   std::string pool_name;
 488   std::string namespace_name;
 489   std::string image_name;
 490   std::string snap_name;
 491   r = utils::get_pool_image_snapshot_names(
 492     vm, at::ARGUMENT_MODIFIER_NONE, &arg_index, &pool_name, &namespace_name,
 493     &image_name, &snap_name, true, utils::SNAPSHOT_PRESENCE_NONE,
 494     utils::SPEC_VALIDATION_NONE);
 495   if (r < 0) {
 496     return r;
 497   }
 498
 499   size_t sparse_size = utils::RBD_DEFAULT_SPARSE_SIZE;
 500   if (vm.count(at::IMAGE_SPARSE_SIZE)) {
 501     sparse_size = vm[at::IMAGE_SPARSE_SIZE].as<size_t>();
 502   }
 503
 504   librados::Rados rados;
 505   librados::IoCtx io_ctx;
 506   librbd::Image image;
 507   r = utils::init_and_open_image(pool_name, namespace_name, image_name, "", "",
 508                                  false, &rados, &io_ctx, &image);
 509   if (r < 0) {
 510     return r;
 511   }
 512
 513   r = do_import_diff(rados, image, path.c_str(),
 514                      vm[at::NO_PROGRESS].as<bool>(), sparse_size);
 515   if (r == -EDOM) {
 516     r = -EBADMSG;
 517   }
 518   if (r < 0) {
 519     cerr << "rbd: import-diff failed: " << cpp_strerror(r) << std::endl;
 520     return r;
 521   }
 522   return 0;
 523 }
 524
 525 Shell::Action action_diff(
 526   {"import-diff"}, {}, "Import an incremental diff.", "", &get_arguments_diff,
 527   &execute_diff);
 528
 529 class C_Import : public Context {
 530 public:
 531   C_Import(SimpleThrottle &simple_throttle, librbd::Image &image,
 532            bufferlist &bl, uint64_t offset)
 533     : m_throttle(simple_throttle), m_image(image),
 534       m_aio_completion(
 535         new librbd::RBD::AioCompletion(this, &utils::aio_context_callback)),
 536       m_bufferlist(bl), m_offset(offset)
 537   {
 538   }
 539
 540   void send()
 541   {
 542     m_throttle.start_op();
 543
 544     int op_flags = LIBRADOS_OP_FLAG_FADVISE_SEQUENTIAL |
 545                    LIBRADOS_OP_FLAG_FADVISE_NOCACHE;
 546     int r = m_image.aio_write2(m_offset, m_bufferlist.length(), m_bufferlist,
 547                                m_aio_completion, op_flags);
 548     if (r < 0) {
 549       std::cerr << "rbd: error requesting write to destination image"
 550                 << std::endl;
 551       m_aio_completion->release();
 552       m_throttle.end_op(r);
 553     }
 554   }
 555
 556   void finish(int r) override
 557   {
 558     if (r < 0) {
 559       std::cerr << "rbd: error writing to destination image at offset "
 560                 << m_offset << ": " << cpp_strerror(r) << std::endl;
 561     }
 562     m_throttle.end_op(r);
 563   }
 564
 565 private:
 566   SimpleThrottle &m_throttle;
 567   librbd::Image &m_image;
 568   librbd::RBD::AioCompletion *m_aio_completion;
 569   bufferlist m_bufferlist;
 570   uint64_t m_offset;
 571 };
 572
 573 static int decode_and_set_image_option(int fd, uint64_t imageopt, librbd::ImageOptions& opts)
 574 {
 575   int r;
 576   char buf[sizeof(uint64_t)];
 577
 578   r = safe_read_exact(fd, buf, sizeof(buf));
 579   if (r < 0) {
 580     std::cerr << "rbd: failed to decode image option" << std::endl;
 581     return r;
 582   }
 583
 584   bufferlist bl;
 585   bl.append(buf, sizeof(buf));
 586   auto it = bl.cbegin();
 587
 588   uint64_t val;
 589   decode(val, it);
 590
 591   if (opts.get(imageopt, &val) != 0) {
 592     opts.set(imageopt, val);
 593   }
 594
 595   return 0;
 596 }
 597
 598 static int do_import_metadata(int import_format, librbd::Image& image,
 599                               const std::map<std::string, std::string> &imagemetas)
 600 {
 601   int r = 0;
 602
 603   //v1 format
 604   if (import_format == 1) {
 605     return 0;
 606   }
 607
 608   for (std::map<std::string, std::string>::const_iterator it = imagemetas.begin();
 609        it != imagemetas.end(); ++it) {
 610     r = image.metadata_set(it->first, it->second);
 611     if (r < 0)
 612       return r;
 613   }
 614
 615   return 0;
 616 }
 617
 618 static int decode_imagemeta(int fd, uint64_t length, std::map<std::string, std::string>* imagemetas)
 619 {
 620   int r;
 621   string key;
 622   string value;
 623
 624   r = utils::read_string(fd, length, &key);
 625   if (r < 0) {
 626     std::cerr << "rbd: failed to decode metadata key" << std::endl;
 627     return r;
 628   }
 629
 630   r = utils::read_string(fd, length, &value);
 631   if (r < 0) {
 632     std::cerr << "rbd: failed to decode metadata value" << std::endl;
 633     return r;
 634   }
 635
 636   (*imagemetas)[key] = value;
 637   return 0;
 638 }
 639
 640 static int do_import_header(int fd, int import_format, librbd::ImageOptions& opts,
 641                             std::map<std::string, std::string>* imagemetas)
 642 {
 643   // There is no header in v1 image.
 644   if (import_format == 1) {
 645     return 0;
 646   }
 647
 648   int r;
 649   r = validate_banner(fd, utils::RBD_IMAGE_BANNER_V2);
 650   if (r < 0) {
 651     return r;
 652   }
 653
 654   // As V1 format for image is already deprecated, import image in V2 by default.
 655   uint64_t image_format = 2;
 656   if (opts.get(RBD_IMAGE_OPTION_FORMAT, &image_format) != 0) {
 657     opts.set(RBD_IMAGE_OPTION_FORMAT, image_format);
 658   }
 659
 660   while (r == 0) {
 661     __u8 tag;
 662     uint64_t length = 0;
 663     r = read_tag(fd, RBD_EXPORT_IMAGE_END, image_format, &tag, &length);
 664     if (r < 0 || tag == RBD_EXPORT_IMAGE_END) {
 665       break;
 666     }
 667
 668     if (tag == RBD_EXPORT_IMAGE_ORDER) {
 669       r = decode_and_set_image_option(fd, RBD_IMAGE_OPTION_ORDER, opts);
 670     } else if (tag == RBD_EXPORT_IMAGE_FEATURES) {
 671       r = decode_and_set_image_option(fd, RBD_IMAGE_OPTION_FEATURES, opts);
 672     } else if (tag == RBD_EXPORT_IMAGE_STRIPE_UNIT) {
 673       r = decode_and_set_image_option(fd, RBD_IMAGE_OPTION_STRIPE_UNIT, opts);
 674     } else if (tag == RBD_EXPORT_IMAGE_STRIPE_COUNT) {
 675       r = decode_and_set_image_option(fd, RBD_IMAGE_OPTION_STRIPE_COUNT, opts);
 676     } else if (tag == RBD_EXPORT_IMAGE_META) {
 677       r = decode_imagemeta(fd, length, imagemetas);
 678     } else {
 679       std::cerr << "rbd: invalid tag in image properties zone: " << tag << "Skip it."
 680                 << std::endl;
 681       r = skip_tag(fd, length);
 682     }
 683   }
 684
 685   return r;
 686 }
 687
 688 static int do_import_v2(librados::Rados &rados, int fd, librbd::Image &image,
 689                         uint64_t size, size_t imgblklen,
 690                         utils::ProgressContext &pc, size_t sparse_size)
 691 {
 692   int r = 0;
 693   r = validate_banner(fd, utils::RBD_IMAGE_DIFFS_BANNER_V2);
 694   if (r < 0) {
 695     return r;
 696   }
 697
 698   char buf[sizeof(uint64_t)];
 699   r = safe_read_exact(fd, buf, sizeof(buf));
 700   if (r < 0) {
 701     std::cerr << "rbd: failed to decode diff count" << std::endl;
 702     return r;
 703   }
 704   bufferlist bl;
 705   bl.append(buf, sizeof(buf));
 706   auto p = bl.cbegin();
 707   uint64_t diff_num;
 708   decode(diff_num, p);
 709   for (size_t i = 0; i < diff_num; i++) {
 710     r = do_import_diff_fd(rados, image, fd, true, 2, sparse_size);
 711     if (r < 0) {
 712       pc.fail();
 713       std::cerr << "rbd: import-diff failed: " << cpp_strerror(r) << std::endl;
 714       return r;
 715     }
 716     pc.update_progress(i + 1, diff_num);
 717   }
 718
 719   return r;
 720 }
 721
 722 static int do_import_v1(int fd, librbd::Image &image, uint64_t size,
 723                         size_t imgblklen, utils::ProgressContext &pc,
 724                         size_t sparse_size)
 725 {
 726   int r = 0;
 727   size_t reqlen = imgblklen;    // amount requested from read
 728   ssize_t readlen;              // amount received from one read
 729   size_t blklen = 0;            // amount accumulated from reads to fill blk
 730   char *p = new char[imgblklen];
 731   uint64_t image_pos = 0;
 732   bool from_stdin = (fd == STDIN_FILENO);
 733   boost::scoped_ptr<SimpleThrottle> throttle;
 734
 735   if (from_stdin) {
 736     throttle.reset(new SimpleThrottle(1, false));
 737   } else {
 738     throttle.reset(new SimpleThrottle(
 739       g_conf().get_val<uint64_t>("rbd_concurrent_management_ops"), false));
 740   }
 741
 742   reqlen = min<uint64_t>(reqlen, size);
 743   // loop body handles 0 return, as we may have a block to flush
 744   while ((readlen = ::read(fd, p + blklen, reqlen)) >= 0) {
 745     if (throttle->pending_error()) {
 746       break;
 747     }
 748
 749     blklen += readlen;
 750     // if read was short, try again to fill the block before writing
 751     if (readlen && ((size_t)readlen < reqlen)) {
 752       reqlen -= readlen;
 753       continue;
 754     }
 755     if (!from_stdin)
 756       pc.update_progress(image_pos, size);
 757
 758     bufferptr blkptr(p, blklen);
 759     // resize output image by binary expansion as we go for stdin
 760     if (from_stdin && (image_pos + (size_t)blklen) > size) {
 761       size *= 2;
 762       r = image.resize(size);
 763       if (r < 0) {
 764         std::cerr << "rbd: can't resize image during import" << std::endl;
 765         goto out;
 766       }
 767     }
 768
 769     // write as much as we got; perhaps less than imgblklen
 770     // but skip writing zeros to create sparse images
 771     size_t buffer_offset = 0;
 772     while (buffer_offset < blklen) {
 773       size_t write_length = 0;
 774       bool zeroed = false;
 775       utils::calc_sparse_extent(blkptr, sparse_size, buffer_offset, blklen,
 776                                 &write_length, &zeroed);
 777
 778       if (!zeroed) {
 779         bufferlist write_bl;
 780         bufferptr write_ptr(blkptr, buffer_offset, write_length);
 781         write_bl.push_back(write_ptr);
 782         ceph_assert(write_bl.length() == write_length);
 783
 784         C_Import *ctx = new C_Import(*throttle, image, write_bl,
 785                                      image_pos + buffer_offset);
 786         ctx->send();
 787       }
 788
 789       buffer_offset += write_length;
 790     }
 791
 792     // done with whole block, whether written or not
 793     image_pos += blklen;
 794     if (!from_stdin && image_pos >= size)
 795       break;
 796     // if read had returned 0, we're at EOF and should quit
 797     if (readlen == 0)
 798       break;
 799     blklen = 0;
 800     reqlen = imgblklen;
 801   }
 802   r = throttle->wait_for_ret();
 803   if (r < 0) {
 804     goto out;
 805   }
 806
 807   if (fd == STDIN_FILENO) {
 808     r = image.resize(image_pos);
 809     if (r < 0) {
 810       std::cerr << "rbd: final image resize failed" << std::endl;
 811       goto out;
 812     }
 813   }
 814 out:
 815   delete[] p;
 816   return r;
 817 }
 818
 819 static int do_import(librados::Rados &rados, librbd::RBD &rbd,
 820                      librados::IoCtx& io_ctx, const char *imgname,
 821                      const char *path, librbd::ImageOptions& opts,
 822                      bool no_progress, int import_format, size_t sparse_size)
 823 {
 824   int fd, r;
 825   struct stat stat_buf;
 826   utils::ProgressContext pc("Importing image", no_progress);
 827   std::map<std::string, std::string> imagemetas;
 828
 829   ceph_assert(imgname);
 830
 831   uint64_t order;
 832   if (opts.get(RBD_IMAGE_OPTION_ORDER, &order) != 0) {
 833     order = g_conf().get_val<uint64_t>("rbd_default_order");
 834   }
 835
 836   // try to fill whole imgblklen blocks for sparsification
 837   size_t imgblklen = 1 << order;
 838   librbd::Image image;
 839   uint64_t size = 0;
 840
 841   bool from_stdin = !strcmp(path, "-");
 842   if (from_stdin) {
 843     fd = STDIN_FILENO;
 844     size = 1ULL << order;
 845   } else {
 846     if ((fd = open(path, O_RDONLY|O_BINARY)) < 0) {
 847       r = -errno;
 848       std::cerr << "rbd: error opening " << path << std::endl;
 849       goto done2;
 850     }
 851
 852     if ((fstat(fd, &stat_buf)) < 0) {
 853       r = -errno;
 854       std::cerr << "rbd: stat error " << path << std::endl;
 855       goto done;
 856     }
 857     if (S_ISDIR(stat_buf.st_mode)) {
 858       r = -EISDIR;
 859       std::cerr << "rbd: cannot import a directory" << std::endl;
 860       goto done;
 861     }
 862     if (stat_buf.st_size)
 863       size = (uint64_t)stat_buf.st_size;
 864
 865     if (!size) {
 866       int64_t bdev_size = 0;
 867       BlkDev blkdev(fd);
 868       r = blkdev.get_size(&bdev_size);
 869       if (r < 0) {
 870         std::cerr << "rbd: unable to get size of file/block device"
 871                   << std::endl;
 872         goto done;
 873       }
 874       ceph_assert(bdev_size >= 0);
 875       size = (uint64_t) bdev_size;
 876     }
 877 #ifdef HAVE_POSIX_FADVISE
 878     posix_fadvise(fd, 0, 0, POSIX_FADV_SEQUENTIAL);
 879 #endif
 880   }
 881
 882   r = do_import_header(fd, import_format, opts, &imagemetas);
 883   if (r < 0) {
 884     std::cerr << "rbd: import header failed." << std::endl;
 885     goto done;
 886   }
 887
 888   r = rbd.create4(io_ctx, imgname, size, opts);
 889   if (r < 0) {
 890     std::cerr << "rbd: image creation failed" << std::endl;
 891     goto done;
 892   }
 893
 894   r = rbd.open(io_ctx, image, imgname);
 895   if (r < 0) {
 896     std::cerr << "rbd: failed to open image" << std::endl;
 897     goto err;
 898   }
 899
 900   r = do_import_metadata(import_format, image, imagemetas);
 901   if (r < 0) {
 902     std::cerr << "rbd: failed to import image-meta" << std::endl;
 903     goto err;
 904   }
 905
 906   if (import_format == 1) {
 907     r = do_import_v1(fd, image, size, imgblklen, pc, sparse_size);
 908   } else {
 909     r = do_import_v2(rados, fd, image, size, imgblklen, pc, sparse_size);
 910   }
 911   if (r < 0) {
 912     std::cerr << "rbd: failed to import image" << std::endl;
 913     image.close();
 914     goto err;
 915   }
 916
 917   r = image.close();
 918 err:
 919   if (r < 0)
 920     rbd.remove(io_ctx, imgname);
 921 done:
 922   if (r < 0)
 923     pc.fail();
 924   else
 925     pc.finish();
 926   if (!from_stdin)
 927     close(fd);
 928 done2:
 929   return r;
 930 }
 931
 932 void get_arguments(po::options_description *positional,
 933                    po::options_description *options) {
 934   at::add_path_options(positional, options,
 935                        "import file (or '-' for stdin)");
 936   at::add_image_spec_options(positional, options, at::ARGUMENT_MODIFIER_DEST);
 937   at::add_create_image_options(options, true);
 938   at::add_sparse_size_option(options);
 939   at::add_no_progress_option(options);
 940   at::add_export_format_option(options);
 941
 942   // TODO legacy rbd allowed import to accept both 'image'/'dest' and
 943   //      'pool'/'dest-pool'
 944   at::add_pool_option(options, at::ARGUMENT_MODIFIER_NONE, " deprecated[:dest-pool]");
 945   at::add_image_option(options, at::ARGUMENT_MODIFIER_NONE, " deprecated[:dest]");
 946 }
 947
 948 int execute(const po::variables_map &vm,
 949             const std::vector<std::string> &ceph_global_init_args) {
 950   std::string path;
 951   size_t arg_index = 0;
 952   int r = utils::get_path(vm, &arg_index, &path);
 953   if (r < 0) {
 954     return r;
 955   }
 956
 957   // odd check to support legacy / deprecated behavior of import
 958   std::string deprecated_pool_name;
 959   if (vm.count(at::POOL_NAME)) {
 960     deprecated_pool_name = vm[at::POOL_NAME].as<std::string>();
 961   }
 962
 963   std::string deprecated_image_name;
 964   if (vm.count(at::IMAGE_NAME)) {
 965     deprecated_image_name = vm[at::IMAGE_NAME].as<std::string>();
 966   } else {
 967     deprecated_image_name = path.substr(path.find_last_of("/\\") + 1);
 968   }
 969
 970   std::string deprecated_snap_name;
 971   r = utils::extract_spec(deprecated_image_name, &deprecated_pool_name,
 972                           nullptr, &deprecated_image_name,
 973                           &deprecated_snap_name, utils::SPEC_VALIDATION_FULL);
 974   if (r < 0) {
 975     return r;
 976   }
 977
 978   size_t sparse_size = utils::RBD_DEFAULT_SPARSE_SIZE;
 979   if (vm.count(at::IMAGE_SPARSE_SIZE)) {
 980     sparse_size = vm[at::IMAGE_SPARSE_SIZE].as<size_t>();
 981   }
 982
 983   std::string pool_name = deprecated_pool_name;
 984   std::string namespace_name;
 985   std::string image_name;
 986   std::string snap_name = deprecated_snap_name;
 987   r = utils::get_pool_image_snapshot_names(
 988     vm, at::ARGUMENT_MODIFIER_DEST, &arg_index, &pool_name, &namespace_name,
 989     &image_name, &snap_name, false, utils::SNAPSHOT_PRESENCE_NONE,
 990     utils::SPEC_VALIDATION_FULL);
 991   if (r < 0) {
 992     return r;
 993   }
 994
 995   if (image_name.empty()) {
 996     image_name = deprecated_image_name;
 997   }
 998
 999   librbd::ImageOptions opts;
1000   r = utils::get_image_options(vm, true, &opts);
1001   if (r < 0) {
1002     return r;
1003   }
1004
1005   librados::Rados rados;
1006   librados::IoCtx io_ctx;
1007   r = utils::init(pool_name, namespace_name, &rados, &io_ctx);
1008   if (r < 0) {
1009     return r;
1010   }
1011
1012   int format = 1;
1013   if (vm.count("export-format"))
1014     format = vm["export-format"].as<uint64_t>();
1015
1016   librbd::RBD rbd;
1017   r = do_import(rados, rbd, io_ctx, image_name.c_str(), path.c_str(),
1018                 opts, vm[at::NO_PROGRESS].as<bool>(), format, sparse_size);
1019   if (r < 0) {
1020     std::cerr << "rbd: import failed: " << cpp_strerror(r) << std::endl;
1021     return r;
1022   }
1023
1024   return 0;
1025 }
1026
1027 Shell::Action action(
1028   {"import"}, {}, "Import image from file.", at::get_long_features_help(),
1029   &get_arguments, &execute);
1030
1031 } // namespace import
1032 } // namespace action
1033 } // namespace rbd