debian/patches/bitmap-mirror/0001-drive-mirror-add-support-for-sync-bitmap-mode-never.patch

   1 From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
   2 From: John Snow <jsnow@redhat.com>
   3 Date: Mon, 6 Apr 2020 12:17:03 +0200
   4 Subject: [PATCH] drive-mirror: add support for sync=bitmap mode=never
   5 MIME-Version: 1.0
   6 Content-Type: text/plain; charset=UTF-8
   7 Content-Transfer-Encoding: 8bit
   8
   9 This patch adds support for the "BITMAP" sync mode to drive-mirror and
  10 blockdev-mirror. It adds support only for the BitmapSyncMode "never,"
  11 because it's the simplest mode.
  12
  13 This mode simply uses a user-provided bitmap as an initial copy
  14 manifest, and then does not clear any bits in the bitmap at the
  15 conclusion of the operation.
  16
  17 Any new writes dirtied during the operation are copied out, in contrast
  18 to backup. Note that whether these writes are reflected in the bitmap
  19 at the conclusion of the operation depends on whether that bitmap is
  20 actually recording!
  21
  22 This patch was originally based on one by Ma Haocong, but it has since
  23 been modified pretty heavily.
  24
  25 Suggested-by: Ma Haocong <mahaocong@didichuxing.com>
  26 Signed-off-by: Ma Haocong <mahaocong@didichuxing.com>
  27 Signed-off-by: John Snow <jsnow@redhat.com>
  28 Signed-off-by: Fabian Grünbichler <f.gruenbichler@proxmox.com>
  29 Signed-off-by: Thomas Lamprecht <t.lamprecht@proxmox.com>
  30 ---
  31  block/mirror.c                   | 98 +++++++++++++++++++++++++-------
  32  blockdev.c                       | 39 ++++++++++++-
  33  include/block/block_int.h        |  4 +-
  34  qapi/block-core.json             | 29 ++++++++--
  35  tests/unit/test-block-iothread.c |  4 +-
  36  5 files changed, 145 insertions(+), 29 deletions(-)
  37
  38 diff --git a/block/mirror.c b/block/mirror.c
  39 index efec2c7674..f7804638f9 100644
  40 --- a/block/mirror.c
  41 +++ b/block/mirror.c
  42 @@ -50,7 +50,7 @@ typedef struct MirrorBlockJob {
  43      BlockDriverState *to_replace;
  44      /* Used to block operations on the drive-mirror-replace target */
  45      Error *replace_blocker;
  46 -    bool is_none_mode;
  47 +    MirrorSyncMode sync_mode;
  48      BlockMirrorBackingMode backing_mode;
  49      /* Whether the target image requires explicit zero-initialization */
  50      bool zero_target;
  51 @@ -64,6 +64,8 @@ typedef struct MirrorBlockJob {
  52      size_t buf_size;
  53      int64_t bdev_length;
  54      unsigned long *cow_bitmap;
  55 +    BdrvDirtyBitmap *sync_bitmap;
  56 +    BitmapSyncMode bitmap_mode;
  57      BdrvDirtyBitmap *dirty_bitmap;
  58      BdrvDirtyBitmapIter *dbi;
  59      uint8_t *buf;
  60 @@ -695,7 +697,8 @@ static int mirror_exit_common(Job *job)
  61      bdrv_child_refresh_perms(mirror_top_bs, mirror_top_bs->backing,
  62                               &error_abort);
  63      if (!abort && s->backing_mode == MIRROR_SOURCE_BACKING_CHAIN) {
  64 -        BlockDriverState *backing = s->is_none_mode ? src : s->base;
  65 +        BlockDriverState *backing;
  66 +        backing = s->sync_mode == MIRROR_SYNC_MODE_NONE ? src : s->base;
  67          BlockDriverState *unfiltered_target = bdrv_skip_filters(target_bs);
  68
  69          if (bdrv_cow_bs(unfiltered_target) != backing) {
  70 @@ -800,6 +803,16 @@ static void mirror_abort(Job *job)
  71      assert(ret == 0);
  72  }
  73
  74 +/* Always called after commit/abort. */
  75 +static void mirror_clean(Job *job)
  76 +{
  77 +    MirrorBlockJob *s = container_of(job, MirrorBlockJob, common.job);
  78 +
  79 +    if (s->sync_bitmap) {
  80 +        bdrv_dirty_bitmap_set_busy(s->sync_bitmap, false);
  81 +    }
  82 +}
  83 +
  84  static void coroutine_fn mirror_throttle(MirrorBlockJob *s)
  85  {
  86      int64_t now = qemu_clock_get_ns(QEMU_CLOCK_REALTIME);
  87 @@ -979,7 +992,8 @@ static int coroutine_fn mirror_run(Job *job, Error **errp)
  88      mirror_free_init(s);
  89
  90      s->last_pause_ns = qemu_clock_get_ns(QEMU_CLOCK_REALTIME);
  91 -    if (!s->is_none_mode) {
  92 +    if ((s->sync_mode == MIRROR_SYNC_MODE_TOP) ||
  93 +        (s->sync_mode == MIRROR_SYNC_MODE_FULL)) {
  94          ret = mirror_dirty_init(s);
  95          if (ret < 0 || job_is_cancelled(&s->common.job)) {
  96              goto immediate_exit;
  97 @@ -1221,6 +1235,7 @@ static const BlockJobDriver mirror_job_driver = {
  98          .run                    = mirror_run,
  99          .prepare                = mirror_prepare,
 100          .abort                  = mirror_abort,
 101 +        .clean                  = mirror_clean,
 102          .pause                  = mirror_pause,
 103          .complete               = mirror_complete,
 104          .cancel                 = mirror_cancel,
 105 @@ -1237,6 +1252,7 @@ static const BlockJobDriver commit_active_job_driver = {
 106          .run                    = mirror_run,
 107          .prepare                = mirror_prepare,
 108          .abort                  = mirror_abort,
 109 +        .clean                  = mirror_clean,
 110          .pause                  = mirror_pause,
 111          .complete               = mirror_complete,
 112          .cancel                 = commit_active_cancel,
 113 @@ -1602,7 +1618,10 @@ static BlockJob *mirror_start_job(
 114                               BlockCompletionFunc *cb,
 115                               void *opaque,
 116                               const BlockJobDriver *driver,
 117 -                             bool is_none_mode, BlockDriverState *base,
 118 +                             MirrorSyncMode sync_mode,
 119 +                             BdrvDirtyBitmap *bitmap,
 120 +                             BitmapSyncMode bitmap_mode,
 121 +                             BlockDriverState *base,
 122                               bool auto_complete, const char *filter_node_name,
 123                               bool is_mirror, MirrorCopyMode copy_mode,
 124                               Error **errp)
 125 @@ -1614,10 +1633,39 @@ static BlockJob *mirror_start_job(
 126      uint64_t target_perms, target_shared_perms;
 127      int ret;
 128
 129 -    if (granularity == 0) {
 130 +    if (sync_mode == MIRROR_SYNC_MODE_INCREMENTAL) {
 131 +        error_setg(errp, "Sync mode '%s' not supported",
 132 +                   MirrorSyncMode_str(sync_mode));
 133 +        return NULL;
 134 +    } else if (sync_mode == MIRROR_SYNC_MODE_BITMAP) {
 135 +        if (!bitmap) {
 136 +            error_setg(errp, "Must provide a valid bitmap name for '%s'"
 137 +                       " sync mode",
 138 +                       MirrorSyncMode_str(sync_mode));
 139 +            return NULL;
 140 +        } else if (bitmap_mode != BITMAP_SYNC_MODE_NEVER) {
 141 +            error_setg(errp,
 142 +                       "Bitmap Sync Mode '%s' is not supported by Mirror",
 143 +                       BitmapSyncMode_str(bitmap_mode));
 144 +        }
 145 +    } else if (bitmap) {
 146 +        error_setg(errp,
 147 +                   "sync mode '%s' is not compatible with bitmaps",
 148 +                   MirrorSyncMode_str(sync_mode));
 149 +        return NULL;
 150 +    }
 151 +
 152 +    if (bitmap) {
 153 +        if (granularity) {
 154 +            error_setg(errp, "granularity (%d)"
 155 +                       "cannot be specified when a bitmap is provided",
 156 +                       granularity);
 157 +            return NULL;
 158 +        }
 159 +        granularity = bdrv_dirty_bitmap_granularity(bitmap);
 160 +    } else if (granularity == 0) {
 161          granularity = bdrv_get_default_bitmap_granularity(target);
 162      }
 163 -
 164      assert(is_power_of_2(granularity));
 165
 166      if (buf_size < 0) {
 167 @@ -1755,7 +1803,9 @@ static BlockJob *mirror_start_job(
 168      s->replaces = g_strdup(replaces);
 169      s->on_source_error = on_source_error;
 170      s->on_target_error = on_target_error;
 171 -    s->is_none_mode = is_none_mode;
 172 +    s->sync_mode = sync_mode;
 173 +    s->sync_bitmap = bitmap;
 174 +    s->bitmap_mode = bitmap_mode;
 175      s->backing_mode = backing_mode;
 176      s->zero_target = zero_target;
 177      s->copy_mode = copy_mode;
 178 @@ -1776,6 +1826,18 @@ static BlockJob *mirror_start_job(
 179          bdrv_disable_dirty_bitmap(s->dirty_bitmap);
 180      }
 181
 182 +    if (s->sync_bitmap) {
 183 +        bdrv_dirty_bitmap_set_busy(s->sync_bitmap, true);
 184 +    }
 185 +
 186 +    if (s->sync_mode == MIRROR_SYNC_MODE_BITMAP) {
 187 +        bdrv_merge_dirty_bitmap(s->dirty_bitmap, s->sync_bitmap,
 188 +                                NULL, &local_err);
 189 +        if (local_err) {
 190 +            goto fail;
 191 +        }
 192 +    }
 193 +
 194      ret = block_job_add_bdrv(&s->common, "source", bs, 0,
 195                               BLK_PERM_WRITE_UNCHANGED | BLK_PERM_WRITE |
 196                               BLK_PERM_CONSISTENT_READ,
 197 @@ -1853,6 +1915,9 @@ fail:
 198          if (s->dirty_bitmap) {
 199              bdrv_release_dirty_bitmap(s->dirty_bitmap);
 200          }
 201 +        if (s->sync_bitmap) {
 202 +            bdrv_dirty_bitmap_set_busy(s->sync_bitmap, false);
 203 +        }
 204          job_early_fail(&s->common.job);
 205      }
 206
 207 @@ -1870,29 +1935,23 @@ void mirror_start(const char *job_id, BlockDriverState *bs,
 208                    BlockDriverState *target, const char *replaces,
 209                    int creation_flags, int64_t speed,
 210                    uint32_t granularity, int64_t buf_size,
 211 -                  MirrorSyncMode mode, BlockMirrorBackingMode backing_mode,
 212 +                  MirrorSyncMode mode, BdrvDirtyBitmap *bitmap,
 213 +                  BitmapSyncMode bitmap_mode,
 214 +                  BlockMirrorBackingMode backing_mode,
 215                    bool zero_target,
 216                    BlockdevOnError on_source_error,
 217                    BlockdevOnError on_target_error,
 218                    bool unmap, const char *filter_node_name,
 219                    MirrorCopyMode copy_mode, Error **errp)
 220  {
 221 -    bool is_none_mode;
 222      BlockDriverState *base;
 223
 224 -    if ((mode == MIRROR_SYNC_MODE_INCREMENTAL) ||
 225 -        (mode == MIRROR_SYNC_MODE_BITMAP)) {
 226 -        error_setg(errp, "Sync mode '%s' not supported",
 227 -                   MirrorSyncMode_str(mode));
 228 -        return;
 229 -    }
 230 -    is_none_mode = mode == MIRROR_SYNC_MODE_NONE;
 231      base = mode == MIRROR_SYNC_MODE_TOP ? bdrv_backing_chain_next(bs) : NULL;
 232      mirror_start_job(job_id, bs, creation_flags, target, replaces,
 233                       speed, granularity, buf_size, backing_mode, zero_target,
 234                       on_source_error, on_target_error, unmap, NULL, NULL,
 235 -                     &mirror_job_driver, is_none_mode, base, false,
 236 -                     filter_node_name, true, copy_mode, errp);
 237 +                     &mirror_job_driver, mode, bitmap, bitmap_mode, base,
 238 +                     false, filter_node_name, true, copy_mode, errp);
 239  }
 240
 241  BlockJob *commit_active_start(const char *job_id, BlockDriverState *bs,
 242 @@ -1917,7 +1976,8 @@ BlockJob *commit_active_start(const char *job_id, BlockDriverState *bs,
 243                       job_id, bs, creation_flags, base, NULL, speed, 0, 0,
 244                       MIRROR_LEAVE_BACKING_CHAIN, false,
 245                       on_error, on_error, true, cb, opaque,
 246 -                     &commit_active_job_driver, false, base, auto_complete,
 247 +                     &commit_active_job_driver, MIRROR_SYNC_MODE_FULL,
 248 +                     NULL, 0, base, auto_complete,
 249                       filter_node_name, false, MIRROR_COPY_MODE_BACKGROUND,
 250                       errp);
 251      if (!job) {
 252 diff --git a/blockdev.c b/blockdev.c
 253 index b35072644e..9940116fe0 100644
 254 --- a/blockdev.c
 255 +++ b/blockdev.c
 256 @@ -2956,6 +2956,10 @@ static void blockdev_mirror_common(const char *job_id, BlockDriverState *bs,
 257                                     BlockDriverState *target,
 258                                     bool has_replaces, const char *replaces,
 259                                     enum MirrorSyncMode sync,
 260 +                                   bool has_bitmap,
 261 +                                   const char *bitmap_name,
 262 +                                   bool has_bitmap_mode,
 263 +                                   BitmapSyncMode bitmap_mode,
 264                                     BlockMirrorBackingMode backing_mode,
 265                                     bool zero_target,
 266                                     bool has_speed, int64_t speed,
 267 @@ -2975,6 +2979,7 @@ static void blockdev_mirror_common(const char *job_id, BlockDriverState *bs,
 268  {
 269      BlockDriverState *unfiltered_bs;
 270      int job_flags = JOB_DEFAULT;
 271 +    BdrvDirtyBitmap *bitmap = NULL;
 272
 273      if (!has_speed) {
 274          speed = 0;
 275 @@ -3029,6 +3034,29 @@ static void blockdev_mirror_common(const char *job_id, BlockDriverState *bs,
 276          sync = MIRROR_SYNC_MODE_FULL;
 277      }
 278
 279 +    if (has_bitmap) {
 280 +        if (granularity) {
 281 +            error_setg(errp, "Granularity and bitmap cannot both be set");
 282 +            return;
 283 +        }
 284 +
 285 +        if (!has_bitmap_mode) {
 286 +            error_setg(errp, "bitmap-mode must be specified if"
 287 +                       " a bitmap is provided");
 288 +            return;
 289 +        }
 290 +
 291 +        bitmap = bdrv_find_dirty_bitmap(bs, bitmap_name);
 292 +        if (!bitmap) {
 293 +            error_setg(errp, "Dirty bitmap '%s' not found", bitmap_name);
 294 +            return;
 295 +        }
 296 +
 297 +        if (bdrv_dirty_bitmap_check(bitmap, BDRV_BITMAP_ALLOW_RO, errp)) {
 298 +            return;
 299 +        }
 300 +    }
 301 +
 302      if (!has_replaces) {
 303          /* We want to mirror from @bs, but keep implicit filters on top */
 304          unfiltered_bs = bdrv_skip_implicit_filters(bs);
 305 @@ -3075,8 +3103,8 @@ static void blockdev_mirror_common(const char *job_id, BlockDriverState *bs,
 306       * and will allow to check whether the node still exist at mirror completion
 307       */
 308      mirror_start(job_id, bs, target,
 309 -                 has_replaces ? replaces : NULL, job_flags,
 310 -                 speed, granularity, buf_size, sync, backing_mode, zero_target,
 311 +                 has_replaces ? replaces : NULL, job_flags, speed, granularity,
 312 +                 buf_size, sync, bitmap, bitmap_mode, backing_mode, zero_target,
 313                   on_source_error, on_target_error, unmap, filter_node_name,
 314                   copy_mode, errp);
 315  }
 316 @@ -3221,6 +3249,8 @@ void qmp_drive_mirror(DriveMirror *arg, Error **errp)
 317
 318      blockdev_mirror_common(arg->has_job_id ? arg->job_id : NULL, bs, target_bs,
 319                             arg->has_replaces, arg->replaces, arg->sync,
 320 +                           arg->has_bitmap, arg->bitmap,
 321 +                           arg->has_bitmap_mode, arg->bitmap_mode,
 322                             backing_mode, zero_target,
 323                             arg->has_speed, arg->speed,
 324                             arg->has_granularity, arg->granularity,
 325 @@ -3242,6 +3272,8 @@ void qmp_blockdev_mirror(bool has_job_id, const char *job_id,
 326                           const char *device, const char *target,
 327                           bool has_replaces, const char *replaces,
 328                           MirrorSyncMode sync,
 329 +                         bool has_bitmap, const char *bitmap,
 330 +                         bool has_bitmap_mode, BitmapSyncMode bitmap_mode,
 331                           bool has_speed, int64_t speed,
 332                           bool has_granularity, uint32_t granularity,
 333                           bool has_buf_size, int64_t buf_size,
 334 @@ -3291,7 +3323,8 @@ void qmp_blockdev_mirror(bool has_job_id, const char *job_id,
 335      }
 336
 337      blockdev_mirror_common(has_job_id ? job_id : NULL, bs, target_bs,
 338 -                           has_replaces, replaces, sync, backing_mode,
 339 +                           has_replaces, replaces, sync, has_bitmap,
 340 +                           bitmap, has_bitmap_mode, bitmap_mode, backing_mode,
 341                             zero_target, has_speed, speed,
 342                             has_granularity, granularity,
 343                             has_buf_size, buf_size,
 344 diff --git a/include/block/block_int.h b/include/block/block_int.h
 345 index f4c75e8ba9..ee0aeb1414 100644
 346 --- a/include/block/block_int.h
 347 +++ b/include/block/block_int.h
 348 @@ -1287,7 +1287,9 @@ void mirror_start(const char *job_id, BlockDriverState *bs,
 349                    BlockDriverState *target, const char *replaces,
 350                    int creation_flags, int64_t speed,
 351                    uint32_t granularity, int64_t buf_size,
 352 -                  MirrorSyncMode mode, BlockMirrorBackingMode backing_mode,
 353 +                  MirrorSyncMode mode, BdrvDirtyBitmap *bitmap,
 354 +                  BitmapSyncMode bitmap_mode,
 355 +                  BlockMirrorBackingMode backing_mode,
 356                    bool zero_target,
 357                    BlockdevOnError on_source_error,
 358                    BlockdevOnError on_target_error,
 359 diff --git a/qapi/block-core.json b/qapi/block-core.json
 360 index 1d3dd9cb48..da5dca1e3b 100644
 361 --- a/qapi/block-core.json
 362 +++ b/qapi/block-core.json
 363 @@ -1995,10 +1995,19 @@
 364  #        (all the disk, only the sectors allocated in the topmost image, or
 365  #        only new I/O).
 366  #
 367 +# @bitmap: The name of a bitmap to use for sync=bitmap mode. This argument must
 368 +#          be present for bitmap mode and absent otherwise. The bitmap's
 369 +#          granularity is used instead of @granularity (since 4.1).
 370 +#
 371 +# @bitmap-mode: Specifies the type of data the bitmap should contain after
 372 +#               the operation concludes. Must be present if sync is "bitmap".
 373 +#               Must NOT be present otherwise. (Since 4.1)
 374 +#
 375  # @granularity: granularity of the dirty bitmap, default is 64K
 376  #               if the image format doesn't have clusters, 4K if the clusters
 377  #               are smaller than that, else the cluster size.  Must be a
 378 -#               power of 2 between 512 and 64M (since 1.4).
 379 +#               power of 2 between 512 and 64M. Must not be specified if
 380 +#               @bitmap is present (since 1.4).
 381  #
 382  # @buf-size: maximum amount of data in flight from source to
 383  #            target (since 1.4).
 384 @@ -2036,7 +2045,9 @@
 385  { 'struct': 'DriveMirror',
 386    'data': { '*job-id': 'str', 'device': 'str', 'target': 'str',
 387              '*format': 'str', '*node-name': 'str', '*replaces': 'str',
 388 -            'sync': 'MirrorSyncMode', '*mode': 'NewImageMode',
 389 +            'sync': 'MirrorSyncMode', '*bitmap': 'str',
 390 +            '*bitmap-mode': 'BitmapSyncMode',
 391 +            '*mode': 'NewImageMode',
 392              '*speed': 'int', '*granularity': 'uint32',
 393              '*buf-size': 'int', '*on-source-error': 'BlockdevOnError',
 394              '*on-target-error': 'BlockdevOnError',
 395 @@ -2308,10 +2319,19 @@
 396  #        (all the disk, only the sectors allocated in the topmost image, or
 397  #        only new I/O).
 398  #
 399 +# @bitmap: The name of a bitmap to use for sync=bitmap mode. This argument must
 400 +#          be present for bitmap mode and absent otherwise. The bitmap's
 401 +#          granularity is used instead of @granularity (since 4.1).
 402 +#
 403 +# @bitmap-mode: Specifies the type of data the bitmap should contain after
 404 +#               the operation concludes. Must be present if sync is "bitmap".
 405 +#               Must NOT be present otherwise. (Since 4.1)
 406 +#
 407  # @granularity: granularity of the dirty bitmap, default is 64K
 408  #               if the image format doesn't have clusters, 4K if the clusters
 409  #               are smaller than that, else the cluster size.  Must be a
 410 -#               power of 2 between 512 and 64M
 411 +#               power of 2 between 512 and 64M . Must not be specified if
 412 +#               @bitmap is present.
 413  #
 414  # @buf-size: maximum amount of data in flight from source to
 415  #            target
 416 @@ -2360,7 +2380,8 @@
 417  { 'command': 'blockdev-mirror',
 418    'data': { '*job-id': 'str', 'device': 'str', 'target': 'str',
 419              '*replaces': 'str',
 420 -            'sync': 'MirrorSyncMode',
 421 +            'sync': 'MirrorSyncMode', '*bitmap': 'str',
 422 +            '*bitmap-mode': 'BitmapSyncMode',
 423              '*speed': 'int', '*granularity': 'uint32',
 424              '*buf-size': 'int', '*on-source-error': 'BlockdevOnError',
 425              '*on-target-error': 'BlockdevOnError',
 426 diff --git a/tests/unit/test-block-iothread.c b/tests/unit/test-block-iothread.c
 427 index aea660aeed..22b9770a3e 100644
 428 --- a/tests/unit/test-block-iothread.c
 429 +++ b/tests/unit/test-block-iothread.c
 430 @@ -626,8 +626,8 @@ static void test_propagate_mirror(void)
 431
 432      /* Start a mirror job */
 433      mirror_start("job0", src, target, NULL, JOB_DEFAULT, 0, 0, 0,
 434 -                 MIRROR_SYNC_MODE_NONE, MIRROR_OPEN_BACKING_CHAIN, false,
 435 -                 BLOCKDEV_ON_ERROR_REPORT, BLOCKDEV_ON_ERROR_REPORT,
 436 +                 MIRROR_SYNC_MODE_NONE, NULL, 0, MIRROR_OPEN_BACKING_CHAIN,
 437 +                 false, BLOCKDEV_ON_ERROR_REPORT, BLOCKDEV_ON_ERROR_REPORT,
 438                   false, "filter_node", MIRROR_COPY_MODE_BACKGROUND,
 439                   &error_abort);
 440      job = job_get("job0");