debian/patches/bitmap-mirror/0001-drive-mirror-add-support-for-sync-bitmap-mode-never.patch

   1 From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
   2 From: John Snow <jsnow@redhat.com>
   3 Date: Mon, 6 Apr 2020 12:17:03 +0200
   4 Subject: [PATCH] drive-mirror: add support for sync=bitmap mode=never
   5 MIME-Version: 1.0
   6 Content-Type: text/plain; charset=UTF-8
   7 Content-Transfer-Encoding: 8bit
   8
   9 This patch adds support for the "BITMAP" sync mode to drive-mirror and
  10 blockdev-mirror. It adds support only for the BitmapSyncMode "never,"
  11 because it's the simplest mode.
  12
  13 This mode simply uses a user-provided bitmap as an initial copy
  14 manifest, and then does not clear any bits in the bitmap at the
  15 conclusion of the operation.
  16
  17 Any new writes dirtied during the operation are copied out, in contrast
  18 to backup. Note that whether these writes are reflected in the bitmap
  19 at the conclusion of the operation depends on whether that bitmap is
  20 actually recording!
  21
  22 This patch was originally based on one by Ma Haocong, but it has since
  23 been modified pretty heavily.
  24
  25 Suggested-by: Ma Haocong <mahaocong@didichuxing.com>
  26 Signed-off-by: Ma Haocong <mahaocong@didichuxing.com>
  27 Signed-off-by: John Snow <jsnow@redhat.com>
  28 Signed-off-by: Fabian Grünbichler <f.gruenbichler@proxmox.com>
  29 Signed-off-by: Thomas Lamprecht <t.lamprecht@proxmox.com>
  30 ---
  31  block/mirror.c                         | 98 +++++++++++++++++++++-----
  32  blockdev.c                             | 39 +++++++++-
  33  include/block/block_int-global-state.h |  4 +-
  34  qapi/block-core.json                   | 29 ++++++--
  35  tests/unit/test-block-iothread.c       |  4 +-
  36  5 files changed, 145 insertions(+), 29 deletions(-)
  37
  38 diff --git a/block/mirror.c b/block/mirror.c
  39 index 251adc5ae0..8ead5f77a0 100644
  40 --- a/block/mirror.c
  41 +++ b/block/mirror.c
  42 @@ -51,7 +51,7 @@ typedef struct MirrorBlockJob {
  43      BlockDriverState *to_replace;
  44      /* Used to block operations on the drive-mirror-replace target */
  45      Error *replace_blocker;
  46 -    bool is_none_mode;
  47 +    MirrorSyncMode sync_mode;
  48      BlockMirrorBackingMode backing_mode;
  49      /* Whether the target image requires explicit zero-initialization */
  50      bool zero_target;
  51 @@ -65,6 +65,8 @@ typedef struct MirrorBlockJob {
  52      size_t buf_size;
  53      int64_t bdev_length;
  54      unsigned long *cow_bitmap;
  55 +    BdrvDirtyBitmap *sync_bitmap;
  56 +    BitmapSyncMode bitmap_mode;
  57      BdrvDirtyBitmap *dirty_bitmap;
  58      BdrvDirtyBitmapIter *dbi;
  59      uint8_t *buf;
  60 @@ -699,7 +701,8 @@ static int mirror_exit_common(Job *job)
  61      bdrv_child_refresh_perms(mirror_top_bs, mirror_top_bs->backing,
  62                               &error_abort);
  63      if (!abort && s->backing_mode == MIRROR_SOURCE_BACKING_CHAIN) {
  64 -        BlockDriverState *backing = s->is_none_mode ? src : s->base;
  65 +        BlockDriverState *backing;
  66 +        backing = s->sync_mode == MIRROR_SYNC_MODE_NONE ? src : s->base;
  67          BlockDriverState *unfiltered_target = bdrv_skip_filters(target_bs);
  68
  69          if (bdrv_cow_bs(unfiltered_target) != backing) {
  70 @@ -797,6 +800,16 @@ static void mirror_abort(Job *job)
  71      assert(ret == 0);
  72  }
  73
  74 +/* Always called after commit/abort. */
  75 +static void mirror_clean(Job *job)
  76 +{
  77 +    MirrorBlockJob *s = container_of(job, MirrorBlockJob, common.job);
  78 +
  79 +    if (s->sync_bitmap) {
  80 +        bdrv_dirty_bitmap_set_busy(s->sync_bitmap, false);
  81 +    }
  82 +}
  83 +
  84  static void coroutine_fn mirror_throttle(MirrorBlockJob *s)
  85  {
  86      int64_t now = qemu_clock_get_ns(QEMU_CLOCK_REALTIME);
  87 @@ -977,7 +990,8 @@ static int coroutine_fn mirror_run(Job *job, Error **errp)
  88      mirror_free_init(s);
  89
  90      s->last_pause_ns = qemu_clock_get_ns(QEMU_CLOCK_REALTIME);
  91 -    if (!s->is_none_mode) {
  92 +    if ((s->sync_mode == MIRROR_SYNC_MODE_TOP) ||
  93 +        (s->sync_mode == MIRROR_SYNC_MODE_FULL)) {
  94          ret = mirror_dirty_init(s);
  95          if (ret < 0 || job_is_cancelled(&s->common.job)) {
  96              goto immediate_exit;
  97 @@ -1224,6 +1238,7 @@ static const BlockJobDriver mirror_job_driver = {
  98          .run                    = mirror_run,
  99          .prepare                = mirror_prepare,
 100          .abort                  = mirror_abort,
 101 +        .clean                  = mirror_clean,
 102          .pause                  = mirror_pause,
 103          .complete               = mirror_complete,
 104          .cancel                 = mirror_cancel,
 105 @@ -1240,6 +1255,7 @@ static const BlockJobDriver commit_active_job_driver = {
 106          .run                    = mirror_run,
 107          .prepare                = mirror_prepare,
 108          .abort                  = mirror_abort,
 109 +        .clean                  = mirror_clean,
 110          .pause                  = mirror_pause,
 111          .complete               = mirror_complete,
 112          .cancel                 = commit_active_cancel,
 113 @@ -1627,7 +1643,10 @@ static BlockJob *mirror_start_job(
 114                               BlockCompletionFunc *cb,
 115                               void *opaque,
 116                               const BlockJobDriver *driver,
 117 -                             bool is_none_mode, BlockDriverState *base,
 118 +                             MirrorSyncMode sync_mode,
 119 +                             BdrvDirtyBitmap *bitmap,
 120 +                             BitmapSyncMode bitmap_mode,
 121 +                             BlockDriverState *base,
 122                               bool auto_complete, const char *filter_node_name,
 123                               bool is_mirror, MirrorCopyMode copy_mode,
 124                               Error **errp)
 125 @@ -1639,10 +1658,39 @@ static BlockJob *mirror_start_job(
 126      uint64_t target_perms, target_shared_perms;
 127      int ret;
 128
 129 -    if (granularity == 0) {
 130 -        granularity = bdrv_get_default_bitmap_granularity(target);
 131 +    if (sync_mode == MIRROR_SYNC_MODE_INCREMENTAL) {
 132 +        error_setg(errp, "Sync mode '%s' not supported",
 133 +                   MirrorSyncMode_str(sync_mode));
 134 +        return NULL;
 135 +    } else if (sync_mode == MIRROR_SYNC_MODE_BITMAP) {
 136 +        if (!bitmap) {
 137 +            error_setg(errp, "Must provide a valid bitmap name for '%s'"
 138 +                       " sync mode",
 139 +                       MirrorSyncMode_str(sync_mode));
 140 +            return NULL;
 141 +        } else if (bitmap_mode != BITMAP_SYNC_MODE_NEVER) {
 142 +            error_setg(errp,
 143 +                       "Bitmap Sync Mode '%s' is not supported by Mirror",
 144 +                       BitmapSyncMode_str(bitmap_mode));
 145 +        }
 146 +    } else if (bitmap) {
 147 +        error_setg(errp,
 148 +                   "sync mode '%s' is not compatible with bitmaps",
 149 +                   MirrorSyncMode_str(sync_mode));
 150 +        return NULL;
 151      }
 152
 153 +    if (bitmap) {
 154 +        if (granularity) {
 155 +            error_setg(errp, "granularity (%d)"
 156 +                       "cannot be specified when a bitmap is provided",
 157 +                       granularity);
 158 +            return NULL;
 159 +        }
 160 +        granularity = bdrv_dirty_bitmap_granularity(bitmap);
 161 +    } else if (granularity == 0) {
 162 +        granularity = bdrv_get_default_bitmap_granularity(target);
 163 +    }
 164      assert(is_power_of_2(granularity));
 165
 166      if (buf_size < 0) {
 167 @@ -1774,7 +1822,9 @@ static BlockJob *mirror_start_job(
 168      s->replaces = g_strdup(replaces);
 169      s->on_source_error = on_source_error;
 170      s->on_target_error = on_target_error;
 171 -    s->is_none_mode = is_none_mode;
 172 +    s->sync_mode = sync_mode;
 173 +    s->sync_bitmap = bitmap;
 174 +    s->bitmap_mode = bitmap_mode;
 175      s->backing_mode = backing_mode;
 176      s->zero_target = zero_target;
 177      s->copy_mode = copy_mode;
 178 @@ -1795,6 +1845,18 @@ static BlockJob *mirror_start_job(
 179          bdrv_disable_dirty_bitmap(s->dirty_bitmap);
 180      }
 181
 182 +    if (s->sync_bitmap) {
 183 +        bdrv_dirty_bitmap_set_busy(s->sync_bitmap, true);
 184 +    }
 185 +
 186 +    if (s->sync_mode == MIRROR_SYNC_MODE_BITMAP) {
 187 +        bdrv_merge_dirty_bitmap(s->dirty_bitmap, s->sync_bitmap,
 188 +                                NULL, &local_err);
 189 +        if (local_err) {
 190 +            goto fail;
 191 +        }
 192 +    }
 193 +
 194      ret = block_job_add_bdrv(&s->common, "source", bs, 0,
 195                               BLK_PERM_WRITE_UNCHANGED | BLK_PERM_WRITE |
 196                               BLK_PERM_CONSISTENT_READ,
 197 @@ -1872,6 +1934,9 @@ fail:
 198          if (s->dirty_bitmap) {
 199              bdrv_release_dirty_bitmap(s->dirty_bitmap);
 200          }
 201 +        if (s->sync_bitmap) {
 202 +            bdrv_dirty_bitmap_set_busy(s->sync_bitmap, false);
 203 +        }
 204          job_early_fail(&s->common.job);
 205      }
 206
 207 @@ -1889,31 +1954,25 @@ void mirror_start(const char *job_id, BlockDriverState *bs,
 208                    BlockDriverState *target, const char *replaces,
 209                    int creation_flags, int64_t speed,
 210                    uint32_t granularity, int64_t buf_size,
 211 -                  MirrorSyncMode mode, BlockMirrorBackingMode backing_mode,
 212 +                  MirrorSyncMode mode, BdrvDirtyBitmap *bitmap,
 213 +                  BitmapSyncMode bitmap_mode,
 214 +                  BlockMirrorBackingMode backing_mode,
 215                    bool zero_target,
 216                    BlockdevOnError on_source_error,
 217                    BlockdevOnError on_target_error,
 218                    bool unmap, const char *filter_node_name,
 219                    MirrorCopyMode copy_mode, Error **errp)
 220  {
 221 -    bool is_none_mode;
 222      BlockDriverState *base;
 223
 224      GLOBAL_STATE_CODE();
 225
 226 -    if ((mode == MIRROR_SYNC_MODE_INCREMENTAL) ||
 227 -        (mode == MIRROR_SYNC_MODE_BITMAP)) {
 228 -        error_setg(errp, "Sync mode '%s' not supported",
 229 -                   MirrorSyncMode_str(mode));
 230 -        return;
 231 -    }
 232 -    is_none_mode = mode == MIRROR_SYNC_MODE_NONE;
 233      base = mode == MIRROR_SYNC_MODE_TOP ? bdrv_backing_chain_next(bs) : NULL;
 234      mirror_start_job(job_id, bs, creation_flags, target, replaces,
 235                       speed, granularity, buf_size, backing_mode, zero_target,
 236                       on_source_error, on_target_error, unmap, NULL, NULL,
 237 -                     &mirror_job_driver, is_none_mode, base, false,
 238 -                     filter_node_name, true, copy_mode, errp);
 239 +                     &mirror_job_driver, mode, bitmap, bitmap_mode, base,
 240 +                     false, filter_node_name, true, copy_mode, errp);
 241  }
 242
 243  BlockJob *commit_active_start(const char *job_id, BlockDriverState *bs,
 244 @@ -1940,7 +1999,8 @@ BlockJob *commit_active_start(const char *job_id, BlockDriverState *bs,
 245                       job_id, bs, creation_flags, base, NULL, speed, 0, 0,
 246                       MIRROR_LEAVE_BACKING_CHAIN, false,
 247                       on_error, on_error, true, cb, opaque,
 248 -                     &commit_active_job_driver, false, base, auto_complete,
 249 +                     &commit_active_job_driver, MIRROR_SYNC_MODE_FULL,
 250 +                     NULL, 0, base, auto_complete,
 251                       filter_node_name, false, MIRROR_COPY_MODE_BACKGROUND,
 252                       errp);
 253      if (!job) {
 254 diff --git a/blockdev.c b/blockdev.c
 255 index 3f1dec6242..2ee30323cb 100644
 256 --- a/blockdev.c
 257 +++ b/blockdev.c
 258 @@ -2946,6 +2946,10 @@ static void blockdev_mirror_common(const char *job_id, BlockDriverState *bs,
 259                                     BlockDriverState *target,
 260                                     bool has_replaces, const char *replaces,
 261                                     enum MirrorSyncMode sync,
 262 +                                   bool has_bitmap,
 263 +                                   const char *bitmap_name,
 264 +                                   bool has_bitmap_mode,
 265 +                                   BitmapSyncMode bitmap_mode,
 266                                     BlockMirrorBackingMode backing_mode,
 267                                     bool zero_target,
 268                                     bool has_speed, int64_t speed,
 269 @@ -2965,6 +2969,7 @@ static void blockdev_mirror_common(const char *job_id, BlockDriverState *bs,
 270  {
 271      BlockDriverState *unfiltered_bs;
 272      int job_flags = JOB_DEFAULT;
 273 +    BdrvDirtyBitmap *bitmap = NULL;
 274
 275      if (!has_speed) {
 276          speed = 0;
 277 @@ -3019,6 +3024,29 @@ static void blockdev_mirror_common(const char *job_id, BlockDriverState *bs,
 278          sync = MIRROR_SYNC_MODE_FULL;
 279      }
 280
 281 +    if (has_bitmap) {
 282 +        if (granularity) {
 283 +            error_setg(errp, "Granularity and bitmap cannot both be set");
 284 +            return;
 285 +        }
 286 +
 287 +        if (!has_bitmap_mode) {
 288 +            error_setg(errp, "bitmap-mode must be specified if"
 289 +                       " a bitmap is provided");
 290 +            return;
 291 +        }
 292 +
 293 +        bitmap = bdrv_find_dirty_bitmap(bs, bitmap_name);
 294 +        if (!bitmap) {
 295 +            error_setg(errp, "Dirty bitmap '%s' not found", bitmap_name);
 296 +            return;
 297 +        }
 298 +
 299 +        if (bdrv_dirty_bitmap_check(bitmap, BDRV_BITMAP_ALLOW_RO, errp)) {
 300 +            return;
 301 +        }
 302 +    }
 303 +
 304      if (!has_replaces) {
 305          /* We want to mirror from @bs, but keep implicit filters on top */
 306          unfiltered_bs = bdrv_skip_implicit_filters(bs);
 307 @@ -3065,8 +3093,8 @@ static void blockdev_mirror_common(const char *job_id, BlockDriverState *bs,
 308       * and will allow to check whether the node still exist at mirror completion
 309       */
 310      mirror_start(job_id, bs, target,
 311 -                 has_replaces ? replaces : NULL, job_flags,
 312 -                 speed, granularity, buf_size, sync, backing_mode, zero_target,
 313 +                 has_replaces ? replaces : NULL, job_flags, speed, granularity,
 314 +                 buf_size, sync, bitmap, bitmap_mode, backing_mode, zero_target,
 315                   on_source_error, on_target_error, unmap, filter_node_name,
 316                   copy_mode, errp);
 317  }
 318 @@ -3211,6 +3239,8 @@ void qmp_drive_mirror(DriveMirror *arg, Error **errp)
 319
 320      blockdev_mirror_common(arg->has_job_id ? arg->job_id : NULL, bs, target_bs,
 321                             arg->has_replaces, arg->replaces, arg->sync,
 322 +                           arg->has_bitmap, arg->bitmap,
 323 +                           arg->has_bitmap_mode, arg->bitmap_mode,
 324                             backing_mode, zero_target,
 325                             arg->has_speed, arg->speed,
 326                             arg->has_granularity, arg->granularity,
 327 @@ -3232,6 +3262,8 @@ void qmp_blockdev_mirror(bool has_job_id, const char *job_id,
 328                           const char *device, const char *target,
 329                           bool has_replaces, const char *replaces,
 330                           MirrorSyncMode sync,
 331 +                         bool has_bitmap, const char *bitmap,
 332 +                         bool has_bitmap_mode, BitmapSyncMode bitmap_mode,
 333                           bool has_speed, int64_t speed,
 334                           bool has_granularity, uint32_t granularity,
 335                           bool has_buf_size, int64_t buf_size,
 336 @@ -3281,7 +3313,8 @@ void qmp_blockdev_mirror(bool has_job_id, const char *job_id,
 337      }
 338
 339      blockdev_mirror_common(has_job_id ? job_id : NULL, bs, target_bs,
 340 -                           has_replaces, replaces, sync, backing_mode,
 341 +                           has_replaces, replaces, sync, has_bitmap,
 342 +                           bitmap, has_bitmap_mode, bitmap_mode, backing_mode,
 343                             zero_target, has_speed, speed,
 344                             has_granularity, granularity,
 345                             has_buf_size, buf_size,
 346 diff --git a/include/block/block_int-global-state.h b/include/block/block_int-global-state.h
 347 index b49f4eb35b..9d744db618 100644
 348 --- a/include/block/block_int-global-state.h
 349 +++ b/include/block/block_int-global-state.h
 350 @@ -149,7 +149,9 @@ void mirror_start(const char *job_id, BlockDriverState *bs,
 351                    BlockDriverState *target, const char *replaces,
 352                    int creation_flags, int64_t speed,
 353                    uint32_t granularity, int64_t buf_size,
 354 -                  MirrorSyncMode mode, BlockMirrorBackingMode backing_mode,
 355 +                  MirrorSyncMode mode, BdrvDirtyBitmap *bitmap,
 356 +                  BitmapSyncMode bitmap_mode,
 357 +                  BlockMirrorBackingMode backing_mode,
 358                    bool zero_target,
 359                    BlockdevOnError on_source_error,
 360                    BlockdevOnError on_target_error,
 361 diff --git a/qapi/block-core.json b/qapi/block-core.json
 362 index 95ac4fa634..7daaf545be 100644
 363 --- a/qapi/block-core.json
 364 +++ b/qapi/block-core.json
 365 @@ -2000,10 +2000,19 @@
 366  #        (all the disk, only the sectors allocated in the topmost image, or
 367  #        only new I/O).
 368  #
 369 +# @bitmap: The name of a bitmap to use for sync=bitmap mode. This argument must
 370 +#          be present for bitmap mode and absent otherwise. The bitmap's
 371 +#          granularity is used instead of @granularity (since 4.1).
 372 +#
 373 +# @bitmap-mode: Specifies the type of data the bitmap should contain after
 374 +#               the operation concludes. Must be present if sync is "bitmap".
 375 +#               Must NOT be present otherwise. (Since 4.1)
 376 +#
 377  # @granularity: granularity of the dirty bitmap, default is 64K
 378  #               if the image format doesn't have clusters, 4K if the clusters
 379  #               are smaller than that, else the cluster size.  Must be a
 380 -#               power of 2 between 512 and 64M (since 1.4).
 381 +#               power of 2 between 512 and 64M. Must not be specified if
 382 +#               @bitmap is present (since 1.4).
 383  #
 384  # @buf-size: maximum amount of data in flight from source to
 385  #            target (since 1.4).
 386 @@ -2043,7 +2052,9 @@
 387  { 'struct': 'DriveMirror',
 388    'data': { '*job-id': 'str', 'device': 'str', 'target': 'str',
 389              '*format': 'str', '*node-name': 'str', '*replaces': 'str',
 390 -            'sync': 'MirrorSyncMode', '*mode': 'NewImageMode',
 391 +            'sync': 'MirrorSyncMode', '*bitmap': 'str',
 392 +            '*bitmap-mode': 'BitmapSyncMode',
 393 +            '*mode': 'NewImageMode',
 394              '*speed': 'int', '*granularity': 'uint32',
 395              '*buf-size': 'int', '*on-source-error': 'BlockdevOnError',
 396              '*on-target-error': 'BlockdevOnError',
 397 @@ -2322,10 +2333,19 @@
 398  #        (all the disk, only the sectors allocated in the topmost image, or
 399  #        only new I/O).
 400  #
 401 +# @bitmap: The name of a bitmap to use for sync=bitmap mode. This argument must
 402 +#          be present for bitmap mode and absent otherwise. The bitmap's
 403 +#          granularity is used instead of @granularity (since 4.1).
 404 +#
 405 +# @bitmap-mode: Specifies the type of data the bitmap should contain after
 406 +#               the operation concludes. Must be present if sync is "bitmap".
 407 +#               Must NOT be present otherwise. (Since 4.1)
 408 +#
 409  # @granularity: granularity of the dirty bitmap, default is 64K
 410  #               if the image format doesn't have clusters, 4K if the clusters
 411  #               are smaller than that, else the cluster size.  Must be a
 412 -#               power of 2 between 512 and 64M
 413 +#               power of 2 between 512 and 64M . Must not be specified if
 414 +#               @bitmap is present.
 415  #
 416  # @buf-size: maximum amount of data in flight from source to
 417  #            target
 418 @@ -2375,7 +2395,8 @@
 419  { 'command': 'blockdev-mirror',
 420    'data': { '*job-id': 'str', 'device': 'str', 'target': 'str',
 421              '*replaces': 'str',
 422 -            'sync': 'MirrorSyncMode',
 423 +            'sync': 'MirrorSyncMode', '*bitmap': 'str',
 424 +            '*bitmap-mode': 'BitmapSyncMode',
 425              '*speed': 'int', '*granularity': 'uint32',
 426              '*buf-size': 'int', '*on-source-error': 'BlockdevOnError',
 427              '*on-target-error': 'BlockdevOnError',
 428 diff --git a/tests/unit/test-block-iothread.c b/tests/unit/test-block-iothread.c
 429 index 8ca5adec5e..dae80e5a5f 100644
 430 --- a/tests/unit/test-block-iothread.c
 431 +++ b/tests/unit/test-block-iothread.c
 432 @@ -755,8 +755,8 @@ static void test_propagate_mirror(void)
 433
 434      /* Start a mirror job */
 435      mirror_start("job0", src, target, NULL, JOB_DEFAULT, 0, 0, 0,
 436 -                 MIRROR_SYNC_MODE_NONE, MIRROR_OPEN_BACKING_CHAIN, false,
 437 -                 BLOCKDEV_ON_ERROR_REPORT, BLOCKDEV_ON_ERROR_REPORT,
 438 +                 MIRROR_SYNC_MODE_NONE, NULL, 0, MIRROR_OPEN_BACKING_CHAIN,
 439 +                 false, BLOCKDEV_ON_ERROR_REPORT, BLOCKDEV_ON_ERROR_REPORT,
 440                   false, "filter_node", MIRROR_COPY_MODE_BACKGROUND,
 441                   &error_abort);
 442      WITH_JOB_LOCK_GUARD() {