debian/patches/pve/0035-PVE-add-query-pbs-bitmap-info-QMP-call.patch

   1 From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
   2 From: Stefan Reiter <s.reiter@proxmox.com>
   3 Date: Wed, 19 Aug 2020 17:02:00 +0200
   4 Subject: [PATCH] PVE: add query-pbs-bitmap-info QMP call
   5
   6 Returns advanced information about dirty bitmaps used (or not used) for
   7 the latest PBS backup.
   8
   9 Signed-off-by: Stefan Reiter <s.reiter@proxmox.com>
  10 Signed-off-by: Thomas Lamprecht <t.lamprecht@proxmox.com>
  11 ---
  12  monitor/hmp-cmds.c   |  28 ++++++-----
  13  pve-backup.c         | 117 ++++++++++++++++++++++++++++++++-----------
  14  qapi/block-core.json |  56 +++++++++++++++++++++
  15  3 files changed, 159 insertions(+), 42 deletions(-)
  16
  17 diff --git a/monitor/hmp-cmds.c b/monitor/hmp-cmds.c
  18 index 4c1671e289..c1152f55a7 100644
  19 --- a/monitor/hmp-cmds.c
  20 +++ b/monitor/hmp-cmds.c
  21 @@ -200,6 +200,7 @@ void hmp_info_mice(Monitor *mon, const QDict *qdict)
  22  void hmp_info_backup(Monitor *mon, const QDict *qdict)
  23  {
  24      BackupStatus *info;
  25 +    PBSBitmapInfoList *bitmap_info;
  26
  27      info = qmp_query_backup(NULL);
  28
  29 @@ -230,26 +231,29 @@ void hmp_info_backup(Monitor *mon, const QDict *qdict)
  30              // this should not happen normally
  31              monitor_printf(mon, "Total size: %d\n", 0);
  32          } else {
  33 -            bool incremental = false;
  34              size_t total_or_dirty = info->total;
  35 -            if (info->has_transferred) {
  36 -                if (info->has_dirty && info->dirty) {
  37 -                     if (info->dirty < info->total) {
  38 -                        total_or_dirty = info->dirty;
  39 -                        incremental = true;
  40 -                    }
  41 -                }
  42 +            bitmap_info = qmp_query_pbs_bitmap_info(NULL);
  43 +
  44 +            while (bitmap_info) {
  45 +                monitor_printf(mon, "Drive %s:\n",
  46 +                        bitmap_info->value->drive);
  47 +                monitor_printf(mon, "  bitmap action: %s\n",
  48 +                        PBSBitmapAction_str(bitmap_info->value->action));
  49 +                monitor_printf(mon, "  size: %zd\n",
  50 +                        bitmap_info->value->size);
  51 +                monitor_printf(mon, "  dirty: %zd\n",
  52 +                        bitmap_info->value->dirty);
  53 +                bitmap_info = bitmap_info->next;
  54              }
  55
  56 -            int per = (info->transferred * 100)/total_or_dirty;
  57 -
  58 -            monitor_printf(mon, "Backup mode: %s\n", incremental ? "incremental" : "full");
  59 +            qapi_free_PBSBitmapInfoList(bitmap_info);
  60
  61              int zero_per = (info->has_zero_bytes && info->zero_bytes) ?
  62                  (info->zero_bytes * 100)/info->total : 0;
  63              monitor_printf(mon, "Total size: %zd\n", info->total);
  64 +            int trans_per = (info->transferred * 100)/total_or_dirty;
  65              monitor_printf(mon, "Transferred bytes: %zd (%d%%)\n",
  66 -                           info->transferred, per);
  67 +                           info->transferred, trans_per);
  68              monitor_printf(mon, "Zero bytes: %zd (%d%%)\n",
  69                             info->zero_bytes, zero_per);
  70
  71 diff --git a/pve-backup.c b/pve-backup.c
  72 index 4684789813..f90abaa50a 100644
  73 --- a/pve-backup.c
  74 +++ b/pve-backup.c
  75 @@ -46,6 +46,7 @@ static struct PVEBackupState {
  76          size_t transferred;
  77          size_t reused;
  78          size_t zero_bytes;
  79 +        GList *bitmap_list;
  80      } stat;
  81      int64_t speed;
  82      VmaWriter *vmaw;
  83 @@ -672,7 +673,6 @@ static void coroutine_fn pvebackup_co_prepare(void *opaque)
  84      }
  85
  86      size_t total = 0;
  87 -    size_t dirty = 0;
  88
  89      l = di_list;
  90      while (l) {
  91 @@ -693,18 +693,33 @@ static void coroutine_fn pvebackup_co_prepare(void *opaque)
  92
  93      uuid_generate(uuid);
  94
  95 +    qemu_mutex_lock(&backup_state.stat.lock);
  96 +    backup_state.stat.reused = 0;
  97 +
  98 +    /* clear previous backup's bitmap_list */
  99 +    if (backup_state.stat.bitmap_list) {
 100 +        GList *bl = backup_state.stat.bitmap_list;
 101 +        while (bl) {
 102 +            g_free(((PBSBitmapInfo *)bl->data)->drive);
 103 +            g_free(bl->data);
 104 +            bl = g_list_next(bl);
 105 +        }
 106 +        g_list_free(backup_state.stat.bitmap_list);
 107 +        backup_state.stat.bitmap_list = NULL;
 108 +    }
 109 +
 110      if (format == BACKUP_FORMAT_PBS) {
 111          if (!task->has_password) {
 112              error_set(task->errp, ERROR_CLASS_GENERIC_ERROR, "missing parameter 'password'");
 113 -            goto err;
 114 +            goto err_mutex;
 115          }
 116          if (!task->has_backup_id) {
 117              error_set(task->errp, ERROR_CLASS_GENERIC_ERROR, "missing parameter 'backup-id'");
 118 -            goto err;
 119 +            goto err_mutex;
 120          }
 121          if (!task->has_backup_time) {
 122              error_set(task->errp, ERROR_CLASS_GENERIC_ERROR, "missing parameter 'backup-time'");
 123 -            goto err;
 124 +            goto err_mutex;
 125          }
 126
 127          int dump_cb_block_size = PROXMOX_BACKUP_DEFAULT_CHUNK_SIZE; // Hardcoded (4M)
 128 @@ -731,12 +746,12 @@ static void coroutine_fn pvebackup_co_prepare(void *opaque)
 129              error_set(task->errp, ERROR_CLASS_GENERIC_ERROR,
 130                        "proxmox_backup_new failed: %s", pbs_err);
 131              proxmox_backup_free_error(pbs_err);
 132 -            goto err;
 133 +            goto err_mutex;
 134          }
 135
 136          int connect_result = proxmox_backup_co_connect(pbs, task->errp);
 137          if (connect_result < 0)
 138 -            goto err;
 139 +            goto err_mutex;
 140
 141          /* register all devices */
 142          l = di_list;
 143 @@ -747,6 +762,8 @@ static void coroutine_fn pvebackup_co_prepare(void *opaque)
 144              di->block_size = dump_cb_block_size;
 145
 146              const char *devname = bdrv_get_device_name(di->bs);
 147 +            PBSBitmapAction action = PBS_BITMAP_ACTION_NOT_USED;
 148 +            size_t dirty = di->size;
 149
 150              BdrvDirtyBitmap *bitmap = bdrv_find_dirty_bitmap(di->bs, PBS_BITMAP_NAME);
 151              bool expect_only_dirty = false;
 152 @@ -755,49 +772,59 @@ static void coroutine_fn pvebackup_co_prepare(void *opaque)
 153                  if (bitmap == NULL) {
 154                      bitmap = bdrv_create_dirty_bitmap(di->bs, dump_cb_block_size, PBS_BITMAP_NAME, task->errp);
 155                      if (!bitmap) {
 156 -                        goto err;
 157 +                        goto err_mutex;
 158                      }
 159 +                    action = PBS_BITMAP_ACTION_NEW;
 160                  } else {
 161                      expect_only_dirty = proxmox_backup_check_incremental(pbs, devname, di->size) != 0;
 162                  }
 163
 164                  if (expect_only_dirty) {
 165 -                    dirty += bdrv_get_dirty_count(bitmap);
 166 +                    /* track clean chunks as reused */
 167 +                    dirty = MIN(bdrv_get_dirty_count(bitmap), di->size);
 168 +                    backup_state.stat.reused += di->size - dirty;
 169 +                    action = PBS_BITMAP_ACTION_USED;
 170                  } else {
 171                      /* mark entire bitmap as dirty to make full backup */
 172                      bdrv_set_dirty_bitmap(bitmap, 0, di->size);
 173 -                    dirty += di->size;
 174 +                    if (action != PBS_BITMAP_ACTION_NEW) {
 175 +                        action = PBS_BITMAP_ACTION_INVALID;
 176 +                    }
 177                  }
 178                  di->bitmap = bitmap;
 179              } else {
 180 -                dirty += di->size;
 181 -
 182                  /* after a full backup the old dirty bitmap is invalid anyway */
 183                  if (bitmap != NULL) {
 184                      bdrv_release_dirty_bitmap(bitmap);
 185 +                    action = PBS_BITMAP_ACTION_NOT_USED_REMOVED;
 186                  }
 187              }
 188
 189              int dev_id = proxmox_backup_co_register_image(pbs, devname, di->size, expect_only_dirty, task->errp);
 190              if (dev_id < 0) {
 191 -                goto err;
 192 +                goto err_mutex;
 193              }
 194
 195              if (!(di->target = bdrv_backup_dump_create(dump_cb_block_size, di->size, pvebackup_co_dump_pbs_cb, di, task->errp))) {
 196 -                goto err;
 197 +                goto err_mutex;
 198              }
 199
 200              di->dev_id = dev_id;
 201 +
 202 +            PBSBitmapInfo *info = g_malloc(sizeof(*info));
 203 +            info->drive = g_strdup(devname);
 204 +            info->action = action;
 205 +            info->size = di->size;
 206 +            info->dirty = dirty;
 207 +            backup_state.stat.bitmap_list = g_list_append(backup_state.stat.bitmap_list, info);
 208          }
 209      } else if (format == BACKUP_FORMAT_VMA) {
 210 -        dirty = total;
 211 -
 212          vmaw = vma_writer_create(task->backup_file, uuid, &local_err);
 213          if (!vmaw) {
 214              if (local_err) {
 215                  error_propagate(task->errp, local_err);
 216              }
 217 -            goto err;
 218 +            goto err_mutex;
 219          }
 220
 221          /* register all devices for vma writer */
 222 @@ -807,7 +834,7 @@ static void coroutine_fn pvebackup_co_prepare(void *opaque)
 223              l = g_list_next(l);
 224
 225              if (!(di->target = bdrv_backup_dump_create(VMA_CLUSTER_SIZE, di->size, pvebackup_co_dump_vma_cb, di, task->errp))) {
 226 -                goto err;
 227 +                goto err_mutex;
 228              }
 229
 230              const char *devname = bdrv_get_device_name(di->bs);
 231 @@ -815,16 +842,14 @@ static void coroutine_fn pvebackup_co_prepare(void *opaque)
 232              if (di->dev_id <= 0) {
 233                  error_set(task->errp, ERROR_CLASS_GENERIC_ERROR,
 234                            "register_stream failed");
 235 -                goto err;
 236 +                goto err_mutex;
 237              }
 238          }
 239      } else if (format == BACKUP_FORMAT_DIR) {
 240 -        dirty = total;
 241 -
 242          if (mkdir(task->backup_file, 0640) != 0) {
 243              error_setg_errno(task->errp, errno, "can't create directory '%s'\n",
 244                               task->backup_file);
 245 -            goto err;
 246 +            goto err_mutex;
 247          }
 248          backup_dir = task->backup_file;
 249
 250 @@ -841,18 +866,18 @@ static void coroutine_fn pvebackup_co_prepare(void *opaque)
 251                              di->size, flags, false, &local_err);
 252              if (local_err) {
 253                  error_propagate(task->errp, local_err);
 254 -                goto err;
 255 +                goto err_mutex;
 256              }
 257
 258              di->target = bdrv_open(di->targetfile, NULL, NULL, flags, &local_err);
 259              if (!di->target) {
 260                  error_propagate(task->errp, local_err);
 261 -                goto err;
 262 +                goto err_mutex;
 263              }
 264          }
 265      } else {
 266          error_set(task->errp, ERROR_CLASS_GENERIC_ERROR, "unknown backup format");
 267 -        goto err;
 268 +        goto err_mutex;
 269      }
 270
 271
 272 @@ -860,7 +885,7 @@ static void coroutine_fn pvebackup_co_prepare(void *opaque)
 273      if (task->has_config_file) {
 274          if (pvebackup_co_add_config(task->config_file, config_name, format, backup_dir,
 275                                      vmaw, pbs, task->errp) != 0) {
 276 -            goto err;
 277 +            goto err_mutex;
 278          }
 279      }
 280
 281 @@ -868,12 +893,11 @@ static void coroutine_fn pvebackup_co_prepare(void *opaque)
 282      if (task->has_firewall_file) {
 283          if (pvebackup_co_add_config(task->firewall_file, firewall_name, format, backup_dir,
 284                                      vmaw, pbs, task->errp) != 0) {
 285 -            goto err;
 286 +            goto err_mutex;
 287          }
 288      }
 289      /* initialize global backup_state now */
 290 -
 291 -    qemu_mutex_lock(&backup_state.stat.lock);
 292 +    /* note: 'reused' and 'bitmap_list' are initialized earlier */
 293
 294      if (backup_state.stat.error) {
 295          error_free(backup_state.stat.error);
 296 @@ -893,10 +917,9 @@ static void coroutine_fn pvebackup_co_prepare(void *opaque)
 297      char *uuid_str = g_strdup(backup_state.stat.uuid_str);
 298
 299      backup_state.stat.total = total;
 300 -    backup_state.stat.dirty = dirty;
 301 +    backup_state.stat.dirty = total - backup_state.stat.reused;
 302      backup_state.stat.transferred = 0;
 303      backup_state.stat.zero_bytes = 0;
 304 -    backup_state.stat.reused = format == BACKUP_FORMAT_PBS && dirty >= total ? 0 : total - dirty;
 305
 306      qemu_mutex_unlock(&backup_state.stat.lock);
 307
 308 @@ -913,6 +936,9 @@ static void coroutine_fn pvebackup_co_prepare(void *opaque)
 309      task->result = uuid_info;
 310      return;
 311
 312 +err_mutex:
 313 +    qemu_mutex_unlock(&backup_state.stat.lock);
 314 +
 315  err:
 316
 317      l = di_list;
 318 @@ -1076,11 +1102,42 @@ BackupStatus *qmp_query_backup(Error **errp)
 319      return info;
 320  }
 321
 322 +PBSBitmapInfoList *qmp_query_pbs_bitmap_info(Error **errp)
 323 +{
 324 +    PBSBitmapInfoList *head = NULL, **p_next = &head;
 325 +
 326 +    qemu_mutex_lock(&backup_state.stat.lock);
 327 +
 328 +    GList *l = backup_state.stat.bitmap_list;
 329 +    while (l) {
 330 +        PBSBitmapInfo *info = (PBSBitmapInfo *)l->data;
 331 +        l = g_list_next(l);
 332 +
 333 +        /* clone bitmap info to avoid auto free after QMP marshalling */
 334 +        PBSBitmapInfo *info_ret = g_malloc0(sizeof(*info_ret));
 335 +        info_ret->drive = g_strdup(info->drive);
 336 +        info_ret->action = info->action;
 337 +        info_ret->size = info->size;
 338 +        info_ret->dirty = info->dirty;
 339 +
 340 +        PBSBitmapInfoList *info_list = g_malloc0(sizeof(*info_list));
 341 +        info_list->value = info_ret;
 342 +
 343 +        *p_next = info_list;
 344 +        p_next = &info_list->next;
 345 +    }
 346 +
 347 +    qemu_mutex_unlock(&backup_state.stat.lock);
 348 +
 349 +    return head;
 350 +}
 351 +
 352  ProxmoxSupportStatus *qmp_query_proxmox_support(Error **errp)
 353  {
 354      ProxmoxSupportStatus *ret = g_malloc0(sizeof(*ret));
 355      ret->pbs_library_version = g_strdup(proxmox_backup_qemu_version());
 356      ret->pbs_dirty_bitmap = true;
 357      ret->pbs_dirty_bitmap_savevm = true;
 358 +    ret->query_bitmap_info = true;
 359      return ret;
 360  }
 361 diff --git a/qapi/block-core.json b/qapi/block-core.json
 362 index 0b453c61d4..16e184dd28 100644
 363 --- a/qapi/block-core.json
 364 +++ b/qapi/block-core.json
 365 @@ -871,6 +871,8 @@
 366  # @pbs-dirty-bitmap: True if dirty-bitmap-incremental backups to PBS are
 367  #                    supported.
 368  #
 369 +# @query-bitmap-info: True if the 'query-pbs-bitmap-info' QMP call is supported.
 370 +#
 371  # @pbs-dirty-bitmap-savevm: True if 'dirty-bitmaps' migration capability can
 372  #                           safely be set for savevm-async.
 373  #
 374 @@ -879,6 +881,7 @@
 375  ##
 376  { 'struct': 'ProxmoxSupportStatus',
 377    'data': { 'pbs-dirty-bitmap': 'bool',
 378 +            'query-bitmap-info': 'bool',
 379              'pbs-dirty-bitmap-savevm': 'bool',
 380              'pbs-library-version': 'str' } }
 381
 382 @@ -892,6 +895,59 @@
 383  ##
 384  { 'command': 'query-proxmox-support', 'returns': 'ProxmoxSupportStatus' }
 385
 386 +##
 387 +# @PBSBitmapAction:
 388 +#
 389 +# An action taken on a dirty-bitmap when a backup job was started.
 390 +#
 391 +# @not-used: Bitmap mode was not enabled.
 392 +#
 393 +# @not-used-removed: Bitmap mode was not enabled, but a bitmap from a
 394 +#                    previous backup still existed and was removed.
 395 +#
 396 +# @new: A new bitmap was attached to the drive for this backup.
 397 +#
 398 +# @used: An existing bitmap will be used to only backup changed data.
 399 +#
 400 +# @invalid: A bitmap existed, but had to be cleared since it's associated
 401 +#           base snapshot did not match the base given for the current job or
 402 +#           the crypt mode has changed.
 403 +#
 404 +##
 405 +{ 'enum': 'PBSBitmapAction',
 406 +  'data': ['not-used', 'not-used-removed', 'new', 'used', 'invalid'] }
 407 +
 408 +##
 409 +# @PBSBitmapInfo:
 410 +#
 411 +# Contains information about dirty bitmaps used for each drive in a PBS backup.
 412 +#
 413 +# @drive: The underlying drive.
 414 +#
 415 +# @action: The action that was taken when the backup started.
 416 +#
 417 +# @size: The total size of the drive.
 418 +#
 419 +# @dirty: How much of the drive is considered dirty and will be backed up,
 420 +#         or 'size' if everything will be.
 421 +#
 422 +##
 423 +{ 'struct': 'PBSBitmapInfo',
 424 +  'data': { 'drive': 'str', 'action': 'PBSBitmapAction', 'size': 'int',
 425 +            'dirty': 'int' } }
 426 +
 427 +##
 428 +# @query-pbs-bitmap-info:
 429 +#
 430 +# Returns information about dirty bitmaps used on the most recently started
 431 +# backup. Returns nothing when the last backup was not using PBS or if no
 432 +# backup occured in this session.
 433 +#
 434 +# Returns: @PBSBitmapInfo
 435 +#
 436 +##
 437 +{ 'command': 'query-pbs-bitmap-info', 'returns': ['PBSBitmapInfo'] }
 438 +
 439  ##
 440  # @BlockDeviceTimedStats:
 441  #