debian/patches/pve/0040-PVE-add-query-pbs-bitmap-info-QMP-call.patch

   1 From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
   2 From: Stefan Reiter <s.reiter@proxmox.com>
   3 Date: Wed, 19 Aug 2020 17:02:00 +0200
   4 Subject: [PATCH] PVE: add query-pbs-bitmap-info QMP call
   5
   6 Returns advanced information about dirty bitmaps used (or not used) for
   7 the latest PBS backup.
   8
   9 Signed-off-by: Stefan Reiter <s.reiter@proxmox.com>
  10 ---
  11  monitor/hmp-cmds.c   |  28 ++++++-----
  12  pve-backup.c         | 117 ++++++++++++++++++++++++++++++++-----------
  13  qapi/block-core.json |  56 +++++++++++++++++++++
  14  3 files changed, 159 insertions(+), 42 deletions(-)
  15
  16 diff --git a/monitor/hmp-cmds.c b/monitor/hmp-cmds.c
  17 index 604026bb37..95f4e7f5c1 100644
  18 --- a/monitor/hmp-cmds.c
  19 +++ b/monitor/hmp-cmds.c
  20 @@ -198,6 +198,7 @@ void hmp_info_mice(Monitor *mon, const QDict *qdict)
  21  void hmp_info_backup(Monitor *mon, const QDict *qdict)
  22  {
  23      BackupStatus *info;
  24 +    PBSBitmapInfoList *bitmap_info;
  25
  26      info = qmp_query_backup(NULL);
  27
  28 @@ -228,26 +229,29 @@ void hmp_info_backup(Monitor *mon, const QDict *qdict)
  29              // this should not happen normally
  30              monitor_printf(mon, "Total size: %d\n", 0);
  31          } else {
  32 -            bool incremental = false;
  33              size_t total_or_dirty = info->total;
  34 -            if (info->has_transferred) {
  35 -                if (info->has_dirty && info->dirty) {
  36 -                     if (info->dirty < info->total) {
  37 -                        total_or_dirty = info->dirty;
  38 -                        incremental = true;
  39 -                    }
  40 -                }
  41 +            bitmap_info = qmp_query_pbs_bitmap_info(NULL);
  42 +
  43 +            while (bitmap_info) {
  44 +                monitor_printf(mon, "Drive %s:\n",
  45 +                        bitmap_info->value->drive);
  46 +                monitor_printf(mon, "  bitmap action: %s\n",
  47 +                        PBSBitmapAction_str(bitmap_info->value->action));
  48 +                monitor_printf(mon, "  size: %zd\n",
  49 +                        bitmap_info->value->size);
  50 +                monitor_printf(mon, "  dirty: %zd\n",
  51 +                        bitmap_info->value->dirty);
  52 +                bitmap_info = bitmap_info->next;
  53              }
  54
  55 -            int per = (info->transferred * 100)/total_or_dirty;
  56 -
  57 -            monitor_printf(mon, "Backup mode: %s\n", incremental ? "incremental" : "full");
  58 +            qapi_free_PBSBitmapInfoList(bitmap_info);
  59
  60              int zero_per = (info->has_zero_bytes && info->zero_bytes) ?
  61                  (info->zero_bytes * 100)/info->total : 0;
  62              monitor_printf(mon, "Total size: %zd\n", info->total);
  63 +            int trans_per = (info->transferred * 100)/total_or_dirty;
  64              monitor_printf(mon, "Transferred bytes: %zd (%d%%)\n",
  65 -                           info->transferred, per);
  66 +                           info->transferred, trans_per);
  67              monitor_printf(mon, "Zero bytes: %zd (%d%%)\n",
  68                             info->zero_bytes, zero_per);
  69
  70 diff --git a/pve-backup.c b/pve-backup.c
  71 index 40c2697b37..1e7b92a950 100644
  72 --- a/pve-backup.c
  73 +++ b/pve-backup.c
  74 @@ -46,6 +46,7 @@ static struct PVEBackupState {
  75          size_t transferred;
  76          size_t reused;
  77          size_t zero_bytes;
  78 +        GList *bitmap_list;
  79      } stat;
  80      int64_t speed;
  81      VmaWriter *vmaw;
  82 @@ -670,7 +671,6 @@ static void coroutine_fn pvebackup_co_prepare(void *opaque)
  83      }
  84
  85      size_t total = 0;
  86 -    size_t dirty = 0;
  87
  88      l = di_list;
  89      while (l) {
  90 @@ -691,18 +691,33 @@ static void coroutine_fn pvebackup_co_prepare(void *opaque)
  91
  92      uuid_generate(uuid);
  93
  94 +    qemu_mutex_lock(&backup_state.stat.lock);
  95 +    backup_state.stat.reused = 0;
  96 +
  97 +    /* clear previous backup's bitmap_list */
  98 +    if (backup_state.stat.bitmap_list) {
  99 +        GList *bl = backup_state.stat.bitmap_list;
 100 +        while (bl) {
 101 +            g_free(((PBSBitmapInfo *)bl->data)->drive);
 102 +            g_free(bl->data);
 103 +            bl = g_list_next(bl);
 104 +        }
 105 +        g_list_free(backup_state.stat.bitmap_list);
 106 +        backup_state.stat.bitmap_list = NULL;
 107 +    }
 108 +
 109      if (format == BACKUP_FORMAT_PBS) {
 110          if (!task->has_password) {
 111              error_set(task->errp, ERROR_CLASS_GENERIC_ERROR, "missing parameter 'password'");
 112 -            goto err;
 113 +            goto err_mutex;
 114          }
 115          if (!task->has_backup_id) {
 116              error_set(task->errp, ERROR_CLASS_GENERIC_ERROR, "missing parameter 'backup-id'");
 117 -            goto err;
 118 +            goto err_mutex;
 119          }
 120          if (!task->has_backup_time) {
 121              error_set(task->errp, ERROR_CLASS_GENERIC_ERROR, "missing parameter 'backup-time'");
 122 -            goto err;
 123 +            goto err_mutex;
 124          }
 125
 126          int dump_cb_block_size = PROXMOX_BACKUP_DEFAULT_CHUNK_SIZE; // Hardcoded (4M)
 127 @@ -729,12 +744,12 @@ static void coroutine_fn pvebackup_co_prepare(void *opaque)
 128              error_set(task->errp, ERROR_CLASS_GENERIC_ERROR,
 129                        "proxmox_backup_new failed: %s", pbs_err);
 130              proxmox_backup_free_error(pbs_err);
 131 -            goto err;
 132 +            goto err_mutex;
 133          }
 134
 135          int connect_result = proxmox_backup_co_connect(pbs, task->errp);
 136          if (connect_result < 0)
 137 -            goto err;
 138 +            goto err_mutex;
 139
 140          /* register all devices */
 141          l = di_list;
 142 @@ -745,6 +760,8 @@ static void coroutine_fn pvebackup_co_prepare(void *opaque)
 143              di->block_size = dump_cb_block_size;
 144
 145              const char *devname = bdrv_get_device_name(di->bs);
 146 +            PBSBitmapAction action = PBS_BITMAP_ACTION_NOT_USED;
 147 +            size_t dirty = di->size;
 148
 149              BdrvDirtyBitmap *bitmap = bdrv_find_dirty_bitmap(di->bs, PBS_BITMAP_NAME);
 150              bool expect_only_dirty = false;
 151 @@ -753,49 +770,59 @@ static void coroutine_fn pvebackup_co_prepare(void *opaque)
 152                  if (bitmap == NULL) {
 153                      bitmap = bdrv_create_dirty_bitmap(di->bs, dump_cb_block_size, PBS_BITMAP_NAME, task->errp);
 154                      if (!bitmap) {
 155 -                        goto err;
 156 +                        goto err_mutex;
 157                      }
 158 +                    action = PBS_BITMAP_ACTION_NEW;
 159                  } else {
 160                      expect_only_dirty = proxmox_backup_check_incremental(pbs, devname, di->size) != 0;
 161                  }
 162
 163                  if (expect_only_dirty) {
 164 -                    dirty += bdrv_get_dirty_count(bitmap);
 165 +                    /* track clean chunks as reused */
 166 +                    dirty = MIN(bdrv_get_dirty_count(bitmap), di->size);
 167 +                    backup_state.stat.reused += di->size - dirty;
 168 +                    action = PBS_BITMAP_ACTION_USED;
 169                  } else {
 170                      /* mark entire bitmap as dirty to make full backup */
 171                      bdrv_set_dirty_bitmap(bitmap, 0, di->size);
 172 -                    dirty += di->size;
 173 +                    if (action != PBS_BITMAP_ACTION_NEW) {
 174 +                        action = PBS_BITMAP_ACTION_INVALID;
 175 +                    }
 176                  }
 177                  di->bitmap = bitmap;
 178              } else {
 179 -                dirty += di->size;
 180 -
 181                  /* after a full backup the old dirty bitmap is invalid anyway */
 182                  if (bitmap != NULL) {
 183                      bdrv_release_dirty_bitmap(bitmap);
 184 +                    action = PBS_BITMAP_ACTION_NOT_USED_REMOVED;
 185                  }
 186              }
 187
 188              int dev_id = proxmox_backup_co_register_image(pbs, devname, di->size, expect_only_dirty, task->errp);
 189              if (dev_id < 0) {
 190 -                goto err;
 191 +                goto err_mutex;
 192              }
 193
 194              if (!(di->target = bdrv_backup_dump_create(dump_cb_block_size, di->size, pvebackup_co_dump_pbs_cb, di, task->errp))) {
 195 -                goto err;
 196 +                goto err_mutex;
 197              }
 198
 199              di->dev_id = dev_id;
 200 +
 201 +            PBSBitmapInfo *info = g_malloc(sizeof(*info));
 202 +            info->drive = g_strdup(devname);
 203 +            info->action = action;
 204 +            info->size = di->size;
 205 +            info->dirty = dirty;
 206 +            backup_state.stat.bitmap_list = g_list_append(backup_state.stat.bitmap_list, info);
 207          }
 208      } else if (format == BACKUP_FORMAT_VMA) {
 209 -        dirty = total;
 210 -
 211          vmaw = vma_writer_create(task->backup_file, uuid, &local_err);
 212          if (!vmaw) {
 213              if (local_err) {
 214                  error_propagate(task->errp, local_err);
 215              }
 216 -            goto err;
 217 +            goto err_mutex;
 218          }
 219
 220          /* register all devices for vma writer */
 221 @@ -805,7 +832,7 @@ static void coroutine_fn pvebackup_co_prepare(void *opaque)
 222              l = g_list_next(l);
 223
 224              if (!(di->target = bdrv_backup_dump_create(VMA_CLUSTER_SIZE, di->size, pvebackup_co_dump_vma_cb, di, task->errp))) {
 225 -                goto err;
 226 +                goto err_mutex;
 227              }
 228
 229              const char *devname = bdrv_get_device_name(di->bs);
 230 @@ -813,16 +840,14 @@ static void coroutine_fn pvebackup_co_prepare(void *opaque)
 231              if (di->dev_id <= 0) {
 232                  error_set(task->errp, ERROR_CLASS_GENERIC_ERROR,
 233                            "register_stream failed");
 234 -                goto err;
 235 +                goto err_mutex;
 236              }
 237          }
 238      } else if (format == BACKUP_FORMAT_DIR) {
 239 -        dirty = total;
 240 -
 241          if (mkdir(task->backup_file, 0640) != 0) {
 242              error_setg_errno(task->errp, errno, "can't create directory '%s'\n",
 243                               task->backup_file);
 244 -            goto err;
 245 +            goto err_mutex;
 246          }
 247          backup_dir = task->backup_file;
 248
 249 @@ -839,18 +864,18 @@ static void coroutine_fn pvebackup_co_prepare(void *opaque)
 250                              di->size, flags, false, &local_err);
 251              if (local_err) {
 252                  error_propagate(task->errp, local_err);
 253 -                goto err;
 254 +                goto err_mutex;
 255              }
 256
 257              di->target = bdrv_open(di->targetfile, NULL, NULL, flags, &local_err);
 258              if (!di->target) {
 259                  error_propagate(task->errp, local_err);
 260 -                goto err;
 261 +                goto err_mutex;
 262              }
 263          }
 264      } else {
 265          error_set(task->errp, ERROR_CLASS_GENERIC_ERROR, "unknown backup format");
 266 -        goto err;
 267 +        goto err_mutex;
 268      }
 269
 270
 271 @@ -858,7 +883,7 @@ static void coroutine_fn pvebackup_co_prepare(void *opaque)
 272      if (task->has_config_file) {
 273          if (pvebackup_co_add_config(task->config_file, config_name, format, backup_dir,
 274                                      vmaw, pbs, task->errp) != 0) {
 275 -            goto err;
 276 +            goto err_mutex;
 277          }
 278      }
 279
 280 @@ -866,12 +891,11 @@ static void coroutine_fn pvebackup_co_prepare(void *opaque)
 281      if (task->has_firewall_file) {
 282          if (pvebackup_co_add_config(task->firewall_file, firewall_name, format, backup_dir,
 283                                      vmaw, pbs, task->errp) != 0) {
 284 -            goto err;
 285 +            goto err_mutex;
 286          }
 287      }
 288      /* initialize global backup_state now */
 289 -
 290 -    qemu_mutex_lock(&backup_state.stat.lock);
 291 +    /* note: 'reused' and 'bitmap_list' are initialized earlier */
 292
 293      if (backup_state.stat.error) {
 294          error_free(backup_state.stat.error);
 295 @@ -891,10 +915,9 @@ static void coroutine_fn pvebackup_co_prepare(void *opaque)
 296      char *uuid_str = g_strdup(backup_state.stat.uuid_str);
 297
 298      backup_state.stat.total = total;
 299 -    backup_state.stat.dirty = dirty;
 300 +    backup_state.stat.dirty = total - backup_state.stat.reused;
 301      backup_state.stat.transferred = 0;
 302      backup_state.stat.zero_bytes = 0;
 303 -    backup_state.stat.reused = format == BACKUP_FORMAT_PBS && dirty >= total ? 0 : total - dirty;
 304
 305      qemu_mutex_unlock(&backup_state.stat.lock);
 306
 307 @@ -911,6 +934,9 @@ static void coroutine_fn pvebackup_co_prepare(void *opaque)
 308      task->result = uuid_info;
 309      return;
 310
 311 +err_mutex:
 312 +    qemu_mutex_unlock(&backup_state.stat.lock);
 313 +
 314  err:
 315
 316      l = di_list;
 317 @@ -1074,10 +1100,41 @@ BackupStatus *qmp_query_backup(Error **errp)
 318      return info;
 319  }
 320
 321 +PBSBitmapInfoList *qmp_query_pbs_bitmap_info(Error **errp)
 322 +{
 323 +    PBSBitmapInfoList *head = NULL, **p_next = &head;
 324 +
 325 +    qemu_mutex_lock(&backup_state.stat.lock);
 326 +
 327 +    GList *l = backup_state.stat.bitmap_list;
 328 +    while (l) {
 329 +        PBSBitmapInfo *info = (PBSBitmapInfo *)l->data;
 330 +        l = g_list_next(l);
 331 +
 332 +        /* clone bitmap info to avoid auto free after QMP marshalling */
 333 +        PBSBitmapInfo *info_ret = g_malloc0(sizeof(*info_ret));
 334 +        info_ret->drive = g_strdup(info->drive);
 335 +        info_ret->action = info->action;
 336 +        info_ret->size = info->size;
 337 +        info_ret->dirty = info->dirty;
 338 +
 339 +        PBSBitmapInfoList *info_list = g_malloc0(sizeof(*info_list));
 340 +        info_list->value = info_ret;
 341 +
 342 +        *p_next = info_list;
 343 +        p_next = &info_list->next;
 344 +    }
 345 +
 346 +    qemu_mutex_unlock(&backup_state.stat.lock);
 347 +
 348 +    return head;
 349 +}
 350 +
 351  ProxmoxSupportStatus *qmp_query_proxmox_support(Error **errp)
 352  {
 353      ProxmoxSupportStatus *ret = g_malloc0(sizeof(*ret));
 354      ret->pbs_library_version = g_strdup(proxmox_backup_qemu_version());
 355      ret->pbs_dirty_bitmap = true;
 356 +    ret->query_bitmap_info = true;
 357      return ret;
 358  }
 359 diff --git a/qapi/block-core.json b/qapi/block-core.json
 360 index e0a0a60354..b368371e8e 100644
 361 --- a/qapi/block-core.json
 362 +++ b/qapi/block-core.json
 363 @@ -876,11 +876,14 @@
 364  # @pbs-dirty-bitmap: True if dirty-bitmap-incremental backups to PBS are
 365  #                    supported.
 366  #
 367 +# @query-bitmap-info: True if the 'query-pbs-bitmap-info' QMP call is supported.
 368 +#
 369  # @pbs-library-version: Running version of libproxmox-backup-qemu0 library.
 370  #
 371  ##
 372  { 'struct': 'ProxmoxSupportStatus',
 373    'data': { 'pbs-dirty-bitmap': 'bool',
 374 +            'query-bitmap-info': 'bool',
 375              'pbs-library-version': 'str' } }
 376
 377  ##
 378 @@ -893,6 +896,59 @@
 379  ##
 380  { 'command': 'query-proxmox-support', 'returns': 'ProxmoxSupportStatus' }
 381
 382 +##
 383 +# @PBSBitmapAction:
 384 +#
 385 +# An action taken on a dirty-bitmap when a backup job was started.
 386 +#
 387 +# @not-used: Bitmap mode was not enabled.
 388 +#
 389 +# @not-used-removed: Bitmap mode was not enabled, but a bitmap from a
 390 +#                    previous backup still existed and was removed.
 391 +#
 392 +# @new: A new bitmap was attached to the drive for this backup.
 393 +#
 394 +# @used: An existing bitmap will be used to only backup changed data.
 395 +#
 396 +# @invalid: A bitmap existed, but had to be cleared since it's associated
 397 +#           base snapshot did not match the base given for the current job or
 398 +#           the crypt mode has changed.
 399 +#
 400 +##
 401 +{ 'enum': 'PBSBitmapAction',
 402 +  'data': ['not-used', 'not-used-removed', 'new', 'used', 'invalid'] }
 403 +
 404 +##
 405 +# @PBSBitmapInfo:
 406 +#
 407 +# Contains information about dirty bitmaps used for each drive in a PBS backup.
 408 +#
 409 +# @drive: The underlying drive.
 410 +#
 411 +# @action: The action that was taken when the backup started.
 412 +#
 413 +# @size: The total size of the drive.
 414 +#
 415 +# @dirty: How much of the drive is considered dirty and will be backed up,
 416 +#         or 'size' if everything will be.
 417 +#
 418 +##
 419 +{ 'struct': 'PBSBitmapInfo',
 420 +  'data': { 'drive': 'str', 'action': 'PBSBitmapAction', 'size': 'int',
 421 +            'dirty': 'int' } }
 422 +
 423 +##
 424 +# @query-pbs-bitmap-info:
 425 +#
 426 +# Returns information about dirty bitmaps used on the most recently started
 427 +# backup. Returns nothing when the last backup was not using PBS or if no
 428 +# backup occured in this session.
 429 +#
 430 +# Returns: @PBSBitmapInfo
 431 +#
 432 +##
 433 +{ 'command': 'query-pbs-bitmap-info', 'returns': ['PBSBitmapInfo'] }
 434 +
 435  ##
 436  # @BlockDeviceTimedStats:
 437  #