debian/patches/pve/0051-vma-allow-partial-restore.patch

   1 From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
   2 From: Fabian Ebner <f.ebner@proxmox.com>
   3 Date: Thu, 21 Apr 2022 13:26:48 +0200
   4 Subject: [PATCH] vma: allow partial restore
   5
   6 Introduce a new map line for skipping a certain drive, of the form
   7 skip=drive-scsi0
   8
   9 Since in PVE, most archives are compressed and piped to vma for
  10 restore, it's not easily possible to skip reads.
  11
  12 For the reader, a new skip flag for VmaRestoreState is added and the
  13 target is allowed to be NULL if skip is specified when registering. If
  14 the skip flag is set, no writes will be made as well as no check for
  15 duplicate clusters. Therefore, the flag is not set for verify.
  16
  17 Signed-off-by: Fabian Ebner <f.ebner@proxmox.com>
  18 Acked-by: Wolfgang Bumiller <w.bumiller@proxmox.com>
  19 Signed-off-by: Thomas Lamprecht <t.lamprecht@proxmox.com>
  20 ---
  21  vma-reader.c |  64 ++++++++++++---------
  22  vma.c        | 157 +++++++++++++++++++++++++++++----------------------
  23  vma.h        |   2 +-
  24  3 files changed, 126 insertions(+), 97 deletions(-)
  25
  26 diff --git a/vma-reader.c b/vma-reader.c
  27 index e65f1e8415..81a891c6b1 100644
  28 --- a/vma-reader.c
  29 +++ b/vma-reader.c
  30 @@ -28,6 +28,7 @@ typedef struct VmaRestoreState {
  31      bool write_zeroes;
  32      unsigned long *bitmap;
  33      int bitmap_size;
  34 +    bool skip;
  35  }  VmaRestoreState;
  36
  37  struct VmaReader {
  38 @@ -425,13 +426,14 @@ VmaDeviceInfo *vma_reader_get_device_info(VmaReader *vmar, guint8 dev_id)
  39  }
  40
  41  static void allocate_rstate(VmaReader *vmar,  guint8 dev_id,
  42 -                            BlockBackend *target, bool write_zeroes)
  43 +                            BlockBackend *target, bool write_zeroes, bool skip)
  44  {
  45      assert(vmar);
  46      assert(dev_id);
  47
  48      vmar->rstate[dev_id].target = target;
  49      vmar->rstate[dev_id].write_zeroes = write_zeroes;
  50 +    vmar->rstate[dev_id].skip = skip;
  51
  52      int64_t size = vmar->devinfo[dev_id].size;
  53
  54 @@ -446,28 +448,30 @@ static void allocate_rstate(VmaReader *vmar,  guint8 dev_id,
  55  }
  56
  57  int vma_reader_register_bs(VmaReader *vmar, guint8 dev_id, BlockBackend *target,
  58 -                           bool write_zeroes, Error **errp)
  59 +                           bool write_zeroes, bool skip, Error **errp)
  60  {
  61      assert(vmar);
  62 -    assert(target != NULL);
  63 +    assert(target != NULL || skip);
  64      assert(dev_id);
  65 -    assert(vmar->rstate[dev_id].target == NULL);
  66 -
  67 -    int64_t size = blk_getlength(target);
  68 -    int64_t size_diff = size - vmar->devinfo[dev_id].size;
  69 -
  70 -    /* storage types can have different size restrictions, so it
  71 -     * is not always possible to create an image with exact size.
  72 -     * So we tolerate a size difference up to 4MB.
  73 -     */
  74 -    if ((size_diff < 0) || (size_diff > 4*1024*1024)) {
  75 -        error_setg(errp, "vma_reader_register_bs for stream %s failed - "
  76 -                   "unexpected size %zd != %zd", vmar->devinfo[dev_id].devname,
  77 -                   size, vmar->devinfo[dev_id].size);
  78 -        return -1;
  79 +    assert(vmar->rstate[dev_id].target == NULL && !vmar->rstate[dev_id].skip);
  80 +
  81 +    if (target != NULL) {
  82 +        int64_t size = blk_getlength(target);
  83 +        int64_t size_diff = size - vmar->devinfo[dev_id].size;
  84 +
  85 +        /* storage types can have different size restrictions, so it
  86 +         * is not always possible to create an image with exact size.
  87 +         * So we tolerate a size difference up to 4MB.
  88 +         */
  89 +        if ((size_diff < 0) || (size_diff > 4*1024*1024)) {
  90 +            error_setg(errp, "vma_reader_register_bs for stream %s failed - "
  91 +                       "unexpected size %zd != %zd", vmar->devinfo[dev_id].devname,
  92 +                       size, vmar->devinfo[dev_id].size);
  93 +            return -1;
  94 +        }
  95      }
  96
  97 -    allocate_rstate(vmar, dev_id, target, write_zeroes);
  98 +    allocate_rstate(vmar, dev_id, target, write_zeroes, skip);
  99
 100      return 0;
 101  }
 102 @@ -560,19 +564,23 @@ static int restore_extent(VmaReader *vmar, unsigned char *buf,
 103          VmaRestoreState *rstate = &vmar->rstate[dev_id];
 104          BlockBackend *target = NULL;
 105
 106 +        bool skip = rstate->skip;
 107 +
 108          if (dev_id != vmar->vmstate_stream) {
 109              target = rstate->target;
 110 -            if (!verify && !target) {
 111 +            if (!verify && !target && !skip) {
 112                  error_setg(errp, "got wrong dev id %d", dev_id);
 113                  return -1;
 114              }
 115
 116 -            if (vma_reader_get_bitmap(rstate, cluster_num)) {
 117 -                error_setg(errp, "found duplicated cluster %zd for stream %s",
 118 -                          cluster_num, vmar->devinfo[dev_id].devname);
 119 -                return -1;
 120 +            if (!skip) {
 121 +                if (vma_reader_get_bitmap(rstate, cluster_num)) {
 122 +                    error_setg(errp, "found duplicated cluster %zd for stream %s",
 123 +                              cluster_num, vmar->devinfo[dev_id].devname);
 124 +                    return -1;
 125 +                }
 126 +                vma_reader_set_bitmap(rstate, cluster_num, 1);
 127              }
 128 -            vma_reader_set_bitmap(rstate, cluster_num, 1);
 129
 130              max_sector = vmar->devinfo[dev_id].size/BDRV_SECTOR_SIZE;
 131          } else {
 132 @@ -618,7 +626,7 @@ static int restore_extent(VmaReader *vmar, unsigned char *buf,
 133                  return -1;
 134              }
 135
 136 -            if (!verify) {
 137 +            if (!verify && !skip) {
 138                  int nb_sectors = end_sector - sector_num;
 139                  if (restore_write_data(vmar, dev_id, target, vmstate_fd,
 140                                         buf + start, sector_num, nb_sectors,
 141 @@ -654,7 +662,7 @@ static int restore_extent(VmaReader *vmar, unsigned char *buf,
 142                          return -1;
 143                      }
 144
 145 -                    if (!verify) {
 146 +                    if (!verify && !skip) {
 147                          int nb_sectors = end_sector - sector_num;
 148                          if (restore_write_data(vmar, dev_id, target, vmstate_fd,
 149                                                 buf + start, sector_num,
 150 @@ -679,7 +687,7 @@ static int restore_extent(VmaReader *vmar, unsigned char *buf,
 151                              vmar->partial_zero_cluster_data += zero_size;
 152                          }
 153
 154 -                        if (rstate->write_zeroes && !verify) {
 155 +                        if (rstate->write_zeroes && !verify && !skip) {
 156                              if (restore_write_data(vmar, dev_id, target, vmstate_fd,
 157                                                     zero_vma_block, sector_num,
 158                                                     nb_sectors, errp) < 0) {
 159 @@ -850,7 +858,7 @@ int vma_reader_verify(VmaReader *vmar, bool verbose, Error **errp)
 160
 161      for (dev_id = 1; dev_id < 255; dev_id++) {
 162          if (vma_reader_get_device_info(vmar, dev_id)) {
 163 -            allocate_rstate(vmar, dev_id, NULL, false);
 164 +            allocate_rstate(vmar, dev_id, NULL, false, false);
 165          }
 166      }
 167
 168 diff --git a/vma.c b/vma.c
 169 index e8dffb43e0..e6e9ffc7fe 100644
 170 --- a/vma.c
 171 +++ b/vma.c
 172 @@ -138,6 +138,7 @@ typedef struct RestoreMap {
 173      char *throttling_group;
 174      char *cache;
 175      bool write_zero;
 176 +    bool skip;
 177  } RestoreMap;
 178
 179  static bool try_parse_option(char **line, const char *optname, char **out, const char *inbuf) {
 180 @@ -245,47 +246,61 @@ static int extract_content(int argc, char **argv)
 181              char *bps = NULL;
 182              char *group = NULL;
 183              char *cache = NULL;
 184 +            char *devname = NULL;
 185 +            bool skip = false;
 186 +            uint64_t bps_value = 0;
 187 +            const char *path = NULL;
 188 +            bool write_zero = true;
 189 +
 190              if (!line || line[0] == '\0' || !strcmp(line, "done\n")) {
 191                  break;
 192              }
 193              int len = strlen(line);
 194              if (line[len - 1] == '\n') {
 195                  line[len - 1] = '\0';
 196 -                if (len == 1) {
 197 +                len = len - 1;
 198 +                if (len == 0) {
 199                      break;
 200                  }
 201              }
 202
 203 -            while (1) {
 204 -                if (!try_parse_option(&line, "format", &format, inbuf) &&
 205 -                    !try_parse_option(&line, "throttling.bps", &bps, inbuf) &&
 206 -                    !try_parse_option(&line, "throttling.group", &group, inbuf) &&
 207 -                    !try_parse_option(&line, "cache", &cache, inbuf))
 208 -                {
 209 -                    break;
 210 +            if (strncmp(line, "skip", 4) == 0) {
 211 +                if (len < 6 || line[4] != '=') {
 212 +                    g_error("read map failed - option 'skip' has no value ('%s')",
 213 +                            inbuf);
 214 +                } else {
 215 +                    devname = line + 5;
 216 +                    skip = true;
 217 +                }
 218 +            } else {
 219 +                while (1) {
 220 +                    if (!try_parse_option(&line, "format", &format, inbuf) &&
 221 +                        !try_parse_option(&line, "throttling.bps", &bps, inbuf) &&
 222 +                        !try_parse_option(&line, "throttling.group", &group, inbuf) &&
 223 +                        !try_parse_option(&line, "cache", &cache, inbuf))
 224 +                    {
 225 +                        break;
 226 +                    }
 227                  }
 228 -            }
 229
 230 -            uint64_t bps_value = 0;
 231 -            if (bps) {
 232 -                bps_value = verify_u64(bps);
 233 -                g_free(bps);
 234 -            }
 235 +                if (bps) {
 236 +                    bps_value = verify_u64(bps);
 237 +                    g_free(bps);
 238 +                }
 239
 240 -            const char *path;
 241 -            bool write_zero;
 242 -            if (line[0] == '0' && line[1] == ':') {
 243 -                path = line + 2;
 244 -                write_zero = false;
 245 -            } else if (line[0] == '1' && line[1] == ':') {
 246 -                path = line + 2;
 247 -                write_zero = true;
 248 -            } else {
 249 -                g_error("read map failed - parse error ('%s')", inbuf);
 250 +                if (line[0] == '0' && line[1] == ':') {
 251 +                    path = line + 2;
 252 +                    write_zero = false;
 253 +                } else if (line[0] == '1' && line[1] == ':') {
 254 +                    path = line + 2;
 255 +                    write_zero = true;
 256 +                } else {
 257 +                    g_error("read map failed - parse error ('%s')", inbuf);
 258 +                }
 259 +
 260 +                path = extract_devname(path, &devname, -1);
 261              }
 262
 263 -            char *devname = NULL;
 264 -            path = extract_devname(path, &devname, -1);
 265              if (!devname) {
 266                  g_error("read map failed - no dev name specified ('%s')",
 267                          inbuf);
 268 @@ -299,6 +314,7 @@ static int extract_content(int argc, char **argv)
 269              map->throttling_group = group;
 270              map->cache = cache;
 271              map->write_zero = write_zero;
 272 +            map->skip = skip;
 273
 274              g_hash_table_insert(devmap, map->devname, map);
 275
 276 @@ -328,6 +344,7 @@ static int extract_content(int argc, char **argv)
 277              const char *cache = NULL;
 278              int flags = BDRV_O_RDWR;
 279              bool write_zero = true;
 280 +            bool skip = false;
 281
 282              BlockBackend *blk = NULL;
 283
 284 @@ -343,6 +360,7 @@ static int extract_content(int argc, char **argv)
 285                  throttling_group = map->throttling_group;
 286                  cache = map->cache;
 287                  write_zero = map->write_zero;
 288 +                skip = map->skip;
 289              } else {
 290                  devfn = g_strdup_printf("%s/tmp-disk-%s.raw",
 291                                          dirname, di->devname);
 292 @@ -361,57 +379,60 @@ static int extract_content(int argc, char **argv)
 293                  write_zero = false;
 294              }
 295
 296 -           size_t devlen = strlen(devfn);
 297 -           QDict *options = NULL;
 298 -            bool writethrough;
 299 -            if (format) {
 300 -                /* explicit format from commandline */
 301 -                options = qdict_new();
 302 -                qdict_put_str(options, "driver", format);
 303 -            } else if ((devlen > 4 && strcmp(devfn+devlen-4, ".raw") == 0) ||
 304 -                      strncmp(devfn, "/dev/", 5) == 0)
 305 -           {
 306 -                /* This part is now deprecated for PVE as well (just as qemu
 307 -                 * deprecated not specifying an explicit raw format, too.
 308 -                 */
 309 -               /* explicit raw format */
 310 -               options = qdict_new();
 311 -               qdict_put_str(options, "driver", "raw");
 312 -           }
 313 -            if (cache && bdrv_parse_cache_mode(cache, &flags, &writethrough)) {
 314 -                g_error("invalid cache option: %s\n", cache);
 315 -            }
 316 +            if (!skip) {
 317 +                size_t devlen = strlen(devfn);
 318 +                QDict *options = NULL;
 319 +                bool writethrough;
 320 +                if (format) {
 321 +                    /* explicit format from commandline */
 322 +                    options = qdict_new();
 323 +                    qdict_put_str(options, "driver", format);
 324 +                } else if ((devlen > 4 && strcmp(devfn+devlen-4, ".raw") == 0) ||
 325 +                    strncmp(devfn, "/dev/", 5) == 0)
 326 +                {
 327 +                    /* This part is now deprecated for PVE as well (just as qemu
 328 +                     * deprecated not specifying an explicit raw format, too.
 329 +                     */
 330 +                    /* explicit raw format */
 331 +                    options = qdict_new();
 332 +                    qdict_put_str(options, "driver", "raw");
 333 +                }
 334
 335 -           if (errp || !(blk = blk_new_open(devfn, NULL, options, flags, &errp))) {
 336 -                g_error("can't open file %s - %s", devfn,
 337 -                        error_get_pretty(errp));
 338 -            }
 339 +                if (cache && bdrv_parse_cache_mode(cache, &flags, &writethrough)) {
 340 +                    g_error("invalid cache option: %s\n", cache);
 341 +                }
 342
 343 -            if (cache) {
 344 -                blk_set_enable_write_cache(blk, !writethrough);
 345 -            }
 346 +                if (errp || !(blk = blk_new_open(devfn, NULL, options, flags, &errp))) {
 347 +                    g_error("can't open file %s - %s", devfn,
 348 +                            error_get_pretty(errp));
 349 +                }
 350
 351 -            if (throttling_group) {
 352 -                blk_io_limits_enable(blk, throttling_group);
 353 -            }
 354 +                if (cache) {
 355 +                    blk_set_enable_write_cache(blk, !writethrough);
 356 +                }
 357
 358 -            if (throttling_bps) {
 359 -                if (!throttling_group) {
 360 -                    blk_io_limits_enable(blk, devfn);
 361 +                if (throttling_group) {
 362 +                    blk_io_limits_enable(blk, throttling_group);
 363                  }
 364
 365 -                ThrottleConfig cfg;
 366 -                throttle_config_init(&cfg);
 367 -                cfg.buckets[THROTTLE_BPS_WRITE].avg = throttling_bps;
 368 -                Error *err = NULL;
 369 -                if (!throttle_is_valid(&cfg, &err)) {
 370 -                    error_report_err(err);
 371 -                    g_error("failed to apply throttling");
 372 +                if (throttling_bps) {
 373 +                    if (!throttling_group) {
 374 +                        blk_io_limits_enable(blk, devfn);
 375 +                    }
 376 +
 377 +                    ThrottleConfig cfg;
 378 +                    throttle_config_init(&cfg);
 379 +                    cfg.buckets[THROTTLE_BPS_WRITE].avg = throttling_bps;
 380 +                    Error *err = NULL;
 381 +                    if (!throttle_is_valid(&cfg, &err)) {
 382 +                        error_report_err(err);
 383 +                        g_error("failed to apply throttling");
 384 +                    }
 385 +                    blk_set_io_limits(blk, &cfg);
 386                  }
 387 -                blk_set_io_limits(blk, &cfg);
 388              }
 389
 390 -            if (vma_reader_register_bs(vmar, i, blk, write_zero, &errp) < 0) {
 391 +            if (vma_reader_register_bs(vmar, i, blk, write_zero, skip, &errp) < 0) {
 392                  g_error("%s", error_get_pretty(errp));
 393              }
 394
 395 diff --git a/vma.h b/vma.h
 396 index c895c97f6d..1b62859165 100644
 397 --- a/vma.h
 398 +++ b/vma.h
 399 @@ -142,7 +142,7 @@ GList *vma_reader_get_config_data(VmaReader *vmar);
 400  VmaDeviceInfo *vma_reader_get_device_info(VmaReader *vmar, guint8 dev_id);
 401  int vma_reader_register_bs(VmaReader *vmar, guint8 dev_id,
 402                             BlockBackend *target, bool write_zeroes,
 403 -                           Error **errp);
 404 +                           bool skip, Error **errp);
 405  int vma_reader_restore(VmaReader *vmar, int vmstate_fd, bool verbose,
 406                         Error **errp);
 407  int vma_reader_verify(VmaReader *vmar, bool verbose, Error **errp);