debian/patches/pve/0032-PVE-Add-PBS-block-driver-to-map-backup-archives-into.patch

   1 From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
   2 From: Stefan Reiter <s.reiter@proxmox.com>
   3 Date: Wed, 8 Jul 2020 09:50:54 +0200
   4 Subject: [PATCH] PVE: Add PBS block driver to map backup archives into VMs
   5
   6 Signed-off-by: Stefan Reiter <s.reiter@proxmox.com>
   7 [error cleanups, file_open implementation]
   8 Signed-off-by: Dietmar Maurer <dietmar@proxmox.com>
   9 Signed-off-by: Thomas Lamprecht <t.lamprecht@proxmox.com>
  10 [WB: add namespace support]
  11 Signed-off-by: Wolfgang Bumiller <w.bumiller@proxmox.com>
  12 [FE: adapt to changed function signatures
  13      make pbs_co_preadv return values consistent with QEMU
  14      getlength is now a coroutine function]
  15 Signed-off-by: Fiona Ebner <f.ebner@proxmox.com>
  16 ---
  17  block/meson.build    |   3 +
  18  block/pbs.c          | 305 +++++++++++++++++++++++++++++++++++++++++++
  19  configure            |   9 ++
  20  meson.build          |   2 +-
  21  qapi/block-core.json |  13 ++
  22  qapi/pragma.json     |   1 +
  23  6 files changed, 332 insertions(+), 1 deletion(-)
  24  create mode 100644 block/pbs.c
  25
  26 diff --git a/block/meson.build b/block/meson.build
  27 index 5bcebb934b..eece0d5743 100644
  28 --- a/block/meson.build
  29 +++ b/block/meson.build
  30 @@ -54,6 +54,9 @@ block_ss.add(files(
  31    '../pve-backup.c',
  32  ), libproxmox_backup_qemu)
  33
  34 +block_ss.add(when: 'CONFIG_PBS_BDRV', if_true: files('pbs.c'))
  35 +block_ss.add(when: 'CONFIG_PBS_BDRV', if_true: libproxmox_backup_qemu)
  36 +
  37
  38  softmmu_ss.add(when: 'CONFIG_TCG', if_true: files('blkreplay.c'))
  39  softmmu_ss.add(files('block-ram-registrar.c'))
  40 diff --git a/block/pbs.c b/block/pbs.c
  41 new file mode 100644
  42 index 0000000000..a2211e0f3b
  43 --- /dev/null
  44 +++ b/block/pbs.c
  45 @@ -0,0 +1,305 @@
  46 +/*
  47 + * Proxmox Backup Server read-only block driver
  48 + */
  49 +
  50 +#include "qemu/osdep.h"
  51 +#include "qapi/error.h"
  52 +#include "qapi/qmp/qdict.h"
  53 +#include "qapi/qmp/qstring.h"
  54 +#include "qemu/module.h"
  55 +#include "qemu/option.h"
  56 +#include "qemu/cutils.h"
  57 +#include "block/block_int.h"
  58 +#include "block/block-io.h"
  59 +
  60 +#include <proxmox-backup-qemu.h>
  61 +
  62 +#define PBS_OPT_REPOSITORY "repository"
  63 +#define PBS_OPT_NAMESPACE "namespace"
  64 +#define PBS_OPT_SNAPSHOT "snapshot"
  65 +#define PBS_OPT_ARCHIVE "archive"
  66 +#define PBS_OPT_KEYFILE "keyfile"
  67 +#define PBS_OPT_PASSWORD "password"
  68 +#define PBS_OPT_FINGERPRINT "fingerprint"
  69 +#define PBS_OPT_ENCRYPTION_PASSWORD "key_password"
  70 +
  71 +typedef struct {
  72 +    ProxmoxRestoreHandle *conn;
  73 +    char aid;
  74 +    int64_t length;
  75 +
  76 +    char *repository;
  77 +    char *namespace;
  78 +    char *snapshot;
  79 +    char *archive;
  80 +} BDRVPBSState;
  81 +
  82 +static QemuOptsList runtime_opts = {
  83 +    .name = "pbs",
  84 +    .head = QTAILQ_HEAD_INITIALIZER(runtime_opts.head),
  85 +    .desc = {
  86 +        {
  87 +            .name = PBS_OPT_REPOSITORY,
  88 +            .type = QEMU_OPT_STRING,
  89 +            .help = "The server address and repository to connect to.",
  90 +        },
  91 +        {
  92 +            .name = PBS_OPT_NAMESPACE,
  93 +            .type = QEMU_OPT_STRING,
  94 +            .help = "Optional: The snapshot's namespace.",
  95 +        },
  96 +        {
  97 +            .name = PBS_OPT_SNAPSHOT,
  98 +            .type = QEMU_OPT_STRING,
  99 +            .help = "The snapshot to read.",
 100 +        },
 101 +        {
 102 +            .name = PBS_OPT_ARCHIVE,
 103 +            .type = QEMU_OPT_STRING,
 104 +            .help = "Which archive within the snapshot should be accessed.",
 105 +        },
 106 +        {
 107 +            .name = PBS_OPT_PASSWORD,
 108 +            .type = QEMU_OPT_STRING,
 109 +            .help = "Server password. Can be passed as env var 'PBS_PASSWORD'.",
 110 +        },
 111 +        {
 112 +            .name = PBS_OPT_FINGERPRINT,
 113 +            .type = QEMU_OPT_STRING,
 114 +            .help = "Server fingerprint. Can be passed as env var 'PBS_FINGERPRINT'.",
 115 +        },
 116 +        {
 117 +            .name = PBS_OPT_ENCRYPTION_PASSWORD,
 118 +            .type = QEMU_OPT_STRING,
 119 +            .help = "Optional: Key password. Can be passed as env var 'PBS_ENCRYPTION_PASSWORD'.",
 120 +        },
 121 +        {
 122 +            .name = PBS_OPT_KEYFILE,
 123 +            .type = QEMU_OPT_STRING,
 124 +            .help = "Optional: The path to the keyfile to use.",
 125 +        },
 126 +        { /* end of list */ }
 127 +    },
 128 +};
 129 +
 130 +
 131 +// filename format:
 132 +// pbs:repository=<repo>,namespace=<ns>,snapshot=<snap>,password=<pw>,key_password=<kpw>,fingerprint=<fp>,archive=<archive>
 133 +static void pbs_parse_filename(const char *filename, QDict *options,
 134 +                                     Error **errp)
 135 +{
 136 +
 137 +    if (!strstart(filename, "pbs:", &filename)) {
 138 +        if (errp) error_setg(errp, "pbs_parse_filename failed - missing 'pbs:' prefix");
 139 +    }
 140 +
 141 +
 142 +    QemuOpts *opts = qemu_opts_parse_noisily(&runtime_opts, filename, false);
 143 +    if (!opts) {
 144 +        if (errp) error_setg(errp, "pbs_parse_filename failed");
 145 +        return;
 146 +    }
 147 +
 148 +    qemu_opts_to_qdict(opts, options);
 149 +
 150 +    qemu_opts_del(opts);
 151 +}
 152 +
 153 +static int pbs_open(BlockDriverState *bs, QDict *options, int flags,
 154 +                    Error **errp)
 155 +{
 156 +    QemuOpts *opts;
 157 +    BDRVPBSState *s = bs->opaque;
 158 +    char *pbs_error = NULL;
 159 +
 160 +    opts = qemu_opts_create(&runtime_opts, NULL, 0, &error_abort);
 161 +    qemu_opts_absorb_qdict(opts, options, &error_abort);
 162 +
 163 +    s->repository = g_strdup(qemu_opt_get(opts, PBS_OPT_REPOSITORY));
 164 +    s->snapshot = g_strdup(qemu_opt_get(opts, PBS_OPT_SNAPSHOT));
 165 +    s->archive = g_strdup(qemu_opt_get(opts, PBS_OPT_ARCHIVE));
 166 +    const char *keyfile = qemu_opt_get(opts, PBS_OPT_KEYFILE);
 167 +    const char *password = qemu_opt_get(opts, PBS_OPT_PASSWORD);
 168 +    const char *namespace = qemu_opt_get(opts, PBS_OPT_NAMESPACE);
 169 +    const char *fingerprint = qemu_opt_get(opts, PBS_OPT_FINGERPRINT);
 170 +    const char *key_password = qemu_opt_get(opts, PBS_OPT_ENCRYPTION_PASSWORD);
 171 +
 172 +    if (!password) {
 173 +        password = getenv("PBS_PASSWORD");
 174 +    }
 175 +    if (!fingerprint) {
 176 +        fingerprint = getenv("PBS_FINGERPRINT");
 177 +    }
 178 +    if (!key_password) {
 179 +        key_password = getenv("PBS_ENCRYPTION_PASSWORD");
 180 +    }
 181 +    if (namespace) {
 182 +        s->namespace = g_strdup(namespace);
 183 +    }
 184 +
 185 +    /* connect to PBS server in read mode */
 186 +    s->conn = proxmox_restore_new_ns(s->repository, s->snapshot, s->namespace, password,
 187 +        keyfile, key_password, fingerprint, &pbs_error);
 188 +
 189 +    /* invalidates qemu_opt_get char pointers from above */
 190 +    qemu_opts_del(opts);
 191 +
 192 +    if (!s->conn) {
 193 +        if (pbs_error && errp) error_setg(errp, "PBS restore_new failed: %s", pbs_error);
 194 +        if (pbs_error) proxmox_backup_free_error(pbs_error);
 195 +        return -ENOMEM;
 196 +    }
 197 +
 198 +    int ret = proxmox_restore_connect(s->conn, &pbs_error);
 199 +    if (ret < 0) {
 200 +        if (pbs_error && errp) error_setg(errp, "PBS connect failed: %s", pbs_error);
 201 +        if (pbs_error) proxmox_backup_free_error(pbs_error);
 202 +        return -ECONNREFUSED;
 203 +    }
 204 +
 205 +    /* acquire handle and length */
 206 +    s->aid = proxmox_restore_open_image(s->conn, s->archive, &pbs_error);
 207 +    if (s->aid < 0) {
 208 +        if (pbs_error && errp) error_setg(errp, "PBS open_image failed: %s", pbs_error);
 209 +        if (pbs_error) proxmox_backup_free_error(pbs_error);
 210 +        return -ENODEV;
 211 +    }
 212 +    s->length = proxmox_restore_get_image_length(s->conn, s->aid, &pbs_error);
 213 +    if (s->length < 0) {
 214 +        if (pbs_error && errp) error_setg(errp, "PBS get_image_length failed: %s", pbs_error);
 215 +        if (pbs_error) proxmox_backup_free_error(pbs_error);
 216 +        return -EINVAL;
 217 +    }
 218 +
 219 +    return 0;
 220 +}
 221 +
 222 +static int pbs_file_open(BlockDriverState *bs, QDict *options, int flags,
 223 +                         Error **errp)
 224 +{
 225 +    return pbs_open(bs, options, flags, errp);
 226 +}
 227 +
 228 +static void pbs_close(BlockDriverState *bs) {
 229 +    BDRVPBSState *s = bs->opaque;
 230 +    g_free(s->repository);
 231 +    g_free(s->namespace);
 232 +    g_free(s->snapshot);
 233 +    g_free(s->archive);
 234 +    proxmox_restore_disconnect(s->conn);
 235 +}
 236 +
 237 +static coroutine_fn int64_t pbs_co_getlength(BlockDriverState *bs)
 238 +{
 239 +    BDRVPBSState *s = bs->opaque;
 240 +    return s->length;
 241 +}
 242 +
 243 +typedef struct ReadCallbackData {
 244 +    Coroutine *co;
 245 +    AioContext *ctx;
 246 +} ReadCallbackData;
 247 +
 248 +static void read_callback(void *callback_data)
 249 +{
 250 +    ReadCallbackData *rcb = callback_data;
 251 +    aio_co_schedule(rcb->ctx, rcb->co);
 252 +}
 253 +
 254 +static coroutine_fn int pbs_co_preadv(BlockDriverState *bs,
 255 +                                      int64_t offset, int64_t bytes,
 256 +                                      QEMUIOVector *qiov, BdrvRequestFlags flags)
 257 +{
 258 +    BDRVPBSState *s = bs->opaque;
 259 +    int ret;
 260 +    char *pbs_error = NULL;
 261 +    uint8_t *buf;
 262 +    bool inline_buf = true;
 263 +
 264 +    /* for single-buffer IO vectors we can fast-path the write directly to it */
 265 +    if (qiov->niov == 1 && qiov->iov->iov_len >= bytes) {
 266 +        buf = qiov->iov->iov_base;
 267 +    } else {
 268 +        inline_buf = false;
 269 +        buf = g_malloc(bytes);
 270 +    }
 271 +
 272 +    if (offset < 0 || bytes < 0) {
 273 +        fprintf(stderr, "unexpected negative 'offset' or 'bytes' value!\n");
 274 +        return -EIO;
 275 +    }
 276 +
 277 +    ReadCallbackData rcb = {
 278 +        .co = qemu_coroutine_self(),
 279 +        .ctx = bdrv_get_aio_context(bs),
 280 +    };
 281 +
 282 +    proxmox_restore_read_image_at_async(s->conn, s->aid, buf, (uint64_t)offset, (uint64_t)bytes,
 283 +                                        read_callback, (void *) &rcb, &ret, &pbs_error);
 284 +
 285 +    qemu_coroutine_yield();
 286 +
 287 +    if (ret < 0) {
 288 +        fprintf(stderr, "error during PBS read: %s\n", pbs_error ? pbs_error : "unknown error");
 289 +        if (pbs_error) proxmox_backup_free_error(pbs_error);
 290 +        return -EIO;
 291 +    }
 292 +
 293 +    if (!inline_buf) {
 294 +        qemu_iovec_from_buf(qiov, 0, buf, bytes);
 295 +        g_free(buf);
 296 +    }
 297 +
 298 +    return 0;
 299 +}
 300 +
 301 +static coroutine_fn int pbs_co_pwritev(BlockDriverState *bs,
 302 +                                       int64_t offset, int64_t bytes,
 303 +                                       QEMUIOVector *qiov, BdrvRequestFlags flags)
 304 +{
 305 +    fprintf(stderr, "pbs-bdrv: cannot write to backup file, make sure "
 306 +           "any attached disk devices are set to read-only!\n");
 307 +    return -EPERM;
 308 +}
 309 +
 310 +static void pbs_refresh_filename(BlockDriverState *bs)
 311 +{
 312 +    BDRVPBSState *s = bs->opaque;
 313 +    if (s->namespace) {
 314 +        snprintf(bs->exact_filename, sizeof(bs->exact_filename), "%s/%s:%s(%s)",
 315 +                 s->repository, s->namespace, s->snapshot, s->archive);
 316 +    } else {
 317 +        snprintf(bs->exact_filename, sizeof(bs->exact_filename), "%s/%s(%s)",
 318 +                 s->repository, s->snapshot, s->archive);
 319 +    }
 320 +}
 321 +
 322 +static const char *const pbs_strong_runtime_opts[] = {
 323 +    NULL
 324 +};
 325 +
 326 +static BlockDriver bdrv_pbs_co = {
 327 +    .format_name            = "pbs",
 328 +    .protocol_name          = "pbs",
 329 +    .instance_size          = sizeof(BDRVPBSState),
 330 +
 331 +    .bdrv_parse_filename    = pbs_parse_filename,
 332 +
 333 +    .bdrv_file_open         = pbs_file_open,
 334 +    .bdrv_open              = pbs_open,
 335 +    .bdrv_close             = pbs_close,
 336 +    .bdrv_co_getlength      = pbs_co_getlength,
 337 +
 338 +    .bdrv_co_preadv         = pbs_co_preadv,
 339 +    .bdrv_co_pwritev        = pbs_co_pwritev,
 340 +
 341 +    .bdrv_refresh_filename  = pbs_refresh_filename,
 342 +    .strong_runtime_opts    = pbs_strong_runtime_opts,
 343 +};
 344 +
 345 +static void bdrv_pbs_init(void)
 346 +{
 347 +    bdrv_register(&bdrv_pbs_co);
 348 +}
 349 +
 350 +block_init(bdrv_pbs_init);
 351 diff --git a/configure b/configure
 352 index a62a3e6be9..1ac0feb46b 100755
 353 --- a/configure
 354 +++ b/configure
 355 @@ -288,6 +288,7 @@ linux_user=""
 356  bsd_user=""
 357  pie=""
 358  coroutine=""
 359 +pbs_bdrv="yes"
 360  plugins="$default_feature"
 361  meson=""
 362  ninja=""
 363 @@ -873,6 +874,10 @@ for opt do
 364    ;;
 365    --with-coroutine=*) coroutine="$optarg"
 366    ;;
 367 +  --disable-pbs-bdrv) pbs_bdrv="no"
 368 +  ;;
 369 +  --enable-pbs-bdrv) pbs_bdrv="yes"
 370 +  ;;
 371    --with-git=*) git="$optarg"
 372    ;;
 373    --with-git-submodules=*)
 374 @@ -1049,6 +1054,7 @@ cat << EOF
 375    debug-info      debugging information
 376    safe-stack      SafeStack Stack Smash Protection. Depends on
 377                    clang/llvm and requires coroutine backend ucontext.
 378 +  pbs-bdrv        Proxmox backup server read-only block driver support
 379
 380  NOTE: The object files are built at the place where configure is launched
 381  EOF
 382 @@ -2386,6 +2392,9 @@ echo "TARGET_DIRS=$target_list" >> $config_host_mak
 383  if test "$modules" = "yes"; then
 384    echo "CONFIG_MODULES=y" >> $config_host_mak
 385  fi
 386 +if test "$pbs_bdrv" = "yes" ; then
 387 +  echo "CONFIG_PBS_BDRV=y" >> $config_host_mak
 388 +fi
 389
 390  # XXX: suppress that
 391  if [ "$bsd" = "yes" ] ; then
 392 diff --git a/meson.build b/meson.build
 393 index 32ab849ce6..69afe3441b 100644
 394 --- a/meson.build
 395 +++ b/meson.build
 396 @@ -4041,7 +4041,7 @@ summary_info += {'bzip2 support':     libbzip2}
 397  summary_info += {'lzfse support':     liblzfse}
 398  summary_info += {'zstd support':      zstd}
 399  summary_info += {'NUMA host support': numa}
 400 -summary_info += {'capstone':          capstone}
 401 +summary_info += {'PBS bdrv support':  config_host.has_key('CONFIG_PBS_BDRV')}
 402  summary_info += {'libpmem support':   libpmem}
 403  summary_info += {'libdaxctl support': libdaxctl}
 404  summary_info += {'libudev':           libudev}
 405 diff --git a/qapi/block-core.json b/qapi/block-core.json
 406 index 985859ddee..d601fb4ab2 100644
 407 --- a/qapi/block-core.json
 408 +++ b/qapi/block-core.json
 409 @@ -3304,6 +3304,7 @@
 410              'parallels', 'preallocate', 'qcow', 'qcow2', 'qed', 'quorum',
 411              'raw', 'rbd',
 412              { 'name': 'replication', 'if': 'CONFIG_REPLICATION' },
 413 +            'pbs',
 414              'ssh', 'throttle', 'vdi', 'vhdx',
 415              { 'name': 'virtio-blk-vfio-pci', 'if': 'CONFIG_BLKIO' },
 416              { 'name': 'virtio-blk-vhost-user', 'if': 'CONFIG_BLKIO' },
 417 @@ -3380,6 +3381,17 @@
 418  { 'struct': 'BlockdevOptionsNull',
 419    'data': { '*size': 'int', '*latency-ns': 'uint64', '*read-zeroes': 'bool' } }
 420
 421 +##
 422 +# @BlockdevOptionsPbs:
 423 +#
 424 +# Driver specific block device options for the PBS backend.
 425 +#
 426 +##
 427 +{ 'struct': 'BlockdevOptionsPbs',
 428 +  'data': { 'repository': 'str', 'snapshot': 'str', 'archive': 'str',
 429 +            '*keyfile': 'str', '*password': 'str', '*fingerprint': 'str',
 430 +            '*key_password': 'str', '*namespace': 'str' } }
 431 +
 432  ##
 433  # @BlockdevOptionsNVMe:
 434  #
 435 @@ -4753,6 +4765,7 @@
 436        'nfs':        'BlockdevOptionsNfs',
 437        'null-aio':   'BlockdevOptionsNull',
 438        'null-co':    'BlockdevOptionsNull',
 439 +      'pbs':        'BlockdevOptionsPbs',
 440        'nvme':       'BlockdevOptionsNVMe',
 441        'nvme-io_uring': { 'type': 'BlockdevOptionsNvmeIoUring',
 442                           'if': 'CONFIG_BLKIO' },
 443 diff --git a/qapi/pragma.json b/qapi/pragma.json
 444 index 325e684411..b6079f6a0e 100644
 445 --- a/qapi/pragma.json
 446 +++ b/qapi/pragma.json
 447 @@ -45,6 +45,7 @@
 448          'BlockInfo',                # query-block
 449          'BlockdevAioOptions',       # blockdev-add, -blockdev
 450          'BlockdevDriver',           # blockdev-add, query-blockstats, ...
 451 +        'BlockdevOptionsPbs',       # for PBS backwards compat
 452          'BlockdevVmdkAdapterType',  # blockdev-create (to match VMDK spec)
 453          'BlockdevVmdkSubformat',    # blockdev-create (to match VMDK spec)
 454          'ColoCompareProperties',    # object_add, -object