debian/patches/pve/0032-PVE-Add-PBS-block-driver-to-map-backup-archives-into.patch

   1 From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
   2 From: Stefan Reiter <s.reiter@proxmox.com>
   3 Date: Wed, 8 Jul 2020 09:50:54 +0200
   4 Subject: [PATCH] PVE: Add PBS block driver to map backup archives into VMs
   5
   6 Signed-off-by: Stefan Reiter <s.reiter@proxmox.com>
   7 [error cleanups, file_open implementation]
   8 Signed-off-by: Dietmar Maurer <dietmar@proxmox.com>
   9 Signed-off-by: Thomas Lamprecht <t.lamprecht@proxmox.com>
  10 [adapt to changed function signatures]
  11 Signed-off-by: Fabian Ebner <f.ebner@proxmox.com>
  12 ---
  13  block/meson.build    |   3 +
  14  block/pbs.c          | 276 +++++++++++++++++++++++++++++++++++++++++++
  15  configure            |   9 ++
  16  meson.build          |   1 +
  17  qapi/block-core.json |  13 ++
  18  5 files changed, 302 insertions(+)
  19  create mode 100644 block/pbs.c
  20
  21 diff --git a/block/meson.build b/block/meson.build
  22 index 9d3dd5b7c3..8c758c0218 100644
  23 --- a/block/meson.build
  24 +++ b/block/meson.build
  25 @@ -51,6 +51,9 @@ block_ss.add(files(
  26    '../pve-backup.c',
  27  ), libproxmox_backup_qemu)
  28
  29 +block_ss.add(when: 'CONFIG_PBS_BDRV', if_true: files('pbs.c'))
  30 +block_ss.add(when: 'CONFIG_PBS_BDRV', if_true: libproxmox_backup_qemu)
  31 +
  32
  33  softmmu_ss.add(when: 'CONFIG_TCG', if_true: files('blkreplay.c'))
  34
  35 diff --git a/block/pbs.c b/block/pbs.c
  36 new file mode 100644
  37 index 0000000000..0b05ea9080
  38 --- /dev/null
  39 +++ b/block/pbs.c
  40 @@ -0,0 +1,276 @@
  41 +/*
  42 + * Proxmox Backup Server read-only block driver
  43 + */
  44 +
  45 +#include "qemu/osdep.h"
  46 +#include "qapi/error.h"
  47 +#include "qapi/qmp/qdict.h"
  48 +#include "qapi/qmp/qstring.h"
  49 +#include "qemu/module.h"
  50 +#include "qemu/option.h"
  51 +#include "qemu/cutils.h"
  52 +#include "block/block_int.h"
  53 +
  54 +#include <proxmox-backup-qemu.h>
  55 +
  56 +#define PBS_OPT_REPOSITORY "repository"
  57 +#define PBS_OPT_SNAPSHOT "snapshot"
  58 +#define PBS_OPT_ARCHIVE "archive"
  59 +#define PBS_OPT_KEYFILE "keyfile"
  60 +#define PBS_OPT_PASSWORD "password"
  61 +#define PBS_OPT_FINGERPRINT "fingerprint"
  62 +#define PBS_OPT_ENCRYPTION_PASSWORD "key_password"
  63 +
  64 +typedef struct {
  65 +    ProxmoxRestoreHandle *conn;
  66 +    char aid;
  67 +    int64_t length;
  68 +
  69 +    char *repository;
  70 +    char *snapshot;
  71 +    char *archive;
  72 +} BDRVPBSState;
  73 +
  74 +static QemuOptsList runtime_opts = {
  75 +    .name = "pbs",
  76 +    .head = QTAILQ_HEAD_INITIALIZER(runtime_opts.head),
  77 +    .desc = {
  78 +        {
  79 +            .name = PBS_OPT_REPOSITORY,
  80 +            .type = QEMU_OPT_STRING,
  81 +            .help = "The server address and repository to connect to.",
  82 +        },
  83 +        {
  84 +            .name = PBS_OPT_SNAPSHOT,
  85 +            .type = QEMU_OPT_STRING,
  86 +            .help = "The snapshot to read.",
  87 +        },
  88 +        {
  89 +            .name = PBS_OPT_ARCHIVE,
  90 +            .type = QEMU_OPT_STRING,
  91 +            .help = "Which archive within the snapshot should be accessed.",
  92 +        },
  93 +        {
  94 +            .name = PBS_OPT_PASSWORD,
  95 +            .type = QEMU_OPT_STRING,
  96 +            .help = "Server password. Can be passed as env var 'PBS_PASSWORD'.",
  97 +        },
  98 +        {
  99 +            .name = PBS_OPT_FINGERPRINT,
 100 +            .type = QEMU_OPT_STRING,
 101 +            .help = "Server fingerprint. Can be passed as env var 'PBS_FINGERPRINT'.",
 102 +        },
 103 +        {
 104 +            .name = PBS_OPT_ENCRYPTION_PASSWORD,
 105 +            .type = QEMU_OPT_STRING,
 106 +            .help = "Optional: Key password. Can be passed as env var 'PBS_ENCRYPTION_PASSWORD'.",
 107 +        },
 108 +        {
 109 +            .name = PBS_OPT_KEYFILE,
 110 +            .type = QEMU_OPT_STRING,
 111 +            .help = "Optional: The path to the keyfile to use.",
 112 +        },
 113 +        { /* end of list */ }
 114 +    },
 115 +};
 116 +
 117 +
 118 +// filename format:
 119 +// pbs:repository=<repo>,snapshot=<snap>,password=<pw>,key_password=<kpw>,fingerprint=<fp>,archive=<archive>
 120 +static void pbs_parse_filename(const char *filename, QDict *options,
 121 +                                     Error **errp)
 122 +{
 123 +
 124 +    if (!strstart(filename, "pbs:", &filename)) {
 125 +        if (errp) error_setg(errp, "pbs_parse_filename failed - missing 'pbs:' prefix");
 126 +    }
 127 +
 128 +
 129 +    QemuOpts *opts = qemu_opts_parse_noisily(&runtime_opts, filename, false);
 130 +    if (!opts) {
 131 +        if (errp) error_setg(errp, "pbs_parse_filename failed");
 132 +        return;
 133 +    }
 134 +
 135 +    qemu_opts_to_qdict(opts, options);
 136 +
 137 +    qemu_opts_del(opts);
 138 +}
 139 +
 140 +static int pbs_open(BlockDriverState *bs, QDict *options, int flags,
 141 +                    Error **errp)
 142 +{
 143 +    QemuOpts *opts;
 144 +    BDRVPBSState *s = bs->opaque;
 145 +    char *pbs_error = NULL;
 146 +
 147 +    opts = qemu_opts_create(&runtime_opts, NULL, 0, &error_abort);
 148 +    qemu_opts_absorb_qdict(opts, options, &error_abort);
 149 +
 150 +    s->repository = g_strdup(qemu_opt_get(opts, PBS_OPT_REPOSITORY));
 151 +    s->snapshot = g_strdup(qemu_opt_get(opts, PBS_OPT_SNAPSHOT));
 152 +    s->archive = g_strdup(qemu_opt_get(opts, PBS_OPT_ARCHIVE));
 153 +    const char *keyfile = qemu_opt_get(opts, PBS_OPT_KEYFILE);
 154 +    const char *password = qemu_opt_get(opts, PBS_OPT_PASSWORD);
 155 +    const char *fingerprint = qemu_opt_get(opts, PBS_OPT_FINGERPRINT);
 156 +    const char *key_password = qemu_opt_get(opts, PBS_OPT_ENCRYPTION_PASSWORD);
 157 +
 158 +    if (!password) {
 159 +        password = getenv("PBS_PASSWORD");
 160 +    }
 161 +    if (!fingerprint) {
 162 +        fingerprint = getenv("PBS_FINGERPRINT");
 163 +    }
 164 +    if (!key_password) {
 165 +        key_password = getenv("PBS_ENCRYPTION_PASSWORD");
 166 +    }
 167 +
 168 +    /* connect to PBS server in read mode */
 169 +    s->conn = proxmox_restore_new(s->repository, s->snapshot, password,
 170 +        keyfile, key_password, fingerprint, &pbs_error);
 171 +
 172 +    /* invalidates qemu_opt_get char pointers from above */
 173 +    qemu_opts_del(opts);
 174 +
 175 +    if (!s->conn) {
 176 +        if (pbs_error && errp) error_setg(errp, "PBS restore_new failed: %s", pbs_error);
 177 +        if (pbs_error) proxmox_backup_free_error(pbs_error);
 178 +        return -ENOMEM;
 179 +    }
 180 +
 181 +    int ret = proxmox_restore_connect(s->conn, &pbs_error);
 182 +    if (ret < 0) {
 183 +        if (pbs_error && errp) error_setg(errp, "PBS connect failed: %s", pbs_error);
 184 +        if (pbs_error) proxmox_backup_free_error(pbs_error);
 185 +        return -ECONNREFUSED;
 186 +    }
 187 +
 188 +    /* acquire handle and length */
 189 +    s->aid = proxmox_restore_open_image(s->conn, s->archive, &pbs_error);
 190 +    if (s->aid < 0) {
 191 +        if (pbs_error && errp) error_setg(errp, "PBS open_image failed: %s", pbs_error);
 192 +        if (pbs_error) proxmox_backup_free_error(pbs_error);
 193 +        return -ENODEV;
 194 +    }
 195 +    s->length = proxmox_restore_get_image_length(s->conn, s->aid, &pbs_error);
 196 +    if (s->length < 0) {
 197 +        if (pbs_error && errp) error_setg(errp, "PBS get_image_length failed: %s", pbs_error);
 198 +        if (pbs_error) proxmox_backup_free_error(pbs_error);
 199 +        return -EINVAL;
 200 +    }
 201 +
 202 +    return 0;
 203 +}
 204 +
 205 +static int pbs_file_open(BlockDriverState *bs, QDict *options, int flags,
 206 +                         Error **errp)
 207 +{
 208 +    return pbs_open(bs, options, flags, errp);
 209 +}
 210 +
 211 +static void pbs_close(BlockDriverState *bs) {
 212 +    BDRVPBSState *s = bs->opaque;
 213 +    g_free(s->repository);
 214 +    g_free(s->snapshot);
 215 +    g_free(s->archive);
 216 +    proxmox_restore_disconnect(s->conn);
 217 +}
 218 +
 219 +static int64_t pbs_getlength(BlockDriverState *bs)
 220 +{
 221 +    BDRVPBSState *s = bs->opaque;
 222 +    return s->length;
 223 +}
 224 +
 225 +typedef struct ReadCallbackData {
 226 +    Coroutine *co;
 227 +    AioContext *ctx;
 228 +} ReadCallbackData;
 229 +
 230 +static void read_callback(void *callback_data)
 231 +{
 232 +    ReadCallbackData *rcb = callback_data;
 233 +    aio_co_schedule(rcb->ctx, rcb->co);
 234 +}
 235 +
 236 +static coroutine_fn int pbs_co_preadv(BlockDriverState *bs,
 237 +                                      int64_t offset, int64_t bytes,
 238 +                                      QEMUIOVector *qiov, BdrvRequestFlags flags)
 239 +{
 240 +    BDRVPBSState *s = bs->opaque;
 241 +    int ret;
 242 +    char *pbs_error = NULL;
 243 +    uint8_t *buf = malloc(bytes);
 244 +
 245 +    if (offset < 0 || bytes < 0) {
 246 +        fprintf(stderr, "unexpected negative 'offset' or 'bytes' value!\n");
 247 +        return -EINVAL;
 248 +    }
 249 +
 250 +    ReadCallbackData rcb = {
 251 +        .co = qemu_coroutine_self(),
 252 +        .ctx = bdrv_get_aio_context(bs),
 253 +    };
 254 +
 255 +    proxmox_restore_read_image_at_async(s->conn, s->aid, buf, (uint64_t)offset, (uint64_t)bytes,
 256 +                                        read_callback, (void *) &rcb, &ret, &pbs_error);
 257 +
 258 +    qemu_coroutine_yield();
 259 +
 260 +    if (ret < 0) {
 261 +        fprintf(stderr, "error during PBS read: %s\n", pbs_error ? pbs_error : "unknown error");
 262 +        if (pbs_error) proxmox_backup_free_error(pbs_error);
 263 +        return -EIO;
 264 +    }
 265 +
 266 +    qemu_iovec_from_buf(qiov, 0, buf, bytes);
 267 +    free(buf);
 268 +
 269 +    return ret;
 270 +}
 271 +
 272 +static coroutine_fn int pbs_co_pwritev(BlockDriverState *bs,
 273 +                                       int64_t offset, int64_t bytes,
 274 +                                       QEMUIOVector *qiov, BdrvRequestFlags flags)
 275 +{
 276 +    fprintf(stderr, "pbs-bdrv: cannot write to backup file, make sure "
 277 +           "any attached disk devices are set to read-only!\n");
 278 +    return -EPERM;
 279 +}
 280 +
 281 +static void pbs_refresh_filename(BlockDriverState *bs)
 282 +{
 283 +    BDRVPBSState *s = bs->opaque;
 284 +    snprintf(bs->exact_filename, sizeof(bs->exact_filename), "%s/%s(%s)",
 285 +             s->repository, s->snapshot, s->archive);
 286 +}
 287 +
 288 +static const char *const pbs_strong_runtime_opts[] = {
 289 +    NULL
 290 +};
 291 +
 292 +static BlockDriver bdrv_pbs_co = {
 293 +    .format_name            = "pbs",
 294 +    .protocol_name          = "pbs",
 295 +    .instance_size          = sizeof(BDRVPBSState),
 296 +
 297 +    .bdrv_parse_filename    = pbs_parse_filename,
 298 +
 299 +    .bdrv_file_open         = pbs_file_open,
 300 +    .bdrv_open              = pbs_open,
 301 +    .bdrv_close             = pbs_close,
 302 +    .bdrv_getlength         = pbs_getlength,
 303 +
 304 +    .bdrv_co_preadv         = pbs_co_preadv,
 305 +    .bdrv_co_pwritev        = pbs_co_pwritev,
 306 +
 307 +    .bdrv_refresh_filename  = pbs_refresh_filename,
 308 +    .strong_runtime_opts    = pbs_strong_runtime_opts,
 309 +};
 310 +
 311 +static void bdrv_pbs_init(void)
 312 +{
 313 +    bdrv_register(&bdrv_pbs_co);
 314 +}
 315 +
 316 +block_init(bdrv_pbs_init);
 317 diff --git a/configure b/configure
 318 index 48c21775f3..eda4e9225a 100755
 319 --- a/configure
 320 +++ b/configure
 321 @@ -356,6 +356,7 @@ vdi=${default_feature:-yes}
 322  vvfat=${default_feature:-yes}
 323  qed=${default_feature:-yes}
 324  parallels=${default_feature:-yes}
 325 +pbs_bdrv="yes"
 326  debug_mutex="no"
 327  plugins="$default_feature"
 328  rng_none="no"
 329 @@ -1126,6 +1127,10 @@ for opt do
 330    ;;
 331    --enable-parallels) parallels="yes"
 332    ;;
 333 +  --disable-pbs-bdrv) pbs_bdrv="no"
 334 +  ;;
 335 +  --enable-pbs-bdrv) pbs_bdrv="yes"
 336 +  ;;
 337    --disable-vhost-user) vhost_user="no"
 338    ;;
 339    --enable-vhost-user) vhost_user="yes"
 340 @@ -1465,6 +1470,7 @@ cat << EOF
 341    vvfat           vvfat image format support
 342    qed             qed image format support
 343    parallels       parallels image format support
 344 +  pbs-bdrv        Proxmox backup server read-only block driver support
 345    crypto-afalg    Linux AF_ALG crypto backend driver
 346    debug-mutex     mutex debugging support
 347    rng-none        dummy RNG, avoid using /dev/(u)random and getrandom()
 348 @@ -3534,6 +3540,9 @@ if test "$xen" = "enabled" ; then
 349    echo "XEN_CFLAGS=$xen_cflags" >> $config_host_mak
 350    echo "XEN_LIBS=$xen_libs" >> $config_host_mak
 351  fi
 352 +if test "$pbs_bdrv" = "yes" ; then
 353 +  echo "CONFIG_PBS_BDRV=y" >> $config_host_mak
 354 +fi
 355  if test "$vhost_scsi" = "yes" ; then
 356    echo "CONFIG_VHOST_SCSI=y" >> $config_host_mak
 357  fi
 358 diff --git a/meson.build b/meson.build
 359 index 1a4dfab4e2..85b3c63199 100644
 360 --- a/meson.build
 361 +++ b/meson.build
 362 @@ -3448,6 +3448,7 @@ summary_info += {'lzfse support':     liblzfse}
 363  summary_info += {'zstd support':      zstd}
 364  summary_info += {'NUMA host support': config_host.has_key('CONFIG_NUMA')}
 365  summary_info += {'libxml2':           libxml2}
 366 +summary_info += {'PBS bdrv support':  config_host.has_key('CONFIG_PBS_BDRV')}
 367  summary_info += {'capstone':          capstone_opt == 'internal' ? capstone_opt : capstone}
 368  summary_info += {'libpmem support':   libpmem}
 369  summary_info += {'libdaxctl support': libdaxctl}
 370 diff --git a/qapi/block-core.json b/qapi/block-core.json
 371 index 88835ebcf6..fd1ba7ccac 100644
 372 --- a/qapi/block-core.json
 373 +++ b/qapi/block-core.json
 374 @@ -3074,6 +3074,7 @@
 375              'luks', 'nbd', 'nfs', 'null-aio', 'null-co', 'nvme', 'parallels',
 376              'preallocate', 'qcow', 'qcow2', 'qed', 'quorum', 'raw', 'rbd',
 377              { 'name': 'replication', 'if': 'CONFIG_REPLICATION' },
 378 +            'pbs',
 379              'ssh', 'throttle', 'vdi', 'vhdx', 'vmdk', 'vpc', 'vvfat' ] }
 380
 381  ##
 382 @@ -3146,6 +3147,17 @@
 383  { 'struct': 'BlockdevOptionsNull',
 384    'data': { '*size': 'int', '*latency-ns': 'uint64', '*read-zeroes': 'bool' } }
 385
 386 +##
 387 +# @BlockdevOptionsPbs:
 388 +#
 389 +# Driver specific block device options for the PBS backend.
 390 +#
 391 +##
 392 +{ 'struct': 'BlockdevOptionsPbs',
 393 +  'data': { 'repository': 'str', 'snapshot': 'str', 'archive': 'str',
 394 +            '*keyfile': 'str', '*password': 'str', '*fingerprint': 'str',
 395 +            '*key_password': 'str' } }
 396 +
 397  ##
 398  # @BlockdevOptionsNVMe:
 399  #
 400 @@ -4388,6 +4400,7 @@
 401        'nfs':        'BlockdevOptionsNfs',
 402        'null-aio':   'BlockdevOptionsNull',
 403        'null-co':    'BlockdevOptionsNull',
 404 +      'pbs':        'BlockdevOptionsPbs',
 405        'nvme':       'BlockdevOptionsNVMe',
 406        'parallels':  'BlockdevOptionsGenericFormat',
 407        'preallocate':'BlockdevOptionsPreallocate',