debian/patches/pve/0044-PVE-Add-PBS-block-driver-to-map-backup-archives-into.patch

   1 From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
   2 From: Stefan Reiter <s.reiter@proxmox.com>
   3 Date: Wed, 8 Jul 2020 09:50:54 +0200
   4 Subject: [PATCH] PVE: Add PBS block driver to map backup archives into VMs
   5
   6 Signed-off-by: Stefan Reiter <s.reiter@proxmox.com>
   7 [error cleanups, file_open implementation]
   8 Signed-off-by: Dietmar Maurer <dietmar@proxmox.com>
   9 ---
  10  block/Makefile.objs  |   2 +
  11  block/pbs.c          | 271 +++++++++++++++++++++++++++++++++++++++++++
  12  configure            |  10 ++
  13  qapi/block-core.json |  14 ++-
  14  4 files changed, 296 insertions(+), 1 deletion(-)
  15  create mode 100644 block/pbs.c
  16
  17 diff --git a/block/Makefile.objs b/block/Makefile.objs
  18 index 9ea0477d0b..28fb3b7f7c 100644
  19 --- a/block/Makefile.objs
  20 +++ b/block/Makefile.objs
  21 @@ -5,6 +5,7 @@ block-obj-$(CONFIG_CLOOP) += cloop.o
  22  block-obj-$(CONFIG_BOCHS) += bochs.o
  23  block-obj-$(CONFIG_VVFAT) += vvfat.o
  24  block-obj-$(CONFIG_DMG) += dmg.o
  25 +block-obj-$(CONFIG_PBS_BDRV) += pbs.o
  26
  27  block-obj-y += qcow2.o qcow2-refcount.o qcow2-cluster.o qcow2-snapshot.o qcow2-cache.o qcow2-bitmap.o qcow2-threads.o
  28  block-obj-$(CONFIG_QED) += qed.o qed-l2-cache.o qed-table.o qed-cluster.o
  29 @@ -75,3 +76,4 @@ io_uring.o-cflags  := $(LINUX_IO_URING_CFLAGS)
  30  io_uring.o-libs    := $(LINUX_IO_URING_LIBS)
  31  parallels.o-cflags := $(LIBXML2_CFLAGS)
  32  parallels.o-libs   := $(LIBXML2_LIBS)
  33 +pbs.o-libs         := -lproxmox_backup_qemu
  34 diff --git a/block/pbs.c b/block/pbs.c
  35 new file mode 100644
  36 index 0000000000..1481a2bfd1
  37 --- /dev/null
  38 +++ b/block/pbs.c
  39 @@ -0,0 +1,271 @@
  40 +/*
  41 + * Proxmox Backup Server read-only block driver
  42 + */
  43 +
  44 +#include "qemu/osdep.h"
  45 +#include "qapi/error.h"
  46 +#include "qapi/qmp/qdict.h"
  47 +#include "qapi/qmp/qstring.h"
  48 +#include "qemu/module.h"
  49 +#include "qemu/option.h"
  50 +#include "qemu/cutils.h"
  51 +#include "block/block_int.h"
  52 +
  53 +#include <proxmox-backup-qemu.h>
  54 +
  55 +#define PBS_OPT_REPOSITORY "repository"
  56 +#define PBS_OPT_SNAPSHOT "snapshot"
  57 +#define PBS_OPT_ARCHIVE "archive"
  58 +#define PBS_OPT_KEYFILE "keyfile"
  59 +#define PBS_OPT_PASSWORD "password"
  60 +#define PBS_OPT_FINGERPRINT "fingerprint"
  61 +#define PBS_OPT_ENCRYPTION_PASSWORD "key_password"
  62 +
  63 +typedef struct {
  64 +    ProxmoxRestoreHandle *conn;
  65 +    char aid;
  66 +    int64_t length;
  67 +
  68 +    char *repository;
  69 +    char *snapshot;
  70 +    char *archive;
  71 +} BDRVPBSState;
  72 +
  73 +static QemuOptsList runtime_opts = {
  74 +    .name = "pbs",
  75 +    .head = QTAILQ_HEAD_INITIALIZER(runtime_opts.head),
  76 +    .desc = {
  77 +        {
  78 +            .name = PBS_OPT_REPOSITORY,
  79 +            .type = QEMU_OPT_STRING,
  80 +            .help = "The server address and repository to connect to.",
  81 +        },
  82 +        {
  83 +            .name = PBS_OPT_SNAPSHOT,
  84 +            .type = QEMU_OPT_STRING,
  85 +            .help = "The snapshot to read.",
  86 +        },
  87 +        {
  88 +            .name = PBS_OPT_ARCHIVE,
  89 +            .type = QEMU_OPT_STRING,
  90 +            .help = "Which archive within the snapshot should be accessed.",
  91 +        },
  92 +        {
  93 +            .name = PBS_OPT_PASSWORD,
  94 +            .type = QEMU_OPT_STRING,
  95 +            .help = "Server password. Can be passed as env var 'PBS_PASSWORD'.",
  96 +        },
  97 +        {
  98 +            .name = PBS_OPT_FINGERPRINT,
  99 +            .type = QEMU_OPT_STRING,
 100 +            .help = "Server fingerprint. Can be passed as env var 'PBS_FINGERPRINT'.",
 101 +        },
 102 +        {
 103 +            .name = PBS_OPT_ENCRYPTION_PASSWORD,
 104 +            .type = QEMU_OPT_STRING,
 105 +            .help = "Optional: Key password. Can be passed as env var 'PBS_ENCRYPTION_PASSWORD'.",
 106 +        },
 107 +        {
 108 +            .name = PBS_OPT_KEYFILE,
 109 +            .type = QEMU_OPT_STRING,
 110 +            .help = "Optional: The path to the keyfile to use.",
 111 +        },
 112 +        { /* end of list */ }
 113 +    },
 114 +};
 115 +
 116 +
 117 +// filename format:
 118 +// pbs:repository=<repo>,snapshot=<snap>,password=<pw>,key_password=<kpw>,fingerprint=<fp>,archive=<archive>
 119 +static void pbs_parse_filename(const char *filename, QDict *options,
 120 +                                     Error **errp)
 121 +{
 122 +
 123 +    if (!strstart(filename, "pbs:", &filename)) {
 124 +        if (errp) error_setg(errp, "pbs_parse_filename failed - missing 'pbs:' prefix");
 125 +    }
 126 +
 127 +
 128 +    QemuOpts *opts = qemu_opts_parse_noisily(&runtime_opts, filename, false);
 129 +    if (!opts) {
 130 +        if (errp) error_setg(errp, "pbs_parse_filename failed");
 131 +        return;
 132 +    }
 133 +
 134 +    qemu_opts_to_qdict(opts, options);
 135 +
 136 +    qemu_opts_del(opts);
 137 +}
 138 +
 139 +static int pbs_open(BlockDriverState *bs, QDict *options, int flags,
 140 +                    Error **errp)
 141 +{
 142 +    QemuOpts *opts;
 143 +    BDRVPBSState *s = bs->opaque;
 144 +    char *pbs_error = NULL;
 145 +
 146 +    opts = qemu_opts_create(&runtime_opts, NULL, 0, &error_abort);
 147 +    qemu_opts_absorb_qdict(opts, options, &error_abort);
 148 +
 149 +    s->repository = g_strdup(qemu_opt_get(opts, PBS_OPT_REPOSITORY));
 150 +    s->snapshot = g_strdup(qemu_opt_get(opts, PBS_OPT_SNAPSHOT));
 151 +    s->archive = g_strdup(qemu_opt_get(opts, PBS_OPT_ARCHIVE));
 152 +    const char *keyfile = qemu_opt_get(opts, PBS_OPT_KEYFILE);
 153 +    const char *password = qemu_opt_get(opts, PBS_OPT_PASSWORD);
 154 +    const char *fingerprint = qemu_opt_get(opts, PBS_OPT_FINGERPRINT);
 155 +    const char *key_password = qemu_opt_get(opts, PBS_OPT_ENCRYPTION_PASSWORD);
 156 +
 157 +    if (!password) {
 158 +        password = getenv("PBS_PASSWORD");
 159 +    }
 160 +    if (!fingerprint) {
 161 +        fingerprint = getenv("PBS_FINGERPRINT");
 162 +    }
 163 +    if (!key_password) {
 164 +        key_password = getenv("PBS_ENCRYPTION_PASSWORD");
 165 +    }
 166 +
 167 +    /* connect to PBS server in read mode */
 168 +    s->conn = proxmox_restore_new(s->repository, s->snapshot, password,
 169 +        keyfile, key_password, fingerprint, &pbs_error);
 170 +
 171 +    /* invalidates qemu_opt_get char pointers from above */
 172 +    qemu_opts_del(opts);
 173 +
 174 +    if (!s->conn) {
 175 +        if (pbs_error && errp) error_setg(errp, "PBS restore_new failed: %s", pbs_error);
 176 +        if (pbs_error) proxmox_backup_free_error(pbs_error);
 177 +        return -ENOMEM;
 178 +    }
 179 +
 180 +    int ret = proxmox_restore_connect(s->conn, &pbs_error);
 181 +    if (ret < 0) {
 182 +        if (pbs_error && errp) error_setg(errp, "PBS connect failed: %s", pbs_error);
 183 +        if (pbs_error) proxmox_backup_free_error(pbs_error);
 184 +        return -ECONNREFUSED;
 185 +    }
 186 +
 187 +    /* acquire handle and length */
 188 +    s->aid = proxmox_restore_open_image(s->conn, s->archive, &pbs_error);
 189 +    if (s->aid < 0) {
 190 +        if (pbs_error && errp) error_setg(errp, "PBS open_image failed: %s", pbs_error);
 191 +        if (pbs_error) proxmox_backup_free_error(pbs_error);
 192 +        return -ENODEV;
 193 +    }
 194 +    s->length = proxmox_restore_get_image_length(s->conn, s->aid, &pbs_error);
 195 +    if (s->length < 0) {
 196 +        if (pbs_error && errp) error_setg(errp, "PBS get_image_length failed: %s", pbs_error);
 197 +        if (pbs_error) proxmox_backup_free_error(pbs_error);
 198 +        return -EINVAL;
 199 +    }
 200 +
 201 +    return 0;
 202 +}
 203 +
 204 +static int pbs_file_open(BlockDriverState *bs, QDict *options, int flags,
 205 +                         Error **errp)
 206 +{
 207 +    return pbs_open(bs, options, flags, errp);
 208 +}
 209 +
 210 +static void pbs_close(BlockDriverState *bs) {
 211 +    BDRVPBSState *s = bs->opaque;
 212 +    g_free(s->repository);
 213 +    g_free(s->snapshot);
 214 +    g_free(s->archive);
 215 +    proxmox_restore_disconnect(s->conn);
 216 +}
 217 +
 218 +static int64_t pbs_getlength(BlockDriverState *bs)
 219 +{
 220 +    BDRVPBSState *s = bs->opaque;
 221 +    return s->length;
 222 +}
 223 +
 224 +typedef struct ReadCallbackData {
 225 +    Coroutine *co;
 226 +    AioContext *ctx;
 227 +} ReadCallbackData;
 228 +
 229 +static void read_callback(void *callback_data)
 230 +{
 231 +    ReadCallbackData *rcb = callback_data;
 232 +    aio_co_schedule(rcb->ctx, rcb->co);
 233 +}
 234 +
 235 +static coroutine_fn int pbs_co_preadv(BlockDriverState *bs,
 236 +                                      uint64_t offset, uint64_t bytes,
 237 +                                      QEMUIOVector *qiov, int flags)
 238 +{
 239 +    BDRVPBSState *s = bs->opaque;
 240 +    int ret;
 241 +    char *pbs_error = NULL;
 242 +    uint8_t *buf = malloc(bytes);
 243 +
 244 +    ReadCallbackData rcb = {
 245 +        .co = qemu_coroutine_self(),
 246 +        .ctx = qemu_get_current_aio_context(),
 247 +    };
 248 +
 249 +    proxmox_restore_read_image_at_async(s->conn, s->aid, buf, offset, bytes,
 250 +                                        read_callback, (void *) &rcb, &ret, &pbs_error);
 251 +
 252 +    qemu_coroutine_yield();
 253 +
 254 +    if (ret < 0) {
 255 +        fprintf(stderr, "error during PBS read: %s\n", pbs_error ? pbs_error : "unknown error");
 256 +        if (pbs_error) proxmox_backup_free_error(pbs_error);
 257 +        return -EIO;
 258 +    }
 259 +
 260 +    qemu_iovec_from_buf(qiov, 0, buf, bytes);
 261 +    free(buf);
 262 +
 263 +    return ret;
 264 +}
 265 +
 266 +static coroutine_fn int pbs_co_pwritev(BlockDriverState *bs,
 267 +                                       uint64_t offset, uint64_t bytes,
 268 +                                       QEMUIOVector *qiov, int flags)
 269 +{
 270 +    fprintf(stderr, "pbs-bdrv: cannot write to backup file, make sure "
 271 +           "any attached disk devices are set to read-only!\n");
 272 +    return -EPERM;
 273 +}
 274 +
 275 +static void pbs_refresh_filename(BlockDriverState *bs)
 276 +{
 277 +    BDRVPBSState *s = bs->opaque;
 278 +    snprintf(bs->exact_filename, sizeof(bs->exact_filename), "%s/%s(%s)",
 279 +             s->repository, s->snapshot, s->archive);
 280 +}
 281 +
 282 +static const char *const pbs_strong_runtime_opts[] = {
 283 +    NULL
 284 +};
 285 +
 286 +static BlockDriver bdrv_pbs_co = {
 287 +    .format_name            = "pbs",
 288 +    .protocol_name          = "pbs",
 289 +    .instance_size          = sizeof(BDRVPBSState),
 290 +
 291 +    .bdrv_parse_filename    = pbs_parse_filename,
 292 +
 293 +    .bdrv_file_open         = pbs_file_open,
 294 +    .bdrv_open              = pbs_open,
 295 +    .bdrv_close             = pbs_close,
 296 +    .bdrv_getlength         = pbs_getlength,
 297 +
 298 +    .bdrv_co_preadv         = pbs_co_preadv,
 299 +    .bdrv_co_pwritev        = pbs_co_pwritev,
 300 +
 301 +    .bdrv_refresh_filename  = pbs_refresh_filename,
 302 +    .strong_runtime_opts    = pbs_strong_runtime_opts,
 303 +};
 304 +
 305 +static void bdrv_pbs_init(void)
 306 +{
 307 +    bdrv_register(&bdrv_pbs_co);
 308 +}
 309 +
 310 +block_init(bdrv_pbs_init);
 311 diff --git a/configure b/configure
 312 index 2acc4d1465..3fc80d0c2c 100755
 313 --- a/configure
 314 +++ b/configure
 315 @@ -510,6 +510,7 @@ vvfat="yes"
 316  qed="yes"
 317  parallels="yes"
 318  sheepdog="yes"
 319 +pbs_bdrv="yes"
 320  libxml2=""
 321  debug_mutex="no"
 322  libpmem=""
 323 @@ -1576,6 +1577,10 @@ for opt do
 324    ;;
 325    --enable-sheepdog) sheepdog="yes"
 326    ;;
 327 +  --disable-pbs-bdrv) pbs_bdrv="no"
 328 +  ;;
 329 +  --enable-pbs-bdrv) pbs_bdrv="yes"
 330 +  ;;
 331    --disable-vhost-user) vhost_user="no"
 332    ;;
 333    --enable-vhost-user) vhost_user="yes"
 334 @@ -1936,6 +1941,7 @@ disabled with --disable-FEATURE, default is enabled if available:
 335    qed             qed image format support
 336    parallels       parallels image format support
 337    sheepdog        sheepdog block driver support
 338 +  pbs-bdrv        Proxmox backup server read-only block driver support
 339    crypto-afalg    Linux AF_ALG crypto backend driver
 340    capstone        capstone disassembler support
 341    debug-mutex     mutex debugging support
 342 @@ -7009,6 +7015,7 @@ echo "vvfat support     $vvfat"
 343  echo "qed support       $qed"
 344  echo "parallels support $parallels"
 345  echo "sheepdog support  $sheepdog"
 346 +echo "pbs-bdrv support  $pbs_bdrv"
 347  echo "capstone          $capstone"
 348  echo "libpmem support   $libpmem"
 349  echo "libdaxctl support $libdaxctl"
 350 @@ -7885,6 +7892,9 @@ fi
 351  if test "$sheepdog" = "yes" ; then
 352    echo "CONFIG_SHEEPDOG=y" >> $config_host_mak
 353  fi
 354 +if test "$pbs_bdrv" = "yes" ; then
 355 +  echo "CONFIG_PBS_BDRV=y" >> $config_host_mak
 356 +fi
 357  if test "$pty_h" = "yes" ; then
 358    echo "HAVE_PTY_H=y" >> $config_host_mak
 359  fi
 360 diff --git a/qapi/block-core.json b/qapi/block-core.json
 361 index a177fea6cd..f782c2cf96 100644
 362 --- a/qapi/block-core.json
 363 +++ b/qapi/block-core.json
 364 @@ -2951,7 +2951,7 @@
 365              'luks', 'nbd', 'nfs', 'null-aio', 'null-co', 'nvme', 'parallels',
 366              'qcow', 'qcow2', 'qed', 'quorum', 'raw', 'rbd',
 367              { 'name': 'replication', 'if': 'defined(CONFIG_REPLICATION)' },
 368 -            'sheepdog',
 369 +            'sheepdog', 'pbs',
 370              'ssh', 'throttle', 'vdi', 'vhdx', 'vmdk', 'vpc', 'vvfat' ] }
 371
 372  ##
 373 @@ -3015,6 +3015,17 @@
 374  { 'struct': 'BlockdevOptionsNull',
 375    'data': { '*size': 'int', '*latency-ns': 'uint64', '*read-zeroes': 'bool' } }
 376
 377 +##
 378 +# @BlockdevOptionsPbs:
 379 +#
 380 +# Driver specific block device options for the PBS backend.
 381 +#
 382 +##
 383 +{ 'struct': 'BlockdevOptionsPbs',
 384 +  'data': { 'repository': 'str', 'snapshot': 'str', 'archive': 'str',
 385 +            '*keyfile': 'str', '*password': 'str', '*fingerprint': 'str',
 386 +            '*key_password': 'str' } }
 387 +
 388  ##
 389  # @BlockdevOptionsNVMe:
 390  #
 391 @@ -4121,6 +4132,7 @@
 392        'nfs':        'BlockdevOptionsNfs',
 393        'null-aio':   'BlockdevOptionsNull',
 394        'null-co':    'BlockdevOptionsNull',
 395 +      'pbs':        'BlockdevOptionsPbs',
 396        'nvme':       'BlockdevOptionsNVMe',
 397        'parallels':  'BlockdevOptionsGenericFormat',
 398        'qcow2':      'BlockdevOptionsQcow2',