PBS patches: block driver, adapat encrypt/compress param, add query-proxmox-support...
[pve-qemu.git] / debian / patches / pve / 0044-PVE-Add-PBS-block-driver-to-map-backup-archives-into.patch
1 From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
2 From: Stefan Reiter <s.reiter@proxmox.com>
3 Date: Wed, 8 Jul 2020 09:50:54 +0200
4 Subject: [PATCH] PVE: Add PBS block driver to map backup archives into VMs
5
6 Signed-off-by: Stefan Reiter <s.reiter@proxmox.com>
7 [error cleanups, file_open implementation]
8 Signed-off-by: Dietmar Maurer <dietmar@proxmox.com>
9 ---
10 block/Makefile.objs | 2 +
11 block/pbs.c | 271 +++++++++++++++++++++++++++++++++++++++++++
12 configure | 10 ++
13 qapi/block-core.json | 14 ++-
14 4 files changed, 296 insertions(+), 1 deletion(-)
15 create mode 100644 block/pbs.c
16
17 diff --git a/block/Makefile.objs b/block/Makefile.objs
18 index 8af7073c83..9785e4431d 100644
19 --- a/block/Makefile.objs
20 +++ b/block/Makefile.objs
21 @@ -5,6 +5,7 @@ block-obj-$(CONFIG_CLOOP) += cloop.o
22 block-obj-$(CONFIG_BOCHS) += bochs.o
23 block-obj-$(CONFIG_VVFAT) += vvfat.o
24 block-obj-$(CONFIG_DMG) += dmg.o
25 +block-obj-$(CONFIG_PBS_BDRV) += pbs.o
26
27 block-obj-y += qcow2.o qcow2-refcount.o qcow2-cluster.o qcow2-snapshot.o qcow2-cache.o qcow2-bitmap.o qcow2-threads.o
28 block-obj-$(CONFIG_QED) += qed.o qed-l2-cache.o qed-table.o qed-cluster.o
29 @@ -76,3 +77,4 @@ io_uring.o-cflags := $(LINUX_IO_URING_CFLAGS)
30 io_uring.o-libs := $(LINUX_IO_URING_LIBS)
31 parallels.o-cflags := $(LIBXML2_CFLAGS)
32 parallels.o-libs := $(LIBXML2_LIBS)
33 +pbs.o-libs := -lproxmox_backup_qemu
34 diff --git a/block/pbs.c b/block/pbs.c
35 new file mode 100644
36 index 0000000000..1481a2bfd1
37 --- /dev/null
38 +++ b/block/pbs.c
39 @@ -0,0 +1,271 @@
40 +/*
41 + * Proxmox Backup Server read-only block driver
42 + */
43 +
44 +#include "qemu/osdep.h"
45 +#include "qapi/error.h"
46 +#include "qapi/qmp/qdict.h"
47 +#include "qapi/qmp/qstring.h"
48 +#include "qemu/module.h"
49 +#include "qemu/option.h"
50 +#include "qemu/cutils.h"
51 +#include "block/block_int.h"
52 +
53 +#include <proxmox-backup-qemu.h>
54 +
55 +#define PBS_OPT_REPOSITORY "repository"
56 +#define PBS_OPT_SNAPSHOT "snapshot"
57 +#define PBS_OPT_ARCHIVE "archive"
58 +#define PBS_OPT_KEYFILE "keyfile"
59 +#define PBS_OPT_PASSWORD "password"
60 +#define PBS_OPT_FINGERPRINT "fingerprint"
61 +#define PBS_OPT_ENCRYPTION_PASSWORD "key_password"
62 +
63 +typedef struct {
64 + ProxmoxRestoreHandle *conn;
65 + char aid;
66 + int64_t length;
67 +
68 + char *repository;
69 + char *snapshot;
70 + char *archive;
71 +} BDRVPBSState;
72 +
73 +static QemuOptsList runtime_opts = {
74 + .name = "pbs",
75 + .head = QTAILQ_HEAD_INITIALIZER(runtime_opts.head),
76 + .desc = {
77 + {
78 + .name = PBS_OPT_REPOSITORY,
79 + .type = QEMU_OPT_STRING,
80 + .help = "The server address and repository to connect to.",
81 + },
82 + {
83 + .name = PBS_OPT_SNAPSHOT,
84 + .type = QEMU_OPT_STRING,
85 + .help = "The snapshot to read.",
86 + },
87 + {
88 + .name = PBS_OPT_ARCHIVE,
89 + .type = QEMU_OPT_STRING,
90 + .help = "Which archive within the snapshot should be accessed.",
91 + },
92 + {
93 + .name = PBS_OPT_PASSWORD,
94 + .type = QEMU_OPT_STRING,
95 + .help = "Server password. Can be passed as env var 'PBS_PASSWORD'.",
96 + },
97 + {
98 + .name = PBS_OPT_FINGERPRINT,
99 + .type = QEMU_OPT_STRING,
100 + .help = "Server fingerprint. Can be passed as env var 'PBS_FINGERPRINT'.",
101 + },
102 + {
103 + .name = PBS_OPT_ENCRYPTION_PASSWORD,
104 + .type = QEMU_OPT_STRING,
105 + .help = "Optional: Key password. Can be passed as env var 'PBS_ENCRYPTION_PASSWORD'.",
106 + },
107 + {
108 + .name = PBS_OPT_KEYFILE,
109 + .type = QEMU_OPT_STRING,
110 + .help = "Optional: The path to the keyfile to use.",
111 + },
112 + { /* end of list */ }
113 + },
114 +};
115 +
116 +
117 +// filename format:
118 +// pbs:repository=<repo>,snapshot=<snap>,password=<pw>,key_password=<kpw>,fingerprint=<fp>,archive=<archive>
119 +static void pbs_parse_filename(const char *filename, QDict *options,
120 + Error **errp)
121 +{
122 +
123 + if (!strstart(filename, "pbs:", &filename)) {
124 + if (errp) error_setg(errp, "pbs_parse_filename failed - missing 'pbs:' prefix");
125 + }
126 +
127 +
128 + QemuOpts *opts = qemu_opts_parse_noisily(&runtime_opts, filename, false);
129 + if (!opts) {
130 + if (errp) error_setg(errp, "pbs_parse_filename failed");
131 + return;
132 + }
133 +
134 + qemu_opts_to_qdict(opts, options);
135 +
136 + qemu_opts_del(opts);
137 +}
138 +
139 +static int pbs_open(BlockDriverState *bs, QDict *options, int flags,
140 + Error **errp)
141 +{
142 + QemuOpts *opts;
143 + BDRVPBSState *s = bs->opaque;
144 + char *pbs_error = NULL;
145 +
146 + opts = qemu_opts_create(&runtime_opts, NULL, 0, &error_abort);
147 + qemu_opts_absorb_qdict(opts, options, &error_abort);
148 +
149 + s->repository = g_strdup(qemu_opt_get(opts, PBS_OPT_REPOSITORY));
150 + s->snapshot = g_strdup(qemu_opt_get(opts, PBS_OPT_SNAPSHOT));
151 + s->archive = g_strdup(qemu_opt_get(opts, PBS_OPT_ARCHIVE));
152 + const char *keyfile = qemu_opt_get(opts, PBS_OPT_KEYFILE);
153 + const char *password = qemu_opt_get(opts, PBS_OPT_PASSWORD);
154 + const char *fingerprint = qemu_opt_get(opts, PBS_OPT_FINGERPRINT);
155 + const char *key_password = qemu_opt_get(opts, PBS_OPT_ENCRYPTION_PASSWORD);
156 +
157 + if (!password) {
158 + password = getenv("PBS_PASSWORD");
159 + }
160 + if (!fingerprint) {
161 + fingerprint = getenv("PBS_FINGERPRINT");
162 + }
163 + if (!key_password) {
164 + key_password = getenv("PBS_ENCRYPTION_PASSWORD");
165 + }
166 +
167 + /* connect to PBS server in read mode */
168 + s->conn = proxmox_restore_new(s->repository, s->snapshot, password,
169 + keyfile, key_password, fingerprint, &pbs_error);
170 +
171 + /* invalidates qemu_opt_get char pointers from above */
172 + qemu_opts_del(opts);
173 +
174 + if (!s->conn) {
175 + if (pbs_error && errp) error_setg(errp, "PBS restore_new failed: %s", pbs_error);
176 + if (pbs_error) proxmox_backup_free_error(pbs_error);
177 + return -ENOMEM;
178 + }
179 +
180 + int ret = proxmox_restore_connect(s->conn, &pbs_error);
181 + if (ret < 0) {
182 + if (pbs_error && errp) error_setg(errp, "PBS connect failed: %s", pbs_error);
183 + if (pbs_error) proxmox_backup_free_error(pbs_error);
184 + return -ECONNREFUSED;
185 + }
186 +
187 + /* acquire handle and length */
188 + s->aid = proxmox_restore_open_image(s->conn, s->archive, &pbs_error);
189 + if (s->aid < 0) {
190 + if (pbs_error && errp) error_setg(errp, "PBS open_image failed: %s", pbs_error);
191 + if (pbs_error) proxmox_backup_free_error(pbs_error);
192 + return -ENODEV;
193 + }
194 + s->length = proxmox_restore_get_image_length(s->conn, s->aid, &pbs_error);
195 + if (s->length < 0) {
196 + if (pbs_error && errp) error_setg(errp, "PBS get_image_length failed: %s", pbs_error);
197 + if (pbs_error) proxmox_backup_free_error(pbs_error);
198 + return -EINVAL;
199 + }
200 +
201 + return 0;
202 +}
203 +
204 +static int pbs_file_open(BlockDriverState *bs, QDict *options, int flags,
205 + Error **errp)
206 +{
207 + return pbs_open(bs, options, flags, errp);
208 +}
209 +
210 +static void pbs_close(BlockDriverState *bs) {
211 + BDRVPBSState *s = bs->opaque;
212 + g_free(s->repository);
213 + g_free(s->snapshot);
214 + g_free(s->archive);
215 + proxmox_restore_disconnect(s->conn);
216 +}
217 +
218 +static int64_t pbs_getlength(BlockDriverState *bs)
219 +{
220 + BDRVPBSState *s = bs->opaque;
221 + return s->length;
222 +}
223 +
224 +typedef struct ReadCallbackData {
225 + Coroutine *co;
226 + AioContext *ctx;
227 +} ReadCallbackData;
228 +
229 +static void read_callback(void *callback_data)
230 +{
231 + ReadCallbackData *rcb = callback_data;
232 + aio_co_schedule(rcb->ctx, rcb->co);
233 +}
234 +
235 +static coroutine_fn int pbs_co_preadv(BlockDriverState *bs,
236 + uint64_t offset, uint64_t bytes,
237 + QEMUIOVector *qiov, int flags)
238 +{
239 + BDRVPBSState *s = bs->opaque;
240 + int ret;
241 + char *pbs_error = NULL;
242 + uint8_t *buf = malloc(bytes);
243 +
244 + ReadCallbackData rcb = {
245 + .co = qemu_coroutine_self(),
246 + .ctx = qemu_get_current_aio_context(),
247 + };
248 +
249 + proxmox_restore_read_image_at_async(s->conn, s->aid, buf, offset, bytes,
250 + read_callback, (void *) &rcb, &ret, &pbs_error);
251 +
252 + qemu_coroutine_yield();
253 +
254 + if (ret < 0) {
255 + fprintf(stderr, "error during PBS read: %s\n", pbs_error ? pbs_error : "unknown error");
256 + if (pbs_error) proxmox_backup_free_error(pbs_error);
257 + return -EIO;
258 + }
259 +
260 + qemu_iovec_from_buf(qiov, 0, buf, bytes);
261 + free(buf);
262 +
263 + return ret;
264 +}
265 +
266 +static coroutine_fn int pbs_co_pwritev(BlockDriverState *bs,
267 + uint64_t offset, uint64_t bytes,
268 + QEMUIOVector *qiov, int flags)
269 +{
270 + fprintf(stderr, "pbs-bdrv: cannot write to backup file, make sure "
271 + "any attached disk devices are set to read-only!\n");
272 + return -EPERM;
273 +}
274 +
275 +static void pbs_refresh_filename(BlockDriverState *bs)
276 +{
277 + BDRVPBSState *s = bs->opaque;
278 + snprintf(bs->exact_filename, sizeof(bs->exact_filename), "%s/%s(%s)",
279 + s->repository, s->snapshot, s->archive);
280 +}
281 +
282 +static const char *const pbs_strong_runtime_opts[] = {
283 + NULL
284 +};
285 +
286 +static BlockDriver bdrv_pbs_co = {
287 + .format_name = "pbs",
288 + .protocol_name = "pbs",
289 + .instance_size = sizeof(BDRVPBSState),
290 +
291 + .bdrv_parse_filename = pbs_parse_filename,
292 +
293 + .bdrv_file_open = pbs_file_open,
294 + .bdrv_open = pbs_open,
295 + .bdrv_close = pbs_close,
296 + .bdrv_getlength = pbs_getlength,
297 +
298 + .bdrv_co_preadv = pbs_co_preadv,
299 + .bdrv_co_pwritev = pbs_co_pwritev,
300 +
301 + .bdrv_refresh_filename = pbs_refresh_filename,
302 + .strong_runtime_opts = pbs_strong_runtime_opts,
303 +};
304 +
305 +static void bdrv_pbs_init(void)
306 +{
307 + bdrv_register(&bdrv_pbs_co);
308 +}
309 +
310 +block_init(bdrv_pbs_init);
311 diff --git a/configure b/configure
312 index 23b5e93752..67054b5795 100755
313 --- a/configure
314 +++ b/configure
315 @@ -503,6 +503,7 @@ vvfat="yes"
316 qed="yes"
317 parallels="yes"
318 sheepdog="yes"
319 +pbs_bdrv="yes"
320 libxml2=""
321 debug_mutex="no"
322 libpmem=""
323 @@ -1553,6 +1554,10 @@ for opt do
324 ;;
325 --enable-sheepdog) sheepdog="yes"
326 ;;
327 + --disable-pbs-bdrv) pbs_bdrv="no"
328 + ;;
329 + --enable-pbs-bdrv) pbs_bdrv="yes"
330 + ;;
331 --disable-vhost-user) vhost_user="no"
332 ;;
333 --enable-vhost-user) vhost_user="yes"
334 @@ -1889,6 +1894,7 @@ disabled with --disable-FEATURE, default is enabled if available:
335 qed qed image format support
336 parallels parallels image format support
337 sheepdog sheepdog block driver support
338 + pbs-bdrv Proxmox backup server read-only block driver support
339 crypto-afalg Linux AF_ALG crypto backend driver
340 capstone capstone disassembler support
341 debug-mutex mutex debugging support
342 @@ -6726,6 +6732,7 @@ echo "vvfat support $vvfat"
343 echo "qed support $qed"
344 echo "parallels support $parallels"
345 echo "sheepdog support $sheepdog"
346 +echo "pbs-bdrv support $pbs_bdrv"
347 echo "capstone $capstone"
348 echo "libpmem support $libpmem"
349 echo "libudev $libudev"
350 @@ -7578,6 +7585,9 @@ fi
351 if test "$sheepdog" = "yes" ; then
352 echo "CONFIG_SHEEPDOG=y" >> $config_host_mak
353 fi
354 +if test "$pbs_bdrv" = "yes" ; then
355 + echo "CONFIG_PBS_BDRV=y" >> $config_host_mak
356 +fi
357 if test "$fuzzing" = "yes" ; then
358 if test "$have_fuzzer" = "yes"; then
359 FUZZ_LDFLAGS=" -fsanitize=address,fuzzer"
360 diff --git a/qapi/block-core.json b/qapi/block-core.json
361 index 0bc15a5098..13c63d8e6a 100644
362 --- a/qapi/block-core.json
363 +++ b/qapi/block-core.json
364 @@ -2942,7 +2942,7 @@
365 'luks', 'nbd', 'nfs', 'null-aio', 'null-co', 'nvme', 'parallels',
366 'qcow', 'qcow2', 'qed', 'quorum', 'raw', 'rbd',
367 { 'name': 'replication', 'if': 'defined(CONFIG_REPLICATION)' },
368 - 'sheepdog',
369 + 'sheepdog', 'pbs',
370 'ssh', 'throttle', 'vdi', 'vhdx', 'vmdk', 'vpc', 'vvfat', 'vxhs' ] }
371
372 ##
373 @@ -3006,6 +3006,17 @@
374 { 'struct': 'BlockdevOptionsNull',
375 'data': { '*size': 'int', '*latency-ns': 'uint64', '*read-zeroes': 'bool' } }
376
377 +##
378 +# @BlockdevOptionsPbs:
379 +#
380 +# Driver specific block device options for the PBS backend.
381 +#
382 +##
383 +{ 'struct': 'BlockdevOptionsPbs',
384 + 'data': { 'repository': 'str', 'snapshot': 'str', 'archive': 'str',
385 + '*keyfile': 'str', '*password': 'str', '*fingerprint': 'str',
386 + '*key_password': 'str' } }
387 +
388 ##
389 # @BlockdevOptionsNVMe:
390 #
391 @@ -4128,6 +4139,7 @@
392 'nfs': 'BlockdevOptionsNfs',
393 'null-aio': 'BlockdevOptionsNull',
394 'null-co': 'BlockdevOptionsNull',
395 + 'pbs': 'BlockdevOptionsPbs',
396 'nvme': 'BlockdevOptionsNVMe',
397 'parallels': 'BlockdevOptionsGenericFormat',
398 'qcow2': 'BlockdevOptionsQcow2',