]> git.proxmox.com Git - pve-qemu-kvm.git/blame - debian/patches/backup-add-vma-binary.patch
projects / pve-qemu-kvm.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Two more fixes
[pve-qemu-kvm.git] / debian / patches / backup-add-vma-binary.patch
CommitLineData
b668dc80
DM		 1From 6289a43696ca6f713a5d3bb9f95a5adb608a5e13 Mon Sep 17 00:00:00 2001
2From: Dietmar Maurer <dietmar@proxmox.com>
3Date: Tue, 13 Nov 2012 11:11:38 +0100
4Subject: [PATCH v5 4/6] introduce new vma archive format
5
6This is a very simple archive format, see docs/specs/vma_spec.txt
7
8Signed-off-by: Dietmar Maurer <dietmar@proxmox.com>
9---
10 Makefile | 3 +-
11 Makefile.objs | 2 +-
12 blockdev.c | 6 +-
13 docs/specs/vma_spec.txt | 24 ++
14 vma-reader.c | 799 ++++++++++++++++++++++++++++++++++++++++
15 vma-writer.c | 940 +++++++++++++++++++++++++++++++++++++++++++++++
16 vma.c | 561 ++++++++++++++++++++++++++++
17 vma.h | 145 ++++++++
18 8 files changed, 2476 insertions(+), 4 deletions(-)
19 create mode 100644 docs/specs/vma_spec.txt
20 create mode 100644 vma-reader.c
21 create mode 100644 vma-writer.c
22 create mode 100644 vma.c
23 create mode 100644 vma.h
24
25Index: new/Makefile
26===================================================================
24bb7da3
DM		 27--- new.orig/Makefile 2014-11-20 08:12:54.000000000 +0100
28+++ new/Makefile 2014-11-20 08:15:12.000000000 +0100
58117ea6 29@@ -156,7 +156,7 @@
b668dc80
DM		 30 include $(SRC_PATH)/libcacard/Makefile
31 endif
32
88d9366c
DM		 33-all: $(DOCS) $(TOOLS) $(HELPERS-y) recurse-all modules
34+all: $(DOCS) $(TOOLS) vma$(EXESUF) $(HELPERS-y) recurse-all modules
b668dc80 35
fbe817e2
DM		 36 config-host.h: config-host.h-timestamp
37 config-host.h-timestamp: config-host.mak
58117ea6 38@@ -224,6 +224,7 @@
b668dc80
DM		 39 qemu-img$(EXESUF): qemu-img.o $(block-obj-y) libqemuutil.a libqemustub.a
40 qemu-nbd$(EXESUF): qemu-nbd.o $(block-obj-y) libqemuutil.a libqemustub.a
41 qemu-io$(EXESUF): qemu-io.o $(block-obj-y) libqemuutil.a libqemustub.a
2014bc7f 42+vma$(EXESUF): vma.o vma-reader.o $(block-obj-y) libqemuutil.a libqemustub.a
b668dc80
DM		 43
44 qemu-bridge-helper$(EXESUF): qemu-bridge-helper.o
45
b668dc80
DM		 46Index: new/vma-reader.c
47===================================================================
48--- /dev/null 1970-01-01 00:00:00.000000000 +0000
24bb7da3 49+++ new/vma-reader.c 2014-11-20 08:15:12.000000000 +0100
b668dc80
DM		 50@@ -0,0 +1,799 @@
51+/*
52+ * VMA: Virtual Machine Archive
53+ *
54+ * Copyright (C) 2012 Proxmox Server Solutions
55+ *
56+ * Authors:
57+ * Dietmar Maurer (dietmar@proxmox.com)
58+ *
59+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
60+ * See the COPYING file in the top-level directory.
61+ *
62+ */
63+
64+#include <stdio.h>
65+#include <errno.h>
66+#include <unistd.h>
67+#include <stdio.h>
68+#include <string.h>
69+#include <sys/types.h>
70+#include <sys/stat.h>
71+#include <fcntl.h>
72+#include <glib.h>
73+#include <uuid/uuid.h>
74+
75+#include "qemu-common.h"
76+#include "qemu/timer.h"
77+#include "qemu/ratelimit.h"
78+#include "vma.h"
79+#include "block/block.h"
80+
81+#define BITS_PER_LONG (sizeof(unsigned long) * CHAR_BIT)
82+
83+static unsigned char zero_vma_block[VMA_BLOCK_SIZE];
84+
85+typedef struct VmaRestoreState {
86+ BlockDriverState *bs;
87+ bool write_zeroes;
88+ unsigned long *bitmap;
89+ int bitmap_size;
90+} VmaRestoreState;
91+
92+struct VmaReader {
93+ int fd;
94+ GChecksum *md5csum;
95+ GHashTable *blob_hash;
96+ unsigned char *head_data;
97+ VmaDeviceInfo devinfo[256];
98+ VmaRestoreState rstate[256];
99+ GList *cdata_list;
100+ guint8 vmstate_stream;
101+ uint32_t vmstate_clusters;
102+ /* to show restore percentage if run with -v */
103+ time_t start_time;
104+ int64_t cluster_count;
105+ int64_t clusters_read;
106+ int clusters_read_per;
107+};
108+
109+static guint
110+g_int32_hash(gconstpointer v)
111+{
112+ return *(const uint32_t *)v;
113+}
114+
115+static gboolean
116+g_int32_equal(gconstpointer v1, gconstpointer v2)
117+{
118+ return *((const uint32_t *)v1) == *((const uint32_t *)v2);
119+}
120+
121+static int vma_reader_get_bitmap(VmaRestoreState *rstate, int64_t cluster_num)
122+{
123+ assert(rstate);
124+ assert(rstate->bitmap);
125+
126+ unsigned long val, idx, bit;
127+
128+ idx = cluster_num / BITS_PER_LONG;
129+
130+ assert(rstate->bitmap_size > idx);
131+
132+ bit = cluster_num % BITS_PER_LONG;
133+ val = rstate->bitmap[idx];
134+
135+ return !!(val & (1UL << bit));
136+}
137+
138+static void vma_reader_set_bitmap(VmaRestoreState *rstate, int64_t cluster_num,
139+ int dirty)
140+{
141+ assert(rstate);
142+ assert(rstate->bitmap);
143+
144+ unsigned long val, idx, bit;
145+
146+ idx = cluster_num / BITS_PER_LONG;
147+
148+ assert(rstate->bitmap_size > idx);
149+
150+ bit = cluster_num % BITS_PER_LONG;
151+ val = rstate->bitmap[idx];
152+ if (dirty) {
153+ if (!(val & (1UL << bit))) {
154+ val |= 1UL << bit;
155+ }
156+ } else {
157+ if (val & (1UL << bit)) {
158+ val &= ~(1UL << bit);
159+ }
160+ }
161+ rstate->bitmap[idx] = val;
162+}
163+
164+typedef struct VmaBlob {
165+ uint32_t start;
166+ uint32_t len;
167+ void *data;
168+} VmaBlob;
169+
170+static const VmaBlob *get_header_blob(VmaReader *vmar, uint32_t pos)
171+{
172+ assert(vmar);
173+ assert(vmar->blob_hash);
174+
175+ return g_hash_table_lookup(vmar->blob_hash, &pos);
176+}
177+
178+static const char *get_header_str(VmaReader *vmar, uint32_t pos)
179+{
180+ const VmaBlob *blob = get_header_blob(vmar, pos);
181+ if (!blob) {
182+ return NULL;
183+ }
184+ const char *res = (char *)blob->data;
185+ if (res[blob->len-1] != '\0') {
186+ return NULL;
187+ }
188+ return res;
189+}
190+
191+static ssize_t
192+safe_read(int fd, unsigned char *buf, size_t count)
193+{
194+ ssize_t n;
195+
196+ do {
197+ n = read(fd, buf, count);
198+ } while (n < 0 && errno == EINTR);
199+
200+ return n;
201+}
202+
203+static ssize_t
204+full_read(int fd, unsigned char *buf, size_t len)
205+{
206+ ssize_t n;
207+ size_t total;
208+
209+ total = 0;
210+
211+ while (len > 0) {
212+ n = safe_read(fd, buf, len);
213+
214+ if (n == 0) {
215+ return total;
216+ }
217+
218+ if (n <= 0) {
219+ break;
220+ }
221+
222+ buf += n;
223+ total += n;
224+ len -= n;
225+ }
226+
227+ if (len) {
228+ return -1;
229+ }
230+
231+ return total;
232+}
233+
234+void vma_reader_destroy(VmaReader *vmar)
235+{
236+ assert(vmar);
237+
238+ if (vmar->fd >= 0) {
239+ close(vmar->fd);
240+ }
241+
242+ if (vmar->cdata_list) {
243+ g_list_free(vmar->cdata_list);
244+ }
245+
246+ int i;
247+ for (i = 1; i < 256; i++) {
248+ if (vmar->rstate[i].bitmap) {
249+ g_free(vmar->rstate[i].bitmap);
250+ }
251+ }
252+
253+ if (vmar->md5csum) {
254+ g_checksum_free(vmar->md5csum);
255+ }
256+
257+ if (vmar->blob_hash) {
258+ g_hash_table_destroy(vmar->blob_hash);
259+ }
260+
261+ if (vmar->head_data) {
262+ g_free(vmar->head_data);
263+ }
264+
265+ g_free(vmar);
266+
267+};
268+
269+static int vma_reader_read_head(VmaReader *vmar, Error **errp)
270+{
271+ assert(vmar);
272+ assert(errp);
273+ assert(*errp == NULL);
274+
275+ unsigned char md5sum[16];
276+ int i;
277+ int ret = 0;
278+
279+ vmar->head_data = g_malloc(sizeof(VmaHeader));
280+
281+ if (full_read(vmar->fd, vmar->head_data, sizeof(VmaHeader)) !=
282+ sizeof(VmaHeader)) {
283+ error_setg(errp, "can't read vma header - %s",
284+ errno ? g_strerror(errno) : "got EOF");
285+ return -1;
286+ }
287+
288+ VmaHeader *h = (VmaHeader *)vmar->head_data;
289+
290+ if (h->magic != VMA_MAGIC) {
291+ error_setg(errp, "not a vma file - wrong magic number");
292+ return -1;
293+ }
294+
295+ uint32_t header_size = GUINT32_FROM_BE(h->header_size);
296+ int need = header_size - sizeof(VmaHeader);
297+ if (need <= 0) {
298+ error_setg(errp, "wrong vma header size %d", header_size);
299+ return -1;
300+ }
301+
302+ vmar->head_data = g_realloc(vmar->head_data, header_size);
303+ h = (VmaHeader *)vmar->head_data;
304+
305+ if (full_read(vmar->fd, vmar->head_data + sizeof(VmaHeader), need) !=
306+ need) {
307+ error_setg(errp, "can't read vma header data - %s",
308+ errno ? g_strerror(errno) : "got EOF");
309+ return -1;
310+ }
311+
312+ memcpy(md5sum, h->md5sum, 16);
313+ memset(h->md5sum, 0, 16);
314+
315+ g_checksum_reset(vmar->md5csum);
316+ g_checksum_update(vmar->md5csum, vmar->head_data, header_size);
317+ gsize csize = 16;
318+ g_checksum_get_digest(vmar->md5csum, (guint8 *)(h->md5sum), &csize);
319+
320+ if (memcmp(md5sum, h->md5sum, 16) != 0) {
321+ error_setg(errp, "wrong vma header chechsum");
322+ return -1;
323+ }
324+
325+ /* we can modify header data after checksum verify */
326+ h->header_size = header_size;
327+
328+ h->version = GUINT32_FROM_BE(h->version);
329+ if (h->version != 1) {
330+ error_setg(errp, "wrong vma version %d", h->version);
331+ return -1;
332+ }
333+
334+ h->ctime = GUINT64_FROM_BE(h->ctime);
335+ h->blob_buffer_offset = GUINT32_FROM_BE(h->blob_buffer_offset);
336+ h->blob_buffer_size = GUINT32_FROM_BE(h->blob_buffer_size);
337+
338+ uint32_t bstart = h->blob_buffer_offset + 1;
339+ uint32_t bend = h->blob_buffer_offset + h->blob_buffer_size;
340+
341+ if (bstart <= sizeof(VmaHeader)) {
342+ error_setg(errp, "wrong vma blob buffer offset %d",
343+ h->blob_buffer_offset);
344+ return -1;
345+ }
346+
347+ if (bend > header_size) {
348+ error_setg(errp, "wrong vma blob buffer size %d/%d",
349+ h->blob_buffer_offset, h->blob_buffer_size);
350+ return -1;
351+ }
352+
353+ while ((bstart + 2) <= bend) {
354+ uint32_t size = vmar->head_data[bstart] +
355+ (vmar->head_data[bstart+1] << 8);
356+ if ((bstart + size + 2) <= bend) {
357+ VmaBlob *blob = g_new0(VmaBlob, 1);
358+ blob->start = bstart - h->blob_buffer_offset;
359+ blob->len = size;
360+ blob->data = vmar->head_data + bstart + 2;
361+ g_hash_table_insert(vmar->blob_hash, &blob->start, blob);
362+ }
363+ bstart += size + 2;
364+ }
365+
366+
367+ int count = 0;
368+ for (i = 1; i < 256; i++) {
369+ VmaDeviceInfoHeader *dih = &h->dev_info[i];
370+ uint32_t devname_ptr = GUINT32_FROM_BE(dih->devname_ptr);
371+ uint64_t size = GUINT64_FROM_BE(dih->size);
372+ const char *devname = get_header_str(vmar, devname_ptr);
373+
374+ if (size && devname) {
375+ count++;
376+ vmar->devinfo[i].size = size;
377+ vmar->devinfo[i].devname = devname;
378+
379+ if (strcmp(devname, "vmstate") == 0) {
380+ vmar->vmstate_stream = i;
381+ }
382+ }
383+ }
384+
385+ if (!count) {
386+ error_setg(errp, "vma does not contain data");
387+ return -1;
388+ }
389+
390+ for (i = 0; i < VMA_MAX_CONFIGS; i++) {
391+ uint32_t name_ptr = GUINT32_FROM_BE(h->config_names[i]);
392+ uint32_t data_ptr = GUINT32_FROM_BE(h->config_data[i]);
393+
394+ if (!(name_ptr && data_ptr)) {
395+ continue;
396+ }
397+ const char *name = get_header_str(vmar, name_ptr);
398+ const VmaBlob *blob = get_header_blob(vmar, data_ptr);
399+
400+ if (!(name && blob)) {
401+ error_setg(errp, "vma contains invalid data pointers");
402+ return -1;
403+ }
404+
405+ VmaConfigData *cdata = g_new0(VmaConfigData, 1);
406+ cdata->name = name;
407+ cdata->data = blob->data;
408+ cdata->len = blob->len;
409+
410+ vmar->cdata_list = g_list_append(vmar->cdata_list, cdata);
411+ }
412+
413+ return ret;
414+};
415+
416+VmaReader *vma_reader_create(const char *filename, Error **errp)
417+{
418+ assert(filename);
419+ assert(errp);
420+
421+ VmaReader *vmar = g_new0(VmaReader, 1);
422+
423+ if (strcmp(filename, "-") == 0) {
424+ vmar->fd = dup(0);
425+ } else {
426+ vmar->fd = open(filename, O_RDONLY);
427+ }
428+
429+ if (vmar->fd < 0) {
430+ error_setg(errp, "can't open file %s - %s\n", filename,
431+ g_strerror(errno));
432+ goto err;
433+ }
434+
435+ vmar->md5csum = g_checksum_new(G_CHECKSUM_MD5);
436+ if (!vmar->md5csum) {
437+ error_setg(errp, "can't allocate cmsum\n");
438+ goto err;
439+ }
440+
441+ vmar->blob_hash = g_hash_table_new_full(g_int32_hash, g_int32_equal,
442+ NULL, g_free);
443+
444+ if (vma_reader_read_head(vmar, errp) < 0) {
445+ goto err;
446+ }
447+
448+ return vmar;
449+
450+err:
451+ if (vmar) {
452+ vma_reader_destroy(vmar);
453+ }
454+
455+ return NULL;
456+}
457+
458+VmaHeader *vma_reader_get_header(VmaReader *vmar)
459+{
460+ assert(vmar);
461+ assert(vmar->head_data);
462+
463+ return (VmaHeader *)(vmar->head_data);
464+}
465+
466+GList *vma_reader_get_config_data(VmaReader *vmar)
467+{
468+ assert(vmar);
469+ assert(vmar->head_data);
470+
471+ return vmar->cdata_list;
472+}
473+
474+VmaDeviceInfo *vma_reader_get_device_info(VmaReader *vmar, guint8 dev_id)
475+{
476+ assert(vmar);
477+ assert(dev_id);
478+
479+ if (vmar->devinfo[dev_id].size && vmar->devinfo[dev_id].devname) {
480+ return &vmar->devinfo[dev_id];
481+ }
482+
483+ return NULL;
484+}
485+
486+int vma_reader_register_bs(VmaReader *vmar, guint8 dev_id, BlockDriverState *bs,
487+ bool write_zeroes, Error **errp)
488+{
489+ assert(vmar);
490+ assert(bs != NULL);
491+ assert(dev_id);
492+ assert(vmar->rstate[dev_id].bs == NULL);
493+
494+ int64_t size = bdrv_getlength(bs);
495+ if (size != vmar->devinfo[dev_id].size) {
496+ error_setg(errp, "vma_reader_register_bs for stream %s failed - "
497+ "unexpected size %zd != %zd", vmar->devinfo[dev_id].devname,
498+ size, vmar->devinfo[dev_id].size);
499+ return -1;
500+ }
501+
502+ vmar->rstate[dev_id].bs = bs;
503+ vmar->rstate[dev_id].write_zeroes = write_zeroes;
504+
505+ int64_t bitmap_size = (size/BDRV_SECTOR_SIZE) +
506+ (VMA_CLUSTER_SIZE/BDRV_SECTOR_SIZE) * BITS_PER_LONG - 1;
507+ bitmap_size /= (VMA_CLUSTER_SIZE/BDRV_SECTOR_SIZE) * BITS_PER_LONG;
508+
509+ vmar->rstate[dev_id].bitmap_size = bitmap_size;
510+ vmar->rstate[dev_id].bitmap = g_new0(unsigned long, bitmap_size);
511+
512+ vmar->cluster_count += size/VMA_CLUSTER_SIZE;
513+
514+ return 0;
515+}
516+
517+static ssize_t safe_write(int fd, void *buf, size_t count)
518+{
519+ ssize_t n;
520+
521+ do {
522+ n = write(fd, buf, count);
523+ } while (n < 0 && errno == EINTR);
524+
525+ return n;
526+}
527+
528+static size_t full_write(int fd, void *buf, size_t len)
529+{
530+ ssize_t n;
531+ size_t total;
532+
533+ total = 0;
534+
535+ while (len > 0) {
536+ n = safe_write(fd, buf, len);
537+ if (n < 0) {
538+ return n;
539+ }
540+ buf += n;
541+ total += n;
542+ len -= n;
543+ }
544+
545+ if (len) {
546+ /* incomplete write ? */
547+ return -1;
548+ }
549+
550+ return total;
551+}
552+
553+static int restore_write_data(VmaReader *vmar, guint8 dev_id,
554+ BlockDriverState *bs, int vmstate_fd,
555+ unsigned char *buf, int64_t sector_num,
556+ int nb_sectors, Error **errp)
557+{
558+ assert(vmar);
559+
560+ if (dev_id == vmar->vmstate_stream) {
561+ if (vmstate_fd >= 0) {
562+ int len = nb_sectors * BDRV_SECTOR_SIZE;
563+ int res = full_write(vmstate_fd, buf, len);
564+ if (res < 0) {
565+ error_setg(errp, "write vmstate failed %d", res);
566+ return -1;
567+ }
568+ }
569+ } else {
570+ int res = bdrv_write(bs, sector_num, buf, nb_sectors);
571+ if (res < 0) {
572+ error_setg(errp, "bdrv_write to %s failed (%d)",
573+ bdrv_get_device_name(bs), res);
574+ return -1;
575+ }
576+ }
577+ return 0;
578+}
579+static int restore_extent(VmaReader *vmar, unsigned char *buf,
580+ int extent_size, int vmstate_fd,
581+ bool verbose, Error **errp)
582+{
583+ assert(vmar);
584+ assert(buf);
585+
586+ VmaExtentHeader *ehead = (VmaExtentHeader *)buf;
587+ int start = VMA_EXTENT_HEADER_SIZE;
588+ int i;
589+
590+ for (i = 0; i < VMA_BLOCKS_PER_EXTENT; i++) {
591+ uint64_t block_info = GUINT64_FROM_BE(ehead->blockinfo[i]);
592+ uint64_t cluster_num = block_info & 0xffffffff;
593+ uint8_t dev_id = (block_info >> 32) & 0xff;
594+ uint16_t mask = block_info >> (32+16);
595+ int64_t max_sector;
596+
597+ if (!dev_id) {
598+ continue;
599+ }
600+
601+ VmaRestoreState *rstate = &vmar->rstate[dev_id];
602+ BlockDriverState *bs = NULL;
603+
604+ if (dev_id != vmar->vmstate_stream) {
605+ bs = rstate->bs;
606+ if (!bs) {
607+ error_setg(errp, "got wrong dev id %d", dev_id);
608+ return -1;
609+ }
610+
611+ if (vma_reader_get_bitmap(rstate, cluster_num)) {
612+ error_setg(errp, "found duplicated cluster %zd for stream %s",
613+ cluster_num, vmar->devinfo[dev_id].devname);
614+ return -1;
615+ }
616+ vma_reader_set_bitmap(rstate, cluster_num, 1);
617+
618+ max_sector = vmar->devinfo[dev_id].size/BDRV_SECTOR_SIZE;
619+ } else {
620+ max_sector = G_MAXINT64;
621+ if (cluster_num != vmar->vmstate_clusters) {
622+ error_setg(errp, "found out of order vmstate data");
623+ return -1;
624+ }
625+ vmar->vmstate_clusters++;
626+ }
627+
628+ vmar->clusters_read++;
629+
630+ if (verbose) {
631+ time_t duration = time(NULL) - vmar->start_time;
632+ int percent = (vmar->clusters_read*100)/vmar->cluster_count;
633+ if (percent != vmar->clusters_read_per) {
634+ printf("progress %d%% (read %zd bytes, duration %zd sec)\n",
635+ percent, vmar->clusters_read*VMA_CLUSTER_SIZE,
636+ duration);
637+ fflush(stdout);
638+ vmar->clusters_read_per = percent;
639+ }
640+ }
641+
642+ /* try to write whole clusters to speedup restore */
643+ if (mask == 0xffff) {
644+ if ((start + VMA_CLUSTER_SIZE) > extent_size) {
645+ error_setg(errp, "short vma extent - too many blocks");
646+ return -1;
647+ }
648+ int64_t sector_num = (cluster_num * VMA_CLUSTER_SIZE) /
649+ BDRV_SECTOR_SIZE;
650+ int64_t end_sector = sector_num +
651+ VMA_CLUSTER_SIZE/BDRV_SECTOR_SIZE;
652+
653+ if (end_sector > max_sector) {
654+ end_sector = max_sector;
655+ }
656+
657+ if (end_sector <= sector_num) {
658+ error_setg(errp, "got wrong block address - write bejond end");
659+ return -1;
660+ }
661+
662+ int nb_sectors = end_sector - sector_num;
663+ if (restore_write_data(vmar, dev_id, bs, vmstate_fd, buf + start,
664+ sector_num, nb_sectors, errp) < 0) {
665+ return -1;
666+ }
667+
668+ start += VMA_CLUSTER_SIZE;
669+ } else {
670+ int j;
671+ int bit = 1;
672+
673+ for (j = 0; j < 16; j++) {
674+ int64_t sector_num = (cluster_num*VMA_CLUSTER_SIZE +
675+ j*VMA_BLOCK_SIZE)/BDRV_SECTOR_SIZE;
676+
677+ int64_t end_sector = sector_num +
678+ VMA_BLOCK_SIZE/BDRV_SECTOR_SIZE;
679+ if (end_sector > max_sector) {
680+ end_sector = max_sector;
681+ }
682+
683+ if (mask & bit) {
684+ if ((start + VMA_BLOCK_SIZE) > extent_size) {
685+ error_setg(errp, "short vma extent - too many blocks");
686+ return -1;
687+ }
688+
689+ if (end_sector <= sector_num) {
690+ error_setg(errp, "got wrong block address - "
691+ "write bejond end");
692+ return -1;
693+ }
694+
695+ int nb_sectors = end_sector - sector_num;
696+ if (restore_write_data(vmar, dev_id, bs, vmstate_fd,
697+ buf + start, sector_num,
698+ nb_sectors, errp) < 0) {
699+ return -1;
700+ }
701+
702+ start += VMA_BLOCK_SIZE;
703+
704+ } else {
705+
706+ if (rstate->write_zeroes && (end_sector > sector_num)) {
707+ /* Todo: use bdrv_co_write_zeroes (but that need to
708+ * be run inside coroutine?)
709+ */
710+ int nb_sectors = end_sector - sector_num;
711+ if (restore_write_data(vmar, dev_id, bs, vmstate_fd,
712+ zero_vma_block, sector_num,
713+ nb_sectors, errp) < 0) {
714+ return -1;
715+ }
716+ }
717+ }
718+
719+ bit = bit << 1;
720+ }
721+ }
722+ }
723+
724+ if (start != extent_size) {
725+ error_setg(errp, "vma extent error - missing blocks");
726+ return -1;
727+ }
728+
729+ return 0;
730+}
731+
732+int vma_reader_restore(VmaReader *vmar, int vmstate_fd, bool verbose,
733+ Error **errp)
734+{
735+ assert(vmar);
736+ assert(vmar->head_data);
737+
738+ int ret = 0;
739+ unsigned char buf[VMA_MAX_EXTENT_SIZE];
740+ int buf_pos = 0;
741+ unsigned char md5sum[16];
742+ VmaHeader *h = (VmaHeader *)vmar->head_data;
743+
744+ vmar->start_time = time(NULL);
745+
746+ while (1) {
747+ int bytes = full_read(vmar->fd, buf + buf_pos, sizeof(buf) - buf_pos);
748+ if (bytes < 0) {
749+ error_setg(errp, "read failed - %s", g_strerror(errno));
750+ return -1;
751+ }
752+
753+ buf_pos += bytes;
754+
755+ if (!buf_pos) {
756+ break; /* EOF */
757+ }
758+
759+ if (buf_pos < VMA_EXTENT_HEADER_SIZE) {
760+ error_setg(errp, "read short extent (%d bytes)", buf_pos);
761+ return -1;
762+ }
763+
764+ VmaExtentHeader *ehead = (VmaExtentHeader *)buf;
765+
766+ /* extract md5sum */
767+ memcpy(md5sum, ehead->md5sum, sizeof(ehead->md5sum));
768+ memset(ehead->md5sum, 0, sizeof(ehead->md5sum));
769+
770+ g_checksum_reset(vmar->md5csum);
771+ g_checksum_update(vmar->md5csum, buf, VMA_EXTENT_HEADER_SIZE);
772+ gsize csize = 16;
773+ g_checksum_get_digest(vmar->md5csum, ehead->md5sum, &csize);
774+
775+ if (memcmp(md5sum, ehead->md5sum, 16) != 0) {
776+ error_setg(errp, "wrong vma extent header chechsum");
777+ return -1;
778+ }
779+
780+ if (memcmp(h->uuid, ehead->uuid, sizeof(ehead->uuid)) != 0) {
781+ error_setg(errp, "wrong vma extent uuid");
782+ return -1;
783+ }
784+
785+ if (ehead->magic != VMA_EXTENT_MAGIC || ehead->reserved1 != 0) {
786+ error_setg(errp, "wrong vma extent header magic");
787+ return -1;
788+ }
789+
790+ int block_count = GUINT16_FROM_BE(ehead->block_count);
791+ int extent_size = VMA_EXTENT_HEADER_SIZE + block_count*VMA_BLOCK_SIZE;
792+
793+ if (buf_pos < extent_size) {
794+ error_setg(errp, "short vma extent (%d < %d)", buf_pos,
795+ extent_size);
796+ return -1;
797+ }
798+
799+ if (restore_extent(vmar, buf, extent_size, vmstate_fd, verbose,
800+ errp) < 0) {
801+ return -1;
802+ }
803+
804+ if (buf_pos > extent_size) {
805+ memmove(buf, buf + extent_size, buf_pos - extent_size);
806+ buf_pos = buf_pos - extent_size;
807+ } else {
808+ buf_pos = 0;
809+ }
810+ }
811+
812+ bdrv_drain_all();
813+
814+ int i;
815+ for (i = 1; i < 256; i++) {
816+ VmaRestoreState *rstate = &vmar->rstate[i];
817+ if (!rstate->bs) {
818+ continue;
819+ }
820+
821+ if (bdrv_flush(rstate->bs) < 0) {
822+ error_setg(errp, "vma bdrv_flush %s failed",
823+ vmar->devinfo[i].devname);
824+ return -1;
825+ }
826+
827+ if (vmar->devinfo[i].size &&
828+ (strcmp(vmar->devinfo[i].devname, "vmstate") != 0)) {
829+ assert(rstate->bitmap);
830+
831+ int64_t cluster_num, end;
832+
833+ end = (vmar->devinfo[i].size + VMA_CLUSTER_SIZE - 1) /
834+ VMA_CLUSTER_SIZE;
835+
836+ for (cluster_num = 0; cluster_num < end; cluster_num++) {
837+ if (!vma_reader_get_bitmap(rstate, cluster_num)) {
838+ error_setg(errp, "detected missing cluster %zd "
839+ "for stream %s", cluster_num,
840+ vmar->devinfo[i].devname);
841+ return -1;
842+ }
843+ }
844+ }
845+ }
846+
847+ return ret;
848+}
849+
850Index: new/vma-writer.c
851===================================================================
852--- /dev/null 1970-01-01 00:00:00.000000000 +0000
24bb7da3 853+++ new/vma-writer.c 2014-11-20 08:31:06.000000000 +0100
e549ff74 854@@ -0,0 +1,876 @@
b668dc80
DM		 855+/*
856+ * VMA: Virtual Machine Archive
857+ *
858+ * Copyright (C) 2012 Proxmox Server Solutions
859+ *
860+ * Authors:
861+ * Dietmar Maurer (dietmar@proxmox.com)
862+ *
863+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
864+ * See the COPYING file in the top-level directory.
865+ *
866+ */
867+
868+#include <stdio.h>
869+#include <errno.h>
870+#include <unistd.h>
871+#include <stdio.h>
872+#include <string.h>
873+#include <sys/types.h>
874+#include <sys/stat.h>
875+#include <fcntl.h>
876+#include <glib.h>
877+#include <uuid/uuid.h>
878+
879+#include "qemu-common.h"
880+#include "vma.h"
881+#include "block/block.h"
882+#include "monitor/monitor.h"
e549ff74 883+#include "qemu/main-loop.h"
b668dc80
DM		 884+
885+#define DEBUG_VMA 0
886+
887+#define DPRINTF(fmt, ...)\
888+ do { if (DEBUG_VMA) { printf("vma: " fmt, ## __VA_ARGS__); } } while (0)
889+
890+#define WRITE_BUFFERS 5
891+
892+typedef struct VmaAIOCB VmaAIOCB;
893+struct VmaAIOCB {
894+ unsigned char buffer[VMA_MAX_EXTENT_SIZE];
895+ VmaWriter *vmaw;
896+ size_t bytes;
897+ Coroutine *co;
898+};
899+
900+struct VmaWriter {
901+ int fd;
902+ FILE *cmd;
903+ int status;
904+ char errmsg[8192];
905+ uuid_t uuid;
906+ bool header_written;
907+ bool closed;
908+
909+ /* we always write extents */
910+ unsigned char outbuf[VMA_MAX_EXTENT_SIZE];
911+ int outbuf_pos; /* in bytes */
912+ int outbuf_count; /* in VMA_BLOCKS */
913+ uint64_t outbuf_block_info[VMA_BLOCKS_PER_EXTENT];
914+
915+ VmaAIOCB *aiocbs[WRITE_BUFFERS];
916+ CoQueue wqueue;
917+
918+ GChecksum *md5csum;
919+ CoMutex writer_lock;
920+ CoMutex flush_lock;
921+ Coroutine *co_writer;
922+
923+ /* drive informations */
924+ VmaStreamInfo stream_info[256];
925+ guint stream_count;
926+
927+ guint8 vmstate_stream;
928+ uint32_t vmstate_clusters;
929+
930+ /* header blob table */
931+ char *header_blob_table;
932+ uint32_t header_blob_table_size;
933+ uint32_t header_blob_table_pos;
934+
935+ /* store for config blobs */
936+ uint32_t config_names[VMA_MAX_CONFIGS]; /* offset into blob_buffer table */
937+ uint32_t config_data[VMA_MAX_CONFIGS]; /* offset into blob_buffer table */
938+ uint32_t config_count;
939+};
940+
941+void vma_writer_set_error(VmaWriter *vmaw, const char *fmt, ...)
942+{
943+ va_list ap;
944+
945+ if (vmaw->status < 0) {
946+ return;
947+ }
948+
949+ vmaw->status = -1;
950+
951+ va_start(ap, fmt);
952+ g_vsnprintf(vmaw->errmsg, sizeof(vmaw->errmsg), fmt, ap);
953+ va_end(ap);
954+
955+ DPRINTF("vma_writer_set_error: %s\n", vmaw->errmsg);
956+}
957+
958+static uint32_t allocate_header_blob(VmaWriter *vmaw, const char *data,
959+ size_t len)
960+{
961+ if (len > 65535) {
962+ return 0;
963+ }
964+
965+ if (!vmaw->header_blob_table ||
966+ (vmaw->header_blob_table_size <
967+ (vmaw->header_blob_table_pos + len + 2))) {
968+ int newsize = vmaw->header_blob_table_size + ((len + 2 + 511)/512)*512;
969+
970+ vmaw->header_blob_table = g_realloc(vmaw->header_blob_table, newsize);
971+ memset(vmaw->header_blob_table + vmaw->header_blob_table_size,
972+ 0, newsize - vmaw->header_blob_table_size);
973+ vmaw->header_blob_table_size = newsize;
974+ }
975+
976+ uint32_t cpos = vmaw->header_blob_table_pos;
977+ vmaw->header_blob_table[cpos] = len & 255;
978+ vmaw->header_blob_table[cpos+1] = (len >> 8) & 255;
979+ memcpy(vmaw->header_blob_table + cpos + 2, data, len);
980+ vmaw->header_blob_table_pos += len + 2;
981+ return cpos;
982+}
983+
984+static uint32_t allocate_header_string(VmaWriter *vmaw, const char *str)
985+{
986+ assert(vmaw);
987+
988+ size_t len = strlen(str) + 1;
989+
990+ return allocate_header_blob(vmaw, str, len);
991+}
992+
993+int vma_writer_add_config(VmaWriter *vmaw, const char *name, gpointer data,
994+ gsize len)
995+{
996+ assert(vmaw);
997+ assert(!vmaw->header_written);
998+ assert(vmaw->config_count < VMA_MAX_CONFIGS);
999+ assert(name);
1000+ assert(data);
1001+ assert(len);
1002+
80b30085
DM		 1003+ gchar *basename = g_path_get_basename(name);
1004+ uint32_t name_ptr = allocate_header_string(vmaw, basename);
1005+ g_free(basename);
1006+
b668dc80
DM		 1007+ if (!name_ptr) {
1008+ return -1;
1009+ }
1010+
1011+ uint32_t data_ptr = allocate_header_blob(vmaw, data, len);
1012+ if (!data_ptr) {
1013+ return -1;
1014+ }
1015+
1016+ vmaw->config_names[vmaw->config_count] = name_ptr;
1017+ vmaw->config_data[vmaw->config_count] = data_ptr;
1018+
1019+ vmaw->config_count++;
1020+
1021+ return 0;
1022+}
1023+
1024+int vma_writer_register_stream(VmaWriter *vmaw, const char *devname,
1025+ size_t size)
1026+{
1027+ assert(vmaw);
1028+ assert(devname);
1029+ assert(!vmaw->status);
1030+
1031+ if (vmaw->header_written) {
1032+ vma_writer_set_error(vmaw, "vma_writer_register_stream: header "
1033+ "already written");
1034+ return -1;
1035+ }
1036+
1037+ guint n = vmaw->stream_count + 1;
1038+
1039+ /* we can have dev_ids form 1 to 255 (0 reserved)
1040+ * 255(-1) reseverd for safety
1041+ */
1042+ if (n > 254) {
1043+ vma_writer_set_error(vmaw, "vma_writer_register_stream: "
1044+ "too many drives");
1045+ return -1;
1046+ }
1047+
1048+ if (size <= 0) {
1049+ vma_writer_set_error(vmaw, "vma_writer_register_stream: "
1050+ "got strange size %zd", size);
1051+ return -1;
1052+ }
1053+
1054+ DPRINTF("vma_writer_register_stream %s %zu %d\n", devname, size, n);
1055+
1056+ vmaw->stream_info[n].devname = g_strdup(devname);
1057+ vmaw->stream_info[n].size = size;
1058+
1059+ vmaw->stream_info[n].cluster_count = (size + VMA_CLUSTER_SIZE - 1) /
1060+ VMA_CLUSTER_SIZE;
1061+
1062+ vmaw->stream_count = n;
1063+
1064+ if (strcmp(devname, "vmstate") == 0) {
1065+ vmaw->vmstate_stream = n;
1066+ }
1067+
1068+ return n;
1069+}
1070+
1071+static void vma_co_continue_write(void *opaque)
1072+{
1073+ VmaWriter *vmaw = opaque;
1074+
1075+ DPRINTF("vma_co_continue_write\n");
1076+ qemu_coroutine_enter(vmaw->co_writer, NULL);
1077+}
1078+
1079+static ssize_t coroutine_fn
1080+vma_co_write(VmaWriter *vmaw, const void *buf, size_t bytes)
1081+{
1082+ size_t done = 0;
1083+ ssize_t ret;
1084+
1085+ /* atomic writes (we cannot interleave writes) */
1086+ qemu_co_mutex_lock(&vmaw->writer_lock);
1087+
1088+ DPRINTF("vma_co_write enter %zd\n", bytes);
1089+
1090+ assert(vmaw->co_writer == NULL);
1091+
1092+ vmaw->co_writer = qemu_coroutine_self();
1093+
e549ff74 1094+ aio_set_fd_handler(qemu_get_aio_context(), vmaw->fd, NULL, vma_co_continue_write, vmaw);
b668dc80
DM		 1095+
1096+ DPRINTF("vma_co_write wait until writable\n");
1097+ qemu_coroutine_yield();
1098+ DPRINTF("vma_co_write starting %zd\n", bytes);
1099+
1100+ while (done < bytes) {
1101+ ret = write(vmaw->fd, buf + done, bytes - done);
1102+ if (ret > 0) {
1103+ done += ret;
1104+ DPRINTF("vma_co_write written %zd %zd\n", done, ret);
1105+ } else if (ret < 0) {
1106+ if (errno == EAGAIN || errno == EWOULDBLOCK) {
1107+ DPRINTF("vma_co_write yield %zd\n", done);
1108+ qemu_coroutine_yield();
1109+ DPRINTF("vma_co_write restart %zd\n", done);
1110+ } else {
1111+ vma_writer_set_error(vmaw, "vma_co_write write error - %s",
1112+ g_strerror(errno));
1113+ done = -1; /* always return failure for partial writes */
1114+ break;
1115+ }
1116+ } else if (ret == 0) {
1117+ /* should not happen - simply try again */
1118+ }
1119+ }
1120+
e549ff74 1121+ aio_set_fd_handler(qemu_get_aio_context(), vmaw->fd, NULL, NULL, NULL);
b668dc80
DM		 1122+
1123+ vmaw->co_writer = NULL;
1124+
1125+ qemu_co_mutex_unlock(&vmaw->writer_lock);
1126+
1127+ DPRINTF("vma_co_write leave %zd\n", done);
1128+ return done;
1129+}
1130+
1131+static void coroutine_fn vma_co_writer_task(void *opaque)
1132+{
1133+ VmaAIOCB *cb = opaque;
1134+
1135+ DPRINTF("vma_co_writer_task start\n");
1136+
1137+ int64_t done = vma_co_write(cb->vmaw, cb->buffer, cb->bytes);
1138+ DPRINTF("vma_co_writer_task write done %zd\n", done);
1139+
1140+ if (done != cb->bytes) {
1141+ DPRINTF("vma_co_writer_task failed write %zd %zd", cb->bytes, done);
1142+ vma_writer_set_error(cb->vmaw, "vma_co_writer_task failed write %zd",
1143+ done);
1144+ }
1145+
1146+ cb->bytes = 0;
1147+
1148+ qemu_co_queue_next(&cb->vmaw->wqueue);
1149+
1150+ DPRINTF("vma_co_writer_task end\n");
1151+}
1152+
1153+static void coroutine_fn vma_queue_flush(VmaWriter *vmaw)
1154+{
1155+ DPRINTF("vma_queue_flush enter\n");
1156+
1157+ assert(vmaw);
1158+
1159+ while (1) {
1160+ int i;
1161+ VmaAIOCB *cb = NULL;
1162+ for (i = 0; i < WRITE_BUFFERS; i++) {
1163+ if (vmaw->aiocbs[i]->bytes) {
1164+ cb = vmaw->aiocbs[i];
1165+ DPRINTF("FOUND USED AIO BUFFER %d %zd\n", i,
1166+ vmaw->aiocbs[i]->bytes);
1167+ break;
1168+ }
1169+ }
1170+ if (!cb) {
1171+ break;
1172+ }
1173+ qemu_co_queue_wait(&vmaw->wqueue);
1174+ }
1175+
1176+ DPRINTF("vma_queue_flush leave\n");
1177+}
1178+
1179+/**
1180+ * NOTE: pipe buffer size in only 4096 bytes on linux (see 'ulimit -a')
1181+ * So we need to create a coroutione to allow 'parallel' execution.
1182+ */
1183+static ssize_t coroutine_fn
1184+vma_queue_write(VmaWriter *vmaw, const void *buf, size_t bytes)
1185+{
1186+ DPRINTF("vma_queue_write enter %zd\n", bytes);
1187+
1188+ assert(vmaw);
1189+ assert(buf);
1190+ assert(bytes <= VMA_MAX_EXTENT_SIZE);
1191+
1192+ VmaAIOCB *cb = NULL;
1193+ while (!cb) {
1194+ int i;
1195+ for (i = 0; i < WRITE_BUFFERS; i++) {
1196+ if (!vmaw->aiocbs[i]->bytes) {
1197+ cb = vmaw->aiocbs[i];
1198+ break;
1199+ }
1200+ }
1201+ if (!cb) {
1202+ qemu_co_queue_wait(&vmaw->wqueue);
1203+ }
1204+ }
1205+
1206+ memcpy(cb->buffer, buf, bytes);
1207+ cb->bytes = bytes;
1208+ cb->vmaw = vmaw;
1209+
1210+ DPRINTF("vma_queue_write start %zd\n", bytes);
1211+ cb->co = qemu_coroutine_create(vma_co_writer_task);
1212+ qemu_coroutine_enter(cb->co, cb);
1213+
1214+ DPRINTF("vma_queue_write leave\n");
1215+
1216+ return bytes;
1217+}
1218+
1219+VmaWriter *vma_writer_create(const char *filename, uuid_t uuid, Error **errp)
1220+{
1221+ const char *p;
1222+
1223+ assert(sizeof(VmaHeader) == (4096 + 8192));
ea04a0b3
DM		 1224+ assert(G_STRUCT_OFFSET(VmaHeader, config_names) == 2044);
1225+ assert(G_STRUCT_OFFSET(VmaHeader, config_data) == 3068);
1226+ assert(G_STRUCT_OFFSET(VmaHeader, dev_info) == 4096);
b668dc80
DM		 1227+ assert(sizeof(VmaExtentHeader) == 512);
1228+
1229+ VmaWriter *vmaw = g_new0(VmaWriter, 1);
1230+ vmaw->fd = -1;
1231+
1232+ vmaw->md5csum = g_checksum_new(G_CHECKSUM_MD5);
1233+ if (!vmaw->md5csum) {
1234+ error_setg(errp, "can't allocate cmsum\n");
1235+ goto err;
1236+ }
1237+
1238+ if (strstart(filename, "exec:", &p)) {
1239+ vmaw->cmd = popen(p, "w");
1240+ if (vmaw->cmd == NULL) {
1241+ error_setg(errp, "can't popen command '%s' - %s\n", p,
1242+ g_strerror(errno));
1243+ goto err;
1244+ }
1245+ vmaw->fd = fileno(vmaw->cmd);
1246+
1247+ /* try to use O_NONBLOCK and O_DIRECT */
1248+ fcntl(vmaw->fd, F_SETFL, fcntl(vmaw->fd, F_GETFL)|O_NONBLOCK);
1249+ fcntl(vmaw->fd, F_SETFL, fcntl(vmaw->fd, F_GETFL)|O_DIRECT);
1250+
1251+ } else {
1252+ struct stat st;
1253+ int oflags;
1254+ const char *tmp_id_str;
1255+
1256+ if ((stat(filename, &st) == 0) && S_ISFIFO(st.st_mode)) {
1257+ oflags = O_NONBLOCK|O_DIRECT|O_WRONLY;
1258+ vmaw->fd = qemu_open(filename, oflags, 0644);
1259+ } else if (strstart(filename, "/dev/fdset/", &tmp_id_str)) {
1260+ oflags = O_NONBLOCK|O_DIRECT|O_WRONLY;
1261+ vmaw->fd = qemu_open(filename, oflags, 0644);
1262+ } else if (strstart(filename, "/dev/fdname/", &tmp_id_str)) {
1263+ vmaw->fd = monitor_get_fd(cur_mon, tmp_id_str, errp);
1264+ if (vmaw->fd < 0) {
1265+ goto err;
1266+ }
1267+ /* try to use O_NONBLOCK and O_DIRECT */
1268+ fcntl(vmaw->fd, F_SETFL, fcntl(vmaw->fd, F_GETFL)|O_NONBLOCK);
1269+ fcntl(vmaw->fd, F_SETFL, fcntl(vmaw->fd, F_GETFL)|O_DIRECT);
1270+ } else {
1271+ oflags = O_NONBLOCK|O_DIRECT|O_WRONLY|O_CREAT|O_EXCL;
1272+ vmaw->fd = qemu_open(filename, oflags, 0644);
1273+ }
1274+
1275+ if (vmaw->fd < 0) {
1276+ error_setg(errp, "can't open file %s - %s\n", filename,
1277+ g_strerror(errno));
1278+ goto err;
1279+ }
1280+ }
1281+
1282+ /* we use O_DIRECT, so we need to align IO buffers */
1283+ int i;
1284+ for (i = 0; i < WRITE_BUFFERS; i++) {
1285+ vmaw->aiocbs[i] = qemu_memalign(512, sizeof(VmaAIOCB));
1286+ memset(vmaw->aiocbs[i], 0, sizeof(VmaAIOCB));
1287+ }
1288+
1289+ vmaw->outbuf_count = 0;
1290+ vmaw->outbuf_pos = VMA_EXTENT_HEADER_SIZE;
1291+
1292+ vmaw->header_blob_table_pos = 1; /* start at pos 1 */
1293+
1294+ qemu_co_mutex_init(&vmaw->writer_lock);
1295+ qemu_co_mutex_init(&vmaw->flush_lock);
1296+ qemu_co_queue_init(&vmaw->wqueue);
1297+
1298+ uuid_copy(vmaw->uuid, uuid);
1299+
1300+ return vmaw;
1301+
1302+err:
1303+ if (vmaw) {
1304+ if (vmaw->cmd) {
1305+ pclose(vmaw->cmd);
1306+ } else if (vmaw->fd >= 0) {
1307+ close(vmaw->fd);
1308+ }
1309+
1310+ if (vmaw->md5csum) {
1311+ g_checksum_free(vmaw->md5csum);
1312+ }
1313+
1314+ g_free(vmaw);
1315+ }
1316+
1317+ return NULL;
1318+}
1319+
1320+static int coroutine_fn vma_write_header(VmaWriter *vmaw)
1321+{
1322+ assert(vmaw);
1323+ int header_clusters = 8;
1324+ char buf[65536*header_clusters];
1325+ VmaHeader *head = (VmaHeader *)buf;
1326+
1327+ int i;
1328+
1329+ DPRINTF("VMA WRITE HEADER\n");
1330+
1331+ if (vmaw->status < 0) {
1332+ return vmaw->status;
1333+ }
1334+
1335+ memset(buf, 0, sizeof(buf));
1336+
1337+ head->magic = VMA_MAGIC;
1338+ head->version = GUINT32_TO_BE(1); /* v1 */
1339+ memcpy(head->uuid, vmaw->uuid, 16);
1340+
1341+ time_t ctime = time(NULL);
1342+ head->ctime = GUINT64_TO_BE(ctime);
1343+
1344+ if (!vmaw->stream_count) {
1345+ return -1;
1346+ }
1347+
1348+ for (i = 0; i < VMA_MAX_CONFIGS; i++) {
1349+ head->config_names[i] = GUINT32_TO_BE(vmaw->config_names[i]);
1350+ head->config_data[i] = GUINT32_TO_BE(vmaw->config_data[i]);
1351+ }
1352+
1353+ /* 32 bytes per device (12 used currently) = 8192 bytes max */
1354+ for (i = 1; i <= 254; i++) {
1355+ VmaStreamInfo *si = &vmaw->stream_info[i];
1356+ if (si->size) {
1357+ assert(si->devname);
1358+ uint32_t devname_ptr = allocate_header_string(vmaw, si->devname);
1359+ if (!devname_ptr) {
1360+ return -1;
1361+ }
1362+ head->dev_info[i].devname_ptr = GUINT32_TO_BE(devname_ptr);
1363+ head->dev_info[i].size = GUINT64_TO_BE(si->size);
1364+ }
1365+ }
1366+
1367+ uint32_t header_size = sizeof(VmaHeader) + vmaw->header_blob_table_size;
1368+ head->header_size = GUINT32_TO_BE(header_size);
1369+
1370+ if (header_size > sizeof(buf)) {
1371+ return -1; /* just to be sure */
1372+ }
1373+
1374+ uint32_t blob_buffer_offset = sizeof(VmaHeader);
1375+ memcpy(buf + blob_buffer_offset, vmaw->header_blob_table,
1376+ vmaw->header_blob_table_size);
1377+ head->blob_buffer_offset = GUINT32_TO_BE(blob_buffer_offset);
1378+ head->blob_buffer_size = GUINT32_TO_BE(vmaw->header_blob_table_pos);
1379+
1380+ g_checksum_reset(vmaw->md5csum);
1381+ g_checksum_update(vmaw->md5csum, (const guchar *)buf, header_size);
1382+ gsize csize = 16;
1383+ g_checksum_get_digest(vmaw->md5csum, (guint8 *)(head->md5sum), &csize);
1384+
1385+ return vma_queue_write(vmaw, buf, header_size);
1386+}
1387+
1388+static int coroutine_fn vma_writer_flush(VmaWriter *vmaw)
1389+{
1390+ assert(vmaw);
1391+
1392+ int ret;
1393+ int i;
1394+
1395+ if (vmaw->status < 0) {
1396+ return vmaw->status;
1397+ }
1398+
1399+ if (!vmaw->header_written) {
1400+ vmaw->header_written = true;
1401+ ret = vma_write_header(vmaw);
1402+ if (ret < 0) {
1403+ vma_writer_set_error(vmaw, "vma_writer_flush: write header failed");
1404+ return ret;
1405+ }
1406+ }
1407+
1408+ DPRINTF("VMA WRITE FLUSH %d %d\n", vmaw->outbuf_count, vmaw->outbuf_pos);
1409+
1410+
1411+ VmaExtentHeader *ehead = (VmaExtentHeader *)vmaw->outbuf;
1412+
1413+ ehead->magic = VMA_EXTENT_MAGIC;
1414+ ehead->reserved1 = 0;
1415+
1416+ for (i = 0; i < VMA_BLOCKS_PER_EXTENT; i++) {
1417+ ehead->blockinfo[i] = GUINT64_TO_BE(vmaw->outbuf_block_info[i]);
1418+ }
1419+
1420+ guint16 block_count = (vmaw->outbuf_pos - VMA_EXTENT_HEADER_SIZE) /
1421+ VMA_BLOCK_SIZE;
1422+
1423+ ehead->block_count = GUINT16_TO_BE(block_count);
1424+
1425+ memcpy(ehead->uuid, vmaw->uuid, sizeof(ehead->uuid));
1426+ memset(ehead->md5sum, 0, sizeof(ehead->md5sum));
1427+
1428+ g_checksum_reset(vmaw->md5csum);
1429+ g_checksum_update(vmaw->md5csum, vmaw->outbuf, VMA_EXTENT_HEADER_SIZE);
1430+ gsize csize = 16;
1431+ g_checksum_get_digest(vmaw->md5csum, ehead->md5sum, &csize);
1432+
1433+ int bytes = vmaw->outbuf_pos;
1434+ ret = vma_queue_write(vmaw, vmaw->outbuf, bytes);
1435+ if (ret != bytes) {
1436+ vma_writer_set_error(vmaw, "vma_writer_flush: failed write");
1437+ }
1438+
1439+ vmaw->outbuf_count = 0;
1440+ vmaw->outbuf_pos = VMA_EXTENT_HEADER_SIZE;
1441+
1442+ for (i = 0; i < VMA_BLOCKS_PER_EXTENT; i++) {
1443+ vmaw->outbuf_block_info[i] = 0;
1444+ }
1445+
1446+ return vmaw->status;
1447+}
1448+
1449+static int vma_count_open_streams(VmaWriter *vmaw)
1450+{
1451+ g_assert(vmaw != NULL);
1452+
1453+ int i;
1454+ int open_drives = 0;
1455+ for (i = 0; i <= 255; i++) {
1456+ if (vmaw->stream_info[i].size && !vmaw->stream_info[i].finished) {
1457+ open_drives++;
1458+ }
1459+ }
1460+
1461+ return open_drives;
1462+}
1463+
1464+/**
1465+ * all jobs should call this when there is no more data
1466+ * Returns: number of remaining stream (0 ==> finished)
1467+ */
1468+int coroutine_fn
1469+vma_writer_close_stream(VmaWriter *vmaw, uint8_t dev_id)
1470+{
1471+ g_assert(vmaw != NULL);
1472+
1473+ DPRINTF("vma_writer_set_status %d\n", dev_id);
1474+ if (!vmaw->stream_info[dev_id].size) {
1475+ vma_writer_set_error(vmaw, "vma_writer_close_stream: "
1476+ "no such stream %d", dev_id);
1477+ return -1;
1478+ }
1479+ if (vmaw->stream_info[dev_id].finished) {
1480+ vma_writer_set_error(vmaw, "vma_writer_close_stream: "
1481+ "stream already closed %d", dev_id);
1482+ return -1;
1483+ }
1484+
1485+ vmaw->stream_info[dev_id].finished = true;
1486+
1487+ int open_drives = vma_count_open_streams(vmaw);
1488+
1489+ if (open_drives <= 0) {
1490+ DPRINTF("vma_writer_set_status all drives completed\n");
1491+ qemu_co_mutex_lock(&vmaw->flush_lock);
1492+ int ret = vma_writer_flush(vmaw);
1493+ qemu_co_mutex_unlock(&vmaw->flush_lock);
1494+ if (ret < 0) {
1495+ vma_writer_set_error(vmaw, "vma_writer_close_stream: flush failed");
1496+ }
1497+ }
1498+
1499+ return open_drives;
1500+}
1501+
1502+int vma_writer_get_status(VmaWriter *vmaw, VmaStatus *status)
1503+{
1504+ int i;
1505+
1506+ g_assert(vmaw != NULL);
1507+
1508+ if (status) {
1509+ status->status = vmaw->status;
1510+ g_strlcpy(status->errmsg, vmaw->errmsg, sizeof(status->errmsg));
1511+ for (i = 0; i <= 255; i++) {
1512+ status->stream_info[i] = vmaw->stream_info[i];
1513+ }
1514+
1515+ uuid_unparse_lower(vmaw->uuid, status->uuid_str);
1516+ }
1517+
1518+ status->closed = vmaw->closed;
1519+
1520+ return vmaw->status;
1521+}
1522+
1523+static int vma_writer_get_buffer(VmaWriter *vmaw)
1524+{
1525+ int ret = 0;
1526+
1527+ qemu_co_mutex_lock(&vmaw->flush_lock);
1528+
1529+ /* wait until buffer is available */
1530+ while (vmaw->outbuf_count >= (VMA_BLOCKS_PER_EXTENT - 1)) {
1531+ ret = vma_writer_flush(vmaw);
1532+ if (ret < 0) {
1533+ vma_writer_set_error(vmaw, "vma_writer_get_buffer: flush failed");
1534+ break;
1535+ }
1536+ }
1537+
1538+ qemu_co_mutex_unlock(&vmaw->flush_lock);
1539+
1540+ return ret;
1541+}
1542+
1543+
1544+int64_t coroutine_fn
1545+vma_writer_write(VmaWriter *vmaw, uint8_t dev_id, int64_t cluster_num,
1546+ unsigned char *buf, size_t *zero_bytes)
1547+{
1548+ g_assert(vmaw != NULL);
1549+ g_assert(zero_bytes != NULL);
1550+
1551+ *zero_bytes = 0;
1552+
1553+ if (vmaw->status < 0) {
1554+ return vmaw->status;
1555+ }
1556+
1557+ if (!dev_id || !vmaw->stream_info[dev_id].size) {
1558+ vma_writer_set_error(vmaw, "vma_writer_write: "
1559+ "no such stream %d", dev_id);
1560+ return -1;
1561+ }
1562+
1563+ if (vmaw->stream_info[dev_id].finished) {
1564+ vma_writer_set_error(vmaw, "vma_writer_write: "
1565+ "stream already closed %d", dev_id);
1566+ return -1;
1567+ }
1568+
1569+
1570+ if (cluster_num >= (((uint64_t)1)<<32)) {
1571+ vma_writer_set_error(vmaw, "vma_writer_write: "
1572+ "cluster number out of range");
1573+ return -1;
1574+ }
1575+
1576+ if (dev_id == vmaw->vmstate_stream) {
1577+ if (cluster_num != vmaw->vmstate_clusters) {
1578+ vma_writer_set_error(vmaw, "vma_writer_write: "
1579+ "non sequential vmstate write");
1580+ }
1581+ vmaw->vmstate_clusters++;
1582+ } else if (cluster_num >= vmaw->stream_info[dev_id].cluster_count) {
1583+ vma_writer_set_error(vmaw, "vma_writer_write: cluster number too big");
1584+ return -1;
1585+ }
1586+
1587+ /* wait until buffer is available */
1588+ if (vma_writer_get_buffer(vmaw) < 0) {
1589+ vma_writer_set_error(vmaw, "vma_writer_write: "
1590+ "vma_writer_get_buffer failed");
1591+ return -1;
1592+ }
1593+
1594+ DPRINTF("VMA WRITE %d %zd\n", dev_id, cluster_num);
1595+
1596+ uint16_t mask = 0;
1597+
1598+ if (buf) {
1599+ int i;
1600+ int bit = 1;
1601+ for (i = 0; i < 16; i++) {
1602+ unsigned char *vmablock = buf + (i*VMA_BLOCK_SIZE);
1603+ if (!buffer_is_zero(vmablock, VMA_BLOCK_SIZE)) {
1604+ mask |= bit;
1605+ memcpy(vmaw->outbuf + vmaw->outbuf_pos, vmablock,
1606+ VMA_BLOCK_SIZE);
1607+ vmaw->outbuf_pos += VMA_BLOCK_SIZE;
1608+ } else {
1609+ DPRINTF("VMA WRITE %zd ZERO BLOCK %d\n", cluster_num, i);
1610+ vmaw->stream_info[dev_id].zero_bytes += VMA_BLOCK_SIZE;
1611+ *zero_bytes += VMA_BLOCK_SIZE;
1612+ }
1613+
1614+ bit = bit << 1;
1615+ }
1616+ } else {
1617+ DPRINTF("VMA WRITE %zd ZERO CLUSTER\n", cluster_num);
1618+ vmaw->stream_info[dev_id].zero_bytes += VMA_CLUSTER_SIZE;
1619+ *zero_bytes += VMA_CLUSTER_SIZE;
1620+ }
1621+
1622+ uint64_t block_info = ((uint64_t)mask) << (32+16);
1623+ block_info |= ((uint64_t)dev_id) << 32;
1624+ block_info |= (cluster_num & 0xffffffff);
1625+ vmaw->outbuf_block_info[vmaw->outbuf_count] = block_info;
1626+
1627+ DPRINTF("VMA WRITE MASK %zd %zx\n", cluster_num, block_info);
1628+
1629+ vmaw->outbuf_count++;
1630+
1631+ /** NOTE: We allways write whole clusters, but we correctly set
1632+ * transferred bytes. So transferred == size when when everything
1633+ * went OK.
1634+ */
1635+ size_t transferred = VMA_CLUSTER_SIZE;
1636+
1637+ if (dev_id != vmaw->vmstate_stream) {
1638+ uint64_t last = (cluster_num + 1) * VMA_CLUSTER_SIZE;
1639+ if (last > vmaw->stream_info[dev_id].size) {
1640+ uint64_t diff = last - vmaw->stream_info[dev_id].size;
1641+ if (diff >= VMA_CLUSTER_SIZE) {
1642+ vma_writer_set_error(vmaw, "vma_writer_write: "
1643+ "read after last cluster");
1644+ return -1;
1645+ }
1646+ transferred -= diff;
1647+ }
1648+ }
1649+
1650+ vmaw->stream_info[dev_id].transferred += transferred;
1651+
1652+ return transferred;
1653+}
1654+
1655+int vma_writer_close(VmaWriter *vmaw, Error **errp)
1656+{
1657+ g_assert(vmaw != NULL);
1658+
1659+ int i;
1660+
1661+ vma_queue_flush(vmaw);
1662+
1663+ /* this should not happen - just to be sure */
1664+ while (!qemu_co_queue_empty(&vmaw->wqueue)) {
1665+ DPRINTF("vma_writer_close wait\n");
24bb7da3 1666+ co_aio_sleep_ns(qemu_get_aio_context(), QEMU_CLOCK_REALTIME, 1000000);
b668dc80
DM		 1667+ }
1668+
1669+ if (vmaw->cmd) {
1670+ if (pclose(vmaw->cmd) < 0) {
1671+ vma_writer_set_error(vmaw, "vma_writer_close: "
1672+ "pclose failed - %s", g_strerror(errno));
1673+ }
1674+ } else {
1675+ if (close(vmaw->fd) < 0) {
1676+ vma_writer_set_error(vmaw, "vma_writer_close: "
1677+ "close failed - %s", g_strerror(errno));
1678+ }
1679+ }
1680+
1681+ for (i = 0; i <= 255; i++) {
1682+ VmaStreamInfo *si = &vmaw->stream_info[i];
1683+ if (si->size) {
1684+ if (!si->finished) {
1685+ vma_writer_set_error(vmaw, "vma_writer_close: "
1686+ "detected open stream '%s'", si->devname);
1687+ } else if ((si->transferred != si->size) &&
1688+ (i != vmaw->vmstate_stream)) {
1689+ vma_writer_set_error(vmaw, "vma_writer_close: "
1690+ "incomplete stream '%s' (%zd != %zd)",
1691+ si->devname, si->transferred, si->size);
1692+ }
1693+ }
1694+ }
1695+
1696+ for (i = 0; i <= 255; i++) {
1697+ vmaw->stream_info[i].finished = 1; /* mark as closed */
1698+ }
1699+
1700+ vmaw->closed = 1;
1701+
1702+ if (vmaw->status < 0 && *errp == NULL) {
1703+ error_setg(errp, "%s", vmaw->errmsg);
1704+ }
1705+
1706+ return vmaw->status;
1707+}
1708+
1709+void vma_writer_destroy(VmaWriter *vmaw)
1710+{
1711+ assert(vmaw);
1712+
1713+ int i;
1714+
1715+ for (i = 0; i <= 255; i++) {
1716+ if (vmaw->stream_info[i].devname) {
1717+ g_free(vmaw->stream_info[i].devname);
1718+ }
1719+ }
1720+
1721+ if (vmaw->md5csum) {
1722+ g_checksum_free(vmaw->md5csum);
1723+ }
1724+
1725+ for (i = 0; i < WRITE_BUFFERS; i++) {
1726+ free(vmaw->aiocbs[i]);
1727+ }
1728+
1729+ g_free(vmaw);
1730+}
1731Index: new/vma.c
1732===================================================================
1733--- /dev/null 1970-01-01 00:00:00.000000000 +0000
24bb7da3
DM		 1734+++ new/vma.c 2014-11-20 08:47:23.000000000 +0100
1735@@ -0,0 +1,582 @@
b668dc80
DM		 1736+/*
1737+ * VMA: Virtual Machine Archive
1738+ *
1739+ * Copyright (C) 2012-2013 Proxmox Server Solutions
1740+ *
1741+ * Authors:
1742+ * Dietmar Maurer (dietmar@proxmox.com)
1743+ *
1744+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
1745+ * See the COPYING file in the top-level directory.
1746+ *
1747+ */
1748+
1749+#include <stdio.h>
1750+#include <errno.h>
1751+#include <unistd.h>
1752+#include <stdio.h>
1753+#include <string.h>
1754+#include <sys/types.h>
1755+#include <sys/stat.h>
1756+#include <fcntl.h>
1757+#include <glib.h>
1758+
1759+#include "vma.h"
1760+#include "qemu-common.h"
1761+#include "qemu/error-report.h"
1762+#include "qemu/main-loop.h"
1763+
1764+static void help(void)
1765+{
1766+ const char *help_msg =
1767+ "usage: vma command [command options]\n"
1768+ "\n"
1769+ "vma list <filename>\n"
1770+ "vma create <filename> [-c config] <archive> pathname ...\n"
09a78de1 1771+ "vma extract <filename> [-r <fifo>] <targetdir>\n"
b668dc80
DM		 1772+ ;
1773+
1774+ printf("%s", help_msg);
1775+ exit(1);
1776+}
1777+
1778+static const char *extract_devname(const char *path, char **devname, int index)
1779+{
1780+ assert(path);
1781+
1782+ const char *sep = strchr(path, '=');
1783+
1784+ if (sep) {
1785+ *devname = g_strndup(path, sep - path);
1786+ path = sep + 1;
1787+ } else {
1788+ if (index >= 0) {
1789+ *devname = g_strdup_printf("disk%d", index);
1790+ } else {
1791+ *devname = NULL;
1792+ }
1793+ }
1794+
1795+ return path;
1796+}
1797+
1798+static void print_content(VmaReader *vmar)
1799+{
1800+ assert(vmar);
1801+
1802+ VmaHeader *head = vma_reader_get_header(vmar);
1803+
1804+ GList *l = vma_reader_get_config_data(vmar);
1805+ while (l && l->data) {
1806+ VmaConfigData *cdata = (VmaConfigData *)l->data;
1807+ l = g_list_next(l);
1808+ printf("CFG: size: %d name: %s\n", cdata->len, cdata->name);
1809+ }
1810+
1811+ int i;
1812+ VmaDeviceInfo *di;
1813+ for (i = 1; i < 255; i++) {
1814+ di = vma_reader_get_device_info(vmar, i);
1815+ if (di) {
1816+ if (strcmp(di->devname, "vmstate") == 0) {
1817+ printf("VMSTATE: dev_id=%d memory: %zd\n", i, di->size);
1818+ } else {
1819+ printf("DEV: dev_id=%d size: %zd devname: %s\n",
1820+ i, di->size, di->devname);
1821+ }
1822+ }
1823+ }
1824+ /* ctime is the last entry we print */
1825+ printf("CTIME: %s", ctime(&head->ctime));
1826+ fflush(stdout);
1827+}
1828+
1829+static int list_content(int argc, char **argv)
1830+{
1831+ int c, ret = 0;
1832+ const char *filename;
1833+
1834+ for (;;) {
1835+ c = getopt(argc, argv, "h");
1836+ if (c == -1) {
1837+ break;
1838+ }
1839+ switch (c) {
1840+ case '?':
1841+ case 'h':
1842+ help();
1843+ break;
1844+ default:
1845+ g_assert_not_reached();
1846+ }
1847+ }
1848+
1849+ /* Get the filename */
1850+ if ((optind + 1) != argc) {
1851+ help();
1852+ }
1853+ filename = argv[optind++];
1854+
1855+ Error *errp = NULL;
1856+ VmaReader *vmar = vma_reader_create(filename, &errp);
1857+
1858+ if (!vmar) {
1859+ g_error("%s", error_get_pretty(errp));
1860+ }
1861+
1862+ print_content(vmar);
1863+
1864+ vma_reader_destroy(vmar);
1865+
1866+ return ret;
1867+}
1868+
1869+typedef struct RestoreMap {
1870+ char *devname;
1871+ char *path;
1872+ bool write_zero;
1873+} RestoreMap;
1874+
1875+static int extract_content(int argc, char **argv)
1876+{
1877+ int c, ret = 0;
1878+ int verbose = 0;
1879+ const char *filename;
1880+ const char *dirname;
1881+ const char *readmap = NULL;
1882+
1883+ for (;;) {
1884+ c = getopt(argc, argv, "hvr:");
1885+ if (c == -1) {
1886+ break;
1887+ }
1888+ switch (c) {
1889+ case '?':
1890+ case 'h':
1891+ help();
1892+ break;
1893+ case 'r':
1894+ readmap = optarg;
1895+ break;
1896+ case 'v':
1897+ verbose = 1;
1898+ break;
1899+ default:
1900+ help();
1901+ }
1902+ }
1903+
1904+ /* Get the filename */
1905+ if ((optind + 2) != argc) {
1906+ help();
1907+ }
1908+ filename = argv[optind++];
1909+ dirname = argv[optind++];
1910+
1911+ Error *errp = NULL;
1912+ VmaReader *vmar = vma_reader_create(filename, &errp);
1913+
1914+ if (!vmar) {
1915+ g_error("%s", error_get_pretty(errp));
1916+ }
1917+
1918+ if (mkdir(dirname, 0777) < 0) {
1919+ g_error("unable to create target directory %s - %s",
1920+ dirname, g_strerror(errno));
1921+ }
1922+
1923+ GList *l = vma_reader_get_config_data(vmar);
1924+ while (l && l->data) {
1925+ VmaConfigData *cdata = (VmaConfigData *)l->data;
1926+ l = g_list_next(l);
1927+ char *cfgfn = g_strdup_printf("%s/%s", dirname, cdata->name);
1928+ GError *err = NULL;
1929+ if (!g_file_set_contents(cfgfn, (gchar *)cdata->data, cdata->len,
1930+ &err)) {
1931+ g_error("unable to write file: %s", err->message);
1932+ }
1933+ }
1934+
1935+ GHashTable *devmap = g_hash_table_new(g_str_hash, g_str_equal);
1936+
1937+ if (readmap) {
1938+ print_content(vmar);
1939+
1940+ FILE *map = fopen(readmap, "r");
1941+ if (!map) {
1942+ g_error("unable to open fifo %s - %s", readmap, g_strerror(errno));
1943+ }
1944+
1945+ while (1) {
1946+ char inbuf[8192];
1947+ char *line = fgets(inbuf, sizeof(inbuf), map);
1948+ if (!line || line[0] == '\0' || !strcmp(line, "done\n")) {
1949+ break;
1950+ }
1951+ int len = strlen(line);
1952+ if (line[len - 1] == '\n') {
1953+ line[len - 1] = '\0';
1954+ if (len == 1) {
1955+ break;
1956+ }
1957+ }
1958+
1959+ const char *path;
1960+ bool write_zero;
1961+ if (line[0] == '0' && line[1] == ':') {
1962+ path = inbuf + 2;
1963+ write_zero = false;
1964+ } else if (line[0] == '1' && line[1] == ':') {
1965+ path = inbuf + 2;
1966+ write_zero = true;
1967+ } else {
1968+ g_error("read map failed - parse error ('%s')", inbuf);
1969+ }
1970+
1971+ char *devname = NULL;
1972+ path = extract_devname(path, &devname, -1);
1973+ if (!devname) {
1974+ g_error("read map failed - no dev name specified ('%s')",
1975+ inbuf);
1976+ }
1977+
1978+ RestoreMap *map = g_new0(RestoreMap, 1);
1979+ map->devname = g_strdup(devname);
1980+ map->path = g_strdup(path);
1981+ map->write_zero = write_zero;
1982+
1983+ g_hash_table_insert(devmap, map->devname, map);
1984+
1985+ };
1986+ }
1987+
1988+ int i;
1989+ int vmstate_fd = -1;
1990+ guint8 vmstate_stream = 0;
1991+
1992+ for (i = 1; i < 255; i++) {
1993+ VmaDeviceInfo *di = vma_reader_get_device_info(vmar, i);
1994+ if (di && (strcmp(di->devname, "vmstate") == 0)) {
1995+ vmstate_stream = i;
1996+ char *statefn = g_strdup_printf("%s/vmstate.bin", dirname);
1997+ vmstate_fd = open(statefn, O_WRONLY|O_CREAT|O_EXCL, 0644);
1998+ if (vmstate_fd < 0) {
1999+ g_error("create vmstate file '%s' failed - %s", statefn,
2000+ g_strerror(errno));
2001+ }
2002+ g_free(statefn);
2003+ } else if (di) {
2004+ char *devfn = NULL;
2005+ int flags = BDRV_O_RDWR|BDRV_O_CACHE_WB;
2006+ bool write_zero = true;
2007+
2008+ if (readmap) {
2009+ RestoreMap *map;
2010+ map = (RestoreMap *)g_hash_table_lookup(devmap, di->devname);
2011+ if (map == NULL) {
2012+ g_error("no device name mapping for %s", di->devname);
2013+ }
2014+ devfn = map->path;
2015+ write_zero = map->write_zero;
2016+ } else {
2017+ devfn = g_strdup_printf("%s/tmp-disk-%s.raw",
2018+ dirname, di->devname);
2019+ printf("DEVINFO %s %zd\n", devfn, di->size);
2020+
2021+ bdrv_img_create(devfn, "raw", NULL, NULL, NULL, di->size,
2022+ flags, &errp, 0);
fbe817e2 2023+ if (errp) {
b668dc80
DM		 2024+ g_error("can't create file %s: %s", devfn,
2025+ error_get_pretty(errp));
2026+ }
2027+
2028+ /* Note: we created an empty file above, so there is no
2029+ * need to write zeroes (so we generate a sparse file)
2030+ */
2031+ write_zero = false;
2032+ }
2033+
24bb7da3 2034+ BlockDriverState *bs = bdrv_new();
fbe817e2 2035+ if (errp || bdrv_open(&bs, devfn, NULL, NULL, flags, NULL, &errp)) {
b668dc80
DM		 2036+ g_error("can't open file %s - %s", devfn,
2037+ error_get_pretty(errp));
2038+ }
2039+ if (vma_reader_register_bs(vmar, i, bs, write_zero, &errp) < 0) {
2040+ g_error("%s", error_get_pretty(errp));
2041+ }
2042+
2043+ if (!readmap) {
2044+ g_free(devfn);
2045+ }
2046+ }
2047+ }
2048+
2049+ if (vma_reader_restore(vmar, vmstate_fd, verbose, &errp) < 0) {
2050+ g_error("restore failed - %s", error_get_pretty(errp));
2051+ }
2052+
2053+ if (!readmap) {
2054+ for (i = 1; i < 255; i++) {
2055+ VmaDeviceInfo *di = vma_reader_get_device_info(vmar, i);
2056+ if (di && (i != vmstate_stream)) {
2057+ char *tmpfn = g_strdup_printf("%s/tmp-disk-%s.raw",
2058+ dirname, di->devname);
2059+ char *fn = g_strdup_printf("%s/disk-%s.raw",
2060+ dirname, di->devname);
2061+ if (rename(tmpfn, fn) != 0) {
2062+ g_error("rename %s to %s failed - %s",
2063+ tmpfn, fn, g_strerror(errno));
2064+ }
2065+ }
2066+ }
2067+ }
2068+
2069+ vma_reader_destroy(vmar);
2070+
2071+ bdrv_close_all();
2072+
2073+ return ret;
2074+}
2075+
2076+typedef struct BackupJob {
2077+ BlockDriverState *bs;
2078+ int64_t len;
2079+ VmaWriter *vmaw;
2080+ uint8_t dev_id;
2081+} BackupJob;
2082+
b668dc80
DM		 2083+#define BACKUP_SECTORS_PER_CLUSTER (VMA_CLUSTER_SIZE / BDRV_SECTOR_SIZE)
2084+
2085+static void coroutine_fn backup_run(void *opaque)
2086+{
2087+ BackupJob *job = (BackupJob *)opaque;
2088+ struct iovec iov;
2089+ QEMUIOVector qiov;
2090+
2091+ int64_t start, end;
2092+ int ret = 0;
2093+
2094+ unsigned char *buf = qemu_blockalign(job->bs, VMA_CLUSTER_SIZE);
2095+
2096+ start = 0;
2097+ end = DIV_ROUND_UP(job->len / BDRV_SECTOR_SIZE,
2098+ BACKUP_SECTORS_PER_CLUSTER);
2099+
2100+ for (; start < end; start++) {
2101+ iov.iov_base = buf;
2102+ iov.iov_len = VMA_CLUSTER_SIZE;
2103+ qemu_iovec_init_external(&qiov, &iov, 1);
2104+
2105+ ret = bdrv_co_readv(job->bs, start * BACKUP_SECTORS_PER_CLUSTER,
2106+ BACKUP_SECTORS_PER_CLUSTER, &qiov);
2107+ if (ret < 0) {
2108+ vma_writer_set_error(job->vmaw, "read error", -1);
2109+ goto out;
2110+ }
2111+
2112+ size_t zb = 0;
2113+ if (vma_writer_write(job->vmaw, job->dev_id, start, buf, &zb) < 0) {
2114+ vma_writer_set_error(job->vmaw, "backup_dump_cb vma_writer_write failed", -1);
2115+ goto out;
2116+ }
2117+ }
2118+
2119+
2120+out:
2121+ if (vma_writer_close_stream(job->vmaw, job->dev_id) <= 0) {
2122+ Error *err = NULL;
2123+ if (vma_writer_close(job->vmaw, &err) != 0) {
2124+ g_warning("vma_writer_close failed %s", error_get_pretty(err));
2125+ }
2126+ }
2127+}
2128+
2129+static int create_archive(int argc, char **argv)
2130+{
2131+ int i, c, res;
2132+ int verbose = 0;
2133+ const char *archivename;
2134+ GList *config_files = NULL;
2135+
2136+ for (;;) {
2137+ c = getopt(argc, argv, "hvc:");
2138+ if (c == -1) {
2139+ break;
2140+ }
2141+ switch (c) {
2142+ case '?':
2143+ case 'h':
2144+ help();
2145+ break;
2146+ case 'c':
2147+ config_files = g_list_append(config_files, optarg);
2148+ break;
2149+ case 'v':
2150+ verbose = 1;
2151+ break;
2152+ default:
2153+ g_assert_not_reached();
2154+ }
2155+ }
2156+
2157+
2158+ /* make sure we have archive name and at least one path */
2159+ if ((optind + 2) > argc) {
2160+ help();
2161+ }
2162+
2163+ archivename = argv[optind++];
2164+
2165+ uuid_t uuid;
2166+ uuid_generate(uuid);
2167+
2168+ Error *local_err = NULL;
2169+ VmaWriter *vmaw = vma_writer_create(archivename, uuid, &local_err);
2170+
2171+ if (vmaw == NULL) {
2172+ g_error("%s", error_get_pretty(local_err));
2173+ }
2174+
2175+ GList *l = config_files;
2176+ while (l && l->data) {
2177+ char *name = l->data;
2178+ char *cdata = NULL;
2179+ gsize clen = 0;
2180+ GError *err = NULL;
2181+ if (!g_file_get_contents(name, &cdata, &clen, &err)) {
2182+ unlink(archivename);
2183+ g_error("Unable to read file: %s", err->message);
2184+ }
2185+
2186+ if (vma_writer_add_config(vmaw, name, cdata, clen) != 0) {
2187+ unlink(archivename);
2188+ g_error("Unable to append config data %s (len = %zd)",
2189+ name, clen);
2190+ }
2191+ l = g_list_next(l);
2192+ }
2193+
2194+ int ind = 0;
2195+ while (optind < argc) {
2196+ const char *path = argv[optind++];
2197+ char *devname = NULL;
2198+ path = extract_devname(path, &devname, ind++);
2199+
2200+ BlockDriver *drv = NULL;
b668dc80 2201+ Error *errp = NULL;
24bb7da3
DM		 2202+ BlockDriverState *bs = bdrv_new();
2203+
88d9366c 2204+ res = bdrv_open(&bs, path, NULL, NULL, BDRV_O_CACHE_WB , drv, &errp);
b668dc80
DM		 2205+ if (res < 0) {
2206+ unlink(archivename);
2207+ g_error("bdrv_open '%s' failed - %s", path, error_get_pretty(errp));
2208+ }
2209+ int64_t size = bdrv_getlength(bs);
2210+ int dev_id = vma_writer_register_stream(vmaw, devname, size);
2211+ if (dev_id <= 0) {
2212+ unlink(archivename);
2213+ g_error("vma_writer_register_stream '%s' failed", devname);
2214+ }
2215+
2216+ BackupJob *job = g_new0(BackupJob, 1);
2217+ job->len = size;
2218+ job->bs = bs;
2219+ job->vmaw = vmaw;
2220+ job->dev_id = dev_id;
2221+
2222+ Coroutine *co = qemu_coroutine_create(backup_run);
2223+ qemu_coroutine_enter(co, job);
b668dc80
DM		 2224+ }
2225+
2226+ VmaStatus vmastat;
2227+ int percent = 0;
2228+ int last_percent = -1;
2229+
2230+ while (1) {
2231+ main_loop_wait(false);
2232+ vma_writer_get_status(vmaw, &vmastat);
2233+
2234+ if (verbose) {
2235+
2236+ uint64_t total = 0;
2237+ uint64_t transferred = 0;
2238+ uint64_t zero_bytes = 0;
2239+
2240+ int i;
2241+ for (i = 0; i < 256; i++) {
2242+ if (vmastat.stream_info[i].size) {
2243+ total += vmastat.stream_info[i].size;
2244+ transferred += vmastat.stream_info[i].transferred;
2245+ zero_bytes += vmastat.stream_info[i].zero_bytes;
2246+ }
2247+ }
2248+ percent = (transferred*100)/total;
2249+ if (percent != last_percent) {
2250+ fprintf(stderr, "progress %d%% %zd/%zd %zd\n", percent,
2251+ transferred, total, zero_bytes);
2252+ fflush(stderr);
2253+
2254+ last_percent = percent;
2255+ }
2256+ }
2257+
2258+ if (vmastat.closed) {
2259+ break;
2260+ }
2261+ }
2262+
2263+ bdrv_drain_all();
2264+
2265+ vma_writer_get_status(vmaw, &vmastat);
2266+
2267+ if (verbose) {
2268+ for (i = 0; i < 256; i++) {
2269+ VmaStreamInfo *si = &vmastat.stream_info[i];
2270+ if (si->size) {
2271+ fprintf(stderr, "image %s: size=%zd zeros=%zd saved=%zd\n",
2272+ si->devname, si->size, si->zero_bytes,
2273+ si->size - si->zero_bytes);
2274+ }
2275+ }
2276+ }
2277+
2278+ if (vmastat.status < 0) {
2279+ unlink(archivename);
2280+ g_error("creating vma archive failed");
2281+ }
2282+
2283+ return 0;
2284+}
2285+
2286+int main(int argc, char **argv)
2287+{
2288+ const char *cmdname;
24bb7da3 2289+ Error *main_loop_err = NULL;
b668dc80
DM		 2290+
2291+ error_set_progname(argv[0]);
2292+
24bb7da3
DM		 2293+ if (qemu_init_main_loop(&main_loop_err)) {
2294+ g_error("%s", error_get_pretty(main_loop_err));
2295+ }
b668dc80
DM		 2296+
2297+ bdrv_init();
2298+
2299+ if (argc < 2) {
2300+ help();
2301+ }
2302+
2303+ cmdname = argv[1];
2304+ argc--; argv++;
2305+
2306+
2307+ if (!strcmp(cmdname, "list")) {
2308+ return list_content(argc, argv);
2309+ } else if (!strcmp(cmdname, "create")) {
2310+ return create_archive(argc, argv);
2311+ } else if (!strcmp(cmdname, "extract")) {
2312+ return extract_content(argc, argv);
2313+ }
2314+
2315+ help();
2316+ return 0;
2317+}
2318Index: new/vma.h
2319===================================================================
2320--- /dev/null 1970-01-01 00:00:00.000000000 +0000
24bb7da3 2321+++ new/vma.h 2014-11-20 08:15:12.000000000 +0100
ea04a0b3 2322@@ -0,0 +1,146 @@
b668dc80
DM		 2323+/*
2324+ * VMA: Virtual Machine Archive
2325+ *
2326+ * Copyright (C) Proxmox Server Solutions
2327+ *
2328+ * Authors:
2329+ * Dietmar Maurer (dietmar@proxmox.com)
2330+ *
2331+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
2332+ * See the COPYING file in the top-level directory.
2333+ *
2334+ */
2335+
2336+#ifndef BACKUP_VMA_H
2337+#define BACKUP_VMA_H
2338+
2339+#include <uuid/uuid.h>
2340+#include "qapi/error.h"
2341+#include "block/block.h"
2342+
2343+#define VMA_BLOCK_BITS 12
2344+#define VMA_BLOCK_SIZE (1<<VMA_BLOCK_BITS)
2345+#define VMA_CLUSTER_BITS (VMA_BLOCK_BITS+4)
2346+#define VMA_CLUSTER_SIZE (1<<VMA_CLUSTER_BITS)
2347+
2348+#if VMA_CLUSTER_SIZE != 65536
2349+#error unexpected cluster size
2350+#endif
2351+
2352+#define VMA_EXTENT_HEADER_SIZE 512
2353+#define VMA_BLOCKS_PER_EXTENT 59
2354+#define VMA_MAX_CONFIGS 256
2355+
2356+#define VMA_MAX_EXTENT_SIZE \
2357+ (VMA_EXTENT_HEADER_SIZE+VMA_CLUSTER_SIZE*VMA_BLOCKS_PER_EXTENT)
2358+#if VMA_MAX_EXTENT_SIZE != 3867136
2359+#error unexpected VMA_EXTENT_SIZE
2360+#endif
2361+
2362+/* File Format Definitions */
2363+
2364+#define VMA_MAGIC (GUINT32_TO_BE(('V'<<24)|('M'<<16)|('A'<<8)|0x00))
2365+#define VMA_EXTENT_MAGIC (GUINT32_TO_BE(('V'<<24)|('M'<<16)|('A'<<8)|'E'))
2366+
2367+typedef struct VmaDeviceInfoHeader {
2368+ uint32_t devname_ptr; /* offset into blob_buffer table */
2369+ uint32_t reserved0;
2370+ uint64_t size; /* device size in bytes */
2371+ uint64_t reserved1;
2372+ uint64_t reserved2;
2373+} VmaDeviceInfoHeader;
2374+
2375+typedef struct VmaHeader {
2376+ uint32_t magic;
2377+ uint32_t version;
2378+ unsigned char uuid[16];
2379+ int64_t ctime;
2380+ unsigned char md5sum[16];
2381+
2382+ uint32_t blob_buffer_offset;
2383+ uint32_t blob_buffer_size;
2384+ uint32_t header_size;
2385+
2386+ unsigned char reserved[1984];
2387+
2388+ uint32_t config_names[VMA_MAX_CONFIGS]; /* offset into blob_buffer table */
2389+ uint32_t config_data[VMA_MAX_CONFIGS]; /* offset into blob_buffer table */
2390+
ea04a0b3
DM		 2391+ uint32_t reserved1;
2392+
b668dc80
DM		 2393+ VmaDeviceInfoHeader dev_info[256];
2394+} VmaHeader;
2395+
2396+typedef struct VmaExtentHeader {
2397+ uint32_t magic;
2398+ uint16_t reserved1;
2399+ uint16_t block_count;
2400+ unsigned char uuid[16];
2401+ unsigned char md5sum[16];
2402+ uint64_t blockinfo[VMA_BLOCKS_PER_EXTENT];
2403+} VmaExtentHeader;
2404+
2405+/* functions/definitions to read/write vma files */
2406+
2407+typedef struct VmaReader VmaReader;
2408+
2409+typedef struct VmaWriter VmaWriter;
2410+
2411+typedef struct VmaConfigData {
2412+ const char *name;
2413+ const void *data;
2414+ uint32_t len;
2415+} VmaConfigData;
2416+
2417+typedef struct VmaStreamInfo {
2418+ uint64_t size;
2419+ uint64_t cluster_count;
2420+ uint64_t transferred;
2421+ uint64_t zero_bytes;
2422+ int finished;
2423+ char *devname;
2424+} VmaStreamInfo;
2425+
2426+typedef struct VmaStatus {
2427+ int status;
2428+ bool closed;
2429+ char errmsg[8192];
2430+ char uuid_str[37];
2431+ VmaStreamInfo stream_info[256];
2432+} VmaStatus;
2433+
2434+typedef struct VmaDeviceInfo {
2435+ uint64_t size; /* device size in bytes */
2436+ const char *devname;
2437+} VmaDeviceInfo;
2438+
2439+VmaWriter *vma_writer_create(const char *filename, uuid_t uuid, Error **errp);
2440+int vma_writer_close(VmaWriter *vmaw, Error **errp);
2441+void vma_writer_destroy(VmaWriter *vmaw);
2442+int vma_writer_add_config(VmaWriter *vmaw, const char *name, gpointer data,
2443+ size_t len);
2444+int vma_writer_register_stream(VmaWriter *vmaw, const char *devname,
2445+ size_t size);
2446+
2447+int64_t coroutine_fn vma_writer_write(VmaWriter *vmaw, uint8_t dev_id,
2448+ int64_t cluster_num, unsigned char *buf,
2449+ size_t *zero_bytes);
2450+
2451+int coroutine_fn vma_writer_close_stream(VmaWriter *vmaw, uint8_t dev_id);
2452+
2453+int vma_writer_get_status(VmaWriter *vmaw, VmaStatus *status);
2454+void vma_writer_set_error(VmaWriter *vmaw, const char *fmt, ...);
2455+
2456+
2457+VmaReader *vma_reader_create(const char *filename, Error **errp);
2458+void vma_reader_destroy(VmaReader *vmar);
2459+VmaHeader *vma_reader_get_header(VmaReader *vmar);
2460+GList *vma_reader_get_config_data(VmaReader *vmar);
2461+VmaDeviceInfo *vma_reader_get_device_info(VmaReader *vmar, guint8 dev_id);
2462+int vma_reader_register_bs(VmaReader *vmar, guint8 dev_id,
2463+ BlockDriverState *bs, bool write_zeroes,
2464+ Error **errp);
2465+int vma_reader_restore(VmaReader *vmar, int vmstate_fd, bool verbose,
2466+ Error **errp);
2467+
2468+#endif /* BACKUP_VMA_H */
2014bc7f
DM		 2469Index: new/Makefile.objs
2470===================================================================
24bb7da3
DM		 2471--- new.orig/Makefile.objs 2014-11-20 08:12:54.000000000 +0100
2472+++ new/Makefile.objs 2014-11-20 08:15:12.000000000 +0100
2473@@ -13,6 +13,7 @@
2474 block-obj-$(CONFIG_WIN32) += aio-win32.o
2014bc7f 2475 block-obj-y += block/
2014bc7f
DM		 2476 block-obj-y += qemu-io-cmds.o
2477+block-obj-y += vma-writer.o
2478
2479 block-obj-y += qemu-coroutine.o qemu-coroutine-lock.o qemu-coroutine-io.o
2480 block-obj-y += qemu-coroutine-sleep.o
old QEMU for PVE