-From 69a3f26bdbe445ce01f096b9d5e1aaf45ad98b81 Mon Sep 17 00:00:00 2001
+From 5476ae43806488e74cd293bbaa17f130aa53d402 Mon Sep 17 00:00:00 2001
From: Dietmar Maurer <dietmar@proxmox.com>
Date: Tue, 13 Nov 2012 11:11:38 +0100
-Subject: [PATCH v3 4/7] introduce new vma archive format
+Subject: [PATCH v4 4/6] introduce new vma archive format
This is a very simple archive format, see docs/specs/vma_spec.txt
---
Makefile | 3 +-
Makefile.objs | 2 +-
+ backup.h | 1 +
blockdev.c | 6 +-
docs/specs/vma_spec.txt | 24 ++
- vma-reader.c | 801 ++++++++++++++++++++++++++++++++++++++++
- vma-writer.c | 931 +++++++++++++++++++++++++++++++++++++++++++++++
- vma.c | 561 ++++++++++++++++++++++++++++
- vma.h | 146 ++++++++
- 8 files changed, 2470 insertions(+), 4 deletions(-)
+ vma-reader.c | 799 ++++++++++++++++++++++++++++++++++++++++
+ vma-writer.c | 932 +++++++++++++++++++++++++++++++++++++++++++++++
+ vma.c | 559 ++++++++++++++++++++++++++++
+ vma.h | 145 ++++++++
+ 9 files changed, 2467 insertions(+), 4 deletions(-)
create mode 100644 docs/specs/vma_spec.txt
create mode 100644 vma-reader.c
create mode 100644 vma-writer.c
create mode 100644 vma.h
diff --git a/Makefile b/Makefile
-index 9ecbcbb..30a9268 100644
+index 0d9099a..16f1c25 100644
--- a/Makefile
+++ b/Makefile
-@@ -100,7 +100,7 @@ defconfig:
-
- -include config-all-devices.mak
+@@ -115,7 +115,7 @@ ifeq ($(CONFIG_SMARTCARD_NSS),y)
+ include $(SRC_PATH)/libcacard/Makefile
+ endif
-all: $(DOCS) $(TOOLS) $(HELPERS-y) recurse-all
+all: $(DOCS) $(TOOLS) vma$(EXESUF) $(HELPERS-y) recurse-all
config-host.h: config-host.h-timestamp
config-host.h-timestamp: config-host.mak
-@@ -194,6 +194,7 @@ tools-obj-$(CONFIG_POSIX) += compatfd.o
- qemu-img$(EXESUF): qemu-img.o $(tools-obj-y) $(block-obj-y) libqemustub.a
- qemu-nbd$(EXESUF): qemu-nbd.o $(tools-obj-y) $(block-obj-y) libqemustub.a
- qemu-io$(EXESUF): qemu-io.o cmd.o $(tools-obj-y) $(block-obj-y) libqemustub.a
-+vma$(EXESUF): vma.o vma-writer.o vma-reader.o $(tools-obj-y) $(block-obj-y) libqemustub.a
+@@ -167,6 +167,7 @@ qemu-img.o: qemu-img-cmds.h
+ qemu-img$(EXESUF): qemu-img.o $(block-obj-y) libqemuutil.a libqemustub.a
+ qemu-nbd$(EXESUF): qemu-nbd.o $(block-obj-y) libqemuutil.a libqemustub.a
+ qemu-io$(EXESUF): qemu-io.o cmd.o $(block-obj-y) libqemuutil.a libqemustub.a
++vma$(EXESUF): vma.o vma-writer.o vma-reader.o $(block-obj-y) libqemuutil.a libqemustub.a
qemu-bridge-helper$(EXESUF): qemu-bridge-helper.o
diff --git a/Makefile.objs b/Makefile.objs
-index cb46be5..b5732e2 100644
+index df64f70..91f133b 100644
--- a/Makefile.objs
+++ b/Makefile.objs
-@@ -48,7 +48,7 @@ coroutine-obj-$(CONFIG_WIN32) += coroutine-win32.o
- block-obj-y = iov.o cache-utils.o qemu-option.o module.o async.o
- block-obj-y += nbd.o block.o blockjob.o aes.o qemu-config.o
- block-obj-y += thread-pool.o qemu-progress.o qemu-sockets.o uri.o notify.o
+@@ -13,7 +13,7 @@ block-obj-$(CONFIG_POSIX) += aio-posix.o
+ block-obj-$(CONFIG_WIN32) += aio-win32.o
+ block-obj-y += block/
+ block-obj-y += qapi-types.o qapi-visit.o
-block-obj-y += backup.o
+block-obj-y += vma-writer.o backup.o
- block-obj-y += $(coroutine-obj-y) $(qobject-obj-y) $(version-obj-y)
- block-obj-$(CONFIG_POSIX) += event_notifier-posix.o aio-posix.o
- block-obj-$(CONFIG_WIN32) += event_notifier-win32.o aio-win32.o
+
+ block-obj-y += qemu-coroutine.o qemu-coroutine-lock.o qemu-coroutine-io.o
+ block-obj-y += qemu-coroutine-sleep.o
+diff --git a/backup.h b/backup.h
+index c8ba153..406f011 100644
+--- a/backup.h
++++ b/backup.h
+@@ -15,6 +15,7 @@
+ #define QEMU_BACKUP_H
+
+ #include <uuid/uuid.h>
++#include "block/block.h"
+
+ #define BACKUP_CLUSTER_BITS 16
+ #define BACKUP_CLUSTER_SIZE (1<<BACKUP_CLUSTER_BITS)
diff --git a/blockdev.c b/blockdev.c
-index 37edb75..0632c57 100644
+index c340fde..1cfc780 100644
--- a/blockdev.c
+++ b/blockdev.c
@@ -21,6 +21,7 @@
#include "trace.h"
- #include "arch_init.h"
+ #include "sysemu/arch_init.h"
#include "backup.h"
+#include "vma.h"
static QTAILQ_HEAD(drivelist, DriveInfo) drives = QTAILQ_HEAD_INITIALIZER(drives);
-@@ -1507,10 +1508,11 @@ char *qmp_backup(const char *backupfile, bool has_format, BackupFormat format,
+@@ -1530,10 +1531,11 @@ char *qmp_backup(const char *backup_file, bool has_format, BackupFormat format,
/* Todo: try to auto-detect format based on file name */
format = has_format ? format : BACKUP_FORMAT_VMA;
+
diff --git a/vma-reader.c b/vma-reader.c
new file mode 100644
-index 0000000..2217a94
+index 0000000..7e81847
--- /dev/null
+++ b/vma-reader.c
-@@ -0,0 +1,801 @@
+@@ -0,0 +1,799 @@
+/*
+ * VMA: Virtual Machine Archive
+ *
+#include <uuid/uuid.h>
+
+#include "qemu-common.h"
-+#include "qemu_socket.h"
-+#include "qemu-coroutine.h"
-+#include "qemu-aio.h"
++#include "qemu/timer.h"
+#include "qemu/ratelimit.h"
+#include "vma.h"
-+#include "block.h"
++#include "block/block.h"
+
+#define BITS_PER_LONG (sizeof(unsigned long) * 8)
+
+
diff --git a/vma-writer.c b/vma-writer.c
new file mode 100644
-index 0000000..688af4b
+index 0000000..761d7ca
--- /dev/null
+++ b/vma-writer.c
-@@ -0,0 +1,931 @@
+@@ -0,0 +1,932 @@
+/*
+ * VMA: Virtual Machine Archive
+ *
+#include <uuid/uuid.h>
+
+#include "qemu-common.h"
-+#include "qemu_socket.h"
-+#include "qemu-coroutine.h"
-+#include "qemu-aio.h"
-+#include "qemu/ratelimit.h"
+#include "vma.h"
-+#include "block.h"
-+#include "monitor.h"
++#include "block/block.h"
++#include "monitor/monitor.h"
+
+#define DEBUG_VMA 0
+
+
+typedef struct VmaAIOCB VmaAIOCB;
+struct VmaAIOCB {
-+ VmaWriter *vmaw;
+ unsigned char buffer[VMA_MAX_EXTENT_SIZE];
++ VmaWriter *vmaw;
+ size_t bytes;
+ Coroutine *co;
+};
+ int outbuf_count; /* in VMA_BLOCKS */
+ uint64_t outbuf_block_info[VMA_BLOCKS_PER_EXTENT];
+
-+ VmaAIOCB aiocbs[WRITE_BUFFERS];
++ VmaAIOCB *aiocbs[WRITE_BUFFERS];
+ CoQueue wqueue;
+
+ GChecksum *md5csum;
+ CoMutex writer_lock;
+ CoMutex flush_lock;
+ Coroutine *co_writer;
-+ RateLimit limit;
+
+ /* drive informations */
+ VmaStreamInfo stream_info[256];
+{
+ VmaWriter *vmaw = opaque;
+
-+ qemu_aio_set_fd_handler(vmaw->fd, NULL, NULL, NULL, NULL);
-+
+ DPRINTF("vma_co_continue_write\n");
+ qemu_coroutine_enter(vmaw->co_writer, NULL);
+}
+
++static int vma_co_write_finished(void *opaque)
++{
++ VmaWriter *vmaw = opaque;
++
++ return (vmaw->co_writer != 0);
++}
++
+static ssize_t coroutine_fn
+vma_co_write(VmaWriter *vmaw, const void *buf, size_t bytes)
+{
+
+ DPRINTF("vma_co_write enter %zd\n", bytes);
+
++ assert(vmaw->co_writer == NULL);
++
++ vmaw->co_writer = qemu_coroutine_self();
++
++ qemu_aio_set_fd_handler(vmaw->fd, NULL, vma_co_continue_write,
++ vma_co_write_finished, vmaw);
++
++ DPRINTF("vma_co_write wait until writable\n");
++ qemu_coroutine_yield();
++ DPRINTF("vma_co_write starting %zd\n", bytes);
++
+ while (done < bytes) {
+ ret = write(vmaw->fd, buf + done, bytes - done);
+ if (ret > 0) {
+ } else if (ret < 0) {
+ if (errno == EAGAIN || errno == EWOULDBLOCK) {
+ DPRINTF("vma_co_write yield %zd\n", done);
-+
-+ vmaw->co_writer = qemu_coroutine_self();
-+ qemu_aio_set_fd_handler(vmaw->fd, NULL, vma_co_continue_write,
-+ NULL, vmaw);
-+
+ qemu_coroutine_yield();
+ DPRINTF("vma_co_write restart %zd\n", done);
+ } else {
+ }
+ }
+
++ qemu_aio_set_fd_handler(vmaw->fd, NULL, NULL, NULL, NULL);
++
++ vmaw->co_writer = NULL;
++
+ qemu_co_mutex_unlock(&vmaw->writer_lock);
+
+ DPRINTF("vma_co_write leave %zd\n", done);
+ int i;
+ VmaAIOCB *cb = NULL;
+ for (i = 0; i < WRITE_BUFFERS; i++) {
-+ if (vmaw->aiocbs[i].bytes) {
-+ cb = &vmaw->aiocbs[i];
++ if (vmaw->aiocbs[i]->bytes) {
++ cb = vmaw->aiocbs[i];
+ DPRINTF("FOUND USED AIO BUFFER %d %zd\n", i,
-+ vmaw->aiocbs[i].bytes);
++ vmaw->aiocbs[i]->bytes);
+ break;
+ }
+ }
+ while (!cb) {
+ int i;
+ for (i = 0; i < WRITE_BUFFERS; i++) {
-+ if (!vmaw->aiocbs[i].bytes) {
-+ cb = &vmaw->aiocbs[i];
++ if (!vmaw->aiocbs[i]->bytes) {
++ cb = vmaw->aiocbs[i];
+ break;
+ }
+ }
+ return bytes;
+}
+
-+VmaWriter *vma_writer_create(const char *filename, uuid_t uuid, int64_t speed,
-+ Error **errp)
++VmaWriter *vma_writer_create(const char *filename, uuid_t uuid, Error **errp)
+{
+ const char *p;
+
+ goto err;
+ }
+ vmaw->fd = fileno(vmaw->cmd);
-+ socket_set_nonblock(vmaw->fd);
++
++ /* try to use O_NONBLOCK and O_DIRECT */
++ fcntl(vmaw->fd, F_SETFL, fcntl(vmaw->fd, F_GETFL)|O_NONBLOCK);
++ fcntl(vmaw->fd, F_SETFL, fcntl(vmaw->fd, F_GETFL)|O_DIRECT);
+
+ } else {
+ struct stat st;
+ const char *tmp_id_str;
+
+ if ((stat(filename, &st) == 0) && S_ISFIFO(st.st_mode)) {
-+ oflags = O_NONBLOCK|O_WRONLY;
++ oflags = O_NONBLOCK|O_DIRECT|O_WRONLY;
+ vmaw->fd = qemu_open(filename, oflags, 0644);
+ } else if (strstart(filename, "/dev/fdset/", &tmp_id_str)) {
-+ oflags = O_NONBLOCK|O_WRONLY;
++ oflags = O_NONBLOCK|O_DIRECT|O_WRONLY;
+ vmaw->fd = qemu_open(filename, oflags, 0644);
+ } else if (strstart(filename, "/dev/fdname/", &tmp_id_str)) {
+ vmaw->fd = monitor_get_fd(cur_mon, tmp_id_str, errp);
+ if (vmaw->fd < 0) {
+ goto err;
+ }
++ /* try to use O_NONBLOCK and O_DIRECT */
++ fcntl(vmaw->fd, F_SETFL, fcntl(vmaw->fd, F_GETFL)|O_NONBLOCK);
++ fcntl(vmaw->fd, F_SETFL, fcntl(vmaw->fd, F_GETFL)|O_DIRECT);
+ } else {
-+ oflags = O_NONBLOCK|O_WRONLY|O_CREAT|O_EXCL;
++ oflags = O_NONBLOCK|O_DIRECT|O_WRONLY|O_CREAT|O_EXCL;
+ vmaw->fd = qemu_open(filename, oflags, 0644);
+ }
+
+ }
+ }
+
++ /* we use O_DIRECT, so we need to align IO buffers */
++ int i;
++ for (i = 0; i < WRITE_BUFFERS; i++) {
++ vmaw->aiocbs[i] = qemu_memalign(512, sizeof(VmaAIOCB));
++ memset(vmaw->aiocbs[i], 0, sizeof(VmaAIOCB));
++ }
++
+ vmaw->outbuf_count = 0;
+ vmaw->outbuf_pos = VMA_EXTENT_HEADER_SIZE;
+
+
+ uuid_copy(vmaw->uuid, uuid);
+
-+ if (speed <= 0) {
-+ speed = 10*1024*1024*1024LLU; /* default 10GB/s */
-+ }
-+
-+ ratelimit_set_speed(&vmaw->limit, speed, 100000000ULL /* 0.1 sec */);
-+
+ return vmaw;
+
+err:
+ return vmaw->status;
+}
+
-+static int vma_writer_get_buffer(VmaWriter *vmaw, size_t bytes)
++static int vma_writer_get_buffer(VmaWriter *vmaw)
+{
+ int ret = 0;
+
+ qemu_co_mutex_lock(&vmaw->flush_lock);
+
-+ /* rate limit */
-+ uint64_t delay_ns = ratelimit_calculate_delay(&vmaw->limit, bytes);
-+ if (delay_ns) {
-+ DPRINTF("DELAY %zd\n", delay_ns);
-+ co_sleep_ns(rt_clock, delay_ns);
-+ }
-+
+ /* wait until buffer is available */
+ while (vmaw->outbuf_count >= (VMA_BLOCKS_PER_EXTENT - 1)) {
+ ret = vma_writer_flush(vmaw);
+ return -1;
+ }
+
-+ /* detect block containing zeroes */
-+ int i;
-+ int bit = 1;
-+ uint16_t mask = 0;
-+ size_t real_size = 0;
-+ for (i = 0; i < 16; i++) {
-+ unsigned char *vmablock = buf + (i*VMA_BLOCK_SIZE);
-+ if (!buffer_is_zero(vmablock, VMA_BLOCK_SIZE)) {
-+ mask |= bit;
-+ real_size += VMA_BLOCK_SIZE;
-+ }
-+ bit = bit << 1;
-+ }
-+
+ /* wait until buffer is available */
-+ if (vma_writer_get_buffer(vmaw, real_size) < 0) {
++ if (vma_writer_get_buffer(vmaw) < 0) {
+ vma_writer_set_error(vmaw, "vma_writer_write: "
+ "vma_writer_get_buffer failed");
+ return -1;
+
+ DPRINTF("VMA WRITE %d %zd\n", dev_id, cluster_num);
+
-+ bit = 1;
++ int i;
++ int bit = 1;
++ uint16_t mask = 0;
+ for (i = 0; i < 16; i++) {
+ unsigned char *vmablock = buf + (i*VMA_BLOCK_SIZE);
-+ if (mask & bit) {
++ if (!buffer_is_zero(vmablock, VMA_BLOCK_SIZE)) {
++ mask |= bit;
+ memcpy(vmaw->outbuf + vmaw->outbuf_pos, vmablock, VMA_BLOCK_SIZE);
+ vmaw->outbuf_pos += VMA_BLOCK_SIZE;
+ } else {
+ g_checksum_free(vmaw->md5csum);
+ }
+
++ for (i = 0; i < WRITE_BUFFERS; i++) {
++ free(vmaw->aiocbs[i]);
++ }
++
+ g_free(vmaw);
+}
+
+ return vma_writer_add_config(vmaw, name, data, data_len);
+}
+
-+static void *vma_open_cb(const char *filename, uuid_t uuid, int64_t speed,
-+ Error **errp)
++static void *vma_open_cb(const char *filename, uuid_t uuid, Error **errp)
+{
-+ return vma_writer_create(filename, uuid, speed, errp);
++ return vma_writer_create(filename, uuid, errp);
+}
+
+const BackupDriver backup_vma_driver = {
+
diff --git a/vma.c b/vma.c
new file mode 100644
-index 0000000..9b47b92
+index 0000000..b2e276c
--- /dev/null
+++ b/vma.c
-@@ -0,0 +1,561 @@
+@@ -0,0 +1,559 @@
+/*
+ * VMA: Virtual Machine Archive
+ *
+#include <glib.h>
+
+#include "qemu-common.h"
-+#include "qemu-option.h"
-+#include "qemu-error.h"
-+#include "osdep.h"
-+#include "sysemu.h"
-+#include "block_int.h"
-+#include <stdio.h>
++#include "qemu/error-report.h"
+#include "vma.h"
++#include "block/block.h"
+
+static void help(void)
+{
+ dirname, di->devname);
+ printf("DEVINFO %s %zd\n", devfn, di->size);
+
-+ if (bdrv_img_create(devfn, "raw", NULL, NULL, NULL,
-+ di->size, flags)) {
-+ g_error("can't create file %s", devfn);
++ bdrv_img_create(devfn, "raw", NULL, NULL, NULL, di->size,
++ flags, &errp);
++ if (error_is_set(&errp)) {
++ g_error("can't create file %s: %s", devfn,
++ error_get_pretty(errp));
+ }
+
+ /* Note: we created an empty file above, so there is no
+ uuid_generate(uuid);
+
+ Error *local_err = NULL;
-+ VmaWriter *vmaw = vma_writer_create(archivename, uuid, 0, &local_err);
++ VmaWriter *vmaw = vma_writer_create(archivename, uuid, &local_err);
+
+ if (vmaw == NULL) {
+ g_error("%s", error_get_pretty(local_err));
+ bcb->vmaw = vmaw;
+ bcb->dev_id = dev_id;
+
-+ if (backup_job_create(bs, backup_dump_cb, backup_complete_cb,
-+ bcb) < 0) {
++ if (backup_job_create(bs, backup_dump_cb, backup_complete_cb,
++ bcb, 0) < 0) {
+ unlink(archivename);
+ g_error("backup_job_start failed");
+ } else {
-+ backup_job_start(bs);
++ backup_job_start(bs, false);
+ }
+ }
+
+}
diff --git a/vma.h b/vma.h
new file mode 100644
-index 0000000..689e639
+index 0000000..76d0dc8
--- /dev/null
+++ b/vma.h
-@@ -0,0 +1,146 @@
+@@ -0,0 +1,145 @@
+/*
+ * VMA: Virtual Machine Archive
+ *
+
+extern const BackupDriver backup_vma_driver;
+
-+VmaWriter *vma_writer_create(const char *filename, uuid_t uuid, int64_t speed,
-+ Error **errp);
++VmaWriter *vma_writer_create(const char *filename, uuid_t uuid, Error **errp);
+int vma_writer_close(VmaWriter *vmaw, Error **errp);
+void vma_writer_destroy(VmaWriter *vmaw);
+int vma_writer_add_config(VmaWriter *vmaw, const char *name, gpointer data,
projects / pve-qemu-kvm.git / blobdiff