[pve-qemu-kvm.git] / debian / patches / backup-add-vma-verify-command.patch

From acdcd483ac6977e096ef7cde746d22bbf82e04d3 Mon Sep 17 00:00:00 2001
From: Dietmar Maurer <dietmar@proxmox.com>
Date: Mon, 11 Mar 2013 07:07:46 +0100
Subject: [PATCH v5 7/7] vma: add verify command

Users wants to verify the archive after backup.

Examples:

 # vma verify -v test.vma

 # lzop -d -c test.vma.lzo |vma verify -

Signed-off-by: Dietmar Maurer <dietmar@proxmox.com>
---
 vma-reader.c |  118 +++++++++++++++++++++++++++++++++++++++++++---------------
 vma.c        |   57 +++++++++++++++++++++++++++-
 vma.h        |    1 +
 3 files changed, 145 insertions(+), 31 deletions(-)

Index: new/vma-reader.c
===================================================================
--- new.orig/vma-reader.c	2014-11-20 08:15:12.000000000 +0100
+++ new/vma-reader.c	2014-11-20 08:47:30.000000000 +0100
@@ -53,6 +53,8 @@
     time_t start_time;
     int64_t cluster_count;
     int64_t clusters_read;
+    int64_t zero_cluster_data;
+    int64_t partial_zero_cluster_data;
     int clusters_read_per;
 };
 
@@ -433,6 +435,27 @@
     return NULL;
 }
 
+static void allocate_rstate(VmaReader *vmar,  guint8 dev_id,
+                            BlockDriverState *bs, bool write_zeroes)
+{
+    assert(vmar);
+    assert(dev_id);
+
+    vmar->rstate[dev_id].bs = bs;
+    vmar->rstate[dev_id].write_zeroes = write_zeroes;
+
+    int64_t size = vmar->devinfo[dev_id].size;
+
+    int64_t bitmap_size = (size/BDRV_SECTOR_SIZE) +
+        (VMA_CLUSTER_SIZE/BDRV_SECTOR_SIZE) * BITS_PER_LONG - 1;
+    bitmap_size /= (VMA_CLUSTER_SIZE/BDRV_SECTOR_SIZE) * BITS_PER_LONG;
+
+    vmar->rstate[dev_id].bitmap_size = bitmap_size;
+    vmar->rstate[dev_id].bitmap = g_new0(unsigned long, bitmap_size);
+
+    vmar->cluster_count += size/VMA_CLUSTER_SIZE;
+}
+
 int vma_reader_register_bs(VmaReader *vmar, guint8 dev_id, BlockDriverState *bs,
                            bool write_zeroes, Error **errp)
 {
@@ -449,17 +472,7 @@
         return -1;
     }
 
-    vmar->rstate[dev_id].bs = bs;
-    vmar->rstate[dev_id].write_zeroes = write_zeroes;
-
-    int64_t bitmap_size = (size/BDRV_SECTOR_SIZE) +
-        (VMA_CLUSTER_SIZE/BDRV_SECTOR_SIZE) * BITS_PER_LONG - 1;
-    bitmap_size /= (VMA_CLUSTER_SIZE/BDRV_SECTOR_SIZE) * BITS_PER_LONG;
-
-    vmar->rstate[dev_id].bitmap_size = bitmap_size;
-    vmar->rstate[dev_id].bitmap = g_new0(unsigned long, bitmap_size);
-
-    vmar->cluster_count += size/VMA_CLUSTER_SIZE;
+    allocate_rstate(vmar, dev_id, bs, write_zeroes);
 
     return 0;
 }
@@ -526,9 +539,10 @@
     }
     return 0;
 }
+
 static int restore_extent(VmaReader *vmar, unsigned char *buf,
                           int extent_size, int vmstate_fd,
-                          bool verbose, Error **errp)
+                          bool verbose, bool verify, Error **errp)
 {
     assert(vmar);
     assert(buf);
@@ -553,7 +567,7 @@
 
         if (dev_id != vmar->vmstate_stream) {
             bs = rstate->bs;
-            if (!bs) {
+            if (!verify && !bs) {
                 error_setg(errp, "got wrong dev id %d", dev_id);
                 return -1;
             }
@@ -609,10 +623,13 @@
                 return -1;
             }
 
-            int nb_sectors = end_sector - sector_num;
-            if (restore_write_data(vmar, dev_id, bs, vmstate_fd, buf + start,
-                                   sector_num, nb_sectors, errp) < 0) {
-                return -1;
+            if (!verify) {
+                int nb_sectors = end_sector - sector_num;
+                if (restore_write_data(vmar, dev_id, bs, vmstate_fd,
+                                       buf + start, sector_num, nb_sectors,
+                                       errp) < 0) {
+                    return -1;
+                }
             }
 
             start += VMA_CLUSTER_SIZE;
@@ -642,26 +659,37 @@
                         return -1;
                     }
 
-                    int nb_sectors = end_sector - sector_num;
-                    if (restore_write_data(vmar, dev_id, bs, vmstate_fd,
-                                           buf + start, sector_num,
-                                           nb_sectors, errp) < 0) {
-                        return -1;
+                    if (!verify) {
+                        int nb_sectors = end_sector - sector_num;
+                        if (restore_write_data(vmar, dev_id, bs, vmstate_fd,
+                                               buf + start, sector_num,
+                                               nb_sectors, errp) < 0) {
+                            return -1;
+                        }
                     }
 
                     start += VMA_BLOCK_SIZE;
 
                 } else {
 
-                    if (rstate->write_zeroes && (end_sector > sector_num)) {
+
+                    if (end_sector > sector_num) {
                         /* Todo: use bdrv_co_write_zeroes (but that need to
                          * be run inside coroutine?)
                          */
                         int nb_sectors = end_sector - sector_num;
-                        if (restore_write_data(vmar, dev_id, bs, vmstate_fd,
-                                              zero_vma_block, sector_num,
-                                               nb_sectors, errp) < 0) {
-                            return -1;
+                        int zero_size = BDRV_SECTOR_SIZE*nb_sectors;
+                        vmar->zero_cluster_data += zero_size;
+                        if (mask != 0) {
+                            vmar->partial_zero_cluster_data += zero_size;
+                        }
+
+                        if (rstate->write_zeroes && !verify) {
+                            if (restore_write_data(vmar, dev_id, bs, vmstate_fd,
+                                                   zero_vma_block, sector_num,
+                                                   nb_sectors, errp) < 0) {
+                                return -1;
+                            }
                         }
                     }
                 }
@@ -679,8 +707,9 @@
     return 0;
 }
 
-int vma_reader_restore(VmaReader *vmar, int vmstate_fd, bool verbose,
-                       Error **errp)
+static int vma_reader_restore_full(VmaReader *vmar, int vmstate_fd,
+                                   bool verbose, bool verify,
+                                   Error **errp)
 {
     assert(vmar);
     assert(vmar->head_data);
@@ -747,7 +776,7 @@
         }
 
         if (restore_extent(vmar, buf, extent_size, vmstate_fd, verbose,
-                           errp) < 0) {
+                           verify, errp) < 0) {
             return -1;
         }
 
@@ -794,6 +823,38 @@
         }
     }
 
+    if (verbose) {
+        printf("total bytes read %zd, sparse bytes %zd (%.3g%%)\n",
+               vmar->clusters_read*VMA_CLUSTER_SIZE,
+               vmar->zero_cluster_data,
+               (double)(100.0*vmar->zero_cluster_data)/
+               (vmar->clusters_read*VMA_CLUSTER_SIZE));
+
+        int64_t datasize = vmar->clusters_read*VMA_CLUSTER_SIZE-vmar->zero_cluster_data;
+        if (datasize) { // this does not make sense for empty files
+            printf("space reduction due to 4K zero blocks %.3g%%\n",
+                   (double)(100.0*vmar->partial_zero_cluster_data) / datasize);
+        }
+    }
     return ret;
 }
 
+int vma_reader_restore(VmaReader *vmar, int vmstate_fd, bool verbose,
+                       Error **errp)
+{
+    return vma_reader_restore_full(vmar, vmstate_fd, verbose, false, errp);
+}
+
+int vma_reader_verify(VmaReader *vmar, bool verbose, Error **errp)
+{
+    guint8 dev_id;
+
+    for (dev_id = 1; dev_id < 255; dev_id++) {
+        if (vma_reader_get_device_info(vmar, dev_id)) {
+            allocate_rstate(vmar, dev_id, NULL, false);
+        }
+    }
+
+    return vma_reader_restore_full(vmar, -1, verbose, true, errp);
+}
+
Index: new/vma.c
===================================================================
--- new.orig/vma.c	2014-11-20 08:47:23.000000000 +0100
+++ new/vma.c	2014-11-20 08:47:30.000000000 +0100
@@ -34,6 +34,7 @@
         "vma list <filename>\n"
         "vma create <filename> [-c config] <archive> pathname ...\n"
         "vma extract <filename> [-r <fifo>] <targetdir>\n"
+        "vma verify <filename> [-v]\n"
         ;
 
     printf("%s", help_msg);
@@ -338,6 +339,58 @@
     return ret;
 }
 
+static int verify_content(int argc, char **argv)
+{
+    int c, ret = 0;
+    int verbose = 0;
+    const char *filename;
+
+    for (;;) {
+        c = getopt(argc, argv, "hv");
+        if (c == -1) {
+            break;
+        }
+        switch (c) {
+        case '?':
+        case 'h':
+            help();
+            break;
+        case 'v':
+            verbose = 1;
+            break;
+        default:
+            help();
+        }
+    }
+
+    /* Get the filename */
+    if ((optind + 1) != argc) {
+        help();
+    }
+    filename = argv[optind++];
+
+    Error *errp = NULL;
+    VmaReader *vmar = vma_reader_create(filename, &errp);
+
+    if (!vmar) {
+        g_error("%s", error_get_pretty(errp));
+    }
+
+    if (verbose) {
+        print_content(vmar);
+    }
+
+    if (vma_reader_verify(vmar, verbose, &errp) < 0) {
+        g_error("verify failed - %s", error_get_pretty(errp));
+    }
+
+    vma_reader_destroy(vmar);
+
+    bdrv_close_all();
+
+    return ret;
+}
+
 typedef struct BackupJob {
     BlockDriverState *bs;
     int64_t len;
@@ -575,6 +628,8 @@
         return create_archive(argc, argv);
     } else if (!strcmp(cmdname, "extract")) {
         return extract_content(argc, argv);
+    } else if (!strcmp(cmdname, "verify")) {
+        return verify_content(argc, argv);
     }
 
     help();
Index: new/vma.h
===================================================================
--- new.orig/vma.h	2014-11-20 08:15:12.000000000 +0100
+++ new/vma.h	2014-11-20 08:47:30.000000000 +0100
@@ -142,5 +142,6 @@
                            Error **errp);
 int vma_reader_restore(VmaReader *vmar, int vmstate_fd, bool verbose,
                        Error **errp);
+int vma_reader_verify(VmaReader *vmar, bool verbose, Error **errp);
 
 #endif /* BACKUP_VMA_H */
Commit	Line	Data
0795670a DM	1	From acdcd483ac6977e096ef7cde746d22bbf82e04d3 Mon Sep 17 00:00:00 2001
	2	From: Dietmar Maurer <dietmar@proxmox.com>
	3	Date: Mon, 11 Mar 2013 07:07:46 +0100
	4	Subject: [PATCH v5 7/7] vma: add verify command
	5
	6	Users wants to verify the archive after backup.
	7
	8	Examples:
	9
	10	# vma verify -v test.vma
	11
	12	# lzop -d -c test.vma.lzo \|vma verify -
	13
	14	Signed-off-by: Dietmar Maurer <dietmar@proxmox.com>
	15	---
	16	vma-reader.c \| 118 +++++++++++++++++++++++++++++++++++++++++++---------------
	17	vma.c \| 57 +++++++++++++++++++++++++++-
	18	vma.h \| 1 +
	19	3 files changed, 145 insertions(+), 31 deletions(-)
	20
	21	Index: new/vma-reader.c
	22	===================================================================
24bb7da3 DM	23	--- new.orig/vma-reader.c 2014-11-20 08:15:12.000000000 +0100
24bb7da3 DM	24	+++ new/vma-reader.c 2014-11-20 08:47:30.000000000 +0100
0795670a DM	25	@@ -53,6 +53,8 @@
	26	time_t start_time;
	27	int64_t cluster_count;
	28	int64_t clusters_read;
	29	+ int64_t zero_cluster_data;
	30	+ int64_t partial_zero_cluster_data;
	31	int clusters_read_per;
	32	};
	33
	34	@@ -433,6 +435,27 @@
	35	return NULL;
	36	}
	37
	38	+static void allocate_rstate(VmaReader *vmar, guint8 dev_id,
	39	+ BlockDriverState *bs, bool write_zeroes)
	40	+{
	41	+ assert(vmar);
	42	+ assert(dev_id);
	43	+
	44	+ vmar->rstate[dev_id].bs = bs;
	45	+ vmar->rstate[dev_id].write_zeroes = write_zeroes;
	46	+
	47	+ int64_t size = vmar->devinfo[dev_id].size;
	48	+
	49	+ int64_t bitmap_size = (size/BDRV_SECTOR_SIZE) +
	50	+ (VMA_CLUSTER_SIZE/BDRV_SECTOR_SIZE) * BITS_PER_LONG - 1;
	51	+ bitmap_size /= (VMA_CLUSTER_SIZE/BDRV_SECTOR_SIZE) * BITS_PER_LONG;
	52	+
	53	+ vmar->rstate[dev_id].bitmap_size = bitmap_size;
	54	+ vmar->rstate[dev_id].bitmap = g_new0(unsigned long, bitmap_size);
	55	+
	56	+ vmar->cluster_count += size/VMA_CLUSTER_SIZE;
	57	+}
	58	+
	59	int vma_reader_register_bs(VmaReader vmar, guint8 dev_id, BlockDriverState bs,
	60	bool write_zeroes, Error **errp)
	61	{
	62	@@ -449,17 +472,7 @@
	63	return -1;
	64	}
	65
	66	- vmar->rstate[dev_id].bs = bs;
	67	- vmar->rstate[dev_id].write_zeroes = write_zeroes;
	68	-
	69	- int64_t bitmap_size = (size/BDRV_SECTOR_SIZE) +
	70	- (VMA_CLUSTER_SIZE/BDRV_SECTOR_SIZE) * BITS_PER_LONG - 1;
	71	- bitmap_size /= (VMA_CLUSTER_SIZE/BDRV_SECTOR_SIZE) * BITS_PER_LONG;
	72	-
	73	- vmar->rstate[dev_id].bitmap_size = bitmap_size;
	74	- vmar->rstate[dev_id].bitmap = g_new0(unsigned long, bitmap_size);
	75	-
	76	- vmar->cluster_count += size/VMA_CLUSTER_SIZE;
	77	+ allocate_rstate(vmar, dev_id, bs, write_zeroes);
	78
	79	return 0;
	80	}
	81	@@ -526,9 +539,10 @@
	82	}
	83	return 0;
	84	}
	85	+
	86	static int restore_extent(VmaReader vmar, unsigned char buf,
	87	int extent_size, int vmstate_fd,
	88	- bool verbose, Error **errp)
89	+ bool verbose, bool verify, Error **errp)
90	{
91	assert(vmar);
92	assert(buf);
93	@@ -553,7 +567,7 @@
94
95	if (dev_id != vmar->vmstate_stream) {
96	bs = rstate->bs;
97	- if (!bs) {
98	+ if (!verify && !bs) {
99	error_setg(errp, "got wrong dev id %d", dev_id);
100	return -1;
101	}
102	@@ -609,10 +623,13 @@
103	return -1;
104	}
105
106	- int nb_sectors = end_sector - sector_num;
107	- if (restore_write_data(vmar, dev_id, bs, vmstate_fd, buf + start,
108	- sector_num, nb_sectors, errp) < 0) {
109	- return -1;
110	+ if (!verify) {
111	+ int nb_sectors = end_sector - sector_num;
112	+ if (restore_write_data(vmar, dev_id, bs, vmstate_fd,
113	+ buf + start, sector_num, nb_sectors,
114	+ errp) < 0) {
115	+ return -1;
116	+ }
117	}
118
119	start += VMA_CLUSTER_SIZE;
120	@@ -642,26 +659,37 @@
121	return -1;
122	}
123
124	- int nb_sectors = end_sector - sector_num;
125	- if (restore_write_data(vmar, dev_id, bs, vmstate_fd,
126	- buf + start, sector_num,
127	- nb_sectors, errp) < 0) {
128	- return -1;
129	+ if (!verify) {
130	+ int nb_sectors = end_sector - sector_num;
131	+ if (restore_write_data(vmar, dev_id, bs, vmstate_fd,
132	+ buf + start, sector_num,
133	+ nb_sectors, errp) < 0) {
134	+ return -1;
135	+ }
136	}
137
138	start += VMA_BLOCK_SIZE;
139
140	} else {
141
142	- if (rstate->write_zeroes && (end_sector > sector_num)) {
143	+
144	+ if (end_sector > sector_num) {
145	/* Todo: use bdrv_co_write_zeroes (but that need to
146	* be run inside coroutine?)
147	*/
148	int nb_sectors = end_sector - sector_num;
149	- if (restore_write_data(vmar, dev_id, bs, vmstate_fd,
150	- zero_vma_block, sector_num,
151	- nb_sectors, errp) < 0) {
152	- return -1;
153	+ int zero_size = BDRV_SECTOR_SIZE*nb_sectors;
154	+ vmar->zero_cluster_data += zero_size;
155	+ if (mask != 0) {
156	+ vmar->partial_zero_cluster_data += zero_size;
157	+ }
158	+
159	+ if (rstate->write_zeroes && !verify) {
160	+ if (restore_write_data(vmar, dev_id, bs, vmstate_fd,
161	+ zero_vma_block, sector_num,
162	+ nb_sectors, errp) < 0) {
163	+ return -1;
164	+ }
165	}
166	}
167	}
168	@@ -679,8 +707,9 @@
169	return 0;
170	}
171
172	-int vma_reader_restore(VmaReader *vmar, int vmstate_fd, bool verbose,
173	- Error **errp)
174	+static int vma_reader_restore_full(VmaReader *vmar, int vmstate_fd,
175	+ bool verbose, bool verify,
176	+ Error **errp)
177	{
178	assert(vmar);
179	assert(vmar->head_data);
180	@@ -747,7 +776,7 @@
181	}
182
183	if (restore_extent(vmar, buf, extent_size, vmstate_fd, verbose,
184	- errp) < 0) {
185	+ verify, errp) < 0) {
186	return -1;
187	}
188
189	@@ -794,6 +823,38 @@
190	}
191	}
192
193	+ if (verbose) {
194	+ printf("total bytes read %zd, sparse bytes %zd (%.3g%%)\n",
195	+ vmar->clusters_read*VMA_CLUSTER_SIZE,
196	+ vmar->zero_cluster_data,
197	+ (double)(100.0*vmar->zero_cluster_data)/
198	+ (vmar->clusters_read*VMA_CLUSTER_SIZE));
199	+
200	+ int64_t datasize = vmar->clusters_read*VMA_CLUSTER_SIZE-vmar->zero_cluster_data;
201	+ if (datasize) { // this does not make sense for empty files
27f6b7fc	202	+ printf("space reduction due to 4K zero blocks %.3g%%\n",
0795670a DM	203	+ (double)(100.0*vmar->partial_zero_cluster_data) / datasize);
	204	+ }
	205	+ }
	206	return ret;
	207	}
	208
	209	+int vma_reader_restore(VmaReader *vmar, int vmstate_fd, bool verbose,
	210	+ Error **errp)
	211	+{
	212	+ return vma_reader_restore_full(vmar, vmstate_fd, verbose, false, errp);
	213	+}
	214	+
	215	+int vma_reader_verify(VmaReader vmar, bool verbose, Error *errp)
	216	+{
	217	+ guint8 dev_id;
	218	+
	219	+ for (dev_id = 1; dev_id < 255; dev_id++) {
	220	+ if (vma_reader_get_device_info(vmar, dev_id)) {
	221	+ allocate_rstate(vmar, dev_id, NULL, false);
	222	+ }
	223	+ }
	224	+
	225	+ return vma_reader_restore_full(vmar, -1, verbose, true, errp);
	226	+}
	227	+
	228	Index: new/vma.c
	229	===================================================================
24bb7da3 DM	230	--- new.orig/vma.c 2014-11-20 08:47:23.000000000 +0100
24bb7da3 DM	231	+++ new/vma.c 2014-11-20 08:47:30.000000000 +0100
09a78de1	232	@@ -34,6 +34,7 @@
0795670a DM	233	"vma list <filename>\n"
0795670a DM	234	"vma create <filename> [-c config] <archive> pathname ...\n"
09a78de1	235	"vma extract <filename> [-r <fifo>] <targetdir>\n"
0795670a DM	236	+ "vma verify <filename> [-v]\n"
	237	;
	238
	239	printf("%s", help_msg);
	240	@@ -338,6 +339,58 @@
	241	return ret;
	242	}
	243
	244	+static int verify_content(int argc, char **argv)
	245	+{
	246	+ int c, ret = 0;
	247	+ int verbose = 0;
	248	+ const char *filename;
	249	+
	250	+ for (;;) {
	251	+ c = getopt(argc, argv, "hv");
	252	+ if (c == -1) {
	253	+ break;
	254	+ }
	255	+ switch (c) {
	256	+ case '?':
	257	+ case 'h':
	258	+ help();
	259	+ break;
	260	+ case 'v':
	261	+ verbose = 1;
	262	+ break;
	263	+ default:
	264	+ help();
	265	+ }
	266	+ }
	267	+
	268	+ /* Get the filename */
	269	+ if ((optind + 1) != argc) {
	270	+ help();
	271	+ }
	272	+ filename = argv[optind++];
	273	+
	274	+ Error *errp = NULL;
	275	+ VmaReader *vmar = vma_reader_create(filename, &errp);
	276	+
	277	+ if (!vmar) {
	278	+ g_error("%s", error_get_pretty(errp));
	279	+ }
	280	+
	281	+ if (verbose) {
	282	+ print_content(vmar);
	283	+ }
	284	+
	285	+ if (vma_reader_verify(vmar, verbose, &errp) < 0) {
	286	+ g_error("verify failed - %s", error_get_pretty(errp));
	287	+ }
	288	+
	289	+ vma_reader_destroy(vmar);
	290	+
	291	+ bdrv_close_all();
	292	+
	293	+ return ret;
	294	+}
	295	+
	296	typedef struct BackupJob {
	297	BlockDriverState *bs;
	298	int64_t len;
24bb7da3	299	@@ -575,6 +628,8 @@
0795670a DM	300	return create_archive(argc, argv);
	301	} else if (!strcmp(cmdname, "extract")) {
	302	return extract_content(argc, argv);
	303	+ } else if (!strcmp(cmdname, "verify")) {
	304	+ return verify_content(argc, argv);
	305	}
	306
	307	help();
	308	Index: new/vma.h
	309	===================================================================
24bb7da3 DM	310	--- new.orig/vma.h 2014-11-20 08:15:12.000000000 +0100
24bb7da3 DM	311	+++ new/vma.h 2014-11-20 08:47:30.000000000 +0100
af0d302f	312	@@ -142,5 +142,6 @@
0795670a DM	313	Error **errp);
	314	int vma_reader_restore(VmaReader *vmar, int vmstate_fd, bool verbose,
	315	Error **errp);
	316	+int vma_reader_verify(VmaReader vmar, bool verbose, Error *errp);
	317
	318	#endif /* BACKUP_VMA_H */