debian/patches/0005-add-regression-tests-for-backup.patch

   1 From 4fb44b8d03b764db04e7751a14055cbb06a7791d Mon Sep 17 00:00:00 2001
   2 From: Dietmar Maurer <dietmar@proxmox.com>
   3 Date: Wed, 14 Nov 2012 09:57:04 +0100
   4 Subject: [PATCH v4 5/6] add regression tests for backup
   5
   6 Simple regression tests using vma-reader and vma-writer.
   7
   8 Note: the call to g_thread_init() solves problems with g_slice_alloc() - without that call we get arbitrary crashes.
   9
  10 Signed-off-by: Dietmar Maurer <dietmar@proxmox.com>
  11 ---
  12  tests/Makefile      |   11 +-
  13  tests/backup-test.c |  517 +++++++++++++++++++++++++++++++++++++++++++++++++++
  14  2 files changed, 526 insertions(+), 2 deletions(-)
  15  create mode 100644 tests/backup-test.c
  16
  17 diff --git a/tests/Makefile b/tests/Makefile
  18 index 567e36e..136be84 100644
  19 --- a/tests/Makefile
  20 +++ b/tests/Makefile
  21 @@ -59,6 +59,8 @@ gcov-files-test-mul64-y = util/host-utils.c
  22
  23  check-block-$(CONFIG_POSIX) += tests/qemu-iotests-quick.sh
  24
  25 +check-backup-y = tests/backup-test$(EXESUF)
  26 +
  27  # All QTests for now are POSIX-only, but the dependencies are
  28  # really in libqtest, not in the testcases themselves.
  29  check-qtest-i386-y = tests/fdc-test$(EXESUF)
  30 @@ -102,6 +104,7 @@ tests/test-coroutine$(EXESUF): tests/test-coroutine.o $(block-obj-y) libqemuutil
  31  tests/test-aio$(EXESUF): tests/test-aio.o $(block-obj-y) libqemuutil.a libqemustub.a
  32  tests/test-thread-pool$(EXESUF): tests/test-thread-pool.o $(block-obj-y) libqemuutil.a libqemustub.a
  33  tests/test-iov$(EXESUF): tests/test-iov.o libqemuutil.a
  34 +tests/backup-test$(EXESUF): tests/backup-test.o vma-reader.o $(block-obj-y) libqemuutil.a libqemustub.a
  35  tests/test-hbitmap$(EXESUF): tests/test-hbitmap.o libqemuutil.a libqemustub.a
  36  tests/test-x86-cpuid$(EXESUF): tests/test-x86-cpuid.o
  37  tests/test-xbzrle$(EXESUF): tests/test-xbzrle.o xbzrle.o page_cache.o libqemuutil.a
  38 @@ -213,10 +216,14 @@ check-tests/qemu-iotests-quick.sh: tests/qemu-iotests-quick.sh qemu-img$(EXESUF)
  39
  40  # Consolidated targets
  41
  42 -.PHONY: check-qtest check-unit check
  43 +.PHONY: check-backup check-qtest check-unit check
  44  check-qtest: $(patsubst %,check-qtest-%, $(QTEST_TARGETS))
  45  check-unit: $(patsubst %,check-%, $(check-unit-y))
  46  check-block: $(patsubst %,check-%, $(check-block-y))
  47 -check: check-unit check-qtest
  48 +
  49 +check-backup: tests/backup-test$(EXESUF)
  50 +       $<
  51 +
  52 +check: check-unit check-qtest check-backup
  53
  54  -include $(wildcard tests/*.d)
  55 diff --git a/tests/backup-test.c b/tests/backup-test.c
  56 new file mode 100644
  57 index 0000000..5ff6f1d
  58 --- /dev/null
  59 +++ b/tests/backup-test.c
  60 @@ -0,0 +1,517 @@
  61 +/*
  62 + * QEMU backup test suit
  63 + *
  64 + * Copyright (C) Proxmox Server Solutions
  65 + *
  66 + * Authors:
  67 + *  Dietmar Maurer (dietmar@proxmox.com)
  68 + *
  69 + * This work is licensed under the terms of the GNU GPL, version 2.  See
  70 + * the COPYING file in the top-level directory.
  71 + *
  72 + * Fixme: running 'backup-test -l' trigger a bug in g_slice_alloc()
  73 + * Note: 'G_SLICE=always-malloc ./tests/backup-test -l' works
  74 + *
  75 + */
  76 +
  77 +#include <sys/time.h>
  78 +#include <sys/types.h>
  79 +#include <stdarg.h>
  80 +#include <stdio.h>
  81 +#include <getopt.h>
  82 +#include <libgen.h>
  83 +
  84 +#include "qemu-common.h"
  85 +#include "block/block.h"
  86 +
  87 +#include "vma.h"
  88 +
  89 +static int opt_debug;
  90 +static int opt_loop;
  91 +
  92 +#define DPRINTF(fmt, ...) \
  93 +    do { if (opt_debug) { printf(fmt, ## __VA_ARGS__); } } while (0)
  94 +
  95 +#define CLUSTER(x) (x*BACKUP_CLUSTER_SIZE)
  96 +
  97 +#define RUN_TEST(testfunc, speed) \
  98 +    backup_test(#testfunc " speed " #speed, speed, testfunc);
  99 +
 100 +
 101 +static unsigned char buf_sec_pattern_cd[BDRV_SECTOR_SIZE];
 102 +static unsigned char buf_sec_pattern_32[BDRV_SECTOR_SIZE];
 103 +
 104 +#define TEST_IMG_SIZE (6*1024*1024+BDRV_SECTOR_SIZE)
 105 +#define TEST_IMG_NAME "backuptest.raw"
 106 +#define TEST_IMG_RESTORE_NAME "backuptest.raw.restore"
 107 +#define TEST_VMA_NAME "backuptest.vma"
 108 +
 109 +typedef struct BackupCB {
 110 +    VmaWriter *vmaw;
 111 +    uint8_t dev_id;
 112 +} BackupCB;
 113 +
 114 +static int backup_dump_cb(void *opaque, BlockDriverState *bs,
 115 +                          int64_t cluster_num, unsigned char *buf)
 116 +{
 117 +    BackupCB *bcb = opaque;
 118 +
 119 +    DPRINTF("backup_dump_cb C%zd %d\n", cluster_num, bcb->dev_id);
 120 +
 121 +    size_t zb = 0;
 122 +    if (vma_writer_write(bcb->vmaw, bcb->dev_id, cluster_num, buf, &zb) < 0) {
 123 +        printf("backup_dump_cb vma_writer_write failed\n");
 124 +        return -1;
 125 +    }
 126 +
 127 +    return 0;
 128 +}
 129 +
 130 +static void backup_complete_cb(void *opaque, int ret)
 131 +{
 132 +    BackupCB *bcb = opaque;
 133 +
 134 +    DPRINTF("backup_complete_cb %d %d\n", bcb->dev_id, ret);
 135 +
 136 +    if (ret < 0) {
 137 +        vma_writer_set_error(bcb->vmaw, "backup_complete_cb %d", ret);
 138 +    }
 139 +
 140 +    if (vma_writer_close_stream(bcb->vmaw, bcb->dev_id) <= 0) {
 141 +        Error *err = NULL;
 142 +        if (vma_writer_close(bcb->vmaw, &err) != 0) {
 143 +            g_error("vma_writer_close failed %s", error_get_pretty(err));
 144 +        }
 145 +    }
 146 +    DPRINTF("backup_complete_cb finish\n");
 147 +}
 148 +
 149 +static void write_sec_pattern_cd(BlockDriverState *bs, int64_t offset)
 150 +{
 151 +    int ret;
 152 +
 153 +    DPRINTF("write_sec_pattern_cd %zd\n", offset);
 154 +
 155 +    if (offset & 0x1ff) {
 156 +        g_error("write_sec_pattern_cd offset %zd is not sector aligned\n",
 157 +                offset);
 158 +    }
 159 +
 160 +    ret = bdrv_write(bs, offset >> 9, buf_sec_pattern_cd, 1);
 161 +    if (ret < 0) {
 162 +        g_error("write_sec_pattern_cd %zd failed", offset);
 163 +    }
 164 +
 165 +}
 166 +
 167 +static void read_sec(BlockDriverState *bs, int64_t offset, unsigned char *buf)
 168 +{
 169 +    DPRINTF("read_sec C%zd start %zd\n", offset>>VMA_CLUSTER_BITS, offset);
 170 +
 171 +    if (offset & 0x1ff) {
 172 +        g_error("read_sec offset %zd is not sector aligned\n", offset);
 173 +    }
 174 +
 175 +    if (bdrv_read(bs, offset >> 9, buf, 1) < 0) {
 176 +        g_error("bdrv_read failed");
 177 +    }
 178 +}
 179 +
 180 +static bool request_term;
 181 +
 182 +typedef struct TestCB {
 183 +    Coroutine *co;
 184 +    BlockDriverState *bs;
 185 +    bool finished;
 186 +} TestCB;
 187 +
 188 +static TestCB *enter_test_co(BlockDriverState *bs, CoroutineEntry *entry)
 189 +{
 190 +    TestCB *cb = g_new0(TestCB, 1);
 191 +    cb->bs = bs;
 192 +    cb->co = qemu_coroutine_create(entry);
 193 +    qemu_coroutine_enter(cb->co, cb);
 194 +    return cb;
 195 +}
 196 +
 197 +static void test_co_sleep(double sec)
 198 +{
 199 +    co_sleep_ns(rt_clock, (int64_t)(sec*1000000000));
 200 +};
 201 +
 202 +static void test_co_yield(void)
 203 +{
 204 +    co_sleep_ns(rt_clock, (int64_t)(1000));
 205 +};
 206 +
 207 +static void coroutine_fn run_co_test1(void *opaque)
 208 +{
 209 +    assert(opaque);
 210 +    TestCB *cb = (TestCB *)opaque;
 211 +
 212 +    test_co_sleep(0.2);
 213 +    write_sec_pattern_cd(cb->bs, 5*BACKUP_CLUSTER_SIZE);
 214 +    test_co_sleep(0.2);
 215 +    write_sec_pattern_cd(cb->bs, 10*BACKUP_CLUSTER_SIZE);
 216 +    test_co_sleep(0.2);
 217 +    write_sec_pattern_cd(cb->bs, 10*BACKUP_CLUSTER_SIZE);
 218 +
 219 +    cb->finished = true;
 220 +}
 221 +
 222 +static void coroutine_fn run_co_test2(void *opaque)
 223 +{
 224 +    assert(opaque);
 225 +    TestCB *cb = (TestCB *)opaque;
 226 +    unsigned char buf[512];
 227 +
 228 +    test_co_sleep(0.2);
 229 +    read_sec(cb->bs, 5*BACKUP_CLUSTER_SIZE, buf);
 230 +    write_sec_pattern_cd(cb->bs, 6*BACKUP_CLUSTER_SIZE);
 231 +
 232 +    cb->finished = true;
 233 +}
 234 +
 235 +static void coroutine_fn run_co_random_read(void *opaque)
 236 +{
 237 +    assert(opaque);
 238 +    TestCB *cb = (TestCB *)opaque;
 239 +    int64_t sectors = bdrv_getlength(cb->bs)/BDRV_SECTOR_SIZE - 1;
 240 +    unsigned char buf[512];
 241 +
 242 +    while (1) {
 243 +        test_co_yield();
 244 +        if (request_term) {
 245 +            DPRINTF("finish run_co_random_read\n");
 246 +            break;
 247 +        }
 248 +        int64_t s = (rand()*sectors)/RAND_MAX;
 249 +        read_sec(cb->bs, s*BDRV_SECTOR_SIZE, buf);
 250 +    }
 251 +
 252 +    cb->finished = true;
 253 +}
 254 +
 255 +static void coroutine_fn run_co_random_write(void *opaque)
 256 +{
 257 +    assert(opaque);
 258 +    TestCB *cb = (TestCB *)opaque;
 259 +    int64_t sectors = bdrv_getlength(cb->bs)/BDRV_SECTOR_SIZE;
 260 +
 261 +    while (1) {
 262 +        test_co_yield();
 263 +        if (request_term) {
 264 +            DPRINTF("finish run_co_random_write\n");
 265 +            break;
 266 +        }
 267 +        int64_t s = (rand()*sectors)/RAND_MAX;
 268 +        write_sec_pattern_cd(cb->bs, s*BDRV_SECTOR_SIZE);
 269 +    }
 270 +
 271 +    cb->finished = true;
 272 +}
 273 +
 274 +static void fill_test_sector(void *buf, size_t sector_num)
 275 +{
 276 +    int64_t *i64buf = (int64_t *)buf;
 277 +    int i;
 278 +
 279 +    int data = sector_num;
 280 +    if (sector_num >= 8 && sector_num < 8*(2*16+2)) {
 281 +        data = 0;  /* add zero region for testing */
 282 +    }
 283 +
 284 +    for (i = 0; i < (512/sizeof(int64_t)); i++) {
 285 +        i64buf[i] = data;
 286 +    }
 287 +}
 288 +
 289 +static void verify_archive(const char *archive, size_t size)
 290 +{
 291 +    Error *errp = NULL;
 292 +
 293 +    VmaReader *vmar = vma_reader_create(archive, &errp);
 294 +
 295 +    if (!vmar) {
 296 +        g_error("%s", error_get_pretty(errp));
 297 +    }
 298 +
 299 +    VmaDeviceInfo *di = vma_reader_get_device_info(vmar, 1);
 300 +    if (!di || strcmp((char *)di->devname, "hda") || di->size != size) {
 301 +        g_error("got wrong device info");
 302 +    }
 303 +
 304 +    unlink(TEST_IMG_RESTORE_NAME);
 305 +
 306 +    int flags = BDRV_O_NATIVE_AIO|BDRV_O_RDWR|BDRV_O_CACHE_WB;
 307 +
 308 +    bdrv_img_create(TEST_IMG_RESTORE_NAME, "raw", NULL, NULL, NULL,
 309 +                    size, flags, &errp);
 310 +    if (error_is_set(&errp)) {
 311 +        g_error("can't create file %s: %s", TEST_IMG_RESTORE_NAME,
 312 +                error_get_pretty(errp));
 313 +    }
 314 +
 315 +    BlockDriverState *bs = NULL;
 316 +    if (bdrv_file_open(&bs, TEST_IMG_RESTORE_NAME, flags)) {
 317 +        g_error("can't open file %s", TEST_IMG_RESTORE_NAME);
 318 +    }
 319 +    if (vma_reader_register_bs(vmar, 1, bs, false, &errp) < 0) {
 320 +        g_error("%s", error_get_pretty(errp));
 321 +    }
 322 +
 323 +    if (vma_reader_restore(vmar, -1, false, &errp) < 0) {
 324 +        g_error("restore failed - %s", error_get_pretty(errp));
 325 +    }
 326 +
 327 +    size_t i;
 328 +    size_t sectors = size/BDRV_SECTOR_SIZE;
 329 +    int64_t buf[512/sizeof(int64_t)];
 330 +    int64_t buf2[512/sizeof(int64_t)];
 331 +
 332 +    for (i = 0; i < sectors; i++) {
 333 +        if (bdrv_read(bs, i, (uint8_t *)buf, 1) != 0) {
 334 +            g_error("bdrv_read failed");
 335 +        }
 336 +        fill_test_sector(buf2, i);
 337 +        if (bcmp(buf, buf2, sizeof(buf))) {
 338 +            g_error("data is different at sector %zd", i);
 339 +        }
 340 +    }
 341 +
 342 +    vma_reader_destroy(vmar);
 343 +
 344 +    unlink(TEST_IMG_RESTORE_NAME);
 345 +}
 346 +
 347 +static void prepare_vm_image(const char *filename, size_t sectors)
 348 +{
 349 +    int fd = open(filename, O_RDWR|O_CREAT|O_TRUNC, 0644);
 350 +    if (fd < 0) {
 351 +        g_error("can't open file %s\n", filename);
 352 +    }
 353 +
 354 +    size_t i;
 355 +    int64_t buf[512/sizeof(int64_t)];
 356 +
 357 +    for (i = 0; i < sectors; i++) {
 358 +        fill_test_sector(buf, i);
 359 +
 360 +         int res = 0;
 361 +        while (1) {
 362 +            res = write(fd, buf, sizeof(buf));
 363 +            if (!(res < 0 && errno == EINTR)) {
 364 +                break;
 365 +            }
 366 +        }
 367 +        if (res != sizeof(buf)) {
 368 +            g_error("can't initialize file %s - %s %d\n",
 369 +                    filename, strerror(errno), res);
 370 +        }
 371 +    }
 372 +
 373 +    if (close(fd) != 0) {
 374 +        g_error("close failed");
 375 +    }
 376 +}
 377 +
 378 +static GList *simple_test(BlockDriverState *bs)
 379 +{
 380 +    GList *cb_list = NULL;
 381 +
 382 +    cb_list = g_list_append(cb_list, enter_test_co(bs, run_co_test1));
 383 +    cb_list = g_list_append(cb_list, enter_test_co(bs, run_co_test2));
 384 +
 385 +    return cb_list;
 386 +}
 387 +
 388 +static GList *random_read_write_test(BlockDriverState *bs)
 389 +{
 390 +    GList *cb_list = NULL;
 391 +
 392 +    cb_list = g_list_append(cb_list, enter_test_co(bs, run_co_random_read));
 393 +    cb_list = g_list_append(cb_list, enter_test_co(bs, run_co_random_read));
 394 +    cb_list = g_list_append(cb_list, enter_test_co(bs, run_co_random_write));
 395 +    cb_list = g_list_append(cb_list, enter_test_co(bs, run_co_random_write));
 396 +
 397 +    return cb_list;
 398 +}
 399 +
 400 +static void backup_test(const char *testname, int64_t speed,
 401 +                        GList *(*start_test_cb)(BlockDriverState *bs))
 402 +{
 403 +    BlockDriverState *bs = bdrv_new("hda");
 404 +
 405 +    static int test_counter;
 406 +
 407 +    test_counter++;
 408 +
 409 +    printf("starting test #%d '%s'\n", test_counter, testname);
 410 +
 411 +    const char *filename = TEST_IMG_NAME;
 412 +
 413 +    prepare_vm_image(TEST_IMG_NAME, TEST_IMG_SIZE/BDRV_SECTOR_SIZE);
 414 +
 415 +    int flags = BDRV_O_NATIVE_AIO|BDRV_O_RDWR|BDRV_O_CACHE_WB;
 416 +
 417 +    if (bdrv_open(bs, filename, flags, NULL) < 0) {
 418 +        g_error("can't open device %s\n", filename);
 419 +    }
 420 +
 421 +    Error *err = NULL;
 422 +    uuid_t uuid;
 423 +    uuid_generate(uuid);
 424 +
 425 +    unlink(TEST_VMA_NAME);
 426 +
 427 +    VmaWriter *vmaw = vma_writer_create(TEST_VMA_NAME, uuid, &err);
 428 +    if (!vmaw) {
 429 +        g_error("%s", error_get_pretty(err));
 430 +    }
 431 +
 432 +    BackupCB bcb;
 433 +    bcb.vmaw = vmaw;
 434 +    bcb.dev_id = vma_writer_register_stream(vmaw, bdrv_get_device_name(bs),
 435 +                                            bdrv_getlength(bs));
 436 +    if (backup_job_create(bs, backup_dump_cb, backup_complete_cb, &bcb,
 437 +                          speed) < 0) {
 438 +        g_error("backup_job_create failed");
 439 +    } else {
 440 +        backup_job_start(bs, false);
 441 +    }
 442 +
 443 +    request_term = false;
 444 +
 445 +    GList *cb_list = start_test_cb(bs);
 446 +
 447 +    while (1) {
 448 +        main_loop_wait(false);
 449 +
 450 +        VmaStatus vmastat;
 451 +        vma_writer_get_status(vmaw, &vmastat);
 452 +        if (vmastat.closed) {
 453 +            break;
 454 +        }
 455 +    }
 456 +
 457 +    request_term = true;
 458 +
 459 +    while (1) {
 460 +        GList *l = cb_list;
 461 +        bool active = 0;
 462 +        while (l && l->data) {
 463 +            TestCB *cb = (TestCB *)l->data;
 464 +            l = g_list_next(l);
 465 +            if (!cb->finished) {
 466 +                active = true;
 467 +                break;
 468 +            }
 469 +        }
 470 +        if (!active) {
 471 +            DPRINTF("All test coroutines finished\n");
 472 +            break;
 473 +        }
 474 +        main_loop_wait(false);
 475 +    }
 476 +
 477 +    /* Make sure all outstanding requests complete */
 478 +    bdrv_drain_all();
 479 +
 480 +    VmaStatus vmastat;
 481 +    vma_writer_get_status(vmaw, &vmastat);
 482 +    DPRINTF("statistic %zd %zd\n", vmastat.stream_info[1].size,
 483 +            vmastat.stream_info[1].transferred);
 484 +    assert(vmastat.stream_info[1].size == vmastat.stream_info[1].transferred);
 485 +
 486 +    vma_writer_destroy(vmaw);
 487 +
 488 +    bdrv_delete(bs);
 489 +
 490 +    /* start verification */
 491 +    verify_archive(TEST_VMA_NAME, TEST_IMG_SIZE);
 492 +
 493 +    bdrv_close_all();
 494 +
 495 +    unlink(TEST_IMG_NAME);
 496 +    unlink(TEST_VMA_NAME);
 497 +
 498 +    printf("finish test #%d '%s' OK\n", test_counter, testname);
 499 +}
 500 +
 501 +static void help(void)
 502 +{
 503 +    const char *help_msg =
 504 +        "usage: backup-test [options]\n"
 505 +        "\n"
 506 +        "backup-test        run default regression test (fast)\n"
 507 +        "backup-test -l     run long running test loop (endless)\n"
 508 +        "\n"
 509 +        "use option -d to turn on verbose debug output\n"
 510 +        ;
 511 +
 512 +    printf("%s", help_msg);
 513 +    exit(1);
 514 +}
 515 +
 516 +int main(int argc, char **argv)
 517 +{
 518 +    int c;
 519 +
 520 +    g_thread_init(NULL);
 521 +
 522 +    for (;;) {
 523 +        c = getopt(argc, argv, "hdl");
 524 +        if (c == -1) {
 525 +            break;
 526 +        }
 527 +        switch (c) {
 528 +        case '?':
 529 +        case 'h':
 530 +            help();
 531 +            break;
 532 +        case 'd':
 533 +            opt_debug = 1;
 534 +            break;
 535 +        case 'l':
 536 +            opt_loop = 1;
 537 +            break;
 538 +        default:
 539 +            g_assert_not_reached();
 540 +        }
 541 +    }
 542 +
 543 +    memset(buf_sec_pattern_cd, 0xcd, sizeof(buf_sec_pattern_cd));
 544 +    memset(buf_sec_pattern_32, 0x32, sizeof(buf_sec_pattern_32));
 545 +
 546 +    srand(1234);
 547 +
 548 +    /* ignore SIGPIPE */
 549 +    struct sigaction act;
 550 +    sigfillset(&act.sa_mask);
 551 +    act.sa_flags = 0;
 552 +    act.sa_handler = SIG_IGN;
 553 +    sigaction(SIGPIPE, &act, NULL);
 554 +
 555 +    qemu_init_main_loop();
 556 +
 557 +    bdrv_init();
 558 +
 559 +    if (opt_loop) { /* endless test loop */
 560 +        while (1) {
 561 +            RUN_TEST(random_read_write_test, 0);
 562 +        }
 563 +        return 0;
 564 +    }
 565 +
 566 +    if (opt_debug) { /* run simple test (rate limited) */
 567 +        RUN_TEST(simple_test, 1024*1024);
 568 +        return 0;
 569 +    }
 570 +
 571 +    /* run default regression tests at full speed */
 572 +
 573 +    RUN_TEST(simple_test, 0);
 574 +    RUN_TEST(random_read_write_test, 0);
 575 +
 576 +    return 0;
 577 +}
 578 --
 579 1.7.2.5
 580