debian/patches/0005-add-regression-tests-for-backup.patch

   1 From d22192dbb5515b4bf0bf2ceed7d16cf6a18769f9 Mon Sep 17 00:00:00 2001
   2 From: Dietmar Maurer <dietmar@proxmox.com>
   3 Date: Wed, 14 Nov 2012 09:57:04 +0100
   4 Subject: [PATCH v3 5/6] add regression tests for backup
   5
   6 Also added the ratelimit.h fix here, because it is still not upstream.
   7
   8 Signed-off-by: Dietmar Maurer <dietmar@proxmox.com>
   9 ---
  10  include/qemu/ratelimit.h |    2 +-
  11  tests/Makefile           |   11 +-
  12  tests/backup-test.c      |  511 ++++++++++++++++++++++++++++++++++++++++++++++
  13  3 files changed, 521 insertions(+), 3 deletions(-)
  14  create mode 100644 tests/backup-test.c
  15
  16 diff --git a/include/qemu/ratelimit.h b/include/qemu/ratelimit.h
  17 index c6ac281..d1610f1 100644
  18 --- a/include/qemu/ratelimit.h
  19 +++ b/include/qemu/ratelimit.h
  20 @@ -42,7 +42,7 @@ static inline void ratelimit_set_speed(RateLimit *limit, uint64_t speed,
  21                                         uint64_t slice_ns)
  22  {
  23      limit->slice_ns = slice_ns;
  24 -    limit->slice_quota = ((double)speed * 1000000000ULL) / slice_ns;
  25 +    limit->slice_quota = ((double)speed * slice_ns)/1000000000ULL;
  26  }
  27
  28  #endif
  29 diff --git a/tests/Makefile b/tests/Makefile
  30 index b60f0fb..cffbd22 100644
  31 --- a/tests/Makefile
  32 +++ b/tests/Makefile
  33 @@ -20,6 +20,8 @@ check-unit-y += tests/test-thread-pool$(EXESUF)
  34
  35  check-block-$(CONFIG_POSIX) += tests/qemu-iotests-quick.sh
  36
  37 +check-backup-y = tests/backup-test$(EXESUF)
  38 +
  39  # All QTests for now are POSIX-only, but the dependencies are
  40  # really in libqtest, not in the testcases themselves.
  41  check-qtest-i386-y = tests/fdc-test$(EXESUF)
  42 @@ -54,6 +56,7 @@ tests/test-coroutine$(EXESUF): tests/test-coroutine.o $(coroutine-obj-y) $(tools
  43  tests/test-aio$(EXESUF): tests/test-aio.o $(coroutine-obj-y) $(tools-obj-y) $(block-obj-y) libqemustub.a
  44  tests/test-thread-pool$(EXESUF): tests/test-thread-pool.o $(coroutine-obj-y) $(tools-obj-y) $(block-obj-y) libqemustub.a
  45  tests/test-iov$(EXESUF): tests/test-iov.o iov.o
  46 +tests/backup-test$(EXESUF): tests/backup-test.o vma-reader.o $(tools-obj-y) $(block-obj-y) libqemustub.a
  47
  48  tests/test-qapi-types.c tests/test-qapi-types.h :\
  49  $(SRC_PATH)/qapi-schema-test.json $(SRC_PATH)/scripts/qapi-types.py
  50 @@ -146,10 +149,14 @@ check-tests/qemu-iotests-quick.sh: tests/qemu-iotests-quick.sh qemu-img$(EXESUF)
  51
  52  # Consolidated targets
  53
  54 -.PHONY: check-qtest check-unit check
  55 +.PHONY: check-backup check-qtest check-unit check
  56  check-qtest: $(patsubst %,check-qtest-%, $(QTEST_TARGETS))
  57  check-unit: $(patsubst %,check-%, $(check-unit-y))
  58  check-block: $(patsubst %,check-%, $(check-block-y))
  59 -check: check-unit check-qtest
  60 +
  61 +check-backup: tests/backup-test$(EXESUF)
  62 +       $<
  63 +
  64 +check: check-unit check-qtest check-backup
  65
  66  -include $(wildcard tests/*.d)
  67 diff --git a/tests/backup-test.c b/tests/backup-test.c
  68 new file mode 100644
  69 index 0000000..c434c5f
  70 --- /dev/null
  71 +++ b/tests/backup-test.c
  72 @@ -0,0 +1,511 @@
  73 +/*
  74 + * QEMU backup test suit
  75 + *
  76 + * Copyright (C) Proxmox Server Solutions
  77 + *
  78 + * Authors:
  79 + *  Dietmar Maurer (dietmar@proxmox.com)
  80 + *
  81 + * This work is licensed under the terms of the GNU GPL, version 2.  See
  82 + * the COPYING file in the top-level directory.
  83 + *
  84 + * Fixme: running 'backup-test -l' trigger a bug in g_slice_alloc()
  85 + * Note: 'G_SLICE=always-malloc ./tests/backup-test -l' works
  86 + *
  87 + */
  88 +
  89 +#include <sys/time.h>
  90 +#include <sys/types.h>
  91 +#include <stdarg.h>
  92 +#include <stdio.h>
  93 +#include <getopt.h>
  94 +#include <libgen.h>
  95 +
  96 +#include "qemu-common.h"
  97 +#include "main-loop.h"
  98 +#include "block_int.h"
  99 +
 100 +#include "vma.h"
 101 +
 102 +static int opt_debug;
 103 +static int opt_loop;
 104 +
 105 +#define DPRINTF(fmt, ...) \
 106 +    do { if (opt_debug) { printf(fmt, ## __VA_ARGS__); } } while (0)
 107 +
 108 +#define CLUSTER(x) (x*BACKUP_CLUSTER_SIZE)
 109 +
 110 +#define RUN_TEST(testfunc, speed) \
 111 +    backup_test(#testfunc " speed " #speed, speed, testfunc);
 112 +
 113 +
 114 +static unsigned char buf_sec_pattern_cd[BDRV_SECTOR_SIZE];
 115 +static unsigned char buf_sec_pattern_32[BDRV_SECTOR_SIZE];
 116 +
 117 +#define TEST_IMG_SIZE (6*1024*1024+BDRV_SECTOR_SIZE)
 118 +#define TEST_IMG_NAME "backuptest.raw"
 119 +#define TEST_IMG_RESTORE_NAME "backuptest.raw.restore"
 120 +#define TEST_VMA_NAME "backuptest.vma"
 121 +
 122 +typedef struct BackupCB {
 123 +    VmaWriter *vmaw;
 124 +    uint8_t dev_id;
 125 +} BackupCB;
 126 +
 127 +static int backup_dump_cb(void *opaque, BlockDriverState *bs,
 128 +                          int64_t cluster_num, unsigned char *buf)
 129 +{
 130 +    BackupCB *bcb = opaque;
 131 +
 132 +    DPRINTF("backup_dump_cb C%zd %d\n", cluster_num, bcb->dev_id);
 133 +
 134 +    size_t zb = 0;
 135 +    if (vma_writer_write(bcb->vmaw, bcb->dev_id, cluster_num, buf, &zb) < 0) {
 136 +        printf("backup_dump_cb vma_writer_write failed\n");
 137 +        return -1;
 138 +    }
 139 +
 140 +    return 0;
 141 +}
 142 +
 143 +static void backup_complete_cb(void *opaque, int ret)
 144 +{
 145 +    BackupCB *bcb = opaque;
 146 +
 147 +    DPRINTF("backup_complete_cb %d %d\n", bcb->dev_id, ret);
 148 +
 149 +    if (ret < 0) {
 150 +        vma_writer_set_error(bcb->vmaw, "backup_complete_cb %d", ret);
 151 +    }
 152 +
 153 +    if (vma_writer_close_stream(bcb->vmaw, bcb->dev_id) <= 0) {
 154 +        Error *err = NULL;
 155 +        if (vma_writer_close(bcb->vmaw, &err) != 0) {
 156 +            g_error("vma_writer_close failed %s", error_get_pretty(err));
 157 +        }
 158 +    }
 159 +    DPRINTF("backup_complete_cb finish\n");
 160 +}
 161 +
 162 +static void write_sec_pattern_cd(BlockDriverState *bs, int64_t offset)
 163 +{
 164 +    int ret;
 165 +
 166 +    DPRINTF("write_sec_pattern_cd %zd\n", offset);
 167 +
 168 +    if (offset & 0x1ff) {
 169 +        g_error("write_sec_pattern_cd offset %zd is not sector aligned\n",
 170 +                offset);
 171 +    }
 172 +
 173 +    ret = bdrv_write(bs, offset >> 9, buf_sec_pattern_cd, 1);
 174 +    if (ret < 0) {
 175 +        g_error("write_sec_pattern_cd %zd failed", offset);
 176 +    }
 177 +
 178 +}
 179 +
 180 +static void read_sec(BlockDriverState *bs, int64_t offset, unsigned char *buf)
 181 +{
 182 +    DPRINTF("read_sec C%zd start %zd\n", offset>>VMA_CLUSTER_BITS, offset);
 183 +
 184 +    if (offset & 0x1ff) {
 185 +        g_error("read_sec offset %zd is not sector aligned\n", offset);
 186 +    }
 187 +
 188 +    if (bdrv_read(bs, offset >> 9, buf, 1) < 0) {
 189 +        g_error("bdrv_read failed");
 190 +    }
 191 +}
 192 +
 193 +static bool request_term;
 194 +
 195 +typedef struct TestCB {
 196 +    Coroutine *co;
 197 +    BlockDriverState *bs;
 198 +    bool finished;
 199 +} TestCB;
 200 +
 201 +static TestCB *enter_test_co(BlockDriverState *bs, CoroutineEntry *entry)
 202 +{
 203 +    TestCB *cb = g_new0(TestCB, 1);
 204 +    cb->bs = bs;
 205 +    cb->co = qemu_coroutine_create(entry);
 206 +    qemu_coroutine_enter(cb->co, cb);
 207 +    return cb;
 208 +}
 209 +
 210 +static void test_co_sleep(double sec)
 211 +{
 212 +    co_sleep_ns(rt_clock, (int64_t)(sec*1000000000));
 213 +};
 214 +
 215 +static void test_co_yield(void)
 216 +{
 217 +    co_sleep_ns(rt_clock, (int64_t)(1000));
 218 +};
 219 +
 220 +static void coroutine_fn run_co_test1(void *opaque)
 221 +{
 222 +    assert(opaque);
 223 +    TestCB *cb = (TestCB *)opaque;
 224 +
 225 +    test_co_sleep(0.2);
 226 +    write_sec_pattern_cd(cb->bs, 5*BACKUP_CLUSTER_SIZE);
 227 +    test_co_sleep(0.2);
 228 +    write_sec_pattern_cd(cb->bs, 10*BACKUP_CLUSTER_SIZE);
 229 +    test_co_sleep(0.2);
 230 +    write_sec_pattern_cd(cb->bs, 10*BACKUP_CLUSTER_SIZE);
 231 +
 232 +    cb->finished = true;
 233 +}
 234 +
 235 +static void coroutine_fn run_co_test2(void *opaque)
 236 +{
 237 +    assert(opaque);
 238 +    TestCB *cb = (TestCB *)opaque;
 239 +    unsigned char buf[512];
 240 +
 241 +    test_co_sleep(0.2);
 242 +    read_sec(cb->bs, 5*BACKUP_CLUSTER_SIZE, buf);
 243 +    write_sec_pattern_cd(cb->bs, 6*BACKUP_CLUSTER_SIZE);
 244 +
 245 +    cb->finished = true;
 246 +}
 247 +
 248 +static void coroutine_fn run_co_random_read(void *opaque)
 249 +{
 250 +    assert(opaque);
 251 +    TestCB *cb = (TestCB *)opaque;
 252 +    int64_t sectors = bdrv_getlength(cb->bs)/BDRV_SECTOR_SIZE - 1;
 253 +    unsigned char buf[512];
 254 +
 255 +    while (1) {
 256 +        test_co_yield();
 257 +        if (request_term) {
 258 +            DPRINTF("finish run_co_random_read\n");
 259 +            break;
 260 +        }
 261 +        int64_t s = (rand()*sectors)/RAND_MAX;
 262 +        read_sec(cb->bs, s*BDRV_SECTOR_SIZE, buf);
 263 +    }
 264 +
 265 +    cb->finished = true;
 266 +}
 267 +
 268 +static void coroutine_fn run_co_random_write(void *opaque)
 269 +{
 270 +    assert(opaque);
 271 +    TestCB *cb = (TestCB *)opaque;
 272 +    int64_t sectors = bdrv_getlength(cb->bs)/BDRV_SECTOR_SIZE;
 273 +
 274 +    while (1) {
 275 +        test_co_yield();
 276 +        if (request_term) {
 277 +            DPRINTF("finish run_co_random_write\n");
 278 +            break;
 279 +        }
 280 +        int64_t s = (rand()*sectors)/RAND_MAX;
 281 +        write_sec_pattern_cd(cb->bs, s*BDRV_SECTOR_SIZE);
 282 +    }
 283 +
 284 +    cb->finished = true;
 285 +}
 286 +
 287 +static void fill_test_sector(void *buf, size_t sector_num)
 288 +{
 289 +    int64_t *i64buf = (int64_t *)buf;
 290 +    int i;
 291 +
 292 +    int data = sector_num;
 293 +    if (sector_num >= 8 && sector_num < 8*(2*16+2)) {
 294 +        data = 0;  /* add zero region for testing */
 295 +    }
 296 +
 297 +    for (i = 0; i < (512/sizeof(int64_t)); i++) {
 298 +        i64buf[i] = data;
 299 +    }
 300 +}
 301 +
 302 +static void verify_archive(const char *archive, size_t size)
 303 +{
 304 +    Error *errp = NULL;
 305 +
 306 +    VmaReader *vmar = vma_reader_create(archive, &errp);
 307 +
 308 +    if (!vmar) {
 309 +        g_error("%s", error_get_pretty(errp));
 310 +    }
 311 +
 312 +    VmaDeviceInfo *di = vma_reader_get_device_info(vmar, 1);
 313 +    if (!di || strcmp((char *)di->devname, "hda") || di->size != size) {
 314 +        g_error("got wrong device info");
 315 +    }
 316 +
 317 +    unlink(TEST_IMG_RESTORE_NAME);
 318 +
 319 +    int flags = BDRV_O_NATIVE_AIO|BDRV_O_RDWR|BDRV_O_CACHE_WB;
 320 +
 321 +    if (bdrv_img_create(TEST_IMG_RESTORE_NAME, "raw", NULL, NULL, NULL,
 322 +                        size, flags)) {
 323 +        g_error("can't create file %s", TEST_IMG_RESTORE_NAME);
 324 +    }
 325 +
 326 +    BlockDriverState *bs = NULL;
 327 +    if (bdrv_file_open(&bs, TEST_IMG_RESTORE_NAME, flags)) {
 328 +        g_error("can't open file %s", TEST_IMG_RESTORE_NAME);
 329 +    }
 330 +    if (vma_reader_register_bs(vmar, 1, bs, false, &errp) < 0) {
 331 +        g_error("%s", error_get_pretty(errp));
 332 +    }
 333 +
 334 +    if (vma_reader_restore(vmar, -1, &errp) < 0) {
 335 +        g_error("restore failed - %s", error_get_pretty(errp));
 336 +    }
 337 +
 338 +    size_t i;
 339 +    size_t sectors = size/BDRV_SECTOR_SIZE;
 340 +    int64_t buf[512/sizeof(int64_t)];
 341 +    int64_t buf2[512/sizeof(int64_t)];
 342 +
 343 +    for (i = 0; i < sectors; i++) {
 344 +        if (bdrv_read(bs, i, (uint8_t *)buf, 1) != 0) {
 345 +            g_error("bdrv_read failed");
 346 +        }
 347 +        fill_test_sector(buf2, i);
 348 +        if (bcmp(buf, buf2, sizeof(buf))) {
 349 +            g_error("data is different at sector %zd", i);
 350 +        }
 351 +    }
 352 +
 353 +    vma_reader_destroy(vmar);
 354 +
 355 +    unlink(TEST_IMG_RESTORE_NAME);
 356 +}
 357 +
 358 +static void prepare_vm_image(const char *filename, size_t sectors)
 359 +{
 360 +    int fd = open(filename, O_RDWR|O_CREAT|O_TRUNC, 0644);
 361 +    if (fd < 0) {
 362 +        g_error("can't open file %s\n", filename);
 363 +    }
 364 +
 365 +    size_t i;
 366 +    int64_t buf[512/sizeof(int64_t)];
 367 +
 368 +    for (i = 0; i < sectors; i++) {
 369 +        fill_test_sector(buf, i);
 370 +
 371 +         int res = 0;
 372 +        while (1) {
 373 +            res = write(fd, buf, sizeof(buf));
 374 +            if (!(res < 0 && errno == EINTR)) {
 375 +                break;
 376 +            }
 377 +        }
 378 +        if (res != sizeof(buf)) {
 379 +            g_error("can't initialize file %s - %s %d\n",
 380 +                    filename, strerror(errno), res);
 381 +        }
 382 +    }
 383 +
 384 +    if (close(fd) != 0) {
 385 +        g_error("close failed");
 386 +    }
 387 +}
 388 +
 389 +static GList *simple_test(BlockDriverState *bs)
 390 +{
 391 +    GList *cb_list = NULL;
 392 +
 393 +    cb_list = g_list_append(cb_list, enter_test_co(bs, run_co_test1));
 394 +    cb_list = g_list_append(cb_list, enter_test_co(bs, run_co_test2));
 395 +
 396 +    return cb_list;
 397 +}
 398 +
 399 +static GList *random_read_write_test(BlockDriverState *bs)
 400 +{
 401 +    GList *cb_list = NULL;
 402 +
 403 +    cb_list = g_list_append(cb_list, enter_test_co(bs, run_co_random_read));
 404 +    cb_list = g_list_append(cb_list, enter_test_co(bs, run_co_random_read));
 405 +    cb_list = g_list_append(cb_list, enter_test_co(bs, run_co_random_write));
 406 +    cb_list = g_list_append(cb_list, enter_test_co(bs, run_co_random_write));
 407 +
 408 +    return cb_list;
 409 +}
 410 +
 411 +static void backup_test(const char *testname, int64_t speed,
 412 +                        GList *(*start_test_cb)(BlockDriverState *bs))
 413 +{
 414 +    BlockDriverState *bs = bdrv_new("hda");
 415 +
 416 +    static int test_counter;
 417 +
 418 +    test_counter++;
 419 +
 420 +    printf("starting test #%d '%s'\n", test_counter, testname);
 421 +
 422 +    const char *filename = TEST_IMG_NAME;
 423 +
 424 +    prepare_vm_image(TEST_IMG_NAME, TEST_IMG_SIZE/BDRV_SECTOR_SIZE);
 425 +
 426 +    int flags = BDRV_O_NATIVE_AIO|BDRV_O_RDWR|BDRV_O_CACHE_WB;
 427 +
 428 +    if (bdrv_open(bs, filename, flags, NULL) < 0) {
 429 +        g_error("can't open device %s\n", filename);
 430 +    }
 431 +
 432 +    Error *err = NULL;
 433 +    uuid_t uuid;
 434 +    uuid_generate(uuid);
 435 +
 436 +    unlink(TEST_VMA_NAME);
 437 +
 438 +    VmaWriter *vmaw = vma_writer_create(TEST_VMA_NAME, uuid, speed, &err);
 439 +    if (!vmaw) {
 440 +        g_error("%s", error_get_pretty(err));
 441 +    }
 442 +
 443 +    BackupCB bcb;
 444 +    bcb.vmaw = vmaw;
 445 +    bcb.dev_id = vma_writer_register_stream(vmaw, bdrv_get_device_name(bs),
 446 +                                            bdrv_getlength(bs));
 447 +    if (backup_job_start(bs, backup_dump_cb, backup_complete_cb, &bcb) < 0) {
 448 +        g_error("backup_job_start failed");
 449 +    }
 450 +
 451 +    request_term = false;
 452 +
 453 +    GList *cb_list = start_test_cb(bs);
 454 +
 455 +    while (1) {
 456 +        main_loop_wait(false);
 457 +
 458 +        VmaStatus vmastat;
 459 +        vma_writer_get_status(vmaw, &vmastat);
 460 +        if (vmastat.closed) {
 461 +            break;
 462 +        }
 463 +    }
 464 +
 465 +    request_term = true;
 466 +
 467 +    while (1) {
 468 +        GList *l = cb_list;
 469 +        bool active = 0;
 470 +        while (l && l->data) {
 471 +            TestCB *cb = (TestCB *)l->data;
 472 +            l = g_list_next(l);
 473 +            if (!cb->finished) {
 474 +                active = true;
 475 +                break;
 476 +            }
 477 +        }
 478 +        if (!active) {
 479 +            DPRINTF("All test coroutines finished\n");
 480 +            break;
 481 +        }
 482 +        main_loop_wait(false);
 483 +    }
 484 +
 485 +    /* Make sure all outstanding requests complete */
 486 +    bdrv_drain_all();
 487 +
 488 +    VmaStatus vmastat;
 489 +    vma_writer_get_status(vmaw, &vmastat);
 490 +    DPRINTF("statistic %zd %zd\n", vmastat.stream_info[1].size,
 491 +            vmastat.stream_info[1].transferred);
 492 +    assert(vmastat.stream_info[1].size == vmastat.stream_info[1].transferred);
 493 +
 494 +    vma_writer_destroy(vmaw);
 495 +
 496 +    bdrv_delete(bs);
 497 +
 498 +    /* start verification */
 499 +    verify_archive(TEST_VMA_NAME, TEST_IMG_SIZE);
 500 +
 501 +    bdrv_close_all();
 502 +
 503 +    unlink(TEST_IMG_NAME);
 504 +    unlink(TEST_VMA_NAME);
 505 +
 506 +    printf("finish test #%d '%s' OK\n", test_counter, testname);
 507 +}
 508 +
 509 +static void help(void)
 510 +{
 511 +    const char *help_msg =
 512 +        "usage: backup-test [options]\n"
 513 +        "\n"
 514 +        "backup-test        run default regression test (fast)\n"
 515 +        "backup-test -l     run long running test loop (endless)\n"
 516 +        "\n"
 517 +        "use option -d to turn on verbose debug output\n"
 518 +        ;
 519 +
 520 +    printf("%s", help_msg);
 521 +    exit(1);
 522 +}
 523 +
 524 +int main(int argc, char **argv)
 525 +{
 526 +    int c;
 527 +
 528 +    for (;;) {
 529 +        c = getopt(argc, argv, "hdl");
 530 +        if (c == -1) {
 531 +            break;
 532 +        }
 533 +        switch (c) {
 534 +        case '?':
 535 +        case 'h':
 536 +            help();
 537 +            break;
 538 +        case 'd':
 539 +            opt_debug = 1;
 540 +            break;
 541 +        case 'l':
 542 +            opt_loop = 1;
 543 +            break;
 544 +        default:
 545 +            g_assert_not_reached();
 546 +        }
 547 +    }
 548 +
 549 +    memset(buf_sec_pattern_cd, 0xcd, sizeof(buf_sec_pattern_cd));
 550 +    memset(buf_sec_pattern_32, 0x32, sizeof(buf_sec_pattern_32));
 551 +
 552 +    srand(1234);
 553 +
 554 +    /* ignore SIGPIPE */
 555 +    struct sigaction act;
 556 +    sigfillset(&act.sa_mask);
 557 +    act.sa_flags = 0;
 558 +    act.sa_handler = SIG_IGN;
 559 +    sigaction(SIGPIPE, &act, NULL);
 560 +
 561 +    qemu_init_main_loop();
 562 +
 563 +    bdrv_init();
 564 +
 565 +    if (opt_loop) { /* endless test loop */
 566 +        while (1) {
 567 +            RUN_TEST(random_read_write_test, 0);
 568 +        }
 569 +        return 0;
 570 +    }
 571 +
 572 +    if (opt_debug) { /* run simple test (rate limited) */
 573 +        RUN_TEST(simple_test, 1024*1024);
 574 +        return 0;
 575 +    }
 576 +
 577 +    /* run default regression tests at full speed */
 578 +
 579 +    RUN_TEST(simple_test, 0);
 580 +    RUN_TEST(random_read_write_test, 0);
 581 +
 582 +    return 0;
 583 +}
 584 --
 585 1.7.2.5
 586