migration/dirtyrate.c

   1 /*
   2  * Dirtyrate implement code
   3  *
   4  * Copyright (c) 2020 HUAWEI TECHNOLOGIES CO.,LTD.
   5  *
   6  * Authors:
   7  *  Chuan Zheng <zhengchuan@huawei.com>
   8  *
   9  * This work is licensed under the terms of the GNU GPL, version 2 or later.
  10  * See the COPYING file in the top-level directory.
  11  */
  12
  13 #include "qemu/osdep.h"
  14 #include "qemu/error-report.h"
  15 #include <zlib.h>
  16 #include "qapi/error.h"
  17 #include "cpu.h"
  18 #include "exec/ramblock.h"
  19 #include "exec/target_page.h"
  20 #include "exec/ram_addr.h"
  21 #include "qemu/rcu_queue.h"
  22 #include "qemu/main-loop.h"
  23 #include "qapi/qapi-commands-migration.h"
  24 #include "ram.h"
  25 #include "trace.h"
  26 #include "dirtyrate.h"
  27 #include "monitor/hmp.h"
  28 #include "monitor/monitor.h"
  29 #include "qapi/qmp/qdict.h"
  30 #include "sysemu/kvm.h"
  31 #include "sysemu/runstate.h"
  32 #include "exec/memory.h"
  33 #include "qemu/xxhash.h"
  34
  35 /*
  36  * total_dirty_pages is procted by BQL and is used
  37  * to stat dirty pages during the period of two
  38  * memory_global_dirty_log_sync
  39  */
  40 uint64_t total_dirty_pages;
  41
  42 typedef struct DirtyPageRecord {
  43     uint64_t start_pages;
  44     uint64_t end_pages;
  45 } DirtyPageRecord;
  46
  47 static int CalculatingState = DIRTY_RATE_STATUS_UNSTARTED;
  48 static struct DirtyRateStat DirtyStat;
  49 static DirtyRateMeasureMode dirtyrate_mode =
  50                 DIRTY_RATE_MEASURE_MODE_PAGE_SAMPLING;
  51
  52 static int64_t dirty_stat_wait(int64_t msec, int64_t initial_time)
  53 {
  54     int64_t current_time;
  55
  56     current_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
  57     if ((current_time - initial_time) >= msec) {
  58         msec = current_time - initial_time;
  59     } else {
  60         g_usleep((msec + initial_time - current_time) * 1000);
  61     }
  62
  63     return msec;
  64 }
  65
  66 static inline void record_dirtypages(DirtyPageRecord *dirty_pages,
  67                                      CPUState *cpu, bool start)
  68 {
  69     if (start) {
  70         dirty_pages[cpu->cpu_index].start_pages = cpu->dirty_pages;
  71     } else {
  72         dirty_pages[cpu->cpu_index].end_pages = cpu->dirty_pages;
  73     }
  74 }
  75
  76 static int64_t do_calculate_dirtyrate(DirtyPageRecord dirty_pages,
  77                                       int64_t calc_time_ms)
  78 {
  79     uint64_t increased_dirty_pages =
  80         dirty_pages.end_pages - dirty_pages.start_pages;
  81     uint64_t memory_size_MiB = qemu_target_pages_to_MiB(increased_dirty_pages);
  82
  83     return memory_size_MiB * 1000 / calc_time_ms;
  84 }
  85
  86 void global_dirty_log_change(unsigned int flag, bool start)
  87 {
  88     qemu_mutex_lock_iothread();
  89     if (start) {
  90         memory_global_dirty_log_start(flag);
  91     } else {
  92         memory_global_dirty_log_stop(flag);
  93     }
  94     qemu_mutex_unlock_iothread();
  95 }
  96
  97 /*
  98  * global_dirty_log_sync
  99  * 1. sync dirty log from kvm
 100  * 2. stop dirty tracking if needed.
 101  */
 102 static void global_dirty_log_sync(unsigned int flag, bool one_shot)
 103 {
 104     qemu_mutex_lock_iothread();
 105     memory_global_dirty_log_sync();
 106     if (one_shot) {
 107         memory_global_dirty_log_stop(flag);
 108     }
 109     qemu_mutex_unlock_iothread();
 110 }
 111
 112 static DirtyPageRecord *vcpu_dirty_stat_alloc(VcpuStat *stat)
 113 {
 114     CPUState *cpu;
 115     int nvcpu = 0;
 116
 117     CPU_FOREACH(cpu) {
 118         nvcpu++;
 119     }
 120
 121     stat->nvcpu = nvcpu;
 122     stat->rates = g_new0(DirtyRateVcpu, nvcpu);
 123
 124     return g_new0(DirtyPageRecord, nvcpu);
 125 }
 126
 127 static void vcpu_dirty_stat_collect(VcpuStat *stat,
 128                                     DirtyPageRecord *records,
 129                                     bool start)
 130 {
 131     CPUState *cpu;
 132
 133     CPU_FOREACH(cpu) {
 134         record_dirtypages(records, cpu, start);
 135     }
 136 }
 137
 138 int64_t vcpu_calculate_dirtyrate(int64_t calc_time_ms,
 139                                  VcpuStat *stat,
 140                                  unsigned int flag,
 141                                  bool one_shot)
 142 {
 143     DirtyPageRecord *records;
 144     int64_t init_time_ms;
 145     int64_t duration;
 146     int64_t dirtyrate;
 147     int i = 0;
 148     unsigned int gen_id;
 149
 150 retry:
 151     init_time_ms = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
 152
 153     WITH_QEMU_LOCK_GUARD(&qemu_cpu_list_lock) {
 154         gen_id = cpu_list_generation_id_get();
 155         records = vcpu_dirty_stat_alloc(stat);
 156         vcpu_dirty_stat_collect(stat, records, true);
 157     }
 158
 159     duration = dirty_stat_wait(calc_time_ms, init_time_ms);
 160
 161     global_dirty_log_sync(flag, one_shot);
 162
 163     WITH_QEMU_LOCK_GUARD(&qemu_cpu_list_lock) {
 164         if (gen_id != cpu_list_generation_id_get()) {
 165             g_free(records);
 166             g_free(stat->rates);
 167             cpu_list_unlock();
 168             goto retry;
 169         }
 170         vcpu_dirty_stat_collect(stat, records, false);
 171     }
 172
 173     for (i = 0; i < stat->nvcpu; i++) {
 174         dirtyrate = do_calculate_dirtyrate(records[i], duration);
 175
 176         stat->rates[i].id = i;
 177         stat->rates[i].dirty_rate = dirtyrate;
 178
 179         trace_dirtyrate_do_calculate_vcpu(i, dirtyrate);
 180     }
 181
 182     g_free(records);
 183
 184     return duration;
 185 }
 186
 187 static bool is_sample_period_valid(int64_t sec)
 188 {
 189     if (sec < MIN_FETCH_DIRTYRATE_TIME_SEC ||
 190         sec > MAX_FETCH_DIRTYRATE_TIME_SEC) {
 191         return false;
 192     }
 193
 194     return true;
 195 }
 196
 197 static bool is_sample_pages_valid(int64_t pages)
 198 {
 199     return pages >= MIN_SAMPLE_PAGE_COUNT &&
 200            pages <= MAX_SAMPLE_PAGE_COUNT;
 201 }
 202
 203 static int dirtyrate_set_state(int *state, int old_state, int new_state)
 204 {
 205     assert(new_state < DIRTY_RATE_STATUS__MAX);
 206     trace_dirtyrate_set_state(DirtyRateStatus_str(new_state));
 207     if (qatomic_cmpxchg(state, old_state, new_state) == old_state) {
 208         return 0;
 209     } else {
 210         return -1;
 211     }
 212 }
 213
 214 static struct DirtyRateInfo *query_dirty_rate_info(void)
 215 {
 216     int i;
 217     int64_t dirty_rate = DirtyStat.dirty_rate;
 218     struct DirtyRateInfo *info = g_new0(DirtyRateInfo, 1);
 219     DirtyRateVcpuList *head = NULL, **tail = &head;
 220
 221     info->status = CalculatingState;
 222     info->start_time = DirtyStat.start_time;
 223     info->calc_time = DirtyStat.calc_time;
 224     info->sample_pages = DirtyStat.sample_pages;
 225     info->mode = dirtyrate_mode;
 226
 227     if (qatomic_read(&CalculatingState) == DIRTY_RATE_STATUS_MEASURED) {
 228         info->has_dirty_rate = true;
 229         info->dirty_rate = dirty_rate;
 230
 231         if (dirtyrate_mode == DIRTY_RATE_MEASURE_MODE_DIRTY_RING) {
 232             /*
 233              * set sample_pages with 0 to indicate page sampling
 234              * isn't enabled
 235              **/
 236             info->sample_pages = 0;
 237             info->has_vcpu_dirty_rate = true;
 238             for (i = 0; i < DirtyStat.dirty_ring.nvcpu; i++) {
 239                 DirtyRateVcpu *rate = g_new0(DirtyRateVcpu, 1);
 240                 rate->id = DirtyStat.dirty_ring.rates[i].id;
 241                 rate->dirty_rate = DirtyStat.dirty_ring.rates[i].dirty_rate;
 242                 QAPI_LIST_APPEND(tail, rate);
 243             }
 244             info->vcpu_dirty_rate = head;
 245         }
 246
 247         if (dirtyrate_mode == DIRTY_RATE_MEASURE_MODE_DIRTY_BITMAP) {
 248             info->sample_pages = 0;
 249         }
 250     }
 251
 252     trace_query_dirty_rate_info(DirtyRateStatus_str(CalculatingState));
 253
 254     return info;
 255 }
 256
 257 static void init_dirtyrate_stat(int64_t start_time,
 258                                 struct DirtyRateConfig config)
 259 {
 260     DirtyStat.dirty_rate = -1;
 261     DirtyStat.start_time = start_time;
 262     DirtyStat.calc_time = config.sample_period_seconds;
 263     DirtyStat.sample_pages = config.sample_pages_per_gigabytes;
 264
 265     switch (config.mode) {
 266     case DIRTY_RATE_MEASURE_MODE_PAGE_SAMPLING:
 267         DirtyStat.page_sampling.total_dirty_samples = 0;
 268         DirtyStat.page_sampling.total_sample_count = 0;
 269         DirtyStat.page_sampling.total_block_mem_MB = 0;
 270         break;
 271     case DIRTY_RATE_MEASURE_MODE_DIRTY_RING:
 272         DirtyStat.dirty_ring.nvcpu = -1;
 273         DirtyStat.dirty_ring.rates = NULL;
 274         break;
 275     default:
 276         break;
 277     }
 278 }
 279
 280 static void cleanup_dirtyrate_stat(struct DirtyRateConfig config)
 281 {
 282     /* last calc-dirty-rate qmp use dirty ring mode */
 283     if (dirtyrate_mode == DIRTY_RATE_MEASURE_MODE_DIRTY_RING) {
 284         free(DirtyStat.dirty_ring.rates);
 285         DirtyStat.dirty_ring.rates = NULL;
 286     }
 287 }
 288
 289 static void update_dirtyrate_stat(struct RamblockDirtyInfo *info)
 290 {
 291     DirtyStat.page_sampling.total_dirty_samples += info->sample_dirty_count;
 292     DirtyStat.page_sampling.total_sample_count += info->sample_pages_count;
 293     /* size of total pages in MB */
 294     DirtyStat.page_sampling.total_block_mem_MB +=
 295         qemu_target_pages_to_MiB(info->ramblock_pages);
 296 }
 297
 298 static void update_dirtyrate(uint64_t msec)
 299 {
 300     uint64_t dirtyrate;
 301     uint64_t total_dirty_samples = DirtyStat.page_sampling.total_dirty_samples;
 302     uint64_t total_sample_count = DirtyStat.page_sampling.total_sample_count;
 303     uint64_t total_block_mem_MB = DirtyStat.page_sampling.total_block_mem_MB;
 304
 305     dirtyrate = total_dirty_samples * total_block_mem_MB *
 306                 1000 / (total_sample_count * msec);
 307
 308     DirtyStat.dirty_rate = dirtyrate;
 309 }
 310
 311 /*
 312  * Compute hash of a single page of size TARGET_PAGE_SIZE.
 313  */
 314 static uint32_t compute_page_hash(void *ptr)
 315 {
 316     size_t page_size = qemu_target_page_size();
 317     uint32_t i;
 318     uint64_t v1, v2, v3, v4;
 319     uint64_t res;
 320     const uint64_t *p = ptr;
 321
 322     v1 = QEMU_XXHASH_SEED + XXH_PRIME64_1 + XXH_PRIME64_2;
 323     v2 = QEMU_XXHASH_SEED + XXH_PRIME64_2;
 324     v3 = QEMU_XXHASH_SEED + 0;
 325     v4 = QEMU_XXHASH_SEED - XXH_PRIME64_1;
 326     for (i = 0; i < page_size / 8; i += 4) {
 327         v1 = XXH64_round(v1, p[i + 0]);
 328         v2 = XXH64_round(v2, p[i + 1]);
 329         v3 = XXH64_round(v3, p[i + 2]);
 330         v4 = XXH64_round(v4, p[i + 3]);
 331     }
 332     res = XXH64_mergerounds(v1, v2, v3, v4);
 333     res += page_size;
 334     res = XXH64_avalanche(res);
 335     return (uint32_t)(res & UINT32_MAX);
 336 }
 337
 338
 339 /*
 340  * get hash result for the sampled memory with length of TARGET_PAGE_SIZE
 341  * in ramblock, which starts from ramblock base address.
 342  */
 343 static uint32_t get_ramblock_vfn_hash(struct RamblockDirtyInfo *info,
 344                                       uint64_t vfn)
 345 {
 346     uint32_t hash;
 347
 348     hash = compute_page_hash(info->ramblock_addr +
 349                              vfn * qemu_target_page_size());
 350
 351     trace_get_ramblock_vfn_hash(info->idstr, vfn, hash);
 352     return hash;
 353 }
 354
 355 static bool save_ramblock_hash(struct RamblockDirtyInfo *info)
 356 {
 357     unsigned int sample_pages_count;
 358     int i;
 359     GRand *rand;
 360
 361     sample_pages_count = info->sample_pages_count;
 362
 363     /* ramblock size less than one page, return success to skip this ramblock */
 364     if (unlikely(info->ramblock_pages == 0 || sample_pages_count == 0)) {
 365         return true;
 366     }
 367
 368     info->hash_result = g_try_malloc0_n(sample_pages_count,
 369                                         sizeof(uint32_t));
 370     if (!info->hash_result) {
 371         return false;
 372     }
 373
 374     info->sample_page_vfn = g_try_malloc0_n(sample_pages_count,
 375                                             sizeof(uint64_t));
 376     if (!info->sample_page_vfn) {
 377         g_free(info->hash_result);
 378         return false;
 379     }
 380
 381     rand  = g_rand_new();
 382     for (i = 0; i < sample_pages_count; i++) {
 383         info->sample_page_vfn[i] = g_rand_int_range(rand, 0,
 384                                                     info->ramblock_pages - 1);
 385         info->hash_result[i] = get_ramblock_vfn_hash(info,
 386                                                      info->sample_page_vfn[i]);
 387     }
 388     g_rand_free(rand);
 389
 390     return true;
 391 }
 392
 393 static void get_ramblock_dirty_info(RAMBlock *block,
 394                                     struct RamblockDirtyInfo *info,
 395                                     struct DirtyRateConfig *config)
 396 {
 397     uint64_t sample_pages_per_gigabytes = config->sample_pages_per_gigabytes;
 398
 399     /* Right shift 30 bits to calc ramblock size in GB */
 400     info->sample_pages_count = (qemu_ram_get_used_length(block) *
 401                                 sample_pages_per_gigabytes) >> 30;
 402     /* Right shift TARGET_PAGE_BITS to calc page count */
 403     info->ramblock_pages = qemu_ram_get_used_length(block) >>
 404                            TARGET_PAGE_BITS;
 405     info->ramblock_addr = qemu_ram_get_host_addr(block);
 406     strcpy(info->idstr, qemu_ram_get_idstr(block));
 407 }
 408
 409 static void free_ramblock_dirty_info(struct RamblockDirtyInfo *infos, int count)
 410 {
 411     int i;
 412
 413     if (!infos) {
 414         return;
 415     }
 416
 417     for (i = 0; i < count; i++) {
 418         g_free(infos[i].sample_page_vfn);
 419         g_free(infos[i].hash_result);
 420     }
 421     g_free(infos);
 422 }
 423
 424 static bool skip_sample_ramblock(RAMBlock *block)
 425 {
 426     /*
 427      * Sample only blocks larger than MIN_RAMBLOCK_SIZE.
 428      */
 429     if (qemu_ram_get_used_length(block) < (MIN_RAMBLOCK_SIZE << 10)) {
 430         trace_skip_sample_ramblock(block->idstr,
 431                                    qemu_ram_get_used_length(block));
 432         return true;
 433     }
 434
 435     return false;
 436 }
 437
 438 static bool record_ramblock_hash_info(struct RamblockDirtyInfo **block_dinfo,
 439                                       struct DirtyRateConfig config,
 440                                       int *block_count)
 441 {
 442     struct RamblockDirtyInfo *info = NULL;
 443     struct RamblockDirtyInfo *dinfo = NULL;
 444     RAMBlock *block = NULL;
 445     int total_count = 0;
 446     int index = 0;
 447     bool ret = false;
 448
 449     RAMBLOCK_FOREACH_MIGRATABLE(block) {
 450         if (skip_sample_ramblock(block)) {
 451             continue;
 452         }
 453         total_count++;
 454     }
 455
 456     dinfo = g_try_malloc0_n(total_count, sizeof(struct RamblockDirtyInfo));
 457     if (dinfo == NULL) {
 458         goto out;
 459     }
 460
 461     RAMBLOCK_FOREACH_MIGRATABLE(block) {
 462         if (skip_sample_ramblock(block)) {
 463             continue;
 464         }
 465         if (index >= total_count) {
 466             break;
 467         }
 468         info = &dinfo[index];
 469         get_ramblock_dirty_info(block, info, &config);
 470         if (!save_ramblock_hash(info)) {
 471             goto out;
 472         }
 473         index++;
 474     }
 475     ret = true;
 476
 477 out:
 478     *block_count = index;
 479     *block_dinfo = dinfo;
 480     return ret;
 481 }
 482
 483 static void calc_page_dirty_rate(struct RamblockDirtyInfo *info)
 484 {
 485     uint32_t hash;
 486     int i;
 487
 488     for (i = 0; i < info->sample_pages_count; i++) {
 489         hash = get_ramblock_vfn_hash(info, info->sample_page_vfn[i]);
 490         if (hash != info->hash_result[i]) {
 491             trace_calc_page_dirty_rate(info->idstr, hash, info->hash_result[i]);
 492             info->sample_dirty_count++;
 493         }
 494     }
 495 }
 496
 497 static struct RamblockDirtyInfo *
 498 find_block_matched(RAMBlock *block, int count,
 499                   struct RamblockDirtyInfo *infos)
 500 {
 501     int i;
 502
 503     for (i = 0; i < count; i++) {
 504         if (!strcmp(infos[i].idstr, qemu_ram_get_idstr(block))) {
 505             break;
 506         }
 507     }
 508
 509     if (i == count) {
 510         return NULL;
 511     }
 512
 513     if (infos[i].ramblock_addr != qemu_ram_get_host_addr(block) ||
 514         infos[i].ramblock_pages !=
 515             (qemu_ram_get_used_length(block) >> TARGET_PAGE_BITS)) {
 516         trace_find_page_matched(block->idstr);
 517         return NULL;
 518     }
 519
 520     return &infos[i];
 521 }
 522
 523 static bool compare_page_hash_info(struct RamblockDirtyInfo *info,
 524                                   int block_count)
 525 {
 526     struct RamblockDirtyInfo *block_dinfo = NULL;
 527     RAMBlock *block = NULL;
 528
 529     RAMBLOCK_FOREACH_MIGRATABLE(block) {
 530         if (skip_sample_ramblock(block)) {
 531             continue;
 532         }
 533         block_dinfo = find_block_matched(block, block_count, info);
 534         if (block_dinfo == NULL) {
 535             continue;
 536         }
 537         calc_page_dirty_rate(block_dinfo);
 538         update_dirtyrate_stat(block_dinfo);
 539     }
 540
 541     if (DirtyStat.page_sampling.total_sample_count == 0) {
 542         return false;
 543     }
 544
 545     return true;
 546 }
 547
 548 static inline void record_dirtypages_bitmap(DirtyPageRecord *dirty_pages,
 549                                             bool start)
 550 {
 551     if (start) {
 552         dirty_pages->start_pages = total_dirty_pages;
 553     } else {
 554         dirty_pages->end_pages = total_dirty_pages;
 555     }
 556 }
 557
 558 static inline void dirtyrate_manual_reset_protect(void)
 559 {
 560     RAMBlock *block = NULL;
 561
 562     WITH_RCU_READ_LOCK_GUARD() {
 563         RAMBLOCK_FOREACH_MIGRATABLE(block) {
 564             memory_region_clear_dirty_bitmap(block->mr, 0,
 565                                              block->used_length);
 566         }
 567     }
 568 }
 569
 570 static void calculate_dirtyrate_dirty_bitmap(struct DirtyRateConfig config)
 571 {
 572     int64_t msec = 0;
 573     int64_t start_time;
 574     DirtyPageRecord dirty_pages;
 575
 576     qemu_mutex_lock_iothread();
 577     memory_global_dirty_log_start(GLOBAL_DIRTY_DIRTY_RATE);
 578
 579     /*
 580      * 1'round of log sync may return all 1 bits with
 581      * KVM_DIRTY_LOG_INITIALLY_SET enable
 582      * skip it unconditionally and start dirty tracking
 583      * from 2'round of log sync
 584      */
 585     memory_global_dirty_log_sync();
 586
 587     /*
 588      * reset page protect manually and unconditionally.
 589      * this make sure kvm dirty log be cleared if
 590      * KVM_DIRTY_LOG_MANUAL_PROTECT_ENABLE cap is enabled.
 591      */
 592     dirtyrate_manual_reset_protect();
 593     qemu_mutex_unlock_iothread();
 594
 595     record_dirtypages_bitmap(&dirty_pages, true);
 596
 597     start_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
 598     DirtyStat.start_time = start_time / 1000;
 599
 600     msec = config.sample_period_seconds * 1000;
 601     msec = dirty_stat_wait(msec, start_time);
 602     DirtyStat.calc_time = msec / 1000;
 603
 604     /*
 605      * do two things.
 606      * 1. fetch dirty bitmap from kvm
 607      * 2. stop dirty tracking
 608      */
 609     global_dirty_log_sync(GLOBAL_DIRTY_DIRTY_RATE, true);
 610
 611     record_dirtypages_bitmap(&dirty_pages, false);
 612
 613     DirtyStat.dirty_rate = do_calculate_dirtyrate(dirty_pages, msec);
 614 }
 615
 616 static void calculate_dirtyrate_dirty_ring(struct DirtyRateConfig config)
 617 {
 618     int64_t duration;
 619     uint64_t dirtyrate = 0;
 620     uint64_t dirtyrate_sum = 0;
 621     int i = 0;
 622
 623     /* start log sync */
 624     global_dirty_log_change(GLOBAL_DIRTY_DIRTY_RATE, true);
 625
 626     DirtyStat.start_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME) / 1000;
 627
 628     /* calculate vcpu dirtyrate */
 629     duration = vcpu_calculate_dirtyrate(config.sample_period_seconds * 1000,
 630                                         &DirtyStat.dirty_ring,
 631                                         GLOBAL_DIRTY_DIRTY_RATE,
 632                                         true);
 633
 634     DirtyStat.calc_time = duration / 1000;
 635
 636     /* calculate vm dirtyrate */
 637     for (i = 0; i < DirtyStat.dirty_ring.nvcpu; i++) {
 638         dirtyrate = DirtyStat.dirty_ring.rates[i].dirty_rate;
 639         DirtyStat.dirty_ring.rates[i].dirty_rate = dirtyrate;
 640         dirtyrate_sum += dirtyrate;
 641     }
 642
 643     DirtyStat.dirty_rate = dirtyrate_sum;
 644 }
 645
 646 static void calculate_dirtyrate_sample_vm(struct DirtyRateConfig config)
 647 {
 648     struct RamblockDirtyInfo *block_dinfo = NULL;
 649     int block_count = 0;
 650     int64_t msec = 0;
 651     int64_t initial_time;
 652
 653     rcu_read_lock();
 654     initial_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
 655     if (!record_ramblock_hash_info(&block_dinfo, config, &block_count)) {
 656         goto out;
 657     }
 658     rcu_read_unlock();
 659
 660     msec = config.sample_period_seconds * 1000;
 661     msec = dirty_stat_wait(msec, initial_time);
 662     DirtyStat.start_time = initial_time / 1000;
 663     DirtyStat.calc_time = msec / 1000;
 664
 665     rcu_read_lock();
 666     if (!compare_page_hash_info(block_dinfo, block_count)) {
 667         goto out;
 668     }
 669
 670     update_dirtyrate(msec);
 671
 672 out:
 673     rcu_read_unlock();
 674     free_ramblock_dirty_info(block_dinfo, block_count);
 675 }
 676
 677 static void calculate_dirtyrate(struct DirtyRateConfig config)
 678 {
 679     if (config.mode == DIRTY_RATE_MEASURE_MODE_DIRTY_BITMAP) {
 680         calculate_dirtyrate_dirty_bitmap(config);
 681     } else if (config.mode == DIRTY_RATE_MEASURE_MODE_DIRTY_RING) {
 682         calculate_dirtyrate_dirty_ring(config);
 683     } else {
 684         calculate_dirtyrate_sample_vm(config);
 685     }
 686
 687     trace_dirtyrate_calculate(DirtyStat.dirty_rate);
 688 }
 689
 690 void *get_dirtyrate_thread(void *arg)
 691 {
 692     struct DirtyRateConfig config = *(struct DirtyRateConfig *)arg;
 693     int ret;
 694     rcu_register_thread();
 695
 696     ret = dirtyrate_set_state(&CalculatingState, DIRTY_RATE_STATUS_UNSTARTED,
 697                               DIRTY_RATE_STATUS_MEASURING);
 698     if (ret == -1) {
 699         error_report("change dirtyrate state failed.");
 700         return NULL;
 701     }
 702
 703     calculate_dirtyrate(config);
 704
 705     ret = dirtyrate_set_state(&CalculatingState, DIRTY_RATE_STATUS_MEASURING,
 706                               DIRTY_RATE_STATUS_MEASURED);
 707     if (ret == -1) {
 708         error_report("change dirtyrate state failed.");
 709     }
 710
 711     rcu_unregister_thread();
 712     return NULL;
 713 }
 714
 715 void qmp_calc_dirty_rate(int64_t calc_time,
 716                          bool has_sample_pages,
 717                          int64_t sample_pages,
 718                          bool has_mode,
 719                          DirtyRateMeasureMode mode,
 720                          Error **errp)
 721 {
 722     static struct DirtyRateConfig config;
 723     QemuThread thread;
 724     int ret;
 725     int64_t start_time;
 726
 727     /*
 728      * If the dirty rate is already being measured, don't attempt to start.
 729      */
 730     if (qatomic_read(&CalculatingState) == DIRTY_RATE_STATUS_MEASURING) {
 731         error_setg(errp, "the dirty rate is already being measured.");
 732         return;
 733     }
 734
 735     if (!is_sample_period_valid(calc_time)) {
 736         error_setg(errp, "calc-time is out of range[%d, %d].",
 737                          MIN_FETCH_DIRTYRATE_TIME_SEC,
 738                          MAX_FETCH_DIRTYRATE_TIME_SEC);
 739         return;
 740     }
 741
 742     if (!has_mode) {
 743         mode =  DIRTY_RATE_MEASURE_MODE_PAGE_SAMPLING;
 744     }
 745
 746     if (has_sample_pages && mode != DIRTY_RATE_MEASURE_MODE_PAGE_SAMPLING) {
 747         error_setg(errp, "sample-pages is used only in page-sampling mode");
 748         return;
 749     }
 750
 751     if (has_sample_pages) {
 752         if (!is_sample_pages_valid(sample_pages)) {
 753             error_setg(errp, "sample-pages is out of range[%d, %d].",
 754                             MIN_SAMPLE_PAGE_COUNT,
 755                             MAX_SAMPLE_PAGE_COUNT);
 756             return;
 757         }
 758     } else {
 759         sample_pages = DIRTYRATE_DEFAULT_SAMPLE_PAGES;
 760     }
 761
 762     /*
 763      * dirty ring mode only works when kvm dirty ring is enabled.
 764      * on the contrary, dirty bitmap mode is not.
 765      */
 766     if (((mode == DIRTY_RATE_MEASURE_MODE_DIRTY_RING) &&
 767         !kvm_dirty_ring_enabled()) ||
 768         ((mode == DIRTY_RATE_MEASURE_MODE_DIRTY_BITMAP) &&
 769          kvm_dirty_ring_enabled())) {
 770         error_setg(errp, "mode %s is not enabled, use other method instead.",
 771                          DirtyRateMeasureMode_str(mode));
 772          return;
 773     }
 774
 775     /*
 776      * Init calculation state as unstarted.
 777      */
 778     ret = dirtyrate_set_state(&CalculatingState, CalculatingState,
 779                               DIRTY_RATE_STATUS_UNSTARTED);
 780     if (ret == -1) {
 781         error_setg(errp, "init dirty rate calculation state failed.");
 782         return;
 783     }
 784
 785     config.sample_period_seconds = calc_time;
 786     config.sample_pages_per_gigabytes = sample_pages;
 787     config.mode = mode;
 788
 789     cleanup_dirtyrate_stat(config);
 790
 791     /*
 792      * update dirty rate mode so that we can figure out what mode has
 793      * been used in last calculation
 794      **/
 795     dirtyrate_mode = mode;
 796
 797     start_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME) / 1000;
 798     init_dirtyrate_stat(start_time, config);
 799
 800     qemu_thread_create(&thread, "get_dirtyrate", get_dirtyrate_thread,
 801                        (void *)&config, QEMU_THREAD_DETACHED);
 802 }
 803
 804 struct DirtyRateInfo *qmp_query_dirty_rate(Error **errp)
 805 {
 806     return query_dirty_rate_info();
 807 }
 808
 809 void hmp_info_dirty_rate(Monitor *mon, const QDict *qdict)
 810 {
 811     DirtyRateInfo *info = query_dirty_rate_info();
 812
 813     monitor_printf(mon, "Status: %s\n",
 814                    DirtyRateStatus_str(info->status));
 815     monitor_printf(mon, "Start Time: %"PRIi64" (ms)\n",
 816                    info->start_time);
 817     if (info->mode == DIRTY_RATE_MEASURE_MODE_PAGE_SAMPLING) {
 818         monitor_printf(mon, "Sample Pages: %"PRIu64" (per GB)\n",
 819                        info->sample_pages);
 820     }
 821     monitor_printf(mon, "Period: %"PRIi64" (sec)\n",
 822                    info->calc_time);
 823     monitor_printf(mon, "Mode: %s\n",
 824                    DirtyRateMeasureMode_str(info->mode));
 825     monitor_printf(mon, "Dirty rate: ");
 826     if (info->has_dirty_rate) {
 827         monitor_printf(mon, "%"PRIi64" (MB/s)\n", info->dirty_rate);
 828         if (info->has_vcpu_dirty_rate) {
 829             DirtyRateVcpuList *rate, *head = info->vcpu_dirty_rate;
 830             for (rate = head; rate != NULL; rate = rate->next) {
 831                 monitor_printf(mon, "vcpu[%"PRIi64"], Dirty rate: %"PRIi64
 832                                " (MB/s)\n", rate->value->id,
 833                                rate->value->dirty_rate);
 834             }
 835         }
 836     } else {
 837         monitor_printf(mon, "(not ready)\n");
 838     }
 839
 840     qapi_free_DirtyRateVcpuList(info->vcpu_dirty_rate);
 841     g_free(info);
 842 }
 843
 844 void hmp_calc_dirty_rate(Monitor *mon, const QDict *qdict)
 845 {
 846     int64_t sec = qdict_get_try_int(qdict, "second", 0);
 847     int64_t sample_pages = qdict_get_try_int(qdict, "sample_pages_per_GB", -1);
 848     bool has_sample_pages = (sample_pages != -1);
 849     bool dirty_ring = qdict_get_try_bool(qdict, "dirty_ring", false);
 850     bool dirty_bitmap = qdict_get_try_bool(qdict, "dirty_bitmap", false);
 851     DirtyRateMeasureMode mode = DIRTY_RATE_MEASURE_MODE_PAGE_SAMPLING;
 852     Error *err = NULL;
 853
 854     if (!sec) {
 855         monitor_printf(mon, "Incorrect period length specified!\n");
 856         return;
 857     }
 858
 859     if (dirty_ring && dirty_bitmap) {
 860         monitor_printf(mon, "Either dirty ring or dirty bitmap "
 861                        "can be specified!\n");
 862         return;
 863     }
 864
 865     if (dirty_bitmap) {
 866         mode = DIRTY_RATE_MEASURE_MODE_DIRTY_BITMAP;
 867     } else if (dirty_ring) {
 868         mode = DIRTY_RATE_MEASURE_MODE_DIRTY_RING;
 869     }
 870
 871     qmp_calc_dirty_rate(sec, has_sample_pages, sample_pages, true,
 872                         mode, &err);
 873     if (err) {
 874         hmp_handle_error(mon, err);
 875         return;
 876     }
 877
 878     monitor_printf(mon, "Starting dirty rate measurement with period %"PRIi64
 879                    " seconds\n", sec);
 880     monitor_printf(mon, "[Please use 'info dirty_rate' to check results]\n");
 881 }